diff --git a/.claude/skills/ec2-cluster-provision/SKILL.md b/.claude/skills/ec2-cluster-provision/SKILL.md new file mode 100644 index 00000000..6a995d48 --- /dev/null +++ b/.claude/skills/ec2-cluster-provision/SKILL.md @@ -0,0 +1,62 @@ +--- +name: ec2-cluster-provision +description: uses the code present in this repository for provision an EC2 cluster for benchmarking purposes +--- + +This project uses a remote benchmarks EC2 cluster constructed with AWS CDK located at `benchmarks/cdk`. + +There's a package.json file in `benchmarks/cdk/package.json` with relevant commands about deploying. + +All the commands in this skill need to be prefixed with whatever the user declared in `./claude/settings.local.json` +in the `aws-commands-prefix` key, typically for providing the commands with the correct permissions. +(e.g., `$aws-commands-prefix npm run cdk deploy` or `$aws-commands-prexfix aws ssm ...`) + +Running `npm run cdk deploy` will provision the cluster with the resources specified in `benchmarks/cdk/lib/`. +This takes a while typically (~5 mins). If the user data of the EC2 machines was changed, and you want those changes +to take effect you will need to prepend the deployment command with `USER_DATA_CAUSES_REPLACEMENT=true`. + +Once the deployment is complete, the list of instance IDs will be printed to stdout. + +It's usually necessary to verify that everything was deployed correctly, and it's running fine. For that +it's necessary to perform the following steps for the following engines: + +## Distributed DataFusion + +1. Port forward the 9000 port in a background terminal: + `aws ssm start-session --target $INSTANCE_ID --document-name AWS-StartPortForwardingSession --parameters "portNumber=9000,localPortNumber=9000"` +2. Issue a command to /info to see what was deployed: + `curl http://localhost:9000/info | jq .` +3. If everything's fine, you should see the list of listening workers and the build time + +## Trino + +1. Port forward the 8080 port in a background terminal: + `aws ssm start-session --target $INSTANCE_ID --document-name AWS-StartPortForwardingSession --parameters "portNumber=8080,localPortNumber=8080"` +2. Issue a command to /v1/node to see what nodes are available for listening: + `curl -s -H "X-Trino-User: admin" http://localhost:8080/v1/node | jq .` + `curl -s -H "X-Trino-User: admin" http://localhost:8080/v1/info | jq .` +3. Make sure that the response of the above is consistent with what is supposed to be deployed by + `benchmarks/cdk/lib/trino.ts` + +## Spark + +1. Port forward the 9003 port in a background terminal (this is a custom Python server `benchmarks/cdk/bin/spark_http.py`): + `aws ssm start-session --target $INSTANCE_ID --document-name AWS-StartPortForwardingSession --parameters "portNumber=9003,localPortNumber=9003"` +2. You can issue curl queries to `http://localhost:9003/health` and `http://localhost:9003/query` to double-check that + everything is consistent with what's expected from `benchmarks/cdk/lib/spark.ts` + +## Ballista + +1. Port forward the 9002 port in a background terminal: + `aws ssm start-session --target $INSTANCE_ID --document-name AWS-StartPortForwardingSession --parameters "portNumber=9002,localPortNumber=9002"` +2. Verify that there are in fact the expected workers connected to the scheduler. For this, all workers need to + have the appropriate instance IP configured in their launch command, which might not have been done by default, + because this happens in a post cloud-init command that might not run because of timeout. +3. Verify with systemctl in each remote machine that all workers are running successfully. One thing that happens + very often is that `--external-host` is incorrectly set to localhost rather than the actual scheduler EC2 private IP + +Remember that for running port forward commands in the background, they take like 5 secs until the +"waiting for connections" message appears. Until then, the port is still not forwarded. + +If at some point you need to run a command in all machines and get its output, you can do it +with `npm run send-command your custom command` diff --git a/.claude/skills/remote-benchmark/SKILL.md b/.claude/skills/remote-benchmark/SKILL.md new file mode 100644 index 00000000..f9e763db --- /dev/null +++ b/.claude/skills/remote-benchmark/SKILL.md @@ -0,0 +1,85 @@ +--- +name: remote-benchmark +description: deploys the code to a remote EC2 cluster with the commands available in the package.json, port-forwards + a machine port, and runs benchmarks against it. +--- + +This project uses a remote benchmarks EC2 cluster constructed with AWS CDK located at `benchmarks/cdk`. + +There's a package.json file in `benchmarks/cdk/package.json` with relevant commands about benchmarking. + +All the commands in this skill need to be prefixed with whatever the user declared in `./claude/settings.local.json` +in the `aws-commands-prefix` key, typically for providing the commands with the correct permissions. +(e.g., `$aws-commands-prefix npm run fast-deploy` or `$aws-commands-prexfix aws ssm ...`) + +You can assume that the cluster is already there, and the only thing necessary is to execute the `npm run fast-deploy` +command for deploying the current code to the EC2 cluster. Remember that all npm commands need to be run from the +`benchmarks/cdk` folder. + +The `npm run fast-deploy` command will compile the current code and deploy it to the EC2 machines. If it fails, +prompt the user to fix it. It will output several EC2 instance IDs: pick the first one, that's the one we will port +forward locally in order to issue queries to it. + +Once the deployment is completed, port forward the first instance id to the local port 9000: + +```shell +aws ssm start-session --target i-0000000000000 --document-name AWS-StartPortForwardingSession --parameters "portNumber=9000,localPortNumber=9000" +``` + +Remember to run that in the background, as that will block in place. + +Once the port is correctly listening locally (you will see a "waiting for connections" message), it's fine to start +the benchmarks. + +You can start the benchmarks with `npm run datafusion-bench`, you can learn how this command works by running: + +```shell +$ npm run datafusion-bench -- --help + +Usage: datafusion-bench [options] + +Options: + --dataset Dataset to run queries on + -i, --iterations Number of iterations (default: "3") + --bytes-processed-per-partition How many bytes each partition is expected to process (default: "134217728") + --batch-size Standard Batch coalescing size (number of rows) (default: "32768") + --shuffle-batch-size Shuffle batch coalescing size (number of rows) (default: "32768") + --children-isolator-unions Use children isolator unions (default: "true") + --broadcast-joins Use broadcast joins (default: "false") + --collect-metrics Propagates metric collection (default: "true") + --compression Compression algo to use within workers (lz4, zstd, none) (default: "lz4") + --queries Specific queries to run + --debug Print the generated plans to stdout + -h, --help display help for command +``` + +The --dataset command is mandatory, and its value can be any of the folder names in `benchmarks/data`, for example: +clickbench_0-100, tpcds_sf1, tpch_sf1, tpch_sf10 or tpch_sf100. + +Also, the --queries argument can be used for executing just a partial subset of queries, for example: +```shell +--queries q1,q2,q3 +``` + +When benchmarking a very specific feature, it's convenient to choose wisely a relevant query and just execute that one. + +The user provided the following arguments: $ARGUMENTS + +parse those and make sure you parse them correctly, for example `tpch_sf100 q1,q2,q4` means +`--dataset tpch_sf100 --queries q1,q2,q4`. Note that the user might also give natural language instructions in the +arguments, be smart while parsing those. + +### analyzing results + +results for individual queries will be dumped in the respective dataset folders, for example: + +`benchmarks/data/tpch_sf10/.results-remote/datafusion-distributed-main/q1.json` +or +`benchmarks/data/tpch_sf1/.results-remote/datafusion-distributed-new-branch/q2.json` + +You can inspect the results and the plan by reading the JSONs. Tip: use jq for printing nice results. + +As the results of previous branches are already stored in disk, they usually can be analyzed without re-running them +again, that can be done by either: +- Just looking at the latencies and plans in the output folders. +- Running the `npm run datafusion-bench -- compare --dataset tpch_sfX datafusion-distributed- datafusion-distributed-` \ No newline at end of file diff --git a/.gitattributes b/.gitattributes index b57f642b..5363338f 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1 +1,2 @@ *.parquet filter=lfs diff=lfs merge=lfs -text +*.png filter=lfs diff=lfs merge=lfs -text diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 101ea138..5df9ed01 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -12,6 +12,9 @@ inputs: runs: using: 'composite' steps: + - name: Free up disk space + shell: bash + run: ${{ github.action_path }}/free_disk_space.sh - shell: bash run: | echo $HOME @@ -26,3 +29,4 @@ runs: - uses: Swatinem/rust-cache@v2 with: key: ${{ runner.os }}-${{ inputs.targets }}-rust-cache + prefix-key: v1-rust diff --git a/.github/actions/setup/free_disk_space.sh b/.github/actions/setup/free_disk_space.sh new file mode 100755 index 00000000..15524194 --- /dev/null +++ b/.github/actions/setup/free_disk_space.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +# Script to free up disk space on CI systems +# Source: https://github.com/apache/flink/blob/02d30ace69dc18555a5085eccf70ee884e73a16e/tools/azure-pipelines/free_disk_space.sh + +echo "Freeing up disk space on CI system" + +echo "Listing 100 largest packages" +dpkg-query -Wf '${Installed-Size}\t${Package}\n' | sort -n | tail -n 100 +df -h + +echo "Removing large packages" +sudo apt-get remove -y '^ghc-8.*' +sudo apt-get remove -y '^dotnet-.*' +sudo apt-get remove -y '^llvm-.*' +sudo apt-get remove -y 'php.*' +sudo apt-get remove -y azure-cli google-cloud-sdk hhvm google-chrome-stable firefox powershell mono-devel +sudo apt-get autoremove -y +sudo apt-get clean +df -h + +echo "Removing large directories" +rm -rf /usr/share/dotnet/ +df -h \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f93d2927..4a23c11b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,12 +9,20 @@ on: env: CARGO_TERM_COLOR: always RUST_BACKTRACE: 1 + TPCDS_SCALE_FACTOR: 0.5 # 0.5 scale factor produces a data dir which is 124MB concurrency: group: ${{ github.ref }} cancel-in-progress: true jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/setup + - run: cargo build + clippy: runs-on: ubuntu-latest steps: @@ -22,7 +30,7 @@ jobs: - uses: ./.github/actions/setup with: components: clippy - - run: cargo clippy --all-targets --features integration + - run: cargo clippy --all-targets --all-features -- -D warnings unit-test: runs-on: ubuntu-latest @@ -38,7 +46,29 @@ jobs: steps: - uses: actions/checkout@v4 - uses: ./.github/actions/setup - - run: cargo test --features tpch --test tpch_validation_test + - run: cargo test --features tpch --test 'tpch_*' + + tpcds-test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/setup + - uses: actions/cache@v4 + with: + path: testdata/tpcds/main.zip + key: "main.zip" + - run: cargo test --features tpcds --test 'tpcds_*' + + clickbench-test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/setup + - uses: actions/cache@v4 + with: + path: testdata/clickbench/ + key: "data" + - run: cargo test --features clickbench --test 'clickbench_*' format-check: runs-on: ubuntu-latest diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 00000000..78f827a0 --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,55 @@ +name: Deploy Documentation + +on: + push: + branches: + - main + workflow_dispatch: + +permissions: + contents: read + pages: write + id-token: write + +concurrency: + group: "pages" + cancel-in-progress: false + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + lfs: true + + - name: Setup Python + uses: actions/setup-python@v6 + with: + python-version: '3.x' + + - name: Install dependencies + run: | + pip install -r docs/requirements.txt + + - name: Build documentation + run: | + cd docs + make html + + - name: Upload artifact + uses: actions/upload-pages-artifact@v4 + with: + path: docs/build/html + + deploy: + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + runs-on: ubuntu-latest + needs: build + steps: + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v4 diff --git a/.gitignore b/.gitignore index be0175de..9aecd384 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,12 @@ /.idea /target /benchmarks/data/ -testdata/tpch/data/ \ No newline at end of file +testdata/tpch/* +!testdata/tpch/queries +testdata/tpcds/* +!testdata/tpcds/queries +!testdata/tpcds/README.md +testdata/clickbench/* +!testdata/clickbench/queries +src/observability/gen/target/* +src/observability/gen/Cargo.lock diff --git a/Cargo.lock b/Cargo.lock index c0a40ea9..25a870af 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,9 +4,9 @@ version = 4 [[package]] name = "addr2line" -version = "0.24.2" +version = "0.25.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" +checksum = "1b5d307320b3181d6d7954e663bd7c774a838b8220fe0593c86d9fb09f498b4b" dependencies = [ "gimli", ] @@ -17,6 +17,17 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures", +] + [[package]] name = "ahash" version = "0.8.12" @@ -25,7 +36,7 @@ checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" dependencies = [ "cfg-if", "const-random", - "getrandom 0.3.3", + "getrandom 0.3.4", "once_cell", "version_check", "zerocopy", @@ -33,9 +44,9 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "1.1.3" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" dependencies = [ "memchr", ] @@ -70,6 +81,12 @@ dependencies = [ "libc", ] +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + [[package]] name = "ansi_term" version = "0.12.1" @@ -81,9 +98,9 @@ dependencies = [ [[package]] name = "anstream" -version = "0.6.19" +version = "0.6.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "301af1932e46185686725e0fad2f8f2aa7da69dd70bf6ecc44d6b703844a3933" +checksum = "43d5b281e737544384e969a5ccad3f1cdd24b48086a0fc1b2a5262a26b8f4f4a" dependencies = [ "anstyle", "anstyle-parse", @@ -96,9 +113,9 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.11" +version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "862ed96ca487e809f1c8e5a8447f6ee2cf102f846893800b20cebdf541fc6bbd" +checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78" [[package]] name = "anstyle-parse" @@ -111,29 +128,76 @@ dependencies = [ [[package]] name = "anstyle-query" -version = "1.1.3" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c8bdeb6047d8983be085bab0ba1472e6dc604e7041dbf6fcd5e71523014fae9" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] name = "anstyle-wincon" -version = "3.0.9" +version = "3.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "403f75924867bb1033c59fbf0797484329750cfbe3c4325cd33127941fabc882" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] name = "anyhow" -version = "1.0.98" +version = "1.0.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" + +[[package]] +name = "apache-avro" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36fa98bc79671c7981272d91a8753a928ff6a1cd8e4f20a44c45bd5d313840bf" +dependencies = [ + "bigdecimal", + "bon", + "bzip2", + "crc32fast", + "digest", + "liblzma", + "log", + "miniz_oxide", + "num-bigint", + "quad-rand", + "rand 0.9.2", + "regex-lite", + "serde", + "serde_bytes", + "serde_json", + "snap", + "strum", + "strum_macros", + "thiserror 2.0.18", + "uuid", + "zstd", +] + +[[package]] +name = "ar_archive_writer" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e16d2d3311acee920a9eb8d33b8cbc1787ce4a264e85f964c2404b969bdcd487" +checksum = "7eb93bbb63b9c227414f6eb3a0adfddca591a8ce1e9b60661bb08969b87e340b" +dependencies = [ + "object", +] + +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" +dependencies = [ + "derive_arbitrary", +] [[package]] name = "arrayref" @@ -149,9 +213,9 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "arrow" -version = "56.2.0" +version = "57.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e833808ff2d94ed40d9379848a950d995043c7fb3e81a30b383f4c6033821cc" +checksum = "2a2b10dcb159faf30d3f81f6d56c1211a5bea2ca424eabe477648a44b993320e" dependencies = [ "arrow-arith", "arrow-array", @@ -170,23 +234,23 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "56.2.0" +version = "57.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad08897b81588f60ba983e3ca39bda2b179bdd84dced378e7df81a5313802ef8" +checksum = "288015089e7931843c80ed4032c5274f02b37bcb720c4a42096d50b390e70372" dependencies = [ "arrow-array", "arrow-buffer", "arrow-data", "arrow-schema", "chrono", - "num", + "num-traits", ] [[package]] name = "arrow-array" -version = "56.2.0" +version = "57.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8548ca7c070d8db9ce7aa43f37393e4bfcf3f2d3681df278490772fd1673d08d" +checksum = "65ca404ea6191e06bf30956394173337fa9c35f445bd447fe6c21ab944e1a23c" dependencies = [ "ahash", "arrow-buffer", @@ -195,30 +259,34 @@ dependencies = [ "chrono", "chrono-tz", "half", - "hashbrown 0.16.0", - "num", + "hashbrown 0.16.1", + "num-complex", + "num-integer", + "num-traits", ] [[package]] name = "arrow-buffer" -version = "56.2.0" +version = "57.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e003216336f70446457e280807a73899dd822feaf02087d31febca1363e2fccc" +checksum = "36356383099be0151dacc4245309895f16ba7917d79bdb71a7148659c9206c56" dependencies = [ "bytes", "half", - "num", + "num-bigint", + "num-traits", ] [[package]] name = "arrow-cast" -version = "56.2.0" +version = "57.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "919418a0681298d3a77d1a315f625916cb5678ad0d74b9c60108eb15fd083023" +checksum = "9c8e372ed52bd4ee88cc1e6c3859aa7ecea204158ac640b10e187936e7e87074" dependencies = [ "arrow-array", "arrow-buffer", "arrow-data", + "arrow-ord", "arrow-schema", "arrow-select", "atoi", @@ -227,15 +295,15 @@ dependencies = [ "comfy-table", "half", "lexical-core", - "num", + "num-traits", "ryu", ] [[package]] name = "arrow-csv" -version = "56.2.0" +version = "57.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa9bf02705b5cf762b6f764c65f04ae9082c7cfc4e96e0c33548ee3f67012eb" +checksum = "8e4100b729fe656f2e4fb32bc5884f14acf9118d4ad532b7b33c1132e4dce896" dependencies = [ "arrow-array", "arrow-cast", @@ -248,21 +316,22 @@ dependencies = [ [[package]] name = "arrow-data" -version = "56.2.0" +version = "57.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5c64fff1d142f833d78897a772f2e5b55b36cb3e6320376f0961ab0db7bd6d0" +checksum = "bf87f4ff5fc13290aa47e499a8b669a82c5977c6a1fedce22c7f542c1fd5a597" dependencies = [ "arrow-buffer", "arrow-schema", "half", - "num", + "num-integer", + "num-traits", ] [[package]] name = "arrow-flight" -version = "56.2.0" +version = "57.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c8b0ba0784d56bc6266b79f5de7a24b47024e7b3a0045d2ad4df3d9b686099f" +checksum = "f63654f21676be802d446c6c4bc54f6a47e18d55f9ae6f7195a6f6faf2ecdbeb" dependencies = [ "arrow-array", "arrow-buffer", @@ -275,13 +344,14 @@ dependencies = [ "prost", "prost-types", "tonic", + "tonic-prost", ] [[package]] name = "arrow-ipc" -version = "56.2.0" +version = "57.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d3594dcddccc7f20fd069bc8e9828ce37220372680ff638c5e00dea427d88f5" +checksum = "eb3ca63edd2073fcb42ba112f8ae165df1de935627ead6e203d07c99445f2081" dependencies = [ "arrow-array", "arrow-buffer", @@ -295,9 +365,9 @@ dependencies = [ [[package]] name = "arrow-json" -version = "56.2.0" +version = "57.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88cf36502b64a127dc659e3b305f1d993a544eab0d48cce704424e62074dc04b" +checksum = "a36b2332559d3310ebe3e173f75b29989b4412df4029a26a30cc3f7da0869297" dependencies = [ "arrow-array", "arrow-buffer", @@ -307,19 +377,21 @@ dependencies = [ "chrono", "half", "indexmap", + "itoa", "lexical-core", "memchr", - "num", - "serde", + "num-traits", + "ryu", + "serde_core", "serde_json", "simdutf8", ] [[package]] name = "arrow-ord" -version = "56.2.0" +version = "57.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c8f82583eb4f8d84d4ee55fd1cb306720cddead7596edce95b50ee418edf66f" +checksum = "13c4e0530272ca755d6814218dffd04425c5b7854b87fa741d5ff848bf50aa39" dependencies = [ "arrow-array", "arrow-buffer", @@ -330,9 +402,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "56.2.0" +version = "57.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d07ba24522229d9085031df6b94605e0f4b26e099fb7cdeec37abd941a73753" +checksum = "b07f52788744cc71c4628567ad834cadbaeb9f09026ff1d7a4120f69edf7abd3" dependencies = [ "arrow-array", "arrow-buffer", @@ -343,33 +415,33 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "56.2.0" +version = "57.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3aa9e59c611ebc291c28582077ef25c97f1975383f1479b12f3b9ffee2ffabe" +checksum = "6bb63203e8e0e54b288d0d8043ca8fa1013820822a27692ef1b78a977d879f2c" dependencies = [ - "serde", + "serde_core", "serde_json", ] [[package]] name = "arrow-select" -version = "56.2.0" +version = "57.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c41dbbd1e97bfcaee4fcb30e29105fb2c75e4d82ae4de70b792a5d3f66b2e7a" +checksum = "c96d8a1c180b44ecf2e66c9a2f2bbcb8b1b6f14e165ce46ac8bde211a363411b" dependencies = [ "ahash", "arrow-array", "arrow-buffer", "arrow-data", "arrow-schema", - "num", + "num-traits", ] [[package]] name = "arrow-string" -version = "56.2.0" +version = "57.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53f5183c150fbc619eede22b861ea7c0eebed8eaac0333eaa7f6da5205fd504d" +checksum = "a8ad6a81add9d3ea30bf8374ee8329992c7fd246ffd8b7e2f48a3cea5aa0cc9a" dependencies = [ "arrow-array", "arrow-buffer", @@ -377,26 +449,32 @@ dependencies = [ "arrow-schema", "arrow-select", "memchr", - "num", + "num-traits", "regex", "regex-syntax", ] [[package]] name = "async-compression" -version = "0.4.19" +version = "0.4.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06575e6a9673580f52661c92107baabffbf41e2141373441cbcdc47cb733003c" +checksum = "d10e4f991a553474232bc0a31799f6d24b034a84c0971d80d2e2f78b2e576e40" dependencies = [ - "bzip2 0.5.2", - "flate2", - "futures-core", - "memchr", + "compression-codecs", + "compression-core", "pin-project-lite", "tokio", - "xz2", - "zstd", - "zstd-safe", +] + +[[package]] +name = "async-lock" +version = "3.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290f7f2596bd5b78a9fec8088ccd89180d7f9f55b94b0576823bbbdc72ee8311" +dependencies = [ + "event-listener", + "event-listener-strategy", + "pin-project-lite", ] [[package]] @@ -407,7 +485,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] @@ -419,6 +497,15 @@ dependencies = [ "num-traits", ] +[[package]] +name = "atomic" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89cbf775b137e9b968e67227ef7f775587cde3fd31b0d8599dbd0f598a48340" +dependencies = [ + "bytemuck", +] + [[package]] name = "atomic-waker" version = "1.1.2" @@ -431,7 +518,7 @@ version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" dependencies = [ - "hermit-abi", + "hermit-abi 0.1.19", "libc", "winapi", ] @@ -443,222 +530,763 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" [[package]] -name = "axum" -version = "0.8.4" +name = "aws-config" +version = "1.8.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "021e862c184ae977658b36c4500f7feac3221ca5da43e3f25bd04ab6c79a29b5" +checksum = "96571e6996817bf3d58f6b569e4b9fd2e9d2fcf9f7424eed07b2ce9bb87535e5" dependencies = [ - "axum-core", + "aws-credential-types", + "aws-runtime", + "aws-sdk-sso", + "aws-sdk-ssooidc", + "aws-sdk-sts", + "aws-smithy-async", + "aws-smithy-http 0.62.6", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", "bytes", - "futures-util", - "http", - "http-body", - "http-body-util", - "itoa", - "matchit", - "memchr", - "mime", - "percent-encoding", - "pin-project-lite", - "rustversion", - "serde", - "sync_wrapper", - "tower", - "tower-layer", - "tower-service", + "fastrand", + "hex", + "http 1.4.0", + "ring", + "time", + "tokio", + "tracing", + "url", + "zeroize", ] [[package]] -name = "axum-core" -version = "0.5.2" +name = "aws-credential-types" +version = "1.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68464cd0412f486726fb3373129ef5d2993f90c34bc2bc1c1e9943b2f4fc7ca6" +checksum = "3cd362783681b15d136480ad555a099e82ecd8e2d10a841e14dfd0078d67fee3" dependencies = [ - "bytes", - "futures-core", - "http", - "http-body", - "http-body-util", - "mime", - "pin-project-lite", - "rustversion", - "sync_wrapper", - "tower-layer", - "tower-service", + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "zeroize", ] [[package]] -name = "backtrace" -version = "0.3.75" +name = "aws-lc-rs" +version = "1.15.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6806a6321ec58106fea15becdad98371e28d92ccbc7c8f1b3b6dd724fe8f1002" +checksum = "7b7b6141e96a8c160799cc2d5adecd5cbbe5054cb8c7c4af53da0f83bb7ad256" dependencies = [ - "addr2line", - "cfg-if", - "libc", - "miniz_oxide", - "object", - "rustc-demangle", - "windows-targets 0.52.6", + "aws-lc-sys", + "zeroize", ] [[package]] -name = "base64" -version = "0.22.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" - -[[package]] -name = "bigdecimal" -version = "0.4.8" +name = "aws-lc-sys" +version = "0.37.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a22f228ab7a1b23027ccc6c350b72868017af7ea8356fbdf19f8d991c690013" +checksum = "5c34dda4df7017c8db52132f0f8a2e0f8161649d15723ed63fc00c82d0f2081a" dependencies = [ - "autocfg", - "libm", - "num-bigint", - "num-integer", - "num-traits", + "cc", + "cmake", + "dunce", + "fs_extra", ] [[package]] -name = "bitflags" -version = "1.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" - -[[package]] -name = "bitflags" -version = "2.9.1" +name = "aws-runtime" +version = "1.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967" +checksum = "959dab27ce613e6c9658eb3621064d0e2027e5f2acb65bc526a43577facea557" +dependencies = [ + "aws-credential-types", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-http 0.62.6", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 0.2.12", + "http-body 0.4.6", + "percent-encoding", + "pin-project-lite", + "tracing", + "uuid", +] [[package]] -name = "blake2" -version = "0.10.6" +name = "aws-sdk-ec2" +version = "1.206.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" +checksum = "45900e23181a7783ee0a247ccf255070b07d112968d2dc0dfc6b1facd6871c11" dependencies = [ - "digest", + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http 0.62.6", + "aws-smithy-json", + "aws-smithy-observability", + "aws-smithy-query", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "fastrand", + "http 0.2.12", + "regex-lite", + "tracing", ] [[package]] -name = "blake3" -version = "1.8.2" +name = "aws-sdk-sso" +version = "1.92.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3888aaa89e4b2a40fca9848e400f6a658a5a3978de7be858e209cafa8be9a4a0" +checksum = "b7d63bd2bdeeb49aa3f9b00c15e18583503b778b2e792fc06284d54e7d5b6566" dependencies = [ - "arrayref", - "arrayvec", - "cc", - "cfg-if", - "constant_time_eq", + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http 0.62.6", + "aws-smithy-json", + "aws-smithy-observability", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 0.2.12", + "regex-lite", + "tracing", ] [[package]] -name = "block-buffer" -version = "0.10.4" +name = "aws-sdk-ssooidc" +version = "1.94.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +checksum = "532d93574bf731f311bafb761366f9ece345a0416dbcc273d81d6d1a1205239b" dependencies = [ - "generic-array", + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http 0.62.6", + "aws-smithy-json", + "aws-smithy-observability", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 0.2.12", + "regex-lite", + "tracing", ] [[package]] -name = "brotli" -version = "8.0.1" +name = "aws-sdk-sts" +version = "1.96.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9991eea70ea4f293524138648e41ee89b0b2b12ddef3b255effa43c8056e0e0d" +checksum = "357e9a029c7524db6a0099cd77fbd5da165540339e7296cca603531bc783b56c" dependencies = [ - "alloc-no-stdlib", - "alloc-stdlib", - "brotli-decompressor", + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http 0.62.6", + "aws-smithy-json", + "aws-smithy-observability", + "aws-smithy-query", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "fastrand", + "http 0.2.12", + "regex-lite", + "tracing", ] [[package]] -name = "brotli-decompressor" -version = "5.0.0" +name = "aws-sigv4" +version = "1.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "874bb8112abecc98cbd6d81ea4fa7e94fb9449648c93cc89aa40c81c24d7de03" +checksum = "69e523e1c4e8e7e8ff219d732988e22bfeae8a1cafdbe6d9eca1546fa080be7c" dependencies = [ - "alloc-no-stdlib", - "alloc-stdlib", + "aws-credential-types", + "aws-smithy-http 0.62.6", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "form_urlencoded", + "hex", + "hmac", + "http 0.2.12", + "http 1.4.0", + "percent-encoding", + "sha2", + "time", + "tracing", ] [[package]] -name = "bumpalo" -version = "3.19.0" +name = "aws-smithy-async" +version = "1.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" +checksum = "52eec3db979d18cb807fc1070961cc51d87d069abe9ab57917769687368a8c6c" +dependencies = [ + "futures-util", + "pin-project-lite", + "tokio", +] [[package]] -name = "byteorder" -version = "1.5.0" +name = "aws-smithy-http" +version = "0.62.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" +checksum = "826141069295752372f8203c17f28e30c464d22899a43a0c9fd9c458d469c88b" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "bytes-utils", + "futures-core", + "futures-util", + "http 0.2.12", + "http 1.4.0", + "http-body 0.4.6", + "percent-encoding", + "pin-project-lite", + "pin-utils", + "tracing", +] [[package]] -name = "bytes" -version = "1.10.1" +name = "aws-smithy-http" +version = "0.63.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" +checksum = "630e67f2a31094ffa51b210ae030855cb8f3b7ee1329bdd8d085aaf61e8b97fc" +dependencies = [ + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "bytes-utils", + "futures-core", + "futures-util", + "http 1.4.0", + "http-body 1.0.1", + "http-body-util", + "percent-encoding", + "pin-project-lite", + "pin-utils", + "tracing", +] [[package]] -name = "bzip2" -version = "0.5.2" +name = "aws-smithy-http-client" +version = "1.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49ecfb22d906f800d4fe833b6282cf4dc1c298f5057ca0b5445e5c209735ca47" +checksum = "12fb0abf49ff0cab20fd31ac1215ed7ce0ea92286ba09e2854b42ba5cabe7525" dependencies = [ - "bzip2-sys", + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "h2 0.3.27", + "h2 0.4.13", + "http 0.2.12", + "http 1.4.0", + "http-body 0.4.6", + "hyper 0.14.32", + "hyper 1.8.1", + "hyper-rustls 0.24.2", + "hyper-rustls 0.27.7", + "hyper-util", + "pin-project-lite", + "rustls 0.21.12", + "rustls 0.23.36", + "rustls-native-certs", + "rustls-pki-types", + "tokio", + "tokio-rustls 0.26.4", + "tower", + "tracing", ] [[package]] -name = "bzip2" -version = "0.6.0" +name = "aws-smithy-json" +version = "0.61.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bea8dcd42434048e4f7a304411d9273a411f647446c1234a65ce0554923f4cff" +checksum = "49fa1213db31ac95288d981476f78d05d9cbb0353d22cdf3472cc05bb02f6551" dependencies = [ - "libbz2-rs-sys", + "aws-smithy-types", ] [[package]] -name = "bzip2-sys" -version = "0.1.13+1.0.8" +name = "aws-smithy-observability" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14" +checksum = "c0a46543fbc94621080b3cf553eb4cbbdc41dd9780a30c4756400f0139440a1d" dependencies = [ - "cc", - "pkg-config", + "aws-smithy-runtime-api", ] [[package]] -name = "cc" -version = "1.2.30" +name = "aws-smithy-query" +version = "0.60.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "deec109607ca693028562ed836a5f1c4b8bd77755c4e132fc5ce11b0b6211ae7" +checksum = "0cebbddb6f3a5bd81553643e9c7daf3cc3dc5b0b5f398ac668630e8a84e6fff0" dependencies = [ - "jobserver", - "libc", - "shlex", + "aws-smithy-types", + "urlencoding", ] [[package]] -name = "cfg-if" -version = "1.0.1" +name = "aws-smithy-runtime" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9555578bc9e57714c812a1f84e4fc5b4d21fcb063490c624de019f7464c91268" +checksum = "f3df87c14f0127a0d77eb261c3bc45d5b4833e2a1f63583ebfb728e4852134ee" +dependencies = [ + "aws-smithy-async", + "aws-smithy-http 0.63.3", + "aws-smithy-http-client", + "aws-smithy-observability", + "aws-smithy-runtime-api", + "aws-smithy-types", + "bytes", + "fastrand", + "http 0.2.12", + "http 1.4.0", + "http-body 0.4.6", + "http-body 1.0.1", + "http-body-util", + "pin-project-lite", + "pin-utils", + "tokio", + "tracing", +] [[package]] -name = "chrono" -version = "0.4.42" +name = "aws-smithy-runtime-api" +version = "1.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "145052bdd345b87320e369255277e3fb5152762ad123a901ef5c262dd38fe8d2" +checksum = "49952c52f7eebb72ce2a754d3866cc0f87b97d2a46146b79f80f3a93fb2b3716" dependencies = [ - "iana-time-zone", - "js-sys", - "num-traits", - "wasm-bindgen", - "windows-link 0.2.0", + "aws-smithy-async", + "aws-smithy-types", + "bytes", + "http 0.2.12", + "http 1.4.0", + "pin-project-lite", + "tokio", + "tracing", + "zeroize", +] + +[[package]] +name = "aws-smithy-types" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b3a26048eeab0ddeba4b4f9d51654c79af8c3b32357dc5f336cee85ab331c33" +dependencies = [ + "base64-simd", + "bytes", + "bytes-utils", + "futures-core", + "http 0.2.12", + "http 1.4.0", + "http-body 0.4.6", + "http-body 1.0.1", + "http-body-util", + "itoa", + "num-integer", + "pin-project-lite", + "pin-utils", + "ryu", + "serde", + "time", + "tokio", + "tokio-util", +] + +[[package]] +name = "aws-smithy-xml" +version = "0.60.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11b2f670422ff42bf7065031e72b45bc52a3508bd089f743ea90731ca2b6ea57" +dependencies = [ + "xmlparser", +] + +[[package]] +name = "aws-types" +version = "1.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d980627d2dd7bfc32a3c025685a033eeab8d365cc840c631ef59d1b8f428164" +dependencies = [ + "aws-credential-types", + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "rustc_version", + "tracing", +] + +[[package]] +name = "axum" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f" +dependencies = [ + "async-trait", + "axum-core 0.4.5", + "bytes", + "futures-util", + "http 1.4.0", + "http-body 1.0.1", + "http-body-util", + "hyper 1.8.1", + "hyper-util", + "itoa", + "matchit 0.7.3", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b52af3cb4058c895d37317bb27508dccc8e5f2d39454016b297bf4a400597b8" +dependencies = [ + "axum-core 0.5.6", + "bytes", + "futures-util", + "http 1.4.0", + "http-body 1.0.1", + "http-body-util", + "itoa", + "matchit 0.8.4", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "serde_core", + "sync_wrapper", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http 1.4.0", + "http-body 1.0.1", + "http-body-util", + "mime", + "pin-project-lite", + "rustversion", + "sync_wrapper", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-core" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" +dependencies = [ + "bytes", + "futures-core", + "http 1.4.0", + "http-body 1.0.1", + "http-body-util", + "mime", + "pin-project-lite", + "sync_wrapper", + "tower-layer", + "tower-service", +] + +[[package]] +name = "backtrace" +version = "0.3.76" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb531853791a215d7c62a30daf0dde835f381ab5de4589cfe7c649d2cbe92bd6" +dependencies = [ + "addr2line", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", + "windows-link", +] + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "base64-simd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "339abbe78e73178762e23bea9dfd08e697eb3f3301cd4be981c0f78ba5859195" +dependencies = [ + "outref", + "vsimd", +] + +[[package]] +name = "bigdecimal" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d6867f1565b3aad85681f1015055b087fcfd840d6aeee6eee7f2da317603695" +dependencies = [ + "autocfg", + "libm", + "num-bigint", + "num-integer", + "num-traits", + "serde", +] + +[[package]] +name = "bit-set" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0700ddab506f33b20a03b13996eccd309a48e5ff77d0d95926aa0210fb4e95f1" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3" + +[[package]] +name = "blake2" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" +dependencies = [ + "digest", +] + +[[package]] +name = "blake3" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2468ef7d57b3fb7e16b576e8377cdbde2320c60e1491e961d11da40fc4f02a2d" +dependencies = [ + "arrayref", + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq 0.4.2", + "cpufeatures", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bon" +version = "3.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "234655ec178edd82b891e262ea7cf71f6584bcd09eff94db786be23f1821825c" +dependencies = [ + "bon-macros", + "rustversion", +] + +[[package]] +name = "bon-macros" +version = "3.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89ec27229c38ed0eb3c0feee3d2c1d6a4379ae44f418a29a658890e062d8f365" +dependencies = [ + "darling", + "ident_case", + "prettyplease", + "proc-macro2", + "quote", + "rustversion", + "syn 2.0.114", +] + +[[package]] +name = "brotli" +version = "8.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bd8b9603c7aa97359dbd97ecf258968c95f3adddd6db2f7e7a5bef101c84560" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "5.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "874bb8112abecc98cbd6d81ea4fa7e94fb9449648c93cc89aa40c81c24d7de03" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + +[[package]] +name = "built" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4ad8f11f288f48ca24471bbd51ac257aaeaaa07adae295591266b792902ae64" +dependencies = [ + "chrono", + "git2", +] + +[[package]] +name = "bumpalo" +version = "3.19.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dd9dc738b7a8311c7ade152424974d8115f2cdad61e8dab8dac9f2362298510" + +[[package]] +name = "bytemuck" +version = "1.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b35204fbdc0b3f4446b89fc1ac2cf84a8a68971995d0bf2e925ec7cd960f9cb3" + +[[package]] +name = "bytes-utils" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dafe3a8757b027e2be6e4e5601ed563c55989fcf1546e933c66c8eb3a058d35" +dependencies = [ + "bytes", + "either", +] + +[[package]] +name = "bzip2" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3a53fac24f34a81bc9954b5d6cfce0c21e18ec6959f44f56e8e90e4bb7c346c" +dependencies = [ + "libbz2-rs-sys", +] + +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + +[[package]] +name = "castaway" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a" +dependencies = [ + "rustversion", +] + +[[package]] +name = "cc" +version = "1.2.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b26a0954ae34af09b50f0de26458fa95369a0d478d8236d3f93082b219bd29" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + +[[package]] +name = "chrono" +version = "0.4.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fac4744fb15ae8337dc853fee7fb3f4e48c0fbaa23d0afe49c447b4fab126118" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "serde", + "wasm-bindgen", + "windows-link", ] [[package]] @@ -668,7 +1296,44 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6139a8597ed92cf816dfb33f5dd6cf0bb93a6adc938f11039f371bc5bcd26c3" dependencies = [ "chrono", - "phf", + "phf 0.12.1", +] + +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common", + "inout", ] [[package]] @@ -680,12 +1345,97 @@ dependencies = [ "ansi_term", "atty", "bitflags 1.3.2", - "strsim", + "strsim 0.8.0", "textwrap", "unicode-width 0.1.14", "vec_map", ] +[[package]] +name = "clap" +version = "4.5.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75ca66430e33a14957acc24c5077b503e7d374151b2b4b3a10c83b4ceb4be0e" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.5.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793207c7fa6300a0608d1080b858e5fdbe713cdc1c8db9fb17777d8a13e63df0" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim 0.11.1", +] + +[[package]] +name = "clap_derive" +version = "4.5.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a92793da1a46a5f2a02a6f4c46c6496b28c43638adea8306fcb0caa1634f24e5" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "clap_lex" +version = "0.7.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3e64b0cc0439b12df2fa678eae89a1c56a529fd067a9115f7827f1fffd22b32" + +[[package]] +name = "clipboard-win" +version = "5.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bde03770d3df201d4fb868f2c9c59e66a3e4e2bd06692a0fe701e7103c7e84d4" +dependencies = [ + "error-code", +] + +[[package]] +name = "cmake" +version = "0.1.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75443c44cd6b379beb8c5b45d85d0773baf31cce901fe7bb252f4eff3008ef7d" +dependencies = [ + "cc", +] + +[[package]] +name = "color-eyre" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5920befb47832a6d61ee3a3a846565cfa39b331331e68a3b1d1116630f2f26d" +dependencies = [ + "backtrace", + "color-spantrace", + "eyre", + "indenter", + "once_cell", + "owo-colors", + "tracing-error", +] + +[[package]] +name = "color-spantrace" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8b88ea9df13354b55bc7234ebcce36e6ef896aca2e42a15de9e10edce01b427" +dependencies = [ + "once_cell", + "owo-colors", + "tracing-core", + "tracing-error", +] + [[package]] name = "colorchoice" version = "1.0.4" @@ -694,13 +1444,56 @@ checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" [[package]] name = "comfy-table" -version = "7.1.2" +version = "7.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0d05af1e006a2407bedef5af410552494ce5be9090444dbbcb57258c1af3d56" +checksum = "958c5d6ecf1f214b4c2bbbbf6ab9523a864bd136dcf71a7e8904799acfe1ad47" dependencies = [ - "strum", - "strum_macros", - "unicode-width 0.2.1", + "unicode-segmentation", + "unicode-width 0.2.2", +] + +[[package]] +name = "compact_str" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdb1325a1cece981e8a296ab8f0f9b63ae357bd0784a9faaf548cc7b480707a" +dependencies = [ + "castaway", + "cfg-if", + "itoa", + "rustversion", + "ryu", + "static_assertions", +] + +[[package]] +name = "compression-codecs" +version = "0.4.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00828ba6fd27b45a448e57dbfe84f1029d4c9f26b368157e9a448a5f49a2ec2a" +dependencies = [ + "bzip2", + "compression-core", + "flate2", + "liblzma", + "memchr", + "zstd", + "zstd-safe", +] + +[[package]] +name = "compression-core" +version = "0.4.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75984efb6ed102a0d42db99afb6c1948f0380d1d91808d5529916e6c08b49d8d" + +[[package]] +name = "concurrent-queue" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" +dependencies = [ + "crossbeam-utils", ] [[package]] @@ -730,7 +1523,7 @@ version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" dependencies = [ - "getrandom 0.2.16", + "getrandom 0.2.17", "once_cell", "tiny-keccak", ] @@ -741,6 +1534,41 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" +[[package]] +name = "constant_time_eq" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" + +[[package]] +name = "convert_case" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "633458d4ef8c78b72454de2d54fd6ab2e60f9e02be22f3c6104cdc8a4e0fceb9" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation-sys" version = "0.8.7" @@ -765,12 +1593,112 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "criterion" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" +dependencies = [ + "anes", + "cast", + "ciborium", + "clap 4.5.56", + "criterion-plot", + "is-terminal", + "itertools 0.10.5", + "num-traits", + "once_cell", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_derive", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" +dependencies = [ + "cast", + "itertools 0.10.5", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-queue" +version = "0.3.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-utils" version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" +[[package]] +name = "crossterm" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8b9f2e4c67f833b660cdb0a3523065869fb35570177239812ed4c905aeff87b" +dependencies = [ + "bitflags 2.10.0", + "crossterm_winapi", + "derive_more", + "document-features", + "mio", + "parking_lot", + "rustix", + "signal-hook", + "signal-hook-mio", + "winapi", +] + +[[package]] +name = "crossterm_winapi" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acdd7c62a3665c7f6830a51635d9ac9b23ed385797f70a83bb8bafe9c572ab2b" +dependencies = [ + "winapi", +] + [[package]] name = "crunchy" version = "0.2.4" @@ -778,34 +1706,78 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" [[package]] -name = "crypto-common" -version = "0.1.6" +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "csscolorparser" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb2a7d3066da2de787b7f032c736763eb7ae5d355f81a68bab2675a96008b0bf" +dependencies = [ + "lab", + "phf 0.11.3", +] + +[[package]] +name = "csv" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52cd9d68cf7efc6ddfaaee42e7288d3a99d613d4b50f76ce9827ae0c6e14f938" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde_core", +] + +[[package]] +name = "csv-core" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704a3c26996a80471189265814dbc2c257598b96b8a7feae2d31ace646bb9782" +dependencies = [ + "memchr", +] + +[[package]] +name = "darling" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" dependencies = [ - "generic-array", - "typenum", + "darling_core", + "darling_macro", ] [[package]] -name = "csv" -version = "1.3.1" +name = "darling_core" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acdc4883a9c96732e4733212c01447ebd805833b7275a73ca3ee080fd77afdaf" +checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0" dependencies = [ - "csv-core", - "itoa", - "ryu", - "serde", + "ident_case", + "proc-macro2", + "quote", + "strsim 0.11.1", + "syn 2.0.114", ] [[package]] -name = "csv-core" -version = "0.1.12" +name = "darling_macro" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d02f3b0da4c6504f86e9cd789d8dbafab48c2321be74e9987593de5a894d93d" +checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" dependencies = [ - "memchr", + "darling_core", + "quote", + "syn 2.0.114", ] [[package]] @@ -824,44 +1796,97 @@ dependencies = [ [[package]] name = "datafusion" -version = "50.0.0" +version = "52.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=15d82292e899bc06adbbdbfacea67070bc89fb95#15d82292e899bc06adbbdbfacea67070bc89fb95" +dependencies = [ + "arrow", + "arrow-schema", + "async-trait", + "bytes", + "chrono", + "datafusion-catalog 52.0.0", + "datafusion-catalog-listing 52.0.0", + "datafusion-common 52.0.0", + "datafusion-common-runtime 52.0.0", + "datafusion-datasource 52.0.0", + "datafusion-datasource-arrow 52.0.0", + "datafusion-datasource-avro 52.0.0", + "datafusion-datasource-csv 52.0.0", + "datafusion-datasource-json 52.0.0", + "datafusion-datasource-parquet 52.0.0", + "datafusion-execution 52.0.0", + "datafusion-expr 52.0.0", + "datafusion-expr-common 52.0.0", + "datafusion-functions 52.0.0", + "datafusion-functions-aggregate 52.0.0", + "datafusion-functions-nested 52.0.0", + "datafusion-functions-table 52.0.0", + "datafusion-functions-window 52.0.0", + "datafusion-optimizer 52.0.0", + "datafusion-physical-expr 52.0.0", + "datafusion-physical-expr-adapter 52.0.0", + "datafusion-physical-expr-common 52.0.0", + "datafusion-physical-optimizer 52.0.0", + "datafusion-physical-plan 52.0.0", + "datafusion-session 52.0.0", + "datafusion-sql 52.0.0", + "futures", + "itertools 0.14.0", + "log", + "object_store", + "parking_lot", + "parquet", + "rand 0.9.2", + "regex", + "sqlparser", + "tempfile", + "tokio", + "url", + "uuid", +] + +[[package]] +name = "datafusion" +version = "52.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "481d0c1cad7606cee11233abcdff8eec46e43dd25abda007db6d5d26ae8483c4" +checksum = "d12ee9fdc6cdb5898c7691bb994f0ba606c4acc93a2258d78bb9f26ff8158bb3" dependencies = [ "arrow", - "arrow-ipc", "arrow-schema", "async-trait", "bytes", - "bzip2 0.6.0", + "bzip2", "chrono", - "datafusion-catalog", - "datafusion-catalog-listing", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", - "datafusion-datasource-csv", - "datafusion-datasource-json", - "datafusion-datasource-parquet", - "datafusion-execution", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-functions", - "datafusion-functions-aggregate", - "datafusion-functions-nested", - "datafusion-functions-table", - "datafusion-functions-window", - "datafusion-optimizer", - "datafusion-physical-expr", - "datafusion-physical-expr-adapter", - "datafusion-physical-expr-common", - "datafusion-physical-optimizer", - "datafusion-physical-plan", - "datafusion-session", - "datafusion-sql", + "datafusion-catalog 52.1.0", + "datafusion-catalog-listing 52.1.0", + "datafusion-common 52.1.0", + "datafusion-common-runtime 52.1.0", + "datafusion-datasource 52.1.0", + "datafusion-datasource-arrow 52.1.0", + "datafusion-datasource-avro 52.1.0", + "datafusion-datasource-csv 52.1.0", + "datafusion-datasource-json 52.1.0", + "datafusion-datasource-parquet 52.1.0", + "datafusion-execution 52.1.0", + "datafusion-expr 52.1.0", + "datafusion-expr-common 52.1.0", + "datafusion-functions 52.1.0", + "datafusion-functions-aggregate 52.1.0", + "datafusion-functions-nested 52.1.0", + "datafusion-functions-table 52.1.0", + "datafusion-functions-window 52.1.0", + "datafusion-optimizer 52.1.0", + "datafusion-physical-expr 52.1.0", + "datafusion-physical-expr-adapter 52.1.0", + "datafusion-physical-expr-common 52.1.0", + "datafusion-physical-optimizer 52.1.0", + "datafusion-physical-plan 52.1.0", + "datafusion-session 52.1.0", + "datafusion-sql 52.1.0", "flate2", "futures", - "itertools", + "itertools 0.14.0", + "liblzma", "log", "object_store", "parking_lot", @@ -873,30 +1898,52 @@ dependencies = [ "tokio", "url", "uuid", - "xz2", "zstd", ] [[package]] name = "datafusion-catalog" -version = "50.0.0" +version = "52.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=15d82292e899bc06adbbdbfacea67070bc89fb95#15d82292e899bc06adbbdbfacea67070bc89fb95" +dependencies = [ + "arrow", + "async-trait", + "dashmap", + "datafusion-common 52.0.0", + "datafusion-common-runtime 52.0.0", + "datafusion-datasource 52.0.0", + "datafusion-execution 52.0.0", + "datafusion-expr 52.0.0", + "datafusion-physical-expr 52.0.0", + "datafusion-physical-plan 52.0.0", + "datafusion-session 52.0.0", + "futures", + "itertools 0.14.0", + "log", + "object_store", + "parking_lot", + "tokio", +] + +[[package]] +name = "datafusion-catalog" +version = "52.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d70327e81ab3a1f5832d8b372d55fa607851d7cea6d1f8e65ff0c98fcc32d222" +checksum = "462dc9ef45e5d688aeaae49a7e310587e81b6016b9d03bace5626ad0043e5a9e" dependencies = [ "arrow", "async-trait", "dashmap", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr", - "datafusion-physical-plan", - "datafusion-session", - "datafusion-sql", + "datafusion-common 52.1.0", + "datafusion-common-runtime 52.1.0", + "datafusion-datasource 52.1.0", + "datafusion-execution 52.1.0", + "datafusion-expr 52.1.0", + "datafusion-physical-expr 52.1.0", + "datafusion-physical-plan 52.1.0", + "datafusion-session 52.1.0", "futures", - "itertools", + "itertools 0.14.0", "log", "object_store", "parking_lot", @@ -905,40 +1952,114 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" -version = "50.0.0" +version = "52.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=15d82292e899bc06adbbdbfacea67070bc89fb95#15d82292e899bc06adbbdbfacea67070bc89fb95" +dependencies = [ + "arrow", + "async-trait", + "datafusion-catalog 52.0.0", + "datafusion-common 52.0.0", + "datafusion-datasource 52.0.0", + "datafusion-execution 52.0.0", + "datafusion-expr 52.0.0", + "datafusion-physical-expr 52.0.0", + "datafusion-physical-expr-adapter 52.0.0", + "datafusion-physical-expr-common 52.0.0", + "datafusion-physical-plan 52.0.0", + "futures", + "itertools 0.14.0", + "log", + "object_store", +] + +[[package]] +name = "datafusion-catalog-listing" +version = "52.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b96dbf1d728fc321817b744eb5080cdd75312faa6980b338817f68f3caa4208" +dependencies = [ + "arrow", + "async-trait", + "datafusion-catalog 52.1.0", + "datafusion-common 52.1.0", + "datafusion-datasource 52.1.0", + "datafusion-execution 52.1.0", + "datafusion-expr 52.1.0", + "datafusion-physical-expr 52.1.0", + "datafusion-physical-expr-adapter 52.1.0", + "datafusion-physical-expr-common 52.1.0", + "datafusion-physical-plan 52.1.0", + "futures", + "itertools 0.14.0", + "log", + "object_store", +] + +[[package]] +name = "datafusion-cli" +version = "52.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "268819e6bb20ba70a664abddc20deac604f30d3267f8c91847064542a8c0720c" +checksum = "e765fd9f2ce5b451861a7858987f47ce11bb963dff7785f667e68dcea212c3de" dependencies = [ "arrow", "async-trait", - "datafusion-catalog", - "datafusion-common", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-session", + "aws-config", + "aws-credential-types", + "chrono", + "clap 4.5.56", + "datafusion 52.1.0", + "datafusion-common 52.1.0", + "dirs", + "env_logger", "futures", "log", + "mimalloc", + "object_store", + "parking_lot", + "parquet", + "regex", + "rustyline", + "tokio", + "url", +] + +[[package]] +name = "datafusion-common" +version = "52.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=15d82292e899bc06adbbdbfacea67070bc89fb95#15d82292e899bc06adbbdbfacea67070bc89fb95" +dependencies = [ + "ahash", + "apache-avro", + "arrow", + "arrow-ipc", + "chrono", + "half", + "hashbrown 0.16.1", + "indexmap", + "libc", + "log", "object_store", + "parquet", + "paste", + "sqlparser", "tokio", + "web-time", ] [[package]] name = "datafusion-common" -version = "50.0.0" +version = "52.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "054873d5563f115f83ef4270b560ac2ce4de713905e825a40cac49d6ff348254" +checksum = "3237a6ff0d2149af4631290074289cae548c9863c885d821315d54c6673a074a" dependencies = [ "ahash", + "apache-avro", "arrow", "arrow-ipc", - "base64", "chrono", "half", - "hashbrown 0.14.5", + "hashbrown 0.16.1", + "hex", "indexmap", "libc", "log", @@ -953,71 +2074,213 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" -version = "50.0.0" +version = "52.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=15d82292e899bc06adbbdbfacea67070bc89fb95#15d82292e899bc06adbbdbfacea67070bc89fb95" +dependencies = [ + "futures", + "log", + "tokio", +] + +[[package]] +name = "datafusion-common-runtime" +version = "52.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8a1d1bc69aaaadb8008b65329ed890b33e845dc063225c190f77b20328fbe1d" +checksum = "70b5e34026af55a1bfccb1ef0a763cf1f64e77c696ffcf5a128a278c31236528" +dependencies = [ + "futures", + "log", + "tokio", +] + +[[package]] +name = "datafusion-datasource" +version = "52.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=15d82292e899bc06adbbdbfacea67070bc89fb95#15d82292e899bc06adbbdbfacea67070bc89fb95" dependencies = [ + "arrow", + "async-trait", + "bytes", + "chrono", + "datafusion-common 52.0.0", + "datafusion-common-runtime 52.0.0", + "datafusion-execution 52.0.0", + "datafusion-expr 52.0.0", + "datafusion-physical-expr 52.0.0", + "datafusion-physical-expr-adapter 52.0.0", + "datafusion-physical-expr-common 52.0.0", + "datafusion-physical-plan 52.0.0", + "datafusion-session 52.0.0", "futures", + "glob", + "itertools 0.14.0", "log", + "object_store", + "rand 0.9.2", "tokio", + "url", ] [[package]] name = "datafusion-datasource" -version = "50.0.0" +version = "52.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d855160469020982880fd9bd0962e033d2f4728f56f85a83d8c90785638b6519" +checksum = "1b2a6be734cc3785e18bbf2a7f2b22537f6b9fb960d79617775a51568c281842" dependencies = [ "arrow", "async-compression", "async-trait", "bytes", - "bzip2 0.6.0", + "bzip2", "chrono", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr", - "datafusion-physical-expr-adapter", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-session", + "datafusion-common 52.1.0", + "datafusion-common-runtime 52.1.0", + "datafusion-execution 52.1.0", + "datafusion-expr 52.1.0", + "datafusion-physical-expr 52.1.0", + "datafusion-physical-expr-adapter 52.1.0", + "datafusion-physical-expr-common 52.1.0", + "datafusion-physical-plan 52.1.0", + "datafusion-session 52.1.0", "flate2", "futures", "glob", - "itertools", + "itertools 0.14.0", + "liblzma", "log", "object_store", - "parquet", "rand 0.9.2", - "tempfile", "tokio", "tokio-util", "url", - "xz2", "zstd", ] +[[package]] +name = "datafusion-datasource-arrow" +version = "52.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=15d82292e899bc06adbbdbfacea67070bc89fb95#15d82292e899bc06adbbdbfacea67070bc89fb95" +dependencies = [ + "arrow", + "arrow-ipc", + "async-trait", + "bytes", + "datafusion-common 52.0.0", + "datafusion-common-runtime 52.0.0", + "datafusion-datasource 52.0.0", + "datafusion-execution 52.0.0", + "datafusion-expr 52.0.0", + "datafusion-physical-expr-common 52.0.0", + "datafusion-physical-plan 52.0.0", + "datafusion-session 52.0.0", + "futures", + "itertools 0.14.0", + "object_store", + "tokio", +] + +[[package]] +name = "datafusion-datasource-arrow" +version = "52.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1739b9b07c9236389e09c74f770e88aff7055250774e9def7d3f4f56b3dcc7be" +dependencies = [ + "arrow", + "arrow-ipc", + "async-trait", + "bytes", + "datafusion-common 52.1.0", + "datafusion-common-runtime 52.1.0", + "datafusion-datasource 52.1.0", + "datafusion-execution 52.1.0", + "datafusion-expr 52.1.0", + "datafusion-physical-expr-common 52.1.0", + "datafusion-physical-plan 52.1.0", + "datafusion-session 52.1.0", + "futures", + "itertools 0.14.0", + "object_store", + "tokio", +] + +[[package]] +name = "datafusion-datasource-avro" +version = "52.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=15d82292e899bc06adbbdbfacea67070bc89fb95#15d82292e899bc06adbbdbfacea67070bc89fb95" +dependencies = [ + "apache-avro", + "arrow", + "async-trait", + "bytes", + "datafusion-common 52.0.0", + "datafusion-datasource 52.0.0", + "datafusion-physical-expr-common 52.0.0", + "datafusion-physical-plan 52.0.0", + "datafusion-session 52.0.0", + "futures", + "num-traits", + "object_store", +] + +[[package]] +name = "datafusion-datasource-avro" +version = "52.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "828088c2fb681cc0e06fb42f541f76c82a0c10278f9fd6334e22c8d1e3574ee7" +dependencies = [ + "apache-avro", + "arrow", + "async-trait", + "bytes", + "datafusion-common 52.1.0", + "datafusion-datasource 52.1.0", + "datafusion-physical-expr-common 52.1.0", + "datafusion-physical-plan 52.1.0", + "datafusion-session 52.1.0", + "futures", + "num-traits", + "object_store", +] + +[[package]] +name = "datafusion-datasource-csv" +version = "52.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=15d82292e899bc06adbbdbfacea67070bc89fb95#15d82292e899bc06adbbdbfacea67070bc89fb95" +dependencies = [ + "arrow", + "async-trait", + "bytes", + "datafusion-common 52.0.0", + "datafusion-common-runtime 52.0.0", + "datafusion-datasource 52.0.0", + "datafusion-execution 52.0.0", + "datafusion-expr 52.0.0", + "datafusion-physical-expr-common 52.0.0", + "datafusion-physical-plan 52.0.0", + "datafusion-session 52.0.0", + "futures", + "object_store", + "regex", + "tokio", +] + [[package]] name = "datafusion-datasource-csv" -version = "50.0.0" +version = "52.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ec3aa7575378d23aae96b955b5233bea6f9d461648174f6ccc8f3c160f2b7a7" +checksum = "61c73bc54b518bbba7c7650299d07d58730293cfba4356f6f428cc94c20b7600" dependencies = [ "arrow", "async-trait", "bytes", - "datafusion-catalog", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-session", + "datafusion-common 52.1.0", + "datafusion-common-runtime 52.1.0", + "datafusion-datasource 52.1.0", + "datafusion-execution 52.1.0", + "datafusion-expr 52.1.0", + "datafusion-physical-expr-common 52.1.0", + "datafusion-physical-plan 52.1.0", + "datafusion-session 52.1.0", "futures", "object_store", "regex", @@ -1026,59 +2289,103 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" -version = "50.0.0" +version = "52.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=15d82292e899bc06adbbdbfacea67070bc89fb95#15d82292e899bc06adbbdbfacea67070bc89fb95" +dependencies = [ + "arrow", + "async-trait", + "bytes", + "datafusion-common 52.0.0", + "datafusion-common-runtime 52.0.0", + "datafusion-datasource 52.0.0", + "datafusion-execution 52.0.0", + "datafusion-expr 52.0.0", + "datafusion-physical-expr-common 52.0.0", + "datafusion-physical-plan 52.0.0", + "datafusion-session 52.0.0", + "futures", + "object_store", + "tokio", +] + +[[package]] +name = "datafusion-datasource-json" +version = "52.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00cfb8f33e2864eeb3188b6818acf5546d56a5a487d423cce9b684a554caabfa" +checksum = "37812c8494c698c4d889374ecfabbff780f1f26d9ec095dd1bddfc2a8ca12559" dependencies = [ "arrow", "async-trait", "bytes", - "datafusion-catalog", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-session", + "datafusion-common 52.1.0", + "datafusion-common-runtime 52.1.0", + "datafusion-datasource 52.1.0", + "datafusion-execution 52.1.0", + "datafusion-expr 52.1.0", + "datafusion-physical-expr-common 52.1.0", + "datafusion-physical-plan 52.1.0", + "datafusion-session 52.1.0", "futures", "object_store", - "serde_json", "tokio", ] [[package]] name = "datafusion-datasource-parquet" -version = "50.0.0" +version = "52.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=15d82292e899bc06adbbdbfacea67070bc89fb95#15d82292e899bc06adbbdbfacea67070bc89fb95" +dependencies = [ + "arrow", + "async-trait", + "bytes", + "datafusion-common 52.0.0", + "datafusion-common-runtime 52.0.0", + "datafusion-datasource 52.0.0", + "datafusion-execution 52.0.0", + "datafusion-expr 52.0.0", + "datafusion-functions-aggregate-common 52.0.0", + "datafusion-physical-expr 52.0.0", + "datafusion-physical-expr-adapter 52.0.0", + "datafusion-physical-expr-common 52.0.0", + "datafusion-physical-plan 52.0.0", + "datafusion-pruning 52.0.0", + "datafusion-session 52.0.0", + "futures", + "itertools 0.14.0", + "log", + "object_store", + "parking_lot", + "parquet", + "tokio", +] + +[[package]] +name = "datafusion-datasource-parquet" +version = "52.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab3bfb48fb4ff42ac1485a12ea56434eaab53f7da8f00b2443b1a3d35a0b6d10" +checksum = "2210937ecd9f0e824c397e73f4b5385c97cd1aff43ab2b5836fcfd2d321523fb" dependencies = [ "arrow", "async-trait", "bytes", - "datafusion-catalog", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions-aggregate", - "datafusion-physical-expr", - "datafusion-physical-expr-adapter", - "datafusion-physical-expr-common", - "datafusion-physical-optimizer", - "datafusion-physical-plan", - "datafusion-pruning", - "datafusion-session", + "datafusion-common 52.1.0", + "datafusion-common-runtime 52.1.0", + "datafusion-datasource 52.1.0", + "datafusion-execution 52.1.0", + "datafusion-expr 52.1.0", + "datafusion-functions-aggregate-common 52.1.0", + "datafusion-physical-expr 52.1.0", + "datafusion-physical-expr-adapter 52.1.0", + "datafusion-physical-expr-common 52.1.0", + "datafusion-physical-plan 52.1.0", + "datafusion-pruning 52.1.0", + "datafusion-session 52.1.0", "futures", - "itertools", + "itertools 0.14.0", "log", "object_store", "parking_lot", "parquet", - "rand 0.9.2", "tokio", ] @@ -1088,76 +2395,161 @@ version = "0.1.0" dependencies = [ "arrow", "arrow-flight", + "arrow-ipc", + "arrow-select", "async-trait", "bytes", "chrono", + "crossbeam-queue", "dashmap", - "datafusion", + "datafusion 52.0.0", "datafusion-proto", "delegate", "futures", - "http", + "http 1.4.0", "hyper-util", "insta", - "itertools", + "itertools 0.14.0", + "moka", "object_store", "parquet", "pin-project", + "pretty_assertions", "prost", - "rand 0.8.5", + "rand 0.9.2", + "reqwest", + "sketches-ddsketch", "structopt", + "test-case", "tokio", "tokio-stream", + "tokio-util", "tonic", + "tonic-prost", "tower", "tpchgen", "tpchgen-arrow", "url", "uuid", + "zip", ] [[package]] name = "datafusion-distributed-benchmarks" version = "0.1.0" dependencies = [ + "arrow-flight", "async-trait", + "aws-config", + "aws-sdk-ec2", + "axum 0.7.9", + "built", "chrono", + "clap 4.5.56", + "criterion", "dashmap", - "datafusion", + "datafusion 52.0.0", "datafusion-distributed", "datafusion-proto", "env_logger", "futures", "log", + "mimalloc", + "object_store", + "openssl", "parquet", "prost", "serde", "serde_json", "structopt", + "sysinfo", + "tokio", + "tonic", + "url", +] + +[[package]] +name = "datafusion-distributed-cli" +version = "0.1.0" +dependencies = [ + "arrow-flight", + "async-trait", + "clap 4.5.56", + "datafusion 52.0.0", + "datafusion-cli", + "datafusion-distributed", + "dirs", + "env_logger", + "hyper-util", + "tokio", + "tokio-stream", + "tonic", + "tower", + "url", +] + +[[package]] +name = "datafusion-distributed-console" +version = "0.1.0" +dependencies = [ + "arrow-flight", + "color-eyre", + "crossterm", + "datafusion-distributed", + "ratatui", + "structopt", "tokio", + "tonic", +] + +[[package]] +name = "datafusion-doc" +version = "52.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=15d82292e899bc06adbbdbfacea67070bc89fb95#15d82292e899bc06adbbdbfacea67070bc89fb95" + +[[package]] +name = "datafusion-doc" +version = "52.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c825f969126bc2ef6a6a02d94b3c07abff871acf4d6dd759ce1255edb7923ce" + +[[package]] +name = "datafusion-execution" +version = "52.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=15d82292e899bc06adbbdbfacea67070bc89fb95#15d82292e899bc06adbbdbfacea67070bc89fb95" +dependencies = [ + "arrow", + "async-trait", + "chrono", + "dashmap", + "datafusion-common 52.0.0", + "datafusion-expr 52.0.0", + "futures", + "log", + "object_store", + "parking_lot", + "rand 0.9.2", + "tempfile", + "url", ] -[[package]] -name = "datafusion-doc" -version = "50.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fbf41013cf55c2369b5229594898e8108c8a1beeb49d97feb5e0cce9933eb8f" - [[package]] name = "datafusion-execution" -version = "50.0.0" +version = "52.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26fd0c1ffe3885687758f985ed548184bf63b17b2a7a5ae695de422ad6432118" +checksum = "fa03ef05a2c2f90dd6c743e3e111078e322f4b395d20d4b4d431a245d79521ae" dependencies = [ "arrow", "async-trait", + "chrono", "dashmap", - "datafusion-common", - "datafusion-expr", + "datafusion-common 52.1.0", + "datafusion-expr 52.1.0", "futures", "log", "object_store", "parking_lot", + "parquet", "rand 0.9.2", "tempfile", "url", @@ -1165,20 +2557,42 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "50.0.0" +version = "52.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=15d82292e899bc06adbbdbfacea67070bc89fb95#15d82292e899bc06adbbdbfacea67070bc89fb95" +dependencies = [ + "arrow", + "async-trait", + "chrono", + "datafusion-common 52.0.0", + "datafusion-doc 52.0.0", + "datafusion-expr-common 52.0.0", + "datafusion-functions-aggregate-common 52.0.0", + "datafusion-functions-window-common 52.0.0", + "datafusion-physical-expr-common 52.0.0", + "indexmap", + "itertools 0.14.0", + "paste", + "serde_json", + "sqlparser", +] + +[[package]] +name = "datafusion-expr" +version = "52.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c4fe6411218a9dab656437b1e69b00a470a7a2d7db087867a366c145eb164a7" +checksum = "ef33934c1f98ee695cc51192cc5f9ed3a8febee84fdbcd9131bf9d3a9a78276f" dependencies = [ "arrow", "async-trait", "chrono", - "datafusion-common", - "datafusion-doc", - "datafusion-expr-common", - "datafusion-functions-aggregate-common", - "datafusion-functions-window-common", - "datafusion-physical-expr-common", + "datafusion-common 52.1.0", + "datafusion-doc 52.1.0", + "datafusion-expr-common 52.1.0", + "datafusion-functions-aggregate-common 52.1.0", + "datafusion-functions-window-common 52.1.0", + "datafusion-physical-expr-common 52.1.0", "indexmap", + "itertools 0.14.0", "paste", "recursive", "serde_json", @@ -1187,22 +2601,60 @@ dependencies = [ [[package]] name = "datafusion-expr-common" -version = "50.0.0" +version = "52.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=15d82292e899bc06adbbdbfacea67070bc89fb95#15d82292e899bc06adbbdbfacea67070bc89fb95" +dependencies = [ + "arrow", + "datafusion-common 52.0.0", + "indexmap", + "itertools 0.14.0", + "paste", +] + +[[package]] +name = "datafusion-expr-common" +version = "52.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a45bee7d2606bfb41ceb1d904ba7cecf69bd5a6f8f3e6c57c3f5a83d84bdd97" +checksum = "000c98206e3dd47d2939a94b6c67af4bfa6732dd668ac4fafdbde408fd9134ea" dependencies = [ "arrow", - "datafusion-common", + "datafusion-common 52.1.0", "indexmap", - "itertools", + "itertools 0.14.0", "paste", ] [[package]] name = "datafusion-functions" -version = "50.0.0" +version = "52.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=15d82292e899bc06adbbdbfacea67070bc89fb95#15d82292e899bc06adbbdbfacea67070bc89fb95" +dependencies = [ + "arrow", + "arrow-buffer", + "base64", + "chrono", + "chrono-tz", + "datafusion-common 52.0.0", + "datafusion-doc 52.0.0", + "datafusion-execution 52.0.0", + "datafusion-expr 52.0.0", + "datafusion-expr-common 52.0.0", + "datafusion-macros 52.0.0", + "hex", + "itertools 0.14.0", + "log", + "num-traits", + "rand 0.9.2", + "regex", + "unicode-segmentation", + "uuid", +] + +[[package]] +name = "datafusion-functions" +version = "52.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c7e1c532ff9d14f291160bca23e55ffd4899800301dd2389786c2f02d76904a" +checksum = "379b01418ab95ca947014066248c22139fe9af9289354de10b445bd000d5d276" dependencies = [ "arrow", "arrow-buffer", @@ -1210,16 +2662,18 @@ dependencies = [ "blake2", "blake3", "chrono", - "datafusion-common", - "datafusion-doc", - "datafusion-execution", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-macros", + "chrono-tz", + "datafusion-common 52.1.0", + "datafusion-doc 52.1.0", + "datafusion-execution 52.1.0", + "datafusion-expr 52.1.0", + "datafusion-expr-common 52.1.0", + "datafusion-macros 52.1.0", "hex", - "itertools", + "itertools 0.14.0", "log", "md-5", + "num-traits", "rand 0.9.2", "regex", "sha2", @@ -1229,20 +2683,40 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" -version = "50.0.0" +version = "52.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=15d82292e899bc06adbbdbfacea67070bc89fb95#15d82292e899bc06adbbdbfacea67070bc89fb95" +dependencies = [ + "ahash", + "arrow", + "datafusion-common 52.0.0", + "datafusion-doc 52.0.0", + "datafusion-execution 52.0.0", + "datafusion-expr 52.0.0", + "datafusion-functions-aggregate-common 52.0.0", + "datafusion-macros 52.0.0", + "datafusion-physical-expr 52.0.0", + "datafusion-physical-expr-common 52.0.0", + "half", + "log", + "paste", +] + +[[package]] +name = "datafusion-functions-aggregate" +version = "52.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b05d47426645aef1e73b1a034c75ab2401bc504175feb191accbe211ec24a342" +checksum = "fd00d5454ba4c3f8ebbd04bd6a6a9dc7ced7c56d883f70f2076c188be8459e4c" dependencies = [ "ahash", "arrow", - "datafusion-common", - "datafusion-doc", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions-aggregate-common", - "datafusion-macros", - "datafusion-physical-expr", - "datafusion-physical-expr-common", + "datafusion-common 52.1.0", + "datafusion-doc 52.1.0", + "datafusion-execution 52.1.0", + "datafusion-expr 52.1.0", + "datafusion-functions-aggregate-common 52.1.0", + "datafusion-macros 52.1.0", + "datafusion-physical-expr 52.1.0", + "datafusion-physical-expr-common 52.1.0", "half", "log", "paste", @@ -1250,108 +2724,212 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" -version = "50.0.0" +version = "52.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=15d82292e899bc06adbbdbfacea67070bc89fb95#15d82292e899bc06adbbdbfacea67070bc89fb95" +dependencies = [ + "ahash", + "arrow", + "datafusion-common 52.0.0", + "datafusion-expr-common 52.0.0", + "datafusion-physical-expr-common 52.0.0", +] + +[[package]] +name = "datafusion-functions-aggregate-common" +version = "52.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05c99f648b2b1743de0c1c19eef07e8cc5a085237f172b2e20bf6934e0a804e4" +checksum = "aec06b380729a87210a4e11f555ec2d729a328142253f8d557b87593622ecc9f" dependencies = [ "ahash", "arrow", - "datafusion-common", - "datafusion-expr-common", - "datafusion-physical-expr-common", + "datafusion-common 52.1.0", + "datafusion-expr-common 52.1.0", + "datafusion-physical-expr-common 52.1.0", +] + +[[package]] +name = "datafusion-functions-nested" +version = "52.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=15d82292e899bc06adbbdbfacea67070bc89fb95#15d82292e899bc06adbbdbfacea67070bc89fb95" +dependencies = [ + "arrow", + "arrow-ord", + "datafusion-common 52.0.0", + "datafusion-doc 52.0.0", + "datafusion-execution 52.0.0", + "datafusion-expr 52.0.0", + "datafusion-expr-common 52.0.0", + "datafusion-functions 52.0.0", + "datafusion-functions-aggregate 52.0.0", + "datafusion-functions-aggregate-common 52.0.0", + "datafusion-macros 52.0.0", + "datafusion-physical-expr-common 52.0.0", + "itertools 0.14.0", + "log", + "paste", ] [[package]] name = "datafusion-functions-nested" -version = "50.0.0" +version = "52.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4227782023f4fb68d3d5c5eb190665212f43c9a0b437553e4b938b379aff6cf6" +checksum = "904f48d45e0f1eb7d0eb5c0f80f2b5c6046a85454364a6b16a2e0b46f62e7dff" dependencies = [ "arrow", "arrow-ord", - "datafusion-common", - "datafusion-doc", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions", - "datafusion-functions-aggregate", - "datafusion-functions-aggregate-common", - "datafusion-macros", - "datafusion-physical-expr-common", - "itertools", + "datafusion-common 52.1.0", + "datafusion-doc 52.1.0", + "datafusion-execution 52.1.0", + "datafusion-expr 52.1.0", + "datafusion-expr-common 52.1.0", + "datafusion-functions 52.1.0", + "datafusion-functions-aggregate 52.1.0", + "datafusion-functions-aggregate-common 52.1.0", + "datafusion-macros 52.1.0", + "datafusion-physical-expr-common 52.1.0", + "itertools 0.14.0", "log", "paste", ] [[package]] name = "datafusion-functions-table" -version = "50.0.0" +version = "52.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=15d82292e899bc06adbbdbfacea67070bc89fb95#15d82292e899bc06adbbdbfacea67070bc89fb95" +dependencies = [ + "arrow", + "async-trait", + "datafusion-catalog 52.0.0", + "datafusion-common 52.0.0", + "datafusion-expr 52.0.0", + "datafusion-physical-plan 52.0.0", + "parking_lot", + "paste", +] + +[[package]] +name = "datafusion-functions-table" +version = "52.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d902b1769f69058236e89f04f3bff2cf62f24311adb7bf3c6c3e945c9451076" +checksum = "e9a0d20e2b887e11bee24f7734d780a2588b925796ac741c3118dd06d5aa77f0" dependencies = [ "arrow", "async-trait", - "datafusion-catalog", - "datafusion-common", - "datafusion-expr", - "datafusion-physical-plan", + "datafusion-catalog 52.1.0", + "datafusion-common 52.1.0", + "datafusion-expr 52.1.0", + "datafusion-physical-plan 52.1.0", "parking_lot", "paste", ] [[package]] name = "datafusion-functions-window" -version = "50.0.0" +version = "52.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=15d82292e899bc06adbbdbfacea67070bc89fb95#15d82292e899bc06adbbdbfacea67070bc89fb95" +dependencies = [ + "arrow", + "datafusion-common 52.0.0", + "datafusion-doc 52.0.0", + "datafusion-expr 52.0.0", + "datafusion-functions-window-common 52.0.0", + "datafusion-macros 52.0.0", + "datafusion-physical-expr 52.0.0", + "datafusion-physical-expr-common 52.0.0", + "log", + "paste", +] + +[[package]] +name = "datafusion-functions-window" +version = "52.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b8ee43974c92eb9920fe8e97e0fab48675e93b062abcb48bef4c1d4305b6ee4" +checksum = "d3414b0a07e39b6979fe3a69c7aa79a9f1369f1d5c8e52146e66058be1b285ee" dependencies = [ "arrow", - "datafusion-common", - "datafusion-doc", - "datafusion-expr", - "datafusion-functions-window-common", - "datafusion-macros", - "datafusion-physical-expr", - "datafusion-physical-expr-common", + "datafusion-common 52.1.0", + "datafusion-doc 52.1.0", + "datafusion-expr 52.1.0", + "datafusion-functions-window-common 52.1.0", + "datafusion-macros 52.1.0", + "datafusion-physical-expr 52.1.0", + "datafusion-physical-expr-common 52.1.0", "log", "paste", ] [[package]] name = "datafusion-functions-window-common" -version = "50.0.0" +version = "52.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=15d82292e899bc06adbbdbfacea67070bc89fb95#15d82292e899bc06adbbdbfacea67070bc89fb95" +dependencies = [ + "datafusion-common 52.0.0", + "datafusion-physical-expr-common 52.0.0", +] + +[[package]] +name = "datafusion-functions-window-common" +version = "52.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1e149d36cdd44fb425dc815c5fac55025aa9a592dd65cb3c421881096292c02" +checksum = "5bf2feae63cd4754e31add64ce75cae07d015bce4bb41cd09872f93add32523a" +dependencies = [ + "datafusion-common 52.1.0", + "datafusion-physical-expr-common 52.1.0", +] + +[[package]] +name = "datafusion-macros" +version = "52.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=15d82292e899bc06adbbdbfacea67070bc89fb95#15d82292e899bc06adbbdbfacea67070bc89fb95" dependencies = [ - "datafusion-common", - "datafusion-physical-expr-common", + "datafusion-doc 52.0.0", + "quote", + "syn 2.0.114", ] [[package]] name = "datafusion-macros" -version = "50.0.0" +version = "52.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07c9faa0cdefb6e6e756482b846397b5c2d84d369e30b009472b9ab9b1430fbd" +checksum = "c4fe888aeb6a095c4bcbe8ac1874c4b9a4c7ffa2ba849db7922683ba20875aaf" dependencies = [ - "datafusion-expr", + "datafusion-doc 52.1.0", "quote", - "syn 2.0.106", + "syn 2.0.114", +] + +[[package]] +name = "datafusion-optimizer" +version = "52.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=15d82292e899bc06adbbdbfacea67070bc89fb95#15d82292e899bc06adbbdbfacea67070bc89fb95" +dependencies = [ + "arrow", + "chrono", + "datafusion-common 52.0.0", + "datafusion-expr 52.0.0", + "datafusion-expr-common 52.0.0", + "datafusion-physical-expr 52.0.0", + "indexmap", + "itertools 0.14.0", + "log", + "regex", + "regex-syntax", ] [[package]] name = "datafusion-optimizer" -version = "50.0.0" +version = "52.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f16a4f7059302ad1de6e97ab0eebb5c34405917b1f80806a30a66e38ad118251" +checksum = "8a6527c063ae305c11be397a86d8193936f4b84d137fe40bd706dfc178cf733c" dependencies = [ "arrow", "chrono", - "datafusion-common", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-physical-expr", + "datafusion-common 52.1.0", + "datafusion-expr 52.1.0", + "datafusion-expr-common 52.1.0", + "datafusion-physical-expr 52.1.0", "indexmap", - "itertools", + "itertools 0.14.0", "log", "recursive", "regex", @@ -1360,101 +2938,203 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "50.0.0" +version = "52.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=15d82292e899bc06adbbdbfacea67070bc89fb95#15d82292e899bc06adbbdbfacea67070bc89fb95" +dependencies = [ + "ahash", + "arrow", + "datafusion-common 52.0.0", + "datafusion-expr 52.0.0", + "datafusion-expr-common 52.0.0", + "datafusion-functions-aggregate-common 52.0.0", + "datafusion-physical-expr-common 52.0.0", + "half", + "hashbrown 0.16.1", + "indexmap", + "itertools 0.14.0", + "parking_lot", + "paste", + "petgraph", + "tokio", +] + +[[package]] +name = "datafusion-physical-expr" +version = "52.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10bb87a605d8ce9672d5347c0293c12211b0c03923fc12fbdc665fe76e6f9e01" +checksum = "0bb028323dd4efd049dd8a78d78fe81b2b969447b39c51424167f973ac5811d9" dependencies = [ "ahash", "arrow", - "datafusion-common", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-functions-aggregate-common", - "datafusion-physical-expr-common", + "datafusion-common 52.1.0", + "datafusion-expr 52.1.0", + "datafusion-expr-common 52.1.0", + "datafusion-functions-aggregate-common 52.1.0", + "datafusion-physical-expr-common 52.1.0", "half", - "hashbrown 0.14.5", + "hashbrown 0.16.1", "indexmap", - "itertools", - "log", + "itertools 0.14.0", "parking_lot", "paste", "petgraph", + "recursive", + "tokio", +] + +[[package]] +name = "datafusion-physical-expr-adapter" +version = "52.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=15d82292e899bc06adbbdbfacea67070bc89fb95#15d82292e899bc06adbbdbfacea67070bc89fb95" +dependencies = [ + "arrow", + "datafusion-common 52.0.0", + "datafusion-expr 52.0.0", + "datafusion-functions 52.0.0", + "datafusion-physical-expr 52.0.0", + "datafusion-physical-expr-common 52.0.0", + "itertools 0.14.0", ] [[package]] name = "datafusion-physical-expr-adapter" -version = "50.0.0" +version = "52.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2da3a7429a555dd5ff0bec4d24bd5532ec43876764088da635cad55b2f178dc2" +checksum = "78fe0826aef7eab6b4b61533d811234a7a9e5e458331ebbf94152a51fc8ab433" +dependencies = [ + "arrow", + "datafusion-common 52.1.0", + "datafusion-expr 52.1.0", + "datafusion-functions 52.1.0", + "datafusion-physical-expr 52.1.0", + "datafusion-physical-expr-common 52.1.0", + "itertools 0.14.0", +] + +[[package]] +name = "datafusion-physical-expr-common" +version = "52.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=15d82292e899bc06adbbdbfacea67070bc89fb95#15d82292e899bc06adbbdbfacea67070bc89fb95" dependencies = [ + "ahash", "arrow", - "datafusion-common", - "datafusion-expr", - "datafusion-functions", - "datafusion-physical-expr", - "datafusion-physical-expr-common", - "itertools", + "chrono", + "datafusion-common 52.0.0", + "datafusion-expr-common 52.0.0", + "hashbrown 0.16.1", + "indexmap", + "itertools 0.14.0", + "parking_lot", ] [[package]] name = "datafusion-physical-expr-common" -version = "50.0.0" +version = "52.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "845eb44ef1e04d2a15c6d955cb146b40a41814a7be4377f0a541857d3e257d6f" +checksum = "cfccd388620734c661bd8b7ca93c44cdd59fecc9b550eea416a78ffcbb29475f" dependencies = [ "ahash", "arrow", - "datafusion-common", - "datafusion-expr-common", - "hashbrown 0.14.5", - "itertools", + "chrono", + "datafusion-common 52.1.0", + "datafusion-expr-common 52.1.0", + "hashbrown 0.16.1", + "indexmap", + "itertools 0.14.0", + "parking_lot", +] + +[[package]] +name = "datafusion-physical-optimizer" +version = "52.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=15d82292e899bc06adbbdbfacea67070bc89fb95#15d82292e899bc06adbbdbfacea67070bc89fb95" +dependencies = [ + "arrow", + "datafusion-common 52.0.0", + "datafusion-execution 52.0.0", + "datafusion-expr 52.0.0", + "datafusion-expr-common 52.0.0", + "datafusion-physical-expr 52.0.0", + "datafusion-physical-expr-common 52.0.0", + "datafusion-physical-plan 52.0.0", + "datafusion-pruning 52.0.0", + "itertools 0.14.0", ] [[package]] name = "datafusion-physical-optimizer" -version = "50.0.0" +version = "52.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32b9b648ee2785722c79eae366528e52e93ece6808aef9297cf8e5521de381da" +checksum = "bde5fa10e73259a03b705d5fddc136516814ab5f441b939525618a4070f5a059" dependencies = [ "arrow", - "datafusion-common", - "datafusion-execution", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-physical-expr", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "datafusion-pruning", - "itertools", - "log", + "datafusion-common 52.1.0", + "datafusion-execution 52.1.0", + "datafusion-expr 52.1.0", + "datafusion-expr-common 52.1.0", + "datafusion-physical-expr 52.1.0", + "datafusion-physical-expr-common 52.1.0", + "datafusion-physical-plan 52.1.0", + "datafusion-pruning 52.1.0", + "itertools 0.14.0", "recursive", ] [[package]] name = "datafusion-physical-plan" -version = "50.0.0" +version = "52.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=15d82292e899bc06adbbdbfacea67070bc89fb95#15d82292e899bc06adbbdbfacea67070bc89fb95" +dependencies = [ + "ahash", + "arrow", + "arrow-ord", + "arrow-schema", + "async-trait", + "datafusion-common 52.0.0", + "datafusion-common-runtime 52.0.0", + "datafusion-execution 52.0.0", + "datafusion-expr 52.0.0", + "datafusion-functions 52.0.0", + "datafusion-functions-aggregate-common 52.0.0", + "datafusion-functions-window-common 52.0.0", + "datafusion-physical-expr 52.0.0", + "datafusion-physical-expr-common 52.0.0", + "futures", + "half", + "hashbrown 0.16.1", + "indexmap", + "itertools 0.14.0", + "log", + "parking_lot", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "datafusion-physical-plan" +version = "52.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e6688d17b78104e169d7069749832c20ff50f112be853d2c058afe46c889064" +checksum = "0e1098760fb29127c24cc9ade3277051dc73c9ed0ac0131bd7bcd742e0ad7470" dependencies = [ "ahash", "arrow", "arrow-ord", "arrow-schema", "async-trait", - "chrono", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions-aggregate-common", - "datafusion-functions-window-common", - "datafusion-physical-expr", - "datafusion-physical-expr-common", + "datafusion-common 52.1.0", + "datafusion-common-runtime 52.1.0", + "datafusion-execution 52.1.0", + "datafusion-expr 52.1.0", + "datafusion-functions 52.1.0", + "datafusion-functions-aggregate-common 52.1.0", + "datafusion-functions-window-common 52.1.0", + "datafusion-physical-expr 52.1.0", + "datafusion-physical-expr-common 52.1.0", "futures", "half", - "hashbrown 0.14.5", + "hashbrown 0.16.1", "indexmap", - "itertools", + "itertools 0.14.0", "log", "parking_lot", "pin-project-lite", @@ -1463,101 +3143,206 @@ dependencies = [ [[package]] name = "datafusion-proto" -version = "50.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "521cd45740788e751bf59b25d2879d162b157a45fb9b5fa2ec03034923f3658a" +version = "52.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=15d82292e899bc06adbbdbfacea67070bc89fb95#15d82292e899bc06adbbdbfacea67070bc89fb95" dependencies = [ "arrow", "chrono", - "datafusion", - "datafusion-common", - "datafusion-expr", + "datafusion-catalog 52.0.0", + "datafusion-catalog-listing 52.0.0", + "datafusion-common 52.0.0", + "datafusion-datasource 52.0.0", + "datafusion-datasource-arrow 52.0.0", + "datafusion-datasource-csv 52.0.0", + "datafusion-datasource-json 52.0.0", + "datafusion-datasource-parquet 52.0.0", + "datafusion-execution 52.0.0", + "datafusion-expr 52.0.0", + "datafusion-functions-table 52.0.0", + "datafusion-physical-expr 52.0.0", + "datafusion-physical-expr-common 52.0.0", + "datafusion-physical-plan 52.0.0", "datafusion-proto-common", "object_store", "prost", + "rand 0.9.2", ] [[package]] name = "datafusion-proto-common" -version = "50.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c15ede0e0f1e51d5b5bea1a196db35e00aa3cae9e58cc12df3cc900e36328437" +version = "52.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=15d82292e899bc06adbbdbfacea67070bc89fb95#15d82292e899bc06adbbdbfacea67070bc89fb95" dependencies = [ "arrow", - "datafusion-common", + "datafusion-common 52.0.0", "prost", ] [[package]] name = "datafusion-pruning" -version = "50.0.0" +version = "52.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=15d82292e899bc06adbbdbfacea67070bc89fb95#15d82292e899bc06adbbdbfacea67070bc89fb95" +dependencies = [ + "arrow", + "datafusion-common 52.0.0", + "datafusion-datasource 52.0.0", + "datafusion-expr-common 52.0.0", + "datafusion-physical-expr 52.0.0", + "datafusion-physical-expr-common 52.0.0", + "datafusion-physical-plan 52.0.0", + "itertools 0.14.0", + "log", +] + +[[package]] +name = "datafusion-pruning" +version = "52.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a893a46c56f5f190085e13949eb8ec163672c7ec2ac33bdb82c84572e71ca73" +checksum = "64d0fef4201777b52951edec086c21a5b246f3c82621569ddb4a26f488bc38a9" dependencies = [ "arrow", - "arrow-schema", - "datafusion-common", - "datafusion-datasource", - "datafusion-expr-common", - "datafusion-physical-expr", - "datafusion-physical-expr-common", - "datafusion-physical-plan", - "itertools", + "datafusion-common 52.1.0", + "datafusion-datasource 52.1.0", + "datafusion-expr-common 52.1.0", + "datafusion-physical-expr 52.1.0", + "datafusion-physical-expr-common 52.1.0", + "datafusion-physical-plan 52.1.0", + "itertools 0.14.0", + "log", +] + +[[package]] +name = "datafusion-session" +version = "52.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=15d82292e899bc06adbbdbfacea67070bc89fb95#15d82292e899bc06adbbdbfacea67070bc89fb95" +dependencies = [ + "async-trait", + "datafusion-common 52.0.0", + "datafusion-execution 52.0.0", + "datafusion-expr 52.0.0", + "datafusion-physical-plan 52.0.0", + "parking_lot", +] + +[[package]] +name = "datafusion-session" +version = "52.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f71f1e39e8f2acbf1c63b0e93756c2e970a64729dab70ac789587d6237c4fde0" +dependencies = [ + "async-trait", + "datafusion-common 52.1.0", + "datafusion-execution 52.1.0", + "datafusion-expr 52.1.0", + "datafusion-physical-plan 52.1.0", + "parking_lot", +] + +[[package]] +name = "datafusion-sql" +version = "52.0.0" +source = "git+https://github.com/DataDog/datafusion?rev=15d82292e899bc06adbbdbfacea67070bc89fb95#15d82292e899bc06adbbdbfacea67070bc89fb95" +dependencies = [ + "arrow", + "bigdecimal", + "chrono", + "datafusion-common 52.0.0", + "datafusion-expr 52.0.0", + "indexmap", + "log", + "regex", + "sqlparser", +] + +[[package]] +name = "datafusion-sql" +version = "52.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f44693cfcaeb7a9f12d71d1c576c3a6dc025a12cef209375fa2d16fb3b5670ee" +dependencies = [ + "arrow", + "bigdecimal", + "chrono", + "datafusion-common 52.1.0", + "datafusion-expr 52.1.0", + "indexmap", "log", + "recursive", + "regex", + "sqlparser", +] + +[[package]] +name = "deflate64" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26bf8fc351c5ed29b5c2f0cbbac1b209b74f60ecd62e675a998df72c49af5204" + +[[package]] +name = "delegate" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "780eb241654bf097afb00fc5f054a09b687dad862e485fdcf8399bb056565370" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "deltae" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5729f5117e208430e437df2f4843f5e5952997175992d1414f94c57d61e270b4" + +[[package]] +name = "deranged" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ececcb659e7ba858fb4f10388c250a7252eb0a27373f1a72b8748afdd248e587" +dependencies = [ + "powerfmt", ] [[package]] -name = "datafusion-session" -version = "50.0.0" +name = "derive_arbitrary" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8b62684c7a1db6121a8c83100209cffa1e664a8d9ced87e1a32f8cdc2fff3c2" +checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" dependencies = [ - "arrow", - "async-trait", - "dashmap", - "datafusion-common", - "datafusion-common-runtime", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr", - "datafusion-physical-plan", - "datafusion-sql", - "futures", - "itertools", - "log", - "object_store", - "parking_lot", - "tokio", + "proc-macro2", + "quote", + "syn 2.0.114", ] [[package]] -name = "datafusion-sql" -version = "50.0.0" +name = "derive_more" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f09cff94b8242843e1da5d069e9d2cfc53807f1f00b1c0da78c297f47c21456e" +checksum = "d751e9e49156b02b44f9c1815bcb94b984cdcc4396ecc32521c739452808b134" dependencies = [ - "arrow", - "bigdecimal", - "datafusion-common", - "datafusion-expr", - "indexmap", - "log", - "recursive", - "regex", - "sqlparser", + "derive_more-impl", ] [[package]] -name = "delegate" -version = "0.13.4" +name = "derive_more-impl" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6178a82cf56c836a3ba61a7935cdb1c49bfaa6fa4327cd5bf554a503087de26b" +checksum = "799a97264921d8623a957f6c3b9011f3b5492f557bbb7a5a19b7fa6d06ba8dcb" dependencies = [ + "convert_case", "proc-macro2", "quote", - "syn 2.0.106", + "rustc_version", + "syn 2.0.114", ] +[[package]] +name = "diff" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" + [[package]] name = "digest" version = "0.10.7" @@ -1569,6 +3354,27 @@ dependencies = [ "subtle", ] +[[package]] +name = "dirs" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3e8aa94d75141228480295a7d0e7feb620b1a5ad9f12bc40be62411e38cce4e" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab" +dependencies = [ + "libc", + "option-ext", + "redox_users", + "windows-sys 0.61.2", +] + [[package]] name = "displaydoc" version = "0.2.5" @@ -1577,9 +3383,24 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.114", +] + +[[package]] +name = "document-features" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4b8a88685455ed29a21542a33abd9cb6510b6b129abadabdcef0f4c55bc8f61" +dependencies = [ + "litrs", ] +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + [[package]] name = "either" version = "1.15.0" @@ -1592,11 +3413,26 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "endian-type" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d" + [[package]] name = "env_filter" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "186e05a59d4c50738528153b83b0b0194d3a29507dfec16eccd4b342903397d0" +checksum = "1bf3c259d255ca70051b30e2e95b5446cdb8949ac4cd22c0d7fd634d89f568e2" dependencies = [ "log", "regex", @@ -1623,12 +3459,68 @@ checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" [[package]] name = "errno" -version = "0.3.13" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "778e2ac28f6c47af28e4907f13ffd1e1ddbd400980a9abd7c8df189bf578a5ad" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", +] + +[[package]] +name = "error-code" +version = "3.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dea2df4cf52843e0452895c455a1a2cfbb842a1e7329671acf418fdc53ed4c59" + +[[package]] +name = "euclid" +version = "0.22.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df61bf483e837f88d5c2291dcf55c67be7e676b3a51acc48db3a7b163b91ed63" +dependencies = [ + "num-traits", +] + +[[package]] +name = "event-listener" +version = "5.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13b66accf52311f30a0db42147dadea9850cb48cd070028831ae5f5d4b856ab" +dependencies = [ + "concurrent-queue", + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" +dependencies = [ + "event-listener", + "pin-project-lite", +] + +[[package]] +name = "eyre" +version = "0.6.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd915d99f24784cdc19fd37ef22b97e3ff0ae756c7e492e9fbfe897d61e2aec" +dependencies = [ + "indenter", + "once_cell", +] + +[[package]] +name = "fancy-regex" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b95f7c0680e4142284cf8b22c14a476e87d61b004a3a0861872b32ef7ead40a2" +dependencies = [ + "bit-set", + "regex", ] [[package]] @@ -1637,6 +3529,46 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" +[[package]] +name = "fd-lock" +version = "4.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce92ff622d6dadf7349484f42c93271a0d49b7cc4d466a936405bacbe10aa78" +dependencies = [ + "cfg-if", + "rustix", + "windows-sys 0.59.0", +] + +[[package]] +name = "filedescriptor" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e40758ed24c9b2eeb76c35fb0aebc66c626084edd827e07e1552279814c6682d" +dependencies = [ + "libc", + "thiserror 1.0.69", + "winapi", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "finl_unicode" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9844ddc3a6e533d62bba727eb6c28b5d360921d5175e9ff0f1e621a5c590a4d5" + +[[package]] +name = "fixedbitset" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" + [[package]] name = "fixedbitset" version = "0.5.7" @@ -1645,23 +3577,23 @@ checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" [[package]] name = "flatbuffers" -version = "25.2.10" +version = "25.12.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1045398c1bfd89168b5fd3f1fc11f6e70b34f6f66300c87d44d3de849463abf1" +checksum = "35f6839d7b3b98adde531effaf34f0c2badc6f4735d26fe74709d8e513a96ef3" dependencies = [ - "bitflags 2.9.1", + "bitflags 2.10.0", "rustc_version", ] [[package]] name = "flate2" -version = "1.1.2" +version = "1.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a3d7db9596fecd151c5f638c0ee5d5bd487b6e0ea232e5dc96d5250f6f94b1d" +checksum = "b375d6465b98090a5f25b1c7703f3859783755aa9a80433b36e0379a3ec2f369" dependencies = [ "crc32fast", - "libz-rs-sys", "miniz_oxide", + "zlib-rs", ] [[package]] @@ -1676,6 +3608,27 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "form_urlencoded" version = "1.2.2" @@ -1685,6 +3638,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + [[package]] name = "futures" version = "0.3.31" @@ -1741,7 +3700,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] @@ -1786,53 +3745,87 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.16" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" dependencies = [ "cfg-if", "js-sys", "libc", - "wasi 0.11.1+wasi-snapshot-preview1", + "wasi", "wasm-bindgen", ] [[package]] name = "getrandom" -version = "0.3.3" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi", - "wasi 0.14.2+wasi-0.2.4", + "wasip2", + "wasm-bindgen", ] [[package]] name = "gimli" -version = "0.31.1" +version = "0.32.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7" + +[[package]] +name = "git2" +version = "0.20.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" +checksum = "7b88256088d75a56f8ecfa070513a775dd9107f6530ef14919dac831af9cfe2b" +dependencies = [ + "bitflags 2.10.0", + "libc", + "libgit2-sys", + "log", + "url", +] [[package]] name = "glob" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" +checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" [[package]] name = "h2" -version = "0.4.11" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0beca50380b1fc32983fc1cb4587bfa4bb9e78fc259aad4a0032d2080309222d" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http 0.2.12", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "h2" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17da50a276f1e01e0ba6c029e47b7100754904ee8a278f886546e98575380785" +checksum = "2f44da3a8150a6703ed5d34e164b875fd14c2cdab9af1252a9a1020bde2bdc54" dependencies = [ "atomic-waker", "bytes", "fnv", "futures-core", "futures-sink", - "http", + "http 1.4.0", "indexmap", "slab", "tokio", @@ -1842,13 +3835,14 @@ dependencies = [ [[package]] name = "half" -version = "2.6.0" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "459196ed295495a68f7d7fe1d84f6c4b7ff0e21fe3017b2f283c6fac3ad803c9" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" dependencies = [ "cfg-if", "crunchy", "num-traits", + "zerocopy", ] [[package]] @@ -1856,27 +3850,26 @@ name = "hashbrown" version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" -dependencies = [ - "ahash", - "allocator-api2", -] [[package]] name = "hashbrown" -version = "0.15.4" +version = "0.15.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5971ac85611da7067dbfcabef3c70ebb5606018acd9e2a3903a0da507521e0d5" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ - "allocator-api2", - "equivalent", - "foldhash", + "foldhash 0.1.5", ] [[package]] name = "hashbrown" -version = "0.16.0" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5419bdc4f6a9207fbeba6d11b604d481addf78ecd10c11ad51e76c2f6482748d" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash 0.2.0", +] [[package]] name = "heck" @@ -1902,23 +3895,68 @@ dependencies = [ "libc", ] +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + [[package]] name = "hex" version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + +[[package]] +name = "home" +version = "0.5.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc627f471c528ff0c4a49e1d5e60450c8f6461dd6d10ba9dcd3a61d3dff7728d" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "http" -version = "1.3.1" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4a85d31aea989eead29a3aaf9e1115a180df8282431156e533de47660892565" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" dependencies = [ "bytes", "fnv", "itoa", ] +[[package]] +name = "http" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" +dependencies = [ + "bytes", + "http 0.2.12", + "pin-project-lite", +] + [[package]] name = "http-body" version = "1.0.1" @@ -1926,7 +3964,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", - "http", + "http 1.4.0", ] [[package]] @@ -1937,8 +3975,8 @@ checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" dependencies = [ "bytes", "futures-core", - "http", - "http-body", + "http 1.4.0", + "http-body 1.0.1", "pin-project-lite", ] @@ -1956,29 +3994,87 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "humantime" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b112acc8b3adf4b107a8ec20977da0273a8c386765a3ec0229bd500a1443f9f" +checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" [[package]] name = "hyper" -version = "1.6.0" +version = "0.14.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc2b571658e38e0c01b1fdca3bbbe93c00d3d71693ff2770043f8c29bc7d6f80" +checksum = "41dfc780fdec9373c01bae43289ea34c972e40ee3c9f6b3c8801a35f35586ce7" dependencies = [ "bytes", "futures-channel", + "futures-core", "futures-util", - "h2", - "http", - "http-body", + "h2 0.3.27", + "http 0.2.12", + "http-body 0.4.6", "httparse", "httpdate", "itoa", "pin-project-lite", + "socket2 0.5.10", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ab2d4f250c3d7b1c9fcdff1cece94ea4e2dfbec68614f7b87cb205f24ca9d11" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2 0.4.13", + "http 1.4.0", + "http-body 1.0.1", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "pin-utils", "smallvec", "tokio", - "want", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" +dependencies = [ + "futures-util", + "http 0.2.12", + "hyper 0.14.32", + "log", + "rustls 0.21.12", + "tokio", + "tokio-rustls 0.24.1", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" +dependencies = [ + "http 1.4.0", + "hyper 1.8.1", + "hyper-util", + "rustls 0.23.36", + "rustls-native-certs", + "rustls-pki-types", + "tokio", + "tokio-rustls 0.26.4", + "tower-service", ] [[package]] @@ -1987,39 +4083,59 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" dependencies = [ - "hyper", + "hyper 1.8.1", "hyper-util", "pin-project-lite", "tokio", "tower-service", ] +[[package]] +name = "hyper-tls" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" +dependencies = [ + "bytes", + "http-body-util", + "hyper 1.8.1", + "hyper-util", + "native-tls", + "tokio", + "tokio-native-tls", + "tower-service", +] + [[package]] name = "hyper-util" -version = "0.1.16" +version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d9b05277c7e8da2c93a568989bb6207bef0112e8d17df7a6eda4a3cf143bc5e" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" dependencies = [ + "base64", "bytes", "futures-channel", - "futures-core", "futures-util", - "http", - "http-body", - "hyper", + "http 1.4.0", + "http-body 1.0.1", + "hyper 1.8.1", + "ipnet", "libc", + "percent-encoding", "pin-project-lite", - "socket2 0.6.0", + "socket2 0.6.2", + "system-configuration", "tokio", "tower-service", "tracing", + "windows-registry", ] [[package]] name = "iana-time-zone" -version = "0.1.63" +version = "0.1.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0c919e5debc312ad217002b8048a17b7d83f80703865bbfcfebb0458b0b27d8" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" dependencies = [ "android_system_properties", "core-foundation-sys", @@ -2027,7 +4143,7 @@ dependencies = [ "js-sys", "log", "wasm-bindgen", - "windows-core", + "windows-core 0.62.2", ] [[package]] @@ -2041,9 +4157,9 @@ dependencies = [ [[package]] name = "icu_collections" -version = "2.0.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "200072f5d0e3614556f94a9930d5dc3e0662a652823904c3a75dc3b0af7fee47" +checksum = "4c6b649701667bbe825c3b7e6388cb521c23d88644678e83c0c4d0a621a34b43" dependencies = [ "displaydoc", "potential_utf", @@ -2054,9 +4170,9 @@ dependencies = [ [[package]] name = "icu_locale_core" -version = "2.0.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cde2700ccaed3872079a65fb1a78f6c0a36c91570f28755dda67bc8f7d9f00a" +checksum = "edba7861004dd3714265b4db54a3c390e880ab658fec5f7db895fae2046b5bb6" dependencies = [ "displaydoc", "litemap", @@ -2067,11 +4183,10 @@ dependencies = [ [[package]] name = "icu_normalizer" -version = "2.0.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "436880e8e18df4d7bbc06d58432329d6458cc84531f7ac5f024e93deadb37979" +checksum = "5f6c8828b67bf8908d82127b2054ea1b4427ff0230ee9141c54251934ab1b599" dependencies = [ - "displaydoc", "icu_collections", "icu_normalizer_data", "icu_properties", @@ -2082,42 +4197,38 @@ dependencies = [ [[package]] name = "icu_normalizer_data" -version = "2.0.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00210d6893afc98edb752b664b8890f0ef174c8adbb8d0be9710fa66fbbf72d3" +checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a" [[package]] name = "icu_properties" -version = "2.0.1" +version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "016c619c1eeb94efb86809b015c58f479963de65bdb6253345c1a1276f22e32b" +checksum = "020bfc02fe870ec3a66d93e677ccca0562506e5872c650f893269e08615d74ec" dependencies = [ - "displaydoc", "icu_collections", "icu_locale_core", "icu_properties_data", "icu_provider", - "potential_utf", "zerotrie", "zerovec", ] [[package]] name = "icu_properties_data" -version = "2.0.1" +version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "298459143998310acd25ffe6810ed544932242d3f07083eee1084d83a71bd632" +checksum = "616c294cf8d725c6afcd8f55abc17c56464ef6211f9ed59cccffe534129c77af" [[package]] name = "icu_provider" -version = "2.0.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03c80da27b5f4187909049ee2d72f276f0d9f99a42c306bd0131ecfe04d8e5af" +checksum = "85962cf0ce02e1e0a629cc34e7ca3e373ce20dda4c4d7294bbd0bf1fdb59e614" dependencies = [ "displaydoc", "icu_locale_core", - "stable_deref_trait", - "tinystr", "writeable", "yoke", "zerofrom", @@ -2125,6 +4236,12 @@ dependencies = [ "zerovec", ] +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + [[package]] name = "idna" version = "1.1.0" @@ -2146,26 +4263,64 @@ dependencies = [ "icu_properties", ] +[[package]] +name = "indenter" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "964de6e86d545b246d84badc0fef527924ace5134f30641c203ef52ba83f58d5" + [[package]] name = "indexmap" -version = "2.11.4" +version = "2.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b0f83760fb341a774ed326568e19f5a863af4a952def8c39f9ab92fd95b88e5" +checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" dependencies = [ "equivalent", - "hashbrown 0.15.4", + "hashbrown 0.16.1", +] + +[[package]] +name = "indoc" +version = "2.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79cf5c93f93228cf8efb3ba362535fb11199ac548a09ce117c9b1adc3030d706" +dependencies = [ + "rustversion", +] + +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "generic-array", ] [[package]] name = "insta" -version = "1.43.1" +version = "1.46.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "154934ea70c58054b556dd430b99a98c2a7ff5309ac9891597e339b5c28f4371" +checksum = "e82db8c87c7f1ccecb34ce0c24399b8a73081427f3c7c50a5d597925356115e4" dependencies = [ "console", "once_cell", "regex", "similar", + "tempfile", +] + +[[package]] +name = "instability" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "357b7205c6cd18dd2c86ed312d1e70add149aea98e7ef72b9fdf0270e555c11d" +dependencies = [ + "darling", + "indoc", + "proc-macro2", + "quote", + "syn 2.0.114", ] [[package]] @@ -2175,21 +4330,46 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" [[package]] -name = "io-uring" -version = "0.7.9" +name = "ipnet" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" + +[[package]] +name = "iri-string" +version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d93587f37623a1a17d94ef2bc9ada592f5465fe7732084ab7beefabe5c77c0c4" +checksum = "c91338f0783edbd6195decb37bae672fd3b165faffb89bf7b9e6942f8b1a731a" dependencies = [ - "bitflags 2.9.1", - "cfg-if", + "memchr", + "serde", +] + +[[package]] +name = "is-terminal" +version = "0.4.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" +dependencies = [ + "hermit-abi 0.5.2", "libc", + "windows-sys 0.61.2", ] [[package]] name = "is_terminal_polyfill" -version = "1.70.1" +version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] [[package]] name = "itertools" @@ -2202,54 +4382,71 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.15" +version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" +checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2" [[package]] name = "jiff" -version = "0.2.15" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be1f93b8b1eb69c77f24bbb0afdf66f54b632ee39af40ca21c4365a1d7347e49" +checksum = "e67e8da4c49d6d9909fe03361f9b620f58898859f5c7aded68351e85e71ecf50" dependencies = [ "jiff-static", "log", "portable-atomic", "portable-atomic-util", - "serde", + "serde_core", ] [[package]] name = "jiff-static" -version = "0.2.15" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03343451ff899767262ec32146f6d559dd759fdadf42ff0e227c7c48f72594b4" +checksum = "e0c84ee7f197eca9a86c6fd6cb771e55eb991632f15f2bc3ca6ec838929e6e78" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] name = "jobserver" -version = "0.1.33" +version = "0.1.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38f262f097c174adebe41eb73d66ae9c06b2844fb0da69969647bbddd9b0538a" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" dependencies = [ - "getrandom 0.3.3", + "getrandom 0.3.4", "libc", ] [[package]] name = "js-sys" -version = "0.3.77" +version = "0.3.85" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" +checksum = "8c942ebf8e95485ca0d52d97da7c5a2c387d0e7f0ba4c35e93bfcaee045955b3" dependencies = [ "once_cell", "wasm-bindgen", ] +[[package]] +name = "kasuari" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fe90c1150662e858c7d5f945089b7517b0a80d8bf7ba4b1b5ffc984e7230a5b" +dependencies = [ + "hashbrown 0.16.1", + "portable-atomic", + "thiserror 2.0.18", +] + +[[package]] +name = "lab" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf36173d4167ed999940f804952e6b08197cae5ad5d572eb4db150ce8ad5d58f" + [[package]] name = "lazy_static" version = "1.5.0" @@ -2258,9 +4455,9 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "lexical-core" -version = "1.0.5" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b765c31809609075565a70b4b71402281283aeda7ecaf4818ac14a7b2ade8958" +checksum = "7d8d125a277f807e55a77304455eb7b1cb52f2b18c143b60e766c120bd64a594" dependencies = [ "lexical-parse-float", "lexical-parse-integer", @@ -2271,53 +4468,46 @@ dependencies = [ [[package]] name = "lexical-parse-float" -version = "1.0.5" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de6f9cb01fb0b08060209a057c048fcbab8717b4c1ecd2eac66ebfe39a65b0f2" +checksum = "52a9f232fbd6f550bc0137dcb5f99ab674071ac2d690ac69704593cb4abbea56" dependencies = [ "lexical-parse-integer", "lexical-util", - "static_assertions", ] [[package]] name = "lexical-parse-integer" -version = "1.0.5" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72207aae22fc0a121ba7b6d479e42cbfea549af1479c3f3a4f12c70dd66df12e" +checksum = "9a7a039f8fb9c19c996cd7b2fcce303c1b2874fe1aca544edc85c4a5f8489b34" dependencies = [ "lexical-util", - "static_assertions", ] [[package]] name = "lexical-util" -version = "1.0.6" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a82e24bf537fd24c177ffbbdc6ebcc8d54732c35b50a3f28cc3f4e4c949a0b3" -dependencies = [ - "static_assertions", -] +checksum = "2604dd126bb14f13fb5d1bd6a66155079cb9fa655b37f875b3a742c705dbed17" [[package]] name = "lexical-write-float" -version = "1.0.5" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5afc668a27f460fb45a81a757b6bf2f43c2d7e30cb5a2dcd3abf294c78d62bd" +checksum = "50c438c87c013188d415fbabbb1dceb44249ab81664efbd31b14ae55dabb6361" dependencies = [ "lexical-util", "lexical-write-integer", - "static_assertions", ] [[package]] name = "lexical-write-integer" -version = "1.0.5" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "629ddff1a914a836fb245616a7888b62903aae58fa771e1d83943035efa0f978" +checksum = "409851a618475d2d5796377cad353802345cba92c867d9fbcde9cf4eac4e14df" dependencies = [ "lexical-util", - "static_assertions", ] [[package]] @@ -2328,133 +4518,330 @@ checksum = "2c4a545a15244c7d945065b5d392b2d2d7f21526fba56ce51467b06ed445e8f7" [[package]] name = "libc" -version = "0.2.176" +version = "0.2.180" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bcc35a38544a891a5f7c865aca548a982ccb3b8650a5b06d0fd33a10283c56fc" + +[[package]] +name = "libgit2-sys" +version = "0.18.3+1.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9b3acc4b91781bb0b3386669d325163746af5f6e4f73e6d2d630e09a35f3487" +dependencies = [ + "cc", + "libc", + "libz-sys", + "pkg-config", +] + +[[package]] +name = "liblzma" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73c36d08cad03a3fbe2c4e7bb3a9e84c57e4ee4135ed0b065cade3d98480c648" +dependencies = [ + "liblzma-sys", +] + +[[package]] +name = "liblzma-sys" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58f929b4d672ea937a23a1ab494143d968337a5f47e56d0815df1e0890ddf174" +checksum = "9f2db66f3268487b5033077f266da6777d057949b8f93c8ad82e441df25e6186" +dependencies = [ + "cc", + "libc", + "pkg-config", +] [[package]] name = "libm" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" [[package]] -name = "libz-rs-sys" -version = "0.5.1" +name = "libmimalloc-sys" +version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "172a788537a2221661b480fee8dc5f96c580eb34fa88764d3205dc356c7e4221" +checksum = "667f4fec20f29dfc6bc7357c582d91796c169ad7e2fce709468aefeb2c099870" dependencies = [ - "zlib-rs", + "cc", + "libc", +] + +[[package]] +name = "libredox" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d0b95e02c851351f877147b7deea7b1afb1df71b63aa5f8270716e0c5720616" +dependencies = [ + "bitflags 2.10.0", + "libc", +] + +[[package]] +name = "libz-sys" +version = "1.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15d118bbf3771060e7311cc7bb0545b01d08a8b4a7de949198dec1fa0ca1c0f7" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "line-clipping" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f4de44e98ddbf09375cbf4d17714d18f39195f4f4894e8524501726fd9a8a4a" +dependencies = [ + "bitflags 2.10.0", ] [[package]] name = "linux-raw-sys" -version = "0.9.4" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12" +checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" [[package]] name = "litemap" -version = "0.8.0" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77" + +[[package]] +name = "litrs" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "241eaef5fd12c88705a01fc1066c48c4b36e0dd4377dcdc7ec3942cea7a69956" +checksum = "11d3d7f243d5c5a8b9bb5d6dd2b1602c0cb0b9db1621bafc7ed66e35ff9fe092" [[package]] name = "lock_api" -version = "0.4.13" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96936507f153605bddfcda068dd804796c84324ed2510809e5b2a624c81da765" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" dependencies = [ - "autocfg", "scopeguard", ] [[package]] name = "log" -version = "0.4.27" +version = "0.4.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" + +[[package]] +name = "lru" +version = "0.16.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1dc47f592c06f33f8e3aea9591776ec7c9f9e4124778ff8a3c3b87159f7e593" +dependencies = [ + "hashbrown 0.16.1", +] + +[[package]] +name = "lru-slab" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" [[package]] name = "lz4_flex" -version = "0.11.5" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08ab2867e3eeeca90e844d1940eab391c9dc5228783db2ed999acbc0a9ed375a" +checksum = "ab6473172471198271ff72e9379150e9dfd70d8e533e0752a27e515b48dd375e" dependencies = [ "twox-hash", ] [[package]] -name = "lzma-sys" -version = "0.1.20" +name = "mac_address" +version = "1.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0aeb26bf5e836cc1c341c8106051b573f1766dfa05aa87f0b98be5e51b02303" +dependencies = [ + "nix 0.29.0", + "winapi", +] + +[[package]] +name = "matchit" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" + +[[package]] +name = "matchit" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" + +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest", +] + +[[package]] +name = "memchr" +version = "2.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" + +[[package]] +name = "memmem" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a64a92489e2744ce060c349162be1c5f33c6969234104dbd99ddb5feb08b8c15" + +[[package]] +name = "memoffset" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" +dependencies = [ + "autocfg", +] + +[[package]] +name = "mimalloc" +version = "0.1.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1ee66a4b64c74f4ef288bcbb9192ad9c3feaad75193129ac8509af543894fd8" +dependencies = [ + "libmimalloc-sys", +] + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "mio" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fda04ab3764e6cde78b9974eec4f779acaba7c4e84b36eca3cf77c581b85d27" +checksum = "a69bcab0ad47271a0234d9422b131806bf3968021e5dc9328caf2d4cd58557fc" dependencies = [ - "cc", "libc", - "pkg-config", + "log", + "wasi", + "windows-sys 0.61.2", ] [[package]] -name = "matchit" -version = "0.8.4" +name = "moka" +version = "0.12.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" +checksum = "b4ac832c50ced444ef6be0767a008b02c106a909ba79d1d830501e94b96f6b7e" +dependencies = [ + "async-lock", + "crossbeam-channel", + "crossbeam-epoch", + "crossbeam-utils", + "equivalent", + "event-listener", + "futures-util", + "parking_lot", + "portable-atomic", + "smallvec", + "tagptr", + "uuid", +] [[package]] -name = "md-5" -version = "0.10.6" +name = "native-tls" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +checksum = "87de3442987e9dbec73158d5c715e7ad9072fda936bb03d19d7fa10e00520f0e" dependencies = [ - "cfg-if", - "digest", + "libc", + "log", + "openssl", + "openssl-probe 0.1.6", + "openssl-sys", + "schannel", + "security-framework 2.11.1", + "security-framework-sys", + "tempfile", ] [[package]] -name = "memchr" -version = "2.7.5" +name = "nibble_vec" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0" +checksum = "77a5d83df9f36fe23f0c3648c6bbb8b0298bb5f1939c8f2704431371f4b84d43" +dependencies = [ + "smallvec", +] [[package]] -name = "mime" -version = "0.3.17" +name = "nix" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" +dependencies = [ + "bitflags 2.10.0", + "cfg-if", + "cfg_aliases", + "libc", + "memoffset", +] [[package]] -name = "miniz_oxide" -version = "0.8.9" +name = "nix" +version = "0.30.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6" dependencies = [ - "adler2", + "bitflags 2.10.0", + "cfg-if", + "cfg_aliases", + "libc", ] [[package]] -name = "mio" -version = "1.0.4" +name = "nom" +version = "7.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78bed444cc8a2160f01cbcf811ef18cac863ad68ae8ca62092e8db51d51c761c" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" dependencies = [ - "libc", - "wasi 0.11.1+wasi-snapshot-preview1", - "windows-sys 0.59.0", + "memchr", + "minimal-lexical", ] [[package]] -name = "num" -version = "0.4.3" +name = "ntapi" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" +checksum = "c70f219e21142367c70c0b30c6a9e3a14d55b4d12a204d897fbec83a0363f081" dependencies = [ - "num-bigint", - "num-complex", - "num-integer", - "num-iter", - "num-rational", - "num-traits", + "winapi", ] [[package]] @@ -2465,6 +4852,7 @@ checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" dependencies = [ "num-integer", "num-traits", + "serde", ] [[package]] @@ -2477,33 +4865,28 @@ dependencies = [ ] [[package]] -name = "num-integer" -version = "0.1.46" +name = "num-conv" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" -dependencies = [ - "num-traits", -] +checksum = "cf97ec579c3c42f953ef76dbf8d55ac91fb219dde70e49aa4a6b7d74e9919050" [[package]] -name = "num-iter" -version = "0.1.45" +name = "num-derive" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" +checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" dependencies = [ - "autocfg", - "num-integer", - "num-traits", + "proc-macro2", + "quote", + "syn 2.0.114", ] [[package]] -name = "num-rational" -version = "0.4.2" +name = "num-integer" +version = "0.1.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" dependencies = [ - "num-bigint", - "num-integer", "num-traits", ] @@ -2517,31 +4900,53 @@ dependencies = [ "libm", ] +[[package]] +name = "num_threads" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c7398b9c8b70908f6371f47ed36737907c87c52af34c268fed0bf0ceb92ead9" +dependencies = [ + "libc", +] + [[package]] name = "object" -version = "0.36.7" +version = "0.37.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" +checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe" dependencies = [ "memchr", ] [[package]] name = "object_store" -version = "0.12.3" +version = "0.12.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efc4f07659e11cd45a341cd24d71e683e3be65d9ff1f8150061678fe60437496" +checksum = "fbfbfff40aeccab00ec8a910b57ca8ecf4319b335c542f2edcd19dd25a1e2a00" dependencies = [ "async-trait", + "base64", "bytes", "chrono", + "form_urlencoded", "futures", - "http", + "http 1.4.0", + "http-body-util", "humantime", - "itertools", + "hyper 1.8.1", + "itertools 0.14.0", + "md-5", "parking_lot", "percent-encoding", - "thiserror", + "quick-xml", + "rand 0.9.2", + "reqwest", + "ring", + "rustls-pemfile", + "serde", + "serde_json", + "serde_urlencoded", + "thiserror 2.0.18", "tokio", "tracing", "url", @@ -2558,9 +4963,81 @@ checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" [[package]] name = "once_cell_polyfill" -version = "1.70.1" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "oorandom" +version = "11.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" + +[[package]] +name = "openssl" +version = "0.10.75" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08838db121398ad17ab8531ce9de97b244589089e290a384c900cb9ff7434328" +dependencies = [ + "bitflags 2.10.0", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "openssl-probe" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" + +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + +[[package]] +name = "openssl-src" +version = "300.5.5+3.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f1787d533e03597a7934fd0a765f0d28e94ecc5fb7789f8053b1e699a56f709" +dependencies = [ + "cc", +] + +[[package]] +name = "openssl-sys" +version = "0.9.111" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82cab2d520aa75e3c58898289429321eb788c3106963d0dc886ec7a5f4adc321" +dependencies = [ + "cc", + "libc", + "openssl-src", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "option-ext" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4895175b425cb1f87721b59f0f286c2092bd4af812243672510e1ac53e2e0ad" +checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" [[package]] name = "ordered-float" @@ -2571,11 +5048,38 @@ dependencies = [ "num-traits", ] +[[package]] +name = "ordered-float" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bb71e1b3fa6ca1c61f383464aaf2bb0e2f8e772a1f01d486832464de363b951" +dependencies = [ + "num-traits", +] + +[[package]] +name = "outref" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" + +[[package]] +name = "owo-colors" +version = "4.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c6901729fa79e91a0913333229e9ca5dc725089d1c363b2f4b4760709dc4a52" + +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + [[package]] name = "parking_lot" -version = "0.12.4" +version = "0.12.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70d58bf43669b5795d1576d0641cfb6fbb2057bf629506267a92807158584a13" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" dependencies = [ "lock_api", "parking_lot_core", @@ -2583,22 +5087,22 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.11" +version = "0.9.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc838d2a56b5b1a6c25f55575dfc605fabb63bb2365f6c2353ef9159aa69e4a5" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" dependencies = [ "cfg-if", "libc", "redox_syscall", "smallvec", - "windows-targets 0.52.6", + "windows-link", ] [[package]] name = "parquet" -version = "56.2.0" +version = "57.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0dbd48ad52d7dccf8ea1b90a3ddbfaea4f69878dd7683e51c507d4bc52b5b27" +checksum = "5f6a2926a30477c0b95fea6c28c3072712b139337a242c2cc64817bdc20a8854" dependencies = [ "ahash", "arrow-array", @@ -2615,10 +5119,11 @@ dependencies = [ "flate2", "futures", "half", - "hashbrown 0.16.0", + "hashbrown 0.16.1", "lz4_flex", - "num", "num-bigint", + "num-integer", + "num-traits", "object_store", "paste", "ring", @@ -2637,31 +5142,136 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" +[[package]] +name = "pbkdf2" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" +dependencies = [ + "digest", + "hmac", +] + [[package]] name = "percent-encoding" version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +[[package]] +name = "pest" +version = "2.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9eb05c21a464ea704b53158d358a31e6425db2f63a1a7312268b05fe2b75f7" +dependencies = [ + "memchr", + "ucd-trie", +] + +[[package]] +name = "pest_derive" +version = "2.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68f9dbced329c441fa79d80472764b1a2c7e57123553b8519b36663a2fb234ed" +dependencies = [ + "pest", + "pest_generator", +] + +[[package]] +name = "pest_generator" +version = "2.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bb96d5051a78f44f43c8f712d8e810adb0ebf923fc9ed2655a7f66f63ba8ee5" +dependencies = [ + "pest", + "pest_meta", + "proc-macro2", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "pest_meta" +version = "2.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "602113b5b5e8621770cfd490cfd90b9f84ab29bd2b0e49ad83eb6d186cef2365" +dependencies = [ + "pest", + "sha2", +] + [[package]] name = "petgraph" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54acf3a685220b533e437e264e4d932cfbdc4cc7ec0cd232ed73c08d03b8a7ca" +checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" dependencies = [ - "fixedbitset", - "hashbrown 0.15.4", + "fixedbitset 0.5.7", + "hashbrown 0.15.5", "indexmap", "serde", ] +[[package]] +name = "phf" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" +dependencies = [ + "phf_macros", + "phf_shared 0.11.3", +] + [[package]] name = "phf" version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "913273894cec178f401a31ec4b656318d95473527be05c0752cc41cdc32be8b7" dependencies = [ - "phf_shared", + "phf_shared 0.12.1", +] + +[[package]] +name = "phf_codegen" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aef8048c789fa5e851558d709946d6d79a8ff88c0440c587967f8e94bfb1216a" +dependencies = [ + "phf_generator", + "phf_shared 0.11.3", +] + +[[package]] +name = "phf_generator" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" +dependencies = [ + "phf_shared 0.11.3", + "rand 0.8.5", +] + +[[package]] +name = "phf_macros" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f84ac04429c13a7ff43785d75ad27569f2951ce0ffd30a3321230db2fc727216" +dependencies = [ + "phf_generator", + "phf_shared 0.11.3", + "proc-macro2", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "phf_shared" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" +dependencies = [ + "siphasher", ] [[package]] @@ -2690,7 +5300,7 @@ checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] @@ -2711,37 +5321,97 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" +[[package]] +name = "plotters" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" + +[[package]] +name = "plotters-svg" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" +dependencies = [ + "plotters-backend", +] + [[package]] name = "portable-atomic" -version = "1.11.1" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483" +checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" [[package]] name = "portable-atomic-util" -version = "0.2.4" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a9db96d7fa8782dd8c15ce32ffe8680bbd1e978a43bf51a34d39483540495f5" +dependencies = [ + "portable-atomic", +] + +[[package]] +name = "potential_utf" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b73949432f5e2a09657003c25bca5e19a0e9c84f8058ca374f49e0ebe605af77" +dependencies = [ + "zerovec", +] + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "ppmd-rust" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "efca4c95a19a79d1c98f791f10aebd5c1363b473244630bb7dbde1dc98455a24" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8a2f0d8d040d7848a709caf78912debcc3f33ee4b3cac47d73d1e1069e83507" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" dependencies = [ - "portable-atomic", + "zerocopy", ] [[package]] -name = "potential_utf" -version = "0.1.2" +name = "pretty_assertions" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5a7c30837279ca13e7c867e9e40053bc68740f988cb07f7ca6df43cc734b585" +checksum = "3ae130e2f271fbc2ac3a40fb1d07180839cdbbe443c7a27e1e3c13c5cac0116d" dependencies = [ - "zerovec", + "diff", + "yansi", ] [[package]] -name = "ppv-lite86" -version = "0.2.21" +name = "prettyplease" +version = "0.2.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ - "zerocopy", + "proc-macro2", + "syn 2.0.114", ] [[package]] @@ -2770,18 +5440,18 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.95" +version = "1.0.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" dependencies = [ "unicode-ident", ] [[package]] name = "prost" -version = "0.13.5" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" +checksum = "d2ea70524a2f82d518bce41317d0fae74151505651af45faf1ffbd6fd33f0568" dependencies = [ "bytes", "prost-derive", @@ -2789,40 +5459,112 @@ dependencies = [ [[package]] name = "prost-derive" -version = "0.13.5" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" +checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" dependencies = [ "anyhow", - "itertools", + "itertools 0.14.0", "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] name = "prost-types" -version = "0.13.5" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16" +checksum = "8991c4cbdb8bc5b11f0b074ffe286c30e523de90fee5ba8132f1399f23cb3dd7" dependencies = [ "prost", ] [[package]] name = "psm" -version = "0.1.26" +version = "0.1.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e944464ec8536cd1beb0bbfd96987eb5e3b72f2ecdafdc5c769a37f1fa2ae1f" +checksum = "1fa96cb91275ed31d6da3e983447320c4eb219ac180fa1679a0889ff32861e2d" dependencies = [ + "ar_archive_writer", "cc", ] +[[package]] +name = "quad-rand" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a651516ddc9168ebd67b24afd085a718be02f8858fe406591b013d101ce2f40" + +[[package]] +name = "quick-xml" +version = "0.38.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b66c2058c55a409d601666cffe35f04333cf1013010882cec174a7467cd4e21c" +dependencies = [ + "memchr", + "serde", +] + +[[package]] +name = "quinn" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls 0.23.36", + "socket2 0.6.2", + "thiserror 2.0.18", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1906b49b0c3bc04b5fe5d86a77925ae6524a19b816ae38ce1e426255f1d8a31" +dependencies = [ + "bytes", + "getrandom 0.3.4", + "lru-slab", + "rand 0.9.2", + "ring", + "rustc-hash", + "rustls 0.23.36", + "rustls-pki-types", + "slab", + "thiserror 2.0.18", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2 0.6.2", + "tracing", + "windows-sys 0.60.2", +] + [[package]] name = "quote" -version = "1.0.40" +version = "1.0.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" +checksum = "21b2ebcf727b7760c461f091f9f0f539b77b8e87f2fd88131e7f1b433b3cece4" dependencies = [ "proc-macro2", ] @@ -2833,14 +5575,22 @@ version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" +[[package]] +name = "radix_trie" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c069c179fcdc6a2fe24d8d18305cf085fdbd4f922c041943e203685d6a1c58fd" +dependencies = [ + "endian-type", + "nibble_vec", +] + [[package]] name = "rand" version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ - "libc", - "rand_chacha 0.3.1", "rand_core 0.6.4", ] @@ -2850,18 +5600,8 @@ version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" dependencies = [ - "rand_chacha 0.9.0", - "rand_core 0.9.3", -] - -[[package]] -name = "rand_chacha" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" -dependencies = [ - "ppv-lite86", - "rand_core 0.6.4", + "rand_chacha", + "rand_core 0.9.5", ] [[package]] @@ -2871,7 +5611,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" dependencies = [ "ppv-lite86", - "rand_core 0.9.3", + "rand_core 0.9.5", ] [[package]] @@ -2879,17 +5619,119 @@ name = "rand_core" version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" dependencies = [ - "getrandom 0.2.16", + "getrandom 0.3.4", ] [[package]] -name = "rand_core" -version = "0.9.3" +name = "ratatui" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1ce67fb8ba4446454d1c8dbaeda0557ff5e94d39d5e5ed7f10a65eb4c8266bc" +dependencies = [ + "instability", + "ratatui-core", + "ratatui-crossterm", + "ratatui-macros", + "ratatui-termwiz", + "ratatui-widgets", +] + +[[package]] +name = "ratatui-core" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ef8dea09a92caaf73bff7adb70b76162e5937524058a7e5bff37869cbbec293" +dependencies = [ + "bitflags 2.10.0", + "compact_str", + "hashbrown 0.16.1", + "indoc", + "itertools 0.14.0", + "kasuari", + "lru", + "strum", + "thiserror 2.0.18", + "unicode-segmentation", + "unicode-truncate", + "unicode-width 0.2.2", +] + +[[package]] +name = "ratatui-crossterm" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "577c9b9f652b4c121fb25c6a391dd06406d3b092ba68827e6d2f09550edc54b3" +dependencies = [ + "cfg-if", + "crossterm", + "instability", + "ratatui-core", +] + +[[package]] +name = "ratatui-macros" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" +checksum = "a7f1342a13e83e4bb9d0b793d0ea762be633f9582048c892ae9041ef39c936f4" dependencies = [ - "getrandom 0.3.3", + "ratatui-core", + "ratatui-widgets", +] + +[[package]] +name = "ratatui-termwiz" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f76fe0bd0ed4295f0321b1676732e2454024c15a35d01904ddb315afd3d545c" +dependencies = [ + "ratatui-core", + "termwiz", +] + +[[package]] +name = "ratatui-widgets" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7dbfa023cd4e604c2553483820c5fe8aa9d71a42eea5aa77c6e7f35756612db" +dependencies = [ + "bitflags 2.10.0", + "hashbrown 0.16.1", + "indoc", + "instability", + "itertools 0.14.0", + "line-clipping", + "ratatui-core", + "strum", + "time", + "unicode-segmentation", + "unicode-width 0.2.2", +] + +[[package]] +name = "rayon" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", ] [[package]] @@ -2909,23 +5751,34 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "76009fbe0614077fc1a2ce255e3a1881a2e3a3527097d5dc6d8212c585e7e38b" dependencies = [ "quote", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] name = "redox_syscall" -version = "0.5.15" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags 2.10.0", +] + +[[package]] +name = "redox_users" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e8af0dde094006011e6a740d4879319439489813bd0bcdc7d821beaeeff48ec" +checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac" dependencies = [ - "bitflags 2.9.1", + "getrandom 0.2.17", + "libredox", + "thiserror 2.0.18", ] [[package]] name = "regex" -version = "1.11.1" +version = "1.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" +checksum = "843bc0191f75f3e22651ae5f1e72939ab2f72a4bc30fa80a066bd66edefc24d4" dependencies = [ "aho-corasick", "memchr", @@ -2935,20 +5788,73 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.9" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" +checksum = "5276caf25ac86c8d810222b3dbb938e512c55c6831a10f3e6ed1c93b84041f1c" dependencies = [ "aho-corasick", "memchr", "regex-syntax", ] +[[package]] +name = "regex-lite" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d942b98df5e658f56f20d592c7f868833fe38115e65c33003d8cd224b0155da" + [[package]] name = "regex-syntax" -version = "0.8.6" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" + +[[package]] +name = "reqwest" +version = "0.12.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "caf4aa5b0f434c91fe5c7f1ecb6a5ece2130b02ad2a590589dda5146df959001" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2 0.4.13", + "http 1.4.0", + "http-body 1.0.1", + "http-body-util", + "hyper 1.8.1", + "hyper-rustls 0.27.7", + "hyper-tls", + "hyper-util", + "js-sys", + "log", + "mime", + "native-tls", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls 0.23.36", + "rustls-native-certs", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-native-tls", + "tokio-rustls 0.26.4", + "tokio-util", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", +] [[package]] name = "ring" @@ -2958,7 +5864,7 @@ checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" dependencies = [ "cc", "cfg-if", - "getrandom 0.2.16", + "getrandom 0.2.17", "libc", "untrusted", "windows-sys 0.52.0", @@ -2966,9 +5872,15 @@ dependencies = [ [[package]] name = "rustc-demangle" -version = "0.1.25" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b50b8869d9fc858ce7266cce0194bd74df58b9d0e3f6df3a9fc8eb470d95c09d" + +[[package]] +name = "rustc-hash" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "989e6739f80c4ad5b13e0fd7fe89531180375b18520cc8c82080e4dc4035b84f" +checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" [[package]] name = "rustc_version" @@ -2981,28 +5893,130 @@ dependencies = [ [[package]] name = "rustix" -version = "1.0.8" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11181fbabf243db407ef8df94a6ce0b2f9a733bd8be4ad02b4eda9602296cac8" +checksum = "146c9e247ccc180c1f61615433868c99f3de3ae256a30a43b49f67c2d9171f34" dependencies = [ - "bitflags 2.9.1", + "bitflags 2.10.0", "errno", "libc", "linux-raw-sys", - "windows-sys 0.60.2", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls" +version = "0.21.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" +dependencies = [ + "log", + "ring", + "rustls-webpki 0.101.7", + "sct", +] + +[[package]] +name = "rustls" +version = "0.23.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c665f33d38cea657d9614f766881e4d510e0eda4239891eea56b4cadcf01801b" +dependencies = [ + "aws-lc-rs", + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki 0.103.9", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "612460d5f7bea540c490b2b6395d8e34a953e52b491accd6c86c8164c5932a63" +dependencies = [ + "openssl-probe 0.2.1", + "rustls-pki-types", + "schannel", + "security-framework 3.5.1", +] + +[[package]] +name = "rustls-pemfile" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "rustls-pki-types" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be040f8b0a225e40375822a563fa9524378b9d63112f53e19ffff34df5d33fdd" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.101.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7df23109aa6c1567d1c575b9952556388da57401e4ace1d15f79eedad0d8f53" +dependencies = [ + "aws-lc-rs", + "ring", + "rustls-pki-types", + "untrusted", ] [[package]] name = "rustversion" -version = "1.0.21" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "rustyline" +version = "17.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a0d197bd2c9dc6e53b84da9556a69ba4cdfab8619eb41a8bd1cc2027a0f6b1d" +checksum = "e902948a25149d50edc1a8e0141aad50f54e22ba83ff988cf8f7c9ef07f50564" +dependencies = [ + "bitflags 2.10.0", + "cfg-if", + "clipboard-win", + "fd-lock", + "home", + "libc", + "log", + "memchr", + "nix 0.30.1", + "radix_trie", + "unicode-segmentation", + "unicode-width 0.2.2", + "utf8parse", + "windows-sys 0.60.2", +] [[package]] name = "ryu" -version = "1.0.20" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" +checksum = "a50f4cf475b65d88e057964e0e9bb1f0aa9bbb2036dc65c64596b42932536984" [[package]] name = "same-file" @@ -3014,16 +6028,71 @@ dependencies = [ ] [[package]] -name = "scopeguard" -version = "1.2.0" +name = "schannel" +version = "0.1.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891d81b926048e76efe18581bf793546b4c0eaf8448d72be8de2bbee5fd166e1" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "sct" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "security-framework" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" +dependencies = [ + "bitflags 2.10.0", + "core-foundation 0.9.4", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework" +version = "3.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3297343eaf830f66ede390ea39da1d462b6b0c1b000f420d0a83f898bbbe6ef" +dependencies = [ + "bitflags 2.10.0", + "core-foundation 0.10.1", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +checksum = "cc1f0cbffaac4852523ce30d8bd3c5cdc873501d96ff467ca09b6767bb8cd5c0" +dependencies = [ + "core-foundation-sys", + "libc", +] [[package]] name = "semver" -version = "1.0.26" +version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56e6fa9c48d24d85fb3de5ad847117517440f6beceb7798af16b4a87d616b8d0" +checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" [[package]] name = "seq-macro" @@ -3033,36 +6102,91 @@ checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc" [[package]] name = "serde" -version = "1.0.219" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_bytes" +version = "0.11.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8" +dependencies = [ + "serde", + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.219" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] name = "serde_json" -version = "1.0.141" +version = "1.0.149" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30b9eff21ebe718216c6ec64e1d9ac57087aad11efc64e32002bce4a0d4c03d3" +checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" dependencies = [ "itoa", "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", "ryu", "serde", ] +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "sha2" version = "0.10.9" @@ -3074,21 +6198,58 @@ dependencies = [ "digest", ] +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + [[package]] name = "shlex" version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +[[package]] +name = "signal-hook" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d881a16cf4426aa584979d30bd82cb33429027e42122b169753d6ef1085ed6e2" +dependencies = [ + "libc", + "signal-hook-registry", +] + +[[package]] +name = "signal-hook-mio" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b75a19a7a740b25bc7944bdee6172368f988763b744e3d4dfe753f6b4ece40cc" +dependencies = [ + "libc", + "mio", + "signal-hook", +] + [[package]] name = "signal-hook-registry" -version = "1.4.5" +version = "1.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9203b8055f63a2a00e2f593bb0510367fe707d7ff1e5c872de2f537b339e5410" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" dependencies = [ + "errno", "libc", ] +[[package]] +name = "simd-adler32" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2" + [[package]] name = "simdutf8" version = "0.1.5" @@ -3103,15 +6264,21 @@ checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa" [[package]] name = "siphasher" -version = "1.0.1" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2aa850e253778c88a04c3d7323b043aeda9d3e30d5971937c1855769763678e" + +[[package]] +name = "sketches-ddsketch" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d" +checksum = "c1e9a774a6c28142ac54bb25d25562e6bcf957493a184f15ad4eebccb23e410a" [[package]] name = "slab" -version = "0.4.10" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04dc19736151f35336d325007ac991178d504a119863a2fcb3758cdb5e52c50d" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" [[package]] name = "smallvec" @@ -3137,19 +6304,19 @@ dependencies = [ [[package]] name = "socket2" -version = "0.6.0" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "233504af464074f9d066d7b5416c5f9b894a5862a6506e306f7b816cdd6f1807" +checksum = "86f4aa3ad99f2088c990dfa82d367e19cb29268ed67c574d10d0a4bfe71f07e0" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] name = "sqlparser" -version = "0.58.0" +version = "0.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec4b661c54b1e4b603b37873a18c59920e4c51ea8ea2cf527d925424dbd4437c" +checksum = "4591acadbcf52f0af60eafbb2c003232b2b4cd8de5f0e9437cb8b1b59046cc0f" dependencies = [ "log", "recursive", @@ -3164,20 +6331,20 @@ checksum = "da5fc6819faabb412da764b99d3b713bb55083c11e7e0c00144d386cd6a1939c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] name = "stable_deref_trait" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" [[package]] name = "stacker" -version = "0.1.21" +version = "0.1.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cddb07e32ddb770749da91081d8d0ac3a16f1a569a18b20348cd371f5dead06b" +checksum = "e1f8b29fb42aafcea4edeeb6b2f2d7ecd0d969c48b4cf0d2e64aafc471dd6e59" dependencies = [ "cc", "cfg-if", @@ -3198,13 +6365,19 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + [[package]] name = "structopt" version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c6b5c64445ba8094a6ab0c3cd2ad323e07171012d9c98b0b15651daf1787a10" dependencies = [ - "clap", + "clap 2.34.0", "lazy_static", "structopt-derive", ] @@ -3224,21 +6397,23 @@ dependencies = [ [[package]] name = "strum" -version = "0.26.3" +version = "0.27.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" +checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" +dependencies = [ + "strum_macros", +] [[package]] name = "strum_macros" -version = "0.26.4" +version = "0.27.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" +checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7" dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "rustversion", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] @@ -3260,9 +6435,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.106" +version = "2.0.114" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ede7c438028d4436d71104916910f5bb611972c5cfd7f89b8300a8186e6fada6" +checksum = "d4d107df263a3013ef9b1879b0df87d706ff80f65a86ea879bd9c31f9b307c2a" dependencies = [ "proc-macro2", "quote", @@ -3274,6 +6449,9 @@ name = "sync_wrapper" version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] [[package]] name = "synstructure" @@ -3283,20 +6461,158 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.114", +] + +[[package]] +name = "sysinfo" +version = "0.30.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a5b4ddaee55fb2bea2bf0e5000747e5f5c0de765e5a5ff87f4cd106439f4bb3" +dependencies = [ + "cfg-if", + "core-foundation-sys", + "libc", + "ntapi", + "once_cell", + "rayon", + "windows", +] + +[[package]] +name = "system-configuration" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" +dependencies = [ + "bitflags 2.10.0", + "core-foundation 0.9.4", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", ] +[[package]] +name = "tagptr" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" + [[package]] name = "tempfile" -version = "3.20.0" +version = "3.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8a64e3985349f2441a1a9ef0b853f869006c3855f2cda6862a94d26ebb9d6a1" +checksum = "655da9c7eb6305c55742045d5a8d2037996d61d8de95806335c7c86ce0f82e9c" dependencies = [ "fastrand", - "getrandom 0.3.3", + "getrandom 0.3.4", "once_cell", "rustix", - "windows-sys 0.59.0", + "windows-sys 0.61.2", +] + +[[package]] +name = "terminfo" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4ea810f0692f9f51b382fff5893887bb4580f5fa246fde546e0b13e7fcee662" +dependencies = [ + "fnv", + "nom", + "phf 0.11.3", + "phf_codegen", +] + +[[package]] +name = "termios" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "411c5bf740737c7918b8b1fe232dca4dc9f8e754b8ad5e20966814001ed0ac6b" +dependencies = [ + "libc", +] + +[[package]] +name = "termwiz" +version = "0.23.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4676b37242ccbd1aabf56edb093a4827dc49086c0ffd764a5705899e0f35f8f7" +dependencies = [ + "anyhow", + "base64", + "bitflags 2.10.0", + "fancy-regex", + "filedescriptor", + "finl_unicode", + "fixedbitset 0.4.2", + "hex", + "lazy_static", + "libc", + "log", + "memmem", + "nix 0.29.0", + "num-derive", + "num-traits", + "ordered-float 4.6.0", + "pest", + "pest_derive", + "phf 0.11.3", + "sha2", + "signal-hook", + "siphasher", + "terminfo", + "termios", + "thiserror 1.0.69", + "ucd-trie", + "unicode-segmentation", + "vtparse", + "wezterm-bidi", + "wezterm-blob-leases", + "wezterm-color-types", + "wezterm-dynamic", + "wezterm-input-types", + "winapi", +] + +[[package]] +name = "test-case" +version = "3.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb2550dd13afcd286853192af8601920d959b14c401fcece38071d53bf0768a8" +dependencies = [ + "test-case-macros", +] + +[[package]] +name = "test-case-core" +version = "3.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adcb7fd841cd518e279be3d5a3eb0636409487998a4aff22f3de87b81e88384f" +dependencies = [ + "cfg-if", + "proc-macro2", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "test-case-macros" +version = "3.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c89e72a01ed4c579669add59014b9a524d609c0c88c6a585ce37485879f6ffb" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", + "test-case-core", ] [[package]] @@ -3310,22 +6626,51 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.12" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl 2.0.18", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ - "thiserror-impl", + "proc-macro2", + "quote", + "syn 2.0.114", ] [[package]] name = "thiserror-impl" -version = "2.0.12" +version = "2.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.114", +] + +[[package]] +name = "thread_local" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +dependencies = [ + "cfg-if", ] [[package]] @@ -3336,7 +6681,39 @@ checksum = "7e54bc85fc7faa8bc175c4bab5b92ba8d9a3ce893d0e9f42cc455c8ab16a9e09" dependencies = [ "byteorder", "integer-encoding", - "ordered-float", + "ordered-float 2.10.1", +] + +[[package]] +name = "time" +version = "0.3.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9da98b7d9b7dad93488a84b8248efc35352b0b2657397d4167e7ad67e5d535e5" +dependencies = [ + "deranged", + "libc", + "num-conv", + "num_threads", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" + +[[package]] +name = "time-macros" +version = "0.2.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78cc610bac2dcee56805c99642447d4c5dbde4d01f752ffea0199aee1f601dc4" +dependencies = [ + "num-conv", + "time-core", ] [[package]] @@ -3350,61 +6727,114 @@ dependencies = [ [[package]] name = "tinystr" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d4f6d1145dcb577acf783d4e601bc1d76a13337bb54e6233add580b07344c8b" +checksum = "42d3e9c45c09de15d06dd8acf5f4e0e399e85927b7f00711024eb7ae10fa4869" dependencies = [ "displaydoc", "zerovec", ] [[package]] -name = "tokio" -version = "1.47.1" +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "tinyvec" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa5fdc3bce6191a1dbc8c02d5c8bffcf557bafa17c124c5264a458f1b0613fa" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.49.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72a2903cd7736441aac9df9d7688bd0ce48edccaadf181c3b90be801e81d3d86" +dependencies = [ + "bytes", + "libc", + "mio", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "socket2 0.6.2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-rustls" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89e49afdadebb872d3145a5638b59eb0691ea23e46ca484037cfab3b76b95038" +checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" dependencies = [ - "backtrace", - "bytes", - "io-uring", - "libc", - "mio", - "parking_lot", - "pin-project-lite", - "signal-hook-registry", - "slab", - "socket2 0.6.0", - "tokio-macros", - "windows-sys 0.59.0", + "rustls 0.21.12", + "tokio", ] [[package]] -name = "tokio-macros" -version = "2.5.0" +name = "tokio-rustls" +version = "0.26.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.106", + "rustls 0.23.36", + "tokio", ] [[package]] name = "tokio-stream" -version = "0.1.17" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eca58d7bba4a75707817a2c44174253f9236b2d5fbd055602e9d5c07c139a047" +checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" dependencies = [ "futures-core", "pin-project-lite", "tokio", + "tokio-util", ] [[package]] name = "tokio-util" -version = "0.7.16" +version = "0.7.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14307c986784f72ef81c89db7d9e28d6ac26d16213b109ea501696195e6e3ce5" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" dependencies = [ "bytes", "futures-core", @@ -3415,25 +6845,25 @@ dependencies = [ [[package]] name = "tonic" -version = "0.13.1" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e581ba15a835f4d9ea06c55ab1bd4dce26fc53752c69a04aac00703bfb49ba9" +checksum = "a286e33f82f8a1ee2df63f4fa35c0becf4a85a0cb03091a15fd7bf0b402dc94a" dependencies = [ "async-trait", - "axum", + "axum 0.8.8", "base64", "bytes", - "h2", - "http", - "http-body", + "h2 0.4.13", + "http 1.4.0", + "http-body 1.0.1", "http-body-util", - "hyper", + "hyper 1.8.1", "hyper-timeout", "hyper-util", "percent-encoding", "pin-project", - "prost", - "socket2 0.5.10", + "socket2 0.6.2", + "sync_wrapper", "tokio", "tokio-stream", "tower", @@ -3442,11 +6872,22 @@ dependencies = [ "tracing", ] +[[package]] +name = "tonic-prost" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6c55a2d6a14174563de34409c9f92ff981d006f56da9c6ecd40d9d4a31500b0" +dependencies = [ + "bytes", + "prost", + "tonic", +] + [[package]] name = "tower" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" dependencies = [ "futures-core", "futures-util", @@ -3461,6 +6902,24 @@ dependencies = [ "tracing", ] +[[package]] +name = "tower-http" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" +dependencies = [ + "bitflags 2.10.0", + "bytes", + "futures-util", + "http 1.4.0", + "http-body 1.0.1", + "iri-string", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", +] + [[package]] name = "tower-layer" version = "0.3.3" @@ -3476,12 +6935,12 @@ checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" [[package]] name = "tpchgen" version = "2.0.1" -source = "git+https://github.com/clflushopt/tpchgen-rs?rev=482ee68#482ee68404d61a5308ca1fd0095b347b3831a5b4" +source = "git+https://github.com/clflushopt/tpchgen-rs?rev=e83365a5a9101906eb9f78c5607b83bc59849acf#e83365a5a9101906eb9f78c5607b83bc59849acf" [[package]] name = "tpchgen-arrow" version = "2.0.1" -source = "git+https://github.com/clflushopt/tpchgen-rs?rev=482ee68#482ee68404d61a5308ca1fd0095b347b3831a5b4" +source = "git+https://github.com/clflushopt/tpchgen-rs?rev=e83365a5a9101906eb9f78c5607b83bc59849acf#e83365a5a9101906eb9f78c5607b83bc59849acf" dependencies = [ "arrow", "tpchgen", @@ -3489,10 +6948,11 @@ dependencies = [ [[package]] name = "tracing" -version = "0.1.41" +version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" dependencies = [ + "log", "pin-project-lite", "tracing-attributes", "tracing-core", @@ -3500,22 +6960,44 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.30" +version = "0.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81383ab64e72a7a8b8e13130c49e3dab29def6d0c7d76a03087b3cf71c5c6903" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] name = "tracing-core" -version = "0.1.34" +version = "0.1.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9d12581f227e93f094d3af2ae690a574abb8a2b9b7a96e7cfe9647b2b617678" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" dependencies = [ "once_cell", + "valuable", +] + +[[package]] +name = "tracing-error" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b1581020d7a273442f5b45074a6a57d5757ad0a47dac0e9f0bd57b81936f3db" +dependencies = [ + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f30143827ddab0d256fd843b7a66d164e9f271cfa0dde49142c5ca0ca291f1e" +dependencies = [ + "sharded-slab", + "thread_local", + "tracing-core", ] [[package]] @@ -3526,21 +7008,27 @@ checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "twox-hash" -version = "2.1.1" +version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b907da542cbced5261bd3256de1b3a1bf340a3d37f93425a07362a1d687de56" +checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" [[package]] name = "typenum" -version = "1.18.0" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" + +[[package]] +name = "ucd-trie" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f" +checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" [[package]] name = "unicode-ident" -version = "1.0.18" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" +checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" [[package]] name = "unicode-segmentation" @@ -3548,6 +7036,17 @@ version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" +[[package]] +name = "unicode-truncate" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16b380a1238663e5f8a691f9039c73e1cdae598a30e9855f541d29b08b53e9a5" +dependencies = [ + "itertools 0.14.0", + "unicode-segmentation", + "unicode-width 0.2.2", +] + [[package]] name = "unicode-width" version = "0.1.14" @@ -3556,9 +7055,9 @@ checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" [[package]] name = "unicode-width" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a1a07cc7db3810833284e8d372ccdc6da29741639ecc70c9ec107df0fa6154c" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" [[package]] name = "untrusted" @@ -3568,9 +7067,9 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "url" -version = "2.5.7" +version = "2.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08bc136a29a3d1758e07a9cca267be308aeebf5cfd5a10f3f67ab2097683ef5b" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" dependencies = [ "form_urlencoded", "idna", @@ -3578,6 +7077,12 @@ dependencies = [ "serde", ] +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + [[package]] name = "utf8_iter" version = "1.0.4" @@ -3592,15 +7097,29 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.18.1" +version = "1.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f87b8aa10b915a06587d0dec516c282ff295b475d94abf425d62b57710070a2" +checksum = "ee48d38b119b0cd71fe4141b30f5ba9c7c5d9f4e7a3a8b4a674e4b6ef789976f" dependencies = [ - "getrandom 0.3.3", + "atomic", + "getrandom 0.3.4", "js-sys", + "serde_core", "wasm-bindgen", ] +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "vec_map" version = "0.8.2" @@ -3613,6 +7132,21 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "vsimd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" + +[[package]] +name = "vtparse" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d9b2acfb050df409c972a37d3b8e08cdea3bddb0c09db9d53137e504cfabed0" +dependencies = [ + "utf8parse", +] + [[package]] name = "walkdir" version = "2.5.0" @@ -3639,47 +7173,35 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" [[package]] -name = "wasi" -version = "0.14.2+wasi-0.2.4" +name = "wasip2" +version = "1.0.2+wasi-0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3" +checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5" dependencies = [ - "wit-bindgen-rt", + "wit-bindgen", ] [[package]] name = "wasm-bindgen" -version = "0.2.100" +version = "0.2.108" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5" +checksum = "64024a30ec1e37399cf85a7ffefebdb72205ca1c972291c51512360d90bd8566" dependencies = [ "cfg-if", "once_cell", "rustversion", "wasm-bindgen-macro", -] - -[[package]] -name = "wasm-bindgen-backend" -version = "0.2.100" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6" -dependencies = [ - "bumpalo", - "log", - "proc-macro2", - "quote", - "syn 2.0.106", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.50" +version = "0.4.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "555d470ec0bc3bb57890405e5d4322cc9ea83cebb085523ced7be4144dac1e61" +checksum = "70a6e77fd0ae8029c9ea0063f87c46fde723e7d887703d74ad2616d792e51e6f" dependencies = [ "cfg-if", + "futures-util", "js-sys", "once_cell", "wasm-bindgen", @@ -3688,9 +7210,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.100" +version = "0.2.108" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407" +checksum = "008b239d9c740232e71bd39e8ef6429d27097518b6b30bdf9086833bd5b6d608" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -3698,31 +7220,44 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.100" +version = "0.2.108" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" +checksum = "5256bae2d58f54820e6490f9839c49780dff84c65aeab9e772f15d5f0e913a55" dependencies = [ + "bumpalo", "proc-macro2", "quote", - "syn 2.0.106", - "wasm-bindgen-backend", + "syn 2.0.114", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.100" +version = "0.2.108" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d" +checksum = "1f01b580c9ac74c8d8f0c0e4afb04eeef2acf145458e52c03845ee9cd23e3d12" dependencies = [ "unicode-ident", ] +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "web-sys" -version = "0.3.77" +version = "0.3.85" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33b6dd2ef9186f1f2072e409e99cd22a975331a6b3591b12c764e0e55c60d5d2" +checksum = "312e32e551d92129218ea9a2452120f4aabc03529ef03e4d0d82fb2780608598" dependencies = [ "js-sys", "wasm-bindgen", @@ -3738,6 +7273,78 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "wezterm-bidi" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c0a6e355560527dd2d1cf7890652f4f09bb3433b6aadade4c9b5ed76de5f3ec" +dependencies = [ + "log", + "wezterm-dynamic", +] + +[[package]] +name = "wezterm-blob-leases" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "692daff6d93d94e29e4114544ef6d5c942a7ed998b37abdc19b17136ea428eb7" +dependencies = [ + "getrandom 0.3.4", + "mac_address", + "sha2", + "thiserror 1.0.69", + "uuid", +] + +[[package]] +name = "wezterm-color-types" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7de81ef35c9010270d63772bebef2f2d6d1f2d20a983d27505ac850b8c4b4296" +dependencies = [ + "csscolorparser", + "deltae", + "lazy_static", + "wezterm-dynamic", +] + +[[package]] +name = "wezterm-dynamic" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f2ab60e120fd6eaa68d9567f3226e876684639d22a4219b313ff69ec0ccd5ac" +dependencies = [ + "log", + "ordered-float 4.6.0", + "strsim 0.11.1", + "thiserror 1.0.69", + "wezterm-dynamic-derive", +] + +[[package]] +name = "wezterm-dynamic-derive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46c0cf2d539c645b448eaffec9ec494b8b19bd5077d9e58cb1ae7efece8d575b" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "wezterm-input-types" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7012add459f951456ec9d6c7e6fc340b1ce15d6fc9629f8c42853412c029e57e" +dependencies = [ + "bitflags 1.3.2", + "euclid", + "lazy_static", + "serde", + "wezterm-dynamic", +] + [[package]] name = "winapi" version = "0.3.9" @@ -3756,11 +7363,11 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] name = "winapi-util" -version = "0.1.9" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -3769,69 +7376,93 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e48a53791691ab099e5e2ad123536d0fff50652600abaf43bbf952894110d0be" +dependencies = [ + "windows-core 0.52.0", + "windows-targets 0.52.6", +] + [[package]] name = "windows-core" -version = "0.61.2" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-core" +version = "0.62.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" dependencies = [ "windows-implement", "windows-interface", - "windows-link 0.1.3", + "windows-link", "windows-result", "windows-strings", ] [[package]] name = "windows-implement" -version = "0.60.0" +version = "0.60.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] name = "windows-interface" -version = "0.59.1" +version = "0.59.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] name = "windows-link" -version = "0.1.3" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" [[package]] -name = "windows-link" -version = "0.2.0" +name = "windows-registry" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45e46c0661abb7180e7b9c281db115305d49ca1709ab8242adf09666d2173c65" +checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" +dependencies = [ + "windows-link", + "windows-result", + "windows-strings", +] [[package]] name = "windows-result" -version = "0.3.4" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" dependencies = [ - "windows-link 0.1.3", + "windows-link", ] [[package]] name = "windows-strings" -version = "0.4.2" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56e6c93f3a0c3b36176cb1327a4958a0353d5d166c2a35cb268ace15e91d3b57" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" dependencies = [ - "windows-link 0.1.3", + "windows-link", ] [[package]] @@ -3858,7 +7489,16 @@ version = "0.60.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" dependencies = [ - "windows-targets 0.53.2", + "windows-targets 0.53.5", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", ] [[package]] @@ -3879,18 +7519,19 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.53.2" +version = "0.53.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c66f69fcc9ce11da9966ddb31a40968cad001c5bedeb5c2b82ede4253ab48aef" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" dependencies = [ - "windows_aarch64_gnullvm 0.53.0", - "windows_aarch64_msvc 0.53.0", - "windows_i686_gnu 0.53.0", - "windows_i686_gnullvm 0.53.0", - "windows_i686_msvc 0.53.0", - "windows_x86_64_gnu 0.53.0", - "windows_x86_64_gnullvm 0.53.0", - "windows_x86_64_msvc 0.53.0", + "windows-link", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", ] [[package]] @@ -3901,9 +7542,9 @@ checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" [[package]] name = "windows_aarch64_gnullvm" -version = "0.53.0" +version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" [[package]] name = "windows_aarch64_msvc" @@ -3913,9 +7554,9 @@ checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" [[package]] name = "windows_aarch64_msvc" -version = "0.53.0" +version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" [[package]] name = "windows_i686_gnu" @@ -3925,9 +7566,9 @@ checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" [[package]] name = "windows_i686_gnu" -version = "0.53.0" +version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1dc67659d35f387f5f6c479dc4e28f1d4bb90ddd1a5d3da2e5d97b42d6272c3" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" [[package]] name = "windows_i686_gnullvm" @@ -3937,9 +7578,9 @@ checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" [[package]] name = "windows_i686_gnullvm" -version = "0.53.0" +version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" [[package]] name = "windows_i686_msvc" @@ -3949,9 +7590,9 @@ checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" [[package]] name = "windows_i686_msvc" -version = "0.53.0" +version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" [[package]] name = "windows_x86_64_gnu" @@ -3961,9 +7602,9 @@ checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" [[package]] name = "windows_x86_64_gnu" -version = "0.53.0" +version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" [[package]] name = "windows_x86_64_gnullvm" @@ -3973,9 +7614,9 @@ checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" [[package]] name = "windows_x86_64_gnullvm" -version = "0.53.0" +version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" [[package]] name = "windows_x86_64_msvc" @@ -3985,41 +7626,40 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "windows_x86_64_msvc" -version = "0.53.0" +version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" [[package]] -name = "wit-bindgen-rt" -version = "0.39.0" +name = "wit-bindgen" +version = "0.51.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" -dependencies = [ - "bitflags 2.9.1", -] +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" [[package]] name = "writeable" -version = "0.6.1" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea2f10b9bb0928dfb1b42b65e1f9e36f7f54dbdf08457afefb38afcdec4fa2bb" +checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" [[package]] -name = "xz2" -version = "0.1.7" +name = "xmlparser" +version = "0.13.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "388c44dc09d76f1536602ead6d325eb532f5c122f17782bd57fb47baeeb767e2" -dependencies = [ - "lzma-sys", -] +checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" + +[[package]] +name = "yansi" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" [[package]] name = "yoke" -version = "0.8.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f41bb01b8226ef4bfd589436a297c53d118f65921786300e427be8d487695cc" +checksum = "72d6e5c6afb84d73944e5cedb052c4680d5657337201555f9f2a16b7406d4954" dependencies = [ - "serde", "stable_deref_trait", "yoke-derive", "zerofrom", @@ -4027,34 +7667,34 @@ dependencies = [ [[package]] name = "yoke-derive" -version = "0.8.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38da3c9736e16c5d3c8c597a9aaa5d1fa565d0532ae05e27c24aa62fb32c0ab6" +checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.114", "synstructure", ] [[package]] name = "zerocopy" -version = "0.8.26" +version = "0.8.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1039dd0d3c310cf05de012d8a39ff557cb0d23087fd44cad61df08fc31907a2f" +checksum = "7456cf00f0685ad319c5b1693f291a650eaf345e941d082fc4e03df8a03996ac" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.26" +version = "0.8.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ecf5b4cc5364572d7f4c329661bcc82724222973f2cab6f050a4e5c22f75181" +checksum = "1328722bbf2115db7e19d69ebcc15e795719e2d66b60827c6a69a117365e37a0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.114", ] [[package]] @@ -4074,15 +7714,35 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.114", "synstructure", ] +[[package]] +name = "zeroize" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85a5b4158499876c763cb03bc4e49185d3cccbabb15b33c627f7884f43db852e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", +] + [[package]] name = "zerotrie" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36f0bbd478583f79edad978b407914f61b2972f5af6fa089686016be8f9af595" +checksum = "2a59c17a5562d507e4b54960e8569ebee33bee890c70aa3fe7b97e85a9fd7851" dependencies = [ "displaydoc", "yoke", @@ -4091,9 +7751,9 @@ dependencies = [ [[package]] name = "zerovec" -version = "0.11.2" +version = "0.11.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a05eb080e015ba39cc9e23bbe5e7fb04d5fb040350f99f34e338d5fdd294428" +checksum = "6c28719294829477f525be0186d13efa9a3c602f7ec202ca9e353d310fb9a002" dependencies = [ "yoke", "zerofrom", @@ -4102,20 +7762,65 @@ dependencies = [ [[package]] name = "zerovec-derive" -version = "0.11.1" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b96237efa0c878c64bd89c436f661be4e46b2f3eff1ebb976f7ef2321d2f58f" +checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.106", + "syn 2.0.114", +] + +[[package]] +name = "zip" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa8cd6af31c3b31c6631b8f483848b91589021b28fffe50adada48d4f4d2ed1" +dependencies = [ + "aes", + "arbitrary", + "bzip2", + "constant_time_eq 0.3.1", + "crc32fast", + "deflate64", + "flate2", + "getrandom 0.3.4", + "hmac", + "indexmap", + "liblzma", + "memchr", + "pbkdf2", + "ppmd-rust", + "sha1", + "time", + "zeroize", + "zopfli", + "zstd", ] [[package]] name = "zlib-rs" -version = "0.5.1" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40990edd51aae2c2b6907af74ffb635029d5788228222c4bb811e9351c0caad3" + +[[package]] +name = "zmij" +version = "1.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "626bd9fa9734751fc50d6060752170984d7053f5a39061f524cda68023d4db8a" +checksum = "3ff05f8caa9038894637571ae6b9e29466c1f4f829d26c9b28f869a29cbe3445" + +[[package]] +name = "zopfli" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249" +dependencies = [ + "bumpalo", + "crc32fast", + "log", + "simd-adler32", +] [[package]] name = "zstd" @@ -4137,9 +7842,9 @@ dependencies = [ [[package]] name = "zstd-sys" -version = "2.0.15+zstd.1.5.7" +version = "2.0.16+zstd.1.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb81183ddd97d0c74cedf1d50d85c8d08c1b8b68ee863bdee9e706eedba1a237" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" dependencies = [ "cc", "pkg-config", diff --git a/Cargo.toml b/Cargo.toml index c60857d1..f39dc7d0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,9 +1,9 @@ [workspace] -members = ["benchmarks"] +members = ["benchmarks", "cli", "console"] [workspace.dependencies] -datafusion = { version = "50.0.0", default-features = false } -datafusion-proto = { version = "50.0.0" } +datafusion = { git = "https://github.com/DataDog/datafusion", rev = "15d82292e899bc06adbbdbfacea67070bc89fb95", package = "datafusion", default-features = false } +datafusion-proto = { git = "https://github.com/DataDog/datafusion", rev = "15d82292e899bc06adbbdbfacea67070bc89fb95", package = "datafusion-proto" } [package] name = "datafusion-distributed" @@ -12,55 +12,78 @@ edition = "2024" [dependencies] chrono = { version = "0.4.42" } -datafusion = { workspace = true } +datafusion = { workspace = true, features = [ + "parquet", + "sql", + "unicode_expressions", + "datetime_expressions", +] } datafusion-proto = { workspace = true } -arrow-flight = "56.1.0" -async-trait = "0.1.88" -tokio = { version = "1.46.1", features = ["full"] } -# Updated to 0.13.1 to match arrow-flight 56.1.0 -tonic = { version = "0.13.1", features = ["transport"] } +arrow-flight = "57.1.0" +arrow-select = "57.1.0" +arrow-ipc = { version = "57.1.0", features = ["zstd"] } +async-trait = "0.1.89" +tokio = { version = "1.48", features = ["full"] } +tonic = { version = "0.14.1", features = ["transport"] } tower = "0.5.2" http = "1.3.1" itertools = "0.14.0" futures = "0.3.31" -url = "2.5.4" -uuid = "1.17.0" +url = "2.5.7" +uuid = "1.19" delegate = "0.13.4" -dashmap = "6.1.0" -prost = "0.13.5" -rand = "0.8.5" -object_store = "0.12.3" -bytes = "1.10.1" +dashmap = "6.0.1" +prost = "0.14.1" +rand = "0.9" +object_store = "0.12.4" +bytes = "1.11" +pin-project = "1.1.10" +tokio-stream = { version = "0.1.17", features = ["sync"] } +tokio-util = "0.7" +moka = { version = "0.12", features = ["sync", "future"] } +crossbeam-queue = "0.3" +sketches-ddsketch = "0.3.0" # integration_tests deps -insta = { version = "1.43.1", features = ["filters"], optional = true } -tpchgen = { git = "https://github.com/clflushopt/tpchgen-rs", rev = "482ee68", optional = true } -tpchgen-arrow = { git = "https://github.com/clflushopt/tpchgen-rs", rev = "482ee68", optional = true } -parquet = { version = "56.1.0", optional = true } -arrow = { version = "56.1.0", optional = true } -tokio-stream = { version = "0.1.17", optional = true } +insta = { version = "1.46.0", features = ["filters"], optional = true } +tpchgen = { git = "https://github.com/clflushopt/tpchgen-rs", rev = "e83365a5a9101906eb9f78c5607b83bc59849acf", optional = true } +tpchgen-arrow = { git = "https://github.com/clflushopt/tpchgen-rs", rev = "e83365a5a9101906eb9f78c5607b83bc59849acf", optional = true } +parquet = { version = "57.1.0", optional = true } +arrow = { version = "57.1.0", optional = true } hyper-util = { version = "0.1.16", optional = true } -pin-project = "1.1.10" +pretty_assertions = { version = "1.4", optional = true } +reqwest = { version = "0.12", optional = true } +zip = { version = "4.0", optional = true } +tonic-prost = "0.14.2" [features] +avro = ["datafusion/avro"] integration = [ - "insta", - "tpchgen", - "tpchgen-arrow", - "parquet", - "arrow", - "tokio-stream", - "hyper-util", + "insta", + "tpchgen", + "tpchgen-arrow", + "parquet", + "arrow", + "hyper-util", + "pretty_assertions", + "reqwest", + "zip", ] tpch = ["integration"] +tpcds = ["integration"] +clickbench = ["integration"] [dev-dependencies] structopt = "0.3" -insta = { version = "1.43.1", features = ["filters"] } -tpchgen = { git = "https://github.com/clflushopt/tpchgen-rs", rev = "482ee68" } -tpchgen-arrow = { git = "https://github.com/clflushopt/tpchgen-rs", rev = "482ee68" } -parquet = "56.1.0" -arrow = "56.1.0" -tokio-stream = "0.1.17" +insta = { version = "1.46.0", features = ["filters"] } +tpchgen = { git = "https://github.com/clflushopt/tpchgen-rs", rev = "e83365a5a9101906eb9f78c5607b83bc59849acf" } +tpchgen-arrow = { git = "https://github.com/clflushopt/tpchgen-rs", rev = "e83365a5a9101906eb9f78c5607b83bc59849acf" } +parquet = "57.1.0" +arrow = "57.1.0" +tokio-stream = { version = "0.1.17", features = ["sync"] } hyper-util = "0.1.16" +pretty_assertions = "1.4" +reqwest = "0.12" +zip = "4.0" +test-case = "3.3.1" diff --git a/README.md b/README.md index bcfabe8d..15bd39b4 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,14 @@ # DataFusion Distributed -Library that brings distributed execution capabilities to [DataFusion](https://github.com/apache/datafusion). +Library that brings distributed execution capabilities to [Apache DataFusion](https://github.com/apache/datafusion). -> [!WARNING] -> This project is currently under construction and is not yet ready for production use. +> [!NOTE] +> This is project is not part of Apache DataFusion ## What can you do with this crate? -This crate is a toolkit that extends [DataFusion](https://github.com/apache/datafusion) with distributed capabilities, +This crate is a toolkit that extends [Apache DataFusion](https://github.com/apache/datafusion) with distributed +capabilities, providing a developer experience as close as possible to vanilla DataFusion while being unopinionated about the networking stack used for hosting the different workers involved in a query. @@ -22,288 +23,43 @@ capabilities with minimal changes. expected to leverage their own infrastructure for hosting DataFusion nodes. - No coordinator-worker architecture. To keep infrastructure simple, any node can act as a coordinator or a worker. -## Architecture +# Benchmarks -Before diving into the architecture, it's important to clarify some terms and what they mean: +The benchmarking code is public an open for anyone to easily reproduce. It uses AWS CDK for automating the creation +of the benchmarking cluster so that anyone can reproduce the same results in their own AWS account. The code can +be found in the [benchmarks/cdk](./benchmarks/cdk) directory. -- `worker`: a physical machine listening to serialized execution plans over an Arrow Flight interface. -- `network boundary`: a node in the plan that streams data from a network interface rather than directly from its - children. Implemented as DataFusion `ExecutionPlan`s: `NeworkShuffle` and `NetworkCoalesce`. -- `stage`: a portion of the plan separated by a network boundary from other parts of the plan. - Implemented as a DataFusion `ExecutionPlan`: `StageExec`. -- `task`: a unit of work in a stage that executes the inner plan in parallel to other tasks. -- `subplan`: a slice of the overall plan +### TPC-H SF1 -A distributed DataFusion query is executed in a very similar fashion as a normal DataFusion query with one key -difference: +![benchmarks_sf1.png](https://github.com/user-attachments/assets/2f922066-7382-4c31-9e76-74b1ca053bfc) -The physical plan is divided into stages and each stage is assigned tasks that run the inner plan in parallel in -different workers. All of this is done at the physical plan level, implemented as a `PhysicalOptimizerRule` that: +### TPC-H SF10 -1. Inspects the non-distributed plan, placing network boundaries (`NetworkShuffle` and `NetworkCoalesce` nodes) in - the appropriate places. -2. Based on the placed network boundaries, divides the plan into stages and assigns tasks to them. +![benchmarks_sf10.png](https://github.com/user-attachments/assets/08fd3090-92bf-43fd-b80c-12e3a127e724) -### Types of network boundaries +# Docs -There are different flavors of network boundaries that can stream data over the wire across stages: +The user and contributor guide can be found here: -- `NetworkShuffle`: the equivalent to a `RepartitionExec` that spreads out different partitions to different tasks - across stages. Refer to the [network_shuffle.rs](./src/execution_plans/network_shuffle.rs) docs for a more detailed - explanation. -- `NetworkCoalesce`: the equivalent to a `CoalescePartitionsExec` but with tasks. It collapses P partitions across - N tasks into a single task with N*P partitions. Refer to - the [network_coalesce.rs](./src/execution_plans/network_coalesce.rs) - docs for a more detailed explanation - -### Example plan distribution - -For example, imagine we have a plan that looks like this: - -``` -┌───────────────────────┐ -│CoalescePartitionsExec │ -└───────────────────────┘ - ▲ - │ -┌───────────────────────┐ -│ ProjectionExec │ -└───────────────────────┘ - ▲ - │ -┌───────────────────────┐ -│ AggregateExec │ -│ (final) │ -└───────────────────────┘ - ▲ - │ -┌───────────────────────┐ -│ RepartitionExec │ -└───────────────────────┘ - ▲ - │ -┌───────────────────────┐ -│ AggregateExec │ -│ (partial) │ -└───────────────────────┘ - ▲ - │ -┌───────────────────────┐ -│ DataSourceExec │ -└───────────────────────┘ -``` - -By looking at the existing nodes, we can decide to place some network boundaries like this: - -``` -┌───────────────────────┐ -│CoalescePartitionsExec │ -└───────────────────────┘ - ▲ - │ -┌───────────────────────┐ -│ NetworkCoalesceExec │ <- injected during distributed planning. This will collapse all tasks into one, -└───────────────────────┘ wihtout performing any changes in the overall partitioning scheme. - ▲ - │ -┌───────────────────────┐ -│ ProjectionExec │ -└───────────────────────┘ - ▲ - │ -┌───────────────────────┐ -│ AggregateExec │ -│ (final) │ -└───────────────────────┘ - ▲ - │ -┌───────────────────────┐ -│ NetworkShuffleExec │ <- injected during distributed planning. This will shuffle the data across the network, -└───────────────────────┘ fanning out the different partitions to different workers. - ▲ - │ -┌───────────────────────┐ -│ RepartitionExec │ -└───────────────────────┘ - ▲ - │ -┌───────────────────────┐ -│ AggregateExec │ -│ (partial) │ -└───────────────────────┘ - ▲ - │ -┌───────────────────────┐ -│ ParititonIsolatorExec │ <- injected during distributed planning. As this lower part of the plan will run in multiple -└───────────────────────┘ tasks in parallel, this node makes sure no two same partitions from the underlying - ▲ DataSourceExec are read in two different tasks. - │ -┌───────────────────────┐ -│ DataSourceExec │ -└───────────────────────┘ -``` - -Once the network boundaries are properly placed, the distributed planner will break down the plan into stages, -and will assign tasks to them: - -``` -┌ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ Stage 3 - ┌───────────────────────┐ -│ │CoalescePartitionsExec │ │ - └───────────────────────┘ -│ ▲ │ - │ -│ ┌───────────────────────┐ │ - │ NetworkCoalesceExec │ -│ └───────────────────────┘ │ - ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ▲ ─ ▲ ─ ▲ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ - ┌────────────────┘ │ └────────────────┐ -┌ ─ ─ ─ ─ ─ │ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─│─ ─ ─ ─ ─ ─ ─ ─ ─ ─ │ ─ ─ Stage 2 - ┌───────────────────┐┌───────────────────┐┌───────────────────┐ -│ │ ProjectionExec ││ ProjectionExec ││ ProjectionExec │ │ - └───────────────────┘└───────────────────┘└───────────────────┘ -│ ▲ ▲ ▲ │ - │ │ │ -│ ┌───────────────────┐┌───────────────────┐┌───────────────────┐ │ - │ AggregateExec ││ AggregateExec ││ AggregateExec │ -│ │ (final) ││ (final) ││ (final) │ │ - └───────────────────┘└───────────────────┘└───────────────────┘ -│ ▲ ▲ ▲ │ - │ │ │ -│ ┌───────────────────┐┌───────────────────┐┌───────────────────┐ │ - │NetworkShuffleExec ││NetworkShuffleExec ││NetworkShuffleExec │ -│ └───────────────────┘└───────────────────┘└───────────────────┘ │ - ─ ─ ─ ─ ─ ─▲─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ▲ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─▲─ ─ ─ ─ ─ ─ - └────────┬───────────┴──────────┬─────────┘ -┌ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─│─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ │ ─ ─ ─ ─ ─ ─ ─ Stage 1 - ┌─────────────────────┐┌─────────────────────┐ -│ │ RepartitionExec ││ RepartitionExec │ │ - └─────────────────────┘└─────────────────────┘ -│ ▲ ▲ │ - │ │ -│ ┌─────────────────────┐┌─────────────────────┐ │ - │ AggregateExec ││ AggregateExec │ -│ │ (partial) ││ (partial) │ │ - └─────────────────────┘└─────────────────────┘ -│ ▲ ▲ │ - │ │ -│ ┌─────────────────────┐┌─────────────────────┐ │ - │PartitionIsolatorExec││PartitionIsolatorExec│ -│ └─────────────────────┘└─────────────────────┘ │ - ▲ ▲ -│ │ │ │ - ┌─────────────────────┐┌─────────────────────┐ -│ │ DataSourceExec ││ DataSourceExec │ │ - └─────────────────────┘└─────────────────────┘ -└ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ┘ -``` - -The plan is immediately executable, and the same process that planned the distributed query can start executing the head -stage (stage 3). The `NetworkCoalesceExec` in that stage will know from which tasks to gather data from stage 2, and -will issue 3 concurrent Arrow Flight requests to the appropriate physical nodes. Same goes from stage 2 to stage 1, -but with the difference that this time data is repartitioned and shuffled, that way each task in stage 2 handles a -different set of partitions. - -This means that: - -1. The head stage is executed normally as if the query was not distributed. -2. Upon calling `.execute()` on `NetworkCoalesceExec`, instead of propagating the `.execute()` call on its child, - the subplan is serialized and sent over the wire to be executed on another worker. -3. The next worker, which is hosting an Arrow Flight Endpoint listening for gRPC requests over an HTTP server, will pick - up the request containing the serialized chunk of the overall plan and execute it. -4. This is repeated for each stage, and data will start flowing from bottom to top until it reaches the head stage. +https://datafusion-contrib.github.io/datafusion-distributed ## Getting familiar with distributed DataFusion There are some runnable examples showcasing how to provide a localhost implementation for Distributed DataFusion in [examples/](examples): -- [localhost_worker.rs](examples/localhost_worker.rs): code that spawns an Arrow Flight Endpoint listening for physical +- [localhost_worker.rs](examples/localhost_worker.rs): code that spawns a Worker listening for physical plans over the network. -- [localhost_run.rs](examples/localhost_run.rs): code that distributes a query across the spawned Arrow Flight Endpoints - and executes it. +- [localhost_run.rs](examples/localhost_run.rs): code that distributes a query across the spawned Workers and executes + it. The integration tests also provide an idea about how to use the library and what can be achieved with it: -- [tpch_validation_test.rs](tests/tpch_validation_test.rs): executes all TPCH queries and performs assertions over the - distributed plans and the results vs running the queries in single node mode with a small scale factor. +- [tpch_validation_test.rs](tests/tpch_plans_test.rs): executes all TPCH queries and performs assertions over the + distributed plans. - [custom_config_extension.rs](tests/custom_config_extension.rs): showcases how to propagate custom DataFusion config extensions. - [custom_extension_codec.rs](tests/custom_extension_codec.rs): showcases how to propagate custom physical extension codecs. - [distributed_aggregation.rs](tests/distributed_aggregation.rs): showcases how to manually place `ArrowFlightReadExec` nodes in a plan and build a distributed query out of it. - -## Development - -### Prerequisites - -- Rust 1.85.1 or later (specified in `rust-toolchain.toml`) -- Git LFS for test data - -### Setup - -1. **Clone the repository:** - ```bash - git clone git@github.com:datafusion-contrib/datafusion-distributed - cd datafusion-distributed - ``` - -2. **Install Git LFS and fetch test data:** - ```bash - git lfs install - git lfs checkout - ``` - -### Running Tests - -**Unit and integration tests:** - -```bash -cargo test --features integration -``` - -### Running Examples - -**Start localhost workers:** - -```bash -# Terminal 1 -cargo run --example localhost_worker -- 8080 --cluster-ports 8080,8081 - -# Terminal 2 -cargo run --example localhost_worker -- 8081 --cluster-ports 8080,8081 -``` - -**Execute distributed queries:** - -```bash -cargo run --example localhost_run -- 'SELECT count(*) FROM weather' --cluster-ports 8080,8081 -``` - -### Benchmarks - -**Generate TPC-H benchmark data:** - -```bash -cd benchmarks -./gen-tpch.sh -``` - -**Run TPC-H benchmarks:** - -```bash -cargo run -p datafusion-distributed-benchmarks --release -- tpch --path benchmarks/data/tpch_sf1 -``` - -### Project Structure - -- `src/` - Core library code - - `flight_service/` - Arrow Flight service implementation - - `plan/` - Physical plan extensions and operators - - `stage/` - Execution stage management - - `common/` - Shared utilities -- `examples/` - Usage examples -- `tests/` - Integration tests -- `benchmarks/` - Performance benchmarks -- `testdata/` - Test datasets diff --git a/benchmarks/Cargo.toml b/benchmarks/Cargo.toml index 9c939f06..3127cb4e 100644 --- a/benchmarks/Cargo.toml +++ b/benchmarks/Cargo.toml @@ -8,19 +8,44 @@ default-run = "dfbench" datafusion = { workspace = true } datafusion-proto = { workspace = true } datafusion-distributed = { path = "..", features = ["integration"] } -tokio = { version = "1.46.1", features = ["full"] } -parquet = { version = "56.1.0" } +tokio = { version = "1.48", features = ["full"] } +parquet = { version = "57.1.0" } structopt = { version = "0.3.26" } log = "0.4.27" serde = "1.0.219" serde_json = "1.0.141" env_logger = "0.11.8" -async-trait = "0.1.88" -chrono = "0.4.41" +async-trait = "0.1.89" +chrono = "0.4.42" futures = "0.3.31" -dashmap = "6.1.0" -prost = "0.13.5" +dashmap = "6.0.1" +prost = "0.14.1" +url = "2.5.7" +arrow-flight = "57.1.0" +tonic = { version = "0.14.1", features = ["transport"] } +axum = "0.7" +object_store = { version = "0.12.4", features = ["aws"] } +aws-config = "1" +aws-sdk-ec2 = "1" +openssl = { version = "0.10", features = ["vendored"] } +clap = "4.5" +mimalloc = "0.1" + +[dev-dependencies] +criterion = "0.5" +sysinfo = "0.30" + +[build-dependencies] +built = { version = "0.8" , features = ["git2", "chrono"]} [[bin]] name = "dfbench" -path = "src/bin/dfbench.rs" +path = "src/main.rs" + +[[bin]] +name = "worker" +path = "cdk/bin/worker.rs" + +[[bench]] +name = "broadcast_cache_scenarios" +harness = false diff --git a/benchmarks/README.md b/benchmarks/README.md index e411e900..926268ff 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -1,81 +1,41 @@ # Distributed DataFusion Benchmarks -### Generating TPCH data +### Generating Benchmarking data -Generate TPCH data into the `data/` dir +Generate datasets into `benchmarks/data/`. ```shell +# TPC-H (default: SCALE_FACTOR=1, PARTITIONS=16 - override by setting these environment variables) ./gen-tpch.sh -``` - -### Running TPCH benchmarks in single-node mode -After generating the data with the command above, the benchmarks can be run with - -```shell -cargo run -p datafusion-distributed-benchmarks --release -- tpch +# TPC-DS (only SCALE_FACTOR=1 is supported) +./gen-tpcds.sh ``` -For preloading the TPCH data in-memory, the `-m` flag can be passed - -```shell -cargo run -p datafusion-distributed-benchmarks --release -- tpch -m -``` +### Running Benchmarks in single-node mode -For running the benchmarks with using just a specific amount of physical threads: +After generating the data with the command above, the benchmarks can be run with: ```shell -cargo run -p datafusion-distributed-benchmarks --release -- tpch -m --threads 3 +WORKERS=0 ./benchmarks/run.sh --threads 2 --dataset tpch_sf1 ``` -### Running TPCH benchmarks in distributed mode +- `--threads`: This is the physical threads that the Tokio runtime will use for executing the + binary. It's recommended to set `--threads` to something small, like `2`, for throttling each + individual process running queries, and simulate how adding throttled workers can speed up the + queries. +- `--dataset`: Dataset directory name under `benchmarks/data/` (e.g. `tpch_sf1`, `tpcds_sf1`). -Running the benchmarks in distributed mode implies: +### Running Benchmarks benchmarks in distributed mode -- running 1 or more workers in separate terminals -- running the benchmarks in an additional terminal - -The workers can be spawned by passing the `--spawn ` flag, for example, for spawning 3 workers: - -```shell -cargo run -p datafusion-distributed-benchmarks --release -- tpch --spawn 8000 -``` - -```shell -cargo run -p datafusion-distributed-benchmarks --release -- tpch --spawn 8001 -``` +The same script is used for running distributed benchmarks: ```shell -cargo run -p datafusion-distributed-benchmarks --release -- tpch --spawn 8002 +WORKERS=8 ./benchmarks/run.sh --threads 2 --dataset tpch_sf1 --files-per-task 2 ``` -With the three workers running in separate terminals, the TPCH benchmarks can be run in distributed mode with: - -```shell -cargo run -p datafusion-distributed-benchmarks --release -- tpch --workers 8000,8001,8002 -``` - -A good way of measuring the impact of distribution is to limit the physical threads each worker can use. For example, -it's expected that running 8 workers with 2 physical threads each one (8 * 2 = 16 total) is faster than running in -single-node with just 2 threads (1 * 3 = 2 total). - -```shell -cargo run -p datafusion-distributed-benchmarks --release -- tpch -m --threads 2 --spawn 8000 & -cargo run -p datafusion-distributed-benchmarks --release -- tpch -m --threads 2 --spawn 8001 & -cargo run -p datafusion-distributed-benchmarks --release -- tpch -m --threads 2 --spawn 8002 & -cargo run -p datafusion-distributed-benchmarks --release -- tpch -m --threads 2 --spawn 8003 & -cargo run -p datafusion-distributed-benchmarks --release -- tpch -m --threads 2 --spawn 8004 & -cargo run -p datafusion-distributed-benchmarks --release -- tpch -m --threads 2 --spawn 8005 & -cargo run -p datafusion-distributed-benchmarks --release -- tpch -m --threads 2 --spawn 8006 & -cargo run -p datafusion-distributed-benchmarks --release -- tpch -m --threads 2 --spawn 8007 & -``` - -```shell -cargo run -p datafusion-distributed-benchmarks --release -- tpch -m --threads 2 --workers 8000,8001,8002,8003,8004,8005,8006,8007 -``` - -The `run.sh` script already does this for you in a more ergonomic way: - -```shell -WORKERS=8 run.sh --threads 2 -m -``` \ No newline at end of file +- `WORKERS`: Env variable that sets the amount of localhost workers used in the query. +- `--threads`: Sets the Tokio runtime threads for each individual worker and for the benchmarking + binary. +- `--dataset`: Dataset directory name under `benchmarks/data/`. +- `--files-per-task`: How many files each distributed task will handle. diff --git a/benchmarks/benches/broadcast_cache_scenarios.rs b/benchmarks/benches/broadcast_cache_scenarios.rs new file mode 100644 index 00000000..24324c29 --- /dev/null +++ b/benchmarks/benches/broadcast_cache_scenarios.rs @@ -0,0 +1,436 @@ +use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; +use datafusion::arrow::array::UInt8Array; +use datafusion::arrow::datatypes::{DataType, Field, Schema, SchemaRef}; +use datafusion::arrow::record_batch::RecordBatch; +use datafusion::common::Statistics; +use datafusion::error::Result; +use datafusion::execution::{SendableRecordBatchStream, TaskContext}; +use datafusion::physical_expr::EquivalenceProperties; +use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType}; +use datafusion::physical_plan::stream::RecordBatchStreamAdapter; +use datafusion::physical_plan::{ + DisplayAs, DisplayFormatType, ExecutionPlan, Partitioning, PlanProperties, +}; +use datafusion::prelude::SessionContext; +use datafusion_distributed::BroadcastExec; +use futures::{StreamExt, stream}; +use std::any::Any; +use std::sync::{ + Arc, + atomic::{AtomicBool, AtomicU64, Ordering}, +}; +use std::thread; +use std::time::{Duration, Instant}; +use sysinfo::{System, get_current_pid}; +use tokio::runtime::Builder as RuntimeBuilder; + +#[derive(Clone, Copy)] +enum ConsumerBehavior { + Fast, + Slow(Duration), + CancelAfter(usize), +} + +#[derive(Clone)] +struct ConsumerSpec { + partition: usize, + behavior: ConsumerBehavior, +} + +#[derive(Clone)] +struct Scenario { + name: &'static str, + input_partitions: usize, + consumer_tasks: usize, + rows_per_batch: usize, + num_batches: usize, + consumers: Vec, +} + +struct PeakRssSampler { + stop: Arc, + peak_kb: Arc, + handle: Option>, +} + +impl PeakRssSampler { + fn start(interval: Duration) -> Self { + let stop = Arc::new(AtomicBool::new(false)); + let peak_kb = Arc::new(AtomicU64::new(0)); + let stop_clone = Arc::clone(&stop); + let peak_clone = Arc::clone(&peak_kb); + let handle = thread::spawn(move || { + let mut sys = System::new(); + let pid = get_current_pid().expect("pid"); + while !stop_clone.load(Ordering::Relaxed) { + sys.refresh_process(pid); + if let Some(proc) = sys.process(pid) { + let mem_kb = proc.memory(); + peak_clone.fetch_max(mem_kb, Ordering::Relaxed); + } + thread::sleep(interval); + } + }); + Self { + stop, + peak_kb, + handle: Some(handle), + } + } + + fn stop(mut self) -> u64 { + self.stop.store(true, Ordering::Relaxed); + if let Some(handle) = self.handle.take() { + let _ = handle.join(); + } + self.peak_kb.load(Ordering::Relaxed) + } +} + +#[derive(Debug)] +struct SyntheticExec { + schema: SchemaRef, + partitions: usize, + batches: Arc>>, + properties: PlanProperties, +} + +impl SyntheticExec { + fn new(schema: SchemaRef, partitions: usize, batches: Arc>>) -> Self { + let properties = PlanProperties::new( + EquivalenceProperties::new(Arc::clone(&schema)), + Partitioning::UnknownPartitioning(partitions), + EmissionType::Incremental, + Boundedness::Bounded, + ); + Self { + schema, + partitions, + batches, + properties, + } + } +} + +impl DisplayAs for SyntheticExec { + fn fmt_as(&self, t: DisplayFormatType, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match t { + DisplayFormatType::Default | DisplayFormatType::Verbose => write!(f, "SyntheticExec"), + DisplayFormatType::TreeRender => write!(f, ""), + } + } +} + +impl ExecutionPlan for SyntheticExec { + fn name(&self) -> &str { + "SyntheticExec" + } + + fn as_any(&self) -> &dyn Any { + self + } + + fn properties(&self) -> &PlanProperties { + &self.properties + } + + fn children(&self) -> Vec<&Arc> { + vec![] + } + + fn with_new_children( + self: Arc, + _children: Vec>, + ) -> Result> { + unimplemented!() + } + + fn execute( + &self, + partition: usize, + _context: Arc, + ) -> Result { + assert!(partition < self.partitions); + let schema = Arc::clone(&self.schema); + let batches = Arc::clone(&self.batches); + let len = batches.len(); + + let stream = stream::iter((0..len).map(move |idx| { + let batch = &batches[idx]; + Ok(batch.as_ref().clone()) + })); + + Ok(Box::pin(RecordBatchStreamAdapter::new(schema, stream))) + } + + fn statistics(&self) -> Result { + Ok(Statistics::new_unknown(&self.schema)) + } + + fn partition_statistics(&self, _partition: Option) -> Result { + Ok(Statistics::new_unknown(&self.schema)) + } +} + +async fn consume_partition( + plan: Arc, + partition: usize, + task_ctx: Arc, + behavior: ConsumerBehavior, +) -> Result<()> { + let mut stream = plan.execute(partition, task_ctx)?; + let mut seen = 0usize; + while let Some(batch) = stream.next().await { + let _ = batch?; + seen += 1; + match behavior { + ConsumerBehavior::Fast => {} + ConsumerBehavior::Slow(delay) => tokio::time::sleep(delay).await, + ConsumerBehavior::CancelAfter(limit) => { + if seen >= limit { + break; + } + } + } + } + Ok(()) +} + +async fn run_scenario( + scenario: &Scenario, + schema: Arc, + batches: Arc>>, + task_ctx: Arc, + sample_rss: bool, +) -> Result<(Duration, u64)> { + let input: Arc = Arc::new(SyntheticExec::new( + Arc::clone(&schema), + scenario.input_partitions, + batches, + )); + + let broadcast = Arc::new(BroadcastExec::new( + Arc::clone(&input), + scenario.consumer_tasks, + )); + + let sampler = sample_rss.then(|| PeakRssSampler::start(Duration::from_millis(25))); + let start = Instant::now(); + + let mut join_set = tokio::task::JoinSet::new(); + for consumer in &scenario.consumers { + let plan = Arc::clone(&broadcast) as Arc; + let task_ctx = Arc::clone(&task_ctx); + let behavior = consumer.behavior; + let partition = consumer.partition; + join_set.spawn(async move { consume_partition(plan, partition, task_ctx, behavior).await }); + } + + while let Some(res) = join_set.join_next().await { + let res = + res.map_err(|err| datafusion::error::DataFusionError::Execution(err.to_string()))?; + res?; + } + + let elapsed = start.elapsed(); + let peak_kb = sampler.map(|s| s.stop()).unwrap_or(0); + Ok((elapsed, peak_kb)) +} + +fn all_fast_consumers(output_partitions: usize) -> Vec { + (0..output_partitions) + .map(|partition| ConsumerSpec { + partition, + behavior: ConsumerBehavior::Fast, + }) + .collect() +} + +fn scenario_matrix() -> Vec { + let input_partitions = 16; + let consumer_tasks = 4; + let output_partitions = input_partitions * consumer_tasks; + + let mut scenarios = vec![ + // All consumers read at full speed. Baseline overhead without cancellations or lag. + // Example: Broadcast of a small dimension table, everything goes smoothly. + Scenario { + name: "all_fast", + input_partitions, + consumer_tasks, + rows_per_batch: 500_000, + num_batches: 100, + consumers: all_fast_consumers(output_partitions), + }, + // One consumer is slow, creating backpressure or queue buildup. + // Example: Broadcast build side for a join where one probe task is a straggler + // due to skewed partition. + Scenario { + name: "one_slow", + input_partitions, + consumer_tasks, + rows_per_batch: 500_000, + num_batches: 100, + consumers: { + let mut consumers = all_fast_consumers(output_partitions); + if let Some(first) = consumers.first_mut() { + first.behavior = ConsumerBehavior::Slow(Duration::from_millis(2)); + } + consumers + }, + }, + // One consumer cancels early (TopK/LIMIT finishes upstream, task killed). + // Example: SELECT ... FROM store_sales JOIN store ON ... ORDER BY ... LIMIT 100. + // The broadcast build side (store) is still fully materialized, but the LIMIT + // above the join can cancel probe-side consumers once enough rows are produced. + Scenario { + name: "one_cancel", + input_partitions, + consumer_tasks, + rows_per_batch: 500_000, + num_batches: 100, + consumers: { + let mut consumers = all_fast_consumers(output_partitions); + if let Some(first) = consumers.first_mut() { + first.behavior = ConsumerBehavior::CancelAfter(5); + } + consumers + }, + }, + // One virtual partition is never executed (no consumer spawned). + // Example: Fault tolerance, a consumer task fails. + Scenario { + name: "unused_partition", + input_partitions, + consumer_tasks, + rows_per_batch: 500_000, + num_batches: 100, + consumers: { + let mut consumers = all_fast_consumers(output_partitions); + consumers.pop(); + consumers + }, + }, + ]; + + let many_consumers = 16usize; + let many_output = input_partitions * many_consumers; + scenarios.push(Scenario { + name: "many_consumers", + input_partitions, + consumer_tasks: many_consumers, + rows_per_batch: 8192, + num_batches: 100, + consumers: all_fast_consumers(many_output), + }); + + scenarios +} + +fn verbose_enabled() -> bool { + match std::env::var("BROADCAST_BENCH_VERBOSE") { + Ok(val) => { + let val = val.to_ascii_lowercase(); + val == "1" || val == "true" || val == "yes" + } + Err(_) => false, + } +} + +fn rss_enabled() -> bool { + match std::env::var("BROADCAST_BENCH_RSS") { + Ok(val) => { + let val = val.to_ascii_lowercase(); + val == "1" || val == "true" || val == "yes" + } + Err(_) => false, + } +} + +fn runtime_threads() -> Option { + std::env::var("BROADCAST_BENCH_THREADS") + .ok() + .and_then(|val| val.parse::().ok()) + .filter(|threads| *threads > 0) +} + +fn bench_broadcast_cache(c: &mut Criterion) { + let mut rt_builder = RuntimeBuilder::new_multi_thread(); + if let Some(threads) = runtime_threads() { + rt_builder.worker_threads(threads); + } + let rt = rt_builder.enable_all().build().expect("tokio runtime"); + + let mut group = c.benchmark_group("broadcast_cache_scenarios"); + group.sample_size(10); + let verbose = verbose_enabled(); + let sample_rss = rss_enabled(); + let task_ctx = SessionContext::new().task_ctx(); + + for scenario in scenario_matrix() { + let schema = Arc::new(Schema::new(vec![Field::new( + "bytes", + DataType::UInt8, + false, + )])); + let batches = (0..scenario.num_batches) + .map(|_| { + let data = vec![0u8; scenario.rows_per_batch]; + let array = UInt8Array::from(data); + let batch = RecordBatch::try_new(Arc::clone(&schema), vec![Arc::new(array)]) + .expect("batch"); + Arc::new(batch) + }) + .collect::>(); + let batches = Arc::new(batches); + + group.bench_function(BenchmarkId::new("scenario", scenario.name), |b| { + b.iter_custom(|iters| { + let mut total = Duration::ZERO; + let mut peaks = Vec::with_capacity(iters as usize); + for i in 0..iters { + let (elapsed, peak_kb) = rt + .block_on(run_scenario( + &scenario, + Arc::clone(&schema), + Arc::clone(&batches), + Arc::clone(&task_ctx), + sample_rss, + )) + .expect("scenario"); + if verbose || sample_rss { + eprintln!( + "scenario={} iter={} peak_rss_kb={} elapsed_ms={}", + scenario.name, + i, + peak_kb, + elapsed.as_millis() + ); + } + peaks.push(peak_kb); + total += elapsed; + } + if sample_rss && !peaks.is_empty() { + peaks.sort_unstable(); + let min = peaks[0]; + let max = peaks[peaks.len() - 1]; + let median = peaks[peaks.len() / 2]; + eprintln!( + "scenario={} peak_rss_kb[min/median/max]={}/{}/{} runs={}", + scenario.name, + min, + median, + max, + peaks.len() + ); + } + total + }); + }); + } + + group.finish(); +} + +criterion_group!(benches, bench_broadcast_cache); +criterion_main!(benches); diff --git a/benchmarks/build.rs b/benchmarks/build.rs new file mode 100644 index 00000000..397ae7cb --- /dev/null +++ b/benchmarks/build.rs @@ -0,0 +1,4 @@ +fn main() { + println!("cargo::rerun-if-env-changed=FORCE_REBUILD"); + built::write_built_file().expect("Failed to acquire build-time information"); +} diff --git a/benchmarks/cdk/.gitignore b/benchmarks/cdk/.gitignore new file mode 100644 index 00000000..f60797b6 --- /dev/null +++ b/benchmarks/cdk/.gitignore @@ -0,0 +1,8 @@ +*.js +!jest.config.js +*.d.ts +node_modules + +# CDK asset staging directory +.cdk.staging +cdk.out diff --git a/benchmarks/cdk/.npmignore b/benchmarks/cdk/.npmignore new file mode 100644 index 00000000..c1d6d45d --- /dev/null +++ b/benchmarks/cdk/.npmignore @@ -0,0 +1,6 @@ +*.ts +!*.d.ts + +# CDK asset staging directory +.cdk.staging +cdk.out diff --git a/benchmarks/cdk/README.md b/benchmarks/cdk/README.md new file mode 100644 index 00000000..5c9e9a9d --- /dev/null +++ b/benchmarks/cdk/README.md @@ -0,0 +1,126 @@ +# AWS CDK code for DataFusion distributed benchmarks + +Creates automatically the appropriate infrastructure in AWS for running benchmarks. + +--- + +# Deploy + +## Prerequisites + +Cargo zigbuild needs to be installed in the system for cross-compiling to Linux x86_64, which +is what the benchmarking machines in AWS run on. + +```shell +cargo install --locked cargo-zigbuild +``` + +Make sure to also have the `x86_64-unknown-linux-gnu` target installed in +your Rust toolchain: + +```shell +rustup target add x86_64-unknown-linux-gnu +``` + +Ensure that you can cross-compile to Linux x86_64 before performing any deployments: + +```shell +cargo zigbuild -p datafusion-distributed-benchmarks --release --bin worker --target x86_64-unknown-linux-gnu +``` + +## CDK deploy + +This only needs to be done once, after that, `npm run fast-deploy` can be used for fast deploying new code in +seconds. + +```shell +npm run cdk deploy +``` + +## Populating the bucket with TPCH data + +```shell +npm run sync-bucket +``` + +--- + +# Connect to instances + +## Prerequisites + +The session manager plugin for the AWS CLI needs to be installed, as that's what is used for +connecting to the EC2 machines instead of SSH. + +These are the docs with installation instructions: + +https://docs.aws.amazon.com/systems-manager/latest/userguide/session-manager-working-with-install-plugin.html + +On Mac with an Apple Silicon processor, it can be installed with: + +```shell +curl "https://s3.amazonaws.com/session-manager-downloads/plugin/latest/mac_arm64/session-manager-plugin.pkg" -o "session-manager-plugin.pkg" +sudo installer -pkg session-manager-plugin.pkg -target +sudo ln -s /usr/local/sessionmanagerplugin/bin/session-manager-plugin /usr/local/bin/session-manager-plugin +``` + +## Port Forward + +After performing a CDK deploy, a CNF output will be printed to stdout with instructions for port-forwarding to them. + +```shell +export INSTANCE_ID=i-0000000000000000 + +aws ssm start-session --target $INSTANCE_ID --document-name AWS-StartPortForwardingSession --parameters "portNumber=9000,localPortNumber=9000" +``` + +Just port-forwarding the first instance is enough for issuing queries. + +## Connect + +After performing a CDK deploy, a CNF output will be printed to stdout with instructions for connecting +to all the machines, something like this: + +```shell +export INSTANCE_ID=i-0000000000000000 + +aws ssm start-session --target $INSTANCE_ID +``` + +The logs can be streamed with: + +```shell +sudo journalctl -u worker.service -f -o cat +``` + +--- + +# Running benchmarks + +There's a script that will run the TPCH benchmarks against the remote cluster: + +In one terminal, perform a port-forward of one machine in the cluster, something like this: + +```shell +export INSTANCE_ID=i-0000000000000000 +aws ssm start-session --target $INSTANCE_ID --document-name AWS-StartPortForwardingSession --parameters "portNumber=9000,localPortNumber=9000" +``` + +In another terminal, navigate to the benchmarks/cdk folder: + +```shell +cd benchmarks/cdk +``` + +And run the benchmarking script + +```shell +npm run datafusion-bench +``` + +Several arguments can be passed for running the benchmarks against different scale factors and with different configs, +for example: + +```shell +npm run datafusion-bench -- --datset tpch_sf10 +``` \ No newline at end of file diff --git a/benchmarks/cdk/ballista/.gitignore b/benchmarks/cdk/ballista/.gitignore new file mode 100644 index 00000000..44709884 --- /dev/null +++ b/benchmarks/cdk/ballista/.gitignore @@ -0,0 +1,2 @@ +target/ +Cargo.lock \ No newline at end of file diff --git a/benchmarks/cdk/ballista/Cargo.toml b/benchmarks/cdk/ballista/Cargo.toml new file mode 100644 index 00000000..a404ac29 --- /dev/null +++ b/benchmarks/cdk/ballista/Cargo.toml @@ -0,0 +1,35 @@ +[package] +name = "ballista-benchmarks" +version = "0.1.0" +edition = "2024" + +# Empty workspace table to detach from root workspace +[workspace] + +[[bin]] +name = "ballista-http" +path = "src/ballista_http.rs" + +[[bin]] +name = "ballista-scheduler" +path = "src/ballista_scheduler.rs" + +[[bin]] +name = "ballista-executor" +path = "src/ballista_executor.rs" + +[dependencies] +ballista = "51.0.0" +ballista-core = "51.0.0" +ballista-executor = "51.0.0" +ballista-scheduler = "51.0.0" +object_store = { version = "0.12", features = ["aws"] } +tokio = { version = "1", features = ["full"] } +url = "2" +clap = { version = "4", features = ["derive"] } +structopt = "0.3" +axum = "0.8" +serde = { version = "1", features = ["derive"] } +futures = "0.3" +log = "0.4" +env_logger = "0.11" diff --git a/benchmarks/cdk/ballista/src/ballista_executor.rs b/benchmarks/cdk/ballista/src/ballista_executor.rs new file mode 100644 index 00000000..87f660c0 --- /dev/null +++ b/benchmarks/cdk/ballista/src/ballista_executor.rs @@ -0,0 +1,36 @@ +use ballista::datafusion::execution::runtime_env::RuntimeEnv; +use ballista::datafusion::prelude::SessionConfig; +use ballista_executor::config::Config; +use ballista_executor::executor_process::{ExecutorProcessConfig, start_executor_process}; +use clap::Parser; +use object_store::aws::AmazonS3Builder; +use std::env; +use std::sync::Arc; +use url::Url; + +#[tokio::main] +async fn main() -> Result<(), Box> { + let opt = Config::parse(); + + let mut config: ExecutorProcessConfig = opt.try_into()?; + + let bucket = env::var("BUCKET").unwrap_or("datafusion-distributed-benchmarks".to_string()); + let s3_url = Url::parse(&format!("s3://{bucket}"))?; + + let s3 = Arc::new( + AmazonS3Builder::from_env() + .with_bucket_name(s3_url.host().unwrap().to_string()) + .build()?, + ); + let runtime_env = Arc::new(RuntimeEnv::default()); + runtime_env.register_object_store(&s3_url, s3); + + config.override_runtime_producer = Some(Arc::new( + move |_: &SessionConfig| -> ballista::datafusion::common::Result> { + Ok(runtime_env.clone()) + }, + )); + + start_executor_process(Arc::new(config)).await?; + Ok(()) +} diff --git a/benchmarks/cdk/ballista/src/ballista_http.rs b/benchmarks/cdk/ballista/src/ballista_http.rs new file mode 100644 index 00000000..948aa1e1 --- /dev/null +++ b/benchmarks/cdk/ballista/src/ballista_http.rs @@ -0,0 +1,124 @@ +use axum::{Json, Router, extract::Query, http::StatusCode, routing::get}; +use ballista::datafusion::common::instant::Instant; +use ballista::datafusion::execution::SessionStateBuilder; +use ballista::datafusion::execution::runtime_env::RuntimeEnv; +use ballista::datafusion::physical_plan::displayable; +use ballista::datafusion::physical_plan::execute_stream; +use ballista::datafusion::prelude::SessionConfig; +use ballista::datafusion::prelude::SessionContext; +use ballista::prelude::*; +use futures::{StreamExt, TryFutureExt}; +use log::{error, info}; +use object_store::aws::AmazonS3Builder; +use serde::Serialize; +use std::collections::HashMap; +use std::error::Error; +use std::fmt::Display; +use std::sync::Arc; +use structopt::StructOpt; +use url::Url; + +#[derive(Serialize)] +struct QueryResult { + plan: String, + count: usize, +} + +#[derive(Debug, StructOpt, Clone)] +#[structopt(about = "worker spawn command")] +struct Cmd { + /// The bucket name. + #[structopt(long, default_value = "datafusion-distributed-benchmarks")] + bucket: String, +} + +#[tokio::main] +async fn main() -> Result<(), Box> { + env_logger::builder() + .filter_level(log::LevelFilter::Info) + .parse_default_env() + .init(); + + let cmd = Cmd::from_args(); + + const LISTENER_ADDR: &str = "0.0.0.0:9002"; + + info!("Starting HTTP listener on {LISTENER_ADDR}..."); + let listener = tokio::net::TcpListener::bind(LISTENER_ADDR).await?; + + // Register S3 object store + let s3_url = Url::parse(&format!("s3://{}", cmd.bucket))?; + + info!("Building shared SessionContext for the whole lifetime of the HTTP listener..."); + let s3 = Arc::new( + AmazonS3Builder::from_env() + .with_bucket_name(s3_url.host().unwrap().to_string()) + .build()?, + ); + let runtime_env = Arc::new(RuntimeEnv::default()); + runtime_env.register_object_store(&s3_url, s3); + + let config = SessionConfig::new_with_ballista().with_ballista_job_name("Benchmarks"); + + let state = SessionStateBuilder::new() + .with_config(config) + .with_default_features() + .with_runtime_env(Arc::clone(&runtime_env)) + .build(); + let ctx = SessionContext::remote_with_state("df://localhost:50050", state).await?; + + let http_server = axum::serve( + listener, + Router::new().route( + "/", + get(move |Query(params): Query>| { + let ctx = ctx.clone(); + + async move { + let sql = params.get("sql").ok_or(err("Missing 'sql' parameter"))?; + + let mut df_opt = None; + for sql in sql.split(";") { + if sql.trim().is_empty() { + continue; + } + let df = ctx.sql(sql).await.map_err(err)?; + df_opt = Some(df); + } + let Some(df) = df_opt else { + return Err(err("Empty 'sql' parameter")); + }; + + let start = Instant::now(); + + info!("Executing query..."); + let physical = df.create_physical_plan().await.map_err(err)?; + let mut stream = + execute_stream(physical.clone(), ctx.task_ctx()).map_err(err)?; + let mut count = 0; + while let Some(batch) = stream.next().await { + count += batch.map_err(err)?.num_rows(); + info!("Gathered {count} rows, query still in progress..") + } + let plan = displayable(physical.as_ref()).indent(true).to_string(); + let elapsed = start.elapsed(); + let ms = elapsed.as_secs_f64() * 1000.0; + info!("Returned {count} rows in {ms} ms"); + + Ok::<_, (StatusCode, String)>(Json(QueryResult { count, plan })) + } + .inspect_err(|(_, msg)| { + error!("Error executing query: {msg}"); + }) + }), + ), + ); + + info!("Started listener HTTP server in {LISTENER_ADDR}"); + http_server.await?; + Ok(()) +} + +fn err(s: impl Display) -> (StatusCode, String) { + (StatusCode::INTERNAL_SERVER_ERROR, s.to_string()) +} diff --git a/benchmarks/cdk/ballista/src/ballista_scheduler.rs b/benchmarks/cdk/ballista/src/ballista_scheduler.rs new file mode 100644 index 00000000..ba0b1f2d --- /dev/null +++ b/benchmarks/cdk/ballista/src/ballista_scheduler.rs @@ -0,0 +1,62 @@ +use ballista::datafusion::execution::runtime_env::RuntimeEnv; +use ballista::datafusion::execution::{SessionState, SessionStateBuilder}; +use ballista::datafusion::prelude::SessionConfig; +use ballista_core::error::BallistaError; +use ballista_core::extension::SessionConfigExt; +use ballista_scheduler::cluster::BallistaCluster; +use ballista_scheduler::config::{Config, SchedulerConfig}; +use ballista_scheduler::scheduler_process::start_server; +use clap::Parser; +use object_store::aws::AmazonS3Builder; +use std::env; +use std::sync::Arc; +use url::Url; + +fn main() -> Result<(), Box> { + let runtime = tokio::runtime::Builder::new_multi_thread() + .enable_io() + .enable_time() + .thread_stack_size(32 * 1024 * 1024) // 32MB + .build()?; + + runtime.block_on(inner()) +} + +async fn inner() -> Result<(), Box> { + let opt = Config::parse(); + + let addr = format!("{}:{}", opt.bind_host, opt.bind_port); + let addr = addr + .parse() + .map_err(|e: std::net::AddrParseError| BallistaError::Configuration(e.to_string()))?; + + let bucket = env::var("BUCKET").unwrap_or("datafusion-distributed-benchmarks".to_string()); + let s3_url = Url::parse(&format!("s3://{bucket}"))?; + + let s3 = Arc::new( + AmazonS3Builder::from_env() + .with_bucket_name(s3_url.host().unwrap().to_string()) + .build()?, + ); + let runtime_env = Arc::new(RuntimeEnv::default()); + runtime_env.register_object_store(&s3_url, s3); + + let config: SchedulerConfig = opt.try_into()?; + let config = config.with_override_config_producer(Arc::new(|| { + SessionConfig::new_with_ballista().with_information_schema(true) + })); + let config = config.with_override_session_builder(Arc::new( + move |cfg: SessionConfig| -> ballista::datafusion::common::Result { + Ok(SessionStateBuilder::new() + .with_config(cfg) + .with_runtime_env(runtime_env.clone()) + .with_default_features() + .build()) + }, + )); + + let cluster = BallistaCluster::new_from_config(&config).await?; + start_server(cluster, addr, Arc::new(config)).await?; + + Ok(()) +} diff --git a/benchmarks/cdk/bin/@bench-common.ts b/benchmarks/cdk/bin/@bench-common.ts new file mode 100644 index 00000000..04604325 --- /dev/null +++ b/benchmarks/cdk/bin/@bench-common.ts @@ -0,0 +1,150 @@ +import path from "path"; +import fs from "fs/promises"; +import {BenchmarkRun, BenchResult} from "./@results"; + +export const ROOT = path.join(__dirname, '../../..') +export const BUCKET = 's3://datafusion-distributed-benchmarks' // hardcoded in CDK code + +export interface TableSpec { + schema: string + name: string + s3Path: string +} + +export interface BenchmarkRunner { + createTables(s3Paths: TableSpec[]): Promise; + + executeQuery(query: string): Promise<{ rowCount: number, plan: string }>; +} + +async function tablePathsForDataset(dataset: string): Promise { + const datasetPath = path.join(ROOT, "benchmarks", "data", dataset) + + const result: TableSpec[] = [] + for (const entryName of await fs.readdir(datasetPath)) { + const dir = path.join(datasetPath, entryName) + if (await isDirWithAllParquetFiles(dir)) { + result.push({ + name: entryName, + schema: dataset, + s3Path: `${BUCKET}/${dataset}/${entryName}/` + }) + } + } + return result +} + +async function isDirWithAllParquetFiles(dir: string): Promise { + let readDir + try { + readDir = await fs.readdir(dir) + } catch (e) { + return false + } + for (const file of readDir) { + if (!file.endsWith(".parquet")) { + return false + } + } + return true +} + +async function queriesForDataset(dataset: string): Promise<{ id: string, sql: string }[]> { + const datasetSuffix = dataset.split("_")[0] + const queriesPath = path.join(ROOT, "testdata", datasetSuffix, "queries") + + const queries = [] + for (const fileName of await fs.readdir(queriesPath)) { + const sql = await fs.readFile(path.join(queriesPath, fileName), 'utf-8'); + queries.push({ id: fileName.replace(".sql", ""), sql }) + } + queries.sort((a, b) => numericId(a.id) > numericId(b.id) ? 1 : -1) + return queries +} + +function numericId(queryName: string): number { + return parseInt([...queryName.matchAll(/(\d+)/g)][0][0]) +} + +export async function runBenchmark( + runner: BenchmarkRunner, + options: { + dataset: string + engine: string, + iterations: number; + queries: string[]; + debug: boolean; + warmup: boolean; + } +) { + const { dataset, engine, iterations, queries, warmup, debug } = options; + + const benchmarkRun = new BenchmarkRun(dataset, engine) + + console.log("Creating tables..."); + const s3Paths = await tablePathsForDataset(dataset) + await runner.createTables(s3Paths); + + for (const { id, sql } of await queriesForDataset(dataset)) { + if (queries.length > 0 && !queries.includes(id)) { + continue; + } + + const result = new BenchResult(dataset, engine, id) + + if (warmup) { + console.log(`Warming up query ${id}...`) + try { + await runner.executeQuery(sql); + } catch (e: any) { + result.iterations.push({ + elapsed: 0, + rowCount: 0, + error: e.toString(), + plan: "" + }) + console.error(`Query ${id} failed: ${e.toString()}`) + continue + } + } + + for (let i = 0; i < iterations; i++) { + const start = new Date() + let response + try { + response = await runner.executeQuery(sql); + } catch (e: any) { + result.iterations.push({ + elapsed: 0, + rowCount: 0, + error: e.toString(), + plan: "" + }) + console.error(`Query ${id} failed: ${e.toString()}`) + break + } + const elapsed = Math.round(new Date().getTime() - start.getTime()) + + if (debug) { + console.log(response.plan) + } + result.iterations.push({ + elapsed, + rowCount: response.rowCount, + plan: response.plan + }) + + console.log( + `Query ${id} iteration ${i} took ${elapsed} ms and returned ${response.rowCount} rows` + ); + } + + console.log(`Query ${id} avg time: ${result.avg()} ms`); + + benchmarkRun.results.push(result) + } + + // Write results and compare + benchmarkRun.compareWithPrevious() + benchmarkRun.store() +} diff --git a/benchmarks/cdk/bin/@ops.ts b/benchmarks/cdk/bin/@ops.ts new file mode 100644 index 00000000..7a4e705e --- /dev/null +++ b/benchmarks/cdk/bin/@ops.ts @@ -0,0 +1,89 @@ +import { exec } from 'child_process'; +import { promisify } from 'util'; + +const execAsync = promisify(exec); + +export const STACK_NAME = 'DataFusionDistributedBenchmarks'; + +export async function getStackOutput(outputKey: string): Promise { + try { + const { stdout } = await execAsync( + `aws cloudformation describe-stacks --stack-name ${STACK_NAME} --query "Stacks[0].Outputs[?OutputKey=='${outputKey}'].OutputValue" --output text` + ); + const value = stdout.trim(); + return value && value !== 'None' ? value : undefined; + } catch { + return undefined; + } +} + +function sleep(ms: number): Promise { + return new Promise(resolve => setTimeout(resolve, ms)); +} + +export interface SsmCommandResult { + success: boolean; + stdout: string; + stderr: string; +} + +async function waitForCommand(commandId: string, instanceId: string): Promise { + const maxAttempts = 60; + const pollInterval = 2000; + const SUCCESS = 'Success' + const FAILED = 'Failed' + const CANCELLED = 'Cancelled' + const TIMED_OUT = 'TimedOut' + + for (let attempt = 0; attempt < maxAttempts; attempt++) { + try { + const { stdout } = await execAsync( + `aws ssm get-command-invocation --command-id "${commandId}" --instance-id "${instanceId}" --output json` + ); + const result = JSON.parse(stdout); + const status = result.Status; + + if (status === SUCCESS) { + return { + success: true, + stdout: result.StandardOutputContent || '', + stderr: result.StandardErrorContent || '', + }; + } else if (status === FAILED || status === CANCELLED || status === TIMED_OUT) { + console.error(` ${instanceId}: Command ${status} - ${result.StatusDetails}`); + return { + success: false, + stdout: result.StandardOutputContent || '', + stderr: result.StandardErrorContent || '', + }; + } + } catch { + // Command invocation might not be ready yet, wait and retry + } + await sleep(pollInterval); + } + + console.error(` ${instanceId}: Timed out waiting for command`); + return { success: false, stdout: '', stderr: '' }; +} + +export async function sendSsmCommand(instanceId: string, commands: string[]): Promise { + console.log(`Sending commands to ${instanceId}...`); + try { + const { stdout } = await execAsync( + `aws ssm send-command --instance-ids "${instanceId}" --document-name "AWS-RunShellScript" --parameters '{"commands":${JSON.stringify(commands)}}' --query "Command.CommandId" --output text` + ); + const commandId = stdout.trim(); + console.log(` ${instanceId}: Command ID ${commandId}, waiting for completion...`); + + const result = await waitForCommand(commandId, instanceId); + if (result.success) { + console.log(` ${instanceId}: Success`); + } + return result; + } catch (error) { + console.error(` ${instanceId}: Failed to send command:`, error); + return { success: false, stdout: '', stderr: String(error) }; + } +} + diff --git a/benchmarks/cdk/bin/@results.ts b/benchmarks/cdk/bin/@results.ts new file mode 100644 index 00000000..7a62e204 --- /dev/null +++ b/benchmarks/cdk/bin/@results.ts @@ -0,0 +1,277 @@ +import * as fs from 'fs'; +import * as path from 'path'; +import { z } from "zod"; + +// Assuming DATA_PATH is defined elsewhere or passed as parameter +export const DATA_PATH = path.join(__dirname, '../../data'); +export const RESULTS_DIR = ".results-remote" + +// Interface for a single iteration of a benchmark query +export interface QueryIter { + plan: string; + rowCount: number; + elapsed: number; // Duration in milliseconds + error?: string; +} + +// Class for collecting benchmark run data +export class BenchmarkRun { + startTime: number; + dataset: string; + engine: string; + results: BenchResult[]; + + constructor(dataset: string, engine: string) { + this.dataset = dataset; + this.engine = engine; + this.startTime = Math.floor(Date.now() / 1000); // Unix timestamp in seconds + this.results = []; + } + + loadPrevious(): BenchmarkRun | null { + const previousPath = path.join(DATA_PATH, this.dataset, `previous-remote.json`); + + try { + const prevData = fs.readFileSync(previousPath, 'utf-8'); + const prevOutput = JSON.parse(prevData) as BenchmarkRun; + + // Create new instance and load results + const instance = new BenchmarkRun(prevOutput.dataset, prevOutput.engine); + instance.startTime = prevOutput.startTime; + instance.loadResults(); + + return instance; + } catch { + return null; + } + } + + loadResults(): void { + this.results = BenchResult.loadMany(this.dataset, this.engine); + } + + // Write data as JSON into output path if it exists + store(): void { + const outputPath = path.join(DATA_PATH, this.dataset, `previous-remote.json`); + + // Ensure directory exists + const dir = path.dirname(outputPath); + if (!fs.existsSync(dir)) { + fs.mkdirSync(dir, { recursive: true }); + } + + // Custom serialization to handle results + const toSerialize = { + ...this, + results: [] // Empty array for serialization + }; + + const json = JSON.stringify(toSerialize, null, 2); + fs.writeFileSync(outputPath, json); + + // Store individual results + for (const result of this.results) { + result.store(); + } + } + + compare(other: BenchmarkRun): void { + console.log(`=== Comparing ${this.dataset} results from engine '${other.engine}' [prev] with '${this.engine}' [new] ===`); + let totalTimePrev = 0 + let totalTimeNew = 0 + for (const query of this.results) { + const prevQuery = other.results.find(v => v.id === query.id); + if (!prevQuery) { + continue; + } + const timePrev = prevQuery.totalTime() + const timeNew = query.totalTime() + if (timePrev && timeNew) { + totalTimePrev += timePrev + totalTimeNew += timeNew + } + + query.compare(prevQuery); + } + + let f, tag, emoji + if (totalTimeNew < totalTimePrev) { + f = totalTimePrev / totalTimeNew; + tag = "faster"; + emoji = f > 1.2 ? "✅" : "✔"; + } else { + f = totalTimeNew / totalTimePrev; + tag = "slower"; + emoji = f > 1.2 ? "❌" : "✖"; + } + console.log( + `${"TOTAL".padStart(8)}: prev=${totalTimePrev.toString()} ms, new=${totalTimeNew.toString()} ms, diff=${f.toFixed(2)} ${tag} ${emoji}` + ); + } + + compareWithPrevious(): void { + const previous = this.loadPrevious(); + if (!previous) { + return; + } + this.compare(previous) + } +} + +// Class for a single benchmark case +export class BenchResult { + id: string; + dataset: string; + engine: string; + iterations: QueryIter[]; + + constructor(dataset: string, engine: string, id: string) { + this.dataset = dataset; + this.engine = engine; + this.id = id; + this.iterations = []; + } + + avg(): number { + if (this.iterations.length === 0) { + return 0; + } + + const sum = this.iterations.reduce((acc, iter) => acc + iter.elapsed, 0); + return Math.floor(sum / this.iterations.length); + } + + totalTime(): undefined | number { + let totalTime = 0; + for (const iter of this.iterations) { + if (iter.error) return undefined + totalTime += iter.elapsed + } + return totalTime + } + + compare(prevQuery: BenchResult): void { + const prevErr = prevQuery.iterations.find(v => v.error)?.error; + const newErr = this.iterations.find(v => v.error)?.error; + + if (prevErr && !newErr) { + console.log(`${this.id}: Previously failed, but now succeeded 🟠`); + return; + } + if (!prevErr && newErr) { + console.log(`${this.id}: Previously succeeded, but now failed ❌`); + return; + } + if (prevErr && newErr) { + console.log(`${this.id}: Previously failed, and now also failed ❌`); + return; + } + + const avgPrev = prevQuery.avg(); + const avg = this.avg(); + + let f: number; + let tag: string; + let emoji: string; + + if (avg < avgPrev) { + f = avgPrev / avg; + tag = "faster"; + emoji = f > 1.2 ? "✅" : "✔"; + } else { + f = avg / avgPrev; + tag = "slower"; + emoji = f > 1.2 ? "❌" : "✖"; + } + + console.log( + `${this.id.padStart(8)}: prev=${avgPrev.toString().padStart(4)} ms, new=${avg.toString().padStart(4)} ms, diff=${f.toFixed(2)} ${tag} ${emoji}` + ); + } + + store(): void { + const filePath = path.join( + DATA_PATH, + this.dataset, + RESULTS_DIR, + this.engine, + `${this.id}.json` + ); + + // Ensure directory exists + const dir = path.dirname(filePath); + if (!fs.existsSync(dir)) { + fs.mkdirSync(dir, { recursive: true }); + } + + const json = JSON.stringify(this, null, 2); + fs.writeFileSync(filePath, json); + } + + static load(dataset: string, engine: string, id: string): BenchResult | null { + const filePath = path.join( + DATA_PATH, + dataset, + RESULTS_DIR, + engine, + `${id}.json` + ); + + try { + const parser = z.object({ + dataset: z.string(), + engine: z.string(), + id: z.string(), + iterations: z.object({ + rowCount: z.number(), + elapsed: z.number(), + error: z.string().optional(), + plan: z.string() + }).array(), + }) + const data = fs.readFileSync(filePath, 'utf-8'); + const parsed = parser.parse(JSON.parse(data)) + const result = new BenchResult( + parsed.dataset, + parsed.engine, + parsed.id + ) + result.iterations = parsed.iterations + return result; + } catch { + return null; + } + } + + static loadMany(dataset: string, engine: string): BenchResult[] { + const resultsDir = path.join(DATA_PATH, dataset, RESULTS_DIR, engine); + const results: BenchResult[] = []; + + try { + const files = fs.readdirSync(resultsDir); + + for (const fileName of files) { + if (!fileName.endsWith('.json')) { + continue; + } + + const id = fileName.slice(0, -5); // Remove .json extension + const result = BenchResult.load(dataset, engine, id); + + if (result) { + results.push(result); + } + } + } catch { + // Directory doesn't exist or can't be read + return results; + } + + results.sort((a, b) => numericId(a.id) > numericId(b.id) ? 1 : -1) + return results; + } +} + +function numericId(queryName: string): number { + return parseInt([...queryName.matchAll(/(\d+)/g)][0][0]) +} diff --git a/benchmarks/cdk/bin/ballista-bench.ts b/benchmarks/cdk/bin/ballista-bench.ts new file mode 100644 index 00000000..075a3641 --- /dev/null +++ b/benchmarks/cdk/bin/ballista-bench.ts @@ -0,0 +1,99 @@ +import { Command } from "commander"; +import { z } from 'zod'; +import { BenchmarkRunner, runBenchmark, TableSpec } from "./@bench-common"; + +// Remember to port-forward the ballista HTTP server with +// aws ssm start-session --target {host-id} --document-name AWS-StartPortForwardingSession --parameters "portNumber=9002,localPortNumber=9002" + +async function main() { + const program = new Command(); + + program + .option('--dataset ', 'Dataset to run queries on') + .option('-i, --iterations ', 'Number of iterations', '3') + .option('--queries ', 'Specific queries to run', undefined) + .option('--debug ', 'Print the generated plans to stdout') + .option('--warmup ', 'Perform a warmup query before the benchmarks') + .parse(process.argv); + + const options = program.opts(); + + const dataset: string = options.dataset + const iterations = parseInt(options.iterations); + const queries = options.queries?.split(",") ?? [] + const debug = options.debug === 'true' || options.debug === 1 + const warmup = options.warmup === 'true' || options.debug === 1 + + const runner = new BallistaRunner({}); + + await runBenchmark(runner, { + dataset, + engine: 'ballista', + iterations, + queries, + debug, + warmup + }); +} + +const QueryResponse = z.object({ + count: z.number(), + plan: z.string() +}) +type QueryResponse = z.infer + +class BallistaRunner implements BenchmarkRunner { + private url = 'http://localhost:9002'; + + constructor(private readonly options: {}) { + } + + async executeQuery(sql: string): Promise<{ rowCount: number, plan: string }> { + let response + if (sql.includes("create view")) { + // This is query 15 + let [createView, query, dropView] = sql.split(";") + await this.query(createView); + response = await this.query(query) + await this.query(dropView); + } else { + response = await this.query(sql) + } + + return { rowCount: response.count, plan: response.plan }; + } + + private async query(sql: string): Promise { + const url = new URL(this.url); + url.searchParams.set('sql', sql); + + const response = await fetch(url.toString()); + + if (!response.ok) { + const msg = await response.text(); + throw new Error(`Query failed: ${response.status} ${msg}`); + } + + const unparsed = await response.json(); + return QueryResponse.parse(unparsed); + } + + async createTables(tables: TableSpec[]): Promise { + let stmt = ''; + for (const table of tables) { + // language=SQL format=false + stmt += ` + DROP TABLE IF EXISTS ${table.name}; + CREATE EXTERNAL TABLE IF NOT EXISTS ${table.name} STORED AS PARQUET LOCATION '${table.s3Path}'; + `; + } + await this.query(stmt); + } + +} + +main() + .catch(err => { + console.error(err) + process.exit(1) + }) \ No newline at end of file diff --git a/benchmarks/cdk/bin/cdk.ts b/benchmarks/cdk/bin/cdk.ts new file mode 100644 index 00000000..673e30cd --- /dev/null +++ b/benchmarks/cdk/bin/cdk.ts @@ -0,0 +1,22 @@ +#!/usr/bin/env node +import * as cdk from 'aws-cdk-lib/core'; +import {CdkStack} from '../lib/cdk-stack'; +import {DATAFUSION_DISTRIBUTED_ENGINE} from "../lib/datafusion-distributed"; +import {TRINO_ENGINE} from "../lib/trino"; +import {SPARK_ENGINE} from "../lib/spark"; +import { BALLISTA_ENGINE } from "../lib/ballista"; + +const app = new cdk.App(); + +const config = { + instanceType: 'c5n.2xlarge', // Non-burstable, network-optimized + instanceCount: 4, + engines: [ + DATAFUSION_DISTRIBUTED_ENGINE, + SPARK_ENGINE, + TRINO_ENGINE, + BALLISTA_ENGINE + ] +}; + +new CdkStack(app, 'DataFusionDistributedBenchmarks', { config }); diff --git a/benchmarks/cdk/bin/compare.ts b/benchmarks/cdk/bin/compare.ts new file mode 100644 index 00000000..cedf143f --- /dev/null +++ b/benchmarks/cdk/bin/compare.ts @@ -0,0 +1,30 @@ +import {Command} from "commander"; +import { BenchmarkRun, BenchResult } from "./@results"; + +async function main() { + const program = new Command(); + + program + .requiredOption('--dataset ', 'Dataset to run queries on') + .argument("", "the base engine") + .argument("", "the engine to compare to") + .parse(process.argv); + + const options = program.opts(); + if (program.args.length != 2) { + throw new Error(`Expected exactly 2 arguments, got ${program.args.length}`) + } + + const prevRun = new BenchmarkRun(options.dataset, program.args[0]) + prevRun.loadResults() + const newRun = new BenchmarkRun(options.dataset, program.args[1]) + newRun.loadResults() + + newRun.compare(prevRun) +} + +main() + .catch(err => { + console.error(err) + process.exit(1) + }) diff --git a/benchmarks/cdk/bin/datafusion-bench.ts b/benchmarks/cdk/bin/datafusion-bench.ts new file mode 100644 index 00000000..55e7d3fe --- /dev/null +++ b/benchmarks/cdk/bin/datafusion-bench.ts @@ -0,0 +1,153 @@ +import { Command } from "commander"; +import { z } from 'zod'; +import { BenchmarkRunner, runBenchmark, TableSpec } from "./@bench-common"; +import { execSync } from "child_process"; + +// Remember to port-forward a worker with +// aws ssm start-session --target {host-id} --document-name AWS-StartPortForwardingSession --parameters "portNumber=9000,localPortNumber=9000" + +async function main() { + const program = new Command(); + + program + .requiredOption('--dataset ', 'Dataset to run queries on') + .option('-i, --iterations ', 'Number of iterations', '3') + .option('--files-per-task ', 'Files per task', '8') + .option('--cardinality-task-sf ', 'Cardinality task scale factor', '1') + .option('--batch-size ', 'Standard Batch coalescing size (number of rows)', '32768') + .option('--shuffle-batch-size ', 'Shuffle batch coalescing size (number of rows)', '32768') + .option('--children-isolator-unions ', 'Use children isolator unions', 'true') + .option('--broadcast-joins ', 'Use broadcast joins', 'false') + .option('--collect-metrics ', 'Propagates metric collection', 'true') + .option('--compression ', 'Compression algo to use within workers (lz4, zstd, none)', 'lz4') + .option('--queries ', 'Specific queries to run', undefined) + .option('--debug ', 'Print the generated plans to stdout') + .option('--warmup ', 'Perform a warmup query before the benchmarks') + .parse(process.argv); + + const options = program.opts(); + + const dataset: string = options.dataset + const iterations = parseInt(options.iterations); + const filesPerTask = parseInt(options.filesPerTask); + const cardinalityTaskSf = parseInt(options.cardinalityTaskSf); + const batchSize = parseInt(options.batchSize); + const shuffleBatchSize = parseInt(options.shuffleBatchSize); + const compression = options.compression; + const queries = options.queries?.split(",") ?? [] + const collectMetrics = options.collectMetrics === 'true' || options.collectMetrics === 1 + const childrenIsolatorUnions = options.childrenIsolatorUnions === 'true' || options.childrenIsolatorUnions === 1 + const broadcastJoins = options.broadcastJoins === 'true' || options.broadcastJoins === 1 + const debug = options.debug === 'true' || options.debug === 1 + const warmup = options.warmup === 'true' || options.debug === 1 + + const runner = new DataFusionRunner({ + filesPerTask, + cardinalityTaskSf, + batchSize, + shuffleBatchSize, + collectMetrics, + childrenIsolatorUnions, + compression, + broadcastJoins + }); + + await runBenchmark(runner, { + dataset, + engine: `datafusion-distributed-${getCurrentBranch()}`, + iterations, + queries, + debug, + warmup + }); +} + +const QueryResponse = z.object({ + count: z.number(), + plan: z.string() +}) +type QueryResponse = z.infer + +class DataFusionRunner implements BenchmarkRunner { + private url = 'http://localhost:9000'; + + constructor(private readonly options: { + filesPerTask: number; + cardinalityTaskSf: number; + batchSize: number; + shuffleBatchSize: number; + collectMetrics: boolean; + compression: string; + childrenIsolatorUnions: boolean; + broadcastJoins: boolean; + }) { + } + + async executeQuery(sql: string): Promise<{ rowCount: number, plan: string }> { + let response + if (sql.includes("create view")) { + // This is query 15 + let [createView, query, dropView] = sql.split(";") + await this.query(createView); + response = await this.query(query) + await this.query(dropView); + } else { + response = await this.query(sql) + } + + return { rowCount: response.count, plan: response.plan }; + } + + private async query(sql: string): Promise { + const url = new URL(this.url); + url.searchParams.set('sql', sql); + + const response = await fetch(url.toString()); + + if (!response.ok) { + const msg = await response.text(); + throw new Error(`Query failed: ${response.status} ${msg}`); + } + + const unparsed = await response.json(); + return QueryResponse.parse(unparsed); + } + + async createTables(tables: TableSpec[]): Promise { + let stmt = ''; + for (const table of tables) { + // language=SQL format=false + stmt += ` + DROP TABLE IF EXISTS ${table.name}; + CREATE EXTERNAL TABLE IF NOT EXISTS ${table.name} STORED AS PARQUET LOCATION '${table.s3Path}'; + `; + } + await this.query(stmt); + await this.query(` + SET distributed.files_per_task=${this.options.filesPerTask}; + SET distributed.cardinality_task_count_factor=${this.options.cardinalityTaskSf}; + SET datafusion.execution.batch_size=${this.options.batchSize}; + SET distributed.shuffle_batch_size=${this.options.shuffleBatchSize}; + SET distributed.collect_metrics=${this.options.collectMetrics}; + SET distributed.compression=${this.options.compression}; + SET distributed.children_isolator_unions=${this.options.childrenIsolatorUnions}; + SET distributed.broadcast_joins=${this.options.broadcastJoins}; + `); + } +} + +function getCurrentBranch(): string { + try { + // Try to get current git branch. For branches with a slash prefix, keep the last entry. + return execSync('git rev-parse --abbrev-ref HEAD', { encoding: 'utf-8' }).trim().split("/").slice(-1)[0]; + } catch { + // Fallback if git command fails + return 'unknown'; + } +} + +main() + .catch(err => { + console.error(err) + process.exit(1) + }) diff --git a/benchmarks/cdk/bin/fast-deploy.ts b/benchmarks/cdk/bin/fast-deploy.ts new file mode 100644 index 00000000..0bcd8fac --- /dev/null +++ b/benchmarks/cdk/bin/fast-deploy.ts @@ -0,0 +1,63 @@ +import { execSync } from 'child_process'; +import * as path from 'path'; +import { getStackOutput, sendSsmCommand, STACK_NAME } from "./@ops"; + +const ROOT = path.join(__dirname, '..', '..', '..'); +const WORKER_BINARY_PATH = path.join(ROOT, 'target/x86_64-unknown-linux-gnu/release/worker'); + +async function main() { + // Step 1: Build the worker binary + console.log('Building worker binary...'); + execSync('cargo zigbuild -p datafusion-distributed-benchmarks --release --bin worker --target x86_64-unknown-linux-gnu', { + cwd: ROOT, + stdio: 'inherit', + env: { ...process.env, FORCE_REBUILD: Date.now().toString() }, + }); + console.log('Worker binary built successfully.\n'); + + // Step 2: Fetch stack outputs (in parallel) + console.log(`Fetching outputs from stack ${STACK_NAME}...`); + const [instanceIdsStr, s3Bucket, s3Key] = await Promise.all([ + getStackOutput('WorkerInstanceIds'), + getStackOutput('WorkerBinaryS3Bucket'), + getStackOutput('WorkerBinaryS3Key'), + ]); + + if (!instanceIdsStr || !s3Bucket || !s3Key) { + console.error('Error: Required outputs not found in CloudFormation stack.'); + console.error('Make sure the stack was deployed with DataFusion Distributed engine.'); + console.error(` WorkerInstanceIds: ${instanceIdsStr ?? 'not found'}`); + console.error(` WorkerBinaryS3Bucket: ${s3Bucket ?? 'not found'}`); + console.error(` WorkerBinaryS3Key: ${s3Key ?? 'not found'}`); + process.exit(1); + } + + const instanceIds = instanceIdsStr.split(','); + console.log(`Target instances: ${instanceIds.join(', ')}\n`); + + // Step 3: Upload the binary to S3 + console.log(`Uploading worker binary to s3://${s3Bucket}/${s3Key}...`); + execSync(`aws s3 cp ${WORKER_BINARY_PATH} s3://${s3Bucket}/${s3Key}`, { stdio: 'inherit' }); + console.log('Upload complete.\n'); + + // Step 4: Send SSM commands to all instances (in parallel) + const commands = [ + `aws s3 cp s3://${s3Bucket}/${s3Key} /usr/local/bin/worker`, + 'chmod +x /usr/local/bin/worker', + 'systemctl restart worker', + ]; + + const results = await Promise.all(instanceIds.map(id => sendSsmCommand(id, commands))); + const successCount = results.filter(r => r).length; + const failCount = results.length - successCount; + + console.log(`\nFast deploy complete: ${successCount} succeeded, ${failCount} failed.`); + if (failCount > 0) { + process.exit(1); + } +} + +main().catch((error) => { + console.error('Fast deploy failed:', error); + process.exit(1); +}); diff --git a/benchmarks/cdk/bin/send-command.ts b/benchmarks/cdk/bin/send-command.ts new file mode 100644 index 00000000..42b13095 --- /dev/null +++ b/benchmarks/cdk/bin/send-command.ts @@ -0,0 +1,37 @@ +import { getStackOutput, sendSsmCommand, STACK_NAME } from "./@ops"; + +async function main() { + // Step 1: Fetch stack outputs + console.log(`Fetching outputs from stack ${STACK_NAME}...`); + const instanceIdsStr = await getStackOutput('WorkerInstanceIds') + + if (!instanceIdsStr) { + console.error('Error: Required outputs not found in CloudFormation stack.'); + console.error('Make sure the stack was deployed with DataFusion Distributed engine.'); + console.error(` WorkerInstanceIds: ${instanceIdsStr ?? 'not found'}`); + process.exit(1); + } + + const instanceIds = instanceIdsStr.split(','); + console.log(`Target instances: ${instanceIds.join(', ')}\n`); + + const command = process.argv.slice(2).join(" ") + + const results = await Promise.all(instanceIds.map(id => sendSsmCommand(id, [command]))); + + for (let i = 0; i < instanceIds.length; i++) { + const result = results[i]; + console.log(`\n--- ${instanceIds[i]} ---`); + if (result.stdout) { + console.log('STDOUT:', result.stdout); + } + if (result.stderr) { + console.log('STDERR:', result.stderr); + } + } +} + +main().catch((error) => { + console.error('Fast deploy failed:', error); + process.exit(1); +}); diff --git a/benchmarks/cdk/bin/spark-bench.ts b/benchmarks/cdk/bin/spark-bench.ts new file mode 100644 index 00000000..243624f3 --- /dev/null +++ b/benchmarks/cdk/bin/spark-bench.ts @@ -0,0 +1,111 @@ +import { Command } from "commander"; +import { z } from 'zod'; +import { BenchmarkRunner, runBenchmark, TableSpec } from "./@bench-common"; + +// Remember to port-forward the Spark HTTP server with +// aws ssm start-session --target {host-id} --document-name AWS-StartPortForwardingSession --parameters "portNumber=9003,localPortNumber=9003" + +async function main() { + const program = new Command(); + + program + .requiredOption('--dataset ', 'Dataset to run queries on') + .option('-i, --iterations ', 'Number of iterations', '3') + .option('--queries ', 'Specific queries to run', undefined) + .option('--debug ', 'Print the generated plans to stdout') + .option('--warmup ', 'Perform a warmup query before the benchmarks') + .parse(process.argv); + + const options = program.opts(); + + const dataset: string = options.dataset + const iterations = parseInt(options.iterations); + const queries = options.queries?.split(",") ?? [] + const debug = options.debug === 'true' || options.debug === 1 + const warmup = options.warmup === 'true' || options.debug === 1 + + const runner = new SparkRunner({}); + + await runBenchmark(runner, { + dataset, + engine: 'spark', + iterations, + queries, + debug, + warmup + }); +} + +const QueryResponse = z.object({ + count: z.number() +}) +type QueryResponse = z.infer + +class SparkRunner implements BenchmarkRunner { + private url = 'http://localhost:9003'; + + constructor(private readonly options: {}) { + } + + async executeQuery(sql: string): Promise<{ rowCount: number, plan: string }> { + // Fix TPCH query 4: Add DATE prefix to date literals + sql = sql.replace(/(? { + const response = await fetch(`${this.url}/query`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + query: sql.trim().replace(/;+$/, '') + }) + }); + + if (!response.ok) { + const msg = await response.text(); + throw new Error(`Query failed: ${response.status} ${msg}`); + } + + return QueryResponse.parse(await response.json()); + } + + async createTables(tables: TableSpec[]): Promise { + for (const table of tables) { + // Spark requires s3a:// protocol, not s3:// + const s3aPath = table.s3Path.replace('s3://', 's3a://'); + + // Create temporary view from Parquet files + const createViewStmt = ` + CREATE OR REPLACE TEMPORARY VIEW ${table.name} + USING parquet + OPTIONS (path '${s3aPath}') + `; + await this.query(createViewStmt); + } + } + +} + +main() + .catch(err => { + console.error(err) + process.exit(1) + }) diff --git a/benchmarks/cdk/bin/spark_http.py b/benchmarks/cdk/bin/spark_http.py new file mode 100644 index 00000000..1b398173 --- /dev/null +++ b/benchmarks/cdk/bin/spark_http.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +import os +from flask import Flask, request, jsonify +from pyspark.sql import SparkSession + +app = Flask(__name__) + +# Initialize Spark session +spark = None + +def get_spark(): + global spark + if spark is None: + master_host = os.environ.get('SPARK_MASTER_HOST', 'localhost') + spark_jars = os.environ.get('SPARK_JARS', '/opt/spark/jars/hadoop-aws-3.4.1.jar,/opt/spark/jars/bundle-2.29.52.jar,/opt/spark/jars/aws-java-sdk-bundle-1.12.262.jar') + spark = SparkSession.builder \ + .appName("SparkHTTPServer") \ + .master(f"spark://{master_host}:7077") \ + .config("spark.jars", spark_jars) \ + .config("spark.sql.catalogImplementation", "hive") \ + .config("spark.hadoop.fs.s3a.impl", "org.apache.hadoop.fs.s3a.S3AFileSystem") \ + .config("spark.hadoop.fs.s3a.aws.credentials.provider", "com.amazonaws.auth.InstanceProfileCredentialsProvider") \ + .enableHiveSupport() \ + .getOrCreate() + + # Set log level to reduce noise + spark.sparkContext.setLogLevel("WARN") + return spark + +@app.route('/health', methods=['GET']) +def health(): + """Health check endpoint""" + return jsonify({"status": "healthy"}), 200 + +@app.route('/query', methods=['POST']) +def execute_query(): + """Execute a SQL query on Spark""" + try: + data = request.get_json() + if not data or 'query' not in data: + return jsonify({"error": "Missing 'query' in request body"}), 400 + + query = data['query'] + + # Execute the query + spark_session = get_spark() + df = spark_session.sql(query) + + # Get row count without collecting all data + count = df.count() + + return jsonify({"count": count}), 200 + + except Exception as e: + return str(e), 500 + +if __name__ == '__main__': + # Run Flask server on port 9000 + port = int(os.environ.get('HTTP_PORT', 9003)) + app.run(host='0.0.0.0', port=port, debug=False) diff --git a/benchmarks/cdk/bin/trino-bench.ts b/benchmarks/cdk/bin/trino-bench.ts new file mode 100644 index 00000000..368dc227 --- /dev/null +++ b/benchmarks/cdk/bin/trino-bench.ts @@ -0,0 +1,829 @@ +import { Command } from "commander"; +import { BenchmarkRunner, runBenchmark, TableSpec } from "./@bench-common"; + +// Remember to port-forward Trino coordinator with +// aws ssm start-session --target {instance-0-id} --document-name AWS-StartPortForwardingSession --parameters "portNumber=8080,localPortNumber=8080" + +async function main() { + const program = new Command(); + + program + .requiredOption('--dataset ', 'Scale factor', '1') + .option('-i, --iterations ', 'Number of iterations', '3') + .option('--queries ', 'Specific queries to run', undefined) + .option('--debug ', 'Print the generated plans to stdout') + .option('--warmup ', 'Perform a warmup query before the benchmarks') + .parse(process.argv); + + const options = program.opts(); + + const dataset: string = options.dataset + const iterations = parseInt(options.iterations); + const queries = options.queries?.split(",") ?? [] + const debug = options.debug === 'true' || options.debug === 1 + const warmup = options.warmup === 'true' || options.debug === 1 + + const runner = new TrinoRunner(); + + await runBenchmark(runner, { + dataset, + engine: 'trino', + iterations, + queries, + debug, + warmup + }); +} + +class TrinoRunner implements BenchmarkRunner { + private trinoUrl = 'http://localhost:8080'; + private schema?: string + + async executeQuery(sql: string): Promise<{ rowCount: number, plan: string }> { + // Fix TPCH query 4: Add DATE prefix to date literals that don't have it. + sql = sql.replace(/(? { + if (!this.schema) { + throw new Error("No schema available, where the tables created?") + } + + // Submit query + const submitResponse = await fetch(`${this.trinoUrl}/v1/statement`, { + method: 'POST', + headers: { + 'X-Trino-User': 'benchmark', + 'X-Trino-Catalog': 'hive', + 'X-Trino-Schema': this.schema ?? '', + }, + body: sql.trim().replace(/;+$/, ''), + }); + + if (!submitResponse.ok) { + const msg = await submitResponse.text(); + throw new Error(`Query submission failed: ${submitResponse.status} ${msg}`); + } + + let result: any = await submitResponse.json(); + let rowCount = 0; + let plan = ""; + + // Poll for results + while (result.nextUri) { + const pollResponse = await fetch(result.nextUri); + + if (!pollResponse.ok) { + const msg = await pollResponse.text(); + throw new Error(`Query polling failed: ${pollResponse.status} ${msg}`); + } + + result = await pollResponse.json(); + + // Count rows if data is present + if (result.data) { + if (typeof result.data?.[0]?.[0] === 'string') { + plan = result.data[0][0] + // Extract row count from EXPLAIN ANALYZE output + const outputMatch = plan.match(/Output.*?(\d+)\s+rows/i); + if (outputMatch) { + rowCount = parseInt(outputMatch[1]); + } + } else { + rowCount += result.data.length; + } + } + + // Check for errors + if (result.error) { + throw new Error(`Query failed: ${result.error.message}`); + } + } + + return { rowCount, plan }; + } + + async createTables(tables: TableSpec[]): Promise { + if (tables.length === 0) { + throw new Error("No table passed") + } + let schema = tables[0].schema + let basePath = tables[0].s3Path.split('/').slice(0, -1).join("/") + + this.schema = schema + + await this.executeSingleStatement(` + CREATE SCHEMA IF NOT EXISTS hive."${schema}" WITH (location = '${basePath}')`); + + for (const table of tables) { + await this.executeSingleStatement(` + DROP TABLE IF EXISTS hive."${table.schema}"."${table.name}"`); + + await this.executeSingleStatement(` + CREATE TABLE hive."${table.schema}"."${table.name}" ${getSchema(table)} + WITH (external_location = '${table.s3Path}', format = 'PARQUET')`); + } + } +} + +const SCHEMAS: Record> = { + tpch: { + customer: `( + c_custkey bigint, + c_name varchar(25), + c_address varchar(40), + c_nationkey bigint, + c_phone varchar(15), + c_acctbal decimal(15, 2), + c_mktsegment varchar(10), + c_comment varchar(117) +)`, + lineitem: `( + l_orderkey bigint, + l_partkey bigint, + l_suppkey bigint, + l_linenumber integer, + l_quantity decimal(15, 2), + l_extendedprice decimal(15, 2), + l_discount decimal(15, 2), + l_tax decimal(15, 2), + l_returnflag varchar(1), + l_linestatus varchar(1), + l_shipdate date, + l_commitdate date, + l_receiptdate date, + l_shipinstruct varchar(25), + l_shipmode varchar(10), + l_comment varchar(44) +)`, + nation: `( + n_nationkey bigint, + n_name varchar(25), + n_regionkey bigint, + n_comment varchar(152) +)`, + orders: `( + o_orderkey bigint, + o_custkey bigint, + o_orderstatus varchar(1), + o_totalprice decimal(15, 2), + o_orderdate date, + o_orderpriority varchar(15), + o_clerk varchar(15), + o_shippriority integer, + o_comment varchar(79) +)`, + part: `( + p_partkey bigint, + p_name varchar(55), + p_mfgr varchar(25), + p_brand varchar(10), + p_type varchar(25), + p_size integer, + p_container varchar(10), + p_retailprice decimal(15, 2), + p_comment varchar(23) +)`, + partsupp: `( + ps_partkey bigint, + ps_suppkey bigint, + ps_availqty integer, + ps_supplycost decimal(15, 2), + ps_comment varchar(199) +)`, + region: `( + r_regionkey bigint, + r_name varchar(25), + r_comment varchar(152) +)`, + supplier: `( + s_suppkey bigint, + s_name varchar(25), + s_address varchar(40), + s_nationkey bigint, + s_phone varchar(15), + s_acctbal decimal(15, 2), + s_comment varchar(101) +)` + }, + clickbench: { + hits: `( + WatchID bigint, + JavaEnable smallint, + Title varchar, + GoodEvent smallint, + EventTime bigint, + EventDate date, + CounterID integer, + ClientIP integer, + RegionID integer, + UserID bigint, + CounterClass smallint, + OS smallint, + UserAgent smallint, + URL varchar, + Referer varchar, + IsRefresh smallint, + RefererCategoryID smallint, + RefererRegionID integer, + URLCategoryID smallint, + URLRegionID integer, + ResolutionWidth smallint, + ResolutionHeight smallint, + ResolutionDepth smallint, + FlashMajor smallint, + FlashMinor smallint, + FlashMinor2 varchar, + NetMajor smallint, + NetMinor smallint, + UserAgentMajor smallint, + UserAgentMinor varchar(255), + CookieEnable smallint, + JavascriptEnable smallint, + IsMobile smallint, + MobilePhone smallint, + MobilePhoneModel varchar, + Params varchar, + IPNetworkID integer, + TraficSourceID smallint, + SearchEngineID smallint, + SearchPhrase varchar, + AdvEngineID smallint, + IsArtifical smallint, + WindowClientWidth smallint, + WindowClientHeight smallint, + ClientTimeZone smallint, + ClientEventTime bigint, + SilverlightVersion1 smallint, + SilverlightVersion2 smallint, + SilverlightVersion3 integer, + SilverlightVersion4 smallint, + PageCharset varchar, + CodeVersion integer, + IsLink smallint, + IsDownload smallint, + IsNotBounce smallint, + FUniqID bigint, + OriginalURL varchar, + HID integer, + IsOldCounter smallint, + IsEvent smallint, + IsParameter smallint, + DontCountHits smallint, + WithHash smallint, + HitColor varchar(1), + LocalEventTime bigint, + Age smallint, + Sex smallint, + Income smallint, + Interests smallint, + Robotness smallint, + RemoteIP integer, + WindowName integer, + OpenerName integer, + HistoryLength smallint, + BrowserLanguage varchar, + BrowserCountry varchar, + SocialNetwork varchar, + SocialAction varchar, + HTTPError smallint, + SendTiming integer, + DNSTiming integer, + ConnectTiming integer, + ResponseStartTiming integer, + ResponseEndTiming integer, + FetchTiming integer, + SocialSourceNetworkID smallint, + SocialSourcePage varchar, + ParamPrice bigint, + ParamOrderID varchar, + ParamCurrency varchar, + ParamCurrencyID smallint, + OpenstatServiceName varchar, + OpenstatCampaignID varchar, + OpenstatAdID varchar, + OpenstatSourceID varchar, + UTMSource varchar, + UTMMedium varchar, + UTMCampaign varchar, + UTMContent varchar, + UTMTerm varchar, + FromTag varchar, + HasGCLID smallint, + RefererHash bigint, + URLHash bigint, + CLID integer +)` + }, + tpcds: { + call_center: `( + cc_call_center_sk integer, + cc_call_center_id varchar, + cc_rec_start_date date, + cc_rec_end_date date, + cc_closed_date_sk double, + cc_open_date_sk integer, + cc_name varchar, + cc_class varchar, + cc_employees integer, + cc_sq_ft integer, + cc_hours varchar, + cc_manager varchar, + cc_mkt_id integer, + cc_mkt_class varchar, + cc_mkt_desc varchar, + cc_market_manager varchar, + cc_division integer, + cc_division_name varchar, + cc_company integer, + cc_company_name varchar, + cc_street_number varchar, + cc_street_name varchar, + cc_street_type varchar, + cc_suite_number varchar, + cc_city varchar, + cc_county varchar, + cc_state varchar, + cc_zip varchar, + cc_country varchar, + cc_gmt_offset decimal(3, 2), + cc_tax_percentage decimal(2, 2) +)`, + catalog_page: `( + cp_catalog_page_sk integer, + cp_catalog_page_id varchar, + cp_start_date_sk double, + cp_end_date_sk double, + cp_department varchar, + cp_catalog_number double, + cp_catalog_page_number double, + cp_description varchar, + cp_type varchar +)`, + catalog_returns: `( + cr_returned_date_sk integer, + cr_returned_time_sk integer, + cr_item_sk integer, + cr_refunded_customer_sk double, + cr_refunded_cdemo_sk double, + cr_refunded_hdemo_sk double, + cr_refunded_addr_sk double, + cr_returning_customer_sk double, + cr_returning_cdemo_sk double, + cr_returning_hdemo_sk double, + cr_returning_addr_sk double, + cr_call_center_sk double, + cr_catalog_page_sk double, + cr_ship_mode_sk double, + cr_warehouse_sk double, + cr_reason_sk double, + cr_order_number integer, + cr_return_quantity double, + cr_return_amount decimal(7, 2), + cr_return_tax decimal(6, 2), + cr_return_amt_inc_tax decimal(7, 2), + cr_fee decimal(5, 2), + cr_return_ship_cost decimal(7, 2), + cr_refunded_cash decimal(7, 2), + cr_reversed_charge decimal(7, 2), + cr_store_credit decimal(7, 2), + cr_net_loss decimal(7, 2) +)`, + catalog_sales: `( + cs_sold_date_sk double, + cs_sold_time_sk double, + cs_ship_date_sk double, + cs_bill_customer_sk double, + cs_bill_cdemo_sk double, + cs_bill_hdemo_sk double, + cs_bill_addr_sk double, + cs_ship_customer_sk double, + cs_ship_cdemo_sk double, + cs_ship_hdemo_sk double, + cs_ship_addr_sk double, + cs_call_center_sk double, + cs_catalog_page_sk double, + cs_ship_mode_sk double, + cs_warehouse_sk double, + cs_item_sk integer, + cs_promo_sk double, + cs_order_number integer, + cs_quantity double, + cs_wholesale_cost decimal(5, 2), + cs_list_price decimal(5, 2), + cs_sales_price decimal(5, 2), + cs_ext_discount_amt decimal(7, 2), + cs_ext_sales_price decimal(7, 2), + cs_ext_wholesale_cost decimal(7, 2), + cs_ext_list_price decimal(7, 2), + cs_ext_tax decimal(6, 2), + cs_coupon_amt decimal(7, 2), + cs_ext_ship_cost decimal(7, 2), + cs_net_paid decimal(7, 2), + cs_net_paid_inc_tax decimal(7, 2), + cs_net_paid_inc_ship decimal(7, 2), + cs_net_paid_inc_ship_tax decimal(7, 2), + cs_net_profit decimal(7, 2) +)`, + customer: `( + c_customer_sk integer, + c_customer_id varchar, + c_current_cdemo_sk double, + c_current_hdemo_sk double, + c_current_addr_sk integer, + c_first_shipto_date_sk double, + c_first_sales_date_sk double, + c_salutation varchar, + c_first_name varchar, + c_last_name varchar, + c_preferred_cust_flag varchar, + c_birth_day double, + c_birth_month double, + c_birth_year double, + c_birth_country varchar, + c_login varchar, + c_email_address varchar, + c_last_review_date double +)`, + customer_address: `( + ca_address_sk integer, + ca_address_id varchar, + ca_street_number varchar, + ca_street_name varchar, + ca_street_type varchar, + ca_suite_number varchar, + ca_city varchar, + ca_county varchar, + ca_state varchar, + ca_zip varchar, + ca_country varchar, + ca_gmt_offset decimal(4, 2), + ca_location_type varchar +)`, + customer_demographics: `( + cd_demo_sk integer, + cd_gender varchar, + cd_marital_status varchar, + cd_education_status varchar, + cd_purchase_estimate integer, + cd_credit_rating varchar, + cd_dep_count integer, + cd_dep_employed_count integer, + cd_dep_college_count integer +)`, + date_dim: `( + d_date_sk integer, + d_date_id varchar, + d_date date, + d_month_seq integer, + d_week_seq integer, + d_quarter_seq integer, + d_year integer, + d_dow integer, + d_moy integer, + d_dom integer, + d_qoy integer, + d_fy_year integer, + d_fy_quarter_seq integer, + d_fy_week_seq integer, + d_day_name varchar, + d_quarter_name varchar, + d_holiday varchar, + d_weekend varchar, + d_following_holiday varchar, + d_first_dom integer, + d_last_dom integer, + d_same_day_ly integer, + d_same_day_lq integer, + d_current_day varchar, + d_current_week varchar, + d_current_month varchar, + d_current_quarter varchar, + d_current_year varchar +)`, + household_demographics: `( + hd_demo_sk integer, + hd_income_band_sk integer, + hd_buy_potential varchar, + hd_dep_count integer, + hd_vehicle_count integer +)`, + income_band: `( + ib_income_band_sk integer, + ib_lower_bound integer, + ib_upper_bound integer +)`, + inventory: `( + inv_date_sk integer, + inv_item_sk integer, + inv_warehouse_sk integer, + inv_quantity_on_hand double +)`, + item: `( + i_item_sk integer, + i_item_id varchar, + i_rec_start_date date, + i_rec_end_date date, + i_item_desc varchar, + i_current_price decimal(4, 2), + i_wholesale_cost decimal(4, 2), + i_brand_id double, + i_brand varchar, + i_class_id double, + i_class varchar, + i_category_id double, + i_category varchar, + i_manufact_id double, + i_manufact varchar, + i_size varchar, + i_formulation varchar, + i_color varchar, + i_units varchar, + i_container varchar, + i_manager_id double, + i_product_name varchar +)`, + promotion: `( + p_promo_sk integer, + p_promo_id varchar, + p_start_date_sk double, + p_end_date_sk double, + p_item_sk double, + p_cost decimal(6, 2), + p_response_target double, + p_promo_name varchar, + p_channel_dmail varchar, + p_channel_email varchar, + p_channel_catalog varchar, + p_channel_tv varchar, + p_channel_radio varchar, + p_channel_press varchar, + p_channel_event varchar, + p_channel_demo varchar, + p_channel_details varchar, + p_purpose varchar, + p_discount_active varchar +)`, + reason: `( + r_reason_sk integer, + r_reason_id varchar, + r_reason_desc varchar +)`, + ship_mode: `( + sm_ship_mode_sk integer, + sm_ship_mode_id varchar, + sm_type varchar, + sm_code varchar, + sm_carrier varchar, + sm_contract varchar +)`, + store: `( + s_store_sk integer, + s_store_id varchar, + s_rec_start_date date, + s_rec_end_date date, + s_closed_date_sk double, + s_store_name varchar, + s_number_employees integer, + s_floor_space integer, + s_hours varchar, + s_manager varchar, + s_market_id integer, + s_geography_class varchar, + s_market_desc varchar, + s_market_manager varchar, + s_division_id integer, + s_division_name varchar, + s_company_id integer, + s_company_name varchar, + s_street_number varchar, + s_street_name varchar, + s_street_type varchar, + s_suite_number varchar, + s_city varchar, + s_county varchar, + s_state varchar, + s_zip varchar, + s_country varchar, + s_gmt_offset decimal(3, 2), + s_tax_precentage decimal(2, 2) +)`, + store_returns: `( + sr_returned_date_sk double, + sr_return_time_sk double, + sr_item_sk integer, + sr_customer_sk double, + sr_cdemo_sk double, + sr_hdemo_sk double, + sr_addr_sk double, + sr_store_sk double, + sr_reason_sk double, + sr_ticket_number integer, + sr_return_quantity double, + sr_return_amt decimal(7, 2), + sr_return_tax decimal(6, 2), + sr_return_amt_inc_tax decimal(7, 2), + sr_fee decimal(5, 2), + sr_return_ship_cost decimal(6, 2), + sr_refunded_cash decimal(7, 2), + sr_reversed_charge decimal(7, 2), + sr_store_credit decimal(7, 2), + sr_net_loss decimal(6, 2) +)`, + store_sales: `( + ss_sold_date_sk double, + ss_sold_time_sk double, + ss_item_sk integer, + ss_customer_sk double, + ss_cdemo_sk double, + ss_hdemo_sk double, + ss_addr_sk double, + ss_store_sk double, + ss_promo_sk double, + ss_ticket_number integer, + ss_quantity double, + ss_wholesale_cost decimal(5, 2), + ss_list_price decimal(5, 2), + ss_sales_price decimal(5, 2), + ss_ext_discount_amt decimal(7, 2), + ss_ext_sales_price decimal(7, 2), + ss_ext_wholesale_cost decimal(7, 2), + ss_ext_list_price decimal(7, 2), + ss_ext_tax decimal(6, 2), + ss_coupon_amt decimal(7, 2), + ss_net_paid decimal(7, 2), + ss_net_paid_inc_tax decimal(7, 2), + ss_net_profit decimal(6, 2) +)`, + time_dim: `( + t_time_sk integer, + t_time_id varchar, + t_time integer, + t_hour integer, + t_minute integer, + t_second integer, + t_am_pm varchar, + t_shift varchar, + t_sub_shift varchar, + t_meal_time varchar +)`, + warehouse: `( + w_warehouse_sk integer, + w_warehouse_id varchar, + w_warehouse_name varchar, + w_warehouse_sq_ft integer, + w_street_number varchar, + w_street_name varchar, + w_street_type varchar, + w_suite_number varchar, + w_city varchar, + w_county varchar, + w_state varchar, + w_zip varchar, + w_country varchar, + w_gmt_offset decimal(3, 2) +)`, + web_page: `( + wp_web_page_sk integer, + wp_web_page_id varchar, + wp_rec_start_date date, + wp_rec_end_date date, + wp_creation_date_sk integer, + wp_access_date_sk integer, + wp_autogen_flag varchar, + wp_customer_sk double, + wp_url varchar, + wp_type varchar, + wp_char_count integer, + wp_link_count integer, + wp_image_count integer, + wp_max_ad_count integer +)`, + web_returns: `( + wr_returned_date_sk double, + wr_returned_time_sk double, + wr_item_sk integer, + wr_refunded_customer_sk double, + wr_refunded_cdemo_sk double, + wr_refunded_hdemo_sk double, + wr_refunded_addr_sk double, + wr_returning_customer_sk double, + wr_returning_cdemo_sk double, + wr_returning_hdemo_sk double, + wr_returning_addr_sk double, + wr_web_page_sk double, + wr_reason_sk double, + wr_order_number integer, + wr_return_quantity double, + wr_return_amt decimal(7, 2), + wr_return_tax decimal(6, 2), + wr_return_amt_inc_tax decimal(7, 2), + wr_fee decimal(5, 2), + wr_return_ship_cost decimal(7, 2), + wr_refunded_cash decimal(7, 2), + wr_reversed_charge decimal(7, 2), + wr_account_credit decimal(7, 2), + wr_net_loss decimal(7, 2) +)`, + web_sales: `( + ws_sold_date_sk double, + ws_sold_time_sk double, + ws_ship_date_sk double, + ws_item_sk integer, + ws_bill_customer_sk double, + ws_bill_cdemo_sk double, + ws_bill_hdemo_sk double, + ws_bill_addr_sk double, + ws_ship_customer_sk double, + ws_ship_cdemo_sk double, + ws_ship_hdemo_sk double, + ws_ship_addr_sk double, + ws_web_page_sk double, + ws_web_site_sk double, + ws_ship_mode_sk double, + ws_warehouse_sk double, + ws_promo_sk double, + ws_order_number integer, + ws_quantity double, + ws_wholesale_cost decimal(5, 2), + ws_list_price decimal(5, 2), + ws_sales_price decimal(5, 2), + ws_ext_discount_amt decimal(7, 2), + ws_ext_sales_price decimal(7, 2), + ws_ext_wholesale_cost decimal(7, 2), + ws_ext_list_price decimal(7, 2), + ws_ext_tax decimal(6, 2), + ws_coupon_amt decimal(7, 2), + ws_ext_ship_cost decimal(7, 2), + ws_net_paid decimal(7, 2), + ws_net_paid_inc_tax decimal(7, 2), + ws_net_paid_inc_ship decimal(7, 2), + ws_net_paid_inc_ship_tax decimal(7, 2), + ws_net_profit decimal(7, 2) +)`, + web_site: `( + web_site_sk integer, + web_site_id varchar, + web_rec_start_date date, + web_rec_end_date date, + web_name varchar, + web_open_date_sk double, + web_close_date_sk double, + web_class varchar, + web_manager varchar, + web_mkt_id integer, + web_mkt_class varchar, + web_mkt_desc varchar, + web_market_manager varchar, + web_company_id integer, + web_company_name varchar, + web_street_number varchar, + web_street_name varchar, + web_street_type varchar, + web_suite_number varchar, + web_city varchar, + web_county varchar, + web_state varchar, + web_zip varchar, + web_country varchar, + web_gmt_offset decimal(3, 2), + web_tax_percentage decimal(2, 2) +)` + } +} + +function getSchema(table: TableSpec): string { + const tableSchema = SCHEMAS[table.schema.split("_")[0]]?.[table.name] + if (!tableSchema) { + throw new Error(`Could not find table ${table.name} in schema ${table.schema}`) + } + return tableSchema +} + +main() + .catch(err => { + console.error(err) + process.exit(1) + }) diff --git a/benchmarks/cdk/bin/worker.rs b/benchmarks/cdk/bin/worker.rs new file mode 100644 index 00000000..26fa275c --- /dev/null +++ b/benchmarks/cdk/bin/worker.rs @@ -0,0 +1,322 @@ +use async_trait::async_trait; +use aws_config::BehaviorVersion; +use aws_sdk_ec2::Client as Ec2Client; +use axum::{Json, Router, extract::Query, http::StatusCode, routing::get}; +use datafusion::common::DataFusionError; +use datafusion::common::instant::Instant; +use datafusion::common::runtime::SpawnedTask; +use datafusion::execution::SessionStateBuilder; +use datafusion::execution::runtime_env::RuntimeEnv; +use datafusion::physical_plan::execute_stream; +use datafusion::prelude::SessionContext; +use datafusion_distributed::{ + ChannelResolver, DistributedExt, DistributedMetricsFormat, DistributedPhysicalOptimizerRule, + Worker, WorkerResolver, display_plan_ascii, get_distributed_channel_resolver, + get_distributed_worker_resolver, rewrite_distributed_plan_with_metrics, +}; +use futures::{StreamExt, TryFutureExt}; +use log::{error, info, warn}; +use object_store::aws::AmazonS3Builder; +use serde::Serialize; +use std::collections::HashMap; +use std::error::Error; +use std::fmt::Display; +use std::sync::atomic::AtomicBool; +use std::sync::{Arc, RwLock}; +use std::time::Duration; +use structopt::StructOpt; +use tonic::transport::Server; +use url::Url; + +#[global_allocator] +static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc; + +pub(crate) mod built_info { + // The file has been placed there by the build script. + include!(concat!(env!("OUT_DIR"), "/built.rs")); +} + +#[derive(Serialize)] +struct QueryResult { + plan: String, + count: usize, +} + +#[derive(Serialize)] +struct WorkerInfo { + worker_urls: Vec, + git_commit_hash: String, + build_time_utc: String, + errors: Vec, +} + +#[derive(Debug, StructOpt, Clone)] +#[structopt(about = "worker spawn command")] +struct Cmd { + /// The bucket name. + #[structopt(long, default_value = "datafusion-distributed-benchmarks")] + bucket: String, + + // Turns broadcast joins on. + #[structopt(long)] + broadcast_joins: bool, +} + +#[tokio::main] +async fn main() -> Result<(), Box> { + env_logger::builder() + .filter_level(log::LevelFilter::Info) + .parse_default_env() + .init(); + + let cmd = Cmd::from_args(); + + const LISTENER_ADDR: &str = "0.0.0.0:9000"; + const WORKER_ADDR: &str = "0.0.0.0:9001"; + + info!("Starting HTTP listener on {LISTENER_ADDR}..."); + let listener = tokio::net::TcpListener::bind(LISTENER_ADDR).await?; + + // Register S3 object store + let s3_url = Url::parse(&format!("s3://{}", cmd.bucket))?; + + info!("Building shared SessionContext for the whole lifetime of the HTTP listener..."); + let s3 = Arc::new( + AmazonS3Builder::from_env() + .with_bucket_name(s3_url.host().unwrap().to_string()) + .build()?, + ); + let runtime_env = Arc::new(RuntimeEnv::default()); + runtime_env.register_object_store(&s3_url, s3); + + let state = SessionStateBuilder::new() + .with_default_features() + .with_runtime_env(Arc::clone(&runtime_env)) + .with_distributed_worker_resolver(Ec2WorkerResolver::new()) + .with_physical_optimizer_rule(Arc::new(DistributedPhysicalOptimizerRule)) + .with_distributed_broadcast_joins(cmd.broadcast_joins)? + .build(); + let ctx = SessionContext::from(state); + let ctx_clone = ctx.clone(); + + let worker = Worker::default().with_runtime_env(runtime_env); + let http_server = axum::serve( + listener, + Router::new() + .route( + "/info", + get(move || async move { + let ctx = ctx_clone.clone(); + + let worker_resolver = + get_distributed_worker_resolver(ctx.state_ref().read().config()) + .map_err(err)?; + let channel_resolver = + get_distributed_channel_resolver(ctx.task_ctx().as_ref()); + + let mut worker_urls = vec![]; + let mut errors = vec![]; + for worker_url in worker_resolver.get_urls().map_err(err)? { + if let Err(err) = channel_resolver + .get_flight_client_for_url(&worker_url) + .await + { + errors.push(err.to_string()) + } else { + worker_urls.push(worker_url); + }; + } + let worker_urls = worker_urls.into_iter().map(|v| v.to_string()).collect(); + + Ok::<_, (StatusCode, String)>(Json(WorkerInfo { + worker_urls, + git_commit_hash: built_info::GIT_COMMIT_HASH + .unwrap_or_default() + .to_string(), + build_time_utc: built_info::BUILT_TIME_UTC.to_string(), + errors, + })) + }), + ) + .route( + "/", + get(move |Query(params): Query>| { + let ctx = ctx.clone(); + + async move { + let sql = params.get("sql").ok_or(err("Missing 'sql' parameter"))?; + + let mut df_opt = None; + for sql in sql.split(";") { + if sql.trim().is_empty() { + continue; + } + let df = ctx.sql(sql).await.map_err(err)?; + df_opt = Some(df); + } + let Some(df) = df_opt else { + return Err(err("Empty 'sql' parameter")); + }; + + let start = Instant::now(); + + info!("Executing query..."); + let abort_notifier = AbortNotifier::new("Query aborted"); + let abort_notifier_clone = abort_notifier.clone(); + let task = SpawnedTask::spawn(async move { + let _ = abort_notifier_clone; + loop { + tokio::time::sleep(Duration::from_secs(5)).await; + info!("Query still running..."); + } + }); + let physical = df.create_physical_plan().await.map_err(err)?; + let mut stream = + execute_stream(physical.clone(), ctx.task_ctx()).map_err(err)?; + let mut count = 0; + while let Some(batch) = stream.next().await { + count += batch.map_err(err)?.num_rows(); + info!("Gathered {count} rows, query still in progress..") + } + let physical = rewrite_distributed_plan_with_metrics( + physical, + DistributedMetricsFormat::PerTask, + ) + .map_err(err)?; + let plan = display_plan_ascii(physical.as_ref(), true); + drop(task); + + let elapsed = start.elapsed(); + let ms = elapsed.as_secs_f64() * 1000.0; + info!("Finished executing query:\n{sql}\n\n{plan}"); + info!("Returned {count} rows in {ms} ms"); + abort_notifier.finished(); + + Ok::<_, (StatusCode, String)>(Json(QueryResult { count, plan })) + } + .inspect_err(|(_, msg)| { + error!("Error executing query: {msg}"); + }) + }), + ), + ); + let grpc_server = Server::builder() + .add_service(worker.into_flight_server()) + .serve(WORKER_ADDR.parse()?); + + info!("Started listener HTTP server in {LISTENER_ADDR}"); + info!("Started distributed DataFusion worker in {WORKER_ADDR}"); + + tokio::select! { + result = http_server => result?, + result = grpc_server => result?, + } + + Ok(()) +} + +struct AbortNotifier { + aborted: AtomicBool, + msg: String, +} + +impl AbortNotifier { + fn new(msg: impl Display) -> Arc { + Arc::new(AbortNotifier { + aborted: AtomicBool::new(true), + msg: msg.to_string(), + }) + } + + fn finished(&self) { + self.aborted + .store(false, std::sync::atomic::Ordering::Relaxed) + } +} + +impl Drop for AbortNotifier { + fn drop(&mut self) { + if self.aborted.load(std::sync::atomic::Ordering::Relaxed) { + warn!("{}", self.msg); + } + } +} + +fn err(s: impl Display) -> (StatusCode, String) { + (StatusCode::INTERNAL_SERVER_ERROR, s.to_string()) +} + +#[derive(Clone)] +struct Ec2WorkerResolver { + urls: Arc>>, +} + +async fn background_ec2_worker_resolver(urls: Arc>>) { + tokio::spawn(async move { + let config = aws_config::load_defaults(BehaviorVersion::latest()).await; + let ec2_client = Ec2Client::new(&config); + + loop { + let result = match ec2_client + .describe_instances() + .filters( + aws_sdk_ec2::types::Filter::builder() + .name("tag:BenchmarkCluster") + .values("datafusion") + .build(), + ) + .filters( + aws_sdk_ec2::types::Filter::builder() + .name("instance-state-name") + .values("running") + .build(), + ) + .send() + .await + { + Ok(v) => v, + Err(err) => { + eprintln!("Error discovering workers: {}", err.into_service_error()); + continue; + } + }; + + let mut workers = Vec::new(); + for reservation in result.reservations() { + for instance in reservation.instances() { + if let Some(private_ip) = instance.private_ip_address() { + let url = Url::parse(&format!("http://{private_ip}:9001")).unwrap(); + workers.push(url); + } + } + } + if !urls.read().unwrap().eq(&workers) { + info!( + "New set of workers found: {}", + workers + .iter() + .map(|url| url.to_string()) + .collect::>() + .join(", ") + ); + *urls.write().unwrap() = workers; + } + tokio::time::sleep(Duration::from_secs(1)).await; + } + }); +} + +impl Ec2WorkerResolver { + fn new() -> Self { + let urls = Arc::new(RwLock::new(Vec::new())); + tokio::spawn(background_ec2_worker_resolver(urls.clone())); + Self { urls } + } +} + +#[async_trait] +impl WorkerResolver for Ec2WorkerResolver { + fn get_urls(&self) -> Result, DataFusionError> { + Ok(self.urls.read().unwrap().clone()) + } +} diff --git a/benchmarks/cdk/cdk.context.json b/benchmarks/cdk/cdk.context.json new file mode 100644 index 00000000..e26be244 --- /dev/null +++ b/benchmarks/cdk/cdk.context.json @@ -0,0 +1,5 @@ +{ + "acknowledged-issue-numbers": [ + 34892 + ] +} diff --git a/benchmarks/cdk/cdk.json b/benchmarks/cdk/cdk.json new file mode 100644 index 00000000..c1f04baf --- /dev/null +++ b/benchmarks/cdk/cdk.json @@ -0,0 +1,101 @@ +{ + "app": "npx ts-node --prefer-ts-exts bin/cdk.ts", + "watch": { + "include": [ + "**" + ], + "exclude": [ + "README.md", + "cdk*.json", + "**/*.d.ts", + "**/*.js", + "tsconfig.json", + "package*.json", + "yarn.lock", + "node_modules", + "test" + ] + }, + "context": { + "@aws-cdk/aws-signer:signingProfileNamePassedToCfn": true, + "@aws-cdk/aws-ecs-patterns:secGroupsDisablesImplicitOpenListener": true, + "@aws-cdk/aws-lambda:recognizeLayerVersion": true, + "@aws-cdk/core:checkSecretUsage": true, + "@aws-cdk/core:target-partitions": [ + "aws", + "aws-cn" + ], + "@aws-cdk-containers/ecs-service-extensions:enableDefaultLogDriver": true, + "@aws-cdk/aws-ec2:uniqueImdsv2TemplateName": true, + "@aws-cdk/aws-ecs:arnFormatIncludesClusterName": true, + "@aws-cdk/aws-iam:minimizePolicies": true, + "@aws-cdk/core:validateSnapshotRemovalPolicy": true, + "@aws-cdk/aws-codepipeline:crossAccountKeyAliasStackSafeResourceName": true, + "@aws-cdk/aws-s3:createDefaultLoggingPolicy": true, + "@aws-cdk/aws-sns-subscriptions:restrictSqsDescryption": true, + "@aws-cdk/aws-apigateway:disableCloudWatchRole": true, + "@aws-cdk/core:enablePartitionLiterals": true, + "@aws-cdk/aws-events:eventsTargetQueueSameAccount": true, + "@aws-cdk/aws-ecs:disableExplicitDeploymentControllerForCircuitBreaker": true, + "@aws-cdk/aws-iam:importedRoleStackSafeDefaultPolicyName": true, + "@aws-cdk/aws-s3:serverAccessLogsUseBucketPolicy": true, + "@aws-cdk/aws-route53-patters:useCertificate": true, + "@aws-cdk/customresources:installLatestAwsSdkDefault": false, + "@aws-cdk/aws-rds:databaseProxyUniqueResourceName": true, + "@aws-cdk/aws-codedeploy:removeAlarmsFromDeploymentGroup": true, + "@aws-cdk/aws-apigateway:authorizerChangeDeploymentLogicalId": true, + "@aws-cdk/aws-ec2:launchTemplateDefaultUserData": true, + "@aws-cdk/aws-secretsmanager:useAttachedSecretResourcePolicyForSecretTargetAttachments": true, + "@aws-cdk/aws-redshift:columnId": true, + "@aws-cdk/aws-stepfunctions-tasks:enableEmrServicePolicyV2": true, + "@aws-cdk/aws-ec2:restrictDefaultSecurityGroup": true, + "@aws-cdk/aws-apigateway:requestValidatorUniqueId": true, + "@aws-cdk/aws-kms:aliasNameRef": true, + "@aws-cdk/aws-kms:applyImportedAliasPermissionsToPrincipal": true, + "@aws-cdk/aws-autoscaling:generateLaunchTemplateInsteadOfLaunchConfig": true, + "@aws-cdk/core:includePrefixInUniqueNameGeneration": true, + "@aws-cdk/aws-efs:denyAnonymousAccess": true, + "@aws-cdk/aws-opensearchservice:enableOpensearchMultiAzWithStandby": true, + "@aws-cdk/aws-lambda-nodejs:useLatestRuntimeVersion": true, + "@aws-cdk/aws-efs:mountTargetOrderInsensitiveLogicalId": true, + "@aws-cdk/aws-rds:auroraClusterChangeScopeOfInstanceParameterGroupWithEachParameters": true, + "@aws-cdk/aws-appsync:useArnForSourceApiAssociationIdentifier": true, + "@aws-cdk/aws-rds:preventRenderingDeprecatedCredentials": true, + "@aws-cdk/aws-codepipeline-actions:useNewDefaultBranchForCodeCommitSource": true, + "@aws-cdk/aws-cloudwatch-actions:changeLambdaPermissionLogicalIdForLambdaAction": true, + "@aws-cdk/aws-codepipeline:crossAccountKeysDefaultValueToFalse": true, + "@aws-cdk/aws-codepipeline:defaultPipelineTypeToV2": true, + "@aws-cdk/aws-kms:reduceCrossAccountRegionPolicyScope": true, + "@aws-cdk/aws-eks:nodegroupNameAttribute": true, + "@aws-cdk/aws-ec2:ebsDefaultGp3Volume": true, + "@aws-cdk/aws-ecs:removeDefaultDeploymentAlarm": true, + "@aws-cdk/custom-resources:logApiResponseDataPropertyTrueDefault": false, + "@aws-cdk/aws-s3:keepNotificationInImportedBucket": false, + "@aws-cdk/core:explicitStackTags": true, + "@aws-cdk/aws-ecs:enableImdsBlockingDeprecatedFeature": false, + "@aws-cdk/aws-ecs:disableEcsImdsBlocking": true, + "@aws-cdk/aws-ecs:reduceEc2FargateCloudWatchPermissions": true, + "@aws-cdk/aws-dynamodb:resourcePolicyPerReplica": true, + "@aws-cdk/aws-ec2:ec2SumTImeoutEnabled": true, + "@aws-cdk/aws-appsync:appSyncGraphQLAPIScopeLambdaPermission": true, + "@aws-cdk/aws-rds:setCorrectValueForDatabaseInstanceReadReplicaInstanceResourceId": true, + "@aws-cdk/core:cfnIncludeRejectComplexResourceUpdateCreatePolicyIntrinsics": true, + "@aws-cdk/aws-lambda-nodejs:sdkV3ExcludeSmithyPackages": true, + "@aws-cdk/aws-stepfunctions-tasks:fixRunEcsTaskPolicy": true, + "@aws-cdk/aws-ec2:bastionHostUseAmazonLinux2023ByDefault": true, + "@aws-cdk/aws-route53-targets:userPoolDomainNameMethodWithoutCustomResource": true, + "@aws-cdk/aws-elasticloadbalancingV2:albDualstackWithoutPublicIpv4SecurityGroupRulesDefault": true, + "@aws-cdk/aws-iam:oidcRejectUnauthorizedConnections": true, + "@aws-cdk/core:enableAdditionalMetadataCollection": true, + "@aws-cdk/aws-lambda:createNewPoliciesWithAddToRolePolicy": false, + "@aws-cdk/aws-s3:setUniqueReplicationRoleName": true, + "@aws-cdk/aws-events:requireEventBusPolicySid": true, + "@aws-cdk/core:aspectPrioritiesMutating": true, + "@aws-cdk/aws-dynamodb:retainTableReplica": true, + "@aws-cdk/aws-stepfunctions:useDistributedMapResultWriterV2": true, + "@aws-cdk/s3-notifications:addS3TrustKeyPolicyForSnsSubscriptions": true, + "@aws-cdk/aws-ec2:requirePrivateSubnetsForEgressOnlyInternetGateway": true, + "@aws-cdk/aws-s3:publicAccessBlockedByDefault": true, + "@aws-cdk/aws-lambda:useCdkManagedLogGroup": true + } +} diff --git a/benchmarks/cdk/jest.config.js b/benchmarks/cdk/jest.config.js new file mode 100644 index 00000000..08263b89 --- /dev/null +++ b/benchmarks/cdk/jest.config.js @@ -0,0 +1,8 @@ +module.exports = { + testEnvironment: 'node', + roots: ['/test'], + testMatch: ['**/*.test.ts'], + transform: { + '^.+\\.tsx?$': 'ts-jest' + } +}; diff --git a/benchmarks/cdk/lib/ballista.ts b/benchmarks/cdk/lib/ballista.ts new file mode 100644 index 00000000..430cccd8 --- /dev/null +++ b/benchmarks/cdk/lib/ballista.ts @@ -0,0 +1,194 @@ +import { + AfterEc2MachinesContext, + BeforeEc2MachinesContext, + QueryEngine, + ROOT, + sendCommandsUnconditionally, + OnEc2MachinesContext +} from "./cdk-stack"; +import * as s3assets from "aws-cdk-lib/aws-s3-assets"; +import path from "path"; +import {execSync} from "child_process"; + +let ballistaServerBinary: s3assets.Asset +let ballistaSchedulerBinary: s3assets.Asset +let ballistaExecutorBinary: s3assets.Asset + +// Ballista is built as a standalone project, separate from the main workspace +const BALLISTA_DIR = path.join(ROOT, 'benchmarks/cdk/ballista') + +export const BALLISTA_ENGINE: QueryEngine = { + beforeEc2Machines(ctx: BeforeEc2MachinesContext): void { + console.log('Building Ballista binaries (standalone project)...'); + execSync('cargo zigbuild --release --target x86_64-unknown-linux-gnu', { + cwd: BALLISTA_DIR, + stdio: 'inherit', + }); + console.log('Ballista binaries built successfully'); + + const targetDir = path.join(BALLISTA_DIR, 'target/x86_64-unknown-linux-gnu/release') + + ballistaServerBinary = new s3assets.Asset(ctx.scope, 'BallistaServerBinary', { + path: path.join(targetDir, 'ballista-http'), + }) + + ballistaSchedulerBinary = new s3assets.Asset(ctx.scope, 'BallistaSchedulerBinary', { + path: path.join(targetDir, 'ballista-scheduler'), + }) + + ballistaExecutorBinary = new s3assets.Asset(ctx.scope, 'BallistaExecutorBinary', { + path: path.join(targetDir, 'ballista-executor'), + }) + + ballistaServerBinary.grantRead(ctx.role) + ballistaSchedulerBinary.grantRead(ctx.role) + ballistaExecutorBinary.grantRead(ctx.role) + }, + onEc2Machine(ctx: OnEc2MachinesContext): void { + const isScheduler = ctx.instanceIdx === 0; + ctx.instanceUserData.addCommands( + // Download pre-compiled Ballista binaries from S3 + `aws s3 cp s3://${ballistaSchedulerBinary.s3BucketName}/${ballistaSchedulerBinary.s3ObjectKey} /usr/local/bin/ballista-scheduler`, + 'chmod +x /usr/local/bin/ballista-scheduler', + `aws s3 cp s3://${ballistaExecutorBinary.s3BucketName}/${ballistaExecutorBinary.s3ObjectKey} /usr/local/bin/ballista-executor`, + 'chmod +x /usr/local/bin/ballista-executor', + `aws s3 cp s3://${ballistaServerBinary.s3BucketName}/${ballistaServerBinary.s3ObjectKey} /usr/local/bin/ballista-http`, + 'chmod +x /usr/local/bin/ballista-http', + + // Create Ballista directories + 'mkdir -p /var/ballista/scheduler', + 'mkdir -p /var/ballista/executor', + 'mkdir -p /var/ballista/logs', + + // Create Ballista scheduler systemd service (coordinator only) + ...(isScheduler ? [ + `cat > /etc/systemd/system/ballista-scheduler.service << 'BALLISTA_EOF' +[Unit] +Description=Ballista Scheduler +After=network.target +[Service] +Type=simple +ExecStart=/usr/local/bin/ballista-scheduler \\ + --bind-host 0.0.0.0 \\ + --bind-port 50050 +Restart=on-failure +RestartSec=5 +User=root +WorkingDirectory=/var/ballista/scheduler +StandardOutput=append:/var/ballista/logs/scheduler.log +StandardError=append:/var/ballista/logs/scheduler.log +[Install] +WantedBy=multi-user.target +BALLISTA_EOF` + ] : []), + + // Create Ballista executor systemd service (all nodes, will be reconfigured for workers) + `cat > /etc/systemd/system/ballista-executor.service << 'BALLISTA_EOF' +[Unit] +Description=Ballista Executor +After=network.target${isScheduler ? ' ballista-scheduler.service' : ''} +${isScheduler ? 'Requires=ballista-scheduler.service' : ''} +[Service] +Type=simple +ExecStart=/usr/local/bin/ballista-executor \\ + --bind-host 0.0.0.0 \\ + --bind-port 50051 \\ + --work-dir /var/ballista/executor \\ + --scheduler-host localhost \\ + --scheduler-port 50050 +Restart=on-failure +RestartSec=5 +User=root +Environment="BUCKET=${ctx.bucketName}" +WorkingDirectory=/var/ballista/executor +StandardOutput=append:/var/ballista/logs/executor.log +StandardError=append:/var/ballista/logs/executor.log +[Install] +WantedBy=multi-user.target +BALLISTA_EOF`, + + // Create HTTP server systemd service (coordinator only for now) + ...(isScheduler ? [ + `aws s3 cp s3://${ballistaServerBinary.s3BucketName}/${ballistaServerBinary.s3ObjectKey} /usr/local/bin/ballista-http`, + 'chmod +x /usr/local/bin/ballista-http', + `cat > /etc/systemd/system/ballista-http.service << 'BALLISTA_EOF' +[Unit] +Description=Ballista HTTP Server +After=network.target ballista-scheduler.service +Requires=ballista-scheduler.service +[Service] +Type=simple +ExecStart=/usr/local/bin/ballista-http --bucket ${ctx.bucketName} +Restart=on-failure +RestartSec=5 +User=root +Environment="RUST_LOG=info" +WorkingDirectory=/var/ballista +StandardOutput=append:/var/ballista/logs/http.log +StandardError=append:/var/ballista/logs/http.log +[Install] +WantedBy=multi-user.target +BALLISTA_EOF` + ] : []), + + // Reload systemd and enable services + 'systemctl daemon-reload', + + // Enable and start scheduler (coordinator only) + ...(isScheduler ? [ + 'systemctl enable ballista-scheduler', + 'systemctl start ballista-scheduler', + // Wait for scheduler to be ready + 'sleep 5' + ] : []), + + // Enable and start executor (all nodes) + 'systemctl enable ballista-executor', + 'systemctl start ballista-executor', + + // Enable and start HTTP server (coordinator only) + ...(isScheduler ? [ + 'systemctl enable ballista-http', + 'systemctl start ballista-http' + ] : []) + ) + + }, + afterEc2Machines(ctx: AfterEc2MachinesContext) { + const [scheduler, ...executors] = ctx.instances + + // Reconfigure executors on worker nodes to point to scheduler. The executor in the machine holding the scheduler + // communicates to it using localhost, so no need to update it with scheduler.instancePrivateIp. + sendCommandsUnconditionally( + ctx.scope, + "ConfigureBallistaExecutors", + [scheduler, ...executors], + [ + `cat > /etc/systemd/system/ballista-executor.service << 'BALLISTA_EOF' +[Unit] +Description=Ballista Executor +After=network.target +[Service] +Type=simple +ExecStart=/usr/local/bin/ballista-executor \\ + --bind-host 0.0.0.0 \\ + --bind-port 50051 \\ + --work-dir /var/ballista/executor \\ + --scheduler-host ${scheduler.instancePrivateIp} \\ + --scheduler-port 50050 +Restart=on-failure +RestartSec=5 +User=root +Environment="BUCKET=${ctx.bucketName}" +WorkingDirectory=/var/ballista/executor +StandardOutput=append:/var/ballista/logs/executor.log +StandardError=append:/var/ballista/logs/executor.log +[Install] +WantedBy=multi-user.target +BALLISTA_EOF`, + 'systemctl daemon-reload', + 'systemctl restart ballista-executor', + ] + ) + } +} diff --git a/benchmarks/cdk/lib/cdk-stack.ts b/benchmarks/cdk/lib/cdk-stack.ts new file mode 100644 index 00000000..1417c464 --- /dev/null +++ b/benchmarks/cdk/lib/cdk-stack.ts @@ -0,0 +1,238 @@ +import {CfnOutput, RemovalPolicy, Stack, StackProps, Tags} from 'aws-cdk-lib'; +import * as ec2 from 'aws-cdk-lib/aws-ec2'; +import * as s3 from 'aws-cdk-lib/aws-s3'; +import * as iam from 'aws-cdk-lib/aws-iam'; +import {Construct} from 'constructs'; +import path from "path"; +import * as cr from "aws-cdk-lib/custom-resources"; + +const USER_DATA_CAUSES_REPLACEMENT = process.env['USER_DATA_CAUSES_REPLACEMENT'] == 'true' +if (USER_DATA_CAUSES_REPLACEMENT) { + console.warn("Instances will forcefully get replaced") +} + +export const ROOT = path.join(__dirname, '../../..') + +export interface BeforeEc2MachinesContext { + scope: Construct + role: iam.Role +} + +export interface OnEc2MachinesContext { + instanceIdx: number + instanceUserData: ec2.UserData + region: string + bucketName: string +} + +export interface AfterEc2MachinesContext { + scope: Construct + instances: ec2.Instance[] + bucketName: string + region: string +} + +export interface QueryEngine { + /** Runs before instantiating any EC2 machine */ + beforeEc2Machines(ctx: BeforeEc2MachinesContext): void + + /** Runs for each instantiated EC2 machine */ + onEc2Machine(ctx: OnEc2MachinesContext): void + + /** Runs after all EC2 machines have been instantiated */ + afterEc2Machines(ctx: AfterEc2MachinesContext): void +} + + +interface CdkStackProps extends StackProps { + config: { + instanceType: string; + instanceCount: number; + engines: QueryEngine[] + }; +} + +export class CdkStack extends Stack { + constructor(scope: Construct, id: string, props: CdkStackProps) { + super(scope, id, props); + + const { config } = props; + + // Create VPC with public subnets only (for internet access without NAT gateway) + const vpc = new ec2.Vpc(this, 'BenchmarkVPC', { + maxAzs: 1, + natGateways: 0, + subnetConfiguration: [ + { + name: 'Public', + subnetType: ec2.SubnetType.PUBLIC, + cidrMask: 24, + }, + ], + }); + + // Create security group that allows instances to communicate + const securityGroup = new ec2.SecurityGroup(this, 'BenchmarkSG', { + vpc, + allowAllOutbound: true, + }); + + // Allow all traffic between instances in the same security group + securityGroup.addIngressRule( + securityGroup, + ec2.Port.allTraffic(), + 'Allow all traffic between benchmark instances' + ); + + // Create S3 bucket + const bucket = new s3.Bucket(this, 'BenchmarkBucket', { + bucketName: "datafusion-distributed-benchmarks", + autoDeleteObjects: true, + removalPolicy: RemovalPolicy.DESTROY + }); + + // Create IAM role for EC2 instances + const role = new iam.Role(this, 'BenchmarkInstanceRole', { + assumedBy: new iam.ServicePrincipal('ec2.amazonaws.com'), + managedPolicies: [ + iam.ManagedPolicy.fromAwsManagedPolicyName('AmazonSSMManagedInstanceCore'), + ], + }); + + // Grant permissions to describe EC2 instances (for peer discovery) + role.addToPolicy(new iam.PolicyStatement({ + actions: ['ec2:DescribeInstances'], + resources: ['*'], + })); + + // Grant Glue permissions for Trino Hive metastore + role.addToPolicy(new iam.PolicyStatement({ + actions: [ + 'glue:GetDatabase', + 'glue:GetDatabases', + 'glue:GetTable', + 'glue:GetTables', + 'glue:GetPartition', + 'glue:GetPartitions', + 'glue:CreateTable', + 'glue:UpdateTable', + 'glue:DeleteTable', + 'glue:CreateDatabase', + 'glue:UpdateDatabase', + 'glue:DeleteDatabase', + ], + resources: ['*'], + })); + + // Grant read access to the bucket and worker binary + bucket.grantRead(role); + + for (const engine of config.engines) { + engine.beforeEc2Machines({ + scope: this, + role + }) + } + + // Create EC2 instances + const instances: ec2.Instance[] = []; + for (let i = 0; i < config.instanceCount; i++) { + const userData = ec2.UserData.forLinux(); + + for (const engine of config.engines) { + engine.onEc2Machine({ + bucketName: bucket.bucketName, + instanceIdx: i, + instanceUserData: userData, + region: this.region + }) + } + + const instance = new ec2.Instance(this, `BenchmarkInstance${i}`, { + vpc, + vpcSubnets: { subnetType: ec2.SubnetType.PUBLIC }, + instanceName: `instance-${i}`, + instanceType: new ec2.InstanceType(config.instanceType), + machineImage: ec2.MachineImage.latestAmazonLinux2023(), + securityGroup, + role, + userData, + userDataCausesReplacement: USER_DATA_CAUSES_REPLACEMENT, + blockDevices: [{ + deviceName: '/dev/xvda', + volume: ec2.BlockDeviceVolume.ebs(200, { + volumeType: ec2.EbsDeviceVolumeType.GP3, + deleteOnTermination: true, + }), + }], + }); + + // Tag for peer discovery + Tags.of(instance).add('BenchmarkCluster', 'datafusion'); + instances.push(instance); + } + + // Output Session Manager commands for all instances + new CfnOutput(this, 'ConnectCommands', { + value: ` +# === select one instance to connect to === +${instances.map(_ => `export INSTANCE_ID=${_.instanceId}`).join("\n")} + +# === port forward the HTTP endpoint === +aws ssm start-session --target $INSTANCE_ID --document-name AWS-StartPortForwardingSession --parameters "portNumber=9000,localPortNumber=9000" + +# === open a sh session in the remote machine === +aws ssm start-session --target $INSTANCE_ID + +# === See worker logs inside a sh session === +sudo journalctl -u worker.service -f -o cat + +`, + description: 'Session Manager commands to connect to instances', + }); + + for (const engine of config.engines) { + engine.afterEc2Machines({ + scope: this, + instances, + region: this.region, + bucketName: bucket.bucketName + }) + } + } +} + +export function sendCommandsUnconditionally( + construct: Construct, + name: string, + instances: ec2.Instance[], + commands: string[] +) { + const cmd = new cr.AwsCustomResource(construct, name, { + onUpdate: { + service: 'SSM', + action: 'sendCommand', + parameters: { + DocumentName: 'AWS-RunShellScript', + InstanceIds: instances.map(inst => inst.instanceId), + Parameters: { + commands: [ + 'cloud-init status --wait', + ...commands + ] + }, + }, + physicalResourceId: cr.PhysicalResourceId.of(`${name}-${Date.now()}`), + ignoreErrorCodesMatching: '.*', + }, + policy: cr.AwsCustomResourcePolicy.fromStatements([ + new iam.PolicyStatement({ + actions: ['ssm:SendCommand'], + resources: ['*'], + }), + ]), + }); + + // Ensure instances are created before restarting + cmd.node.addDependency(...instances) +} diff --git a/benchmarks/cdk/lib/datafusion-distributed.ts b/benchmarks/cdk/lib/datafusion-distributed.ts new file mode 100644 index 00000000..71ecf69f --- /dev/null +++ b/benchmarks/cdk/lib/datafusion-distributed.ts @@ -0,0 +1,86 @@ +import { + AfterEc2MachinesContext, + BeforeEc2MachinesContext, + OnEc2MachinesContext, + QueryEngine, + ROOT, + sendCommandsUnconditionally +} from "./cdk-stack"; +import {execSync} from "child_process"; +import * as s3assets from "aws-cdk-lib/aws-s3-assets"; +import * as cdk from "aws-cdk-lib"; +import path from "path"; + +let workerBinary: s3assets.Asset + +export const DATAFUSION_DISTRIBUTED_ENGINE: QueryEngine = { + beforeEc2Machines(ctx: BeforeEc2MachinesContext): void { + console.log('Building worker binary...'); + execSync('cargo zigbuild -p datafusion-distributed-benchmarks --release --bin worker --target x86_64-unknown-linux-gnu', { + cwd: ROOT, + stdio: 'inherit', + }); + console.log('Worker binary built successfully'); + + + // Upload worker binary as an asset + workerBinary = new s3assets.Asset(ctx.scope, 'WorkerBinary', { + path: path.join(ROOT, 'target/x86_64-unknown-linux-gnu/release/worker'), + }); + + workerBinary.grantRead(ctx.role) + }, + onEc2Machine(ctx: OnEc2MachinesContext): void { + ctx.instanceUserData.addCommands( + 'yum install -y gcc openssl-devel', + 'curl --proto \'=https\' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y', + + `aws s3 cp s3://${workerBinary.s3BucketName}/${workerBinary.s3ObjectKey} /usr/local/bin/worker`, + 'chmod +x /usr/local/bin/worker', + // Create systemd service + `cat > /etc/systemd/system/worker.service << 'EOF' +[Unit] +Description=DataFusion Distributed Worker +After=network.target + +[Service] +Type=simple +ExecStart=/usr/local/bin/worker --bucket ${ctx.bucketName} +Restart=always +User=root + +[Install] +WantedBy=multi-user.target +EOF`, + + // Enable and start the service + 'systemctl daemon-reload', + 'systemctl enable worker', + 'systemctl start worker', + ) + }, + afterEc2Machines(ctx: AfterEc2MachinesContext): void { + // Downloads the latest version of the worker binary and restarts the systemd service. + // This is done instead of the userData.addS3Download() so that the instance does not need + // to restart every time a new worker binary is available. + sendCommandsUnconditionally(ctx.scope, 'RestartWorkerService', + ctx.instances, + [ + `aws s3 cp s3://${workerBinary.s3BucketName}/${workerBinary.s3ObjectKey} /usr/local/bin/worker`, + 'chmod +x /usr/local/bin/worker', + 'systemctl restart worker', + ]) + + // Output data needed for fast-deploy + const instanceIds = ctx.instances.map(instance => instance.instanceId); + new cdk.CfnOutput(ctx.scope, 'WorkerInstanceIds', { + value: cdk.Fn.join(',', instanceIds), + }); + new cdk.CfnOutput(ctx.scope, 'WorkerBinaryS3Bucket', { + value: workerBinary.s3BucketName, + }); + new cdk.CfnOutput(ctx.scope, 'WorkerBinaryS3Key', { + value: workerBinary.s3ObjectKey, + }); + } +} \ No newline at end of file diff --git a/benchmarks/cdk/lib/spark.ts b/benchmarks/cdk/lib/spark.ts new file mode 100644 index 00000000..bf8a759f --- /dev/null +++ b/benchmarks/cdk/lib/spark.ts @@ -0,0 +1,342 @@ +import { + AfterEc2MachinesContext, + BeforeEc2MachinesContext, + QueryEngine, + ROOT, + sendCommandsUnconditionally, + OnEc2MachinesContext +} from "./cdk-stack"; +import * as s3assets from "aws-cdk-lib/aws-s3-assets"; +import path from "path"; + +const SPARK_VERSION = "4.0.0" +const HADOOP_VERSION = "3" + +let sparkHttpScript: s3assets.Asset +let sparkRequirements: s3assets.Asset + +export const SPARK_ENGINE: QueryEngine = { + beforeEc2Machines(ctx: BeforeEc2MachinesContext): void { + // Upload Python HTTP server script to S3 + sparkHttpScript = new s3assets.Asset(ctx.scope, 'SparkHttpScript', { + path: path.join(ROOT, 'benchmarks/cdk/bin/spark_http.py'), + }) + + // Upload Python requirements file to S3 + sparkRequirements = new s3assets.Asset(ctx.scope, 'SparkRequirements', { + path: path.join(ROOT, 'benchmarks/cdk/requirements.txt'), + }) + + sparkHttpScript.grantRead(ctx.role) + sparkRequirements.grantRead(ctx.role) + }, + onEc2Machine(ctx: OnEc2MachinesContext): void { + const isMaster = ctx.instanceIdx === 0; + ctx.instanceUserData.addCommands( + // Install Java 17 for Spark + 'yum install -y java-17-amazon-corretto-headless python3 python3-pip', + + // Download and install Spark + 'cd /opt', + `curl -L -o spark.tgz https://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz`, + 'tar -xzf spark.tgz', + `mv spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION} spark`, + 'rm spark.tgz', + + // Download AWS JARs for S3 access (Hadoop 3.4.1 with AWS SDK V2 for Spark 4.0) + 'cd /opt/spark/jars', + 'curl -L -O https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/3.4.1/hadoop-aws-3.4.1.jar', + 'curl -L -O https://repo1.maven.org/maven2/software/amazon/awssdk/bundle/2.29.52/bundle-2.29.52.jar', + 'curl -L -O https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-bundle/1.12.262/aws-java-sdk-bundle-1.12.262.jar', + + // Create Spark directories + 'mkdir -p /var/spark/logs', + 'mkdir -p /var/spark/work', + 'mkdir -p /var/spark/http', + + // Download Python scripts from S3 + `aws s3 cp s3://${sparkHttpScript.s3BucketName}/${sparkHttpScript.s3ObjectKey} /var/spark/http/spark_http.py`, + 'chmod +x /var/spark/http/spark_http.py', + `aws s3 cp s3://${sparkRequirements.s3BucketName}/${sparkRequirements.s3ObjectKey} /var/spark/http/requirements.txt`, + + // Install Python dependencies + 'pip3 install -r /var/spark/http/requirements.txt', + + // Configure Spark defaults + `cat > /opt/spark/conf/spark-defaults.conf << 'SPARK_EOF' +spark.master spark://localhost:7077 +spark.executor.memory 4g +spark.driver.memory 2g +spark.sql.warehouse.dir /var/spark/warehouse +spark.jars /opt/spark/jars/hadoop-aws-3.4.1.jar,/opt/spark/jars/bundle-2.29.52.jar,/opt/spark/jars/aws-java-sdk-bundle-1.12.262.jar +spark.hadoop.fs.s3a.impl org.apache.hadoop.fs.s3a.S3AFileSystem +spark.hadoop.fs.s3a.aws.credentials.provider com.amazonaws.auth.InstanceProfileCredentialsProvider +spark.hadoop.fs.s3a.connection.timeout 60000 +spark.hadoop.fs.s3a.connection.establish.timeout 60000 +spark.hadoop.fs.s3a.attempts.maximum 10 +spark.sql.catalogImplementation hive +spark.hadoop.hive.metastore.client.factory.class com.amazonaws.glue.catalog.metastore.AWSGlueDataCatalogHiveClientFactory +spark.sql.hive.metastore.version 2.3.9 +spark.sql.hive.metastore.jars builtin +SPARK_EOF`, + + // Configure Hadoop core-site.xml for S3A with numeric timeouts + `cat > /opt/spark/conf/core-site.xml << 'CORE_SITE_EOF' + + + + fs.s3a.impl + org.apache.hadoop.fs.s3a.S3AFileSystem + + + fs.s3a.aws.credentials.provider + com.amazonaws.auth.InstanceProfileCredentialsProvider + + + fs.s3a.connection.timeout + 60000 + + + fs.s3a.connection.establish.timeout + 60000 + + + fs.s3a.connection.request.timeout + 60000 + + + fs.s3a.attempts.maximum + 10 + + + fs.s3a.retry.interval + 500 + + + fs.s3a.retry.limit + 10 + + +CORE_SITE_EOF`, + + // Configure Spark environment + `cat > /opt/spark/conf/spark-env.sh << 'SPARK_EOF' +#!/usr/bin/env bash +export SPARK_WORKER_DIR=/var/spark/work +export SPARK_LOG_DIR=/var/spark/logs +export SPARK_MASTER_HOST=localhost +export SPARK_MASTER_PORT=7077 +export SPARK_MASTER_WEBUI_PORT=8082 +export SPARK_WORKER_WEBUI_PORT=8083 +export JAVA_HOME=/usr/lib/jvm/java-17-amazon-corretto.x86_64 +SPARK_EOF`, + 'chmod +x /opt/spark/conf/spark-env.sh', + + // Create Spark master systemd service (master only) + ...(isMaster ? [ + `cat > /etc/systemd/system/spark-master.service << 'SPARK_EOF' +[Unit] +Description=Spark Master +After=network.target + +[Service] +Type=forking +ExecStart=/opt/spark/sbin/start-master.sh +ExecStop=/opt/spark/sbin/stop-master.sh +Restart=on-failure +RestartSec=5 +User=root +WorkingDirectory=/opt/spark +Environment="JAVA_HOME=/usr/lib/jvm/java-17-amazon-corretto.x86_64" +Environment="SPARK_LOG_DIR=/var/spark/logs" + +[Install] +WantedBy=multi-user.target +SPARK_EOF` + ] : []), + + // Create Spark worker systemd service (will be reconfigured for workers) + `cat > /etc/systemd/system/spark-worker.service << 'SPARK_EOF' +[Unit] +Description=Spark Worker +After=network.target${isMaster ? ' spark-master.service' : ''} +${isMaster ? 'Requires=spark-master.service' : ''} + +[Service] +Type=forking +ExecStart=/opt/spark/sbin/start-worker.sh spark://localhost:7077 +ExecStop=/opt/spark/sbin/stop-worker.sh +Restart=on-failure +RestartSec=5 +User=root +WorkingDirectory=/opt/spark +Environment="JAVA_HOME=/usr/lib/jvm/java-17-amazon-corretto.x86_64" +Environment="SPARK_LOG_DIR=/var/spark/logs" +Environment="SPARK_WORKER_DIR=/var/spark/work" + +[Install] +WantedBy=multi-user.target +SPARK_EOF`, + + // Create HTTP server systemd service (master only) + ...(isMaster ? [ + `cat > /etc/systemd/system/spark-http.service << 'SPARK_EOF' +[Unit] +Description=Spark HTTP Server +After=network.target spark-master.service +Requires=spark-master.service + +[Service] +Type=simple +ExecStart=/usr/bin/python3 /var/spark/http/spark_http.py +Restart=on-failure +RestartSec=5 +User=root +WorkingDirectory=/var/spark/http +Environment="SPARK_MASTER_HOST=localhost" +Environment="HTTP_PORT=9003" +Environment="JAVA_HOME=/usr/lib/jvm/java-17-amazon-corretto.x86_64" +Environment="SPARK_JARS=/opt/spark/jars/hadoop-aws-3.4.1.jar,/opt/spark/jars/bundle-2.29.52.jar,/opt/spark/jars/aws-java-sdk-bundle-1.12.262.jar" +StandardOutput=append:/var/spark/logs/http.log +StandardError=append:/var/spark/logs/http.log + +[Install] +WantedBy=multi-user.target +SPARK_EOF` + ] : []), + + // Reload systemd and enable services + 'systemctl daemon-reload', + + // Enable and start master (master only) + ...(isMaster ? [ + 'systemctl enable spark-master', + 'systemctl start spark-master', + 'sleep 5' + ] : []), + + // Enable and start worker (all nodes) + 'systemctl enable spark-worker', + 'systemctl start spark-worker', + + // Enable and start HTTP server (master only) + ...(isMaster ? [ + 'systemctl enable spark-http', + 'systemctl start spark-http' + ] : []) + ) + }, + afterEc2Machines(ctx: AfterEc2MachinesContext): void { + const [master, ...workers] = ctx.instances + + // Reconfigure all workers to point to master IP + sendCommandsUnconditionally( + ctx.scope, + 'ConfigureSparkWorkers', + workers, + [ + `cat > /opt/spark/conf/spark-env.sh << 'SPARK_EOF' +#!/usr/bin/env bash +export SPARK_WORKER_DIR=/var/spark/work +export SPARK_LOG_DIR=/var/spark/logs +export SPARK_MASTER_HOST=${master.instancePrivateIp} +export SPARK_MASTER_PORT=7077 +export SPARK_MASTER_WEBUI_PORT=8082 +export SPARK_WORKER_WEBUI_PORT=8081 +export JAVA_HOME=/usr/lib/jvm/java-17-amazon-corretto.x86_64 +SPARK_EOF`, + 'chmod +x /opt/spark/conf/spark-env.sh', + `cat > /etc/systemd/system/spark-worker.service << 'SPARK_EOF' +[Unit] +Description=Spark Worker +After=network.target + +[Service] +Type=forking +ExecStart=/opt/spark/sbin/start-worker.sh spark://${master.instancePrivateIp}:7077 +ExecStop=/opt/spark/sbin/stop-worker.sh +Restart=on-failure +RestartSec=5 +User=root +WorkingDirectory=/opt/spark +Environment="JAVA_HOME=/usr/lib/jvm/java-17-amazon-corretto.x86_64" +Environment="SPARK_LOG_DIR=/var/spark/logs" +Environment="SPARK_WORKER_DIR=/var/spark/work" + +[Install] +WantedBy=multi-user.target +SPARK_EOF`, + 'systemctl daemon-reload', + 'systemctl restart spark-worker' + ] + ) + + // Also update master's spark-env.sh to use its own IP for proper cluster formation + sendCommandsUnconditionally( + ctx.scope, + 'ConfigureSparkMaster', + [master], + [ + `cat > /opt/spark/conf/spark-env.sh << 'SPARK_EOF' +#!/usr/bin/env bash +export SPARK_WORKER_DIR=/var/spark/work +export SPARK_LOG_DIR=/var/spark/logs +export SPARK_MASTER_HOST=${master.instancePrivateIp} +export SPARK_MASTER_PORT=7077 +export SPARK_MASTER_WEBUI_PORT=8082 +export SPARK_WORKER_WEBUI_PORT=8081 +export JAVA_HOME=/usr/lib/jvm/java-17-amazon-corretto.x86_64 +SPARK_EOF`, + 'chmod +x /opt/spark/conf/spark-env.sh', + `cat > /etc/systemd/system/spark-worker.service << 'SPARK_EOF' +[Unit] +Description=Spark Worker +After=network.target spark-master.service +Requires=spark-master.service + +[Service] +Type=forking +ExecStart=/opt/spark/sbin/start-worker.sh spark://${master.instancePrivateIp}:7077 +ExecStop=/opt/spark/sbin/stop-worker.sh +Restart=on-failure +RestartSec=5 +User=root +WorkingDirectory=/opt/spark +Environment="JAVA_HOME=/usr/lib/jvm/java-17-amazon-corretto.x86_64" +Environment="SPARK_LOG_DIR=/var/spark/logs" +Environment="SPARK_WORKER_DIR=/var/spark/work" + +[Install] +WantedBy=multi-user.target +SPARK_EOF`, + `cat > /etc/systemd/system/spark-http.service << 'SPARK_EOF' +[Unit] +Description=Spark HTTP Server +After=network.target spark-master.service +Requires=spark-master.service + +[Service] +Type=simple +ExecStart=/usr/bin/python3 /var/spark/http/spark_http.py +Restart=on-failure +RestartSec=5 +User=root +WorkingDirectory=/var/spark/http +Environment="SPARK_MASTER_HOST=${master.instancePrivateIp}" +Environment="HTTP_PORT=9003" +Environment="JAVA_HOME=/usr/lib/jvm/java-17-amazon-corretto.x86_64" +Environment="SPARK_JARS=/opt/spark/jars/hadoop-aws-3.4.1.jar,/opt/spark/jars/bundle-2.29.52.jar,/opt/spark/jars/aws-java-sdk-bundle-1.12.262.jar" +StandardOutput=append:/var/spark/logs/http.log +StandardError=append:/var/spark/logs/http.log + +[Install] +WantedBy=multi-user.target +SPARK_EOF`, + 'systemctl daemon-reload', + 'systemctl restart spark-master', + 'systemctl restart spark-worker', + 'systemctl restart spark-http' + ] + ) + } +} diff --git a/benchmarks/cdk/lib/trino.ts b/benchmarks/cdk/lib/trino.ts new file mode 100644 index 00000000..2a3a8b3e --- /dev/null +++ b/benchmarks/cdk/lib/trino.ts @@ -0,0 +1,155 @@ +import { + AfterEc2MachinesContext, + QueryEngine, + OnEc2MachinesContext, + sendCommandsUnconditionally, +} from "./cdk-stack"; + +const TRINO_VERSION = 476 +const JVM_MEMORY_GB = 10 + +const JVM_CONFIG = `\ +-server +-Xmx${JVM_MEMORY_GB}G +-XX:+UseG1GC +-XX:G1HeapRegionSize=32M +-XX:+ExplicitGCInvokesConcurrent +-XX:+HeapDumpOnOutOfMemoryError +-XX:+ExitOnOutOfMemoryError +-Djdk.attach.allowAttachSelf=true +` +const COORDINATOR_CONFIG = `\ +coordinator=true +node-scheduler.include-coordinator=true +http-server.http.port=8080 +discovery.uri=http://localhost:8080 +query.max-memory=80GB +query.max-total-memory=80GB +query.max-memory-per-node=${JVM_MEMORY_GB-2}GB +memory.heap-headroom-per-node=1.5GB +` + +const INSTANCE_ID_PLACEHOLDER = '{instance-ip}' +const WORKER_CONFIG = `\ +coordinator=false +http-server.http.port=8080 +discovery.uri=http://${INSTANCE_ID_PLACEHOLDER}:8080 +query.max-memory=80GB +query.max-total-memory=80GB +query.max-memory-per-node=${JVM_MEMORY_GB-2}GB +memory.heap-headroom-per-node=1.5GB +` + +export const TRINO_ENGINE: QueryEngine = { + beforeEc2Machines(): void { + // nothing to compile here + }, + onEc2Machine(ctx: OnEc2MachinesContext): void { + const isCoordinator = ctx.instanceIdx === 0; + ctx.instanceUserData.addCommands( + // Install Java 24 for Trino (Trino 478 requires Java 24+) + 'yum install -y java-24-amazon-corretto-headless python', + + // Download and install Trino 478 (latest version) + 'cd /opt', + `curl -L -o trino-server.tar.gz https://repo1.maven.org/maven2/io/trino/trino-server/${TRINO_VERSION}/trino-server-${TRINO_VERSION}.tar.gz`, + 'tar -xzf trino-server.tar.gz', + `mv trino-server-${TRINO_VERSION} trino-server`, + 'rm trino-server.tar.gz', + + // Create Trino directories + 'mkdir -p /var/trino/data', + 'mkdir -p /opt/trino-server/etc/catalog', + + // Configure Trino node properties + `cat > /opt/trino-server/etc/node.properties << 'TRINO_EOF' +node.environment=benchmark +node.id=instance-${ctx.instanceIdx} +node.data-dir=/var/trino/data +TRINO_EOF`, + + // Configure Trino JVM settings (10GB heap to support 8GB query memory) + `cat > /opt/trino-server/etc/jvm.config << 'TRINO_EOF' +${JVM_CONFIG} +TRINO_EOF`, + + // Configure Trino config.properties (workers will be reconfigured during lazy startup) + isCoordinator + ? `cat > /opt/trino-server/etc/config.properties << 'TRINO_EOF' +${COORDINATOR_CONFIG} +TRINO_EOF` + : `cat > /opt/trino-server/etc/config.properties << 'TRINO_EOF' +${WORKER_CONFIG.replace(INSTANCE_ID_PLACEHOLDER, "localhost")} +TRINO_EOF`, + + // Configure Hive catalog with AWS Glue metastore + `cat > /opt/trino-server/etc/catalog/hive.properties << 'TRINO_EOF' +connector.name=hive +hive.metastore=glue +hive.metastore.glue.region=${ctx.region} +fs.native-s3.enabled=true +s3.region=${ctx.region} +TRINO_EOF`, + + // Configure TPCH catalog for reference + `cat > /opt/trino-server/etc/catalog/tpch.properties << 'TRINO_EOF' +connector.name=tpch +TRINO_EOF`, + + // Download Trino CLI + 'curl -L -o /usr/local/bin/trino https://repo1.maven.org/maven2/io/trino/trino-cli/478/trino-cli-478-executable.jar', + 'chmod +x /usr/local/bin/trino', + + // Create Trino systemd service + `cat > /etc/systemd/system/trino.service << 'TRINO_EOF' +[Unit] +Description=Trino Server +After=network.target + +[Service] +Type=forking +ExecStart=/opt/trino-server/bin/launcher start +ExecStop=/opt/trino-server/bin/launcher stop +Restart=on-failure +User=root +WorkingDirectory=/opt/trino-server + +[Install] +WantedBy=multi-user.target +TRINO_EOF`, + + // Enable Trino (but don't start yet - will be started lazily after all instances are up) + 'systemctl daemon-reload', + 'systemctl enable trino', + 'systemctl start trino' + ) + }, + afterEc2Machines(ctx: AfterEc2MachinesContext): void { + const [coordinator, ...workers] = ctx.instances + // Then start workers (they will discover the coordinator) + sendCommandsUnconditionally(ctx.scope, 'TrinoCoordinatorCommands', + [coordinator], + [ + `cat > /opt/trino-server/etc/jvm.config << 'TRINO_EOF' +${JVM_CONFIG} +TRINO_EOF`, + `cat > /opt/trino-server/etc/config.properties << 'TRINO_EOF' +${COORDINATOR_CONFIG} +TRINO_EOF`, + 'systemctl start trino', + ] + ) + sendCommandsUnconditionally(ctx.scope, 'TrinoWorkerCommands', + workers, + [ + `cat > /opt/trino-server/etc/jvm.config << 'TRINO_EOF' +${JVM_CONFIG} +TRINO_EOF`, + `cat > /opt/trino-server/etc/config.properties << TRINO_EOF +${WORKER_CONFIG.replace(INSTANCE_ID_PLACEHOLDER, coordinator.instancePrivateIp)} +TRINO_EOF`, + 'systemctl restart trino', + ] + ) + } +} \ No newline at end of file diff --git a/benchmarks/cdk/package-lock.json b/benchmarks/cdk/package-lock.json new file mode 100644 index 00000000..7c55a38b --- /dev/null +++ b/benchmarks/cdk/package-lock.json @@ -0,0 +1,4455 @@ +{ + "name": "cdk", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "cdk", + "version": "0.1.0", + "dependencies": { + "aws-cdk-lib": "2.215.0", + "commander": "^14.0.2", + "constructs": "^10.0.0", + "zod": "^4.1.12" + }, + "bin": { + "cdk": "bin/cdk.js" + }, + "devDependencies": { + "@types/jest": "^29.5.14", + "@types/node": "22.7.9", + "aws-cdk": "2.1031.2", + "jest": "^29.7.0", + "ts-jest": "^29.2.5", + "ts-node": "^10.9.2", + "typescript": "~5.6.3" + } + }, + "node_modules/@aws-cdk/asset-awscli-v1": { + "version": "2.2.242", + "resolved": "https://registry.npmjs.org/@aws-cdk/asset-awscli-v1/-/asset-awscli-v1-2.2.242.tgz", + "integrity": "sha512-4c1bAy2ISzcdKXYS1k4HYZsNrgiwbiDzj36ybwFVxEWZXVAP0dimQTCaB9fxu7sWzEjw3d+eaw6Fon+QTfTIpQ==", + "license": "Apache-2.0" + }, + "node_modules/@aws-cdk/asset-node-proxy-agent-v6": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@aws-cdk/asset-node-proxy-agent-v6/-/asset-node-proxy-agent-v6-2.1.0.tgz", + "integrity": "sha512-7bY3J8GCVxLupn/kNmpPc5VJz8grx+4RKfnnJiO1LG+uxkZfANZG3RMHhE+qQxxwkyQ9/MfPtTpf748UhR425A==", + "license": "Apache-2.0" + }, + "node_modules/@aws-cdk/cloud-assembly-schema": { + "version": "48.18.0", + "resolved": "https://registry.npmjs.org/@aws-cdk/cloud-assembly-schema/-/cloud-assembly-schema-48.18.0.tgz", + "integrity": "sha512-KJHmkji59QboUIytK/imIY/ajt/r7OOXdyvZz25e0jssreyyxx9HUswZpcB0BM/9ZkVI7IdzYlotWe6opXdapg==", + "bundleDependencies": [ + "jsonschema", + "semver" + ], + "license": "Apache-2.0", + "dependencies": { + "jsonschema": "~1.4.1", + "semver": "^7.7.2" + }, + "engines": { + "node": ">= 18.0.0" + } + }, + "node_modules/@aws-cdk/cloud-assembly-schema/node_modules/jsonschema": { + "version": "1.4.1", + "inBundle": true, + "license": "MIT", + "engines": { + "node": "*" + } + }, + "node_modules/@aws-cdk/cloud-assembly-schema/node_modules/semver": { + "version": "7.7.2", + "inBundle": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/@babel/code-frame": { + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.27.1.tgz", + "integrity": "sha512-cjQ7ZlQ0Mv3b47hABuTevyTuYN4i+loJKGeV9flcCgIK37cCXRh+L1bd3iBHlynerhQ7BhCkn2BPbQUL+rGqFg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-validator-identifier": "^7.27.1", + "js-tokens": "^4.0.0", + "picocolors": "^1.1.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/compat-data": { + "version": "7.28.5", + "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.28.5.tgz", + "integrity": "sha512-6uFXyCayocRbqhZOB+6XcuZbkMNimwfVGFji8CTZnCzOHVGvDqzvitu1re2AU5LROliz7eQPhB8CpAMvnx9EjA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/core": { + "version": "7.28.5", + "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.28.5.tgz", + "integrity": "sha512-e7jT4DxYvIDLk1ZHmU/m/mB19rex9sv0c2ftBtjSBv+kVM/902eh0fINUzD7UwLLNR+jU585GxUJ8/EBfAM5fw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.27.1", + "@babel/generator": "^7.28.5", + "@babel/helper-compilation-targets": "^7.27.2", + "@babel/helper-module-transforms": "^7.28.3", + "@babel/helpers": "^7.28.4", + "@babel/parser": "^7.28.5", + "@babel/template": "^7.27.2", + "@babel/traverse": "^7.28.5", + "@babel/types": "^7.28.5", + "@jridgewell/remapping": "^2.3.5", + "convert-source-map": "^2.0.0", + "debug": "^4.1.0", + "gensync": "^1.0.0-beta.2", + "json5": "^2.2.3", + "semver": "^6.3.1" + }, + "engines": { + "node": ">=6.9.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/babel" + } + }, + "node_modules/@babel/generator": { + "version": "7.28.5", + "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.28.5.tgz", + "integrity": "sha512-3EwLFhZ38J4VyIP6WNtt2kUdW9dokXA9Cr4IVIFHuCpZ3H8/YFOl5JjZHisrn1fATPBmKKqXzDFvh9fUwHz6CQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.28.5", + "@babel/types": "^7.28.5", + "@jridgewell/gen-mapping": "^0.3.12", + "@jridgewell/trace-mapping": "^0.3.28", + "jsesc": "^3.0.2" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-compilation-targets": { + "version": "7.27.2", + "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.27.2.tgz", + "integrity": "sha512-2+1thGUUWWjLTYTHZWK1n8Yga0ijBz1XAhUXcKy81rd5g6yh7hGqMp45v7cadSbEHc9G3OTv45SyneRN3ps4DQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/compat-data": "^7.27.2", + "@babel/helper-validator-option": "^7.27.1", + "browserslist": "^4.24.0", + "lru-cache": "^5.1.1", + "semver": "^6.3.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-globals": { + "version": "7.28.0", + "resolved": "https://registry.npmjs.org/@babel/helper-globals/-/helper-globals-7.28.0.tgz", + "integrity": "sha512-+W6cISkXFa1jXsDEdYA8HeevQT/FULhxzR99pxphltZcVaugps53THCeiWA8SguxxpSp3gKPiuYfSWopkLQ4hw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-module-imports": { + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.27.1.tgz", + "integrity": "sha512-0gSFWUPNXNopqtIPQvlD5WgXYI5GY2kP2cCvoT8kczjbfcfuIljTbcWrulD1CIPIX2gt1wghbDy08yE1p+/r3w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/traverse": "^7.27.1", + "@babel/types": "^7.27.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-module-transforms": { + "version": "7.28.3", + "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.28.3.tgz", + "integrity": "sha512-gytXUbs8k2sXS9PnQptz5o0QnpLL51SwASIORY6XaBKF88nsOT0Zw9szLqlSGQDP/4TljBAD5y98p2U1fqkdsw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-module-imports": "^7.27.1", + "@babel/helper-validator-identifier": "^7.27.1", + "@babel/traverse": "^7.28.3" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/@babel/helper-plugin-utils": { + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.27.1.tgz", + "integrity": "sha512-1gn1Up5YXka3YYAHGKpbideQ5Yjf1tDa9qYcgysz+cNCXukyLl6DjPXhD3VRwSb8c0J9tA4b2+rHEZtc6R0tlw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-string-parser": { + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.27.1.tgz", + "integrity": "sha512-qMlSxKbpRlAridDExk92nSobyDdpPijUq2DW6oDnUqd0iOGxmQjyqhMIihI9+zv4LPyZdRje2cavWPbCbWm3eA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-identifier": { + "version": "7.28.5", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.28.5.tgz", + "integrity": "sha512-qSs4ifwzKJSV39ucNjsvc6WVHs6b7S03sOh2OcHF9UHfVPqWWALUsNUVzhSBiItjRZoLHx7nIarVjqKVusUZ1Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-option": { + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-option/-/helper-validator-option-7.27.1.tgz", + "integrity": "sha512-YvjJow9FxbhFFKDSuFnVCe2WxXk1zWc22fFePVNEaWJEu8IrZVlda6N0uHwzZrUM1il7NC9Mlp4MaJYbYd9JSg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helpers": { + "version": "7.28.4", + "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.28.4.tgz", + "integrity": "sha512-HFN59MmQXGHVyYadKLVumYsA9dBFun/ldYxipEjzA4196jpLZd8UjEEBLkbEkvfYreDqJhZxYAWFPtrfhNpj4w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/template": "^7.27.2", + "@babel/types": "^7.28.4" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/parser": { + "version": "7.28.5", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.28.5.tgz", + "integrity": "sha512-KKBU1VGYR7ORr3At5HAtUQ+TV3SzRCXmA/8OdDZiLDBIZxVyzXuztPjfLd3BV1PRAQGCMWWSHYhL0F8d5uHBDQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/types": "^7.28.5" + }, + "bin": { + "parser": "bin/babel-parser.js" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@babel/plugin-syntax-async-generators": { + "version": "7.8.4", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-async-generators/-/plugin-syntax-async-generators-7.8.4.tgz", + "integrity": "sha512-tycmZxkGfZaxhMRbXlPXuVFpdWlXpir2W4AMhSJgRKzk/eDlIXOhb2LHWoLpDF7TEHylV5zNhykX6KAgHJmTNw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-bigint": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-bigint/-/plugin-syntax-bigint-7.8.3.tgz", + "integrity": "sha512-wnTnFlG+YxQm3vDxpGE57Pj0srRU4sHE/mDkt1qv2YJJSeUAec2ma4WLUnUPeKjyrfntVwe/N6dCXpU+zL3Npg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-class-properties": { + "version": "7.12.13", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-class-properties/-/plugin-syntax-class-properties-7.12.13.tgz", + "integrity": "sha512-fm4idjKla0YahUNgFNLCB0qySdsoPiZP3iQE3rky0mBUtMZ23yDJ9SJdg6dXTSDnulOVqiF3Hgr9nbXvXTQZYA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.12.13" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-class-static-block": { + "version": "7.14.5", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-class-static-block/-/plugin-syntax-class-static-block-7.14.5.tgz", + "integrity": "sha512-b+YyPmr6ldyNnM6sqYeMWE+bgJcJpO6yS4QD7ymxgH34GBPNDM/THBh8iunyvKIZztiwLH4CJZ0RxTk9emgpjw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.14.5" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-import-attributes": { + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-import-attributes/-/plugin-syntax-import-attributes-7.27.1.tgz", + "integrity": "sha512-oFT0FrKHgF53f4vOsZGi2Hh3I35PfSmVs4IBFLFj4dnafP+hIWDLg3VyKmUHfLoLHlyxY4C7DGtmHuJgn+IGww==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.27.1" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-import-meta": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-import-meta/-/plugin-syntax-import-meta-7.10.4.tgz", + "integrity": "sha512-Yqfm+XDx0+Prh3VSeEQCPU81yC+JWZ2pDPFSS4ZdpfZhp4MkFMaDC1UqseovEKwSUpnIL7+vK+Clp7bfh0iD7g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.10.4" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-json-strings": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-json-strings/-/plugin-syntax-json-strings-7.8.3.tgz", + "integrity": "sha512-lY6kdGpWHvjoe2vk4WrAapEuBR69EMxZl+RoGRhrFGNYVK8mOPAW8VfbT/ZgrFbXlDNiiaxQnAtgVCZ6jv30EA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-jsx": { + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-jsx/-/plugin-syntax-jsx-7.27.1.tgz", + "integrity": "sha512-y8YTNIeKoyhGd9O0Jiyzyyqk8gdjnumGTQPsz0xOZOQ2RmkVJeZ1vmmfIvFEKqucBG6axJGBZDE/7iI5suUI/w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.27.1" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-logical-assignment-operators": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-logical-assignment-operators/-/plugin-syntax-logical-assignment-operators-7.10.4.tgz", + "integrity": "sha512-d8waShlpFDinQ5MtvGU9xDAOzKH47+FFoney2baFIoMr952hKOLp1HR7VszoZvOsV/4+RRszNY7D17ba0te0ig==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.10.4" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-nullish-coalescing-operator": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-nullish-coalescing-operator/-/plugin-syntax-nullish-coalescing-operator-7.8.3.tgz", + "integrity": "sha512-aSff4zPII1u2QD7y+F8oDsz19ew4IGEJg9SVW+bqwpwtfFleiQDMdzA/R+UlWDzfnHFCxxleFT0PMIrR36XLNQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-numeric-separator": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-numeric-separator/-/plugin-syntax-numeric-separator-7.10.4.tgz", + "integrity": "sha512-9H6YdfkcK/uOnY/K7/aA2xpzaAgkQn37yzWUMRK7OaPOqOpGS1+n0H5hxT9AUw9EsSjPW8SVyMJwYRtWs3X3ug==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.10.4" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-object-rest-spread": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-object-rest-spread/-/plugin-syntax-object-rest-spread-7.8.3.tgz", + "integrity": "sha512-XoqMijGZb9y3y2XskN+P1wUGiVwWZ5JmoDRwx5+3GmEplNyVM2s2Dg8ILFQm8rWM48orGy5YpI5Bl8U1y7ydlA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-optional-catch-binding": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-optional-catch-binding/-/plugin-syntax-optional-catch-binding-7.8.3.tgz", + "integrity": "sha512-6VPD0Pc1lpTqw0aKoeRTMiB+kWhAoT24PA+ksWSBrFtl5SIRVpZlwN3NNPQjehA2E/91FV3RjLWoVTglWcSV3Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-optional-chaining": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-optional-chaining/-/plugin-syntax-optional-chaining-7.8.3.tgz", + "integrity": "sha512-KoK9ErH1MBlCPxV0VANkXW2/dw4vlbGDrFgz8bmUsBGYkFRcbRwMh6cIJubdPrkxRwuGdtCk0v/wPTKbQgBjkg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.8.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-private-property-in-object": { + "version": "7.14.5", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-private-property-in-object/-/plugin-syntax-private-property-in-object-7.14.5.tgz", + "integrity": "sha512-0wVnp9dxJ72ZUJDV27ZfbSj6iHLoytYZmh3rFcxNnvsJF3ktkzLDZPy/mA17HGsaQT3/DQsWYX1f1QGWkCoVUg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.14.5" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-top-level-await": { + "version": "7.14.5", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-top-level-await/-/plugin-syntax-top-level-await-7.14.5.tgz", + "integrity": "sha512-hx++upLv5U1rgYfwe1xBQUhRmU41NEvpUvrp8jkrSCdvGSnM5/qdRMtylJ6PG5OFkBaHkbTAKTnd3/YyESRHFw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.14.5" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-syntax-typescript": { + "version": "7.27.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-typescript/-/plugin-syntax-typescript-7.27.1.tgz", + "integrity": "sha512-xfYCBMxveHrRMnAWl1ZlPXOZjzkN82THFvLhQhFXFt81Z5HnN+EtUkZhv/zcKpmT3fzmWZB0ywiBrbC3vogbwQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.27.1" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/template": { + "version": "7.27.2", + "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.27.2.tgz", + "integrity": "sha512-LPDZ85aEJyYSd18/DkjNh4/y1ntkE5KwUHWTiqgRxruuZL2F1yuHligVHLvcHY2vMHXttKFpJn6LwfI7cw7ODw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.27.1", + "@babel/parser": "^7.27.2", + "@babel/types": "^7.27.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/traverse": { + "version": "7.28.5", + "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.28.5.tgz", + "integrity": "sha512-TCCj4t55U90khlYkVV/0TfkJkAkUg3jZFA3Neb7unZT8CPok7iiRfaX0F+WnqWqt7OxhOn0uBKXCw4lbL8W0aQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.27.1", + "@babel/generator": "^7.28.5", + "@babel/helper-globals": "^7.28.0", + "@babel/parser": "^7.28.5", + "@babel/template": "^7.27.2", + "@babel/types": "^7.28.5", + "debug": "^4.3.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/types": { + "version": "7.28.5", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.28.5.tgz", + "integrity": "sha512-qQ5m48eI/MFLQ5PxQj4PFaprjyCTLI37ElWMmNs0K8Lk3dVeOdNpB3ks8jc7yM5CDmVC73eMVk/trk3fgmrUpA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-string-parser": "^7.27.1", + "@babel/helper-validator-identifier": "^7.28.5" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@bcoe/v8-coverage": { + "version": "0.2.3", + "resolved": "https://registry.npmjs.org/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz", + "integrity": "sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@cspotcode/source-map-support": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", + "integrity": "sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/trace-mapping": "0.3.9" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/@cspotcode/source-map-support/node_modules/@jridgewell/trace-mapping": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.9.tgz", + "integrity": "sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/resolve-uri": "^3.0.3", + "@jridgewell/sourcemap-codec": "^1.4.10" + } + }, + "node_modules/@istanbuljs/load-nyc-config": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@istanbuljs/load-nyc-config/-/load-nyc-config-1.1.0.tgz", + "integrity": "sha512-VjeHSlIzpv/NyD3N0YuHfXOPDIixcA1q2ZV98wsMqcYlPmv2n3Yb2lYP9XMElnaFVXg5A7YLTeLu6V84uQDjmQ==", + "dev": true, + "license": "ISC", + "dependencies": { + "camelcase": "^5.3.1", + "find-up": "^4.1.0", + "get-package-type": "^0.1.0", + "js-yaml": "^3.13.1", + "resolve-from": "^5.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/@istanbuljs/schema": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/@istanbuljs/schema/-/schema-0.1.3.tgz", + "integrity": "sha512-ZXRY4jNvVgSVQ8DL3LTcakaAtXwTVUxE81hslsyD2AtoXW/wVob10HkOJ1X/pAlcI7D+2YoZKg5do8G/w6RYgA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/@jest/console": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/console/-/console-29.7.0.tgz", + "integrity": "sha512-5Ni4CU7XHQi32IJ398EEP4RrB8eV09sXP2ROqD4bksHrnTree52PsxvX8tpL8LvTZ3pFzXyPbNQReSN41CAhOg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/types": "^29.6.3", + "@types/node": "*", + "chalk": "^4.0.0", + "jest-message-util": "^29.7.0", + "jest-util": "^29.7.0", + "slash": "^3.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jest/core": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/core/-/core-29.7.0.tgz", + "integrity": "sha512-n7aeXWKMnGtDA48y8TLWJPJmLmmZ642Ceo78cYWEpiD7FzDgmNDV/GCVRorPABdXLJZ/9wzzgZAlHjXjxDHGsg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/console": "^29.7.0", + "@jest/reporters": "^29.7.0", + "@jest/test-result": "^29.7.0", + "@jest/transform": "^29.7.0", + "@jest/types": "^29.6.3", + "@types/node": "*", + "ansi-escapes": "^4.2.1", + "chalk": "^4.0.0", + "ci-info": "^3.2.0", + "exit": "^0.1.2", + "graceful-fs": "^4.2.9", + "jest-changed-files": "^29.7.0", + "jest-config": "^29.7.0", + "jest-haste-map": "^29.7.0", + "jest-message-util": "^29.7.0", + "jest-regex-util": "^29.6.3", + "jest-resolve": "^29.7.0", + "jest-resolve-dependencies": "^29.7.0", + "jest-runner": "^29.7.0", + "jest-runtime": "^29.7.0", + "jest-snapshot": "^29.7.0", + "jest-util": "^29.7.0", + "jest-validate": "^29.7.0", + "jest-watcher": "^29.7.0", + "micromatch": "^4.0.4", + "pretty-format": "^29.7.0", + "slash": "^3.0.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + }, + "peerDependencies": { + "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0" + }, + "peerDependenciesMeta": { + "node-notifier": { + "optional": true + } + } + }, + "node_modules/@jest/environment": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/environment/-/environment-29.7.0.tgz", + "integrity": "sha512-aQIfHDq33ExsN4jP1NWGXhxgQ/wixs60gDiKO+XVMd8Mn0NWPWgc34ZQDTb2jKaUWQ7MuwoitXAsN2XVXNMpAw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/fake-timers": "^29.7.0", + "@jest/types": "^29.6.3", + "@types/node": "*", + "jest-mock": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jest/expect": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/expect/-/expect-29.7.0.tgz", + "integrity": "sha512-8uMeAMycttpva3P1lBHB8VciS9V0XAr3GymPpipdyQXbBcuhkLQOSe8E/p92RyAdToS6ZD1tFkX+CkhoECE0dQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "expect": "^29.7.0", + "jest-snapshot": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jest/expect-utils": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/expect-utils/-/expect-utils-29.7.0.tgz", + "integrity": "sha512-GlsNBWiFQFCVi9QVSx7f5AgMeLxe9YCCs5PuP2O2LdjDAA8Jh9eX7lA1Jq/xdXw3Wb3hyvlFNfZIfcRetSzYcA==", + "dev": true, + "license": "MIT", + "dependencies": { + "jest-get-type": "^29.6.3" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jest/fake-timers": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/fake-timers/-/fake-timers-29.7.0.tgz", + "integrity": "sha512-q4DH1Ha4TTFPdxLsqDXK1d3+ioSL7yL5oCMJZgDYm6i+6CygW5E5xVr/D1HdsGxjt1ZWSfUAs9OxSB/BNelWrQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/types": "^29.6.3", + "@sinonjs/fake-timers": "^10.0.2", + "@types/node": "*", + "jest-message-util": "^29.7.0", + "jest-mock": "^29.7.0", + "jest-util": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jest/globals": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/globals/-/globals-29.7.0.tgz", + "integrity": "sha512-mpiz3dutLbkW2MNFubUGUEVLkTGiqW6yLVTA+JbP6fI6J5iL9Y0Nlg8k95pcF8ctKwCS7WVxteBs29hhfAotzQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/environment": "^29.7.0", + "@jest/expect": "^29.7.0", + "@jest/types": "^29.6.3", + "jest-mock": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jest/reporters": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/reporters/-/reporters-29.7.0.tgz", + "integrity": "sha512-DApq0KJbJOEzAFYjHADNNxAE3KbhxQB1y5Kplb5Waqw6zVbuWatSnMjE5gs8FUgEPmNsnZA3NCWl9NG0ia04Pg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@bcoe/v8-coverage": "^0.2.3", + "@jest/console": "^29.7.0", + "@jest/test-result": "^29.7.0", + "@jest/transform": "^29.7.0", + "@jest/types": "^29.6.3", + "@jridgewell/trace-mapping": "^0.3.18", + "@types/node": "*", + "chalk": "^4.0.0", + "collect-v8-coverage": "^1.0.0", + "exit": "^0.1.2", + "glob": "^7.1.3", + "graceful-fs": "^4.2.9", + "istanbul-lib-coverage": "^3.0.0", + "istanbul-lib-instrument": "^6.0.0", + "istanbul-lib-report": "^3.0.0", + "istanbul-lib-source-maps": "^4.0.0", + "istanbul-reports": "^3.1.3", + "jest-message-util": "^29.7.0", + "jest-util": "^29.7.0", + "jest-worker": "^29.7.0", + "slash": "^3.0.0", + "string-length": "^4.0.1", + "strip-ansi": "^6.0.0", + "v8-to-istanbul": "^9.0.1" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + }, + "peerDependencies": { + "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0" + }, + "peerDependenciesMeta": { + "node-notifier": { + "optional": true + } + } + }, + "node_modules/@jest/schemas": { + "version": "29.6.3", + "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-29.6.3.tgz", + "integrity": "sha512-mo5j5X+jIZmJQveBKeS/clAueipV7KgiX1vMgCxam1RNYiqE1w62n0/tJJnHtjW8ZHcQco5gY85jA3mi0L+nSA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@sinclair/typebox": "^0.27.8" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jest/source-map": { + "version": "29.6.3", + "resolved": "https://registry.npmjs.org/@jest/source-map/-/source-map-29.6.3.tgz", + "integrity": "sha512-MHjT95QuipcPrpLM+8JMSzFx6eHp5Bm+4XeFDJlwsvVBjmKNiIAvasGK2fxz2WbGRlnvqehFbh07MMa7n3YJnw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/trace-mapping": "^0.3.18", + "callsites": "^3.0.0", + "graceful-fs": "^4.2.9" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jest/test-result": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/test-result/-/test-result-29.7.0.tgz", + "integrity": "sha512-Fdx+tv6x1zlkJPcWXmMDAG2HBnaR9XPSd5aDWQVsfrZmLVT3lU1cwyxLgRmXR9yrq4NBoEm9BMsfgFzTQAbJYA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/console": "^29.7.0", + "@jest/types": "^29.6.3", + "@types/istanbul-lib-coverage": "^2.0.0", + "collect-v8-coverage": "^1.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jest/test-sequencer": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/test-sequencer/-/test-sequencer-29.7.0.tgz", + "integrity": "sha512-GQwJ5WZVrKnOJuiYiAF52UNUJXgTZx1NHjFSEB0qEMmSZKAkdMoIzw/Cj6x6NF4AvV23AUqDpFzQkN/eYCYTxw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/test-result": "^29.7.0", + "graceful-fs": "^4.2.9", + "jest-haste-map": "^29.7.0", + "slash": "^3.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jest/transform": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/@jest/transform/-/transform-29.7.0.tgz", + "integrity": "sha512-ok/BTPFzFKVMwO5eOHRrvnBVHdRy9IrsrW1GpMaQ9MCnilNLXQKmAX8s1YXDFaai9xJpac2ySzV0YeRRECr2Vw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/core": "^7.11.6", + "@jest/types": "^29.6.3", + "@jridgewell/trace-mapping": "^0.3.18", + "babel-plugin-istanbul": "^6.1.1", + "chalk": "^4.0.0", + "convert-source-map": "^2.0.0", + "fast-json-stable-stringify": "^2.1.0", + "graceful-fs": "^4.2.9", + "jest-haste-map": "^29.7.0", + "jest-regex-util": "^29.6.3", + "jest-util": "^29.7.0", + "micromatch": "^4.0.4", + "pirates": "^4.0.4", + "slash": "^3.0.0", + "write-file-atomic": "^4.0.2" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jest/types": { + "version": "29.6.3", + "resolved": "https://registry.npmjs.org/@jest/types/-/types-29.6.3.tgz", + "integrity": "sha512-u3UPsIilWKOM3F9CXtrG8LEJmNxwoCQC/XVj4IKYXvvpx7QIi/Kg1LI5uDmDpKlac62NUtX7eLjRh+jVZcLOzw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/schemas": "^29.6.3", + "@types/istanbul-lib-coverage": "^2.0.0", + "@types/istanbul-reports": "^3.0.0", + "@types/node": "*", + "@types/yargs": "^17.0.8", + "chalk": "^4.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/@jridgewell/gen-mapping": { + "version": "0.3.13", + "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz", + "integrity": "sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.0", + "@jridgewell/trace-mapping": "^0.3.24" + } + }, + "node_modules/@jridgewell/remapping": { + "version": "2.3.5", + "resolved": "https://registry.npmjs.org/@jridgewell/remapping/-/remapping-2.3.5.tgz", + "integrity": "sha512-LI9u/+laYG4Ds1TDKSJW2YPrIlcVYOwi2fUC6xB43lueCjgxV4lffOCZCtYFiH6TNOX+tQKXx97T4IKHbhyHEQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/gen-mapping": "^0.3.5", + "@jridgewell/trace-mapping": "^0.3.24" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.5.5", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz", + "integrity": "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==", + "dev": true, + "license": "MIT" + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.31", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.31.tgz", + "integrity": "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/resolve-uri": "^3.1.0", + "@jridgewell/sourcemap-codec": "^1.4.14" + } + }, + "node_modules/@sinclair/typebox": { + "version": "0.27.8", + "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.27.8.tgz", + "integrity": "sha512-+Fj43pSMwJs4KRrH/938Uf+uAELIgVBmQzg/q1YG10djyfA3TnrU8N8XzqCh/okZdszqBQTZf96idMfE5lnwTA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@sinonjs/commons": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-3.0.1.tgz", + "integrity": "sha512-K3mCHKQ9sVh8o1C9cxkwxaOmXoAMlDxC1mYyHrjqOWEcBjYr76t96zL2zlj5dUGZ3HSw240X1qgH3Mjf1yJWpQ==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "type-detect": "4.0.8" + } + }, + "node_modules/@sinonjs/fake-timers": { + "version": "10.3.0", + "resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-10.3.0.tgz", + "integrity": "sha512-V4BG07kuYSUkTCSBHG8G8TNhM+F19jXFWnQtzj+we8DrkpSBCee9Z3Ms8yiGer/dlmhe35/Xdgyo3/0rQKg7YA==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "@sinonjs/commons": "^3.0.0" + } + }, + "node_modules/@tsconfig/node10": { + "version": "1.0.12", + "resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.12.tgz", + "integrity": "sha512-UCYBaeFvM11aU2y3YPZ//O5Rhj+xKyzy7mvcIoAjASbigy8mHMryP5cK7dgjlz2hWxh1g5pLw084E0a/wlUSFQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/@tsconfig/node12": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/@tsconfig/node12/-/node12-1.0.11.tgz", + "integrity": "sha512-cqefuRsh12pWyGsIoBKJA9luFu3mRxCA+ORZvA4ktLSzIuCUtWVxGIuXigEwO5/ywWFMZ2QEGKWvkZG1zDMTag==", + "dev": true, + "license": "MIT" + }, + "node_modules/@tsconfig/node14": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@tsconfig/node14/-/node14-1.0.3.tgz", + "integrity": "sha512-ysT8mhdixWK6Hw3i1V2AeRqZ5WfXg1G43mqoYlM2nc6388Fq5jcXyr5mRsqViLx/GJYdoL0bfXD8nmF+Zn/Iow==", + "dev": true, + "license": "MIT" + }, + "node_modules/@tsconfig/node16": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@tsconfig/node16/-/node16-1.0.4.tgz", + "integrity": "sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/babel__core": { + "version": "7.20.5", + "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.20.5.tgz", + "integrity": "sha512-qoQprZvz5wQFJwMDqeseRXWv3rqMvhgpbXFfVyWhbx9X47POIA6i/+dXefEmZKoAgOaTdaIgNSMqMIU61yRyzA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.20.7", + "@babel/types": "^7.20.7", + "@types/babel__generator": "*", + "@types/babel__template": "*", + "@types/babel__traverse": "*" + } + }, + "node_modules/@types/babel__generator": { + "version": "7.27.0", + "resolved": "https://registry.npmjs.org/@types/babel__generator/-/babel__generator-7.27.0.tgz", + "integrity": "sha512-ufFd2Xi92OAVPYsy+P4n7/U7e68fex0+Ee8gSG9KX7eo084CWiQ4sdxktvdl0bOPupXtVJPY19zk6EwWqUQ8lg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/types": "^7.0.0" + } + }, + "node_modules/@types/babel__template": { + "version": "7.4.4", + "resolved": "https://registry.npmjs.org/@types/babel__template/-/babel__template-7.4.4.tgz", + "integrity": "sha512-h/NUaSyG5EyxBIp8YRxo4RMe2/qQgvyowRwVMzhYhBCONbW8PUsg4lkFMrhgZhUe5z3L3MiLDuvyJ/CaPa2A8A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.1.0", + "@babel/types": "^7.0.0" + } + }, + "node_modules/@types/babel__traverse": { + "version": "7.28.0", + "resolved": "https://registry.npmjs.org/@types/babel__traverse/-/babel__traverse-7.28.0.tgz", + "integrity": "sha512-8PvcXf70gTDZBgt9ptxJ8elBeBjcLOAcOtoO/mPJjtji1+CdGbHgm77om1GrsPxsiE+uXIpNSK64UYaIwQXd4Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/types": "^7.28.2" + } + }, + "node_modules/@types/graceful-fs": { + "version": "4.1.9", + "resolved": "https://registry.npmjs.org/@types/graceful-fs/-/graceful-fs-4.1.9.tgz", + "integrity": "sha512-olP3sd1qOEe5dXTSaFvQG+02VdRXcdytWLAZsAq1PecU8uqQAhkrnbli7DagjtXKW/Bl7YJbUsa8MPcuc8LHEQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/istanbul-lib-coverage": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.6.tgz", + "integrity": "sha512-2QF/t/auWm0lsy8XtKVPG19v3sSOQlJe/YHZgfjb/KBBHOGSV+J2q/S671rcq9uTBrLAXmZpqJiaQbMT+zNU1w==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/istanbul-lib-report": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@types/istanbul-lib-report/-/istanbul-lib-report-3.0.3.tgz", + "integrity": "sha512-NQn7AHQnk/RSLOxrBbGyJM/aVQ+pjj5HCgasFxc0K/KhoATfQ/47AyUl15I2yBUpihjmas+a+VJBOqecrFH+uA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/istanbul-lib-coverage": "*" + } + }, + "node_modules/@types/istanbul-reports": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@types/istanbul-reports/-/istanbul-reports-3.0.4.tgz", + "integrity": "sha512-pk2B1NWalF9toCRu6gjBzR69syFjP4Od8WRAX+0mmf9lAjCRicLOWc+ZrxZHx/0XRjotgkF9t6iaMJ+aXcOdZQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/istanbul-lib-report": "*" + } + }, + "node_modules/@types/jest": { + "version": "29.5.14", + "resolved": "https://registry.npmjs.org/@types/jest/-/jest-29.5.14.tgz", + "integrity": "sha512-ZN+4sdnLUbo8EVvVc2ao0GFW6oVrQRPn4K2lglySj7APvSrgzxHiNNK99us4WDMi57xxA2yggblIAMNhXOotLQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "expect": "^29.0.0", + "pretty-format": "^29.0.0" + } + }, + "node_modules/@types/node": { + "version": "22.7.9", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.7.9.tgz", + "integrity": "sha512-jrTfRC7FM6nChvU7X2KqcrgquofrWLFDeYC1hKfwNWomVvrn7JIksqf344WN2X/y8xrgqBd2dJATZV4GbatBfg==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~6.19.2" + } + }, + "node_modules/@types/stack-utils": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@types/stack-utils/-/stack-utils-2.0.3.tgz", + "integrity": "sha512-9aEbYZ3TbYMznPdcdr3SmIrLXwC/AKZXQeCf9Pgao5CKb8CyHuEX5jzWPTkvregvhRJHcpRO6BFoGW9ycaOkYw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/yargs": { + "version": "17.0.35", + "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.35.tgz", + "integrity": "sha512-qUHkeCyQFxMXg79wQfTtfndEC+N9ZZg76HJftDJp+qH2tV7Gj4OJi7l+PiWwJ+pWtW8GwSmqsDj/oymhrTWXjg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/yargs-parser": "*" + } + }, + "node_modules/@types/yargs-parser": { + "version": "21.0.3", + "resolved": "https://registry.npmjs.org/@types/yargs-parser/-/yargs-parser-21.0.3.tgz", + "integrity": "sha512-I4q9QU9MQv4oEOz4tAHJtNz1cwuLxn2F3xcc2iV5WdqLPpUnj30aUuxt1mAxYTG+oe8CZMV/+6rU4S4gRDzqtQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/acorn": { + "version": "8.15.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz", + "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", + "dev": true, + "license": "MIT", + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/acorn-walk": { + "version": "8.3.4", + "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.3.4.tgz", + "integrity": "sha512-ueEepnujpqee2o5aIYnvHU6C0A42MNdsIDeqy5BydrkuC5R1ZuUFnm27EeFJGoEHJQgn3uleRvmTXaJgfXbt4g==", + "dev": true, + "license": "MIT", + "dependencies": { + "acorn": "^8.11.0" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/ansi-escapes": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-4.3.2.tgz", + "integrity": "sha512-gKXj5ALrKWQLsYG9jlTRmR/xKluxHV+Z9QEwNIgCfM1/uwPMCuzVVnh5mwTd+OuBZcwSIMbqssNWRm1lE51QaQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "type-fest": "^0.21.3" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/anymatch": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz", + "integrity": "sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==", + "dev": true, + "license": "ISC", + "dependencies": { + "normalize-path": "^3.0.0", + "picomatch": "^2.0.4" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/arg": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/arg/-/arg-4.1.3.tgz", + "integrity": "sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==", + "dev": true, + "license": "MIT" + }, + "node_modules/argparse": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "dev": true, + "license": "MIT", + "dependencies": { + "sprintf-js": "~1.0.2" + } + }, + "node_modules/aws-cdk": { + "version": "2.1031.2", + "resolved": "https://registry.npmjs.org/aws-cdk/-/aws-cdk-2.1031.2.tgz", + "integrity": "sha512-FI8XkslwC1Vatjdu5MXu2ww++FcZPkPt45/DJklApxMF+aGcCKOuLf+COc12QYK88GOrLBeCED6lDjNc9m/ueA==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "cdk": "bin/cdk" + }, + "engines": { + "node": ">= 18.0.0" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/aws-cdk-lib": { + "version": "2.215.0", + "resolved": "https://registry.npmjs.org/aws-cdk-lib/-/aws-cdk-lib-2.215.0.tgz", + "integrity": "sha512-DvRiUEsZSc4voeqfkYGrjP0f4NJ1u94JvbVCUQ+Fci4rf25xLRSEVPzyaH9Tf7V38i0Otts1+u7seunqe+R19g==", + "bundleDependencies": [ + "@balena/dockerignore", + "case", + "fs-extra", + "ignore", + "jsonschema", + "minimatch", + "punycode", + "semver", + "table", + "yaml", + "mime-types" + ], + "license": "Apache-2.0", + "dependencies": { + "@aws-cdk/asset-awscli-v1": "2.2.242", + "@aws-cdk/asset-node-proxy-agent-v6": "^2.1.0", + "@aws-cdk/cloud-assembly-schema": "^48.6.0", + "@balena/dockerignore": "^1.0.2", + "case": "1.6.3", + "fs-extra": "^11.3.1", + "ignore": "^5.3.2", + "jsonschema": "^1.5.0", + "mime-types": "^2.1.35", + "minimatch": "^3.1.2", + "punycode": "^2.3.1", + "semver": "^7.7.2", + "table": "^6.9.0", + "yaml": "1.10.2" + }, + "engines": { + "node": ">= 18.0.0" + }, + "peerDependencies": { + "constructs": "^10.0.0" + } + }, + "node_modules/aws-cdk-lib/node_modules/@balena/dockerignore": { + "version": "1.0.2", + "inBundle": true, + "license": "Apache-2.0" + }, + "node_modules/aws-cdk-lib/node_modules/ajv": { + "version": "8.17.1", + "inBundle": true, + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/aws-cdk-lib/node_modules/ansi-regex": { + "version": "5.0.1", + "inBundle": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/aws-cdk-lib/node_modules/ansi-styles": { + "version": "4.3.0", + "inBundle": true, + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/aws-cdk-lib/node_modules/astral-regex": { + "version": "2.0.0", + "inBundle": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/aws-cdk-lib/node_modules/balanced-match": { + "version": "1.0.2", + "inBundle": true, + "license": "MIT" + }, + "node_modules/aws-cdk-lib/node_modules/brace-expansion": { + "version": "1.1.12", + "inBundle": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/aws-cdk-lib/node_modules/case": { + "version": "1.6.3", + "inBundle": true, + "license": "(MIT OR GPL-3.0-or-later)", + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/aws-cdk-lib/node_modules/color-convert": { + "version": "2.0.1", + "inBundle": true, + "license": "MIT", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/aws-cdk-lib/node_modules/color-name": { + "version": "1.1.4", + "inBundle": true, + "license": "MIT" + }, + "node_modules/aws-cdk-lib/node_modules/concat-map": { + "version": "0.0.1", + "inBundle": true, + "license": "MIT" + }, + "node_modules/aws-cdk-lib/node_modules/emoji-regex": { + "version": "8.0.0", + "inBundle": true, + "license": "MIT" + }, + "node_modules/aws-cdk-lib/node_modules/fast-deep-equal": { + "version": "3.1.3", + "inBundle": true, + "license": "MIT" + }, + "node_modules/aws-cdk-lib/node_modules/fast-uri": { + "version": "3.1.0", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "inBundle": true, + "license": "BSD-3-Clause" + }, + "node_modules/aws-cdk-lib/node_modules/fs-extra": { + "version": "11.3.1", + "inBundle": true, + "license": "MIT", + "dependencies": { + "graceful-fs": "^4.2.0", + "jsonfile": "^6.0.1", + "universalify": "^2.0.0" + }, + "engines": { + "node": ">=14.14" + } + }, + "node_modules/aws-cdk-lib/node_modules/graceful-fs": { + "version": "4.2.11", + "inBundle": true, + "license": "ISC" + }, + "node_modules/aws-cdk-lib/node_modules/ignore": { + "version": "5.3.2", + "inBundle": true, + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, + "node_modules/aws-cdk-lib/node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "inBundle": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/aws-cdk-lib/node_modules/json-schema-traverse": { + "version": "1.0.0", + "inBundle": true, + "license": "MIT" + }, + "node_modules/aws-cdk-lib/node_modules/jsonfile": { + "version": "6.2.0", + "inBundle": true, + "license": "MIT", + "dependencies": { + "universalify": "^2.0.0" + }, + "optionalDependencies": { + "graceful-fs": "^4.1.6" + } + }, + "node_modules/aws-cdk-lib/node_modules/jsonschema": { + "version": "1.5.0", + "inBundle": true, + "license": "MIT", + "engines": { + "node": "*" + } + }, + "node_modules/aws-cdk-lib/node_modules/lodash.truncate": { + "version": "4.4.2", + "inBundle": true, + "license": "MIT" + }, + "node_modules/aws-cdk-lib/node_modules/mime-db": { + "version": "1.52.0", + "inBundle": true, + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/aws-cdk-lib/node_modules/mime-types": { + "version": "2.1.35", + "inBundle": true, + "license": "MIT", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/aws-cdk-lib/node_modules/minimatch": { + "version": "3.1.2", + "inBundle": true, + "license": "ISC", + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/aws-cdk-lib/node_modules/punycode": { + "version": "2.3.1", + "inBundle": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/aws-cdk-lib/node_modules/require-from-string": { + "version": "2.0.2", + "inBundle": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/aws-cdk-lib/node_modules/semver": { + "version": "7.7.2", + "inBundle": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/aws-cdk-lib/node_modules/slice-ansi": { + "version": "4.0.0", + "inBundle": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "astral-regex": "^2.0.0", + "is-fullwidth-code-point": "^3.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/slice-ansi?sponsor=1" + } + }, + "node_modules/aws-cdk-lib/node_modules/string-width": { + "version": "4.2.3", + "inBundle": true, + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/aws-cdk-lib/node_modules/strip-ansi": { + "version": "6.0.1", + "inBundle": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/aws-cdk-lib/node_modules/table": { + "version": "6.9.0", + "inBundle": true, + "license": "BSD-3-Clause", + "dependencies": { + "ajv": "^8.0.1", + "lodash.truncate": "^4.4.2", + "slice-ansi": "^4.0.0", + "string-width": "^4.2.3", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=10.0.0" + } + }, + "node_modules/aws-cdk-lib/node_modules/universalify": { + "version": "2.0.1", + "inBundle": true, + "license": "MIT", + "engines": { + "node": ">= 10.0.0" + } + }, + "node_modules/aws-cdk-lib/node_modules/yaml": { + "version": "1.10.2", + "inBundle": true, + "license": "ISC", + "engines": { + "node": ">= 6" + } + }, + "node_modules/babel-jest": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/babel-jest/-/babel-jest-29.7.0.tgz", + "integrity": "sha512-BrvGY3xZSwEcCzKvKsCi2GgHqDqsYkOP4/by5xCgIwGXQxIEh+8ew3gmrE1y7XRR6LHZIj6yLYnUi/mm2KXKBg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/transform": "^29.7.0", + "@types/babel__core": "^7.1.14", + "babel-plugin-istanbul": "^6.1.1", + "babel-preset-jest": "^29.6.3", + "chalk": "^4.0.0", + "graceful-fs": "^4.2.9", + "slash": "^3.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + }, + "peerDependencies": { + "@babel/core": "^7.8.0" + } + }, + "node_modules/babel-plugin-istanbul": { + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/babel-plugin-istanbul/-/babel-plugin-istanbul-6.1.1.tgz", + "integrity": "sha512-Y1IQok9821cC9onCx5otgFfRm7Lm+I+wwxOx738M/WLPZ9Q42m4IG5W0FNX8WLL2gYMZo3JkuXIH2DOpWM+qwA==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "@babel/helper-plugin-utils": "^7.0.0", + "@istanbuljs/load-nyc-config": "^1.0.0", + "@istanbuljs/schema": "^0.1.2", + "istanbul-lib-instrument": "^5.0.4", + "test-exclude": "^6.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/babel-plugin-istanbul/node_modules/istanbul-lib-instrument": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/istanbul-lib-instrument/-/istanbul-lib-instrument-5.2.1.tgz", + "integrity": "sha512-pzqtp31nLv/XFOzXGuvhCb8qhjmTVo5vjVk19XE4CRlSWz0KoeJ3bw9XsA7nOp9YBf4qHjwBxkDzKcME/J29Yg==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "@babel/core": "^7.12.3", + "@babel/parser": "^7.14.7", + "@istanbuljs/schema": "^0.1.2", + "istanbul-lib-coverage": "^3.2.0", + "semver": "^6.3.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/babel-plugin-jest-hoist": { + "version": "29.6.3", + "resolved": "https://registry.npmjs.org/babel-plugin-jest-hoist/-/babel-plugin-jest-hoist-29.6.3.tgz", + "integrity": "sha512-ESAc/RJvGTFEzRwOTT4+lNDk/GNHMkKbNzsvT0qKRfDyyYTskxB5rnU2njIDYVxXCBHHEI1c0YwHob3WaYujOg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/template": "^7.3.3", + "@babel/types": "^7.3.3", + "@types/babel__core": "^7.1.14", + "@types/babel__traverse": "^7.0.6" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/babel-preset-current-node-syntax": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/babel-preset-current-node-syntax/-/babel-preset-current-node-syntax-1.2.0.tgz", + "integrity": "sha512-E/VlAEzRrsLEb2+dv8yp3bo4scof3l9nR4lrld+Iy5NyVqgVYUJnDAmunkhPMisRI32Qc4iRiz425d8vM++2fg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/plugin-syntax-async-generators": "^7.8.4", + "@babel/plugin-syntax-bigint": "^7.8.3", + "@babel/plugin-syntax-class-properties": "^7.12.13", + "@babel/plugin-syntax-class-static-block": "^7.14.5", + "@babel/plugin-syntax-import-attributes": "^7.24.7", + "@babel/plugin-syntax-import-meta": "^7.10.4", + "@babel/plugin-syntax-json-strings": "^7.8.3", + "@babel/plugin-syntax-logical-assignment-operators": "^7.10.4", + "@babel/plugin-syntax-nullish-coalescing-operator": "^7.8.3", + "@babel/plugin-syntax-numeric-separator": "^7.10.4", + "@babel/plugin-syntax-object-rest-spread": "^7.8.3", + "@babel/plugin-syntax-optional-catch-binding": "^7.8.3", + "@babel/plugin-syntax-optional-chaining": "^7.8.3", + "@babel/plugin-syntax-private-property-in-object": "^7.14.5", + "@babel/plugin-syntax-top-level-await": "^7.14.5" + }, + "peerDependencies": { + "@babel/core": "^7.0.0 || ^8.0.0-0" + } + }, + "node_modules/babel-preset-jest": { + "version": "29.6.3", + "resolved": "https://registry.npmjs.org/babel-preset-jest/-/babel-preset-jest-29.6.3.tgz", + "integrity": "sha512-0B3bhxR6snWXJZtR/RliHTDPRgn1sNHOR0yVtq/IiQFyuOVjFS+wuio/R4gSNkyYmKmJB4wGZv2NZanmKmTnNA==", + "dev": true, + "license": "MIT", + "dependencies": { + "babel-plugin-jest-hoist": "^29.6.3", + "babel-preset-current-node-syntax": "^1.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/balanced-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", + "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", + "dev": true, + "license": "MIT" + }, + "node_modules/baseline-browser-mapping": { + "version": "2.8.28", + "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.8.28.tgz", + "integrity": "sha512-gYjt7OIqdM0PcttNYP2aVrr2G0bMALkBaoehD4BuRGjAOtipg0b6wHg1yNL+s5zSnLZZrGHOw4IrND8CD+3oIQ==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "baseline-browser-mapping": "dist/cli.js" + } + }, + "node_modules/brace-expansion": { + "version": "1.1.12", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", + "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/braces": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", + "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", + "dev": true, + "license": "MIT", + "dependencies": { + "fill-range": "^7.1.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/browserslist": { + "version": "4.28.0", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.28.0.tgz", + "integrity": "sha512-tbydkR/CxfMwelN0vwdP/pLkDwyAASZ+VfWm4EOwlB6SWhx1sYnWLqo8N5j0rAzPfzfRaxt0mM/4wPU/Su84RQ==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "baseline-browser-mapping": "^2.8.25", + "caniuse-lite": "^1.0.30001754", + "electron-to-chromium": "^1.5.249", + "node-releases": "^2.0.27", + "update-browserslist-db": "^1.1.4" + }, + "bin": { + "browserslist": "cli.js" + }, + "engines": { + "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" + } + }, + "node_modules/bs-logger": { + "version": "0.2.6", + "resolved": "https://registry.npmjs.org/bs-logger/-/bs-logger-0.2.6.tgz", + "integrity": "sha512-pd8DCoxmbgc7hyPKOvxtqNcjYoOsABPQdcCUjGp3d42VR2CX1ORhk2A87oqqu5R1kk+76nsxZupkmyd+MVtCog==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-json-stable-stringify": "2.x" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/bser": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/bser/-/bser-2.1.1.tgz", + "integrity": "sha512-gQxTNE/GAfIIrmHLUE3oJyp5FO6HRBfhjnw4/wMmA63ZGDJnWBmgY/lyQBpnDUkGmAhbSe39tx2d/iTOAfglwQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "node-int64": "^0.4.0" + } + }, + "node_modules/buffer-from": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", + "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/callsites": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", + "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/camelcase": { + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz", + "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/caniuse-lite": { + "version": "1.0.30001754", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001754.tgz", + "integrity": "sha512-x6OeBXueoAceOmotzx3PO4Zpt4rzpeIFsSr6AAePTZxSkXiYDUmpypEl7e2+8NCd9bD7bXjqyef8CJYPC1jfxg==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/caniuse-lite" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "CC-BY-4.0" + }, + "node_modules/chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/char-regex": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/char-regex/-/char-regex-1.0.2.tgz", + "integrity": "sha512-kWWXztvZ5SBQV+eRgKFeh8q5sLuZY2+8WUIzlxWVTg+oGwY14qylx1KbKzHd8P6ZYkAg0xyIDU9JMHhyJMZ1jw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + } + }, + "node_modules/ci-info": { + "version": "3.9.0", + "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-3.9.0.tgz", + "integrity": "sha512-NIxF55hv4nSqQswkAeiOi1r83xy8JldOFDTWiug55KBu9Jnblncd2U6ViHmYgHf01TPZS77NJBhBMKdWj9HQMQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/sibiraj-s" + } + ], + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/cjs-module-lexer": { + "version": "1.4.3", + "resolved": "https://registry.npmjs.org/cjs-module-lexer/-/cjs-module-lexer-1.4.3.tgz", + "integrity": "sha512-9z8TZaGM1pfswYeXrUpzPrkx8UnWYdhJclsiYMm6x/w5+nN+8Tf/LnAgfLGQCm59qAOxU8WwHEq2vNwF6i4j+Q==", + "dev": true, + "license": "MIT" + }, + "node_modules/cliui": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", + "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", + "dev": true, + "license": "ISC", + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.1", + "wrap-ansi": "^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/co": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz", + "integrity": "sha512-QVb0dM5HvG+uaxitm8wONl7jltx8dqhfU33DcqtOZcLSVIKSDDLDi7+0LbAKiyI8hD9u42m2YxXSkMGWThaecQ==", + "dev": true, + "license": "MIT", + "engines": { + "iojs": ">= 1.0.0", + "node": ">= 0.12.0" + } + }, + "node_modules/collect-v8-coverage": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/collect-v8-coverage/-/collect-v8-coverage-1.0.3.tgz", + "integrity": "sha512-1L5aqIkwPfiodaMgQunkF1zRhNqifHBmtbbbxcr6yVxxBnliw4TDOW6NxpO8DJLgJ16OT+Y4ztZqP6p/FtXnAw==", + "dev": true, + "license": "MIT" + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true, + "license": "MIT" + }, + "node_modules/commander": { + "version": "14.0.2", + "resolved": "https://registry.npmjs.org/commander/-/commander-14.0.2.tgz", + "integrity": "sha512-TywoWNNRbhoD0BXs1P3ZEScW8W5iKrnbithIl0YH+uCmBd0QpPOA8yc82DS3BIE5Ma6FnBVUsJ7wVUDz4dvOWQ==", + "license": "MIT", + "engines": { + "node": ">=20" + } + }, + "node_modules/concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", + "dev": true, + "license": "MIT" + }, + "node_modules/constructs": { + "version": "10.4.3", + "resolved": "https://registry.npmjs.org/constructs/-/constructs-10.4.3.tgz", + "integrity": "sha512-3+ZB67qWGM1vEstNpj6pGaLNN1qz4gxC1CBhEUhZDZk0PqzQWY65IzC1Doq17MGPa9xa2wJ1G/DJ3swU8kWAHQ==", + "license": "Apache-2.0" + }, + "node_modules/convert-source-map": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", + "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==", + "dev": true, + "license": "MIT" + }, + "node_modules/create-jest": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/create-jest/-/create-jest-29.7.0.tgz", + "integrity": "sha512-Adz2bdH0Vq3F53KEMJOoftQFutWCukm6J24wbPWRO4k1kMY7gS7ds/uoJkNuV8wDCtWWnuwGcJwpWcih+zEW1Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/types": "^29.6.3", + "chalk": "^4.0.0", + "exit": "^0.1.2", + "graceful-fs": "^4.2.9", + "jest-config": "^29.7.0", + "jest-util": "^29.7.0", + "prompts": "^2.0.1" + }, + "bin": { + "create-jest": "bin/create-jest.js" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/create-require": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz", + "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "dev": true, + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/dedent": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/dedent/-/dedent-1.7.0.tgz", + "integrity": "sha512-HGFtf8yhuhGhqO07SV79tRp+br4MnbdjeVxotpn1QBl30pcLLCQjX5b2295ll0fv8RKDKsmWYrl05usHM9CewQ==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "babel-plugin-macros": "^3.1.0" + }, + "peerDependenciesMeta": { + "babel-plugin-macros": { + "optional": true + } + } + }, + "node_modules/deepmerge": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-4.3.1.tgz", + "integrity": "sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/detect-newline": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/detect-newline/-/detect-newline-3.1.0.tgz", + "integrity": "sha512-TLz+x/vEXm/Y7P7wn1EJFNLxYpUD4TgMosxY6fAVJUnJMbupHBOncxyWUG9OpTaH9EBD7uFI5LfEgmMOc54DsA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/diff": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz", + "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.3.1" + } + }, + "node_modules/diff-sequences": { + "version": "29.6.3", + "resolved": "https://registry.npmjs.org/diff-sequences/-/diff-sequences-29.6.3.tgz", + "integrity": "sha512-EjePK1srD3P08o2j4f0ExnylqRs5B9tJjcp9t1krH2qRi8CCdsYfwe9JgSLurFBWwq4uOlipzfk5fHNvwFKr8Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/electron-to-chromium": { + "version": "1.5.253", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.253.tgz", + "integrity": "sha512-O0tpQ/35rrgdiGQ0/OFWhy1itmd9A6TY9uQzlqj3hKSu/aYpe7UIn5d7CU2N9myH6biZiWF3VMZVuup8pw5U9w==", + "dev": true, + "license": "ISC" + }, + "node_modules/emittery": { + "version": "0.13.1", + "resolved": "https://registry.npmjs.org/emittery/-/emittery-0.13.1.tgz", + "integrity": "sha512-DeWwawk6r5yR9jFgnDKYt4sLS0LmHJJi3ZOnb5/JdbYwj3nW+FxQnHIjhBKz8YLC7oRNPVM9NQ47I3CVx34eqQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sindresorhus/emittery?sponsor=1" + } + }, + "node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true, + "license": "MIT" + }, + "node_modules/error-ex": { + "version": "1.3.4", + "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.4.tgz", + "integrity": "sha512-sqQamAnR14VgCr1A618A3sGrygcpK+HEbenA/HiEAkkUwcZIIB/tgWqHFxWgOyDh4nB4JCRimh79dR5Ywc9MDQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-arrayish": "^0.2.1" + } + }, + "node_modules/escalade": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", + "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/escape-string-regexp": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-2.0.0.tgz", + "integrity": "sha512-UpzcLCXolUWcNu5HtVMHYdXJjArjsF9C0aNnquZYY4uW/Vu0miy5YoWvbV345HauVvcAUnpRuhMMcqTcGOY2+w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/esprima": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", + "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", + "dev": true, + "license": "BSD-2-Clause", + "bin": { + "esparse": "bin/esparse.js", + "esvalidate": "bin/esvalidate.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/execa": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/execa/-/execa-5.1.1.tgz", + "integrity": "sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg==", + "dev": true, + "license": "MIT", + "dependencies": { + "cross-spawn": "^7.0.3", + "get-stream": "^6.0.0", + "human-signals": "^2.1.0", + "is-stream": "^2.0.0", + "merge-stream": "^2.0.0", + "npm-run-path": "^4.0.1", + "onetime": "^5.1.2", + "signal-exit": "^3.0.3", + "strip-final-newline": "^2.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sindresorhus/execa?sponsor=1" + } + }, + "node_modules/exit": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/exit/-/exit-0.1.2.tgz", + "integrity": "sha512-Zk/eNKV2zbjpKzrsQ+n1G6poVbErQxJ0LBOJXaKZ1EViLzH+hrLu9cdXI4zw9dBQJslwBEpbQ2P1oS7nDxs6jQ==", + "dev": true, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/expect": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/expect/-/expect-29.7.0.tgz", + "integrity": "sha512-2Zks0hf1VLFYI1kbh0I5jP3KHHyCHpkfyHBzsSXRFgl/Bg9mWYfMW8oD+PdMPlEwy5HNsR9JutYy6pMeOh61nw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/expect-utils": "^29.7.0", + "jest-get-type": "^29.6.3", + "jest-matcher-utils": "^29.7.0", + "jest-message-util": "^29.7.0", + "jest-util": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/fast-json-stable-stringify": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", + "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", + "dev": true, + "license": "MIT" + }, + "node_modules/fb-watchman": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/fb-watchman/-/fb-watchman-2.0.2.tgz", + "integrity": "sha512-p5161BqbuCaSnB8jIbzQHOlpgsPmK5rJVDfDKO91Axs5NC1uu3HRQm6wt9cd9/+GtQQIO53JdGXXoyDpTAsgYA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "bser": "2.1.1" + } + }, + "node_modules/fill-range": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", + "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", + "dev": true, + "license": "MIT", + "dependencies": { + "to-regex-range": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/find-up": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", + "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", + "dev": true, + "license": "MIT", + "dependencies": { + "locate-path": "^5.0.0", + "path-exists": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/fs.realpath": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", + "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", + "dev": true, + "license": "ISC" + }, + "node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/gensync": { + "version": "1.0.0-beta.2", + "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", + "integrity": "sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "dev": true, + "license": "ISC", + "engines": { + "node": "6.* || 8.* || >= 10.*" + } + }, + "node_modules/get-package-type": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/get-package-type/-/get-package-type-0.1.0.tgz", + "integrity": "sha512-pjzuKtY64GYfWizNAJ0fr9VqttZkNiK2iS430LtIHzjBEr6bX8Am2zm4sW4Ro5wjWW5cAlRL1qAMTcXbjNAO2Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/get-stream": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz", + "integrity": "sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/glob": { + "version": "7.2.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", + "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", + "deprecated": "Glob versions prior to v9 are no longer supported", + "dev": true, + "license": "ISC", + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.1.1", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/graceful-fs": { + "version": "4.2.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/handlebars": { + "version": "4.7.8", + "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.7.8.tgz", + "integrity": "sha512-vafaFqs8MZkRrSX7sFVUdo3ap/eNiLnb4IakshzvP56X5Nr1iGKAIqdX6tMlm6HcNRIkr6AxO5jFEoJzzpT8aQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "minimist": "^1.2.5", + "neo-async": "^2.6.2", + "source-map": "^0.6.1", + "wordwrap": "^1.0.0" + }, + "bin": { + "handlebars": "bin/handlebars" + }, + "engines": { + "node": ">=0.4.7" + }, + "optionalDependencies": { + "uglify-js": "^3.1.4" + } + }, + "node_modules/has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/hasown": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", + "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/html-escaper": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-2.0.2.tgz", + "integrity": "sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==", + "dev": true, + "license": "MIT" + }, + "node_modules/human-signals": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-2.1.0.tgz", + "integrity": "sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=10.17.0" + } + }, + "node_modules/import-local": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/import-local/-/import-local-3.2.0.tgz", + "integrity": "sha512-2SPlun1JUPWoM6t3F0dw0FkCF/jWY8kttcY4f599GLTSjh2OCuuhdTkJQsEcZzBqbXZGKMK2OqW1oZsjtf/gQA==", + "dev": true, + "license": "MIT", + "dependencies": { + "pkg-dir": "^4.2.0", + "resolve-cwd": "^3.0.0" + }, + "bin": { + "import-local-fixture": "fixtures/cli.js" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/imurmurhash": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", + "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.8.19" + } + }, + "node_modules/inflight": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", + "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", + "deprecated": "This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful.", + "dev": true, + "license": "ISC", + "dependencies": { + "once": "^1.3.0", + "wrappy": "1" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/is-arrayish": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", + "integrity": "sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==", + "dev": true, + "license": "MIT" + }, + "node_modules/is-core-module": { + "version": "2.16.1", + "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.16.1.tgz", + "integrity": "sha512-UfoeMA6fIJ8wTYFEUjelnaGI67v6+N7qXJEvQuIGa99l4xsCruSYOVSQ0uPANn4dAzm8lkYPaKLrrijLq7x23w==", + "dev": true, + "license": "MIT", + "dependencies": { + "hasown": "^2.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/is-generator-fn": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-generator-fn/-/is-generator-fn-2.1.0.tgz", + "integrity": "sha512-cTIB4yPYL/Grw0EaSzASzg6bBy9gqCofvWN8okThAYIxKJZC+udlRAmGbM0XLeniEJSs8uEgHPGuHSe1XsOLSQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/is-number": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", + "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.12.0" + } + }, + "node_modules/is-stream": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", + "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "dev": true, + "license": "ISC" + }, + "node_modules/istanbul-lib-coverage": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/istanbul-lib-coverage/-/istanbul-lib-coverage-3.2.2.tgz", + "integrity": "sha512-O8dpsF+r0WV/8MNRKfnmrtCWhuKjxrq2w+jpzBL5UZKTi2LeVWnWOmWRxFlesJONmc+wLAGvKQZEOanko0LFTg==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=8" + } + }, + "node_modules/istanbul-lib-instrument": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/istanbul-lib-instrument/-/istanbul-lib-instrument-6.0.3.tgz", + "integrity": "sha512-Vtgk7L/R2JHyyGW07spoFlB8/lpjiOLTjMdms6AFMraYt3BaJauod/NGrfnVG/y4Ix1JEuMRPDPEj2ua+zz1/Q==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "@babel/core": "^7.23.9", + "@babel/parser": "^7.23.9", + "@istanbuljs/schema": "^0.1.3", + "istanbul-lib-coverage": "^3.2.0", + "semver": "^7.5.4" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/istanbul-lib-instrument/node_modules/semver": { + "version": "7.7.3", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.3.tgz", + "integrity": "sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/istanbul-lib-report": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/istanbul-lib-report/-/istanbul-lib-report-3.0.1.tgz", + "integrity": "sha512-GCfE1mtsHGOELCU8e/Z7YWzpmybrx/+dSTfLrvY8qRmaY6zXTKWn6WQIjaAFw069icm6GVMNkgu0NzI4iPZUNw==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "istanbul-lib-coverage": "^3.0.0", + "make-dir": "^4.0.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/istanbul-lib-source-maps": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/istanbul-lib-source-maps/-/istanbul-lib-source-maps-4.0.1.tgz", + "integrity": "sha512-n3s8EwkdFIJCG3BPKBYvskgXGoy88ARzvegkitk60NxRdwltLOTaH7CUiMRXvwYorl0Q712iEjcWB+fK/MrWVw==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "debug": "^4.1.1", + "istanbul-lib-coverage": "^3.0.0", + "source-map": "^0.6.1" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/istanbul-reports": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/istanbul-reports/-/istanbul-reports-3.2.0.tgz", + "integrity": "sha512-HGYWWS/ehqTV3xN10i23tkPkpH46MLCIMFNCaaKNavAXTF1RkqxawEPtnjnGZ6XKSInBKkiOA5BKS+aZiY3AvA==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "html-escaper": "^2.0.0", + "istanbul-lib-report": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/jest": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest/-/jest-29.7.0.tgz", + "integrity": "sha512-NIy3oAFp9shda19hy4HK0HRTWKtPJmGdnvywu01nOqNC2vZg+Z+fvJDxpMQA88eb2I9EcafcdjYgsDthnYTvGw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/core": "^29.7.0", + "@jest/types": "^29.6.3", + "import-local": "^3.0.2", + "jest-cli": "^29.7.0" + }, + "bin": { + "jest": "bin/jest.js" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + }, + "peerDependencies": { + "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0" + }, + "peerDependenciesMeta": { + "node-notifier": { + "optional": true + } + } + }, + "node_modules/jest-changed-files": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-changed-files/-/jest-changed-files-29.7.0.tgz", + "integrity": "sha512-fEArFiwf1BpQ+4bXSprcDc3/x4HSzL4al2tozwVpDFpsxALjLYdyiIK4e5Vz66GQJIbXJ82+35PtysofptNX2w==", + "dev": true, + "license": "MIT", + "dependencies": { + "execa": "^5.0.0", + "jest-util": "^29.7.0", + "p-limit": "^3.1.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-circus": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-circus/-/jest-circus-29.7.0.tgz", + "integrity": "sha512-3E1nCMgipcTkCocFwM90XXQab9bS+GMsjdpmPrlelaxwD93Ad8iVEjX/vvHPdLPnFf+L40u+5+iutRdA1N9myw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/environment": "^29.7.0", + "@jest/expect": "^29.7.0", + "@jest/test-result": "^29.7.0", + "@jest/types": "^29.6.3", + "@types/node": "*", + "chalk": "^4.0.0", + "co": "^4.6.0", + "dedent": "^1.0.0", + "is-generator-fn": "^2.0.0", + "jest-each": "^29.7.0", + "jest-matcher-utils": "^29.7.0", + "jest-message-util": "^29.7.0", + "jest-runtime": "^29.7.0", + "jest-snapshot": "^29.7.0", + "jest-util": "^29.7.0", + "p-limit": "^3.1.0", + "pretty-format": "^29.7.0", + "pure-rand": "^6.0.0", + "slash": "^3.0.0", + "stack-utils": "^2.0.3" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-cli": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-cli/-/jest-cli-29.7.0.tgz", + "integrity": "sha512-OVVobw2IubN/GSYsxETi+gOe7Ka59EFMR/twOU3Jb2GnKKeMGJB5SGUUrEz3SFVmJASUdZUzy83sLNNQ2gZslg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/core": "^29.7.0", + "@jest/test-result": "^29.7.0", + "@jest/types": "^29.6.3", + "chalk": "^4.0.0", + "create-jest": "^29.7.0", + "exit": "^0.1.2", + "import-local": "^3.0.2", + "jest-config": "^29.7.0", + "jest-util": "^29.7.0", + "jest-validate": "^29.7.0", + "yargs": "^17.3.1" + }, + "bin": { + "jest": "bin/jest.js" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + }, + "peerDependencies": { + "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0" + }, + "peerDependenciesMeta": { + "node-notifier": { + "optional": true + } + } + }, + "node_modules/jest-config": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-config/-/jest-config-29.7.0.tgz", + "integrity": "sha512-uXbpfeQ7R6TZBqI3/TxCU4q4ttk3u0PJeC+E0zbfSoSjq6bJ7buBPxzQPL0ifrkY4DNu4JUdk0ImlBUYi840eQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/core": "^7.11.6", + "@jest/test-sequencer": "^29.7.0", + "@jest/types": "^29.6.3", + "babel-jest": "^29.7.0", + "chalk": "^4.0.0", + "ci-info": "^3.2.0", + "deepmerge": "^4.2.2", + "glob": "^7.1.3", + "graceful-fs": "^4.2.9", + "jest-circus": "^29.7.0", + "jest-environment-node": "^29.7.0", + "jest-get-type": "^29.6.3", + "jest-regex-util": "^29.6.3", + "jest-resolve": "^29.7.0", + "jest-runner": "^29.7.0", + "jest-util": "^29.7.0", + "jest-validate": "^29.7.0", + "micromatch": "^4.0.4", + "parse-json": "^5.2.0", + "pretty-format": "^29.7.0", + "slash": "^3.0.0", + "strip-json-comments": "^3.1.1" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + }, + "peerDependencies": { + "@types/node": "*", + "ts-node": ">=9.0.0" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "ts-node": { + "optional": true + } + } + }, + "node_modules/jest-diff": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-diff/-/jest-diff-29.7.0.tgz", + "integrity": "sha512-LMIgiIrhigmPrs03JHpxUh2yISK3vLFPkAodPeo0+BuF7wA2FoQbkEg1u8gBYBThncu7e1oEDUfIXVuTqLRUjw==", + "dev": true, + "license": "MIT", + "dependencies": { + "chalk": "^4.0.0", + "diff-sequences": "^29.6.3", + "jest-get-type": "^29.6.3", + "pretty-format": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-docblock": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-docblock/-/jest-docblock-29.7.0.tgz", + "integrity": "sha512-q617Auw3A612guyaFgsbFeYpNP5t2aoUNLwBUbc/0kD1R4t9ixDbyFTHd1nok4epoVFpr7PmeWHrhvuV3XaJ4g==", + "dev": true, + "license": "MIT", + "dependencies": { + "detect-newline": "^3.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-each": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-each/-/jest-each-29.7.0.tgz", + "integrity": "sha512-gns+Er14+ZrEoC5fhOfYCY1LOHHr0TI+rQUHZS8Ttw2l7gl+80eHc/gFf2Ktkw0+SIACDTeWvpFcv3B04VembQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/types": "^29.6.3", + "chalk": "^4.0.0", + "jest-get-type": "^29.6.3", + "jest-util": "^29.7.0", + "pretty-format": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-environment-node": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-environment-node/-/jest-environment-node-29.7.0.tgz", + "integrity": "sha512-DOSwCRqXirTOyheM+4d5YZOrWcdu0LNZ87ewUoywbcb2XR4wKgqiG8vNeYwhjFMbEkfju7wx2GYH0P2gevGvFw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/environment": "^29.7.0", + "@jest/fake-timers": "^29.7.0", + "@jest/types": "^29.6.3", + "@types/node": "*", + "jest-mock": "^29.7.0", + "jest-util": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-get-type": { + "version": "29.6.3", + "resolved": "https://registry.npmjs.org/jest-get-type/-/jest-get-type-29.6.3.tgz", + "integrity": "sha512-zrteXnqYxfQh7l5FHyL38jL39di8H8rHoecLH3JNxH3BwOrBsNeabdap5e0I23lD4HHI8W5VFBZqG4Eaq5LNcw==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-haste-map": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-haste-map/-/jest-haste-map-29.7.0.tgz", + "integrity": "sha512-fP8u2pyfqx0K1rGn1R9pyE0/KTn+G7PxktWidOBTqFPLYX0b9ksaMFkhK5vrS3DVun09pckLdlx90QthlW7AmA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/types": "^29.6.3", + "@types/graceful-fs": "^4.1.3", + "@types/node": "*", + "anymatch": "^3.0.3", + "fb-watchman": "^2.0.0", + "graceful-fs": "^4.2.9", + "jest-regex-util": "^29.6.3", + "jest-util": "^29.7.0", + "jest-worker": "^29.7.0", + "micromatch": "^4.0.4", + "walker": "^1.0.8" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + }, + "optionalDependencies": { + "fsevents": "^2.3.2" + } + }, + "node_modules/jest-leak-detector": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-leak-detector/-/jest-leak-detector-29.7.0.tgz", + "integrity": "sha512-kYA8IJcSYtST2BY9I+SMC32nDpBT3J2NvWJx8+JCuCdl/CR1I4EKUJROiP8XtCcxqgTTBGJNdbB1A8XRKbTetw==", + "dev": true, + "license": "MIT", + "dependencies": { + "jest-get-type": "^29.6.3", + "pretty-format": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-matcher-utils": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-matcher-utils/-/jest-matcher-utils-29.7.0.tgz", + "integrity": "sha512-sBkD+Xi9DtcChsI3L3u0+N0opgPYnCRPtGcQYrgXmR+hmt/fYfWAL0xRXYU8eWOdfuLgBe0YCW3AFtnRLagq/g==", + "dev": true, + "license": "MIT", + "dependencies": { + "chalk": "^4.0.0", + "jest-diff": "^29.7.0", + "jest-get-type": "^29.6.3", + "pretty-format": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-message-util": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-message-util/-/jest-message-util-29.7.0.tgz", + "integrity": "sha512-GBEV4GRADeP+qtB2+6u61stea8mGcOT4mCtrYISZwfu9/ISHFJ/5zOMXYbpBE9RsS5+Gb63DW4FgmnKJ79Kf6w==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.12.13", + "@jest/types": "^29.6.3", + "@types/stack-utils": "^2.0.0", + "chalk": "^4.0.0", + "graceful-fs": "^4.2.9", + "micromatch": "^4.0.4", + "pretty-format": "^29.7.0", + "slash": "^3.0.0", + "stack-utils": "^2.0.3" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-mock": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-mock/-/jest-mock-29.7.0.tgz", + "integrity": "sha512-ITOMZn+UkYS4ZFh83xYAOzWStloNzJFO2s8DWrE4lhtGD+AorgnbkiKERe4wQVBydIGPx059g6riW5Btp6Llnw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/types": "^29.6.3", + "@types/node": "*", + "jest-util": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-pnp-resolver": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/jest-pnp-resolver/-/jest-pnp-resolver-1.2.3.tgz", + "integrity": "sha512-+3NpwQEnRoIBtx4fyhblQDPgJI0H1IEIkX7ShLUjPGA7TtUTvI1oiKi3SR4oBR0hQhQR80l4WAe5RrXBwWMA8w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + }, + "peerDependencies": { + "jest-resolve": "*" + }, + "peerDependenciesMeta": { + "jest-resolve": { + "optional": true + } + } + }, + "node_modules/jest-regex-util": { + "version": "29.6.3", + "resolved": "https://registry.npmjs.org/jest-regex-util/-/jest-regex-util-29.6.3.tgz", + "integrity": "sha512-KJJBsRCyyLNWCNBOvZyRDnAIfUiRJ8v+hOBQYGn8gDyF3UegwiP4gwRR3/SDa42g1YbVycTidUF3rKjyLFDWbg==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-resolve": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-resolve/-/jest-resolve-29.7.0.tgz", + "integrity": "sha512-IOVhZSrg+UvVAshDSDtHyFCCBUl/Q3AAJv8iZ6ZjnZ74xzvwuzLXid9IIIPgTnY62SJjfuupMKZsZQRsCvxEgA==", + "dev": true, + "license": "MIT", + "dependencies": { + "chalk": "^4.0.0", + "graceful-fs": "^4.2.9", + "jest-haste-map": "^29.7.0", + "jest-pnp-resolver": "^1.2.2", + "jest-util": "^29.7.0", + "jest-validate": "^29.7.0", + "resolve": "^1.20.0", + "resolve.exports": "^2.0.0", + "slash": "^3.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-resolve-dependencies": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-resolve-dependencies/-/jest-resolve-dependencies-29.7.0.tgz", + "integrity": "sha512-un0zD/6qxJ+S0et7WxeI3H5XSe9lTBBR7bOHCHXkKR6luG5mwDDlIzVQ0V5cZCuoTgEdcdwzTghYkTWfubi+nA==", + "dev": true, + "license": "MIT", + "dependencies": { + "jest-regex-util": "^29.6.3", + "jest-snapshot": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-runner": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-runner/-/jest-runner-29.7.0.tgz", + "integrity": "sha512-fsc4N6cPCAahybGBfTRcq5wFR6fpLznMg47sY5aDpsoejOcVYFb07AHuSnR0liMcPTgBsA3ZJL6kFOjPdoNipQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/console": "^29.7.0", + "@jest/environment": "^29.7.0", + "@jest/test-result": "^29.7.0", + "@jest/transform": "^29.7.0", + "@jest/types": "^29.6.3", + "@types/node": "*", + "chalk": "^4.0.0", + "emittery": "^0.13.1", + "graceful-fs": "^4.2.9", + "jest-docblock": "^29.7.0", + "jest-environment-node": "^29.7.0", + "jest-haste-map": "^29.7.0", + "jest-leak-detector": "^29.7.0", + "jest-message-util": "^29.7.0", + "jest-resolve": "^29.7.0", + "jest-runtime": "^29.7.0", + "jest-util": "^29.7.0", + "jest-watcher": "^29.7.0", + "jest-worker": "^29.7.0", + "p-limit": "^3.1.0", + "source-map-support": "0.5.13" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-runtime": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-runtime/-/jest-runtime-29.7.0.tgz", + "integrity": "sha512-gUnLjgwdGqW7B4LvOIkbKs9WGbn+QLqRQQ9juC6HndeDiezIwhDP+mhMwHWCEcfQ5RUXa6OPnFF8BJh5xegwwQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/environment": "^29.7.0", + "@jest/fake-timers": "^29.7.0", + "@jest/globals": "^29.7.0", + "@jest/source-map": "^29.6.3", + "@jest/test-result": "^29.7.0", + "@jest/transform": "^29.7.0", + "@jest/types": "^29.6.3", + "@types/node": "*", + "chalk": "^4.0.0", + "cjs-module-lexer": "^1.0.0", + "collect-v8-coverage": "^1.0.0", + "glob": "^7.1.3", + "graceful-fs": "^4.2.9", + "jest-haste-map": "^29.7.0", + "jest-message-util": "^29.7.0", + "jest-mock": "^29.7.0", + "jest-regex-util": "^29.6.3", + "jest-resolve": "^29.7.0", + "jest-snapshot": "^29.7.0", + "jest-util": "^29.7.0", + "slash": "^3.0.0", + "strip-bom": "^4.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-snapshot": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-snapshot/-/jest-snapshot-29.7.0.tgz", + "integrity": "sha512-Rm0BMWtxBcioHr1/OX5YCP8Uov4riHvKPknOGs804Zg9JGZgmIBkbtlxJC/7Z4msKYVbIJtfU+tKb8xlYNfdkw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/core": "^7.11.6", + "@babel/generator": "^7.7.2", + "@babel/plugin-syntax-jsx": "^7.7.2", + "@babel/plugin-syntax-typescript": "^7.7.2", + "@babel/types": "^7.3.3", + "@jest/expect-utils": "^29.7.0", + "@jest/transform": "^29.7.0", + "@jest/types": "^29.6.3", + "babel-preset-current-node-syntax": "^1.0.0", + "chalk": "^4.0.0", + "expect": "^29.7.0", + "graceful-fs": "^4.2.9", + "jest-diff": "^29.7.0", + "jest-get-type": "^29.6.3", + "jest-matcher-utils": "^29.7.0", + "jest-message-util": "^29.7.0", + "jest-util": "^29.7.0", + "natural-compare": "^1.4.0", + "pretty-format": "^29.7.0", + "semver": "^7.5.3" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-snapshot/node_modules/semver": { + "version": "7.7.3", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.3.tgz", + "integrity": "sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/jest-util": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-29.7.0.tgz", + "integrity": "sha512-z6EbKajIpqGKU56y5KBUgy1dt1ihhQJgWzUlZHArA/+X2ad7Cb5iF+AK1EWVL/Bo7Rz9uurpqw6SiBCefUbCGA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/types": "^29.6.3", + "@types/node": "*", + "chalk": "^4.0.0", + "ci-info": "^3.2.0", + "graceful-fs": "^4.2.9", + "picomatch": "^2.2.3" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-validate": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-validate/-/jest-validate-29.7.0.tgz", + "integrity": "sha512-ZB7wHqaRGVw/9hST/OuFUReG7M8vKeq0/J2egIGLdvjHCmYqGARhzXmtgi+gVeZ5uXFF219aOc3Ls2yLg27tkw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/types": "^29.6.3", + "camelcase": "^6.2.0", + "chalk": "^4.0.0", + "jest-get-type": "^29.6.3", + "leven": "^3.1.0", + "pretty-format": "^29.7.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-validate/node_modules/camelcase": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-6.3.0.tgz", + "integrity": "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/jest-watcher": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-watcher/-/jest-watcher-29.7.0.tgz", + "integrity": "sha512-49Fg7WXkU3Vl2h6LbLtMQ/HyB6rXSIX7SqvBLQmssRBGN9I0PNvPmAmCWSOY6SOvrjhI/F7/bGAv9RtnsPA03g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/test-result": "^29.7.0", + "@jest/types": "^29.6.3", + "@types/node": "*", + "ansi-escapes": "^4.2.1", + "chalk": "^4.0.0", + "emittery": "^0.13.1", + "jest-util": "^29.7.0", + "string-length": "^4.0.1" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-worker": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/jest-worker/-/jest-worker-29.7.0.tgz", + "integrity": "sha512-eIz2msL/EzL9UFTFFx7jBTkeZfku0yUAyZZZmJ93H2TYEiroIx2PQjEXcwYtYl8zXCxb+PAmA2hLIt/6ZEkPHw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*", + "jest-util": "^29.7.0", + "merge-stream": "^2.0.0", + "supports-color": "^8.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/jest-worker/node_modules/supports-color": { + "version": "8.1.1", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz", + "integrity": "sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/supports-color?sponsor=1" + } + }, + "node_modules/js-tokens": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", + "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/js-yaml": { + "version": "3.14.2", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.2.tgz", + "integrity": "sha512-PMSmkqxr106Xa156c2M265Z+FTrPl+oxd/rgOQy2tijQeK5TxQ43psO1ZCwhVOSdnn+RzkzlRz/eY4BgJBYVpg==", + "dev": true, + "license": "MIT", + "dependencies": { + "argparse": "^1.0.7", + "esprima": "^4.0.0" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, + "node_modules/jsesc": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-3.1.0.tgz", + "integrity": "sha512-/sM3dO2FOzXjKQhJuo0Q173wf2KOo8t4I8vHy6lF9poUp7bKT0/NHE8fPX23PwfhnykfqnC2xRxOnVw5XuGIaA==", + "dev": true, + "license": "MIT", + "bin": { + "jsesc": "bin/jsesc" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/json-parse-even-better-errors": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz", + "integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==", + "dev": true, + "license": "MIT" + }, + "node_modules/json5": { + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz", + "integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==", + "dev": true, + "license": "MIT", + "bin": { + "json5": "lib/cli.js" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/kleur": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/kleur/-/kleur-3.0.3.tgz", + "integrity": "sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/leven": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/leven/-/leven-3.1.0.tgz", + "integrity": "sha512-qsda+H8jTaUaN/x5vzW2rzc+8Rw4TAQ/4KjB46IwK5VH+IlVeeeje/EoZRpiXvIqjFgK84QffqPztGI3VBLG1A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/lines-and-columns": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz", + "integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==", + "dev": true, + "license": "MIT" + }, + "node_modules/locate-path": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", + "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-locate": "^4.1.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/lodash.memoize": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/lodash.memoize/-/lodash.memoize-4.1.2.tgz", + "integrity": "sha512-t7j+NzmgnQzTAYXcsHYLgimltOV1MXHtlOWf6GjL9Kj8GK5FInw5JotxvbOs+IvV1/Dzo04/fCGfLVs7aXb4Ag==", + "dev": true, + "license": "MIT" + }, + "node_modules/lru-cache": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", + "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==", + "dev": true, + "license": "ISC", + "dependencies": { + "yallist": "^3.0.2" + } + }, + "node_modules/make-dir": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-4.0.0.tgz", + "integrity": "sha512-hXdUTZYIVOt1Ex//jAQi+wTZZpUpwBj/0QsOzqegb3rGMMeJiSEu5xLHnYfBrRV4RH2+OCSOO95Is/7x1WJ4bw==", + "dev": true, + "license": "MIT", + "dependencies": { + "semver": "^7.5.3" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/make-dir/node_modules/semver": { + "version": "7.7.3", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.3.tgz", + "integrity": "sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/make-error": { + "version": "1.3.6", + "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz", + "integrity": "sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==", + "dev": true, + "license": "ISC" + }, + "node_modules/makeerror": { + "version": "1.0.12", + "resolved": "https://registry.npmjs.org/makeerror/-/makeerror-1.0.12.tgz", + "integrity": "sha512-JmqCvUhmt43madlpFzG4BQzG2Z3m6tvQDNKdClZnO3VbIudJYmxsT0FNJMeiB2+JTSlTQTSbU8QdesVmwJcmLg==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "tmpl": "1.0.5" + } + }, + "node_modules/merge-stream": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz", + "integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==", + "dev": true, + "license": "MIT" + }, + "node_modules/micromatch": { + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz", + "integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==", + "dev": true, + "license": "MIT", + "dependencies": { + "braces": "^3.0.3", + "picomatch": "^2.3.1" + }, + "engines": { + "node": ">=8.6" + } + }, + "node_modules/mimic-fn": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz", + "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "license": "ISC", + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/minimist": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true, + "license": "MIT" + }, + "node_modules/natural-compare": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", + "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==", + "dev": true, + "license": "MIT" + }, + "node_modules/neo-async": { + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/neo-async/-/neo-async-2.6.2.tgz", + "integrity": "sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==", + "dev": true, + "license": "MIT" + }, + "node_modules/node-int64": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz", + "integrity": "sha512-O5lz91xSOeoXP6DulyHfllpq+Eg00MWitZIbtPfoSEvqIHdl5gfcY6hYzDWnj0qD5tz52PI08u9qUvSVeUBeHw==", + "dev": true, + "license": "MIT" + }, + "node_modules/node-releases": { + "version": "2.0.27", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.27.tgz", + "integrity": "sha512-nmh3lCkYZ3grZvqcCH+fjmQ7X+H0OeZgP40OierEaAptX4XofMh5kwNbWh7lBduUzCcV/8kZ+NDLCwm2iorIlA==", + "dev": true, + "license": "MIT" + }, + "node_modules/normalize-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", + "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/npm-run-path": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-4.0.1.tgz", + "integrity": "sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw==", + "dev": true, + "license": "MIT", + "dependencies": { + "path-key": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "dev": true, + "license": "ISC", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/onetime": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/onetime/-/onetime-5.1.2.tgz", + "integrity": "sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==", + "dev": true, + "license": "MIT", + "dependencies": { + "mimic-fn": "^2.1.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-limit": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "yocto-queue": "^0.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-locate": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", + "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-limit": "^2.2.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/p-locate/node_modules/p-limit": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", + "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", + "dev": true, + "license": "MIT", + "dependencies": { + "p-try": "^2.0.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-try": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", + "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/parse-json": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-5.2.0.tgz", + "integrity": "sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.0.0", + "error-ex": "^1.3.1", + "json-parse-even-better-errors": "^2.3.0", + "lines-and-columns": "^1.1.6" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-is-absolute": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", + "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/path-parse": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", + "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", + "dev": true, + "license": "MIT" + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "dev": true, + "license": "ISC" + }, + "node_modules/picomatch": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", + "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8.6" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/pirates": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/pirates/-/pirates-4.0.7.tgz", + "integrity": "sha512-TfySrs/5nm8fQJDcBDuUng3VOUKsd7S+zqvbOTiGXHfxX4wK31ard+hoNuvkicM/2YFzlpDgABOevKSsB4G/FA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 6" + } + }, + "node_modules/pkg-dir": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-4.2.0.tgz", + "integrity": "sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "find-up": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/pretty-format": { + "version": "29.7.0", + "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-29.7.0.tgz", + "integrity": "sha512-Pdlw/oPxN+aXdmM9R00JVC9WVFoCLTKJvDVLgmJ+qAffBMxsV85l/Lu7sNx4zSzPyoL2euImuEwHhOXdEgNFZQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jest/schemas": "^29.6.3", + "ansi-styles": "^5.0.0", + "react-is": "^18.0.0" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || >=18.0.0" + } + }, + "node_modules/pretty-format/node_modules/ansi-styles": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", + "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/prompts": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/prompts/-/prompts-2.4.2.tgz", + "integrity": "sha512-NxNv/kLguCA7p3jE8oL2aEBsrJWgAakBpgmgK6lpPWV+WuOmY6r2/zbAVnP+T8bQlA0nzHXSJSJW0Hq7ylaD2Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "kleur": "^3.0.3", + "sisteransi": "^1.0.5" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/pure-rand": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/pure-rand/-/pure-rand-6.1.0.tgz", + "integrity": "sha512-bVWawvoZoBYpp6yIoQtQXHZjmz35RSVHnUOTefl8Vcjr8snTPY1wnpSPMWekcFwbxI6gtmT7rSYPFvz71ldiOA==", + "dev": true, + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/dubzzz" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fast-check" + } + ], + "license": "MIT" + }, + "node_modules/react-is": { + "version": "18.3.1", + "resolved": "https://registry.npmjs.org/react-is/-/react-is-18.3.1.tgz", + "integrity": "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg==", + "dev": true, + "license": "MIT" + }, + "node_modules/require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/resolve": { + "version": "1.22.11", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.11.tgz", + "integrity": "sha512-RfqAvLnMl313r7c9oclB1HhUEAezcpLjz95wFH4LVuhk9JF/r22qmVP9AMmOU4vMX7Q8pN8jwNg/CSpdFnMjTQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-core-module": "^2.16.1", + "path-parse": "^1.0.7", + "supports-preserve-symlinks-flag": "^1.0.0" + }, + "bin": { + "resolve": "bin/resolve" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/resolve-cwd": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/resolve-cwd/-/resolve-cwd-3.0.0.tgz", + "integrity": "sha512-OrZaX2Mb+rJCpH/6CpSqt9xFVpN++x01XnN2ie9g6P5/3xelLAkXWVADpdz1IHD/KFfEXyE6V0U01OQ3UO2rEg==", + "dev": true, + "license": "MIT", + "dependencies": { + "resolve-from": "^5.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/resolve-from": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-5.0.0.tgz", + "integrity": "sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/resolve.exports": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/resolve.exports/-/resolve.exports-2.0.3.tgz", + "integrity": "sha512-OcXjMsGdhL4XnbShKpAcSqPMzQoYkYyhbEaeSko47MjRP9NfEQMhZkXL1DoFlt9LWQn4YttrdnV6X2OiyzBi+A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + } + }, + "node_modules/semver": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dev": true, + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/signal-exit": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", + "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/sisteransi": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/sisteransi/-/sisteransi-1.0.5.tgz", + "integrity": "sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg==", + "dev": true, + "license": "MIT" + }, + "node_modules/slash": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz", + "integrity": "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/source-map-support": { + "version": "0.5.13", + "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.13.tgz", + "integrity": "sha512-SHSKFHadjVA5oR4PPqhtAVdcBWwRYVd6g6cAXnIbRiIwc2EhPrTuKUBdSLvlEKyIP3GCf89fltvcZiP9MMFA1w==", + "dev": true, + "license": "MIT", + "dependencies": { + "buffer-from": "^1.0.0", + "source-map": "^0.6.0" + } + }, + "node_modules/sprintf-js": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", + "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==", + "dev": true, + "license": "BSD-3-Clause" + }, + "node_modules/stack-utils": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/stack-utils/-/stack-utils-2.0.6.tgz", + "integrity": "sha512-XlkWvfIm6RmsWtNJx+uqtKLS8eqFbxUg0ZzLXqY0caEy9l7hruX8IpiDnjsLavoBgqCCR71TqWO8MaXYheJ3RQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "escape-string-regexp": "^2.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/string-length": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/string-length/-/string-length-4.0.2.tgz", + "integrity": "sha512-+l6rNN5fYHNhZZy41RXsYptCjA2Igmq4EG7kZAYFQI1E1VTXarr6ZPXBg6eq7Y6eK4FEhY6AJlyuFIb/v/S0VQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "char-regex": "^1.0.2", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-bom": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-4.0.0.tgz", + "integrity": "sha512-3xurFv5tEgii33Zi8Jtp55wEIILR9eh34FAW00PZf+JnSsTmV/ioewSgQl97JHvgjoRGwPShsWm+IdrxB35d0w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-final-newline": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/strip-final-newline/-/strip-final-newline-2.0.0.tgz", + "integrity": "sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/strip-json-comments": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", + "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "license": "MIT", + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/supports-preserve-symlinks-flag": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz", + "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/test-exclude": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/test-exclude/-/test-exclude-6.0.0.tgz", + "integrity": "sha512-cAGWPIyOHU6zlmg88jwm7VRyXnMN7iV68OGAbYDk/Mh/xC/pzVPlQtY6ngoIH/5/tciuhGfvESU8GrHrcxD56w==", + "dev": true, + "license": "ISC", + "dependencies": { + "@istanbuljs/schema": "^0.1.2", + "glob": "^7.1.4", + "minimatch": "^3.0.4" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/tmpl": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/tmpl/-/tmpl-1.0.5.tgz", + "integrity": "sha512-3f0uOEAQwIqGuWW2MVzYg8fV/QNnc/IpuJNG837rLuczAaLVHslWHZQj4IGiEl5Hs3kkbhwL9Ab7Hrsmuj+Smw==", + "dev": true, + "license": "BSD-3-Clause" + }, + "node_modules/to-regex-range": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", + "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-number": "^7.0.0" + }, + "engines": { + "node": ">=8.0" + } + }, + "node_modules/ts-jest": { + "version": "29.4.5", + "resolved": "https://registry.npmjs.org/ts-jest/-/ts-jest-29.4.5.tgz", + "integrity": "sha512-HO3GyiWn2qvTQA4kTgjDcXiMwYQt68a1Y8+JuLRVpdIzm+UOLSHgl/XqR4c6nzJkq5rOkjc02O2I7P7l/Yof0Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "bs-logger": "^0.2.6", + "fast-json-stable-stringify": "^2.1.0", + "handlebars": "^4.7.8", + "json5": "^2.2.3", + "lodash.memoize": "^4.1.2", + "make-error": "^1.3.6", + "semver": "^7.7.3", + "type-fest": "^4.41.0", + "yargs-parser": "^21.1.1" + }, + "bin": { + "ts-jest": "cli.js" + }, + "engines": { + "node": "^14.15.0 || ^16.10.0 || ^18.0.0 || >=20.0.0" + }, + "peerDependencies": { + "@babel/core": ">=7.0.0-beta.0 <8", + "@jest/transform": "^29.0.0 || ^30.0.0", + "@jest/types": "^29.0.0 || ^30.0.0", + "babel-jest": "^29.0.0 || ^30.0.0", + "jest": "^29.0.0 || ^30.0.0", + "jest-util": "^29.0.0 || ^30.0.0", + "typescript": ">=4.3 <6" + }, + "peerDependenciesMeta": { + "@babel/core": { + "optional": true + }, + "@jest/transform": { + "optional": true + }, + "@jest/types": { + "optional": true + }, + "babel-jest": { + "optional": true + }, + "esbuild": { + "optional": true + }, + "jest-util": { + "optional": true + } + } + }, + "node_modules/ts-jest/node_modules/semver": { + "version": "7.7.3", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.3.tgz", + "integrity": "sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/ts-jest/node_modules/type-fest": { + "version": "4.41.0", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-4.41.0.tgz", + "integrity": "sha512-TeTSQ6H5YHvpqVwBRcnLDCBnDOHWYu7IvGbHT6N8AOymcr9PJGjc1GTtiWZTYg0NCgYwvnYWEkVChQAr9bjfwA==", + "dev": true, + "license": "(MIT OR CC0-1.0)", + "engines": { + "node": ">=16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/ts-node": { + "version": "10.9.2", + "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.9.2.tgz", + "integrity": "sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@cspotcode/source-map-support": "^0.8.0", + "@tsconfig/node10": "^1.0.7", + "@tsconfig/node12": "^1.0.7", + "@tsconfig/node14": "^1.0.0", + "@tsconfig/node16": "^1.0.2", + "acorn": "^8.4.1", + "acorn-walk": "^8.1.1", + "arg": "^4.1.0", + "create-require": "^1.1.0", + "diff": "^4.0.1", + "make-error": "^1.1.1", + "v8-compile-cache-lib": "^3.0.1", + "yn": "3.1.1" + }, + "bin": { + "ts-node": "dist/bin.js", + "ts-node-cwd": "dist/bin-cwd.js", + "ts-node-esm": "dist/bin-esm.js", + "ts-node-script": "dist/bin-script.js", + "ts-node-transpile-only": "dist/bin-transpile.js", + "ts-script": "dist/bin-script-deprecated.js" + }, + "peerDependencies": { + "@swc/core": ">=1.2.50", + "@swc/wasm": ">=1.2.50", + "@types/node": "*", + "typescript": ">=2.7" + }, + "peerDependenciesMeta": { + "@swc/core": { + "optional": true + }, + "@swc/wasm": { + "optional": true + } + } + }, + "node_modules/type-detect": { + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.0.8.tgz", + "integrity": "sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/type-fest": { + "version": "0.21.3", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.21.3.tgz", + "integrity": "sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w==", + "dev": true, + "license": "(MIT OR CC0-1.0)", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/typescript": { + "version": "5.6.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.6.3.tgz", + "integrity": "sha512-hjcS1mhfuyi4WW8IWtjP7brDrG2cuDZukyrYrSauoXGNgx0S7zceP07adYkJycEr56BOUTNPzbInooiN3fn1qw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/uglify-js": { + "version": "3.19.3", + "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.19.3.tgz", + "integrity": "sha512-v3Xu+yuwBXisp6QYTcH4UbH+xYJXqnq2m/LtQVWKWzYc1iehYnLixoQDN9FH6/j9/oybfd6W9Ghwkl8+UMKTKQ==", + "dev": true, + "license": "BSD-2-Clause", + "optional": true, + "bin": { + "uglifyjs": "bin/uglifyjs" + }, + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/undici-types": { + "version": "6.19.8", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.19.8.tgz", + "integrity": "sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==", + "dev": true, + "license": "MIT" + }, + "node_modules/update-browserslist-db": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.1.4.tgz", + "integrity": "sha512-q0SPT4xyU84saUX+tomz1WLkxUbuaJnR1xWt17M7fJtEJigJeWUNGUqrauFXsHnqev9y9JTRGwk13tFBuKby4A==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "escalade": "^3.2.0", + "picocolors": "^1.1.1" + }, + "bin": { + "update-browserslist-db": "cli.js" + }, + "peerDependencies": { + "browserslist": ">= 4.21.0" + } + }, + "node_modules/v8-compile-cache-lib": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz", + "integrity": "sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==", + "dev": true, + "license": "MIT" + }, + "node_modules/v8-to-istanbul": { + "version": "9.3.0", + "resolved": "https://registry.npmjs.org/v8-to-istanbul/-/v8-to-istanbul-9.3.0.tgz", + "integrity": "sha512-kiGUalWN+rgBJ/1OHZsBtU4rXZOfj/7rKQxULKlIzwzQSvMJUUNgPwJEEh7gU6xEVxC0ahoOBvN2YI8GH6FNgA==", + "dev": true, + "license": "ISC", + "dependencies": { + "@jridgewell/trace-mapping": "^0.3.12", + "@types/istanbul-lib-coverage": "^2.0.1", + "convert-source-map": "^2.0.0" + }, + "engines": { + "node": ">=10.12.0" + } + }, + "node_modules/walker": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/walker/-/walker-1.0.8.tgz", + "integrity": "sha512-ts/8E8l5b7kY0vlWLewOkDXMmPdLcVV4GmOQLyxuSswIJsweeFZtAsMF7k1Nszz+TYBQrlYRmzOnr398y1JemQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "makeerror": "1.0.12" + } + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dev": true, + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/wordwrap": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-1.0.0.tgz", + "integrity": "sha512-gvVzJFlPycKc5dZN4yPkP8w7Dc37BtP1yczEneOb4uq34pXZcvrtRTmWV8W+Ume+XCxKgbjM+nevkyFPMybd4Q==", + "dev": true, + "license": "MIT" + }, + "node_modules/wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "dev": true, + "license": "ISC" + }, + "node_modules/write-file-atomic": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-4.0.2.tgz", + "integrity": "sha512-7KxauUdBmSdWnmpaGFg+ppNjKF8uNLry8LyzjauQDOVONfFLNKrKvQOxZ/VuTIcS/gge/YNahf5RIIQWTSarlg==", + "dev": true, + "license": "ISC", + "dependencies": { + "imurmurhash": "^0.1.4", + "signal-exit": "^3.0.7" + }, + "engines": { + "node": "^12.13.0 || ^14.15.0 || >=16.0.0" + } + }, + "node_modules/y18n": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=10" + } + }, + "node_modules/yallist": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", + "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==", + "dev": true, + "license": "ISC" + }, + "node_modules/yargs": { + "version": "17.7.2", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", + "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", + "dev": true, + "license": "MIT", + "dependencies": { + "cliui": "^8.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.3", + "y18n": "^5.0.5", + "yargs-parser": "^21.1.1" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/yargs-parser": { + "version": "21.1.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", + "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", + "dev": true, + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/yn": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz", + "integrity": "sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/yocto-queue": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", + "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/zod": { + "version": "4.1.12", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.1.12.tgz", + "integrity": "sha512-JInaHOamG8pt5+Ey8kGmdcAcg3OL9reK8ltczgHTAwNhMys/6ThXHityHxVV2p3fkw/c+MAvBHFVYHFZDmjMCQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + } + } +} diff --git a/benchmarks/cdk/package.json b/benchmarks/cdk/package.json new file mode 100644 index 00000000..62b45910 --- /dev/null +++ b/benchmarks/cdk/package.json @@ -0,0 +1,36 @@ +{ + "name": "cdk", + "version": "0.1.0", + "bin": { + "cdk": "bin/cdk.js" + }, + "scripts": { + "build": "tsc", + "watch": "tsc -w", + "test": "jest", + "cdk": "cdk", + "sync-bucket": "aws s3 sync ../data s3://datafusion-distributed-benchmarks/", + "compare": "npx ts-node bin/compare.ts", + "datafusion-bench": "npx ts-node bin/datafusion-bench.ts", + "trino-bench": "npx ts-node bin/trino-bench.ts", + "spark-bench": "npx ts-node bin/spark-bench.ts", + "ballista-bench": "npx ts-node bin/ballista-bench.ts", + "fast-deploy": "npx ts-node bin/fast-deploy.ts", + "send-command": "npx ts-node bin/send-command.ts" + }, + "devDependencies": { + "@types/jest": "^29.5.14", + "@types/node": "22.7.9", + "aws-cdk": "2.1031.2", + "jest": "^29.7.0", + "ts-jest": "^29.2.5", + "ts-node": "^10.9.2", + "typescript": "~5.6.3" + }, + "dependencies": { + "aws-cdk-lib": "2.215.0", + "constructs": "^10.0.0", + "commander": "^14.0.2", + "zod": "^4.1.12" + } +} diff --git a/benchmarks/cdk/requirements.txt b/benchmarks/cdk/requirements.txt new file mode 100644 index 00000000..1ee67330 --- /dev/null +++ b/benchmarks/cdk/requirements.txt @@ -0,0 +1,2 @@ +flask==3.0.0 +pyspark==4.0.0 diff --git a/benchmarks/cdk/test/cdk.test.ts b/benchmarks/cdk/test/cdk.test.ts new file mode 100644 index 00000000..aad8e382 --- /dev/null +++ b/benchmarks/cdk/test/cdk.test.ts @@ -0,0 +1,15 @@ +import * as cdk from 'aws-cdk-lib/core'; +import * as Cdk from '../lib/cdk-stack'; + +const config = { + instanceType: 't3.xlarge', + instanceCount: 4, +}; + +test('it builds', () => { + const app = new cdk.App(); + // WHEN + new Cdk.CdkStack(app, 'MyTestStack', { + config + }); +}); diff --git a/benchmarks/cdk/tsconfig.json b/benchmarks/cdk/tsconfig.json new file mode 100644 index 00000000..bfc61bf8 --- /dev/null +++ b/benchmarks/cdk/tsconfig.json @@ -0,0 +1,32 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "lib": [ + "es2022" + ], + "declaration": true, + "strict": true, + "noImplicitAny": true, + "strictNullChecks": true, + "noImplicitThis": true, + "alwaysStrict": true, + "noUnusedLocals": false, + "noUnusedParameters": false, + "noImplicitReturns": true, + "noFallthroughCasesInSwitch": false, + "inlineSourceMap": true, + "inlineSources": true, + "experimentalDecorators": true, + "strictPropertyInitialization": false, + "skipLibCheck": true, + "typeRoots": [ + "./node_modules/@types" + ] + }, + "exclude": [ + "node_modules", + "cdk.out" + ] +} diff --git a/benchmarks/gen-clickbench.sh b/benchmarks/gen-clickbench.sh new file mode 100755 index 00000000..c0b6d0af --- /dev/null +++ b/benchmarks/gen-clickbench.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +set -e + +PARTITION_START=${PARTITION_START:-0} +PARTITION_END=${PARTITION_END:-100} + +echo "Generating ClickBench dataset" + + +# https://stackoverflow.com/questions/59895/how-do-i-get-the-directory-where-a-bash-script-is-located-from-within-the-script +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +DATA_DIR=${DATA_DIR:-$SCRIPT_DIR/data} +CARGO_COMMAND=${CARGO_COMMAND:-"cargo run --release"} +CLICKBENCH_DIR="${DATA_DIR}/clickbench_${PARTITION_START}-${PARTITION_END}" + +echo "Creating clickbench dataset from partition ${PARTITION_START} to ${PARTITION_END}" + +# Ensure the target data directory exists +mkdir -p "${CLICKBENCH_DIR}" + +$CARGO_COMMAND -- prepare-clickbench --output "${CLICKBENCH_DIR}" --partition-start "$PARTITION_START" --partition-end "$PARTITION_END" diff --git a/benchmarks/gen-tpcds.sh b/benchmarks/gen-tpcds.sh new file mode 100755 index 00000000..7c8138a3 --- /dev/null +++ b/benchmarks/gen-tpcds.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +set -e + +SCALE_FACTOR=${SCALE_FACTOR:-1} +PARTITIONS=${PARTITIONS:-16} + +echo "Generating TPC-DS dataset with SCALE_FACTOR=${SCALE_FACTOR} and PARTITIONS=${PARTITIONS}" + +# https://stackoverflow.com/questions/59895/how-do-i-get-the-directory-where-a-bash-script-is-located-from-within-the-script +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +DATA_DIR=${DATA_DIR:-$SCRIPT_DIR/data} +CARGO_COMMAND=${CARGO_COMMAND:-"cargo run -p datafusion-distributed-benchmarks --release"} +TPCDS_DIR="${DATA_DIR}/tpcds_sf${SCALE_FACTOR}" + +echo "Creating tpcds dataset at Scale Factor ${SCALE_FACTOR} in ${TPCDS_DIR}..." + +# Ensure the target data directory exists +mkdir -p "${TPCDS_DIR}" + +$CARGO_COMMAND -- prepare-tpcds --output "${TPCDS_DIR}" --partitions "$PARTITIONS" diff --git a/benchmarks/gen-tpch.sh b/benchmarks/gen-tpch.sh index 98ec9c88..4a26f8db 100755 --- a/benchmarks/gen-tpch.sh +++ b/benchmarks/gen-tpch.sh @@ -2,18 +2,15 @@ set -e -SCALE_FACTOR=1 +SCALE_FACTOR=${SCALE_FACTOR:-1} +PARTITIONS=${PARTITIONS:-16} + +echo "Generating TPCH dataset with SCALE_FACTOR=${SCALE_FACTOR} and PARTITIONS=${PARTITIONS}" # https://stackoverflow.com/questions/59895/how-do-i-get-the-directory-where-a-bash-script-is-located-from-within-the-script SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) DATA_DIR=${DATA_DIR:-$SCRIPT_DIR/data} CARGO_COMMAND=${CARGO_COMMAND:-"cargo run --release"} - -if [ -z "$SCALE_FACTOR" ] ; then - echo "Internal error: Scale factor not specified" - exit 1 -fi - TPCH_DIR="${DATA_DIR}/tpch_sf${SCALE_FACTOR}" echo "Creating tpch dataset at Scale Factor ${SCALE_FACTOR} in ${TPCH_DIR}..." @@ -29,16 +26,6 @@ else docker run -v "${TPCH_DIR}":/data -it --rm ghcr.io/scalytics/tpch-docker:main -vf -s "${SCALE_FACTOR}" fi -# Copy expected answers into the ./data/answers directory if it does not already exist -FILE="${TPCH_DIR}/answers/q1.out" -if test -f "${FILE}"; then - echo " Expected answers exist (${FILE} exists)." -else - echo " Copying answers to ${TPCH_DIR}/answers" - mkdir -p "${TPCH_DIR}/answers" - docker run -v "${TPCH_DIR}":/data -it --entrypoint /bin/bash --rm ghcr.io/scalytics/tpch-docker:main -c "cp -f /opt/tpch/2.18.0_rc2/dbgen/answers/* /data/answers/" -fi - # Create 'parquet' files from tbl FILE="${TPCH_DIR}/supplier" if test -d "${FILE}"; then @@ -46,18 +33,6 @@ if test -d "${FILE}"; then else echo " creating parquet files using benchmark binary ..." pushd "${SCRIPT_DIR}" > /dev/null - $CARGO_COMMAND -- tpch-convert --input "${TPCH_DIR}" --output "${TPCH_DIR}" --format parquet + $CARGO_COMMAND -- prepare-tpch --input "${TPCH_DIR}" --output "${TPCH_DIR}" --partitions "$PARTITIONS" popd > /dev/null fi - -# Create 'csv' files from tbl -FILE="${TPCH_DIR}/csv/supplier" -if test -d "${FILE}"; then - echo " csv files exist ($FILE exists)." -else - echo " creating csv files using benchmark binary ..." - pushd "${SCRIPT_DIR}" > /dev/null - $CARGO_COMMAND -- tpch-convert --input "${TPCH_DIR}" --output "${TPCH_DIR}/csv" --format csv - popd > /dev/null -fi - diff --git a/benchmarks/run.sh b/benchmarks/run.sh index 7cb0151a..e97f69b2 100755 --- a/benchmarks/run.sh +++ b/benchmarks/run.sh @@ -8,7 +8,7 @@ WORKERS=${WORKERS:-8} SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) if [ "$WORKERS" == "0" ]; then - cargo run -p datafusion-distributed-benchmarks --release -- tpch "$@" + cargo run -p datafusion-distributed-benchmarks --release -- run "$@" exit fi @@ -38,7 +38,7 @@ cargo build -p datafusion-distributed-benchmarks --release trap cleanup EXIT INT TERM for i in $(seq 0 $((WORKERS-1))); do - "$SCRIPT_DIR"/../target/release/dfbench tpch --spawn $((8000+i)) "$@" & + "$SCRIPT_DIR"/../target/release/dfbench run --spawn $((8000+i)) "$@" & done echo "Waiting for worker ports to be ready..." @@ -46,4 +46,4 @@ for i in $(seq 0 $((WORKERS-1))); do wait_for_port $((8000+i)) done -"$SCRIPT_DIR"/../target/release/dfbench tpch --workers $(seq -s, 8000 $((8000+WORKERS-1))) "$@" +"$SCRIPT_DIR"/../target/release/dfbench run --workers $(seq -s, 8000 $((8000+WORKERS-1))) "$@" diff --git a/benchmarks/src/bin/dfbench.rs b/benchmarks/src/bin/dfbench.rs deleted file mode 100644 index d626f79e..00000000 --- a/benchmarks/src/bin/dfbench.rs +++ /dev/null @@ -1,43 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! DataFusion Distributed benchmark runner -use datafusion::error::Result; - -use structopt::StructOpt; - -use datafusion_distributed_benchmarks::tpch; - -#[derive(Debug, StructOpt)] -#[structopt(about = "benchmark command")] -enum Options { - Tpch(tpch::RunOpt), - TpchConvert(tpch::ConvertOpt), -} - -// Main benchmark runner entrypoint -pub fn main() -> Result<()> { - env_logger::init(); - - match Options::from_args() { - Options::Tpch(opt) => opt.run(), - Options::TpchConvert(opt) => { - let rt = tokio::runtime::Runtime::new()?; - rt.block_on(async { opt.run().await }) - } - } -} diff --git a/benchmarks/src/compare.rs b/benchmarks/src/compare.rs new file mode 100644 index 00000000..dfda32e0 --- /dev/null +++ b/benchmarks/src/compare.rs @@ -0,0 +1,40 @@ +use crate::results::BenchResult; +use datafusion::common::{Result, internal_err}; +use structopt::StructOpt; + +/// Compare different runs of the tpch benchmarks. +#[derive(Debug, StructOpt, Clone)] +#[structopt(verbatim_doc_comment)] +pub struct CompareOpt { + /// Query number. If not specified, runs all queries + #[structopt(name = "BRANCHES", required = true)] + pub branches: Vec, + + /// Path to data files + #[structopt(long)] + dataset: String, +} + +impl CompareOpt { + pub fn run(&self) -> Result<()> { + let (base, new) = match self.branches.as_slice() { + [one, two] => (one, two), + rest => { + return internal_err!("Exactly two branches must be specified, got: {rest:?}"); + } + }; + println!( + "=== Comparing {} results from branch '{}' [prev] with '{}' [new] ===", + self.dataset, base, new + ); + let base = BenchResult::load_many(&self.dataset, base); + let new = BenchResult::load_many(&self.dataset, new); + for query in new { + let Some(prev) = base.iter().find(|v| v.id == query.id) else { + continue; + }; + query.compare(prev) + } + Ok(()) + } +} diff --git a/benchmarks/src/lib.rs b/benchmarks/src/lib.rs deleted file mode 100644 index 5ede4af0..00000000 --- a/benchmarks/src/lib.rs +++ /dev/null @@ -1,2 +0,0 @@ -pub mod tpch; -mod util; diff --git a/benchmarks/src/main.rs b/benchmarks/src/main.rs new file mode 100644 index 00000000..00592043 --- /dev/null +++ b/benchmarks/src/main.rs @@ -0,0 +1,50 @@ +//! DataFusion Distributed benchmark runner +mod compare; +mod prepare_clickbench; +mod prepare_tpcds; +mod prepare_tpch; +mod results; +mod run; + +use datafusion::error::Result; +use structopt::StructOpt; + +pub(crate) mod built_info { + // The file has been placed there by the build script. + include!(concat!(env!("OUT_DIR"), "/built.rs")); +} + +pub(crate) const DATA_PATH: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/data"); +pub(crate) const RESULTS_DIR: &str = ".results"; + +#[derive(Debug, StructOpt)] +#[structopt(about = "benchmark command")] +enum Options { + Run(run::RunOpt), + Compare(compare::CompareOpt), + PrepareTpch(prepare_tpch::PrepareTpchOpt), + PrepareTpcds(prepare_tpcds::PrepareTpcdsOpt), + PrepareClickbench(prepare_clickbench::PrepareClickBenchOpt), +} + +// Main benchmark runner entrypoint +pub fn main() -> Result<()> { + env_logger::init(); + + match Options::from_args() { + Options::Run(opt) => opt.run(), + Options::Compare(opt) => opt.run(), + Options::PrepareTpch(opt) => { + let rt = tokio::runtime::Runtime::new()?; + rt.block_on(async { opt.run().await }) + } + Options::PrepareTpcds(opt) => { + let rt = tokio::runtime::Runtime::new()?; + rt.block_on(async { opt.run().await }) + } + Options::PrepareClickbench(opt) => { + let rt = tokio::runtime::Runtime::new()?; + rt.block_on(async { opt.run().await }) + } + } +} diff --git a/benchmarks/src/prepare_clickbench.rs b/benchmarks/src/prepare_clickbench.rs new file mode 100644 index 00000000..cb18c5bb --- /dev/null +++ b/benchmarks/src/prepare_clickbench.rs @@ -0,0 +1,33 @@ +use datafusion::error::DataFusionError; +use datafusion_distributed::test_utils::clickbench; +use std::path::{Path, PathBuf}; +use structopt::StructOpt; + +/// Prepare ClickBench parquet files for benchmarks +#[derive(Debug, StructOpt)] +pub struct PrepareClickBenchOpt { + /// Output path + #[structopt(parse(from_os_str), required = true, short = "o", long = "output")] + output_path: PathBuf, + + /// Clickbench dataset is partitioned in 100 files. You may not want to use all the files for + /// the benchmark, so this allows setting from which file partition to start. + #[structopt(long, default_value = "0")] + partition_start: usize, + + /// Clickbench dataset is partitioned in 100 files. You may not want to use all the files for + /// the benchmark, so this allows setting a maximum in the file partition index. + #[structopt(long, default_value = "100")] + partition_end: usize, +} + +impl PrepareClickBenchOpt { + pub async fn run(self) -> datafusion::common::Result<()> { + clickbench::generate_clickbench_data( + Path::new(&self.output_path), + self.partition_start..self.partition_end, + ) + .await + .map_err(|e| DataFusionError::Internal(format!("{e:?}"))) + } +} diff --git a/benchmarks/src/prepare_tpcds.rs b/benchmarks/src/prepare_tpcds.rs new file mode 100644 index 00000000..a1ab1b47 --- /dev/null +++ b/benchmarks/src/prepare_tpcds.rs @@ -0,0 +1,28 @@ +use datafusion::error::DataFusionError; +use datafusion_distributed::test_utils::tpcds; +use std::path::{Path, PathBuf}; +use structopt::StructOpt; + +/// Prepare TPC-DS parquet files for benchmarks +#[derive(Debug, StructOpt)] +pub struct PrepareTpcdsOpt { + /// Output path + #[structopt(parse(from_os_str), required = true, short = "o", long = "output")] + output_path: PathBuf, + + /// Number of partitions to produce. By default, uses only 1 partition. + #[structopt(short = "n", long = "partitions", default_value = "1")] + partitions: usize, + + /// Scale factor of the TPC-DS data + #[structopt(long, default_value = "1")] + sf: f64, +} + +impl PrepareTpcdsOpt { + pub async fn run(self) -> datafusion::common::Result<()> { + tpcds::generate_data(Path::new(&self.output_path), self.sf, self.partitions) + .await + .map_err(|e| DataFusionError::Internal(format!("{e:?}"))) + } +} diff --git a/benchmarks/src/tpch/mod.rs b/benchmarks/src/prepare_tpch.rs similarity index 52% rename from benchmarks/src/tpch/mod.rs rename to benchmarks/src/prepare_tpch.rs index e8f558c9..d7acde24 100644 --- a/benchmarks/src/tpch/mod.rs +++ b/benchmarks/src/prepare_tpch.rs @@ -15,42 +15,118 @@ // specific language governing permissions and limitations // under the License. -//! Benchmark derived from TPC-H. This is not an official TPC-H benchmark. - -use datafusion::arrow::datatypes::SchemaBuilder; -use datafusion::{ - arrow::datatypes::{DataType, Field, Schema}, - common::plan_err, - error::Result, -}; -mod run; -pub use run::RunOpt; +use datafusion::arrow::datatypes::{DataType, Field, Schema}; +use datafusion::common::instant::Instant; +use datafusion::error::Result; +use datafusion::logical_expr::select_expr::SelectExpr; +use datafusion::parquet::basic::Compression; +use datafusion::parquet::file::properties::WriterProperties; +use datafusion::prelude::*; +use std::fs; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use structopt::StructOpt; + +/// Prepare TPCH parquet files for benchmarks +#[derive(Debug, StructOpt)] +pub struct PrepareTpchOpt { + /// Path to unprepared files + #[structopt(parse(from_os_str), required = true, short = "i", long = "input")] + input_path: PathBuf, + + /// Output path + #[structopt(parse(from_os_str), required = true, short = "o", long = "output")] + output_path: PathBuf, + + /// Number of partitions to produce. By default, uses only 1 partition. + #[structopt(short = "n", long = "partitions", default_value = "1")] + partitions: usize, + + /// Sort each table by its first column in ascending order. + #[structopt(short = "t", long = "sort")] + sort: bool, +} -mod convert; -pub use convert::ConvertOpt; +impl PrepareTpchOpt { + pub async fn run(self) -> Result<()> { + let input_path = self.input_path.to_str().unwrap(); + let output_path = self.output_path.to_str().unwrap(); + + let output_root_path = Path::new(output_path); + for table in TPCH_TABLES { + let start = Instant::now(); + let schema = get_tpch_table_schema(table); + let key_column_name = schema.fields()[0].name(); + let csv_schema = tpch_csv_schema_with_padding(&schema); + + let input_path = format!("{input_path}/{table}.tbl"); + let options = CsvReadOptions::new() + .schema(&csv_schema) + .has_header(false) + .delimiter(b'|') + .file_extension(".tbl"); + let options = if self.sort { + // indicated that the file is already sorted by its first column to speed up the conversion + options.file_sort_order(vec![vec![col(key_column_name).sort(true, false)]]) + } else { + options + }; + + let config = SessionConfig::new().with_target_partitions(self.partitions); + let ctx = SessionContext::new_with_config(config); + + // build plan to read the TBL file + let mut csv = ctx.read_csv(&input_path, options).await?; + + // Select all apart from the padding column + let selection = csv + .schema() + .iter() + .take(schema.fields().len()) + .map(Expr::from) + .map(SelectExpr::from) + .collect::>(); + + csv = csv.select(selection)?; + csv = csv.repartition(Partitioning::RoundRobinBatch(self.partitions))?; + let csv = if self.sort { + csv.sort_by(vec![col(key_column_name)])? + } else { + csv + }; + + // create the physical plan + let csv = csv.create_physical_plan().await?; + + let output_path = output_root_path.join(table); + let output_path = output_path.to_str().unwrap().to_owned(); + fs::create_dir_all(&output_path)?; + println!( + "Converting '{}' to parquet files in directory '{}'", + &input_path, &output_path + ); + let props = WriterProperties::builder() + .set_compression(Compression::ZSTD(Default::default())) + .build(); + ctx.write_parquet(csv, output_path, Some(props)).await?; + println!("Conversion completed in {} ms", start.elapsed().as_millis()); + } + + Ok(()) + } +} -pub const TPCH_TABLES: &[&str] = &[ +const TPCH_TABLES: &[&str] = &[ "part", "supplier", "partsupp", "customer", "orders", "lineitem", "nation", "region", ]; -pub const TPCH_QUERY_START_ID: usize = 1; -pub const TPCH_QUERY_END_ID: usize = 22; - -/// The `.tbl` file contains a trailing column -pub fn get_tbl_tpch_table_schema(table: &str) -> Schema { - let mut schema = SchemaBuilder::from(get_tpch_table_schema(table).fields); - schema.push(Field::new("__placeholder", DataType::Utf8, false)); - schema.finish() -} - -/// Get the schema for the benchmarks derived from TPC-H -pub fn get_tpch_table_schema(table: &str) -> Schema { +pub fn get_tpch_table_schema(table: &str) -> datafusion::arrow::datatypes::Schema { // note that the schema intentionally uses signed integers so that any generated Parquet // files can also be used to benchmark tools that only support signed integers, such as // Apache Spark match table { - "part" => Schema::new(vec![ + "part" => datafusion::arrow::datatypes::Schema::new(vec![ Field::new("p_partkey", DataType::Int64, false), Field::new("p_name", DataType::Utf8, false), Field::new("p_mfgr", DataType::Utf8, false), @@ -62,7 +138,7 @@ pub fn get_tpch_table_schema(table: &str) -> Schema { Field::new("p_comment", DataType::Utf8, false), ]), - "supplier" => Schema::new(vec![ + "supplier" => datafusion::arrow::datatypes::Schema::new(vec![ Field::new("s_suppkey", DataType::Int64, false), Field::new("s_name", DataType::Utf8, false), Field::new("s_address", DataType::Utf8, false), @@ -72,7 +148,7 @@ pub fn get_tpch_table_schema(table: &str) -> Schema { Field::new("s_comment", DataType::Utf8, false), ]), - "partsupp" => Schema::new(vec![ + "partsupp" => datafusion::arrow::datatypes::Schema::new(vec![ Field::new("ps_partkey", DataType::Int64, false), Field::new("ps_suppkey", DataType::Int64, false), Field::new("ps_availqty", DataType::Int32, false), @@ -80,7 +156,7 @@ pub fn get_tpch_table_schema(table: &str) -> Schema { Field::new("ps_comment", DataType::Utf8, false), ]), - "customer" => Schema::new(vec![ + "customer" => datafusion::arrow::datatypes::Schema::new(vec![ Field::new("c_custkey", DataType::Int64, false), Field::new("c_name", DataType::Utf8, false), Field::new("c_address", DataType::Utf8, false), @@ -91,7 +167,7 @@ pub fn get_tpch_table_schema(table: &str) -> Schema { Field::new("c_comment", DataType::Utf8, false), ]), - "orders" => Schema::new(vec![ + "orders" => datafusion::arrow::datatypes::Schema::new(vec![ Field::new("o_orderkey", DataType::Int64, false), Field::new("o_custkey", DataType::Int64, false), Field::new("o_orderstatus", DataType::Utf8, false), @@ -103,7 +179,7 @@ pub fn get_tpch_table_schema(table: &str) -> Schema { Field::new("o_comment", DataType::Utf8, false), ]), - "lineitem" => Schema::new(vec![ + "lineitem" => datafusion::arrow::datatypes::Schema::new(vec![ Field::new("l_orderkey", DataType::Int64, false), Field::new("l_partkey", DataType::Int64, false), Field::new("l_suppkey", DataType::Int64, false), @@ -122,14 +198,14 @@ pub fn get_tpch_table_schema(table: &str) -> Schema { Field::new("l_comment", DataType::Utf8, false), ]), - "nation" => Schema::new(vec![ + "nation" => datafusion::arrow::datatypes::Schema::new(vec![ Field::new("n_nationkey", DataType::Int64, false), Field::new("n_name", DataType::Utf8, false), Field::new("n_regionkey", DataType::Int64, false), Field::new("n_comment", DataType::Utf8, false), ]), - "region" => Schema::new(vec![ + "region" => datafusion::arrow::datatypes::Schema::new(vec![ Field::new("r_regionkey", DataType::Int64, false), Field::new("r_name", DataType::Utf8, false), Field::new("r_comment", DataType::Utf8, false), @@ -139,50 +215,8 @@ pub fn get_tpch_table_schema(table: &str) -> Schema { } } -fn get_benchmark_queries_dir() -> std::path::PathBuf { - std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../testdata/tpch/queries") -} - -/// Get the SQL statements from the specified query file -pub fn get_query_sql(query: usize) -> Result> { - if query > 0 && query < 23 { - let queries_dir = get_benchmark_queries_dir(); - let contents = datafusion_distributed::test_utils::tpch::tpch_query_from_dir( - &queries_dir, - query as u8, - ); - Ok(contents - .split(';') - .map(|s| s.trim()) - .filter(|s| !s.is_empty()) - .map(|s| s.to_string()) - .collect()) - } else { - plan_err!("invalid query. Expected value between 1 and 22") - } +fn tpch_csv_schema_with_padding(schema: &Schema) -> Schema { + let mut fields = schema.fields().iter().cloned().collect::>(); + fields.push(Arc::new(Field::new("__padding", DataType::Utf8, true))); + Schema::new(fields) } - -pub const QUERY_LIMIT: [Option; 22] = [ - None, - Some(100), - Some(10), - None, - None, - None, - None, - None, - None, - Some(20), - None, - None, - None, - None, - None, - None, - None, - Some(100), - None, - None, - Some(100), - None, -]; diff --git a/benchmarks/src/results.rs b/benchmarks/src/results.rs new file mode 100644 index 00000000..fcdc7f22 --- /dev/null +++ b/benchmarks/src/results.rs @@ -0,0 +1,292 @@ +use crate::{DATA_PATH, RESULTS_DIR}; +use datafusion::common::utils::get_available_parallelism; +use datafusion::common::{Result, internal_datafusion_err}; +use serde::ser::SerializeSeq; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; +use std::fs; +use std::path::PathBuf; +use std::process::Command; +use std::time::{Duration, SystemTime}; + +/// A single iteration of a benchmark query +#[derive(Debug, Serialize, Deserialize)] +pub struct QueryIter { + pub row_count: usize, + pub n_tasks: usize, + #[serde( + serialize_with = "serialize_elapsed", + deserialize_with = "deserialize_elapsed" + )] + pub elapsed: Duration, + pub error: Option, +} + +/// A single benchmark case +#[derive(Debug, Serialize, Deserialize)] +pub struct BenchResult { + pub id: String, + pub dataset: String, + pub iterations: Vec, +} + +/// collects benchmark run data and then serializes it at the end +#[derive(Debug, Serialize, Deserialize)] +pub struct BenchmarkRun { + /// Number of workers involved in a distributed query + pub workers: usize, + /// Number of physical threads used per worker + pub threads: usize, + /// Start time + #[serde( + serialize_with = "serialize_start_time", + deserialize_with = "deserialize_start_time" + )] + pub start_time: SystemTime, + pub dataset: String, + pub branch: String, + #[serde(serialize_with = "serialize_bench_results")] + pub results: Vec, +} + +impl BenchmarkRun { + pub fn new(dataset: String, workers: usize, threads: usize) -> Self { + Self { + workers, + threads, + dataset, + branch: get_current_branch(), + start_time: SystemTime::now(), + results: vec![], + } + } + + pub fn load_previous(dataset: &str) -> Option { + let path = PathBuf::from(DATA_PATH).join(dataset).join("previous.json"); + let Ok(prev) = fs::read(path) else { + return None; + }; + + let Ok(mut prev_output) = serde_json::from_slice::(&prev) else { + return None; + }; + + prev_output.results = BenchResult::load_many(&prev_output.dataset, &prev_output.branch); + Some(prev_output) + } + + /// Write data as json into output path if it exists. + pub fn store(&self) -> Result<()> { + let path = PathBuf::from(DATA_PATH) + .join(&self.dataset) + .join("previous.json"); + let json = serde_json::to_string_pretty(&self).unwrap(); + + let _ = fs::create_dir_all(path.parent().unwrap()); + + fs::write(path, json)?; + for result in &self.results { + result.store()?; + } + Ok(()) + } + + pub fn compare_with_previous(&self) -> Result<()> { + let Some(previous) = Self::load_previous(&self.dataset) else { + return Ok(()); + }; + + let header = format!( + "=== Comparing {} results from branch '{}' [prev] with '{}' [new] ===", + self.dataset, previous.branch, self.branch + ); + println!("{header}"); + // Print machine information + println!("os: {}", std::env::consts::OS); + println!("arch: {}", std::env::consts::ARCH); + println!("cpu cores: {}", get_available_parallelism()); + println!("threads: {} -> {}", previous.threads, self.threads); + println!("workers: {} -> {}", previous.workers, self.workers); + println!("{}", "=".repeat(header.len())); + for query in self.results.iter() { + let Some(prev_query) = previous.results.iter().find(|v| v.id == query.id) else { + continue; + }; + query.compare(prev_query) + } + + Ok(()) + } +} + +fn get_current_branch() -> String { + let output = Command::new("git") + .args(["rev-parse", "--abbrev-ref", "HEAD"]) + .output() + .expect("failed to execute git command"); + + let branch_name = String::from_utf8(output.stdout) + .expect("git output is not valid UTF-8") + .trim() + .to_string(); + + branch_name.split("/").last().unwrap().to_string() +} + +impl BenchResult { + pub fn avg(&self) -> u128 { + self.iterations + .iter() + .map(|v| v.elapsed.as_millis()) + .sum::() + / self.iterations.len() as u128 + } + + pub fn store(&self) -> Result<()> { + let path = PathBuf::from(DATA_PATH) + .join(&self.dataset) + .join(RESULTS_DIR) + .join(get_current_branch()) + .join(format!("{}.json", self.id)); + + let _ = fs::create_dir_all(path.parent().unwrap()); + + let result_string = + serde_json::to_string_pretty(self).map_err(|err| internal_datafusion_err!("{err}"))?; + fs::write(path, result_string)?; + + Ok(()) + } + + pub fn load_many(dataset: &str, branch: &str) -> Vec { + let dir = PathBuf::from(DATA_PATH) + .join(dataset) + .join(RESULTS_DIR) + .join(branch); + + let Ok(dir) = fs::read_dir(dir) else { + return vec![]; + }; + + let mut results = vec![]; + for file in dir { + let Ok(file) = file else { continue }; + let file_name = file.file_name().to_string_lossy().to_string(); + let id = if file_name.ends_with(".json") { + file_name.trim_end_matches(".json") + } else { + continue; + }; + let Ok(result) = BenchResult::load(dataset, branch, id) else { + continue; + }; + results.push(result); + } + results.sort_by(|a, b| { + let extract_number = |s: &str| -> Option { + s.chars() + .filter(|c| c.is_ascii_digit()) + .collect::() + .parse::() + .ok() + }; + + match (extract_number(&a.id), extract_number(&b.id)) { + (Some(num_a), Some(num_b)) => num_a.cmp(&num_b), + (Some(_), None) => std::cmp::Ordering::Less, + (None, Some(_)) => std::cmp::Ordering::Greater, + (None, None) => a.id.cmp(&b.id), + } + }); + results + } + + pub fn load(dataset: &str, branch: &str, id: &str) -> Result { + let path = PathBuf::from(DATA_PATH) + .join(dataset) + .join(RESULTS_DIR) + .join(branch) + .join(format!("{id}.json")); + + let read = fs::read(path)?; + let read = + serde_json::from_slice(&read).map_err(|err| internal_datafusion_err!("{err}"))?; + Ok(read) + } + + pub fn compare(&self, prev_query: &Self) { + let prev_err = prev_query.iterations.iter().find_map(|v| v.error.clone()); + let new_err = self.iterations.iter().find_map(|v| v.error.clone()); + match (prev_err, new_err) { + (Some(_prev_err), None) => { + println!("{}: Previously failed, but now succeeded 🟠", self.id); + return; + } + (None, Some(_new_err)) => { + println!("{}: Previously succeeded, but now failed ❌", self.id); + return; + } + (Some(_prev_err), Some(_new_err)) => { + println!("{}: Previously failed, and now also failed ❌", self.id); + return; + } + (None, None) => {} + } + + let avg_prev = prev_query.avg(); + let avg = self.avg(); + let (f, tag, emoji) = if avg < avg_prev { + let f = avg_prev as f64 / avg as f64; + (f, "faster", if f > 1.2 { "✅" } else { "✔" }) + } else { + let f = avg as f64 / avg_prev as f64; + (f, "slower", if f > 1.2 { "❌" } else { "✖" }) + }; + println!( + "{:>8}: prev={avg_prev:>4} ms, new={avg:>4} ms, diff={f:.2} {tag} {emoji}", + self.id + ); + } +} + +fn serialize_bench_results( + _bench_result: &[BenchResult], + ser: S, +) -> Result { + // We want to avoid serializing these here on purpose. + ser.serialize_seq(Some(0))?.end() +} + +fn serialize_start_time(start_time: &SystemTime, ser: S) -> Result +where + S: Serializer, +{ + ser.serialize_u64( + start_time + .duration_since(SystemTime::UNIX_EPOCH) + .expect("current time is later than UNIX_EPOCH") + .as_secs(), + ) +} +fn deserialize_start_time<'de, D>(des: D) -> Result +where + D: Deserializer<'de>, +{ + let secs = u64::deserialize(des)?; + Ok(SystemTime::UNIX_EPOCH + Duration::from_secs(secs)) +} + +fn serialize_elapsed(elapsed: &Duration, ser: S) -> Result +where + S: Serializer, +{ + let ms = elapsed.as_secs_f64() * 1000.0; + ser.serialize_f64(ms) +} + +fn deserialize_elapsed<'de, D>(des: D) -> Result +where + D: Deserializer<'de>, +{ + let ms = f64::deserialize(des)?; + Ok(Duration::from_secs_f64(ms / 1000.0)) +} diff --git a/benchmarks/src/run.rs b/benchmarks/src/run.rs new file mode 100644 index 00000000..d1672cc1 --- /dev/null +++ b/benchmarks/src/run.rs @@ -0,0 +1,347 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use crate::results::{BenchResult, BenchmarkRun, QueryIter}; +use datafusion::arrow::ipc::CompressionType; +use datafusion::arrow::record_batch::RecordBatch; +use datafusion::common::instant::Instant; +use datafusion::common::tree_node::{Transformed, TreeNode}; +use datafusion::common::utils::get_available_parallelism; +use datafusion::common::{config_err, exec_err, not_impl_err}; +use datafusion::error::{DataFusionError, Result}; +use datafusion::execution::SessionStateBuilder; +use datafusion::physical_plan::display::DisplayableExecutionPlan; +use datafusion::physical_plan::{collect, displayable}; +use datafusion::prelude::*; +use datafusion_distributed::test_utils::benchmarks_common; +use datafusion_distributed::test_utils::localhost::LocalHostWorkerResolver; +use datafusion_distributed::test_utils::{clickbench, tpcds, tpch}; +use datafusion_distributed::{ + DistributedExt, DistributedPhysicalOptimizerRule, NetworkBoundaryExt, Worker, +}; +use std::error::Error; +use std::fs; +use std::path::PathBuf; +use std::sync::Arc; +use std::time::Duration; +use structopt::StructOpt; +use tokio::net::TcpListener; +use tonic::codegen::tokio_stream; +use tonic::transport::Server; + +/// Run the tpch benchmark. +/// +/// This benchmarks is derived from the [TPC-H][1] version +/// [2.17.1]. The data and answers are generated using `tpch-gen` from +/// [2]. +/// +/// [1]: http://www.tpc.org/tpch/ +/// [2]: https://github.com/databricks/tpch-dbgen.git +/// [2.17.1]: https://www.tpc.org/tpc_documents_current_versions/pdf/tpc-h_v2.17.1.pdf +#[derive(Debug, StructOpt, Clone)] +#[structopt(verbatim_doc_comment)] +pub struct RunOpt { + /// Query number. If not specified, runs all queries + #[structopt(short, long, use_delimiter = true)] + pub query: Vec, + + /// Path to data files + #[structopt(long)] + dataset: String, + + /// Spawns a worker in the specified port. + #[structopt(long)] + spawn: Option, + + /// The ports of all the workers involved in the query. + #[structopt(long, use_delimiter = true)] + workers: Vec, + + /// Number of physical threads per worker. + #[structopt(long)] + threads: Option, + + /// Number of files per each distributed task. + #[structopt(long)] + files_per_task: Option, + + /// Task count scale factor for when nodes in stages change the cardinality of the data + #[structopt(long)] + cardinality_task_sf: Option, + + /// Use children isolator UNIONs for distributing UNION operations. + #[structopt(long)] + children_isolator_unions: bool, + + /// Turns on broadcast joins. + #[structopt(long = "broadcast-joins")] + broadcast_joins: bool, + + /// Collects metrics across network boundaries + #[structopt(long)] + collect_metrics: bool, + + /// Collects metrics across network boundaries + #[structopt(long, default_value = "lz4")] + compression: String, + + /// Number of iterations of each test run + #[structopt(short = "i", long = "iterations", default_value = "3")] + iterations: usize, + + /// Number of partitions to process in parallel. Defaults to number of available cores. + /// Should typically be less or equal than --threads. + #[structopt(short = "n", long = "partitions")] + partitions: Option, + + /// Batch size when reading CSV or Parquet files + #[structopt(short = "s", long = "batch-size")] + batch_size: Option, + + /// Activate debug mode to see more details + #[structopt(short, long)] + debug: bool, +} + +fn queries_for_dataset(dataset: &str) -> Result, DataFusionError> { + match dataset { + "tpch" => tpch::get_queries() + .into_iter() + .map(|id| Ok((id.clone(), tpch::get_query(&id)?))) + .collect(), + "tpcds" => tpcds::get_queries() + .into_iter() + .filter(|id| id != "q72") // 72 is terribly slow + .map(|id| Ok((id.clone(), tpcds::get_query(&id)?))) + .collect(), + "clickbench" => clickbench::get_queries() + .into_iter() + .map(|id| Ok((id.clone(), clickbench::get_query(&id)?))) + .collect(), + _ => not_impl_err!("Unknown benchmark dataset {dataset}"), + } +} + +impl RunOpt { + fn config(&self) -> Result { + SessionConfig::from_env().map(|mut config| { + if let Some(batch_size) = self.batch_size { + config = config.with_batch_size(batch_size); + } + config.with_target_partitions(self.partitions()) + }) + } + + pub fn run(self) -> Result<()> { + let rt = tokio::runtime::Builder::new_multi_thread() + .worker_threads(self.threads.unwrap_or(get_available_parallelism())) + .enable_all() + .build()?; + + if let Some(port) = self.spawn { + rt.block_on(async move { + let listener = TcpListener::bind(format!("127.0.0.1:{port}")).await?; + println!("Listening on {}...", listener.local_addr().unwrap()); + let incoming = tokio_stream::wrappers::TcpListenerStream::new(listener); + Ok::<_, Box>( + Server::builder() + .add_service(Worker::default().into_flight_server()) + .serve_with_incoming(incoming) + .await?, + ) + })?; + } else { + rt.block_on(self.run_local())?; + } + Ok(()) + } + + async fn run_local(self) -> Result<()> { + let state = SessionStateBuilder::new() + .with_default_features() + .with_config(self.config()?) + .with_distributed_worker_resolver(LocalHostWorkerResolver::new(self.workers.clone())) + .with_physical_optimizer_rule(Arc::new(DistributedPhysicalOptimizerRule)) + .with_distributed_files_per_task( + self.files_per_task.unwrap_or(get_available_parallelism()), + )? + .with_distributed_cardinality_effect_task_scale_factor( + self.cardinality_task_sf.unwrap_or(1.0), + )? + .with_distributed_compression(match self.compression.as_str() { + "zstd" => Some(CompressionType::ZSTD), + "lz4" => Some(CompressionType::LZ4_FRAME), + "none" => None, + v => return config_err!("Unknown compression type {v}"), + })? + .with_distributed_children_isolator_unions(self.children_isolator_unions)? + .with_distributed_broadcast_joins(self.broadcast_joins)? + .with_distributed_metrics_collection(self.collect_metrics)? + .build(); + let ctx = SessionContext::new_with_state(state); + benchmarks_common::register_tables(&ctx, &self.get_path()?).await?; + + println!("Running benchmarks with the following options: {self:?}"); + let mut benchmark_run = BenchmarkRun::new( + self.dataset.clone(), + self.workers.len(), + self.threads.unwrap_or(get_available_parallelism()), + ); + + let dataset_prefix = self.dataset.split("_").next().unwrap(); + for (id, sql) in queries_for_dataset(dataset_prefix)? { + if !self.query.is_empty() && !self.query.contains(&id.to_string()) { + continue; + } + let query_id = format!("{} {id}", self.dataset); + let query_run = self.benchmark_query(&query_id, &sql, &ctx).await; + if let Err(e) = &query_run { + eprintln!("{query_id} failed: {e:?}"); + } + benchmark_run.results.push(query_run?); + } + + benchmark_run.compare_with_previous()?; + benchmark_run.store()?; + Ok(()) + } + + async fn benchmark_query( + &self, + id: &str, + sql: &str, + ctx: &SessionContext, + ) -> Result { + let mut bench_query = BenchResult { + id: id.to_string(), + dataset: self.dataset.clone(), + iterations: vec![], + }; + + 'outer: for i in 0..self.iterations { + let start = Instant::now(); + + for query in sql.split(";").map(|v| v.trim()) { + if query.starts_with("create") || query.starts_with("drop") { + self.execute_query(ctx, query).await?; + continue; + } else if query.is_empty() { + continue; + } + + match self.execute_query(ctx, query).await { + Ok((result, n_tasks)) => { + let elapsed = start.elapsed(); + let ms = elapsed.as_secs_f64() * 1000.0; + let row_count = result.iter().map(|b| b.num_rows()).sum(); + println!( + "Query {id} iteration {i} took {ms:.1} ms and returned {row_count} rows" + ); + + bench_query.iterations.push(QueryIter { + elapsed, + row_count, + n_tasks, + error: None, + }); + } + Err(err) => { + println!("Query {id} iteration {i} failed: {err}"); + bench_query.iterations.push(QueryIter { + elapsed: Duration::from_millis(0), + row_count: 0, + n_tasks: 0, + error: Some(err.to_string()), + }); + continue 'outer; + } + } + } + } + println!("Query {id} avg time: {:.2} ms", bench_query.avg()); + + Ok(bench_query) + } + + async fn execute_query( + &self, + ctx: &SessionContext, + sql: &str, + ) -> Result<(Vec, usize)> { + let plan = ctx.sql(sql).await?; + let (state, plan) = plan.into_parts(); + + if self.debug { + println!("=== Logical plan ===\n{plan}\n"); + } + + let plan = state.optimize(&plan)?; + if self.debug { + println!("=== Optimized logical plan ===\n{plan}\n"); + } + let physical_plan = state.create_physical_plan(&plan).await?; + if self.debug { + println!( + "=== Physical plan ===\n{}\n", + displayable(physical_plan.as_ref()).indent(true) + ); + } + let mut n_tasks = 0; + physical_plan.clone().transform_down(|node| { + if let Some(node) = node.as_network_boundary() { + n_tasks += node.input_stage().tasks.len() + } + Ok(Transformed::no(node)) + })?; + let result = collect(physical_plan.clone(), state.task_ctx()).await?; + if self.debug { + println!( + "=== Physical plan with metrics ===\n{}\n", + DisplayableExecutionPlan::with_metrics(physical_plan.as_ref()).indent(true) + ); + } + Ok((result, n_tasks)) + } + + fn get_path(&self) -> Result { + let data_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("data") + .join(&self.dataset); + if !data_path.exists() { + return exec_err!( + "--dataset {} doesn't exist. Was it generated?", + self.dataset + ); + } + + let entries = fs::read_dir(&data_path)?.collect::, _>>()?; + if entries.is_empty() { + return exec_err!("Dataset {} is empty", self.dataset); + } + Ok(data_path) + } + + fn partitions(&self) -> usize { + if let Some(partitions) = self.partitions { + return partitions; + } + if let Some(threads) = self.threads { + return threads; + } + get_available_parallelism() + } +} diff --git a/benchmarks/src/tpch/convert.rs b/benchmarks/src/tpch/convert.rs deleted file mode 100644 index 2371cdd4..00000000 --- a/benchmarks/src/tpch/convert.rs +++ /dev/null @@ -1,161 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use datafusion::common::instant::Instant; -use datafusion::logical_expr::select_expr::SelectExpr; -use std::fs; -use std::path::{Path, PathBuf}; - -use datafusion::common::not_impl_err; - -use super::TPCH_TABLES; -use super::get_tbl_tpch_table_schema; -use datafusion::error::Result; -use datafusion::prelude::*; -use parquet::basic::Compression; -use parquet::file::properties::WriterProperties; -use structopt::StructOpt; - -/// Convert tpch .slt files to .parquet or .csv files -#[derive(Debug, StructOpt)] -pub struct ConvertOpt { - /// Path to csv files - #[structopt(parse(from_os_str), required = true, short = "i", long = "input")] - input_path: PathBuf, - - /// Output path - #[structopt(parse(from_os_str), required = true, short = "o", long = "output")] - output_path: PathBuf, - - /// Output file format: `csv` or `parquet` - #[structopt(short = "f", long = "format")] - file_format: String, - - /// Compression to use when writing Parquet files - #[structopt(short = "c", long = "compression", default_value = "zstd")] - compression: String, - - /// Number of partitions to produce - #[structopt(short = "n", long = "partitions", default_value = "1")] - partitions: usize, - - /// Batch size when reading CSV or Parquet files - #[structopt(short = "s", long = "batch-size", default_value = "8192")] - batch_size: usize, - - /// Sort each table by its first column in ascending order. - #[structopt(short = "t", long = "sort")] - sort: bool, -} - -impl ConvertOpt { - pub async fn run(self) -> Result<()> { - let compression = self.compression()?; - - let input_path = self.input_path.to_str().unwrap(); - let output_path = self.output_path.to_str().unwrap(); - - let output_root_path = Path::new(output_path); - for table in TPCH_TABLES { - let start = Instant::now(); - let schema = get_tbl_tpch_table_schema(table); - let key_column_name = schema.fields()[0].name(); - - let input_path = format!("{input_path}/{table}.tbl"); - let options = CsvReadOptions::new() - .schema(&schema) - .has_header(false) - .delimiter(b'|') - .file_extension(".tbl"); - let options = if self.sort { - // indicated that the file is already sorted by its first column to speed up the conversion - options.file_sort_order(vec![vec![col(key_column_name).sort(true, false)]]) - } else { - options - }; - - let config = SessionConfig::new().with_batch_size(self.batch_size); - let ctx = SessionContext::new_with_config(config); - - // build plan to read the TBL file - let mut csv = ctx.read_csv(&input_path, options).await?; - - // Select all apart from the padding column - let selection = csv - .schema() - .iter() - .take(schema.fields.len() - 1) - .map(Expr::from) - .map(SelectExpr::from) - .collect::>(); - - csv = csv.select(selection)?; - // optionally, repartition the file - let partitions = self.partitions; - if partitions > 1 { - csv = csv.repartition(Partitioning::RoundRobinBatch(partitions))? - } - let csv = if self.sort { - csv.sort_by(vec![col(key_column_name)])? - } else { - csv - }; - - // create the physical plan - let csv = csv.create_physical_plan().await?; - - let output_path = output_root_path.join(table); - let output_path = output_path.to_str().unwrap().to_owned(); - fs::create_dir_all(&output_path)?; - println!( - "Converting '{}' to {} files in directory '{}'", - &input_path, self.file_format, &output_path - ); - match self.file_format.as_str() { - "csv" => ctx.write_csv(csv, output_path).await?, - "parquet" => { - let props = WriterProperties::builder() - .set_compression(compression) - .build(); - ctx.write_parquet(csv, output_path, Some(props)).await? - } - other => { - return not_impl_err!("Invalid output format: {other}"); - } - } - println!("Conversion completed in {} ms", start.elapsed().as_millis()); - } - - Ok(()) - } - - /// return the compression method to use when writing parquet - fn compression(&self) -> Result { - Ok(match self.compression.as_str() { - "none" => Compression::UNCOMPRESSED, - "snappy" => Compression::SNAPPY, - "brotli" => Compression::BROTLI(Default::default()), - "gzip" => Compression::GZIP(Default::default()), - "lz4" => Compression::LZ4, - "lz0" => Compression::LZO, - "zstd" => Compression::ZSTD(Default::default()), - other => { - return not_impl_err!("Invalid compression format: {other}"); - } - }) - } -} diff --git a/benchmarks/src/tpch/run.rs b/benchmarks/src/tpch/run.rs deleted file mode 100644 index 015b6603..00000000 --- a/benchmarks/src/tpch/run.rs +++ /dev/null @@ -1,447 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use super::{ - TPCH_QUERY_END_ID, TPCH_QUERY_START_ID, TPCH_TABLES, get_query_sql, get_tbl_tpch_table_schema, - get_tpch_table_schema, -}; -use crate::util::{ - BenchmarkRun, CommonOpt, InMemoryCacheExecCodec, InMemoryDataSourceRule, QueryIter, - WarmingUpMarker, -}; -use async_trait::async_trait; -use datafusion::arrow::record_batch::RecordBatch; -use datafusion::arrow::util::pretty::pretty_format_batches; -use datafusion::common::instant::Instant; -use datafusion::common::tree_node::{Transformed, TreeNode}; -use datafusion::common::utils::get_available_parallelism; -use datafusion::common::{DEFAULT_CSV_EXTENSION, DEFAULT_PARQUET_EXTENSION, exec_err}; -use datafusion::datasource::TableProvider; -use datafusion::datasource::file_format::FileFormat; -use datafusion::datasource::file_format::csv::CsvFormat; -use datafusion::datasource::file_format::parquet::ParquetFormat; -use datafusion::datasource::listing::{ - ListingOptions, ListingTable, ListingTableConfig, ListingTableUrl, -}; -use datafusion::error::{DataFusionError, Result}; -use datafusion::execution::{SessionState, SessionStateBuilder}; -use datafusion::physical_plan::display::DisplayableExecutionPlan; -use datafusion::physical_plan::{collect, displayable}; -use datafusion::prelude::*; -use datafusion_distributed::test_utils::localhost::{ - LocalHostChannelResolver, spawn_flight_service, -}; -use datafusion_distributed::{ - DistributedExt, DistributedPhysicalOptimizerRule, DistributedSessionBuilder, - DistributedSessionBuilderContext, NetworkBoundaryExt, -}; -use log::info; -use std::fs; -use std::path::PathBuf; -use std::sync::Arc; -use structopt::StructOpt; -use tokio::net::TcpListener; - -/// Run the tpch benchmark. -/// -/// This benchmarks is derived from the [TPC-H][1] version -/// [2.17.1]. The data and answers are generated using `tpch-gen` from -/// [2]. -/// -/// [1]: http://www.tpc.org/tpch/ -/// [2]: https://github.com/databricks/tpch-dbgen.git -/// [2.17.1]: https://www.tpc.org/tpc_documents_current_versions/pdf/tpc-h_v2.17.1.pdf -#[derive(Debug, StructOpt, Clone)] -#[structopt(verbatim_doc_comment)] -pub struct RunOpt { - /// Query number. If not specified, runs all queries - #[structopt(short, long)] - pub query: Option, - - /// Common options - #[structopt(flatten)] - common: CommonOpt, - - /// Path to data files - #[structopt(parse(from_os_str), short = "p", long = "path")] - path: Option, - - /// File format: `csv` or `parquet` - #[structopt(short = "f", long = "format", default_value = "parquet")] - file_format: String, - - /// Load the data into a MemTable before executing the query - #[structopt(short = "m", long = "mem-table")] - mem_table: bool, - - /// Path to machine readable output file - #[structopt(parse(from_os_str), short = "o", long = "output")] - output_path: Option, - - /// Whether to disable collection of statistics (and cost based optimizations) or not. - #[structopt(short = "S", long = "disable-statistics")] - disable_statistics: bool, - - /// Mark the first column of each table as sorted in ascending order. - /// The tables should have been created with the `--sort` option for this to have any effect. - #[structopt(short = "t", long = "sorted")] - sorted: bool, - - /// Upon shuffling data, this defines how many tasks are employed into performing the shuffling. - /// ```text - /// ( task 1 ) ( task 2 ) ( task 3 ) - /// ▲ ▲ ▲ - /// └────┬──────┴─────┬────┘ - /// ( task 1 ) ( task 2 ) N tasks - /// ``` - /// This parameter defines N - #[structopt(long)] - shuffle_tasks: Option, - - /// Upon merging multiple tasks into one, this defines how many tasks are merged. - /// ```text - /// ( task 1 ) - /// ▲ - /// ┌───────────┴──────────┐ - /// ( task 1 ) ( task 2 ) ( task 3 ) N tasks - /// ``` - /// This parameter defines N - #[structopt(long)] - coalesce_tasks: Option, - - /// Spawns a worker in the specified port. - #[structopt(long)] - spawn: Option, - - /// The ports of all the workers involved in the query. - #[structopt(long, use_delimiter = true)] - workers: Vec, - - /// Number of physical threads per worker. - #[structopt(long)] - threads: Option, -} - -#[async_trait] -impl DistributedSessionBuilder for RunOpt { - async fn build_session_state( - &self, - ctx: DistributedSessionBuilderContext, - ) -> Result { - let mut builder = SessionStateBuilder::new().with_default_features(); - - let config = self - .common - .config()? - .with_collect_statistics(!self.disable_statistics) - .with_distributed_user_codec(InMemoryCacheExecCodec) - .with_distributed_channel_resolver(LocalHostChannelResolver::new(self.workers.clone())) - .with_distributed_option_extension_from_headers::(&ctx.headers)? - .with_target_partitions(self.partitions()); - - let rt_builder = self.common.runtime_env_builder()?; - - if self.mem_table { - builder = builder.with_physical_optimizer_rule(Arc::new(InMemoryDataSourceRule)); - } - if !self.workers.is_empty() { - let rule = DistributedPhysicalOptimizerRule::new() - .with_network_coalesce_tasks(self.coalesce_tasks.unwrap_or(self.workers.len())) - .with_network_shuffle_tasks(self.shuffle_tasks.unwrap_or(self.workers.len())); - builder = builder.with_physical_optimizer_rule(Arc::new(rule)); - } - - Ok(builder - .with_config(config) - .with_runtime_env(rt_builder.build_arc()?) - .build()) - } -} - -impl RunOpt { - pub fn run(self) -> Result<()> { - let rt = tokio::runtime::Builder::new_multi_thread() - .worker_threads(self.threads.unwrap_or(get_available_parallelism())) - .enable_all() - .build()?; - - if let Some(port) = self.spawn { - rt.block_on(async move { - let listener = TcpListener::bind(format!("127.0.0.1:{port}")).await?; - println!("Listening on {}...", listener.local_addr().unwrap()); - spawn_flight_service(self, listener).await - })?; - } else { - rt.block_on(self.run_local())?; - } - Ok(()) - } - - async fn run_local(mut self) -> Result<()> { - let state = self.build_session_state(Default::default()).await?; - let ctx = SessionContext::new_with_state(state); - self.register_tables(&ctx).await?; - - println!("Running benchmarks with the following options: {self:?}"); - let query_range = match self.query { - Some(query_id) => query_id..=query_id, - None => TPCH_QUERY_START_ID..=TPCH_QUERY_END_ID, - }; - - self.output_path - .get_or_insert(self.get_path()?.join("results.json")); - let mut benchmark_run = BenchmarkRun::new( - self.workers.len(), - self.threads.unwrap_or(get_available_parallelism()), - ); - - // Warmup the cache for the in-memory mode. - if self.mem_table { - for query_id in query_range.clone() { - // put the WarmingUpMarker in the context, otherwise, queries will fail as the - // InMemoryCacheExec node will think they should already be warmed up. - let ctx = ctx - .clone() - .with_distributed_option_extension(WarmingUpMarker::warming_up())?; - for query in get_query_sql(query_id)? { - self.execute_query(&ctx, &query).await?; - } - println!("Query {query_id} data loaded in memory"); - } - } - - for query_id in query_range { - benchmark_run.start_new_case(&format!("Query {query_id}")); - let query_run = self.benchmark_query(query_id, &ctx).await; - match query_run { - Ok(query_results) => { - for iter in query_results { - benchmark_run.write_iter(iter); - } - } - Err(e) => { - benchmark_run.mark_failed(); - eprintln!("Query {query_id} failed: {e:?}"); - } - } - } - benchmark_run.maybe_compare_with_previous(self.output_path.as_ref())?; - benchmark_run.maybe_write_json(self.output_path.as_ref())?; - benchmark_run.maybe_print_failures(); - Ok(()) - } - - async fn benchmark_query( - &self, - query_id: usize, - ctx: &SessionContext, - ) -> Result> { - let mut millis = vec![]; - // run benchmark - let mut query_results = vec![]; - - let sql = &get_query_sql(query_id)?; - - let mut n_tasks = 0; - for i in 0..self.iterations() { - let start = Instant::now(); - let mut result = vec![]; - - // query 15 is special, with 3 statements. the second statement is the one from which we - // want to capture the results - let result_stmt = if query_id == 15 { 1 } else { sql.len() - 1 }; - - for (i, query) in sql.iter().enumerate() { - if i == result_stmt { - (result, n_tasks) = self.execute_query(ctx, query).await?; - } else { - self.execute_query(ctx, query).await?; - } - } - - let elapsed = start.elapsed(); - let ms = elapsed.as_secs_f64() * 1000.0; - millis.push(ms); - info!("output:\n\n{}\n\n", pretty_format_batches(&result)?); - let row_count = result.iter().map(|b| b.num_rows()).sum(); - println!( - "Query {query_id} iteration {i} took {ms:.1} ms and returned {row_count} rows" - ); - - query_results.push(QueryIter { - elapsed, - row_count, - n_tasks, - }); - } - - let avg = millis.iter().sum::() / millis.len() as f64; - println!("Query {query_id} avg time: {avg:.2} ms"); - if n_tasks > 0 { - println!("Query {query_id} number of tasks: {n_tasks}"); - } - - Ok(query_results) - } - - async fn register_tables(&self, ctx: &SessionContext) -> Result<()> { - for table in TPCH_TABLES { - ctx.register_table(*table, self.get_table(ctx, table).await?)?; - } - Ok(()) - } - - async fn execute_query( - &self, - ctx: &SessionContext, - sql: &str, - ) -> Result<(Vec, usize)> { - let debug = self.common.debug; - let plan = ctx.sql(sql).await?; - let (state, plan) = plan.into_parts(); - - if debug { - println!("=== Logical plan ===\n{plan}\n"); - } - - let plan = state.optimize(&plan)?; - if debug { - println!("=== Optimized logical plan ===\n{plan}\n"); - } - let physical_plan = state.create_physical_plan(&plan).await?; - if debug { - println!( - "=== Physical plan ===\n{}\n", - displayable(physical_plan.as_ref()).indent(true) - ); - } - let mut n_tasks = 0; - physical_plan.clone().transform_down(|node| { - if let Some(node) = node.as_network_boundary() { - n_tasks += node.input_stage().map(|v| v.tasks.len()).unwrap_or(0) - } - Ok(Transformed::no(node)) - })?; - let result = collect(physical_plan.clone(), state.task_ctx()).await?; - if debug { - println!( - "=== Physical plan with metrics ===\n{}\n", - DisplayableExecutionPlan::with_metrics(physical_plan.as_ref()).indent(true) - ); - } - Ok((result, n_tasks)) - } - - fn get_path(&self) -> Result { - if let Some(path) = &self.path { - return Ok(path.clone()); - } - let crate_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")); - let data_path = crate_path.join("data"); - let entries = fs::read_dir(&data_path)?.collect::, _>>()?; - if entries.is_empty() { - exec_err!( - "No TPCH dataset present in '{data_path:?}'. Generate one with ./benchmarks/gen-tpch.sh" - ) - } else if entries.len() == 1 { - Ok(entries[0].path()) - } else { - exec_err!( - "Multiple TPCH datasets present in '{data_path:?}'. One must be selected with --path" - ) - } - } - - async fn get_table(&self, ctx: &SessionContext, table: &str) -> Result> { - let path = self.get_path()?; - let path = path.to_str().unwrap(); - let table_format = self.file_format.as_str(); - let target_partitions = self.partitions(); - - // Obtain a snapshot of the SessionState - let state = ctx.state(); - let (format, path, extension): (Arc, String, &'static str) = - match table_format { - // dbgen creates .tbl ('|' delimited) files without header - "tbl" => { - let path = format!("{path}/{table}.tbl"); - - let format = CsvFormat::default() - .with_delimiter(b'|') - .with_has_header(false); - - (Arc::new(format), path, ".tbl") - } - "csv" => { - let path = format!("{path}/csv/{table}"); - let format = CsvFormat::default() - .with_delimiter(b',') - .with_has_header(true); - - (Arc::new(format), path, DEFAULT_CSV_EXTENSION) - } - "parquet" => { - let path = format!("{path}/{table}"); - let format = ParquetFormat::default() - .with_options(ctx.state().table_options().parquet.clone()); - - (Arc::new(format), path, DEFAULT_PARQUET_EXTENSION) - } - other => { - unimplemented!("Invalid file format '{}'", other); - } - }; - - let table_path = ListingTableUrl::parse(path)?; - let options = ListingOptions::new(format) - .with_file_extension(extension) - .with_target_partitions(target_partitions) - .with_collect_stat(state.config().collect_statistics()); - let schema = match table_format { - "parquet" => options.infer_schema(&state, &table_path).await?, - "tbl" => Arc::new(get_tbl_tpch_table_schema(table)), - "csv" => Arc::new(get_tpch_table_schema(table)), - _ => unreachable!(), - }; - let options = if self.sorted { - let key_column_name = schema.fields()[0].name(); - options.with_file_sort_order(vec![vec![col(key_column_name).sort(true, false)]]) - } else { - options - }; - - let config = ListingTableConfig::new(table_path) - .with_listing_options(options) - .with_schema(schema); - - Ok(Arc::new(ListingTable::try_new(config)?)) - } - - fn iterations(&self) -> usize { - self.common.iterations - } - - fn partitions(&self) -> usize { - if let Some(partitions) = self.common.partitions { - return partitions; - } - if let Some(threads) = self.threads { - return threads; - } - get_available_parallelism() - } -} diff --git a/benchmarks/src/util/memory.rs b/benchmarks/src/util/memory.rs deleted file mode 100644 index 61f7e451..00000000 --- a/benchmarks/src/util/memory.rs +++ /dev/null @@ -1,215 +0,0 @@ -use dashmap::{DashMap, Entry}; -use datafusion::arrow::record_batch::RecordBatch; -use datafusion::common::tree_node::{Transformed, TreeNode}; -use datafusion::common::{exec_err, extensions_options, plan_err}; -use datafusion::config::{ConfigExtension, ConfigOptions}; -use datafusion::error::DataFusionError; -use datafusion::execution::{FunctionRegistry, SendableRecordBatchStream, TaskContext}; -use datafusion::physical_optimizer::PhysicalOptimizerRule; -use datafusion::physical_plan::stream::RecordBatchStreamAdapter; -use datafusion::physical_plan::{ - DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties, displayable, -}; -use datafusion_proto::physical_plan::PhysicalExtensionCodec; -use futures::{FutureExt, StreamExt}; -use prost::Message; -use std::any::Any; -use std::fmt::Formatter; -use std::sync::{Arc, LazyLock}; -use tokio::sync::OnceCell; - -type Key = (String, usize); -type Value = Arc>>; -static CACHE: LazyLock> = LazyLock::new(DashMap::default); - -/// Caches all the record batches in a global [CACHE] on the first run, and serves -/// them from the cache in any subsequent run. -#[derive(Debug, Clone)] -pub struct InMemoryCacheExec { - inner: Arc, -} - -extensions_options! { - /// Marker used by the [InMemoryCacheExec] that determines wether its fine - /// to load data from disk because we are warming up, or not. - /// - /// If this marker is not present during InMemoryCacheExec::execute(), and - /// the data was not loaded in-memory already, the query will fail. - pub struct WarmingUpMarker { - is_warming_up: bool, default = false - } -} - -impl ConfigExtension for WarmingUpMarker { - const PREFIX: &'static str = "in-memory-cache-exec"; -} - -impl WarmingUpMarker { - pub fn warming_up() -> Self { - Self { - is_warming_up: true, - } - } -} - -impl ExecutionPlan for InMemoryCacheExec { - fn name(&self) -> &str { - "InMemoryDataSourceExec" - } - - fn as_any(&self) -> &dyn Any { - self - } - - fn properties(&self) -> &PlanProperties { - self.inner.properties() - } - - fn children(&self) -> Vec<&Arc> { - vec![&self.inner] - } - - fn with_new_children( - self: Arc, - children: Vec>, - ) -> datafusion::common::Result> { - Ok(Arc::new(Self { - inner: children[0].clone(), - })) - } - - fn execute( - &self, - partition: usize, - context: Arc, - ) -> datafusion::common::Result { - let once = { - let inner_display = displayable(self.inner.as_ref()).one_line().to_string(); - let entry = CACHE.entry((inner_display, partition)); - if matches!(entry, Entry::Vacant(_)) - && !context - .session_config() - .options() - .extensions - .get::() - .map(|v| v.is_warming_up) - .unwrap_or_default() - { - return exec_err!("InMemoryCacheExec is not yet warmed up"); - } - let once = entry.or_insert(Arc::new(OnceCell::new())); - once.value().clone() - }; - - let inner = Arc::clone(&self.inner); - - let stream = async move { - let batches = once - .get_or_try_init(|| async move { - let mut stream = inner.execute(partition, context)?; - let mut batches = vec![]; - while let Some(batch) = stream.next().await { - batches.push(batch?); - } - Ok::<_, DataFusionError>(batches) - }) - .await?; - Ok(batches.clone()) - } - .into_stream() - .map(|v| match v { - Ok(batch) => futures::stream::iter(batch.into_iter().map(Ok)).boxed(), - Err(err) => futures::stream::once(async { Err(err) }).boxed(), - }) - .flatten(); - - Ok(Box::pin(RecordBatchStreamAdapter::new( - self.inner.schema(), - stream, - ))) - } -} - -impl DisplayAs for InMemoryCacheExec { - fn fmt_as(&self, _t: DisplayFormatType, f: &mut Formatter) -> std::fmt::Result { - writeln!(f, "InMemoryDataSourceExec") - } -} - -#[derive(Clone, PartialEq, ::prost::Message)] -struct InMemoryCacheExecProto { - #[prost(string, tag = "1")] - name: String, -} - -#[derive(Debug)] -pub struct InMemoryCacheExecCodec; - -impl PhysicalExtensionCodec for InMemoryCacheExecCodec { - fn try_decode( - &self, - buf: &[u8], - inputs: &[Arc], - _registry: &dyn FunctionRegistry, - ) -> datafusion::common::Result> { - let Ok(proto) = InMemoryCacheExecProto::decode(buf) else { - return plan_err!("no InMemoryDataSourceExecProto"); - }; - if proto.name != "InMemoryDataSourceExec" { - return plan_err!("unsupported InMemoryDataSourceExec proto: {:?}", proto.name); - }; - Ok(Arc::new(InMemoryCacheExec { - inner: inputs[0].clone(), - })) - } - - fn try_encode( - &self, - node: Arc, - buf: &mut Vec, - ) -> datafusion::common::Result<()> { - if !node.as_any().is::() { - return plan_err!("no InMemoryDataSourceExec"); - }; - let proto = InMemoryCacheExecProto { - name: "InMemoryDataSourceExec".to_string(), - }; - let Ok(_) = proto.encode(buf) else { - return plan_err!("no InMemoryDataSourceExecProto"); - }; - - Ok(()) - } -} - -/// Wraps any plan without children with an [InMemoryCacheExec] node. -#[derive(Debug)] -pub struct InMemoryDataSourceRule; - -impl PhysicalOptimizerRule for InMemoryDataSourceRule { - fn optimize( - &self, - plan: Arc, - _config: &ConfigOptions, - ) -> datafusion::common::Result> { - Ok(plan - .transform_up(|plan| { - if plan.children().is_empty() { - Ok(Transformed::yes(Arc::new(InMemoryCacheExec { - inner: plan.clone(), - }))) - } else { - Ok(Transformed::no(plan)) - } - })? - .data) - } - - fn name(&self) -> &str { - "InMemoryDataSourceRule" - } - - fn schema_check(&self) -> bool { - true - } -} diff --git a/benchmarks/src/util/mod.rs b/benchmarks/src/util/mod.rs deleted file mode 100644 index 40d8d956..00000000 --- a/benchmarks/src/util/mod.rs +++ /dev/null @@ -1,25 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! Shared benchmark utilities -mod memory; -mod options; -mod run; - -pub use memory::{InMemoryCacheExecCodec, InMemoryDataSourceRule, WarmingUpMarker}; -pub use options::CommonOpt; -pub use run::{BenchmarkRun, QueryIter}; diff --git a/benchmarks/src/util/options.rs b/benchmarks/src/util/options.rs deleted file mode 100644 index 0d838685..00000000 --- a/benchmarks/src/util/options.rs +++ /dev/null @@ -1,154 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use std::{num::NonZeroUsize, sync::Arc}; - -use datafusion::common::{DataFusionError, Result}; -use datafusion::{ - execution::{ - disk_manager::DiskManagerBuilder, - memory_pool::{FairSpillPool, GreedyMemoryPool, MemoryPool, TrackConsumersPool}, - runtime_env::RuntimeEnvBuilder, - }, - prelude::SessionConfig, -}; -use structopt::StructOpt; - -// Common benchmark options (don't use doc comments otherwise this doc -// shows up in help files) -#[derive(Debug, StructOpt, Clone)] -pub struct CommonOpt { - /// Number of iterations of each test run - #[structopt(short = "i", long = "iterations", default_value = "3")] - pub iterations: usize, - - /// Number of partitions to process in parallel. Defaults to number of available cores. - /// Should typically be less or equal than --threads. - #[structopt(short = "n", long = "partitions")] - pub partitions: Option, - - /// Batch size when reading CSV or Parquet files - #[structopt(short = "s", long = "batch-size")] - pub batch_size: Option, - - /// The memory pool type to use, should be one of "fair" or "greedy" - #[structopt(long = "mem-pool-type", default_value = "fair")] - pub mem_pool_type: String, - - /// Memory limit (e.g. '100M', '1.5G'). If not specified, run all pre-defined memory limits for given query - /// if there's any, otherwise run with no memory limit. - #[structopt(long = "memory-limit", parse(try_from_str = parse_memory_limit))] - pub memory_limit: Option, - - /// The amount of memory to reserve for sort spill operations. DataFusion's default value will be used - /// if not specified. - #[structopt(long = "sort-spill-reservation-bytes", parse(try_from_str = parse_memory_limit))] - pub sort_spill_reservation_bytes: Option, - - /// Activate debug mode to see more details - #[structopt(short, long)] - pub debug: bool, -} - -impl CommonOpt { - /// Return an appropriately configured `SessionConfig` - pub fn config(&self) -> Result { - SessionConfig::from_env().map(|config| self.update_config(config)) - } - - /// Modify the existing config appropriately - pub fn update_config(&self, mut config: SessionConfig) -> SessionConfig { - if let Some(batch_size) = self.batch_size { - config = config.with_batch_size(batch_size); - } - - if let Some(partitions) = self.partitions { - config = config.with_target_partitions(partitions); - } - - if let Some(sort_spill_reservation_bytes) = self.sort_spill_reservation_bytes { - config = config.with_sort_spill_reservation_bytes(sort_spill_reservation_bytes); - } - - config - } - - /// Return an appropriately configured `RuntimeEnvBuilder` - pub fn runtime_env_builder(&self) -> Result { - let mut rt_builder = RuntimeEnvBuilder::new(); - const NUM_TRACKED_CONSUMERS: usize = 5; - if let Some(memory_limit) = self.memory_limit { - let pool: Arc = match self.mem_pool_type.as_str() { - "fair" => Arc::new(TrackConsumersPool::new( - FairSpillPool::new(memory_limit), - NonZeroUsize::new(NUM_TRACKED_CONSUMERS).unwrap(), - )), - "greedy" => Arc::new(TrackConsumersPool::new( - GreedyMemoryPool::new(memory_limit), - NonZeroUsize::new(NUM_TRACKED_CONSUMERS).unwrap(), - )), - _ => { - return Err(DataFusionError::Configuration(format!( - "Invalid memory pool type: {}", - self.mem_pool_type - ))); - } - }; - rt_builder = rt_builder - .with_memory_pool(pool) - .with_disk_manager_builder(DiskManagerBuilder::default()); - } - Ok(rt_builder) - } -} - -/// Parse memory limit from string to number of bytes -/// e.g. '1.5G', '100M' -> 1572864 -fn parse_memory_limit(limit: &str) -> Result { - let (number, unit) = limit.split_at(limit.len() - 1); - let number: f64 = number - .parse() - .map_err(|_| format!("Failed to parse number from memory limit '{limit}'"))?; - - match unit { - "K" => Ok((number * 1024.0) as usize), - "M" => Ok((number * 1024.0 * 1024.0) as usize), - "G" => Ok((number * 1024.0 * 1024.0 * 1024.0) as usize), - _ => Err(format!( - "Unsupported unit '{unit}' in memory limit '{limit}'" - )), - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_parse_memory_limit_all() { - // Test valid inputs - assert_eq!(parse_memory_limit("100K").unwrap(), 102400); - assert_eq!(parse_memory_limit("1.5M").unwrap(), 1572864); - assert_eq!(parse_memory_limit("2G").unwrap(), 2147483648); - - // Test invalid unit - assert!(parse_memory_limit("500X").is_err()); - - // Test invalid number - assert!(parse_memory_limit("abcM").is_err()); - } -} diff --git a/benchmarks/src/util/run.rs b/benchmarks/src/util/run.rs deleted file mode 100644 index 48569e8c..00000000 --- a/benchmarks/src/util/run.rs +++ /dev/null @@ -1,270 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use chrono::{DateTime, Utc}; -use datafusion::common::utils::get_available_parallelism; -use datafusion::{DATAFUSION_VERSION, error::Result}; -use serde::{Deserialize, Deserializer, Serialize, Serializer}; -use std::{ - path::Path, - time::{Duration, SystemTime}, -}; - -fn serialize_start_time(start_time: &SystemTime, ser: S) -> Result -where - S: Serializer, -{ - ser.serialize_u64( - start_time - .duration_since(SystemTime::UNIX_EPOCH) - .expect("current time is later than UNIX_EPOCH") - .as_secs(), - ) -} -fn deserialize_start_time<'de, D>(des: D) -> Result -where - D: Deserializer<'de>, -{ - let secs = u64::deserialize(des)?; - Ok(SystemTime::UNIX_EPOCH + Duration::from_secs(secs)) -} - -fn serialize_elapsed(elapsed: &Duration, ser: S) -> Result -where - S: Serializer, -{ - let ms = elapsed.as_secs_f64() * 1000.0; - ser.serialize_f64(ms) -} -fn deserialize_elapsed<'de, D>(des: D) -> Result -where - D: Deserializer<'de>, -{ - let ms = f64::deserialize(des)?; - Ok(Duration::from_secs_f64(ms / 1000.0)) -} -#[derive(Debug, Serialize, Deserialize)] -pub struct RunContext { - /// Benchmark crate version - pub benchmark_version: String, - /// DataFusion crate version - pub datafusion_version: String, - /// Number of CPU cores - pub num_cpus: usize, - /// Number of workers involved in a distributed query - pub workers: usize, - /// Number of physical threads used per worker - pub threads: usize, - /// Start time - #[serde( - serialize_with = "serialize_start_time", - deserialize_with = "deserialize_start_time" - )] - pub start_time: SystemTime, - /// CLI arguments - pub arguments: Vec, -} - -impl RunContext { - pub fn new(workers: usize, threads: usize) -> Self { - Self { - benchmark_version: env!("CARGO_PKG_VERSION").to_owned(), - datafusion_version: DATAFUSION_VERSION.to_owned(), - num_cpus: get_available_parallelism(), - workers, - threads, - start_time: SystemTime::now(), - arguments: std::env::args().skip(1).collect::>(), - } - } -} - -/// A single iteration of a benchmark query -#[derive(Debug, Serialize, Deserialize)] -pub struct QueryIter { - #[serde( - serialize_with = "serialize_elapsed", - deserialize_with = "deserialize_elapsed" - )] - pub elapsed: Duration, - pub row_count: usize, - pub n_tasks: usize, -} -/// A single benchmark case -#[derive(Debug, Serialize, Deserialize)] -pub struct BenchQuery { - query: String, - iterations: Vec, - #[serde( - serialize_with = "serialize_start_time", - deserialize_with = "deserialize_start_time" - )] - start_time: SystemTime, - success: bool, -} - -/// collects benchmark run data and then serializes it at the end -#[derive(Debug, Serialize, Deserialize)] -pub struct BenchmarkRun { - context: RunContext, - queries: Vec, - current_case: Option, -} - -impl BenchmarkRun { - // create new - pub fn new(workers: usize, threads: usize) -> Self { - Self { - context: RunContext::new(workers, threads), - queries: vec![], - current_case: None, - } - } - /// begin a new case. iterations added after this will be included in the new case - pub fn start_new_case(&mut self, id: &str) { - self.queries.push(BenchQuery { - query: id.to_owned(), - iterations: vec![], - start_time: SystemTime::now(), - success: true, - }); - if let Some(c) = self.current_case.as_mut() { - *c += 1; - } else { - self.current_case = Some(0); - } - } - /// Write a new iteration to the current case - pub fn write_iter(&mut self, query_iter: QueryIter) { - if let Some(idx) = self.current_case { - self.queries[idx].iterations.push(query_iter) - } else { - panic!("no cases existed yet"); - } - } - - /// Print the names of failed queries, if any - pub fn maybe_print_failures(&self) { - let failed_queries: Vec<&str> = self - .queries - .iter() - .filter_map(|q| (!q.success).then_some(q.query.as_str())) - .collect(); - - if !failed_queries.is_empty() { - println!("Failed Queries: {}", failed_queries.join(", ")); - } - } - - /// Mark current query - pub fn mark_failed(&mut self) { - if let Some(idx) = self.current_case { - self.queries[idx].success = false; - } else { - unreachable!("Cannot mark failure: no current case"); - } - } - - /// Stringify data into formatted json - pub fn to_json(&self) -> String { - serde_json::to_string_pretty(&self).unwrap() - } - - /// Write data as json into output path if it exists. - pub fn maybe_write_json(&self, maybe_path: Option>) -> Result<()> { - if let Some(path) = maybe_path { - std::fs::write(path, self.to_json())?; - }; - Ok(()) - } - - pub fn maybe_compare_with_previous(&self, maybe_path: Option>) -> Result<()> { - let Some(path) = maybe_path else { - return Ok(()); - }; - let Ok(prev) = std::fs::read(path) else { - return Ok(()); - }; - - let Ok(prev_output) = serde_json::from_slice::(&prev) else { - return Ok(()); - }; - - let mut header_printed = false; - for query in self.queries.iter() { - let Some(prev_query) = prev_output.queries.iter().find(|v| v.query == query.query) - else { - continue; - }; - if prev_query.iterations.is_empty() { - continue; - } - if query.iterations.is_empty() { - println!("{}: Failed ❌", query.query); - continue; - } - - let avg_prev = prev_query.avg(); - let avg = query.avg(); - let (f, tag, emoji) = if avg < avg_prev { - let f = avg_prev as f64 / avg as f64; - (f, "faster", if f > 1.2 { "✅" } else { "✔" }) - } else { - let f = avg as f64 / avg_prev as f64; - (f, "slower", if f > 1.2 { "❌" } else { "✖" }) - }; - if !header_printed { - header_printed = true; - let datetime: DateTime = prev_query.start_time.into(); - let header = format!( - "==== Comparison with the previous benchmark from {} ====", - datetime.format("%Y-%m-%d %H:%M:%S UTC") - ); - println!("{header}"); - // Print machine information - println!("os: {}", std::env::consts::OS); - println!("arch: {}", std::env::consts::ARCH); - println!("cpu cores: {}", get_available_parallelism()); - println!( - "threads: {} -> {}", - prev_output.context.threads, self.context.threads - ); - println!( - "workers: {} -> {}", - prev_output.context.workers, self.context.workers - ); - println!("{}", "=".repeat(header.len())) - } - println!( - "{:>8}: prev={avg_prev:>4} ms, new={avg:>4} ms, diff={f:.2} {tag} {emoji}", - query.query - ); - } - - Ok(()) - } -} - -impl BenchQuery { - fn avg(&self) -> u128 { - self.iterations - .iter() - .map(|v| v.elapsed.as_millis()) - .sum::() - / self.iterations.len() as u128 - } -} diff --git a/cli/Cargo.toml b/cli/Cargo.toml new file mode 100644 index 00000000..5e355dec --- /dev/null +++ b/cli/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "datafusion-distributed-cli" +version = "0.1.0" +edition = "2024" + +[dependencies] +datafusion = { workspace = true } +datafusion-distributed = { path = "..", features = ["avro", "integration"] } +datafusion-cli = { version = "52", default-features = false } +tokio = { version = "1.48", features = ["full"] } +clap = { version = "4", features = ["derive"] } +env_logger = "0.11" +dirs = "6" +arrow-flight = "57.1.0" +tonic = { version = "0.14.1", features = ["transport"] } +tower = "0.5.2" +hyper-util = "0.1.16" +tokio-stream = "0.1.17" +async-trait = "0.1.89" +url = "2.5.7" + +[[bin]] +name = "datafusion-distributed-cli" +path = "src/main.rs" diff --git a/cli/README.md b/cli/README.md new file mode 100644 index 00000000..a82122f4 --- /dev/null +++ b/cli/README.md @@ -0,0 +1,37 @@ +# Distributed DataFusion CLI + +This is the same CLI as https://github.com/apache/datafusion/blob/main/datafusion-cli/, +but enriched with distributed execution capabilities. + +## Installation + +The CLI can be installed from source by cloning this repository: + +```bash +git clone https://github.com/datafusion-contrib/datafusion-distributed +cd datafusion-distributed +``` + +And running: + +```bash +cargo install --path cli +``` + +## Usage + +The CLI can be invoked by running: + +```bash +datafusion-distributed-cli +``` + +The best way of trying distributed queries is by issuing queries against the parquet files in the +`testdata` directory. For maximum parallelism, set the following config options: + +```sql +SET distributed.files_per_task = 1; +``` + +The usage is exactly the same as the original CLI: +https://datafusion.apache.org/user-guide/cli/usage.html \ No newline at end of file diff --git a/cli/src/main.rs b/cli/src/main.rs new file mode 100644 index 00000000..fee4fc45 --- /dev/null +++ b/cli/src/main.rs @@ -0,0 +1,264 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// File mainly copied from https://github.com/apache/datafusion/blob/main/datafusion-cli/src/main.rs + +use clap::Parser; +use datafusion::common::config_err; +use datafusion::config::ConfigOptions; +use datafusion::error::{DataFusionError, Result}; +use datafusion::execution::SessionStateBuilder; +use datafusion::execution::context::SessionConfig; +use datafusion::execution::runtime_env::RuntimeEnvBuilder; +use datafusion::logical_expr::ExplainFormat; +use datafusion::prelude::SessionContext; +use datafusion_cli::catalog::DynamicObjectStoreCatalog; +use datafusion_cli::object_storage::instrumented::InstrumentedObjectStoreRegistry; +use datafusion_cli::{ + DATAFUSION_CLI_VERSION, exec, + print_format::PrintFormat, + print_options::{MaxRows, PrintOptions}, +}; +use datafusion_distributed::test_utils::in_memory_channel_resolver::{ + InMemoryChannelResolver, InMemoryWorkerResolver, +}; +use datafusion_distributed::{DistributedExt, DistributedPhysicalOptimizerRule}; +use std::env; +use std::path::Path; +use std::process::ExitCode; +use std::sync::Arc; + +#[derive(Debug, Parser, PartialEq)] +#[clap(author, version, about, long_about= None)] +struct Args { + #[clap( + short = 'p', + long, + help = "Path to your data, default to current directory", + value_parser(parse_valid_data_dir) + )] + data_path: Option, + + #[clap( + short = 'b', + long, + help = "The batch size of each query, or use DataFusion default", + value_parser(parse_batch_size) + )] + batch_size: Option, + + #[clap( + short = 'c', + long, + num_args = 0.., + help = "Execute the given command string(s), then exit. Commands are expected to be non empty.", + value_parser(parse_command) + )] + command: Vec, + + #[clap( + short, + long, + num_args = 0.., + help = "Execute commands from file(s), then exit", + value_parser(parse_valid_file) + )] + file: Vec, + + #[clap( + short = 'r', + long, + num_args = 0.., + help = "Run the provided files on startup instead of ~/.datafusionrc", + value_parser(parse_valid_file), + conflicts_with = "file" + )] + rc: Option>, + + #[clap(long, value_enum, default_value_t = PrintFormat::Automatic)] + format: PrintFormat, + + #[clap( + short, + long, + help = "Reduce printing other than the results and work quietly" + )] + quiet: bool, + + #[clap( + long, + help = "The max number of rows to display for 'Table' format\n[possible values: numbers(0/10/...), inf(no limit)]", + default_value = "40" + )] + maxrows: MaxRows, + + #[clap(long, help = "Enables console syntax highlighting")] + color: bool, +} + +#[tokio::main] +/// Calls [`main_inner`], then handles printing errors and returning the correct exit code +pub async fn main() -> ExitCode { + if let Err(e) = main_inner().await { + println!("Error: {e}"); + return ExitCode::FAILURE; + } + + ExitCode::SUCCESS +} + +/// Main CLI entrypoint +async fn main_inner() -> Result<()> { + env_logger::init(); + let args = Args::parse(); + + if !args.quiet { + println!("Distributed DataFusion CLI v{DATAFUSION_CLI_VERSION}"); + } + + if let Some(ref path) = args.data_path { + let p = Path::new(path); + env::set_current_dir(p)?; + }; + + let session_config = get_session_config(&args)?; + + let mut rt_builder = RuntimeEnvBuilder::new(); + + let instrumented_registry = Arc::new(InstrumentedObjectStoreRegistry::new()); + rt_builder = rt_builder.with_object_store_registry(instrumented_registry.clone()); + + let runtime_env = rt_builder.build_arc()?; + + let state = SessionStateBuilder::new() + .with_default_features() + .with_config(session_config) + .with_runtime_env(runtime_env) + .with_physical_optimizer_rule(Arc::new(DistributedPhysicalOptimizerRule)) + .with_distributed_worker_resolver(InMemoryWorkerResolver::new(16)) + .with_distributed_channel_resolver(InMemoryChannelResolver::default()) + .build(); + + // enable dynamic file query + let ctx = SessionContext::from(state).enable_url_table(); + ctx.refresh_catalogs().await?; + // install dynamic catalog provider that can register required object stores + ctx.register_catalog_list(Arc::new(DynamicObjectStoreCatalog::new( + ctx.state().catalog_list().clone(), + ctx.state_weak_ref(), + ))); + + let mut print_options = PrintOptions { + format: args.format, + quiet: args.quiet, + maxrows: args.maxrows, + color: args.color, + instrumented_registry: Arc::clone(&instrumented_registry), + }; + + let commands = args.command; + let files = args.file; + let rc = args.rc.unwrap_or_else(|| { + let mut files = Vec::new(); + let home = dirs::home_dir(); + if let Some(p) = home { + let home_rc = p.join(".datafusionrc"); + if home_rc.exists() { + files.push(home_rc.into_os_string().into_string().unwrap()); + } + } + files + }); + + if commands.is_empty() && files.is_empty() { + if !rc.is_empty() { + exec::exec_from_files(&ctx, rc, &print_options).await?; + } + return exec::exec_from_repl(&ctx, &mut print_options) + .await + .map_err(|e| DataFusionError::External(Box::new(e))); + } + + if !files.is_empty() { + exec::exec_from_files(&ctx, files, &print_options).await?; + } + + if !commands.is_empty() { + exec::exec_from_commands(&ctx, commands, &print_options).await?; + } + + Ok(()) +} + +/// Get the session configuration based on the provided arguments +/// and environment settings. +fn get_session_config(args: &Args) -> Result { + // Read options from environment variables and merge with command line options + let mut config_options = ConfigOptions::from_env()?; + + if let Some(batch_size) = args.batch_size { + if batch_size == 0 { + return config_err!("batch_size must be greater than 0"); + } + config_options.execution.batch_size = batch_size; + }; + + // use easier to understand "tree" mode by default + // if the user hasn't specified an explain format in the environment + if env::var_os("DATAFUSION_EXPLAIN_FORMAT").is_none() { + config_options.explain.format = ExplainFormat::Indent; + } + + // in the CLI, we want to show NULL values rather the empty strings + if env::var_os("DATAFUSION_FORMAT_NULL").is_none() { + config_options.format.null = String::from("NULL"); + } + + let session_config = SessionConfig::from(config_options).with_information_schema(true); + Ok(session_config) +} + +fn parse_valid_file(dir: &str) -> Result { + if Path::new(dir).is_file() { + Ok(dir.to_string()) + } else { + Err(format!("Invalid file '{dir}'")) + } +} + +fn parse_valid_data_dir(dir: &str) -> Result { + if Path::new(dir).is_dir() { + Ok(dir.to_string()) + } else { + Err(format!("Invalid data directory '{dir}'")) + } +} + +fn parse_batch_size(size: &str) -> Result { + match size.parse::() { + Ok(size) if size > 0 => Ok(size), + _ => Err(format!("Invalid batch size '{size}'")), + } +} + +fn parse_command(command: &str) -> Result { + if !command.is_empty() { + Ok(command.to_string()) + } else { + Err("-c flag expects only non empty commands".to_string()) + } +} diff --git a/console/.gitignore b/console/.gitignore new file mode 100644 index 00000000..24d76ea8 --- /dev/null +++ b/console/.gitignore @@ -0,0 +1 @@ +testdata/ diff --git a/console/Cargo.toml b/console/Cargo.toml new file mode 100644 index 00000000..d5ab0fff --- /dev/null +++ b/console/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "datafusion-distributed-console" +version = "0.1.0" +edition = "2024" + +[dependencies] +color-eyre = "0.6.5" +crossterm = "0.29.0" +ratatui = "0.30.0" +tokio = { version = "1.49.0", features = ["full"] } +tonic = "0.14.2" +datafusion-distributed = { path = "..", features = ["integration"] } +arrow-flight = "57.1.0" +structopt = "0.3.26" diff --git a/console/examples/README.md b/console/examples/README.md new file mode 100644 index 00000000..a497eaac --- /dev/null +++ b/console/examples/README.md @@ -0,0 +1,31 @@ +# Console Example + +This example runs a shared worker instance that exposes the observability +service and then starts the console UI that connects to it. + +## Run the worker + +In one terminal from the repository root: + +```bash +cargo run -p datafusion-distributed-console --example console_worker +``` + +The worker listens on `127.0.0.1:8080` by default. Use `--port` to change it. +If you change the port, update the address in `console/src/main.rs`. + +## Run the console + +In a second terminal from the repository root: + +```bash +cargo run -p datafusion-distributed-console +``` + +This following message should appear: + +```bash +ping ok (value: 1) +``` + +Press any key to exit the console. diff --git a/console/examples/console_worker.rs b/console/examples/console_worker.rs new file mode 100644 index 00000000..7dc827bf --- /dev/null +++ b/console/examples/console_worker.rs @@ -0,0 +1,31 @@ +use datafusion_distributed::{ObservabilityServiceServer, Worker}; +use std::error::Error; +use std::net::{IpAddr, Ipv4Addr, SocketAddr}; +use structopt::StructOpt; +use tonic::transport::Server; + +#[derive(StructOpt)] +#[structopt( + name = "console_worker", + about = "A localhost DataFusion worker with observability" +)] +struct Args { + #[structopt(default_value = "8080")] + port: u16, +} + +#[tokio::main] +async fn main() -> Result<(), Box> { + let args = Args::from_args(); + + let worker = Worker::default(); + let observability_service = ObservabilityServiceServer::new(worker.observability_service()); + + Server::builder() + .add_service(worker.into_flight_server()) + .add_service(observability_service) + .serve(SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), args.port)) + .await?; + + Ok(()) +} diff --git a/console/src/main.rs b/console/src/main.rs new file mode 100644 index 00000000..0e56128f --- /dev/null +++ b/console/src/main.rs @@ -0,0 +1,29 @@ +use datafusion_distributed::ObservabilityServiceClient; +use datafusion_distributed::PingRequest; +use ratatui::widgets::Paragraph; +use ratatui::{DefaultTerminal, Frame}; + +#[tokio::main] +async fn main() -> color_eyre::Result<()> { + let mut client = + ObservabilityServiceClient::connect("http://127.0.0.1:8080".to_string()).await?; + let resp = client.ping(PingRequest {}).await?; + let message = format!("ping ok (value: {})", resp.into_inner().value); + + color_eyre::install()?; + ratatui::run(|terminal| app(terminal, &message))?; + Ok(()) +} + +fn app(terminal: &mut DefaultTerminal, message: &str) -> std::io::Result<()> { + loop { + terminal.draw(|frame| render(frame, message))?; + if crossterm::event::read()?.is_key_press() { + break Ok(()); + } + } +} + +fn render(frame: &mut Frame, message: &str) { + frame.render_widget(Paragraph::new(message), frame.area()); +} diff --git a/docs/.gitignore b/docs/.gitignore new file mode 100644 index 00000000..5b478e67 --- /dev/null +++ b/docs/.gitignore @@ -0,0 +1,3 @@ +build/ +venv/ +temp/ diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 00000000..198eedd8 --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,19 @@ +# Minimal makefile for Sphinx documentation + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= -W +SPHINXBUILD ?= sphinx-build +SOURCEDIR = source +BUILDDIR = build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 00000000..a7f29074 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,45 @@ +# DataFusion Distributed Documentation + +This directory contains the documentation for DataFusion Distributed, built using [Sphinx](https://www.sphinx-doc.org/). + +## Building the Documentation + +### Prerequisites + +Install the required dependencies: + +```bash +pip install -r requirements.txt +``` + +### Build HTML Documentation + +```bash +make html +``` + +The generated documentation will be available in `build/html/index.html`. + +### Clean Build Files + +```bash +make clean +``` + +## Documentation Structure + +- `source/` - Documentation source files (reStructuredText and Markdown) + - `user-guide/` - User-facing documentation + - `architecture/` - Architecture documentation + - `contributor-guide/` - Contributor documentation + - `_static/` - Static files (images, CSS, etc.) + - `_templates/` - Custom templates + +## Contributing + +When adding new documentation: + +1. Create new `.md` or `.rst` files in the appropriate subdirectory +2. Add references to new files in the relevant `index.rst` or `index.md` +3. Build and preview your changes locally +4. Ensure all links and references work correctly diff --git a/docs/build.sh b/docs/build.sh new file mode 100755 index 00000000..a0aef503 --- /dev/null +++ b/docs/build.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +make clean +make html diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 00000000..2a9bb3af --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,7 @@ +sphinx==8.2.3 +sphinx-reredirects==1.0.0 +pydata-sphinx-theme==0.16.1 +myst-parser==4.0.1 +maturin==1.10.2 +jinja2==3.1.6 +setuptools==80.9.0 diff --git a/docs/source/_static/images/concepts.png b/docs/source/_static/images/concepts.png new file mode 100644 index 00000000..60c89f0c --- /dev/null +++ b/docs/source/_static/images/concepts.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3714f57b3815db2004a1f6445298256b3f63cac9882e5186f901357e688fcbee +size 262763 diff --git a/docs/source/_static/images/dist-df-vs-df-vs-trino.png b/docs/source/_static/images/dist-df-vs-df-vs-trino.png new file mode 100644 index 00000000..0075bb32 --- /dev/null +++ b/docs/source/_static/images/dist-df-vs-df-vs-trino.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:71aa64a287979d6a7c4ee5871e10c28eb83134e9ab3960ac1d4aa3534876a923 +size 86500 diff --git a/docs/source/_static/images/img.png b/docs/source/_static/images/img.png new file mode 100644 index 00000000..a6d31ba4 --- /dev/null +++ b/docs/source/_static/images/img.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ca45af071afb0d46fcc63cafd55c02725cc82fbf79fe642613428308799a72f9 +size 100712 diff --git a/docs/source/_static/images/img_1.png b/docs/source/_static/images/img_1.png new file mode 100644 index 00000000..338066e9 --- /dev/null +++ b/docs/source/_static/images/img_1.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b878a69b0e3fba19e6901a35007e47502689ff320c7c18b2f184818da7eb944c +size 139713 diff --git a/docs/source/_static/images/img_2.png b/docs/source/_static/images/img_2.png new file mode 100644 index 00000000..338066e9 --- /dev/null +++ b/docs/source/_static/images/img_2.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b878a69b0e3fba19e6901a35007e47502689ff320c7c18b2f184818da7eb944c +size 139713 diff --git a/docs/source/_static/images/img_3.png b/docs/source/_static/images/img_3.png new file mode 100644 index 00000000..7ce125dd --- /dev/null +++ b/docs/source/_static/images/img_3.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:74443b0130d4d583c9e899ebccd952e5fe20c10685b92894ef5ff8413981ff77 +size 181525 diff --git a/docs/source/_static/images/img_4.png b/docs/source/_static/images/img_4.png new file mode 100644 index 00000000..51137725 --- /dev/null +++ b/docs/source/_static/images/img_4.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5e2ad1163809b7ba54258925dd10eea593f8543e1627279469942e9032babb58 +size 212238 diff --git a/docs/source/_static/images/img_5.png b/docs/source/_static/images/img_5.png new file mode 100644 index 00000000..a45ac37d --- /dev/null +++ b/docs/source/_static/images/img_5.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:562371508231c0532dca522997ad1153466003c01ebe86367df818db267a72aa +size 258118 diff --git a/docs/source/_static/images/img_6.png b/docs/source/_static/images/img_6.png new file mode 100644 index 00000000..6f28232a --- /dev/null +++ b/docs/source/_static/images/img_6.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:292388062f20615b842a08ac1e73e09ab309f8f4a340e7e9d9db8a5b5b3661dc +size 307749 diff --git a/docs/source/_static/images/img_7.png b/docs/source/_static/images/img_7.png new file mode 100644 index 00000000..418b2481 --- /dev/null +++ b/docs/source/_static/images/img_7.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eb71ed8bcd09c75d5555a21701cd71b8f675f69a81c75a1863296ee7179b84d1 +size 359725 diff --git a/docs/source/_static/images/img_8.png b/docs/source/_static/images/img_8.png new file mode 100644 index 00000000..192b21d8 --- /dev/null +++ b/docs/source/_static/images/img_8.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ad0fed45f388743d753c79746e6a0b2bd7885f5f30f26368ee3b154c525074bc +size 403514 diff --git a/docs/source/_static/theme_overrides.css b/docs/source/_static/theme_overrides.css new file mode 100644 index 00000000..e6b68470 --- /dev/null +++ b/docs/source/_static/theme_overrides.css @@ -0,0 +1,144 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +/* Customizing with theme CSS variables */ + +:root { + --pst-color-active-navigation: 215, 70, 51; + --pst-color-link-hover: 215, 70, 51; + --pst-color-headerlink: 215, 70, 51; + /* Use normal text color (like h3, ..) instead of primary color */ + --pst-color-h1: var(--color-text-base); + --pst-color-h2: var(--color-text-base); + /* Use softer blue from bootstrap's default info color */ + --pst-color-info: 23, 162, 184; +} + +code { + color: rgb(215, 70, 51); +} + +.footer { + text-align: center; +} + +/* Limit both light and dark mode logos in the navbar */ +.logo__image { + height: 32px; + width: auto; + max-height: 2.5rem; +} + +/* Display appropriate logo for dark and light mode */ +.light-logo { + display: inline; +} + +.dark-logo { + display: none; +} + +html[data-theme="dark"] .light-logo { + display: none; +} + +html[data-theme="dark"] .dark-logo { + display: inline; + background-color: transparent !important; +} + +/* Align search bar & theme switch right */ +.navbar-header-items__end { + margin-left: auto; +} + +/* This is the bootstrap CSS style for "table-striped". Since the theme does +not yet provide an easy way to configure this globally, it easier to simply +include this snippet here than updating each table in all rst files to +add ":class: table-striped" */ + +.table tbody tr:nth-of-type(odd) { + background-color: rgba(0, 0, 0, 0.05); +} + + +/* Limit the max height of the sidebar navigation section. Because in our +customized template, there is more content above the navigation, i.e. +larger logo: if we don't decrease the max-height, it will overlap with +the footer. +Details: 8rem for search box etc*/ + +@media (min-width: 720px) { + @supports (position:-webkit-sticky) or (position:sticky) { + .bd-links { + max-height: calc(100vh - 8rem) + } + } +} + + +/* Fix table text wrapping in RTD theme, + * see https://rackerlabs.github.io/docs-rackspace/tools/rtd-tables.html + */ + +@media screen { + table.docutils td { + /* !important prevents the common CSS stylesheets from overriding + this as on RTD they are loaded after this stylesheet */ + white-space: normal !important; + } +} + +/* Make wide tables scroll within the content area to avoid overlapping the + right sidebar. Prevents tables from bleeding underneath the sticky sidebar. */ +.bd-content table { + display: block; + overflow-x: auto; + -webkit-overflow-scrolling: touch; + max-width: 100%; +} + +/* Make table container width fit content instead of spanning full width. */ +.pst-scrollable-table-container { + display: inline-block; + overflow-x: auto; + max-width: 100%; +} + +/* Restore proper table display to maintain column alignment */ +.bd-content table thead, +.bd-content table tbody { + display: table-row-group; +} + +.bd-content table tr { + display: table-row; +} + +.bd-content table th, +.bd-content table td { + display: table-cell; + white-space: normal; +} + +/* Maintain striped styling when table scrolls */ +.bd-content table tbody tr:nth-of-type(odd) { + background-color: rgba(0, 0, 0, 0.03); +} diff --git a/docs/source/_templates/docs-sidebar.html b/docs/source/_templates/docs-sidebar.html new file mode 100644 index 00000000..fa3cd96b --- /dev/null +++ b/docs/source/_templates/docs-sidebar.html @@ -0,0 +1,10 @@ + \ No newline at end of file diff --git a/docs/source/_templates/layout.html b/docs/source/_templates/layout.html new file mode 100644 index 00000000..2d116dc4 --- /dev/null +++ b/docs/source/_templates/layout.html @@ -0,0 +1,8 @@ +{% extends "pydata_sphinx_theme/layout.html" %} + + +{% block footer %} + +{% endblock %} diff --git a/docs/source/conf.py b/docs/source/conf.py new file mode 100644 index 00000000..eac0ccfe --- /dev/null +++ b/docs/source/conf.py @@ -0,0 +1,122 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Configuration file for the Sphinx documentation builder. +# +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +import os +import sys + +# To pickup rustdoc_trim.py +sys.path.insert(0, os.path.abspath("..")) + +# -- Project information ----------------------------------------------------- + +project = "Distributed DataFusion" + +# -- General configuration --------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + "sphinx.ext.autodoc", + "sphinx.ext.autosummary", + "sphinx.ext.doctest", + "sphinx.ext.ifconfig", + "sphinx.ext.mathjax", + "sphinx.ext.viewcode", + "sphinx.ext.napoleon", + "myst_parser", + "sphinx_reredirects", +] + +source_suffix = { + ".rst": "restructuredtext", + ".md": "markdown", +} + +# Add any paths that contain templates here, relative to this directory. +templates_path = ["_templates"] + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = [] + +# Show members for classes in .. autosummary +autodoc_default_options = { + "members": None, + "undoc-members": None, + "show-inheritance": None, + "inherited-members": None, +} + +autosummary_generate = True + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = "pydata_sphinx_theme" + +html_theme_options = { + "logo": { + }, + "use_edit_page_button": True, + "navbar_center": [], + "navbar_end": ["theme-switcher"], +} + +html_context = { + "github_user": "apache", + "github_repo": "arrow-datafusion", + "github_version": "main", + "doc_path": "docs/source", +} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ["_static"] + +html_css_files = ["theme_overrides.css"] + +html_sidebars = { + "**": ["docs-sidebar.html"], +} + +# tell myst_parser to auto-generate anchor links for headers h1, h2, h3 +myst_heading_anchors = 3 + +# enable nice rendering of checkboxes for the task lists +myst_enable_extensions = ["colon_fence", "deflist", "tasklist"] + +# Some code blocks (sql) are not being highlighted correctly, due to the +# presence of some special characters like: 🚀, å, {,... But this isn’t a major +# issue for our documentation. So, suppress these warnings to keep our build +# log cleaner. +suppress_warnings = ["misc.highlighting_failure"] diff --git a/docs/source/contributor-guide/benchmarks.md b/docs/source/contributor-guide/benchmarks.md new file mode 100644 index 00000000..7176323c --- /dev/null +++ b/docs/source/contributor-guide/benchmarks.md @@ -0,0 +1,41 @@ +# Benchmarks + +There are two kinds of benchmarks in this project. + +## Local Benchmarks + +It's recommended to run these benchmarks locally when contributing to ensure there are no performance regressions. + +### Generating Test Data + +First, a TPCH dataset must be generated: + +```bash +cd benchmarks +SCALE_FACTOR=10 ./gen-tpch.sh +``` + +This might take a while. + +### Running Benchmarks + +After generating the data, it's recommended to use the `run.sh` script to run the benchmarks. +A good setup is to run 8 workers throttled at 2 physical threads per worker. This provides a relatively +accurate benchmarking environment for a distributed system locally. + +```bash +WORKERS=8 ./benchmarks/run.sh --threads 2 --path benchmarks/data/tpch_sf10 +``` + +Subsequent runs will compare results against the previous one, so a useful trick to measure the impact of a PR +is to first run the benchmarks on `main`, and then on the PR branch. + +More information about these benchmarks can be found in the [benchmarks README](https://github.com/datafusion-contrib/datafusion-distributed/blob/main/benchmarks/README.md). + +## Remote Benchmarks + +These benchmarks run on a remote EC2 cluster against parquet files stored in S3. These are the most realistic +benchmarks, but also the most expensive to run in terms of development iteration cycles (it requires AWS CDK deploys for +every code change) and cost, as it uses a real EC2 cluster. + +For running these benchmarks, refer to the [CDK benchmarks README](https://github.com/datafusion-contrib/datafusion-distributed/blob/main/benchmarks/cdk/README.md). diff --git a/docs/source/contributor-guide/index.md b/docs/source/contributor-guide/index.md new file mode 100644 index 00000000..75721058 --- /dev/null +++ b/docs/source/contributor-guide/index.md @@ -0,0 +1,7 @@ +# Index + +Welcome to the DataFusion Distributed contributor guide! + +- [Setup](setup.md) - Getting started with development +- [Tests](tests.md) - Running unit and integration tests +- [Benchmarks](benchmarks.md) - Local and remote performance benchmarks diff --git a/docs/source/contributor-guide/setup.md b/docs/source/contributor-guide/setup.md new file mode 100644 index 00000000..90db1274 --- /dev/null +++ b/docs/source/contributor-guide/setup.md @@ -0,0 +1,33 @@ +# Setup + +## Prerequisites + +- Rust toolchain (see `rust-toolchain.toml` for the required version) +- Git LFS for test data + +## Clone and Setup + +Clone the repository and set up test data: + +```bash +git clone https://github.com/datafusion-contrib/datafusion-distributed +cd datafusion-distributed +git lfs install +git lfs checkout +``` + +## Running Examples + +```bash +# In-memory cluster example +cargo run --example in_memory_cluster -- 'SELECT * FROM weather LIMIT 10' + +# Localhost workers (requires starting workers first in separate terminals) +cargo run --example localhost_worker -- 8080 --cluster-ports 8080,8081 +cargo run --example localhost_run -- 'SELECT * FROM weather LIMIT 10' --cluster-ports 8080,8081 +``` + +## Resources + +- [Examples directory](https://github.com/datafusion-contrib/datafusion-distributed/tree/main/examples) - Full working examples +- [Integration tests](https://github.com/datafusion-contrib/datafusion-distributed/tree/main/tests) - Feature-specific examples diff --git a/docs/source/contributor-guide/tests.md b/docs/source/contributor-guide/tests.md new file mode 100644 index 00000000..b61a0ffb --- /dev/null +++ b/docs/source/contributor-guide/tests.md @@ -0,0 +1,36 @@ +# Tests + +When submitting code, make sure it's always covered by tests. For every important feature, +it's recommended to add a dedicated integration test that tests it end-to-end. + +Please note that LLMs like to make very verbose and redundant tests even for simple things, +so before committing LLM-generated tests, review them and simplify them as much as possible. + +## Running Unit Tests + +Running unit tests provides the shortest feedback loop during development. + +```bash +# Run unit tests +cargo test +``` + +## Running Integration Tests + +Integration tests are slower but cover a wide range of functionality. + +```bash +# Run unit and integration tests +cargo test --features integration + +# Run TPCH integration tests +cargo test --features tpch + +# Run TPC-DS integration tests +cargo test --features tpc-ds +``` + +## Resources + +- [Integration tests directory](https://github.com/datafusion-contrib/datafusion-distributed/tree/main/tests) - + Feature-specific test examples diff --git a/docs/source/index.rst b/docs/source/index.rst new file mode 100644 index 00000000..ea80de49 --- /dev/null +++ b/docs/source/index.rst @@ -0,0 +1,32 @@ +======================== +DataFusion Distributed +======================== + +DataFusion Distributed is a library that enhances `Apache DataFusion `_ with distributed +capabilities. + +These docs will guide you towards using the library for building your own Distributed DataFusion cluster, and +how to contribute changes to the library yourself. + +.. _toc.guide: +.. toctree:: + :maxdepth: 2 + :caption: User Guide + + user-guide/concepts + user-guide/getting-started + user-guide/worker + user-guide/worker-resolver + user-guide/channel-resolver + user-guide/task-estimator + user-guide/how-a-distributed-plan-is-built + +.. _toc.contributor-guide: +.. toctree:: + :maxdepth: 2 + :caption: Contributor Guide + + contributor-guide/index + contributor-guide/setup + contributor-guide/tests + contributor-guide/benchmarks diff --git a/docs/source/user-guide/channel-resolver.md b/docs/source/user-guide/channel-resolver.md new file mode 100644 index 00000000..749cabf0 --- /dev/null +++ b/docs/source/user-guide/channel-resolver.md @@ -0,0 +1,65 @@ +# Building a ChannelResolver + +This trait is optional—a sensible default implementation exists that handles most use cases. + +The `ChannelResolver` trait controls how Distributed DataFusion builds Arrow Flight clients backed by +[Tonic](https://github.com/hyperium/tonic) channels for worker URLs. + +The default implementation connects to each URL, builds an Arrow Flight client, and caches it for reuse on +subsequent requests to the same URL. + +## Providing your own ChannelResolver + +For providing your own implementation, you'll need to take into account the following points: + +- You will need to provide your own implementation in two places: + - in the `SessionContext` that first initiates and plans your queries. + - while instantiating the `Worker` with the `from_session_builder()` constructor. +- If building from scratch, ensure Arrow Flight clients are reused across requests rather than recreated each time. +- You can extend `DefaultChannelResolver` as a foundation for custom implementations. This automatically handles + gRPC channel reuse. + +```rust +#[derive(Clone)] +struct CustomChannelResolver; + +#[async_trait] +impl ChannelResolver for CustomChannelResolver { + async fn get_flight_client_for_url( + &self, + url: &Url, + ) -> Result, DataFusionError> { + // Build a custom FlightServiceClient wrapped with tower + // layers or something similar. + todo!() + } +} + +async fn main() { + // Build a single instance for your application's lifetime + // to enable Arrow Flight client reuse across queries. + let channel_resolver = CustomChannelResolver; + + let state = SessionStateBuilder::new() + // these two are mandatory. + .with_distributed_worker_resolver(todo!()) + .with_physical_optimizer_rule(Arc::new(DistributedPhysicalOptimizerRule)) + // the CustomChannelResolver needs to be passed here once... + .with_distributed_channel_resolver(channel_resolver.clone()) + .build(); + + // ... and here for each query the Worker handles. + let endpoint = Worker::from_session_builder(move |ctx: WorkerQueryContext| { + let channel_resolver = channel_resolver.clone(); + async move { + Ok(ctx.builder.with_distributed_channel_resolver(channel_resolver).build()) + } + }); + Server::builder() + .add_service(endpoint.into_flight_server()) + .serve(SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 8000)) + .await?; + + Ok(()) +} +``` \ No newline at end of file diff --git a/docs/source/user-guide/concepts.md b/docs/source/user-guide/concepts.md new file mode 100644 index 00000000..88cbd0aa --- /dev/null +++ b/docs/source/user-guide/concepts.md @@ -0,0 +1,72 @@ +# Concepts + +This library is a collection of DataFusion extensions that enable distributed query execution. You can think of +it as normal DataFusion, with the addition that some nodes are capable of streaming data over the network using +Arrow Flight instead of through in-memory communication. + +Key terminology: + +- `Stage`: a portion of the plan separated by a network boundary from other parts of the plan. A plan contains + one or more stages, each separated by network boundaries. +- `Task`: a unit of work in a stage that executes the inner plan in parallel to other tasks within the stage. Each task + in a stage executes a structurally identical plan in different worker, passing a `task_index` as a contextual value + for making choices about what data should be returned. +- `Network Boundary`: a node in the plan that streams data from a network interface rather than directly from its + children nodes. +- `Worker`: a physical machine listening to serialized execution plans over an Arrow Flight interface. A task is + executed by exactly one worker, but one worker executes many tasks concurrently. + +![concepts.png](../_static/images/concepts.png) + +You'll see these concepts mentioned extensively across the documentation and the code itself. + +# Public API + +Some other more tangible concepts are the structs and traits exposed publicly, the most important are: + +## [DistributedPhysicalOptimizerRule](https://github.com/datafusion-contrib/datafusion-distributed/blob/main/src/distributed_planner/distributed_physical_optimizer_rule.rs) + +A physical optimizer rule that transforms single-node DataFusion query plans into distributed query plans. It reads +a fully formed physical plan and injects the appropriate nodes to execute the query in a distributed fashion. + +It builds the distributed plan from bottom to top, injecting network boundaries at appropriate locations based on +the nodes present in the original plan. + +## [Worker](https://github.com/datafusion-contrib/datafusion-distributed/blob/main/src/flight_service/worker.rs) + +Arrow Flight server implementation that integrates with the Tonic ecosystem and listens to serialized plans that get +executed over the wire. + +Users are expected to build these and spawn them in ports so that the network boundary nodes can reach them. + +## [WorkerResolver](https://github.com/datafusion-contrib/datafusion-distributed/blob/main/src/networking/worker_resolver.rs) + +Determines the available workers in the Distributed DataFusion cluster by returning their URLs. + +Different organizations have different networking requirements—from Kubernetes deployments to cloud provider +solutions. This trait allows Distributed DataFusion to adapt to various scenarios. + +## [TaskEstimator](https://github.com/datafusion-contrib/datafusion-distributed/blob/main/src/distributed_planner/task_estimator.rs) + +Estimates the number of tasks required in the leaf stage of a distributed query. + +The number of tasks each stage has is determined from bottom to top. This means that leaf stages will decide how many +tasks they need to execute based on the amount of data their leaf nodes will pull. Upper stages +will have their number of tasks reduced or increased depending on how much the cardinality of the data was reduced in +previous stages. + +## [DistributedTaskContext](https://github.com/datafusion-contrib/datafusion-distributed/blob/main/src/stage.rs) + +An extension present during the `ExecutionPlan::execute()` that contains information about the current task in +which the plan is being executed. + +As a user, you will need to interact with this type in your custom leaf nodes, as depending on which task index +you are in, you might want to return a different set of data. + +For example, if you are on the task with index 0 of a 3-task stage, you might want to return only the first 1/3 of the +data. If you are on the task with index 2, you might want to return the last 1/3 of the data, and so on. + +## [ChannelResolver](https://github.com/datafusion-contrib/datafusion-distributed/blob/main/src/networking/channel_resolver.rs) + +Optional extension trait that allows to customize how connections are established to workers. Given one of the +URLs returned by the `WorkerResolver`, it builds an Arrow Flight client ready for serving queries. diff --git a/docs/source/user-guide/getting-started.md b/docs/source/user-guide/getting-started.md new file mode 100644 index 00000000..33f5fa02 --- /dev/null +++ b/docs/source/user-guide/getting-started.md @@ -0,0 +1,86 @@ +# Getting Started + +Think of this library as vanilla DataFusion, except that certain nodes execute their children on remote +machines and retrieve data via the Arrow Flight protocol. + +This library aims to provide an experience as close as possible to vanilla DataFusion. + +## How to use Distributed DataFusion + +Rather than imposing constraints on your infrastructure or query serving patterns, Distributed DataFusion +allows you to plug in your own networking stack and spawn your own gRPC servers that act as workers in the cluster. + +This project heavily relies on the [Tonic](https://github.com/hyperium/tonic) ecosystem for the networking layer. +Users of this library are responsible for building their own Tonic server, adding the Arrow Flight distributed +DataFusion service to it and spawning it on a port so that it can be reached by other workers in the cluster. A very +basic example of this would be: + +```rust +#[tokio::main] +async fn main() -> Result<(), Box> { + let worker = Worker::default(); + + Server::builder() + .add_service(worker.into_flight_server()) + .serve(SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 8000)) + .await?; + + Ok(()) +} +``` + +Distributed DataFusion requires knowledge of worker locations. Implement the `WorkerResolver` trait to provide +this information. Here is a simple example of what this would look like with localhost workers: + +```rust +#[derive(Clone)] +struct LocalhostWorkerResolver { + ports: Vec, +} + +impl WorkerResolver for LocalhostWorkerResolver { + fn get_urls(&self) -> Result, DataFusionError> { + Ok(self + .ports + .iter() + .map(|port| Url::parse(&format!("http://localhost:{port}")).unwrap()) + .collect()) + } +} +``` + +Register both the `WorkerResolver` implementation and the `DistributedPhysicalOptimizerRule` in DataFusion's +`SessionStateBuilder` to enable distributed query planning: + +```rs +let localhost_worker_resolver = LocalhostWorkerResolver { + ports: vec![8000, 8001, 8002] +} + +let state = SessionStateBuilder::new() + .with_distributed_worker_resolver(localhost_worker_resolver) + .with_physical_optimizer_rule(Arc::new(DistributedPhysicalOptimizerRule)) + .build(); + +let ctx = SessionContext::from(state); +``` + +This will leave a DataFusion `SessionContext` ready for executing distributed queries. + +> NOTE: This example is not production-ready and is meant to showcase the basic concepts of the library. + +## Next steps + +Depending on your needs, your setup can get more complicated, for example: + +- You may want to resolve worker URLs dynamically using the Kubernetes API. +- You may want to wrap the Arrow Flight clients that connect workers with an observability layer. +- You may want to be able to execute your own custom ExecutionPlans in a distributed manner. +- etc... + +To learn how to do all that, it's recommended to: + +- [Continue reading this guide](worker.md) +- [Look at examples in the project](https://github.com/datafusion-contrib/datafusion-distributed/tree/main/examples) +- [Look at the integration tests for finer grained examples](https://github.com/datafusion-contrib/datafusion-distributed/tree/main/tests) + diff --git a/docs/source/user-guide/how-a-distributed-plan-is-built.md b/docs/source/user-guide/how-a-distributed-plan-is-built.md new file mode 100644 index 00000000..7b1efbd5 --- /dev/null +++ b/docs/source/user-guide/how-a-distributed-plan-is-built.md @@ -0,0 +1,97 @@ +# How a Distributed Plan is Built + +This page walks through how the distributed DataFusion planner transforms a query into a distributed execution plan. + +Everything starts with a simple single-node plan, for example: + +```shell +ProjectionExec: expr=[...] + AggregateExec: mode=FinalPartitioned, gby=[...], aggr=[...] + CoalesceBatchesExec: target_batch_size=8192 + RepartitionExec: partitioning=Hash([...], 12), input_partitions=12 + AggregateExec: mode=Partial, gby=[...], aggr=[...] + DataSourceExec: files=[data1, data2, data3, data4] +``` + +To better understand what happens with the plan in the distribution process, we will represent it graphically: + +![img.png](../_static/images/img.png) + +Note how the underlying `DataSourceExec` contains multiple non-overlapping pieces of data, represented with colors +in the image. Depending on the underlying `DataSource` implementation in the `DataSourceExec` node, these can +represent different things, such as different parquet files, or an external API from which we can gather data for different +time ranges. + +The first step is to split the leaf node into different tasks: + +![img_2.png](../_static/images/img_2.png) + +Each task will handle a different non-overlapping piece of data. + +The number of tasks that will be used for executing leaf nodes is determined by a `TaskEstimator` implementation. +A default implementation exists for file-based `DataSourceExec` nodes. However, since `DataSourceExec` can be +customized to represent any data source, users with custom implementations should also provide a corresponding +`TaskEstimator`. + +In the case above, a `TaskEstimator` decided to use four tasks for the leaf node. Note that even if we are distributing +the data across different tasks, each task will also distribute its data across partitions using the vanilla DataFusion +partitioning mechanism. A partition is a split of data processed by a single thread on a single machine, whereas a +task is a split of data processed by an entire machine within a cluster. + +After that, we can continue reconstructing the plan: + +![img_3.png](../_static/images/img_3.png) + +Nothing special to consider for now—the partial aggregation can simply be executed in parallel across different +workers without further considerations. + +Let's keep constructing the plan: + +![img_4.png](../_static/images/img_4.png) + +At this point, the plan encounters a `RepartitionExec` node, which requires repartitioning data so each partition +handles a non-overlapping subset of grouping keys for the aggregation. + +While `RepartitionExec` redistributes data across threads on a single machine in vanilla DataFusion, it redistributes +data across threads on different machines in the distributed context—requiring a network shuffle. + +As we are about to send data over the network, it's convenient to coalesce smaller batches into larger ones to avoid +the overhead of sending many small messages, and instead send fewer but larger messages: + +![img_5.png](../_static/images/img_5.png) + +After this, we are ready to perform the shuffle over the network. For that, a new `ExecutionPlan` implementation is +provided: `NetworkShuffleExec`: + +![img_6.png](../_static/images/img_6.png) + +A `NetworkShuffleExec`, instead of calling `execute()` on its child node, will execute it remotely through Arrow +Flight, and each `NetworkShuffleExec` instance will know from which partitions and machines it should gather data. + +Note how this means that we have just built the first stage, as the first network boundary was introduced. We are now +in the process of building the second stage, and note how it has just two tasks. + +If the number of tasks in a leaf stage is driven by the hints given by `TaskEstimator`s, the number of tasks in upper +stages is driven by the nodes in between that reduce or increase the cardinality of the data. + +In this case, the leaf stage is performing a partial aggregation before sending data to the next stage, so we can +assume that less compute will be needed, and therefore, we can reduce the width of the next stage to just two +tasks. + +The rest of the plan can be formed as normal: + +![img_7.png](../_static/images/img_7.png) + +One final step remains: the plan's head is currently distributed across two machines, but the final result must be +consolidated on a single node. In the same way that vanilla DataFusion coalesces all partitions into one in the head +node for the user, we also need to do that, but not only across partitions on a single machine, but across tasks on +different machines. + +For that, the `NetworkCoalesceExec` network boundary is introduced: it coalesces P partitions across N tasks into +N*P partitions in one task. This does not imply repartitioning, or shuffling, or anything like that. The partitions +are the same but joined into a single task: + +![img_8.png](../_static/images/img_8.png) + +Note how at this point, what the user sees is just an `ExecutionPlan` that can be executed as any other normal plan, +but it will happen to be distributed under the hood. diff --git a/docs/source/user-guide/task-estimator.md b/docs/source/user-guide/task-estimator.md new file mode 100644 index 00000000..b2d42f4d --- /dev/null +++ b/docs/source/user-guide/task-estimator.md @@ -0,0 +1,30 @@ +# Building a TaskEstimator + +The `TaskEstimator` trait controls how many distributed tasks the planner allocates to each stage of the query plan. + +The number of tasks is assigned to the different stages in a bottom-up fashion. You can refer to the +[Plan Annotation docs](https://github.com/datafusion-contrib/datafusion-distributed/blob/75b4e73e9052c6596b9d1744ce2bdfa6cbc010d3/src/distributed_planner/plan_annotator.rs) +for an explanation on how this works. A `TaskEstimator` is what hints this process how many tasks should be +used. + +While a default implementation exists for file-based `DataSourceExec` nodes (those backed by `FileScanConfig`), you +can provide custom `TaskEstimator` implementations for your own `ExecutionPlan` types. + +## Providing your own TaskEstimator + +Providing a `TaskEstimator` allows you to do two things: + +1. Tell the distributed planner how many tasks should be used for your own `ExecutionPlan`s. +2. Tell the distributed planner how to "scale up" your `ExecutionPlan` in order to account for it running in + multiple distributed tasks. + +If your custom nodes will execute in a distributed manner, you must handle this during execution. When your +`TaskEstimator` specifies N tasks for a node, your execution logic must respond to the +[DistributedTaskContext](https://github.com/datafusion-contrib/datafusion-distributed/blob/75b4e73e9052c6596b9d1744ce2bdfa6cbc010d3/src/stage.rs#L137-L137) +present in DataFusion's `TaskContext` to determine which subset of data this task should process. + +There's an example of how to do that in the `examples/` folder: + +- [custom_execution_plan.rs](https://github.com/datafusion-contrib/datafusion-distributed/blob/main/examples/custom_execution_plan.rs) - + A complete example showing how to implement a custom execution plan (`numbers(start, end)` table function) + that works with distributed DataFusion, including a custom codec and TaskEstimator. diff --git a/docs/source/user-guide/worker-resolver.md b/docs/source/user-guide/worker-resolver.md new file mode 100644 index 00000000..a17e5915 --- /dev/null +++ b/docs/source/user-guide/worker-resolver.md @@ -0,0 +1,73 @@ +# Implementing a WorkerResolver + +The `WorkerResolver` trait provides Distributed DataFusion with the locations (URLs) of your worker nodes. This +information is used in two different places: + +1. **During planning**: The number of available workers (i.e., `Vec.len()`) determines how the plan scales. + The planner will not allocate more tasks per stage than available workers. +2. **Before execution**: Each task in a distributed plan needs its worker URL assignment populated right before + execution. + +You need to pass your own `WorkerResolver` to DataFusion's `SessionStateBuilder` so that it's available in the +`SesionContext`: + +```rust +struct CustomWorkerResolver; + +#[async_trait] +impl WorkerResolver for CustomWorkerResolver { + fn get_urls(&self) -> Result, DataFusionError> { + todo!() + } +} + +async fn main() { + let state = SessionStateBuilder::new() + .with_distributed_worker_resolver(CustomWorkerResolver) + .with_physical_optimizer_rule(Arc::new(DistributedPhysicalOptimizerRule)) + .build(); +} +``` + +> NOTE: It's not necessary to pass a WorkerResolver to the Worker session builder, it's just necessary on the +> SessionState that initiates and plans the query. + +## Static WorkerResolver + +This is the simplest approach, though it doesn't accommodate dynamic worker discovery. An example of this can be +seen in the +[localhost_worker.rs](https://github.com/datafusion-contrib/datafusion-distributed/blob/fad9fa222d65b7d2ddae92fbc20082b5c434e4ff/examples/localhost_run.rs) +example: + +```rust +#[derive(Clone)] +struct LocalhostChannelResolver { + ports: Vec, +} + +#[async_trait] +impl WorkerResolver for LocalhostChannelResolver { + fn get_urls(&self) -> Result, DataFusionError> { + Ok(self + .ports + .iter() + .map(|port| Url::parse(&format!("http://localhost:{port}")).unwrap()) + .collect()) + } +} +``` + +## Dynamic WorkerResolver + +In a typical setup, you might have different pods running in a Kubernetes cluster, or you might be using a cloud +provider for hosting your Distributed DataFusion workers. + +It's up to you to decide how the URLs should be resolved. One important implementation note is: + +- Since planning is synchronous, `get_urls()` must return immediately. For dynamic worker discovery, spawn + background tasks that periodically refresh the worker URL list, storing results that `get_urls()` can access + synchronously. + +A good example can be found +in [benchmarks/cdk/bin/worker.rs](https://github.com/datafusion-contrib/datafusion-distributed/blob/main/benchmarks/cdk/bin/worker.rs), +where a cluster of AWS EC2 machines is discovered identified by tags with the AWS Rust SDK. \ No newline at end of file diff --git a/docs/source/user-guide/worker.md b/docs/source/user-guide/worker.md new file mode 100644 index 00000000..e57bc329 --- /dev/null +++ b/docs/source/user-guide/worker.md @@ -0,0 +1,97 @@ +# Spawn a Worker + +The `Worker` is a gRPC server implementing the Arrow Flight protocol for distributed query execution. Worker nodes +run these endpoints to receive execution plans, execute them, and stream results back. + +## Overview + +The `Worker` is the core worker component in Distributed DataFusion. It: + +- Receives serialized execution plans via Arrow Flight's `do_get` method +- Deserializes plans using protobuf and user-provided codecs +- Executes plans using the local DataFusion runtime +- Streams results back as Arrow record batches through the gRPC Arrow Flight interface + +## Launching the Arrow Flight server + +The default `Worker` implementation satisfies most basic use cases: + +```rust +use datafusion_distributed::Worker; + +async fn main() { + let endpoint = Worker::default(); + + Server::builder() + .add_service(endpoint.into_flight_server()) + .serve(SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 8000)) + .await?; + + Ok(()) +} +``` + +However, most DataFusion deployments include custom UDFs, execution nodes, or configuration options. + +You'll need to tell the `Worker` how to build your DataFusion sessions: + +```rust +async fn build_state(ctx: WorkerQueryContext) -> Result { + Ok(ctx + .builder + .with_scalar_functions(vec![your_custom_udf()]) + .build()) +} + +async fn main() { + let endpoint = Worker::from_session_builder(build_sate); + + Server::builder() + .add_service(endpoint.into_flight_server()) + .serve(SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 8000)) + .await?; + + Ok(()) +} +``` + +## WorkerSessionBuilder + +The `WorkerSessionBuilder` is a closure or type that implements: + +```rust +#[async_trait] +pub trait WorkerSessionBuilder { + async fn build_session_state( + &self, + ctx: WorkerQueryContext, + ) -> Result; +} +``` + +It receives a `WorkerQueryContext` containing: + +- `SessionStateBuilder`: A pre-populated session state builder in which you can inject your custom stuff +- `headers`: HTTP headers from the incoming request (useful for passing metadata like authentication tokens or + configuration) + +## Serving the Endpoint + +Convert the endpoint to a gRPC service and serve it: + +```rust +use tonic::transport::Server; +use datafusion_distributed::Worker; +use std::net::{IpAddr, Ipv4Addr, SocketAddr}; + +async fn main() { + let endpoint = Worker::default(); + + Server::builder() + .add_service(endpoint.into_flight_server()) + .serve(SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 8080)) + .await?; +} +``` + +The `into_flight_server()` method builds a `FlightServiceServer` ready to be added as a Tonic service. diff --git a/examples/custom_execution_plan.md b/examples/custom_execution_plan.md new file mode 100644 index 00000000..49f2366a --- /dev/null +++ b/examples/custom_execution_plan.md @@ -0,0 +1,148 @@ +# Custom Execution Plan Example + +Demonstrates how to create a distributed custom execution plan with a `numbers(start, end)` table function. + +## Components + +**NumbersTableFunction** – Table function callable in SQL: `SELECT * FROM numbers(1, 100)` + +**NumbersExec** – Execution plan with `ranges_per_task: Vec>` storing one range per task. +Uses `DistributedTaskContext` to determine which range to generate. + +**NumbersExecCodec** – Protobuf-based serialization implementing `PhysicalExtensionCodec`. +Must be registered in the `SessionStateBuilder` that initiates the query as well as the one used by `Worker`s. + +**NumbersTaskEstimator** – Controls distributed parallelism: + +- `task_estimation()` - Returns how many tasks needed based on range size and config +- `scale_up_leaf_node()` - Splits single range of numbers into N per-task ranges + +**NumbersConfig** – Custom config extension for controlling distributed parallelism (`numbers_per_task: usize`) + +## Usage + +This example imports the `InMemoryWorkerResolver` and the `InMemoryChannelResolver` used during integration testing +of this project, so it needs the `--features integration` flag on. + +This example demonstrates how the bigger the range of numbers is queried, the more tasks are used in executing +the query, for example: + +```bash +cargo run \ + --features integration \ + --example custom_execution_plan \ + "SELECT DISTINCT number FROM numbers(0, 10) ORDER BY number" \ + --show-distributed-plan +``` + +``` +SortPreservingMergeExec: [number@0 ASC NULLS LAST] + SortExec: expr=[number@0 ASC NULLS LAST], preserve_partitioning=[true] + AggregateExec: mode=FinalPartitioned, gby=[number@0 as number], aggr=[] + CoalesceBatchesExec: target_batch_size=8192 + RepartitionExec: partitioning=Hash([number@0], 16), input_partitions=16 + AggregateExec: mode=Partial, gby=[number@0 as number], aggr=[] + RepartitionExec: partitioning=RoundRobinBatch(16), input_partitions=1 + CooperativeExec + NumbersExec: t0:[0-10) +``` + +This will print a non-distributed plan, as the range of numbers we are querying (`numbers(0, 10)`) is small. + +The config parameter `numbers.numbers_per_task` is the one that controls how many distributed tasks are used in the +query, and it's default value is `10`, so querying 10 numbers will not distribute the plan. + +However, if we try to query 11 numbers: + +```bash +cargo run \ + --features integration \ + --example custom_execution_plan \ + "SELECT DISTINCT number FROM numbers(0, 11) ORDER BY number" \ + --show-distributed-plan +``` + +``` +┌───── DistributedExec ── Tasks: t0:[p0] +│ SortPreservingMergeExec: [number@0 ASC NULLS LAST] +│ [Stage 2] => NetworkCoalesceExec: output_partitions=32, input_tasks=2 +└────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p15] t1:[p0..p15] + │ SortExec: expr=[number@0 ASC NULLS LAST], preserve_partitioning=[true] + │ AggregateExec: mode=FinalPartitioned, gby=[number@0 as number], aggr=[] + │ [Stage 1] => NetworkShuffleExec: output_partitions=16, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p31] t1:[p0..p31] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([number@0], 32), input_partitions=16 + │ AggregateExec: mode=Partial, gby=[number@0 as number], aggr=[] + │ RepartitionExec: partitioning=RoundRobinBatch(16), input_partitions=1 + │ CooperativeExec + │ NumbersExec: t0:[0-6), t1:[6-11) + └────────────────────────────────────────────────── +``` + +The distribution rule kicks in, and the plan gets distributed. + +Note that the parallelism in the plan has an upper threshold, so for example, if we query 100 numbers: + +```bash +cargo run \ + --features integration \ + --example custom_execution_plan \ + "SELECT DISTINCT number FROM numbers(0, 100) ORDER BY number" \ + --show-distributed-plan +``` + +``` +┌───── DistributedExec ── Tasks: t0:[p0] +│ SortPreservingMergeExec: [number@0 ASC NULLS LAST] +│ [Stage 2] => NetworkCoalesceExec: output_partitions=48, input_tasks=3 +└────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p15] t1:[p0..p15] t2:[p0..p15] + │ SortExec: expr=[number@0 ASC NULLS LAST], preserve_partitioning=[true] + │ AggregateExec: mode=FinalPartitioned, gby=[number@0 as number], aggr=[] + │ [Stage 1] => NetworkShuffleExec: output_partitions=16, input_tasks=4 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p47] t1:[p0..p47] t2:[p0..p47] t3:[p0..p47] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([number@0], 48), input_partitions=16 + │ AggregateExec: mode=Partial, gby=[number@0 as number], aggr=[] + │ RepartitionExec: partitioning=RoundRobinBatch(16), input_partitions=1 + │ CooperativeExec + │ NumbersExec: t0:[0-25), t1:[25-50), t2:[50-75), t3:[75-100) + └────────────────────────────────────────────────── +``` + +We do not get 100/10 = 10 distributed tasks, we just get 4. This is because the example is configured by default to +simulate a 4-worker cluster. If we increase the worker count, we get a highly distributed plan out with 10 tasks: + +```bash +cargo run \ + --features integration \ + --example custom_execution_plan \ + "SELECT DISTINCT number FROM numbers(0, 100) ORDER BY number" \ + --workers 10 \ + --show-distributed-plan +``` + +``` +┌───── DistributedExec ── Tasks: t0:[p0] +│ SortPreservingMergeExec: [number@0 ASC NULLS LAST] +│ [Stage 2] => NetworkCoalesceExec: output_partitions=112, input_tasks=7 +└────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p15] t1:[p0..p15] t2:[p0..p15] t3:[p0..p15] t4:[p0..p15] t5:[p0..p15] t6:[p0..p15] + │ SortExec: expr=[number@0 ASC NULLS LAST], preserve_partitioning=[true] + │ AggregateExec: mode=FinalPartitioned, gby=[number@0 as number], aggr=[] + │ [Stage 1] => NetworkShuffleExec: output_partitions=16, input_tasks=10 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p111] t1:[p0..p111] t2:[p0..p111] t3:[p0..p111] t4:[p0..p111] t5:[p0..p111] t6:[p0..p111] t7:[p0..p111] t8:[p0..p111] t9:[p0..p111] + │ CoalesceBatchesExec: target_batch_size=8192 + │ RepartitionExec: partitioning=Hash([number@0], 112), input_partitions=16 + │ AggregateExec: mode=Partial, gby=[number@0 as number], aggr=[] + │ RepartitionExec: partitioning=RoundRobinBatch(16), input_partitions=1 + │ CooperativeExec + │ NumbersExec: t0:[0-10), t1:[10-20), t2:[20-30), t3:[30-40), t4:[40-50), t5:[50-60), t6:[60-70), t7:[70-80), t8:[80-90), t9:[90-100) + └────────────────────────────────────────────────── +``` + diff --git a/examples/custom_execution_plan.rs b/examples/custom_execution_plan.rs new file mode 100644 index 00000000..10c16d28 --- /dev/null +++ b/examples/custom_execution_plan.rs @@ -0,0 +1,408 @@ +//! This example demonstrates how to create a custom execution plan that works with +//! Distributed DataFusion. It implements a `numbers(start, end)` table function that +//! generates a sequence of numbers and can be distributed across multiple workers. +//! +//! This example includes: +//! - Custom TableFunction for accepting the `numbers(start, end)` in SQL +//! - Custom TableProvider for mapping the table function to an execution plan +//! - Custom ExecutionPlan for returning the requested number range +//! - Custom PhysicalExtensionCodec for serialization across the network +//! - Custom TaskEstimator to control parallelism +//! +//! Run this example with: +//! ```bash +//! cargo run --features integration --example custom_execution_plan "SELECT DISTINCT number FROM numbers(0, 10) ORDER BY number" --show-distributed-plan +//! cargo run --features integration --example custom_execution_plan "SELECT DISTINCT number FROM numbers(0, 11) ORDER BY number" --show-distributed-plan +//! cargo run --features integration --example custom_execution_plan "SELECT DISTINCT number FROM numbers(0, 100) ORDER BY number" --show-distributed-plan +//! cargo run --features integration --example custom_execution_plan "SELECT DISTINCT number FROM numbers(0, 100) ORDER BY number" --workers 10 --show-distributed-plan +//! ``` + +use arrow::array::{ArrayRef, Int64Array, RecordBatch}; +use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; +use arrow::record_batch::RecordBatchOptions; +use arrow::util::pretty::pretty_format_batches; +use async_trait::async_trait; +use datafusion::catalog::{Session, TableFunctionImpl}; +use datafusion::common::{ + DataFusionError, Result, ScalarValue, exec_err, extensions_options, internal_err, plan_err, +}; +use datafusion::config::ConfigExtension; +use datafusion::datasource::{TableProvider, TableType}; +use datafusion::execution::{SendableRecordBatchStream, SessionStateBuilder, TaskContext}; +use datafusion::logical_expr::Expr; +use datafusion::physical_expr::EquivalenceProperties; +use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType}; +use datafusion::physical_plan::stream::RecordBatchStreamAdapter; +use datafusion::physical_plan::{DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties}; +use datafusion::prelude::{SessionConfig, SessionContext}; +use datafusion_distributed::test_utils::in_memory_channel_resolver::{ + InMemoryChannelResolver, InMemoryWorkerResolver, +}; +use datafusion_distributed::{ + DistributedExt, DistributedPhysicalOptimizerRule, DistributedTaskContext, TaskEstimation, + TaskEstimator, WorkerQueryContext, display_plan_ascii, +}; +use datafusion_proto::physical_plan::PhysicalExtensionCodec; +use datafusion_proto::protobuf; +use datafusion_proto::protobuf::proto_error; +use futures::{TryStreamExt, stream}; +use prost::Message; +use std::any::Any; +use std::fmt::{self, Formatter}; +use std::ops::Range; +use std::sync::Arc; +use structopt::StructOpt; + +/// Table function that generates a sequence of numbers from start to end. +/// Can be called in SQL as: SELECT * FROM numbers(start, end) +#[derive(Debug)] +struct NumbersTableFunction; + +impl TableFunctionImpl for NumbersTableFunction { + fn call(&self, exprs: &[Expr]) -> Result> { + if exprs.len() != 2 { + return plan_err!( + "numbers() requires exactly 2 arguments (start, end), got {}", + exprs.len() + ); + } + fn get_number(expr: &Expr) -> Result { + match &expr { + Expr::Literal(ScalarValue::Int64(Some(v)), _) => Ok(*v), + Expr::Literal(ScalarValue::Int32(Some(v)), _) => Ok(*v as i64), + v => plan_err!("numbers() arguments must be integer literals, got {v:?}"), + } + } + Ok(Arc::new(NumbersTableProvider { + start: get_number(&exprs[0])?, + end: get_number(&exprs[1])?, + })) + } +} + +fn numbers_schema() -> SchemaRef { + Arc::new(Schema::new(vec![Field::new( + "number", + DataType::Int64, + false, + )])) +} + +/// TableProvider that generates a sequence of numbers from start to end. +#[derive(Debug)] +struct NumbersTableProvider { + start: i64, + end: i64, +} + +#[async_trait] +impl TableProvider for NumbersTableProvider { + fn as_any(&self) -> &dyn Any { + self + } + + fn schema(&self) -> SchemaRef { + numbers_schema() + } + + fn table_type(&self) -> TableType { + TableType::Base + } + + async fn scan( + &self, + _state: &dyn Session, + projection: Option<&Vec>, + _filters: &[Expr], + _limit: Option, + ) -> Result> { + let schema = match projection { + Some(indices) => Arc::new(self.schema().project(indices)?), + None => self.schema(), + }; + + #[allow(clippy::single_range_in_vec_init)] + Ok(Arc::new(NumbersExec::new([self.start..self.end], schema))) + } +} + +/// Custom execution plan that generates numbers from start to end. +/// When distributed, each task generates a subset of numbers based on its task_index. +#[derive(Debug, Clone)] +struct NumbersExec { + ranges_per_task: Vec>, + plan_properties: PlanProperties, +} + +impl NumbersExec { + fn new(ranges_per_task: impl IntoIterator>, schema: SchemaRef) -> Self { + let plan_properties = PlanProperties::new( + EquivalenceProperties::new(schema.clone()), + datafusion::physical_expr::Partitioning::UnknownPartitioning(1), + EmissionType::Incremental, + Boundedness::Bounded, + ); + Self { + ranges_per_task: ranges_per_task.into_iter().collect(), + plan_properties, + } + } +} + +impl DisplayAs for NumbersExec { + fn fmt_as(&self, _t: DisplayFormatType, f: &mut Formatter) -> fmt::Result { + write!(f, "NumbersExec: ")?; + for (task_i, range) in self.ranges_per_task.iter().enumerate() { + write!(f, "t{task_i}:[{}-{})", range.start, range.end)?; + if task_i < self.ranges_per_task.len() - 1 { + write!(f, ", ")?; + } + } + Ok(()) + } +} + +impl ExecutionPlan for NumbersExec { + fn name(&self) -> &str { + "NumbersExec" + } + + fn as_any(&self) -> &dyn Any { + self + } + + fn properties(&self) -> &PlanProperties { + &self.plan_properties + } + + fn children(&self) -> Vec<&Arc> { + vec![] + } + + fn with_new_children( + self: Arc, + _: Vec>, + ) -> Result> { + Ok(self) + } + + fn execute( + &self, + _partition: usize, + context: Arc, + ) -> Result { + // Get the distributed task context to determine which subset of numbers + // this task should generate + let dist_ctx = DistributedTaskContext::from_ctx(&context); + + let Some(range) = self.ranges_per_task.get(dist_ctx.task_index) else { + return exec_err!("Task index out of range"); + }; + + // Calculate which numbers this task should generate + let numbers: Vec = range.clone().collect(); + let row_count = numbers.len(); + + // Create batch matching the schema (may be empty for COUNT queries) + let batch = if self.schema().fields().is_empty() { + // For COUNT queries, return batch with correct row count but no columns + let mut options = RecordBatchOptions::new(); + options.row_count = Some(row_count); + RecordBatch::try_new_with_options(self.schema(), vec![], &options)? + } else { + let array: ArrayRef = Arc::new(Int64Array::from(numbers)); + RecordBatch::try_new(self.schema(), vec![array])? + }; + + Ok(Box::pin(RecordBatchStreamAdapter::new( + self.schema(), + stream::once(async { Ok(batch) }), + ))) + } +} + +/// Custom codec for serializing/deserializing NumbersExec across the network. As the NumbersExec +/// plan will be sent over the wire during distributed queries, both the SessionContext that +/// initiates the query and each Worker need to know how to (de)serialize it. +#[derive(Debug)] +struct NumbersExecCodec; + +#[derive(Clone, PartialEq, ::prost::Message)] +struct NumbersExecProto { + #[prost(message, optional, tag = "1")] + schema: Option, + #[prost(repeated, message, tag = "2")] + ranges: Vec, +} + +#[derive(Clone, PartialEq, ::prost::Message)] +struct RangeProto { + #[prost(int64, tag = "1")] + start: i64, + #[prost(int64, tag = "2")] + end: i64, +} + +impl PhysicalExtensionCodec for NumbersExecCodec { + fn try_decode( + &self, + buf: &[u8], + inputs: &[Arc], + _ctx: &TaskContext, + ) -> Result> { + if !inputs.is_empty() { + return internal_err!("NumbersExec should have no children, got {}", inputs.len()); + } + + let proto = NumbersExecProto::decode(buf) + .map_err(|e| proto_error(format!("Failed to decode NumbersExec: {e}")))?; + + let schema: Schema = proto + .schema + .as_ref() + .map(|s| s.try_into()) + .ok_or(proto_error("NetworkShuffleExec is missing schema"))??; + + Ok(Arc::new(NumbersExec::new( + proto.ranges.iter().map(|v| v.start..v.end), + Arc::new(schema), + ))) + } + + fn try_encode(&self, node: Arc, buf: &mut Vec) -> Result<()> { + let Some(exec) = node.as_any().downcast_ref::() else { + return internal_err!("Expected plan to be NumbersExec, but was {}", node.name()); + }; + + let proto = NumbersExecProto { + schema: Some(node.schema().try_into()?), + ranges: exec + .ranges_per_task + .iter() + .map(|v| RangeProto { + start: v.start, + end: v.end, + }) + .collect(), + }; + + proto + .encode(buf) + .map_err(|e| proto_error(format!("Failed to encode NumbersExec: {e}"))) + } +} + +extensions_options! { + /// Custom ConfigExtension for configuring NumbersExec distributed task estimation behavior + /// at runtime with SET statements. + struct NumbersConfig { + /// how many numbers each task will produce + numbers_per_task: usize, default = 10 + } +} + +impl ConfigExtension for NumbersConfig { + const PREFIX: &'static str = "numbers"; +} + +/// Custom TaskEstimator that tells the planner how to distribute NumbersExec. +#[derive(Debug)] +struct NumbersTaskEstimator; + +impl TaskEstimator for NumbersTaskEstimator { + fn task_estimation( + &self, + plan: &Arc, + cfg: &datafusion::config::ConfigOptions, + ) -> Option { + let plan = plan.as_any().downcast_ref::()?; + let cfg: &NumbersConfig = cfg.extensions.get()?; + let task_count = (plan.ranges_per_task[0].end - plan.ranges_per_task[0].start) as f64 + / cfg.numbers_per_task as f64; + + Some(TaskEstimation::desired(task_count.ceil() as usize)) + } + + fn scale_up_leaf_node( + &self, + plan: &Arc, + task_count: usize, + _cfg: &datafusion::config::ConfigOptions, + ) -> Option> { + let plan = plan.as_any().downcast_ref::()?; + let range = &plan.ranges_per_task[0]; + let chunk_size = ((range.end - range.start) as f64 / task_count as f64).ceil() as i64; + + let ranges_per_task = (0..task_count).map(|i| { + let start = range.start + (i as i64 * chunk_size); + let end = (start + chunk_size).min(range.end); + start..end + }); + + Some(Arc::new(NumbersExec::new(ranges_per_task, plan.schema()))) + } +} + +#[derive(StructOpt)] +#[structopt( + name = "custom_execution_plan", + about = "Example demonstrating custom execution plans with Distributed DataFusion" +)] +struct Args { + /// The SQL query to run. + #[structopt()] + query: String, + + /// Number of distributed workers to simulate. + #[structopt(long, default_value = "4")] + workers: usize, + + /// Whether the distributed plan should be rendered instead of executing the query. + #[structopt(long)] + show_distributed_plan: bool, +} + +#[tokio::main] +async fn main() -> Result<(), Box> { + let args = Args::from_args(); + + let worker_resolver = InMemoryWorkerResolver::new(args.workers); + let channel_resolver = + InMemoryChannelResolver::from_session_builder(|ctx: WorkerQueryContext| async move { + Ok(ctx + .builder + .with_distributed_user_codec(NumbersExecCodec) + .build()) + }); + + let config = SessionConfig::new().with_option_extension(NumbersConfig::default()); + + let state = SessionStateBuilder::new() + .with_default_features() + .with_config(config) + .with_distributed_worker_resolver(worker_resolver) + .with_distributed_channel_resolver(channel_resolver) + .with_physical_optimizer_rule(Arc::new(DistributedPhysicalOptimizerRule)) + .with_distributed_user_codec(NumbersExecCodec) + .with_distributed_task_estimator(NumbersTaskEstimator) + .build(); + + let ctx = SessionContext::from(state); + ctx.register_udtf("numbers", Arc::new(NumbersTableFunction)); + + let mut df = None; + for query in args.query.split(';') { + df = Some(ctx.sql(query).await?); + } + let df = df.unwrap(); + if args.show_distributed_plan { + let plan = df.create_physical_plan().await?; + println!("{}", display_plan_ascii(plan.as_ref(), false)); + } else { + let stream = df.execute_stream().await?; + let batches = stream.try_collect::>().await?; + let formatted = pretty_format_batches(&batches)?; + println!("{formatted}"); + } + Ok(()) +} diff --git a/examples/in_memory.md b/examples/in_memory.md index c12818ad..aea65caa 100644 --- a/examples/in_memory.md +++ b/examples/in_memory.md @@ -1,9 +1,9 @@ # In-memory cluster example -This examples shows how queries can be run in a distributed context without making any +This example shows how queries can be run in a distributed context without making any network IO for communicating between workers. -This is specially useful for testing, as no servers need to be spawned in localhost ports, +This is especially useful for testing, as no servers need to be spawned in localhost ports, the setup is quite easy, and the code coverage for running in this mode is the same as running in an actual distributed cluster. @@ -13,38 +13,26 @@ This example queries a couple of test parquet we have for integration tests, and so pulling the first is necessary. ```shell -git intall checkout +git install checkout git lfs checkout ``` ### Issuing a distributed SQL query +The `--show-distributed-plan` flag can be passed to render the distributed plan: + ```shell -cargo run --example in_memory_cluster -- 'SELECT count(*), "MinTemp" FROM weather GROUP BY "MinTemp"' +cargo run --example in_memory_cluster -- 'SELECT count(*), "MinTemp" FROM weather GROUP BY "MinTemp"' --show-distributed-plan ``` -Additionally, the `--explain` flag can be passed to render the distributed plan: +Not passing the flag will execute the query: ```shell -cargo run --example in_memory_cluster -- 'SELECT count(*), "MinTemp" FROM weather GROUP BY "MinTemp"' --explain +cargo run --example in_memory_cluster -- 'SELECT count(*), "MinTemp" FROM weather GROUP BY "MinTemp"' ``` ### Available tables -Two tables are available in this example: - -- `flights_1m`: Flight data with 1m rows - -``` -FL_DATE [INT32] -DEP_DELAY [INT32] -ARR_DELAY [INT32] -AIR_TIME [INT32] -DISTANCE [INT32] -DEP_TIME [FLOAT] -ARR_TIME [FLOAT] -``` - - `weather`: Small dataset of weather data ``` diff --git a/examples/in_memory_cluster.rs b/examples/in_memory_cluster.rs index 97787610..e5f02fdc 100644 --- a/examples/in_memory_cluster.rs +++ b/examples/in_memory_cluster.rs @@ -1,14 +1,12 @@ use arrow::util::pretty::pretty_format_batches; use arrow_flight::flight_service_client::FlightServiceClient; -use arrow_flight::flight_service_server::FlightServiceServer; use async_trait::async_trait; use datafusion::common::DataFusionError; use datafusion::execution::SessionStateBuilder; -use datafusion::physical_plan::displayable; use datafusion::prelude::{ParquetReadOptions, SessionContext}; use datafusion_distributed::{ - ArrowFlightEndpoint, BoxCloneSyncChannel, ChannelResolver, DistributedExt, - DistributedPhysicalOptimizerRule, DistributedSessionBuilderContext, + BoxCloneSyncChannel, ChannelResolver, DistributedExt, DistributedPhysicalOptimizerRule, Worker, + WorkerQueryContext, WorkerResolver, create_flight_client, display_plan_ascii, }; use futures::TryStreamExt; use hyper_util::rt::TokioIo; @@ -20,14 +18,16 @@ use tonic::transport::{Endpoint, Server}; #[derive(StructOpt)] #[structopt( name = "run", - about = "An in-memory cluster Distributed DataFusion runner" + about = "Run a query in an in-memory Distributed DataFusion cluster" )] struct Args { + /// The SQL query to run. #[structopt()] query: String, + /// Whether the distributed plan should be rendered instead of executing the query. #[structopt(long)] - explain: bool, + show_distributed_plan: bool, } #[tokio::main] @@ -36,27 +36,21 @@ async fn main() -> Result<(), Box> { let state = SessionStateBuilder::new() .with_default_features() + .with_distributed_worker_resolver(InMemoryWorkerResolver) .with_distributed_channel_resolver(InMemoryChannelResolver::new()) - .with_physical_optimizer_rule(Arc::new(DistributedPhysicalOptimizerRule::new())) + .with_physical_optimizer_rule(Arc::new(DistributedPhysicalOptimizerRule)) + .with_distributed_files_per_task(1)? .build(); let ctx = SessionContext::from(state); - ctx.register_parquet( - "flights_1m", - "testdata/flights-1m.parquet", - ParquetReadOptions::default(), - ) - .await?; - ctx.register_parquet("weather", "testdata/weather", ParquetReadOptions::default()) .await?; let df = ctx.sql(&args.query).await?; - if args.explain { + if args.show_distributed_plan { let plan = df.create_physical_plan().await?; - let display = displayable(plan.as_ref()).indent(true).to_string(); - println!("{display}"); + println!("{}", display_plan_ascii(plan.as_ref(), false)); } else { let stream = df.execute_stream().await?; let batches = stream.try_collect::>().await?; @@ -90,26 +84,18 @@ impl InMemoryChannelResolver { })); let this = Self { - channel: FlightServiceClient::new(BoxCloneSyncChannel::new(channel)), + channel: create_flight_client(BoxCloneSyncChannel::new(channel)), }; let this_clone = this.clone(); - let endpoint = - ArrowFlightEndpoint::try_new(move |ctx: DistributedSessionBuilderContext| { - let this = this.clone(); - async move { - let builder = SessionStateBuilder::new() - .with_default_features() - .with_distributed_channel_resolver(this) - .with_runtime_env(ctx.runtime_env.clone()); - Ok(builder.build()) - } - }) - .unwrap(); + let endpoint = Worker::from_session_builder(move |ctx: WorkerQueryContext| { + let this = this.clone(); + async move { Ok(ctx.builder.with_distributed_channel_resolver(this).build()) } + }); tokio::spawn(async move { Server::builder() - .add_service(FlightServiceServer::new(endpoint)) + .add_service(endpoint.into_flight_server()) .serve_with_incoming(tokio_stream::once(Ok::<_, std::io::Error>(server))) .await }); @@ -120,10 +106,6 @@ impl InMemoryChannelResolver { #[async_trait] impl ChannelResolver for InMemoryChannelResolver { - fn get_urls(&self) -> Result, DataFusionError> { - Ok(vec![url::Url::parse(DUMMY_URL).unwrap()]) - } - async fn get_flight_client_for_url( &self, _: &url::Url, @@ -131,3 +113,11 @@ impl ChannelResolver for InMemoryChannelResolver { Ok(self.channel.clone()) } } + +struct InMemoryWorkerResolver; + +impl WorkerResolver for InMemoryWorkerResolver { + fn get_urls(&self) -> Result, DataFusionError> { + Ok(vec![url::Url::parse(DUMMY_URL).unwrap(); 16]) // simulate 16 workers. + } +} diff --git a/examples/localhost.md b/examples/localhost.md index 63719268..497e61f4 100644 --- a/examples/localhost.md +++ b/examples/localhost.md @@ -16,18 +16,17 @@ git lfs checkout ### Spawning the workers -In two different terminals spawn two ArrowFlightEndpoints +In two different terminals spawn two workers ```shell -cargo run --example localhost_worker -- 8080 --cluster-ports 8080,8081 +cargo run --example localhost_worker -- 8080 ``` ```shell -cargo run --example localhost_worker -- 8081 --cluster-ports 8080,8081 +cargo run --example localhost_worker -- 8081 ``` -- The positional numeric argument is the port in which each Arrow Flight endpoint will listen -- The `--cluster-ports` parameter tells the Arrow Flight endpoint all the available localhost workers in the cluster +The positional numeric argument is the port in which each worker will listen to. ### Issuing a distributed SQL query @@ -43,7 +42,7 @@ command, but further stages will be delegated to the workers running on ports 80 Additionally, the `--explain` flag can be passed to render the distributed plan: ```shell -cargo run --example localhost_run -- 'SELECT count(*), "MinTemp" FROM weather GROUP BY "MinTemp"' --cluster-ports 8080,8081 --explain +cargo run --example localhost_run -- 'SELECT count(*), "MinTemp" FROM weather GROUP BY "MinTemp"' --cluster-ports 8080,8081 --show-distributed-plan ``` ### Available tables diff --git a/examples/localhost_run.rs b/examples/localhost_run.rs index bfdb87dd..eae9b6cd 100644 --- a/examples/localhost_run.rs +++ b/examples/localhost_run.rs @@ -1,77 +1,57 @@ use arrow::util::pretty::pretty_format_batches; -use arrow_flight::flight_service_client::FlightServiceClient; use async_trait::async_trait; -use dashmap::{DashMap, Entry}; use datafusion::common::DataFusionError; use datafusion::execution::SessionStateBuilder; -use datafusion::physical_plan::displayable; use datafusion::prelude::{ParquetReadOptions, SessionContext}; use datafusion_distributed::{ - BoxCloneSyncChannel, ChannelResolver, DistributedExt, DistributedPhysicalOptimizerRule, + DistributedExt, DistributedPhysicalOptimizerRule, WorkerResolver, display_plan_ascii, }; use futures::TryStreamExt; use std::error::Error; use std::sync::Arc; use structopt::StructOpt; -use tonic::transport::Channel; use url::Url; #[derive(StructOpt)] #[structopt(name = "run", about = "A localhost Distributed DataFusion runner")] struct Args { + /// The SQL query to run. #[structopt()] query: String, - // --cluster-ports 8080,8081,8082 + /// The ports holding Distributed DataFusion workers. #[structopt(long = "cluster-ports", use_delimiter = true)] cluster_ports: Vec, + /// Whether the distributed plan should be rendered instead of executing the query. #[structopt(long)] - explain: bool, - - #[structopt(long, default_value = "3")] - network_shuffle_tasks: usize, - - #[structopt(long, default_value = "3")] - network_coalesce_tasks: usize, + show_distributed_plan: bool, } #[tokio::main] async fn main() -> Result<(), Box> { let args = Args::from_args(); - let localhost_resolver = LocalhostChannelResolver { + let localhost_resolver = LocalhostWorkerResolver { ports: args.cluster_ports, - cached: DashMap::new(), }; let state = SessionStateBuilder::new() .with_default_features() - .with_distributed_channel_resolver(localhost_resolver) - .with_physical_optimizer_rule(Arc::new( - DistributedPhysicalOptimizerRule::new() - .with_network_coalesce_tasks(args.network_coalesce_tasks) - .with_network_shuffle_tasks(args.network_shuffle_tasks), - )) + .with_distributed_worker_resolver(localhost_resolver) + .with_physical_optimizer_rule(Arc::new(DistributedPhysicalOptimizerRule)) + .with_distributed_files_per_task(1)? .build(); let ctx = SessionContext::from(state); - ctx.register_parquet( - "flights_1m", - "testdata/flights-1m.parquet", - ParquetReadOptions::default(), - ) - .await?; - ctx.register_parquet("weather", "testdata/weather", ParquetReadOptions::default()) .await?; let df = ctx.sql(&args.query).await?; - if args.explain { + if args.show_distributed_plan { let plan = df.create_physical_plan().await?; - let display = displayable(plan.as_ref()).indent(true).to_string(); - println!("{display}"); + println!("{}", display_plan_ascii(plan.as_ref(), false)); } else { let stream = df.execute_stream().await?; let batches = stream.try_collect::>().await?; @@ -82,13 +62,12 @@ async fn main() -> Result<(), Box> { } #[derive(Clone)] -struct LocalhostChannelResolver { +struct LocalhostWorkerResolver { ports: Vec, - cached: DashMap>, } #[async_trait] -impl ChannelResolver for LocalhostChannelResolver { +impl WorkerResolver for LocalhostWorkerResolver { fn get_urls(&self) -> Result, DataFusionError> { Ok(self .ports @@ -96,21 +75,4 @@ impl ChannelResolver for LocalhostChannelResolver { .map(|port| Url::parse(&format!("http://localhost:{port}")).unwrap()) .collect()) } - - async fn get_flight_client_for_url( - &self, - url: &Url, - ) -> Result, DataFusionError> { - match self.cached.entry(url.clone()) { - Entry::Occupied(v) => Ok(v.get().clone()), - Entry::Vacant(v) => { - let channel = Channel::from_shared(url.to_string()) - .unwrap() - .connect_lazy(); - let channel = FlightServiceClient::new(BoxCloneSyncChannel::new(channel)); - v.insert(channel.clone()); - Ok(channel) - } - } - } } diff --git a/examples/localhost_worker.rs b/examples/localhost_worker.rs index 809175d7..58d329e0 100644 --- a/examples/localhost_worker.rs +++ b/examples/localhost_worker.rs @@ -1,88 +1,24 @@ -use arrow_flight::flight_service_client::FlightServiceClient; -use arrow_flight::flight_service_server::FlightServiceServer; -use async_trait::async_trait; -use dashmap::{DashMap, Entry}; -use datafusion::common::DataFusionError; -use datafusion::execution::SessionStateBuilder; -use datafusion_distributed::{ - ArrowFlightEndpoint, BoxCloneSyncChannel, ChannelResolver, DistributedExt, - DistributedSessionBuilderContext, -}; +use datafusion_distributed::Worker; use std::error::Error; use std::net::{IpAddr, Ipv4Addr, SocketAddr}; use structopt::StructOpt; -use tonic::transport::{Channel, Server}; -use url::Url; +use tonic::transport::Server; #[derive(StructOpt)] #[structopt(name = "localhost_worker", about = "A localhost DataFusion worker")] struct Args { #[structopt(default_value = "8080")] port: u16, - - // --cluster-ports 8080,8081,8082 - #[structopt(long = "cluster-ports", use_delimiter = true)] - cluster_ports: Vec, } #[tokio::main] async fn main() -> Result<(), Box> { let args = Args::from_args(); - let localhost_resolver = LocalhostChannelResolver { - ports: args.cluster_ports, - cached: DashMap::new(), - }; - - let endpoint = ArrowFlightEndpoint::try_new(move |ctx: DistributedSessionBuilderContext| { - let local_host_resolver = localhost_resolver.clone(); - async move { - Ok(SessionStateBuilder::new() - .with_runtime_env(ctx.runtime_env) - .with_distributed_channel_resolver(local_host_resolver) - .with_default_features() - .build()) - } - })?; - Server::builder() - .add_service(FlightServiceServer::new(endpoint)) + .add_service(Worker::default().into_flight_server()) .serve(SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), args.port)) .await?; Ok(()) } - -#[derive(Clone)] -struct LocalhostChannelResolver { - ports: Vec, - cached: DashMap>, -} - -#[async_trait] -impl ChannelResolver for LocalhostChannelResolver { - fn get_urls(&self) -> Result, DataFusionError> { - Ok(self - .ports - .iter() - .map(|port| Url::parse(&format!("http://localhost:{port}")).unwrap()) - .collect()) - } - - async fn get_flight_client_for_url( - &self, - url: &Url, - ) -> Result, DataFusionError> { - match self.cached.entry(url.clone()) { - Entry::Occupied(v) => Ok(v.get().clone()), - Entry::Vacant(v) => { - let channel = Channel::from_shared(url.to_string()) - .unwrap() - .connect_lazy(); - let channel = FlightServiceClient::new(BoxCloneSyncChannel::new(channel)); - v.insert(channel.clone()); - Ok(channel) - } - } - } -} diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 4bd1ff45..ab40f4f4 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,3 +1,3 @@ [toolchain] -channel = "1.86.0" +channel = "1.88.0" profile = "default" diff --git a/src/channel_resolver_ext.rs b/src/channel_resolver_ext.rs deleted file mode 100644 index c8c1c5b0..00000000 --- a/src/channel_resolver_ext.rs +++ /dev/null @@ -1,61 +0,0 @@ -use arrow_flight::flight_service_client::FlightServiceClient; -use async_trait::async_trait; -use datafusion::common::exec_datafusion_err; -use datafusion::error::DataFusionError; -use datafusion::prelude::SessionConfig; -use std::sync::Arc; -use tonic::body::Body; -use url::Url; - -pub(crate) fn set_distributed_channel_resolver( - cfg: &mut SessionConfig, - channel_resolver: impl ChannelResolver + Send + Sync + 'static, -) { - cfg.set_extension(Arc::new(ChannelResolverExtension(Arc::new( - channel_resolver, - )))); -} - -pub(crate) fn get_distributed_channel_resolver( - cfg: &SessionConfig, -) -> Result, DataFusionError> { - cfg.get_extension::() - .map(|cm| cm.0.clone()) - .ok_or_else(|| exec_datafusion_err!("ChannelResolver not present in the session config")) -} - -#[derive(Clone)] -struct ChannelResolverExtension(Arc); - -pub type BoxCloneSyncChannel = tower::util::BoxCloneSyncService< - http::Request, - http::Response, - tonic::transport::Error, ->; - -/// Abstracts networking details so that users can implement their own network resolution -/// mechanism. -#[async_trait] -pub trait ChannelResolver { - /// Gets all available worker URLs. Used during stage assignment. - fn get_urls(&self) -> Result, DataFusionError>; - /// For a given URL, get an Arrow Flight client for communicating to it. - async fn get_flight_client_for_url( - &self, - url: &Url, - ) -> Result, DataFusionError>; -} - -#[async_trait] -impl ChannelResolver for Arc { - fn get_urls(&self) -> Result, DataFusionError> { - self.as_ref().get_urls() - } - - async fn get_flight_client_for_url( - &self, - url: &Url, - ) -> Result, DataFusionError> { - self.as_ref().get_flight_client_for_url(url).await - } -} diff --git a/src/common/children_helpers.rs b/src/common/children_helpers.rs new file mode 100644 index 00000000..c7b4e3ce --- /dev/null +++ b/src/common/children_helpers.rs @@ -0,0 +1,18 @@ +use datafusion::common::{DataFusionError, plan_err}; +use datafusion::physical_plan::ExecutionPlan; +use std::borrow::Borrow; +use std::sync::Arc; + +pub(crate) fn require_one_child( + children: L, +) -> Result, DataFusionError> +where + L: AsRef<[T]>, + T: Borrow>, +{ + let children = children.as_ref(); + if children.len() != 1 { + return plan_err!("Expected exactly 1 children, got {}", children.len()); + } + Ok(children[0].borrow().clone()) +} diff --git a/src/common/map_last_stream.rs b/src/common/map_last_stream.rs index d0eb7798..0017520e 100644 --- a/src/common/map_last_stream.rs +++ b/src/common/map_last_stream.rs @@ -1,13 +1,12 @@ use futures::{Stream, StreamExt, stream}; use std::task::Poll; -/// Maps the last element of the provided stream. +/// Maps each element of the provided stream, providing an additional flag that determines +/// whether the mapped element is the last one or not. pub(crate) fn map_last_stream( mut input: impl Stream + Unpin, - map_f: impl FnOnce(T) -> T, + map_f: impl Fn(T, bool) -> T, ) -> impl Stream + Unpin { - let mut final_closure = Some(map_f); - // this is used to peek the new value so that we can map upon emitting the last message let mut current_value = None; @@ -23,8 +22,7 @@ pub(crate) fn map_last_stream( Some(existing) => { current_value = Some(new_val); - - Poll::Ready(Some(existing)) + Poll::Ready(Some(map_f(existing, false))) } } } @@ -33,12 +31,7 @@ pub(crate) fn map_last_stream( Some(existing) => { // make sure we wake ourselves to finish the stream cx.waker().wake_by_ref(); - - if let Some(closure) = final_closure.take() { - Poll::Ready(Some(closure(existing))) - } else { - unreachable!("the closure is only executed once") - } + Poll::Ready(Some(map_f(existing, true))) } None => Poll::Ready(None), }, @@ -53,7 +46,7 @@ mod tests { #[tokio::test] async fn test_map_last_stream_empty_stream() { let input = stream::empty::(); - let mapped = map_last_stream(input, |x| x + 10); + let mapped = map_last_stream(input, |x, _| x + 10); let result: Vec = mapped.collect().await; assert_eq!(result, Vec::::new()); } @@ -61,7 +54,7 @@ mod tests { #[tokio::test] async fn test_map_last_stream_single_element() { let input = stream::iter(vec![5]); - let mapped = map_last_stream(input, |x| x * 2); + let mapped = map_last_stream(input, |x, _| x * 2); let result: Vec = mapped.collect().await; assert_eq!(result, vec![10]); } @@ -69,7 +62,7 @@ mod tests { #[tokio::test] async fn test_map_last_stream_multiple_elements() { let input = stream::iter(vec![1, 2, 3, 4]); - let mapped = map_last_stream(input, |x| x + 100); + let mapped = map_last_stream(input, |x, is_last| if is_last { x + 100 } else { x }); let result: Vec = mapped.collect().await; assert_eq!(result, vec![1, 2, 3, 104]); // Only the last element is transformed } @@ -77,7 +70,7 @@ mod tests { #[tokio::test] async fn test_map_last_stream_preserves_order() { let input = stream::iter(vec![10, 20, 30, 40, 50]); - let mapped = map_last_stream(input, |x| x - 50); + let mapped = map_last_stream(input, |x, is_last| if is_last { x - 50 } else { x }); let result: Vec = mapped.collect().await; assert_eq!(result, vec![10, 20, 30, 40, 0]); // Last element: 50 - 50 = 0 } diff --git a/src/common/mod.rs b/src/common/mod.rs index ae81b10b..da6d83ec 100644 --- a/src/common/mod.rs +++ b/src/common/mod.rs @@ -1,4 +1,7 @@ +mod children_helpers; mod map_last_stream; -pub mod ttl_map; +mod on_drop_stream; +pub(crate) use children_helpers::require_one_child; pub(crate) use map_last_stream::map_last_stream; +pub(crate) use on_drop_stream::on_drop_stream; diff --git a/src/common/on_drop_stream.rs b/src/common/on_drop_stream.rs new file mode 100644 index 00000000..c01f8874 --- /dev/null +++ b/src/common/on_drop_stream.rs @@ -0,0 +1,119 @@ +use futures::Stream; +use pin_project::pin_project; +use std::pin::Pin; +use std::task::{Context, Poll}; + +/// Wraps a stream and fires a callback when the stream is dropped. +/// +/// This is useful for cleanup operations like releasing memory reservations, +/// cancelling background tasks, or logging when a stream consumer stops early. +/// +/// # Example +/// ```ignore +/// let stream = on_drop_stream(inner_stream, || { +/// println!("Stream was dropped!"); +/// }); +/// ``` +pub(crate) fn on_drop_stream(inner: S, on_drop: F) -> OnDropStream +where + S: Stream, + F: FnOnce(), +{ + OnDropStream { + inner, + on_drop: Some(on_drop), + } +} + +/// A stream wrapper that fires a callback when dropped. +#[pin_project(PinnedDrop)] +pub(crate) struct OnDropStream { + #[pin] + inner: S, + on_drop: Option, +} + +impl Stream for OnDropStream +where + S: Stream, + F: FnOnce(), +{ + type Item = S::Item; + + fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + self.project().inner.poll_next(cx) + } + + fn size_hint(&self) -> (usize, Option) { + self.inner.size_hint() + } +} + +#[pin_project::pinned_drop] +impl PinnedDrop for OnDropStream { + fn drop(self: Pin<&mut Self>) { + let this = self.project(); + if let Some(on_drop) = this.on_drop.take() { + on_drop(); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use futures::StreamExt; + use std::sync::Arc; + use std::sync::atomic::{AtomicBool, Ordering}; + + #[tokio::test] + async fn fires_on_drop_when_fully_consumed() { + let dropped = Arc::new(AtomicBool::new(false)); + let dropped_clone = Arc::clone(&dropped); + + let stream = futures::stream::iter(vec![1, 2, 3]); + let stream = on_drop_stream(stream, move || { + dropped_clone.store(true, Ordering::SeqCst); + }); + + // Fully consume the stream + let items: Vec<_> = stream.collect().await; + assert_eq!(items, vec![1, 2, 3]); + assert!(dropped.load(Ordering::SeqCst)); + } + + #[tokio::test] + async fn fires_on_drop_when_partially_consumed() { + let dropped = Arc::new(AtomicBool::new(false)); + let dropped_clone = Arc::clone(&dropped); + + let stream = futures::stream::iter(vec![1, 2, 3, 4, 5]); + let mut stream = on_drop_stream(stream, move || { + dropped_clone.store(true, Ordering::SeqCst); + }); + + // Only consume part of the stream + assert_eq!(stream.next().await, Some(1)); + assert_eq!(stream.next().await, Some(2)); + assert!(!dropped.load(Ordering::SeqCst)); + + // Drop the stream + drop(stream); + assert!(dropped.load(Ordering::SeqCst)); + } + + #[tokio::test] + async fn fires_on_drop_when_never_consumed() { + let dropped = Arc::new(AtomicBool::new(false)); + let dropped_clone = Arc::clone(&dropped); + + let stream = futures::stream::iter(vec![1, 2, 3]); + let stream = on_drop_stream(stream, move || { + dropped_clone.store(true, Ordering::SeqCst); + }); + + assert!(!dropped.load(Ordering::SeqCst)); + drop(stream); + assert!(dropped.load(Ordering::SeqCst)); + } +} diff --git a/src/common/ttl_map.rs b/src/common/ttl_map.rs deleted file mode 100644 index b3878ada..00000000 --- a/src/common/ttl_map.rs +++ /dev/null @@ -1,544 +0,0 @@ -//! TTLMap is a DashMap that automatically removes entries after a specified time-to-live (TTL). -//! -//! How the Time Wheel Works -//! -//! Time Buckets: [0] [1] [2] [3] [4] [5] [6] [7] ... -//! Current Time: ^ -//! | -//! time % buckets.len() -//! -//! When inserting key "A" at time=2: -//! - Key "A" goes into bucket[(2-1) % 8] = bucket[1] -//! - Key "A" will be expired when time advances to bucket[1] again -//! -//! Generally, keys in a bucket expire when the wheel makes a full rotation, making -//! the total TTL equal to the tick duration * buckets.len(). -//! -//! Usage -//! ```rust,ignore -//! use std::time::Duration; -//! use datafusion_distributed::common::ttl_map::{TTLMapConfig, TTLMap}; -//! let config = TTLMapConfig { tick: Duration::from_secs(5), ttl: Duration::from_secs(60) }; -//! let ttl_map = TTLMap::try_new(config).unwrap(); -//! let value = ttl_map.get_or_init("key", || "value"); -//! ``` -//! TODO(#101): If an existing entry is accessed, reset its TTL timer. -use dashmap::{DashMap, Entry}; -use datafusion::error::DataFusionError; -use std::collections::HashSet; -use std::hash::Hash; -use std::sync::Arc; -use std::sync::atomic::AtomicU64; -#[cfg(test)] -use std::sync::atomic::{AtomicUsize, Ordering::Relaxed}; -use std::time::Duration; -use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender}; - -/// A bucket is a set of keys that are managed edited asynchronously. -#[derive(Clone)] -struct Bucket { - /// tx is used to send a BucketOp to the task - tx: UnboundedSender>, - /// This task is responsible for managing keys - _task: Arc>, -} - -/// BucketOp is used to communicate with the task responsible for managing keys. -enum BucketOp { - Insert { key: K }, - Clear, -} - -impl Bucket -where - K: Hash + Eq + Send + Sync + Clone + 'static, -{ - /// new creates a new Bucket - fn new(data: Arc>) -> Self - where - V: Send + Sync + 'static, - { - // TODO: To avoid unbounded growth, consider making this bounded. Alternatively, we can - // introduce a dynamic GC period to ensure that GC can keep up. - let (tx, rx) = tokio::sync::mpsc::unbounded_channel(); - Self { - tx, - _task: Arc::new(tokio::spawn( - async move { Bucket::task(rx, data.clone()).await }, - )), - } - } - - fn register_key(&self, key: K) { - // We can safely ignore the error here because the receiver is never closed. - // If the receiver is dropped, it means this struct is being dropped. - let _ = self.tx.send(BucketOp::Insert { key }); - } - - fn clear(&self) { - // We can safely ignore the error here because the receiver is never closed. - // If the receiver is dropped, it means this struct is being dropped. - let _ = self.tx.send(BucketOp::Clear); - } - - /// task is responsible for managing a subset of keys in the map. - async fn task(mut rx: UnboundedReceiver>, data: Arc>) - where - V: Send + Sync + 'static, - { - let mut shard = HashSet::new(); - while let Some(op) = rx.recv().await { - match op { - BucketOp::Insert { key } => { - shard.insert(key); - } - BucketOp::Clear => { - let keys_to_delete = std::mem::take(&mut shard); - for key in keys_to_delete { - data.remove(&key); - } - } - } - } - } -} - -/// TTLMap is a key-value store that automatically removes entries after a specified time-to-live. -pub struct TTLMap { - /// The buckets in the time wheel - buckets: Vec>, - - /// gc_scheduler_task is responsible scheduling GC operations among the Buckets in the time wheel. - gc_scheduler_task: Option>>, - - /// The actual key-value storage using DashMap for concurrent access - data: Arc>, - - /// Atomic counter tracking the current time. Incremented by the background GC task every `tick` duration. - time: Arc, - - config: TTLMapConfig, - - #[cfg(test)] - metrics: TTLMapMetrics, -} - -#[cfg(test)] -#[derive(Default)] -struct TTLMapMetrics { - dash_map_lock_contention_time: AtomicUsize, - ttl_accounting_time: AtomicUsize, -} - -/// TTLMapConfig configures the TTLMap parameters such as the TTL and tick period. -pub struct TTLMapConfig { - /// How often the map is checks for expired entries. - /// This must be less than `ttl`. It's recommended to set `ttl` to a multiple - /// of `tick`. - pub tick: Duration, - /// The time-to-live for entries - pub ttl: Duration, -} - -impl Default for TTLMapConfig { - fn default() -> Self { - Self { - tick: Duration::from_secs(3), - ttl: Duration::from_secs(60), - } - } -} - -impl TTLMapConfig { - fn is_valid(&self, tick: Duration, ttl: Duration) -> Result<(), DataFusionError> { - if tick > ttl && !tick.is_zero() { - return Err(DataFusionError::Configuration( - "`tick` must be nonzero and <= `ttl`".to_string(), - )); - } - Ok(()) - } -} - -impl TTLMap -where - K: Eq + Hash + Send + Sync + Clone + 'static, - V: Default + Clone + Send + Sync + 'static, -{ - // try_new creates a new TTLMap. - pub fn try_new(config: TTLMapConfig) -> Result { - config.is_valid(config.tick, config.ttl)?; - let mut map = Self::_new(config); - map._start_gc(); - Ok(map) - } - - fn _new(config: TTLMapConfig) -> Self { - let stage_targets = Arc::new(DashMap::new()); - let bucket_count = (config.ttl.as_nanos() / config.tick.as_nanos()) as usize; - let mut buckets = Vec::with_capacity(bucket_count); - for _ in 0..bucket_count { - buckets.push(Bucket::new(stage_targets.clone())); - } - - Self { - buckets, - data: stage_targets, - time: Arc::new(AtomicU64::new(0)), - gc_scheduler_task: None, - config, - #[cfg(test)] - metrics: Default::default(), - } - } - - // Start and set the background GC task. - fn _start_gc(&mut self) { - let time = self.time.clone(); - let buckets = self.buckets.clone(); - let period = self.config.tick; - - let gc_task = tokio::spawn(async move { - Self::run_gc_loop(time, period, &buckets).await; - }); - - self.gc_scheduler_task = Some(vec![gc_task]); - } - - /// get_or_default executes the provided closure with a reference to the map entry for the given key. - /// If the key does not exist, it inserts a new entry with the default value. - pub fn get_or_init(&self, key: K, init: F) -> V - where - F: FnOnce() -> V, - { - let mut new_entry = false; - - #[cfg(test)] - let start = std::time::Instant::now(); - - let value = match self.data.entry(key.clone()) { - Entry::Vacant(entry) => { - let value = init(); - entry.insert(value.clone()); - new_entry = true; - value - } - Entry::Occupied(entry) => entry.get().clone(), - }; - - #[cfg(test)] - self.metrics - .dash_map_lock_contention_time - .fetch_add(start.elapsed().as_nanos() as usize, Relaxed); - - // Insert the key into the previous bucket, meaning the key will be evicted - // when the wheel completes a full rotation. - if new_entry { - #[cfg(test)] - let start = std::time::Instant::now(); - - let time = self.time.load(std::sync::atomic::Ordering::SeqCst); - let bucket_index = (time.wrapping_sub(1)) % self.buckets.len() as u64; - self.buckets[bucket_index as usize].register_key(key); - - #[cfg(test)] - self.metrics - .ttl_accounting_time - .fetch_add(start.elapsed().as_nanos() as usize, Relaxed); - } - - value - } - - /// Removes the key from the map. - /// TODO: Consider removing the key from the time bucket as well. We would need to know which - /// bucket the key was in to do this. One idea is to store the bucket idx in the map value. - pub fn remove(&self, key: K) { - self.data.remove(&key); - } - - /// Returns the number of entries currently stored in the map - #[cfg(test)] - pub fn len(&self) -> usize { - self.data.len() - } - - /// Returns an iterator over the keys currently stored in the map - #[cfg(test)] - pub fn keys(&self) -> impl Iterator + '_ { - self.data.iter().map(|entry| entry.key().clone()) - } - - /// run_gc_loop will continuously clear expired entries from the map, checking every `period`. The - /// function terminates if `shutdown` is signalled. - async fn run_gc_loop(time: Arc, period: Duration, buckets: &[Bucket]) { - loop { - tokio::time::sleep(period).await; - Self::gc(time.clone(), buckets); - } - } - - fn gc(time: Arc, buckets: &[Bucket]) { - let index = time.load(std::sync::atomic::Ordering::SeqCst) % buckets.len() as u64; - buckets[index as usize].clear(); - time.fetch_add(1, std::sync::atomic::Ordering::SeqCst); - } -} - -#[cfg(test)] -mod tests { - use super::*; - use std::sync::atomic::Ordering; - use tokio::time::Duration; - - #[tokio::test] - async fn test_basic_insert_and_get() { - let ttl_map = TTLMap::::_new(TTLMapConfig::default()); - - ttl_map.get_or_init("key1".to_string(), || 42); - - let value = ttl_map.get_or_init("key1".to_string(), || 0); - assert_eq!(value, 42); - } - - #[tokio::test] - async fn test_time_wheel_bucket_calculation() { - let ttl_map = TTLMap::::_new(TTLMapConfig { - ttl: Duration::from_secs(1), - tick: Duration::from_millis(100), - }); - - // With 1s TTL and 100ms tick, we should have 10 buckets - assert_eq!(ttl_map.buckets.len(), 10); - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 4)] - async fn test_gc_expiration() { - // Create map with 10 buckets. - let ttl_map = TTLMap::::_new(TTLMapConfig { - ttl: Duration::from_secs(1), - tick: Duration::from_millis(100), - }); - - // Initial batch of entries - ttl_map.get_or_init("key1".to_string(), || 42); - ttl_map.get_or_init("key2".to_string(), || 84); - assert_eq!(ttl_map.data.len(), 2); - - // Run partial GC cycles (should not expire yet) - for _ in 0..5 { - TTLMap::::gc(ttl_map.time.clone(), &ttl_map.buckets); - } - assert_eventually(|| ttl_map.data.len() == 2, Duration::from_millis(100)).await; // Still there - - // Add more entries mid-cycle - ttl_map.get_or_init("key3".to_string(), || 168); - ttl_map.get_or_init("key4".to_string(), || 0); // Default value (0) - ttl_map.get_or_init("key5".to_string(), || 210); - assert_eq!(ttl_map.data.len(), 5); - - // Verify default value was set - let default_value = ttl_map.get_or_init("key4".to_string(), || 0); - assert_eq!(default_value, 0); - - // Complete the first rotation to expire initial entries - for _ in 5..10 { - TTLMap::::gc(ttl_map.time.clone(), &ttl_map.buckets); - } - - assert_eventually(|| ttl_map.data.len() == 3, Duration::from_millis(100)).await; // Initial entries expired, new entries still alive - - // Add entries after expiration - ttl_map.get_or_init("new_key1".to_string(), || 999); - ttl_map.get_or_init("new_key2".to_string(), || 0); // Default value - assert_eq!(ttl_map.data.len(), 5); // 3 from mid-cycle + 2 new ones - - // Verify values - let value1 = ttl_map.get_or_init("new_key1".to_string(), || 0); - assert_eq!(value1, 999); - let value2 = ttl_map.get_or_init("new_key2".to_string(), || 0); - assert_eq!(value2, 0); - - // Run additional GC cycles to expire remaining entries - // Mid-cycle entries (bucket 4) expire at time=14, late entries (bucket 9) expire at time=19 - for _ in 10..20 { - TTLMap::::gc(ttl_map.time.clone(), &ttl_map.buckets); - } - assert_eventually(|| ttl_map.data.is_empty(), Duration::from_millis(100)).await; - // All entries expired - } - - // assert_eventually checks a condition every 10ms for a maximum of timeout - async fn assert_eventually(assertion: F, timeout: Duration) - where - F: Fn() -> bool, - { - let start = std::time::Instant::now(); - while start.elapsed() < timeout { - if assertion() { - return; - } - tokio::time::sleep(Duration::from_millis(10)).await; - } - panic!("Assertion failed within {:?}", timeout); - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 8)] - #[ignore] // the test is flaky, uncomment once flakyness is solved - async fn test_concurrent_gc_and_access() { - let ttl_map = TTLMap::::try_new(TTLMapConfig { - ttl: Duration::from_millis(10), - tick: Duration::from_millis(2), - }) - .unwrap(); - - assert!(ttl_map.gc_scheduler_task.is_some()); - - let ttl_map = Arc::new(ttl_map); - - // Spawn 5 concurrent tasks - let mut handles = Vec::new(); - for task_id in 0..10 { - let map = Arc::clone(&ttl_map); - handles.push(tokio::spawn(async move { - for i in 0..100 { - let key = format!("task{}_key{}", task_id, i % 10); - map.get_or_init(key.clone(), || task_id * 100 + i); - } - })); - let map2 = Arc::clone(&ttl_map); - handles.push(tokio::spawn(async move { - // Remove some keys which may or may not exist. - for i in 0..50 { - let key = format!("task{}_key{}", task_id, i % 15); - map2.remove(key) - } - })); - } - - // Wait for all tasks to complete - for handle in handles { - handle.await.unwrap(); - } - - assert_eventually(|| ttl_map.data.is_empty(), Duration::from_millis(20)).await; - } - - #[tokio::test] - async fn test_wraparound_time() { - let ttl_map = TTLMap::::_new(TTLMapConfig { - ttl: Duration::from_millis(20), - tick: Duration::from_millis(10), - }); - - // Manually set time near overflow - ttl_map.time.store(u64::MAX - 2, Ordering::SeqCst); - - ttl_map.get_or_init("test_key".to_string(), || 999); - - // Run GC to cause time wraparound - for _ in 0..5 { - TTLMap::::gc(ttl_map.time.clone(), &ttl_map.buckets); - } - - // Entry should be expired and time should have wrapped - assert_eventually(|| ttl_map.data.is_empty(), Duration::from_millis(100)).await; - let final_time = ttl_map.time.load(Ordering::SeqCst); - assert!(final_time < 100); - } - - // Run with `cargo test bench_lock_contention --release -- --nocapture` to see output. - #[tokio::test(flavor = "multi_thread", worker_threads = 16)] - async fn bench_lock_contention() { - use std::time::Instant; - - let config = TTLMapConfig { - tick: Duration::from_micros(1), - ttl: Duration::from_micros(2), - }; - - let ttl_map = TTLMap::::try_new(config).unwrap(); - - let ttl_map = Arc::new(ttl_map); - - let start_time = Instant::now(); - let task_count = 100_000; - - // Spawn 10 tasks that repeatedly read the same keys - let mut handles = Vec::new(); - for task_id in 0..task_count { - let map = Arc::clone(&ttl_map); - let handle = tokio::spawn(async move { - // All tasks fight for the same keys - maximum contention - let start = Instant::now(); - let _value = map.get_or_init(rand::random(), || task_id * 1000); - start.elapsed().as_nanos() - }); - handles.push(handle); - } - - // Wait for all tasks and collect operation counts - let mut avg_time = 0; - for handle in handles { - avg_time += handle.await.unwrap(); - } - avg_time /= task_count as u128; - - let elapsed = start_time.elapsed(); - - println!("\n=== TTLMap Lock Contention Benchmark ==="); - println!("Tasks: {}", task_count); - println!("Total time: {:.2?}", elapsed); - println!("Average latency: {:.2} ns per operation", avg_time); - println!("Entries remaining: {}", ttl_map.data.len()); - println!( - "DashMap Lock contention time: {}ms", - ttl_map - .metrics - .dash_map_lock_contention_time - .load(Ordering::SeqCst) - / 1_000_000 - ); - println!( - "Accounting time: {}ms", - ttl_map.metrics.ttl_accounting_time.load(Ordering::SeqCst) / 1_000_000 - ); - } - - #[tokio::test] - async fn test_remove_with_manual_gc() { - let ttl_map = TTLMap::::_new(TTLMapConfig { - ttl: Duration::from_millis(50), - tick: Duration::from_millis(10), - }); - - ttl_map.get_or_init("key1".to_string(), || 100); - ttl_map.get_or_init("key2".to_string(), || 200); - ttl_map.get_or_init("key3".to_string(), || 300); - assert_eq!(ttl_map.data.len(), 3); - - // Remove key2 and verify the others remain. - ttl_map.remove("key2".to_string()); - assert_eq!(ttl_map.data.len(), 2); - let val1 = ttl_map.get_or_init("key1".to_string(), || 999); - assert_eq!(val1, 100); - let val3 = ttl_map.get_or_init("key3".to_string(), || 999); - assert_eq!(val3, 300); - - // key2 should be recreated with new value - let val2 = ttl_map.get_or_init("key2".to_string(), || 999); - assert_eq!(val2, 999); // New value since it was removed - assert_eq!(ttl_map.data.len(), 3); - let val3 = ttl_map.get_or_init("key2".to_string(), || 200); - assert_eq!(val3, 999); - - // Remove key1 before GCing. - ttl_map.remove("key1".to_string()); - - // Run GC and verify the map is empty. - for _ in 0..5 { - TTLMap::::gc(ttl_map.time.clone(), &ttl_map.buckets); - } - assert_eventually(|| ttl_map.data.is_empty(), Duration::from_millis(100)).await; - } -} diff --git a/src/config_extension_ext.rs b/src/config_extension_ext.rs index a287bf2c..88556ffe 100644 --- a/src/config_extension_ext.rs +++ b/src/config_extension_ext.rs @@ -1,101 +1,126 @@ use datafusion::common::{DataFusionError, internal_datafusion_err}; use datafusion::config::ConfigExtension; -use datafusion::execution::TaskContext; use datafusion::prelude::SessionConfig; use http::{HeaderMap, HeaderName}; use std::error::Error; use std::str::FromStr; use std::sync::Arc; -const FLIGHT_METADATA_CONFIG_PREFIX: &str = "x-datafusion-distributed-config-"; +pub(crate) const FLIGHT_METADATA_CONFIG_PREFIX: &str = "x-datafusion-distributed-config-"; pub(crate) fn set_distributed_option_extension( cfg: &mut SessionConfig, t: T, -) -> Result<(), DataFusionError> { - fn parse_err(err: impl Error) -> DataFusionError { - DataFusionError::Internal(format!("Failed to add config extension: {err}")) - } - let mut meta = HeaderMap::new(); - - for entry in t.entries() { - if let Some(value) = entry.value { - meta.insert( - HeaderName::from_str(&format!( - "{}{}.{}", - FLIGHT_METADATA_CONFIG_PREFIX, - T::PREFIX, - entry.key - )) - .map_err(parse_err)?, - value.parse().map_err(parse_err)?, - ); - } - } - let flight_metadata = ContextGrpcMetadata(meta); - match cfg.get_extension::() { - None => cfg.set_extension(Arc::new(flight_metadata)), - Some(prev) => { - let prev = prev.as_ref().clone(); - cfg.set_extension(Arc::new(prev.merge(flight_metadata))) - } - } +) { cfg.options_mut().extensions.insert(t); - Ok(()) + let mut propagation_ctx = match cfg.get_extension::() { + None => ConfigExtensionPropagationContext::default(), + Some(prev) => prev.as_ref().clone(), + }; + propagation_ctx.prefixes.push(T::PREFIX); + cfg.set_extension(Arc::new(propagation_ctx)); } -pub(crate) fn set_distributed_option_extension_from_headers( - cfg: &mut SessionConfig, +pub(crate) fn set_distributed_option_extension_from_headers<'a, T: ConfigExtension + Default>( + cfg: &'a mut SessionConfig, headers: &HeaderMap, -) -> Result<(), DataFusionError> { - let mut result = T::default(); - let mut found_some = false; +) -> Result<&'a T, DataFusionError> { + enum MutOrOwned<'a, T> { + Mut(&'a mut T), + Owned(T), + } + + impl<'a, T> MutOrOwned<'a, T> { + fn as_mut(&mut self) -> &mut T { + match self { + MutOrOwned::Mut(v) => v, + MutOrOwned::Owned(v) => v, + } + } + } + + let mut propagation_ctx = match cfg.get_extension::() { + None => ConfigExtensionPropagationContext::default(), + Some(prev) => prev.as_ref().clone(), + }; + propagation_ctx.prefixes.push(T::PREFIX); + cfg.set_extension(Arc::new(propagation_ctx)); + + // If the config extension existed before, we want to modify instead of adding a new one from + // scratch. If not, we'll start from scratch with a new one. + let mut result = match cfg.options_mut().extensions.get_mut::() { + Some(v) => MutOrOwned::Mut(v), + None => MutOrOwned::Owned(T::default()), + }; + for (k, v) in headers.iter() { let key = k.as_str().trim_start_matches(FLIGHT_METADATA_CONFIG_PREFIX); let prefix = format!("{}.", T::PREFIX); if key.starts_with(&prefix) { - found_some = true; - result.set( + result.as_mut().set( key.trim_start_matches(&prefix), v.to_str() .map_err(|err| internal_datafusion_err!("Cannot parse header value: {err}"))?, )?; } } - if !found_some { - return Ok(()); + + // Only insert the extension if it is not already there. If this is otherwise MutOrOwned::Mut it + // means that the extension was already there, and we already modified it. + if let MutOrOwned::Owned(v) = result { + cfg.options_mut().extensions.insert(v); } - cfg.options_mut().extensions.insert(result); - Ok(()) + cfg.options().extensions.get().ok_or_else(|| { + internal_datafusion_err!("ProgrammingError: a config option extension was just inserted, but it was not immediately retrievable") + }) } #[derive(Clone, Debug, Default)] -pub(crate) struct ContextGrpcMetadata(pub HeaderMap); +struct ConfigExtensionPropagationContext { + prefixes: Vec<&'static str>, +} -impl ContextGrpcMetadata { - fn merge(mut self, other: Self) -> Self { - for (k, v) in other.0.into_iter() { - let Some(k) = k else { continue }; - self.0.insert(k, v); - } - self +pub(crate) fn get_config_extension_propagation_headers( + cfg: &SessionConfig, +) -> Result { + fn parse_err(err: impl Error) -> DataFusionError { + DataFusionError::Internal(format!("Failed to add config extension: {err}")) } + let prefixes_to_send = cfg + .get_extension::() + .unwrap_or_default(); - pub fn headers_from_ctx(ctx: &Arc) -> HeaderMap { - ctx.session_config() - .get_extension::() - .as_ref() - .map(|v| v.as_ref().clone()) - .unwrap_or_default() - .0 + if prefixes_to_send.prefixes.is_empty() { + return Ok(HeaderMap::new()); + } + + let mut headers = HeaderMap::new(); + + for (prefix, extension) in cfg.options().extensions.iter() { + if !prefixes_to_send.prefixes.contains(&prefix) { + continue; + } + for entry in extension.entries() { + let Some(value) = entry.value else { + continue; + }; + + let key = entry.key; + headers.insert( + HeaderName::from_str(&format!("{FLIGHT_METADATA_CONFIG_PREFIX}{prefix}.{key}")) + .map_err(parse_err)?, + value.parse().map_err(parse_err)?, + ); + } } + Ok(headers) } #[cfg(test)] mod tests { use crate::config_extension_ext::{ - ContextGrpcMetadata, set_distributed_option_extension, - set_distributed_option_extension_from_headers, + ConfigExtensionPropagationContext, get_config_extension_propagation_headers, + set_distributed_option_extension, set_distributed_option_extension_from_headers, }; use datafusion::common::extensions_options; use datafusion::config::ConfigExtension; @@ -113,12 +138,12 @@ mod tests { baz: false, }; - set_distributed_option_extension(&mut config, opt)?; - let metadata = config.get_extension::().unwrap(); + set_distributed_option_extension(&mut config, opt); + let headers = get_config_extension_propagation_headers(&config)?; let mut new_config = SessionConfig::new(); set_distributed_option_extension_from_headers::( &mut new_config, - &metadata.0, + &headers, )?; let opt = get_ext::(&config); @@ -136,17 +161,17 @@ mod tests { let mut config = SessionConfig::new(); let opt = CustomExtension::default(); - set_distributed_option_extension(&mut config, opt)?; + set_distributed_option_extension(&mut config, opt); - let flight_metadata = config.get_extension::(); + let flight_metadata = config.get_extension::(); assert!(flight_metadata.is_some()); - let metadata = &flight_metadata.unwrap().0; - assert!(metadata.contains_key("x-datafusion-distributed-config-custom.foo")); - assert!(metadata.contains_key("x-datafusion-distributed-config-custom.bar")); - assert!(metadata.contains_key("x-datafusion-distributed-config-custom.baz")); + let headers = get_config_extension_propagation_headers(&config)?; + assert!(headers.contains_key("x-datafusion-distributed-config-custom.foo")); + assert!(headers.contains_key("x-datafusion-distributed-config-custom.bar")); + assert!(headers.contains_key("x-datafusion-distributed-config-custom.baz")); - let get = |key: &str| metadata.get(key).unwrap().to_str().unwrap(); + let get = |key: &str| headers.get(key).unwrap().to_str().unwrap(); assert_eq!(get("x-datafusion-distributed-config-custom.foo"), ""); assert_eq!(get("x-datafusion-distributed-config-custom.bar"), "0"); assert_eq!(get("x-datafusion-distributed-config-custom.baz"), "false"); @@ -162,18 +187,17 @@ mod tests { foo: "first".to_string(), ..Default::default() }; - set_distributed_option_extension(&mut config, opt1)?; + set_distributed_option_extension(&mut config, opt1); let opt2 = CustomExtension { bar: 42, ..Default::default() }; - set_distributed_option_extension(&mut config, opt2)?; + set_distributed_option_extension(&mut config, opt2); - let flight_metadata = config.get_extension::().unwrap(); - let metadata = &flight_metadata.0; + let headers = get_config_extension_propagation_headers(&config)?; - let get = |key: &str| metadata.get(key).unwrap().to_str().unwrap(); + let get = |key: &str| headers.get(key).unwrap().to_str().unwrap(); assert_eq!(get("x-datafusion-distributed-config-custom.foo"), ""); assert_eq!(get("x-datafusion-distributed-config-custom.bar"), "42"); assert_eq!(get("x-datafusion-distributed-config-custom.baz"), "false"); @@ -190,8 +214,15 @@ mod tests { &Default::default(), )?; - let extension = config.options().extensions.get::(); - assert!(extension.is_none()); + let extension = config + .options() + .extensions + .get::() + .unwrap(); + let default = CustomExtension::default(); + assert_eq!(extension.foo, default.foo); + assert_eq!(extension.bar, default.bar); + assert_eq!(extension.baz, default.baz); Ok(()) } @@ -201,14 +232,21 @@ mod tests { let mut config = SessionConfig::new(); let mut header_map = HeaderMap::new(); header_map.insert( - HeaderName::from_str("x-datafusion-distributed-config-other.setting").unwrap(), - HeaderValue::from_str("value").unwrap(), + HeaderName::from_str("x-datafusion-distributed-config-other.setting")?, + HeaderValue::from_str("value")?, ); set_distributed_option_extension_from_headers::(&mut config, &header_map)?; - let extension = config.options().extensions.get::(); - assert!(extension.is_none()); + let extension = config + .options() + .extensions + .get::() + .unwrap(); + let default = CustomExtension::default(); + assert_eq!(extension.foo, default.foo); + assert_eq!(extension.bar, default.bar); + assert_eq!(extension.baz, default.baz); Ok(()) } @@ -229,18 +267,17 @@ mod tests { ..Default::default() }; - set_distributed_option_extension(&mut config, custom_opt)?; - set_distributed_option_extension(&mut config, another_opt)?; + set_distributed_option_extension(&mut config, custom_opt); + set_distributed_option_extension(&mut config, another_opt); - let flight_metadata = config.get_extension::().unwrap(); - let metadata = &flight_metadata.0; + let headers = get_config_extension_propagation_headers(&config)?; - assert!(metadata.contains_key("x-datafusion-distributed-config-custom.foo")); - assert!(metadata.contains_key("x-datafusion-distributed-config-custom.bar")); - assert!(metadata.contains_key("x-datafusion-distributed-config-another.setting1")); - assert!(metadata.contains_key("x-datafusion-distributed-config-another.setting2")); + assert!(headers.contains_key("x-datafusion-distributed-config-custom.foo")); + assert!(headers.contains_key("x-datafusion-distributed-config-custom.bar")); + assert!(headers.contains_key("x-datafusion-distributed-config-another.setting1")); + assert!(headers.contains_key("x-datafusion-distributed-config-another.setting2")); - let get = |key: &str| metadata.get(key).unwrap().to_str().unwrap(); + let get = |key: &str| headers.get(key).unwrap().to_str().unwrap(); assert_eq!( get("x-datafusion-distributed-config-custom.foo"), @@ -259,11 +296,11 @@ mod tests { let mut new_config = SessionConfig::new(); set_distributed_option_extension_from_headers::( &mut new_config, - metadata, + &headers, )?; set_distributed_option_extension_from_headers::( &mut new_config, - metadata, + &headers, )?; let propagated_custom = get_ext::(&new_config); @@ -282,7 +319,8 @@ mod tests { let mut config = SessionConfig::new(); let extension = InvalidExtension::default(); - let result = set_distributed_option_extension(&mut config, extension); + set_distributed_option_extension(&mut config, extension); + let result = get_config_extension_propagation_headers(&config); assert!(result.is_err()); } @@ -291,7 +329,8 @@ mod tests { let mut config = SessionConfig::new(); let extension = InvalidValueExtension::default(); - let result = set_distributed_option_extension(&mut config, extension); + set_distributed_option_extension(&mut config, extension); + let result = get_config_extension_propagation_headers(&config); assert!(result.is_err()); } diff --git a/src/distributed_ext.rs b/src/distributed_ext.rs index 4a7b483f..7efea431 100644 --- a/src/distributed_ext.rs +++ b/src/distributed_ext.rs @@ -1,9 +1,12 @@ -use crate::ChannelResolver; -use crate::channel_resolver_ext::set_distributed_channel_resolver; use crate::config_extension_ext::{ set_distributed_option_extension, set_distributed_option_extension_from_headers, }; +use crate::distributed_planner::set_distributed_task_estimator; +use crate::networking::{set_distributed_channel_resolver, set_distributed_worker_resolver}; +use crate::passthrough_headers::set_passthrough_headers; use crate::protobuf::{set_distributed_user_codec, set_distributed_user_codec_arc}; +use crate::{ChannelResolver, DistributedConfig, TaskEstimator, WorkerResolver}; +use arrow_ipc::CompressionType; use datafusion::common::DataFusionError; use datafusion::config::ConfigExtension; use datafusion::execution::{SessionState, SessionStateBuilder}; @@ -31,7 +34,7 @@ pub trait DistributedExt: Sized { /// # use datafusion::config::ConfigExtension; /// # use datafusion::execution::{SessionState, SessionStateBuilder}; /// # use datafusion::prelude::SessionConfig; - /// # use datafusion_distributed::{DistributedExt, DistributedSessionBuilder, DistributedSessionBuilderContext}; + /// # use datafusion_distributed::{DistributedExt, WorkerSessionBuilder, WorkerQueryContext}; /// /// extensions_options! { /// pub struct CustomExtension { @@ -48,35 +51,33 @@ pub trait DistributedExt: Sized { /// let mut my_custom_extension = CustomExtension::default(); /// // Now, the CustomExtension will be able to cross network boundaries. Upon making an Arrow /// // Flight request, it will be sent through gRPC metadata. - /// let mut config = SessionConfig::new() - /// .with_distributed_option_extension(my_custom_extension).unwrap(); + /// let state = SessionStateBuilder::new() + /// .with_distributed_option_extension(my_custom_extension) + /// .build(); /// - /// async fn build_state(ctx: DistributedSessionBuilderContext) -> Result { - /// // This function can be provided to an ArrowFlightEndpoint in order to tell it how to + /// async fn build_state(ctx: WorkerQueryContext) -> Result { + /// // This function can be provided to a Worker to tell it how to /// // build sessions that retrieve the CustomExtension from gRPC metadata. - /// Ok(SessionStateBuilder::new() + /// Ok(ctx + /// .builder /// .with_distributed_option_extension_from_headers::(&ctx.headers)? /// .build()) /// } /// ``` - fn with_distributed_option_extension( - self, - t: T, - ) -> Result; + fn with_distributed_option_extension(self, t: T) -> Self; /// Same as [DistributedExt::with_distributed_option_extension] but with an in-place mutation - fn set_distributed_option_extension( - &mut self, - t: T, - ) -> Result<(), DataFusionError>; + fn set_distributed_option_extension(&mut self, t: T); /// Adds the provided [ConfigExtension] to the distributed context. The [ConfigExtension] will /// be serialized using gRPC metadata and sent across tasks. Users are expected to call this /// method with their own extensions to be able to access them in any place in the /// plan. /// - /// This method also adds the provided [ConfigExtension] to the current session option - /// extensions, the same as calling [SessionConfig::with_option_extension]. + /// - If there was a [ConfigExtension] of the same type already present, it's updated with an + /// in-place mutation based on the headers that came over the wire. + /// - If there was no [ConfigExtension] set before, it will get added, as if + /// [SessionConfig::with_option_extension] was being called. /// /// Example: /// @@ -86,7 +87,7 @@ pub trait DistributedExt: Sized { /// # use datafusion::config::ConfigExtension; /// # use datafusion::execution::{SessionState, SessionStateBuilder}; /// # use datafusion::prelude::SessionConfig; - /// # use datafusion_distributed::{DistributedExt, DistributedSessionBuilder, DistributedSessionBuilderContext}; + /// # use datafusion_distributed::{DistributedExt, WorkerSessionBuilder, WorkerQueryContext}; /// /// extensions_options! { /// pub struct CustomExtension { @@ -103,13 +104,15 @@ pub trait DistributedExt: Sized { /// let mut my_custom_extension = CustomExtension::default(); /// // Now, the CustomExtension will be able to cross network boundaries. Upon making an Arrow /// // Flight request, it will be sent through gRPC metadata. - /// let mut config = SessionConfig::new() - /// .with_distributed_option_extension(my_custom_extension).unwrap(); + /// let state = SessionStateBuilder::new() + /// .with_distributed_option_extension(my_custom_extension) + /// .build(); /// - /// async fn build_state(ctx: DistributedSessionBuilderContext) -> Result { - /// // This function can be provided to an ArrowFlightEndpoint in order to tell it how to + /// async fn build_state(ctx: WorkerQueryContext) -> Result { + /// // This function can be provided to a Worker to tell it how to /// // build sessions that retrieve the CustomExtension from gRPC metadata. - /// Ok(SessionStateBuilder::new() + /// Ok(ctx + /// .builder /// .with_distributed_option_extension_from_headers::(&ctx.headers)? /// .build()) /// } @@ -134,17 +137,17 @@ pub trait DistributedExt: Sized { /// ``` /// # use std::sync::Arc; /// # use datafusion::common::DataFusionError; - /// # use datafusion::execution::{SessionState, FunctionRegistry, SessionStateBuilder}; + /// # use datafusion::execution::{SessionState, FunctionRegistry, SessionStateBuilder, TaskContext}; /// # use datafusion::physical_plan::ExecutionPlan; /// # use datafusion::prelude::SessionConfig; /// # use datafusion_proto::physical_plan::PhysicalExtensionCodec; - /// # use datafusion_distributed::{DistributedExt, DistributedSessionBuilderContext}; + /// # use datafusion_distributed::{DistributedExt, WorkerQueryContext}; /// /// #[derive(Debug)] /// struct CustomExecCodec; /// /// impl PhysicalExtensionCodec for CustomExecCodec { - /// fn try_decode(&self, buf: &[u8], inputs: &[Arc], registry: &dyn FunctionRegistry) -> datafusion::common::Result> { + /// fn try_decode(&self, buf: &[u8], inputs: &[Arc], ctx: &TaskContext) -> datafusion::common::Result> { /// todo!() /// } /// @@ -153,10 +156,12 @@ pub trait DistributedExt: Sized { /// } /// } /// - /// let config = SessionConfig::new().with_distributed_user_codec(CustomExecCodec); + /// let state = SessionStateBuilder::new() + /// .with_distributed_user_codec(CustomExecCodec) + /// .build(); /// - /// async fn build_state(ctx: DistributedSessionBuilderContext) -> Result { - /// // This function can be provided to an ArrowFlightEndpoint in order to tell it how to + /// async fn build_state(ctx: WorkerQueryContext) -> Result { + /// // This function can be provided to a Worker to tell it how to /// // encode/decode CustomExec nodes. /// Ok(SessionStateBuilder::new() /// .with_distributed_user_codec(CustomExecCodec) @@ -174,8 +179,14 @@ pub trait DistributedExt: Sized { /// Same as [DistributedExt::set_distributed_user_codec] but with a dynamic argument. fn set_distributed_user_codec_arc(&mut self, codec: Arc); - /// Injects a [ChannelResolver] implementation for Distributed DataFusion to resolve worker - /// nodes. When running in distributed mode, setting a [ChannelResolver] is required. + /// This is what tells Distributed DataFusion the URLs of the workers available for serving queries. + /// + /// It injects a [WorkerResolver] implementation for Distributed DataFusion to resolve worker + /// nodes in the cluster. When running in distributed mode, setting a [WorkerResolver] is required. + /// + /// Even if this is required to be present in the [SessionContext] that first initiates and + /// plans the query, it's not necessary to be present in a Worker's session state builder, + /// as no planning happens there. /// /// Example: /// @@ -186,27 +197,79 @@ pub trait DistributedExt: Sized { /// # use datafusion::execution::{SessionState, SessionStateBuilder}; /// # use datafusion::prelude::SessionConfig; /// # use url::Url; - /// # use datafusion_distributed::{BoxCloneSyncChannel, ChannelResolver, DistributedExt, DistributedSessionBuilderContext}; + /// # use std::sync::Arc; + /// # use datafusion_distributed::{BoxCloneSyncChannel, WorkerResolver, DistributedExt, DistributedPhysicalOptimizerRule, WorkerQueryContext}; /// - /// struct CustomChannelResolver; + /// struct CustomWorkerResolver; /// /// #[async_trait] - /// impl ChannelResolver for CustomChannelResolver { + /// impl WorkerResolver for CustomWorkerResolver { /// fn get_urls(&self) -> Result, DataFusionError> { /// todo!() /// } + /// } + /// + /// // This tweaks the SessionState so that it can plan for distributed queries and execute them. + /// let state = SessionStateBuilder::new() + /// .with_distributed_worker_resolver(CustomWorkerResolver) + /// // the DistributedPhysicalOptimizerRule also needs to be passed so that query plans + /// // get distributed. + /// .with_physical_optimizer_rule(Arc::new(DistributedPhysicalOptimizerRule)) + /// .build(); + /// ``` + fn with_distributed_worker_resolver( + self, + resolver: T, + ) -> Self; + + /// Same as [DistributedExt::with_distributed_channel_resolver] but with an in-place mutation. + fn set_distributed_worker_resolver( + &mut self, + resolver: T, + ); + + /// This is what tells Distributed DataFusion how to build an Arrow Flight client out of a worker URL. + /// + /// There's a default implementation that caches the Arrow Flight client instances so that there's + /// only one per URL, but users can decide to override that behavior in favor of their own solution. + /// + /// Example: + /// + /// ``` + /// # use arrow_flight::flight_service_client::FlightServiceClient; + /// # use async_trait::async_trait; + /// # use datafusion::common::DataFusionError; + /// # use datafusion::execution::{SessionState, SessionStateBuilder}; + /// # use datafusion::prelude::SessionConfig; + /// # use url::Url; + /// # use std::sync::Arc; + /// # use datafusion_distributed::{BoxCloneSyncChannel, ChannelResolver, DistributedExt, DistributedPhysicalOptimizerRule, WorkerQueryContext}; /// + /// struct CustomChannelResolver; + /// + /// #[async_trait] + /// impl ChannelResolver for CustomChannelResolver { /// async fn get_flight_client_for_url(&self, url: &Url) -> Result, DataFusionError> { + /// // Build a custom FlightServiceClient wrapped with tower layers or something similar. /// todo!() /// } /// } /// - /// let config = SessionConfig::new().with_distributed_channel_resolver(CustomChannelResolver); + /// // This tweaks the SessionState so that it can plan for distributed queries and execute them. + /// let state = SessionStateBuilder::new() + /// .with_distributed_channel_resolver(CustomChannelResolver) + /// // the DistributedPhysicalOptimizerRule also needs to be passed so that query plans + /// // get distributed. + /// .with_physical_optimizer_rule(Arc::new(DistributedPhysicalOptimizerRule)) + /// .build(); /// - /// async fn build_state(ctx: DistributedSessionBuilderContext) -> Result { - /// // This function can be provided to an ArrowFlightEndpoint so that it knows how to - /// // resolve tonic channels from URLs upon making network calls to other nodes. - /// Ok(SessionStateBuilder::new() + /// // This function can be provided to a Worker so that, upon receiving a distributed + /// // part of a plan, it knows how to resolve gRPC channels from URLs for making network calls to other nodes. + /// async fn build_state(ctx: WorkerQueryContext) -> Result { + /// // If you have a custom channel resolver, it should also be passed in the + /// // Worker session builder. + /// Ok(ctx + /// .builder /// .with_distributed_channel_resolver(CustomChannelResolver) /// .build()) /// } @@ -221,13 +284,246 @@ pub trait DistributedExt: Sized { &mut self, resolver: T, ); + + /// Adds a distributed task count estimator. [TaskEstimator]s are executed on each node + /// sequentially until one returns an estimation on the number of tasks that should be + /// used for the stage containing that node. + /// + /// Many nodes might decide to provide an estimation, so a reconciliation between all of them + /// is performed internally during planning. + /// + /// ```text + /// ┌───────────────────────┐ + /// │SortPreservingMergeExec│ + /// └───────────────────────┘ + /// ▲ + /// ┌ ─ ─ ─ ─ ─ ─ ─ ┼ ─ ─ ─ ─ ─ ─ ─ ─ Stage 2 + /// ┌───────────┴───────────┐ │ + /// │ │ SortExec │ + /// └───────────────────────┘ │ + /// │ ┌───────────────────────┐ + /// │ AggregateExec │ │ + /// │ └───────────────────────┘ + /// ─ ─ ─ ─ ─ ─ ─ ─▲─ ─ ─ ─ ─ ─ ─ ─ ┘ + /// ┌ ─ ─ ─ ─ ─ ─ ─ ┴ ─ ─ ─ ─ ─ ─ ─ ─ Stage 1 + /// ┌───────────────────────┐ │ + /// │ │ FilterExec │ + /// └───────────────────────┘ │ + /// │ ┌───────────────────────┐ a TaskEstimator estimates the amount of tasks + /// │ SomeExec │◀───┼── based on how much data will be pulled. + /// │ └───────────────────────┘ + /// ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ┘ + /// ``` + fn with_distributed_task_estimator( + self, + estimator: T, + ) -> Self; + + /// Same as [DistributedExt::with_distributed_task_estimator] but with an in-place mutation. + fn set_distributed_task_estimator( + &mut self, + estimator: T, + ); + + /// Sets the maximum number of files each task in a stage with a FileScanConfig node will + /// handle. Reducing this number will increment the amount of tasks. By default, this + /// is close to the number of cores in the machine. + /// + /// ```text + /// ┌───────────────────────┐ + /// │SortPreservingMergeExec│ + /// └───────────────────────┘ + /// ▲ + /// ┌ ─ ─ ─ ─ ─ ─ ─ ┼ ─ ─ ─ ─ ─ ─ ─ ─ Stage 2 + /// ┌───────────┴───────────┐ │ + /// │ │ SortExec │ + /// └───────────────────────┘ │ + /// │ ┌───────────────────────┐ + /// │ AggregateExec │ │ + /// │ └───────────────────────┘ + /// ─ ─ ─ ─ ─ ─ ─ ─▲─ ─ ─ ─ ─ ─ ─ ─ ┘ + /// ┌ ─ ─ ─ ─ ─ ─ ─ ┴ ─ ─ ─ ─ ─ ─ ─ ─ Stage 1 + /// ┌───────────────────────┐ │ + /// │ │ FilterExec │ + /// └───────────────────────┘ │ + /// │ ┌───────────────────────┐ Sets the max number of files + /// │ FileScanConfig │◀───┼─ each task will handle. Less + /// │ └───────────────────────┘ files_per_task == more tasks + /// ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ┘ + ///``` + fn with_distributed_files_per_task( + self, + files_per_task: usize, + ) -> Result; + + /// Same as [DistributedExt::with_distributed_files_per_task] but with an in-place mutation. + fn set_distributed_files_per_task( + &mut self, + files_per_task: usize, + ) -> Result<(), DataFusionError>; + + /// The number of tasks in each stage is calculated in a bottom-to-top fashion. + /// + /// Bottom stages containing leaf nodes will provide an estimation of the amount of tasks + /// for those stages, but upper stages might see a reduction (or increment) in the amount + /// of tasks based on the cardinality effect bottom stages have in the data. + /// + /// For example: If there are two stages, and the leaf stage is estimated to use 10 tasks, + /// the upper stage might use less (e.g. 5) if it sees that the leaf stage is returning + /// less data because of filters or aggregations. + /// + /// This function sets the scale factor for when encountering these nodes that change the + /// cardinality of the data. For example, if a stage with 10 tasks contains an AggregateExec + /// node, and the scale factor is 2.0, the following stage will use 10 / 2.0 = 5 tasks. + /// + /// ```text + /// ┌───────────────────────┐ + /// │SortPreservingMergeExec│ + /// └───────────────────────┘ + /// ▲ + /// ┌ ─ ─ ─ ─ ─ ─ ─ ┼ ─ ─ ─ ─ ─ ─ ─ ─ Stage 2 (N/scale_factor tasks) + /// ┌───────────┴───────────┐ │ + /// │ │ SortExec │ + /// └───────────────────────┘ │ + /// │ ┌───────────────────────┐ + /// │ AggregateExec │ │ + /// │ └───────────────────────┘ + /// ─ ─ ─ ─ ─ ─ ─ ─▲─ ─ ─ ─ ─ ─ ─ ─ ┘ + /// ┌ ─ ─ ─ ─ ─ ─ ─ ┴ ─ ─ ─ ─ ─ ─ ─ ─ Stage 1 (N tasks) + /// ┌───────────────────────┐ │ A filter reduces cardinality, + /// │ │ FilterExec │◀────────therefore the next stage will have + /// └───────────────────────┘ │ less tasks according to this factor + /// │ ┌───────────────────────┐ + /// │ FileScanConfig │ │ + /// │ └───────────────────────┘ + /// ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ┘ + /// ``` + fn with_distributed_cardinality_effect_task_scale_factor( + self, + factor: f64, + ) -> Result; + + /// Same as [DistributedExt::with_distributed_cardinality_effect_task_scale_factor] but with + /// an in-place mutation. + fn set_distributed_cardinality_effect_task_scale_factor( + &mut self, + factor: f64, + ) -> Result<(), DataFusionError>; + + /// Enables metrics collection across network boundaries so that all the metrics gather in + /// each node are accessible from the head stage that started running the query. + fn with_distributed_metrics_collection(self, enabled: bool) -> Result; + + /// Same as [DistributedExt::with_distributed_metrics_collection] but with an in-place mutation. + fn set_distributed_metrics_collection(&mut self, enabled: bool) -> Result<(), DataFusionError>; + + /// Enables children isolator unions for distributing UNION operations across as many tasks as + /// the sum of all the tasks required for each child. + /// + /// For example, if there is a UNION with 3 children, requiring one task each, it will result + /// in a plan with 3 tasks where each task runs one child: + /// + /// ```text + /// ┌─────────────────────────────┐┌─────────────────────────────┐┌─────────────────────────────┐ + /// │ Task 1 ││ Task 2 ││ Task 3 │ + /// │┌───────────────────────────┐││┌───────────────────────────┐││┌───────────────────────────┐│ + /// ││ ChildrenIsolatorUnionExec ││││ ChildrenIsolatorUnionExec ││││ ChildrenIsolatorUnionExec ││ + /// │└───▲─────────▲─────────▲───┘││└───▲─────────▲─────────▲───┘││└───▲─────────▲─────────▲───┘│ + /// │ │ ││ │ ││ │ │ + /// │┌───┴───┐ ┌ ─│ ─ ┌ ─│ ─ ││┌ ─│ ─ ┌───┴───┐ ┌ ─│ ─ ││┌ ─│ ─ ┌ ─│ ─ ┌───┴───┐│ + /// ││Child 1│ Child 2│ Child 3│││ Child 1│ │Child 2│ Child 3│││ Child 1│ Child 2│ │Child 3││ + /// │└───────┘ └ ─ ─ └ ─ ─ ││└ ─ ─ └───────┘ └ ─ ─ ││└ ─ ─ └ ─ ─ └───────┘│ + /// └─────────────────────────────┘└─────────────────────────────┘└─────────────────────────────┘ + /// ``` + fn with_distributed_children_isolator_unions( + self, + enabled: bool, + ) -> Result; + + /// Same as [DistributedExt::with_distributed_children_isolator_unions] but with an in-place mutation. + fn set_distributed_children_isolator_unions( + &mut self, + enabled: bool, + ) -> Result<(), DataFusionError>; + + /// Enables broadcast joins for CollectLeft hash joins. When enabled, the build side of + /// a CollectLeft join is broadcast to all consumer tasks instead of being coalesced + /// into a single partition. + /// + /// Note: This option is disabled by default until the implementation is smarter about when to + /// broadcast. + fn with_distributed_broadcast_joins(self, enabled: bool) -> Result; + + /// Same as [DistributedExt::with_distributed_broadcast_joins_enabled] but with an in-place mutation. + fn set_distributed_broadcast_joins(&mut self, enabled: bool) -> Result<(), DataFusionError>; + + /// The compression type to use for sending data over the wire. + /// + /// The default is [CompressionType::LZ4_FRAME]. + fn with_distributed_compression( + self, + compression: Option, + ) -> Result; + + /// Same as [DistributedExt::with_distributed_compression] but with an in-place mutation. + fn set_distributed_compression( + &mut self, + compression: Option, + ) -> Result<(), DataFusionError>; + + /// How many rows to collect in each record batch before sending it over the wire in a + /// shuffle operation. This value defaults to the same as `datafusion.execution.batch_size`. + /// + /// Setting it to something smaller than `datafusion.execution.batch_size` has no effect. + /// + /// It's preferable to set `datafusion.execution.batch_size` directly instead of this + /// parameter if the specific use case allows it. + fn with_distributed_shuffle_batch_size( + self, + batch_size: usize, + ) -> Result; + + /// Same as [DistributedExt::with_distributed_shuffle_batch_size] but with an in-place mutation. + fn set_distributed_shuffle_batch_size( + &mut self, + batch_size: usize, + ) -> Result<(), DataFusionError>; + + /// Sets arbitrary HTTP headers that will be forwarded unchanged to worker nodes. + /// These headers are included in outgoing Arrow Flight requests to workers. + /// + /// Returns an error if any header name starts with the reserved prefix + /// `x-datafusion-distributed-config-`, which is used internally. + /// + /// Example: + /// + /// ```rust + /// # use datafusion::execution::SessionStateBuilder; + /// # use datafusion_distributed::DistributedExt; + /// # use http::HeaderMap; + /// + /// let mut passthrough = HeaderMap::new(); + /// passthrough.insert("x-custom-priority", "high".parse().unwrap()); + /// + /// let state = SessionStateBuilder::new() + /// .with_distributed_passthrough_headers(passthrough) + /// .unwrap() + /// .build(); + /// ``` + fn with_distributed_passthrough_headers( + self, + headers: HeaderMap, + ) -> Result; + + /// Same as [DistributedExt::with_distributed_passthrough_headers] but with an in-place mutation. + fn set_distributed_passthrough_headers( + &mut self, + headers: HeaderMap, + ) -> Result<(), DataFusionError>; } impl DistributedExt for SessionConfig { - fn set_distributed_option_extension( - &mut self, - t: T, - ) -> Result<(), DataFusionError> { + fn set_distributed_option_extension(&mut self, t: T) { set_distributed_option_extension(self, t) } @@ -235,7 +531,8 @@ impl DistributedExt for SessionConfig { &mut self, headers: &HeaderMap, ) -> Result<(), DataFusionError> { - set_distributed_option_extension_from_headers::(self, headers) + set_distributed_option_extension_from_headers::(self, headers)?; + Ok(()) } fn set_distributed_user_codec(&mut self, codec: T) { @@ -246,18 +543,100 @@ impl DistributedExt for SessionConfig { set_distributed_user_codec_arc(self, codec) } + fn set_distributed_worker_resolver( + &mut self, + resolver: T, + ) { + set_distributed_worker_resolver(self, resolver); + } + fn set_distributed_channel_resolver( &mut self, resolver: T, ) { - set_distributed_channel_resolver(self, resolver) + set_distributed_channel_resolver(self, resolver); + } + + fn set_distributed_task_estimator( + &mut self, + estimator: T, + ) { + set_distributed_task_estimator(self, estimator) + } + + fn set_distributed_files_per_task( + &mut self, + files_per_task: usize, + ) -> Result<(), DataFusionError> { + let d_cfg = DistributedConfig::from_config_options_mut(self.options_mut())?; + d_cfg.files_per_task = files_per_task; + Ok(()) + } + + fn set_distributed_cardinality_effect_task_scale_factor( + &mut self, + factor: f64, + ) -> Result<(), DataFusionError> { + let d_cfg = DistributedConfig::from_config_options_mut(self.options_mut())?; + d_cfg.cardinality_task_count_factor = factor; + Ok(()) + } + + fn set_distributed_metrics_collection(&mut self, enabled: bool) -> Result<(), DataFusionError> { + let d_cfg = DistributedConfig::from_config_options_mut(self.options_mut())?; + d_cfg.collect_metrics = enabled; + Ok(()) + } + + fn set_distributed_children_isolator_unions( + &mut self, + enabled: bool, + ) -> Result<(), DataFusionError> { + let d_cfg = DistributedConfig::from_config_options_mut(self.options_mut())?; + d_cfg.children_isolator_unions = enabled; + Ok(()) + } + + fn set_distributed_broadcast_joins(&mut self, enabled: bool) -> Result<(), DataFusionError> { + let d_cfg = DistributedConfig::from_config_options_mut(self.options_mut())?; + d_cfg.broadcast_joins = enabled; + Ok(()) + } + + fn set_distributed_compression( + &mut self, + compression: Option, + ) -> Result<(), DataFusionError> { + let d_cfg = DistributedConfig::from_config_options_mut(self.options_mut())?; + d_cfg.compression = match compression { + Some(CompressionType::ZSTD) => "zstd".to_string(), + Some(CompressionType::LZ4_FRAME) => "lz4".to_string(), + _ => "none".to_string(), + }; + Ok(()) + } + + fn set_distributed_shuffle_batch_size( + &mut self, + batch_size: usize, + ) -> Result<(), DataFusionError> { + let d_cfg = DistributedConfig::from_config_options_mut(self.options_mut())?; + d_cfg.shuffle_batch_size = batch_size; + Ok(()) + } + + fn set_distributed_passthrough_headers( + &mut self, + headers: HeaderMap, + ) -> Result<(), DataFusionError> { + set_passthrough_headers(self, headers) } delegate! { to self { #[call(set_distributed_option_extension)] - #[expr($?;Ok(self))] - fn with_distributed_option_extension(mut self, t: T) -> Result; + #[expr($;self)] + fn with_distributed_option_extension(mut self, t: T) -> Self; #[call(set_distributed_option_extension_from_headers)] #[expr($?;Ok(self))] @@ -271,9 +650,49 @@ impl DistributedExt for SessionConfig { #[expr($;self)] fn with_distributed_user_codec_arc(mut self, codec: Arc) -> Self; + #[call(set_distributed_worker_resolver)] + #[expr($;self)] + fn with_distributed_worker_resolver(mut self, resolver: T) -> Self; + #[call(set_distributed_channel_resolver)] #[expr($;self)] fn with_distributed_channel_resolver(mut self, resolver: T) -> Self; + + #[call(set_distributed_task_estimator)] + #[expr($;self)] + fn with_distributed_task_estimator(mut self, estimator: T) -> Self; + + #[call(set_distributed_files_per_task)] + #[expr($?;Ok(self))] + fn with_distributed_files_per_task(mut self, files_per_task: usize) -> Result; + + #[call(set_distributed_cardinality_effect_task_scale_factor)] + #[expr($?;Ok(self))] + fn with_distributed_cardinality_effect_task_scale_factor(mut self, factor: f64) -> Result; + + #[call(set_distributed_metrics_collection)] + #[expr($?;Ok(self))] + fn with_distributed_metrics_collection(mut self, enabled: bool) -> Result; + + #[call(set_distributed_children_isolator_unions)] + #[expr($?;Ok(self))] + fn with_distributed_children_isolator_unions(mut self, enabled: bool) -> Result; + + #[call(set_distributed_broadcast_joins)] + #[expr($?;Ok(self))] + fn with_distributed_broadcast_joins(mut self, enabled: bool) -> Result; + + #[call(set_distributed_compression)] + #[expr($?;Ok(self))] + fn with_distributed_compression(mut self, compression: Option) -> Result; + + #[call(set_distributed_shuffle_batch_size)] + #[expr($?;Ok(self))] + fn with_distributed_shuffle_batch_size(mut self, batch_size: usize) -> Result; + + #[call(set_distributed_passthrough_headers)] + #[expr($?;Ok(self))] + fn with_distributed_passthrough_headers(mut self, headers: HeaderMap) -> Result; } } } @@ -281,10 +700,10 @@ impl DistributedExt for SessionConfig { impl DistributedExt for SessionStateBuilder { delegate! { to self.config().get_or_insert_default() { - fn set_distributed_option_extension(&mut self, t: T) -> Result<(), DataFusionError>; + fn set_distributed_option_extension(&mut self, t: T); #[call(set_distributed_option_extension)] - #[expr($?;Ok(self))] - fn with_distributed_option_extension(mut self, t: T) -> Result; + #[expr($;self)] + fn with_distributed_option_extension(mut self, t: T) -> Self; fn set_distributed_option_extension_from_headers(&mut self, h: &HeaderMap) -> Result<(), DataFusionError>; #[call(set_distributed_option_extension_from_headers)] @@ -301,10 +720,60 @@ impl DistributedExt for SessionStateBuilder { #[expr($;self)] fn with_distributed_user_codec_arc(mut self, codec: Arc) -> Self; + fn set_distributed_worker_resolver(&mut self, resolver: T); + #[call(set_distributed_worker_resolver)] + #[expr($;self)] + fn with_distributed_worker_resolver(mut self, resolver: T) -> Self; + fn set_distributed_channel_resolver(&mut self, resolver: T); #[call(set_distributed_channel_resolver)] #[expr($;self)] fn with_distributed_channel_resolver(mut self, resolver: T) -> Self; + + fn set_distributed_task_estimator(&mut self, estimator: T); + #[call(set_distributed_task_estimator)] + #[expr($;self)] + fn with_distributed_task_estimator(mut self, estimator: T) -> Self; + + fn set_distributed_files_per_task(&mut self, files_per_task: usize) -> Result<(), DataFusionError>; + #[call(set_distributed_files_per_task)] + #[expr($?;Ok(self))] + fn with_distributed_files_per_task(mut self, files_per_task: usize) -> Result; + + fn set_distributed_cardinality_effect_task_scale_factor(&mut self, factor: f64) -> Result<(), DataFusionError>; + #[call(set_distributed_cardinality_effect_task_scale_factor)] + #[expr($?;Ok(self))] + fn with_distributed_cardinality_effect_task_scale_factor(mut self, factor: f64) -> Result; + + fn set_distributed_metrics_collection(&mut self, enabled: bool) -> Result<(), DataFusionError>; + #[call(set_distributed_metrics_collection)] + #[expr($?;Ok(self))] + fn with_distributed_metrics_collection(mut self, enabled: bool) -> Result; + + fn set_distributed_children_isolator_unions(&mut self, enabled: bool) -> Result<(), DataFusionError>; + #[call(set_distributed_children_isolator_unions)] + #[expr($?;Ok(self))] + fn with_distributed_children_isolator_unions(mut self, enabled: bool) -> Result; + + fn set_distributed_broadcast_joins(&mut self, enabled: bool) -> Result<(), DataFusionError>; + #[call(set_distributed_broadcast_joins)] + #[expr($?;Ok(self))] + fn with_distributed_broadcast_joins(mut self, enabled: bool) -> Result; + + fn set_distributed_compression(&mut self, compression: Option) -> Result<(), DataFusionError>; + #[call(set_distributed_compression)] + #[expr($?;Ok(self))] + fn with_distributed_compression(mut self, compression: Option) -> Result; + + fn set_distributed_shuffle_batch_size(&mut self, batch_size: usize) -> Result<(), DataFusionError>; + #[call(set_distributed_shuffle_batch_size)] + #[expr($?;Ok(self))] + fn with_distributed_shuffle_batch_size(mut self, batch_size: usize) -> Result; + + fn set_distributed_passthrough_headers(&mut self, headers: HeaderMap) -> Result<(), DataFusionError>; + #[call(set_distributed_passthrough_headers)] + #[expr($?;Ok(self))] + fn with_distributed_passthrough_headers(mut self, headers: HeaderMap) -> Result; } } } @@ -312,10 +781,10 @@ impl DistributedExt for SessionStateBuilder { impl DistributedExt for SessionState { delegate! { to self.config_mut() { - fn set_distributed_option_extension(&mut self, t: T) -> Result<(), DataFusionError>; + fn set_distributed_option_extension(&mut self, t: T); #[call(set_distributed_option_extension)] - #[expr($?;Ok(self))] - fn with_distributed_option_extension(mut self, t: T) -> Result; + #[expr($;self)] + fn with_distributed_option_extension(mut self, t: T) -> Self; fn set_distributed_option_extension_from_headers(&mut self, h: &HeaderMap) -> Result<(), DataFusionError>; #[call(set_distributed_option_extension_from_headers)] @@ -332,10 +801,60 @@ impl DistributedExt for SessionState { #[expr($;self)] fn with_distributed_user_codec_arc(mut self, codec: Arc) -> Self; + fn set_distributed_worker_resolver(&mut self, resolver: T); + #[call(set_distributed_worker_resolver)] + #[expr($;self)] + fn with_distributed_worker_resolver(mut self, resolver: T) -> Self; + fn set_distributed_channel_resolver(&mut self, resolver: T); #[call(set_distributed_channel_resolver)] #[expr($;self)] fn with_distributed_channel_resolver(mut self, resolver: T) -> Self; + + fn set_distributed_task_estimator(&mut self, estimator: T); + #[call(set_distributed_task_estimator)] + #[expr($;self)] + fn with_distributed_task_estimator(mut self, estimator: T) -> Self; + + fn set_distributed_files_per_task(&mut self, files_per_task: usize) -> Result<(), DataFusionError>; + #[call(set_distributed_files_per_task)] + #[expr($?;Ok(self))] + fn with_distributed_files_per_task(mut self, files_per_task: usize) -> Result; + + fn set_distributed_cardinality_effect_task_scale_factor(&mut self, factor: f64) -> Result<(), DataFusionError>; + #[call(set_distributed_cardinality_effect_task_scale_factor)] + #[expr($?;Ok(self))] + fn with_distributed_cardinality_effect_task_scale_factor(mut self, factor: f64) -> Result; + + fn set_distributed_metrics_collection(&mut self, enabled: bool) -> Result<(), DataFusionError>; + #[call(set_distributed_metrics_collection)] + #[expr($?;Ok(self))] + fn with_distributed_metrics_collection(mut self, enabled: bool) -> Result; + + fn set_distributed_children_isolator_unions(&mut self, enabled: bool) -> Result<(), DataFusionError>; + #[call(set_distributed_children_isolator_unions)] + #[expr($?;Ok(self))] + fn with_distributed_children_isolator_unions(mut self, enabled: bool) -> Result; + + fn set_distributed_broadcast_joins(&mut self, enabled: bool) -> Result<(), DataFusionError>; + #[call(set_distributed_broadcast_joins)] + #[expr($?;Ok(self))] + fn with_distributed_broadcast_joins(mut self, enabled: bool) -> Result; + + fn set_distributed_compression(&mut self, compression: Option) -> Result<(), DataFusionError>; + #[call(set_distributed_compression)] + #[expr($?;Ok(self))] + fn with_distributed_compression(mut self, compression: Option) -> Result; + + fn set_distributed_shuffle_batch_size(&mut self, batch_size: usize) -> Result<(), DataFusionError>; + #[call(set_distributed_shuffle_batch_size)] + #[expr($?;Ok(self))] + fn with_distributed_shuffle_batch_size(mut self, batch_size: usize) -> Result; + + fn set_distributed_passthrough_headers(&mut self, headers: HeaderMap) -> Result<(), DataFusionError>; + #[call(set_distributed_passthrough_headers)] + #[expr($?;Ok(self))] + fn with_distributed_passthrough_headers(mut self, headers: HeaderMap) -> Result; } } } @@ -343,10 +862,10 @@ impl DistributedExt for SessionState { impl DistributedExt for SessionContext { delegate! { to self.state_ref().write().config_mut() { - fn set_distributed_option_extension(&mut self, t: T) -> Result<(), DataFusionError>; + fn set_distributed_option_extension(&mut self, t: T); #[call(set_distributed_option_extension)] - #[expr($?;Ok(self))] - fn with_distributed_option_extension(self, t: T) -> Result; + #[expr($;self)] + fn with_distributed_option_extension(self, t: T) -> Self; fn set_distributed_option_extension_from_headers(&mut self, h: &HeaderMap) -> Result<(), DataFusionError>; #[call(set_distributed_option_extension_from_headers)] @@ -363,10 +882,60 @@ impl DistributedExt for SessionContext { #[expr($;self)] fn with_distributed_user_codec_arc(self, codec: Arc) -> Self; + fn set_distributed_worker_resolver(&mut self, resolver: T); + #[call(set_distributed_worker_resolver)] + #[expr($;self)] + fn with_distributed_worker_resolver(self, resolver: T) -> Self; + fn set_distributed_channel_resolver(&mut self, resolver: T); #[call(set_distributed_channel_resolver)] #[expr($;self)] fn with_distributed_channel_resolver(self, resolver: T) -> Self; + + fn set_distributed_task_estimator(&mut self, estimator: T); + #[call(set_distributed_task_estimator)] + #[expr($;self)] + fn with_distributed_task_estimator(self, estimator: T) -> Self; + + fn set_distributed_files_per_task(&mut self, files_per_task: usize) -> Result<(), DataFusionError>; + #[call(set_distributed_files_per_task)] + #[expr($?;Ok(self))] + fn with_distributed_files_per_task(self, files_per_task: usize) -> Result; + + fn set_distributed_cardinality_effect_task_scale_factor(&mut self, factor: f64) -> Result<(), DataFusionError>; + #[call(set_distributed_cardinality_effect_task_scale_factor)] + #[expr($?;Ok(self))] + fn with_distributed_cardinality_effect_task_scale_factor(self, factor: f64) -> Result; + + fn set_distributed_metrics_collection(&mut self, enabled: bool) -> Result<(), DataFusionError>; + #[call(set_distributed_metrics_collection)] + #[expr($?;Ok(self))] + fn with_distributed_metrics_collection(self, enabled: bool) -> Result; + + fn set_distributed_children_isolator_unions(&mut self, enabled: bool) -> Result<(), DataFusionError>; + #[call(set_distributed_children_isolator_unions)] + #[expr($?;Ok(self))] + fn with_distributed_children_isolator_unions(self, enabled: bool) -> Result; + + fn set_distributed_broadcast_joins(&mut self, enabled: bool) -> Result<(), DataFusionError>; + #[call(set_distributed_broadcast_joins)] + #[expr($?;Ok(self))] + fn with_distributed_broadcast_joins(self, enabled: bool) -> Result; + + fn set_distributed_compression(&mut self, compression: Option) -> Result<(), DataFusionError>; + #[call(set_distributed_compression)] + #[expr($?;Ok(self))] + fn with_distributed_compression(self, compression: Option) -> Result; + + fn set_distributed_shuffle_batch_size(&mut self, batch_size: usize) -> Result<(), DataFusionError>; + #[call(set_distributed_shuffle_batch_size)] + #[expr($?;Ok(self))] + fn with_distributed_shuffle_batch_size(self, batch_size: usize) -> Result; + + fn set_distributed_passthrough_headers(&mut self, headers: HeaderMap) -> Result<(), DataFusionError>; + #[call(set_distributed_passthrough_headers)] + #[expr($?;Ok(self))] + fn with_distributed_passthrough_headers(self, headers: HeaderMap) -> Result; } } } diff --git a/src/distributed_physical_optimizer_rule.rs b/src/distributed_physical_optimizer_rule.rs deleted file mode 100644 index 90cfd0fb..00000000 --- a/src/distributed_physical_optimizer_rule.rs +++ /dev/null @@ -1,663 +0,0 @@ -use super::{NetworkShuffleExec, PartitionIsolatorExec}; -use crate::execution_plans::{DistributedExec, NetworkCoalesceExec}; -use crate::stage::Stage; -use datafusion::common::plan_err; -use datafusion::common::tree_node::TreeNodeRecursion; -use datafusion::datasource::source::DataSourceExec; -use datafusion::error::DataFusionError; -use datafusion::physical_expr::Partitioning; -use datafusion::physical_plan::ExecutionPlanProperties; -use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec; -use datafusion::physical_plan::joins::{HashJoinExec, PartitionMode}; -use datafusion::physical_plan::sorts::sort_preserving_merge::SortPreservingMergeExec; -use datafusion::{ - common::tree_node::{Transformed, TreeNode}, - config::ConfigOptions, - error::Result, - physical_optimizer::PhysicalOptimizerRule, - physical_plan::{ExecutionPlan, repartition::RepartitionExec}, -}; -use std::error::Error; -use std::fmt::{Display, Formatter}; -use std::sync::Arc; -use uuid::Uuid; - -/// Physical optimizer rule that inspects the plan, places the appropriate network -/// boundaries and breaks it down into stages that can be executed in a distributed manner. -/// -/// The rule has two steps: -/// -/// 1. Inject the appropriate distributed execution nodes in the appropriate places. -/// -/// This is done by looking at specific nodes in the original plan and enhancing them -/// with new additional nodes: -/// - a [DataSourceExec] is wrapped with a [PartitionIsolatorExec] for exposing just a subset -/// of the [DataSourceExec] partitions to the rest of the plan. -/// - a [CoalescePartitionsExec] is followed by a [NetworkCoalesceExec] so that all tasks in the -/// previous stage collapse into just 1 in the next stage. -/// - a [SortPreservingMergeExec] is followed by a [NetworkCoalesceExec] for the same reasons as -/// above -/// - a [RepartitionExec] with a hash partition is wrapped with a [NetworkShuffleExec] for -/// shuffling data to different tasks. -/// -/// -/// 2. Break down the plan into stages -/// -/// Based on the network boundaries ([NetworkShuffleExec], [NetworkCoalesceExec], ...) placed in -/// the plan by the first step, the plan is divided into stages and tasks are assigned to each -/// stage. -/// -/// This step might decide to not respect the amount of tasks each network boundary is requesting, -/// like when a plan is not parallelizable in different tasks (e.g. a collect left [HashJoinExec]) -/// or when a [DataSourceExec] has not enough partitions to be spread across tasks. -#[derive(Debug, Default)] -pub struct DistributedPhysicalOptimizerRule { - /// Upon shuffling data, this defines how many tasks are employed into performing the shuffling. - /// ```text - /// ( task 1 ) ( task 2 ) ( task 3 ) - /// ▲ ▲ ▲ - /// └────┬──────┴─────┬────┘ - /// ( task 1 ) ( task 2 ) N tasks - /// ``` - /// This parameter defines N - network_shuffle_tasks: Option, - /// Upon merging multiple tasks into one, this defines how many tasks are merged. - /// ```text - /// ( task 1 ) - /// ▲ - /// ┌───────────┴──────────┐ - /// ( task 1 ) ( task 2 ) ( task 3 ) N tasks - /// ``` - /// This parameter defines N - network_coalesce_tasks: Option, -} - -impl DistributedPhysicalOptimizerRule { - pub fn new() -> Self { - DistributedPhysicalOptimizerRule { - network_shuffle_tasks: None, - network_coalesce_tasks: None, - } - } - - /// Sets the amount of tasks employed in performing shuffles. - pub fn with_network_shuffle_tasks(mut self, tasks: usize) -> Self { - self.network_shuffle_tasks = Some(tasks); - self - } - - /// Sets the amount of input tasks for every task coalescing operation. - pub fn with_network_coalesce_tasks(mut self, tasks: usize) -> Self { - self.network_coalesce_tasks = Some(tasks); - self - } -} - -impl PhysicalOptimizerRule for DistributedPhysicalOptimizerRule { - fn optimize( - &self, - plan: Arc, - _config: &ConfigOptions, - ) -> Result> { - // We can only optimize plans that are not already distributed - let plan = self.apply_network_boundaries(plan)?; - Self::distribute_plan(plan) - } - - fn name(&self) -> &str { - "DistributedPhysicalOptimizer" - } - - fn schema_check(&self) -> bool { - true - } -} - -impl DistributedPhysicalOptimizerRule { - fn apply_network_boundaries( - &self, - mut plan: Arc, - ) -> Result, DataFusionError> { - if plan.output_partitioning().partition_count() > 1 { - // Coalescing partitions here will allow us to put a NetworkCoalesceExec on top - // of the plan, executing it in parallel. - plan = Arc::new(CoalescePartitionsExec::new(plan)) - } - - let result = - plan.transform_up(|plan| { - // If this node is a DataSourceExec, we need to wrap it with PartitionIsolatorExec so - // that not all tasks have access to all partitions of the underlying DataSource. - if plan.as_any().is::() { - let node = PartitionIsolatorExec::new(plan); - - return Ok(Transformed::yes(Arc::new(node))); - } - - // If this is a hash RepartitionExec, introduce a shuffle. - if let (Some(node), Some(tasks)) = ( - plan.as_any().downcast_ref::(), - self.network_shuffle_tasks, - ) { - if !matches!(node.partitioning(), Partitioning::Hash(_, _)) { - return Ok(Transformed::no(plan)); - } - let node = NetworkShuffleExec::try_new(plan, tasks)?; - - return Ok(Transformed::yes(Arc::new(node))); - } - - // If this is a CoalescePartitionsExec, it means that the original plan is trying to - // merge all partitions into one. We need to go one step ahead and also merge all tasks - // into one. - if let (Some(node), Some(tasks)) = ( - plan.as_any().downcast_ref::(), - self.network_coalesce_tasks, - ) { - // If the immediate child is a PartitionIsolatorExec, it means that the rest of the - // plan is just a couple of non-computational nodes that are probably not worth - // distributing. - if node.input().as_any().is::() { - return Ok(Transformed::no(plan)); - } - - let plan = plan.clone().with_new_children(vec![Arc::new( - NetworkCoalesceExec::new(Arc::clone(node.input()), tasks), - )])?; - - return Ok(Transformed::yes(plan)); - } - - // The SortPreservingMergeExec node will try to coalesce all partitions into just 1. - // We need to account for it and help it by also coalescing all tasks into one, therefore - // a NetworkCoalesceExec is introduced. - if let (Some(node), Some(tasks)) = ( - plan.as_any().downcast_ref::(), - self.network_coalesce_tasks, - ) { - let plan = plan.clone().with_new_children(vec![Arc::new( - NetworkCoalesceExec::new(Arc::clone(node.input()), tasks), - )])?; - - return Ok(Transformed::yes(plan)); - } - - Ok(Transformed::no(plan)) - })?; - Ok(result.data) - } - - /// Takes a plan with certain network boundaries in it ([NetworkShuffleExec], [NetworkCoalesceExec], ...) - /// and breaks it down into stages. - /// - /// This can be used a standalone function for distributing arbitrary plans in which users have - /// manually placed network boundaries, or as part of the [DistributedPhysicalOptimizerRule] that - /// places the network boundaries automatically as a standard [PhysicalOptimizerRule]. - pub fn distribute_plan( - plan: Arc, - ) -> Result, DataFusionError> { - let stage = Self::_distribute_plan_inner(Uuid::new_v4(), plan, &mut 1, 0, 1)?; - let plan = stage.plan.decoded()?; - Ok(Arc::new(DistributedExec::new(Arc::clone(plan)))) - } - - fn _distribute_plan_inner( - query_id: Uuid, - plan: Arc, - num: &mut usize, - depth: usize, - n_tasks: usize, - ) -> Result { - let mut distributed = plan.clone().transform_down(|plan| { - // We cannot break down CollectLeft hash joins into more than 1 task, as these need - // a full materialized build size with all the data in it. - // - // Maybe in the future these can be broadcast joins? - if let Some(node) = plan.as_any().downcast_ref::() { - if n_tasks > 1 && node.mode == PartitionMode::CollectLeft { - return Err(limit_tasks_err(1)); - } - } - - if let Some(node) = plan.as_any().downcast_ref::() { - // If there's only 1 task, no need to perform any isolation. - if n_tasks == 1 { - return Ok(Transformed::yes(Arc::clone(plan.children().first().unwrap()))); - } - let node = node.ready(n_tasks)?; - return Ok(Transformed::new(Arc::new(node), true, TreeNodeRecursion::Jump)); - } - - let Some(mut dnode) = plan.as_network_boundary().map(Referenced::Borrowed) else { - return Ok(Transformed::no(plan)); - }; - - let stage = loop { - let input_stage_info = dnode.as_ref().get_input_stage_info(n_tasks)?; - // If the current stage has just 1 task, and the next stage is only going to have - // 1 task, there's no point in having a network boundary in between, they can just - // communicate in memory. - if n_tasks == 1 && input_stage_info.task_count == 1 { - let mut n = dnode.as_ref().rollback()?; - if let Some(node) = n.as_any().downcast_ref::() { - // Also trim PartitionIsolatorExec out of the plan. - n = Arc::clone(node.children().first().unwrap()); - } - return Ok(Transformed::yes(n)); - } - match Self::_distribute_plan_inner(query_id, input_stage_info.plan, num, depth + 1, input_stage_info.task_count) { - Ok(v) => break v, - Err(e) => match get_distribute_plan_err(&e) { - None => return Err(e), - Some(DistributedPlanError::LimitTasks(limit)) => { - // While attempting to build a new stage, a failure was raised stating - // that no more than `limit` tasks can be used for it, so we are going - // to limit the amount of tasks to the requested number and try building - // the stage again. - if input_stage_info.task_count == *limit { - return plan_err!("A node requested {limit} tasks for the stage its in, but that stage already has that many tasks"); - } - dnode = Referenced::Arced(dnode.as_ref().with_input_task_count(*limit)?); - } - }, - } - }; - let node = dnode.as_ref().with_input_stage(stage)?; - Ok(Transformed::new(node, true, TreeNodeRecursion::Jump)) - })?; - - // The head stage is executable, and upon execution, it will lazily assign worker URLs to - // all tasks. This must only be done once, so the executable StageExec must only be called - // once on 1 partition. - if depth == 0 && distributed.data.output_partitioning().partition_count() > 1 { - distributed.data = Arc::new(CoalescePartitionsExec::new(distributed.data)); - } - - let stage = Stage::new(query_id, *num, distributed.data, n_tasks); - *num += 1; - Ok(stage) - } -} - -/// Necessary information for building a [Stage] during distributed planning. -/// -/// [NetworkBoundary]s return this piece of data so that the distributed planner know how to -/// build the next [Stage] from which the [NetworkBoundary] is going to receive data. -/// -/// Some network boundaries might perform some modifications in their children, like scaling -/// up the number of partitions, or injecting a specific [ExecutionPlan] on top. -pub struct InputStageInfo { - /// The head plan of the [Stage] that is about to be built. - pub plan: Arc, - /// The amount of tasks the [Stage] will have. - pub task_count: usize, -} - -/// This trait represents a node that introduces the necessity of a network boundary in the plan. -/// The distributed planner, upon stepping into one of these, will break the plan and build a stage -/// out of it. -pub trait NetworkBoundary: ExecutionPlan { - /// Returns the information necessary for building the next stage from which this - /// [NetworkBoundary] is going to collect data. - fn get_input_stage_info(&self, task_count: usize) -> Result; - - /// re-assigns a different number of input tasks to the current [NetworkBoundary]. - /// - /// This will be called if upon building a stage, a [DistributedPlanError::LimitTasks] error - /// is returned, prompting the [NetworkBoundary] to choose a different number of input tasks. - fn with_input_task_count(&self, input_tasks: usize) -> Result>; - - /// Called when a [Stage] is correctly formed. The [NetworkBoundary] can use this - /// information to perform any internal transformations necessary for distributed execution. - /// - /// Typically, [NetworkBoundary]s will use this call for transitioning from "Pending" to "ready". - fn with_input_stage(&self, input_stage: Stage) -> Result>; - - /// Returns the assigned input [Stage], if any. - fn input_stage(&self) -> Option<&Stage>; - - /// The planner might decide to remove this [NetworkBoundary] from the plan if it decides that - /// it's not going to bring any benefit. The [NetworkBoundary] will be replaced with whatever - /// this function returns. - fn rollback(&self) -> Result> { - let children = self.children(); - if children.len() != 1 { - return plan_err!( - "Expected distributed node {} to have exactly 1 children, but got {}", - self.name(), - children.len() - ); - } - Ok(Arc::clone(children.first().unwrap())) - } -} - -/// Extension trait for downcasting dynamic types to [NetworkBoundary]. -pub trait NetworkBoundaryExt { - /// Downcasts self to a [NetworkBoundary] if possible. - fn as_network_boundary(&self) -> Option<&dyn NetworkBoundary>; - /// Returns whether self is a [NetworkBoundary] or not. - fn is_network_boundary(&self) -> bool { - self.as_network_boundary().is_some() - } -} - -impl NetworkBoundaryExt for dyn ExecutionPlan { - fn as_network_boundary(&self) -> Option<&dyn NetworkBoundary> { - if let Some(node) = self.as_any().downcast_ref::() { - Some(node) - } else if let Some(node) = self.as_any().downcast_ref::() { - Some(node) - } else { - None - } - } -} - -/// Helper enum for storing either borrowed or owned trait object references -enum Referenced<'a, T: ?Sized> { - Borrowed(&'a T), - Arced(Arc), -} - -impl Referenced<'_, T> { - fn as_ref(&self) -> &T { - match self { - Self::Borrowed(r) => r, - Self::Arced(arc) => arc.as_ref(), - } - } -} - -/// Error thrown during distributed planning that prompts the planner to change something and -/// try again. -#[derive(Debug)] -enum DistributedPlanError { - /// Prompts the planner to limit the amount of tasks used in the stage that is currently - /// being planned. - LimitTasks(usize), -} - -impl Display for DistributedPlanError { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - match self { - DistributedPlanError::LimitTasks(n) => { - write!(f, "LimitTasksErr: {n}") - } - } - } -} - -impl Error for DistributedPlanError {} - -/// Builds a [DistributedPlanError::LimitTasks] error. This error prompts the distributed planner -/// to try rebuilding the current stage with a limited amount of tasks. -pub fn limit_tasks_err(limit: usize) -> DataFusionError { - DataFusionError::External(Box::new(DistributedPlanError::LimitTasks(limit))) -} - -fn get_distribute_plan_err(err: &DataFusionError) -> Option<&DistributedPlanError> { - let DataFusionError::External(err) = err else { - return None; - }; - err.downcast_ref() -} - -#[cfg(test)] -mod tests { - use crate::distributed_physical_optimizer_rule::DistributedPhysicalOptimizerRule; - use crate::test_utils::parquet::register_parquet_tables; - use crate::{assert_snapshot, display_plan_ascii}; - use datafusion::error::DataFusionError; - use datafusion::execution::SessionStateBuilder; - use datafusion::prelude::{SessionConfig, SessionContext}; - use std::sync::Arc; - - /* shema for the "weather" table - - MinTemp [type=DOUBLE] [repetitiontype=OPTIONAL] - MaxTemp [type=DOUBLE] [repetitiontype=OPTIONAL] - Rainfall [type=DOUBLE] [repetitiontype=OPTIONAL] - Evaporation [type=DOUBLE] [repetitiontype=OPTIONAL] - Sunshine [type=BYTE_ARRAY] [convertedtype=UTF8] [repetitiontype=OPTIONAL] - WindGustDir [type=BYTE_ARRAY] [convertedtype=UTF8] [repetitiontype=OPTIONAL] - WindGustSpeed [type=BYTE_ARRAY] [convertedtype=UTF8] [repetitiontype=OPTIONAL] - WindDir9am [type=BYTE_ARRAY] [convertedtype=UTF8] [repetitiontype=OPTIONAL] - WindDir3pm [type=BYTE_ARRAY] [convertedtype=UTF8] [repetitiontype=OPTIONAL] - WindSpeed9am [type=BYTE_ARRAY] [convertedtype=UTF8] [repetitiontype=OPTIONAL] - WindSpeed3pm [type=INT64] [convertedtype=INT_64] [repetitiontype=OPTIONAL] - Humidity9am [type=INT64] [convertedtype=INT_64] [repetitiontype=OPTIONAL] - Humidity3pm [type=INT64] [convertedtype=INT_64] [repetitiontype=OPTIONAL] - Pressure9am [type=DOUBLE] [repetitiontype=OPTIONAL] - Pressure3pm [type=DOUBLE] [repetitiontype=OPTIONAL] - Cloud9am [type=INT64] [convertedtype=INT_64] [repetitiontype=OPTIONAL] - Cloud3pm [type=INT64] [convertedtype=INT_64] [repetitiontype=OPTIONAL] - Temp9am [type=DOUBLE] [repetitiontype=OPTIONAL] - Temp3pm [type=DOUBLE] [repetitiontype=OPTIONAL] - RainToday [type=BYTE_ARRAY] [convertedtype=UTF8] [repetitiontype=OPTIONAL] - RISK_MM [type=DOUBLE] [repetitiontype=OPTIONAL] - RainTomorrow [type=BYTE_ARRAY] [convertedtype=UTF8] [repetitiontype=OPTIONAL] - */ - - #[tokio::test] - async fn test_select_all() { - let query = r#"SELECT * FROM weather"#; - let plan = sql_to_explain(query, 1).await.unwrap(); - assert_snapshot!(plan, @r" - ┌───── DistributedExec ── Tasks: t0:[p0] - │ CoalescePartitionsExec - │ DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MinTemp, MaxTemp, Rainfall, Evaporation, Sunshine, WindGustDir, WindGustSpeed, WindDir9am, WindDir3pm, WindSpeed9am, WindSpeed3pm, Humidity9am, Humidity3pm, Pressure9am, Pressure3pm, Cloud9am, Cloud3pm, Temp9am, Temp3pm, RainToday, RISK_MM, RainTomorrow], file_type=parquet - └────────────────────────────────────────────────── - "); - } - - #[tokio::test] - async fn test_aggregation() { - let query = - r#"SELECT count(*), "RainToday" FROM weather GROUP BY "RainToday" ORDER BY count(*)"#; - let plan = sql_to_explain(query, 2).await.unwrap(); - assert_snapshot!(plan, @r" - ┌───── DistributedExec ── Tasks: t0:[p0] - │ ProjectionExec: expr=[count(*)@0 as count(*), RainToday@1 as RainToday] - │ SortPreservingMergeExec: [count(Int64(1))@2 ASC NULLS LAST] - │ [Stage 2] => NetworkCoalesceExec: output_partitions=8, input_tasks=2 - └────────────────────────────────────────────────── - ┌───── Stage 2 ── Tasks: t0:[p0,p1,p2,p3] t1:[p0,p1,p2,p3] - │ SortExec: expr=[count(*)@0 ASC NULLS LAST], preserve_partitioning=[true] - │ ProjectionExec: expr=[count(Int64(1))@1 as count(*), RainToday@0 as RainToday, count(Int64(1))@1 as count(Int64(1))] - │ AggregateExec: mode=FinalPartitioned, gby=[RainToday@0 as RainToday], aggr=[count(Int64(1))] - │ CoalesceBatchesExec: target_batch_size=8192 - │ [Stage 1] => NetworkShuffleExec: output_partitions=4, input_tasks=2 - └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0,p1,p2,p3,p4,p5,p6,p7] t1:[p0,p1,p2,p3,p4,p5,p6,p7] - │ RepartitionExec: partitioning=Hash([RainToday@0], 8), input_partitions=4 - │ RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=2 - │ AggregateExec: mode=Partial, gby=[RainToday@0 as RainToday], aggr=[count(Int64(1))] - │ PartitionIsolatorExec: t0:[p0,p1,__] t1:[__,__,p0] - │ DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[RainToday], file_type=parquet - └────────────────────────────────────────────────── - "); - } - - #[tokio::test] - async fn test_aggregation_with_partitions_per_task() { - let query = - r#"SELECT count(*), "RainToday" FROM weather GROUP BY "RainToday" ORDER BY count(*)"#; - let plan = sql_to_explain(query, 2).await.unwrap(); - assert_snapshot!(plan, @r" - ┌───── DistributedExec ── Tasks: t0:[p0] - │ ProjectionExec: expr=[count(*)@0 as count(*), RainToday@1 as RainToday] - │ SortPreservingMergeExec: [count(Int64(1))@2 ASC NULLS LAST] - │ [Stage 2] => NetworkCoalesceExec: output_partitions=8, input_tasks=2 - └────────────────────────────────────────────────── - ┌───── Stage 2 ── Tasks: t0:[p0,p1,p2,p3] t1:[p0,p1,p2,p3] - │ SortExec: expr=[count(*)@0 ASC NULLS LAST], preserve_partitioning=[true] - │ ProjectionExec: expr=[count(Int64(1))@1 as count(*), RainToday@0 as RainToday, count(Int64(1))@1 as count(Int64(1))] - │ AggregateExec: mode=FinalPartitioned, gby=[RainToday@0 as RainToday], aggr=[count(Int64(1))] - │ CoalesceBatchesExec: target_batch_size=8192 - │ [Stage 1] => NetworkShuffleExec: output_partitions=4, input_tasks=2 - └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0,p1,p2,p3,p4,p5,p6,p7] t1:[p0,p1,p2,p3,p4,p5,p6,p7] - │ RepartitionExec: partitioning=Hash([RainToday@0], 8), input_partitions=4 - │ RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=2 - │ AggregateExec: mode=Partial, gby=[RainToday@0 as RainToday], aggr=[count(Int64(1))] - │ PartitionIsolatorExec: t0:[p0,p1,__] t1:[__,__,p0] - │ DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[RainToday], file_type=parquet - └────────────────────────────────────────────────── - "); - } - - #[tokio::test] - async fn test_left_join() { - let query = r#"SELECT a."MinTemp", b."MaxTemp" FROM weather a LEFT JOIN weather b ON a."RainToday" = b."RainToday" "#; - let plan = sql_to_explain(query, 2).await.unwrap(); - assert_snapshot!(plan, @r" - ┌───── DistributedExec ── Tasks: t0:[p0] - │ CoalescePartitionsExec - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Left, on=[(RainToday@1, RainToday@1)], projection=[MinTemp@0, MaxTemp@2] - │ CoalescePartitionsExec - │ DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MinTemp, RainToday], file_type=parquet - │ DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MaxTemp, RainToday], file_type=parquet - └────────────────────────────────────────────────── - "); - } - - #[tokio::test] - async fn test_left_join_distributed() { - let query = r#" - WITH a AS ( - SELECT - AVG("MinTemp") as "MinTemp", - "RainTomorrow" - FROM weather - WHERE "RainToday" = 'yes' - GROUP BY "RainTomorrow" - ), b AS ( - SELECT - AVG("MaxTemp") as "MaxTemp", - "RainTomorrow" - FROM weather - WHERE "RainToday" = 'no' - GROUP BY "RainTomorrow" - ) - SELECT - a."MinTemp", - b."MaxTemp" - FROM a - LEFT JOIN b - ON a."RainTomorrow" = b."RainTomorrow" - - "#; - let plan = sql_to_explain(query, 2).await.unwrap(); - assert_snapshot!(plan, @r" - ┌───── DistributedExec ── Tasks: t0:[p0] - │ CoalescePartitionsExec - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Left, on=[(RainTomorrow@1, RainTomorrow@1)], projection=[MinTemp@0, MaxTemp@2] - │ CoalescePartitionsExec - │ [Stage 2] => NetworkCoalesceExec: output_partitions=8, input_tasks=2 - │ ProjectionExec: expr=[avg(weather.MaxTemp)@1 as MaxTemp, RainTomorrow@0 as RainTomorrow] - │ AggregateExec: mode=FinalPartitioned, gby=[RainTomorrow@0 as RainTomorrow], aggr=[avg(weather.MaxTemp)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ [Stage 3] => NetworkShuffleExec: output_partitions=4, input_tasks=2 - └────────────────────────────────────────────────── - ┌───── Stage 2 ── Tasks: t0:[p0,p1,p2,p3] t1:[p0,p1,p2,p3] - │ ProjectionExec: expr=[avg(weather.MinTemp)@1 as MinTemp, RainTomorrow@0 as RainTomorrow] - │ AggregateExec: mode=FinalPartitioned, gby=[RainTomorrow@0 as RainTomorrow], aggr=[avg(weather.MinTemp)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ [Stage 1] => NetworkShuffleExec: output_partitions=4, input_tasks=2 - └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0,p1,p2,p3,p4,p5,p6,p7] t1:[p0,p1,p2,p3,p4,p5,p6,p7] - │ RepartitionExec: partitioning=Hash([RainTomorrow@0], 8), input_partitions=4 - │ AggregateExec: mode=Partial, gby=[RainTomorrow@1 as RainTomorrow], aggr=[avg(weather.MinTemp)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: RainToday@1 = yes, projection=[MinTemp@0, RainTomorrow@2] - │ RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=2 - │ PartitionIsolatorExec: t0:[p0,p1,__] t1:[__,__,p0] - │ DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MinTemp, RainToday, RainTomorrow], file_type=parquet, predicate=RainToday@1 = yes, pruning_predicate=RainToday_null_count@2 != row_count@3 AND RainToday_min@0 <= yes AND yes <= RainToday_max@1, required_guarantees=[RainToday in (yes)] - └────────────────────────────────────────────────── - ┌───── Stage 3 ── Tasks: t0:[p0,p1,p2,p3] t1:[p0,p1,p2,p3] - │ RepartitionExec: partitioning=Hash([RainTomorrow@0], 4), input_partitions=4 - │ AggregateExec: mode=Partial, gby=[RainTomorrow@1 as RainTomorrow], aggr=[avg(weather.MaxTemp)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: RainToday@1 = no, projection=[MaxTemp@0, RainTomorrow@2] - │ RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=2 - │ PartitionIsolatorExec: t0:[p0,p1,__] t1:[__,__,p0] - │ DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MaxTemp, RainToday, RainTomorrow], file_type=parquet, predicate=RainToday@1 = no, pruning_predicate=RainToday_null_count@2 != row_count@3 AND RainToday_min@0 <= no AND no <= RainToday_max@1, required_guarantees=[RainToday in (no)] - └────────────────────────────────────────────────── - "); - } - - #[tokio::test] - async fn test_sort() { - let query = r#"SELECT * FROM weather ORDER BY "MinTemp" DESC "#; - let plan = sql_to_explain(query, 2).await.unwrap(); - assert_snapshot!(plan, @r" - ┌───── DistributedExec ── Tasks: t0:[p0] - │ SortPreservingMergeExec: [MinTemp@0 DESC] - │ [Stage 1] => NetworkCoalesceExec: output_partitions=4, input_tasks=2 - └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0,p1] t1:[p2,p3] - │ SortExec: expr=[MinTemp@0 DESC], preserve_partitioning=[true] - │ PartitionIsolatorExec: t0:[p0,p1,__] t1:[__,__,p0] - │ DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MinTemp, MaxTemp, Rainfall, Evaporation, Sunshine, WindGustDir, WindGustSpeed, WindDir9am, WindDir3pm, WindSpeed9am, WindSpeed3pm, Humidity9am, Humidity3pm, Pressure9am, Pressure3pm, Cloud9am, Cloud3pm, Temp9am, Temp3pm, RainToday, RISK_MM, RainTomorrow], file_type=parquet - └────────────────────────────────────────────────── - "); - } - - #[tokio::test] - async fn test_distinct() { - let query = r#"SELECT DISTINCT "RainToday", "WindGustDir" FROM weather"#; - let plan = sql_to_explain(query, 2).await.unwrap(); - assert_snapshot!(plan, @r" - ┌───── DistributedExec ── Tasks: t0:[p0] - │ CoalescePartitionsExec - │ [Stage 2] => NetworkCoalesceExec: output_partitions=8, input_tasks=2 - └────────────────────────────────────────────────── - ┌───── Stage 2 ── Tasks: t0:[p0,p1,p2,p3] t1:[p0,p1,p2,p3] - │ AggregateExec: mode=FinalPartitioned, gby=[RainToday@0 as RainToday, WindGustDir@1 as WindGustDir], aggr=[] - │ CoalesceBatchesExec: target_batch_size=8192 - │ [Stage 1] => NetworkShuffleExec: output_partitions=4, input_tasks=2 - └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0,p1,p2,p3,p4,p5,p6,p7] t1:[p0,p1,p2,p3,p4,p5,p6,p7] - │ RepartitionExec: partitioning=Hash([RainToday@0, WindGustDir@1], 8), input_partitions=4 - │ RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=2 - │ AggregateExec: mode=Partial, gby=[RainToday@0 as RainToday, WindGustDir@1 as WindGustDir], aggr=[] - │ PartitionIsolatorExec: t0:[p0,p1,__] t1:[__,__,p0] - │ DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[RainToday, WindGustDir], file_type=parquet - └────────────────────────────────────────────────── - "); - } - - async fn sql_to_explain(query: &str, tasks: usize) -> Result { - sql_to_explain_with_rule( - query, - DistributedPhysicalOptimizerRule::new() - .with_network_shuffle_tasks(tasks) - .with_network_coalesce_tasks(tasks), - ) - .await - } - - async fn sql_to_explain_with_rule( - query: &str, - rule: DistributedPhysicalOptimizerRule, - ) -> Result { - let config = SessionConfig::new().with_target_partitions(4); - - let state = SessionStateBuilder::new() - .with_default_features() - .with_physical_optimizer_rule(Arc::new(rule)) - .with_config(config) - .build(); - - let ctx = SessionContext::new_with_state(state); - register_parquet_tables(&ctx).await?; - - let df = ctx.sql(query).await?; - - let physical_plan = df.create_physical_plan().await?; - Ok(display_plan_ascii(physical_plan.as_ref())) - } -} diff --git a/src/distributed_planner/batch_coalescing_below_network_boundaries.rs b/src/distributed_planner/batch_coalescing_below_network_boundaries.rs new file mode 100644 index 00000000..80da921a --- /dev/null +++ b/src/distributed_planner/batch_coalescing_below_network_boundaries.rs @@ -0,0 +1,177 @@ +use crate::common::require_one_child; +use crate::{DistributedConfig, NetworkBoundaryExt}; +use datafusion::common::DataFusionError; +use datafusion::common::tree_node::{Transformed, TreeNode}; +use datafusion::config::ConfigOptions; +use datafusion::physical_plan::ExecutionPlan; +use datafusion::physical_plan::coalesce_batches::CoalesceBatchesExec; +use std::sync::Arc; + +/// Rearranges the [CoalesceBatchesExec] nodes in the plan so that they are placed right below +/// the network boundaries, so that fewer but bigger record batches are sent over the wire across +/// stages. +pub(crate) fn batch_coalescing_below_network_boundaries( + plan: Arc, + cfg: &ConfigOptions, +) -> Result, DataFusionError> { + let d_cfg = DistributedConfig::from_config_options(cfg)?; + + // Only apply this rule if the normal execution batch size is not already bigger than the + // shuffle batch size. Exchanging data between workers is better done with batches as big as + // possible, and if the normal execution batch size is already big, we don't want to proactively + // reduce it. + if d_cfg.shuffle_batch_size <= cfg.execution.batch_size { + return Ok(plan); + } + + let transformed = plan.transform_up(|plan| { + if !plan.is_network_boundary() { + return Ok(Transformed::no(plan)); + } + + let input = require_one_child(plan.children())?; + if let Some(existing_coalesce) = input.as_any().downcast_ref::() { + // There was already a CoalesceBatchesExec below... + if existing_coalesce.target_batch_size() == d_cfg.shuffle_batch_size { + // ...so either leave it alone if the batch size is correctly set... + Ok(Transformed::no(plan)) + } else { + // ... or replace it with one with the correct batch size. + let coalesce_input = existing_coalesce.input(); + let new_coalesce = + CoalesceBatchesExec::new(Arc::clone(coalesce_input), d_cfg.shuffle_batch_size) + .with_fetch(existing_coalesce.fetch()); + let new_plan = plan.with_new_children(vec![Arc::new(new_coalesce)])?; + Ok(Transformed::yes(new_plan)) + } + } else { + // No CoalesceBatchesExec below, need to put one. + let coalesce_input = input; + let new_coalesce = CoalesceBatchesExec::new(coalesce_input, d_cfg.shuffle_batch_size); + let new_plan = plan.with_new_children(vec![Arc::new(new_coalesce)])?; + Ok(Transformed::yes(new_plan)) + } + })?; + + Ok(transformed.data) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_utils::in_memory_channel_resolver::InMemoryWorkerResolver; + use crate::test_utils::parquet::register_parquet_tables; + use crate::{ + DistributedExt, DistributedPhysicalOptimizerRule, assert_snapshot, display_plan_ascii, + }; + use datafusion::execution::SessionStateBuilder; + use datafusion::prelude::{SessionConfig, SessionContext}; + use itertools::Itertools; + + #[tokio::test] + async fn same_batch_size_and_shuffle_batch_size() { + let query = r#" + SET datafusion.execution.batch_size=100; + SET distributed.shuffle_batch_size=100; + SELECT DISTINCT "RainToday", "WindGustDir" FROM weather + "#; + let explain = sql_to_explain(query).await; + // No CoalesceBatchExec is placed before sending data over the network. + assert_snapshot!(explain, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=8, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p3] t1:[p0..p3] + │ AggregateExec: mode=FinalPartitioned, gby=[RainToday@0 as RainToday, WindGustDir@1 as WindGustDir], aggr=[] + │ [Stage 1] => NetworkShuffleExec: output_partitions=4, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p7] t1:[p0..p7] t2:[p0..p7] + │ RepartitionExec: partitioning=Hash([RainToday@0, WindGustDir@1], 8), input_partitions=4 + │ AggregateExec: mode=Partial, gby=[RainToday@0 as RainToday, WindGustDir@1 as WindGustDir], aggr=[] + │ RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 + │ PartitionIsolatorExec: t0:[p0,__,__] t1:[__,p0,__] t2:[__,__,p0] + │ DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[RainToday, WindGustDir], file_type=parquet + └────────────────────────────────────────────────── + "); + } + + #[tokio::test] + async fn batch_size_greater_than_shuffle_batch_size() { + let query = r#" + SET datafusion.execution.batch_size=101; + SET distributed.shuffle_batch_size=100; + SELECT DISTINCT "RainToday", "WindGustDir" FROM weather + "#; + let explain = sql_to_explain(query).await; + // No CoalesceBatchExec is placed before sending data over the network. + assert_snapshot!(explain, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=8, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p3] t1:[p0..p3] + │ AggregateExec: mode=FinalPartitioned, gby=[RainToday@0 as RainToday, WindGustDir@1 as WindGustDir], aggr=[] + │ [Stage 1] => NetworkShuffleExec: output_partitions=4, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p7] t1:[p0..p7] t2:[p0..p7] + │ RepartitionExec: partitioning=Hash([RainToday@0, WindGustDir@1], 8), input_partitions=4 + │ AggregateExec: mode=Partial, gby=[RainToday@0 as RainToday, WindGustDir@1 as WindGustDir], aggr=[] + │ RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 + │ PartitionIsolatorExec: t0:[p0,__,__] t1:[__,p0,__] t2:[__,__,p0] + │ DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[RainToday, WindGustDir], file_type=parquet + └────────────────────────────────────────────────── + "); + } + + #[tokio::test] + async fn shuffle_batch_size_greater_than_batch_size() { + let query = r#" + SET datafusion.execution.batch_size=100; + SET distributed.shuffle_batch_size=101; + SELECT DISTINCT "RainToday", "WindGustDir" FROM weather + "#; + let explain = sql_to_explain(query).await; + // CoalesceBatchExec is placed before sending data over the network. + assert_snapshot!(explain, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=8, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p3] t1:[p0..p3] + │ CoalesceBatchesExec: target_batch_size=101 + │ AggregateExec: mode=FinalPartitioned, gby=[RainToday@0 as RainToday, WindGustDir@1 as WindGustDir], aggr=[] + │ [Stage 1] => NetworkShuffleExec: output_partitions=4, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p7] t1:[p0..p7] t2:[p0..p7] + │ CoalesceBatchesExec: target_batch_size=101 + │ RepartitionExec: partitioning=Hash([RainToday@0, WindGustDir@1], 8), input_partitions=4 + │ AggregateExec: mode=Partial, gby=[RainToday@0 as RainToday, WindGustDir@1 as WindGustDir], aggr=[] + │ RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 + │ PartitionIsolatorExec: t0:[p0,__,__] t1:[__,p0,__] t2:[__,__,p0] + │ DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[RainToday, WindGustDir], file_type=parquet + └────────────────────────────────────────────────── + "); + } + + async fn sql_to_explain(query: &str) -> String { + let state = SessionStateBuilder::new() + .with_default_features() + .with_config(SessionConfig::new().with_target_partitions(4)) + .with_physical_optimizer_rule(Arc::new(DistributedPhysicalOptimizerRule)) + .with_distributed_worker_resolver(InMemoryWorkerResolver::new(3)) + .build(); + + let ctx = SessionContext::new_with_state(state); + let mut queries = query.split(";").collect_vec(); + let last_query = queries.pop().unwrap(); + for query in queries { + ctx.sql(query).await.unwrap(); + } + register_parquet_tables(&ctx).await.unwrap(); + let df = ctx.sql(last_query).await.unwrap(); + + let physical_plan = df.create_physical_plan().await.unwrap(); + display_plan_ascii(physical_plan.as_ref(), false) + } +} diff --git a/src/distributed_planner/distributed_config.rs b/src/distributed_planner/distributed_config.rs new file mode 100644 index 00000000..f3c3f8c6 --- /dev/null +++ b/src/distributed_planner/distributed_config.rs @@ -0,0 +1,168 @@ +use crate::TaskEstimator; +use crate::distributed_planner::task_estimator::CombinedTaskEstimator; +use crate::networking::{ChannelResolverExtension, WorkerResolverExtension}; +use datafusion::common::utils::get_available_parallelism; +use datafusion::common::{DataFusionError, extensions_options, not_impl_err, plan_err}; +use datafusion::config::{ConfigExtension, ConfigField, ConfigOptions, Visit}; +use std::fmt::{Debug, Formatter}; +use std::sync::Arc; + +extensions_options! { + /// Configuration for the distributed planner. + pub struct DistributedConfig { + /// Sets the maximum amount of files that will be assigned to each task. Reducing this + /// number will spawn more tasks for the same number of files. This only applies when + /// estimating tasks for stages containing `DataSourceExec` nodes with `FileScanConfig` + /// implementations. + pub files_per_task: usize, default = files_per_task_default() + /// Task multiplying factor for when a node declares that it changes the cardinality + /// of the data: + /// - If a node is increasing the cardinality of the data, this factor will increase. + /// - If a node reduces the cardinality of the data, this factor will decrease. + /// - In any other situation, this factor is left intact. + pub cardinality_task_count_factor: f64, default = cardinality_task_count_factor_default() + /// Upon shuffling over the network, data streams need to be disassembled in a lot of output + /// partitions, which means the resulting streams might contain a lot of tiny record batches + /// to be sent over the wire. This parameter controls the batch size in number of rows for + /// the CoalesceBatchExec operator that is placed at the top of the stage for sending bigger + /// batches over the wire. + /// If set to 0, batch coalescing is disabled on network shuffle operations. + pub shuffle_batch_size: usize, default = 8192 + /// When encountering a UNION operation, isolate its children depending on the task context. + /// For example, on a UNION operation with 3 children running in 3 distributed tasks, + /// instead of executing the 3 children in each 3 tasks with a DistributedTaskContext of + /// 1/3, 2/3, and 3/3 respectively, Execute: + /// - The first child in the first task with a DistributedTaskContext of 1/1 + /// - The second child in the second task with a DistributedTaskContext of 1/1 + /// - The third child in the third task with a DistributedTaskContext of 1/1 + pub children_isolator_unions: bool, default = true + /// Propagate collected metrics from all nodes in the plan across network boundaries + /// so that they can be reconstructed on the head node of the plan. + pub collect_metrics: bool, default = true + /// Enable broadcast joins for CollectLeft hash joins. When enabled, the build side of + /// a CollectLeft join is broadcast to all consumer tasks. + /// TODO: This option exists temporarily until we become smarter about when to actually + /// use broadcasting like checking build side size. + /// For now, broadcasting all CollectLeft joins is not always beneficial. + pub broadcast_joins: bool, default = false + /// The compression used for sending data over the network between workers. + /// It can be set to either `zstd`, `lz4` or `none`. + pub compression: String, default = "lz4".to_string() + /// Collection of [TaskEstimator]s that will be applied to leaf nodes in order to + /// estimate how many tasks should be spawned for the [Stage] containing the leaf node. + pub(crate) __private_task_estimator: CombinedTaskEstimator, default = CombinedTaskEstimator::default() + /// [ChannelResolver] implementation that tells the distributed planner information about + /// the available workers ready to execute distributed tasks. + pub(crate) __private_channel_resolver: ChannelResolverExtension, default = ChannelResolverExtension::default() + /// [WorkerResolver] implementation that tells the distributed planner information about + /// the available workers ready to execute distributed tasks. + pub(crate) __private_worker_resolver: WorkerResolverExtension, default = WorkerResolverExtension::not_implemented() + } +} + +fn files_per_task_default() -> usize { + if cfg!(test) || cfg!(feature = "integration") { + 1 + } else { + get_available_parallelism() + } +} + +fn cardinality_task_count_factor_default() -> f64 { + if cfg!(test) || cfg!(feature = "integration") { + 1.5 + } else { + 1.0 + } +} + +impl DistributedConfig { + /// Appends a [TaskEstimator] to the list. [TaskEstimator] will be executed sequentially in + /// order on leaf nodes, and the first one to provide a value is the one that gets to decide + /// how many tasks are used for that [Stage]. + pub fn with_task_estimator( + mut self, + task_estimator: impl TaskEstimator + Send + Sync + 'static, + ) -> Self { + self.__private_task_estimator + .user_provided + .push(Arc::new(task_estimator)); + self + } + + /// Gets the [DistributedConfig] from the [ConfigOptions]'s extensions. + pub fn from_config_options(cfg: &ConfigOptions) -> Result<&Self, DataFusionError> { + let Some(distributed_cfg) = cfg.extensions.get::() else { + return plan_err!("DistributedConfig is not in ConfigOptions.extensions"); + }; + Ok(distributed_cfg) + } + + /// Gets the [DistributedConfig] from the [ConfigOptions]'s extensions. + pub fn from_config_options_mut(cfg: &mut ConfigOptions) -> Result<&mut Self, DataFusionError> { + let Some(distributed_cfg) = cfg.extensions.get_mut::() else { + return plan_err!("DistributedConfig is not in ConfigOptions.extensions"); + }; + Ok(distributed_cfg) + } +} + +impl ConfigExtension for DistributedConfig { + const PREFIX: &'static str = "distributed"; +} + +// FIXME: Ideally, both ChannelResolverExtension and TaskEstimators would be passed as +// extensions in SessionConfig's AnyMap instead of the ConfigOptions. However, we need +// to pass this as ConfigOptions as we need these two fields to be present during +// planning in the DistributedPhysicalOptimizerRule, and the signature of the optimize() +// method there accepts a ConfigOptions instead of a SessionConfig. +// The following PR addresses this: https://github.com/apache/datafusion/pull/18168 +// but it still has not been accepted or merged. +// Because of this, all the boilerplate trait implementations below are needed. +impl ConfigField for ChannelResolverExtension { + fn visit(&self, _: &mut V, _: &str, _: &'static str) { + // nothing to do. + } + + fn set(&mut self, _: &str, _: &str) -> datafusion::common::Result<()> { + not_impl_err!("Not implemented") + } +} + +impl Debug for ChannelResolverExtension { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "ChannelResolverExtension") + } +} + +impl ConfigField for WorkerResolverExtension { + fn visit(&self, _: &mut V, _: &str, _: &'static str) { + // nothing to do. + } + + fn set(&mut self, _: &str, _: &str) -> datafusion::common::Result<()> { + not_impl_err!("Not implemented") + } +} + +impl Debug for WorkerResolverExtension { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "WorkerResolverExtension") + } +} + +impl ConfigField for CombinedTaskEstimator { + fn visit(&self, _: &mut V, _: &str, _: &'static str) { + //nothing to do. + } + + fn set(&mut self, _: &str, _: &str) -> Result<(), DataFusionError> { + not_impl_err!("not implemented") + } +} + +impl Debug for CombinedTaskEstimator { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "TaskEstimators") + } +} diff --git a/src/distributed_planner/distributed_physical_optimizer_rule.rs b/src/distributed_planner/distributed_physical_optimizer_rule.rs new file mode 100644 index 00000000..1f31e171 --- /dev/null +++ b/src/distributed_planner/distributed_physical_optimizer_rule.rs @@ -0,0 +1,946 @@ +use crate::common::require_one_child; +use crate::distributed_planner::batch_coalescing_below_network_boundaries; +use crate::distributed_planner::plan_annotator::{ + AnnotatedPlan, PlanOrNetworkBoundary, annotate_plan, +}; +use crate::{ + DistributedConfig, DistributedExec, NetworkBroadcastExec, NetworkCoalesceExec, + NetworkShuffleExec, TaskEstimator, +}; +use datafusion::config::ConfigOptions; +use datafusion::error::DataFusionError; +use datafusion::physical_optimizer::PhysicalOptimizerRule; +use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec; +use datafusion::physical_plan::{ExecutionPlan, ExecutionPlanProperties}; +use std::fmt::Debug; +use std::ops::AddAssign; +use std::sync::Arc; +use uuid::Uuid; + +use super::insert_broadcast::insert_broadcast_execs; + +/// Physical optimizer rule that inspects the plan, places the appropriate network +/// boundaries, and breaks it down into stages that can be executed in a distributed manner. +/// +/// The rule has three steps: +/// +/// 1. Annotate the plan with [annotate_plan]: adds some annotations to each node about how +/// many distributed tasks should be used in the stage containing them, and whether they +/// need a network boundary below or not. +/// For more information about this step, read [annotate_plan] docs. +/// +/// 2. Based on the [AnnotatedPlan] returned by [annotate_plan], place all the appropriate +/// network boundaries ([NetworkShuffleExec] and [NetworkCoalesceExec]) with the task count +/// assignation that the annotations required. After this, the plan is already a distributed +/// executable plan. +/// +/// 3. Place the [CoalesceBatchesExec] in the appropriate places (just below network boundaries), +/// so that we send fewer and bigger record batches over the wire instead of a lot of small ones. +#[derive(Debug, Default)] +pub struct DistributedPhysicalOptimizerRule; + +impl PhysicalOptimizerRule for DistributedPhysicalOptimizerRule { + fn optimize( + &self, + original: Arc, + cfg: &ConfigOptions, + ) -> datafusion::common::Result> { + if original.as_any().is::() { + return Ok(original); + } + + let mut plan = Arc::clone(&original); + if original.output_partitioning().partition_count() > 1 { + plan = Arc::new(CoalescePartitionsExec::new(plan)) + } + + plan = insert_broadcast_execs(plan, cfg)?; + + let annotated = annotate_plan(plan, cfg)?; + + let mut stage_id = 1; + let distributed = distribute_plan(annotated, cfg, Uuid::new_v4(), &mut stage_id)?; + if stage_id == 1 { + return Ok(original); + } + let distributed = batch_coalescing_below_network_boundaries(distributed, cfg)?; + + Ok(Arc::new(DistributedExec::new(distributed))) + } + + fn name(&self) -> &str { + "DistributedPhysicalOptimizer" + } + + fn schema_check(&self) -> bool { + true + } +} + +/// Takes an [AnnotatedPlan] and returns a modified [ExecutionPlan] with all the network boundaries +/// appropriately placed. This step performs the following modifications to the original +/// [ExecutionPlan]: +/// - The leaf nodes are scaled up in parallelism based on the number of distributed tasks in +/// which they are going to run. This is configurable by the user via the [TaskEstimator] trait. +/// - The appropriate network boundaries are placed in the plan depending on how it was annotated, +/// so new nodes like [NetworkBroadcastExec], [NetworkCoalesceExec] and [NetworkShuffleExec] will be present. +fn distribute_plan( + annotated_plan: AnnotatedPlan, + cfg: &ConfigOptions, + query_id: Uuid, + stage_id: &mut usize, +) -> Result, DataFusionError> { + let d_cfg = DistributedConfig::from_config_options(cfg)?; + let children = annotated_plan.children; + let task_count = annotated_plan.task_count.as_usize(); + let max_child_task_count = children.iter().map(|v| v.task_count.as_usize()).max(); + let new_children = children + .into_iter() + .map(|child| distribute_plan(child, cfg, query_id, stage_id)) + .collect::, _>>()?; + match annotated_plan.plan_or_nb { + // This is a leaf node. It needs to be scaled up in order to account for it running in + // multiple tasks. + PlanOrNetworkBoundary::Plan(plan) if plan.children().is_empty() => { + let scaled_up = d_cfg.__private_task_estimator.scale_up_leaf_node( + &plan, + annotated_plan.task_count.as_usize(), + cfg, + ); + Ok(scaled_up.unwrap_or(plan)) + } + // This is a normal intermediate plan, just pass it through with the mapped children. + PlanOrNetworkBoundary::Plan(plan) => plan.with_new_children(new_children), + // This is a shuffle, so inject a NetworkShuffleExec here in the plan. + PlanOrNetworkBoundary::Shuffle => { + // It would need a network boundary, but on both sides of the boundary there is just 1 task, + // so we are fine with not introducing any network boundary. + if task_count == 1 && max_child_task_count == Some(1) { + return require_one_child(new_children); + } + let node = Arc::new(NetworkShuffleExec::try_new( + require_one_child(new_children)?, + query_id, + *stage_id, + task_count, + max_child_task_count.unwrap_or(1), + )?); + stage_id.add_assign(1); + Ok(node) + } + // DataFusion is trying to coalesce multiple partitions into one, so we should do the + // same with tasks. + PlanOrNetworkBoundary::Coalesce => { + // It would need a network boundary, but on both sides of the boundary there is just 1 task, + // so we are fine with not introducing any network boundary. + if task_count == 1 && max_child_task_count == Some(1) { + return require_one_child(new_children); + } + let node = Arc::new(NetworkCoalesceExec::try_new( + require_one_child(new_children)?, + query_id, + *stage_id, + task_count, + max_child_task_count.unwrap_or(1), + )?); + stage_id.add_assign(1); + Ok(node) + } + // This is a CollectLeft HashJoinExec with the build side marked as being broadcast. we + // need to insert a NetworkBroadcastExec and scale up the BroadcastExec consumer_tasks. + PlanOrNetworkBoundary::Broadcast => { + // It would need a network boundary, but on both sides of the boundary there is just 1 task, + // so we are fine with not introducing any network boundary. + if task_count == 1 && max_child_task_count == Some(1) { + return require_one_child(new_children); + } + let node = Arc::new(NetworkBroadcastExec::try_new( + require_one_child(new_children)?, + query_id, + *stage_id, + task_count, + max_child_task_count.unwrap_or(1), + )?); + stage_id.add_assign(1); + Ok(node) + } + } +} + +#[cfg(test)] +mod tests { + use crate::test_utils::in_memory_channel_resolver::InMemoryWorkerResolver; + use crate::test_utils::plans::{ + BuildSideOneTaskEstimator, TestPlanOptions, base_session_builder, context_with_query, + sql_to_physical_plan, + }; + use crate::{ + DistributedExt, DistributedPhysicalOptimizerRule, assert_snapshot, display_plan_ascii, + }; + use datafusion::execution::SessionStateBuilder; + use datafusion::physical_plan::displayable; + use std::sync::Arc; + /* schema for the "weather" table + + MinTemp [type=DOUBLE] [repetitiontype=OPTIONAL] + MaxTemp [type=DOUBLE] [repetitiontype=OPTIONAL] + Rainfall [type=DOUBLE] [repetitiontype=OPTIONAL] + Evaporation [type=DOUBLE] [repetitiontype=OPTIONAL] + Sunshine [type=BYTE_ARRAY] [convertedtype=UTF8] [repetitiontype=OPTIONAL] + WindGustDir [type=BYTE_ARRAY] [convertedtype=UTF8] [repetitiontype=OPTIONAL] + WindGustSpeed [type=BYTE_ARRAY] [convertedtype=UTF8] [repetitiontype=OPTIONAL] + WindDir9am [type=BYTE_ARRAY] [convertedtype=UTF8] [repetitiontype=OPTIONAL] + WindDir3pm [type=BYTE_ARRAY] [convertedtype=UTF8] [repetitiontype=OPTIONAL] + WindSpeed9am [type=BYTE_ARRAY] [convertedtype=UTF8] [repetitiontype=OPTIONAL] + WindSpeed3pm [type=INT64] [convertedtype=INT_64] [repetitiontype=OPTIONAL] + Humidity9am [type=INT64] [convertedtype=INT_64] [repetitiontype=OPTIONAL] + Humidity3pm [type=INT64] [convertedtype=INT_64] [repetitiontype=OPTIONAL] + Pressure9am [type=DOUBLE] [repetitiontype=OPTIONAL] + Pressure3pm [type=DOUBLE] [repetitiontype=OPTIONAL] + Cloud9am [type=INT64] [convertedtype=INT_64] [repetitiontype=OPTIONAL] + Cloud3pm [type=INT64] [convertedtype=INT_64] [repetitiontype=OPTIONAL] + Temp9am [type=DOUBLE] [repetitiontype=OPTIONAL] + Temp3pm [type=DOUBLE] [repetitiontype=OPTIONAL] + RainToday [type=BYTE_ARRAY] [convertedtype=UTF8] [repetitiontype=OPTIONAL] + RISK_MM [type=DOUBLE] [repetitiontype=OPTIONAL] + RainTomorrow [type=BYTE_ARRAY] [convertedtype=UTF8] [repetitiontype=OPTIONAL] + */ + + #[tokio::test] + async fn test_select_all() { + let query = r#" + SELECT * FROM weather + "#; + let plan = sql_to_explain(query, |b| { + b.with_distributed_worker_resolver(InMemoryWorkerResolver::new(3)) + }) + .await; + assert_snapshot!(plan, @"DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MinTemp, MaxTemp, Rainfall, Evaporation, Sunshine, WindGustDir, WindGustSpeed, WindDir9am, WindDir3pm, WindSpeed9am, WindSpeed3pm, Humidity9am, Humidity3pm, Pressure9am, Pressure3pm, Cloud9am, Cloud3pm, Temp9am, Temp3pm, RainToday, RISK_MM, RainTomorrow], file_type=parquet"); + } + + #[tokio::test] + async fn test_aggregation() { + let query = r#" + SELECT count(*), "RainToday" FROM weather GROUP BY "RainToday" ORDER BY count(*) + "#; + let plan = sql_to_explain(query, |b| { + b.with_distributed_worker_resolver(InMemoryWorkerResolver::new(3)) + }) + .await; + assert_snapshot!(plan, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ ProjectionExec: expr=[count(*)@0 as count(*), RainToday@1 as RainToday] + │ SortPreservingMergeExec: [count(Int64(1))@2 ASC NULLS LAST] + │ [Stage 2] => NetworkCoalesceExec: output_partitions=8, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p3] t1:[p0..p3] + │ SortExec: expr=[count(*)@0 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[count(Int64(1))@1 as count(*), RainToday@0 as RainToday, count(Int64(1))@1 as count(Int64(1))] + │ AggregateExec: mode=FinalPartitioned, gby=[RainToday@0 as RainToday], aggr=[count(Int64(1))] + │ [Stage 1] => NetworkShuffleExec: output_partitions=4, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p7] t1:[p0..p7] t2:[p0..p7] + │ RepartitionExec: partitioning=Hash([RainToday@0], 8), input_partitions=1 + │ AggregateExec: mode=Partial, gby=[RainToday@0 as RainToday], aggr=[count(Int64(1))] + │ PartitionIsolatorExec: t0:[p0,__,__] t1:[__,p0,__] t2:[__,__,p0] + │ DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[RainToday], file_type=parquet + └────────────────────────────────────────────────── + "); + } + + #[tokio::test] + async fn test_aggregation_with_fewer_workers_than_files() { + let query = r#" + SELECT count(*), "RainToday" FROM weather GROUP BY "RainToday" ORDER BY count(*) + "#; + let plan = sql_to_explain(query, |b| { + b.with_distributed_worker_resolver(InMemoryWorkerResolver::new(2)) + }) + .await; + assert_snapshot!(plan, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ ProjectionExec: expr=[count(*)@0 as count(*), RainToday@1 as RainToday] + │ SortPreservingMergeExec: [count(Int64(1))@2 ASC NULLS LAST] + │ [Stage 2] => NetworkCoalesceExec: output_partitions=8, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p3] t1:[p0..p3] + │ SortExec: expr=[count(*)@0 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[count(Int64(1))@1 as count(*), RainToday@0 as RainToday, count(Int64(1))@1 as count(Int64(1))] + │ AggregateExec: mode=FinalPartitioned, gby=[RainToday@0 as RainToday], aggr=[count(Int64(1))] + │ [Stage 1] => NetworkShuffleExec: output_partitions=4, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p7] t1:[p0..p7] + │ RepartitionExec: partitioning=Hash([RainToday@0], 8), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[RainToday@0 as RainToday], aggr=[count(Int64(1))] + │ PartitionIsolatorExec: t0:[p0,p1,__] t1:[__,__,p0] + │ DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[RainToday], file_type=parquet + └────────────────────────────────────────────────── + "); + } + + #[tokio::test] + async fn test_aggregation_with_0_workers() { + let query = r#" + SELECT count(*), "RainToday" FROM weather GROUP BY "RainToday" ORDER BY count(*) + "#; + let plan = sql_to_explain(query, |b| { + b.with_distributed_worker_resolver(InMemoryWorkerResolver::new(0)) + }) + .await; + assert_snapshot!(plan, @r" + ProjectionExec: expr=[count(*)@0 as count(*), RainToday@1 as RainToday] + SortPreservingMergeExec: [count(Int64(1))@2 ASC NULLS LAST] + SortExec: expr=[count(*)@0 ASC NULLS LAST], preserve_partitioning=[true] + ProjectionExec: expr=[count(Int64(1))@1 as count(*), RainToday@0 as RainToday, count(Int64(1))@1 as count(Int64(1))] + AggregateExec: mode=FinalPartitioned, gby=[RainToday@0 as RainToday], aggr=[count(Int64(1))] + RepartitionExec: partitioning=Hash([RainToday@0], 4), input_partitions=3 + AggregateExec: mode=Partial, gby=[RainToday@0 as RainToday], aggr=[count(Int64(1))] + DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[RainToday], file_type=parquet + "); + } + + #[tokio::test] + async fn test_aggregation_with_high_cardinality_factor() { + let query = r#" + SELECT count(*), "RainToday" FROM weather GROUP BY "RainToday" ORDER BY count(*) + "#; + let plan = sql_to_explain(query, |b| { + b.with_distributed_worker_resolver(InMemoryWorkerResolver::new(3)) + .with_distributed_cardinality_effect_task_scale_factor(3.0) + .unwrap() + }) + .await; + assert_snapshot!(plan, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ ProjectionExec: expr=[count(*)@0 as count(*), RainToday@1 as RainToday] + │ SortPreservingMergeExec: [count(Int64(1))@2 ASC NULLS LAST] + │ SortExec: expr=[count(*)@0 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[count(Int64(1))@1 as count(*), RainToday@0 as RainToday, count(Int64(1))@1 as count(Int64(1))] + │ AggregateExec: mode=FinalPartitioned, gby=[RainToday@0 as RainToday], aggr=[count(Int64(1))] + │ [Stage 1] => NetworkShuffleExec: output_partitions=4, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p3] t1:[p0..p3] t2:[p0..p3] + │ RepartitionExec: partitioning=Hash([RainToday@0], 4), input_partitions=1 + │ AggregateExec: mode=Partial, gby=[RainToday@0 as RainToday], aggr=[count(Int64(1))] + │ PartitionIsolatorExec: t0:[p0,__,__] t1:[__,p0,__] t2:[__,__,p0] + │ DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[RainToday], file_type=parquet + └────────────────────────────────────────────────── + "); + } + + #[tokio::test] + async fn test_aggregation_with_a_lot_of_files_per_task() { + let query = r#" + SELECT count(*), "RainToday" FROM weather GROUP BY "RainToday" ORDER BY count(*) + "#; + let plan = sql_to_explain(query, |b| { + b.with_distributed_worker_resolver(InMemoryWorkerResolver::new(3)) + .with_distributed_files_per_task(3) + .unwrap() + }) + .await; + assert_snapshot!(plan, @r" + ProjectionExec: expr=[count(*)@0 as count(*), RainToday@1 as RainToday] + SortPreservingMergeExec: [count(Int64(1))@2 ASC NULLS LAST] + SortExec: expr=[count(*)@0 ASC NULLS LAST], preserve_partitioning=[true] + ProjectionExec: expr=[count(Int64(1))@1 as count(*), RainToday@0 as RainToday, count(Int64(1))@1 as count(Int64(1))] + AggregateExec: mode=FinalPartitioned, gby=[RainToday@0 as RainToday], aggr=[count(Int64(1))] + RepartitionExec: partitioning=Hash([RainToday@0], 4), input_partitions=3 + AggregateExec: mode=Partial, gby=[RainToday@0 as RainToday], aggr=[count(Int64(1))] + DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[RainToday], file_type=parquet + "); + } + + #[tokio::test] + async fn test_aggregation_with_partitions_per_task() { + let query = r#" + SELECT count(*), "RainToday" FROM weather GROUP BY "RainToday" ORDER BY count(*) + "#; + let plan = sql_to_explain(query, |b| { + b.with_distributed_worker_resolver(InMemoryWorkerResolver::new(3)) + }) + .await; + assert_snapshot!(plan, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ ProjectionExec: expr=[count(*)@0 as count(*), RainToday@1 as RainToday] + │ SortPreservingMergeExec: [count(Int64(1))@2 ASC NULLS LAST] + │ [Stage 2] => NetworkCoalesceExec: output_partitions=8, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p3] t1:[p0..p3] + │ SortExec: expr=[count(*)@0 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[count(Int64(1))@1 as count(*), RainToday@0 as RainToday, count(Int64(1))@1 as count(Int64(1))] + │ AggregateExec: mode=FinalPartitioned, gby=[RainToday@0 as RainToday], aggr=[count(Int64(1))] + │ [Stage 1] => NetworkShuffleExec: output_partitions=4, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p7] t1:[p0..p7] t2:[p0..p7] + │ RepartitionExec: partitioning=Hash([RainToday@0], 8), input_partitions=1 + │ AggregateExec: mode=Partial, gby=[RainToday@0 as RainToday], aggr=[count(Int64(1))] + │ PartitionIsolatorExec: t0:[p0,__,__] t1:[__,p0,__] t2:[__,__,p0] + │ DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[RainToday], file_type=parquet + └────────────────────────────────────────────────── + "); + } + + #[tokio::test] + async fn test_left_join() { + let query = r#" + SELECT a."MinTemp", b."MaxTemp" FROM weather a LEFT JOIN weather b ON a."RainToday" = b."RainToday" + "#; + let plan = sql_to_explain(query, |b| { + b.with_distributed_worker_resolver(InMemoryWorkerResolver::new(3)) + }) + .await; + assert_snapshot!(plan, @r" + HashJoinExec: mode=CollectLeft, join_type=Left, on=[(RainToday@1, RainToday@1)], projection=[MinTemp@0, MaxTemp@2] + CoalescePartitionsExec + DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MinTemp, RainToday], file_type=parquet + DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MaxTemp, RainToday], file_type=parquet + "); + } + + #[tokio::test] + async fn test_left_join_distributed() { + let query = r#" + WITH a AS ( + SELECT + AVG("MinTemp") as "MinTemp", + "RainTomorrow" + FROM weather + WHERE "RainToday" = 'yes' + GROUP BY "RainTomorrow" + ), b AS ( + SELECT + AVG("MaxTemp") as "MaxTemp", + "RainTomorrow" + FROM weather + WHERE "RainToday" = 'no' + GROUP BY "RainTomorrow" + ) + SELECT + a."MinTemp", + b."MaxTemp" + FROM a + LEFT JOIN b + ON a."RainTomorrow" = b."RainTomorrow" + "#; + let plan = sql_to_explain(query, |b| { + b.with_distributed_worker_resolver(InMemoryWorkerResolver::new(3)) + }) + .await; + assert_snapshot!(plan, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ CoalescePartitionsExec + │ HashJoinExec: mode=CollectLeft, join_type=Left, on=[(RainTomorrow@1, RainTomorrow@1)], projection=[MinTemp@0, MaxTemp@2] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=8, input_tasks=2 + │ ProjectionExec: expr=[avg(weather.MaxTemp)@1 as MaxTemp, RainTomorrow@0 as RainTomorrow] + │ AggregateExec: mode=FinalPartitioned, gby=[RainTomorrow@0 as RainTomorrow], aggr=[avg(weather.MaxTemp)] + │ [Stage 3] => NetworkShuffleExec: output_partitions=4, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p3] t1:[p0..p3] + │ ProjectionExec: expr=[avg(weather.MinTemp)@1 as MinTemp, RainTomorrow@0 as RainTomorrow] + │ AggregateExec: mode=FinalPartitioned, gby=[RainTomorrow@0 as RainTomorrow], aggr=[avg(weather.MinTemp)] + │ [Stage 1] => NetworkShuffleExec: output_partitions=4, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p7] t1:[p0..p7] t2:[p0..p7] + │ RepartitionExec: partitioning=Hash([RainTomorrow@0], 8), input_partitions=4 + │ AggregateExec: mode=Partial, gby=[RainTomorrow@1 as RainTomorrow], aggr=[avg(weather.MinTemp)] + │ FilterExec: RainToday@1 = yes, projection=[MinTemp@0, RainTomorrow@2] + │ RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 + │ PartitionIsolatorExec: t0:[p0,__,__] t1:[__,p0,__] t2:[__,__,p0] + │ DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MinTemp, RainToday, RainTomorrow], file_type=parquet, predicate=RainToday@19 = yes, pruning_predicate=RainToday_null_count@2 != row_count@3 AND RainToday_min@0 <= yes AND yes <= RainToday_max@1, required_guarantees=[RainToday in (yes)] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p3] t1:[p0..p3] t2:[p0..p3] + │ RepartitionExec: partitioning=Hash([RainTomorrow@0], 4), input_partitions=4 + │ AggregateExec: mode=Partial, gby=[RainTomorrow@1 as RainTomorrow], aggr=[avg(weather.MaxTemp)] + │ FilterExec: RainToday@1 = no, projection=[MaxTemp@0, RainTomorrow@2] + │ RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 + │ PartitionIsolatorExec: t0:[p0,__,__] t1:[__,p0,__] t2:[__,__,p0] + │ DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MaxTemp, RainToday, RainTomorrow], file_type=parquet, predicate=RainToday@19 = no, pruning_predicate=RainToday_null_count@2 != row_count@3 AND RainToday_min@0 <= no AND no <= RainToday_max@1, required_guarantees=[RainToday in (no)] + └────────────────────────────────────────────────── + "); + } + + #[tokio::test] + async fn test_sort() { + let query = r#" + SELECT * FROM weather ORDER BY "MinTemp" DESC + "#; + let plan = sql_to_explain(query, |b| { + b.with_distributed_worker_resolver(InMemoryWorkerResolver::new(3)) + }) + .await; + assert_snapshot!(plan, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [MinTemp@0 DESC] + │ [Stage 1] => NetworkCoalesceExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0] t1:[p1] t2:[p2] + │ SortExec: expr=[MinTemp@0 DESC], preserve_partitioning=[true] + │ PartitionIsolatorExec: t0:[p0,__,__] t1:[__,p0,__] t2:[__,__,p0] + │ DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MinTemp, MaxTemp, Rainfall, Evaporation, Sunshine, WindGustDir, WindGustSpeed, WindDir9am, WindDir3pm, WindSpeed9am, WindSpeed3pm, Humidity9am, Humidity3pm, Pressure9am, Pressure3pm, Cloud9am, Cloud3pm, Temp9am, Temp3pm, RainToday, RISK_MM, RainTomorrow], file_type=parquet + └────────────────────────────────────────────────── + "); + } + + #[tokio::test] + async fn test_distinct() { + let query = r#" + SELECT DISTINCT "RainToday", "WindGustDir" FROM weather + "#; + let plan = sql_to_explain(query, |b| { + b.with_distributed_worker_resolver(InMemoryWorkerResolver::new(3)) + }) + .await; + assert_snapshot!(plan, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=8, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p3] t1:[p0..p3] + │ AggregateExec: mode=FinalPartitioned, gby=[RainToday@0 as RainToday, WindGustDir@1 as WindGustDir], aggr=[] + │ [Stage 1] => NetworkShuffleExec: output_partitions=4, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p7] t1:[p0..p7] t2:[p0..p7] + │ RepartitionExec: partitioning=Hash([RainToday@0, WindGustDir@1], 8), input_partitions=1 + │ AggregateExec: mode=Partial, gby=[RainToday@0 as RainToday, WindGustDir@1 as WindGustDir], aggr=[] + │ PartitionIsolatorExec: t0:[p0,__,__] t1:[__,p0,__] t2:[__,__,p0] + │ DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[RainToday, WindGustDir], file_type=parquet + └────────────────────────────────────────────────── + "); + } + + #[tokio::test] + async fn test_show_columns() { + let query = r#" + SHOW COLUMNS from weather + "#; + let plan = sql_to_explain(query, |b| { + b.with_distributed_worker_resolver(InMemoryWorkerResolver::new(3)) + }) + .await; + assert_snapshot!(plan, @r" + ProjectionExec: expr=[table_catalog@0 as table_catalog, table_schema@1 as table_schema, table_name@2 as table_name, column_name@3 as column_name, data_type@5 as data_type, is_nullable@4 as is_nullable] + FilterExec: table_name@2 = weather + RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 + StreamingTableExec: partition_sizes=1, projection=[table_catalog, table_schema, table_name, column_name, is_nullable, data_type] + "); + } + + #[tokio::test] + #[ignore] // FIXME: fix this test + async fn test_limited_by_worker() { + let query = r#" + SET datafusion.execution.target_partitions=2; + SELECT 1 FROM weather + UNION ALL + SELECT 1 FROM flights_1m + "#; + let plan = sql_to_explain(query, |b| { + b.with_distributed_worker_resolver(InMemoryWorkerResolver::new(2)) + }) + .await; + assert_snapshot!(plan, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=4, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] + │ UnionExec + │ ProjectionExec: expr=[1 as Int64(1)] + │ PartitionIsolatorExec: t0:[p0,__] t1:[__,p0] + │ DataSourceExec: file_groups={2 groups: [[/testdata/weather/result-000000.parquet, /testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, file_type=parquet + │ ProjectionExec: expr=[1 as Int64(1)] + │ PartitionIsolatorExec: t0:[p0] t1:[__] + │ DataSourceExec: file_groups={1 group: [[/testdata/flights-1m.parquet]]}, file_type=parquet + └────────────────────────────────────────────────── + "); + } + + #[tokio::test] + async fn test_unioning_2_tables() { + let query = r#" + set distributed.children_isolator_unions=true; + SELECT "MinTemp", "RainToday" FROM weather WHERE "MinTemp" > 10.0 + UNION ALL + SELECT "MaxTemp", "RainToday" FROM weather WHERE "MaxTemp" < 30.0 + "#; + let plan = sql_to_explain(query, |b| { + b.with_distributed_worker_resolver(InMemoryWorkerResolver::new(6)) + }) + .await; + assert_snapshot!(plan, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=24, input_tasks=6 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p3] t1:[p4..p7] t2:[p8..p11] t3:[p12..p15] t4:[p16..p19] t5:[p20..p23] + │ DistributedUnionExec: t0:[c0(0/3)] t1:[c0(1/3)] t2:[c0(2/3)] t3:[c1(0/3)] t4:[c1(1/3)] t5:[c1(2/3)] + │ FilterExec: MinTemp@0 > 10 + │ RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 + │ PartitionIsolatorExec: t0:[p0,__,__] t1:[__,p0,__] t2:[__,__,p0] + │ DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MinTemp, RainToday], file_type=parquet, predicate=MinTemp@0 > 10, pruning_predicate=MinTemp_null_count@1 != row_count@2 AND MinTemp_max@0 > 10, required_guarantees=[] + │ ProjectionExec: expr=[MaxTemp@0 as MinTemp, RainToday@1 as RainToday] + │ FilterExec: MaxTemp@0 < 30 + │ RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 + │ PartitionIsolatorExec: t0:[p0,__,__] t1:[__,p0,__] t2:[__,__,p0] + │ DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=MaxTemp@1 < 30, pruning_predicate=MaxTemp_null_count@1 != row_count@2 AND MaxTemp_min@0 < 30, required_guarantees=[] + └────────────────────────────────────────────────── + "); + } + + #[tokio::test] + async fn test_unioning_2_tables_limited_workers() { + let query = r#" + set distributed.children_isolator_unions=true; + SELECT "MinTemp", "RainToday" FROM weather WHERE "MinTemp" > 10.0 + UNION ALL + SELECT "MaxTemp", "RainToday" FROM weather WHERE "MaxTemp" < 30.0 + "#; + let plan = sql_to_explain(query, |b| { + b.with_distributed_worker_resolver(InMemoryWorkerResolver::new(3)) + }) + .await; + assert_snapshot!(plan, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=12, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p3] t1:[p4..p7] t2:[p8..p11] + │ DistributedUnionExec: t0:[c0] t1:[c1(0/2)] t2:[c1(1/2)] + │ FilterExec: MinTemp@0 > 10 + │ RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=3 + │ DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MinTemp, RainToday], file_type=parquet, predicate=MinTemp@0 > 10, pruning_predicate=MinTemp_null_count@1 != row_count@2 AND MinTemp_max@0 > 10, required_guarantees=[] + │ ProjectionExec: expr=[MaxTemp@0 as MinTemp, RainToday@1 as RainToday] + │ FilterExec: MaxTemp@0 < 30 + │ RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__] t1:[__,__,p0] + │ DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=MaxTemp@1 < 30, pruning_predicate=MaxTemp_null_count@1 != row_count@2 AND MaxTemp_min@0 < 30, required_guarantees=[] + └────────────────────────────────────────────────── + "); + } + + #[tokio::test] + async fn test_unioning_3_tables() { + let query = r#" + set distributed.children_isolator_unions=true; + SELECT "MinTemp", "RainToday" FROM weather WHERE "MinTemp" > 10.0 + UNION ALL + SELECT "MaxTemp", "RainToday" FROM weather WHERE "MaxTemp" < 30.0 + UNION ALL + SELECT "Temp9am", "RainToday" FROM weather WHERE "Temp9am" > 15.0 + "#; + let plan = sql_to_explain(query, |b| { + b.with_distributed_worker_resolver(InMemoryWorkerResolver::new(3)) + }) + .await; + assert_snapshot!(plan, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=12, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p3] t1:[p4..p7] t2:[p8..p11] + │ DistributedUnionExec: t0:[c0] t1:[c1] t2:[c2] + │ FilterExec: MinTemp@0 > 10 + │ RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=3 + │ DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MinTemp, RainToday], file_type=parquet, predicate=MinTemp@0 > 10, pruning_predicate=MinTemp_null_count@1 != row_count@2 AND MinTemp_max@0 > 10, required_guarantees=[] + │ ProjectionExec: expr=[MaxTemp@0 as MinTemp, RainToday@1 as RainToday] + │ FilterExec: MaxTemp@0 < 30 + │ RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=3 + │ DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=MaxTemp@1 < 30, pruning_predicate=MaxTemp_null_count@1 != row_count@2 AND MaxTemp_min@0 < 30, required_guarantees=[] + │ ProjectionExec: expr=[Temp9am@0 as MinTemp, RainToday@1 as RainToday] + │ FilterExec: Temp9am@0 > 15 + │ RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=3 + │ DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[Temp9am, RainToday], file_type=parquet, predicate=Temp9am@17 > 15, pruning_predicate=Temp9am_null_count@1 != row_count@2 AND Temp9am_max@0 > 15, required_guarantees=[] + └────────────────────────────────────────────────── + "); + } + + #[tokio::test] + async fn test_unioning_5_tables() { + let query = r#" + set distributed.children_isolator_unions=true; + SELECT "MinTemp", "RainToday" FROM weather WHERE "MinTemp" > 10.0 + UNION ALL + SELECT "MaxTemp", "RainToday" FROM weather WHERE "MaxTemp" < 30.0 + UNION ALL + SELECT "Temp9am", "RainToday" FROM weather WHERE "Temp9am" > 15.0 + UNION ALL + SELECT "Temp3pm", "RainToday" FROM weather WHERE "Temp3pm" < 25.0 + UNION ALL + SELECT "Rainfall", "RainToday" FROM weather WHERE "Rainfall" > 5.0 + "#; + let plan = sql_to_explain(query, |b| { + b.with_distributed_worker_resolver(InMemoryWorkerResolver::new(3)) + }) + .await; + assert_snapshot!(plan, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=24, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p7] t1:[p8..p15] t2:[p16..p23] + │ DistributedUnionExec: t0:[c0, c1] t1:[c2, c3] t2:[c4] + │ FilterExec: MinTemp@0 > 10 + │ RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=3 + │ DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MinTemp, RainToday], file_type=parquet, predicate=MinTemp@0 > 10, pruning_predicate=MinTemp_null_count@1 != row_count@2 AND MinTemp_max@0 > 10, required_guarantees=[] + │ ProjectionExec: expr=[MaxTemp@0 as MinTemp, RainToday@1 as RainToday] + │ FilterExec: MaxTemp@0 < 30 + │ RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=3 + │ DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=MaxTemp@1 < 30, pruning_predicate=MaxTemp_null_count@1 != row_count@2 AND MaxTemp_min@0 < 30, required_guarantees=[] + │ ProjectionExec: expr=[Temp9am@0 as MinTemp, RainToday@1 as RainToday] + │ FilterExec: Temp9am@0 > 15 + │ RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=3 + │ DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[Temp9am, RainToday], file_type=parquet, predicate=Temp9am@17 > 15, pruning_predicate=Temp9am_null_count@1 != row_count@2 AND Temp9am_max@0 > 15, required_guarantees=[] + │ ProjectionExec: expr=[Temp3pm@0 as MinTemp, RainToday@1 as RainToday] + │ FilterExec: Temp3pm@0 < 25 + │ RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=3 + │ DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[Temp3pm, RainToday], file_type=parquet, predicate=Temp3pm@18 < 25, pruning_predicate=Temp3pm_null_count@1 != row_count@2 AND Temp3pm_min@0 < 25, required_guarantees=[] + │ ProjectionExec: expr=[Rainfall@0 as MinTemp, RainToday@1 as RainToday] + │ FilterExec: Rainfall@0 > 5 + │ RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=3 + │ DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[Rainfall, RainToday], file_type=parquet, predicate=Rainfall@2 > 5, pruning_predicate=Rainfall_null_count@1 != row_count@2 AND Rainfall_max@0 > 5, required_guarantees=[] + └────────────────────────────────────────────────── + "); + } + + #[tokio::test] + async fn test_broadcast_join() { + let query = r#" + SELECT a."MinTemp", b."MaxTemp" + FROM weather a INNER JOIN weather b + ON a."RainToday" = b."RainToday" + "#; + let annotated = sql_to_explain_with_broadcast(query, 3, true).await; + assert_snapshot!(annotated, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0] t1:[p1] t2:[p2] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(RainToday@1, RainToday@1)], projection=[MinTemp@0, MaxTemp@2] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=1, stage_partitions=3, input_tasks=3 + │ PartitionIsolatorExec: t0:[p0,__,__] t1:[__,p0,__] t2:[__,__,p0] + │ DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] t2:[p6..p8] + │ BroadcastExec: input_partitions=1, consumer_tasks=3, output_partitions=3 + │ PartitionIsolatorExec: t0:[p0,__,__] t1:[__,p0,__] t2:[__,__,p0] + │ DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MinTemp, RainToday], file_type=parquet + └────────────────────────────────────────────────── + ") + } + + #[tokio::test] + async fn test_broadcast_nested_joins() { + let query = r#" + SELECT a."MinTemp", b."MaxTemp", c."Rainfall" + FROM weather a + INNER JOIN weather b ON a."RainToday" = b."RainToday" + INNER JOIN weather c ON b."RainToday" = c."RainToday" + "#; + let plan = sql_to_explain_with_broadcast(query, 3, true).await; + assert_snapshot!(plan, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkCoalesceExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0] t1:[p1] t2:[p2] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(RainToday@2, RainToday@1)], projection=[MinTemp@0, MaxTemp@1, Rainfall@3] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=1, stage_partitions=3, input_tasks=3 + │ PartitionIsolatorExec: t0:[p0,__,__] t1:[__,p0,__] t2:[__,__,p0] + │ DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[Rainfall, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p3..p5] t2:[p6..p8] + │ BroadcastExec: input_partitions=1, consumer_tasks=3, output_partitions=3 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(RainToday@1, RainToday@1)], projection=[MinTemp@0, MaxTemp@2, RainToday@3] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=1, stage_partitions=3, input_tasks=3 + │ PartitionIsolatorExec: t0:[p0,__,__] t1:[__,p0,__] t2:[__,__,p0] + │ DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] t2:[p6..p8] + │ BroadcastExec: input_partitions=1, consumer_tasks=3, output_partitions=3 + │ PartitionIsolatorExec: t0:[p0,__,__] t1:[__,p0,__] t2:[__,__,p0] + │ DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MinTemp, RainToday], file_type=parquet + └────────────────────────────────────────────────── + ") + } + + #[tokio::test] + async fn test_broadcast_datasource_as_build_child() { + let query = r#" + SELECT a."MinTemp", b."MaxTemp" + FROM weather a INNER JOIN weather b + ON a."RainToday" = b."RainToday" + "#; + + let physical_plan = sql_to_physical_plan(query, 4, 3).await; + assert_snapshot!(physical_plan, @r" + HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(RainToday@1, RainToday@1)], projection=[MinTemp@0, MaxTemp@2] + CoalescePartitionsExec + DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MinTemp, RainToday], file_type=parquet + DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ] + "); + + let plan = sql_to_explain_with_broadcast(query, 3, true).await; + assert_snapshot!(plan, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0] t1:[p1] t2:[p2] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(RainToday@1, RainToday@1)], projection=[MinTemp@0, MaxTemp@2] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=1, stage_partitions=3, input_tasks=3 + │ PartitionIsolatorExec: t0:[p0,__,__] t1:[__,p0,__] t2:[__,__,p0] + │ DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] t2:[p6..p8] + │ BroadcastExec: input_partitions=1, consumer_tasks=3, output_partitions=3 + │ PartitionIsolatorExec: t0:[p0,__,__] t1:[__,p0,__] t2:[__,__,p0] + │ DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MinTemp, RainToday], file_type=parquet + └────────────────────────────────────────────────── + ") + } + + #[tokio::test] + async fn test_broadcast_union_children_isolator_plan() { + let query = r#" + SET distributed.children_isolator_unions = true; + SELECT a."MinTemp", b."MaxTemp" + FROM weather a INNER JOIN weather b + ON a."RainToday" = b."RainToday" + UNION ALL + SELECT a."MinTemp", b."MaxTemp" + FROM weather a INNER JOIN weather b + ON a."RainToday" = b."RainToday" + UNION ALL + SELECT a."MinTemp", b."MaxTemp" + FROM weather a INNER JOIN weather b + ON a."RainToday" = b."RainToday" + "#; + let plan = sql_to_explain_with_broadcast(query, 3, true).await; + assert_snapshot!(plan, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=9, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] t2:[p6..p8] + │ DistributedUnionExec: t0:[c0] t1:[c1] t2:[c2] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(RainToday@1, RainToday@1)], projection=[MinTemp@0, MaxTemp@2] + │ CoalescePartitionsExec + │ BroadcastExec: input_partitions=3, consumer_tasks=1, output_partitions=3 + │ DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MinTemp, RainToday], file_type=parquet + │ DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(RainToday@1, RainToday@1)], projection=[MinTemp@0, MaxTemp@2] + │ CoalescePartitionsExec + │ BroadcastExec: input_partitions=3, consumer_tasks=1, output_partitions=3 + │ DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MinTemp, RainToday], file_type=parquet + │ DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(RainToday@1, RainToday@1)], projection=[MinTemp@0, MaxTemp@2] + │ CoalescePartitionsExec + │ BroadcastExec: input_partitions=3, consumer_tasks=1, output_partitions=3 + │ DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MinTemp, RainToday], file_type=parquet + │ DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + "); + } + + #[tokio::test] + async fn test_broadcast_one_to_many_plan() { + let query = r#" + SELECT a."MinTemp", b."MaxTemp" + FROM weather a INNER JOIN weather b + ON a."RainToday" = b."RainToday" + "#; + let plan = sql_to_explain_with_broadcast_one_to_many(query, 3).await; + assert_snapshot!(plan, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0] t1:[p1] t2:[p2] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(RainToday@1, RainToday@1)], projection=[MinTemp@0, MaxTemp@2] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=1 + │ PartitionIsolatorExec: t0:[p0,__,__] t1:[__,p0,__] t2:[__,__,p0] + │ DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p8] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MinTemp, RainToday], file_type=parquet + └────────────────────────────────────────────────── + "); + } + + async fn sql_to_explain( + query: &str, + f: impl FnOnce(SessionStateBuilder) -> SessionStateBuilder, + ) -> String { + explain_test_plan(query, TestPlanOptions::default(), true, f).await + } + + async fn sql_to_explain_with_broadcast( + query: &str, + num_workers: usize, + broadcast_enabled: bool, + ) -> String { + sql_to_plan_with_options(query, num_workers, broadcast_enabled, true).await + } + + async fn sql_to_explain_with_broadcast_one_to_many(query: &str, num_workers: usize) -> String { + let options = TestPlanOptions { + target_partitions: 4, + num_workers, + broadcast_enabled: true, + }; + explain_test_plan(query, options, true, |b| { + b.with_distributed_task_estimator(BuildSideOneTaskEstimator) + }) + .await + } + + async fn sql_to_plan_with_options( + query: &str, + num_workers: usize, + broadcast_enabled: bool, + use_optimizer: bool, + ) -> String { + let options = TestPlanOptions { + target_partitions: 4, + num_workers, + broadcast_enabled, + }; + explain_test_plan(query, options, use_optimizer, |b| b).await + } + + async fn explain_test_plan( + query: &str, + options: TestPlanOptions, + use_optimizer: bool, + configure: impl FnOnce(SessionStateBuilder) -> SessionStateBuilder, + ) -> String { + let mut builder = base_session_builder( + options.target_partitions, + options.num_workers, + options.broadcast_enabled, + ); + if use_optimizer { + builder = + builder.with_physical_optimizer_rule(Arc::new(DistributedPhysicalOptimizerRule)); + } + let builder = configure(builder); + let (ctx, query) = context_with_query(builder, query).await; + let df = ctx.sql(&query).await.unwrap(); + let physical_plan = df.create_physical_plan().await.unwrap(); + + if use_optimizer { + display_plan_ascii(physical_plan.as_ref(), false) + } else { + format!("{}", displayable(physical_plan.as_ref()).indent(true)) + } + } +} diff --git a/src/distributed_planner/insert_broadcast.rs b/src/distributed_planner/insert_broadcast.rs new file mode 100644 index 00000000..b5b8e941 --- /dev/null +++ b/src/distributed_planner/insert_broadcast.rs @@ -0,0 +1,290 @@ +use std::sync::Arc; + +use datafusion::common::JoinType; +use datafusion::common::tree_node::{Transformed, TreeNode}; +use datafusion::config::ConfigOptions; +use datafusion::error::DataFusionError; +use datafusion::physical_plan::ExecutionPlan; +use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec; +use datafusion::physical_plan::joins::{HashJoinExec, PartitionMode}; + +use crate::BroadcastExec; + +use super::DistributedConfig; + +/// This is a top-down traversal of a [ExecutionPlan] that inserts [BroadcastExec] opeerators where +/// appropriate. +/// +/// # What is it doing? +/// The pass searches for CollectLeft [HashJoinExec]s that are either "Right" or Inner join type. +/// Then does one of two things: +/// 1. If the build child is a [CoalescePartitionsExec] -> Insert a [BroadcastExec] directly +/// below it. +/// 2. Otherwise (means its already single partitioned going into the join) -> Insert a +/// [BroadcastExec] -> [CoalescePartitionsExec] below the [HashJoinExec] but above its +/// orginal build child. +/// ```text +/// ┌──────────────────────┐ ┌──────────────────────┐ +/// │ CoalesceBatches │ │ CoalesceBatches │ +/// └───────────▲──────────┘ └───────────▲──────────┘ +/// │ │ +/// ┌───────────┴──────────┐ ┌───────────┴──────────┐ +/// │ HashJoin │ │ HashJoin │ +/// │ (CollectLeft) │ │ (CollectLeft) │ +/// └────▲────────────▲────┘ └────▲────────────▲────┘ +/// │ │ │ │ +/// ┌─────────┘ └──────────┐ ┌─────────┘ └──────────┐ +/// Build Side Probe Side Build Side Probe Side +/// │ │ │ │ +/// ┌───────────┴──────────┐ ┌───────────┴──────────┐ ┌───────────┴──────────┐ ┌───────────┴──────────┐ +/// │ CoalescePartitions │ │ Projection │ │ CoalescePartitions │ │ Projection │ +/// └───▲────▲────▲────▲───┘ └───────────▲──────────┘ └───────────▲──────────┘ └───────────▲──────────┘ +/// │ │ │ │ │ │ │ +/// ┌───┴────┴────┴────┴───┐ ┌───────────┴──────────┐ ┌───────────┴──────────┐ ┌───────────┴──────────┐ +/// │ DataSource │ │ Aggregation │ ───────────────▶ │ BroadcastExec │ │ Aggregation │ +/// └──────────────────────┘ └───────────▲──────────┘ └──▲────▲────▲────▲────┘ └───────────▲──────────┘ +/// │ │ │ │ │ │ +/// ┌───────────┴──────────┐ ┌──┴────┴────┴────┴────┐ ┌───────────┴──────────┐ +/// │ Repartition │ │ DataSource │ │ Repartition │ +/// └───────────▲──────────┘ └──────────────────────┘ └───────────▲──────────┘ +/// │ │ +/// ┌───────────┴──────────┐ ┌───────────┴──────────┐ +/// │ Aggregation │ │ Aggregation │ +/// │ (Partial) │ │ (Partial) │ +/// └───────────▲──────────┘ └───────────▲──────────┘ +/// │ │ +/// ┌───────────┴──────────┐ ┌───────────┴──────────┐ +/// │ DataSource │ │ DataSource │ +/// └──────────────────────┘ └──────────────────────┘ +/// ``` +/// +/// # Why a Right or Inner join type? +/// Left and Full join types allow build side rows to be emitted. This creates complicatioins when +/// broadcasting your build side to all workers as it can cause incorrect and duplicate data. Here +/// is an example: +/// +/// Say there is arbitrary tables containing information on customers and their orders. +/// ```text +/// Probe Side +/// ┌──────────┬─────────────┐ +/// Build Side │ order_id │ customer_id │ +/// ┌─────────────┬─────────┐ ├──────────┼─────────────┤ +/// │ customer_id │ Name │ │ 100 │ 1 │ +/// ├─────────────┼─────────┤ ├──────────┼─────────────┤ +/// │ 1 │ John │ │ 200 │ 1 │ +/// ├─────────────┼─────────┤ ├──────────┼─────────────┤ +/// │ 2 │ Alice │ │ 300 │ 1 │ +/// ├─────────────┼─────────┤ ├──────────┼─────────────┤ +/// │ 3 │ Bob │ │ 400 │ 2 │ +/// └─────────────┴─────────┘ ├──────────┼─────────────┤ +/// │ 500 │ 2 │ +/// └──────────┴─────────────┘ +/// ``` +/// Then want to execute the query: +/// SELECT * FROM customers c +/// WHERE NOT EXISTS (SELECT 1 FROM orders o WHERE o.customer_id = c.customer_id); +/// +/// This query is selecting all customers that have not made an order. It does this by using a +/// LeftAnti join which will emit all the rows from our build side which do not have a matching +/// join key (in this case customer_id) on the probe side. +/// +/// In a single node this would produce the correct result: (3, Bob) +/// +/// In a distributed context broadcasting the build table would create incorrect results: +/// ```text +/// ┌──────────────────────────────────────────────────────────┐ ┌──────────────────────────────────────────────────────────┐ +/// │ Worker 1 │ │ Worker 1 │ +/// │ ┌─────────────┬─────────┐ ┌──────────┬─────────────┐ │ │ ┌─────────────┬─────────┐ │ +/// │ │ customer_id │ Name │ │ order_id │ customer_id │ │ │ │ customer_id │ Name │ ┌──────────┬─────────────┐ │ +/// │ ├─────────────┼─────────┤ ├──────────┼─────────────┤ │ │ ├─────────────┼─────────┤ │ order_id │ customer_id │ │ +/// │ │ 1 │ John │ │ 100 │ 1 │ │ │ │ 1 │ John │ ├──────────┼─────────────┤ │ +/// │ ├─────────────┼─────────┤ ├──────────┼─────────────┤ │ │ ├─────────────┼─────────┤ │ 400 │ 2 │ │ +/// │ │ 2 │ Alice │ │ 200 │ 1 │ │ │ │ 2 │ Alice │ ├──────────┼─────────────┤ │ +/// │ ├─────────────┼─────────┤ ├──────────┼─────────────┤ │ │ ├─────────────┼─────────┤ │ 500 │ 2 │ │ +/// │ │ 3 │ Bob │ │ 300 │ 1 │ │ │ │ 3 │ Bob │ └──────────┴─────────────┘ │ +/// │ └─────────────┴─────────┘ └──────────┴─────────────┘ │ │ └─────────────┴─────────┘ │ +/// └──────────────────────────────────────────────────────────┘ └──────────────────────────────────────────────────────────┘ +/// ``` +/// Worker 1 would emit: (2, Alice), (3, Bob) +/// Worker 2 would emit: (1, John), (3, Bob) +/// Thus when unioning results: (2, Alice), (3, Bob), (1, John), (3, Bob) +/// ``` +/// +/// ``` +pub(super) fn insert_broadcast_execs( + plan: Arc, + cfg: &ConfigOptions, +) -> Result, DataFusionError> { + let d_cfg = DistributedConfig::from_config_options(cfg)?; + if !d_cfg.broadcast_joins { + return Ok(plan); + } + + plan.transform_down(|node| { + let Some(hash_join) = node.as_any().downcast_ref::() else { + return Ok(Transformed::no(node)); + }; + if hash_join.partition_mode() != &PartitionMode::CollectLeft { + return Ok(Transformed::no(node)); + } + + // Only broadcast when output is driven by the probe side. + // Joins that can emit build-side rows (left/left-semi/left-anti/left-mark/full) would + // duplicate output if the build is broadcast, thus are excluded. + let join_type = hash_join.join_type(); + if !matches!( + join_type, + JoinType::Inner + | JoinType::Right + | JoinType::RightSemi + | JoinType::RightAnti + | JoinType::RightMark + ) { + return Ok(Transformed::no(node)); + } + + let children = node.children(); + let Some(build_child) = children.first() else { + return Ok(Transformed::no(node)); + }; + + // If build child is CoalescePartitionsExec get its input + // Otherwise, use the build child directly (DataSourceExec) + let broadcast_input = if let Some(coalesce) = build_child + .as_any() + .downcast_ref::() + { + Arc::clone(coalesce.input()) + } else { + Arc::clone(build_child) + }; + + // Insert BroadcastExec. consumer_task_count=1 is a placeholder and + // will be corrected during optimizer rule. + let broadcast = Arc::new(BroadcastExec::new( + broadcast_input, + 1, // placeholder + )); + + // Always wrap with CoalescePartitionsExec + let new_build_child: Arc = + Arc::new(CoalescePartitionsExec::new(broadcast)); + + let mut new_children: Vec> = children.into_iter().cloned().collect(); + new_children[0] = new_build_child; + Ok(Transformed::yes(node.with_new_children(new_children)?)) + }) + .map(|transformed| transformed.data) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::assert_snapshot; + use crate::test_utils::plans::{ + TestPlanOptions, base_session_builder, context_with_query, sql_to_physical_plan, + }; + use datafusion::physical_plan::displayable; + + #[tokio::test] + async fn test_insert_broadcast_with_existing_coalesce_build_child() { + let query = r#" + SELECT a."MinTemp", b."MaxTemp" + FROM weather a INNER JOIN weather b + ON a."RainToday" = b."RainToday" + "#; + let physical_plan = sql_to_physical_plan(query, 4, 4).await; + assert_snapshot!(physical_plan, @r" + HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(RainToday@1, RainToday@1)], projection=[MinTemp@0, MaxTemp@2] + CoalescePartitionsExec + DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MinTemp, RainToday], file_type=parquet + DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ] + "); + let plan = sql_to_plan_with_broadcast(query, true, 4).await; + assert_snapshot!(plan, @r" + HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(RainToday@1, RainToday@1)], projection=[MinTemp@0, MaxTemp@2] + CoalescePartitionsExec + BroadcastExec: input_partitions=3, consumer_tasks=1, output_partitions=3 + DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MinTemp, RainToday], file_type=parquet + DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ] + "); + } + + #[tokio::test] + async fn test_insert_broadcast_without_existing_coalesce_build_child() { + let query = r#" + SELECT a."MinTemp", b."MaxTemp" + FROM weather a INNER JOIN weather b + ON a."RainToday" = b."RainToday" + "#; + let physical_plan = sql_to_physical_plan(query, 1, 4).await; + assert_snapshot!(physical_plan, @r" + HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(RainToday@1, RainToday@1)], projection=[MinTemp@0, MaxTemp@2] + DataSourceExec: file_groups={1 group: [[/testdata/weather/result-000000.parquet, /testdata/weather/result-000001.parquet, /testdata/weather/result-000002.parquet]]}, projection=[MinTemp, RainToday], file_type=parquet + DataSourceExec: file_groups={1 group: [[/testdata/weather/result-000000.parquet, /testdata/weather/result-000001.parquet, /testdata/weather/result-000002.parquet]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ] + "); + let plan = sql_to_plan_with_broadcast(query, true, 1).await; + assert_snapshot!(plan, @r" + HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(RainToday@1, RainToday@1)], projection=[MinTemp@0, MaxTemp@2] + CoalescePartitionsExec + BroadcastExec: input_partitions=1, consumer_tasks=1, output_partitions=1 + DataSourceExec: file_groups={1 group: [[/testdata/weather/result-000000.parquet, /testdata/weather/result-000001.parquet, /testdata/weather/result-000002.parquet]]}, projection=[MinTemp, RainToday], file_type=parquet + DataSourceExec: file_groups={1 group: [[/testdata/weather/result-000000.parquet, /testdata/weather/result-000001.parquet, /testdata/weather/result-000002.parquet]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ] + "); + } + + #[tokio::test] + async fn test_no_broadcast_left_join() { + let query = r#" + SELECT a."MinTemp", b."MaxTemp" + FROM weather a LEFT JOIN weather b + ON a."RainToday" = b."RainToday" + "#; + let plan = sql_to_plan_with_broadcast(query, true, 4).await; + assert_snapshot!(plan, @r" + HashJoinExec: mode=CollectLeft, join_type=Left, on=[(RainToday@1, RainToday@1)], projection=[MinTemp@0, MaxTemp@2] + CoalescePartitionsExec + DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MinTemp, RainToday], file_type=parquet + DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MaxTemp, RainToday], file_type=parquet + "); + } + + #[tokio::test] + async fn test_no_broadcast_when_disabled() { + let query = r#" + SELECT a."MinTemp", b."MaxTemp" + FROM weather a INNER JOIN weather b + ON a."RainToday" = b."RainToday" + "#; + let plan = sql_to_plan_with_broadcast(query, false, 4).await; + assert_snapshot!(plan, @r" + HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(RainToday@1, RainToday@1)], projection=[MinTemp@0, MaxTemp@2] + CoalescePartitionsExec + DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MinTemp, RainToday], file_type=parquet + DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ] + "); + } + + async fn sql_to_plan_with_broadcast( + query: &str, + broadcast_enabled: bool, + target_partitions: usize, + ) -> String { + let options = TestPlanOptions { + target_partitions, + num_workers: 4, + broadcast_enabled, + }; + let builder = base_session_builder( + options.target_partitions, + options.num_workers, + options.broadcast_enabled, + ); + let (ctx, query) = context_with_query(builder, query).await; + let df = ctx.sql(&query).await.unwrap(); + let plan = df.create_physical_plan().await.unwrap(); + let plan = insert_broadcast_execs(plan, ctx.state_ref().read().config_options().as_ref()) + .expect("failed to insert broadcasts"); + format!("{}", displayable(plan.as_ref()).indent(true)) + } +} diff --git a/src/distributed_planner/mod.rs b/src/distributed_planner/mod.rs new file mode 100644 index 00000000..2f9b2895 --- /dev/null +++ b/src/distributed_planner/mod.rs @@ -0,0 +1,14 @@ +mod batch_coalescing_below_network_boundaries; +mod distributed_config; +mod distributed_physical_optimizer_rule; +mod insert_broadcast; +mod network_boundary; +mod plan_annotator; +mod task_estimator; + +pub(crate) use batch_coalescing_below_network_boundaries::batch_coalescing_below_network_boundaries; +pub use distributed_config::DistributedConfig; +pub use distributed_physical_optimizer_rule::DistributedPhysicalOptimizerRule; +pub use network_boundary::{NetworkBoundary, NetworkBoundaryExt}; +pub(crate) use task_estimator::set_distributed_task_estimator; +pub use task_estimator::{TaskCountAnnotation, TaskEstimation, TaskEstimator}; diff --git a/src/distributed_planner/network_boundary.rs b/src/distributed_planner/network_boundary.rs new file mode 100644 index 00000000..fcb0b676 --- /dev/null +++ b/src/distributed_planner/network_boundary.rs @@ -0,0 +1,44 @@ +use crate::{NetworkBroadcastExec, NetworkCoalesceExec, NetworkShuffleExec, Stage}; +use datafusion::physical_plan::ExecutionPlan; +use std::sync::Arc; + +/// This trait represents a node that introduces the necessity of a network boundary in the plan. +/// The distributed planner, upon stepping into one of these, will break the plan and build a stage +/// out of it. +pub trait NetworkBoundary: ExecutionPlan { + /// Called when a [Stage] is correctly formed. The [NetworkBoundary] can use this + /// information to perform any internal transformations necessary for distributed execution. + /// + /// Typically, [NetworkBoundary]s will use this call for transitioning from "Pending" to "ready". + fn with_input_stage( + &self, + input_stage: Stage, + ) -> datafusion::common::Result>; + + /// Returns the assigned input [Stage], if any. + fn input_stage(&self) -> &Stage; +} + +/// Extension trait for downcasting dynamic types to [NetworkBoundary]. +pub trait NetworkBoundaryExt { + /// Downcasts self to a [NetworkBoundary] if possible. + fn as_network_boundary(&self) -> Option<&dyn NetworkBoundary>; + /// Returns whether self is a [NetworkBoundary] or not. + fn is_network_boundary(&self) -> bool { + self.as_network_boundary().is_some() + } +} + +impl NetworkBoundaryExt for dyn ExecutionPlan { + fn as_network_boundary(&self) -> Option<&dyn NetworkBoundary> { + if let Some(node) = self.as_any().downcast_ref::() { + Some(node) + } else if let Some(node) = self.as_any().downcast_ref::() { + Some(node) + } else if let Some(node) = self.as_any().downcast_ref::() { + Some(node) + } else { + None + } + } +} diff --git a/src/distributed_planner/plan_annotator.rs b/src/distributed_planner/plan_annotator.rs new file mode 100644 index 00000000..f54b4271 --- /dev/null +++ b/src/distributed_planner/plan_annotator.rs @@ -0,0 +1,987 @@ +use crate::TaskCountAnnotation::{Desired, Maximum}; +use crate::execution_plans::ChildrenIsolatorUnionExec; +use crate::{BroadcastExec, DistributedConfig, TaskCountAnnotation, TaskEstimator}; +use datafusion::common::{DataFusionError, plan_datafusion_err}; +use datafusion::config::ConfigOptions; +use datafusion::physical_expr::Partitioning; +use datafusion::physical_plan::ExecutionPlan; +use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec; +use datafusion::physical_plan::execution_plan::CardinalityEffect; +use datafusion::physical_plan::joins::{HashJoinExec, PartitionMode}; +use datafusion::physical_plan::repartition::RepartitionExec; +use datafusion::physical_plan::sorts::sort_preserving_merge::SortPreservingMergeExec; +use datafusion::physical_plan::union::UnionExec; +use std::fmt::{Debug, Formatter}; +use std::sync::Arc; + +/// Annotation attached to a single [ExecutionPlan] that determines the kind of network boundary +/// needed just below itself. +pub(super) enum PlanOrNetworkBoundary { + Plan(Arc), + Shuffle, + Coalesce, + Broadcast, +} + +impl Debug for PlanOrNetworkBoundary { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + Self::Plan(plan) => write!(f, "{}", plan.name()), + Self::Shuffle => write!(f, "[NetworkBoundary] Shuffle"), + Self::Coalesce => write!(f, "[NetworkBoundary] Coalesce"), + Self::Broadcast => write!(f, "[NetworkBoundary] Broadcast"), + } + } +} + +impl PlanOrNetworkBoundary { + fn is_network_boundary(&self) -> bool { + matches!(self, Self::Shuffle | Self::Coalesce | Self::Broadcast) + } +} + +/// Wraps an [ExecutionPlan] and annotates it with information about how many distributed tasks +/// it should run on, and whether it needs a network boundary below or not. +pub(super) struct AnnotatedPlan { + /// The annotated [ExecutionPlan]. + pub(super) plan_or_nb: PlanOrNetworkBoundary, + /// The annotated children of this [ExecutionPlan]. This will always hold the same nodes as + /// `self.plan.children()` but annotated. + pub(super) children: Vec, + + // annotation fields + /// How many distributed tasks this plan should run on. + pub(super) task_count: TaskCountAnnotation, +} + +impl Debug for AnnotatedPlan { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + fn fmt_dbg(f: &mut Formatter<'_>, plan: &AnnotatedPlan, depth: usize) -> std::fmt::Result { + write!( + f, + "{}{:?}: task_count={:?}", + " ".repeat(depth * 2), + plan.plan_or_nb, + plan.task_count + )?; + writeln!(f)?; + for child in plan.children.iter() { + fmt_dbg(f, child, depth + 1)?; + } + Ok(()) + } + + fmt_dbg(f, self, 0) + } +} + +/// Annotates recursively an [ExecutionPlan] and its children with information about how many +/// distributed tasks it should run on, and whether it needs a network boundary below it or not. +/// +/// This is the first step of the distribution process, where the plan structure is still left +/// untouched and the existing nodes are just annotated for future steps to perform the distribution. +/// +/// The plans are annotated in a bottom-to-top manner, starting with the leaf nodes all the way +/// to the head of the plan: +/// +/// 1. Leaf nodes have the opportunity to provide an estimation of how many distributed tasks should +/// be used for the whole stage that will execute them. +/// +/// 2. If a stage contains multiple leaf nodes, and all provide a task count estimation, the +/// biggest is taken. +/// +/// 3. When traversing the plan in a bottom-to-top fashion, this function looks for nodes that +/// either increase or reduce cardinality: +/// - If there's a node that increases cardinality, the next stage will spawn more tasks than +/// the current one. +/// - If there's a node that reduces cardinality, the next stage will spawn fewer tasks than the +/// current one. +/// +/// 4. At a certain point, the function will reach a node that needs a network boundary below; in +/// that case, the node is annotated with a [PlanOrNetworkBoundary] value. At this point, all +/// the nodes below must reach a consensus about the final task count for the stage below the +/// network boundary. +/// +/// 5. This process is repeated recursively until all nodes are annotated. +/// +/// ## Example: +/// +/// Following the process above, an annotated plan will look like this: +/// +/// ```text +/// +/// ┌──────────────────────┐ task_count: Desired(3) (inherited from child) +/// │ CoalesceBatches │ network_boundary: None +/// └───────────▲──────────┘ +/// │ +/// ┌───────────┴──────────┐ task_count: Desired(3) (inherits from probe child) +/// │ HashJoin │ network_boundary: None +/// │ (CollectLeft Inner) │ +/// └─▲──────────────────▲─┘ +/// │ │ +/// │ Probe Side +/// │ │ +/// │ ┌───────────┴──────────┐ task_count: Desired(3) (inherited from child) +/// │ │ Projection │ network_boundary: None +/// │ └───────────▲──────────┘ +/// │ │ +/// │ ┌───────────┴──────────┐ task_count: Desired(3) (as this node requires a network boundary below, +/// │ │ Aggregation │ and the stage below reduces the cardinality of the data because of the +/// │ │ (Final) │ partial aggregation, we can choose a smaller amount of tasks) +/// │ └───────────▲──────────┘ network_boundary: Some(Shuffle) (because the child is a repartition) +/// │ │ +/// │ ┌───────────┴──────────┐ task_count: Desired(4) (inherited from child) +/// │ │ Repartition │ network_boundary: None +/// Build Side └───────────▲──────────┘ +/// │ │ +/// │ ┌───────────┴──────────┐ +/// │ │ Aggregation │ task_count: Desired(4) (inherited from child) +/// │ │ (Partial) │ network_boundary: None +/// │ └───────────▲──────────┘ +/// │ │ +/// │ ┌───────────┴──────────┐ task_count: Desired(4) (this was set by a TaskEstimator implementation) +/// │ │ DataSource │ network_boundary: None +/// │ └──────────────────────┘ +/// │ +/// │ +/// ┌───────────┴──────────┐ task_count: Desired(3) (inherits from the probe side) +/// │ CoalescePartitions │ network_boundary: Broadcast +/// └───────────▲──────────┘ +/// │ +/// ┌───────────┴──────────┐ task_count: Desired(2) (inherited from child) +/// │ BroadcastExec │ network_boundary: None +/// └───────────▲──────────┘ +/// │ +/// ┌───────────┴──────────┐ task_count: Desired(2) (this was set by a TaskEstimator implementation) +/// │ DataSource │ network_boundary: None +/// └──────────────────────┘ └──────────────────────┘ +/// ``` +/// +/// ``` +pub(super) fn annotate_plan( + plan: Arc, + cfg: &ConfigOptions, +) -> Result { + _annotate_plan(plan, None, cfg, true) +} + +fn _annotate_plan( + plan: Arc, + parent: Option<&Arc>, + cfg: &ConfigOptions, + root: bool, +) -> Result { + let d_cfg = DistributedConfig::from_config_options(cfg)?; + let broadcast_joins = d_cfg.broadcast_joins; + let estimator = &d_cfg.__private_task_estimator; + let n_workers = d_cfg.__private_worker_resolver.0.get_urls()?.len().max(1); + + let annotated_children = plan + .children() + .iter() + .map(|child| _annotate_plan(Arc::clone(child), Some(&plan), cfg, false)) + .collect::, _>>()?; + + if plan.children().is_empty() { + // This is a leaf node, maybe a DataSourceExec, or maybe something else custom from the + // user. We need to estimate how many tasks are needed for this leaf node, and we'll take + // this decision into account when deciding how many tasks will be actually used. + return if let Some(estimate) = estimator.task_estimation(&plan, cfg) { + Ok(AnnotatedPlan { + plan_or_nb: PlanOrNetworkBoundary::Plan(plan), + children: Vec::new(), + task_count: estimate.task_count.limit(n_workers), + }) + } else { + // We could not determine how many tasks this leaf node should run on, so + // assume it cannot be distributed and use just 1 task. + Ok(AnnotatedPlan { + plan_or_nb: PlanOrNetworkBoundary::Plan(plan), + children: Vec::new(), + task_count: Maximum(1), + }) + }; + } + + let mut task_count = estimator + .task_estimation(&plan, cfg) + .map_or(Desired(1), |v| v.task_count); + if d_cfg.children_isolator_unions && plan.as_any().is::() { + // Unions have the chance to decide how many tasks they should run on. If there's a union + // with a bunch of children, the user might want to increase parallelism and increase the + // task count for the stage running that. + let mut count = 0; + for annotated_child in annotated_children.iter() { + count += annotated_child.task_count.as_usize(); + } + task_count = Desired(count); + } else if let Some(node) = plan.as_any().downcast_ref::() + && node.mode == PartitionMode::CollectLeft + && !broadcast_joins + { + // Only distriubte CollectLeft HashJoins after we broadcast more intelligently or when it + // is explicitly enabled. + task_count = Maximum(1); + } else { + // The task count for this plan is decided by the biggest task count from the children; unless + // a child specifies a maximum task count, in that case, the maximum is respected. Some + // nodes can only run in one task. If there is a subplan with a single node declaring that + // it can only run in one task, all the rest of the nodes in the stage need to respect it. + for annotated_child in annotated_children.iter() { + task_count = match (task_count, &annotated_child.task_count) { + (Desired(desired), Desired(child)) => Desired(desired.max(*child)), + (Maximum(max), Desired(_)) => Maximum(max), + (Desired(_), Maximum(max)) => Maximum(*max), + (Maximum(max_1), Maximum(max_2)) => Maximum(max_1.min(*max_2)), + }; + } + } + + task_count = task_count.limit(n_workers); + + // Wrap the node with a boundary node if the parent marks it. + let mut annotation = AnnotatedPlan { + plan_or_nb: PlanOrNetworkBoundary::Plan(Arc::clone(&plan)), + children: annotated_children, + task_count: task_count.clone(), + }; + + // Upon reaching a hash repartition, we need to introduce a shuffle right above it. + if let Some(r_exec) = plan.as_any().downcast_ref::() { + if matches!(r_exec.partitioning(), Partitioning::Hash(_, _)) { + annotation = AnnotatedPlan { + plan_or_nb: PlanOrNetworkBoundary::Shuffle, + children: vec![annotation], + task_count, + }; + } + } else if let Some(parent) = parent + // If this node is a leaf node, putting a network boundary above is a bit wasteful, so + // we don't want to do it. + && !plan.children().is_empty() + // If the parent is trying to coalesce all partitions into one, we need to introduce + // a network coalesce right below it (or in other words, above the current node) + && (parent.as_any().is::() + || parent.as_any().is::()) + { + // A BroadcastExec underneath a coalesce parent means the build side will cross stages. + if plan.as_any().is::() { + annotation = AnnotatedPlan { + plan_or_nb: PlanOrNetworkBoundary::Broadcast, + children: vec![annotation], + task_count, + }; + } else { + annotation = AnnotatedPlan { + plan_or_nb: PlanOrNetworkBoundary::Coalesce, + children: vec![annotation], + task_count, + }; + } + } + + // The plan needs a NetworkBoundary. At this point we have all the info we need for choosing + // the right size for the stage below, so what we need to do is take the calculated final + // task count and propagate to all the children that will eventually be part of the stage. + fn propagate_task_count( + annotation: &mut AnnotatedPlan, + task_count: &TaskCountAnnotation, + d_cfg: &DistributedConfig, + ) -> Result<(), DataFusionError> { + annotation.task_count = task_count.clone(); + let plan = match &annotation.plan_or_nb { + // If it's a normal plan, continue with the propagation. + PlanOrNetworkBoundary::Plan(plan) => plan, + // Broadcast is a stage split only propagate a Maximum cap into the build stage. + PlanOrNetworkBoundary::Broadcast => { + if let Maximum(max) = task_count { + for child in annotation.children.iter_mut() { + let child_task_count = child.task_count.clone().limit(*max); + propagate_task_count(child, &child_task_count, d_cfg)?; + } + } + return Ok(()); + } + // This is a network boundary. + // + // Nothing to propagate here, all the nodes below the network boundary were already + // assigned a task count, we do not want to overwrite it. + PlanOrNetworkBoundary::Shuffle => return Ok(()), + PlanOrNetworkBoundary::Coalesce => return Ok(()), + }; + + if d_cfg.children_isolator_unions && plan.as_any().is::() { + // Propagating through ChildrenIsolatorUnionExec is not that easy, each child will + // be executed in its own task, and therefore, they will act as if they were in executing + // in a non-distributed context. The ChildrenIsolatorUnionExec itself will make sure to + // determine which children to run and which to exclude depending on the task index in + // which it's running. + let c_i_union = ChildrenIsolatorUnionExec::from_children_and_task_counts( + plan.children().into_iter().cloned(), + annotation.children.iter().map(|v| v.task_count.as_usize()), + task_count.as_usize(), + )?; + for children_and_tasks in c_i_union.task_idx_map.iter() { + for (child_i, task_ctx) in children_and_tasks { + if let Some(child) = annotation.children.get_mut(*child_i) { + propagate_task_count(child, &Maximum(task_ctx.task_count), d_cfg)? + }; + } + } + annotation.plan_or_nb = PlanOrNetworkBoundary::Plan(Arc::new(c_i_union)); + } else { + for child in &mut annotation.children { + propagate_task_count(child, task_count, d_cfg)?; + } + } + Ok(()) + } + + if annotation.plan_or_nb.is_network_boundary() { + // The plan is a network boundary, so everything below it belongs to the same stage. This + // means that we need to propagate the task count to all the nodes in that stage. + for annotated_child in annotation.children.iter_mut() { + propagate_task_count(annotated_child, &annotation.task_count, d_cfg)?; + } + + // If the current plan that needs a NetworkBoundary boundary below is either a + // CoalescePartitionsExec or a SortPreservingMergeExec, then we are sure that all the stage + // that they are going to be part of needs to run in exactly one task. + if matches!(annotation.plan_or_nb, PlanOrNetworkBoundary::Coalesce) { + annotation.task_count = Maximum(1); + return Ok(annotation); + } + + // From now and up in the plan, a new task count needs to be calculated for the next stage. + // Depending on the number of nodes that reduce/increase cardinality, the task count will be + // calculated based on the previous task count multiplied by a factor. + fn calculate_scale_factor(annotation: &AnnotatedPlan, f: f64) -> f64 { + let PlanOrNetworkBoundary::Plan(plan) = &annotation.plan_or_nb else { + return 1.0; + }; + + let mut sf = None; + for plan in &annotation.children { + sf = match sf { + None => Some(calculate_scale_factor(plan, f)), + Some(sf) => Some(sf.max(calculate_scale_factor(plan, f))), + } + } + + let sf = sf.unwrap_or(1.0); + match plan.cardinality_effect() { + CardinalityEffect::LowerEqual => sf / f, + CardinalityEffect::GreaterEqual => sf * f, + _ => sf, + } + } + let sf = calculate_scale_factor( + annotation.children.first().ok_or_else(|| { + plan_datafusion_err!("missing child in a plan annotated with a network boundary") + })?, + d_cfg.cardinality_task_count_factor, + ); + let prev_task_count = annotation.task_count.as_usize() as f64; + annotation.task_count = Desired((prev_task_count * sf).ceil() as usize); + Ok(annotation) + } else if root { + // If this is the root node, it means that we have just finished annotating nodes for the + // subplan belonging to the head stage, so propagate the task count to all children. + let task_count = annotation.task_count.clone(); + propagate_task_count(&mut annotation, &task_count, d_cfg)?; + Ok(annotation) + } else { + // If this is not the root node, and it's also not a network boundary, then we don't need + // to do anything else. + Ok(annotation) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::distributed_planner::insert_broadcast::insert_broadcast_execs; + use crate::test_utils::plans::{ + BuildSideOneTaskEstimator, TestPlanOptions, base_session_builder, context_with_query, + sql_to_physical_plan, + }; + use crate::{DistributedExt, TaskEstimation, TaskEstimator, assert_snapshot}; + use datafusion::config::ConfigOptions; + use datafusion::execution::SessionStateBuilder; + use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec; + use datafusion::physical_plan::filter::FilterExec; + /* schema for the "weather" table + + MinTemp [type=DOUBLE] [repetitiontype=OPTIONAL] + MaxTemp [type=DOUBLE] [repetitiontype=OPTIONAL] + Rainfall [type=DOUBLE] [repetitiontype=OPTIONAL] + Evaporation [type=DOUBLE] [repetitiontype=OPTIONAL] + Sunshine [type=BYTE_ARRAY] [convertedtype=UTF8] [repetitiontype=OPTIONAL] + WindGustDir [type=BYTE_ARRAY] [convertedtype=UTF8] [repetitiontype=OPTIONAL] + WindGustSpeed [type=BYTE_ARRAY] [convertedtype=UTF8] [repetitiontype=OPTIONAL] + WindDir9am [type=BYTE_ARRAY] [convertedtype=UTF8] [repetitiontype=OPTIONAL] + WindDir3pm [type=BYTE_ARRAY] [convertedtype=UTF8] [repetitiontype=OPTIONAL] + WindSpeed9am [type=BYTE_ARRAY] [convertedtype=UTF8] [repetitiontype=OPTIONAL] + WindSpeed3pm [type=INT64] [convertedtype=INT_64] [repetitiontype=OPTIONAL] + Humidity9am [type=INT64] [convertedtype=INT_64] [repetitiontype=OPTIONAL] + Humidity3pm [type=INT64] [convertedtype=INT_64] [repetitiontype=OPTIONAL] + Pressure9am [type=DOUBLE] [repetitiontype=OPTIONAL] + Pressure3pm [type=DOUBLE] [repetitiontype=OPTIONAL] + Cloud9am [type=INT64] [convertedtype=INT_64] [repetitiontype=OPTIONAL] + Cloud3pm [type=INT64] [convertedtype=INT_64] [repetitiontype=OPTIONAL] + Temp9am [type=DOUBLE] [repetitiontype=OPTIONAL] + Temp3pm [type=DOUBLE] [repetitiontype=OPTIONAL] + RainToday [type=BYTE_ARRAY] [convertedtype=UTF8] [repetitiontype=OPTIONAL] + RISK_MM [type=DOUBLE] [repetitiontype=OPTIONAL] + RainTomorrow [type=BYTE_ARRAY] [convertedtype=UTF8] [repetitiontype=OPTIONAL] + */ + + #[tokio::test] + async fn test_select_all() { + let query = r#" + SELECT * FROM weather + "#; + let annotated = sql_to_annotated(query).await; + assert_snapshot!(annotated, @"DataSourceExec: task_count=Desired(3)") + } + + #[tokio::test] + async fn test_aggregation() { + let query = r#" + SELECT count(*), "RainToday" FROM weather GROUP BY "RainToday" ORDER BY count(*) + "#; + let annotated = sql_to_annotated(query).await; + assert_snapshot!(annotated, @r" + ProjectionExec: task_count=Maximum(1) + SortPreservingMergeExec: task_count=Maximum(1) + [NetworkBoundary] Coalesce: task_count=Maximum(1) + SortExec: task_count=Desired(2) + ProjectionExec: task_count=Desired(2) + AggregateExec: task_count=Desired(2) + [NetworkBoundary] Shuffle: task_count=Desired(2) + RepartitionExec: task_count=Desired(3) + AggregateExec: task_count=Desired(3) + DataSourceExec: task_count=Desired(3) + ") + } + + #[tokio::test] + async fn test_left_join() { + let query = r#" + SELECT a."MinTemp", b."MaxTemp" FROM weather a LEFT JOIN weather b ON a."RainToday" = b."RainToday" + "#; + let annotated = sql_to_annotated(query).await; + assert_snapshot!(annotated, @r" + HashJoinExec: task_count=Maximum(1) + CoalescePartitionsExec: task_count=Maximum(1) + DataSourceExec: task_count=Maximum(1) + DataSourceExec: task_count=Maximum(1) + ") + } + + #[tokio::test] + async fn test_left_join_distributed() { + let query = r#" + WITH a AS ( + SELECT + AVG("MinTemp") as "MinTemp", + "RainTomorrow" + FROM weather + WHERE "RainToday" = 'yes' + GROUP BY "RainTomorrow" + ), b AS ( + SELECT + AVG("MaxTemp") as "MaxTemp", + "RainTomorrow" + FROM weather + WHERE "RainToday" = 'no' + GROUP BY "RainTomorrow" + ) + SELECT + a."MinTemp", + b."MaxTemp" + FROM a + LEFT JOIN b + ON a."RainTomorrow" = b."RainTomorrow" + "#; + let annotated = sql_to_annotated(query).await; + assert_snapshot!(annotated, @r" + HashJoinExec: task_count=Maximum(1) + CoalescePartitionsExec: task_count=Maximum(1) + [NetworkBoundary] Coalesce: task_count=Maximum(1) + ProjectionExec: task_count=Desired(2) + AggregateExec: task_count=Desired(2) + [NetworkBoundary] Shuffle: task_count=Desired(2) + RepartitionExec: task_count=Desired(3) + AggregateExec: task_count=Desired(3) + FilterExec: task_count=Desired(3) + RepartitionExec: task_count=Desired(3) + DataSourceExec: task_count=Desired(3) + ProjectionExec: task_count=Maximum(1) + AggregateExec: task_count=Maximum(1) + [NetworkBoundary] Shuffle: task_count=Maximum(1) + RepartitionExec: task_count=Desired(3) + AggregateExec: task_count=Desired(3) + FilterExec: task_count=Desired(3) + RepartitionExec: task_count=Desired(3) + DataSourceExec: task_count=Desired(3) + ") + } + + // TODO: should be changed once broadcasting is done more intelligently and not behind a + // feature flag. + #[tokio::test] + async fn test_inner_join() { + let query = r#" + SELECT a."MinTemp", b."MaxTemp" FROM weather a INNER JOIN weather b ON a."RainToday" = b."RainToday" + "#; + let annotated = sql_to_annotated(query).await; + assert_snapshot!(annotated, @r" + HashJoinExec: task_count=Maximum(1) + CoalescePartitionsExec: task_count=Maximum(1) + DataSourceExec: task_count=Maximum(1) + DataSourceExec: task_count=Maximum(1) + ") + } + + #[tokio::test] + async fn test_distinct() { + let query = r#" + SELECT DISTINCT "RainToday" FROM weather + "#; + let annotated = sql_to_annotated(query).await; + assert_snapshot!(annotated, @r" + AggregateExec: task_count=Desired(2) + [NetworkBoundary] Shuffle: task_count=Desired(2) + RepartitionExec: task_count=Desired(3) + AggregateExec: task_count=Desired(3) + DataSourceExec: task_count=Desired(3) + ") + } + + #[tokio::test] + async fn test_union_all() { + let query = r#" + SELECT "MinTemp" FROM weather WHERE "RainToday" = 'yes' + UNION ALL + SELECT "MaxTemp" FROM weather WHERE "RainToday" = 'no' + "#; + let annotated = sql_to_annotated(query).await; + assert_snapshot!(annotated, @r" + ChildrenIsolatorUnionExec: task_count=Desired(4) + FilterExec: task_count=Maximum(2) + RepartitionExec: task_count=Maximum(2) + DataSourceExec: task_count=Maximum(2) + ProjectionExec: task_count=Maximum(2) + FilterExec: task_count=Maximum(2) + RepartitionExec: task_count=Maximum(2) + DataSourceExec: task_count=Maximum(2) + ") + } + + #[tokio::test] + async fn test_subquery() { + let query = r#" + SELECT * FROM ( + SELECT "MinTemp", "MaxTemp" FROM weather WHERE "RainToday" = 'yes' + ) AS subquery WHERE "MinTemp" > 5 + "#; + let annotated = sql_to_annotated(query).await; + assert_snapshot!(annotated, @r" + FilterExec: task_count=Desired(3) + RepartitionExec: task_count=Desired(3) + DataSourceExec: task_count=Desired(3) + ") + } + + #[tokio::test] + async fn test_window_function() { + let query = r#" + SELECT "MinTemp", ROW_NUMBER() OVER (PARTITION BY "RainToday" ORDER BY "MinTemp") as rn + FROM weather + "#; + let annotated = sql_to_annotated(query).await; + assert_snapshot!(annotated, @r" + ProjectionExec: task_count=Desired(3) + BoundedWindowAggExec: task_count=Desired(3) + SortExec: task_count=Desired(3) + [NetworkBoundary] Shuffle: task_count=Desired(3) + RepartitionExec: task_count=Desired(3) + DataSourceExec: task_count=Desired(3) + ") + } + + #[tokio::test] + async fn test_children_isolator_union() { + let query = r#" + SET distributed.children_isolator_unions = true; + SET distributed.files_per_task = 1; + SELECT "MinTemp" FROM weather WHERE "RainToday" = 'yes' + UNION ALL + SELECT "MaxTemp" FROM weather WHERE "RainToday" = 'no' + UNION ALL + SELECT "Rainfall" FROM weather WHERE "RainTomorrow" = 'yes' + "#; + let annotated = sql_to_annotated(query).await; + assert_snapshot!(annotated, @r" + ChildrenIsolatorUnionExec: task_count=Desired(4) + FilterExec: task_count=Maximum(1) + RepartitionExec: task_count=Maximum(1) + DataSourceExec: task_count=Maximum(1) + ProjectionExec: task_count=Maximum(1) + FilterExec: task_count=Maximum(1) + RepartitionExec: task_count=Maximum(1) + DataSourceExec: task_count=Maximum(1) + ProjectionExec: task_count=Maximum(2) + FilterExec: task_count=Maximum(2) + RepartitionExec: task_count=Maximum(2) + DataSourceExec: task_count=Maximum(2) + ") + } + + #[tokio::test] + async fn test_intermediate_task_estimator() { + let query = r#" + SELECT DISTINCT "RainToday" FROM weather + "#; + let annotated = sql_to_annotated_with_estimator(query, |_: &RepartitionExec| { + Some(TaskEstimation::maximum(1)) + }) + .await; + assert_snapshot!(annotated, @r" + AggregateExec: task_count=Desired(1) + [NetworkBoundary] Shuffle: task_count=Desired(1) + RepartitionExec: task_count=Maximum(1) + AggregateExec: task_count=Maximum(1) + DataSourceExec: task_count=Maximum(1) + ") + } + + #[tokio::test] + async fn test_union_all_limited_by_intermediate_estimator() { + let query = r#" + SELECT "MinTemp" FROM weather WHERE "RainToday" = 'yes' + UNION ALL + SELECT "MaxTemp" FROM weather WHERE "RainToday" = 'no' + "#; + let annotated = sql_to_annotated_with_estimator(query, |_: &FilterExec| { + Some(TaskEstimation::maximum(1)) + }) + .await; + assert_snapshot!(annotated, @r" + ChildrenIsolatorUnionExec: task_count=Desired(2) + FilterExec: task_count=Maximum(1) + RepartitionExec: task_count=Maximum(1) + DataSourceExec: task_count=Maximum(1) + ProjectionExec: task_count=Maximum(1) + FilterExec: task_count=Maximum(1) + RepartitionExec: task_count=Maximum(1) + DataSourceExec: task_count=Maximum(1) + ") + } + + #[tokio::test] + async fn test_broadcast_join_annotation() { + let query = r#" + SELECT a."MinTemp", b."MaxTemp" + FROM weather a INNER JOIN weather b + ON a."RainToday" = b."RainToday" + "#; + let annotated = sql_to_annotated_broadcast(query, 4, 4, true).await; + assert_snapshot!(annotated, @r" + HashJoinExec: task_count=Desired(3) + CoalescePartitionsExec: task_count=Desired(3) + [NetworkBoundary] Broadcast: task_count=Desired(3) + BroadcastExec: task_count=Desired(3) + DataSourceExec: task_count=Desired(3) + DataSourceExec: task_count=Desired(3) + ") + } + + #[tokio::test] + async fn test_broadcast_datasource_as_build_child() { + let query = r#" + SELECT a."MinTemp", b."MaxTemp" + FROM weather a INNER JOIN weather b + ON a."RainToday" = b."RainToday" + "#; + + // Check physical plan before insertion, shouldn't have CoalescePartitionsExec + let physical_plan = sql_to_physical_plan(query, 1, 4).await; + assert_snapshot!(physical_plan, @r" + HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(RainToday@1, RainToday@1)], projection=[MinTemp@0, MaxTemp@2] + DataSourceExec: file_groups={1 group: [[/testdata/weather/result-000000.parquet, /testdata/weather/result-000001.parquet, /testdata/weather/result-000002.parquet]]}, projection=[MinTemp, RainToday], file_type=parquet + DataSourceExec: file_groups={1 group: [[/testdata/weather/result-000000.parquet, /testdata/weather/result-000001.parquet, /testdata/weather/result-000002.parquet]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=DynamicFilter [ empty ] + "); + + // With target_partitions=1, there is no CoalescePartitionsExec initially + // With broadcast, should create one and insert BroadcastExec below it + let annotated = sql_to_annotated_broadcast(query, 1, 4, true).await; + assert!(annotated.contains("Broadcast")); + assert_snapshot!(annotated, @r" + HashJoinExec: task_count=Desired(3) + CoalescePartitionsExec: task_count=Desired(3) + [NetworkBoundary] Broadcast: task_count=Desired(3) + BroadcastExec: task_count=Desired(3) + DataSourceExec: task_count=Desired(3) + DataSourceExec: task_count=Desired(3) + "); + } + + #[tokio::test] + async fn test_broadcast_one_to_many() { + let query = r#" + SELECT a."MinTemp", b."MaxTemp" + FROM weather a INNER JOIN weather b + ON a."RainToday" = b."RainToday" + "#; + let annotated = + sql_to_annotated_broadcast_with_estimator(query, 3, BuildSideOneTaskEstimator).await; + assert_snapshot!(annotated, @r" + HashJoinExec: task_count=Desired(3) + CoalescePartitionsExec: task_count=Desired(3) + [NetworkBoundary] Broadcast: task_count=Desired(3) + BroadcastExec: task_count=Maximum(1) + DataSourceExec: task_count=Maximum(1) + DataSourceExec: task_count=Desired(3) + "); + } + + #[tokio::test] + async fn test_broadcast_build_coalesce_caps_join_stage() { + let query = r#" + SELECT a."MinTemp", b."MaxTemp" + FROM weather a INNER JOIN weather b + ON a."RainToday" = b."RainToday" + "#; + let annotated = + sql_to_annotated_broadcast_with_estimator(query, 3, BroadcastBuildCoalesceMaxEstimator) + .await; + assert_snapshot!(annotated, @r" + HashJoinExec: task_count=Maximum(1) + CoalescePartitionsExec: task_count=Maximum(1) + [NetworkBoundary] Broadcast: task_count=Maximum(1) + BroadcastExec: task_count=Desired(1) + DataSourceExec: task_count=Desired(1) + DataSourceExec: task_count=Maximum(1) + "); + } + + #[tokio::test] + async fn test_broadcast_disabled_default() { + let query = r#" + SELECT a."MinTemp", b."MaxTemp" + FROM weather a INNER JOIN weather b + ON a."RainToday" = b."RainToday" + "#; + let annotated = sql_to_annotated_broadcast(query, 4, 4, false).await; + // With broadcast disabled, no broadcast annotation should appear + assert!(!annotated.contains("Broadcast")); + assert_snapshot!(annotated, @r" + HashJoinExec: task_count=Maximum(1) + CoalescePartitionsExec: task_count=Maximum(1) + DataSourceExec: task_count=Maximum(1) + DataSourceExec: task_count=Maximum(1) + ") + } + + #[tokio::test] + async fn test_broadcast_multi_join_chain() { + let query = r#" + SELECT a."MinTemp", b."MaxTemp", c."Rainfall" + FROM weather a + INNER JOIN weather b ON a."RainToday" = b."RainToday" + INNER JOIN weather c ON b."RainToday" = c."RainToday" + "#; + let annotated = sql_to_annotated_broadcast(query, 4, 4, true).await; + assert_snapshot!(annotated, @r" + HashJoinExec: task_count=Desired(3) + CoalescePartitionsExec: task_count=Desired(3) + [NetworkBoundary] Broadcast: task_count=Desired(3) + BroadcastExec: task_count=Desired(3) + HashJoinExec: task_count=Desired(3) + CoalescePartitionsExec: task_count=Desired(3) + [NetworkBoundary] Broadcast: task_count=Desired(3) + BroadcastExec: task_count=Desired(3) + DataSourceExec: task_count=Desired(3) + DataSourceExec: task_count=Desired(3) + DataSourceExec: task_count=Desired(3) + ") + } + + #[tokio::test] + async fn test_broadcast_union_children_isolator_annotation() { + let query = r#" + SET distributed.children_isolator_unions = true; + SELECT a."MinTemp", b."MaxTemp" + FROM weather a INNER JOIN weather b + ON a."RainToday" = b."RainToday" + UNION ALL + SELECT a."MinTemp", b."MaxTemp" + FROM weather a INNER JOIN weather b + ON a."RainToday" = b."RainToday" + UNION ALL + SELECT a."MinTemp", b."MaxTemp" + FROM weather a INNER JOIN weather b + ON a."RainToday" = b."RainToday" + "#; + let annotated = sql_to_annotated_broadcast(query, 4, 4, true).await; + // With ChildrenIsolatorUnionExec, each broadcast task_count should be limited to their + // context. + assert_snapshot!(annotated, @r" + ChildrenIsolatorUnionExec: task_count=Desired(4) + HashJoinExec: task_count=Maximum(1) + CoalescePartitionsExec: task_count=Maximum(1) + [NetworkBoundary] Broadcast: task_count=Maximum(1) + BroadcastExec: task_count=Desired(1) + DataSourceExec: task_count=Desired(1) + DataSourceExec: task_count=Maximum(1) + HashJoinExec: task_count=Maximum(1) + CoalescePartitionsExec: task_count=Maximum(1) + [NetworkBoundary] Broadcast: task_count=Maximum(1) + BroadcastExec: task_count=Desired(1) + DataSourceExec: task_count=Desired(1) + DataSourceExec: task_count=Maximum(1) + HashJoinExec: task_count=Maximum(2) + CoalescePartitionsExec: task_count=Maximum(2) + [NetworkBoundary] Broadcast: task_count=Maximum(2) + BroadcastExec: task_count=Desired(2) + DataSourceExec: task_count=Desired(2) + DataSourceExec: task_count=Maximum(2) + "); + } + + #[allow(clippy::type_complexity)] + struct CallbackEstimator { + f: Arc Option + Send + Sync>, + } + + impl CallbackEstimator { + fn new( + f: impl Fn(&T) -> Option + Send + Sync + 'static, + ) -> Self { + let f = Arc::new(move |plan: &dyn ExecutionPlan| -> Option { + if let Some(plan) = plan.as_any().downcast_ref::() { + f(plan) + } else { + None + } + }); + Self { f } + } + } + + impl TaskEstimator for CallbackEstimator { + fn task_estimation( + &self, + plan: &Arc, + _: &ConfigOptions, + ) -> Option { + (self.f)(plan.as_ref()) + } + + fn scale_up_leaf_node( + &self, + _: &Arc, + _: usize, + _: &ConfigOptions, + ) -> Option> { + None + } + } + + #[derive(Debug)] + struct BroadcastBuildCoalesceMaxEstimator; + + impl TaskEstimator for BroadcastBuildCoalesceMaxEstimator { + fn task_estimation( + &self, + plan: &Arc, + _: &ConfigOptions, + ) -> Option { + let coalesce = plan.as_any().downcast_ref::()?; + if coalesce.input().as_any().is::() { + Some(TaskEstimation::maximum(1)) + } else { + None + } + } + + fn scale_up_leaf_node( + &self, + _: &Arc, + _: usize, + _: &ConfigOptions, + ) -> Option> { + None + } + } + + async fn sql_to_annotated(query: &str) -> String { + annotate_test_plan(query, TestPlanOptions::default(), |b| b).await + } + + async fn sql_to_annotated_broadcast( + query: &str, + target_partitions: usize, + num_workers: usize, + broadcast_enabled: bool, + ) -> String { + let options = TestPlanOptions { + target_partitions, + num_workers, + broadcast_enabled, + }; + annotate_test_plan(query, options, |b| b).await + } + + async fn sql_to_annotated_with_estimator( + query: &str, + estimator: impl Fn(&T) -> Option + Send + Sync + 'static, + ) -> String { + let options = TestPlanOptions::default(); + annotate_test_plan(query, options, |b| { + b.with_distributed_task_estimator(CallbackEstimator::new(estimator)) + }) + .await + } + + async fn sql_to_annotated_broadcast_with_estimator( + query: &str, + num_workers: usize, + estimator: impl TaskEstimator + Send + Sync + 'static, + ) -> String { + let options = TestPlanOptions { + target_partitions: 4, + num_workers, + broadcast_enabled: true, + }; + annotate_test_plan(query, options, |b| { + b.with_distributed_task_estimator(estimator) + }) + .await + } + + async fn annotate_test_plan( + query: &str, + options: TestPlanOptions, + configure: impl FnOnce(SessionStateBuilder) -> SessionStateBuilder, + ) -> String { + let builder = base_session_builder( + options.target_partitions, + options.num_workers, + options.broadcast_enabled, + ); + let builder = configure(builder); + let (ctx, query) = context_with_query(builder, query).await; + let df = ctx.sql(&query).await.unwrap(); + let mut plan = df.create_physical_plan().await.unwrap(); + + plan = insert_broadcast_execs(plan, ctx.state_ref().read().config_options().as_ref()) + .expect("failed to insert broadcasts"); + + let annotated = annotate_plan(plan, ctx.state_ref().read().config_options().as_ref()) + .expect("failed to annotate plan"); + format!("{annotated:?}") + } +} diff --git a/src/distributed_planner/task_estimator.rs b/src/distributed_planner/task_estimator.rs new file mode 100644 index 00000000..ae22ae0a --- /dev/null +++ b/src/distributed_planner/task_estimator.rs @@ -0,0 +1,404 @@ +use crate::config_extension_ext::set_distributed_option_extension; +use crate::{DistributedConfig, PartitionIsolatorExec}; +use datafusion::catalog::memory::DataSourceExec; +use datafusion::config::ConfigOptions; +use datafusion::datasource::physical_plan::FileScanConfig; +use datafusion::physical_plan::ExecutionPlan; +use datafusion::prelude::SessionConfig; +use delegate::delegate; +use std::fmt::Debug; +use std::sync::Arc; + +/// Annotation attached to a single [ExecutionPlan] that determines how many distributed tasks +/// it should run on. +#[derive(Debug, Clone)] +pub enum TaskCountAnnotation { + /// The desired number of distributed tasks for this node. The final task count for the + /// annotated node might not be exactly this number, it is more like a hint, so depending + /// on the desired task count of adjacent nodes, the final task count might change. + Desired(usize), + /// Sets a maximum number of distributed tasks for this node. Typically used with the inner + /// value of 1, stating that this node cannot be executed in a distributed fashion. + Maximum(usize), +} + +impl From for usize { + fn from(annotation: TaskCountAnnotation) -> Self { + annotation.as_usize() + } +} + +impl TaskCountAnnotation { + pub fn as_usize(&self) -> usize { + match self { + Self::Desired(desired) => *desired, + Self::Maximum(maximum) => *maximum, + } + } + + pub(crate) fn limit(self, limit: usize) -> Self { + match self { + Self::Desired(desired) => Self::Desired(desired.min(limit)), + Self::Maximum(maximum) => Self::Maximum(maximum.min(limit)), + } + } +} + +/// Result of running a [TaskEstimator] on a leaf node. It tells the distributed planner hints +/// about how many tasks should be used in [Stage]s that contain leaf nodes. +pub struct TaskEstimation { + /// The number of tasks that should be used in the [Stage] containing the leaf node. + /// + /// Even if implementations get to decide this number, there are situations where it can + /// get overridden: + /// - If a [Stage] contains multiple leaf nodes, the one that declares the biggest + /// task_count wins. + /// - If there are less available workers than this number, the number of available workers + /// is chosen. + pub task_count: TaskCountAnnotation, +} + +impl TaskEstimation { + /// Tells the distributed planner that the evaluated stage can have **at maximum** the provided + /// number of tasks, setting a hard upper limit. + /// + /// Returning `TaskEstimation::maximum(1)` tells the distributed planner that the evaluated + /// stage cannot be distributed. + /// + /// Even if a `TaskEstimation::maximum(N)` is provided, any other node in the same stage + /// providing a value of `TaskEstimation::maximum(M)` where `M` < `N` will have preference. + pub fn maximum(value: usize) -> Self { + TaskEstimation { + task_count: TaskCountAnnotation::Maximum(value), + } + } + + /// Tells the distributed planner that the evaluated can **optimally** have the provided + /// number of tasks, setting a soft task count hint that can be overridden by others. + /// + /// The provided `TaskEstimation::desired(N)` can be overridden by: + /// - Other nodes providing a `TaskEstimation::desired(M)` where `M` > `N`. + /// - Any other node providing a `TaskEstimation::maximum(M)` where `M` can be anything. + pub fn desired(value: usize) -> Self { + TaskEstimation { + task_count: TaskCountAnnotation::Desired(value), + } + } +} + +/// Given a leaf node, provides an estimation about how many tasks should be used in the +/// stage containing it, and if the leaf node should be replaced by some other. +/// +/// The distributed planner will try many [TaskEstimator]s in order until one provides an +/// estimation for a specific leaf node. Once that's done, upper stages will get their task +/// count calculated based on whether lower stages are reducing the cardinality of the data +/// or increasing it. +pub trait TaskEstimator { + /// Function applied to each node that returns a [TaskEstimation] hinting how many + /// tasks should be used in the [Stage] containing that node. + /// + /// All the [TaskEstimator] registered in the session will be applied to the node + /// until one returns an estimation. + /// + /// + /// If no estimation is returned from any of the registered [TaskEstimator]s, then: + /// - If the node is a leaf node,`Maximum(1)` is assumed, hinting the distributed planner + /// that the leaf node cannot be distributed across tasks. + /// - If the node is a normal node in the plan, then the maximum task count from its children + /// is inherited. + fn task_estimation( + &self, + plan: &Arc, + cfg: &ConfigOptions, + ) -> Option; + + /// After a final task_count is decided, taking into account all the leaf nodes in the [Stage], + /// this allows performing a transformation in the leaf nodes for accounting for the fact that + /// they are going to run in multiple tasks. + fn scale_up_leaf_node( + &self, + plan: &Arc, + task_count: usize, + cfg: &ConfigOptions, + ) -> Option>; +} + +impl TaskEstimator for usize { + fn task_estimation( + &self, + inputs: &Arc, + _: &ConfigOptions, + ) -> Option { + if inputs.children().is_empty() { + Some(TaskEstimation { + task_count: TaskCountAnnotation::Desired(*self), + }) + } else { + None + } + } + + fn scale_up_leaf_node( + &self, + _: &Arc, + _: usize, + _: &ConfigOptions, + ) -> Option> { + None + } +} + +impl TaskEstimator for Arc { + delegate! { + to self.as_ref() { + fn task_estimation(&self, plan: &Arc, cfg: &ConfigOptions) -> Option; + fn scale_up_leaf_node(&self, plan: &Arc, task_count: usize, cfg: &ConfigOptions) -> Option>; + } + } +} + +impl TaskEstimator for Arc { + delegate! { + to self.as_ref() { + fn task_estimation(&self, plan: &Arc, cfg: &ConfigOptions) -> Option; + fn scale_up_leaf_node(&self, plan: &Arc, task_count: usize, cfg: &ConfigOptions) -> Option>; + } + } +} + +pub(crate) fn set_distributed_task_estimator( + cfg: &mut SessionConfig, + estimator: impl TaskEstimator + Send + Sync + 'static, +) { + let opts = cfg.options_mut(); + if let Some(distributed_cfg) = opts.extensions.get_mut::() { + distributed_cfg + .__private_task_estimator + .user_provided + .push(Arc::new(estimator)); + } else { + let mut estimators = CombinedTaskEstimator::default(); + estimators.user_provided.push(Arc::new(estimator)); + set_distributed_option_extension( + cfg, + DistributedConfig { + __private_task_estimator: estimators, + ..Default::default() + }, + ) + } +} + +/// [TaskEstimator] implementation that acts on [DataSourceExec] nodes that contain +/// [FileScanConfig]s data sources (e.g., Parquet or CSV files). it will read the +/// [DistributedConfig].`files_per_task` field and assigns as many tasks as needed so that +/// no task handles more than the configured files. +#[derive(Debug)] +struct FileScanConfigTaskEstimator; + +impl TaskEstimator for FileScanConfigTaskEstimator { + fn task_estimation( + &self, + plan: &Arc, + cfg: &ConfigOptions, + ) -> Option { + let dse: &DataSourceExec = plan.as_any().downcast_ref()?; + let file_scan: &FileScanConfig = dse.data_source().as_any().downcast_ref()?; + + let d_cfg = cfg.extensions.get::()?; + + // Count how many partitioned files we have in the FileScanConfig. + let mut partitioned_files = 0; + for file_group in &file_scan.file_groups { + partitioned_files += file_group.len(); + } + + // Based on the user-provided files_per_task configuration, do the math to calculate + // how many tasks should be used, without surpassing the number of available workers. + let task_count = partitioned_files.div_ceil(d_cfg.files_per_task); + + Some(TaskEstimation { + task_count: TaskCountAnnotation::Desired(task_count), + }) + } + + fn scale_up_leaf_node( + &self, + plan: &Arc, + task_count: usize, + _cfg: &ConfigOptions, + ) -> Option> { + if task_count == 1 { + return Some(Arc::clone(plan)); + } + // Based on the task count, attempt to scale up the partitions in the DataSourceExec by + // repartitioning it. This will result in a DataSourceExec with potentially a lot of + // partitions, but as we are going to wrap it with PartitionIsolatorExec, that's fine. + let dse: &DataSourceExec = plan.as_any().downcast_ref()?; + let file_scan: &FileScanConfig = dse.data_source().as_any().downcast_ref()?; + + let mut new_file_scan = file_scan.clone(); + new_file_scan.file_groups.clear(); + for file_group in file_scan.file_groups.clone() { + new_file_scan + .file_groups + .extend(file_group.split_files(task_count)); + } + let plan = DataSourceExec::from_data_source(new_file_scan); + Some(Arc::new(PartitionIsolatorExec::new(plan, task_count))) + } +} + +/// Tries multiple user-provided [TaskEstimator]s until one returns an estimation. If none +/// returns an estimation, a set of default [TaskEstimation] implementations is tried. Right +/// now the only default [TaskEstimation] is [FileScanConfigTaskEstimator]. +#[derive(Clone, Default)] +pub(crate) struct CombinedTaskEstimator { + pub(crate) user_provided: Vec>, +} + +impl TaskEstimator for CombinedTaskEstimator { + fn task_estimation( + &self, + plan: &Arc, + cfg: &ConfigOptions, + ) -> Option { + for estimator in &self.user_provided { + if let Some(result) = estimator.task_estimation(plan, cfg) { + return Some(result); + } + } + // We want to execute the default estimators last so that the user-provided ones have + // a chance of providing an estimation. + // If none of the user-provided returned an estimation, the default ones are used. + for default_estimator in [&FileScanConfigTaskEstimator as &dyn TaskEstimator] { + if let Some(result) = default_estimator.task_estimation(plan, cfg) { + return Some(result); + } + } + None + } + + fn scale_up_leaf_node( + &self, + plan: &Arc, + task_count: usize, + cfg: &ConfigOptions, + ) -> Option> { + for estimator in &self.user_provided { + if let Some(result) = estimator.scale_up_leaf_node(plan, task_count, cfg) { + return Some(result); + } + } + // We want to execute the default estimators last so that the user-provided ones have + // a chance of providing an estimation. + // If none of the user-provided returned an estimation, the default ones are used. + for default_estimator in [&FileScanConfigTaskEstimator as &dyn TaskEstimator] { + if let Some(result) = default_estimator.scale_up_leaf_node(plan, task_count, cfg) { + return Some(result); + } + } + None + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::networking::WorkerResolverExtension; + use crate::test_utils::in_memory_channel_resolver::InMemoryWorkerResolver; + use crate::test_utils::parquet::register_parquet_tables; + use datafusion::error::DataFusionError; + use datafusion::prelude::SessionContext; + + #[tokio::test] + async fn test_first_user_estimator_wins() -> Result<(), DataFusionError> { + let mut combined = CombinedTaskEstimator::default(); + combined.push(10); + combined.push(20); + + let node = make_data_source_exec().await?; + assert_eq!(combined.task_count(node, |cfg| cfg), 10); + Ok(()) + } + + #[tokio::test] + async fn test_continues_until_some() -> Result<(), DataFusionError> { + let mut combined = CombinedTaskEstimator::default(); + combined.push(|_: &Arc, _: &ConfigOptions| None); + combined.push(30); + + let node = make_data_source_exec().await?; + assert_eq!(combined.task_count(node, |cfg| cfg), 30); + Ok(()) + } + + #[tokio::test] + async fn test_defaults_to_file_scan_config_task_estimator() -> Result<(), DataFusionError> { + let mut combined = CombinedTaskEstimator::default(); + combined.push(|_: &Arc, _: &ConfigOptions| None); + + let node = make_data_source_exec().await?; + assert_eq!(combined.task_count(node, |cfg| cfg), 3); + Ok(()) + } + + impl CombinedTaskEstimator { + fn push(&mut self, value: impl TaskEstimator + Send + Sync + 'static) { + self.user_provided.push(Arc::new(value)); + } + + fn task_count( + &self, + node: Arc, + f: impl FnOnce(DistributedConfig) -> DistributedConfig, + ) -> usize { + let mut cfg = ConfigOptions::default(); + let d_cfg = DistributedConfig { + files_per_task: 1, + __private_worker_resolver: WorkerResolverExtension(Arc::new( + InMemoryWorkerResolver::new(3), + )), + ..Default::default() + }; + cfg.extensions.insert(f(d_cfg)); + self.task_estimation(&node, &cfg) + .unwrap() + .task_count + .as_usize() + } + } + + async fn make_data_source_exec() -> Result, DataFusionError> { + let ctx = SessionContext::new(); + register_parquet_tables(&ctx).await?; + let mut plan = ctx + .sql("SELECT * FROM weather") + .await? + .create_physical_plan() + .await?; + while !plan.children().is_empty() { + plan = Arc::clone(plan.children()[0]) + } + Ok(plan) + } + + impl, &ConfigOptions) -> Option> TaskEstimator for F { + fn task_estimation( + &self, + plan: &Arc, + cfg: &ConfigOptions, + ) -> Option { + self(plan, cfg) + } + + fn scale_up_leaf_node( + &self, + _plan: &Arc, + _task_count: usize, + _cfg: &ConfigOptions, + ) -> Option> { + None + } + } +} diff --git a/src/execution_plans/broadcast.rs b/src/execution_plans/broadcast.rs new file mode 100644 index 00000000..915a5e70 --- /dev/null +++ b/src/execution_plans/broadcast.rs @@ -0,0 +1,559 @@ +use crate::common::require_one_child; +use crossbeam_queue::SegQueue; +use datafusion::arrow::datatypes::SchemaRef; +use datafusion::common::runtime::SpawnedTask; +use datafusion::error::{DataFusionError, Result}; +use datafusion::execution::memory_pool::MemoryConsumer; +use datafusion::execution::{SendableRecordBatchStream, TaskContext}; +use datafusion::physical_plan::stream::RecordBatchStreamAdapter; +use datafusion::physical_plan::{ + DisplayAs, DisplayFormatType, ExecutionPlan, Partitioning, PlanProperties, internal_err, +}; +use futures::{Stream, StreamExt}; +use std::any::Any; +use std::fmt::Formatter; +use std::pin::Pin; +use std::sync::{Arc, Mutex, OnceLock}; +use std::task::{Context, Poll}; +use tokio_stream::wrappers::WatchStream; + +/// [ExecutionPlan] that scales up partitions for network broadcasting. +/// +/// This plan takes N input partitions and exposes N*M output partitions, +/// where M is the number of consumer tasks. Each virtual partition `i` +/// returns the cached result of input partition `i % N`. +/// +/// This allows each consumer task to fetch a unique set of partition numbers, +/// the virtual partitions, while all receiving the same data via the actual partitions. +/// This structure maintains the invariant that each partition is executed exactly +/// once by the framework. +/// +/// Broadcast is used in a 1 to many context, like this: +/// ```text +/// ┌────────────────────────┐ ┌────────────────────────┐ ┌────────────────────────┐ ■ +/// │ NetworkBroadcastExec │ │ NetworkBroadcastExec │ ... │ NetworkBroadcastExec │ │ +/// │ (task 1) │ │ (task 2) │ │ (task M) │ Stage N+1 +/// └┬─┬─────┬───┬───────────┘ └───────┬─┬─────┬────┬───┘ └─────┬──────┬─────┬────┬┘ │ +/// │0│ │N-1│ │N│ │2N-1│ │(M-1)N│ │MN-1│ │ +/// └▲┘ ... └▲──┘ └▲┘ ... └──▲─┘ └───▲──┘ ... └──▲─┘ ■ +/// │ │ Populates │ │ │ │ +/// │ └────Cache Index ───┐ Cache Hit Cache Hit ┌──Cache Hit────┘ │ +/// │ N-1 │ Index 0 Index N-1 │ │ +/// └────Populates ─────┐ │ │ │ │ ┌───Cache Hit──┘ +/// Cache Index 0 │ │ │ │ │ │ +/// ┌┴┐ ... ┌┴──┐ ┌┴┐ ... ┌──┴─┐ ... ┌───┴──┐ ... ┌───┴┐ ■ +/// │0│ │N-1│ │N│ │2N-1│ │(M-1)N│ │MN-1│ │ +/// ┌┴─┴─────┴───┴──────┴─┴─────┴────┴───────────────────┴──────┴─────┴────┴┐ │ +/// │ BroadcastExec │ │ +/// │ ┌───────────────────────────┐ │ │ +/// │ │ Batch Cache │ │ Stage N +/// │ │┌─────────┐ ┌─────────┐│ │ │ +/// │ ││ index 0 │ ... │index N-1││ │ │ +/// │ │└─────────┘ └─────────┘│ │ │ +/// │ └───────────────────────────┘ │ │ +/// └───────────────────────────┬─┬──────────┬───┬──────────────────────────┘ ■ +/// │0│ │N-1│ +/// └▲┘ ... └─▲─┘ +/// │ │ +/// ┌──┘ └──┐ +/// │ │ ■ +/// ┌┴┐ ... ┌──┴┐ │ +/// │0│ │N-1│ Stage N-1 +/// ┌┴─┴───────────────┴───┴┐ │ +/// │Arc │ │ +/// └───────────────────────┘ ■ +/// ``` +/// +/// Notice that the first consumer task, [NetworkBroadcastExec] task 1, triggers the execution of +/// the operator below the [BroadCastExec] and populates each cache index with the respective +/// partition. Subsequent consumer tasks, rather than executing the same partitions, read the +/// data from the cache for each partition. +#[derive(Debug)] +pub struct BroadcastExec { + input: Arc, + consumer_task_count: usize, + properties: PlanProperties, + queues: Vec>>>, +} + +type StreamAndTask = (SegQueue, Arc>); + +impl BroadcastExec { + pub fn new(input: Arc, consumer_task_count: usize) -> Self { + let input_partition_count = input.properties().partitioning.partition_count(); + let output_partition_count = input_partition_count * consumer_task_count; + + let properties = input + .properties() + .clone() + .with_partitioning(Partitioning::UnknownPartitioning(output_partition_count)); + + let queues = (0..input_partition_count) + .map(|_| OnceLock::new()) + .collect(); + + Self { + input, + consumer_task_count, + properties, + queues, + } + } + + pub fn input_partition_count(&self) -> usize { + self.input.properties().partitioning.partition_count() + } + + pub fn consumer_task_count(&self) -> usize { + self.consumer_task_count + } +} + +impl DisplayAs for BroadcastExec { + fn fmt_as(&self, _t: DisplayFormatType, f: &mut Formatter) -> std::fmt::Result { + let input_partition_count = self.input_partition_count(); + write!( + f, + "BroadcastExec: input_partitions={}, consumer_tasks={}, output_partitions={}", + input_partition_count, + self.consumer_task_count, + input_partition_count * self.consumer_task_count + ) + } +} + +impl ExecutionPlan for BroadcastExec { + fn name(&self) -> &str { + "BroadcastExec" + } + + fn as_any(&self) -> &dyn Any { + self + } + + fn properties(&self) -> &PlanProperties { + &self.properties + } + + fn children(&self) -> Vec<&Arc> { + vec![&self.input] + } + + fn with_new_children( + self: Arc, + children: Vec>, + ) -> Result> { + Ok(Arc::new(Self::new( + require_one_child(children)?, + self.consumer_task_count, + ))) + } + + fn execute( + &self, + partition: usize, + context: Arc, + ) -> Result { + let real_partition = partition % self.input_partition_count(); + + let input = Arc::clone(&self.input); + + let queue_or_err = self.queues[real_partition].get_or_init(|| { + let queue = BroadcastQueue::new(); + let consumers = SegQueue::new(); + for _ in 0..self.consumer_task_count { + consumers.push(Box::pin(RecordBatchStreamAdapter::new( + self.schema(), + queue.new_consumer().map(|msg| match msg { + Ok((batch, _reservation)) => Ok(batch), + Err(e) => Err(DataFusionError::Shared(e)), + }), + )) as SendableRecordBatchStream); + } + + let pool = Arc::clone(context.memory_pool()); + let mut stream = input.execute(real_partition, context).map_err(Arc::new)?; + let task = SpawnedTask::spawn(async move { + let mem_consumer = MemoryConsumer::new(format!("BroadcastExec[{real_partition}]")); + + while let Some(msg) = stream.next().await { + match msg { + Ok(record_batch) => { + let mut reservation = mem_consumer.clone_with_new_id().register(&pool); + reservation.grow(record_batch.get_array_memory_size()); + queue.push(Ok((record_batch, Arc::new(reservation)))); + } + Err(err) => { + queue.push(Err(Arc::new(err))); + break; + } + } + } + }); + + Ok::<_, Arc>((consumers, Arc::new(task))) + }); + let (consumer, task) = match queue_or_err { + Ok((consumers, task)) => (consumers.pop(), Arc::clone(task)), + Err(err) => return Err(DataFusionError::Shared(Arc::clone(err))), + }; + let Some(consumer) = consumer else { + return internal_err!("Too many consumers for real partition {real_partition}"); + }; + Ok(Box::pin(RecordBatchStreamAdapter::new( + self.schema(), + consumer.inspect(move |_| { + let _ = &task; + }), + ))) + } + + fn schema(&self) -> SchemaRef { + self.input.schema() + } +} + +#[derive(Debug, Clone, Copy)] +struct BroadcastState { + len: usize, + closed: bool, +} + +#[derive(Debug)] +struct BroadcastQueue { + entries: Arc>>, + notify: tokio::sync::watch::Sender, +} + +impl BroadcastQueue { + fn new() -> Self { + let (notify, _rx) = tokio::sync::watch::channel(BroadcastState { + len: 0, + closed: false, + }); + Self { + entries: Arc::new(Mutex::new(vec![])), + notify, + } + } + + fn new_consumer(&self) -> BroadcastConsumer { + let rx = self.notify.subscribe(); + let state = *rx.borrow(); + BroadcastConsumer { + index: 0, + entries: Arc::clone(&self.entries), + notify: WatchStream::new(rx), + state, + } + } + + fn push(&self, entry: T) { + let len = { + let mut entries = self.entries.lock().unwrap(); + entries.push(entry); + entries.len() + }; + let mut state = *self.notify.borrow(); + state.len = len; + let _ = self.notify.send(state); + } +} + +impl Drop for BroadcastQueue { + fn drop(&mut self) { + let mut state = *self.notify.borrow(); + state.closed = true; + let _ = self.notify.send(state); + } +} + +/// A consumer stream that reads from the broadcast queue. +struct BroadcastConsumer { + index: usize, + entries: Arc>>, + notify: WatchStream, + state: BroadcastState, +} + +impl Stream for BroadcastConsumer { + type Item = T; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + loop { + if self.index < self.state.len { + let entry = self.entries.lock().unwrap().get(self.index).cloned(); + if let Some(v) = entry { + self.index += 1; + return Poll::Ready(Some(v)); + } + } + + if self.state.closed { + return Poll::Ready(None); + } + + match Pin::new(&mut self.notify).poll_next(cx) { + Poll::Ready(Some(state)) => { + self.state = state; + } + Poll::Ready(None) => { + self.state.closed = true; + } + Poll::Pending => return Poll::Pending, + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_utils::mock_exec::MockExec; + use datafusion::arrow::array::Int32Array; + use datafusion::arrow::datatypes::{DataType, Field, Schema}; + use datafusion::arrow::record_batch::RecordBatch; + use datafusion::prelude::SessionContext; + use futures::StreamExt; + use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + use tokio::sync::Notify; + use tokio::time::{Duration, sleep}; + + fn assert_int32_batch_values(batch: &RecordBatch, expected: &[i32]) { + let values = batch + .column(0) + .as_any() + .downcast_ref::() + .expect("int32 column"); + assert_eq!(values.len(), expected.len()); + for (idx, expected_value) in expected.iter().enumerate() { + assert_eq!(values.value(idx), *expected_value); + } + } + + #[tokio::test] + async fn broadcast_exec_reuses_queue_for_virtual_partitions() -> Result<()> { + let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)])); + let counts = Arc::new(vec![AtomicUsize::new(0)]); + let batch = RecordBatch::try_new( + Arc::clone(&schema), + vec![Arc::new(Int32Array::from(vec![0]))], + )?; + let input = Arc::new( + MockExec::new_partitioned(vec![vec![Ok(batch)]], Arc::clone(&schema)) + .with_execute_counts(Arc::clone(&counts)), + ); + let broadcast = Arc::new(BroadcastExec::new(input, 2)); + + let ctx = SessionContext::new(); + let task_ctx = ctx.task_ctx(); + + let batches0 = + datafusion::physical_plan::common::collect(broadcast.execute(0, task_ctx.clone())?) + .await?; + let batches1 = + datafusion::physical_plan::common::collect(broadcast.execute(1, task_ctx)?).await?; + + // Only executes the partition once, second batch is read from the queue + assert_eq!(counts[0].load(Ordering::SeqCst), 1); + assert_eq!(batches0.len(), 1); + assert_eq!(batches1.len(), 1); + assert_eq!(batches0[0].num_rows(), 1); + assert_eq!(batches1[0].num_rows(), 1); + assert_int32_batch_values(&batches0[0], &[0]); + assert_int32_batch_values(&batches1[0], &[0]); + + Ok(()) + } + + #[tokio::test] + async fn broadcast_exec_maps_partitions_by_modulo() -> Result<()> { + let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)])); + let counts = Arc::new(vec![AtomicUsize::new(0), AtomicUsize::new(0)]); + let batch0 = RecordBatch::try_new( + Arc::clone(&schema), + vec![Arc::new(Int32Array::from(vec![0]))], + )?; + let batch1 = RecordBatch::try_new( + Arc::clone(&schema), + vec![Arc::new(Int32Array::from(vec![1]))], + )?; + let input = Arc::new( + MockExec::new_partitioned( + vec![vec![Ok(batch0)], vec![Ok(batch1)]], + Arc::clone(&schema), + ) + .with_execute_counts(Arc::clone(&counts)), + ); + let broadcast = Arc::new(BroadcastExec::new(input, 2)); + + let ctx = SessionContext::new(); + let task_ctx = ctx.task_ctx(); + + // Should map to real partition 0 + let batches0 = + datafusion::physical_plan::common::collect(broadcast.execute(0, task_ctx.clone())?) + .await?; + // Should map to real partition 1 + let batches1 = + datafusion::physical_plan::common::collect(broadcast.execute(1, task_ctx.clone())?) + .await?; + // Should map to real partition 0 + let batches2 = + datafusion::physical_plan::common::collect(broadcast.execute(2, task_ctx.clone())?) + .await?; + // Should map to real partition 1 + let batches3 = + datafusion::physical_plan::common::collect(broadcast.execute(3, task_ctx)?).await?; + + assert_eq!(counts[0].load(Ordering::SeqCst), 1); + assert_eq!(counts[1].load(Ordering::SeqCst), 1); + + assert_eq!(batches0.len(), 1); + assert_eq!(batches1.len(), 1); + assert_eq!(batches2.len(), 1); + assert_eq!(batches3.len(), 1); + assert_int32_batch_values(&batches0[0], &[0]); + assert_int32_batch_values(&batches1[0], &[1]); + assert_int32_batch_values(&batches2[0], &[0]); + assert_int32_batch_values(&batches3[0], &[1]); + + Ok(()) + } + + #[tokio::test] + async fn broadcast_exec_queue_survives_cancellation() -> Result<()> { + let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)])); + let execute_counts = Arc::new(vec![AtomicUsize::new(0)]); + let permit_open = Arc::new(AtomicBool::new(false)); + let permit_notify = Arc::new(Notify::new()); + + let batch = RecordBatch::try_new( + Arc::clone(&schema), + vec![Arc::new(Int32Array::from(vec![1, 2, 3]))], + )?; + let input = Arc::new( + MockExec::new_partitioned(vec![vec![Ok(batch)]], Arc::clone(&schema)) + .with_execute_counts(Arc::clone(&execute_counts)) + .with_gate(Arc::clone(&permit_open), Arc::clone(&permit_notify)), + ); + + // Has two consumers that will execute the same real partition + let broadcast = Arc::new(BroadcastExec::new(input, 2)); + + let ctx = SessionContext::new(); + let task_ctx = ctx.task_ctx(); + + // Execute is called synchronously, so execute_counts should increment immediately + let mut stream1 = broadcast.execute(0, task_ctx.clone())?; + assert_eq!(execute_counts[0].load(Ordering::SeqCst), 1); + + let handle = tokio::spawn(async move { stream1.next().await }); + + // Cancel this consumer (simulates a cancellation like a TopK) + handle.abort(); + let _ = handle.await; + + // Execute with a different virtual partition but maps to same real partition and allow + // full execution + let stream2 = broadcast.execute(1, task_ctx)?; + permit_open.store(true, Ordering::SeqCst); + permit_notify.notify_waiters(); + + let batches: Vec = datafusion::physical_plan::common::collect(stream2).await?; + assert_eq!(batches.len(), 1); + assert_int32_batch_values(&batches[0], &[1, 2, 3]); + + // Partition should only be executed a single time, second stream should've pulled from + // queue + assert_eq!(execute_counts[0].load(Ordering::SeqCst), 1); + + Ok(()) + } + + #[tokio::test] + async fn broadcast_exec_continues_after_consumer_cancel() -> Result<()> { + let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)])); + let batches = vec![ + Ok(RecordBatch::try_new( + Arc::clone(&schema), + vec![Arc::new(Int32Array::from(vec![0]))], + )?), + Ok(RecordBatch::try_new( + Arc::clone(&schema), + vec![Arc::new(Int32Array::from(vec![1]))], + )?), + Ok(RecordBatch::try_new( + Arc::clone(&schema), + vec![Arc::new(Int32Array::from(vec![2]))], + )?), + ]; + let input = Arc::new( + MockExec::new_partitioned(vec![batches], Arc::clone(&schema)) + .with_delay_between_batches(Duration::from_millis(10)), + ); + let broadcast = Arc::new(BroadcastExec::new(input, 2)); + + let ctx = SessionContext::new(); + let task_ctx = ctx.task_ctx(); + + let mut stream1 = broadcast.execute(0, task_ctx.clone())?; + let stream2 = broadcast.execute(1, task_ctx)?; + + let first = stream1.next().await.transpose()?.expect("first batch"); + assert_int32_batch_values(&first, &[0]); + drop(stream1); + + let batches: Vec = datafusion::physical_plan::common::collect(stream2).await?; + assert_eq!(batches.len(), 3); + assert_int32_batch_values(&batches[0], &[0]); + assert_int32_batch_values(&batches[1], &[1]); + assert_int32_batch_values(&batches[2], &[2]); + + Ok(()) + } + + #[tokio::test] + async fn broadcast_exec_replay_for_late_consumer() -> Result<()> { + let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)])); + let batches = vec![ + Ok(RecordBatch::try_new( + Arc::clone(&schema), + vec![Arc::new(Int32Array::from(vec![0]))], + )?), + Ok(RecordBatch::try_new( + Arc::clone(&schema), + vec![Arc::new(Int32Array::from(vec![1]))], + )?), + Ok(RecordBatch::try_new( + Arc::clone(&schema), + vec![Arc::new(Int32Array::from(vec![2]))], + )?), + ]; + let input = Arc::new( + MockExec::new_partitioned(vec![batches], Arc::clone(&schema)) + .with_delay_between_batches(Duration::from_millis(10)), + ); + let broadcast = Arc::new(BroadcastExec::new(input, 2)); + + let ctx = SessionContext::new(); + let task_ctx = ctx.task_ctx(); + + let mut stream0 = broadcast.execute(0, task_ctx.clone())?; + let batch0 = stream0.next().await.transpose()?.expect("batch 0"); + assert_int32_batch_values(&batch0, &[0]); + let batch1 = stream0.next().await.transpose()?.expect("batch 1"); + assert_int32_batch_values(&batch1, &[1]); + + // Late consumer joins after producer has already emitted some batches. + sleep(Duration::from_millis(5)).await; + let stream1 = broadcast.execute(1, task_ctx)?; + let batches: Vec = datafusion::physical_plan::common::collect(stream1).await?; + assert_eq!(batches.len(), 3); + assert_int32_batch_values(&batches[0], &[0]); + assert_int32_batch_values(&batches[1], &[1]); + assert_int32_batch_values(&batches[2], &[2]); + + Ok(()) + } +} diff --git a/src/execution_plans/children_isolator_union.rs b/src/execution_plans/children_isolator_union.rs new file mode 100644 index 00000000..6f0dad1b --- /dev/null +++ b/src/execution_plans/children_isolator_union.rs @@ -0,0 +1,541 @@ +use crate::DistributedTaskContext; +use datafusion::arrow::array::RecordBatch; +use datafusion::arrow::datatypes::SchemaRef; +use datafusion::common::{internal_err, plan_err}; +use datafusion::error::DataFusionError; +use datafusion::execution::{RecordBatchStream, SendableRecordBatchStream, TaskContext}; +use datafusion::physical_plan::metrics::{BaselineMetrics, ExecutionPlanMetricsSet, MetricsSet}; +use datafusion::physical_plan::union::UnionExec; +use datafusion::physical_plan::{ + DisplayAs, DisplayFormatType, EmptyRecordBatchStream, ExecutionPlan, ExecutionPlanProperties, + Partitioning, PlanProperties, +}; +use futures::{Stream, StreamExt}; +use itertools::Itertools; +use std::any::Any; +use std::fmt::Formatter; +use std::pin::Pin; +use std::sync::Arc; +use std::task::{Context, Poll}; +use std::vec; + +/// Distributed version of the vanilla [UnionExec] node that is capable of spreading the execution +/// of its children across multiple distributed tasks. +/// +/// Without [ChildrenIsolatorUnionExec], distributing a normal [UnionExec] implies scaling up +/// in partitions all the child leaf nodes and executing them all in all the assigned tasks, +/// passing a [DistributedTaskContext] so that each child knows how to distribute its work. +/// +/// With [ChildrenIsolatorUnionExec], its children are isolated per task, meaning that each +/// child will potentially be executed as if it was running in a single-node setup, and +/// [ChildrenIsolatorUnionExec] will figure out which children to execute depending on the +/// [DistributedTaskContext]. +/// +/// It's easy to think about this node in the case that the task count is equal to the number +/// of children. However, it gets a bit more complicated in case there are fewer tasks than children, +/// or more tasks than children. +/// +/// ## Case when task_count == 3 and children.len() == 3 +/// +/// ```text +/// ┌─────────────────────────────┐┌─────────────────────────────┐┌─────────────────────────────┐ +/// │ Task 1 ││ Task 2 ││ Task 3 │ +/// │┌───────────────────────────┐││┌───────────────────────────┐││┌───────────────────────────┐│ +/// ││ ChildrenIsolatorUnionExec ││││ ChildrenIsolatorUnionExec ││││ ChildrenIsolatorUnionExec ││ +/// │└───▲─────────▲─────────▲───┘││└───▲─────────▲─────────▲───┘││└───▲─────────▲─────────▲───┘│ +/// │ │ ││ │ ││ │ │ +/// │┌───┴───┐ ┌ ─│ ─ ┌ ─│ ─ ││┌ ─│ ─ ┌───┴───┐ ┌ ─│ ─ ││┌ ─│ ─ ┌ ─│ ─ ┌───┴───┐│ +/// ││Child 1│ Child 2│ Child 3│││ Child 1│ │Child 2│ Child 3│││ Child 1│ Child 2│ │Child 3││ +/// │└───────┘ └ ─ ─ └ ─ ─ ││└ ─ ─ └───────┘ └ ─ ─ ││└ ─ ─ └ ─ ─ └───────┘│ +/// └─────────────────────────────┘└─────────────────────────────┘└─────────────────────────────┘ +/// ``` +/// +/// ## Case when task_count == 2 and children.len() == 3 +/// +/// ```text +/// ┌─────────────────────────────┐┌─────────────────────────────┐ +/// │ Task 1 ││ Task 2 │ +/// │┌───────────────────────────┐││┌───────────────────────────┐│ +/// ││ ChildrenIsolatorUnionExec ││││ ChildrenIsolatorUnionExec ││ +/// │└───▲─────────▲─────────▲───┘││└───▲─────────▲─────────▲───┘│ +/// │ │ │ ││ │ │ +/// │┌───┴───┐ ┌───┴───┐ ┌ ─│ ─ ││┌ ─│ ─ ┌ ─ ┴ ─ ┐ ┌───┴───┐│ +/// ││Child 1│ │Child 2│ Child 3│││ Child 1│ Child 2 │Child 3││ +/// │└───────┘ └───────┘ └ ─ ─ ││└ ─ ─ └ ─ ─ ─ ┘ └───────┘│ +/// └─────────────────────────────┘└─────────────────────────────┘ +///``` +/// +/// ## Case when task_count == 4 and children.len() == 3 +/// +/// ```text +/// ┌─────────────────────────────┐┌─────────────────────────────┐┌─────────────────────────────┐┌─────────────────────────────┐ +/// │ Task 1 ││ Task 2 ││ Task 3 ││ Task 4 │ +/// │┌───────────────────────────┐││┌───────────────────────────┐││┌───────────────────────────┐││┌───────────────────────────┐│ +/// ││ ChildrenIsolatorUnionExec ││││ ChildrenIsolatorUnionExec ││││ ChildrenIsolatorUnionExec ││││ ChildrenIsolatorUnionExec ││ +/// │└───▲─────────▲─────────▲───┘││└───▲─────────▲─────────▲───┘││└───▲─────────▲─────────▲───┘││└───▲─────────▲─────────▲───┘│ +/// │ │ ││ │ ││ │ ││ │ │ +/// │┌───┴───┐ ┌ ─│ ─ ┌ ─│ ─ ││┌───┴───┐ ┌ ─│ ─ ┌ ─│ ─ ││┌ ─│ ─ ┌───┴───┐ ┌ ─│ ─ ││┌ ─│ ─ ┌ ─│ ─ ┌───┴───┐│ +/// ││Child 1│ Child 2│ Child 3││││Child 1│ Child 2│ Child 3│││ Child 1│ │Child 2│ Child 3│││ Child 1│ Child 2│ │Child 3││ +/// ││ (1/2) │ └ ─ ─ └ ─ ─ │││ (2/2) │ └ ─ ─ └ ─ ─ ││└ ─ ─ └───────┘ └ ─ ─ ││└ ─ ─ └ ─ ─ └───────┘│ +/// │└───────┘ ││└───────┘ ││ ││ │ +/// └─────────────────────────────┘└─────────────────────────────┘└─────────────────────────────┘└─────────────────────────────┘ +/// ``` +#[derive(Debug, Clone)] +pub struct ChildrenIsolatorUnionExec { + pub(crate) properties: PlanProperties, + pub(crate) metrics: ExecutionPlanMetricsSet, + pub(crate) children: Vec>, + pub(crate) task_idx_map: Vec< + /* outer distributed task idx */ + Vec<( + /* child index */ usize, + /* inner distributed task ctx for the isolated child*/ DistributedTaskContext, + )>, + >, +} + +impl ChildrenIsolatorUnionExec { + pub(crate) fn from_children_and_task_counts( + children: impl IntoIterator>, + children_task_count: impl IntoIterator, + task_count: usize, + ) -> Result { + let children = children.into_iter().collect_vec(); + let task_count_per_children = children_task_count.into_iter().collect_vec(); + + if children.len() != task_count_per_children.len() { + return internal_err!( + "ChildrenIsolatorUnionExec received {} children but a vec of {} positions for those children. This is a bug in the distributed planning logic, please report it", + children.len(), + task_count_per_children.len() + ); + } + + let task_idx_map = split_children(task_count_per_children, task_count)?; + + // Because different children might return a different number of partitions, and we might + // execute a different number of children in different tasks, the reality is that this node, + // depending on which task index is running, it will have a different number of partitions. + // + // We want to hide that to the outside and just advertise as many partitions as the task + // that will handle the greatest number of partitions, and just return empty streams for + // remainder partitions in tasks that will execute fewer partitions. + let mut partition_counts = vec![0; task_idx_map.len()]; + for (t, children_in_task) in task_idx_map.iter().enumerate() { + for (child_idx, _) in children_in_task { + partition_counts[t] += children[*child_idx].output_partitioning().partition_count(); + } + } + let Some(partition_count) = partition_counts.iter().max() else { + return internal_err!( + "ChildrenIsolatorUnionExec built an empty task_idx_map. This is a bug in the distributed planning logic, please report it" + ); + }; + + // It's not supper efficient to build a UnionExec just to get the properties out, but the + // other solution is to copy-paste a bunch of code from upstream for computing the properties + // of a union, so we prefer to just reuse it like this. + let mut properties = UnionExec::try_new(children.clone())?.properties().clone(); + properties.partitioning = Partitioning::UnknownPartitioning(*partition_count); + Ok(Self { + properties, + metrics: ExecutionPlanMetricsSet::default(), + children, + task_idx_map, + }) + } +} + +impl DisplayAs for ChildrenIsolatorUnionExec { + fn fmt_as(&self, t: DisplayFormatType, f: &mut Formatter) -> std::fmt::Result { + match t { + DisplayFormatType::Default | DisplayFormatType::Verbose => { + write!(f, "DistributedUnionExec:")?; + for (task_i, children_in_task) in self.task_idx_map.iter().enumerate() { + write!(f, " t{task_i}:[")?; + for (i, (child_idx, child_task_ctx)) in children_in_task.iter().enumerate() { + if child_task_ctx.task_count > 1 { + write!( + f, + "c{child_idx}({}/{})", + child_task_ctx.task_index, child_task_ctx.task_count + )?; + } else { + write!(f, "c{child_idx}")?; + } + if i < children_in_task.len() - 1 { + write!(f, ", ")?; + } + } + write!(f, "]")?; + } + + Ok(()) + } + DisplayFormatType::TreeRender => Ok(()), + } + } +} + +impl ExecutionPlan for ChildrenIsolatorUnionExec { + fn name(&self) -> &str { + "ChildrenIsolatorUnionExec" + } + + fn as_any(&self) -> &dyn Any { + self + } + + fn properties(&self) -> &PlanProperties { + &self.properties + } + + fn with_new_children( + self: Arc, + children: Vec>, + ) -> datafusion::common::Result> { + if children.len() != self.children.len() { + return plan_err!( + "Number of children must match the original plan, have {} but expected {}", + children.len(), + self.children.len() + ); + } + let mut clone = self.as_ref().clone(); + clone.children = children; + Ok(Arc::new(clone)) + } + + fn children(&self) -> Vec<&Arc> { + self.children.iter().collect() + } + + fn metrics(&self) -> Option { + Some(self.metrics.clone_inner()) + } + + fn execute( + &self, + mut partition: usize, + context: Arc, + ) -> datafusion::common::Result { + let d_ctx = DistributedTaskContext::from_ctx(&context); + + let children = self.task_idx_map[d_ctx.task_index].clone(); + + let baseline_metrics = BaselineMetrics::new(&self.metrics, partition); + + let elapsed_compute = baseline_metrics.elapsed_compute().clone(); + let _timer = elapsed_compute.timer(); // record on drop + + for (child_idx, child_task_ctx) in children { + let Some(input) = self.children.get(child_idx) else { + return internal_err!("Could not find child with index {child_idx}"); + }; + // Calculate whether a partition belongs to the current partition + if partition < input.output_partitioning().partition_count() { + // We need to intercept the DistributedTaskContext and insert a modified one that + // tells the child that is running in "isolation" (see the beginning of this file + // for a longer explanation) + let context = Arc::new(TaskContext::new( + context.task_id(), + context.session_id(), + context + .session_config() + .clone() + .with_extension(Arc::new(child_task_ctx)), + context.scalar_functions().clone(), + context.aggregate_functions().clone(), + context.window_functions().clone(), + context.runtime_env(), + )); + + let stream = input.execute(partition, context)?; + + return Ok(Box::pin(ObservedStream::new( + stream, + baseline_metrics, + None, + ))); + } else { + partition -= input.output_partitioning().partition_count(); + } + } + + Ok(Box::pin(EmptyRecordBatchStream::new(self.schema()))) + } +} + +// Struct copied from https://github.com/apache/datafusion/blob/2c3566ce856bf7c87508567119bc3834f007e94b/datafusion/physical-plan/src/stream.rs#L506-L506 +// It's what allows a UnionExec to have metrics. +pub(crate) struct ObservedStream { + inner: SendableRecordBatchStream, + baseline_metrics: BaselineMetrics, + fetch: Option, + produced: usize, +} + +impl ObservedStream { + pub fn new( + inner: SendableRecordBatchStream, + baseline_metrics: BaselineMetrics, + fetch: Option, + ) -> Self { + Self { + inner, + baseline_metrics, + fetch, + produced: 0, + } + } + + fn limit_reached( + &mut self, + poll: Poll>>, + ) -> Poll>> { + let Some(fetch) = self.fetch else { return poll }; + + if self.produced >= fetch { + return Poll::Ready(None); + } + + if let Poll::Ready(Some(Ok(batch))) = &poll { + if self.produced + batch.num_rows() > fetch { + let batch = batch.slice(0, fetch.saturating_sub(self.produced)); + self.produced += batch.num_rows(); + return Poll::Ready(Some(Ok(batch))); + }; + self.produced += batch.num_rows() + } + poll + } +} + +impl RecordBatchStream for ObservedStream { + fn schema(&self) -> SchemaRef { + self.inner.schema() + } +} + +impl Stream for ObservedStream { + type Item = datafusion::common::Result; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let mut poll = self.inner.poll_next_unpin(cx); + if self.fetch.is_some() { + poll = self.limit_reached(poll); + } + self.baseline_metrics.record_poll(poll) + } +} + +/// Given a list of children with a different number of tasks assigned each, it redistributes them +/// and re-assign tasks numbers to them so that they fit in the provided `task_count`. +/// +/// For example, given these inputs: +/// `task_count_per_children`: [1, 1, 1] +/// `task_count`: 3 +/// It returns the following output: +/// `[[(0, 0/1)], [(1, 0/1)], [(2, 0/1)]]` +/// That means that: +/// - Task 0 will execute task 0 from child 0 +/// - Task 1 will execute task 0 from child 1 +/// - Task 2 will execute task 0 from child 2 +/// +/// Things can get more complicated if the sum of all the tasks from the children is greater than +/// the `task_count` passed: +/// +/// For example, given these inputs: +/// `task_count_per_children`: [2, 1, 1] +/// `task_count`: 3 +/// It returns the following output: +/// `[[(0, 0/1)], [(1, 0/1)], [(2, 0/1)]]` +/// As we have 3 tasks available for executing 3 children with a total sum of 2+1+1=4 tasks, we +/// need to tell that first child to be executed in 1 task. +/// +/// In this other case: +/// `task_count_per_children`: [2, 3, 1] +/// `task_count`: 5 +/// It returns the following output: +/// `[[(0, 0/2)], [(0, 1/2)], [(1, 0/2)], [(1, 1/2)], [(2, 0/1)]]` +/// Note how in this case there are 2+3+1=6 tasks to be executed, but we only have 5 tasks +/// available. We need to trim a task from somewhere, and the only candidates are child 0 and 1. +/// As child 1 is the one with the greatest task count (3), we prefer to trim the task from there, +/// as that's what will yield the most even distribution of tasks given the 5 tasks budget. +/// +/// Going to the other extreme, given this input: +/// `task_count_per_children`: [1, 1, 1, 1, 1] +/// `task_count`: 2 +/// It returns the following output: +/// `[[(0, 0/1), (1, 0/1), (2, 0/1)] , [(3, 0/1), (4, 0/1)]]` +/// In this case, we have very few `task_count` available, so we cannot afford to execute one +/// child per task. We need to group children so that one task executes several of them, and the +/// other tasks execute the several other remaining. +/// +/// The function looks pretty much like the solution of a LeetCode problem, so it's not super +/// easy to understand just be looking at the code. The best way to get a grasp of what its doing +/// is by looking at the tests. +fn split_children( + mut task_count_per_children: Vec, + task_count_budget: usize, +) -> Result< + // Task idx. This Vec will have `task_count_budget` length. + Vec< + // For this task, the child indexes and DistributedTaskContext that should be executed. + Vec<( + /* Child index */ usize, + /* Distributed task ctx for the child */ DistributedTaskContext, + )>, + >, + DataFusionError, +> { + let total_children_tasks = task_count_per_children.iter().sum::(); + if task_count_budget > total_children_tasks { + return internal_err!( + "ChildrenIsolatorUnionExec had a task count {task_count_budget}, which is greater than the sum of child task counts {total_children_tasks}. This is a bug in the distributed planning logic, please report it" + ); + } else if task_count_budget == 0 { + return internal_err!( + "ChildrenIsolatorUnionExec had a task count {task_count_budget}. This is a bug in the distributed planning logic, please report it" + ); + } + + let mut tasks_to_trim = total_children_tasks - task_count_budget; + while tasks_to_trim > 0 { + let mut max_child_task_count_idx = 0; + let mut max_child_task_count_value = 1; + for (i, child_task_count) in task_count_per_children.iter().enumerate() { + if child_task_count > &max_child_task_count_value { + max_child_task_count_idx = i; + max_child_task_count_value = *child_task_count; + } + } + if max_child_task_count_value == 1 { + break; + } + task_count_per_children[max_child_task_count_idx] -= 1; + tasks_to_trim -= 1; + } + + let total_child_tasks: usize = task_count_per_children.iter().sum(); + let base_per_task = total_child_tasks / task_count_budget; + let mut extra = total_child_tasks % task_count_budget; + + let mut result = vec![vec![]; task_count_budget]; + let mut task_idx = 0; + let mut current_task_count = 0; + let mut current_task_capacity = base_per_task; + if extra > 0 { + extra -= 1; + current_task_capacity += 1 + } + + for (child_idx, &child_task_count) in task_count_per_children.iter().enumerate() { + for task_i in 0..child_task_count { + result[task_idx].push(( + child_idx, + DistributedTaskContext { + task_index: task_i, + task_count: child_task_count, + }, + )); + current_task_count += 1; + + if current_task_count >= current_task_capacity && task_idx < task_count_budget - 1 { + task_idx += 1; + current_task_count = 0; + current_task_capacity = base_per_task; + if extra > 0 { + extra -= 1; + current_task_capacity += 1 + } + } + } + } + + Ok(result) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn children_split_all_1_task() -> Result<(), Box> { + assert_eq!( + split_children(vec![1, 1, 1], 3)?, + vec![ + vec![(0, ctx(0, 1))], + vec![(1, ctx(0, 1))], + vec![(2, ctx(0, 1))] + ] + ); + assert_eq!( + split_children(vec![1, 1, 1], 2)?, + vec![vec![(0, ctx(0, 1)), (1, ctx(0, 1))], vec![(2, ctx(0, 1))]] + ); + assert_eq!( + split_children(vec![1, 1, 1], 1)?, + vec![vec![(0, ctx(0, 1)), (1, ctx(0, 1)), (2, ctx(0, 1))]] + ); + Ok(()) + } + + #[test] + fn split_children_different_tasks() -> Result<(), Box> { + assert_eq!( + split_children(vec![1, 2, 3], 6)?, + vec![ + vec![(0, ctx(0, 1))], + vec![(1, ctx(0, 2))], + vec![(1, ctx(1, 2))], + vec![(2, ctx(0, 3))], + vec![(2, ctx(1, 3))], + vec![(2, ctx(2, 3))] + ] + ); + assert_eq!( + split_children(vec![1, 2, 3], 5)?, + vec![ + vec![(0, ctx(0, 1))], + vec![(1, ctx(0, 2))], + vec![(1, ctx(1, 2))], + vec![(2, ctx(0, 2))], + vec![(2, ctx(1, 2))], + ] + ); + assert_eq!( + split_children(vec![1, 2, 3], 4)?, + vec![ + vec![(0, ctx(0, 1))], + vec![(1, ctx(0, 1))], + vec![(2, ctx(0, 2))], + vec![(2, ctx(1, 2))], + ] + ); + assert_eq!( + split_children(vec![1, 2, 3], 3)?, + vec![ + vec![(0, ctx(0, 1))], + vec![(1, ctx(0, 1))], + vec![(2, ctx(0, 1))], + ] + ); + assert_eq!( + split_children(vec![1, 2, 3], 2)?, + vec![vec![(0, ctx(0, 1)), (1, ctx(0, 1))], vec![(2, ctx(0, 1))]] + ); + assert_eq!( + split_children(vec![1, 2, 3], 1)?, + vec![vec![(0, ctx(0, 1)), (1, ctx(0, 1)), (2, ctx(0, 1))]] + ); + Ok(()) + } + + fn ctx(task_index: usize, task_count: usize) -> DistributedTaskContext { + DistributedTaskContext { + task_index, + task_count, + } + } +} diff --git a/src/execution_plans/common.rs b/src/execution_plans/common.rs index f085b3e5..a1557a23 100644 --- a/src/execution_plans/common.rs +++ b/src/execution_plans/common.rs @@ -1,22 +1,5 @@ -use datafusion::common::{DataFusionError, plan_err}; use datafusion::physical_expr::Partitioning; -use datafusion::physical_plan::{ExecutionPlan, PlanProperties}; -use std::borrow::Borrow; -use std::sync::Arc; - -pub(super) fn require_one_child( - children: L, -) -> Result, DataFusionError> -where - L: AsRef<[T]>, - T: Borrow>, -{ - let children = children.as_ref(); - if children.len() != 1 { - return plan_err!("Expected exactly 1 children, got {}", children.len()); - } - Ok(children[0].borrow().clone()) -} +use datafusion::physical_plan::PlanProperties; pub(super) fn scale_partitioning_props( props: &PlanProperties, diff --git a/src/execution_plans/distributed.rs b/src/execution_plans/distributed.rs index 9fa4aab8..5e290f2c 100644 --- a/src/execution_plans/distributed.rs +++ b/src/execution_plans/distributed.rs @@ -1,10 +1,12 @@ -use crate::channel_resolver_ext::get_distributed_channel_resolver; -use crate::distributed_physical_optimizer_rule::NetworkBoundaryExt; -use crate::execution_plans::common::require_one_child; +use crate::common::require_one_child; +use crate::distributed_planner::NetworkBoundaryExt; +use crate::networking::get_distributed_worker_resolver; use crate::protobuf::DistributedCodec; use crate::stage::{ExecutionTask, Stage}; use datafusion::common::exec_err; +use datafusion::common::internal_datafusion_err; use datafusion::common::tree_node::{Transformed, TreeNode}; +use datafusion::error::DataFusionError; use datafusion::execution::{SendableRecordBatchStream, TaskContext}; use datafusion::physical_plan::{DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties}; use datafusion_proto::physical_plan::PhysicalExtensionCodec; @@ -12,6 +14,7 @@ use rand::Rng; use std::any::Any; use std::fmt::Formatter; use std::sync::Arc; +use std::sync::Mutex; use url::Url; /// [ExecutionPlan] that executes the inner plan in distributed mode. @@ -20,14 +23,30 @@ use url::Url; /// channel resolver and assigned to each task in each stage. /// 2. Encodes all the plans in protobuf format so that network boundary nodes can send them /// over the wire. -#[derive(Debug, Clone)] +#[derive(Debug)] pub struct DistributedExec { pub plan: Arc, + pub prepared_plan: Arc>>>, } impl DistributedExec { pub fn new(plan: Arc) -> Self { - Self { plan } + Self { + plan, + prepared_plan: Arc::new(Mutex::new(None)), + } + } + + /// Returns the plan which is lazily prepared on execute() and actually gets executed. + /// It is updated on every call to execute(). Returns an error if .execute() has not been called. + pub(crate) fn prepared_plan(&self) -> Result, DataFusionError> { + self.prepared_plan + .lock() + .map_err(|e| internal_datafusion_err!("Failed to lock prepared plan: {}", e))? + .clone() + .ok_or_else(|| { + internal_datafusion_err!("No prepared plan found. Was execute() called?") + }) } fn prepare_plan( @@ -40,15 +59,10 @@ impl DistributedExec { return Ok(Transformed::no(plan)); }; - let mut rng = rand::thread_rng(); - let start_idx = rng.gen_range(0..urls.len()); + let mut rng = rand::rng(); + let start_idx = rng.random_range(0..urls.len()); - let Some(stage) = plan.input_stage() else { - return exec_err!( - "NetworkBoundary '{}' has not been assigned a stage", - plan.name() - ); - }; + let stage = plan.input_stage(); let ready_stage = Stage { query_id: stage.query_id, @@ -98,7 +112,8 @@ impl ExecutionPlan for DistributedExec { children: Vec>, ) -> datafusion::common::Result> { Ok(Arc::new(DistributedExec { - plan: require_one_child(children)?, + plan: require_one_child(&children)?, + prepared_plan: self.prepared_plan.clone(), })) } @@ -117,10 +132,18 @@ impl ExecutionPlan for DistributedExec { ); } - let channel_resolver = get_distributed_channel_resolver(context.session_config())?; + let worker_resolver = get_distributed_worker_resolver(context.session_config())?; let codec = DistributedCodec::new_combined_with_user(context.session_config()); - let plan = self.prepare_plan(&channel_resolver.get_urls()?, &codec)?; - plan.execute(partition, context) + let prepared = self.prepare_plan(&worker_resolver.get_urls()?, &codec)?; + { + let mut guard = self + .prepared_plan + .lock() + .map_err(|e| internal_datafusion_err!("Failed to lock prepared plan: {}", e))?; + *guard = Some(prepared.clone()); + } + + prepared.execute(partition, context) } } diff --git a/src/execution_plans/metrics.rs b/src/execution_plans/metrics.rs index b424c67a..c31a80e5 100644 --- a/src/execution_plans/metrics.rs +++ b/src/execution_plans/metrics.rs @@ -5,33 +5,27 @@ use datafusion::error::Result; use datafusion::execution::{SendableRecordBatchStream, TaskContext}; use datafusion::physical_plan::ExecutionPlan; use datafusion::physical_plan::{DisplayAs, DisplayFormatType, PlanProperties}; +use delegate::delegate; use std::any::Any; use std::fmt::{Debug, Formatter}; /// A transparent wrapper that delegates all execution to its child but returns custom metrics. This node is invisible during display. /// The structure of a plan tree is closely tied to the [TaskMetricsRewriter]. -pub struct MetricsWrapperExec { +pub(crate) struct MetricsWrapperExec { inner: Arc, /// metrics for this plan node. metrics: MetricsSet, - /// children is initially None. When used by the [TaskMetricsRewriter], the children will be updated - /// to point at other wrapped nodes. - children: Option>>, } impl MetricsWrapperExec { - pub fn new(inner: Arc, metrics: MetricsSet) -> Self { - Self { - inner, - metrics, - children: None, - } + pub(crate) fn new(inner: Arc, metrics: MetricsSet) -> Self { + Self { inner, metrics } } } /// MetricsWrapperExec is invisible during display. impl DisplayAs for MetricsWrapperExec { - fn fmt_as(&self, t: DisplayFormatType, f: &mut std::fmt::Formatter) -> std::fmt::Result { + fn fmt_as(&self, t: DisplayFormatType, f: &mut Formatter) -> std::fmt::Result { self.inner.fmt_as(t, f) } } @@ -44,23 +38,16 @@ impl Debug for MetricsWrapperExec { } impl ExecutionPlan for MetricsWrapperExec { - fn name(&self) -> &str { - "MetricsWrapperExec" - } - - fn as_any(&self) -> &dyn Any { - self - } - - fn properties(&self) -> &PlanProperties { - unimplemented!("MetricsWrapperExec does not implement properties") + delegate! { + to self.inner { + fn name(&self) -> &str; + fn properties(&self) -> &PlanProperties; + fn as_any(&self) -> &dyn Any; + } } fn children(&self) -> Vec<&Arc> { - match &self.children { - Some(children) => children.iter().collect(), - None => self.inner.children(), - } + self.inner.children() } fn with_new_children( @@ -68,9 +55,8 @@ impl ExecutionPlan for MetricsWrapperExec { children: Vec>, ) -> Result> { Ok(Arc::new(MetricsWrapperExec { - inner: self.inner.clone(), + inner: Arc::clone(&self.inner).with_new_children(children.clone())?, metrics: self.metrics.clone(), - children: Some(children), })) } diff --git a/src/execution_plans/mod.rs b/src/execution_plans/mod.rs index 6a64763c..cdf27ae1 100644 --- a/src/execution_plans/mod.rs +++ b/src/execution_plans/mod.rs @@ -1,12 +1,18 @@ +mod broadcast; +mod children_isolator_union; mod common; mod distributed; mod metrics; +mod network_broadcast; mod network_coalesce; mod network_shuffle; mod partition_isolator; +pub use broadcast::BroadcastExec; +pub use children_isolator_union::ChildrenIsolatorUnionExec; pub use distributed::DistributedExec; -pub use metrics::MetricsWrapperExec; -pub use network_coalesce::{NetworkCoalesceExec, NetworkCoalesceReady}; -pub use network_shuffle::{NetworkShuffleExec, NetworkShuffleReadyExec}; +pub(crate) use metrics::MetricsWrapperExec; +pub use network_broadcast::NetworkBroadcastExec; +pub use network_coalesce::NetworkCoalesceExec; +pub use network_shuffle::NetworkShuffleExec; pub use partition_isolator::PartitionIsolatorExec; diff --git a/src/execution_plans/network_broadcast.rs b/src/execution_plans/network_broadcast.rs new file mode 100644 index 00000000..119267f1 --- /dev/null +++ b/src/execution_plans/network_broadcast.rs @@ -0,0 +1,271 @@ +use crate::DistributedTaskContext; +use crate::common::require_one_child; +use crate::distributed_planner::NetworkBoundary; +use crate::flight_service::WorkerConnectionPool; +use crate::metrics::proto::MetricsSetProto; +use crate::protobuf::{AppMetadata, StageKey}; +use crate::stage::{MaybeEncodedPlan, Stage}; +use dashmap::DashMap; +use datafusion::common::internal_datafusion_err; +use datafusion::error::DataFusionError; +use datafusion::execution::{SendableRecordBatchStream, TaskContext}; +use datafusion::physical_expr_common::metrics::MetricsSet; +use datafusion::physical_plan::stream::RecordBatchStreamAdapter; +use datafusion::physical_plan::{ + DisplayAs, DisplayFormatType, ExecutionPlan, Partitioning, PlanProperties, +}; +use std::any::Any; +use std::fmt::Formatter; +use std::sync::Arc; +use uuid::Uuid; + +/// Network boundary for broadcasting data to all consumer tasks. +/// +/// This operator works with [BroadcastExec] which scales up partitions so each +/// consumer task fetches a unique set of partition numbers. Each partition request +/// is sent to all stage tasks because PartitionIsolatorExec maps the same logical +/// partition to different actual data on each task. +/// +/// Here are some examples of how [NetworkBroadcastExec] distributes data: +/// +/// # 1 to many +/// +/// ```text +/// ┌────────────────────────┐ ┌────────────────────────┐ ■ +/// │ NetworkBroadcastExec │ │ NetworkBroadcastExec │ │ +/// │ (task 1) │ ... │ (task M) │ │ +/// │ │ │ │ Stage N +/// │ Populates Caches │ │ Populates Caches │ │ +/// └────────┬─┬┬─┬┬─┬───────┘ └────────┬─┬┬─┬┬─┬───────┘ │ +/// │0││1││2│ │0││1││2│ │ +/// └▲┘└▲┘└▲┘ └▲┘└▲┘└▲┘ ■ +/// │ │ │ │ │ │ +/// │ │ │ │ │ │ +/// │ │ │ │ │ │ +/// │ │ └─────────────┐ ┌──────────────────┘ │ │ +/// │ └─────────────┐ │ │ ┌───────────────┘ │ +/// └─────────────┐ │ │ │ │ ┌─────────────┘ +/// │ │ │ │ │ │ +/// ┌┴┐┌┴┐┌┴┐ ... ┌───┴┐┌───┴┐┌──┴─┐ +/// │1││2││3│ │NM-3││NM-2││NM-1│ ■ +/// ┌┴─┴┴─┴┴─┴─────┴────┴┴────┴┴────┴─┐ │ +/// │ BroadcastExec │ │ +/// │ ┌───────────────┐ │ Stage N-1 +/// │ │ Batch Cache │ │ │ +/// │ │ ┌─┐ ┌─┐ ┌─┐ │ │ │ +/// │ │ │0│ │1│ │2│ │ │ │ +/// │ │ └─┘ └─┘ └─┘ │ │ │ +/// │ └───────────────┘ │ │ +/// └───────────┬─┬─┬─┬─┬─┬───────────┘ │ +/// │0│ │1│ │2│ │ +/// └▲┘ └▲┘ └▲┘ ■ +/// │ │ │ +/// │ │ │ +/// │ │ │ +/// ┌┴┐ ┌┴┐ ┌┴┐ ■ +/// │0│ │1│ │2│ │ +/// ┌──────┴─┴─┴─┴─┴─┴──────┐ Stage N-2 +/// │Arc │ │ +/// │ (task 1) │ │ +/// └───────────────────────┘ ■ +/// ``` +/// +/// # Many to many +/// +/// ```text +/// ┌────────────────────────┐ ┌────────────────────────┐ ■ +/// │ NetworkBroadcastExec │ │ NetworkBroadcastExec │ │ +/// │ (task 1) │ │ (task M) │ │ +/// │ │ ... │ │ Stage N +/// │ Populates Caches │ │ Cache Hits │ │ +/// └────────┬─┬┬─┬┬─┬───────┘ └────────┬─┬┬─┬┬─┬───────┘ │ +/// │0││1││2│ │0││1││2│ │ +/// └▲┘└▲┘└▲┘ └▲┘└▲┘└▲┘ ■ +/// │ │ │ │ │ │ +/// ┌──────────┴──┼──┼────────────────────────────────┐ │ │ │ +/// │ ┌──────────┴──┼────────────────────────────────┼──┐ │ │ │ +/// │ │ ┌──────────┴────────────────────────────────┼──┼──┐ │ │ │ +/// │ │ │ │ │ │ │ │ │ +/// │ │ │ ┌─────────────────────────────────┼──┼──┼────┴──┼─┐│ +/// │ │ │ │ ┌───────────────────────────┼──┼──┼───────┴─┼┼─────┐ +/// │ │ │ │ │ ┌─────────────────────┼──┼──┼─────────┼┴─────┼────┐ +/// │ │ │ │ │ │ │ │ │ │ │ │ +/// ┌┴┐┌┴┐┌┴┐ ... ┌──┴─┐┌──┴─┐┌──┴─┐ ┌┴┐┌┴┐┌┴┐ ... ┌──┴─┐┌───┴┐┌──┴─┐ ■ +/// │0││1││2│ │3M-3││3M-2││3M-1│ │0││1││2│ │3M-3││3M-2││3M-1│ │ +/// ┌┴─┴┴─┴┴─┴─────┴────┴┴────┴┴────┴┐ ┌┴─┴┴─┴┴─┴─────┴────┴┴────┴┴────┴┐ │ +/// │ BroadcastExec │ │ BroadcastExec │ │ +/// │ ┌───────────────┐ │ │ ┌───────────────┐ │ │ +/// │ │ Batch Cache │ │ │ │ Batch Cache │ │ │ +/// │ │ ┌─┐ ┌─┐ ┌─┐ │ │ ... │ │ ┌─┐ ┌─┐ ┌─┐ │ │ Stage N-1 +/// │ │ │0│ │1│ │2│ │ │ │ │ │0│ │1│ │2│ │ │ │ +/// │ │ └─┘ └─┘ └─┘ │ │ │ │ └─┘ └─┘ └─┘ │ │ │ +/// │ └───────────────┘ │ │ └───────────────┘ │ │ +/// └───────────┬─┬─┬─┬─┬─┬──────────┘ └───────────┬─┬─┬─┬─┬─┬──────────┘ │ +/// │0│ │1│ │2│ │0│ │1│ │2│ │ +/// └▲┘ └▲┘ └▲┘ └▲┘ └▲┘ └▲┘ ■ +/// │ │ │ │ │ │ +/// │ │ │ │ │ │ +/// │ │ │ │ │ │ +/// ┌┴┐ ┌┴┐ ┌┴┐ ┌┴┐ ┌┴┐ ┌┴┐ ■ +/// │0│ │1│ │2│ │0│ │1│ │2│ │ +/// ┌──────┴─┴─┴─┴─┴─┴──────┐ ┌──────┴─┴─┴─┴─┴─┴──────┐ Stage N-2 +/// │Arc │ ... │Arc │ │ +/// │ (task 1) │ │ (task N) │ │ +/// └───────────────────────┘ └───────────────────────┘ ■ +/// ``` +/// +/// Notice in this diagram that each [NetworkBroadcastExec] sends a request to fetch data from each +/// [BroadcastExec] in the stage below per partition. This is because each [BroadcastExec] has its +/// own cache which contains partial results for the partition. It is the [NetworkBroadcastExec]'s +/// job to merge these partial partitions to then broadcast complete data to the consumers. +#[derive(Debug, Clone)] +pub struct NetworkBroadcastExec { + pub(crate) properties: PlanProperties, + pub(crate) input_stage: Stage, + pub(crate) worker_connections: WorkerConnectionPool, + pub(crate) metrics_collection: Arc>>, +} + +impl NetworkBroadcastExec { + /// Creates a [NetworkBroadcastExec]. + /// + /// Extracts its child, a BroadcastExec, and creates a new BroadcastExec with + /// the correct consumer_task_count. + pub fn try_new( + input: Arc, + query_id: Uuid, + stage_num: usize, + consumer_task_count: usize, + input_task_count: usize, + ) -> Result { + let Some(broadcast) = input.as_any().downcast_ref::() else { + return Err(internal_datafusion_err!( + "NetworkBroadcastExec requires a BroadcastExec input, found: {}", + input.name() + )); + }; + + let child = require_one_child(broadcast.children())?; + let input_partition_count = child.properties().partitioning.partition_count(); + let broadcast_exec: Arc = + Arc::new(super::BroadcastExec::new(child, consumer_task_count)); + + let input_stage = Stage::new(query_id, stage_num, broadcast_exec, input_task_count); + let properties = input_stage + .plan + .decoded()? + .properties() + .clone() + .with_partitioning(Partitioning::UnknownPartitioning(input_partition_count)); + + Ok(Self { + properties, + input_stage, + worker_connections: WorkerConnectionPool::new(input_task_count), + metrics_collection: Default::default(), + }) + } +} + +impl NetworkBoundary for NetworkBroadcastExec { + fn with_input_stage( + &self, + input_stage: Stage, + ) -> Result, DataFusionError> { + let mut self_clone = self.clone(); + self_clone.input_stage = input_stage; + Ok(Arc::new(self_clone)) + } + + fn input_stage(&self) -> &Stage { + &self.input_stage + } +} + +impl DisplayAs for NetworkBroadcastExec { + fn fmt_as(&self, _t: DisplayFormatType, f: &mut Formatter) -> std::fmt::Result { + let input_tasks = self.input_stage.tasks.len(); + let stage = self.input_stage.num; + let consumer_partitions = self.properties.partitioning.partition_count(); + let stage_partitions = self + .input_stage + .plan + .decoded() + .map(|p| p.properties().partitioning.partition_count()) + .unwrap_or(0); + write!( + f, + "[Stage {stage}] => NetworkBroadcastExec: partitions_per_consumer={consumer_partitions}, stage_partitions={stage_partitions}, input_tasks={input_tasks}", + ) + } +} + +impl ExecutionPlan for NetworkBroadcastExec { + fn name(&self) -> &str { + "NetworkBroadcastExec" + } + + fn as_any(&self) -> &dyn Any { + self + } + + fn properties(&self) -> &PlanProperties { + &self.properties + } + + fn children(&self) -> Vec<&Arc> { + match &self.input_stage.plan { + MaybeEncodedPlan::Decoded(plan) => vec![plan], + MaybeEncodedPlan::Encoded(_) => vec![], + } + } + + fn with_new_children( + self: Arc, + children: Vec>, + ) -> Result, DataFusionError> { + let mut self_clone = self.as_ref().clone(); + self_clone.input_stage.plan = MaybeEncodedPlan::Decoded(require_one_child(children)?); + Ok(Arc::new(self_clone)) + } + + fn execute( + &self, + partition: usize, + context: Arc, + ) -> Result { + let task_context = DistributedTaskContext::from_ctx(&context); + let off = self.properties.partitioning.partition_count() * task_context.task_index; + let mut streams = Vec::with_capacity(self.input_stage.tasks.len()); + + for input_task_index in 0..self.input_stage.tasks.len() { + let worker_connection = self.worker_connections.get_or_init_worker_connection( + &self.input_stage, + off..(off + self.properties.partitioning.partition_count()), + input_task_index, + &context, + )?; + + let metrics_collection = Arc::clone(&self.metrics_collection); + let stream = worker_connection.stream_partition(off + partition, move |meta| { + if let Some(AppMetadata::MetricsCollection(m)) = meta.content { + for task_metrics in m.tasks { + if let Some(stage_key) = task_metrics.stage_key { + metrics_collection.insert(stage_key, task_metrics.metrics); + }; + } + } + })?; + streams.push(stream); + } + + Ok(Box::pin(RecordBatchStreamAdapter::new( + self.schema(), + futures::stream::select_all(streams), + ))) + } + + fn metrics(&self) -> Option { + Some(self.worker_connections.metrics.clone_inner()) + } +} diff --git a/src/execution_plans/network_coalesce.rs b/src/execution_plans/network_coalesce.rs index 3a873ed1..8ff9b91f 100644 --- a/src/execution_plans/network_coalesce.rs +++ b/src/execution_plans/network_coalesce.rs @@ -1,40 +1,31 @@ -use crate::channel_resolver_ext::get_distributed_channel_resolver; -use crate::config_extension_ext::ContextGrpcMetadata; -use crate::distributed_physical_optimizer_rule::{ - InputStageInfo, NetworkBoundary, limit_tasks_err, -}; -use crate::execution_plans::common::{require_one_child, scale_partitioning_props}; -use crate::flight_service::DoGet; -use crate::metrics::MetricsCollectingStream; +use crate::common::require_one_child; +use crate::distributed_planner::NetworkBoundary; +use crate::execution_plans::common::scale_partitioning_props; +use crate::flight_service::WorkerConnectionPool; use crate::metrics::proto::MetricsSetProto; -use crate::protobuf::{StageKey, map_flight_to_datafusion_error, map_status_to_datafusion_error}; +use crate::protobuf::{AppMetadata, StageKey}; use crate::stage::{MaybeEncodedPlan, Stage}; -use crate::{ChannelResolver, DistributedTaskContext}; -use arrow_flight::Ticket; -use arrow_flight::decode::FlightRecordBatchStream; -use arrow_flight::error::FlightError; -use bytes::Bytes; +use crate::{DistributedTaskContext, ExecutionTask}; use dashmap::DashMap; -use datafusion::common::{exec_err, internal_err, plan_err}; -use datafusion::datasource::schema_adapter::DefaultSchemaAdapterFactory; -use datafusion::error::DataFusionError; +use datafusion::common::{exec_err, plan_err}; +use datafusion::error::Result; use datafusion::execution::{SendableRecordBatchStream, TaskContext}; +use datafusion::physical_expr_common::metrics::MetricsSet; use datafusion::physical_plan::stream::RecordBatchStreamAdapter; -use datafusion::physical_plan::{DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties}; -use futures::{StreamExt, TryFutureExt, TryStreamExt}; -use http::Extensions; -use prost::Message; +use datafusion::physical_plan::{ + DisplayAs, DisplayFormatType, EmptyRecordBatchStream, ExecutionPlan, PlanProperties, + internal_err, +}; use std::any::Any; -use std::fmt::Formatter; +use std::fmt::{Debug, Formatter}; use std::sync::Arc; -use tonic::Request; -use tonic::metadata::MetadataMap; +use uuid::Uuid; -/// [ExecutionPlan] that coalesces partitions from multiple tasks into a single task without +/// [ExecutionPlan] that coalesces partitions from multiple tasks into a one or more task without /// performing any repartition, and maintaining the same partitioning scheme. /// /// This is the equivalent of a [CoalescePartitionsExec] but coalescing tasks across the network -/// into one. +/// between distributed stages. /// /// ```text /// ┌───────────────────────────┐ ■ @@ -56,139 +47,103 @@ use tonic::metadata::MetadataMap; /// /// The communication between two stages across a [NetworkCoalesceExec] has two implications: /// -/// - Stage N+1 must have exactly 1 task. The distributed planner ensures this is true. -/// - The amount of partitions in the single task of Stage N+1 is equal to the sum of all -/// partitions in all tasks in Stage N+1 (e.g. (1,2,3,4,5,6,7,8,9) = (1,2,3)+(4,5,6)+(7,8,9) ) +/// - Stage N+1 may have one or more tasks. Each consumer task reads a contiguous group of upstream +/// tasks from Stage N. +/// - Output partitioning for Stage N+1 is sized based on the maximum upstream-group size. When +/// groups are uneven, consumer tasks with smaller groups return empty streams for the “extra” +/// partitions. +/// ```text +/// ┌───────────────────────────┐ ┌───────────────────────────┐ ■ +/// │ NetworkCoalesceExec │ │ NetworkCoalesceExec │ │ +/// │ (task 1) │ │ (task 2) │ │ +/// └┬─┬┬─┬┬─┬┬─┬┬─┬┬─┬─────────┘ └┬─┬┬─┬┬─┬┬─┬┬─┬┬─┬─────────┘ Stage N+1 +/// │1││2││3││4││5││6│ │7││8││9││_││_││_│ │ +/// └─┘└─┘└─┘└─┘└─┘└─┘ └─┘└─┘└─┘└─┘└─┘└─┘ │ +/// ▲ ▲ ▲ ▲ ▲ ▲ ▲ ▲ ▲ ■ +/// ┌──┬──┬────────────┴──┴──┘ └──┴──┴─────┬──┬──┐ └──┴──┴────────────────┬──┬──┐ +/// │ │ │ │ │ │ │ │ │ ■ +/// ┌─┐┌─┐┌─┐ ┌─┐┌─┐┌─┐ ┌─┐┌─┐┌─┐ │ +/// │1││2││3│ │4││5││6│ │7││8││9│ │ +/// ┌┴─┴┴─┴┴─┴──────────────────┐ ┌─────────┴─┴┴─┴┴─┴─────────┐ ┌──────────────────┴─┴┴─┴┴─┴┐ Stage N +/// │ Arc │ │ Arc │ │ Arc │ │ +/// │ (task 1) │ │ (task 2) │ │ (task 3) │ │ +/// └───────────────────────────┘ └───────────────────────────┘ └───────────────────────────┘ ■ +/// ``` /// /// This node has two variants. -/// 1. Pending: it acts as a placeholder for the distributed optimization step to mark it as ready. +/// 1. Pending: acts as a placeholder for the distributed optimization step to mark it as ready. /// 2. Ready: runs within a distributed stage and queries the next input stage over the network /// using Arrow Flight. #[derive(Debug, Clone)] -pub enum NetworkCoalesceExec { - Pending(NetworkCoalescePending), - Ready(NetworkCoalesceReady), -} - -/// Placeholder version of the [NetworkCoalesceExec] node. It acts as a marker for the -/// distributed optimization step, which will replace it with the appropriate -/// [NetworkCoalesceReady] node. -#[derive(Debug, Clone)] -pub struct NetworkCoalescePending { - properties: PlanProperties, - input_tasks: usize, - input: Arc, -} - -/// Ready version of the [NetworkCoalesceExec] node. This node can be created in -/// just two ways: -/// - by the distributed optimization step based on an original [NetworkCoalescePending] -/// - deserialized from a protobuf plan sent over the network. -#[derive(Debug, Clone)] -pub struct NetworkCoalesceReady { +pub struct NetworkCoalesceExec { /// the properties we advertise for this execution plan pub(crate) properties: PlanProperties, pub(crate) input_stage: Stage, - /// metrics_collection is used to collect metrics from child tasks. It is empty when an - /// is instantiated (deserialized, created via [NetworkCoalesceExec::new_ready] etc...). - /// Metrics are populated in this map via [NetworkCoalesceExec::execute]. + pub(crate) worker_connections: WorkerConnectionPool, + /// metrics_collection is used to collect metrics from child tasks. It is initially + /// instantiated as an empty [DashMap] (see `try_decode` in `distributed_codec.rs`). + /// Metrics are populated here via [NetworkCoalesceExec::execute]. /// /// An instance may receive metrics for 0 to N child tasks, where N is the number of tasks in - /// the stage it is reading from. This is because, by convention, the ArrowFlightEndpoint - /// sends metrics for a task to the last NetworkCoalesceExec to read from it, which may or may - /// not be this instance. + /// the stage it is reading from. This is because, by convention, the Worker sends metrics for + /// a task to the last NetworkCoalesceExec to read from it, which may or may not be this + /// instance. pub(crate) metrics_collection: Arc>>, } impl NetworkCoalesceExec { /// Builds a new [NetworkCoalesceExec] in "Pending" state. /// - /// Typically, this node should be place right after nodes that coalesce all the input + /// Typically, this node should be placed right after nodes that coalesce all the input /// partitions into one, for example: /// - [CoalescePartitionsExec] /// - [SortPreservingMergeExec] - pub fn new(input: Arc, input_tasks: usize) -> Self { - Self::Pending(NetworkCoalescePending { - properties: input.properties().clone(), - input_tasks, - input, - }) - } -} - -impl NetworkBoundary for NetworkCoalesceExec { - fn get_input_stage_info(&self, n_tasks: usize) -> Result { - let Self::Pending(pending) = self else { - return plan_err!("can only return wrapped child if on Pending state"); - }; - - // As this node coalesces multiple tasks into 1, it must run in a stage with 1 task. - if n_tasks > 1 { - return Err(limit_tasks_err(1)); + pub fn try_new( + input: Arc, + query_id: Uuid, + num: usize, + task_count: usize, + input_task_count: usize, + ) -> Result { + if task_count == 0 { + return plan_err!("NetworkCoalesceExec cannot be executed with task_count=0"); } - Ok(InputStageInfo { - plan: Arc::clone(&pending.input), - task_count: pending.input_tasks, + // Each output task coalesces a group of input tasks. We size the output partition count + // per output task based on the maximum group size, returning empty streams for tasks with + // smaller groups. + let max_input_task_count = input_task_count.div_ceil(task_count).max(1); + Ok(Self { + properties: scale_partitioning_props(input.properties(), |p| p * max_input_task_count), + input_stage: Stage { + query_id, + num, + plan: MaybeEncodedPlan::Decoded(input), + tasks: vec![ExecutionTask { url: None }; input_task_count], + }, + worker_connections: WorkerConnectionPool::new(input_task_count), + metrics_collection: Default::default(), }) } +} - fn with_input_stage( - &self, - input_stage: Stage, - ) -> Result, DataFusionError> { - match self { - Self::Pending(pending) => { - let properties = input_stage.plan.decoded()?.properties(); - let ready = NetworkCoalesceReady { - properties: scale_partitioning_props(properties, |p| p * pending.input_tasks), - input_stage, - metrics_collection: Default::default(), - }; - - Ok(Arc::new(Self::Ready(ready))) - } - Self::Ready(ready) => { - let mut ready = ready.clone(); - ready.input_stage = input_stage; - Ok(Arc::new(Self::Ready(ready))) - } - } - } - - fn input_stage(&self) -> Option<&Stage> { - match self { - Self::Pending(_) => None, - Self::Ready(v) => Some(&v.input_stage), - } +impl NetworkBoundary for NetworkCoalesceExec { + fn input_stage(&self) -> &Stage { + &self.input_stage } - fn with_input_task_count( - &self, - input_tasks: usize, - ) -> Result, DataFusionError> { - Ok(Arc::new(match self { - Self::Pending(pending) => Self::Pending(NetworkCoalescePending { - properties: pending.properties.clone(), - input_tasks, - input: pending.input.clone(), - }), - Self::Ready(_) => { - plan_err!("Self can only re-assign input tasks if in 'Pending' state")? - } - })) + fn with_input_stage(&self, input_stage: Stage) -> Result> { + let mut self_clone = self.clone(); + self_clone.input_stage = input_stage; + Ok(Arc::new(self_clone)) } } impl DisplayAs for NetworkCoalesceExec { fn fmt_as(&self, _t: DisplayFormatType, f: &mut Formatter) -> std::fmt::Result { - let Self::Ready(self_ready) = self else { - return write!(f, "NetworkCoalesceExec"); - }; - - let input_tasks = self_ready.input_stage.tasks.len(); - let partitions = self_ready.properties.partitioning.partition_count(); - let stage = self_ready.input_stage.num; + let input_tasks = self.input_stage.tasks.len(); + let partitions = self.properties.partitioning.partition_count(); + let stage = self.input_stage.num; write!( f, "[Stage {stage}] => NetworkCoalesceExec: output_partitions={partitions}, input_tasks={input_tasks}", @@ -206,127 +161,306 @@ impl ExecutionPlan for NetworkCoalesceExec { } fn properties(&self) -> &PlanProperties { - match self { - NetworkCoalesceExec::Pending(v) => &v.properties, - NetworkCoalesceExec::Ready(v) => &v.properties, - } + &self.properties } fn children(&self) -> Vec<&Arc> { - match self { - NetworkCoalesceExec::Pending(v) => vec![&v.input], - NetworkCoalesceExec::Ready(v) => match &v.input_stage.plan { - MaybeEncodedPlan::Decoded(v) => vec![v], - MaybeEncodedPlan::Encoded(_) => vec![], - }, + match &self.input_stage.plan { + MaybeEncodedPlan::Decoded(v) => vec![v], + MaybeEncodedPlan::Encoded(_) => vec![], } } fn with_new_children( self: Arc, children: Vec>, - ) -> Result, DataFusionError> { - match self.as_ref() { - Self::Pending(v) => { - let mut v = v.clone(); - v.input = require_one_child(children)?; - Ok(Arc::new(Self::Pending(v))) - } - Self::Ready(v) => { - let mut v = v.clone(); - v.input_stage.plan = MaybeEncodedPlan::Decoded(require_one_child(children)?); - Ok(Arc::new(Self::Ready(v))) - } - } + ) -> Result> { + let mut self_clone = self.as_ref().clone(); + self_clone.input_stage.plan = MaybeEncodedPlan::Decoded(require_one_child(children)?); + Ok(Arc::new(self_clone)) } fn execute( &self, partition: usize, context: Arc, - ) -> Result { - let NetworkCoalesceExec::Ready(self_ready) = self else { + ) -> Result { + let task_context = DistributedTaskContext::from_ctx(&context); + if task_context.task_index >= task_context.task_count { return exec_err!( - "NetworkCoalesceExec is not ready, was the distributed optimization step performed?" + "NetworkCoalesceExec invalid task context: task_index={} >= task_count={}", + task_context.task_index, + task_context.task_count ); - }; - - // get the channel manager and current stage from our context - let channel_resolver = get_distributed_channel_resolver(context.session_config())?; - - let input_stage = &self_ready.input_stage; - let encoded_input_plan = input_stage.plan.encoded()?; + } - let context_headers = ContextGrpcMetadata::headers_from_ctx(&context); - let task_context = DistributedTaskContext::from_ctx(&context); - if task_context.task_index > 0 { - return exec_err!("NetworkCoalesceExec cannot be executed in more than one task"); + let partitions_per_task = self + .properties() + .partitioning + .partition_count() + .checked_div( + self.input_stage + .tasks + .len() + .div_ceil(task_context.task_count) + .max(1), + ) + .unwrap_or(0); + if partitions_per_task == 0 { + return exec_err!("NetworkCoalesceExec has 0 partitions per input task"); } - let partitions_per_task = - self.properties().partitioning.partition_count() / input_stage.tasks.len(); + let input_task_count = self.input_stage.tasks.len(); + let group = task_group( + input_task_count, + task_context.task_index, + task_context.task_count, + ); - let target_task = partition / partitions_per_task; + let input_task_offset = partition / partitions_per_task; let target_partition = partition % partitions_per_task; - let ticket = Request::from_parts( - MetadataMap::from_headers(context_headers.clone()), - Extensions::default(), - Ticket { - ticket: DoGet { - plan_proto: encoded_input_plan.clone(), - target_partition: target_partition as u64, - stage_key: Some(StageKey { - query_id: Bytes::from(input_stage.query_id.as_bytes().to_vec()), - stage_id: input_stage.num as u64, - task_number: target_task as u64, - }), - target_task_index: target_task as u64, - target_task_count: input_stage.tasks.len() as u64, - } - .encode_to_vec() - .into(), - }, - ); + // Some consumer tasks are assigned fewer upstream tasks when + // `input_task_count % task_count != 0` (uneven grouping). + // We still size partitions based on the maximum group size, so partitions that + // would map to a missing upstream task slot are treated as padding and return + // an empty stream (no network call). + if input_task_offset >= group.len { + return Ok(Box::pin(EmptyRecordBatchStream::new(self.schema()))); + } - let Some(task) = input_stage.tasks.get(target_task) else { - return internal_err!("ProgrammingError: Task {target_task} not found"); - }; + // This should never happen. + if input_task_offset >= group.max_len { + return internal_err!( + "NetworkCoalesceExec input_task_offset={} >= group.max_len={}", + input_task_offset, + group.max_len + ); + } - let Some(url) = task.url.clone() else { - return internal_err!("NetworkCoalesceExec: task is unassigned, cannot proceed"); - }; + let target_task = group.start_task + input_task_offset; - let metrics_collection_capture = self_ready.metrics_collection.clone(); - let adapter = DefaultSchemaAdapterFactory::from_schema(self.schema()); - let (mapper, _indices) = adapter.map_schema(&self.schema())?; - let stream = async move { - let mut client = channel_resolver.get_flight_client_for_url(&url).await?; - let stream = client - .do_get(ticket) - .await - .map_err(map_status_to_datafusion_error)? - .into_inner() - .map_err(|err| FlightError::Tonic(Box::new(err))); - - let metrics_collecting_stream = - MetricsCollectingStream::new(stream, metrics_collection_capture); - - Ok( - FlightRecordBatchStream::new_from_flight_data(metrics_collecting_stream) - .map_err(map_flight_to_datafusion_error) - .map(move |batch| { - let batch = batch?; - - mapper.map_batch(batch) - }), - ) - } - .try_flatten_stream(); + let worker_connection = self.worker_connections.get_or_init_worker_connection( + &self.input_stage, + 0..partitions_per_task, + target_task, + &context, + )?; + + let metrics_collection = Arc::clone(&self.metrics_collection); + + let stream = worker_connection.stream_partition(target_partition, move |meta| { + if let Some(AppMetadata::MetricsCollection(m)) = meta.content { + for task_metrics in m.tasks { + if let Some(stage_key) = task_metrics.stage_key { + metrics_collection.insert(stage_key, task_metrics.metrics); + }; + } + } + })?; Ok(Box::pin(RecordBatchStreamAdapter::new( self.schema(), stream, ))) } + + fn metrics(&self) -> Option { + Some(self.worker_connections.metrics.clone_inner()) + } +} + +#[derive(Debug, Clone, Copy)] +struct TaskGroup { + /// The first input task index in this group. + start_task: usize, + /// The number of input tasks in this group. + len: usize, + /// The maximum possible group size across all groups. + /// + /// When groups are uneven (input_tasks % task_count != 0), some groups are shorter. We still + /// size the output partitioning based on this max and return empty streams for the extra + /// partitions in smaller groups. + max_len: usize, +} + +/// Returns the contiguous group of input tasks assigned to DistributedTaskContext::task_index. +fn task_group(input_task_count: usize, task_index: usize, task_count: usize) -> TaskGroup { + if task_count == 0 { + return TaskGroup { + start_task: 0, + len: 0, + max_len: 0, + }; + } + + // Split `input_task_count` into `task_count` contiguous groups. + // - base_tasks_per_group: floor(input_task_count / task_count) + // - groups_with_extra_task: first N groups that get one extra task (remainder) + let base_tasks_per_group = input_task_count / task_count; + let groups_with_extra_task = input_task_count % task_count; + + let len = base_tasks_per_group + usize::from(task_index < groups_with_extra_task); + let start_task = (task_index * base_tasks_per_group) + task_index.min(groups_with_extra_task); + let max_len = base_tasks_per_group + usize::from(groups_with_extra_task > 0); + + TaskGroup { + start_task, + len, + max_len, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use datafusion::arrow::datatypes::Schema; + use datafusion::physical_plan::empty::EmptyExec; + + #[derive(Clone, Copy)] + struct Case { + name: &'static str, + input_tasks: usize, + consumer_tasks: usize, + } + + fn expected_groups(input_tasks: usize, consumer_tasks: usize) -> Vec<(usize, usize)> { + assert!(consumer_tasks > 0, "consumer_tasks must be non-zero"); + + let base_tasks_per_group = input_tasks / consumer_tasks; + let groups_with_extra_task = input_tasks % consumer_tasks; + let mut groups = Vec::with_capacity(consumer_tasks); + let mut start_task = 0; + + for task_index in 0..consumer_tasks { + let len = base_tasks_per_group + usize::from(task_index < groups_with_extra_task); + groups.push((start_task, len)); + start_task += len; + } + + groups + } + + fn assert_case(case: Case) -> Result<()> { + const STAGE_NUM: usize = 1; + + // Child plan used only for properties/schema (we won't reach network codepaths). + let child: Arc = Arc::new(EmptyExec::new(Arc::new(Schema::empty()))); + let child_partitions = child.properties().partitioning.partition_count(); + + let exec = NetworkCoalesceExec::try_new( + Arc::clone(&child), + Uuid::nil(), + STAGE_NUM, + case.consumer_tasks, + case.input_tasks, + )?; + + // Output partitions are sized by the maximum group size. + let max_group_size = case.input_tasks.div_ceil(case.consumer_tasks).max(1); + assert_eq!( + exec.properties().partitioning.partition_count(), + child_partitions * max_group_size + ); + + let groups = expected_groups(case.input_tasks, case.consumer_tasks); + assert_eq!(groups.len(), case.consumer_tasks); + + let mut seen = vec![false; case.input_tasks]; + let mut expected_start = 0; + let mut padding_slots = 0; + + for (index, (start, len)) in groups.into_iter().enumerate() { + assert_eq!( + start, expected_start, + "case {} group {} should be contiguous", + case.name, index + ); + assert!( + start + len <= case.input_tasks, + "case {} group {} exceeds input task count", + case.name, + index + ); + + for (offset, seen_task) in seen.iter_mut().skip(start).take(len).enumerate() { + let task = start + offset; + assert!( + !*seen_task, + "case {} input task {} appears twice", + case.name, task + ); + *seen_task = true; + } + + expected_start = start + len; + padding_slots += max_group_size - len; + } + + assert_eq!( + expected_start, case.input_tasks, + "case {} groups should cover all input tasks", + case.name + ); + assert!( + seen.iter().all(|v| *v), + "case {} missing at least one input task", + case.name + ); + + let total_slots = case.consumer_tasks * max_group_size; + let total_padding = total_slots - case.input_tasks; + assert_eq!( + padding_slots, total_padding, + "case {} padding slots mismatch", + case.name + ); + + Ok(()) + } + + const ONE_TO_MANY_INPUT: usize = 1; + const ONE_TO_MANY_OUTPUT: usize = 3; + const MANY_TO_ONE_INPUT: usize = 4; + const MANY_TO_ONE_OUTPUT: usize = 1; + const MANY_TO_FEWER_INPUT: usize = 5; + const MANY_TO_FEWER_OUTPUT: usize = 2; + const FEWER_TO_MANY_INPUT: usize = 2; + const FEWER_TO_MANY_OUTPUT: usize = 5; + + #[test] + fn validates_partition_coverage_one_to_many() -> Result<()> { + assert_case(Case { + name: "1_to_n", + input_tasks: ONE_TO_MANY_INPUT, + consumer_tasks: ONE_TO_MANY_OUTPUT, + }) + } + + #[test] + fn validates_partition_coverage_many_to_one() -> Result<()> { + assert_case(Case { + name: "n_to_1", + input_tasks: MANY_TO_ONE_INPUT, + consumer_tasks: MANY_TO_ONE_OUTPUT, + }) + } + + #[test] + fn validates_partition_coverage_many_to_fewer() -> Result<()> { + assert_case(Case { + name: "n_to_m_n_gt_m", + input_tasks: MANY_TO_FEWER_INPUT, + consumer_tasks: MANY_TO_FEWER_OUTPUT, + }) + } + + #[test] + fn validates_partition_coverage_fewer_to_many() -> Result<()> { + assert_case(Case { + name: "m_to_n_n_gt_m", + input_tasks: FEWER_TO_MANY_INPUT, + consumer_tasks: FEWER_TO_MANY_OUTPUT, + }) + } } diff --git a/src/execution_plans/network_shuffle.rs b/src/execution_plans/network_shuffle.rs index 92413c81..c574d6f1 100644 --- a/src/execution_plans/network_shuffle.rs +++ b/src/execution_plans/network_shuffle.rs @@ -1,46 +1,35 @@ -use crate::channel_resolver_ext::get_distributed_channel_resolver; -use crate::config_extension_ext::ContextGrpcMetadata; -use crate::distributed_physical_optimizer_rule::{InputStageInfo, NetworkBoundary}; -use crate::execution_plans::common::{require_one_child, scale_partitioning}; -use crate::flight_service::DoGet; -use crate::metrics::MetricsCollectingStream; +use crate::common::require_one_child; +use crate::execution_plans::common::scale_partitioning; +use crate::flight_service::WorkerConnectionPool; use crate::metrics::proto::MetricsSetProto; -use crate::protobuf::StageKey; -use crate::protobuf::{map_flight_to_datafusion_error, map_status_to_datafusion_error}; +use crate::protobuf::{AppMetadata, StageKey}; use crate::stage::{MaybeEncodedPlan, Stage}; -use crate::{ChannelResolver, DistributedTaskContext}; -use arrow_flight::Ticket; -use arrow_flight::decode::FlightRecordBatchStream; -use arrow_flight::error::FlightError; -use bytes::Bytes; +use crate::{DistributedTaskContext, ExecutionTask, NetworkBoundary}; use dashmap::DashMap; -use datafusion::common::{exec_err, internal_datafusion_err, plan_err}; -use datafusion::datasource::schema_adapter::DefaultSchemaAdapterFactory; +use datafusion::common::tree_node::{Transformed, TreeNode, TreeNodeRecursion}; +use datafusion::common::{Result, plan_err}; use datafusion::error::DataFusionError; use datafusion::execution::{SendableRecordBatchStream, TaskContext}; use datafusion::physical_expr::Partitioning; +use datafusion::physical_expr_common::metrics::MetricsSet; use datafusion::physical_plan::repartition::RepartitionExec; use datafusion::physical_plan::stream::RecordBatchStreamAdapter; use datafusion::physical_plan::{ DisplayAs, DisplayFormatType, ExecutionPlan, ExecutionPlanProperties, PlanProperties, }; -use futures::{StreamExt, TryFutureExt, TryStreamExt}; -use http::Extensions; -use prost::Message; use std::any::Any; use std::fmt::Formatter; use std::sync::Arc; -use tonic::Request; -use tonic::metadata::MetadataMap; +use uuid::Uuid; /// [ExecutionPlan] implementation that shuffles data across the network in a distributed context. /// /// The easiest way of thinking about this node is as a plan [RepartitionExec] node that is /// capable of fanning out the different produced partitions to different tasks. -/// This allows redistributing data across different tasks in different stages, that way different +/// This allows redistributing data across different tasks in different stages, so that different /// physical machines can make progress on different non-overlapping sets of data. /// -/// This node allows fanning out data from N tasks to M tasks, being N and M arbitrary non-zero +/// This node allows fanning out of data from N tasks to M tasks, with N and M being arbitrary non-zero /// positive numbers. Here are some examples of how data can be shuffled in different scenarios: /// /// # 1 to many @@ -105,46 +94,28 @@ use tonic::metadata::MetadataMap; /// /// The communication between two stages across a [NetworkShuffleExec] has two implications: /// -/// - Each task in Stage N+1 gather data from all tasks in Stage N -/// - The sum of the number of partitions in all tasks in Stage N+1 is equal to the +/// - Each task in Stage N+1 gathers data from all tasks in Stage N +/// - The total number of partitions across all tasks in Stage N+1 is equal to the /// number of partitions in a single task in Stage N. (e.g. (1,2,3,4)+(5,6,7,8) = (1,2,3,4,5,6,7,8) ) /// /// This node has two variants. -/// 1. Pending: it acts as a placeholder for the distributed optimization step to mark it as ready. +/// 1. Pending: acts as a placeholder for the distributed optimization step to mark it as ready. /// 2. Ready: runs within a distributed stage and queries the next input stage over the network /// using Arrow Flight. #[derive(Debug, Clone)] -pub enum NetworkShuffleExec { - Pending(NetworkShufflePendingExec), - Ready(NetworkShuffleReadyExec), -} - -/// Placeholder version of the [NetworkShuffleExec] node. It acts as a marker for the -/// distributed optimization step, which will replace it with the appropriate -/// [NetworkShuffleReadyExec] node. -#[derive(Debug, Clone)] -pub struct NetworkShufflePendingExec { - input: Arc, - input_tasks: usize, -} - -/// Ready version of the [NetworkShuffleExec] node. This node can be created in -/// just two ways: -/// - by the distributed optimization step based on an original [NetworkShufflePendingExec] -/// - deserialized from a protobuf plan sent over the network. -#[derive(Debug, Clone)] -pub struct NetworkShuffleReadyExec { +pub struct NetworkShuffleExec { /// the properties we advertise for this execution plan pub(crate) properties: PlanProperties, pub(crate) input_stage: Stage, - /// metrics_collection is used to collect metrics from child tasks. It is empty when an - /// is instantiated (deserialized, created via [NetworkShuffleExec::new_ready] etc...). - /// Metrics are populated in this map via [NetworkShuffleExec::execute]. + pub(crate) worker_connections: WorkerConnectionPool, + /// metrics_collection is used to collect metrics from child tasks. It is initially + /// instantiated as an empty [DashMap] (see `try_decode` in `distributed_codec.rs`). + /// Metrics are populated here via [NetworkCoalesceExec::execute]. /// /// An instance may receive metrics for 0 to N child tasks, where N is the number of tasks in - /// the stage it is reading from. This is because, by convention, the ArrowFlightEndpoint - /// sends metrics for a task to the last NetworkShuffleExec to read from it, which may or may - /// not be this instance. + /// the stage it is reading from. This is because, by convention, the Worker sends metrics for + /// a task to the last NetworkCoalesceExec to read from it, which may or may not be this + /// instance. pub(crate) metrics_collection: Arc>>, } @@ -155,93 +126,67 @@ impl NetworkShuffleExec { /// node is a [RepartitionExec] with a [Partitioning::Hash] partition scheme. pub fn try_new( input: Arc, - input_tasks: usize, + query_id: Uuid, + num: usize, + task_count: usize, + input_task_count: usize, ) -> Result { if !matches!(input.output_partitioning(), Partitioning::Hash(_, _)) { return plan_err!("NetworkShuffleExec input must be hash partitioned"); } - Ok(Self::Pending(NetworkShufflePendingExec { - input, - input_tasks, - })) - } -} - -impl NetworkBoundary for NetworkShuffleExec { - fn get_input_stage_info(&self, n_tasks: usize) -> Result { - let Self::Pending(pending) = self else { - return plan_err!("cannot only return wrapped child if on Pending state"); - }; - - // TODO: Avoid downcasting once https://github.com/apache/datafusion/pull/17990 is shipped. - let Some(r_exe) = pending.input.as_any().downcast_ref::() else { - return plan_err!("NetworkShuffleExec.input must always be RepartitionExec"); - }; - - let next_stage_plan = Arc::new(RepartitionExec::try_new( - require_one_child(r_exe.children())?, - scale_partitioning(r_exe.partitioning(), |p| p * n_tasks), - )?); - Ok(InputStageInfo { - plan: next_stage_plan, - task_count: pending.input_tasks, + let transformed = Arc::clone(&input).transform_down(|plan| { + if let Some(r_exe) = plan.as_any().downcast_ref::() { + // Scale the input RepartitionExec to account for all the tasks to which it will + // need to fan data out. + let scaled = Arc::new(RepartitionExec::try_new( + require_one_child(r_exe.children())?, + scale_partitioning(r_exe.partitioning(), |p| p * task_count), + )?); + Ok(Transformed::new(scaled, true, TreeNodeRecursion::Stop)) + } else if matches!(plan.output_partitioning(), Partitioning::Hash(_, _)) { + // This might be a passthrough node, like a CoalesceBatchesExec or something like that. + // This is fine, we can let the node be here. + Ok(Transformed::no(plan)) + } else { + return plan_err!( + "NetworkShuffleExec input must be hash partitioned, but {} is not", + plan.name() + ); + } + })?; + + Ok(Self { + input_stage: Stage { + query_id, + num, + plan: MaybeEncodedPlan::Decoded(transformed.data), + tasks: vec![ExecutionTask { url: None }; input_task_count], + }, + worker_connections: WorkerConnectionPool::new(input_task_count), + properties: input.properties().clone(), + metrics_collection: Default::default(), }) } +} - fn with_input_task_count( - &self, - input_tasks: usize, - ) -> Result, DataFusionError> { - Ok(Arc::new(match self { - Self::Pending(prev) => Self::Pending(NetworkShufflePendingExec { - input: Arc::clone(&prev.input), - input_tasks, - }), - Self::Ready(_) => plan_err!( - "NetworkShuffleExec can only re-assign input tasks if in 'Pending' state" - )?, - })) +impl NetworkBoundary for NetworkShuffleExec { + fn input_stage(&self) -> &Stage { + &self.input_stage } - fn with_input_stage( - &self, - input_stage: Stage, - ) -> Result, DataFusionError> { - match self { - Self::Pending(pending) => { - let ready = NetworkShuffleReadyExec { - properties: pending.input.properties().clone(), - input_stage, - metrics_collection: Default::default(), - }; - Ok(Arc::new(Self::Ready(ready))) - } - Self::Ready(ready) => { - let mut ready = ready.clone(); - ready.input_stage = input_stage; - Ok(Arc::new(Self::Ready(ready))) - } - } - } - - fn input_stage(&self) -> Option<&Stage> { - match self { - Self::Pending(_) => None, - Self::Ready(v) => Some(&v.input_stage), - } + fn with_input_stage(&self, input_stage: Stage) -> Result> { + let mut self_clone = self.clone(); + self_clone.input_stage = input_stage; + Ok(Arc::new(self_clone)) } } impl DisplayAs for NetworkShuffleExec { fn fmt_as(&self, _t: DisplayFormatType, f: &mut Formatter) -> std::fmt::Result { - let Self::Ready(self_ready) = self else { - return write!(f, "NetworkShuffleExec: Pending"); - }; - - let input_tasks = self_ready.input_stage.tasks.len(); - let partitions = self_ready.properties.partitioning.partition_count(); - let stage = self_ready.input_stage.num; + let input_tasks = self.input_stage.tasks.len(); + let partitions = self.properties.partitioning.partition_count(); + let stage = self.input_stage.num; write!( f, "[Stage {stage}] => NetworkShuffleExec: output_partitions={partitions}, input_tasks={input_tasks}", @@ -259,19 +204,13 @@ impl ExecutionPlan for NetworkShuffleExec { } fn properties(&self) -> &PlanProperties { - match self { - NetworkShuffleExec::Pending(v) => v.input.properties(), - NetworkShuffleExec::Ready(v) => &v.properties, - } + &self.properties } fn children(&self) -> Vec<&Arc> { - match self { - NetworkShuffleExec::Pending(v) => vec![&v.input], - NetworkShuffleExec::Ready(v) => match &v.input_stage.plan { - MaybeEncodedPlan::Decoded(v) => vec![v], - MaybeEncodedPlan::Encoded(_) => vec![], - }, + match &self.input_stage.plan { + MaybeEncodedPlan::Decoded(v) => vec![v], + MaybeEncodedPlan::Encoded(_) => vec![], } } @@ -279,18 +218,9 @@ impl ExecutionPlan for NetworkShuffleExec { self: Arc, children: Vec>, ) -> Result, DataFusionError> { - match self.as_ref() { - Self::Pending(v) => { - let mut v = v.clone(); - v.input = require_one_child(children)?; - Ok(Arc::new(Self::Pending(v))) - } - Self::Ready(v) => { - let mut v = v.clone(); - v.input_stage.plan = MaybeEncodedPlan::Decoded(require_one_child(children)?); - Ok(Arc::new(Self::Ready(v))) - } - } + let mut self_clone = self.as_ref().clone(); + self_clone.input_stage.plan = MaybeEncodedPlan::Decoded(require_one_child(children)?); + Ok(Arc::new(self_clone)) } fn execute( @@ -298,88 +228,38 @@ impl ExecutionPlan for NetworkShuffleExec { partition: usize, context: Arc, ) -> Result { - let NetworkShuffleExec::Ready(self_ready) = self else { - return exec_err!( - "NetworkShuffleExec is not ready, was the distributed optimization step performed?" - ); - }; - - // get the channel manager and current stage from our context - let channel_resolver = get_distributed_channel_resolver(context.session_config())?; - - let input_stage = &self_ready.input_stage; - let encoded_input_plan = input_stage.plan.encoded()?; - - let input_stage_tasks = input_stage.tasks.to_vec(); - let input_task_count = input_stage_tasks.len(); - let input_stage_num = input_stage.num as u64; - let query_id = Bytes::from(input_stage.query_id.as_bytes().to_vec()); - - let context_headers = ContextGrpcMetadata::headers_from_ctx(&context); let task_context = DistributedTaskContext::from_ctx(&context); - let off = self_ready.properties.partitioning.partition_count() * task_context.task_index; - - let adapter = DefaultSchemaAdapterFactory::from_schema(self.schema()); - let (mapper, _indices) = adapter.map_schema(&self.schema())?; - - let stream = input_stage_tasks.into_iter().enumerate().map(|(i, task)| { - let channel_resolver = Arc::clone(&channel_resolver); - let mapper = mapper.clone(); - - let ticket = Request::from_parts( - MetadataMap::from_headers(context_headers.clone()), - Extensions::default(), - Ticket { - ticket: DoGet { - plan_proto: encoded_input_plan.clone(), - target_partition: (off + partition) as u64, - stage_key: Some(StageKey { - query_id: query_id.clone(), - stage_id: input_stage_num, - task_number: i as u64, - }), - target_task_index: i as u64, - target_task_count: input_task_count as u64, + let off = self.properties.partitioning.partition_count() * task_context.task_index; + + let mut streams = Vec::with_capacity(self.input_stage.tasks.len()); + for input_task_index in 0..self.input_stage.tasks.len() { + let worker_connection = self.worker_connections.get_or_init_worker_connection( + &self.input_stage, + off..(off + self.properties.partitioning.partition_count()), + input_task_index, + &context, + )?; + + let metrics_collection = Arc::clone(&self.metrics_collection); + let stream = worker_connection.stream_partition(off + partition, move |meta| { + if let Some(AppMetadata::MetricsCollection(m)) = meta.content { + for task_metrics in m.tasks { + if let Some(stage_key) = task_metrics.stage_key { + metrics_collection.insert(stage_key, task_metrics.metrics); + }; } - .encode_to_vec() - .into(), - }, - ); - - let metrics_collection_capture = self_ready.metrics_collection.clone(); - async move { - let url = task.url.ok_or(internal_datafusion_err!( - "NetworkShuffleExec: task is unassigned, cannot proceed" - ))?; - - let mut client = channel_resolver.get_flight_client_for_url(&url).await?; - let stream = client - .do_get(ticket) - .await - .map_err(map_status_to_datafusion_error)? - .into_inner() - .map_err(|err| FlightError::Tonic(Box::new(err))); - - let metrics_collecting_stream = - MetricsCollectingStream::new(stream, metrics_collection_capture); - - Ok( - FlightRecordBatchStream::new_from_flight_data(metrics_collecting_stream) - .map_err(map_flight_to_datafusion_error) - .map(move |batch| { - let batch = batch?; - - mapper.map_batch(batch) - }), - ) - } - .try_flatten_stream() - .boxed() - }); + } + })?; + streams.push(stream); + } Ok(Box::pin(RecordBatchStreamAdapter::new( self.schema(), - futures::stream::select_all(stream), + futures::stream::select_all(streams), ))) } + + fn metrics(&self) -> Option { + Some(self.worker_connections.metrics.clone_inner()) + } } diff --git a/src/execution_plans/partition_isolator.rs b/src/execution_plans/partition_isolator.rs index afa4cbf0..e543c6b9 100644 --- a/src/execution_plans/partition_isolator.rs +++ b/src/execution_plans/partition_isolator.rs @@ -1,9 +1,10 @@ use crate::DistributedTaskContext; -use crate::distributed_physical_optimizer_rule::limit_tasks_err; -use datafusion::common::{exec_err, plan_err}; -use datafusion::error::DataFusionError; +use crate::common::require_one_child; use datafusion::execution::TaskContext; +use datafusion::physical_expr_common::metrics::MetricsSet; use datafusion::physical_plan::ExecutionPlanProperties; +use datafusion::physical_plan::metrics::{ExecutionPlanMetricsSet, MetricBuilder}; +use datafusion::physical_plan::stream::RecordBatchStreamAdapter; use datafusion::{ error::Result, execution::SendableRecordBatchStream, @@ -12,6 +13,7 @@ use datafusion::{ PlanProperties, }, }; +use futures::TryStreamExt; use std::{fmt::Formatter, sync::Arc}; /// This is a simple [ExecutionPlan] that isolates a set of N partitions from an input @@ -50,61 +52,33 @@ use std::{fmt::Formatter, sync::Arc}; /// └───────────────────────────┘ └───────────────────────────┘ └───────────────────────────┘ ■ /// ``` #[derive(Debug)] -pub enum PartitionIsolatorExec { - Pending(PartitionIsolatorPendingExec), - Ready(PartitionIsolatorReadyExec), -} - -#[derive(Debug)] -pub struct PartitionIsolatorPendingExec { - input: Arc, -} - -#[derive(Debug)] -pub struct PartitionIsolatorReadyExec { +pub struct PartitionIsolatorExec { pub(crate) input: Arc, pub(crate) properties: PlanProperties, pub(crate) n_tasks: usize, + pub(crate) metrics: ExecutionPlanMetricsSet, } impl PartitionIsolatorExec { - pub fn new(input: Arc) -> Self { - PartitionIsolatorExec::Pending(PartitionIsolatorPendingExec { input }) - } - - pub(crate) fn ready(&self, n_tasks: usize) -> Result { - let Self::Pending(pending) = self else { - return plan_err!("PartitionIsolatorExec is already ready"); - }; - - let input_partitions = pending.input.properties().partitioning.partition_count(); - if n_tasks > input_partitions { - return Err(limit_tasks_err(input_partitions)); - } + pub fn new(input: Arc, n_tasks: usize) -> Self { + let input_partitions = input.properties().partitioning.partition_count(); let partition_count = Self::partition_groups(input_partitions, n_tasks)[0].len(); - let properties = pending - .input + let properties = input .properties() .clone() .with_partitioning(Partitioning::UnknownPartitioning(partition_count)); - Ok(Self::Ready(PartitionIsolatorReadyExec { - input: pending.input.clone(), + Self { + input: input.clone(), properties, n_tasks, - })) - } - - pub(crate) fn new_ready( - input: Arc, - n_tasks: usize, - ) -> Result { - Self::new(input).ready(n_tasks) + metrics: ExecutionPlanMetricsSet::new(), + } } - pub(crate) fn partition_groups(input_partitions: usize, n_tasks: usize) -> Vec> { + fn partition_groups(input_partitions: usize, n_tasks: usize) -> Vec> { let q = input_partitions / n_tasks; let r = input_partitions % n_tasks; @@ -126,27 +100,16 @@ impl PartitionIsolatorExec { ) -> Vec { Self::partition_groups(input_partitions, n_tasks)[task_i].clone() } - - pub(crate) fn input(&self) -> &Arc { - match self { - PartitionIsolatorExec::Pending(v) => &v.input, - PartitionIsolatorExec::Ready(v) => &v.input, - } - } } impl DisplayAs for PartitionIsolatorExec { fn fmt_as(&self, _t: DisplayFormatType, f: &mut Formatter) -> std::fmt::Result { - let PartitionIsolatorExec::Ready(self_ready) = self else { - return write!(f, "PartitionIsolatorExec"); - }; - let input_partitions = self.input().output_partitioning().partition_count(); - let partition_groups = - PartitionIsolatorExec::partition_groups(input_partitions, self_ready.n_tasks); + let input_partitions = self.input.output_partitioning().partition_count(); + let partition_groups = Self::partition_groups(input_partitions, self.n_tasks); let n: usize = partition_groups.iter().map(|v| v.len()).sum(); let mut partitions = vec![]; - for _ in 0..self_ready.n_tasks { + for _ in 0..self.n_tasks { partitions.push(vec!["__".to_string(); n]); } @@ -171,33 +134,19 @@ impl ExecutionPlan for PartitionIsolatorExec { } fn properties(&self) -> &PlanProperties { - match self { - PartitionIsolatorExec::Pending(pending) => pending.input.properties(), - PartitionIsolatorExec::Ready(ready) => &ready.properties, - } + &self.properties } fn children(&self) -> Vec<&Arc> { - vec![self.input()] + vec![&self.input] } fn with_new_children( self: Arc, children: Vec>, ) -> Result> { - if children.len() != 1 { - return plan_err!( - "PartitionIsolatorExec wrong number of children, expected 1, got {}", - children.len() - ); - } - - Ok(Arc::new(match self.as_ref() { - PartitionIsolatorExec::Pending(_) => Self::new(children[0].clone()), - PartitionIsolatorExec::Ready(ready) => { - Self::new(children[0].clone()).ready(ready.n_tasks)? - } - })) + let input = require_one_child(children)?; + Ok(Arc::new(Self::new(input, self.n_tasks))) } fn execute( @@ -205,13 +154,10 @@ impl ExecutionPlan for PartitionIsolatorExec { partition: usize, context: Arc, ) -> Result { - let Self::Ready(self_ready) = self else { - return exec_err!("PartitionIsolatorExec is not ready"); - }; - let task_context = DistributedTaskContext::from_ctx(&context); - let input_partitions = self_ready.input.output_partitioning().partition_count(); + let metric = MetricBuilder::new(&self.metrics).output_rows(partition); + let input_partitions = self.input.output_partitioning().partition_count(); let partition_group = Self::partition_group( input_partitions, @@ -223,24 +169,28 @@ impl ExecutionPlan for PartitionIsolatorExec { // then look up that index in our group and execute that partition, in this // example partition 8 - let output_stream = match partition_group.get(partition) { + match partition_group.get(partition) { Some(actual_partition_number) => { if *actual_partition_number >= input_partitions { //trace!("{} returning empty stream", ctx_name); - Ok( - Box::pin(EmptyRecordBatchStream::new(self_ready.input.schema())) - as SendableRecordBatchStream, - ) + Ok(Box::pin(EmptyRecordBatchStream::new(self.input.schema())) + as SendableRecordBatchStream) } else { - self_ready.input.execute(*actual_partition_number, context) + Ok(Box::pin(RecordBatchStreamAdapter::new( + self.schema(), + self.input + .execute(*actual_partition_number, context)? + .inspect_ok(move |v| metric.add(v.num_rows())), + ))) } } - None => Ok( - Box::pin(EmptyRecordBatchStream::new(self_ready.input.schema())) - as SendableRecordBatchStream, - ), - }; - output_stream + None => Ok(Box::pin(EmptyRecordBatchStream::new(self.input.schema())) + as SendableRecordBatchStream), + } + } + + fn metrics(&self) -> Option { + Some(self.metrics.clone_inner()) } } diff --git a/src/flight_service/do_get.rs b/src/flight_service/do_get.rs index da894f2f..55e83120 100644 --- a/src/flight_service/do_get.rs +++ b/src/flight_service/do_get.rs @@ -1,33 +1,40 @@ -use crate::DistributedTaskContext; -use crate::common::map_last_stream; -use crate::config_extension_ext::ContextGrpcMetadata; -use crate::flight_service::service::ArrowFlightEndpoint; -use crate::flight_service::session_builder::DistributedSessionBuilderContext; +use crate::common::{map_last_stream, on_drop_stream}; +use crate::config_extension_ext::set_distributed_option_extension_from_headers; +use crate::flight_service::session_builder::WorkerQueryContext; +use crate::flight_service::worker::Worker; use crate::metrics::TaskMetricsCollector; use crate::metrics::proto::df_metrics_set_to_proto; use crate::protobuf::{ AppMetadata, DistributedCodec, FlightAppMetadata, MetricsCollection, StageKey, TaskMetrics, datafusion_error_to_tonic_status, }; -use arrow_flight::FlightData; +use crate::{DistributedConfig, DistributedTaskContext}; use arrow_flight::Ticket; -use arrow_flight::encode::FlightDataEncoderBuilder; +use arrow_flight::encode::{DictionaryHandling, FlightDataEncoder, FlightDataEncoderBuilder}; use arrow_flight::error::FlightError; use arrow_flight::flight_service_server::FlightService; +use arrow_select::dictionary::garbage_collect_any_dictionary; use bytes::Bytes; +use datafusion::arrow::array::{Array, AsArray, RecordBatch}; +use crate::flight_service::spawn_select_all::spawn_select_all; +use datafusion::arrow::ipc::CompressionType; +use datafusion::arrow::ipc::writer::IpcWriteOptions; use datafusion::common::exec_datafusion_err; use datafusion::error::DataFusionError; +use datafusion::execution::{SendableRecordBatchStream, SessionStateBuilder}; use datafusion::physical_plan::ExecutionPlan; -use datafusion::prelude::SessionContext; use datafusion_proto::physical_plan::AsExecutionPlan; use datafusion_proto::protobuf::PhysicalPlanNode; use futures::TryStreamExt; use prost::Message; use std::sync::Arc; -use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use tonic::{Request, Response, Status}; +/// How many record batches to buffer from the plan execution. +const RECORD_BATCH_BUFFER_SIZE: usize = 2; + #[derive(Clone, PartialEq, ::prost::Message)] pub struct DoGet { /// The [Arc] we are going to execute encoded as protobuf bytes. @@ -38,14 +45,17 @@ pub struct DoGet { pub target_task_index: u64, #[prost(uint64, tag = "3")] pub target_task_count: u64, - /// the partition number we want to execute + /// lower bound for the list of partitions to execute (inclusive). #[prost(uint64, tag = "4")] - pub target_partition: u64, + pub target_partition_start: u64, + /// upper bound for the list of partitions to execute (exclusive). + #[prost(uint64, tag = "5")] + pub target_partition_end: u64, /// The stage key that identifies the stage. This is useful to keep /// outside of the stage proto as it is used to store the stage /// and we may not need to deserialize the entire stage proto /// if we already have stored it - #[prost(message, optional, tag = "5")] + #[prost(message, optional, tag = "6")] pub stage_key: Option, } @@ -62,39 +72,44 @@ pub struct TaskData { num_partitions_remaining: Arc, } -impl ArrowFlightEndpoint { +impl Worker { pub(super) async fn get( &self, request: Request, - ) -> Result::DoGetStream>, Status> { + ) -> Result::DoGetStream>, Status> { let (metadata, _ext, body) = request.into_parts(); let doget = DoGet::decode(body.ticket).map_err(|err| { Status::invalid_argument(format!("Cannot decode DoGet message: {err}")) })?; + let headers = metadata.into_headers(); let mut session_state = self .session_builder - .build_session_state(DistributedSessionBuilderContext { - runtime_env: Arc::clone(&self.runtime), - headers: metadata.clone().into_headers(), + .build_session_state(WorkerQueryContext { + builder: SessionStateBuilder::new() + .with_default_features() + .with_runtime_env(Arc::clone(&self.runtime)), + headers: headers.clone(), }) .await .map_err(|err| datafusion_error_to_tonic_status(&err))?; let codec = DistributedCodec::new_combined_with_user(session_state.config()); - let ctx = SessionContext::new_with_state(session_state.clone()); + let task_ctx = session_state.task_ctx(); - // There's only 1 `StageExec` responsible for all requests that share the same `stage_key`, - // so here we either retrieve the existing one or create a new one if it does not exist. let key = doget.stage_key.ok_or_else(missing("stage_key"))?; let once = self .task_data_entries - .get_or_init(key.clone(), Default::default); + .get_with(key.clone(), async { Default::default() }) + .await; let stage_data = once .get_or_try_init(|| async { let proto_node = PhysicalPlanNode::try_decode(doget.plan_proto.as_ref())?; - let plan = proto_node.try_into_physical_plan(&ctx, &self.runtime, &codec)?; + let mut plan = proto_node.try_into_physical_plan(&task_ctx, &codec)?; + for hook in self.hooks.on_plan.iter() { + plan = hook(plan) + } // Initialize partition count to the number of partitions in the stage let total_partitions = plan.properties().partitioning.partition_count(); @@ -107,45 +122,111 @@ impl ArrowFlightEndpoint { .map_err(|err| Status::invalid_argument(format!("Cannot decode stage proto: {err}")))?; let plan = Arc::clone(&stage_data.plan); - // Find out which partition group we are executing let cfg = session_state.config_mut(); - cfg.set_extension(Arc::new(ContextGrpcMetadata(metadata.into_headers()))); + let d_cfg = + set_distributed_option_extension_from_headers::(cfg, &headers) + .map_err(|err| datafusion_error_to_tonic_status(&err))?; + let compression = match d_cfg.compression.as_str() { + "lz4" => Some(CompressionType::LZ4_FRAME), + "zstd" => Some(CompressionType::ZSTD), + "none" => None, + v => Err(Status::invalid_argument(format!( + "Unknown compression type {v}" + )))?, + }; + let send_metrics = d_cfg.collect_metrics; cfg.set_extension(Arc::new(DistributedTaskContext { task_index: doget.target_task_index as usize, task_count: doget.target_task_count as usize, })); let partition_count = plan.properties().partitioning.partition_count(); - let target_partition = doget.target_partition as usize; let plan_name = plan.name(); - if target_partition >= partition_count { - return Err(datafusion_error_to_tonic_status(&exec_datafusion_err!( - "partition {target_partition} not available. The head plan {plan_name} of the stage just has {partition_count} partitions" - ))); - } - // Rather than executing the `StageExec` itself, we want to execute the inner plan instead, - // as executing `StageExec` performs some worker assignation that should have already been - // done in the head stage. - let stream = plan - .execute(doget.target_partition as usize, session_state.task_ctx()) - .map_err(|err| Status::internal(format!("Error executing stage plan: {err:#?}")))?; - - let stream = FlightDataEncoderBuilder::new() - .with_schema(stream.schema().clone()) - .build(stream.map_err(|err| { - FlightError::Tonic(Box::new(datafusion_error_to_tonic_status(&err))) - })); - - let task_data_entries = Arc::clone(&self.task_data_entries); - let num_partitions_remaining = Arc::clone(&stage_data.num_partitions_remaining); - - let stream = map_last_stream(stream, move |last| { - if num_partitions_remaining.fetch_sub(1, Ordering::SeqCst) == 1 { - task_data_entries.remove(key.clone()); + // Execute all the requested partitions at once, and collect all the streams so that they + // can be merged into a single one at the end of this function. + let n_streams = doget.target_partition_end - doget.target_partition_start; + let mut streams = Vec::with_capacity(n_streams as usize); + for partition in doget.target_partition_start..doget.target_partition_end { + if partition >= partition_count as u64 { + return Err(datafusion_error_to_tonic_status(&exec_datafusion_err!( + "partition {partition} not available. The head plan {plan_name} of the stage just has {partition_count} partitions" + ))); } - last.and_then(|el| collect_and_create_metrics_flight_data(key, plan, el)) - }); + + let stream = plan + .execute(partition as usize, session_state.task_ctx()) + .map_err(|err| Status::internal(format!("Error executing stage plan: {err:#?}")))?; + + let stream = build_flight_data_stream(stream, compression)?; + + let task_data_entries = Arc::clone(&self.task_data_entries); + let num_partitions_remaining = Arc::clone(&stage_data.num_partitions_remaining); + + let key = key.clone(); + let key_clone = key.clone(); + let plan = Arc::clone(&plan); + let fully_finished = Arc::new(AtomicBool::new(false)); + let fully_finished_cloned = Arc::clone(&fully_finished); + let stream = map_last_stream(stream, move |msg, last_msg_in_stream| { + // For each FlightData produced by this stream, mark it with the appropriate + // partition. This stream will be merged with several others from other partitions, + // so marking it with the original partition allows it to be deconstructed into + // the original per-partition streams in later steps. + let mut flight_data = FlightAppMetadata::new(partition); + + if last_msg_in_stream { + // If it's the last message from the last partition, clean up the entry from + // the cache and send the collected metrics. + if num_partitions_remaining.fetch_sub(1, Ordering::SeqCst) == 1 { + let entries = Arc::clone(&task_data_entries); + let k = key.clone(); + tokio::spawn(async move { + entries.invalidate(&k).await; + }); + if send_metrics { + // Last message of the last partition. This is the moment to send + // the metrics back. + flight_data.set_content(collect_and_create_metrics_flight_data( + key.clone(), + plan.clone(), + )?); + } + } + fully_finished.store(true, Ordering::SeqCst); + } + + msg.map(|v| v.with_app_metadata(flight_data.encode_to_vec())) + }); + + let num_partitions_remaining = Arc::clone(&stage_data.num_partitions_remaining); + let task_data_entries = Arc::clone(&self.task_data_entries); + // When the stream is dropped before fully consumed (e.g. LIMIT on the client side), + // metrics piggybacked on the last FlightData message are lost. + // See https://github.com/datafusion-contrib/datafusion-distributed/issues/187 + let stream = on_drop_stream(stream, move || { + if !fully_finished_cloned.load(Ordering::SeqCst) { + // If the stream was not fully consumed, but it was dropped (abandoned), we + // still need to remove the entry from `task_data_entries`, otherwise we + // might leak memory until the cache automatically evicts it after the TTL expires. + if num_partitions_remaining.fetch_sub(1, Ordering::SeqCst) == 1 { + let entries = Arc::clone(&task_data_entries); + let k = key_clone.clone(); + // Fire-and-forget background tokio task to handle async + // invalidate() within synchronous on_drop_stream. + tokio::spawn(async move { + entries.invalidate(&k).await; + }); + } + } + }); + streams.push(stream) + } + + // Merge all the per-partition streams into one. Each message in the stream is marked with + // the original partition, so they can be reconstructed at the other side of the boundary. + let memory_pool = Arc::clone(&session_state.runtime_env().memory_pool); + let stream = spawn_select_all(streams, memory_pool, RECORD_BATCH_BUFFER_SIZE); Ok(Response::new(Box::pin(stream.map_err(|err| match err { FlightError::Tonic(status) => *status, @@ -154,6 +235,46 @@ impl ArrowFlightEndpoint { } } +fn build_flight_data_stream( + stream: SendableRecordBatchStream, + compression_type: Option, +) -> Result { + let stream = FlightDataEncoderBuilder::new() + .with_options( + IpcWriteOptions::default() + .try_with_compression(compression_type) + .map_err(|err| Status::internal(err.to_string()))?, + ) + .with_schema(stream.schema()) + // This tells the encoder to send dictionaries across the wire as-is. + // The alternative (`DictionaryHandling::Hydrate`) would expand the dictionaries + // into their value types, which can potentially blow up the size of the data transfer. + // The main reason to use `DictionaryHandling::Hydrate` is for compatibility with clients + // that do not support dictionaries, but since we are using the same server/client on both + // sides, we can safely use `DictionaryHandling::Resend`. + // Note that we do garbage collection of unused dictionary values above, so we are not sending + // unused dictionary values over the wire. + .with_dictionary_handling(DictionaryHandling::Resend) + // Set max flight data size to unlimited. + // This requires servers and clients to also be configured to handle unlimited sizes. + // Using unlimited sizes avoids splitting RecordBatches into multiple FlightData messages, + // which could add significant overhead for large RecordBatches. + // The only reason to split them really is if the client/server are configured with a message size limit, + // which mainly makes sense in a public network scenario where you want to avoid DoS attacks. + // Since all of our Arrow Flight communication happens within trusted data plane networks, + // we can safely use unlimited sizes here. + .with_max_flight_data_size(usize::MAX) + .build( + stream + // Apply garbage collection of dictionary and view arrays before sending over the network + .and_then(|rb| std::future::ready(garbage_collect_arrays(rb))) + .map_err(|err| { + FlightError::Tonic(Box::new(datafusion_error_to_tonic_status(&err))) + }), + ); + Ok(stream) +} + fn missing(field: &'static str) -> impl FnOnce() -> Status { move || Status::invalid_argument(format!("Missing field '{field}'")) } @@ -162,9 +283,8 @@ fn missing(field: &'static str) -> impl FnOnce() -> Status { fn collect_and_create_metrics_flight_data( stage_key: StageKey, plan: Arc, - incoming: FlightData, -) -> Result { - // Get the metrics for the task executed on this worker. Separately, collect metrics for child tasks. +) -> Result { + // Get the metrics for the task executed on this worker + child tasks. let mut result = TaskMetricsCollector::new() .collect(plan) .map_err(|err| FlightError::ProtocolError(err.to_string()))?; @@ -192,24 +312,42 @@ fn collect_and_create_metrics_flight_data( }); } - let flight_app_metadata = FlightAppMetadata { - content: Some(AppMetadata::MetricsCollection(MetricsCollection { - tasks: task_metrics_set, - })), - }; + Ok(AppMetadata::MetricsCollection(MetricsCollection { + tasks: task_metrics_set, + })) +} - let mut buf = vec![]; - flight_app_metadata - .encode(&mut buf) - .map_err(|err| FlightError::ProtocolError(err.to_string()))?; +/// Garbage collects values sub-arrays. +/// +/// We apply this before sending RecordBatches over the network to avoid sending +/// values that are not referenced by any dictionary keys or buffers that are not used. +/// +/// Unused values can arise from operations such as filtering, where +/// some keys may no longer be referenced in the filtered result. +fn garbage_collect_arrays(batch: RecordBatch) -> Result { + let (schema, arrays, _row_count) = batch.into_parts(); + + let arrays = arrays + .into_iter() + .map(|array| { + if let Some(array) = array.as_any_dictionary_opt() { + garbage_collect_any_dictionary(array) + } else if let Some(array) = array.as_string_view_opt() { + Ok(Arc::new(array.gc()) as Arc) + } else if let Some(array) = array.as_binary_view_opt() { + Ok(Arc::new(array.gc()) as Arc) + } else { + Ok(array) + } + }) + .collect::, _>>()?; - Ok(incoming.with_app_metadata(buf)) + Ok(RecordBatch::try_new(schema, arrays)?) } #[cfg(test)] mod tests { use super::*; - use crate::flight_service::session_builder::DefaultSessionBuilder; use crate::stage::ExecutionTask; use arrow::datatypes::{Schema, SchemaRef}; use arrow_flight::Ticket; @@ -224,9 +362,15 @@ mod tests { #[tokio::test] async fn test_task_data_partition_counting() { - // Create ArrowFlightEndpoint with DefaultSessionBuilder - let endpoint = - ArrowFlightEndpoint::try_new(DefaultSessionBuilder).expect("Failed to create endpoint"); + let mut endpoint = Worker::default(); + let plans_received = Arc::new(AtomicUsize::default()); + { + let plans_received = Arc::clone(&plans_received); + endpoint.add_on_plan_hook(move |plan| { + plans_received.fetch_add(1, Ordering::SeqCst); + plan + }); + } // Create 3 tasks with 3 partitions each. let num_tasks = 3; @@ -246,23 +390,10 @@ mod tests { .encode_to_vec() .into(); - let task_keys = [ - StageKey { - query_id: query_id.clone(), - stage_id, - task_number: 0, - }, - StageKey { - query_id: query_id.clone(), - stage_id, - task_number: 1, - }, - StageKey { - query_id: query_id.clone(), - stage_id, - task_number: 2, - }, - ]; + let task_keys: Vec<_> = (0..3) + .map(|i| StageKey::new(query_id.clone(), stage_id, i)) + .collect(); + let plan_proto_for_closure = plan_proto.clone(); let endpoint_ref = &endpoint; @@ -272,7 +403,8 @@ mod tests { plan_proto, target_task_index: task_number, target_task_count: num_tasks, - target_partition: partition, + target_partition_start: partition, + target_partition_end: partition + 1, stage_key: Some(stage_key), }; @@ -293,18 +425,29 @@ mod tests { for (task_number, task_key) in task_keys.iter().enumerate() { for partition in 0..num_partitions_per_task - 1 { let result = do_get(partition as u64, task_number as u64, task_key.clone()).await; - assert!(result.is_ok()); + if let Err(err) = result { + panic!("do_get call failed with error: {err}") + } } } + // As many plans as tasks should have been received. + assert_eq!(plans_received.load(Ordering::SeqCst), task_keys.len()); // Check that the endpoint has not evicted any task states. - assert_eq!(endpoint.task_data_entries.len(), num_tasks as usize); + assert_eq!( + endpoint.task_data_entries.iter().count(), + num_tasks as usize + ); // Run the last partition of task 0. Any partition number works. Verify that the task state // is evicted because all partitions have been processed. let result = do_get(2, 0, task_keys[0].clone()).await; assert!(result.is_ok()); - let stored_stage_keys = endpoint.task_data_entries.keys().collect::>(); + let stored_stage_keys = endpoint + .task_data_entries + .iter() + .map(|(k, _)| (*k).clone()) + .collect::>(); assert_eq!(stored_stage_keys.len(), 2); assert!(stored_stage_keys.contains(&task_keys[1])); assert!(stored_stage_keys.contains(&task_keys[2])); @@ -312,14 +455,22 @@ mod tests { // Run the last partition of task 1. let result = do_get(2, 1, task_keys[1].clone()).await; assert!(result.is_ok()); - let stored_stage_keys = endpoint.task_data_entries.keys().collect::>(); + let stored_stage_keys = endpoint + .task_data_entries + .iter() + .map(|(k, _)| (*k).clone()) + .collect::>(); assert_eq!(stored_stage_keys.len(), 1); assert!(stored_stage_keys.contains(&task_keys[2])); // Run the last partition of the last task. let result = do_get(2, 2, task_keys[2].clone()).await; assert!(result.is_ok()); - let stored_stage_keys = endpoint.task_data_entries.keys().collect::>(); + let stored_stage_keys = endpoint + .task_data_entries + .iter() + .map(|(k, _)| (*k).clone()) + .collect::>(); assert_eq!(stored_stage_keys.len(), 0); } diff --git a/src/flight_service/mod.rs b/src/flight_service/mod.rs index 96ea6090..565633ed 100644 --- a/src/flight_service/mod.rs +++ b/src/flight_service/mod.rs @@ -1,10 +1,15 @@ mod do_get; -mod service; mod session_builder; -pub(crate) use do_get::DoGet; +mod spawn_select_all; +mod worker; +mod worker_connection_pool; + +pub(crate) use worker_connection_pool::WorkerConnectionPool; -pub use service::ArrowFlightEndpoint; pub use session_builder::{ - DefaultSessionBuilder, DistributedSessionBuilder, DistributedSessionBuilderContext, - MappedDistributedSessionBuilder, MappedDistributedSessionBuilderExt, + DefaultSessionBuilder, MappedWorkerSessionBuilder, MappedWorkerSessionBuilderExt, + WorkerQueryContext, WorkerSessionBuilder, }; +pub use worker::Worker; + +pub use do_get::TaskData; diff --git a/src/flight_service/service.rs b/src/flight_service/service.rs deleted file mode 100644 index df01be73..00000000 --- a/src/flight_service/service.rs +++ /dev/null @@ -1,122 +0,0 @@ -use crate::common::ttl_map::{TTLMap, TTLMapConfig}; -use crate::flight_service::DistributedSessionBuilder; -use crate::flight_service::do_get::TaskData; -use crate::protobuf::StageKey; -use arrow_flight::flight_service_server::FlightService; -use arrow_flight::{ - Action, ActionType, Criteria, Empty, FlightData, FlightDescriptor, FlightInfo, - HandshakeRequest, HandshakeResponse, PollInfo, PutResult, SchemaResult, Ticket, -}; -use async_trait::async_trait; -use datafusion::error::DataFusionError; -use datafusion::execution::runtime_env::RuntimeEnv; -use futures::stream::BoxStream; -use std::sync::Arc; -use tokio::sync::OnceCell; -use tonic::{Request, Response, Status, Streaming}; - -pub struct ArrowFlightEndpoint { - pub(super) runtime: Arc, - pub(super) task_data_entries: Arc>>>, - pub(super) session_builder: Arc, -} - -impl ArrowFlightEndpoint { - pub fn try_new( - session_builder: impl DistributedSessionBuilder + Send + Sync + 'static, - ) -> Result { - let ttl_map = TTLMap::try_new(TTLMapConfig::default())?; - Ok(Self { - runtime: Arc::new(RuntimeEnv::default()), - task_data_entries: Arc::new(ttl_map), - session_builder: Arc::new(session_builder), - }) - } -} - -#[async_trait] -impl FlightService for ArrowFlightEndpoint { - type HandshakeStream = BoxStream<'static, Result>; - - async fn handshake( - &self, - _: Request>, - ) -> Result, Status> { - Err(Status::unimplemented("Not yet implemented")) - } - - type ListFlightsStream = BoxStream<'static, Result>; - - async fn list_flights( - &self, - _: Request, - ) -> Result, Status> { - Err(Status::unimplemented("Not yet implemented")) - } - - async fn get_flight_info( - &self, - _: Request, - ) -> Result, Status> { - Err(Status::unimplemented("Not yet implemented")) - } - - async fn poll_flight_info( - &self, - _: Request, - ) -> Result, Status> { - Err(Status::unimplemented("Not yet implemented")) - } - - async fn get_schema( - &self, - _: Request, - ) -> Result, Status> { - Err(Status::unimplemented("Not yet implemented")) - } - - type DoGetStream = BoxStream<'static, Result>; - - async fn do_get( - &self, - request: Request, - ) -> Result, Status> { - self.get(request).await - } - - type DoPutStream = BoxStream<'static, Result>; - - async fn do_put( - &self, - _: Request>, - ) -> Result, Status> { - Err(Status::unimplemented("Not yet implemented")) - } - - type DoExchangeStream = BoxStream<'static, Result>; - - async fn do_exchange( - &self, - _: Request>, - ) -> Result, Status> { - Err(Status::unimplemented("Not yet implemented")) - } - - type DoActionStream = BoxStream<'static, Result>; - - async fn do_action( - &self, - _: Request, - ) -> Result, Status> { - Err(Status::unimplemented("Not yet implemented")) - } - - type ListActionsStream = BoxStream<'static, Result>; - - async fn list_actions( - &self, - _: Request, - ) -> Result, Status> { - Err(Status::unimplemented("Not yet implemented")) - } -} diff --git a/src/flight_service/session_builder.rs b/src/flight_service/session_builder.rs index 3cf617bc..c0f0bf41 100644 --- a/src/flight_service/session_builder.rs +++ b/src/flight_service/session_builder.rs @@ -1,20 +1,18 @@ use async_trait::async_trait; use datafusion::error::DataFusionError; -use datafusion::execution::runtime_env::RuntimeEnv; use datafusion::execution::{SessionState, SessionStateBuilder}; use http::HeaderMap; use std::sync::Arc; -#[derive(Debug, Clone, Default)] -pub struct DistributedSessionBuilderContext { - pub runtime_env: Arc, +#[derive(Debug, Default)] +pub struct WorkerQueryContext { + pub builder: SessionStateBuilder, pub headers: HeaderMap, } -/// Trait called by the Arrow Flight endpoint that handles distributed parts of a DataFusion -/// plan for building a DataFusion's [SessionState]. +/// builds a DataFusion's [SessionState] in each query issued to a worker. #[async_trait] -pub trait DistributedSessionBuilder { +pub trait WorkerSessionBuilder { /// Builds a custom [SessionState] scoped to a single ArrowFlight gRPC call, allowing the /// users to provide a customized DataFusion session with things like custom extension codecs, /// custom physical optimization rules, UDFs, UDAFs, config extensions, etc... @@ -25,16 +23,16 @@ pub trait DistributedSessionBuilder { /// # use std::sync::Arc; /// # use async_trait::async_trait; /// # use datafusion::error::DataFusionError; - /// # use datafusion::execution::{FunctionRegistry, SessionState, SessionStateBuilder}; + /// # use datafusion::execution::{FunctionRegistry, SessionState, SessionStateBuilder, TaskContext}; /// # use datafusion::physical_plan::ExecutionPlan; /// # use datafusion_proto::physical_plan::PhysicalExtensionCodec; - /// # use datafusion_distributed::{DistributedExt, DistributedSessionBuilder, DistributedSessionBuilderContext}; + /// # use datafusion_distributed::{DistributedExt, WorkerSessionBuilder, WorkerQueryContext}; /// /// #[derive(Debug)] /// struct CustomExecCodec; /// /// impl PhysicalExtensionCodec for CustomExecCodec { - /// fn try_decode(&self, buf: &[u8], inputs: &[Arc], registry: &dyn FunctionRegistry) -> datafusion::common::Result> { + /// fn try_decode(&self, buf: &[u8], inputs: &[Arc], ctx: &TaskContext) -> datafusion::common::Result> { /// todo!() /// } /// @@ -47,69 +45,64 @@ pub trait DistributedSessionBuilder { /// struct CustomSessionBuilder; /// /// #[async_trait] - /// impl DistributedSessionBuilder for CustomSessionBuilder { - /// async fn build_session_state(&self, ctx: DistributedSessionBuilderContext) -> Result { - /// let mut builder = SessionStateBuilder::new() - /// .with_runtime_env(ctx.runtime_env.clone()) - /// .with_default_features(); - /// builder.set_distributed_user_codec(CustomExecCodec); - /// // Add your UDFs, optimization rules, etc... - /// - /// Ok(builder.build()) + /// impl WorkerSessionBuilder for CustomSessionBuilder { + /// async fn build_session_state(&self, ctx: WorkerQueryContext) -> Result { + /// Ok(ctx + /// .builder + /// .with_distributed_user_codec(CustomExecCodec) + /// // Add your UDFs, optimization rules, etc... + /// .build()) /// } /// } /// ``` async fn build_session_state( &self, - ctx: DistributedSessionBuilderContext, + ctx: WorkerQueryContext, ) -> Result; } -/// Noop implementation of the [DistributedSessionBuilder]. Used by default if no [DistributedSessionBuilder] is provided -/// while building the Arrow Flight endpoint. +/// Noop implementation of the [WorkerSessionBuilder]. Used by default if no [WorkerSessionBuilder] +/// is provided while building the Worker. #[derive(Debug, Clone)] pub struct DefaultSessionBuilder; #[async_trait] -impl DistributedSessionBuilder for DefaultSessionBuilder { +impl WorkerSessionBuilder for DefaultSessionBuilder { async fn build_session_state( &self, - ctx: DistributedSessionBuilderContext, + ctx: WorkerQueryContext, ) -> Result { - Ok(SessionStateBuilder::new() - .with_runtime_env(ctx.runtime_env.clone()) - .with_default_features() - .build()) + Ok(ctx.builder.build()) } } -/// Implementation of [DistributedSessionBuilder] for any async function that returns a [Result] +/// Implementation of [WorkerSessionBuilder] for any async function that returns a [Result] #[async_trait] -impl DistributedSessionBuilder for F +impl WorkerSessionBuilder for F where - F: Fn(DistributedSessionBuilderContext) -> Fut + Send + Sync + 'static, + F: Fn(WorkerQueryContext) -> Fut + Send + Sync + 'static, Fut: std::future::Future> + Send + 'static, { async fn build_session_state( &self, - ctx: DistributedSessionBuilderContext, + ctx: WorkerQueryContext, ) -> Result { self(ctx).await } } -pub trait MappedDistributedSessionBuilderExt { - /// Maps an existing [DistributedSessionBuilder] allowing to add further extensions +pub trait MappedWorkerSessionBuilderExt { + /// Maps an existing [WorkerSessionBuilder] allowing to add further extensions /// to its already built [SessionStateBuilder]. /// - /// Useful if there's already a [DistributedSessionBuilder] that needs to be extended + /// Useful if there's already a [WorkerSessionBuilder] that needs to be extended /// with further capabilities. /// /// Example: /// /// ```rust /// # use datafusion::execution::SessionStateBuilder; - /// # use datafusion_distributed::{DefaultSessionBuilder, MappedDistributedSessionBuilderExt}; + /// # use datafusion_distributed::{DefaultSessionBuilder, MappedWorkerSessionBuilderExt}; /// /// let session_builder = DefaultSessionBuilder /// .map(|b: SessionStateBuilder| { @@ -117,30 +110,30 @@ pub trait MappedDistributedSessionBuilderExt { /// Ok(b.build()) /// }); /// ``` - fn map(self, f: F) -> MappedDistributedSessionBuilder + fn map(self, f: F) -> MappedWorkerSessionBuilder where Self: Sized, F: Fn(SessionStateBuilder) -> Result; } -impl MappedDistributedSessionBuilderExt for T { - fn map(self, f: F) -> MappedDistributedSessionBuilder +impl MappedWorkerSessionBuilderExt for T { + fn map(self, f: F) -> MappedWorkerSessionBuilder where Self: Sized, { - MappedDistributedSessionBuilder { + MappedWorkerSessionBuilder { inner: self, f: Arc::new(f), } } } -pub struct MappedDistributedSessionBuilder { +pub struct MappedWorkerSessionBuilder { inner: T, f: Arc, } -impl Clone for MappedDistributedSessionBuilder { +impl Clone for MappedWorkerSessionBuilder { fn clone(&self) -> Self { Self { inner: self.inner.clone(), @@ -150,14 +143,14 @@ impl Clone for MappedDistributedSessionBuilder { } #[async_trait] -impl DistributedSessionBuilder for MappedDistributedSessionBuilder +impl WorkerSessionBuilder for MappedWorkerSessionBuilder where - T: DistributedSessionBuilder + Send + Sync + 'static, + T: WorkerSessionBuilder + Send + Sync + 'static, F: Fn(SessionStateBuilder) -> Result + Send + Sync, { async fn build_session_state( &self, - ctx: DistributedSessionBuilderContext, + ctx: WorkerQueryContext, ) -> Result { let state = self.inner.build_session_state(ctx).await?; let builder = SessionStateBuilder::new_from_existing(state); diff --git a/src/flight_service/spawn_select_all.rs b/src/flight_service/spawn_select_all.rs new file mode 100644 index 00000000..7ae67cb6 --- /dev/null +++ b/src/flight_service/spawn_select_all.rs @@ -0,0 +1,171 @@ +use arrow_flight::FlightData; +use datafusion::arrow::array::RecordBatch; +use datafusion::common::runtime::SpawnedTask; +use datafusion::execution::memory_pool::{MemoryConsumer, MemoryPool}; +use futures::{Stream, StreamExt}; +use std::sync::Arc; +use tokio_stream::wrappers::ReceiverStream; + +/// Consumes all the provided streams in parallel sending their produced messages to a single +/// queue in random order. The resulting queue is returned as a stream. +/// +/// Uses a bounded channel with send timeout to detect when the client has stopped consuming +/// (e.g., due to disconnect), allowing for prompt cleanup of resources. +pub(crate) fn spawn_select_all( + inner: Vec, + pool: Arc, + queue_size: usize, +) -> impl Stream> +where + T: Stream> + Send + Unpin + 'static, + El: MemoryFootPrint + Send + 'static, + Err: Send + 'static, +{ + let (tx, rx) = tokio::sync::mpsc::channel(queue_size); + + let mut tasks = vec![]; + for mut t in inner { + let tx = tx.clone(); + let pool = Arc::clone(&pool); + let consumer = MemoryConsumer::new("NetworkBoundary"); + + tasks.push(SpawnedTask::spawn(async move { + loop { + // Capture the closed() event as soon as possible. We don't want to do + // extra work if we know nobody is going to listen to it. + let msg = tokio::select! { + biased; + _ = tx.closed() => return, + msg = t.next() => msg + }; + let Some(msg) = msg else { return }; + + let mut reservation = consumer.clone_with_new_id().register(&pool); + if let Ok(msg) = &msg { + reservation.grow(msg.get_memory_size()); + } + + if tx.send((msg, reservation)).await.is_err() { + return; + }; + } + })) + } + + ReceiverStream::new(rx).map(move |(msg, _reservation)| { + // keep the tasks alive as long as the stream lives + let _ = &tasks; + msg + }) +} + +pub(crate) trait MemoryFootPrint { + fn get_memory_size(&self) -> usize; +} + +impl MemoryFootPrint for RecordBatch { + fn get_memory_size(&self) -> usize { + self.get_array_memory_size() + } +} + +impl MemoryFootPrint for FlightData { + fn get_memory_size(&self) -> usize { + self.data_header.len() + self.data_body.len() + self.app_metadata.len() + } +} + +#[cfg(test)] +mod tests { + use super::{MemoryFootPrint, spawn_select_all}; + use datafusion::execution::memory_pool::{MemoryPool, UnboundedMemoryPool}; + use std::error::Error; + use std::sync::Arc; + use tokio_stream::StreamExt; + + #[tokio::test] + async fn memory_reservation() -> Result<(), Box> { + let pool: Arc = Arc::new(UnboundedMemoryPool::default()); + + let mut stream = spawn_select_all( + vec![ + futures::stream::iter(vec![Ok::<_, String>(1), Ok(2), Ok(3)]), + futures::stream::iter(vec![Ok(4), Ok(5)]), + ], + Arc::clone(&pool), + 5, + ); + tokio::time::sleep(tokio::time::Duration::from_millis(1)).await; + let reserved = pool.reserved(); + assert_eq!(reserved, 15); + + for i in [1, 2, 3] { + let n = stream.next().await.unwrap()?; + assert_eq!(i, n) + } + + let reserved = pool.reserved(); + assert_eq!(reserved, 9); + + drop(stream); + + let reserved = pool.reserved(); + assert_eq!(reserved, 0); + + Ok(()) + } + + #[tokio::test] + async fn memory_reservation_backpressure() -> Result<(), Box> { + let pool: Arc = Arc::new(UnboundedMemoryPool::default()); + + let mut stream = spawn_select_all( + vec![futures::stream::iter(vec![ + Ok::<_, String>(1), + Ok(2), + Ok(3), + ])], + Arc::clone(&pool), + 1, + ); + // First two messages are buffered (1+2) + tokio::time::sleep(tokio::time::Duration::from_millis(1)).await; + let reserved = pool.reserved(); + assert_eq!(reserved, 3); + + // First message is pulled + let n = stream.next().await.unwrap()?; + assert_eq!(n, 1); + + // The third message is buffered, but the first one came out (2+3) + tokio::time::sleep(tokio::time::Duration::from_millis(1)).await; + let reserved = pool.reserved(); + assert_eq!(reserved, 5); + + // Second message is pulled + let n = stream.next().await.unwrap()?; + assert_eq!(n, 2); + + // Only the third message is buffered + tokio::time::sleep(tokio::time::Duration::from_millis(1)).await; + let reserved = pool.reserved(); + assert_eq!(reserved, 3); + + // The third message is pulled + let n = stream.next().await.unwrap()?; + assert_eq!(n, 3); + + // Nothing remains in the pool + tokio::time::sleep(tokio::time::Duration::from_millis(1)).await; + let reserved = pool.reserved(); + assert_eq!(reserved, 0); + + Ok(()) + } + + impl MemoryFootPrint for usize { + fn get_memory_size(&self) -> usize { + *self + } + } +} diff --git a/src/flight_service/worker.rs b/src/flight_service/worker.rs new file mode 100644 index 00000000..6d4c187e --- /dev/null +++ b/src/flight_service/worker.rs @@ -0,0 +1,232 @@ +use crate::flight_service::WorkerSessionBuilder; +use crate::flight_service::do_get::TaskData; +use crate::protobuf::StageKey; +use crate::{DefaultSessionBuilder, ObservabilityServiceImpl}; +use arrow_flight::flight_service_server::{FlightService, FlightServiceServer}; +use arrow_flight::{ + Action, ActionType, Criteria, Empty, FlightData, FlightDescriptor, FlightInfo, + HandshakeRequest, HandshakeResponse, PollInfo, PutResult, SchemaResult, Ticket, +}; +use async_trait::async_trait; +use datafusion::execution::runtime_env::RuntimeEnv; +use datafusion::physical_plan::ExecutionPlan; +use futures::stream::BoxStream; +use moka::future::Cache; +use std::sync::Arc; +use std::time::Duration; +use tokio::sync::OnceCell; +use tonic::{Request, Response, Status, Streaming}; + +#[allow(clippy::type_complexity)] +#[derive(Clone, Default)] +pub(super) struct WorkerHooks { + pub(super) on_plan: + Vec) -> Arc + Sync + Send>>, +} + +#[derive(Clone)] +pub struct Worker { + pub(super) runtime: Arc, + /// TTL-based cache for task execution data. Entries are automatically evicted after 60 seconds. + /// This prevents memory leaks from abandoned or incomplete queries while allowing concurrent + /// access to task results across multiple partition requests. + pub(super) task_data_entries: Arc>>>, + pub(super) session_builder: Arc, + pub(super) hooks: WorkerHooks, + pub(super) max_message_size: Option, +} + +impl Default for Worker { + fn default() -> Self { + let cache = Cache::builder() + .time_to_idle(Duration::from_secs(60)) + .build(); + Self { + runtime: Arc::new(RuntimeEnv::default()), + task_data_entries: Arc::new(cache), + session_builder: Arc::new(DefaultSessionBuilder), + hooks: WorkerHooks::default(), + max_message_size: Some(usize::MAX), + } + } +} + +impl Worker { + /// Builds a [Worker] with a custom [WorkerSessionBuilder]. Use this + /// method whenever you need to add custom stuff to the `SessionContext` that executes the query. + pub fn from_session_builder( + session_builder: impl WorkerSessionBuilder + Send + Sync + 'static, + ) -> Self { + Self { + session_builder: Arc::new(session_builder), + ..Default::default() + } + } + + /// Sets a [RuntimeEnv] to be used in all the queries this [Worker] will handle during + /// its lifetime. + pub fn with_runtime_env(mut self, runtime_env: Arc) -> Self { + self.runtime = runtime_env; + self + } + + /// Adds a callback for when an [ExecutionPlan] is received in the `do_get` call. + /// + /// The callback takes the plan and returns another plan that must be either the same, + /// or equivalent in terms of execution. Mutating the plan by adding nodes or removing them + /// will make the query blow up in unexpected ways. + pub fn add_on_plan_hook( + &mut self, + hook: impl Fn(Arc) -> Arc + Sync + Send + 'static, + ) { + self.hooks.on_plan.push(Arc::new(hook)); + } + + /// Set the maximum message size for FlightData chunks. + /// + /// Defaults to `usize::MAX` to minimize chunking overhead for internal communication. + /// See [`FlightDataEncoderBuilder::with_max_flight_data_size`] for details. + /// + /// If you change this to a lower value, ensure you configure the server's + /// max_encoding_message_size and max_decoding_message_size to at least 2x this value + /// to allow for overhead. For most use cases, the default of `usize::MAX` is appropriate. + /// + /// [`FlightDataEncoderBuilder::with_max_flight_data_size`]: https://arrow.apache.org/rust/arrow_flight/encode/struct.FlightDataEncoderBuilder.html#structfield.max_flight_data_size + pub fn with_max_message_size(mut self, size: usize) -> Self { + self.max_message_size = Some(size); + self + } + + /// Converts this [Worker] into a [`FlightServiceServer`] with high default message size limits. + /// + /// This is a convenience method that wraps the endpoint in a [`FlightServiceServer`] and + /// configures it with `max_decoding_message_size(usize::MAX)` and + /// `max_encoding_message_size(usize::MAX)` to avoid message size limitations for internal + /// communication. + /// + /// You can further customize the returned server by chaining additional tonic methods. + /// + /// # Example + /// + /// ``` + /// # use datafusion_distributed::Worker; + /// # use tonic::transport::Server; + /// # use std::net::{IpAddr, Ipv4Addr, SocketAddr}; + /// # async fn f() { + /// + /// let worker = Worker::default(); + /// let server = worker.into_flight_server(); + /// + /// Server::builder() + /// .add_service(Worker::default().into_flight_server()) + /// .serve(SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 8080)) + /// .await; + /// + /// # } + /// ``` + pub fn into_flight_server(self) -> FlightServiceServer { + FlightServiceServer::new(self) + .max_decoding_message_size(usize::MAX) + .max_encoding_message_size(usize::MAX) + } + + pub fn observability_service(&self) -> ObservabilityServiceImpl { + ObservabilityServiceImpl::new(self.task_data_entries.clone()) + } + + /// Returns the number of cached task entries currently held by this worker. + #[cfg(any(test, feature = "integration"))] + pub async fn tasks_running(&self) -> usize { + // Use `run_pending_tasks()` to migigate inaccuracy from potential stale + // `entry_count()` task data. + self.task_data_entries.run_pending_tasks().await; + self.task_data_entries.entry_count() as usize + } +} + +#[async_trait] +impl FlightService for Worker { + type HandshakeStream = BoxStream<'static, Result>; + + async fn handshake( + &self, + _: Request>, + ) -> Result, Status> { + Err(Status::unimplemented("Not yet implemented")) + } + + type ListFlightsStream = BoxStream<'static, Result>; + + async fn list_flights( + &self, + _: Request, + ) -> Result, Status> { + Err(Status::unimplemented("Not yet implemented")) + } + + async fn get_flight_info( + &self, + _: Request, + ) -> Result, Status> { + Err(Status::unimplemented("Not yet implemented")) + } + + async fn poll_flight_info( + &self, + _: Request, + ) -> Result, Status> { + Err(Status::unimplemented("Not yet implemented")) + } + + async fn get_schema( + &self, + _: Request, + ) -> Result, Status> { + Err(Status::unimplemented("Not yet implemented")) + } + + type DoGetStream = BoxStream<'static, Result>; + + async fn do_get( + &self, + request: Request, + ) -> Result, Status> { + self.get(request).await + } + + type DoPutStream = BoxStream<'static, Result>; + + async fn do_put( + &self, + _: Request>, + ) -> Result, Status> { + Err(Status::unimplemented("Not yet implemented")) + } + + type DoExchangeStream = BoxStream<'static, Result>; + + async fn do_exchange( + &self, + _: Request>, + ) -> Result, Status> { + Err(Status::unimplemented("Not yet implemented")) + } + + type DoActionStream = BoxStream<'static, Result>; + + async fn do_action( + &self, + _: Request, + ) -> Result, Status> { + Err(Status::unimplemented("Not yet implemented")) + } + + type ListActionsStream = BoxStream<'static, Result>; + + async fn list_actions( + &self, + _: Request, + ) -> Result, Status> { + Err(Status::unimplemented("Not yet implemented")) + } +} diff --git a/src/flight_service/worker_connection_pool.rs b/src/flight_service/worker_connection_pool.rs new file mode 100644 index 00000000..9513d020 --- /dev/null +++ b/src/flight_service/worker_connection_pool.rs @@ -0,0 +1,667 @@ +use crate::Stage; +use crate::common::on_drop_stream; +use crate::config_extension_ext::get_config_extension_propagation_headers; +use crate::flight_service::do_get::DoGet; +use crate::metrics::latency_tracker::LatencyTracker; +use crate::networking::get_distributed_channel_resolver; +use crate::passthrough_headers::get_passthrough_headers; +use crate::protobuf::{ + FlightAppMetadata, StageKey, datafusion_error_to_tonic_status, map_flight_to_datafusion_error, +}; +use arrow_flight::decode::FlightRecordBatchStream; +use arrow_flight::error::FlightError; +use arrow_flight::{FlightData, Ticket}; +use bytes::Bytes; +use dashmap::DashMap; +use datafusion::arrow::array::RecordBatch; +use datafusion::common::instant::Instant; +use datafusion::common::runtime::SpawnedTask; +use datafusion::common::{DataFusionError, Result, internal_err}; +use datafusion::execution::TaskContext; +use datafusion::execution::memory_pool::{MemoryConsumer, MemoryReservation}; +use datafusion::physical_expr_common::metrics::{ExecutionPlanMetricsSet, MetricValue}; +use datafusion::physical_plan::metrics::{MetricBuilder, Time}; +use futures::{Stream, TryStreamExt}; +use http::Extensions; +use pin_project::{pin_project, pinned_drop}; +use prost::Message; +use std::fmt::{Debug, Formatter}; +use std::ops::Range; +use std::pin::Pin; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, OnceLock}; +use std::task::{Context, Poll}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; +use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender}; +use tokio_stream::StreamExt; +use tokio_stream::wrappers::UnboundedReceiverStream; +use tokio_util::sync::CancellationToken; +use tonic::metadata::MetadataMap; +use tonic::{Request, Status}; + +/// Holds a list of lazily initialized [WorkerConnection]s. Each position in the underlying +/// `connections` vector corresponds to the connection to one worker. It assumes a 1:1 mapping +/// between worker and tasks, and upon calling [WorkerConnectionPool::get_or_init_worker_connection] +/// it will initialize the corresponding position in the vector matching the provided `target_task` +/// index. +pub(crate) struct WorkerConnectionPool { + connections: Vec>>>, + pub(crate) metrics: ExecutionPlanMetricsSet, +} + +impl WorkerConnectionPool { + /// Builds a new [WorkerConnectionPool] with as many empty slots for [WorkerConnection]s as + /// the provided `input_tasks`. + pub(crate) fn new(input_tasks: usize) -> Self { + let mut connections = Vec::with_capacity(input_tasks); + for _ in 0..input_tasks { + connections.push(OnceLock::new()); + } + Self { + connections, + metrics: ExecutionPlanMetricsSet::default(), + } + } + + /// Lazily initializes the [WorkerConnection] corresponding to the provided `target_task` + /// (therefore maintaining one independent [WorkerConnection] per `target_task`), and + /// returns it. + pub(crate) fn get_or_init_worker_connection( + &self, + input_stage: &Stage, + target_partitions: Range, + target_task: usize, + ctx: &Arc, + ) -> Result<&WorkerConnection> { + let Some(worker_connection) = self.connections.get(target_task) else { + return internal_err!( + "WorkerConnections: Task index {target_task} not found, only have {} tasks", + self.connections.len() + ); + }; + + let conn = worker_connection.get_or_init(|| { + WorkerConnection::init( + input_stage, + target_partitions, + target_task, + ctx, + &self.metrics, + ) + .map_err(Arc::new) + }); + + match conn { + Ok(v) => Ok(v), + Err(err) => Err(DataFusionError::Shared(Arc::clone(err))), + } + } +} + +type WorkerMsg = Result<(FlightData, FlightAppMetadata, MemoryReservation), Status>; + +/// Represents a connection to one [Worker]. Network boundaries will use this for streaming +/// data from single partitions while the actual network communication is handling all the partitions +/// under the hood. +/// +/// This is done so that, rather than issuing one gRPC stream per partition, we issue one gRPC stream +/// per group of partitions, and we multiplex streamed record batches locally to in-memory channels. +/// +/// Even if Tonic can perfectly multiplex and interleave messages from different gRPC streams through +/// the same underlying TCP connection, there do is some overhead in having one gRPC stream per +/// partition VS a single gRPC stream interleaving multiple partitions. The whole serialized plan +/// needs to be sent over the wire on every gRPC call, so the less gRPC calls we do the better. +pub(crate) struct WorkerConnection { + task: Arc>, + not_consumed_streams: Arc, + cancel_token: CancellationToken, + per_partition_rx: DashMap>, + + // Metrics collection stuff. + curr_mem_used: Arc, + elapsed_compute: Time, +} + +impl WorkerConnection { + fn init( + input_stage: &Stage, + target_partition_range: Range, + target_task: usize, + ctx: &Arc, + metrics: &ExecutionPlanMetricsSet, + ) -> Result { + let channel_resolver = get_distributed_channel_resolver(ctx.as_ref()); + + // Stuff for collecting metrics. + let curr_mem_used = Arc::new(AtomicUsize::new(0)); + let curr_mem_used_clone = Arc::clone(&curr_mem_used); + + // Track the maximum memory used to buffer recieved messages. + let mut curr_max_mem = 0; + let max_mem_used = MetricBuilder::new(metrics).global_gauge("max_mem_used"); + // Track the total encoded size of all recieved messages. + let bytes_transferred = MetricBuilder::new(metrics).global_counter("bytes_transferred"); + // Track end-to-end network latency distribution for all messages. + let network_latency_metrics = NetworkLatencyMetrics::new(metrics); + // Track the total CPU time spent in polling messages over the network + decoding them. + let elapsed_compute = Time::new(); + let elapsed_compute_clone = elapsed_compute.clone(); + MetricBuilder::new(metrics).build(MetricValue::ElapsedCompute(elapsed_compute.clone())); + + // Building the actual request that will be sent to the worker. + let mut headers = get_config_extension_propagation_headers(ctx.session_config())?; + headers.extend(get_passthrough_headers(ctx.session_config())); + let ticket = Request::from_parts( + MetadataMap::from_headers(headers), + Extensions::default(), + Ticket { + ticket: DoGet { + plan_proto: Bytes::clone(input_stage.plan.encoded()?), + target_partition_start: target_partition_range.start as u64, + target_partition_end: target_partition_range.end as u64, + stage_key: Some(StageKey::new( + Bytes::from(input_stage.query_id.as_bytes().to_vec()), + input_stage.num as u64, + target_task as u64, + )), + target_task_index: target_task as u64, + target_task_count: input_stage.tasks.len() as u64, + } + .encode_to_vec() + .into(), + }, + ); + + let Some(task) = input_stage.tasks.get(target_task) else { + return internal_err!("ProgrammingError: Task {target_task} not found"); + }; + let Some(url) = task.url.clone() else { + return internal_err!("ProgrammingError: task is unassigned, cannot proceed"); + }; + + // The senders and receivers are unbounded queues used for multiplexing the record + // batches sent through the single gRPC stream into one stream per partition. + // The received record batches contain information of the partition to which they belong, + // so we use that for determining where to put them. + let mut per_partition_tx = Vec::with_capacity(target_partition_range.len()); + let per_partition_rx = DashMap::with_capacity(target_partition_range.len()); + for partition in target_partition_range.clone() { + let (tx, rx) = tokio::sync::mpsc::unbounded_channel::(); + per_partition_tx.push(tx); + per_partition_rx.insert(partition, rx); + } + + // We are retaining record batches in memory until they are consumed, so we need to account + // for them in the memory pool. + let memory_pool = Arc::clone(ctx.memory_pool()); + + // Cancellation token allows us to stop the background task promptly when all partition + // streams are dropped (e.g., when the query is cancelled). + let cancel_token = CancellationToken::new(); + let cancel = cancel_token.clone(); + + // This task will pull data from all the partitions in `target_partition_range`, and will + // fan them out to the appropriate `per_partition_rx` based on the "partition" declared + // in each individual record batch flight metadata. + let task = SpawnedTask::spawn(async move { + let mut client = match channel_resolver.get_flight_client_for_url(&url).await { + Ok(v) => v, + Err(err) => { + return fanout(&per_partition_tx, datafusion_error_to_tonic_status(&err)); + } + }; + + let mut interleaved_stream = match client.do_get(ticket).await { + Ok(v) => v.into_inner(), + Err(err) => return fanout(&per_partition_tx, err), + }; + + // Recorder updates network latency metrics when it's dropped. + let mut network_latency_recorder = + NetworkLatencyRecorder::new(network_latency_metrics); + + let consumer = MemoryConsumer::new("WorkerConnection"); + + loop { + // Check for cancellation while waiting for the next message. + let msg = tokio::select! { + biased; + _ = cancel.cancelled() => return, + msg = interleaved_stream.next() => { + match msg { + Some(Ok(v)) => v, + Some(Err(err)) => return fanout(&per_partition_tx, err), + None => return, // Stream exhausted + } + } + }; + + // Earliest time at which the msg was received. + let msg_received_time = SystemTime::now(); + + let flight_metadata = match FlightAppMetadata::decode(msg.app_metadata.as_ref()) { + Ok(v) => v, + Err(err) => { + return fanout(&per_partition_tx, Status::internal(err.to_string())); + } + }; + + // Update the running latency tracker. + network_latency_recorder.record_from_metadata( + &flight_metadata, + msg_received_time, + ); + let partition = flight_metadata.partition as usize; + // the `per_partition_tx` variable is using a normal `Vec` for storing the + // channel transmitters, so we need to subtract the `target_partition_range.start` + // to the `partition` in order to offset it to the appropriate index. + let sender_i = partition - target_partition_range.start; + + let Some(o_tx) = per_partition_tx.get(sender_i) else { + let msg = format!( + "Received partition {partition} in Flight metadata, but available partitions are {target_partition_range:?}" + ); + return fanout(&per_partition_tx, Status::internal(msg)); + }; + + // We need to send the memory reservation in the same tuple as the actual message + // so that it gets dropped as soon as the message leaves the queue. Dropping the + // memory reservation means releasing the memory from the pool for that specific + // message + let mut reservation = consumer.clone_with_new_id().register(&memory_pool); + let size = msg.encoded_len(); + + // Update memory related metrics. + bytes_transferred.add(size); + let curr_mem_used = curr_mem_used.fetch_add(size, Ordering::Relaxed); + if curr_mem_used > curr_max_mem { + curr_max_mem = curr_mem_used; + max_mem_used.set(curr_max_mem); + } + + reservation.grow(size); + if o_tx.send(Ok((msg, flight_metadata, reservation))).is_err() { + return; // channel closed + }; + } + }.with_elapsed_compute(elapsed_compute)); + + Ok(Self { + task: Arc::new(task), + cancel_token, + not_consumed_streams: Arc::new(AtomicUsize::new(per_partition_rx.len())), + per_partition_rx, + + // metrics stuff + curr_mem_used: curr_mem_used_clone, + elapsed_compute: elapsed_compute_clone, + }) + } + + /// Streams the provided `partition` from the remote worker. + /// + /// Note that this does not issue a network request, the actual network request happened before + /// in the init step, and is in charge of handling not only this `partition`, but also all the + /// partitions passed in `target_partition_range`. This method just streams all the record + /// batches belonging to the provided `partition` from an in-memory queue, but what populates + /// this queue is [WorkerConnection::init]. + /// + /// When the returned stream is dropped (e.g., due to query cancellation), the background task + /// pulling from the Flight stream will be cancelled promptly. + pub(crate) fn stream_partition( + &self, + partition: usize, + on_metadata: impl Fn(FlightAppMetadata) + Send + Sync + 'static, + ) -> Result> + 'static> { + let Some((_, partition_receiver)) = self.per_partition_rx.remove(&partition) else { + return internal_err!( + "WorkerConnection has no stream for target partition {partition}. Was it already consumed?" + ); + }; + let task = Arc::clone(&self.task); + let cancel_token = self.cancel_token.clone(); + let curr_mem_used = Arc::clone(&self.curr_mem_used); + + let stream = UnboundedReceiverStream::new(partition_receiver); + let stream = stream.map_err(|err| FlightError::Tonic(Box::new(err))); + let stream = stream.map_ok(move |(data, meta, reservation)| { + curr_mem_used.fetch_sub(reservation.size(), Ordering::Relaxed); + drop(reservation); // <- drop the reservation, freeing memory on the memory pool. + let _ = &task; // <- keep the task that polls data from the network alive. + on_metadata(meta); + data + }); + let stream = FlightRecordBatchStream::new_from_flight_data(stream); + let stream = stream.map_err(map_flight_to_datafusion_error); + let stream = stream.with_elapsed_compute(self.elapsed_compute.clone()); + + // When the stream is dropped, cancel the background task to ensure prompt cleanup. + let not_consumed_streams = Arc::clone(&self.not_consumed_streams); + Ok(on_drop_stream(stream, move || { + let remaining_streams = not_consumed_streams.fetch_sub(1, Ordering::SeqCst) - 1; + if remaining_streams == 0 { + cancel_token.cancel(); + } + })) + } +} + +fn fanout(o_txs: &[UnboundedSender], err: Status) { + for o_tx in o_txs { + let _ = o_tx.send(Err(err.clone())); + } +} + +/// Aggregates network latency metrics ourselves since DataFusion's default aggregation for +/// all [Time] metrics is sum. +#[derive(Clone)] +struct NetworkLatencyMetrics { + first: Time, + min: Time, + max: Time, + avg: Time, + p99: Time, +} + +impl NetworkLatencyMetrics { + fn new(metrics: &ExecutionPlanMetricsSet) -> Self { + Self { + first: build_time_metric(metrics, "network_latency_first"), + min: build_time_metric(metrics, "network_latency_min"), + max: build_time_metric(metrics, "network_latency_max"), + avg: build_time_metric(metrics, "network_latency_avg"), + p99: build_time_metric(metrics, "network_latency_p99"), + } + } +} + +fn build_time_metric(metrics: &ExecutionPlanMetricsSet, name: &'static str) -> Time { + let time = Time::new(); + MetricBuilder::new(metrics).build(MetricValue::Time { + name: name.into(), + time: time.clone(), + }); + time +} + +/// Tracks message network latencies and publishes metrics on drop. +struct NetworkLatencyRecorder { + tracker: LatencyTracker, + metrics: NetworkLatencyMetrics, +} + +impl NetworkLatencyRecorder { + fn new(metrics: NetworkLatencyMetrics) -> Self { + Self { + tracker: LatencyTracker::new(), + metrics, + } + } + + fn record_from_metadata( + &mut self, + flight_metadata: &FlightAppMetadata, + message_recieved_time: SystemTime, + ) { + if flight_metadata.created_timestamp_unix_nanos == 0 { + return; + } + + let sent_time = + UNIX_EPOCH + Duration::from_nanos(flight_metadata.created_timestamp_unix_nanos); + if let Ok(delta) = message_recieved_time.duration_since(sent_time) { + self.tracker.track(delta); + } + } +} + +impl Drop for NetworkLatencyRecorder { + fn drop(&mut self) { + if let Some(first) = self.tracker.first() { + self.metrics.first.add_duration(first); + } + if let Some(min) = self.tracker.min() { + self.metrics.min.add_duration(min); + } + if let Some(max) = self.tracker.max() { + self.metrics.max.add_duration(max); + } + if let Some(avg) = self.tracker.avg() { + self.metrics.avg.add_duration(avg); + } + if let Some(p99) = self.tracker.quantile(0.99) { + self.metrics.p99.add_duration(p99); + } + } +} + +impl Debug for WorkerConnectionPool { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("WorkerConnections") + .field("num_connections", &self.connections.len()) + .finish() + } +} + +impl Clone for WorkerConnectionPool { + fn clone(&self) -> Self { + Self::new(self.connections.len()) + } +} + +impl Debug for WorkerConnection { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("WorkerConnection").finish() + } +} + +trait ElapsedComputeFutureExt: Future + Sized { + fn with_elapsed_compute(self, elapsed_compute: Time) -> ElapsedComputeFuture; +} + +trait ElapsedComputeStreamExt: Stream + Sized { + fn with_elapsed_compute(self, elapsed_compute: Time) -> ElapsedComputeStream; +} + +impl> ElapsedComputeFutureExt for F { + fn with_elapsed_compute(self, elapsed_compute: Time) -> ElapsedComputeFuture { + ElapsedComputeFuture { + inner: self, + curr: Duration::default(), + elapsed_compute, + } + } +} + +impl> ElapsedComputeStreamExt for S { + fn with_elapsed_compute(self, elapsed_compute: Time) -> ElapsedComputeStream { + ElapsedComputeStream { + inner: self, + curr: Duration::default(), + elapsed_compute, + } + } +} + +#[pin_project(PinnedDrop)] +struct ElapsedComputeStream { + #[pin] + inner: T, + curr: Duration, + elapsed_compute: Time, +} + +/// Drop implementation that ensures that any accumulated time is properly dumped to the metric +/// in case the stream gets dropped before completion. +#[pinned_drop] +impl PinnedDrop for ElapsedComputeStream { + fn drop(self: Pin<&mut Self>) { + if self.curr > Duration::default() { + let self_projected = self.project(); + self_projected + .elapsed_compute + .add_duration(*self_projected.curr); + } + } +} + +impl> Stream for ElapsedComputeStream { + type Item = O; + + fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let self_projected = self.project(); + let start = Instant::now(); + let result = self_projected.inner.poll_next(cx); + *self_projected.curr += start.elapsed(); + if result.is_ready() { + self_projected + .elapsed_compute + .add_duration(*self_projected.curr); + *self_projected.curr = Duration::default(); + } + result + } +} + +#[pin_project(PinnedDrop)] +struct ElapsedComputeFuture { + #[pin] + inner: T, + curr: Duration, + elapsed_compute: Time, +} + +/// Drop implementation that ensures that any accumulated time is properly dumped to the metric +/// in case the future gets dropped before completion. +#[pinned_drop] +impl PinnedDrop for ElapsedComputeFuture { + fn drop(self: Pin<&mut Self>) { + if self.curr > Duration::default() { + let self_projected = self.project(); + self_projected + .elapsed_compute + .add_duration(*self_projected.curr); + } + } +} + +impl> Future for ElapsedComputeFuture { + type Output = O; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let self_projected = self.project(); + let start = Instant::now(); + let result = self_projected.inner.poll(cx); + *self_projected.curr += start.elapsed(); + if result.is_ready() { + self_projected + .elapsed_compute + .add_duration(*self_projected.curr); + *self_projected.curr = Duration::default(); + } + result + } +} + +#[cfg(test)] +mod tests { + use super::*; + use futures::StreamExt; + use futures::stream::unfold; + + #[tokio::test] + async fn elapsed_compute_future() { + async fn cheap() { + tokio::time::sleep(Duration::from_millis(1)).await; + } + + async fn expensive() { + let mut _count = 0f64; + for i in 0..100000 { + tokio::task::yield_now().await; + _count /= i as f64 + } + } + + let cheap_time = Time::new(); + cheap().with_elapsed_compute(cheap_time.clone()).await; + println!("cheap future: {}", cheap_time.value()); + + let expensive_time = Time::new(); + expensive() + .with_elapsed_compute(expensive_time.clone()) + .await; + println!("expensive future: {}", expensive_time.value()); + + assert!(expensive_time.value() > cheap_time.value()); + } + + #[tokio::test] + async fn elapsed_compute_stream() { + fn cheap() -> impl Stream { + unfold(0i64, |state| async move { + if state < 10 { + tokio::time::sleep(Duration::from_micros(10)).await; + Some((state, state + 1)) + } else { + None + } + }) + } + + fn expensive() -> impl Stream { + unfold(0i64, |state| async move { + if state < 10 { + // Simulate expensive computation + let mut _count = 0f64; + for i in 1..100000 { + _count += (i as f64).sqrt(); + } + tokio::task::yield_now().await; + Some((state, state + 1)) + } else { + None + } + }) + } + + let cheap_time = Time::new(); + cheap() + .with_elapsed_compute(cheap_time.clone()) + .collect::>() + .await; + println!("cheap future: {}", cheap_time.value()); + + let expensive_time = Time::new(); + expensive() + .with_elapsed_compute(expensive_time.clone()) + .collect::>() + .await; + println!("expensive future: {}", expensive_time.value()); + + assert!(expensive_time.value() > cheap_time.value()); + } + + #[test] + fn network_latency_updates_on_drop() { + let metrics = NetworkLatencyMetrics::new(&ExecutionPlanMetricsSet::default()); + + let sent = UNIX_EPOCH + Duration::from_secs(5); + let received = UNIX_EPOCH + Duration::from_secs(8); + let mut flight_metadata = FlightAppMetadata::new(0); + flight_metadata.created_timestamp_unix_nanos = + sent.duration_since(UNIX_EPOCH).unwrap().as_nanos() as u64; + + { + let mut recorder = NetworkLatencyRecorder::new(metrics.clone()); + recorder.record_from_metadata(&flight_metadata, received); + } + + let expected = 3_000_000_000usize; + assert_eq!(metrics.first.value(), expected); + assert_eq!(metrics.min.value(), expected); + assert_eq!(metrics.max.value(), expected); + assert_eq!(metrics.avg.value(), expected); + assert!(metrics.p99.value() >= expected); + } +} diff --git a/src/lib.rs b/src/lib.rs index 5d0c492e..b15c743f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,32 +1,46 @@ #![deny(clippy::all)] -mod channel_resolver_ext; mod common; mod config_extension_ext; mod distributed_ext; -mod distributed_physical_optimizer_rule; mod execution_plans; mod flight_service; mod metrics; +mod passthrough_headers; mod stage; +mod distributed_planner; +mod networking; +mod observability; mod protobuf; #[cfg(any(feature = "integration", test))] pub mod test_utils; -pub use channel_resolver_ext::{BoxCloneSyncChannel, ChannelResolver}; +pub use arrow_ipc::CompressionType; pub use distributed_ext::DistributedExt; -pub use distributed_physical_optimizer_rule::{ - DistributedPhysicalOptimizerRule, InputStageInfo, NetworkBoundary, NetworkBoundaryExt, +pub use distributed_planner::{ + DistributedConfig, DistributedPhysicalOptimizerRule, NetworkBoundary, NetworkBoundaryExt, + TaskCountAnnotation, TaskEstimation, TaskEstimator, }; pub use execution_plans::{ - DistributedExec, NetworkCoalesceExec, NetworkShuffleExec, PartitionIsolatorExec, + BroadcastExec, DistributedExec, NetworkBroadcastExec, NetworkCoalesceExec, NetworkShuffleExec, + PartitionIsolatorExec, }; pub use flight_service::{ - ArrowFlightEndpoint, DefaultSessionBuilder, DistributedSessionBuilder, - DistributedSessionBuilderContext, MappedDistributedSessionBuilder, - MappedDistributedSessionBuilderExt, + DefaultSessionBuilder, MappedWorkerSessionBuilder, MappedWorkerSessionBuilderExt, TaskData, + Worker, WorkerQueryContext, WorkerSessionBuilder, +}; +pub use metrics::{DistributedMetricsFormat, rewrite_distributed_plan_with_metrics}; +pub use networking::{ + BoxCloneSyncChannel, ChannelResolver, DefaultChannelResolver, WorkerResolver, + create_flight_client, get_distributed_channel_resolver, get_distributed_worker_resolver, }; pub use stage::{ DistributedTaskContext, ExecutionTask, Stage, display_plan_ascii, display_plan_graphviz, + explain_analyze, +}; + +pub use observability::{ + ObservabilityService, ObservabilityServiceClient, ObservabilityServiceImpl, + ObservabilityServiceServer, PingRequest, PingResponse, }; diff --git a/src/metrics/latency_tracker.rs b/src/metrics/latency_tracker.rs new file mode 100644 index 00000000..e98bba3b --- /dev/null +++ b/src/metrics/latency_tracker.rs @@ -0,0 +1,134 @@ +use sketches_ddsketch::{Config, DDSketch}; +use std::time::Duration; + +/// Tracks latency stats using running aggregates and a DDSketch for quantiles. +#[derive(Clone)] +pub struct LatencyTracker { + // first sample + first_nanos: Option, + // DDSketch provides relative-error quantiles with bounded size. + sketch: DDSketch, +} + +impl Default for LatencyTracker { + fn default() -> Self { + Self::new() + } +} + +impl LatencyTracker { + /// Creates a new empty tracker. + pub fn new() -> Self { + Self { + first_nanos: None, + sketch: DDSketch::new(Config::defaults()), + } + } + + /// Adds a latency sample to the tracker. + pub fn track(&mut self, duration: Duration) { + let nanos = duration.as_nanos() as u64; + + if self.sketch.count() == 0 { + self.first_nanos = Some(nanos); + } + + self.sketch.add(nanos as f64); + } + + /// Returns the number of samples recorded. + pub fn count(&self) -> u64 { + self.sketch.count() as u64 + } + + /// Returns the first recorded latency, if any. + pub fn first(&self) -> Option { + self.first_nanos.map(Duration::from_nanos) + } + + /// Returns the minimum latency seen, if any. + pub fn min(&self) -> Option { + let value = self.sketch.min()?; + if value.is_finite() && value >= 0.0 { + Some(Duration::from_nanos(value.round() as u64)) + } else { + None + } + } + + /// Returns the maximum latency seen, if any. + pub fn max(&self) -> Option { + let value = self.sketch.max()?; + if value.is_finite() && value >= 0.0 { + Some(Duration::from_nanos(value.round() as u64)) + } else { + None + } + } + + /// Returns the average latency, if any. + pub fn avg(&self) -> Option { + let sum = self.sketch.sum()?; + let count = self.count(); + if count == 0 { + return None; + } + let avg = sum / count as f64; + if avg.is_finite() && avg >= 0.0 { + Some(Duration::from_nanos(avg.round() as u64)) + } else { + None + } + } + + /// Returns the estimated latency at the given quantile (0.0..=1.0). + pub fn quantile(&self, q: f64) -> Option { + if self.count() == 0 { + return None; + } + let q = q.clamp(0.0, 1.0); + let value = self.sketch.quantile(q).ok().flatten()?; + if value.is_finite() && value >= 0.0 { + Some(Duration::from_nanos(value.round() as u64)) + } else { + None + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn empty_tracker_returns_none() { + let tracker = LatencyTracker::new(); + assert_eq!(tracker.count(), 0); + assert!(tracker.first().is_none()); + assert!(tracker.min().is_none()); + assert!(tracker.max().is_none()); + assert!(tracker.avg().is_none()); + assert!(tracker.quantile(0.90).is_none()); + assert!(tracker.quantile(0.99).is_none()); + } + + #[test] + fn basic_stats_and_quantiles() { + let mut tracker = LatencyTracker::new(); + for nanos in [10u64, 20, 30, 40, 50] { + tracker.track(Duration::from_nanos(nanos)); + } + + assert_eq!(tracker.count(), 5); + assert_eq!(tracker.first(), Some(Duration::from_nanos(10))); + assert_eq!(tracker.min(), Some(Duration::from_nanos(10))); + assert_eq!(tracker.max(), Some(Duration::from_nanos(50))); + assert_eq!(tracker.avg(), Some(Duration::from_nanos(30))); + let p90 = tracker.quantile(0.90).unwrap().as_nanos() as i64; + let p99 = tracker.quantile(0.99).unwrap().as_nanos() as i64; + // DDSketch provides approximate quantiles with bounded relative error, + // so we use a wider range to account for this. + assert!((40..=60).contains(&p90)); + assert!((40..=60).contains(&p99)); + } +} diff --git a/src/metrics/metrics_collecting_stream.rs b/src/metrics/metrics_collecting_stream.rs deleted file mode 100644 index 7e2e4e67..00000000 --- a/src/metrics/metrics_collecting_stream.rs +++ /dev/null @@ -1,264 +0,0 @@ -use std::pin::Pin; -use std::task::{Context, Poll}; - -use crate::metrics::proto::MetricsSetProto; -use crate::protobuf::StageKey; -use crate::protobuf::{AppMetadata, FlightAppMetadata}; -use arrow_flight::{FlightData, error::FlightError}; -use dashmap::DashMap; -use futures::stream::Stream; -use pin_project::pin_project; -use prost::Message; -use std::sync::Arc; - -/// MetricsCollectingStream wraps a FlightData stream and extracts metrics from app_metadata -/// while passing through all the other FlightData unchanged. -#[pin_project] -pub struct MetricsCollectingStream -where - S: Stream> + Send, -{ - #[pin] - inner: S, - metrics_collection: Arc>>, -} - -impl MetricsCollectingStream -where - S: Stream> + Send, -{ - pub fn new( - stream: S, - metrics_collection: Arc>>, - ) -> Self { - Self { - inner: stream, - metrics_collection, - } - } - - fn extract_metrics_from_flight_data( - metrics_collection: Arc>>, - flight_data: &mut FlightData, - ) -> Result<(), FlightError> { - if flight_data.app_metadata.is_empty() { - return Ok(()); - } - - let metadata = - FlightAppMetadata::decode(flight_data.app_metadata.as_ref()).map_err(|e| { - FlightError::ProtocolError(format!("failed to decode app_metadata: {}", e)) - })?; - - let Some(content) = metadata.content else { - return Err(FlightError::ProtocolError( - "expected Some content in app_metadata".to_string(), - )); - }; - - let AppMetadata::MetricsCollection(task_metrics_set) = content; - - for task_metrics in task_metrics_set.tasks { - let Some(stage_key) = task_metrics.stage_key else { - return Err(FlightError::ProtocolError( - "expected Some StageKey in MetricsCollectingStream, got None".to_string(), - )); - }; - metrics_collection.insert(stage_key, task_metrics.metrics); - } - flight_data.app_metadata.clear(); - - Ok(()) - } -} - -impl Stream for MetricsCollectingStream -where - S: Stream> + Send, -{ - type Item = Result; - - fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - let this = self.project(); - match this.inner.poll_next(cx) { - Poll::Ready(Some(Ok(mut flight_data))) => { - // Extract metrics from app_metadata if present. - match Self::extract_metrics_from_flight_data( - this.metrics_collection.clone(), - &mut flight_data, - ) { - Ok(_) => Poll::Ready(Some(Ok(flight_data))), - Err(e) => Poll::Ready(Some(Err(e))), - } - } - Poll::Ready(Some(Err(err))) => Poll::Ready(Some(Err(err))), - Poll::Ready(None) => Poll::Ready(None), - Poll::Pending => Poll::Pending, - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::protobuf::{ - AppMetadata, FlightAppMetadata, MetricsCollection, StageKey, TaskMetrics, - }; - use crate::test_utils::metrics::make_test_metrics_set_proto_from_seed; - use arrow_flight::FlightData; - use futures::stream::{self, StreamExt}; - use prost::{Message, bytes::Bytes}; - - fn assert_protocol_error(result: Result, expected_msg: &str) { - let FlightError::ProtocolError(msg) = result.unwrap_err() else { - panic!("expected FlightError::ProtocolError"); - }; - assert!(msg.contains(expected_msg)); - } - - fn make_flight_data(data: Vec, metadata: Option) -> FlightData { - let metadata_bytes = match metadata { - Some(metadata) => metadata.encode_to_vec().into(), - None => Bytes::new(), - }; - FlightData { - flight_descriptor: None, - data_header: Bytes::new(), - app_metadata: metadata_bytes, - data_body: data.into(), - } - } - - // Tests that metrics are extracted from FlightData. Metrics are always stored per task, so this - // tests creates some random metrics and tasks and puts them in the app_metadata field of - // FlightData message. Then, it streams these messages through the MetricsCollectingStream - // and asserts that the metrics are collected correctly. - #[tokio::test] - async fn test_metrics_collecting_stream_extracts_and_removes_metadata() { - let stage_keys = vec![ - StageKey { - query_id: Bytes::from("test_query"), - stage_id: 1, - task_number: 1, - }, - StageKey { - query_id: Bytes::from("test_query"), - stage_id: 1, - task_number: 2, - }, - ]; - - let app_metadatas = stage_keys - .iter() - .map(|stage_key| FlightAppMetadata { - content: Some(AppMetadata::MetricsCollection(MetricsCollection { - tasks: vec![TaskMetrics { - stage_key: Some(stage_key.clone()), - // use the task number to seed the test metrics set for convenience - metrics: vec![make_test_metrics_set_proto_from_seed(stage_key.task_number)], - }], - })), - }) - .collect::>(); - - // Create test FlightData messages - some with metadata, some without - let flight_messages = vec![ - make_flight_data(vec![1], Some(app_metadatas[0].clone())), - make_flight_data(vec![2], None), - make_flight_data(vec![3], Some(app_metadatas[1].clone())), - ] - .into_iter() - .map(Ok); - - // Collect all messages from the stream. All should have empty app_metadata. - let metrics_collection = Arc::new(DashMap::new()); - let input_stream = stream::iter(flight_messages); - let collecting_stream = - MetricsCollectingStream::new(input_stream, metrics_collection.clone()); - let collected_messages: Vec = collecting_stream - .map(|result| result.unwrap()) - .collect() - .await; - - // Assert the data is unchanged and app_metadata is cleared - assert_eq!(collected_messages.len(), 3); - assert!( - collected_messages - .iter() - .all(|msg| msg.app_metadata.is_empty()) - ); - - // Verify the data in the messages. - assert_eq!(collected_messages[0].data_body, vec![1]); - assert_eq!(collected_messages[1].data_body, vec![2]); - assert_eq!(collected_messages[2].data_body, vec![3]); - - // Verify the correct metrics were collected - assert_eq!(metrics_collection.len(), 2); - for stage_key in stage_keys { - let collected_metrics = metrics_collection.get(&stage_key).unwrap(); - assert_eq!(collected_metrics.len(), 1); - assert_eq!( - collected_metrics[0], - make_test_metrics_set_proto_from_seed(stage_key.task_number) - ); - } - } - - #[tokio::test] - async fn test_metrics_collecting_stream_error_missing_stage_key() { - let task_metrics_with_no_stage_key = TaskMetrics { - stage_key: None, - metrics: vec![make_test_metrics_set_proto_from_seed(1)], - }; - - let invalid_app_metadata = FlightAppMetadata { - content: Some(AppMetadata::MetricsCollection(MetricsCollection { - tasks: vec![task_metrics_with_no_stage_key], - })), - }; - - let invalid_flight_data = make_flight_data(vec![1], Some(invalid_app_metadata)); - - let error_stream = stream::iter(vec![Ok(invalid_flight_data)]); - let mut collecting_stream = - MetricsCollectingStream::new(error_stream, Arc::new(DashMap::new())); - - let result = collecting_stream.next().await.unwrap(); - assert_protocol_error( - result, - "expected Some StageKey in MetricsCollectingStream, got None", - ); - } - - #[tokio::test] - async fn test_metrics_collecting_stream_error_decoding_metadata() { - let flight_data_with_invalid_metadata = FlightData { - flight_descriptor: None, - data_header: Bytes::new(), - app_metadata: vec![0xFF, 0xFF, 0xFF, 0xFF].into(), // Invalid protobuf data - data_body: vec![4, 5, 6].into(), - }; - - let error_stream = stream::iter(vec![Ok(flight_data_with_invalid_metadata)]); - let mut collecting_stream = - MetricsCollectingStream::new(error_stream, Arc::new(DashMap::new())); - - let result = collecting_stream.next().await.unwrap(); - assert_protocol_error(result, "failed to decode app_metadata"); - } - - #[tokio::test] - async fn test_metrics_collecting_stream_error_propagation() { - let metrics_collection = Arc::new(DashMap::new()); - - // Create a stream that emits an error - should be propagated through - let stream_error = FlightError::ProtocolError("stream error from inner stream".to_string()); - let error_stream = stream::iter(vec![Err(stream_error)]); - let mut collecting_stream = - MetricsCollectingStream::new(error_stream, metrics_collection.clone()); - - let result = collecting_stream.next().await.unwrap(); - assert_protocol_error(result, "stream error from inner stream"); - } -} diff --git a/src/metrics/mod.rs b/src/metrics/mod.rs index b82d8c0c..94b5ad69 100644 --- a/src/metrics/mod.rs +++ b/src/metrics/mod.rs @@ -1,6 +1,10 @@ -mod metrics_collecting_stream; +pub(crate) mod latency_tracker; pub(crate) mod proto; mod task_metrics_collector; mod task_metrics_rewriter; -pub(crate) use metrics_collecting_stream::MetricsCollectingStream; -pub(crate) use task_metrics_collector::TaskMetricsCollector; +pub(crate) use task_metrics_collector::{MetricsCollectorResult, TaskMetricsCollector}; +pub use task_metrics_rewriter::{DistributedMetricsFormat, rewrite_distributed_plan_with_metrics}; + +/// Label used to annotate metrics in execution plan nodes with the task in which they were executed. +/// Note that the same task id may be used in multiple stages. +pub const DISTRIBUTED_DATAFUSION_TASK_ID_LABEL: &str = "task_id"; diff --git a/src/metrics/proto.rs b/src/metrics/proto.rs index ad954382..9897d29e 100644 --- a/src/metrics/proto.rs +++ b/src/metrics/proto.rs @@ -3,18 +3,22 @@ use datafusion::common::internal_err; use datafusion::error::DataFusionError; use datafusion::physical_plan::metrics::{Count, Gauge, Label, Time, Timestamp}; use datafusion::physical_plan::metrics::{Metric, MetricValue, MetricsSet}; +use datafusion::physical_plan::metrics::{PruningMetrics as DfPruningMetrics, RatioMetrics}; use std::borrow::Cow; use std::sync::Arc; /// A MetricProto is a protobuf mirror of [datafusion::physical_plan::metrics::Metric]. #[derive(Clone, PartialEq, ::prost::Message)] pub struct MetricProto { - #[prost(oneof = "MetricValueProto", tags = "1,2,3,4,5,6,7,8,9,10,11")] + #[prost( + oneof = "MetricValueProto", + tags = "1,2,3,4,5,6,7,8,9,10,11,12,13,14,15" + )] // This field is *always* set. It is marked optional due to protobuf "oneof" requirements. pub metric: Option, - #[prost(message, repeated, tag = "12")] + #[prost(message, repeated, tag = "16")] pub labels: Vec, - #[prost(uint64, optional, tag = "13")] + #[prost(uint64, optional, tag = "17")] pub partition: Option, } @@ -26,6 +30,18 @@ pub struct MetricsSetProto { pub metrics: Vec, } +impl MetricsSetProto { + pub fn new() -> Self { + Self { + metrics: Vec::new(), + } + } + + pub fn push(&mut self, metric: MetricProto) { + self.metrics.push(metric) + } +} + /// MetricValueProto is a protobuf mirror of the [datafusion::physical_plan::metrics::MetricValue] enum. #[derive(Clone, PartialEq, Eq, ::prost::Oneof)] pub enum MetricValueProto { @@ -51,6 +67,14 @@ pub enum MetricValueProto { StartTimestamp(StartTimestamp), #[prost(message, tag = "11")] EndTimestamp(EndTimestamp), + #[prost(message, tag = "12")] + OutputBytes(OutputBytes), + #[prost(message, tag = "13")] + OutputBatches(OutputBatches), + #[prost(message, tag = "14")] + PruningMetrics(NamedPruningMetrics), + #[prost(message, tag = "15")] + Ratio(NamedRatio), } #[derive(Clone, PartialEq, Eq, ::prost::Message)] @@ -125,6 +149,38 @@ pub struct EndTimestamp { pub value: Option, } +#[derive(Clone, PartialEq, Eq, ::prost::Message)] +pub struct OutputBytes { + #[prost(uint64, tag = "1")] + pub value: u64, +} + +#[derive(Clone, PartialEq, Eq, ::prost::Message)] +pub struct OutputBatches { + #[prost(uint64, tag = "1")] + pub value: u64, +} + +#[derive(Clone, PartialEq, Eq, ::prost::Message)] +pub struct NamedPruningMetrics { + #[prost(string, tag = "1")] + pub name: String, + #[prost(uint64, tag = "2")] + pub pruned: u64, + #[prost(uint64, tag = "3")] + pub matched: u64, +} + +#[derive(Clone, PartialEq, Eq, ::prost::Message)] +pub struct NamedRatio { + #[prost(string, tag = "1")] + pub name: String, + #[prost(uint64, tag = "2")] + pub part: u64, + #[prost(uint64, tag = "3")] + pub total: u64, +} + /// A ProtoLabel mirrors [datafusion::physical_plan::metrics::Label]. #[derive(Clone, PartialEq, Eq, ::prost::Message)] pub struct ProtoLabel { @@ -148,8 +204,8 @@ pub fn df_metrics_set_to_proto( Err(err) => { // Check if this is the specific custom metrics error we want to filter out if let DataFusionError::Internal(msg) = &err { - if msg == CUSTOM_METRICS_NOT_SUPPORTED { - // Filter out custom metrics error - continue processing other metrics + if msg == CUSTOM_METRICS_NOT_SUPPORTED || msg == UNSUPPORTED_METRICS { + // Filter out custom/unsupported metrics error - continue processing other metrics continue; } } @@ -179,6 +235,9 @@ pub fn metrics_set_proto_to_df( const CUSTOM_METRICS_NOT_SUPPORTED: &str = "custom metrics are not supported in metrics proto conversion"; +/// New DataFusion metrics that are not yet supported in proto conversion. +const UNSUPPORTED_METRICS: &str = "metric type not supported in proto conversion"; + /// df_metric_to_proto converts a `datafusion::physical_plan::metrics::Metric` to a `MetricProto`. It does not consume the Arc. pub fn df_metric_to_proto(metric: Arc) -> Result { let partition = metric.partition().map(|p| p as u64); @@ -273,6 +332,34 @@ pub fn df_metric_to_proto(metric: Arc) -> Result internal_err!("{}", CUSTOM_METRICS_NOT_SUPPORTED), + MetricValue::OutputBytes(count) => Ok(MetricProto { + metric: Some(MetricValueProto::OutputBytes(OutputBytes { value: count.value() as u64 })), + partition, + labels, + }), + MetricValue::OutputBatches(count) => Ok(MetricProto { + metric: Some(MetricValueProto::OutputBatches(OutputBatches { value: count.value() as u64 })), + partition, + labels, + }), + MetricValue::PruningMetrics { name, pruning_metrics } => Ok(MetricProto { + metric: Some(MetricValueProto::PruningMetrics(NamedPruningMetrics { + name: name.to_string(), + pruned: pruning_metrics.pruned() as u64, + matched: pruning_metrics.matched() as u64, + })), + partition, + labels, + }), + MetricValue::Ratio { name, ratio_metrics } => Ok(MetricProto { + metric: Some(MetricValueProto::Ratio(NamedRatio { + name: name.to_string(), + part: ratio_metrics.part() as u64, + total: ratio_metrics.total() as u64, + })), + partition, + labels, + }), } } @@ -398,6 +485,50 @@ pub fn metric_proto_to_df(metric: MetricProto) -> Result, DataFusion labels, ))) } + Some(MetricValueProto::OutputBytes(output_bytes)) => { + let count = Count::new(); + count.add(output_bytes.value as usize); + Ok(Arc::new(Metric::new_with_labels( + MetricValue::OutputBytes(count), + partition, + labels, + ))) + } + Some(MetricValueProto::OutputBatches(output_batches)) => { + let count = Count::new(); + count.add(output_batches.value as usize); + Ok(Arc::new(Metric::new_with_labels( + MetricValue::OutputBatches(count), + partition, + labels, + ))) + } + Some(MetricValueProto::PruningMetrics(named_pruning)) => { + let pruning_metrics = DfPruningMetrics::new(); + pruning_metrics.add_pruned(named_pruning.pruned as usize); + pruning_metrics.add_matched(named_pruning.matched as usize); + Ok(Arc::new(Metric::new_with_labels( + MetricValue::PruningMetrics { + name: Cow::Owned(named_pruning.name), + pruning_metrics, + }, + partition, + labels, + ))) + } + Some(MetricValueProto::Ratio(named_ratio)) => { + let ratio_metrics = RatioMetrics::new(); + ratio_metrics.set_part(named_ratio.part as usize); + ratio_metrics.set_total(named_ratio.total as usize); + Ok(Arc::new(Metric::new_with_labels( + MetricValue::Ratio { + name: Cow::Owned(named_ratio.name), + ratio_metrics, + }, + partition, + labels, + ))) + } None => internal_err!("proto metric is missing the metric field"), } } @@ -408,6 +539,7 @@ mod tests { use datafusion::physical_plan::metrics::CustomMetricValue; use datafusion::physical_plan::metrics::{Count, Gauge, Label, MetricsSet, Time, Timestamp}; use datafusion::physical_plan::metrics::{Metric, MetricValue}; + use datafusion::physical_plan::metrics::{PruningMetrics as DfPruningMetrics, RatioMetrics}; use std::borrow::Cow; use std::sync::Arc; @@ -421,8 +553,7 @@ mod tests { let roundtrip_count = roundtrip_metrics_set.iter().count(); assert_eq!( original_count, roundtrip_count, - "roundtrip should preserve metrics count for {}", - test_name + "roundtrip should preserve metrics count for {test_name}" ); // Verify equivalence of each metric. @@ -430,29 +561,25 @@ mod tests { assert_eq!( original.partition(), roundtrip.partition(), - "partition mismatch in {}", - test_name + "partition mismatch in {test_name}" ); assert_eq!( original.labels().len(), roundtrip.labels().len(), - "label count mismatch in {}", - test_name + "label count mismatch in {test_name}" ); for (orig_label, rt_label) in original.labels().iter().zip(roundtrip.labels().iter()) { assert_eq!( orig_label.name(), rt_label.name(), - "label name mismatch in {}", - test_name + "label name mismatch in {test_name}" ); assert_eq!( orig_label.value(), rt_label.value(), - "label value mismatch in {}", - test_name + "label value mismatch in {test_name}" ); } @@ -520,6 +647,40 @@ mod tests { rt.value().map(|dt| dt.timestamp_nanos_opt().unwrap()) ); } + (MetricValue::OutputBytes(orig), MetricValue::OutputBytes(rt)) => { + assert_eq!(orig.value(), rt.value()); + } + (MetricValue::OutputBatches(orig), MetricValue::OutputBatches(rt)) => { + assert_eq!(orig.value(), rt.value()); + } + ( + MetricValue::PruningMetrics { + name: n1, + pruning_metrics: p1, + }, + MetricValue::PruningMetrics { + name: n2, + pruning_metrics: p2, + }, + ) => { + assert_eq!(n1.as_ref(), n2.as_ref()); + assert_eq!(p1.pruned(), p2.pruned()); + assert_eq!(p1.matched(), p2.matched()); + } + ( + MetricValue::Ratio { + name: n1, + ratio_metrics: r1, + }, + MetricValue::Ratio { + name: n2, + ratio_metrics: r2, + }, + ) => { + assert_eq!(n1.as_ref(), n2.as_ref()); + assert_eq!(r1.part(), r2.part()); + assert_eq!(r1.total(), r2.total()); + } _ => panic!( "mismatched metric types in roundtrip test {}: {:?} vs {:?}", test_name, @@ -865,4 +1026,68 @@ mod tests { "should successfully roundtrip default timestamp" ); } + + #[test] + fn test_output_bytes_roundtrip() { + let mut metrics_set = MetricsSet::new(); + let count = Count::new(); + count.add(8192); + let labels = vec![Label::new("source", "parquet")]; + metrics_set.push(Arc::new(Metric::new_with_labels( + MetricValue::OutputBytes(count), + Some(0), + labels, + ))); + test_roundtrip_helper(metrics_set, "output_bytes"); + } + + #[test] + fn test_output_batches_roundtrip() { + let mut metrics_set = MetricsSet::new(); + let count = Count::new(); + count.add(42); + let labels = vec![Label::new("operator", "filter")]; + metrics_set.push(Arc::new(Metric::new_with_labels( + MetricValue::OutputBatches(count), + Some(1), + labels, + ))); + test_roundtrip_helper(metrics_set, "output_batches"); + } + + #[test] + fn test_pruning_metrics_roundtrip() { + let mut metrics_set = MetricsSet::new(); + let pruning_metrics = DfPruningMetrics::new(); + pruning_metrics.add_pruned(100); + pruning_metrics.add_matched(50); + let labels = vec![Label::new("predicate", "range")]; + metrics_set.push(Arc::new(Metric::new_with_labels( + MetricValue::PruningMetrics { + name: Cow::Borrowed("row_groups"), + pruning_metrics, + }, + Some(2), + labels, + ))); + test_roundtrip_helper(metrics_set, "pruning_metrics"); + } + + #[test] + fn test_ratio_metrics_roundtrip() { + let mut metrics_set = MetricsSet::new(); + let ratio_metrics = RatioMetrics::new(); + ratio_metrics.set_part(75); + ratio_metrics.set_total(100); + let labels = vec![Label::new("type", "cache_hit")]; + metrics_set.push(Arc::new(Metric::new_with_labels( + MetricValue::Ratio { + name: Cow::Borrowed("cache_hit_ratio"), + ratio_metrics, + }, + Some(3), + labels, + ))); + test_roundtrip_helper(metrics_set, "ratio_metrics"); + } } diff --git a/src/metrics/task_metrics_collector.rs b/src/metrics/task_metrics_collector.rs index bd10d960..31033b80 100644 --- a/src/metrics/task_metrics_collector.rs +++ b/src/metrics/task_metrics_collector.rs @@ -1,3 +1,4 @@ +use crate::NetworkBroadcastExec; use crate::execution_plans::NetworkCoalesceExec; use crate::execution_plans::NetworkShuffleExec; use crate::metrics::proto::MetricsSetProto; @@ -37,23 +38,24 @@ impl TreeNodeRewriter for TaskMetricsCollector { type Node = Arc; fn f_down(&mut self, plan: Self::Node) -> Result> { - // If the plan is an NetworkShuffleExec, assume it has collected metrics already + // For plan nodes in this task, collect metrics. + match plan.metrics() { + Some(metrics) => self.task_metrics.push(metrics.clone()), + None => { + // TODO: Consider using a more efficent encoding scheme to avoid empty slots in the vec. + self.task_metrics.push(MetricsSet::new()) + } + } + + // If the plan is a network boundary, assume it has collected metrics already // from child tasks. let metrics_collection = if let Some(node) = plan.as_any().downcast_ref::() { - let NetworkShuffleExec::Ready(ready) = node else { - return internal_err!( - "unexpected NetworkShuffleExec::Pending during metrics collection" - ); - }; - Some(Arc::clone(&ready.metrics_collection)) + Some(Arc::clone(&node.metrics_collection)) } else if let Some(node) = plan.as_any().downcast_ref::() { - let NetworkCoalesceExec::Ready(ready) = node else { - return internal_err!( - "unexpected NetworkCoalesceExec::Pending during metrics collection" - ); - }; - Some(Arc::clone(&ready.metrics_collection)) + Some(Arc::clone(&node.metrics_collection)) + } else if let Some(node) = plan.as_any().downcast_ref::() { + Some(Arc::clone(&node.metrics_collection)) } else { None }; @@ -82,14 +84,6 @@ impl TreeNodeRewriter for TaskMetricsCollector { return Ok(Transformed::new(plan, false, TreeNodeRecursion::Jump)); } - // For plan nodes in this task, collect metrics. - match plan.metrics() { - Some(metrics) => self.task_metrics.push(metrics.clone()), - None => { - // TODO: Consider using a more efficent encoding scheme to avoid empty slots in the vec. - self.task_metrics.push(MetricsSet::new()) - } - } Ok(Transformed::new(plan, false, TreeNodeRecursion::Continue)) } } @@ -127,13 +121,16 @@ mod tests { use datafusion::arrow::record_batch::RecordBatch; use futures::StreamExt; - use crate::DistributedExt; - use crate::DistributedPhysicalOptimizerRule; use crate::execution_plans::DistributedExec; - use crate::metrics::proto::metrics_set_proto_to_df; - use crate::test_utils::in_memory_channel_resolver::InMemoryChannelResolver; - use crate::test_utils::plans::{count_plan_nodes, get_stages_and_stage_keys}; + use crate::test_utils::in_memory_channel_resolver::{ + InMemoryChannelResolver, InMemoryWorkerResolver, + }; + use crate::test_utils::parquet::register_parquet_tables; + use crate::test_utils::plans::{ + count_plan_nodes_up_to_network_boundary, get_stages_and_stage_keys, + }; use crate::test_utils::session_context::register_temp_parquet_table; + use crate::{DistributedExt, DistributedPhysicalOptimizerRule}; use datafusion::execution::{SessionStateBuilder, context::SessionContext}; use datafusion::prelude::SessionConfig; use datafusion::{ @@ -152,12 +149,12 @@ mod tests { let state = SessionStateBuilder::new() .with_default_features() .with_config(config) - .with_distributed_channel_resolver(InMemoryChannelResolver::new()) - .with_physical_optimizer_rule(Arc::new( - DistributedPhysicalOptimizerRule::default() - .with_network_coalesce_tasks(2) - .with_network_shuffle_tasks(2), - )) + .with_distributed_worker_resolver(InMemoryWorkerResolver::new(10)) + .with_distributed_channel_resolver(InMemoryChannelResolver::default()) + .with_physical_optimizer_rule(Arc::new(DistributedPhysicalOptimizerRule)) + .with_distributed_task_estimator(2) + .with_distributed_metrics_collection(true) + .unwrap() .build(); let ctx = SessionContext::from(state); @@ -232,25 +229,7 @@ mod tests { ctx } - /// runs a sql query and returns the coordinator StageExec - async fn plan_sql(ctx: &SessionContext, sql: &str) -> DistributedExec { - let df = ctx.sql(sql).await.unwrap(); - let physical_distributed = df.create_physical_plan().await.unwrap(); - - let stage_exec = match physical_distributed - .as_any() - .downcast_ref::() - { - Some(stage_exec) => stage_exec.clone(), - None => panic!( - "expected StageExec from distributed optimization, got: {}", - physical_distributed.name() - ), - }; - stage_exec - } - - async fn execute_plan(stage_exec: &DistributedExec, ctx: &SessionContext) { + async fn execute_plan(stage_exec: Arc, ctx: &SessionContext) { let task_ctx = ctx.task_ctx(); let stream = stage_exec.execute(0, task_ctx).unwrap(); @@ -270,39 +249,44 @@ mod tests { async fn run_metrics_collection_e2e_test(sql: &str) { // Plan and execute the query let ctx = make_test_ctx().await; - let stage_exec = plan_sql(&ctx, sql).await; - execute_plan(&stage_exec, &ctx).await; + let df = ctx.sql(sql).await.unwrap(); + let plan = df.create_physical_plan().await.unwrap(); + execute_plan(plan.clone(), &ctx).await; + + let dist_exec = plan + .as_any() + .downcast_ref::() + .expect("expected DistributedExec"); // Assert to ensure the distributed test case is sufficiently complex. - let (stages, expected_stage_keys) = get_stages_and_stage_keys(&stage_exec); + let (stages, expected_stage_keys) = get_stages_and_stage_keys(dist_exec); assert!( expected_stage_keys.len() > 1, "expected more than 1 stage key in test. the plan was not distributed):\n{}", - DisplayableExecutionPlan::new(&stage_exec).indent(true) + DisplayableExecutionPlan::new(plan.as_ref()).indent(true) ); // Collect metrics for all tasks from the root StageExec. let collector = TaskMetricsCollector::new(); - let result = collector.collect(stage_exec.plan.clone()).unwrap(); + let result = collector.collect(dist_exec.plan.clone()).unwrap(); // Ensure that there's metrics for each node for each task for each stage. for expected_stage_key in expected_stage_keys { // Get the collected metrics for this task. let actual_metrics = result.input_task_metrics.get(&expected_stage_key).unwrap(); - // Assert that there's metrics for each node in this task. + // Verify that metrics were collected for all nodes. Some nodes may legitimately have + // empty metrics (e.g., custom execution plans without metrics), which is fine - we + // just verify that a metrics set exists for each node. The count assertion above + // ensures all nodes are included in the metrics collection. let stage = stages.get(&(expected_stage_key.stage_id as usize)).unwrap(); + let stage_plan = stage.plan.decoded().unwrap(); assert_eq!( actual_metrics.len(), - count_plan_nodes(stage.plan.decoded().unwrap()) + count_plan_nodes_up_to_network_boundary(stage_plan), + "Mismatch between collected metrics and actual nodes for {expected_stage_key:?}" ); - - // Ensure each node has at least one metric which was collected. - for metrics_set in actual_metrics.iter() { - let metrics_set = metrics_set_proto_to_df(metrics_set).unwrap(); - assert!(metrics_set.iter().count() > 0); - } } } @@ -313,7 +297,6 @@ mod tests { // Skip this test, it's failing after upgrading to datafusion 50 // See https://github.com/datafusion-contrib/datafusion-distributed/pull/146#issuecomment-3356621629 - #[ignore] #[tokio::test] async fn test_metrics_collection_e2e_2() { run_metrics_collection_e2e_test( @@ -349,8 +332,129 @@ mod tests { .await; } + /// Skipped due to https://github.com/apache/datafusion/issues/14218 + /// + /// When aggregating on a dictionary column (ex. `company` in this case which is Dict), + /// the aggregation seems to be outputting Utf8. Some assertion fails due to this, even in + /// single node execution: + /// "column types must match schema types, expected Dictionary(UInt16, Utf8) but found Utf8 at column index 0" #[tokio::test] + #[ignore] async fn test_metrics_collection_e2e_4() { run_metrics_collection_e2e_test("SELECT distinct company from table2").await; } + + /// Tests whether metrics are lost when a LIMIT causes early stream termination. + /// + /// Issue: https://github.com/datafusion-contrib/datafusion-distributed/issues/187 + /// + /// Metrics are piggybacked on the last FlightData message of the last partition stream + /// (see `do_get.rs`). If a LIMIT causes the client-side stream to be dropped before the + /// worker finishes, the last message (carrying metrics) is never received. + /// + /// This uses the `flights_1m` dataset (1M rows) so the worker is still producing data + /// when the LIMIT causes the client to drop the stream. + /// + /// This test is ignored because it demonstrates a known issue: metrics are lost when the + /// client drops the stream early. Un-ignore it to reproduce the issue or to verify a fix. + #[tokio::test] + #[ignore] + async fn test_metrics_collection_with_limit_causing_early_stream_termination() { + let ctx = make_test_ctx().await; + register_parquet_tables(&ctx).await.unwrap(); + + // GROUP BY forces a network shuffle; LIMIT 1 causes early stream termination. + let sql = + "SELECT \"FL_DATE\", COUNT(*) as cnt FROM flights_1m GROUP BY \"FL_DATE\" LIMIT 1"; + + let df = ctx.sql(sql).await.unwrap(); + let plan = df.create_physical_plan().await.unwrap(); + + let dist_exec = plan + .as_any() + .downcast_ref::() + .expect("expected DistributedExec"); + + let (stages, expected_stage_keys) = get_stages_and_stage_keys(dist_exec); + assert!( + expected_stage_keys.len() > 1, + "expected more than 1 stage key. Plan was not distributed:\n{}", + DisplayableExecutionPlan::new(plan.as_ref()).indent(true) + ); + + execute_plan(plan.clone(), &ctx).await; + + let collector = TaskMetricsCollector::new(); + let result = collector.collect(dist_exec.plan.clone()).unwrap(); + + for expected_stage_key in expected_stage_keys { + let actual_metrics = result + .input_task_metrics + .get(&expected_stage_key) + .unwrap_or_else(|| { + panic!( + "Missing metrics for stage key {expected_stage_key:?}. \ + The LIMIT caused the stream to be dropped before the worker \ + sent the last FlightData message with metrics." + ) + }); + + let stage = stages.get(&(expected_stage_key.stage_id as usize)).unwrap(); + let stage_plan = stage.plan.decoded().unwrap(); + assert_eq!( + actual_metrics.len(), + count_plan_nodes_up_to_network_boundary(stage_plan), + "Mismatch between collected metrics and actual nodes for {expected_stage_key:?}" + ); + } + } + + /// Test that verifies PartitionIsolatorExec nodes are preserved during metrics collection. + /// This tests the corner case where PartitionIsolatorExec nodes (which have no metrics) + /// must still be included in the metrics collection to maintain correct node-to-metric mapping. + #[tokio::test] + async fn test_metrics_collection_with_partition_isolator() { + let ctx = make_test_ctx().await; + ctx.sql("SET distributed.children_isolator_unions=true;") + .await + .unwrap(); + + // Use weather dataset (already available) + register_parquet_tables(&ctx).await.unwrap(); + + // UNION query that creates PartitionIsolatorExec nodes + let query = r#" + SELECT "MinTemp" FROM weather WHERE "RainToday" = 'yes' + UNION ALL + SELECT "MaxTemp" FROM weather WHERE "RainToday" = 'no' + "#; + + let df = ctx.sql(query).await.unwrap(); + let plan = df.create_physical_plan().await.unwrap(); + execute_plan(plan.clone(), &ctx).await; + + let dist_exec = plan + .as_any() + .downcast_ref::() + .expect("expected DistributedExec"); + + let (stages, expected_stage_keys) = get_stages_and_stage_keys(dist_exec); + let collector = TaskMetricsCollector::new(); + let result = collector.collect(dist_exec.plan.clone()).unwrap(); + + // Verify all nodes (including PartitionIsolatorExec) are preserved in metrics collection + for expected_stage_key in expected_stage_keys { + let actual_metrics = result.input_task_metrics.get(&expected_stage_key).unwrap(); + let stage = stages.get(&(expected_stage_key.stage_id as usize)).unwrap(); + let stage_plan = stage.plan.decoded().unwrap(); + + // Verify metrics count matches - this ensures all nodes are included in metrics collection + // regardless of whether they have metrics or not (some nodes may have empty metrics sets) + assert_eq!( + actual_metrics.len(), + count_plan_nodes_up_to_network_boundary(stage_plan), + "Metrics count must match plan nodes for stage {expected_stage_key:?}" + ); + } + } } diff --git a/src/metrics/task_metrics_rewriter.rs b/src/metrics/task_metrics_rewriter.rs index 529237ee..83cf0374 100644 --- a/src/metrics/task_metrics_rewriter.rs +++ b/src/metrics/task_metrics_rewriter.rs @@ -1,115 +1,334 @@ +use crate::distributed_planner::NetworkBoundaryExt; +use crate::execution_plans::DistributedExec; use crate::execution_plans::MetricsWrapperExec; -use crate::execution_plans::NetworkCoalesceExec; -use crate::execution_plans::NetworkShuffleExec; -use crate::metrics::proto::MetricsSetProto; -use crate::metrics::proto::metrics_set_proto_to_df; +use crate::metrics::DISTRIBUTED_DATAFUSION_TASK_ID_LABEL; +use crate::metrics::MetricsCollectorResult; +use crate::metrics::TaskMetricsCollector; +use crate::metrics::proto::{MetricsSetProto, metrics_set_proto_to_df}; +use crate::protobuf::StageKey; +use crate::stage::Stage; +use bytes::Bytes; +use datafusion::common::HashMap; use datafusion::common::tree_node::Transformed; use datafusion::common::tree_node::TreeNode; use datafusion::common::tree_node::TreeNodeRecursion; -use datafusion::common::tree_node::TreeNodeRewriter; use datafusion::error::Result; use datafusion::physical_plan::ExecutionPlan; use datafusion::physical_plan::internal_err; +use datafusion::physical_plan::metrics::{Label, Metric, MetricsSet}; use std::sync::Arc; +use std::vec; -/// TaskMetricsRewriter is used to enrich a task with metrics by re-writing the plan using [MetricsWrapperExec] nodes. -/// -/// Ex. for a plan with the form -/// AggregateExec -/// └── ProjectionExec -/// └── NetworkShuffleExec -/// -/// the task will be rewritten as -/// -/// MetricsWrapperExec (wrapped: AggregateExec) -/// └── MetricsWrapperExec (wrapped: ProjectionExec) -/// └── NetworkShuffleExec -/// (Note that the NetworkShuffleExec node is not wrapped) -pub struct TaskMetricsRewriter { - metrics: Vec, - idx: usize, +/// Format to use when displaying metrics for a distributed plan. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DistributedMetricsFormat { + /// Metrics are aggregated across all tasks. ex. a `output_rows=X` represents the output rows for all tasks. + Aggregated, + + /// Metric names are rewritten to include the task id. ex. `output_rows` -> `output_rows_0`, `output_rows_1` etc. + PerTask, } -impl TaskMetricsRewriter { - /// Create a new TaskMetricsRewriter. The provided metrics will be used to enrich the plan. - #[allow(dead_code)] - pub fn new(metrics: Vec) -> Self { - Self { metrics, idx: 0 } - } - - /// enrich_task_with_metrics rewrites the plan by wrapping nodes. If the length of the provided metrics set vec does not - /// match the number of nodes in the plan, an error will be returned. - #[allow(dead_code)] - pub fn enrich_task_with_metrics( - mut self, - plan: Arc, - ) -> Result> { - let transformed = plan.rewrite(&mut self)?; - if self.idx != self.metrics.len() { - return internal_err!( - "too many metrics sets provided to rewrite task: {} metrics sets provided, {} nodes in plan", - self.metrics.len(), - self.idx - ); +impl DistributedMetricsFormat { + pub(crate) fn to_rewrite_ctx(self, task_id: u64) -> RewriteCtx { + match self { + DistributedMetricsFormat::Aggregated => RewriteCtx::default(), + DistributedMetricsFormat::PerTask => RewriteCtx::from_task_id(task_id), } - Ok(transformed.data) } } -impl TreeNodeRewriter for TaskMetricsRewriter { - type Node = Arc; +/// Rewrites a distributed plan with metrics. Does nothing if the root node is not a [DistributedExec]. +/// Returns an error if the distributed plan was not executed. +pub fn rewrite_distributed_plan_with_metrics( + plan: Arc, + format: DistributedMetricsFormat, +) -> Result> { + let Some(distributed_exec) = plan.as_any().downcast_ref::() else { + return Ok(plan); + }; + + // Collect metrics from the DistributedExec's prepared plan. + let MetricsCollectorResult { + task_metrics, // Metrics for the DistributedExec plan + input_task_metrics, // Metrics for all child stages / tasks. + } = TaskMetricsCollector::new().collect(distributed_exec.prepared_plan()?)?; + + // Rewrite the DistributedExec's child plan with metrics. + let dist_exec_plan_with_metrics = rewrite_local_plan_with_metrics( + format.to_rewrite_ctx(0), // Task id is 0 for the DistributedExec plan + plan.children()[0].clone(), + task_metrics, + )?; + let plan = plan.with_new_children(vec![dist_exec_plan_with_metrics])?; - fn f_down(&mut self, plan: Self::Node) -> Result> { - if plan.as_any().is::() { - return Ok(Transformed::new(plan, false, TreeNodeRecursion::Jump)); + let metrics_collection = Arc::new(input_task_metrics); + + let transformed = plan.transform_down(|plan| { + // Transform all stages using NetworkShuffleExec and NetworkCoalesceExec as barriers. + if let Some(network_boundary) = plan.as_network_boundary() { + let stage = network_boundary.input_stage(); + // This transform is a bit inefficient because we traverse the plan nodes twice + // For now, we are okay with trading off performance for simplicity. + let plan_with_metrics = + stage_metrics_rewriter(stage, metrics_collection.clone(), format)?; + let network_boundary = network_boundary.with_input_stage(Stage::new( + stage.query_id, + stage.num, + plan_with_metrics, + stage.tasks.len(), + ))?; + let network_boundary = + MetricsWrapperExec::new(network_boundary, plan.metrics().unwrap_or_default()); + return Ok(Transformed::yes(Arc::new(network_boundary))); } - if plan.as_any().is::() { - return Ok(Transformed::new(plan, false, TreeNodeRecursion::Jump)); + + Ok(Transformed::no(plan)) + })?; + Ok(transformed.data) +} + +/// Extra information for rewriting local plans. +#[derive(Default)] +pub struct RewriteCtx { + /// Used to rename metrics for the current task. + pub task_id: Option, +} + +impl RewriteCtx { + pub(crate) fn from_task_id(task_id: u64) -> RewriteCtx { + RewriteCtx { + task_id: Some(task_id), } - if self.idx >= self.metrics.len() { - return internal_err!( - "not enough metrics provided to rewrite task: {} metrics provided", - self.metrics.len() - ); + } + + /// Rewrites the [MetricsSet] depending on the context. + pub(crate) fn maybe_rewrite_node_metics(&self, node_metrics: MetricsSet) -> MetricsSet { + if let Some(task_id) = self.task_id { + return annotate_metrics_set_with_task_id(node_metrics, task_id); + } + node_metrics + } +} + +/// Adds task id labels to all metrics in the provided [MetricsSet]. +/// +/// TODO: This re-allocates the vec of metrics by creating a new [MetricsSet]. It also +/// reallocates the labels vec for each metric. Can we avoid this? +/// See https://github.com/apache/datafusion/issues/19959 +pub fn annotate_metrics_set_with_task_id(metrics_set: MetricsSet, task_id: u64) -> MetricsSet { + let mut result = MetricsSet::new(); + + for metric in metrics_set.iter() { + let mut labels = metric.labels().to_vec(); + labels.push(Label::new( + DISTRIBUTED_DATAFUSION_TASK_ID_LABEL, + task_id.to_string(), + )); + result.push(Arc::new(Metric::new_with_labels( + metric.value().clone(), + metric.partition(), + labels, + ))); + } + + result +} + +/// Rewrites a local plan with metrics, stopping at network boundaries. +/// +/// Example: +/// +/// AggregateExec [output_rows = 1, elapsed_compute = 100] +/// └── ProjectionExec [output_rows = 2, elapsed_compute = 200] +/// └── NetworkShuffleExec [bytes_transferred = 100, max_mem_used = 100] +/// +/// The result will be: +/// +/// MetricsWrapperExec (wrapped: AggregateExec) [output_rows = 1, elapsed_compute = 100] +/// └── MetricsWrapperExec (wrapped: ProjectionExec) [output_rows = 2, elapsed_compute = 200] +/// └── MetricsWrapperExec (wrapped: NetworkShuffleExec) [bytes_transferred = 100, max_mem_used = 100] +pub fn rewrite_local_plan_with_metrics( + ctx: RewriteCtx, + plan: Arc, + metrics: Vec, +) -> Result> { + let mut idx = 0; + Ok(plan + .transform_down(|node| { + if idx >= metrics.len() { + return internal_err!("not enough metrics provided to rewrite plan"); + } + let mut node_metrics = metrics[idx].clone(); + + node_metrics = ctx.maybe_rewrite_node_metics(node_metrics); + + idx += 1; + Ok(Transformed::new( + Arc::new(MetricsWrapperExec::new(node.clone(), node_metrics)), + true, + if node.is_network_boundary() { + TreeNodeRecursion::Jump + } else { + TreeNodeRecursion::Continue + }, + )) + })? + .data) +} + +/// Enriches a stage with metrics from each task by re-writing the plan using +/// [MetricsWrapperExec] nodes. +/// +/// Example: +/// +/// For a stage with 2 tasks: +/// +/// Task 1: +/// AggregateExec [output_rows = 1, elapsed_compute = 100] +/// └── ProjectionExec [output_rows = 2, elapsed_compute = 200] +/// └── NetworkShuffleExec [bytes_transferred = 100, max_mem_used = 100] +/// +/// Task 2: +/// AggregateExec [output_rows = 3, elapsed_compute = 300] +/// └── ProjectionExec [output_rows = 4, elapsed_compute = 400] +/// └── NetworkShuffleExec [bytes_transferred = 200, max_mem_used = 200] +/// +/// The result will be: +/// +/// MetricsWrapperExec (wrapped: AggregateExec) [output_rows = 1, output_rows = 3, elapsed_compute = 100, elapsed_compute = 300] +/// └── MetricsWrapperExec (wrapped: ProjectionExec) [output_rows = 2, output_rows = 4, elapsed_compute = 200, elapsed_compute = 400] +/// └── MetricsWrapperExec (wrapped: NetworkShuffleExec) [bytes_transferred = 100, bytes_transferred = 200, max_mem_used = 100, max_mem_used = 200] +/// +/// Note: Metrics may be aggregated by name (ex. output_rows) automatically by various datafusion utils. +pub fn stage_metrics_rewriter( + stage: &Stage, + metrics_collection: Arc>>, + format: DistributedMetricsFormat, +) -> Result> { + let mut node_idx = 0; + + let plan = stage.plan.decoded()?; + + plan.clone().transform_down(|plan| { + // Collect metrics for this node. It should contain metrics from each task. + let mut stage_metrics = MetricsSet::new(); + + for task_id in 0..stage.tasks.len() { + let stage_key = StageKey::new(Bytes::from(stage.query_id.as_bytes().to_vec()), stage.num as u64, task_id as u64); + match metrics_collection.get(&stage_key) { + Some(task_metrics) => { + if node_idx >= task_metrics.len() { + return internal_err!( + "not enough metrics provided to rewrite task: {} metrics provided", + task_metrics.len() + ); + } + let node_metrics_protos = task_metrics[node_idx].clone(); + let mut node_metrics = metrics_set_proto_to_df(&node_metrics_protos)?; + + let rewrite_ctx = format.to_rewrite_ctx(task_id as u64); + node_metrics = rewrite_ctx.maybe_rewrite_node_metics(node_metrics); + + for metric in node_metrics.iter().map(Arc::clone) { + stage_metrics.push(metric); + } + } + None => { + return internal_err!( + "not enough metrics provided to rewrite task: missing metrics for task {} in stage {}", + task_id, + stage.num + ); + } + } } - let proto_metrics = &self.metrics[self.idx]; + + node_idx += 1; let wrapped_plan_node: Arc = Arc::new(MetricsWrapperExec::new( plan.clone(), - metrics_set_proto_to_df(proto_metrics)?, + stage_metrics, )); - let result = Transformed::new(wrapped_plan_node, true, TreeNodeRecursion::Continue); - self.idx += 1; - Ok(result) - } + Ok(Transformed::new( + wrapped_plan_node, + true, + if plan.is_network_boundary() { + TreeNodeRecursion::Jump + } else { + TreeNodeRecursion::Continue + } + )) + }).map(|v| v.data) } #[cfg(test)] mod tests { - use crate::metrics::proto::MetricsSetProto; - use crate::metrics::proto::df_metrics_set_to_proto; - use crate::metrics::task_metrics_collector::TaskMetricsCollector; - use crate::metrics::task_metrics_rewriter::TaskMetricsRewriter; + use crate::Stage; + use crate::metrics::DISTRIBUTED_DATAFUSION_TASK_ID_LABEL; + use crate::metrics::proto::{ + MetricsSetProto, df_metrics_set_to_proto, metrics_set_proto_to_df, + }; + use crate::metrics::task_metrics_rewriter::{ + annotate_metrics_set_with_task_id, stage_metrics_rewriter, + }; + use crate::metrics::{DistributedMetricsFormat, rewrite_distributed_plan_with_metrics}; + use crate::protobuf::StageKey; + use crate::test_utils::in_memory_channel_resolver::{ + InMemoryChannelResolver, InMemoryWorkerResolver, + }; use crate::test_utils::metrics::make_test_metrics_set_proto_from_seed; - use crate::test_utils::plans::count_plan_nodes; + use crate::test_utils::plans::count_plan_nodes_up_to_network_boundary; use crate::test_utils::session_context::register_temp_parquet_table; + use crate::{DistributedExec, DistributedPhysicalOptimizerRule}; + use bytes::Bytes; use datafusion::arrow::array::{Int32Array, StringArray}; use datafusion::arrow::datatypes::{DataType, Field, Schema}; use datafusion::arrow::record_batch::RecordBatch; + use datafusion::common::HashMap; use datafusion::execution::SessionStateBuilder; + use datafusion::physical_plan::metrics::{Count, Label, Metric, MetricValue, MetricsSet}; + use test_case::test_case; + + use datafusion::physical_plan::{ExecutionPlan, collect}; + use itertools::Itertools; + use uuid::Uuid; + + use crate::DistributedExt; + use crate::metrics::task_metrics_rewriter::MetricsWrapperExec; + use datafusion::physical_plan::empty::EmptyExec; use datafusion::prelude::SessionConfig; use datafusion::prelude::SessionContext; use std::sync::Arc; + async fn make_test_ctx() -> SessionContext { + make_test_ctx_inner(false).await + } + + async fn make_test_distributed_ctx() -> SessionContext { + make_test_ctx_inner(true).await + } + /// Creates a non-distributed session context and registers two tables: /// - table1 (id: int, name: string) /// - table2 (id: int, name: string, phone: string, balance: float64) - async fn make_test_ctx() -> SessionContext { - let config = SessionConfig::new().with_target_partitions(2); - let state = SessionStateBuilder::new() + async fn make_test_ctx_inner(distributed: bool) -> SessionContext { + let config = SessionConfig::new().with_target_partitions(4); + let mut builder = SessionStateBuilder::new() .with_default_features() - .with_config(config) - .build(); + .with_config(config); + + if distributed { + builder = builder + .with_distributed_worker_resolver(InMemoryWorkerResolver::new(10)) + .with_distributed_channel_resolver(InMemoryChannelResolver::default()) + .with_distributed_metrics_collection(true) + .unwrap() + .with_physical_optimizer_rule(Arc::new(DistributedPhysicalOptimizerRule)) + .with_distributed_task_estimator(2) + } + + let state = builder.build(); let ctx = SessionContext::from(state); // Create test data for table1 @@ -172,12 +391,30 @@ mod tests { ctx } + fn make_test_stage(plan: Arc) -> Stage { + Stage::new(Uuid::new_v4(), 2, plan, 4) + } + + fn collect_metrics_from_plan(plan: &Arc, metrics: &mut Vec) { + metrics.extend(plan.metrics()); + for child in plan.children() { + collect_metrics_from_plan(child, metrics); + } + } + + fn metrics_set_eq(a: &MetricsSet, b: &MetricsSet) -> bool { + println!("a: {a:?}"); + println!("b: {b:?}"); + // Check equality by converting to proto representation. + df_metrics_set_to_proto(a).unwrap() == df_metrics_set_to_proto(b).unwrap() + } + /// Asserts that we successfully re-write the metrics of a plan generated from the provided SQL query. /// Also asserts that the order which metrics are collected from a plan matches the order which /// they are re-written (ie. ensures we don't assign metrics to the wrong nodes) /// /// Only tests single node plans since the [TaskMetricsRewriter] stops on [NetworkBoundary]. - async fn run_metrics_rewriter_test(sql: &str) { + async fn run_stage_metrics_rewriter_test(sql: &str, format: DistributedMetricsFormat) { // Generate the plan let ctx = make_test_ctx().await; let plan = ctx @@ -188,48 +425,104 @@ mod tests { .await .unwrap(); - // Generate metrics for each plan node. - let expected_metrics = (0..count_plan_nodes(&plan)) - .map(|i| make_test_metrics_set_proto_from_seed(i as u64 + 10)) - .collect::>(); + let stage = make_test_stage(plan.clone()); - // Rewrite the metrics. - let rewriter = TaskMetricsRewriter::new(expected_metrics.clone()); - let rewritten_plan = rewriter.enrich_task_with_metrics(plan.clone()).unwrap(); + let num_metrics_per_task_per_node = 4; - // Collect metrics - let actual_metrics = TaskMetricsCollector::new() - .collect(rewritten_plan) - .unwrap() - .task_metrics; - - // Assert that all the metrics are present and in the same order. - assert_eq!(actual_metrics.len(), expected_metrics.len()); - for (actual_metrics_set, expected_metrics_set) in actual_metrics - .iter() - .map(|m| df_metrics_set_to_proto(m).unwrap()) - .zip(expected_metrics) - { - assert_eq!(actual_metrics_set, expected_metrics_set); + // Generate metrics for each task and store them in the map. + let mut metrics_collection = HashMap::new(); + for task_id in 0..stage.tasks.len() { + let stage_key = StageKey::new( + Bytes::from(stage.query_id.as_bytes().to_vec()), + stage.num as u64, + task_id as u64, + ); + let metrics = (0..count_plan_nodes_up_to_network_boundary(&plan)) + .map(|node_id| { + make_test_metrics_set_proto_from_seed( + (node_id * task_id) as u64, + num_metrics_per_task_per_node, + ) + }) + .collect::>(); + + metrics_collection.insert(stage_key, metrics); + } + let metrics_collection = Arc::new(metrics_collection); + + // Rewrite the plan. + let rewritten_plan = + stage_metrics_rewriter(&stage, metrics_collection.clone(), format).unwrap(); + + // Collect metrics from the plan. + let mut actual_metrics = vec![]; + collect_metrics_from_plan(&rewritten_plan, &mut actual_metrics); + assert_eq!( + actual_metrics.len(), + count_plan_nodes_up_to_network_boundary(&plan) + ); + + // Assert that metrics from all tasks are present. + // actual_stage_node_metrics_set contains metrics for all task ex. [output_rows=1, elapsed_compute=1, output_rows=2, elapsed_compute=2...] + for (node_id, actual_stage_node_metrics_set) in actual_metrics.iter().enumerate() { + // actual_task_node_metrics_set contains metrics for one task ex. [output_rows=1, elapsed_compute=1] + for (task_id, actual_task_node_metrics_set) in actual_stage_node_metrics_set + .iter() + .chunks(num_metrics_per_task_per_node) + .into_iter() + .enumerate() + { + let expected_task_node_metrics = metrics_collection + .get(&StageKey::new( + Bytes::from(stage.query_id.as_bytes().to_vec()), + stage.num as u64, + task_id as u64, + )) + .unwrap()[node_id] + .clone(); + + let mut actual_metrics_set = MetricsSet::new(); + actual_task_node_metrics_set + .for_each(|metric| actual_metrics_set.push(metric.clone())); + + // Convert from proto to check for equality. + let mut expected_metrics_set = + metrics_set_proto_to_df(&expected_task_node_metrics).unwrap(); + + if format == DistributedMetricsFormat::PerTask { + // Add task ids labels. We expect the actual metrics to be annotated by the + // rewriter when using DistributedMetricsFormat::PerTask + expected_metrics_set = + annotate_metrics_set_with_task_id(expected_metrics_set, task_id as u64); + } + assert!(metrics_set_eq(&actual_metrics_set, &expected_metrics_set)); + } } } + #[test_case(DistributedMetricsFormat::Aggregated ; "aggregated_metrics")] + #[test_case(DistributedMetricsFormat::PerTask ; "per_task_metrics")] #[tokio::test] - async fn test_metrics_rewriter_1() { - run_metrics_rewriter_test( + async fn test_stage_metrics_rewriter_1(format: DistributedMetricsFormat) { + run_stage_metrics_rewriter_test( "SELECT sum(balance) / 7.0 as avg_yearly from table2 group by name", + format, ) .await; } + #[test_case(DistributedMetricsFormat::Aggregated ; "aggregated_metrics")] + #[test_case(DistributedMetricsFormat::PerTask ; "per_task_metrics")] #[tokio::test] - async fn test_metrics_rewriter_2() { - run_metrics_rewriter_test("SELECT id, COUNT(*) as count FROM table1 WHERE id > 1 GROUP BY id ORDER BY id LIMIT 10").await; + async fn test_stage_metrics_rewriter_2(format: DistributedMetricsFormat) { + run_stage_metrics_rewriter_test("SELECT id, COUNT(*) as count FROM table1 WHERE id > 1 GROUP BY id ORDER BY id LIMIT 10", format).await; } + #[test_case(DistributedMetricsFormat::Aggregated ; "aggregated_metrics")] + #[test_case(DistributedMetricsFormat::PerTask ; "per_task_metrics")] #[tokio::test] - async fn test_metrics_rewriter_3() { - run_metrics_rewriter_test( + async fn test_stage_metrics_rewriter_3(format: DistributedMetricsFormat) { + run_stage_metrics_rewriter_test( "SELECT sum(balance) / 7.0 as avg_yearly FROM table2 WHERE name LIKE 'customer%' @@ -238,7 +531,112 @@ mod tests { FROM table2 t2_inner WHERE t2_inner.id = table2.id )", + format, ) .await; } + + #[tokio::test] + async fn test_rewrite_unexecuted_distributed_plan_with_metrics_err() { + let ctx = make_test_distributed_ctx().await; + let plan = ctx + .sql("SELECT id, COUNT(*) as count FROM table1 WHERE id > 1 GROUP BY id ORDER BY id LIMIT 10") + .await + .unwrap() + .create_physical_plan() + .await + .unwrap(); + assert!(plan.as_any().is::()); + assert!( + rewrite_distributed_plan_with_metrics(plan, DistributedMetricsFormat::Aggregated) + .is_err() + ); + } + + // Assert every plan node has at least one metric except partition isolators, network boundary nodes, and the root DistributedExec node. + fn assert_metrics_present_in_plan(plan: &Arc) { + if let Some(metrics) = plan.metrics() { + assert!(metrics.iter().count() > 0); + } else { + assert!(plan.as_any().is::()); + } + for child in plan.children() { + assert_metrics_present_in_plan(child); + } + } + + #[tokio::test] + async fn test_executed_distributed_plan_has_metrics() { + let ctx = make_test_distributed_ctx().await; + let plan = ctx + .sql("SELECT id, COUNT(*) as count FROM table1 WHERE id > 1 GROUP BY id ORDER BY id LIMIT 10") + .await + .unwrap() + .create_physical_plan() + .await + .unwrap(); + collect(plan.clone(), ctx.task_ctx()).await.unwrap(); + assert!(plan.as_any().is::()); + let rewritten_plan = + rewrite_distributed_plan_with_metrics(plan, DistributedMetricsFormat::Aggregated) + .unwrap(); + assert_metrics_present_in_plan(&rewritten_plan); + } + + #[test] + // An important feature of DF execution plans which we want to preserve is the ability + // to traverse a plan and collect metrics from specific nodes. To do this, the wrapper must + // allow access to the inner node. This test asserts that we support this. + fn test_wrapped_node_is_accessible() { + let example_node = Arc::new(EmptyExec::new(Arc::new(Schema::new(vec![Field::new( + "id", + DataType::Int32, + false, + )])))); + + let wrapped = MetricsWrapperExec::new(example_node, MetricsSet::new()); + assert_eq!(wrapped.name(), "EmptyExec"); + assert!(wrapped.as_any().is::()); + } + + #[test] + fn test_annotate_metrics_set_with_task_id_output_rows() { + // Create a MetricsSet with an OutputRows metric + let mut metrics_set = MetricsSet::new(); + let count = Count::new(); + count.add(1234); + let labels = vec![Label::new("operator", "scan")]; + metrics_set.push(Arc::new(Metric::new_with_labels( + MetricValue::OutputRows(count), + Some(0), + labels, + ))); + + let task_id = 42; + let annotated = annotate_metrics_set_with_task_id(metrics_set, task_id); + + // Verify we have one metric + assert_eq!(annotated.iter().count(), 1); + + let metric = annotated.iter().next().unwrap(); + + // Verify metric type is preserved (OutputRows) + match metric.value() { + MetricValue::OutputRows(count) => { + assert_eq!(count.value(), 1234); + } + other => panic!("Expected OutputRows, got {:?}", other.name()), + } + + // Verify partition is preserved + assert_eq!(metric.partition(), Some(0)); + + // Verify original labels are preserved and task_id label is added + let labels: Vec<_> = metric.labels().iter().collect(); + assert_eq!(labels.len(), 2); + assert_eq!(labels[0].name(), "operator"); + assert_eq!(labels[0].value(), "scan"); + assert_eq!(labels[1].name(), DISTRIBUTED_DATAFUSION_TASK_ID_LABEL); + assert_eq!(labels[1].value(), "42"); + } } diff --git a/src/networking/channel_resolver.rs b/src/networking/channel_resolver.rs new file mode 100644 index 00000000..5e44546b --- /dev/null +++ b/src/networking/channel_resolver.rs @@ -0,0 +1,298 @@ +use crate::DistributedConfig; +use crate::config_extension_ext::set_distributed_option_extension; +use arrow_flight::flight_service_client::FlightServiceClient; +use async_trait::async_trait; +use datafusion::common::{DataFusionError, config_datafusion_err, exec_datafusion_err}; +use datafusion::execution::TaskContext; +use datafusion::prelude::SessionConfig; +use futures::FutureExt; +use futures::future::Shared; +use std::sync::{Arc, LazyLock}; +use std::time::Duration; +use tonic::body::Body; +use tonic::codegen::BoxFuture; +use tonic::transport::Channel; +use tower::ServiceExt; +use url::Url; + +/// Allows users to customize the way Arrow Flight clients are created. A common use case is to +/// wrap the client with tower layers or schedule it in an IO-specific tokio runtime. +/// +/// There is a default implementation of this trait that should be enough for the most common +/// use-cases. +/// +/// # Implementation Notes +/// - This is called per Arrow Flight request, so implementors of this trait should make sure that +/// clients are reused across method calls instead of building a new Arrow Flight client +/// every time. +/// +/// - When implementing `get_flight_client_for_url`, it is recommended to use the +/// [`create_flight_client`] helper function to ensure clients are configured with +/// appropriate message size limits for internal communication. This helps avoid message +/// size errors when transferring large datasets. +#[async_trait] +pub trait ChannelResolver { + /// For a given URL, get an Arrow Flight client for communicating to it. + /// + /// *WARNING*: This method is called for every Arrow Flight gRPC request, so to not create + /// one client connection for each request, users are required to reuse generated clients. + /// It's recommended to rely on [DefaultChannelResolver] either by delegating method calls + /// to it or by copying the implementation. + /// + /// Consider using [`create_flight_client`] to create the client with appropriate + /// default message size limits. + async fn get_flight_client_for_url( + &self, + url: &Url, + ) -> Result, DataFusionError>; +} + +pub(crate) fn set_distributed_channel_resolver( + cfg: &mut SessionConfig, + channel_resolver: impl ChannelResolver + Send + Sync + 'static, +) { + let opts = cfg.options_mut(); + let channel_resolver = ChannelResolverExtension(Some(Arc::new(channel_resolver))); + if let Some(distributed_cfg) = opts.extensions.get_mut::() { + distributed_cfg.__private_channel_resolver = channel_resolver; + } else { + set_distributed_option_extension( + cfg, + DistributedConfig { + __private_channel_resolver: channel_resolver, + ..Default::default() + }, + ) + } +} + +// Unlike TaskContext, a DataFusion RuntimeEnv does not allow to introduce user-defined extensions. +// For the default implementation of the ChannelResolvers, we cannot inject one DefaultChannelResolver +// per TaskContext, as this holds reference to Tonic channels that must outlive a single TaskContext. +// +// The Tonic channels need to be established and reused under a whole RuntimeEnv scope, not a single +// TaskContext, which forces us to put the default implementation in a static global variable that +// stores and reuses tonic channels per RuntimeEnv's pointer address. +static DEFAULT_CHANNEL_RESOLVER_PER_RUNTIME: LazyLock< + moka::sync::Cache< + /* Arc pointer address */ usize, + /* ChannelResolver that reuses built channels */ Arc, + >, +> = LazyLock::new(|| moka::sync::Cache::builder().max_capacity(256).build()); + +pub fn get_distributed_channel_resolver( + task_ctx: &TaskContext, +) -> Arc { + let opts = task_ctx.session_config().options(); + if let Some(distributed_cfg) = opts.extensions.get::() { + if let Some(cr) = &distributed_cfg.__private_channel_resolver.0 { + return Arc::clone(cr); + } + } + let runtime_addr = Arc::as_ptr(&task_ctx.runtime_env()) as usize; + DEFAULT_CHANNEL_RESOLVER_PER_RUNTIME + .get_with(runtime_addr, || Arc::new(DefaultChannelResolver::default())) +} + +pub type BoxCloneSyncChannel = tower::util::BoxCloneSyncService< + http::Request, + http::Response, + tonic::transport::Error, +>; + +type ChannelCacheValue = Shared>>; + +#[derive(Clone, Default)] +pub(crate) struct ChannelResolverExtension(Option>); + +/// Default implementation of a [ChannelResolver] that connects to the workers given the URL once +/// and stores the connection instance in a TTI cache. +/// +/// Sane default over which other [ChannelResolver] can be built for better customization of the +/// [FlightServiceClient]s. +#[derive(Clone)] +pub struct DefaultChannelResolver { + cache: Arc>, +} + +impl Default for DefaultChannelResolver { + fn default() -> Self { + Self { + cache: Arc::new( + moka::sync::Cache::builder() + // Use an unrealistic max capacity, just in case there is a logic error on the + // user part that produces an unreasonable amount of URLs. + .max_capacity(64556) + // If a channel has not been used in 5 mins, delete it. + .time_to_idle(Duration::from_secs(5 * 60)) + .build(), + ), + } + } +} + +impl DefaultChannelResolver { + /// Gets the cached [BoxCloneSyncChannel] for the given URL, or builds a new one. + pub async fn get_channel(&self, url: &Url) -> Result { + let channel = self.cache.get_with_by_ref(url, move || { + let url = url.to_string(); + async move { + let endpoint = Channel::from_shared(url.clone()).map_err(|err| { + config_datafusion_err!( + "Invalid URL '{url}' returned by WorkerResolver implementation: {err}" + ) + })?; + let mut channel = endpoint.connect().await.map_err(|err| { + DataFusionError::Context( + format!("{err:?}"), + Box::new(exec_datafusion_err!( + "Error connecting to Distributed DataFusion worker on '{url}': {err}" + )), + ) + })?; + channel.ready().await.map_err(|err| { + DataFusionError::Context( + format!("{err:?}"), + Box::new(exec_datafusion_err!( + "Error waiting for Distributed DataFusion channel to be ready on '{url}': {err}" + )), + ) + })?; + Ok(BoxCloneSyncChannel::new(channel)) + } + .boxed() + .shared() + }); + + channel.await.map_err(|err| { + self.cache.invalidate(url); + DataFusionError::Shared(err) + }) + } +} + +#[async_trait] +impl ChannelResolver for DefaultChannelResolver { + async fn get_flight_client_for_url( + &self, + url: &Url, + ) -> Result, DataFusionError> { + self.get_channel(url).await.map(create_flight_client) + } +} + +#[async_trait] +impl ChannelResolver for Arc { + async fn get_flight_client_for_url( + &self, + url: &Url, + ) -> Result, DataFusionError> { + self.as_ref().get_flight_client_for_url(url).await + } +} + +/// Creates a [`FlightServiceClient`] with high default message size limits. +/// +/// This is a convenience function that wraps [`FlightServiceClient::new`] and configures +/// it with `max_decoding_message_size(usize::MAX)` and `max_encoding_message_size(usize::MAX)` +/// to avoid message size limitations for internal communication. +/// +/// Users implementing custom [`ChannelResolver`]s should use this function in their +/// `get_flight_client_for_url` implementations to ensure consistent behavior with built-in +/// implementations. +/// +/// # Example +/// +/// ```rust,ignore +/// use datafusion_distributed::{create_flight_client, BoxCloneSyncChannel, ChannelResolver}; +/// use arrow_flight::flight_service_client::FlightServiceClient; +/// use tonic::transport::Channel; +/// +/// #[async_trait] +/// impl ChannelResolver for MyResolver { +/// async fn get_flight_client_for_url( +/// &self, +/// url: &Url, +/// ) -> Result, DataFusionError> { +/// let channel = Channel::from_shared(url.to_string())?.connect().await?; +/// Ok(create_flight_client(BoxCloneSyncChannel::new(channel))) +/// } +/// } +/// ``` +pub fn create_flight_client( + channel: BoxCloneSyncChannel, +) -> FlightServiceClient { + FlightServiceClient::new(channel) + .max_decoding_message_size(usize::MAX) + .max_encoding_message_size(usize::MAX) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::Worker; + use datafusion::common::assert_contains; + use datafusion::common::runtime::SpawnedTask; + use std::error::Error; + use std::time::Instant; + use tokio::net::TcpListener; + use tonic::transport::Server; + + #[tokio::test] + async fn fails_establishing_connection() -> Result<(), Box> { + let (url, _guard) = spawn_http_localhost_worker().await?; + drop(_guard); + let channel_resolver = DefaultChannelResolver::default(); + let err = channel_resolver.get_channel(&url).await.unwrap_err(); + assert_contains!(err.to_string(), "tcp connect error"); + Ok(()) + } + + #[tokio::test] + async fn can_establish_connection() -> Result<(), Box> { + let (url, _guard) = spawn_http_localhost_worker().await?; + let channel_resolver = DefaultChannelResolver::default(); + channel_resolver.get_channel(&url).await?; + Ok(()) + } + + #[tokio::test] + async fn channel_resolve_is_cached() -> Result<(), Box> { + let (url, _guard) = spawn_http_localhost_worker().await?; + let channel_resolver = DefaultChannelResolver::default(); + + let start = Instant::now(); + channel_resolver.get_channel(&url).await?; + let first_call = start.elapsed(); + + let start = Instant::now(); + channel_resolver.get_channel(&url).await?; + let second_call = start.elapsed(); + + assert!(first_call > second_call); + Ok(()) + } + + async fn spawn_http_localhost_worker() -> Result<(Url, SpawnedTask<()>), Box> { + let listener = TcpListener::bind("127.0.0.1:0").await?; + + let port = listener + .local_addr() + .expect("Failed to get local address") + .port(); + + let task = SpawnedTask::spawn(async { + let worker = Worker::default(); + let incoming = tokio_stream::wrappers::TcpListenerStream::new(listener); + if let Err(err) = Server::builder() + .add_service(worker.into_flight_server()) + .serve_with_incoming(incoming) + .await + { + panic!("{err}") + } + }); + + Ok((Url::parse(&format!("http://127.0.0.1:{port}"))?, task)) + } +} diff --git a/src/networking/mod.rs b/src/networking/mod.rs new file mode 100644 index 00000000..3da47b0d --- /dev/null +++ b/src/networking/mod.rs @@ -0,0 +1,11 @@ +mod channel_resolver; +mod worker_resolver; + +pub use channel_resolver::{ + BoxCloneSyncChannel, ChannelResolver, DefaultChannelResolver, create_flight_client, + get_distributed_channel_resolver, +}; +pub(crate) use channel_resolver::{ChannelResolverExtension, set_distributed_channel_resolver}; + +pub use worker_resolver::{WorkerResolver, get_distributed_worker_resolver}; +pub(crate) use worker_resolver::{WorkerResolverExtension, set_distributed_worker_resolver}; diff --git a/src/networking/worker_resolver.rs b/src/networking/worker_resolver.rs new file mode 100644 index 00000000..75923da0 --- /dev/null +++ b/src/networking/worker_resolver.rs @@ -0,0 +1,73 @@ +use crate::DistributedConfig; +use crate::config_extension_ext::set_distributed_option_extension; +use datafusion::common::{DataFusionError, exec_err, not_impl_err}; +use datafusion::prelude::SessionConfig; +use std::sync::Arc; +use url::Url; + +/// Resolves a list of worker URLs in the cluster available for executing parts of the plan. +pub trait WorkerResolver { + /// Gets all available worker URLs in the cluster. Note how this method is not async, which + /// means that any async operation involved in discovering worker URLs must happen on a + /// background thread and be retrieved by this method synchronously. + /// + /// This method will be called in several places during distributed planning: + /// - During task count assignation for the different stages, for determining the size of + /// the cluster and limiting the amount of tasks per stage to Vec.length(). + /// - Right before execution, for lazily assigning worker URLs to the different tasks in the + /// plan. This is done as close to execution in order to have fresh worker URLs as updated + /// as possible. + fn get_urls(&self) -> Result, DataFusionError>; +} + +pub(crate) fn set_distributed_worker_resolver( + cfg: &mut SessionConfig, + worker_resolver: impl WorkerResolver + Send + Sync + 'static, +) { + let opts = cfg.options_mut(); + let worker_resolver = WorkerResolverExtension(Arc::new(worker_resolver)); + if let Some(distributed_cfg) = opts.extensions.get_mut::() { + distributed_cfg.__private_worker_resolver = worker_resolver; + } else { + set_distributed_option_extension( + cfg, + DistributedConfig { + __private_worker_resolver: worker_resolver, + ..Default::default() + }, + ) + } +} + +pub fn get_distributed_worker_resolver( + cfg: &SessionConfig, +) -> Result, DataFusionError> { + let opts = cfg.options(); + let Some(distributed_cfg) = opts.extensions.get::() else { + return exec_err!("ChannelResolver not present in the session config"); + }; + Ok(Arc::clone(&distributed_cfg.__private_worker_resolver.0)) +} + +#[derive(Clone)] +pub(crate) struct WorkerResolverExtension( + pub(crate) Arc, +); + +impl WorkerResolverExtension { + pub(crate) fn not_implemented() -> Self { + struct NotImplementedWorkerResolver; + impl WorkerResolver for NotImplementedWorkerResolver { + fn get_urls(&self) -> Result, DataFusionError> { + not_impl_err!("WorkerResolver::get_urls() not implemented") + } + } + Self(Arc::new(NotImplementedWorkerResolver)) + } +} + +impl WorkerResolver for Arc { + fn get_urls(&self) -> Result, DataFusionError> { + self.as_ref().get_urls() + } +} diff --git a/src/observability/README.md b/src/observability/README.md new file mode 100644 index 00000000..73dd5389 --- /dev/null +++ b/src/observability/README.md @@ -0,0 +1,12 @@ +# Distributed DataFusion Observability Proto Generation + +--- +This module provides gRPC-based observability services using Protocol Buffers. +The setup is inspired by [Apache DataFusion's proto crate](https://github.com/apache/datafusion/tree/main/datafusion/proto) structure but adapted +for tonic/gRPC services. + +In the root of the datafusion-distribued repo, run: + +```bash +./src/observability/gen/regen.sh +``` diff --git a/src/observability/gen/Cargo.toml b/src/observability/gen/Cargo.toml new file mode 100644 index 00000000..f03b4ffb --- /dev/null +++ b/src/observability/gen/Cargo.toml @@ -0,0 +1,12 @@ +[workspace] +# Empty workspace table to exclude this crate from parent workspace + +[package] +name = "observability-gen" +version = "0.1.0" +edition = "2024" +description = "Protobuf code generation for observability services" +publish = false + +[dependencies] +tonic-prost-build = "0.14.2" diff --git a/src/observability/gen/regen.sh b/src/observability/gen/regen.sh new file mode 100755 index 00000000..d5d48ef8 --- /dev/null +++ b/src/observability/gen/regen.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +set -e + +repo_root=$(git rev-parse --show-toplevel) +cd "$repo_root" && cargo run --manifest-path src/observability/gen/Cargo.toml diff --git a/src/observability/gen/src/main.rs b/src/observability/gen/src/main.rs new file mode 100644 index 00000000..93aef11c --- /dev/null +++ b/src/observability/gen/src/main.rs @@ -0,0 +1,27 @@ +use std::env; +use std::fs; + +fn main() -> Result<(), Box> { + let repo_root = env::current_dir()?; + + let proto_dir = repo_root.join("src/observability/proto"); + let proto_file = proto_dir.join("observability.proto"); + let out_dir = repo_root.join("src/observability/generated"); + + fs::create_dir_all(&out_dir)?; + + println!("Generating protobuf code..."); + println!("Proto dir: {proto_dir:?}"); + println!("Proto file: {proto_file:?}"); + println!("Output dir: {out_dir:?}"); + + tonic_prost_build::configure() + .build_server(true) + .build_client(true) + .out_dir(&out_dir) + .compile_protos(&[proto_file], &[proto_dir])?; + + println!("Successfully generated observability proto code"); + + Ok(()) +} diff --git a/src/observability/generated/mod.rs b/src/observability/generated/mod.rs new file mode 100644 index 00000000..e8cd9143 --- /dev/null +++ b/src/observability/generated/mod.rs @@ -0,0 +1,3 @@ +// This file is @generated by tonic-prost-build + +pub mod observability; diff --git a/src/observability/generated/observability.rs b/src/observability/generated/observability.rs new file mode 100644 index 00000000..e9103bad --- /dev/null +++ b/src/observability/generated/observability.rs @@ -0,0 +1,281 @@ +// This file is @generated by prost-build. +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct PingRequest {} +#[derive(Clone, Copy, PartialEq, Eq, Hash, ::prost::Message)] +pub struct PingResponse { + #[prost(uint32, tag = "1")] + pub value: u32, +} +/// Generated client implementations. +pub mod observability_service_client { + #![allow( + unused_variables, + dead_code, + missing_docs, + clippy::wildcard_imports, + clippy::let_unit_value + )] + use tonic::codegen::http::Uri; + use tonic::codegen::*; + #[derive(Debug, Clone)] + pub struct ObservabilityServiceClient { + inner: tonic::client::Grpc, + } + impl ObservabilityServiceClient { + /// Attempt to create a new client by connecting to a given endpoint. + pub async fn connect(dst: D) -> Result + where + D: TryInto, + D::Error: Into, + { + let conn = tonic::transport::Endpoint::new(dst)?.connect().await?; + Ok(Self::new(conn)) + } + } + impl ObservabilityServiceClient + where + T: tonic::client::GrpcService, + T::Error: Into, + T::ResponseBody: Body + std::marker::Send + 'static, + ::Error: Into + std::marker::Send, + { + pub fn new(inner: T) -> Self { + let inner = tonic::client::Grpc::new(inner); + Self { inner } + } + pub fn with_origin(inner: T, origin: Uri) -> Self { + let inner = tonic::client::Grpc::with_origin(inner, origin); + Self { inner } + } + pub fn with_interceptor( + inner: T, + interceptor: F, + ) -> ObservabilityServiceClient> + where + F: tonic::service::Interceptor, + T::ResponseBody: Default, + T: tonic::codegen::Service< + http::Request, + Response = http::Response< + >::ResponseBody, + >, + >, + >>::Error: + Into + std::marker::Send + std::marker::Sync, + { + ObservabilityServiceClient::new(InterceptedService::new(inner, interceptor)) + } + /// Compress requests with the given encoding. + /// + /// This requires the server to support it otherwise it might respond with an + /// error. + #[must_use] + pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.inner = self.inner.send_compressed(encoding); + self + } + /// Enable decompressing responses. + #[must_use] + pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.inner = self.inner.accept_compressed(encoding); + self + } + /// Limits the maximum size of a decoded message. + /// + /// Default: `4MB` + #[must_use] + pub fn max_decoding_message_size(mut self, limit: usize) -> Self { + self.inner = self.inner.max_decoding_message_size(limit); + self + } + /// Limits the maximum size of an encoded message. + /// + /// Default: `usize::MAX` + #[must_use] + pub fn max_encoding_message_size(mut self, limit: usize) -> Self { + self.inner = self.inner.max_encoding_message_size(limit); + self + } + pub async fn ping( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> { + self.inner.ready().await.map_err(|e| { + tonic::Status::unknown(format!("Service was not ready: {}", e.into())) + })?; + let codec = tonic_prost::ProstCodec::default(); + let path = + http::uri::PathAndQuery::from_static("/observability.ObservabilityService/Ping"); + let mut req = request.into_request(); + req.extensions_mut().insert(GrpcMethod::new( + "observability.ObservabilityService", + "Ping", + )); + self.inner.unary(req, path, codec).await + } + } +} +/// Generated server implementations. +pub mod observability_service_server { + #![allow( + unused_variables, + dead_code, + missing_docs, + clippy::wildcard_imports, + clippy::let_unit_value + )] + use tonic::codegen::*; + /// Generated trait containing gRPC methods that should be implemented for use with ObservabilityServiceServer. + #[async_trait] + pub trait ObservabilityService: std::marker::Send + std::marker::Sync + 'static { + async fn ping( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status>; + } + #[derive(Debug)] + pub struct ObservabilityServiceServer { + inner: Arc, + accept_compression_encodings: EnabledCompressionEncodings, + send_compression_encodings: EnabledCompressionEncodings, + max_decoding_message_size: Option, + max_encoding_message_size: Option, + } + impl ObservabilityServiceServer { + pub fn new(inner: T) -> Self { + Self::from_arc(Arc::new(inner)) + } + pub fn from_arc(inner: Arc) -> Self { + Self { + inner, + accept_compression_encodings: Default::default(), + send_compression_encodings: Default::default(), + max_decoding_message_size: None, + max_encoding_message_size: None, + } + } + pub fn with_interceptor(inner: T, interceptor: F) -> InterceptedService + where + F: tonic::service::Interceptor, + { + InterceptedService::new(Self::new(inner), interceptor) + } + /// Enable decompressing requests with the given encoding. + #[must_use] + pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.accept_compression_encodings.enable(encoding); + self + } + /// Compress responses with the given encoding, if the client supports it. + #[must_use] + pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.send_compression_encodings.enable(encoding); + self + } + /// Limits the maximum size of a decoded message. + /// + /// Default: `4MB` + #[must_use] + pub fn max_decoding_message_size(mut self, limit: usize) -> Self { + self.max_decoding_message_size = Some(limit); + self + } + /// Limits the maximum size of an encoded message. + /// + /// Default: `usize::MAX` + #[must_use] + pub fn max_encoding_message_size(mut self, limit: usize) -> Self { + self.max_encoding_message_size = Some(limit); + self + } + } + impl tonic::codegen::Service> for ObservabilityServiceServer + where + T: ObservabilityService, + B: Body + std::marker::Send + 'static, + B::Error: Into + std::marker::Send + 'static, + { + type Response = http::Response; + type Error = std::convert::Infallible; + type Future = BoxFuture; + fn poll_ready( + &mut self, + _cx: &mut Context<'_>, + ) -> Poll> { + Poll::Ready(Ok(())) + } + fn call(&mut self, req: http::Request) -> Self::Future { + match req.uri().path() { + "/observability.ObservabilityService/Ping" => { + #[allow(non_camel_case_types)] + struct PingSvc(pub Arc); + impl tonic::server::UnaryService for PingSvc { + type Response = super::PingResponse; + type Future = BoxFuture, tonic::Status>; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::ping(&inner, request).await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = PingSvc(inner); + let codec = tonic_prost::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + _ => Box::pin(async move { + let mut response = http::Response::new(tonic::body::Body::default()); + let headers = response.headers_mut(); + headers.insert( + tonic::Status::GRPC_STATUS, + (tonic::Code::Unimplemented as i32).into(), + ); + headers.insert( + http::header::CONTENT_TYPE, + tonic::metadata::GRPC_CONTENT_TYPE, + ); + Ok(response) + }), + } + } + } + impl Clone for ObservabilityServiceServer { + fn clone(&self) -> Self { + let inner = self.inner.clone(); + Self { + inner, + accept_compression_encodings: self.accept_compression_encodings, + send_compression_encodings: self.send_compression_encodings, + max_decoding_message_size: self.max_decoding_message_size, + max_encoding_message_size: self.max_encoding_message_size, + } + } + } + /// Generated gRPC service name + pub const SERVICE_NAME: &str = "observability.ObservabilityService"; + impl tonic::server::NamedService for ObservabilityServiceServer { + const NAME: &'static str = SERVICE_NAME; + } +} diff --git a/src/observability/mod.rs b/src/observability/mod.rs new file mode 100644 index 00000000..8c152d58 --- /dev/null +++ b/src/observability/mod.rs @@ -0,0 +1,9 @@ +mod generated; +mod service; + +pub use generated::observability::observability_service_client::ObservabilityServiceClient; +pub use generated::observability::observability_service_server::{ + ObservabilityService, ObservabilityServiceServer, +}; +pub use generated::observability::{PingRequest, PingResponse}; +pub use service::ObservabilityServiceImpl; diff --git a/src/observability/proto/observability.proto b/src/observability/proto/observability.proto new file mode 100644 index 00000000..c2352170 --- /dev/null +++ b/src/observability/proto/observability.proto @@ -0,0 +1,12 @@ +syntax = "proto3"; +package observability; + +service ObservabilityService { + rpc Ping (PingRequest) returns (PingResponse); +} + +message PingRequest {} + +message PingResponse { + uint32 value = 1; +} diff --git a/src/observability/service.rs b/src/observability/service.rs new file mode 100644 index 00000000..32d32e39 --- /dev/null +++ b/src/observability/service.rs @@ -0,0 +1,26 @@ +use crate::flight_service::TaskData; +use crate::protobuf::StageKey; +use moka::future::Cache; +use std::sync::Arc; +use tokio::sync::OnceCell; +use tonic::{Request, Response, Status}; + +use super::{ObservabilityService, PingRequest, PingResponse}; + +#[allow(dead_code)] // TEMP: will be used in future implementations. +pub struct ObservabilityServiceImpl { + task_data_entries: Arc>>>, +} + +impl ObservabilityServiceImpl { + pub fn new(task_data_entries: Arc>>>) -> Self { + Self { task_data_entries } + } +} + +#[tonic::async_trait] +impl ObservabilityService for ObservabilityServiceImpl { + async fn ping(&self, _request: Request) -> Result, Status> { + Ok(tonic::Response::new(PingResponse { value: 1 })) + } +} diff --git a/src/passthrough_headers.rs b/src/passthrough_headers.rs new file mode 100644 index 00000000..d269e685 --- /dev/null +++ b/src/passthrough_headers.rs @@ -0,0 +1,127 @@ +use crate::config_extension_ext::FLIGHT_METADATA_CONFIG_PREFIX; +use datafusion::common::DataFusionError; +use datafusion::prelude::SessionConfig; +use http::HeaderMap; +use std::sync::Arc; + +/// Stores arbitrary HTTP headers that should be forwarded unchanged to worker nodes. +#[derive(Clone, Default)] +pub(crate) struct PassthroughHeaders(HeaderMap); + +/// Sets passthrough headers on a SessionConfig as an extension. +/// +/// Returns an error if any header name starts with the reserved prefix +/// `x-datafusion-distributed-config-`, which is used internally for config extensions. +pub(crate) fn set_passthrough_headers( + cfg: &mut SessionConfig, + headers: HeaderMap, +) -> Result<(), DataFusionError> { + // Validate that no headers use the reserved internal prefix + for name in headers.keys() { + if name.as_str().starts_with(FLIGHT_METADATA_CONFIG_PREFIX) { + return Err(DataFusionError::Configuration(format!( + "Passthrough header '{name}' uses reserved prefix '{FLIGHT_METADATA_CONFIG_PREFIX}'. \ + This prefix is reserved for internal config extension propagation." + ))); + } + } + + cfg.set_extension(Arc::new(PassthroughHeaders(headers))); + Ok(()) +} + +/// Gets passthrough headers from a SessionConfig extension. +/// Returns an empty HeaderMap if none are set. +pub(crate) fn get_passthrough_headers(cfg: &SessionConfig) -> HeaderMap { + cfg.get_extension::() + .map(|h| h.0.clone()) + .unwrap_or_default() +} + +#[cfg(test)] +mod tests { + use super::*; + use http::{HeaderName, HeaderValue}; + + #[test] + fn test_set_and_get_passthrough_headers() { + let mut config = SessionConfig::new(); + let mut headers = HeaderMap::new(); + headers.insert( + HeaderName::from_static("x-custom-header"), + HeaderValue::from_static("test-value"), + ); + headers.insert( + HeaderName::from_static("x-another-header"), + HeaderValue::from_static("another-value"), + ); + + set_passthrough_headers(&mut config, headers.clone()).unwrap(); + + let retrieved = get_passthrough_headers(&config); + assert_eq!(retrieved.len(), 2); + assert_eq!(retrieved.get("x-custom-header").unwrap(), "test-value"); + assert_eq!(retrieved.get("x-another-header").unwrap(), "another-value"); + } + + #[test] + fn test_get_passthrough_headers_empty() { + let config = SessionConfig::new(); + let retrieved = get_passthrough_headers(&config); + assert!(retrieved.is_empty()); + } + + #[test] + fn test_overwrite_passthrough_headers() { + let mut config = SessionConfig::new(); + + let mut headers1 = HeaderMap::new(); + headers1.insert( + HeaderName::from_static("x-first"), + HeaderValue::from_static("first-value"), + ); + set_passthrough_headers(&mut config, headers1).unwrap(); + + let mut headers2 = HeaderMap::new(); + headers2.insert( + HeaderName::from_static("x-second"), + HeaderValue::from_static("second-value"), + ); + set_passthrough_headers(&mut config, headers2).unwrap(); + + let retrieved = get_passthrough_headers(&config); + assert_eq!(retrieved.len(), 1); + assert!(retrieved.get("x-first").is_none()); + assert_eq!(retrieved.get("x-second").unwrap(), "second-value"); + } + + #[test] + fn test_rejects_reserved_prefix() { + let mut config = SessionConfig::new(); + let mut headers = HeaderMap::new(); + headers.insert( + HeaderName::from_static("x-datafusion-distributed-config-custom.foo"), + HeaderValue::from_static("should-fail"), + ); + + let result = set_passthrough_headers(&mut config, headers); + assert!(result.is_err()); + + let err = result.unwrap_err(); + assert!(err.to_string().contains("reserved prefix")); + } + + #[test] + fn test_accepts_similar_but_different_prefix() { + let mut config = SessionConfig::new(); + let mut headers = HeaderMap::new(); + // This is similar but doesn't match the exact prefix + headers.insert( + HeaderName::from_static("x-datafusion-distributed-other"), + HeaderValue::from_static("should-succeed"), + ); + + let result = set_passthrough_headers(&mut config, headers); + assert!(result.is_ok()); + } +} diff --git a/src/protobuf/app_metadata.rs b/src/protobuf/app_metadata.rs index b0de54d1..07d793dd 100644 --- a/src/protobuf/app_metadata.rs +++ b/src/protobuf/app_metadata.rs @@ -1,5 +1,6 @@ use crate::metrics::proto::MetricsSetProto; use crate::protobuf::distributed_codec::StageKey; +use std::time::{SystemTime, UNIX_EPOCH}; /// A collection of metrics for a set of tasks in an ExecutionPlan. each /// entry should have a distinct [StageKey]. @@ -26,14 +27,40 @@ pub struct TaskMetrics { // FlightAppMetadata represents all types of app_metadata which we use in the distributed execution. #[derive(Clone, PartialEq, ::prost::Message)] pub struct FlightAppMetadata { + #[prost(uint64, tag = "1")] + pub partition: u64, + // Unix timestamp in nanoseconds at which this message was created. + #[prost(uint64, tag = "2")] + pub created_timestamp_unix_nanos: u64, // content should always be Some, but it is optional due to protobuf rules. - #[prost(oneof = "AppMetadata", tags = "1")] + #[prost(oneof = "AppMetadata", tags = "10")] pub content: Option, } +impl FlightAppMetadata { + pub fn new(partition: u64) -> Self { + Self { + partition, + created_timestamp_unix_nanos: current_unix_timestamp_nanos(), + content: None, + } + } + + pub fn set_content(&mut self, content: AppMetadata) { + self.content = Some(content); + } +} + +fn current_unix_timestamp_nanos() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|duration| duration.as_nanos() as u64) + .unwrap_or(0) +} + #[derive(Clone, PartialEq, ::prost::Oneof)] pub enum AppMetadata { - #[prost(message, tag = "1")] + #[prost(message, tag = "10")] MetricsCollection(MetricsCollection), // Note: For every additional enum variant, ensure to add tags to [FlightAppMetadata]. ex. `#[prost(oneof = "AppMetadata", tags = "1,2,3")]` etc. // If you don't the proto will compile but you may encounter errors during serialization/deserialization. diff --git a/src/protobuf/distributed_codec.rs b/src/protobuf/distributed_codec.rs index ec51d8e7..588cb682 100644 --- a/src/protobuf/distributed_codec.rs +++ b/src/protobuf/distributed_codec.rs @@ -1,23 +1,30 @@ use super::get_distributed_user_codecs; -use crate::distributed_physical_optimizer_rule::NetworkBoundary; -use crate::execution_plans::{NetworkCoalesceExec, NetworkCoalesceReady, NetworkShuffleReadyExec}; +use crate::execution_plans::{ + BroadcastExec, ChildrenIsolatorUnionExec, NetworkBroadcastExec, NetworkCoalesceExec, +}; +use crate::flight_service::WorkerConnectionPool; use crate::stage::{ExecutionTask, MaybeEncodedPlan, Stage}; +use crate::{DistributedTaskContext, NetworkBoundary}; use crate::{NetworkShuffleExec, PartitionIsolatorExec}; use bytes::Bytes; use datafusion::arrow::datatypes::Schema; use datafusion::arrow::datatypes::SchemaRef; use datafusion::common::internal_datafusion_err; use datafusion::error::DataFusionError; -use datafusion::execution::FunctionRegistry; +use datafusion::execution::TaskContext; use datafusion::physical_expr::EquivalenceProperties; use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType}; +use datafusion::physical_plan::union::UnionExec; use datafusion::physical_plan::{ExecutionPlan, Partitioning, PlanProperties}; -use datafusion::prelude::{SessionConfig, SessionContext}; +use datafusion::prelude::SessionConfig; use datafusion_proto::physical_plan::from_proto::parse_protobuf_partitioning; use datafusion_proto::physical_plan::to_proto::serialize_partitioning; -use datafusion_proto::physical_plan::{ComposedPhysicalExtensionCodec, PhysicalExtensionCodec}; +use datafusion_proto::physical_plan::{ + ComposedPhysicalExtensionCodec, DefaultPhysicalProtoConverter, PhysicalExtensionCodec, +}; use datafusion_proto::protobuf; use datafusion_proto::protobuf::proto_error; +use itertools::Itertools; use prost::Message; use std::sync::Arc; use url::Url; @@ -40,7 +47,7 @@ impl PhysicalExtensionCodec for DistributedCodec { &self, buf: &[u8], inputs: &[Arc], - _registry: &dyn FunctionRegistry, + ctx: &TaskContext, ) -> datafusion::common::Result> { let DistributedExecProto { node: Some(distributed_exec_node), @@ -51,11 +58,6 @@ impl PhysicalExtensionCodec for DistributedCodec { )); }; - // TODO: The PhysicalExtensionCodec trait doesn't provide access to session state, - // so we create a new SessionContext which loses any custom UDFs, UDAFs, and other - // user configurations. This is a limitation of the current trait design. - let ctx = SessionContext::new(); - fn parse_stage_proto( proto: Option, inputs: &[Arc], @@ -85,7 +87,7 @@ impl PhysicalExtensionCodec for DistributedCodec { Ok(Stage { query_id: uuid::Uuid::from_slice(proto.query_id.as_ref()) - .map_err(|_| proto_error("Invalid query_id in ExecutionStageProto"))?, + .map_err(|_| proto_error("Invalid query_id in StageProto"))?, num: proto.num as usize, plan, tasks: decode_tasks(proto.tasks)?, @@ -105,9 +107,10 @@ impl PhysicalExtensionCodec for DistributedCodec { let partitioning = parse_protobuf_partitioning( partitioning.as_ref(), - &ctx, + ctx, &schema, &DistributedCodec {}, + &DefaultPhysicalProtoConverter {}, )? .ok_or(proto_error("NetworkShuffleExec is missing partitioning"))?; @@ -129,9 +132,10 @@ impl PhysicalExtensionCodec for DistributedCodec { let partitioning = parse_protobuf_partitioning( partitioning.as_ref(), - &ctx, + ctx, &schema, &DistributedCodec {}, + &DefaultPhysicalProtoConverter {}, )? .ok_or(proto_error("NetworkCoalesceExec is missing partitioning"))?; @@ -151,10 +155,87 @@ impl PhysicalExtensionCodec for DistributedCodec { let child = inputs.first().unwrap(); - Ok(Arc::new(PartitionIsolatorExec::new_ready( + Ok(Arc::new(PartitionIsolatorExec::new( child.clone(), n_tasks as usize, - )?)) + ))) + } + DistributedExecNode::NetworkBroadcast(NetworkBroadcastExecProto { + schema, + partitioning, + input_stage, + }) => { + let schema: Schema = schema + .as_ref() + .map(|s| s.try_into()) + .ok_or(proto_error("NetworkBroadcastExec is missing schema"))??; + + let partitioning = parse_protobuf_partitioning( + partitioning.as_ref(), + ctx, + &schema, + &DistributedCodec {}, + &DefaultPhysicalProtoConverter {}, + )? + .ok_or(proto_error("NetworkBroadcastExec is missing partitioning"))?; + + Ok(Arc::new(new_network_broadcast_exec( + partitioning, + Arc::new(schema), + parse_stage_proto(input_stage, inputs)?, + ))) + } + DistributedExecNode::Broadcast(BroadcastExecProto { + consumer_task_count, + }) => { + if inputs.len() != 1 { + return Err(proto_error(format!( + "BroadcastExec expects exactly one child, got {}", + inputs.len() + ))); + } + + let child = inputs.first().unwrap(); + Ok(Arc::new(BroadcastExec::new( + child.clone(), + consumer_task_count as usize, + ))) + } + DistributedExecNode::ChildrenIsolatorUnion(ChildrenIsolatorUnionExecProto { + partition_count, + task_idx_map, + }) => { + // Building a UnionExec just to get the properties out of it is not the most + // efficient thing to do. However, it's the easiest way of getting the properties + // for the ChildrenIsolatorUnionExec without copy-pasting in this project + // all the machinery that builds them for UnionExec. + let mut properties = UnionExec::try_new(inputs.to_vec())?.properties().clone(); + properties.partitioning = + Partitioning::UnknownPartitioning(partition_count as usize); + + Ok(Arc::new(ChildrenIsolatorUnionExec { + properties, + metrics: Default::default(), + children: inputs.to_vec(), + task_idx_map: task_idx_map + .iter() + .map(|entry| { + entry + .child_ctx + .iter() + .map(|child_ctx| { + ( + child_ctx.child_idx as usize, + DistributedTaskContext { + task_index: child_ctx.task_idx as usize, + task_count: child_ctx.task_count as usize, + }, + ) + }) + .collect_vec() + }) + .collect(), + })) } } } @@ -164,10 +245,7 @@ impl PhysicalExtensionCodec for DistributedCodec { node: Arc, buf: &mut Vec, ) -> datafusion::common::Result<()> { - fn encode_stage_proto(stage: Option<&Stage>) -> Result { - let stage = stage.ok_or(proto_error( - "Cannot encode a NetworkBoundary that has no stage assinged", - ))?; + fn encode_stage_proto(stage: &Stage) -> Result { Ok(StageProto { query_id: Bytes::from(stage.query_id.as_bytes().to_vec()), num: stage.num as u64, @@ -185,6 +263,7 @@ impl PhysicalExtensionCodec for DistributedCodec { partitioning: Some(serialize_partitioning( node.properties().output_partitioning(), &DistributedCodec {}, + &DefaultPhysicalProtoConverter {}, )?), input_stage: Some(encode_stage_proto(node.input_stage())?), }; @@ -200,6 +279,7 @@ impl PhysicalExtensionCodec for DistributedCodec { partitioning: Some(serialize_partitioning( node.properties().output_partitioning(), &DistributedCodec {}, + &DefaultPhysicalProtoConverter {}, )?), input_stage: Some(encode_stage_proto(node.input_stage())?), }; @@ -210,19 +290,64 @@ impl PhysicalExtensionCodec for DistributedCodec { wrapper.encode(buf).map_err(|e| proto_error(format!("{e}"))) } else if let Some(node) = node.as_any().downcast_ref::() { - let PartitionIsolatorExec::Ready(ready_node) = node else { - return Err(proto_error( - "deserialized an PartitionIsolatorExec that is not ready", - )); - }; let inner = PartitionIsolatorExecProto { - n_tasks: ready_node.n_tasks as u64, + n_tasks: node.n_tasks as u64, }; let wrapper = DistributedExecProto { node: Some(DistributedExecNode::PartitionIsolator(inner)), }; + wrapper.encode(buf).map_err(|e| proto_error(format!("{e}"))) + } else if let Some(node) = node.as_any().downcast_ref::() { + let inner = NetworkBroadcastExecProto { + schema: Some(node.schema().try_into()?), + partitioning: Some(serialize_partitioning( + node.properties().output_partitioning(), + &DistributedCodec {}, + &DefaultPhysicalProtoConverter {}, + )?), + input_stage: Some(encode_stage_proto(node.input_stage())?), + }; + + let wrapper = DistributedExecProto { + node: Some(DistributedExecNode::NetworkBroadcast(inner)), + }; + + wrapper.encode(buf).map_err(|e| proto_error(format!("{e}"))) + } else if let Some(node) = node.as_any().downcast_ref::() { + let inner = BroadcastExecProto { + consumer_task_count: node.consumer_task_count() as u64, + }; + + let wrapper = DistributedExecProto { + node: Some(DistributedExecNode::Broadcast(inner)), + }; + + wrapper.encode(buf).map_err(|e| proto_error(format!("{e}"))) + } else if let Some(node) = node.as_any().downcast_ref::() { + let inner = ChildrenIsolatorUnionExecProto { + partition_count: node.properties().output_partitioning().partition_count() as u64, + task_idx_map: node + .task_idx_map + .iter() + .map(|v| TaskIdxMapEntryProto { + child_ctx: v + .iter() + .map(|(child_idx, task_ctx)| ChildIdxWithTaskContextProto { + child_idx: *child_idx as u64, + task_idx: task_ctx.task_index as u64, + task_count: task_ctx.task_count as u64, + }) + .collect_vec(), + }) + .collect_vec(), + }; + + let wrapper = DistributedExecProto { + node: Some(DistributedExecNode::ChildrenIsolatorUnion(inner)), + }; + wrapper.encode(buf).map_err(|e| proto_error(format!("{e}"))) } else { Err(proto_error(format!("Unexpected plan {}", node.name()))) @@ -230,7 +355,7 @@ impl PhysicalExtensionCodec for DistributedCodec { } } -/// A key that uniquely identifies a stage in a query +/// A key that uniquely identifies a stage in a query. #[derive(Clone, Hash, Eq, PartialEq, ::prost::Message)] pub struct StageKey { /// Our query id @@ -244,6 +369,17 @@ pub struct StageKey { pub task_number: u64, } +impl StageKey { + /// Creates a new `StageKey`. + pub fn new(query_id: Bytes, stage_id: u64, task_number: u64) -> StageKey { + Self { + query_id, + stage_id, + task_number, + } + } +} + #[derive(Clone, PartialEq, ::prost::Message)] pub struct StageProto { /// Our query id @@ -271,7 +407,7 @@ pub struct ExecutionTaskProto { #[derive(Clone, PartialEq, ::prost::Message)] pub struct DistributedExecProto { - #[prost(oneof = "DistributedExecNode", tags = "1, 2, 3, 4, 5")] + #[prost(oneof = "DistributedExecNode", tags = "1, 2, 3, 4, 5, 6")] pub node: Option, } @@ -283,6 +419,12 @@ pub enum DistributedExecNode { NetworkCoalesceTasks(NetworkCoalesceExecProto), #[prost(message, tag = "3")] PartitionIsolator(PartitionIsolatorExecProto), + #[prost(message, tag = "4")] + ChildrenIsolatorUnion(ChildrenIsolatorUnionExecProto), + #[prost(message, tag = "5")] + NetworkBroadcast(NetworkBroadcastExecProto), + #[prost(message, tag = "6")] + Broadcast(BroadcastExecProto), } #[derive(Clone, PartialEq, ::prost::Message)] @@ -304,21 +446,46 @@ pub struct NetworkShuffleExecProto { input_stage: Option, } +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ChildrenIsolatorUnionExecProto { + #[prost(uint64, tag = "1")] + partition_count: u64, + #[prost(message, repeated, tag = "2")] + task_idx_map: Vec, +} + +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct TaskIdxMapEntryProto { + #[prost(message, repeated, tag = "1")] + child_ctx: Vec, +} + +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ChildIdxWithTaskContextProto { + #[prost(uint64, tag = "1")] + child_idx: u64, + #[prost(uint64, tag = "2")] + task_idx: u64, + #[prost(uint64, tag = "3")] + task_count: u64, +} + fn new_network_hash_shuffle_exec( partitioning: Partitioning, schema: SchemaRef, input_stage: Stage, ) -> NetworkShuffleExec { - NetworkShuffleExec::Ready(NetworkShuffleReadyExec { + NetworkShuffleExec { properties: PlanProperties::new( EquivalenceProperties::new(schema), partitioning, EmissionType::Incremental, Boundedness::Bounded, ), + worker_connections: WorkerConnectionPool::new(input_stage.tasks.len()), input_stage, metrics_collection: Default::default(), - }) + } } /// Protobuf representation of the [NetworkShuffleExec] physical node. It serves as @@ -339,16 +506,51 @@ fn new_network_coalesce_tasks_exec( schema: SchemaRef, input_stage: Stage, ) -> NetworkCoalesceExec { - NetworkCoalesceExec::Ready(NetworkCoalesceReady { + NetworkCoalesceExec { properties: PlanProperties::new( EquivalenceProperties::new(schema), partitioning, EmissionType::Incremental, Boundedness::Bounded, ), + worker_connections: WorkerConnectionPool::new(input_stage.tasks.len()), input_stage, metrics_collection: Default::default(), - }) + } +} + +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct NetworkBroadcastExecProto { + #[prost(message, optional, tag = "1")] + schema: Option, + #[prost(message, optional, tag = "2")] + partitioning: Option, + #[prost(message, optional, tag = "3")] + input_stage: Option, +} + +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct BroadcastExecProto { + #[prost(uint64, tag = "1")] + pub consumer_task_count: u64, +} + +fn new_network_broadcast_exec( + partitioning: Partitioning, + schema: SchemaRef, + input_stage: Stage, +) -> NetworkBroadcastExec { + NetworkBroadcastExec { + properties: PlanProperties::new( + EquivalenceProperties::new(schema), + partitioning, + EmissionType::Incremental, + Boundedness::Bounded, + ), + worker_connections: WorkerConnectionPool::new(input_stage.tasks.len()), + input_stage, + metrics_collection: Default::default(), + } } fn encode_tasks(tasks: &[ExecutionTask]) -> Vec { @@ -383,11 +585,12 @@ mod tests { use datafusion::physical_expr::LexOrdering; use datafusion::physical_plan::empty::EmptyExec; use datafusion::{ - execution::registry::MemoryFunctionRegistry, physical_expr::{Partitioning, PhysicalSortExpr, expressions::Column, expressions::col}, physical_plan::{ExecutionPlan, displayable, sorts::sort::SortExec, union::UnionExec}, }; + use datafusion::prelude::SessionContext; + fn empty_exec() -> Arc { Arc::new(EmptyExec::new(SchemaRef::new(Schema::empty()))) } @@ -409,10 +612,14 @@ mod tests { displayable(plan.as_ref()).indent(true).to_string() } + fn create_context() -> Arc { + SessionContext::new().task_ctx() + } + #[test] fn test_roundtrip_single_flight() -> datafusion::common::Result<()> { let codec = DistributedCodec; - let registry = MemoryFunctionRegistry::new(); + let ctx = create_context(); let schema = schema_i32("a"); let part = Partitioning::Hash(vec![Arc::new(Column::new("a", 0))], 4); @@ -422,7 +629,7 @@ mod tests { let mut buf = Vec::new(); codec.try_encode(plan.clone(), &mut buf)?; - let decoded = codec.try_decode(&buf, &[empty_exec()], ®istry)?; + let decoded = codec.try_decode(&buf, &[empty_exec()], &ctx)?; assert_eq!(repr(&plan), repr(&decoded)); Ok(()) @@ -431,7 +638,7 @@ mod tests { #[test] fn test_roundtrip_isolator_flight() -> datafusion::common::Result<()> { let codec = DistributedCodec; - let registry = MemoryFunctionRegistry::new(); + let ctx = create_context(); let schema = schema_i32("b"); let flight = Arc::new(new_network_hash_shuffle_exec( @@ -440,13 +647,12 @@ mod tests { dummy_stage(), )); - let plan: Arc = - Arc::new(PartitionIsolatorExec::new_ready(flight.clone(), 1)?); + let plan: Arc = Arc::new(PartitionIsolatorExec::new(flight.clone(), 1)); let mut buf = Vec::new(); codec.try_encode(plan.clone(), &mut buf)?; - let decoded = codec.try_decode(&buf, &[flight], ®istry)?; + let decoded = codec.try_decode(&buf, &[flight], &ctx)?; assert_eq!(repr(&plan), repr(&decoded)); Ok(()) @@ -455,7 +661,7 @@ mod tests { #[test] fn test_roundtrip_isolator_union() -> datafusion::common::Result<()> { let codec = DistributedCodec; - let registry = MemoryFunctionRegistry::new(); + let ctx = create_context(); let schema = schema_i32("c"); let left = Arc::new(new_network_hash_shuffle_exec( @@ -469,14 +675,13 @@ mod tests { dummy_stage(), )); - let union = Arc::new(UnionExec::new(vec![left.clone(), right.clone()])); - let plan: Arc = - Arc::new(PartitionIsolatorExec::new_ready(union.clone(), 1)?); + let union = UnionExec::try_new(vec![left.clone(), right.clone()])?; + let plan: Arc = Arc::new(PartitionIsolatorExec::new(union.clone(), 1)); let mut buf = Vec::new(); codec.try_encode(plan.clone(), &mut buf)?; - let decoded = codec.try_decode(&buf, &[union], ®istry)?; + let decoded = codec.try_decode(&buf, &[union], &ctx)?; assert_eq!(repr(&plan), repr(&decoded)); Ok(()) @@ -485,7 +690,7 @@ mod tests { #[test] fn test_roundtrip_isolator_sort_flight() -> datafusion::common::Result<()> { let codec = DistributedCodec; - let registry = MemoryFunctionRegistry::new(); + let ctx = create_context(); let schema = schema_i32("d"); let flight = Arc::new(new_network_hash_shuffle_exec( @@ -503,13 +708,118 @@ mod tests { flight.clone(), )); + let plan: Arc = Arc::new(PartitionIsolatorExec::new(sort.clone(), 1)); + + let mut buf = Vec::new(); + codec.try_encode(plan.clone(), &mut buf)?; + + let decoded = codec.try_decode(&buf, &[sort], &ctx)?; + assert_eq!(repr(&plan), repr(&decoded)); + + Ok(()) + } + + #[test] + fn test_roundtrip_single_flight_coalesce() -> datafusion::common::Result<()> { + let codec = DistributedCodec; + let ctx = create_context(); + + let schema = schema_i32("e"); + let plan: Arc = Arc::new(new_network_coalesce_tasks_exec( + Partitioning::RoundRobinBatch(3), + schema, + dummy_stage(), + )); + + let mut buf = Vec::new(); + codec.try_encode(plan.clone(), &mut buf)?; + + let decoded = codec.try_decode(&buf, &[empty_exec()], &ctx)?; + assert_eq!(repr(&plan), repr(&decoded)); + + Ok(()) + } + + #[test] + fn test_roundtrip_isolator_flight_coalesce() -> datafusion::common::Result<()> { + let codec = DistributedCodec; + let ctx = create_context(); + + let schema = schema_i32("f"); + let flight = Arc::new(new_network_coalesce_tasks_exec( + Partitioning::UnknownPartitioning(1), + schema, + dummy_stage(), + )); + + let plan: Arc = Arc::new(PartitionIsolatorExec::new(flight.clone(), 1)); + + let mut buf = Vec::new(); + codec.try_encode(plan.clone(), &mut buf)?; + + let decoded = codec.try_decode(&buf, &[flight], &ctx)?; + assert_eq!(repr(&plan), repr(&decoded)); + + Ok(()) + } + + #[test] + fn test_roundtrip_isolator_union_coalesce() -> datafusion::common::Result<()> { + let codec = DistributedCodec; + let ctx = create_context(); + + let schema = schema_i32("g"); + let left = Arc::new(new_network_coalesce_tasks_exec( + Partitioning::RoundRobinBatch(2), + schema.clone(), + dummy_stage(), + )); + let right = Arc::new(new_network_coalesce_tasks_exec( + Partitioning::RoundRobinBatch(2), + schema.clone(), + dummy_stage(), + )); + + let union = UnionExec::try_new(vec![left.clone(), right.clone()])?; + let plan: Arc = Arc::new(PartitionIsolatorExec::new(union.clone(), 3)); + + let mut buf = Vec::new(); + codec.try_encode(plan.clone(), &mut buf)?; + + let decoded = codec.try_decode(&buf, &[union], &ctx)?; + assert_eq!(repr(&plan), repr(&decoded)); + + Ok(()) + } + + #[test] + fn test_roundtrip_children_isolator_union() -> datafusion::common::Result<()> { + let codec = DistributedCodec; + let ctx = create_context(); + + let schema = schema_i32("h"); + let left = Arc::new(new_network_hash_shuffle_exec( + Partitioning::RoundRobinBatch(2), + schema.clone(), + dummy_stage(), + )) as Arc; + let right = Arc::new(new_network_hash_shuffle_exec( + Partitioning::RoundRobinBatch(2), + schema.clone(), + dummy_stage(), + )) as Arc; + let plan: Arc = - Arc::new(PartitionIsolatorExec::new_ready(sort.clone(), 1)?); + Arc::new(ChildrenIsolatorUnionExec::from_children_and_task_counts( + vec![left.clone(), right.clone()], + vec![2, 2], + 4, + )?); let mut buf = Vec::new(); codec.try_encode(plan.clone(), &mut buf)?; - let decoded = codec.try_decode(&buf, &[sort], ®istry)?; + let decoded = codec.try_decode(&buf, &[left, right], &ctx)?; assert_eq!(repr(&plan), repr(&decoded)); Ok(()) diff --git a/src/protobuf/errors/arrow_error.rs b/src/protobuf/errors/arrow_error.rs index 01f2a31e..a8630bba 100644 --- a/src/protobuf/errors/arrow_error.rs +++ b/src/protobuf/errors/arrow_error.rs @@ -53,6 +53,8 @@ pub enum ArrowErrorInnerProto { RunEndIndexOverflowError(bool), #[prost(uint64, tag = "20")] OffsetOverflowError(u64), + #[prost(string, tag = "21")] + AvroError(String), } impl ArrowErrorProto { @@ -136,6 +138,10 @@ impl ArrowErrorProto { inner: Some(ArrowErrorInnerProto::OffsetOverflowError(*offset as u64)), ctx: ctx.cloned(), }, + ArrowError::AvroError(msg) => ArrowErrorProto { + inner: Some(ArrowErrorInnerProto::AvroError(msg.to_string())), + ctx: ctx.cloned(), + }, } } @@ -185,6 +191,7 @@ impl ArrowErrorProto { ArrowErrorInnerProto::OffsetOverflowError(offset) => { ArrowError::OffsetOverflowError(*offset as usize) } + ArrowErrorInnerProto::AvroError(msg) => ArrowError::AvroError(msg.to_string()), }; (err, self.ctx.clone()) } @@ -200,10 +207,7 @@ mod tests { fn test_arrow_error_roundtrip() { let test_cases = vec![ ArrowError::NotYetImplemented("test not implemented".to_string()), - ArrowError::ExternalError(Box::new(std::io::Error::new( - ErrorKind::Other, - "external error", - ))), + ArrowError::ExternalError(Box::new(std::io::Error::other("external error"))), ArrowError::CastError("cast error".to_string()), ArrowError::MemoryError("memory error".to_string()), ArrowError::ParseError("parse error".to_string()), @@ -235,8 +239,8 @@ mod tests { let (recovered_error, recovered_ctx) = proto.to_arrow_error(); if original_error.to_string() != recovered_error.to_string() { - println!("original error: {}", original_error); - println!("recovered error: {}", recovered_error); + println!("original error: {original_error}"); + println!("recovered error: {recovered_error}"); } assert_eq!(original_error.to_string(), recovered_error.to_string()); diff --git a/src/protobuf/errors/datafusion_error.rs b/src/protobuf/errors/datafusion_error.rs index d1312b72..2cedbdc0 100644 --- a/src/protobuf/errors/datafusion_error.rs +++ b/src/protobuf/errors/datafusion_error.rs @@ -173,6 +173,13 @@ impl DataFusionErrorProto { DataFusionErrorProto::from_datafusion_error(err.as_ref()), ))), }, + #[cfg(feature = "avro")] + DataFusionError::AvroError(err) => DataFusionErrorProto { + inner: Some(DataFusionErrorInnerProto::External(err.to_string())), + }, + DataFusionError::Ffi(err) => DataFusionErrorProto { + inner: Some(DataFusionErrorInnerProto::External(err.clone())), + }, } } @@ -313,7 +320,7 @@ mod tests { ), DataFusionError::Execution("execution error".to_string()), DataFusionError::ResourcesExhausted("resources exhausted".to_string()), - DataFusionError::External(Box::new(std::io::Error::new(ErrorKind::Other, "external"))), + DataFusionError::External(Box::new(std::io::Error::other("external"))), DataFusionError::Context( "context message".to_string(), Box::new(DataFusionError::Internal("nested".to_string())), diff --git a/src/protobuf/errors/objectstore_error.rs b/src/protobuf/errors/objectstore_error.rs index 45d0ff7e..ea0bb3ff 100644 --- a/src/protobuf/errors/objectstore_error.rs +++ b/src/protobuf/errors/objectstore_error.rs @@ -267,7 +267,7 @@ mod tests { // Use known store names that will be preserved ObjectStoreError::Generic { store: "S3", - source: Box::new(std::io::Error::new(ErrorKind::Other, "generic error")), + source: Box::new(std::io::Error::other("generic error")), }, ObjectStoreError::NotFound { path: "test/path".to_string(), @@ -279,14 +279,14 @@ mod tests { }, ObjectStoreError::Precondition { path: "precondition/path".to_string(), - source: Box::new(std::io::Error::new(ErrorKind::Other, "precondition failed")), + source: Box::new(std::io::Error::other("precondition failed")), }, ObjectStoreError::NotSupported { source: Box::new(std::io::Error::new(ErrorKind::Unsupported, "not supported")), }, ObjectStoreError::NotModified { path: "not/modified".to_string(), - source: Box::new(std::io::Error::new(ErrorKind::Other, "not modified")), + source: Box::new(std::io::Error::other("not modified")), }, ObjectStoreError::NotImplemented, ObjectStoreError::PermissionDenied { @@ -298,7 +298,7 @@ mod tests { }, ObjectStoreError::Unauthenticated { path: "auth/path".to_string(), - source: Box::new(std::io::Error::new(ErrorKind::Other, "unauthenticated")), + source: Box::new(std::io::Error::other("unauthenticated")), }, ObjectStoreError::UnknownConfigurationKey { key: "unknown_key".to_string(), diff --git a/src/protobuf/errors/parquet_error.rs b/src/protobuf/errors/parquet_error.rs index 991e68e5..3ddd6530 100644 --- a/src/protobuf/errors/parquet_error.rs +++ b/src/protobuf/errors/parquet_error.rs @@ -104,10 +104,7 @@ mod tests { ParquetError::EOF("end of file".to_string()), ParquetError::ArrowError("arrow error".to_string()), ParquetError::IndexOutOfBound(42, 100), - ParquetError::External(Box::new(std::io::Error::new( - std::io::ErrorKind::Other, - "external error", - ))), + ParquetError::External(Box::new(std::io::Error::other("external error"))), ParquetError::NeedMoreData(1024), ]; diff --git a/src/protobuf/mod.rs b/src/protobuf/mod.rs index 5de1c155..81b444f1 100644 --- a/src/protobuf/mod.rs +++ b/src/protobuf/mod.rs @@ -3,13 +3,9 @@ mod distributed_codec; mod errors; mod user_codec; -#[allow(unused_imports)] pub(crate) use app_metadata::{AppMetadata, FlightAppMetadata, MetricsCollection, TaskMetrics}; pub(crate) use distributed_codec::{DistributedCodec, StageKey}; -pub(crate) use errors::{ - datafusion_error_to_tonic_status, map_flight_to_datafusion_error, - map_status_to_datafusion_error, -}; +pub(crate) use errors::{datafusion_error_to_tonic_status, map_flight_to_datafusion_error}; pub(crate) use user_codec::{ get_distributed_user_codecs, set_distributed_user_codec, set_distributed_user_codec_arc, }; diff --git a/src/stage.rs b/src/stage.rs index e4db799c..62a7d686 100644 --- a/src/stage.rs +++ b/src/stage.rs @@ -1,30 +1,28 @@ use crate::execution_plans::{DistributedExec, NetworkCoalesceExec}; +use crate::metrics::DISTRIBUTED_DATAFUSION_TASK_ID_LABEL; use crate::{NetworkShuffleExec, PartitionIsolatorExec}; +use datafusion::common::HashMap; use datafusion::common::plan_err; use datafusion::error::Result; use datafusion::execution::TaskContext; +use datafusion::physical_plan::display::DisplayableExecutionPlan; +use datafusion::physical_plan::metrics::{Label, Metric, MetricsSet}; use datafusion::physical_plan::{ExecutionPlan, ExecutionPlanProperties, displayable}; -use itertools::{Either, Itertools}; +use itertools::Either; use std::collections::VecDeque; use std::sync::Arc; use url::Url; use uuid::Uuid; /// A unit of isolation for a portion of a physical execution plan -/// that can be executed independently and across a network boundary. +/// that can be executed independently and across a network boundary. /// It implements [`ExecutionPlan`] and can be executed to produce a /// stream of record batches. /// -/// An ExecutionTask is a finer grained unit of work compared to an ExecutionStage. -/// One ExecutionStage will create one or more ExecutionTasks -/// -/// When an [`StageExec`] is execute()'d if will execute its plan and return a stream -/// of record batches. -/// -/// If the stage has input stages, then it those input stages will be executed on remote resources +/// If a stage has input stages, then those input stages will be executed on remote resources /// and will be provided the remainder of the stage tree. /// -/// For example if our stage tree looks like this: +/// For example, if our stage tree looks like this: /// /// ```text /// ┌─────────┐ @@ -39,35 +37,35 @@ use uuid::Uuid; /// ┌──────┴────────┐ /// ┌────┴────┐ ┌────┴────┐ /// │ stage 4 │ │ Stage 5 │ -/// └─────────┘ └─────────┘ +/// └─────────┘ └─────────┘ /// /// ``` -/// -/// Then executing Stage 1 will run its plan locally. Stage 1 has two inputs, Stage 2 and Stage 3. We -/// know these will execute on remote resources. As such the plan for Stage 1 must contain an -/// [`NetworkShuffleExec`] node that will read the results of Stage 2 and Stage 3 and coalese the +/// +/// Then executing Stage 1 will run its plan locally. Stage 1 has two inputs, Stage 2 and Stage 3. We +/// know these will execute on remote resources. As such, the plan for Stage 1 must contain a +/// [`NetworkShuffleExec`] node that will read the results of Stage 2 and Stage 3 and coalesce the /// results. /// /// When Stage 1's [`NetworkShuffleExec`] node is executed, it makes an ArrowFlightRequest to the -/// host assigned in the Stage. It provides the following Stage tree serialilzed in the body of the +/// host assigned in the Stage. It provides the following Stage tree serialized in the body of the /// Arrow Flight Ticket: /// /// ```text -/// ┌─────────┐ -/// │ Stage 2 │ -/// └────┬────┘ +/// ┌─────────┐ +/// │ Stage 2 │ +/// └────┬────┘ /// │ /// ┌──────┴────────┐ /// ┌────┴────┐ ┌────┴────┐ /// │ Stage 4 │ │ Stage 5 │ -/// └─────────┘ └─────────┘ +/// └─────────┘ └─────────┘ /// /// ``` /// -/// The receiving ArrowFlightEndpoint will then execute Stage 2 and will repeat this process. +/// The receiving Worker will then execute Stage 2 and will repeat this process. /// /// When Stage 4 is executed, it has no input tasks, so it is assumed that the plan included in that -/// Stage can complete on its own; its likely holding a leaf node in the overall phyysical plan and +/// Stage can complete on its own; it's likely holding a leaf node in the overall physical plan and /// producing data from a [`DataSourceExec`]. #[derive(Debug, Clone)] pub struct Stage { @@ -132,7 +130,7 @@ impl MaybeEncodedPlan { } } -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq)] pub struct DistributedTaskContext { pub task_index: usize, pub task_count: usize, @@ -150,8 +148,8 @@ impl DistributedTaskContext { } impl Stage { - /// Creates a new `ExecutionStage` with the given plan and inputs. One task will be created - /// responsible for partitions in the plan. + /// Creates a new `Stage` with the given plan and inputs. `ExecutionTasks` will be created for + /// each of the `n_tasks` specified tasks. pub(crate) fn new( query_id: Uuid, num: usize, @@ -167,17 +165,19 @@ impl Stage { } } -use crate::distributed_physical_optimizer_rule::{NetworkBoundary, NetworkBoundaryExt}; +use crate::{DistributedMetricsFormat, rewrite_distributed_plan_with_metrics}; +use crate::{NetworkBoundary, NetworkBoundaryExt}; use bytes::Bytes; +use datafusion::common::DataFusionError; use datafusion::physical_expr::Partitioning; use datafusion_proto::physical_plan::{AsExecutionPlan, PhysicalExtensionCodec}; use datafusion_proto::protobuf::PhysicalPlanNode; use prost::Message; /// Be able to display a nice tree for stages. /// -/// The challenge to doing this at the moment is that `TreeRenderVistor` +/// The challenge to doing this at the moment is that `TreeRenderVisitor` /// in [`datafusion::physical_plan::display`] is not public, and that it also -/// is specific to a `ExecutionPlan` trait object, which we don't have. +/// is specific to an `ExecutionPlan` trait object, which we don't have. /// /// TODO: try to upstream a change to make rendering of Trees (logical, physical, stages) against /// a generic trait rather than a specific trait object. This would allow us to @@ -187,25 +187,46 @@ use prost::Message; /// the Stage tree. use std::fmt::Write; +/// explain_analyze renders an [ExecutionPlan] with metrics. +pub fn explain_analyze( + executed: Arc, + format: DistributedMetricsFormat, +) -> Result { + match executed.as_any().downcast_ref::() { + None => Ok(DisplayableExecutionPlan::with_metrics(executed.as_ref()) + .indent(true) + .to_string()), + Some(_) => { + let executed = rewrite_distributed_plan_with_metrics(executed.clone(), format)?; + Ok(display_plan_ascii(executed.as_ref(), true)) + } + } +} + // Unicode box-drawing characters for creating borders and connections. const LTCORNER: &str = "┌"; // Left top corner const LDCORNER: &str = "└"; // Left bottom corner const VERTICAL: &str = "│"; // Vertical line const HORIZONTAL: &str = "─"; // Horizontal line - -pub fn display_plan_ascii(plan: &dyn ExecutionPlan) -> String { +pub fn display_plan_ascii(plan: &dyn ExecutionPlan, show_metrics: bool) -> String { if let Some(plan) = plan.as_any().downcast_ref::() { let mut f = String::new(); - display_ascii(Either::Left(plan), 0, &mut f).unwrap(); + display_ascii(Either::Left(plan), 0, show_metrics, &mut f).unwrap(); f } else { - displayable(plan).indent(true).to_string() + match show_metrics { + true => DisplayableExecutionPlan::with_metrics(plan) + .indent(true) + .to_string(), + false => displayable(plan).indent(true).to_string(), + } } } fn display_ascii( stage: Either<&DistributedExec, &Stage>, depth: usize, + show_metrics: bool, f: &mut String, ) -> std::fmt::Result { let plan = match stage { @@ -244,7 +265,7 @@ fn display_ascii( } let mut plan_str = String::new(); - display_inner_ascii(plan, 0, &mut plan_str)?; + display_inner_ascii(plan, 0, show_metrics, &mut plan_str)?; let plan_str = plan_str .split('\n') .filter(|v| !v.is_empty()) @@ -259,7 +280,7 @@ fn display_ascii( HORIZONTAL.repeat(50) )?; for input_stage in find_input_stages(plan.as_ref()) { - display_ascii(Either::Right(input_stage), depth + 1, f)?; + display_ascii(Either::Right(input_stage), depth + 1, show_metrics, f)?; } Ok(()) } @@ -267,21 +288,145 @@ fn display_ascii( fn display_inner_ascii( plan: &Arc, indent: usize, + show_metrics: bool, f: &mut String, ) -> std::fmt::Result { + let metrics_str = if show_metrics { + if let Some(metrics) = plan.metrics() { + let formatted = format_metrics_by_task(&metrics); + if formatted.is_empty() { + ", metrics=[]".to_string() + } else { + format!(", metrics=[{formatted}]") + } + } else { + ", metrics=[]".to_string() + } + } else { + String::new() + }; + let node_str = displayable(plan.as_ref()).one_line().to_string(); - writeln!(f, "{} {node_str}", " ".repeat(indent))?; + writeln!( + f, + "{} {}{metrics_str}", + " ".repeat(indent), + node_str.trim_end() // remove trailing newline + )?; if plan.is_network_boundary() { return Ok(()); } for child in plan.children() { - display_inner_ascii(child, indent + 2, f)?; + display_inner_ascii(child, indent + 2, show_metrics, f)?; } Ok(()) } +/// Aggregates metrics by (name, task_id), preserving the [DISTRIBUTED_DATAFUSION_TASK_ID_LABEL] +/// only. Metrics without a task_id label (ie. non distributed metrics) are aggregated together. +/// +/// For a non-distributed plan, this is equivalent to [MetricsSet::aggregate_by_name] since there +/// will be no task ids. For a distributed plan, it's expected that the metrics rewriter populated +/// task id labels in all metrics. +fn aggregate_by_task_id(metrics: &MetricsSet) -> MetricsSet { + // Key: (metric_name, Option) + let mut map: HashMap<(String, Option), Metric> = HashMap::new(); + + for metric in metrics.iter() { + let name = metric.value().name().to_string(); + let task_id = metric + .labels() + .iter() + .find(|l| l.name() == DISTRIBUTED_DATAFUSION_TASK_ID_LABEL) + .map(|l| l.value().to_string()); + + let key = (name, task_id.clone()); + + map.entry(key) + .and_modify(|accum| { + accum.value_mut().aggregate(metric.value()); + }) + .or_insert_with(|| { + let labels = task_id + .map(|id| vec![Label::new(DISTRIBUTED_DATAFUSION_TASK_ID_LABEL, id)]) + .unwrap_or_default(); + let mut accum = Metric::new_with_labels( + metric.value().new_empty(), + None, // no partition + labels, + ); + accum.value_mut().aggregate(metric.value()); + accum + }); + } + + let mut result = MetricsSet::new(); + for (_, metric) in map { + result.push(Arc::new(metric)); + } + result +} + +/// Sorts metrics by display priority, then name, then by task_id (numerically). +/// +/// For a non-distributed plan, this is equivalent to [MetricsSet::sorted_for_display] since there +/// will be no task ids. For a distributed plan, it's expected that the metrics rewriter populated +/// task id labels in all metrics. +fn sorted_for_display_by_task_id(metrics: MetricsSet) -> MetricsSet { + let mut vec: Vec> = metrics.iter().cloned().collect(); + vec.sort_unstable_by_key(|metric| { + let task_id = metric + .labels() + .iter() + .find(|l| l.name() == DISTRIBUTED_DATAFUSION_TASK_ID_LABEL) + .and_then(|l| l.value().parse::().ok()); + ( + metric.value().display_sort_key(), + metric.value().name().to_owned(), + task_id, + ) + }); + let mut result = MetricsSet::new(); + for m in vec { + result.push(m); + } + result +} + +/// Formats metrics as "{metric_name}_{task_id}={value}, {metric_name}_{task_id}={value}" +/// e.g., "output_rows_0=100, output_rows_1=150, elapsed_compute_0=50ns, elapsed_compute_1=100ns" +/// +/// For a non-distributed plan, this is equivalent to using [ShowMetrics::Aggregated] / +/// [DisplayableExecutionPlan::with_metrics] which aggregates, sorts, removes timestamps, and finally formats +/// the metrics. +/// +/// See +/// https://github.com/apache/datafusion/blob/b463a9f9e3c9603eb2db7113125fea3a1b7f5455/datafusion/physical-plan/src/display.rs#L421. +fn format_metrics_by_task(metrics: &MetricsSet) -> String { + let aggregated = aggregate_by_task_id(metrics); + let sorted = sorted_for_display_by_task_id(aggregated).timestamps_removed(); + + sorted + .iter() + .map(|m| { + let name = m.value().name(); + let task_id = m + .labels() + .iter() + .find(|l| l.name() == DISTRIBUTED_DATAFUSION_TASK_ID_LABEL) + .map(|l| l.value()); + + match task_id { + Some(id) => format!("{name}_{id}={}", m.value()), + None => format!("{name}={}", m.value()), + } + }) + .collect::>() + .join(", ") +} + fn format_tasks_for_stage(n_tasks: usize, head: &Arc) -> String { let partitioning = head.properties().output_partitioning(); let input_partitions = partitioning.partition_count(); @@ -290,9 +435,12 @@ fn format_tasks_for_stage(n_tasks: usize, head: &Arc) -> Stri let mut off = 0; for i in 0..n_tasks { result += &format!("t{i}:["); - result += &(off..(off + input_partitions)) - .map(|v| format!("p{v}")) - .join(","); + let end = off + input_partitions - 1; + if input_partitions == 1 { + result += &format!("p{off}"); + } else { + result += &format!("p{off}..p{end}"); + } result += "] "; off += if hash_shuffle { 0 } else { input_partitions } } @@ -308,7 +456,7 @@ const COLOR_SCHEME: &str = "spectral6"; /// Graphviz dot format. /// You can view them on https://vis-js.com /// -/// Or it is often useful to expertiment with plan output using +/// Or it is often useful to experiment with plan output using /// https://datafusion-fiddle.vercel.app/ pub fn display_plan_graphviz(plan: Arc) -> Result { let mut f = String::new(); @@ -317,10 +465,9 @@ pub fn display_plan_graphviz(plan: Arc) -> Result { f, "digraph G {{ rankdir=BT - edge[colorscheme={}, penwidth=2.0] + edge[colorscheme={COLOR_SCHEME}, penwidth=2.0] splines=false -", - COLOR_SCHEME +" )?; if plan.as_any().is::() { @@ -341,7 +488,7 @@ pub fn display_plan_graphviz(plan: Arc) -> Result { for stage in &all_stages { for i in 0..stage.tasks.iter().len() { let p = display_single_task(stage, i)?; - writeln!(f, "{}", p)?; + writeln!(f, "{p}")?; } } // now draw edges between the tasks @@ -364,7 +511,7 @@ pub fn display_plan_graphviz(plan: Arc) -> Result { // single plan, not a stage tree writeln!(f, "node[shape=none]")?; let p = display_plan(&plan, 0, 1, 0)?; - writeln!(f, "{}", p)?; + writeln!(f, "{p}")?; } writeln!(f, "}}")?; @@ -431,7 +578,7 @@ fn display_plan( while let Some(plan) = queue.pop_front() { node_index += 1; let p = display_single_plan(plan.as_ref(), stage_num, task_i, node_index)?; - writeln!(f, "{}", p)?; + writeln!(f, "{p}")?; if plan.is_network_boundary() { continue; @@ -454,7 +601,7 @@ fn display_plan( node_index += 1; if let Some(node) = plan.as_any().downcast_ref::() { isolator_partition_group = Some(PartitionIsolatorExec::partition_group( - node.input().output_partitioning().partition_count(), + node.input.output_partitioning().partition_count(), task_i, n_tasks, )); @@ -579,7 +726,7 @@ pub fn display_single_plan( )?; for i in 0..output_partitions { - writeln!(f, " ", i)?; + writeln!(f, " ")?; } writeln!( @@ -605,7 +752,7 @@ pub fn display_single_plan( )?; for i in 0..input_partitions { - writeln!(f, " ", i)?; + writeln!(f, " ")?; } writeln!( @@ -640,7 +787,7 @@ fn display_inter_task_edges( while let Some(plan) = queue.pop_front() { index += 1; if let Some(node) = plan.as_any().downcast_ref::() { - if node.input_stage().is_none_or(|v| v.num != input_stage.num) { + if node.input_stage().num != input_stage.num { continue; } // draw the edges to this node pulling data up from its child @@ -664,7 +811,7 @@ fn display_inter_task_edges( } continue; } else if let Some(node) = plan.as_any().downcast_ref::() { - if node.input_stage().is_none_or(|v| v.num != input_stage.num) { + if node.input_stage().num != input_stage.num { continue; } // draw the edges to this node pulling data up from its child @@ -714,9 +861,7 @@ fn find_input_stages(plan: &dyn ExecutionPlan) -> Vec<&Stage> { let mut result = vec![]; for child in plan.children() { if let Some(plan) = child.as_network_boundary() { - if let Some(stage) = plan.input_stage() { - result.push(stage); - } + result.push(plan.input_stage()); } else { result.extend(find_input_stages(child.as_ref())); } @@ -724,12 +869,10 @@ fn find_input_stages(plan: &dyn ExecutionPlan) -> Vec<&Stage> { result } -fn find_all_stages(plan: &Arc) -> Vec<&Stage> { +pub(crate) fn find_all_stages(plan: &Arc) -> Vec<&Stage> { let mut result = vec![]; if let Some(plan) = plan.as_network_boundary() { - if let Some(stage) = plan.input_stage() { - result.push(stage); - } + result.push(plan.input_stage()); } for child in plan.children() { result.extend(find_all_stages(child)); diff --git a/src/test_utils/benchmarks_common.rs b/src/test_utils/benchmarks_common.rs new file mode 100644 index 00000000..4aeb679b --- /dev/null +++ b/src/test_utils/benchmarks_common.rs @@ -0,0 +1,82 @@ +use datafusion::common::{DataFusionError, internal_datafusion_err, internal_err}; +use datafusion::prelude::{ParquetReadOptions, SessionContext}; +use std::fs; +use std::path::Path; + +pub(crate) fn get_queries(path: &str) -> Vec { + let queries_dir = Path::new(env!("CARGO_MANIFEST_DIR")).join(path); + let mut result = vec![]; + for file in queries_dir.read_dir().unwrap() { + let file = file.unwrap(); + let file_name = file.file_name().display().to_string(); + if file_name.ends_with(".sql") { + result.push(file_name.trim_end_matches(".sql").to_string()); + } + } + + // Each element might be something like q12.sql or custom2.sql. + // This orders the string list by the parsed integer number inside an arbitrary string. + result.sort_by(|a, b| { + // Extract numbers from both strings + let extract_number = |s: &str| -> Option { + s.chars() + .filter(|c| c.is_ascii_digit()) + .collect::() + .parse::() + .ok() + }; + + match (extract_number(a), extract_number(b)) { + (Some(num_a), Some(num_b)) => num_a.cmp(&num_b), + (Some(_), None) => std::cmp::Ordering::Less, + (None, Some(_)) => std::cmp::Ordering::Greater, + (None, None) => a.cmp(b), // Fall back to lexicographic ordering + } + }); + result +} + +pub(crate) fn get_query(path: &str, id: &str) -> Result { + let queries_dir = Path::new(env!("CARGO_MANIFEST_DIR")).join(path); + + if !queries_dir.exists() { + return internal_err!( + "Benchmark queries directory not found: {}", + queries_dir.display() + ); + } + + let query_file = queries_dir.join(format!("{id}.sql")); + + if !query_file.exists() { + return internal_err!("Query file not found: {}", query_file.display()); + } + + let query_sql = fs::read_to_string(&query_file) + .map_err(|e| { + internal_datafusion_err!("Failed to read query file {}: {e}", query_file.display()) + })? + .trim() + .to_string(); + + Ok(query_sql) +} + +pub async fn register_tables( + ctx: &SessionContext, + data_path: &Path, +) -> Result<(), DataFusionError> { + for entry in fs::read_dir(data_path)? { + let path = entry?.path(); + if path.is_dir() { + let table_name = path.file_name().unwrap().to_str().unwrap(); + ctx.register_parquet( + table_name, + path.to_str().unwrap(), + ParquetReadOptions::default(), + ) + .await?; + } + } + Ok(()) +} diff --git a/src/test_utils/clickbench.rs b/src/test_utils/clickbench.rs new file mode 100644 index 00000000..dfa4aad0 --- /dev/null +++ b/src/test_utils/clickbench.rs @@ -0,0 +1,68 @@ +use crate::test_utils::benchmarks_common; +use datafusion::common::DataFusionError; +use std::fs; +use std::io::Write; +use std::ops::Range; +use std::path::{Path, PathBuf}; +use tokio::task::JoinSet; + +const URL: &str = + "https://datasets.clickhouse.com/hits_compatible/athena_partitioned/hits_{}.parquet"; + +pub fn get_queries() -> Vec { + benchmarks_common::get_queries("testdata/clickbench/queries") +} + +pub fn get_query(id: &str) -> Result { + benchmarks_common::get_query("testdata/clickbench/queries", id) +} + +/// Downloads the datafusion-benchmarks repository as a zip file +async fn download_benchmark( + dest_path: PathBuf, + i: usize, +) -> Result<(), Box> { + if dest_path.exists() { + return Ok(()); + } + + // Create directory if it doesn't exist + if let Some(parent) = dest_path.parent() { + fs::create_dir_all(parent)?; + } + + // Download the file + let response = reqwest::get(URL.replace("{}", &i.to_string())).await?; + let bytes = response.bytes().await?; + + // Write to file + let mut file = fs::File::create(&dest_path)?; + file.write_all(&bytes)?; + + println!("Downloaded to {}", dest_path.display()); + + Ok(()) +} + +async fn download_partitioned( + dest_path: PathBuf, + range: Range, +) -> Result<(), Box> { + let mut join_set = JoinSet::new(); + for i in range { + let dest_path = dest_path.clone(); + join_set.spawn(async move { + download_benchmark(dest_path.join("hits").join(format!("{i}.parquet")), i).await + }); + } + join_set.join_all().await; + Ok(()) +} + +pub async fn generate_clickbench_data( + dest_path: &Path, + range: Range, +) -> Result<(), Box> { + download_partitioned(dest_path.to_path_buf(), range).await?; + Ok(()) +} diff --git a/src/test_utils/in_memory_channel_resolver.rs b/src/test_utils/in_memory_channel_resolver.rs index cd412fd0..96baef2d 100644 --- a/src/test_utils/in_memory_channel_resolver.rs +++ b/src/test_utils/in_memory_channel_resolver.rs @@ -1,12 +1,11 @@ use crate::{ - ArrowFlightEndpoint, BoxCloneSyncChannel, ChannelResolver, DistributedExt, - DistributedSessionBuilderContext, + BoxCloneSyncChannel, ChannelResolver, DefaultSessionBuilder, DistributedExt, + MappedWorkerSessionBuilderExt, Worker, WorkerResolver, WorkerSessionBuilder, + create_flight_client, }; use arrow_flight::flight_service_client::FlightServiceClient; -use arrow_flight::flight_service_server::FlightServiceServer; use async_trait::async_trait; use datafusion::common::DataFusionError; -use datafusion::execution::SessionStateBuilder; use hyper_util::rt::TokioIo; use tonic::transport::{Endpoint, Server}; @@ -19,14 +18,13 @@ pub struct InMemoryChannelResolver { channel: FlightServiceClient, } -impl Default for InMemoryChannelResolver { - fn default() -> Self { - Self::new() - } -} - impl InMemoryChannelResolver { - pub fn new() -> Self { + /// Build an [InMemoryChannelResolver] with a custom [WorkerSessionBuilder]. + /// This allows you to inject your own DataFusion extensions in the in-memory worker + /// spawned by this method. + pub fn from_session_builder( + builder: impl WorkerSessionBuilder + Send + Sync + 'static, + ) -> Self { let (client, server) = tokio::io::duplex(1024 * 1024); let mut client = Some(client); @@ -40,26 +38,18 @@ impl InMemoryChannelResolver { })); let this = Self { - channel: FlightServiceClient::new(BoxCloneSyncChannel::new(channel)), + channel: create_flight_client(BoxCloneSyncChannel::new(channel)), }; let this_clone = this.clone(); - let endpoint = - ArrowFlightEndpoint::try_new(move |ctx: DistributedSessionBuilderContext| { - let this = this.clone(); - async move { - let builder = SessionStateBuilder::new() - .with_default_features() - .with_distributed_channel_resolver(this) - .with_runtime_env(ctx.runtime_env.clone()); - Ok(builder.build()) - } - }) - .unwrap(); + let endpoint = Worker::from_session_builder(builder.map(move |builder| { + let this = this.clone(); + Ok(builder.with_distributed_channel_resolver(this).build()) + })); tokio::spawn(async move { Server::builder() - .add_service(FlightServiceServer::new(endpoint)) + .add_service(endpoint.into_flight_server()) .serve_with_incoming(tokio_stream::once(Ok::<_, std::io::Error>(server))) .await }); @@ -68,12 +58,14 @@ impl InMemoryChannelResolver { } } -#[async_trait] -impl ChannelResolver for InMemoryChannelResolver { - fn get_urls(&self) -> Result, DataFusionError> { - Ok(vec![url::Url::parse(DUMMY_URL).unwrap()]) +impl Default for InMemoryChannelResolver { + fn default() -> Self { + Self::from_session_builder(DefaultSessionBuilder) } +} +#[async_trait] +impl ChannelResolver for InMemoryChannelResolver { async fn get_flight_client_for_url( &self, _: &url::Url, @@ -81,3 +73,21 @@ impl ChannelResolver for InMemoryChannelResolver { Ok(self.channel.clone()) } } + +pub struct InMemoryWorkerResolver { + n_workers: usize, +} + +impl InMemoryWorkerResolver { + pub fn new(n_workers: usize) -> Self { + Self { n_workers } + } +} + +impl WorkerResolver for InMemoryWorkerResolver { + fn get_urls(&self) -> Result, DataFusionError> { + // Set to a high number so that the distributed planner does not limit the maximum + // spawned tasks to just 1. + Ok(vec![url::Url::parse(DUMMY_URL).unwrap(); self.n_workers]) + } +} diff --git a/src/test_utils/insta.rs b/src/test_utils/insta.rs index 54c92256..3a829c3d 100644 --- a/src/test_utils/insta.rs +++ b/src/test_utils/insta.rs @@ -23,6 +23,5 @@ pub fn settings() -> insta::Settings { "UUID", ); settings.add_filter(r"\d+\.\.\d+", ".."); - settings } diff --git a/src/test_utils/localhost.rs b/src/test_utils/localhost.rs index f1685312..09a1ee12 100644 --- a/src/test_utils/localhost.rs +++ b/src/test_utils/localhost.rs @@ -1,29 +1,31 @@ use crate::{ - ArrowFlightEndpoint, BoxCloneSyncChannel, ChannelResolver, DistributedExt, - DistributedSessionBuilder, DistributedSessionBuilderContext, - MappedDistributedSessionBuilderExt, + DistributedExt, DistributedPhysicalOptimizerRule, Worker, WorkerResolver, WorkerSessionBuilder, }; -use arrow_flight::flight_service_client::FlightServiceClient; -use arrow_flight::flight_service_server::FlightServiceServer; use async_trait::async_trait; use datafusion::common::DataFusionError; use datafusion::common::runtime::JoinSet; use datafusion::execution::SessionStateBuilder; -use datafusion::execution::runtime_env::RuntimeEnv; use datafusion::prelude::SessionContext; use std::error::Error; use std::sync::Arc; use std::time::Duration; use tokio::net::TcpListener; -use tonic::transport::{Channel, Server}; +use tonic::transport::Server; use url::Url; +/// Create workers and context on localhost with a fixed number of target partitions. +/// +/// Creates `num_workers` listeners, all bound to a random OS decided port on `127.0.0.1`, then +/// attaches a channel resolver that is aware of these addresses to `session_builder` and uses it +/// to spawn a flight service behind each listener. +/// +/// Returns a session context aware of these workers, and a join set of all spawned worker tasks. pub async fn start_localhost_context( num_workers: usize, session_builder: B, -) -> (SessionContext, JoinSet<()>) +) -> (SessionContext, JoinSet<()>, Vec) where - B: DistributedSessionBuilder + Send + Sync + 'static, + B: WorkerSessionBuilder + Send + Sync + 'static, B: Clone, { let listeners = futures::future::try_join_all( @@ -44,42 +46,42 @@ where }) .collect(); - let channel_resolver = LocalHostChannelResolver::new(ports.clone()); - let session_builder = session_builder.map(move |builder: SessionStateBuilder| { - let channel_resolver = channel_resolver.clone(); - Ok(builder - .with_distributed_channel_resolver(channel_resolver) - .build()) - }); let mut join_set = JoinSet::new(); + let mut workers = vec![]; for listener in listeners { let session_builder = session_builder.clone(); + let worker = Worker::from_session_builder(session_builder); + workers.push(worker.clone()); + + let incoming = tokio_stream::wrappers::TcpListenerStream::new(listener); + join_set.spawn(async move { - spawn_flight_service(session_builder, listener) + Server::builder() + .add_service(worker.into_flight_server()) + .serve_with_incoming(incoming) .await .unwrap(); }); } tokio::time::sleep(Duration::from_millis(100)).await; - let mut state = session_builder - .build_session_state(DistributedSessionBuilderContext { - runtime_env: Arc::new(RuntimeEnv::default()), - headers: Default::default(), - }) - .await - .unwrap(); + let worker_resolver = LocalHostWorkerResolver::new(ports); + let mut state = SessionStateBuilder::new() + .with_default_features() + .with_physical_optimizer_rule(Arc::new(DistributedPhysicalOptimizerRule)) + .with_distributed_worker_resolver(worker_resolver) + .build(); state.config_mut().options_mut().execution.target_partitions = 3; - (SessionContext::from(state), join_set) + (SessionContext::from(state), join_set, workers) } #[derive(Clone)] -pub struct LocalHostChannelResolver { +pub struct LocalHostWorkerResolver { ports: Vec, } -impl LocalHostChannelResolver { +impl LocalHostWorkerResolver { pub fn new, I: IntoIterator>(ports: I) -> Self where N::Error: std::fmt::Debug, @@ -91,7 +93,7 @@ impl LocalHostChannelResolver { } #[async_trait] -impl ChannelResolver for LocalHostChannelResolver { +impl WorkerResolver for LocalHostWorkerResolver { fn get_urls(&self) -> Result, DataFusionError> { self.ports .iter() @@ -99,26 +101,18 @@ impl ChannelResolver for LocalHostChannelResolver { .map(|url| Url::parse(&url).map_err(external_err)) .collect::, _>>() } - async fn get_flight_client_for_url( - &self, - url: &Url, - ) -> Result, DataFusionError> { - let endpoint = Channel::from_shared(url.to_string()).map_err(external_err)?; - let channel = endpoint.connect().await.map_err(external_err)?; - Ok(FlightServiceClient::new(BoxCloneSyncChannel::new(channel))) - } } pub async fn spawn_flight_service( - session_builder: impl DistributedSessionBuilder + Send + Sync + 'static, + session_builder: impl WorkerSessionBuilder + Send + Sync + 'static, incoming: TcpListener, ) -> Result<(), Box> { - let endpoint = ArrowFlightEndpoint::try_new(session_builder)?; + let endpoint = Worker::from_session_builder(session_builder); let incoming = tokio_stream::wrappers::TcpListenerStream::new(incoming); Ok(Server::builder() - .add_service(FlightServiceServer::new(endpoint)) + .add_service(endpoint.into_flight_server()) .serve_with_incoming(incoming) .await?) } diff --git a/src/test_utils/metrics.rs b/src/test_utils/metrics.rs index 4335cb73..a768379d 100644 --- a/src/test_utils/metrics.rs +++ b/src/test_utils/metrics.rs @@ -2,36 +2,41 @@ use crate::metrics::proto::{ElapsedCompute, EndTimestamp, OutputRows, StartTimes use crate::metrics::proto::{MetricProto, MetricValueProto, MetricsSetProto}; /// creates a "distinct" set of metrics from the provided seed -pub fn make_test_metrics_set_proto_from_seed(seed: u64) -> MetricsSetProto { +pub fn make_test_metrics_set_proto_from_seed(seed: u64, num_metrics: usize) -> MetricsSetProto { const TEST_TIMESTAMP: i64 = 1758200400000000000; // 2025-09-18 13:00:00 UTC - MetricsSetProto { - metrics: vec![ - MetricProto { - metric: Some(MetricValueProto::OutputRows(OutputRows { value: seed })), + + let mut result = MetricsSetProto { metrics: vec![] }; + + for i in 0..num_metrics { + let value = seed + i as u64; + result.push(match i % 4 { + 0 => MetricProto { + metric: Some(MetricValueProto::OutputRows(OutputRows { value })), labels: vec![], partition: None, }, - MetricProto { - metric: Some(MetricValueProto::ElapsedCompute(ElapsedCompute { - value: seed, - })), + + 1 => MetricProto { + metric: Some(MetricValueProto::ElapsedCompute(ElapsedCompute { value })), labels: vec![], partition: None, }, - MetricProto { + 2 => MetricProto { metric: Some(MetricValueProto::StartTimestamp(StartTimestamp { - value: Some(TEST_TIMESTAMP + (seed as i64 * 1_000_000_000)), + value: Some(TEST_TIMESTAMP + (value as i64 * 1_000_000_000)), })), labels: vec![], partition: None, }, - MetricProto { + 3 => MetricProto { metric: Some(MetricValueProto::EndTimestamp(EndTimestamp { - value: Some(TEST_TIMESTAMP + ((seed as i64 + 1) * 1_000_000_000)), + value: Some(TEST_TIMESTAMP + (value as i64 * 1_000_000_000)), })), labels: vec![], partition: None, }, - ], + _ => unreachable!(), + }) } + result } diff --git a/src/test_utils/mock_exec.rs b/src/test_utils/mock_exec.rs index a8d41266..52678be9 100644 --- a/src/test_utils/mock_exec.rs +++ b/src/test_utils/mock_exec.rs @@ -7,8 +7,14 @@ use datafusion::physical_plan::common::compute_record_batch_statistics; use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType}; use datafusion::physical_plan::stream::{RecordBatchReceiverStream, RecordBatchStreamAdapter}; use datafusion::physical_plan::{DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties}; +use futures::{Stream, stream}; use std::any::Any; +use std::pin::Pin; use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::time::Duration; +use tokio::sync::Notify; +use tokio::time::sleep; // Copied from https://github.com/apache/datafusion/blob/4b9a468cc1949062cf3cd8685ba8ced377fd212e/datafusion/physical-plan/src/test/exec.rs#L121 /// A Mock ExecutionPlan that can be used for writing tests of other @@ -16,11 +22,17 @@ use std::sync::Arc; #[derive(Debug)] pub struct MockExec { /// the results to send back - data: Vec>, + data: Vec>>, schema: SchemaRef, + partitions: usize, /// if true (the default), sends data using a separate task to ensure the /// batches are not available without this stream yielding first use_task: bool, + delay: Option, + start_notify: Option>, + permit_open: Option>, + permit_notify: Option>, + execute_counts: Option>>, cache: PlanProperties, } @@ -33,11 +45,38 @@ impl MockExec { /// ensure any poll loops are correct. This behavior can be /// changed with `with_use_task` pub fn new(data: Vec>, schema: SchemaRef) -> Self { - let cache = Self::compute_properties(Arc::clone(&schema)); + let cache = Self::compute_properties(Arc::clone(&schema), 1); + Self { + data: vec![data], + schema, + partitions: 1, + use_task: true, + delay: None, + start_notify: None, + permit_open: None, + permit_notify: None, + execute_counts: None, + cache, + } + } + + /// Create a new `MockExec` with per-partition data. + pub fn new_partitioned( + data: Vec>>, + schema: SchemaRef, + ) -> Self { + let partitions = data.len().max(1); + let cache = Self::compute_properties(Arc::clone(&schema), partitions); Self { data, schema, + partitions, use_task: true, + delay: None, + start_notify: None, + permit_open: None, + permit_notify: None, + execute_counts: None, cache, } } @@ -50,11 +89,36 @@ impl MockExec { self } + /// Adds a delay between emitted batches (simulates a slow producer). + pub fn with_delay_between_batches(mut self, delay: Duration) -> Self { + self.delay = Some(delay); + self + } + + /// Notify when execute is called (before emitting any batches). + pub fn with_start_notify(mut self, start_notify: Arc) -> Self { + self.start_notify = Some(start_notify); + self + } + + /// Block emission until `permit_open` is true (use with `permit_notify`). + pub fn with_gate(mut self, permit_open: Arc, permit_notify: Arc) -> Self { + self.permit_open = Some(permit_open); + self.permit_notify = Some(permit_notify); + self + } + + /// Track execute calls per partition (for replay/once-only assertions). + pub fn with_execute_counts(mut self, execute_counts: Arc>) -> Self { + self.execute_counts = Some(execute_counts); + self + } + /// This function creates the cache object that stores the plan properties such as schema, equivalence properties, ordering, partitioning, etc. - fn compute_properties(schema: SchemaRef) -> PlanProperties { + fn compute_properties(schema: SchemaRef, partitions: usize) -> PlanProperties { PlanProperties::new( EquivalenceProperties::new(schema), - Partitioning::UnknownPartitioning(1), + Partitioning::UnknownPartitioning(partitions), EmissionType::Incremental, Boundedness::Bounded, ) @@ -105,11 +169,21 @@ impl ExecutionPlan for MockExec { partition: usize, _context: Arc, ) -> datafusion::common::Result { - assert_eq!(partition, 0); + assert!(partition < self.partitions); + + if let Some(counts) = &self.execute_counts { + counts[partition].fetch_add(1, Ordering::SeqCst); + } + + if let Some(start_notify) = &self.start_notify { + start_notify.notify_waiters(); + } // Result doesn't implement clone, so do it ourself let data: Vec<_> = self .data + .get(partition) + .expect("partition data") .iter() .map(|r| match r { Ok(batch) => Ok(batch.clone()), @@ -123,8 +197,22 @@ impl ExecutionPlan for MockExec { // the batches are not available without the stream // yielding). let tx = builder.tx(); + let delay = self.delay; + let permit_open = self.permit_open.clone(); + let permit_notify = self.permit_notify.clone(); builder.spawn(async move { + if let Some(open) = permit_open { + let notify = permit_notify.expect("permit_notify"); + while !open.load(Ordering::SeqCst) { + notify.notified().await; + } + } for batch in data { + if let Some(delay) = delay + && delay > Duration::ZERO + { + sleep(delay).await; + } // println!("Sending batch via delayed stream"); if let Err(e) = tx.send(batch).await { println!("ERROR batch via delayed stream: {e}"); @@ -136,8 +224,40 @@ impl ExecutionPlan for MockExec { // returned stream simply reads off the rx stream Ok(builder.build()) } else { - // make an input that will error - let stream = futures::stream::iter(data); + let delay = self.delay; + let permit_open = self.permit_open.clone(); + let permit_notify = self.permit_notify.clone(); + let stream: Pin< + Box> + Send>, + > = if delay.is_some() || permit_open.is_some() { + Box::pin(stream::unfold( + (data.into_iter(), false), + move |(mut iter, mut gate_done)| { + let permit_open = permit_open.clone(); + let permit_notify = permit_notify.clone(); + async move { + if !gate_done { + if let Some(open) = permit_open { + let notify = permit_notify.expect("permit_notify"); + while !open.load(Ordering::SeqCst) { + notify.notified().await; + } + } + gate_done = true; + } + let batch = iter.next()?; + if let Some(delay) = delay + && delay > Duration::ZERO + { + sleep(delay).await; + } + Some((batch, (iter, gate_done))) + } + }, + )) + } else { + Box::pin(stream::iter(data)) + }; Ok(Box::pin(RecordBatchStreamAdapter::new( self.schema(), stream, @@ -157,18 +277,23 @@ impl ExecutionPlan for MockExec { if partition.is_some() { return Ok(Statistics::new_unknown(&self.schema)); } - let data: datafusion::common::Result> = self + let data: datafusion::common::Result>> = self .data .iter() - .map(|r| match r { - Ok(batch) => Ok(batch.clone()), - Err(e) => Err(clone_error(e)), + .map(|partition_data| { + partition_data + .iter() + .map(|r| match r { + Ok(batch) => Ok(batch.clone()), + Err(e) => Err(clone_error(e)), + }) + .collect() }) .collect(); let data = data?; - Ok(compute_record_batch_statistics(&[data], &self.schema, None)) + Ok(compute_record_batch_statistics(&data, &self.schema, None)) } } diff --git a/src/test_utils/mod.rs b/src/test_utils/mod.rs index a5b3becd..e565d94f 100644 --- a/src/test_utils/mod.rs +++ b/src/test_utils/mod.rs @@ -1,3 +1,5 @@ +pub mod benchmarks_common; +pub mod clickbench; pub mod in_memory_channel_resolver; pub mod insta; pub mod localhost; @@ -5,5 +7,7 @@ pub mod metrics; pub mod mock_exec; pub mod parquet; pub mod plans; +pub mod property_based; pub mod session_context; +pub mod tpcds; pub mod tpch; diff --git a/src/test_utils/parquet.rs b/src/test_utils/parquet.rs index 9f4edded..81062714 100644 --- a/src/test_utils/parquet.rs +++ b/src/test_utils/parquet.rs @@ -1,6 +1,11 @@ use datafusion::error::DataFusionError; use datafusion::prelude::{ParquetReadOptions, SessionContext}; +/// Adds two test tables to the provided [SessionContext]: +/// - `flights_1m`: 1M rows of flight data. +/// - `weather`: smaller dataset with weather forecast. +/// +/// Useful for testing queries using SQL directly. pub async fn register_parquet_tables(ctx: &SessionContext) -> Result<(), DataFusionError> { ctx.register_parquet( "flights_1m", diff --git a/src/test_utils/plans.rs b/src/test_utils/plans.rs index c182c900..0932c8af 100644 --- a/src/test_utils/plans.rs +++ b/src/test_utils/plans.rs @@ -1,30 +1,41 @@ -use bytes::Bytes; +use crate::NetworkBoundaryExt; +use crate::distributed_ext::DistributedExt; +use crate::execution_plans::DistributedExec; +use crate::protobuf::StageKey; +use crate::stage::Stage; +use crate::test_utils::in_memory_channel_resolver::InMemoryWorkerResolver; +#[cfg(test)] +use crate::{DistributedConfig, TaskEstimation, TaskEstimator}; +#[cfg(test)] +use datafusion::config::ConfigOptions; use datafusion::{ common::{HashMap, HashSet}, - physical_plan::ExecutionPlan, + execution::{SessionStateBuilder, context::SessionContext}, + physical_plan::{ExecutionPlan, displayable}, + prelude::SessionConfig, }; +#[cfg(test)] +use itertools::Itertools; use std::sync::Arc; -use crate::distributed_physical_optimizer_rule::NetworkBoundaryExt; -use crate::execution_plans::DistributedExec; -use crate::protobuf::StageKey; -use crate::stage::Stage; +use super::parquet::register_parquet_tables; /// count_plan_nodes counts the number of execution plan nodes in a plan using BFS traversal. /// This does NOT traverse child stages, only the execution plan tree within this stage. -/// Excludes [NetworkBoundary] nodes from the count. -pub fn count_plan_nodes(plan: &Arc) -> usize { +/// Network boundary nodes are counted but their children (which belong to child stages) are not traversed. +pub fn count_plan_nodes_up_to_network_boundary(plan: &Arc) -> usize { let mut count = 0; let mut queue = vec![plan]; while let Some(plan) = queue.pop() { - // Skip [NetworkBoundary] nodes from the count. + // Include the network boundary in the count. + count += 1; + + // Stop at network boundaries - don't traverse into child stages if plan.as_ref().is_network_boundary() { continue; } - count += 1; - // Add children to the queue for BFS traversal for child in plan.children() { queue.push(child); @@ -51,12 +62,11 @@ pub fn get_stages_and_stage_keys( // Add each task. for j in 0..stage.tasks.len() { - let stage_key = StageKey { - query_id: Bytes::from(stage.query_id.as_bytes().to_vec()), - stage_id: stage.num as u64, - task_number: j as u64, - }; - stage_keys.insert(stage_key); + stage_keys.insert(StageKey::new( + stage.query_id.as_bytes().to_vec().into(), + stage.num as u64, + j as u64, + )); } // Add any child stages @@ -69,12 +79,129 @@ fn find_input_stages(plan: &dyn ExecutionPlan) -> Vec<&Stage> { let mut result = vec![]; for child in plan.children() { if let Some(plan) = child.as_network_boundary() { - if let Some(stage) = plan.input_stage() { - result.push(stage); - } + result.push(plan.input_stage()); } else { result.extend(find_input_stages(child.as_ref())); } } result } + +/// Creates a physical plan from SQL without applying broadcast insertion or distribution. +/// Used for snapshotting the baseline physical plan in tests. +pub async fn sql_to_physical_plan( + query: &str, + target_partitions: usize, + num_workers: usize, +) -> String { + let config = SessionConfig::new() + .with_target_partitions(target_partitions) + .with_information_schema(true); + + let state = SessionStateBuilder::new() + .with_default_features() + .with_config(config) + .with_distributed_worker_resolver(InMemoryWorkerResolver::new(num_workers)) + .build(); + + let ctx = SessionContext::new_with_state(state); + register_parquet_tables(&ctx).await.unwrap(); + + let df = ctx.sql(query).await.unwrap(); + let physical_plan = df.create_physical_plan().await.unwrap(); + + format!("{}", displayable(physical_plan.as_ref()).indent(true)) +} + +#[cfg(test)] +pub(crate) fn base_session_builder( + target_partitions: usize, + num_workers: usize, + broadcast_enabled: bool, +) -> SessionStateBuilder { + let mut config = SessionConfig::new() + .with_target_partitions(target_partitions) + .with_information_schema(true); + + let d_cfg = DistributedConfig { + broadcast_joins: broadcast_enabled, + ..Default::default() + }; + config.set_distributed_option_extension(d_cfg); + + SessionStateBuilder::new() + .with_default_features() + .with_config(config) + .with_distributed_worker_resolver(InMemoryWorkerResolver::new(num_workers)) +} + +#[cfg(test)] +#[derive(Clone, Copy, Debug)] +pub(crate) struct TestPlanOptions { + pub(crate) target_partitions: usize, + pub(crate) num_workers: usize, + pub(crate) broadcast_enabled: bool, +} + +#[cfg(test)] +impl Default for TestPlanOptions { + fn default() -> Self { + Self { + target_partitions: 4, + num_workers: 4, + broadcast_enabled: false, + } + } +} + +#[cfg(test)] +pub(crate) async fn context_with_query( + builder: SessionStateBuilder, + query: &str, +) -> (SessionContext, String) { + let state = builder.build(); + let ctx = SessionContext::new_with_state(state); + let mut queries = query.split(';').collect_vec(); + let last_query = queries.pop().unwrap(); + + for query in queries { + ctx.sql(query).await.unwrap(); + } + + register_parquet_tables(&ctx).await.unwrap(); + (ctx, last_query.to_string()) +} + +#[cfg(test)] +#[derive(Debug)] +pub(crate) struct BuildSideOneTaskEstimator; + +#[cfg(test)] +impl TaskEstimator for BuildSideOneTaskEstimator { + fn task_estimation( + &self, + plan: &Arc, + _: &ConfigOptions, + ) -> Option { + if !plan.children().is_empty() { + return None; + } + let schema = plan.schema(); + let has_min_temp = schema.fields().iter().any(|f| f.name() == "MinTemp"); + let has_max_temp = schema.fields().iter().any(|f| f.name() == "MaxTemp"); + if has_min_temp && !has_max_temp { + Some(TaskEstimation::maximum(1)) + } else { + None + } + } + + fn scale_up_leaf_node( + &self, + _: &Arc, + _: usize, + _: &ConfigOptions, + ) -> Option> { + None + } +} diff --git a/src/test_utils/property_based.rs b/src/test_utils/property_based.rs new file mode 100644 index 00000000..8c7e2974 --- /dev/null +++ b/src/test_utils/property_based.rs @@ -0,0 +1,457 @@ +use arrow::{ + array::{ArrayRef, Float16Array, Float32Array, Float64Array, UInt32Array}, + compute::{SortColumn, concat_batches, lexsort_to_indices}, + record_batch::RecordBatch, +}; +use datafusion::{ + common::{internal_datafusion_err, internal_err}, + error::{DataFusionError, Result}, + physical_expr::LexOrdering, + physical_plan::ExecutionPlan, +}; +use std::sync::Arc; + +/// compares the set of record batches for equality +pub fn compare_result_set( + actual_result: &Result>, + expected_result: &Result>, +) -> Result<()> { + let test_batches = match actual_result.as_ref() { + Ok(batches) => batches, + Err(e) => { + if expected_result.is_ok() { + return internal_err!("expected no error but got: {}", e); + } + return Ok(()); // Both errored, so the query is valid + } + }; + + let compare_batches = match expected_result.as_ref() { + Ok(batches) => batches, + Err(e) => { + if actual_result.is_ok() { + return internal_err!("expected error but got none, error: {}", e); + } + return Ok(()); // Both errored, so the query is valid + } + }; + + records_equal_as_sets(test_batches, compare_batches) + .map_err(|e| internal_datafusion_err!("result sets were not equal: {}", e)) +} + +// Ensures that the plans have the same ordering properties and that the actual result is sorted +// correctly. +pub fn compare_ordering( + actual_physical_plan: Arc, + expected_physical_plan: Arc, + actual_result: &Result>, +) -> Result<()> { + // Only validate if the query succeeded + let test_batches = match actual_result.as_ref() { + Ok(batches) => batches, + Err(_) => return Ok(()), + }; + + let actual_ordering = actual_physical_plan.properties().output_ordering(); + let expected_ordering = expected_physical_plan.properties().output_ordering(); + + if actual_ordering != expected_ordering { + return internal_err!( + "ordering mismatch: expected ordering: {:?}, actual ordering: {:?}", + expected_ordering, + actual_ordering + ); + } + + // If there's no ordering, there's nothing to validate. + let Some(lex_ordering) = actual_ordering else { + return Ok(()); + }; + + // Coalesce all batches into a single batch to check ordering across the entire result set + if !test_batches.is_empty() { + let coalesced_batch = if test_batches.len() == 1 { + test_batches[0].clone() + } else { + concat_batches(&test_batches[0].schema(), test_batches)? + }; + + let is_sorted = is_table_same_after_sort(lex_ordering, &coalesced_batch)?; + if !is_sorted { + return internal_err!( + "ordering validation failed: results are not properly sorted according to expected ordering: {:?}", + lex_ordering + ); + } + } + + Ok(()) +} + +/// Compare two sets of record batches for equality (order-independent) +fn records_equal_as_sets( + left: &[RecordBatch], + right: &[RecordBatch], +) -> Result<(), DataFusionError> { + // First check if total row counts match + let left_rows: usize = left.iter().map(|b| b.num_rows()).sum(); + let right_rows: usize = right.iter().map(|b| b.num_rows()).sum(); + + if left_rows != right_rows { + return internal_err!( + "Row counts differ: left={}, right={}", + left_rows, + right_rows + ); + } + + // Check if schemas match + if !left.is_empty() && !right.is_empty() && left[0].schema() != right[0].schema() { + return internal_err!( + "Schemas differ between result sets\nLeft schema: {:?}\nRight schema: {:?}", + left[0].schema(), + right[0].schema() + ); + } + + detailed_batch_comparison(left, right) +} + +/// Perform detailed comparison of record batches +fn detailed_batch_comparison( + left: &[RecordBatch], + right: &[RecordBatch], +) -> Result<(), DataFusionError> { + // Convert both sides to sets of string representations of rows + let left_rows = batch_rows_to_strings(left); + let right_rows = batch_rows_to_strings(right); + + if left_rows.len() != right_rows.len() { + return internal_err!( + "Row set sizes differ: left={}, right={}", + left_rows.len(), + right_rows.len() + ); + } + + let left_set: std::collections::HashSet<_> = left_rows.iter().collect(); + let right_set: std::collections::HashSet<_> = right_rows.iter().collect(); + + if left_set != right_set { + // Get all differences (not just samples) + let left_only: Vec<_> = left_rows + .iter() + .filter(|row| !right_set.contains(row)) + .collect(); + let right_only: Vec<_> = right_rows + .iter() + .filter(|row| !left_set.contains(row)) + .collect(); + + let mut error_msg = format!( + "Row content differs between result sets\nLeft set size: {}, Right set size: {}", + left_set.len(), + right_set.len() + ); + + if !left_only.is_empty() { + error_msg.push_str(&format!( + "\n\nRows only in left ({} total):", + left_only.len() + )); + for row in left_only { + error_msg.push_str(&format!("\n {row}")); + } + } + + if !right_only.is_empty() { + error_msg.push_str(&format!( + "\n\nRows only in right ({} total):", + right_only.len() + )); + for row in right_only { + error_msg.push_str(&format!("\n {row}")); + } + } + + return internal_err!("{}", error_msg); + } + + Ok(()) +} + +/// Convert record batches to string representations of individual rows +fn batch_rows_to_strings(batches: &[RecordBatch]) -> Vec { + use arrow::util::display::array_value_to_string; + + let mut result = Vec::new(); + + for batch in batches { + for row_idx in 0..batch.num_rows() { + let mut row_values = Vec::new(); + + for col_idx in 0..batch.num_columns() { + let array = batch.column(col_idx); + + if array.is_null(row_idx) { + row_values.push("NULL".to_string()); + } else if let Some(arr) = array.as_any().downcast_ref::() { + row_values.push(format!("{:.1$}", arr.value(row_idx), 2)); + } else if let Some(arr) = array.as_any().downcast_ref::() { + row_values.push(format!("{:.1$}", arr.value(row_idx), 2)); + } else if let Some(arr) = array.as_any().downcast_ref::() { + row_values.push(format!("{:.1$}", arr.value(row_idx), 2)); + } else { + // Use Arrow's deterministic string representation + let value_str = array_value_to_string(array, row_idx) + .unwrap_or_else(|_| "ERROR".to_string()); + row_values.push(value_str); + } + } + + // Join columns with a delimiter for deterministic representation + result.push(row_values.join("|")); + } + } + + result +} + +/// Checks if the table remains unchanged when sorted according to the provided ordering. +/// +/// This implementation is copied from datafusion/core/tests/fuzz_cases/equivalence/utils.rs +pub fn is_table_same_after_sort( + required_ordering: &LexOrdering, + batch: &RecordBatch, +) -> Result { + if required_ordering.is_empty() { + return Ok(true); + } + + let n_row = batch.num_rows(); + if n_row == 0 { + return Ok(true); + } + + // Add a unique column of ascending integers to break ties + let unique_column = Arc::new(UInt32Array::from_iter_values(0..n_row as u32)) as ArrayRef; + + let mut columns = batch.columns().to_vec(); + columns.push(unique_column); + + let mut fields: Vec> = + batch.schema().fields().iter().cloned().collect(); + fields.push(Arc::new(arrow::datatypes::Field::new( + "unique_col", + arrow::datatypes::DataType::UInt32, + false, + ))); + + let schema = Arc::new(arrow::datatypes::Schema::new(fields)); + let batch_with_unique = RecordBatch::try_new(schema, columns)?; + + // Convert to sort columns + let mut sort_columns = Vec::new(); + for sort_expr in required_ordering { + let sort_column = sort_expr.evaluate_to_sort_column(&batch_with_unique)?; + sort_columns.push(sort_column); + } + + // Add the unique column sort + sort_columns.push(SortColumn { + values: batch_with_unique + .column(batch_with_unique.num_columns() - 1) + .clone(), + options: Some(arrow::compute::SortOptions::default()), + }); + + // Get sorted indices + let sorted_indices = lexsort_to_indices(&sort_columns, None)?; + + // Check if indices are in natural order [0, 1, 2, ...] + let expected: Vec = (0..n_row as u32).collect(); + let actual: Vec = sorted_indices.values().iter().cloned().collect(); + Ok(actual == expected) +} + +#[cfg(test)] +mod tests { + use super::*; + + use arrow::array::{Int32Array, StringArray}; + use arrow::datatypes::{DataType, Field, Schema}; + use datafusion::physical_plan::collect; + use datafusion::prelude::SessionContext; + + use std::sync::Arc; + + #[tokio::test] + async fn test_records_equal_as_sets_empty() { + let left: Vec = vec![]; + let right: Vec = vec![]; + assert!(records_equal_as_sets(&left, &right).is_ok()); + } + + #[tokio::test] + async fn test_records_equal_as_sets_with_data() { + // Create test data + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new("name", DataType::Utf8, false), + ])); + + let batch1 = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from(vec![1, 2, 3])), + Arc::new(StringArray::from(vec!["a", "b", "c"])), + ], + ) + .unwrap(); + + let batch2 = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from(vec![3, 1, 2])), // Different order + Arc::new(StringArray::from(vec!["c", "a", "b"])), + ], + ) + .unwrap(); + + let left = vec![batch1]; + let right = vec![batch2]; + + // Should be equal as sets (order independent) + assert!(records_equal_as_sets(&left, &right).is_ok()); + } + + #[tokio::test] + async fn test_records_equal_different_counts() { + let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)])); + + let batch1 = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(Int32Array::from(vec![1, 2, 3]))], + ) + .unwrap(); + + let batch2 = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(Int32Array::from(vec![1, 2]))], // Different count + ) + .unwrap(); + + let left = vec![batch1]; + let right = vec![batch2]; + + assert!(records_equal_as_sets(&left, &right).is_err()); + } + + #[tokio::test] + async fn test_batch_rows_to_strings() { + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new("name", DataType::Utf8, true), // Nullable + ])); + + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from(vec![1, 2])), + Arc::new(StringArray::from(vec![Some("a"), None])), + ], + ) + .unwrap(); + + let strings = batch_rows_to_strings(&[batch]); + + assert_eq!(strings.len(), 2); + // The exact format might vary, but we should have 2 strings representing the rows + assert!(!strings[0].is_empty()); + assert!(!strings[1].is_empty()); + assert!(strings[1].contains("NULL")); // Second row has null value + } + + #[tokio::test] + async fn test_detailed_batch_comparison_identical() { + let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)])); + + let batch = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(Int32Array::from(vec![1, 2, 3]))], + ) + .unwrap(); + + let left = vec![batch.clone()]; + let right = vec![batch]; + + assert!(detailed_batch_comparison(&left, &right).is_ok()); + } + + #[tokio::test] + async fn test_detailed_batch_comparison_different() { + let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)])); + + let batch1 = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(Int32Array::from(vec![1, 2, 3]))], + ) + .unwrap(); + + let batch2 = RecordBatch::try_new( + schema.clone(), + vec![Arc::new(Int32Array::from(vec![1, 2, 4]))], // Different last value + ) + .unwrap(); + + let left = vec![batch1]; + let right = vec![batch2]; + + assert!(detailed_batch_comparison(&left, &right).is_err()); + } + + #[tokio::test] + async fn test_ordering_validation() { + let actual_ctx = SessionContext::new(); + let expected_ctx = SessionContext::new(); + + let schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int32, false), + Field::new("value", DataType::Utf8, false), + ])); + + let batch = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from(vec![1, 2, 3])), + Arc::new(StringArray::from(vec!["c", "b", "a"])), + ], + ) + .unwrap(); + + actual_ctx + .register_batch("test_table", batch.clone()) + .unwrap(); + expected_ctx + .register_batch("test_table", batch.clone()) + .unwrap(); + + // Query which sorted by id should pass. + let ordered_query = "SELECT * FROM test_table ORDER BY value"; + + let df = actual_ctx.sql(ordered_query).await.unwrap(); + let task_ctx = actual_ctx.task_ctx(); + let actual_plan = df.create_physical_plan().await.unwrap(); + let results = collect(actual_plan.clone(), task_ctx).await; + + let df = expected_ctx.sql(ordered_query).await.unwrap(); + let expected_plan = df.create_physical_plan().await.unwrap(); + + assert!(compare_ordering(actual_plan.clone(), expected_plan.clone(), &results).is_ok()); + + // This should fail because the batch is not sorted by value + let result = Ok(vec![batch]); + assert!(compare_ordering(actual_plan.clone(), expected_plan.clone(), &result).is_err()); + } +} diff --git a/src/test_utils/session_context.rs b/src/test_utils/session_context.rs index 276010d2..804d30bb 100644 --- a/src/test_utils/session_context.rs +++ b/src/test_utils/session_context.rs @@ -32,7 +32,7 @@ pub async fn register_temp_parquet_table( let temp_dir = std::env::temp_dir(); let file_id = Uuid::new_v4(); - let temp_file_path = temp_dir.join(format!("{}_{}.parquet", table_name, file_id,)); + let temp_file_path = temp_dir.join(format!("{table_name}_{file_id}.parquet",)); let file = std::fs::File::create(&temp_file_path)?; let schema = batches[0].schema(); diff --git a/src/test_utils/tpcds.rs b/src/test_utils/tpcds.rs new file mode 100644 index 00000000..5b8a5e16 --- /dev/null +++ b/src/test_utils/tpcds.rs @@ -0,0 +1,215 @@ +use crate::test_utils::benchmarks_common; +use arrow::datatypes::{DataType, Field}; +use datafusion::common::internal_err; +use datafusion::error::DataFusionError; +use datafusion::physical_expr::Partitioning; +use datafusion::physical_expr::expressions::{CastColumnExpr, Column}; +use datafusion::physical_expr::projection::ProjectionExpr; +use datafusion::physical_plan::ExecutionPlan; +use datafusion::physical_plan::projection::ProjectionExec; +use datafusion::physical_plan::repartition::RepartitionExec; +use datafusion::prelude::{ParquetReadOptions, SessionContext}; +use parquet::file::properties::WriterProperties; +use std::fs; +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +const URL: &str = "https://github.com/apache/datafusion-benchmarks/archive/refs/heads/main.zip"; + +pub fn get_queries() -> Vec { + benchmarks_common::get_queries("testdata/tpcds/queries") +} + +pub fn get_query(id: &str) -> Result { + benchmarks_common::get_query("testdata/tpcds/queries", id) +} + +/// Downloads the datafusion-benchmarks repository as a zip file +async fn download_benchmarks(dest_path: PathBuf) -> Result<(), Box> { + if dest_path.exists() { + return Ok(()); + } + + // Create directory if it doesn't exist + if let Some(parent) = dest_path.parent() { + fs::create_dir_all(parent)?; + } + + // Download the file + let response = reqwest::get(URL).await?; + let bytes = response.bytes().await?; + + // Write to file + let mut file = fs::File::create(&dest_path)?; + file.write_all(&bytes)?; + + Ok(()) +} + +/// Unzips the downloaded benchmarks zip file +fn unzip_benchmarks( + zip_path: PathBuf, + extract_to: PathBuf, +) -> Result<(), Box> { + if extract_to.exists() { + return Ok(()); + } + + let file = fs::File::open(zip_path)?; + let mut archive = zip::ZipArchive::new(file)?; + + for i in 0..archive.len() { + let mut zip_file = archive.by_index(i)?; + let file_name = zip_file.name(); + if !(file_name.contains("tpcds") && file_name.ends_with(".parquet")) { + continue; + } + let outpath = extract_to.join(zip_file.mangled_name().file_name().unwrap()); + + if let Some(parent) = outpath.parent() { + fs::create_dir_all(parent)?; + } + let mut outfile = fs::File::create(&outpath)?; + std::io::copy(&mut zip_file, &mut outfile)?; + } + + Ok(()) +} + +async fn repartition_parquet_file( + file_path: PathBuf, + dest_path: PathBuf, + partitions: usize, + use_dict_encoding: bool, +) -> Result<(), DataFusionError> { + if !file_path.exists() { + return internal_err!("Path {} does not exist", file_path.display()); + } + let file_name = file_path.file_name().unwrap().to_str().unwrap(); + if !file_name.ends_with(".parquet") { + return internal_err!("Path {} is not parquet", file_path.display()); + } + let table_name = file_name.trim_end_matches(".parquet"); + + if let Ok(dir) = fs::read_dir(&dest_path) { + if dir.count() >= 1 { + return Ok(()); + } + } + + let ctx = SessionContext::new(); + ctx.sql("SET datafusion.execution.target_partitions=1") + .await?; + + ctx.register_parquet( + table_name, + &file_path.to_str().unwrap(), + ParquetReadOptions::default(), + ) + .await?; + + let table = ctx.table(table_name).await?; + let mut plan = table.create_physical_plan().await?; + if use_dict_encoding && table_name == "item" { + let cols = ["i_brand", "i_category", "i_class", "i_color", "i_size"]; + plan = project_cols_as_dict(plan, &cols)?; + } else if use_dict_encoding && table_name == "customer" { + let cols = ["c_salutation"]; + plan = project_cols_as_dict(plan, &cols)?; + } else if use_dict_encoding && table_name == "store" { + let cols = ["s_state", "s_country"]; + plan = project_cols_as_dict(plan, &cols)?; + } + + let plan = RepartitionExec::try_new(plan, Partitioning::RoundRobinBatch(partitions))?; + ctx.write_parquet( + Arc::new(plan), + dest_path.to_str().unwrap(), + Some( + WriterProperties::builder() + .set_dictionary_enabled(true) + .build(), + ), + ) + .await?; + + Ok(()) +} + +fn project_cols_as_dict( + plan: Arc, + cols: &[&str], +) -> Result, DataFusionError> { + let project = ProjectionExec::try_new( + plan.schema() + .fields + .iter() + .enumerate() + .map(|(i, f)| ProjectionExpr { + expr: if cols.contains(&f.name().as_str()) { + Arc::new(CastColumnExpr::new( + Arc::new(Column::new(f.name(), i)), + f.clone(), + Arc::new(Field::new( + f.name(), + DataType::Dictionary( + Box::new(DataType::UInt16), + Box::new(DataType::Utf8), + ), + f.is_nullable(), + )), + None, + )) + } else { + Arc::new(Column::new(f.name(), i)) + }, + alias: f.name().to_string(), + }), + plan, + )?; + Ok(Arc::new(project)) +} + +async fn prepare_tables( + data_path: PathBuf, + dest_path: PathBuf, + partitions: usize, +) -> datafusion::common::Result<()> { + for entry in fs::read_dir(data_path)? { + let entry = entry?; + let file_name = entry.file_name(); + let file_name = file_name.to_str().unwrap(); + if !file_name.ends_with(".parquet") { + continue; + } + let table_name = file_name.trim_end_matches(".parquet"); + // Apply dictionary encoding if requested and materialize to disk + /// Tables that should have dictionary encoding applied for testing + const DICT_ENCODING_TABLES: &[&str] = &["item", "customer", "store"]; + + repartition_parquet_file( + entry.path(), + dest_path.join(table_name), + partitions, + DICT_ENCODING_TABLES.contains(&table_name), + ) + .await?; + } + Ok(()) +} + +pub async fn generate_data( + dir: &Path, + sf: f64, + partitions: usize, +) -> Result<(), Box> { + if sf != 1.0 { + Err("Only scale factor 1.0 is supported for TPC-DS")?; + } + let base_path = dir.parent().unwrap(); + download_benchmarks(base_path.join("main.zip")).await?; + unzip_benchmarks(base_path.join("main.zip"), base_path.join("downloaded"))?; + prepare_tables(base_path.join("downloaded"), dir.to_path_buf(), partitions).await?; + Ok(()) +} diff --git a/src/test_utils/tpch.rs b/src/test_utils/tpch.rs index bef375ff..c61b3400 100644 --- a/src/test_utils/tpch.rs +++ b/src/test_utils/tpch.rs @@ -1,14 +1,8 @@ -use std::sync::Arc; - -use datafusion::{ - arrow::datatypes::{DataType, Field, Schema}, - catalog::{MemTable, TableProvider}, -}; - -use std::fs; - +use crate::test_utils::benchmarks_common; use arrow::record_batch::RecordBatch; +use datafusion::error::DataFusionError; use parquet::{arrow::arrow_writer::ArrowWriter, file::properties::WriterProperties}; +use std::fs; use tpchgen::generators::{ CustomerGenerator, LineItemGenerator, NationGenerator, OrderGenerator, PartGenerator, PartSuppGenerator, RegionGenerator, SupplierGenerator, @@ -18,113 +12,12 @@ use tpchgen_arrow::{ SupplierArrow, }; -pub fn tpch_query_from_dir(queries_dir: &std::path::Path, num: u8) -> String { - let query_path = queries_dir.join(format!("q{num}.sql")); - fs::read_to_string(query_path) - .unwrap_or_else(|_| panic!("Failed to read TPCH query file: q{}.sql", num)) - .trim() - .to_string() -} -pub const NUM_QUERIES: u8 = 22; // number of queries in the TPCH benchmark numbered from 1 to 22 - -pub fn tpch_table(name: &str) -> Arc { - let schema = Arc::new(get_tpch_table_schema(name)); - Arc::new(MemTable::try_new(schema, vec![]).unwrap()) +pub fn get_queries() -> Vec { + benchmarks_common::get_queries("testdata/tpch/queries") } -pub fn get_tpch_table_schema(table: &str) -> Schema { - // note that the schema intentionally uses signed integers so that any generated Parquet - // files can also be used to benchmark tools that only support signed integers, such as - // Apache Spark - - match table { - "part" => Schema::new(vec![ - Field::new("p_partkey", DataType::Int64, false), - Field::new("p_name", DataType::Utf8, false), - Field::new("p_mfgr", DataType::Utf8, false), - Field::new("p_brand", DataType::Utf8, false), - Field::new("p_type", DataType::Utf8, false), - Field::new("p_size", DataType::Int32, false), - Field::new("p_container", DataType::Utf8, false), - Field::new("p_retailprice", DataType::Decimal128(15, 2), false), - Field::new("p_comment", DataType::Utf8, false), - ]), - - "supplier" => Schema::new(vec![ - Field::new("s_suppkey", DataType::Int64, false), - Field::new("s_name", DataType::Utf8, false), - Field::new("s_address", DataType::Utf8, false), - Field::new("s_nationkey", DataType::Int64, false), - Field::new("s_phone", DataType::Utf8, false), - Field::new("s_acctbal", DataType::Decimal128(15, 2), false), - Field::new("s_comment", DataType::Utf8, false), - ]), - - "partsupp" => Schema::new(vec![ - Field::new("ps_partkey", DataType::Int64, false), - Field::new("ps_suppkey", DataType::Int64, false), - Field::new("ps_availqty", DataType::Int32, false), - Field::new("ps_supplycost", DataType::Decimal128(15, 2), false), - Field::new("ps_comment", DataType::Utf8, false), - ]), - - "customer" => Schema::new(vec![ - Field::new("c_custkey", DataType::Int64, false), - Field::new("c_name", DataType::Utf8, false), - Field::new("c_address", DataType::Utf8, false), - Field::new("c_nationkey", DataType::Int64, false), - Field::new("c_phone", DataType::Utf8, false), - Field::new("c_acctbal", DataType::Decimal128(15, 2), false), - Field::new("c_mktsegment", DataType::Utf8, false), - Field::new("c_comment", DataType::Utf8, false), - ]), - - "orders" => Schema::new(vec![ - Field::new("o_orderkey", DataType::Int64, false), - Field::new("o_custkey", DataType::Int64, false), - Field::new("o_orderstatus", DataType::Utf8, false), - Field::new("o_totalprice", DataType::Decimal128(15, 2), false), - Field::new("o_orderdate", DataType::Date32, false), - Field::new("o_orderpriority", DataType::Utf8, false), - Field::new("o_clerk", DataType::Utf8, false), - Field::new("o_shippriority", DataType::Int32, false), - Field::new("o_comment", DataType::Utf8, false), - ]), - - "lineitem" => Schema::new(vec![ - Field::new("l_orderkey", DataType::Int64, false), - Field::new("l_partkey", DataType::Int64, false), - Field::new("l_suppkey", DataType::Int64, false), - Field::new("l_linenumber", DataType::Int32, false), - Field::new("l_quantity", DataType::Decimal128(15, 2), false), - Field::new("l_extendedprice", DataType::Decimal128(15, 2), false), - Field::new("l_discount", DataType::Decimal128(15, 2), false), - Field::new("l_tax", DataType::Decimal128(15, 2), false), - Field::new("l_returnflag", DataType::Utf8, false), - Field::new("l_linestatus", DataType::Utf8, false), - Field::new("l_shipdate", DataType::Date32, false), - Field::new("l_commitdate", DataType::Date32, false), - Field::new("l_receiptdate", DataType::Date32, false), - Field::new("l_shipinstruct", DataType::Utf8, false), - Field::new("l_shipmode", DataType::Utf8, false), - Field::new("l_comment", DataType::Utf8, false), - ]), - - "nation" => Schema::new(vec![ - Field::new("n_nationkey", DataType::Int64, false), - Field::new("n_name", DataType::Utf8, false), - Field::new("n_regionkey", DataType::Int64, false), - Field::new("n_comment", DataType::Utf8, false), - ]), - - "region" => Schema::new(vec![ - Field::new("r_regionkey", DataType::Int64, false), - Field::new("r_name", DataType::Utf8, false), - Field::new("r_comment", DataType::Utf8, false), - ]), - - _ => unimplemented!(), - } +pub fn get_query(id: &str) -> Result { + benchmarks_common::get_query("testdata/tpch/queries", id) } // generate_table creates a parquet file in the data directory from an arrow RecordBatch row @@ -137,7 +30,7 @@ fn generate_table( where A: Iterator, { - let output_path = data_dir.join(format!("{}.parquet", table_name)); + let output_path = data_dir.join(format!("{table_name}.parquet")); if let Some(first_batch) = data_source.next() { let file = fs::File::create(&output_path)?; diff --git a/testdata/clickbench/queries/q0.sql b/testdata/clickbench/queries/q0.sql new file mode 100644 index 00000000..35f2b32e --- /dev/null +++ b/testdata/clickbench/queries/q0.sql @@ -0,0 +1,4 @@ +-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591 + +-- set datafusion.execution.parquet.binary_as_string = true +SELECT COUNT(*) FROM hits; diff --git a/testdata/clickbench/queries/q1.sql b/testdata/clickbench/queries/q1.sql new file mode 100644 index 00000000..0bee959e --- /dev/null +++ b/testdata/clickbench/queries/q1.sql @@ -0,0 +1,4 @@ +-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591 +-- set datafusion.execution.parquet.binary_as_string = true + +SELECT COUNT(*) FROM hits WHERE "AdvEngineID" <> 0; diff --git a/testdata/clickbench/queries/q10.sql b/testdata/clickbench/queries/q10.sql new file mode 100644 index 00000000..0f911480 --- /dev/null +++ b/testdata/clickbench/queries/q10.sql @@ -0,0 +1,4 @@ +-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591 +-- set datafusion.execution.parquet.binary_as_string = true + +SELECT "MobilePhoneModel", COUNT(DISTINCT "UserID") AS u FROM hits WHERE "MobilePhoneModel" <> '' GROUP BY "MobilePhoneModel" ORDER BY u DESC LIMIT 10; diff --git a/testdata/clickbench/queries/q11.sql b/testdata/clickbench/queries/q11.sql new file mode 100644 index 00000000..bed8bb21 --- /dev/null +++ b/testdata/clickbench/queries/q11.sql @@ -0,0 +1,4 @@ +-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591 +-- set datafusion.execution.parquet.binary_as_string = true + +SELECT "MobilePhone", "MobilePhoneModel", COUNT(DISTINCT "UserID") AS u FROM hits WHERE "MobilePhoneModel" <> '' GROUP BY "MobilePhone", "MobilePhoneModel" ORDER BY u DESC LIMIT 10; diff --git a/testdata/clickbench/queries/q12.sql b/testdata/clickbench/queries/q12.sql new file mode 100644 index 00000000..8cf09c00 --- /dev/null +++ b/testdata/clickbench/queries/q12.sql @@ -0,0 +1,4 @@ +-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591 +-- set datafusion.execution.parquet.binary_as_string = true + +SELECT "SearchPhrase", COUNT(*) AS c FROM hits WHERE "SearchPhrase" <> '' GROUP BY "SearchPhrase" ORDER BY c DESC LIMIT 10; diff --git a/testdata/clickbench/queries/q13.sql b/testdata/clickbench/queries/q13.sql new file mode 100644 index 00000000..ef6583c8 --- /dev/null +++ b/testdata/clickbench/queries/q13.sql @@ -0,0 +1,4 @@ +-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591 +-- set datafusion.execution.parquet.binary_as_string = true + +SELECT "SearchPhrase", COUNT(DISTINCT "UserID") AS u FROM hits WHERE "SearchPhrase" <> '' GROUP BY "SearchPhrase" ORDER BY u DESC LIMIT 10; diff --git a/testdata/clickbench/queries/q14.sql b/testdata/clickbench/queries/q14.sql new file mode 100644 index 00000000..dd267146 --- /dev/null +++ b/testdata/clickbench/queries/q14.sql @@ -0,0 +1,4 @@ +-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591 +-- set datafusion.execution.parquet.binary_as_string = true + +SELECT "SearchEngineID", "SearchPhrase", COUNT(*) AS c FROM hits WHERE "SearchPhrase" <> '' GROUP BY "SearchEngineID", "SearchPhrase" ORDER BY c DESC LIMIT 10; diff --git a/testdata/clickbench/queries/q15.sql b/testdata/clickbench/queries/q15.sql new file mode 100644 index 00000000..721d924c --- /dev/null +++ b/testdata/clickbench/queries/q15.sql @@ -0,0 +1,4 @@ +-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591 +-- set datafusion.execution.parquet.binary_as_string = true + +SELECT "UserID", COUNT(*) FROM hits GROUP BY "UserID" ORDER BY COUNT(*) DESC LIMIT 10; diff --git a/testdata/clickbench/queries/q16.sql b/testdata/clickbench/queries/q16.sql new file mode 100644 index 00000000..389725d5 --- /dev/null +++ b/testdata/clickbench/queries/q16.sql @@ -0,0 +1,4 @@ +-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591 +-- set datafusion.execution.parquet.binary_as_string = true + +SELECT "UserID", "SearchPhrase", COUNT(*) FROM hits GROUP BY "UserID", "SearchPhrase" ORDER BY COUNT(*) DESC LIMIT 10; diff --git a/testdata/clickbench/queries/q17.sql b/testdata/clickbench/queries/q17.sql new file mode 100644 index 00000000..be9976a0 --- /dev/null +++ b/testdata/clickbench/queries/q17.sql @@ -0,0 +1,4 @@ +-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591 +-- set datafusion.execution.parquet.binary_as_string = true + +SELECT "UserID", "SearchPhrase", COUNT(*) FROM hits GROUP BY "UserID", "SearchPhrase" LIMIT 10; diff --git a/testdata/clickbench/queries/q18.sql b/testdata/clickbench/queries/q18.sql new file mode 100644 index 00000000..d649f1ed --- /dev/null +++ b/testdata/clickbench/queries/q18.sql @@ -0,0 +1,4 @@ +-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591 +-- set datafusion.execution.parquet.binary_as_string = true + +SELECT "UserID", extract(minute FROM to_timestamp_seconds("EventTime")) AS m, "SearchPhrase", COUNT(*) FROM hits GROUP BY "UserID", m, "SearchPhrase" ORDER BY COUNT(*) DESC LIMIT 10; diff --git a/testdata/clickbench/queries/q19.sql b/testdata/clickbench/queries/q19.sql new file mode 100644 index 00000000..8212a765 --- /dev/null +++ b/testdata/clickbench/queries/q19.sql @@ -0,0 +1,4 @@ +-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591 +-- set datafusion.execution.parquet.binary_as_string = true + +SELECT "UserID" FROM hits WHERE "UserID" = 435090932899640449; diff --git a/testdata/clickbench/queries/q2.sql b/testdata/clickbench/queries/q2.sql new file mode 100644 index 00000000..bcdfad84 --- /dev/null +++ b/testdata/clickbench/queries/q2.sql @@ -0,0 +1,4 @@ +-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591 +-- set datafusion.execution.parquet.binary_as_string = true + +SELECT SUM("AdvEngineID"), COUNT(*), AVG("ResolutionWidth") FROM hits; diff --git a/testdata/clickbench/queries/q20.sql b/testdata/clickbench/queries/q20.sql new file mode 100644 index 00000000..a7e488c2 --- /dev/null +++ b/testdata/clickbench/queries/q20.sql @@ -0,0 +1,4 @@ +-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591 +-- set datafusion.execution.parquet.binary_as_string = true + +SELECT COUNT(*) FROM hits WHERE "URL" LIKE '%google%'; diff --git a/testdata/clickbench/queries/q21.sql b/testdata/clickbench/queries/q21.sql new file mode 100644 index 00000000..35516897 --- /dev/null +++ b/testdata/clickbench/queries/q21.sql @@ -0,0 +1,4 @@ +-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591 +-- set datafusion.execution.parquet.binary_as_string = true + +SELECT "SearchPhrase", MIN("URL"), COUNT(*) AS c FROM hits WHERE "URL" LIKE '%google%' AND "SearchPhrase" <> '' GROUP BY "SearchPhrase" ORDER BY c DESC LIMIT 10; diff --git a/testdata/clickbench/queries/q22.sql b/testdata/clickbench/queries/q22.sql new file mode 100644 index 00000000..d5f696e7 --- /dev/null +++ b/testdata/clickbench/queries/q22.sql @@ -0,0 +1,4 @@ +-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591 +-- set datafusion.execution.parquet.binary_as_string = true + +SELECT "SearchPhrase", MIN("URL"), MIN("Title"), COUNT(*) AS c, COUNT(DISTINCT "UserID") FROM hits WHERE "Title" LIKE '%Google%' AND "URL" NOT LIKE '%.google.%' AND "SearchPhrase" <> '' GROUP BY "SearchPhrase" ORDER BY c DESC LIMIT 10; diff --git a/testdata/clickbench/queries/q23.sql b/testdata/clickbench/queries/q23.sql new file mode 100644 index 00000000..ff399ded --- /dev/null +++ b/testdata/clickbench/queries/q23.sql @@ -0,0 +1,4 @@ +-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591 +-- set datafusion.execution.parquet.binary_as_string = true + +SELECT * FROM hits WHERE "URL" LIKE '%google%' ORDER BY "EventTime" LIMIT 10; diff --git a/testdata/clickbench/queries/q24.sql b/testdata/clickbench/queries/q24.sql new file mode 100644 index 00000000..bc7a3641 --- /dev/null +++ b/testdata/clickbench/queries/q24.sql @@ -0,0 +1,4 @@ +-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591 +-- set datafusion.execution.parquet.binary_as_string = true + +SELECT "SearchPhrase" FROM hits WHERE "SearchPhrase" <> '' ORDER BY "EventTime" LIMIT 10; diff --git a/testdata/clickbench/queries/q25.sql b/testdata/clickbench/queries/q25.sql new file mode 100644 index 00000000..5332e345 --- /dev/null +++ b/testdata/clickbench/queries/q25.sql @@ -0,0 +1,4 @@ +-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591 +-- set datafusion.execution.parquet.binary_as_string = true + +SELECT "SearchPhrase" FROM hits WHERE "SearchPhrase" <> '' ORDER BY "SearchPhrase" LIMIT 10; diff --git a/testdata/clickbench/queries/q26.sql b/testdata/clickbench/queries/q26.sql new file mode 100644 index 00000000..bc1108ae --- /dev/null +++ b/testdata/clickbench/queries/q26.sql @@ -0,0 +1,4 @@ +-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591 +-- set datafusion.execution.parquet.binary_as_string = true + +SELECT "SearchPhrase" FROM hits WHERE "SearchPhrase" <> '' ORDER BY "EventTime", "SearchPhrase" LIMIT 10; diff --git a/testdata/clickbench/queries/q27.sql b/testdata/clickbench/queries/q27.sql new file mode 100644 index 00000000..ba234d34 --- /dev/null +++ b/testdata/clickbench/queries/q27.sql @@ -0,0 +1,4 @@ +-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591 +-- set datafusion.execution.parquet.binary_as_string = true + +SELECT "CounterID", AVG(length("URL")) AS l, COUNT(*) AS c FROM hits WHERE "URL" <> '' GROUP BY "CounterID" HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25; diff --git a/testdata/clickbench/queries/q28.sql b/testdata/clickbench/queries/q28.sql new file mode 100644 index 00000000..6a3bd037 --- /dev/null +++ b/testdata/clickbench/queries/q28.sql @@ -0,0 +1,4 @@ +-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591 +-- set datafusion.execution.parquet.binary_as_string = true + +SELECT REGEXP_REPLACE("Referer", '^https?://(?:www\.)?([^/]+)/.*$', '\1') AS k, AVG(length("Referer")) AS l, COUNT(*) AS c, MIN("Referer") FROM hits WHERE "Referer" <> '' GROUP BY k HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25; diff --git a/testdata/clickbench/queries/q29.sql b/testdata/clickbench/queries/q29.sql new file mode 100644 index 00000000..bca1eb7b --- /dev/null +++ b/testdata/clickbench/queries/q29.sql @@ -0,0 +1,4 @@ +-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591 +-- set datafusion.execution.parquet.binary_as_string = true + +SELECT SUM("ResolutionWidth"), SUM("ResolutionWidth" + 1), SUM("ResolutionWidth" + 2), SUM("ResolutionWidth" + 3), SUM("ResolutionWidth" + 4), SUM("ResolutionWidth" + 5), SUM("ResolutionWidth" + 6), SUM("ResolutionWidth" + 7), SUM("ResolutionWidth" + 8), SUM("ResolutionWidth" + 9), SUM("ResolutionWidth" + 10), SUM("ResolutionWidth" + 11), SUM("ResolutionWidth" + 12), SUM("ResolutionWidth" + 13), SUM("ResolutionWidth" + 14), SUM("ResolutionWidth" + 15), SUM("ResolutionWidth" + 16), SUM("ResolutionWidth" + 17), SUM("ResolutionWidth" + 18), SUM("ResolutionWidth" + 19), SUM("ResolutionWidth" + 20), SUM("ResolutionWidth" + 21), SUM("ResolutionWidth" + 22), SUM("ResolutionWidth" + 23), SUM("ResolutionWidth" + 24), SUM("ResolutionWidth" + 25), SUM("ResolutionWidth" + 26), SUM("ResolutionWidth" + 27), SUM("ResolutionWidth" + 28), SUM("ResolutionWidth" + 29), SUM("ResolutionWidth" + 30), SUM("ResolutionWidth" + 31), SUM("ResolutionWidth" + 32), SUM("ResolutionWidth" + 33), SUM("ResolutionWidth" + 34), SUM("ResolutionWidth" + 35), SUM("ResolutionWidth" + 36), SUM("ResolutionWidth" + 37), SUM("ResolutionWidth" + 38), SUM("ResolutionWidth" + 39), SUM("ResolutionWidth" + 40), SUM("ResolutionWidth" + 41), SUM("ResolutionWidth" + 42), SUM("ResolutionWidth" + 43), SUM("ResolutionWidth" + 44), SUM("ResolutionWidth" + 45), SUM("ResolutionWidth" + 46), SUM("ResolutionWidth" + 47), SUM("ResolutionWidth" + 48), SUM("ResolutionWidth" + 49), SUM("ResolutionWidth" + 50), SUM("ResolutionWidth" + 51), SUM("ResolutionWidth" + 52), SUM("ResolutionWidth" + 53), SUM("ResolutionWidth" + 54), SUM("ResolutionWidth" + 55), SUM("ResolutionWidth" + 56), SUM("ResolutionWidth" + 57), SUM("ResolutionWidth" + 58), SUM("ResolutionWidth" + 59), SUM("ResolutionWidth" + 60), SUM("ResolutionWidth" + 61), SUM("ResolutionWidth" + 62), SUM("ResolutionWidth" + 63), SUM("ResolutionWidth" + 64), SUM("ResolutionWidth" + 65), SUM("ResolutionWidth" + 66), SUM("ResolutionWidth" + 67), SUM("ResolutionWidth" + 68), SUM("ResolutionWidth" + 69), SUM("ResolutionWidth" + 70), SUM("ResolutionWidth" + 71), SUM("ResolutionWidth" + 72), SUM("ResolutionWidth" + 73), SUM("ResolutionWidth" + 74), SUM("ResolutionWidth" + 75), SUM("ResolutionWidth" + 76), SUM("ResolutionWidth" + 77), SUM("ResolutionWidth" + 78), SUM("ResolutionWidth" + 79), SUM("ResolutionWidth" + 80), SUM("ResolutionWidth" + 81), SUM("ResolutionWidth" + 82), SUM("ResolutionWidth" + 83), SUM("ResolutionWidth" + 84), SUM("ResolutionWidth" + 85), SUM("ResolutionWidth" + 86), SUM("ResolutionWidth" + 87), SUM("ResolutionWidth" + 88), SUM("ResolutionWidth" + 89) FROM hits; diff --git a/testdata/clickbench/queries/q3.sql b/testdata/clickbench/queries/q3.sql new file mode 100644 index 00000000..09cdaca7 --- /dev/null +++ b/testdata/clickbench/queries/q3.sql @@ -0,0 +1,4 @@ +-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591 +-- set datafusion.execution.parquet.binary_as_string = true + +SELECT AVG("UserID") FROM hits; diff --git a/testdata/clickbench/queries/q30.sql b/testdata/clickbench/queries/q30.sql new file mode 100644 index 00000000..c0d65792 --- /dev/null +++ b/testdata/clickbench/queries/q30.sql @@ -0,0 +1,4 @@ +-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591 +-- set datafusion.execution.parquet.binary_as_string = true + +SELECT "SearchEngineID", "ClientIP", COUNT(*) AS c, SUM("IsRefresh"), AVG("ResolutionWidth") FROM hits WHERE "SearchPhrase" <> '' GROUP BY "SearchEngineID", "ClientIP" ORDER BY c DESC LIMIT 10; diff --git a/testdata/clickbench/queries/q31.sql b/testdata/clickbench/queries/q31.sql new file mode 100644 index 00000000..76ab3622 --- /dev/null +++ b/testdata/clickbench/queries/q31.sql @@ -0,0 +1,4 @@ +-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591 +-- set datafusion.execution.parquet.binary_as_string = true + +SELECT "WatchID", "ClientIP", COUNT(*) AS c, SUM("IsRefresh"), AVG("ResolutionWidth") FROM hits WHERE "SearchPhrase" <> '' GROUP BY "WatchID", "ClientIP" ORDER BY c DESC LIMIT 10; diff --git a/testdata/clickbench/queries/q32.sql b/testdata/clickbench/queries/q32.sql new file mode 100644 index 00000000..88f1e4ce --- /dev/null +++ b/testdata/clickbench/queries/q32.sql @@ -0,0 +1,4 @@ +-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591 +-- set datafusion.execution.parquet.binary_as_string = true + +SELECT "WatchID", "ClientIP", COUNT(*) AS c, SUM("IsRefresh"), AVG("ResolutionWidth") FROM hits GROUP BY "WatchID", "ClientIP" ORDER BY c DESC LIMIT 10; diff --git a/testdata/clickbench/queries/q33.sql b/testdata/clickbench/queries/q33.sql new file mode 100644 index 00000000..3740503b --- /dev/null +++ b/testdata/clickbench/queries/q33.sql @@ -0,0 +1,4 @@ +-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591 +-- set datafusion.execution.parquet.binary_as_string = true + +SELECT "URL", COUNT(*) AS c FROM hits GROUP BY "URL" ORDER BY c DESC LIMIT 10; diff --git a/testdata/clickbench/queries/q34.sql b/testdata/clickbench/queries/q34.sql new file mode 100644 index 00000000..fdb7edbb --- /dev/null +++ b/testdata/clickbench/queries/q34.sql @@ -0,0 +1,4 @@ +-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591 +-- set datafusion.execution.parquet.binary_as_string = true + +SELECT 1, "URL", COUNT(*) AS c FROM hits GROUP BY 1, "URL" ORDER BY c DESC LIMIT 10; diff --git a/testdata/clickbench/queries/q35.sql b/testdata/clickbench/queries/q35.sql new file mode 100644 index 00000000..de7e2256 --- /dev/null +++ b/testdata/clickbench/queries/q35.sql @@ -0,0 +1,4 @@ +-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591 +-- set datafusion.execution.parquet.binary_as_string = true + +SELECT "ClientIP", "ClientIP" - 1, "ClientIP" - 2, "ClientIP" - 3, COUNT(*) AS c FROM hits GROUP BY "ClientIP", "ClientIP" - 1, "ClientIP" - 2, "ClientIP" - 3 ORDER BY c DESC LIMIT 10; diff --git a/testdata/clickbench/queries/q36.sql b/testdata/clickbench/queries/q36.sql new file mode 100644 index 00000000..81b1199b --- /dev/null +++ b/testdata/clickbench/queries/q36.sql @@ -0,0 +1,4 @@ +-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591 +-- set datafusion.execution.parquet.binary_as_string = true + +SELECT "URL", COUNT(*) AS PageViews FROM hits WHERE "CounterID" = 62 AND "EventDate" >= '2013-07-01' AND "EventDate" <= '2013-07-31' AND "DontCountHits" = 0 AND "IsRefresh" = 0 AND "URL" <> '' GROUP BY "URL" ORDER BY PageViews DESC LIMIT 10; diff --git a/testdata/clickbench/queries/q37.sql b/testdata/clickbench/queries/q37.sql new file mode 100644 index 00000000..fa4b85ff --- /dev/null +++ b/testdata/clickbench/queries/q37.sql @@ -0,0 +1,4 @@ +-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591 +-- set datafusion.execution.parquet.binary_as_string = true + +SELECT "Title", COUNT(*) AS PageViews FROM hits WHERE "CounterID" = 62 AND "EventDate" >= '2013-07-01' AND "EventDate" <= '2013-07-31' AND "DontCountHits" = 0 AND "IsRefresh" = 0 AND "Title" <> '' GROUP BY "Title" ORDER BY PageViews DESC LIMIT 10; diff --git a/testdata/clickbench/queries/q38.sql b/testdata/clickbench/queries/q38.sql new file mode 100644 index 00000000..18fafab6 --- /dev/null +++ b/testdata/clickbench/queries/q38.sql @@ -0,0 +1,4 @@ +-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591 +-- set datafusion.execution.parquet.binary_as_string = true + +SELECT "URL", COUNT(*) AS PageViews FROM hits WHERE "CounterID" = 62 AND "EventDate" >= '2013-07-01' AND "EventDate" <= '2013-07-31' AND "IsRefresh" = 0 AND "IsLink" <> 0 AND "IsDownload" = 0 GROUP BY "URL" ORDER BY PageViews DESC LIMIT 10 OFFSET 1000; diff --git a/testdata/clickbench/queries/q39.sql b/testdata/clickbench/queries/q39.sql new file mode 100644 index 00000000..306f0caa --- /dev/null +++ b/testdata/clickbench/queries/q39.sql @@ -0,0 +1,4 @@ +-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591 +-- set datafusion.execution.parquet.binary_as_string = true + +SELECT "TraficSourceID", "SearchEngineID", "AdvEngineID", CASE WHEN ("SearchEngineID" = 0 AND "AdvEngineID" = 0) THEN "Referer" ELSE '' END AS Src, "URL" AS Dst, COUNT(*) AS PageViews FROM hits WHERE "CounterID" = 62 AND "EventDate" >= '2013-07-01' AND "EventDate" <= '2013-07-31' AND "IsRefresh" = 0 GROUP BY "TraficSourceID", "SearchEngineID", "AdvEngineID", Src, Dst ORDER BY PageViews DESC LIMIT 10 OFFSET 1000; diff --git a/testdata/clickbench/queries/q4.sql b/testdata/clickbench/queries/q4.sql new file mode 100644 index 00000000..d89ca78c --- /dev/null +++ b/testdata/clickbench/queries/q4.sql @@ -0,0 +1,4 @@ +-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591 +-- set datafusion.execution.parquet.binary_as_string = true + +SELECT COUNT(DISTINCT "UserID") FROM hits; diff --git a/testdata/clickbench/queries/q40.sql b/testdata/clickbench/queries/q40.sql new file mode 100644 index 00000000..e9d27f59 --- /dev/null +++ b/testdata/clickbench/queries/q40.sql @@ -0,0 +1,4 @@ +-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591 +-- set datafusion.execution.parquet.binary_as_string = true + +SELECT "URLHash", "EventDate", COUNT(*) AS PageViews FROM hits WHERE "CounterID" = 62 AND "EventDate" >= '2013-07-01' AND "EventDate" <= '2013-07-31' AND "IsRefresh" = 0 AND "TraficSourceID" IN (-1, 6) AND "RefererHash" = 3594120000172545465 GROUP BY "URLHash", "EventDate" ORDER BY PageViews DESC LIMIT 10 OFFSET 100; diff --git a/testdata/clickbench/queries/q41.sql b/testdata/clickbench/queries/q41.sql new file mode 100644 index 00000000..0e067e2d --- /dev/null +++ b/testdata/clickbench/queries/q41.sql @@ -0,0 +1,3 @@ +-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591 +-- set datafusion.execution.parquet.binary_as_string = true +SELECT "WindowClientWidth", "WindowClientHeight", COUNT(*) AS PageViews FROM hits WHERE "CounterID" = 62 AND "EventDate" >= '2013-07-01' AND "EventDate" <= '2013-07-31' AND "IsRefresh" = 0 AND "DontCountHits" = 0 AND "URLHash" = 2868770270353813622 GROUP BY "WindowClientWidth", "WindowClientHeight" ORDER BY PageViews DESC LIMIT 10 OFFSET 10000; diff --git a/testdata/clickbench/queries/q42.sql b/testdata/clickbench/queries/q42.sql new file mode 100644 index 00000000..111cc1d3 --- /dev/null +++ b/testdata/clickbench/queries/q42.sql @@ -0,0 +1,4 @@ +-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591 +-- set datafusion.execution.parquet.binary_as_string = true + +SELECT DATE_TRUNC('minute', to_timestamp_seconds("EventTime")) AS M, COUNT(*) AS PageViews FROM hits WHERE "CounterID" = 62 AND "EventDate" >= '2013-07-14' AND "EventDate" <= '2013-07-15' AND "IsRefresh" = 0 AND "DontCountHits" = 0 GROUP BY DATE_TRUNC('minute', to_timestamp_seconds("EventTime")) ORDER BY DATE_TRUNC('minute', M) LIMIT 10 OFFSET 1000; diff --git a/testdata/clickbench/queries/q5.sql b/testdata/clickbench/queries/q5.sql new file mode 100644 index 00000000..d371cfb6 --- /dev/null +++ b/testdata/clickbench/queries/q5.sql @@ -0,0 +1,4 @@ +-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591 +-- set datafusion.execution.parquet.binary_as_string = true + +SELECT COUNT(DISTINCT "SearchPhrase") FROM hits; diff --git a/testdata/clickbench/queries/q6.sql b/testdata/clickbench/queries/q6.sql new file mode 100644 index 00000000..5b4e896a --- /dev/null +++ b/testdata/clickbench/queries/q6.sql @@ -0,0 +1,4 @@ +-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591 +-- set datafusion.execution.parquet.binary_as_string = true + +SELECT MIN("EventDate"), MAX("EventDate") FROM hits; diff --git a/testdata/clickbench/queries/q7.sql b/testdata/clickbench/queries/q7.sql new file mode 100644 index 00000000..afffcb13 --- /dev/null +++ b/testdata/clickbench/queries/q7.sql @@ -0,0 +1,4 @@ +-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591 +-- set datafusion.execution.parquet.binary_as_string = true + +SELECT "AdvEngineID", COUNT(*) FROM hits WHERE "AdvEngineID" <> 0 GROUP BY "AdvEngineID" ORDER BY COUNT(*) DESC; diff --git a/testdata/clickbench/queries/q8.sql b/testdata/clickbench/queries/q8.sql new file mode 100644 index 00000000..097880a9 --- /dev/null +++ b/testdata/clickbench/queries/q8.sql @@ -0,0 +1,4 @@ +-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591 +-- set datafusion.execution.parquet.binary_as_string = true + +SELECT "RegionID", COUNT(DISTINCT "UserID") AS u FROM hits GROUP BY "RegionID" ORDER BY u DESC LIMIT 10; diff --git a/testdata/clickbench/queries/q9.sql b/testdata/clickbench/queries/q9.sql new file mode 100644 index 00000000..cb1b79bf --- /dev/null +++ b/testdata/clickbench/queries/q9.sql @@ -0,0 +1,4 @@ +-- Must set for ClickBench hits_partitioned dataset. See https://github.com/apache/datafusion/issues/16591 +-- set datafusion.execution.parquet.binary_as_string = true + +SELECT "RegionID", SUM("AdvEngineID"), COUNT(*) AS c, AVG("ResolutionWidth"), COUNT(DISTINCT "UserID") FROM hits GROUP BY "RegionID" ORDER BY c DESC LIMIT 10; diff --git a/testdata/join/csv/dim/d_dkey=A/data0.csv b/testdata/join/csv/dim/d_dkey=A/data0.csv new file mode 100644 index 00000000..60818a27 --- /dev/null +++ b/testdata/join/csv/dim/d_dkey=A/data0.csv @@ -0,0 +1,3 @@ +env,service,host +dev,log,host-y + diff --git a/testdata/join/csv/dim/d_dkey=B/data0.csv b/testdata/join/csv/dim/d_dkey=B/data0.csv new file mode 100644 index 00000000..6a883d21 --- /dev/null +++ b/testdata/join/csv/dim/d_dkey=B/data0.csv @@ -0,0 +1,3 @@ +env,service,host +prod,log,host-x + diff --git a/testdata/join/csv/dim/d_dkey=C/data0.csv b/testdata/join/csv/dim/d_dkey=C/data0.csv new file mode 100644 index 00000000..0c8f0cfd --- /dev/null +++ b/testdata/join/csv/dim/d_dkey=C/data0.csv @@ -0,0 +1,3 @@ +env,service,host +dev,trace,host-z + diff --git a/testdata/join/csv/dim/d_dkey=D/data0.csv b/testdata/join/csv/dim/d_dkey=D/data0.csv new file mode 100644 index 00000000..9ea19d80 --- /dev/null +++ b/testdata/join/csv/dim/d_dkey=D/data0.csv @@ -0,0 +1,3 @@ +env,service,host +prod,trace,host-x + diff --git a/testdata/join/csv/fact/f_dkey=A/data0.csv b/testdata/join/csv/fact/f_dkey=A/data0.csv new file mode 100644 index 00000000..ed765e7d --- /dev/null +++ b/testdata/join/csv/fact/f_dkey=A/data0.csv @@ -0,0 +1,9 @@ +timestamp,value +2023-01-01T09:00:00,95.5 +2023-01-01T09:00:10,102.3 +2023-01-01T09:00:20,98.7 +2023-01-01T09:12:20,105.1 +2023-01-01T09:12:30,100.0 +2023-01-01T09:12:40,150.0 +2023-01-01T09:12:50,120.8 + diff --git a/testdata/join/csv/fact/f_dkey=B/data0.csv b/testdata/join/csv/fact/f_dkey=B/data0.csv new file mode 100644 index 00000000..b2d4c205 --- /dev/null +++ b/testdata/join/csv/fact/f_dkey=B/data0.csv @@ -0,0 +1,9 @@ +timestamp,value +2023-01-01T09:00:00,75.2 +2023-01-01T09:00:10,82.4 +2023-01-01T09:00:20,78.9 +2023-01-01T09:00:30,85.6 +2023-01-01T09:12:30,80.0 +2023-01-01T09:12:40,120.0 +2023-01-01T09:12:50,92.3 + diff --git a/testdata/join/csv/fact/f_dkey=C/data0.csv b/testdata/join/csv/fact/f_dkey=C/data0.csv new file mode 100644 index 00000000..0f21f8fa --- /dev/null +++ b/testdata/join/csv/fact/f_dkey=C/data0.csv @@ -0,0 +1,9 @@ +timestamp,value +2023-01-01T10:00:00,310.5 +2023-01-01T10:00:10,225.7 +2023-01-01T10:00:20,380.2 +2023-01-01T10:00:30,205.8 +2023-01-01T10:00:40,350.0 +2023-01-01T10:12:40,200.0 +2023-01-01T10:12:50,205.4 + diff --git a/testdata/join/csv/fact/f_dkey=D/data0.csv b/testdata/join/csv/fact/f_dkey=D/data0.csv new file mode 100644 index 00000000..c15bfc1b --- /dev/null +++ b/testdata/join/csv/fact/f_dkey=D/data0.csv @@ -0,0 +1,5 @@ +timestamp,value +2023-01-01T10:00:00,24.8 +2023-01-01T10:00:10,72.1 +2023-01-01T10:00:20,42.5 + diff --git a/testdata/join/generate_parquet_from_csv.sql b/testdata/join/generate_parquet_from_csv.sql new file mode 100644 index 00000000..e4af6065 --- /dev/null +++ b/testdata/join/generate_parquet_from_csv.sql @@ -0,0 +1,36 @@ +-- datafusion-cli -f testdata/join/generate_parquet_from_csv.sql + +-- Generate parquet dim files from csv files. +COPY (SELECT * FROM "testdata/join/csv/dim/d_dkey=A/data0.csv") +TO "testdata/join/parquet/dim/d_dkey=A/data0.parquet" +STORED AS PARQUET; + +COPY (SELECT * FROM "testdata/join/csv/dim/d_dkey=B/data0.csv") +TO "testdata/join/parquet/dim/d_dkey=B/data0.parquet" +STORED AS PARQUET; + +COPY (SELECT * FROM "testdata/join/csv/dim/d_dkey=C/data0.csv") +TO "testdata/join/parquet/dim/d_dkey=C/data0.parquet" +STORED AS PARQUET; + +COPY (SELECT * FROM "testdata/join/csv/dim/d_dkey=D/data0.csv") +TO "testdata/join/parquet/dim/d_dkey=D/data0.parquet" +STORED AS PARQUET; + +-- Generate parquet fact files from csv files. +COPY (SELECT * FROM "testdata/join/csv/fact/f_dkey=A/data0.csv") +TO "testdata/join/parquet/fact/f_dkey=A/data0.parquet" +STORED AS PARQUET; + +COPY (SELECT * FROM "testdata/join/csv/fact/f_dkey=B/data0.csv") +TO "testdata/join/parquet/fact/f_dkey=B/data0.parquet" +STORED AS PARQUET; + +COPY (SELECT * FROM "testdata/join/csv/fact/f_dkey=C/data0.csv") +TO "testdata/join/parquet/fact/f_dkey=C/data0.parquet" +STORED AS PARQUET; + +COPY (SELECT * FROM "testdata/join/csv/fact/f_dkey=D/data0.csv") +TO "testdata/join/parquet/fact/f_dkey=D/data0.parquet" +STORED AS PARQUET; + diff --git a/testdata/join/parquet/dim/d_dkey=A/data0.parquet b/testdata/join/parquet/dim/d_dkey=A/data0.parquet new file mode 100644 index 00000000..6865b9d2 --- /dev/null +++ b/testdata/join/parquet/dim/d_dkey=A/data0.parquet @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a8752b1efcfb3f541d0481e397fb0920060d1b324972228bf38f2c6547838374 +size 1011 diff --git a/testdata/join/parquet/dim/d_dkey=B/data0.parquet b/testdata/join/parquet/dim/d_dkey=B/data0.parquet new file mode 100644 index 00000000..5d114c63 --- /dev/null +++ b/testdata/join/parquet/dim/d_dkey=B/data0.parquet @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0788861c3959ba7ac5f14d3cfbb9ecfa64fd2e823ea08d23a9b7da65255794d3 +size 1016 diff --git a/testdata/join/parquet/dim/d_dkey=C/data0.parquet b/testdata/join/parquet/dim/d_dkey=C/data0.parquet new file mode 100644 index 00000000..f1451fac --- /dev/null +++ b/testdata/join/parquet/dim/d_dkey=C/data0.parquet @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0cf9e37dc5368a9de0badc5a6392e949e942d96d9e25ddd98217544e2629995c +size 1021 diff --git a/testdata/join/parquet/dim/d_dkey=D/data0.parquet b/testdata/join/parquet/dim/d_dkey=D/data0.parquet new file mode 100644 index 00000000..ecfeae2a --- /dev/null +++ b/testdata/join/parquet/dim/d_dkey=D/data0.parquet @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b4531e8da8141539997e09816af7e1c6090c90a9bcf36ad308973b6029608703 +size 1026 diff --git a/testdata/join/parquet/fact/f_dkey=A/data0.parquet b/testdata/join/parquet/fact/f_dkey=A/data0.parquet new file mode 100644 index 00000000..9bca1622 --- /dev/null +++ b/testdata/join/parquet/fact/f_dkey=A/data0.parquet @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0afda79871479819092957dbb75d6ed17d6953c059d1fea4cfb9df2aa74a5b3e +size 929 diff --git a/testdata/join/parquet/fact/f_dkey=B/data0.parquet b/testdata/join/parquet/fact/f_dkey=B/data0.parquet new file mode 100644 index 00000000..0aed0d21 --- /dev/null +++ b/testdata/join/parquet/fact/f_dkey=B/data0.parquet @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5c1df83afcb61abd11ec6f3e3a75947b7d79cdee64104474234aea1c26c2e553 +size 925 diff --git a/testdata/join/parquet/fact/f_dkey=C/data0.parquet b/testdata/join/parquet/fact/f_dkey=C/data0.parquet new file mode 100644 index 00000000..eccbdf4b --- /dev/null +++ b/testdata/join/parquet/fact/f_dkey=C/data0.parquet @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:58669c93aba8f140ff95039a53d4bdb58f87f8cf7d2a0f95a3454ae017ccb1a7 +size 934 diff --git a/testdata/join/parquet/fact/f_dkey=D/data0.parquet b/testdata/join/parquet/fact/f_dkey=D/data0.parquet new file mode 100644 index 00000000..c2e2f417 --- /dev/null +++ b/testdata/join/parquet/fact/f_dkey=D/data0.parquet @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:264bf30baa3997fa9cd104e58f42929110d78cc6f8cd7f4b4e9328406c43e429 +size 895 diff --git a/testdata/tpcds/README.md b/testdata/tpcds/README.md new file mode 100644 index 00000000..56c5a0eb --- /dev/null +++ b/testdata/tpcds/README.md @@ -0,0 +1,6 @@ +This directory contains 99 TPC-DS queries from https://github.com/duckdb/duckdb + +## Modifications for DataFusion Compatibility + + - Queries 47 and 57 were modified to add explicit ORDER BY d_moy to avg() window function. DataFusion requires explicit ordering in window functions with PARTITION BY for deterministic results. + - Query 72 was modified to support date functions in datafusion diff --git a/testdata/tpcds/queries/q1.sql b/testdata/tpcds/queries/q1.sql new file mode 100644 index 00000000..5140b8b9 --- /dev/null +++ b/testdata/tpcds/queries/q1.sql @@ -0,0 +1,24 @@ +WITH customer_total_return AS + (SELECT sr_customer_sk AS ctr_customer_sk, + sr_store_sk AS ctr_store_sk, + sum(sr_return_amt) AS ctr_total_return + FROM store_returns, + date_dim + WHERE sr_returned_date_sk = d_date_sk + AND d_year = 2000 + GROUP BY sr_customer_sk, + sr_store_sk) +SELECT c_customer_id +FROM customer_total_return ctr1, + store, + customer +WHERE ctr1.ctr_total_return > + (SELECT avg(ctr_total_return)*1.2 + FROM customer_total_return ctr2 + WHERE ctr1.ctr_store_sk = ctr2.ctr_store_sk) + AND s_store_sk = ctr1.ctr_store_sk + AND s_state = 'TN' + AND ctr1.ctr_customer_sk = c_customer_sk +ORDER BY c_customer_id +LIMIT 100; + diff --git a/testdata/tpcds/queries/q10.sql b/testdata/tpcds/queries/q10.sql new file mode 100644 index 00000000..d2a23c27 --- /dev/null +++ b/testdata/tpcds/queries/q10.sql @@ -0,0 +1,66 @@ +SELECT cd_gender, + cd_marital_status, + cd_education_status, + count(*) cnt1, + cd_purchase_estimate, + count(*) cnt2, + cd_credit_rating, + count(*) cnt3, + cd_dep_count, + count(*) cnt4, + cd_dep_employed_count, + count(*) cnt5, + cd_dep_college_count, + count(*) cnt6 +FROM customer c, + customer_address ca, + customer_demographics +WHERE c.c_current_addr_sk = ca.ca_address_sk + AND ca_county IN ('Rush County', + 'Toole County', + 'Jefferson County', + 'Dona Ana County', + 'La Porte County') + AND cd_demo_sk = c.c_current_cdemo_sk + AND EXISTS + (SELECT * + FROM store_sales, + date_dim + WHERE c.c_customer_sk = ss_customer_sk + AND ss_sold_date_sk = d_date_sk + AND d_year = 2002 + AND d_moy BETWEEN 1 AND 1+3) + AND (EXISTS + (SELECT * + FROM web_sales, + date_dim + WHERE c.c_customer_sk = ws_bill_customer_sk + AND ws_sold_date_sk = d_date_sk + AND d_year = 2002 + AND d_moy BETWEEN 1 AND 1+3) + OR EXISTS + (SELECT * + FROM catalog_sales, + date_dim + WHERE c.c_customer_sk = cs_ship_customer_sk + AND cs_sold_date_sk = d_date_sk + AND d_year = 2002 + AND d_moy BETWEEN 1 AND 1+3)) +GROUP BY cd_gender, + cd_marital_status, + cd_education_status, + cd_purchase_estimate, + cd_credit_rating, + cd_dep_count, + cd_dep_employed_count, + cd_dep_college_count +ORDER BY cd_gender, + cd_marital_status, + cd_education_status, + cd_purchase_estimate, + cd_credit_rating, + cd_dep_count, + cd_dep_employed_count, + cd_dep_college_count +LIMIT 100; + diff --git a/testdata/tpcds/queries/q11.sql b/testdata/tpcds/queries/q11.sql new file mode 100644 index 00000000..fe5c69a6 --- /dev/null +++ b/testdata/tpcds/queries/q11.sql @@ -0,0 +1,81 @@ +WITH year_total AS + (SELECT c_customer_id customer_id, + c_first_name customer_first_name, + c_last_name customer_last_name, + c_preferred_cust_flag customer_preferred_cust_flag, + c_birth_country customer_birth_country, + c_login customer_login, + c_email_address customer_email_address, + d_year dyear, + sum(ss_ext_list_price-ss_ext_discount_amt) year_total, + 's' sale_type + FROM customer, + store_sales, + date_dim + WHERE c_customer_sk = ss_customer_sk + AND ss_sold_date_sk = d_date_sk + GROUP BY c_customer_id, + c_first_name, + c_last_name, + c_preferred_cust_flag, + c_birth_country, + c_login, + c_email_address, + d_year + UNION ALL SELECT c_customer_id customer_id, + c_first_name customer_first_name, + c_last_name customer_last_name, + c_preferred_cust_flag customer_preferred_cust_flag, + c_birth_country customer_birth_country, + c_login customer_login, + c_email_address customer_email_address, + d_year dyear, + sum(ws_ext_list_price-ws_ext_discount_amt) year_total, + 'w' sale_type + FROM customer, + web_sales, + date_dim + WHERE c_customer_sk = ws_bill_customer_sk + AND ws_sold_date_sk = d_date_sk + GROUP BY c_customer_id, + c_first_name, + c_last_name, + c_preferred_cust_flag, + c_birth_country, + c_login, + c_email_address, + d_year) +SELECT t_s_secyear.customer_id, + t_s_secyear.customer_first_name, + t_s_secyear.customer_last_name, + t_s_secyear.customer_preferred_cust_flag +FROM year_total t_s_firstyear, + year_total t_s_secyear, + year_total t_w_firstyear, + year_total t_w_secyear +WHERE t_s_secyear.customer_id = t_s_firstyear.customer_id + AND t_s_firstyear.customer_id = t_w_secyear.customer_id + AND t_s_firstyear.customer_id = t_w_firstyear.customer_id + AND t_s_firstyear.sale_type = 's' + AND t_w_firstyear.sale_type = 'w' + AND t_s_secyear.sale_type = 's' + AND t_w_secyear.sale_type = 'w' + AND t_s_firstyear.dyear = 2001 + AND t_s_secyear.dyear = 2001+1 + AND t_w_firstyear.dyear = 2001 + AND t_w_secyear.dyear = 2001+1 + AND t_s_firstyear.year_total > 0 + AND t_w_firstyear.year_total > 0 + AND CASE + WHEN t_w_firstyear.year_total > 0 THEN (t_w_secyear.year_total*1.0000) / t_w_firstyear.year_total + ELSE 0.0 + END > CASE + WHEN t_s_firstyear.year_total > 0 THEN (t_s_secyear.year_total*1.0000) / t_s_firstyear.year_total + ELSE 0.0 + END +ORDER BY t_s_secyear.customer_id NULLS FIRST, + t_s_secyear.customer_first_name NULLS FIRST, + t_s_secyear.customer_last_name NULLS FIRST, + t_s_secyear.customer_preferred_cust_flag NULLS FIRST +LIMIT 100; + diff --git a/testdata/tpcds/queries/q12.sql b/testdata/tpcds/queries/q12.sql new file mode 100644 index 00000000..3af87d9c --- /dev/null +++ b/testdata/tpcds/queries/q12.sql @@ -0,0 +1,28 @@ +SELECT i_item_id, + i_item_desc, + i_category, + i_class, + i_current_price, + sum(ws_ext_sales_price) AS itemrevenue, + sum(ws_ext_sales_price)*100.0000/sum(sum(ws_ext_sales_price)) OVER (PARTITION BY i_class) AS revenueratio +FROM web_sales, + item, + date_dim +WHERE ws_item_sk = i_item_sk + AND i_category IN ('Sports', + 'Books', + 'Home') + AND ws_sold_date_sk = d_date_sk + AND d_date BETWEEN cast('1999-02-22' AS date) AND cast('1999-03-24' AS date) +GROUP BY i_item_id, + i_item_desc, + i_category, + i_class, + i_current_price +ORDER BY i_category, + i_class, + i_item_id, + i_item_desc, + revenueratio +LIMIT 100; + diff --git a/testdata/tpcds/queries/q13.sql b/testdata/tpcds/queries/q13.sql new file mode 100644 index 00000000..eba0f320 --- /dev/null +++ b/testdata/tpcds/queries/q13.sql @@ -0,0 +1,43 @@ + +SELECT avg(ss_quantity) avg1, + avg(ss_ext_sales_price) avg2, + avg(ss_ext_wholesale_cost) avg3, + sum(ss_ext_wholesale_cost) +FROM store_sales , + store , + customer_demographics , + household_demographics , + customer_address , + date_dim +WHERE s_store_sk = ss_store_sk + AND ss_sold_date_sk = d_date_sk + AND d_year = 2001 and((ss_hdemo_sk=hd_demo_sk + AND cd_demo_sk = ss_cdemo_sk + AND cd_marital_status = 'M' + AND cd_education_status = 'Advanced Degree' + AND ss_sales_price BETWEEN 100.00 AND 150.00 + AND hd_dep_count = 3) + OR (ss_hdemo_sk=hd_demo_sk + AND cd_demo_sk = ss_cdemo_sk + AND cd_marital_status = 'S' + AND cd_education_status = 'College' + AND ss_sales_price BETWEEN 50.00 AND 100.00 + AND hd_dep_count = 1 ) + OR (ss_hdemo_sk=hd_demo_sk + AND cd_demo_sk = ss_cdemo_sk + AND cd_marital_status = 'W' + AND cd_education_status = '2 yr Degree' + AND ss_sales_price BETWEEN 150.00 AND 200.00 + AND hd_dep_count = 1)) and((ss_addr_sk = ca_address_sk + AND ca_country = 'United States' + AND ca_state IN ('TX', 'OH', 'TX') + AND ss_net_profit BETWEEN 100 AND 200) + OR (ss_addr_sk = ca_address_sk + AND ca_country = 'United States' + AND ca_state IN ('OR', 'NM', 'KY') + AND ss_net_profit BETWEEN 150 AND 300) + OR (ss_addr_sk = ca_address_sk + AND ca_country = 'United States' + AND ca_state IN ('VA', 'TX', 'MS') + AND ss_net_profit BETWEEN 50 AND 250)) ; + diff --git a/testdata/tpcds/queries/q14.sql b/testdata/tpcds/queries/q14.sql new file mode 100644 index 00000000..0a4667d9 --- /dev/null +++ b/testdata/tpcds/queries/q14.sql @@ -0,0 +1,134 @@ +WITH cross_items AS + (SELECT i_item_sk ss_item_sk + FROM item, + (SELECT iss.i_brand_id brand_id, + iss.i_class_id class_id, + iss.i_category_id category_id + FROM store_sales, + item iss, + date_dim d1 + WHERE ss_item_sk = iss.i_item_sk + AND ss_sold_date_sk = d1.d_date_sk + AND d1.d_year BETWEEN 1999 AND 1999 + 2 INTERSECT + SELECT ics.i_brand_id, + ics.i_class_id, + ics.i_category_id + FROM catalog_sales, + item ics, + date_dim d2 WHERE cs_item_sk = ics.i_item_sk + AND cs_sold_date_sk = d2.d_date_sk + AND d2.d_year BETWEEN 1999 AND 1999 + 2 INTERSECT + SELECT iws.i_brand_id, + iws.i_class_id, + iws.i_category_id + FROM web_sales, + item iws, + date_dim d3 WHERE ws_item_sk = iws.i_item_sk + AND ws_sold_date_sk = d3.d_date_sk + AND d3.d_year BETWEEN 1999 AND 1999 + 2) sq1 + WHERE i_brand_id = brand_id + AND i_class_id = class_id + AND i_category_id = category_id ), + avg_sales AS + (SELECT avg(quantity*list_price) average_sales + FROM + (SELECT ss_quantity quantity, + ss_list_price list_price + FROM store_sales, + date_dim + WHERE ss_sold_date_sk = d_date_sk + AND d_year BETWEEN 1999 AND 1999 + 2 + UNION ALL SELECT cs_quantity quantity, + cs_list_price list_price + FROM catalog_sales, + date_dim + WHERE cs_sold_date_sk = d_date_sk + AND d_year BETWEEN 1999 AND 1999 + 2 + UNION ALL SELECT ws_quantity quantity, + ws_list_price list_price + FROM web_sales, + date_dim + WHERE ws_sold_date_sk = d_date_sk + AND d_year BETWEEN 1999 AND 1999 + 2) sq2) +SELECT channel, + i_brand_id, + i_class_id, + i_category_id, + sum(sales) AS sum_sales, + sum(number_sales) AS sum_number_sales +FROM + (SELECT 'store' channel, + i_brand_id, + i_class_id, + i_category_id, + sum(ss_quantity*ss_list_price) sales, + count(*) number_sales + FROM store_sales, + item, + date_dim + WHERE ss_item_sk IN + (SELECT ss_item_sk + FROM cross_items) + AND ss_item_sk = i_item_sk + AND ss_sold_date_sk = d_date_sk + AND d_year = 1999+2 + AND d_moy = 11 + GROUP BY i_brand_id, + i_class_id, + i_category_id + HAVING sum(ss_quantity*ss_list_price) > + (SELECT average_sales + FROM avg_sales) + UNION ALL SELECT 'catalog' channel, + i_brand_id, + i_class_id, + i_category_id, + sum(cs_quantity*cs_list_price) sales, + count(*) number_sales + FROM catalog_sales, + item, + date_dim + WHERE cs_item_sk IN + (SELECT ss_item_sk + FROM cross_items) + AND cs_item_sk = i_item_sk + AND cs_sold_date_sk = d_date_sk + AND d_year = 1999+2 + AND d_moy = 11 + GROUP BY i_brand_id, + i_class_id, + i_category_id + HAVING sum(cs_quantity*cs_list_price) > + (SELECT average_sales + FROM avg_sales) + UNION ALL SELECT 'web' channel, + i_brand_id, + i_class_id, + i_category_id, + sum(ws_quantity*ws_list_price) sales, + count(*) number_sales + FROM web_sales, + item, + date_dim + WHERE ws_item_sk IN + (SELECT ss_item_sk + FROM cross_items) + AND ws_item_sk = i_item_sk + AND ws_sold_date_sk = d_date_sk + AND d_year = 1999+2 + AND d_moy = 11 + GROUP BY i_brand_id, + i_class_id, + i_category_id + HAVING sum(ws_quantity*ws_list_price) > + (SELECT average_sales + FROM avg_sales)) y +GROUP BY ROLLUP (channel, + i_brand_id, + i_class_id, + i_category_id) +ORDER BY channel NULLS FIRST, + i_brand_id NULLS FIRST, + i_class_id NULLS FIRST, + i_category_id NULLS FIRST +LIMIT 100; diff --git a/testdata/tpcds/queries/q15.sql b/testdata/tpcds/queries/q15.sql new file mode 100644 index 00000000..3eb13aed --- /dev/null +++ b/testdata/tpcds/queries/q15.sql @@ -0,0 +1,28 @@ +SELECT ca_zip, + sum(cs_sales_price) +FROM catalog_sales, + customer, + customer_address, + date_dim +WHERE cs_bill_customer_sk = c_customer_sk + AND c_current_addr_sk = ca_address_sk + AND (SUBSTRING(ca_zip, 1, 5) IN ('85669', + '86197', + '88274', + '83405', + '86475', + '85392', + '85460', + '80348', + '81792') + OR ca_state IN ('CA', + 'WA', + 'GA') + OR cs_sales_price > 500) + AND cs_sold_date_sk = d_date_sk + AND d_qoy = 2 + AND d_year = 2001 +GROUP BY ca_zip +ORDER BY ca_zip NULLS FIRST +LIMIT 100; + diff --git a/testdata/tpcds/queries/q16.sql b/testdata/tpcds/queries/q16.sql new file mode 100644 index 00000000..cfd29a18 --- /dev/null +++ b/testdata/tpcds/queries/q16.sql @@ -0,0 +1,25 @@ +SELECT count(DISTINCT cs_order_number) AS "order count", + sum(cs_ext_ship_cost) AS "total shipping cost", + sum(cs_net_profit) AS "total net profit" +FROM catalog_sales cs1, + date_dim, + customer_address, + call_center +WHERE d_date BETWEEN '2002-02-01' AND cast('2002-04-02' AS date) + AND cs1.cs_ship_date_sk = d_date_sk + AND cs1.cs_ship_addr_sk = ca_address_sk + AND ca_state = 'GA' + AND cs1.cs_call_center_sk = cc_call_center_sk + AND cc_county = 'Williamson County' + AND EXISTS + (SELECT * + FROM catalog_sales cs2 + WHERE cs1.cs_order_number = cs2.cs_order_number + AND cs1.cs_warehouse_sk <> cs2.cs_warehouse_sk) + AND NOT EXISTS + (SELECT * + FROM catalog_returns cr1 + WHERE cs1.cs_order_number = cr1.cr_order_number) +ORDER BY count(DISTINCT cs_order_number) +LIMIT 100; + diff --git a/testdata/tpcds/queries/q17.sql b/testdata/tpcds/queries/q17.sql new file mode 100644 index 00000000..afcadfbb --- /dev/null +++ b/testdata/tpcds/queries/q17.sql @@ -0,0 +1,48 @@ +SELECT i_item_id, + i_item_desc, + s_state, + count(ss_quantity) AS store_sales_quantitycount, + avg(ss_quantity) AS store_sales_quantityave, + stddev_samp(ss_quantity) AS store_sales_quantitystdev, + stddev_samp(ss_quantity)/avg(ss_quantity) AS store_sales_quantitycov, + count(sr_return_quantity) AS store_returns_quantitycount, + avg(sr_return_quantity) AS store_returns_quantityave, + stddev_samp(sr_return_quantity) AS store_returns_quantitystdev, + stddev_samp(sr_return_quantity)/avg(sr_return_quantity) AS store_returns_quantitycov, + count(cs_quantity) AS catalog_sales_quantitycount, + avg(cs_quantity) AS catalog_sales_quantityave, + stddev_samp(cs_quantity) AS catalog_sales_quantitystdev, + stddev_samp(cs_quantity)/avg(cs_quantity) AS catalog_sales_quantitycov +FROM store_sales, + store_returns, + catalog_sales, + date_dim d1, + date_dim d2, + date_dim d3, + store, + item +WHERE d1.d_quarter_name = '2001Q1' + AND d1.d_date_sk = ss_sold_date_sk + AND i_item_sk = ss_item_sk + AND s_store_sk = ss_store_sk + AND ss_customer_sk = sr_customer_sk + AND ss_item_sk = sr_item_sk + AND ss_ticket_number = sr_ticket_number + AND sr_returned_date_sk = d2.d_date_sk + AND d2.d_quarter_name IN ('2001Q1', + '2001Q2', + '2001Q3') + AND sr_customer_sk = cs_bill_customer_sk + AND sr_item_sk = cs_item_sk + AND cs_sold_date_sk = d3.d_date_sk + AND d3.d_quarter_name IN ('2001Q1', + '2001Q2', + '2001Q3') +GROUP BY i_item_id, + i_item_desc, + s_state +ORDER BY i_item_id NULLS FIRST, + i_item_desc NULLS FIRST, + s_state NULLS FIRST +LIMIT 100; + diff --git a/testdata/tpcds/queries/q18.sql b/testdata/tpcds/queries/q18.sql new file mode 100644 index 00000000..d2f759b4 --- /dev/null +++ b/testdata/tpcds/queries/q18.sql @@ -0,0 +1,49 @@ +SELECT i_item_id, + ca_country, + ca_state, + ca_county, + avg(cast(cs_quantity AS decimal(12, 2))) agg1, + avg(cast(cs_list_price AS decimal(12, 2))) agg2, + avg(cast(cs_coupon_amt AS decimal(12, 2))) agg3, + avg(cast(cs_sales_price AS decimal(12, 2))) agg4, + avg(cast(cs_net_profit AS decimal(12, 2))) agg5, + avg(cast(c_birth_year AS decimal(12, 2))) agg6, + avg(cast(cd1.cd_dep_count AS decimal(12, 2))) agg7 +FROM catalog_sales, + customer_demographics cd1, + customer_demographics cd2, + customer, + customer_address, + date_dim, + item +WHERE cs_sold_date_sk = d_date_sk + AND cs_item_sk = i_item_sk + AND cs_bill_cdemo_sk = cd1.cd_demo_sk + AND cs_bill_customer_sk = c_customer_sk + AND cd1.cd_gender = 'F' + AND cd1.cd_education_status = 'Unknown' + AND c_current_cdemo_sk = cd2.cd_demo_sk + AND c_current_addr_sk = ca_address_sk + AND c_birth_month IN (1, + 6, + 8, + 9, + 12, + 2) + AND d_year = 1998 + AND ca_state IN ('MS', + 'IN', + 'ND', + 'OK', + 'NM', + 'VA', + 'MS') +GROUP BY ROLLUP (i_item_id, + ca_country, + ca_state, + ca_county) +ORDER BY ca_country NULLS FIRST, + ca_state NULLS FIRST, + ca_county NULLS FIRST, + i_item_id NULLS FIRST +LIMIT 100; diff --git a/testdata/tpcds/queries/q19.sql b/testdata/tpcds/queries/q19.sql new file mode 100644 index 00000000..765d2dd9 --- /dev/null +++ b/testdata/tpcds/queries/q19.sql @@ -0,0 +1,31 @@ +SELECT i_brand_id brand_id, + i_brand brand, + i_manufact_id, + i_manufact, + sum(ss_ext_sales_price) ext_price +FROM date_dim, + store_sales, + item, + customer, + customer_address, + store +WHERE d_date_sk = ss_sold_date_sk + AND ss_item_sk = i_item_sk + AND i_manager_id=8 + AND d_moy=11 + AND d_year=1998 + AND ss_customer_sk = c_customer_sk + AND c_current_addr_sk = ca_address_sk + AND SUBSTRING(ca_zip, 1, 5) <> SUBSTRING(s_zip, 1, 5) + AND ss_store_sk = s_store_sk +GROUP BY i_brand, + i_brand_id, + i_manufact_id, + i_manufact +ORDER BY ext_price DESC, + i_brand, + i_brand_id, + i_manufact_id, + i_manufact +LIMIT 100 ; + diff --git a/testdata/tpcds/queries/q2.sql b/testdata/tpcds/queries/q2.sql new file mode 100644 index 00000000..8663b362 --- /dev/null +++ b/testdata/tpcds/queries/q2.sql @@ -0,0 +1,79 @@ +WITH wscs AS + (SELECT sold_date_sk, + sales_price + FROM + (SELECT ws_sold_date_sk sold_date_sk, + ws_ext_sales_price sales_price + FROM web_sales + UNION ALL SELECT cs_sold_date_sk sold_date_sk, + cs_ext_sales_price sales_price + FROM catalog_sales) sq1), + wswscs AS + (SELECT d_week_seq, + sum(CASE + WHEN (d_day_name='Sunday') THEN sales_price + ELSE NULL + END) sun_sales, + sum(CASE + WHEN (d_day_name='Monday') THEN sales_price + ELSE NULL + END) mon_sales, + sum(CASE + WHEN (d_day_name='Tuesday') THEN sales_price + ELSE NULL + END) tue_sales, + sum(CASE + WHEN (d_day_name='Wednesday') THEN sales_price + ELSE NULL + END) wed_sales, + sum(CASE + WHEN (d_day_name='Thursday') THEN sales_price + ELSE NULL + END) thu_sales, + sum(CASE + WHEN (d_day_name='Friday') THEN sales_price + ELSE NULL + END) fri_sales, + sum(CASE + WHEN (d_day_name='Saturday') THEN sales_price + ELSE NULL + END) sat_sales + FROM wscs, + date_dim + WHERE d_date_sk = sold_date_sk + GROUP BY d_week_seq) +SELECT d_week_seq1, + round(sun_sales1/sun_sales2, 2) r1, + round(mon_sales1/mon_sales2, 2) r2, + round(tue_sales1/tue_sales2, 2) r3, + round(wed_sales1/wed_sales2, 2) r4, + round(thu_sales1/thu_sales2, 2) r5, + round(fri_sales1/fri_sales2, 2) r6, + round(sat_sales1/sat_sales2, 2) +FROM + (SELECT wswscs.d_week_seq d_week_seq1, + sun_sales sun_sales1, + mon_sales mon_sales1, + tue_sales tue_sales1, + wed_sales wed_sales1, + thu_sales thu_sales1, + fri_sales fri_sales1, + sat_sales sat_sales1 + FROM wswscs, + date_dim + WHERE date_dim.d_week_seq = wswscs.d_week_seq + AND d_year = 2001) y, + (SELECT wswscs.d_week_seq d_week_seq2, + sun_sales sun_sales2, + mon_sales mon_sales2, + tue_sales tue_sales2, + wed_sales wed_sales2, + thu_sales thu_sales2, + fri_sales fri_sales2, + sat_sales sat_sales2 + FROM wswscs, + date_dim + WHERE date_dim.d_week_seq = wswscs.d_week_seq + AND d_year = 2001+1) z +WHERE d_week_seq1 = d_week_seq2-53 +ORDER BY d_week_seq1 NULLS FIRST; diff --git a/testdata/tpcds/queries/q20.sql b/testdata/tpcds/queries/q20.sql new file mode 100644 index 00000000..b262fbf1 --- /dev/null +++ b/testdata/tpcds/queries/q20.sql @@ -0,0 +1,29 @@ + +SELECT i_item_id , + i_item_desc, + i_category, + i_class, + i_current_price , + sum(cs_ext_sales_price) AS itemrevenue, + sum(cs_ext_sales_price)*100.0000/sum(sum(cs_ext_sales_price)) OVER (PARTITION BY i_class) AS revenueratio +FROM catalog_sales , + item, + date_dim +WHERE cs_item_sk = i_item_sk + AND i_category IN ('Sports', + 'Books', + 'Home') + AND cs_sold_date_sk = d_date_sk + AND d_date BETWEEN cast('1999-02-22' AS date) AND cast('1999-03-24' AS date) +GROUP BY i_item_id , + i_item_desc, + i_category , + i_class , + i_current_price +ORDER BY i_category NULLS FIRST, + i_class NULLS FIRST, + i_item_id NULLS FIRST, + i_item_desc NULLS FIRST, + revenueratio NULLS FIRST +LIMIT 100; + diff --git a/testdata/tpcds/queries/q21.sql b/testdata/tpcds/queries/q21.sql new file mode 100644 index 00000000..f1c2fae1 --- /dev/null +++ b/testdata/tpcds/queries/q21.sql @@ -0,0 +1,31 @@ +SELECT * +FROM + (SELECT w_warehouse_name, + i_item_id, + sum(CASE + WHEN (cast(d_date AS date) < CAST ('2000-03-11' AS date)) THEN inv_quantity_on_hand + ELSE 0 + END) AS inv_before, + sum(CASE + WHEN (cast(d_date AS date) >= CAST ('2000-03-11' AS date)) THEN inv_quantity_on_hand + ELSE 0 + END) AS inv_after + FROM inventory, + warehouse, + item, + date_dim + WHERE i_current_price BETWEEN 0.99 AND 1.49 + AND i_item_sk = inv_item_sk + AND inv_warehouse_sk = w_warehouse_sk + AND inv_date_sk = d_date_sk + AND d_date BETWEEN CAST ('2000-02-10' AS date) AND CAST ('2000-04-10' AS date) + GROUP BY w_warehouse_name, + i_item_id) x +WHERE (CASE + WHEN inv_before > 0 THEN (inv_after*1.000) / inv_before + ELSE NULL + END) BETWEEN 2.000/3.000 AND 3.000/2.000 +ORDER BY w_warehouse_name NULLS FIRST, + i_item_id NULLS FIRST +LIMIT 100; + diff --git a/testdata/tpcds/queries/q22.sql b/testdata/tpcds/queries/q22.sql new file mode 100644 index 00000000..13a26a58 --- /dev/null +++ b/testdata/tpcds/queries/q22.sql @@ -0,0 +1,18 @@ +SELECT i_product_name , + i_brand , + i_class , + i_category , + avg(inv_quantity_on_hand) qoh +FROM inventory , + date_dim , + item +WHERE inv_date_sk=d_date_sk + AND inv_item_sk=i_item_sk + AND d_month_seq BETWEEN 1200 AND 1200 + 11 +GROUP BY rollup(i_product_name ,i_brand ,i_class ,i_category) +ORDER BY qoh NULLS FIRST, + i_product_name NULLS FIRST, + i_brand NULLS FIRST, + i_class NULLS FIRST, + i_category NULLS FIRST +LIMIT 100; diff --git a/testdata/tpcds/queries/q23.sql b/testdata/tpcds/queries/q23.sql new file mode 100644 index 00000000..c26704f0 --- /dev/null +++ b/testdata/tpcds/queries/q23.sql @@ -0,0 +1,84 @@ +WITH frequent_ss_items AS + (SELECT itemdesc, + i_item_sk item_sk, + d_date solddate, + count(*) cnt + FROM store_sales, + date_dim, + (SELECT SUBSTRING(i_item_desc, 1, 30) itemdesc, + * + FROM item) sq1 + WHERE ss_sold_date_sk = d_date_sk + AND ss_item_sk = i_item_sk + AND d_year IN (2000, + 2000+1, + 2000+2, + 2000+3) + GROUP BY itemdesc, + i_item_sk, + d_date + HAVING count(*) >4), + max_store_sales AS + (SELECT max(csales) tpcds_cmax + FROM + (SELECT c_customer_sk, + sum(ss_quantity*ss_sales_price) csales + FROM store_sales, + customer, + date_dim + WHERE ss_customer_sk = c_customer_sk + AND ss_sold_date_sk = d_date_sk + AND d_year IN (2000, + 2000+1, + 2000+2, + 2000+3) + GROUP BY c_customer_sk) sq2), + best_ss_customer AS + (SELECT c_customer_sk, + sum(ss_quantity*ss_sales_price) ssales + FROM store_sales, + customer, + max_store_sales + WHERE ss_customer_sk = c_customer_sk + GROUP BY c_customer_sk + HAVING sum(ss_quantity*ss_sales_price) > (50/100.0) * max(tpcds_cmax)) +SELECT c_last_name, + c_first_name, + sales +FROM + (SELECT c_last_name, + c_first_name, + sum(cs_quantity*cs_list_price) sales + FROM catalog_sales, + customer, + date_dim, + frequent_ss_items, + best_ss_customer + WHERE d_year = 2000 + AND d_moy = 2 + AND cs_sold_date_sk = d_date_sk + AND cs_item_sk = item_sk + AND cs_bill_customer_sk = best_ss_customer.c_customer_sk + AND cs_bill_customer_sk = customer.c_customer_sk + GROUP BY c_last_name, + c_first_name + UNION ALL SELECT c_last_name, + c_first_name, + sum(ws_quantity*ws_list_price) sales + FROM web_sales, + customer, + date_dim, + frequent_ss_items, + best_ss_customer + WHERE d_year = 2000 + AND d_moy = 2 + AND ws_sold_date_sk = d_date_sk + AND ws_item_sk = item_sk + AND ws_bill_customer_sk = best_ss_customer.c_customer_sk + AND ws_bill_customer_sk = customer.c_customer_sk + GROUP BY c_last_name, + c_first_name) sq3 +ORDER BY c_last_name NULLS FIRST, + c_first_name NULLS FIRST, + sales NULLS FIRST +LIMIT 100; diff --git a/testdata/tpcds/queries/q24.sql b/testdata/tpcds/queries/q24.sql new file mode 100644 index 00000000..d2438510 --- /dev/null +++ b/testdata/tpcds/queries/q24.sql @@ -0,0 +1,52 @@ +WITH ssales AS + (SELECT c_last_name, + c_first_name, + s_store_name, + ca_state, + s_state, + i_color, + i_current_price, + i_manager_id, + i_units, + i_size, + sum(ss_net_paid) netpaid + FROM store_sales, + store_returns, + store, + item, + customer, + customer_address + WHERE ss_ticket_number = sr_ticket_number + AND ss_item_sk = sr_item_sk + AND ss_customer_sk = c_customer_sk + AND ss_item_sk = i_item_sk + AND ss_store_sk = s_store_sk + AND c_current_addr_sk = ca_address_sk + AND c_birth_country <> upper(ca_country) + AND s_zip = ca_zip + AND s_market_id=8 + GROUP BY c_last_name, + c_first_name, + s_store_name, + ca_state, + s_state, + i_color, + i_current_price, + i_manager_id, + i_units, + i_size) +SELECT c_last_name, + c_first_name, + s_store_name, + sum(netpaid) paid +FROM ssales +WHERE i_color = 'peach' +GROUP BY c_last_name, + c_first_name, + s_store_name +HAVING sum(netpaid) > + (SELECT 0.05*avg(netpaid) + FROM ssales) +ORDER BY c_last_name, + c_first_name, + s_store_name ; diff --git a/testdata/tpcds/queries/q25.sql b/testdata/tpcds/queries/q25.sql new file mode 100644 index 00000000..6c1a95b8 --- /dev/null +++ b/testdata/tpcds/queries/q25.sql @@ -0,0 +1,42 @@ + +SELECT i_item_id , + i_item_desc , + s_store_id , + s_store_name , + sum(ss_net_profit) AS store_sales_profit , + sum(sr_net_loss) AS store_returns_loss , + sum(cs_net_profit) AS catalog_sales_profit +FROM store_sales , + store_returns , + catalog_sales , + date_dim d1 , + date_dim d2 , + date_dim d3 , + store , + item +WHERE d1.d_moy = 4 + AND d1.d_year = 2001 + AND d1.d_date_sk = ss_sold_date_sk + AND i_item_sk = ss_item_sk + AND s_store_sk = ss_store_sk + AND ss_customer_sk = sr_customer_sk + AND ss_item_sk = sr_item_sk + AND ss_ticket_number = sr_ticket_number + AND sr_returned_date_sk = d2.d_date_sk + AND d2.d_moy BETWEEN 4 AND 10 + AND d2.d_year = 2001 + AND sr_customer_sk = cs_bill_customer_sk + AND sr_item_sk = cs_item_sk + AND cs_sold_date_sk = d3.d_date_sk + AND d3.d_moy BETWEEN 4 AND 10 + AND d3.d_year = 2001 +GROUP BY i_item_id , + i_item_desc , + s_store_id , + s_store_name +ORDER BY i_item_id , + i_item_desc , + s_store_id , + s_store_name +LIMIT 100; + diff --git a/testdata/tpcds/queries/q26.sql b/testdata/tpcds/queries/q26.sql new file mode 100644 index 00000000..4ca9fdb9 --- /dev/null +++ b/testdata/tpcds/queries/q26.sql @@ -0,0 +1,24 @@ +SELECT i_item_id, + avg(cs_quantity) agg1, + avg(cs_list_price) agg2, + avg(cs_coupon_amt) agg3, + avg(cs_sales_price) agg4 +FROM catalog_sales, + customer_demographics, + date_dim, + item, + promotion +WHERE cs_sold_date_sk = d_date_sk + AND cs_item_sk = i_item_sk + AND cs_bill_cdemo_sk = cd_demo_sk + AND cs_promo_sk = p_promo_sk + AND cd_gender = 'M' + AND cd_marital_status = 'S' + AND cd_education_status = 'College' + AND (p_channel_email = 'N' + OR p_channel_event = 'N') + AND d_year = 2000 +GROUP BY i_item_id +ORDER BY i_item_id +LIMIT 100; + diff --git a/testdata/tpcds/queries/q27.sql b/testdata/tpcds/queries/q27.sql new file mode 100644 index 00000000..2d5884db --- /dev/null +++ b/testdata/tpcds/queries/q27.sql @@ -0,0 +1,60 @@ +WITH results AS + (SELECT i_item_id, + s_state, + 0 AS g_state, + ss_quantity agg1, + ss_list_price agg2, + ss_coupon_amt agg3, + ss_sales_price agg4 + FROM store_sales, + customer_demographics, + date_dim, + store, + item + WHERE ss_sold_date_sk = d_date_sk + AND ss_item_sk = i_item_sk + AND ss_store_sk = s_store_sk + AND ss_cdemo_sk = cd_demo_sk + AND cd_gender = 'M' + AND cd_marital_status = 'S' + AND cd_education_status = 'College' + AND d_year = 2002 + AND s_state = 'TN' ) +SELECT i_item_id, + s_state, + g_state, + agg1, + agg2, + agg3, + agg4 +FROM + ( SELECT i_item_id, + s_state, + 0 AS g_state, + avg(agg1) agg1, + avg(agg2) agg2, + avg(agg3) agg3, + avg(agg4) agg4 + FROM results + GROUP BY i_item_id , + s_state + UNION ALL SELECT i_item_id, + NULL AS s_state, + 1 AS g_state, + avg(agg1) agg1, + avg(agg2) agg2, + avg(agg3) agg3, + avg(agg4) agg4 + FROM results + GROUP BY i_item_id + UNION ALL SELECT NULL AS i_item_id, + NULL AS s_state, + 1 AS g_state, + avg(agg1) agg1, + avg(agg2) agg2, + avg(agg3) agg3, + avg(agg4) agg4 + FROM results ) foo +ORDER BY i_item_id NULLS FIRST, + s_state NULLS FIRST +LIMIT 100; diff --git a/testdata/tpcds/queries/q28.sql b/testdata/tpcds/queries/q28.sql new file mode 100644 index 00000000..c0a04137 --- /dev/null +++ b/testdata/tpcds/queries/q28.sql @@ -0,0 +1,52 @@ +SELECT * +FROM + (SELECT avg(ss_list_price) B1_LP, + count(ss_list_price) B1_CNT, + count(DISTINCT ss_list_price) B1_CNTD + FROM store_sales + WHERE ss_quantity BETWEEN 0 AND 5 + AND (ss_list_price BETWEEN 8 AND 8+10 + OR ss_coupon_amt BETWEEN 459 AND 459+1000 + OR ss_wholesale_cost BETWEEN 57 AND 57+20)) B1, + (SELECT avg(ss_list_price) B2_LP, + count(ss_list_price) B2_CNT, + count(DISTINCT ss_list_price) B2_CNTD + FROM store_sales + WHERE ss_quantity BETWEEN 6 AND 10 + AND (ss_list_price BETWEEN 90 AND 90+10 + OR ss_coupon_amt BETWEEN 2323 AND 2323+1000 + OR ss_wholesale_cost BETWEEN 31 AND 31+20)) B2, + (SELECT avg(ss_list_price) B3_LP, + count(ss_list_price) B3_CNT, + count(DISTINCT ss_list_price) B3_CNTD + FROM store_sales + WHERE ss_quantity BETWEEN 11 AND 15 + AND (ss_list_price BETWEEN 142 AND 142+10 + OR ss_coupon_amt BETWEEN 12214 AND 12214+1000 + OR ss_wholesale_cost BETWEEN 79 AND 79+20)) B3, + (SELECT avg(ss_list_price) B4_LP, + count(ss_list_price) B4_CNT, + count(DISTINCT ss_list_price) B4_CNTD + FROM store_sales + WHERE ss_quantity BETWEEN 16 AND 20 + AND (ss_list_price BETWEEN 135 AND 135+10 + OR ss_coupon_amt BETWEEN 6071 AND 6071+1000 + OR ss_wholesale_cost BETWEEN 38 AND 38+20)) B4, + (SELECT avg(ss_list_price) B5_LP, + count(ss_list_price) B5_CNT, + count(DISTINCT ss_list_price) B5_CNTD + FROM store_sales + WHERE ss_quantity BETWEEN 21 AND 25 + AND (ss_list_price BETWEEN 122 AND 122+10 + OR ss_coupon_amt BETWEEN 836 AND 836+1000 + OR ss_wholesale_cost BETWEEN 17 AND 17+20)) B5, + (SELECT avg(ss_list_price) B6_LP, + count(ss_list_price) B6_CNT, + count(DISTINCT ss_list_price) B6_CNTD + FROM store_sales + WHERE ss_quantity BETWEEN 26 AND 30 + AND (ss_list_price BETWEEN 154 AND 154+10 + OR ss_coupon_amt BETWEEN 7326 AND 7326+1000 + OR ss_wholesale_cost BETWEEN 7 AND 7+20)) B6 +LIMIT 100; + diff --git a/testdata/tpcds/queries/q29.sql b/testdata/tpcds/queries/q29.sql new file mode 100644 index 00000000..73caa1d4 --- /dev/null +++ b/testdata/tpcds/queries/q29.sql @@ -0,0 +1,42 @@ +SELECT i_item_id, + i_item_desc, + s_store_id, + s_store_name, + sum(ss_quantity) AS store_sales_quantity, + sum(sr_return_quantity) AS store_returns_quantity, + sum(cs_quantity) AS catalog_sales_quantity +FROM store_sales, + store_returns, + catalog_sales, + date_dim d1, + date_dim d2, + date_dim d3, + store, + item +WHERE d1.d_moy = 9 + AND d1.d_year = 1999 + AND d1.d_date_sk = ss_sold_date_sk + AND i_item_sk = ss_item_sk + AND s_store_sk = ss_store_sk + AND ss_customer_sk = sr_customer_sk + AND ss_item_sk = sr_item_sk + AND ss_ticket_number = sr_ticket_number + AND sr_returned_date_sk = d2.d_date_sk + AND d2.d_moy BETWEEN 9 AND 9 + 3 + AND d2.d_year = 1999 + AND sr_customer_sk = cs_bill_customer_sk + AND sr_item_sk = cs_item_sk + AND cs_sold_date_sk = d3.d_date_sk + AND d3.d_year IN (1999, + 1999+1, + 1999+2) +GROUP BY i_item_id, + i_item_desc, + s_store_id, + s_store_name +ORDER BY i_item_id, + i_item_desc, + s_store_id, + s_store_name +LIMIT 100; + diff --git a/testdata/tpcds/queries/q3.sql b/testdata/tpcds/queries/q3.sql new file mode 100644 index 00000000..6cf202dc --- /dev/null +++ b/testdata/tpcds/queries/q3.sql @@ -0,0 +1,19 @@ +SELECT dt.d_year, + item.i_brand_id brand_id, + item.i_brand brand, + sum(ss_ext_sales_price) sum_agg +FROM date_dim dt, + store_sales, + item +WHERE dt.d_date_sk = store_sales.ss_sold_date_sk + AND store_sales.ss_item_sk = item.i_item_sk + AND item.i_manufact_id = 128 + AND dt.d_moy=11 +GROUP BY dt.d_year, + item.i_brand, + item.i_brand_id +ORDER BY dt.d_year, + sum_agg DESC, + brand_id +LIMIT 100; + diff --git a/testdata/tpcds/queries/q30.sql b/testdata/tpcds/queries/q30.sql new file mode 100644 index 00000000..544e8198 --- /dev/null +++ b/testdata/tpcds/queries/q30.sql @@ -0,0 +1,50 @@ +WITH customer_total_return AS + (SELECT wr_returning_customer_sk AS ctr_customer_sk, + ca_state AS ctr_state, + sum(wr_return_amt) AS ctr_total_return + FROM web_returns, + date_dim, + customer_address + WHERE wr_returned_date_sk = d_date_sk + AND d_year = 2002 + AND wr_returning_addr_sk = ca_address_sk + GROUP BY wr_returning_customer_sk, + ca_state) +SELECT c_customer_id, + c_salutation, + c_first_name, + c_last_name, + c_preferred_cust_flag, + c_birth_day, + c_birth_month, + c_birth_year, + c_birth_country, + c_login, + c_email_address, + c_last_review_date_sk, + ctr_total_return +FROM customer_total_return ctr1, + customer_address, + customer +WHERE ctr1.ctr_total_return > + (SELECT avg(ctr_total_return)*1.2 + FROM customer_total_return ctr2 + WHERE ctr1.ctr_state = ctr2.ctr_state) + AND ca_address_sk = c_current_addr_sk + AND ca_state = 'GA' + AND ctr1.ctr_customer_sk = c_customer_sk +ORDER BY c_customer_id NULLS FIRST, + c_salutation NULLS FIRST, + c_first_name NULLS FIRST, + c_last_name NULLS FIRST, + c_preferred_cust_flag NULLS FIRST, + c_birth_day NULLS FIRST, + c_birth_month NULLS FIRST, + c_birth_year NULLS FIRST, + c_birth_country NULLS FIRST, + c_login NULLS FIRST, + c_email_address NULLS FIRST, + c_last_review_date_sk NULLS FIRST, + ctr_total_return NULLS FIRST +LIMIT 100; + diff --git a/testdata/tpcds/queries/q31.sql b/testdata/tpcds/queries/q31.sql new file mode 100644 index 00000000..1d9f87a5 --- /dev/null +++ b/testdata/tpcds/queries/q31.sql @@ -0,0 +1,71 @@ +WITH ss AS + (SELECT ca_county, + d_qoy, + d_year, + sum(ss_ext_sales_price) AS store_sales + FROM store_sales, + date_dim, + customer_address + WHERE ss_sold_date_sk = d_date_sk + AND ss_addr_sk=ca_address_sk + GROUP BY ca_county, + d_qoy, + d_year), + ws AS + (SELECT ca_county, + d_qoy, + d_year, + sum(ws_ext_sales_price) AS web_sales + FROM web_sales, + date_dim, + customer_address + WHERE ws_sold_date_sk = d_date_sk + AND ws_bill_addr_sk=ca_address_sk + GROUP BY ca_county, + d_qoy, + d_year) +SELECT ss1.ca_county , + ss1.d_year , + (ws2.web_sales*1.0000)/ws1.web_sales web_q1_q2_increase , + (ss2.store_sales*1.0000)/ss1.store_sales store_q1_q2_increase , + (ws3.web_sales*1.0000)/ws2.web_sales web_q2_q3_increase , + (ss3.store_sales*1.0000)/ss2.store_sales store_q2_q3_increase +FROM ss ss1 , + ss ss2 , + ss ss3 , + ws ws1 , + ws ws2 , + ws ws3 +WHERE ss1.d_qoy = 1 + AND ss1.d_year = 2000 + AND ss1.ca_county = ss2.ca_county + AND ss2.d_qoy = 2 + AND ss2.d_year = 2000 + AND ss2.ca_county = ss3.ca_county + AND ss3.d_qoy = 3 + AND ss3.d_year = 2000 + AND ss1.ca_county = ws1.ca_county + AND ws1.d_qoy = 1 + AND ws1.d_year = 2000 + AND ws1.ca_county = ws2.ca_county + AND ws2.d_qoy = 2 + AND ws2.d_year = 2000 + AND ws1.ca_county = ws3.ca_county + AND ws3.d_qoy = 3 + AND ws3.d_year = 2000 + AND CASE + WHEN ws1.web_sales > 0 THEN (ws2.web_sales*1.0000)/ws1.web_sales + ELSE NULL + END > CASE + WHEN ss1.store_sales > 0 THEN (ss2.store_sales*1.0000)/ss1.store_sales + ELSE NULL + END + AND CASE + WHEN ws2.web_sales > 0 THEN (ws3.web_sales*1.0000)/ws2.web_sales + ELSE NULL + END > CASE + WHEN ss2.store_sales > 0 THEN (ss3.store_sales*1.0000)/ss2.store_sales + ELSE NULL + END +ORDER BY ss1.ca_county; + diff --git a/testdata/tpcds/queries/q32.sql b/testdata/tpcds/queries/q32.sql new file mode 100644 index 00000000..50e16a34 --- /dev/null +++ b/testdata/tpcds/queries/q32.sql @@ -0,0 +1,17 @@ +SELECT sum(cs_ext_discount_amt) AS "excess discount amount" +FROM catalog_sales , + item , + date_dim +WHERE i_manufact_id = 977 + AND i_item_sk = cs_item_sk + AND d_date BETWEEN '2000-01-27' AND cast('2000-04-26' AS date) + AND d_date_sk = cs_sold_date_sk + AND cs_ext_discount_amt > + ( SELECT 1.3 * avg(cs_ext_discount_amt) + FROM catalog_sales , + date_dim + WHERE cs_item_sk = i_item_sk + AND d_date BETWEEN '2000-01-27' AND cast('2000-04-26' AS date) + AND d_date_sk = cs_sold_date_sk ) +LIMIT 100; + diff --git a/testdata/tpcds/queries/q33.sql b/testdata/tpcds/queries/q33.sql new file mode 100644 index 00000000..76cbd653 --- /dev/null +++ b/testdata/tpcds/queries/q33.sql @@ -0,0 +1,67 @@ +WITH ss AS + ( SELECT i_manufact_id, + sum(ss_ext_sales_price) total_sales + FROM store_sales, + date_dim, + customer_address, + item + WHERE i_manufact_id IN + (SELECT i_manufact_id + FROM item + WHERE i_category IN ('Electronics')) + AND ss_item_sk = i_item_sk + AND ss_sold_date_sk = d_date_sk + AND d_year = 1998 + AND d_moy = 5 + AND ss_addr_sk = ca_address_sk + AND ca_gmt_offset = -5 + GROUP BY i_manufact_id), + cs AS + ( SELECT i_manufact_id, + sum(cs_ext_sales_price) total_sales + FROM catalog_sales, + date_dim, + customer_address, + item + WHERE i_manufact_id IN + (SELECT i_manufact_id + FROM item + WHERE i_category IN ('Electronics')) + AND cs_item_sk = i_item_sk + AND cs_sold_date_sk = d_date_sk + AND d_year = 1998 + AND d_moy = 5 + AND cs_bill_addr_sk = ca_address_sk + AND ca_gmt_offset = -5 + GROUP BY i_manufact_id), + ws AS + ( SELECT i_manufact_id, + sum(ws_ext_sales_price) total_sales + FROM web_sales, + date_dim, + customer_address, + item + WHERE i_manufact_id IN + (SELECT i_manufact_id + FROM item + WHERE i_category IN ('Electronics')) + AND ws_item_sk = i_item_sk + AND ws_sold_date_sk = d_date_sk + AND d_year = 1998 + AND d_moy = 5 + AND ws_bill_addr_sk = ca_address_sk + AND ca_gmt_offset = -5 + GROUP BY i_manufact_id) +SELECT i_manufact_id, + sum(total_sales) total_sales +FROM + (SELECT * + FROM ss + UNION ALL SELECT * + FROM cs + UNION ALL SELECT * + FROM ws) tmp1 +GROUP BY i_manufact_id +ORDER BY total_sales +LIMIT 100; + diff --git a/testdata/tpcds/queries/q34.sql b/testdata/tpcds/queries/q34.sql new file mode 100644 index 00000000..d200b15b --- /dev/null +++ b/testdata/tpcds/queries/q34.sql @@ -0,0 +1,41 @@ +SELECT c_last_name , + c_first_name , + c_salutation , + c_preferred_cust_flag , + ss_ticket_number , + cnt +FROM + (SELECT ss_ticket_number , + ss_customer_sk , + count(*) cnt + FROM store_sales, + date_dim, + store, + household_demographics + WHERE store_sales.ss_sold_date_sk = date_dim.d_date_sk + AND store_sales.ss_store_sk = store.s_store_sk + AND store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk + AND (date_dim.d_dom BETWEEN 1 AND 3 + OR date_dim.d_dom BETWEEN 25 AND 28) + AND (household_demographics.hd_buy_potential = '>10000' + OR household_demographics.hd_buy_potential = 'Unknown') + AND household_demographics.hd_vehicle_count > 0 + AND (CASE + WHEN household_demographics.hd_vehicle_count > 0 THEN (household_demographics.hd_dep_count*1.000)/ household_demographics.hd_vehicle_count + ELSE NULL + END) > 1.2 + AND date_dim.d_year IN (1999, + 1999+1, + 1999+2) + AND store.s_county = 'Williamson County' + GROUP BY ss_ticket_number, + ss_customer_sk) dn, + customer +WHERE ss_customer_sk = c_customer_sk + AND cnt BETWEEN 15 AND 20 +ORDER BY c_last_name NULLS FIRST, + c_first_name NULLS FIRST, + c_salutation NULLS FIRST, + c_preferred_cust_flag DESC NULLS FIRST, + ss_ticket_number NULLS FIRST; + diff --git a/testdata/tpcds/queries/q35.sql b/testdata/tpcds/queries/q35.sql new file mode 100644 index 00000000..c8a136aa --- /dev/null +++ b/testdata/tpcds/queries/q35.sql @@ -0,0 +1,61 @@ +SELECT ca_state, + cd_gender, + cd_marital_status, + cd_dep_count, + count(*) cnt1, + min(cd_dep_count) min1, + max(cd_dep_count) max1, + avg(cd_dep_count) avg1, + cd_dep_employed_count, + count(*) cnt2, + min(cd_dep_employed_count) min2, + max(cd_dep_employed_count) max2, + avg(cd_dep_employed_count) avg2, + cd_dep_college_count, + count(*) cnt3, + min(cd_dep_college_count), + max(cd_dep_college_count), + avg(cd_dep_college_count) +FROM customer c, + customer_address ca, + customer_demographics +WHERE c.c_current_addr_sk = ca.ca_address_sk + AND cd_demo_sk = c.c_current_cdemo_sk + AND EXISTS + (SELECT * + FROM store_sales, + date_dim + WHERE c.c_customer_sk = ss_customer_sk + AND ss_sold_date_sk = d_date_sk + AND d_year = 2002 + AND d_qoy < 4) + AND (EXISTS + (SELECT * + FROM web_sales, + date_dim + WHERE c.c_customer_sk = ws_bill_customer_sk + AND ws_sold_date_sk = d_date_sk + AND d_year = 2002 + AND d_qoy < 4) + OR EXISTS + (SELECT * + FROM catalog_sales, + date_dim + WHERE c.c_customer_sk = cs_ship_customer_sk + AND cs_sold_date_sk = d_date_sk + AND d_year = 2002 + AND d_qoy < 4)) +GROUP BY ca_state, + cd_gender, + cd_marital_status, + cd_dep_count, + cd_dep_employed_count, + cd_dep_college_count +ORDER BY ca_state NULLS FIRST, + cd_gender NULLS FIRST, + cd_marital_status NULLS FIRST, + cd_dep_count NULLS FIRST, + cd_dep_employed_count NULLS FIRST, + cd_dep_college_count NULLS FIRST +LIMIT 100; + diff --git a/testdata/tpcds/queries/q36.sql b/testdata/tpcds/queries/q36.sql new file mode 100644 index 00000000..db6107ef --- /dev/null +++ b/testdata/tpcds/queries/q36.sql @@ -0,0 +1,59 @@ +WITH results AS + (SELECT sum(ss_net_profit) AS ss_net_profit, + sum(ss_ext_sales_price) AS ss_ext_sales_price, + (sum(ss_net_profit)*1.0000)/sum(ss_ext_sales_price) AS gross_margin , + i_category , + i_class , + 0 AS g_category, + 0 AS g_class + FROM store_sales , + date_dim d1 , + item , + store + WHERE d1.d_year = 2001 + AND d1.d_date_sk = ss_sold_date_sk + AND i_item_sk = ss_item_sk + AND s_store_sk = ss_store_sk + AND s_state ='TN' + GROUP BY i_category, + i_class) , + results_rollup AS + (SELECT gross_margin, + i_category, + i_class, + 0 AS t_category, + 0 AS t_class, + 0 AS lochierarchy + FROM results + UNION SELECT (sum(ss_net_profit)*1.0000)/sum(ss_ext_sales_price) AS gross_margin, + i_category, + NULL AS i_class, + 0 AS t_category, + 1 AS t_class, + 1 AS lochierarchy + FROM results + GROUP BY i_category + UNION SELECT (sum(ss_net_profit)*1.0000)/sum(ss_ext_sales_price) AS gross_margin, + NULL AS i_category, + NULL AS i_class, + 1 AS t_category, + 1 AS t_class, + 2 AS lochierarchy + FROM results) +SELECT gross_margin, + i_category, + i_class, + lochierarchy, + rank() OVER ( PARTITION BY lochierarchy, + CASE + WHEN t_class = 0 THEN i_category + END + ORDER BY gross_margin ASC) AS rank_within_parent +FROM results_rollup +ORDER BY lochierarchy DESC NULLS FIRST, + CASE + WHEN lochierarchy = 0 THEN i_category + END NULLS FIRST, + rank_within_parent NULLS FIRST +LIMIT 100; + diff --git a/testdata/tpcds/queries/q37.sql b/testdata/tpcds/queries/q37.sql new file mode 100644 index 00000000..1ae1df0d --- /dev/null +++ b/testdata/tpcds/queries/q37.sql @@ -0,0 +1,23 @@ +SELECT i_item_id, + i_item_desc, + i_current_price +FROM item, + inventory, + date_dim, + catalog_sales +WHERE i_current_price BETWEEN 68 AND 68 + 30 + AND inv_item_sk = i_item_sk + AND d_date_sk=inv_date_sk + AND d_date BETWEEN cast('2000-02-01' AS date) AND cast('2000-04-01' AS date) + AND i_manufact_id IN (677, + 940, + 694, + 808) + AND inv_quantity_on_hand BETWEEN 100 AND 500 + AND cs_item_sk = i_item_sk +GROUP BY i_item_id, + i_item_desc, + i_current_price +ORDER BY i_item_id +LIMIT 100; + diff --git a/testdata/tpcds/queries/q38.sql b/testdata/tpcds/queries/q38.sql new file mode 100644 index 00000000..eda90e83 --- /dev/null +++ b/testdata/tpcds/queries/q38.sql @@ -0,0 +1,29 @@ +SELECT count(*) +FROM + (SELECT DISTINCT c_last_name, + c_first_name, + d_date + FROM store_sales, + date_dim, + customer + WHERE store_sales.ss_sold_date_sk = date_dim.d_date_sk + AND store_sales.ss_customer_sk = customer.c_customer_sk + AND d_month_seq BETWEEN 1200 AND 1200 + 11 INTERSECT + SELECT DISTINCT c_last_name, + c_first_name, + d_date + FROM catalog_sales, + date_dim, + customer WHERE catalog_sales.cs_sold_date_sk = date_dim.d_date_sk + AND catalog_sales.cs_bill_customer_sk = customer.c_customer_sk + AND d_month_seq BETWEEN 1200 AND 1200 + 11 INTERSECT + SELECT DISTINCT c_last_name, + c_first_name, + d_date + FROM web_sales, + date_dim, + customer WHERE web_sales.ws_sold_date_sk = date_dim.d_date_sk + AND web_sales.ws_bill_customer_sk = customer.c_customer_sk + AND d_month_seq BETWEEN 1200 AND 1200 + 11 ) hot_cust +LIMIT 100; + diff --git a/testdata/tpcds/queries/q39.sql b/testdata/tpcds/queries/q39.sql new file mode 100644 index 00000000..62f3137a --- /dev/null +++ b/testdata/tpcds/queries/q39.sql @@ -0,0 +1,59 @@ +WITH inv AS + (SELECT w_warehouse_name, + w_warehouse_sk, + i_item_sk, + d_moy, + stdev, + mean, + CASE mean + WHEN 0 THEN NULL + ELSE stdev/mean + END cov + FROM + (SELECT w_warehouse_name, + w_warehouse_sk, + i_item_sk, + d_moy, + stddev_samp(inv_quantity_on_hand)*1.000 stdev, + avg(inv_quantity_on_hand) mean + FROM inventory, + item, + warehouse, + date_dim + WHERE inv_item_sk = i_item_sk + AND inv_warehouse_sk = w_warehouse_sk + AND inv_date_sk = d_date_sk + AND d_year =2001 + GROUP BY w_warehouse_name, + w_warehouse_sk, + i_item_sk, + d_moy) foo + WHERE CASE mean + WHEN 0 THEN 0 + ELSE stdev/mean + END > 1) +SELECT inv1.w_warehouse_sk wsk1, + inv1.i_item_sk isk1, + inv1.d_moy dmoy1, + inv1.mean mean1, + inv1.cov cov1, + inv2.w_warehouse_sk, + inv2.i_item_sk, + inv2.d_moy, + inv2.mean, + inv2.cov +FROM inv inv1, + inv inv2 +WHERE inv1.i_item_sk = inv2.i_item_sk + AND inv1.w_warehouse_sk = inv2.w_warehouse_sk + AND inv1.d_moy=1 + AND inv2.d_moy=1+1 +ORDER BY inv1.w_warehouse_sk NULLS FIRST, + inv1.i_item_sk NULLS FIRST, + inv1.d_moy NULLS FIRST, + inv1.mean NULLS FIRST, + inv1.cov NULLS FIRST, + inv2.d_moy NULLS FIRST, + inv2.mean NULLS FIRST, + inv2.cov NULLS FIRST; + diff --git a/testdata/tpcds/queries/q4.sql b/testdata/tpcds/queries/q4.sql new file mode 100644 index 00000000..fadb0a6c --- /dev/null +++ b/testdata/tpcds/queries/q4.sql @@ -0,0 +1,120 @@ +WITH year_total AS + (SELECT c_customer_id customer_id, + c_first_name customer_first_name, + c_last_name customer_last_name, + c_preferred_cust_flag customer_preferred_cust_flag, + c_birth_country customer_birth_country, + c_login customer_login, + c_email_address customer_email_address, + d_year dyear, + sum(((ss_ext_list_price-ss_ext_wholesale_cost-ss_ext_discount_amt)+ss_ext_sales_price)/2) year_total, + 's' sale_type + FROM customer, + store_sales, + date_dim + WHERE c_customer_sk = ss_customer_sk + AND ss_sold_date_sk = d_date_sk + GROUP BY c_customer_id, + c_first_name, + c_last_name, + c_preferred_cust_flag, + c_birth_country, + c_login, + c_email_address, + d_year + UNION ALL SELECT c_customer_id customer_id, + c_first_name customer_first_name, + c_last_name customer_last_name, + c_preferred_cust_flag customer_preferred_cust_flag, + c_birth_country customer_birth_country, + c_login customer_login, + c_email_address customer_email_address, + d_year dyear, + sum((((cs_ext_list_price-cs_ext_wholesale_cost-cs_ext_discount_amt)+cs_ext_sales_price)/2)) year_total, + 'c' sale_type + FROM customer, + catalog_sales, + date_dim + WHERE c_customer_sk = cs_bill_customer_sk + AND cs_sold_date_sk = d_date_sk + GROUP BY c_customer_id, + c_first_name, + c_last_name, + c_preferred_cust_flag, + c_birth_country, + c_login, + c_email_address, + d_year + UNION ALL SELECT c_customer_id customer_id, + c_first_name customer_first_name, + c_last_name customer_last_name, + c_preferred_cust_flag customer_preferred_cust_flag, + c_birth_country customer_birth_country, + c_login customer_login, + c_email_address customer_email_address, + d_year dyear, + sum((((ws_ext_list_price-ws_ext_wholesale_cost-ws_ext_discount_amt)+ws_ext_sales_price)/2)) year_total, + 'w' sale_type + FROM customer, + web_sales, + date_dim + WHERE c_customer_sk = ws_bill_customer_sk + AND ws_sold_date_sk = d_date_sk + GROUP BY c_customer_id, + c_first_name, + c_last_name, + c_preferred_cust_flag, + c_birth_country, + c_login, + c_email_address, + d_year) +SELECT t_s_secyear.customer_id, + t_s_secyear.customer_first_name, + t_s_secyear.customer_last_name, + t_s_secyear.customer_preferred_cust_flag +FROM year_total t_s_firstyear, + year_total t_s_secyear, + year_total t_c_firstyear, + year_total t_c_secyear, + year_total t_w_firstyear, + year_total t_w_secyear +WHERE t_s_secyear.customer_id = t_s_firstyear.customer_id + AND t_s_firstyear.customer_id = t_c_secyear.customer_id + AND t_s_firstyear.customer_id = t_c_firstyear.customer_id + AND t_s_firstyear.customer_id = t_w_firstyear.customer_id + AND t_s_firstyear.customer_id = t_w_secyear.customer_id + AND t_s_firstyear.sale_type = 's' + AND t_c_firstyear.sale_type = 'c' + AND t_w_firstyear.sale_type = 'w' + AND t_s_secyear.sale_type = 's' + AND t_c_secyear.sale_type = 'c' + AND t_w_secyear.sale_type = 'w' + AND t_s_firstyear.dyear = 2001 + AND t_s_secyear.dyear = 2001+1 + AND t_c_firstyear.dyear = 2001 + AND t_c_secyear.dyear = 2001+1 + AND t_w_firstyear.dyear = 2001 + AND t_w_secyear.dyear = 2001+1 + AND t_s_firstyear.year_total > 0 + AND t_c_firstyear.year_total > 0 + AND t_w_firstyear.year_total > 0 + AND CASE + WHEN t_c_firstyear.year_total > 0 THEN t_c_secyear.year_total / t_c_firstyear.year_total + ELSE NULL + END > CASE + WHEN t_s_firstyear.year_total > 0 THEN t_s_secyear.year_total / t_s_firstyear.year_total + ELSE NULL + END + AND CASE + WHEN t_c_firstyear.year_total > 0 THEN t_c_secyear.year_total / t_c_firstyear.year_total + ELSE NULL + END > CASE + WHEN t_w_firstyear.year_total > 0 THEN t_w_secyear.year_total / t_w_firstyear.year_total + ELSE NULL + END +ORDER BY t_s_secyear.customer_id NULLS FIRST, + t_s_secyear.customer_first_name NULLS FIRST, + t_s_secyear.customer_last_name NULLS FIRST, + t_s_secyear.customer_preferred_cust_flag NULLS FIRST +LIMIT 100; + diff --git a/testdata/tpcds/queries/q40.sql b/testdata/tpcds/queries/q40.sql new file mode 100644 index 00000000..8419ef3e --- /dev/null +++ b/testdata/tpcds/queries/q40.sql @@ -0,0 +1,26 @@ +SELECT w_state, + i_item_id, + sum(CASE + WHEN (cast(d_date AS date) < CAST ('2000-03-11' AS date)) THEN cs_sales_price - coalesce(cr_refunded_cash,0) + ELSE 0 + END) AS sales_before, + sum(CASE + WHEN (cast(d_date AS date) >= CAST ('2000-03-11' AS date)) THEN cs_sales_price - coalesce(cr_refunded_cash,0) + ELSE 0 + END) AS sales_after +FROM catalog_sales +LEFT OUTER JOIN catalog_returns ON (cs_order_number = cr_order_number + AND cs_item_sk = cr_item_sk) ,warehouse, + item, + date_dim +WHERE i_current_price BETWEEN 0.99 AND 1.49 + AND i_item_sk = cs_item_sk + AND cs_warehouse_sk = w_warehouse_sk + AND cs_sold_date_sk = d_date_sk + AND d_date BETWEEN CAST ('2000-02-10' AS date) AND CAST ('2000-04-10' AS date) +GROUP BY w_state, + i_item_id +ORDER BY w_state, + i_item_id +LIMIT 100; + diff --git a/testdata/tpcds/queries/q41.sql b/testdata/tpcds/queries/q41.sql new file mode 100644 index 00000000..b4cc52a1 --- /dev/null +++ b/testdata/tpcds/queries/q41.sql @@ -0,0 +1,67 @@ +SELECT distinct(i_product_name) +FROM item i1 +WHERE i_manufact_id BETWEEN 738 AND 738+40 + AND + (SELECT count(*) AS item_cnt + FROM item + WHERE (i_manufact = i1.i_manufact + AND ((i_category = 'Women' + AND (i_color = 'powder' + OR i_color = 'khaki') + AND (i_units = 'Ounce' + OR i_units = 'Oz') + AND (i_size = 'medium' + OR i_size = 'extra large')) + OR (i_category = 'Women' + AND (i_color = 'brown' + OR i_color = 'honeydew') + AND (i_units = 'Bunch' + OR i_units = 'Ton') + AND (i_size = 'N/A' + OR i_size = 'small')) + OR (i_category = 'Men' + AND (i_color = 'floral' + OR i_color = 'deep') + AND (i_units = 'N/A' + OR i_units = 'Dozen') + AND (i_size = 'petite' + OR i_size = 'petite')) + OR (i_category = 'Men' + AND (i_color = 'light' + OR i_color = 'cornflower') + AND (i_units = 'Box' + OR i_units = 'Pound') + AND (i_size = 'medium' + OR i_size = 'extra large')))) + OR (i_manufact = i1.i_manufact + AND ((i_category = 'Women' + AND (i_color = 'midnight' + OR i_color = 'snow') + AND (i_units = 'Pallet' + OR i_units = 'Gross') + AND (i_size = 'medium' + OR i_size = 'extra large')) + OR (i_category = 'Women' + AND (i_color = 'cyan' + OR i_color = 'papaya') + AND (i_units = 'Cup' + OR i_units = 'Dram') + AND (i_size = 'N/A' + OR i_size = 'small')) + OR (i_category = 'Men' + AND (i_color = 'orange' + OR i_color = 'frosted') + AND (i_units = 'Each' + OR i_units = 'Tbl') + AND (i_size = 'petite' + OR i_size = 'petite')) + OR (i_category = 'Men' + AND (i_color = 'forest' + OR i_color = 'ghost') + AND (i_units = 'Lb' + OR i_units = 'Bundle') + AND (i_size = 'medium' + OR i_size = 'extra large'))))) > 0 +ORDER BY i_product_name +LIMIT 100; + diff --git a/testdata/tpcds/queries/q42.sql b/testdata/tpcds/queries/q42.sql new file mode 100644 index 00000000..f5dba1e2 --- /dev/null +++ b/testdata/tpcds/queries/q42.sql @@ -0,0 +1,20 @@ +SELECT dt.d_year, + item.i_category_id, + item.i_category, + sum(ss_ext_sales_price) +FROM date_dim dt, + store_sales, + item +WHERE dt.d_date_sk = store_sales.ss_sold_date_sk + AND store_sales.ss_item_sk = item.i_item_sk + AND item.i_manager_id = 1 + AND dt.d_moy=11 + AND dt.d_year=2000 +GROUP BY dt.d_year, + item.i_category_id, + item.i_category +ORDER BY sum(ss_ext_sales_price) DESC,dt.d_year, + item.i_category_id, + item.i_category +LIMIT 100 ; + diff --git a/testdata/tpcds/queries/q43.sql b/testdata/tpcds/queries/q43.sql new file mode 100644 index 00000000..69cfd5ae --- /dev/null +++ b/testdata/tpcds/queries/q43.sql @@ -0,0 +1,51 @@ + +SELECT s_store_name, + s_store_id, + sum(CASE + WHEN (d_day_name='Sunday') THEN ss_sales_price + ELSE NULL + END) sun_sales, + sum(CASE + WHEN (d_day_name='Monday') THEN ss_sales_price + ELSE NULL + END) mon_sales, + sum(CASE + WHEN (d_day_name='Tuesday') THEN ss_sales_price + ELSE NULL + END) tue_sales, + sum(CASE + WHEN (d_day_name='Wednesday') THEN ss_sales_price + ELSE NULL + END) wed_sales, + sum(CASE + WHEN (d_day_name='Thursday') THEN ss_sales_price + ELSE NULL + END) thu_sales, + sum(CASE + WHEN (d_day_name='Friday') THEN ss_sales_price + ELSE NULL + END) fri_sales, + sum(CASE + WHEN (d_day_name='Saturday') THEN ss_sales_price + ELSE NULL + END) sat_sales +FROM date_dim, + store_sales, + store +WHERE d_date_sk = ss_sold_date_sk + AND s_store_sk = ss_store_sk + AND s_gmt_offset = -5 + AND d_year = 2000 +GROUP BY s_store_name, + s_store_id +ORDER BY s_store_name, + s_store_id, + sun_sales, + mon_sales, + tue_sales, + wed_sales, + thu_sales, + fri_sales, + sat_sales +LIMIT 100; + diff --git a/testdata/tpcds/queries/q44.sql b/testdata/tpcds/queries/q44.sql new file mode 100644 index 00000000..0943471b --- /dev/null +++ b/testdata/tpcds/queries/q44.sql @@ -0,0 +1,48 @@ +SELECT asceding.rnk, + i1.i_product_name best_performing, + i2.i_product_name worst_performing +FROM + (SELECT * + FROM + (SELECT item_sk, + rank() OVER ( + ORDER BY rank_col ASC) rnk + FROM + (SELECT ss_item_sk item_sk, + avg(ss_net_profit) rank_col + FROM store_sales ss1 + WHERE ss_store_sk = 4 + GROUP BY ss_item_sk + HAVING avg(ss_net_profit) > 0.9* + (SELECT avg(ss_net_profit) rank_col + FROM store_sales + WHERE ss_store_sk = 4 + AND ss_addr_sk IS NULL + GROUP BY ss_store_sk))V1)V11 + WHERE rnk < 11) asceding, + (SELECT * + FROM + (SELECT item_sk, + rank() OVER ( + ORDER BY rank_col DESC) rnk + FROM + (SELECT ss_item_sk item_sk, + avg(ss_net_profit) rank_col + FROM store_sales ss1 + WHERE ss_store_sk = 4 + GROUP BY ss_item_sk + HAVING avg(ss_net_profit) > 0.9* + (SELECT avg(ss_net_profit) rank_col + FROM store_sales + WHERE ss_store_sk = 4 + AND ss_addr_sk IS NULL + GROUP BY ss_store_sk))V2)V21 + WHERE rnk < 11) descending, + item i1, + item i2 +WHERE asceding.rnk = descending.rnk + AND i1.i_item_sk=asceding.item_sk + AND i2.i_item_sk=descending.item_sk +ORDER BY asceding.rnk +LIMIT 100; + diff --git a/testdata/tpcds/queries/q45.sql b/testdata/tpcds/queries/q45.sql new file mode 100644 index 00000000..a6495a8d --- /dev/null +++ b/testdata/tpcds/queries/q45.sql @@ -0,0 +1,42 @@ +SELECT ca_zip, + ca_city, + sum(ws_sales_price) +FROM web_sales, + customer, + customer_address, + date_dim, + item +WHERE ws_bill_customer_sk = c_customer_sk + AND c_current_addr_sk = ca_address_sk + AND ws_item_sk = i_item_sk + AND (SUBSTRING(ca_zip,1,5) IN ('85669', + '86197', + '88274', + '83405', + '86475', + '85392', + '85460', + '80348', + '81792') + OR i_item_id IN + (SELECT i_item_id + FROM item + WHERE i_item_sk IN (2, + 3, + 5, + 7, + 11, + 13, + 17, + 19, + 23, + 29) )) + AND ws_sold_date_sk = d_date_sk + AND d_qoy = 2 + AND d_year = 2001 +GROUP BY ca_zip, + ca_city +ORDER BY ca_zip, + ca_city +LIMIT 100; + diff --git a/testdata/tpcds/queries/q46.sql b/testdata/tpcds/queries/q46.sql new file mode 100644 index 00000000..e5981e7e --- /dev/null +++ b/testdata/tpcds/queries/q46.sql @@ -0,0 +1,48 @@ + +SELECT c_last_name, + c_first_name, + ca_city, + bought_city, + ss_ticket_number, + amt, + profit +FROM + (SELECT ss_ticket_number, + ss_customer_sk, + ca_city bought_city, + sum(ss_coupon_amt) amt, + sum(ss_net_profit) profit + FROM store_sales, + date_dim, + store, + household_demographics, + customer_address + WHERE store_sales.ss_sold_date_sk = date_dim.d_date_sk + AND store_sales.ss_store_sk = store.s_store_sk + AND store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk + AND store_sales.ss_addr_sk = customer_address.ca_address_sk + AND (household_demographics.hd_dep_count = 4 + OR household_demographics.hd_vehicle_count= 3) + AND date_dim.d_dow IN (6, + 0) + AND date_dim.d_year IN (1999, + 1999+1, + 1999+2) + AND store.s_city IN ('Fairview', + 'Midway') + GROUP BY ss_ticket_number, + ss_customer_sk, + ss_addr_sk, + ca_city) dn, + customer, + customer_address current_addr +WHERE ss_customer_sk = c_customer_sk + AND customer.c_current_addr_sk = current_addr.ca_address_sk + AND current_addr.ca_city <> bought_city +ORDER BY c_last_name NULLS FIRST, + c_first_name NULLS FIRST, + ca_city NULLS FIRST, + bought_city NULLS FIRST, + ss_ticket_number NULLS FIRST +LIMIT 100; + diff --git a/testdata/tpcds/queries/q47.sql b/testdata/tpcds/queries/q47.sql new file mode 100644 index 00000000..3a22bfca --- /dev/null +++ b/testdata/tpcds/queries/q47.sql @@ -0,0 +1,75 @@ +-- TPC-DS Query 47 +-- Modified: Added ORDER BY d_moy to avg() window function for DataFusion compatibility +WITH v1 AS + (SELECT i_category, + i_brand, + s_store_name, + s_company_name, + d_year, + d_moy, + sum(ss_sales_price) sum_sales, + avg(sum(ss_sales_price)) OVER (PARTITION BY i_category, + i_brand, + s_store_name, + s_company_name, + d_year + ORDER BY d_moy) avg_monthly_sales, + rank() OVER (PARTITION BY i_category, + i_brand, + s_store_name, + s_company_name + ORDER BY d_year, + d_moy) rn + FROM item, + store_sales, + date_dim, + store + WHERE ss_item_sk = i_item_sk + AND ss_sold_date_sk = d_date_sk + AND ss_store_sk = s_store_sk + AND (d_year = 1999 + OR (d_year = 1999-1 + AND d_moy =12) + OR (d_year = 1999+1 + AND d_moy =1)) + GROUP BY i_category, + i_brand, + s_store_name, + s_company_name, + d_year, + d_moy), + v2 AS + (SELECT v1.i_category, + v1.i_brand, + v1.s_store_name, + v1.s_company_name, + v1.d_year, + v1.d_moy, + v1.avg_monthly_sales, + v1.sum_sales, + v1_lag.sum_sales psum, + v1_lead.sum_sales nsum + FROM v1, + v1 v1_lag, + v1 v1_lead + WHERE v1.i_category = v1_lag.i_category + AND v1.i_category = v1_lead.i_category + AND v1.i_brand = v1_lag.i_brand + AND v1.i_brand = v1_lead.i_brand + AND v1.s_store_name = v1_lag.s_store_name + AND v1.s_store_name = v1_lead.s_store_name + AND v1.s_company_name = v1_lag.s_company_name + AND v1.s_company_name = v1_lead.s_company_name + AND v1.rn = v1_lag.rn + 1 + AND v1.rn = v1_lead.rn - 1) +SELECT * +FROM v2 +WHERE d_year = 1999 + AND avg_monthly_sales > 0 + AND CASE + WHEN avg_monthly_sales > 0 THEN abs(sum_sales - avg_monthly_sales) / avg_monthly_sales + ELSE NULL + END > 0.1 +ORDER BY sum_sales - avg_monthly_sales, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 +LIMIT 100; + diff --git a/testdata/tpcds/queries/q48.sql b/testdata/tpcds/queries/q48.sql new file mode 100644 index 00000000..607df470 --- /dev/null +++ b/testdata/tpcds/queries/q48.sql @@ -0,0 +1,40 @@ +SELECT SUM (ss_quantity) +FROM store_sales, + store, + customer_demographics, + customer_address, + date_dim +WHERE s_store_sk = ss_store_sk + AND ss_sold_date_sk = d_date_sk + AND d_year = 2000 + AND ((cd_demo_sk = ss_cdemo_sk + AND cd_marital_status = 'M' + AND cd_education_status = '4 yr Degree' + AND ss_sales_price BETWEEN 100.00 AND 150.00) + OR (cd_demo_sk = ss_cdemo_sk + AND cd_marital_status = 'D' + AND cd_education_status = '2 yr Degree' + AND ss_sales_price BETWEEN 50.00 AND 100.00) + OR (cd_demo_sk = ss_cdemo_sk + AND cd_marital_status = 'S' + AND cd_education_status = 'College' + AND ss_sales_price BETWEEN 150.00 AND 200.00)) + AND ((ss_addr_sk = ca_address_sk + AND ca_country = 'United States' + AND ca_state IN ('CO', + 'OH', + 'TX') + AND ss_net_profit BETWEEN 0 AND 2000) + OR (ss_addr_sk = ca_address_sk + AND ca_country = 'United States' + AND ca_state IN ('OR', + 'MN', + 'KY') + AND ss_net_profit BETWEEN 150 AND 3000) + OR (ss_addr_sk = ca_address_sk + AND ca_country = 'United States' + AND ca_state IN ('VA', + 'CA', + 'MS') + AND ss_net_profit BETWEEN 50 AND 25000)) ; + diff --git a/testdata/tpcds/queries/q49.sql b/testdata/tpcds/queries/q49.sql new file mode 100644 index 00000000..dd79e8c6 --- /dev/null +++ b/testdata/tpcds/queries/q49.sql @@ -0,0 +1,103 @@ + +SELECT channel, + item, + return_ratio, + return_rank, + currency_rank +FROM + (SELECT 'web' AS channel, + web.item, + web.return_ratio, + web.return_rank, + web.currency_rank + FROM + (SELECT item, + return_ratio, + currency_ratio, + rank() OVER ( + ORDER BY return_ratio) AS return_rank, + rank() OVER ( + ORDER BY currency_ratio) AS currency_rank + FROM + (SELECT ws.ws_item_sk AS item, + (cast(sum(coalesce(wr.wr_return_quantity,0)) AS decimal(15,4))/ cast(sum(coalesce(ws.ws_quantity,0)) AS decimal(15,4))) AS return_ratio, + (cast(sum(coalesce(wr.wr_return_amt,0)) AS decimal(15,4))/ cast(sum(coalesce(ws.ws_net_paid,0)) AS decimal(15,4))) AS currency_ratio + FROM web_sales ws + LEFT OUTER JOIN web_returns wr ON (ws.ws_order_number = wr.wr_order_number + AND ws.ws_item_sk = wr.wr_item_sk) ,date_dim + WHERE wr.wr_return_amt > 10000 + AND ws.ws_net_profit > 1 + AND ws.ws_net_paid > 0 + AND ws.ws_quantity > 0 + AND ws_sold_date_sk = d_date_sk + AND d_year = 2001 + AND d_moy = 12 + GROUP BY ws.ws_item_sk) in_web) web + WHERE (web.return_rank <= 10 + OR web.currency_rank <= 10) + UNION SELECT 'catalog' AS channel, + catalog.item, + catalog.return_ratio, + catalog.return_rank, + catalog.currency_rank + FROM + (SELECT item, + return_ratio, + currency_ratio, + rank() OVER ( + ORDER BY return_ratio) AS return_rank, + rank() OVER ( + ORDER BY currency_ratio) AS currency_rank + FROM + (SELECT cs.cs_item_sk AS item, + (cast(sum(coalesce(cr.cr_return_quantity,0)) AS decimal(15,4))/ cast(sum(coalesce(cs.cs_quantity,0)) AS decimal(15,4))) AS return_ratio, + (cast(sum(coalesce(cr.cr_return_amount,0)) AS decimal(15,4))/ cast(sum(coalesce(cs.cs_net_paid,0)) AS decimal(15,4))) AS currency_ratio + FROM catalog_sales cs + LEFT OUTER JOIN catalog_returns cr ON (cs.cs_order_number = cr.cr_order_number + AND cs.cs_item_sk = cr.cr_item_sk) ,date_dim + WHERE cr.cr_return_amount > 10000 + AND cs.cs_net_profit > 1 + AND cs.cs_net_paid > 0 + AND cs.cs_quantity > 0 + AND cs_sold_date_sk = d_date_sk + AND d_year = 2001 + AND d_moy = 12 + GROUP BY cs.cs_item_sk) in_cat) CATALOG + WHERE (catalog.return_rank <= 10 + OR catalog.currency_rank <=10) + UNION SELECT 'store' AS channel, + store.item, + store.return_ratio, + store.return_rank, + store.currency_rank + FROM + (SELECT item, + return_ratio, + currency_ratio, + rank() OVER ( + ORDER BY return_ratio) AS return_rank, + rank() OVER ( + ORDER BY currency_ratio) AS currency_rank + FROM + (SELECT sts.ss_item_sk AS item, + (cast(sum(coalesce(sr.sr_return_quantity,0)) AS decimal(15,4))/cast(sum(coalesce(sts.ss_quantity,0)) AS decimal(15,4))) AS return_ratio, + (cast(sum(coalesce(sr.sr_return_amt,0)) AS decimal(15,4))/cast(sum(coalesce(sts.ss_net_paid,0)) AS decimal(15,4))) AS currency_ratio + FROM store_sales sts + LEFT OUTER JOIN store_returns sr ON (sts.ss_ticket_number = sr.sr_ticket_number + AND sts.ss_item_sk = sr.sr_item_sk) ,date_dim + WHERE sr.sr_return_amt > 10000 + AND sts.ss_net_profit > 1 + AND sts.ss_net_paid > 0 + AND sts.ss_quantity > 0 + AND ss_sold_date_sk = d_date_sk + AND d_year = 2001 + AND d_moy = 12 + GROUP BY sts.ss_item_sk) in_store) store + WHERE (store.return_rank <= 10 + OR store.currency_rank <= 10) ) sq1 +ORDER BY 1 NULLS FIRST, + 4 NULLS FIRST, + 5 NULLS FIRST, + 2 NULLS FIRST +LIMIT 100; + diff --git a/testdata/tpcds/queries/q5.sql b/testdata/tpcds/queries/q5.sql new file mode 100644 index 00000000..7c9342dc --- /dev/null +++ b/testdata/tpcds/queries/q5.sql @@ -0,0 +1,112 @@ +WITH ssr AS + (SELECT s_store_id, + sum(sales_price) AS sales, + sum(profit) AS profit, + sum(return_amt) AS returns_, + sum(net_loss) AS profit_loss + FROM + (SELECT ss_store_sk AS store_sk, + ss_sold_date_sk AS date_sk, + ss_ext_sales_price AS sales_price, + ss_net_profit AS profit, + cast(0 AS decimal(7,2)) AS return_amt, + cast(0 AS decimal(7,2)) AS net_loss + FROM store_sales + UNION ALL SELECT sr_store_sk AS store_sk, + sr_returned_date_sk AS date_sk, + cast(0 AS decimal(7,2)) AS sales_price, + cast(0 AS decimal(7,2)) AS profit, + sr_return_amt AS return_amt, + sr_net_loss AS net_loss + FROM store_returns ) salesreturns, + date_dim, + store + WHERE date_sk = d_date_sk + AND d_date BETWEEN cast('2000-08-23' AS date) AND cast('2000-09-06' AS date) + AND store_sk = s_store_sk + GROUP BY s_store_id) , + csr AS + (SELECT cp_catalog_page_id, + sum(sales_price) AS sales, + sum(profit) AS profit, + sum(return_amt) AS returns_, + sum(net_loss) AS profit_loss + FROM + (SELECT cs_catalog_page_sk AS page_sk, + cs_sold_date_sk AS date_sk, + cs_ext_sales_price AS sales_price, + cs_net_profit AS profit, + cast(0 AS decimal(7,2)) AS return_amt, + cast(0 AS decimal(7,2)) AS net_loss + FROM catalog_sales + UNION ALL SELECT cr_catalog_page_sk AS page_sk, + cr_returned_date_sk AS date_sk, + cast(0 AS decimal(7,2)) AS sales_price, + cast(0 AS decimal(7,2)) AS profit, + cr_return_amount AS return_amt, + cr_net_loss AS net_loss + FROM catalog_returns ) salesreturns, + date_dim, + catalog_page + WHERE date_sk = d_date_sk + AND d_date BETWEEN cast('2000-08-23' AS date) AND cast('2000-09-06' AS date) + AND page_sk = cp_catalog_page_sk + GROUP BY cp_catalog_page_id) , + wsr AS + (SELECT web_site_id, + sum(sales_price) AS sales, + sum(profit) AS profit, + sum(return_amt) AS returns_, + sum(net_loss) AS profit_loss + FROM + (SELECT ws_web_site_sk AS wsr_web_site_sk, + ws_sold_date_sk AS date_sk, + ws_ext_sales_price AS sales_price, + ws_net_profit AS profit, + cast(0 AS decimal(7,2)) AS return_amt, + cast(0 AS decimal(7,2)) AS net_loss + FROM web_sales + UNION ALL SELECT ws_web_site_sk AS wsr_web_site_sk, + wr_returned_date_sk AS date_sk, + cast(0 AS decimal(7,2)) AS sales_price, + cast(0 AS decimal(7,2)) AS profit, + wr_return_amt AS return_amt, + wr_net_loss AS net_loss + FROM web_returns + LEFT OUTER JOIN web_sales ON (wr_item_sk = ws_item_sk + AND wr_order_number = ws_order_number) ) salesreturns, + date_dim, + web_site + WHERE date_sk = d_date_sk + AND d_date BETWEEN cast('2000-08-23' AS date) AND cast('2000-09-06' AS date) + AND wsr_web_site_sk = web_site_sk + GROUP BY web_site_id) +SELECT channel , + id , + sum(sales) AS sales , + sum(returns_) AS returns_ , + sum(profit) AS profit +FROM + (SELECT 'store channel' AS channel , + concat('store', s_store_id) AS id , + sales , + returns_ , + (profit - profit_loss) AS profit + FROM ssr + UNION ALL SELECT 'catalog channel' AS channel , + concat('catalog_page', cp_catalog_page_id) AS id , + sales , + returns_ , + (profit - profit_loss) AS profit + FROM csr + UNION ALL SELECT 'web channel' AS channel , + concat('web_site', web_site_id) AS id , + sales , + returns_ , + (profit - profit_loss) AS profit + FROM wsr ) x +GROUP BY ROLLUP (channel, + id) +ORDER BY channel NULLS FIRST, + id NULLS FIRST +LIMIT 100; diff --git a/testdata/tpcds/queries/q50.sql b/testdata/tpcds/queries/q50.sql new file mode 100644 index 00000000..d7a8975f --- /dev/null +++ b/testdata/tpcds/queries/q50.sql @@ -0,0 +1,68 @@ +SELECT s_store_name, + s_company_id, + s_street_number, + s_street_name, + s_street_type, + s_suite_number, + s_city, + s_county, + s_state, + s_zip, + sum(CASE + WHEN (sr_returned_date_sk - ss_sold_date_sk <= 30) THEN 1 + ELSE 0 + END) AS "30 days", + sum(CASE + WHEN (sr_returned_date_sk - ss_sold_date_sk > 30) + AND (sr_returned_date_sk - ss_sold_date_sk <= 60) THEN 1 + ELSE 0 + END) AS "31-60 days", + sum(CASE + WHEN (sr_returned_date_sk - ss_sold_date_sk > 60) + AND (sr_returned_date_sk - ss_sold_date_sk <= 90) THEN 1 + ELSE 0 + END) AS "61-90 days", + sum(CASE + WHEN (sr_returned_date_sk - ss_sold_date_sk > 90) + AND (sr_returned_date_sk - ss_sold_date_sk <= 120) THEN 1 + ELSE 0 + END) AS "91-120 days", + sum(CASE + WHEN (sr_returned_date_sk - ss_sold_date_sk > 120) THEN 1 + ELSE 0 + END) AS ">120 days" +FROM store_sales, + store_returns, + store, + date_dim d1, + date_dim d2 +WHERE d2.d_year = 2001 + AND d2.d_moy = 8 + AND ss_ticket_number = sr_ticket_number + AND ss_item_sk = sr_item_sk + AND ss_sold_date_sk = d1.d_date_sk + AND sr_returned_date_sk = d2.d_date_sk + AND ss_customer_sk = sr_customer_sk + AND ss_store_sk = s_store_sk +GROUP BY s_store_name, + s_company_id, + s_street_number, + s_street_name, + s_street_type, + s_suite_number, + s_city, + s_county, + s_state, + s_zip +ORDER BY s_store_name, + s_company_id, + s_street_number, + s_street_name, + s_street_type, + s_suite_number, + s_city, + s_county, + s_state, + s_zip +LIMIT 100; + diff --git a/testdata/tpcds/queries/q51.sql b/testdata/tpcds/queries/q51.sql new file mode 100644 index 00000000..71d34887 --- /dev/null +++ b/testdata/tpcds/queries/q51.sql @@ -0,0 +1,53 @@ +WITH web_v1 AS + (SELECT ws_item_sk item_sk, + d_date, + sum(sum(ws_sales_price)) OVER (PARTITION BY ws_item_sk + ORDER BY d_date ROWS BETWEEN unbounded preceding AND CURRENT ROW) cume_sales + FROM web_sales, + date_dim + WHERE ws_sold_date_sk=d_date_sk + AND d_month_seq BETWEEN 1200 AND 1200+11 + AND ws_item_sk IS NOT NULL + GROUP BY ws_item_sk, + d_date), + store_v1 AS + (SELECT ss_item_sk item_sk, + d_date, + sum(sum(ss_sales_price)) OVER (PARTITION BY ss_item_sk + ORDER BY d_date ROWS BETWEEN unbounded preceding AND CURRENT ROW) cume_sales + FROM store_sales, + date_dim + WHERE ss_sold_date_sk=d_date_sk + AND d_month_seq BETWEEN 1200 AND 1200+11 + AND ss_item_sk IS NOT NULL + GROUP BY ss_item_sk, + d_date) +SELECT * +FROM + (SELECT item_sk, + d_date, + web_sales, + store_sales, + max(web_sales) OVER (PARTITION BY item_sk + ORDER BY d_date ROWS BETWEEN unbounded preceding AND CURRENT ROW) web_cumulative, + max(store_sales) OVER (PARTITION BY item_sk + ORDER BY d_date ROWS BETWEEN unbounded preceding AND CURRENT ROW) store_cumulative + FROM + (SELECT CASE + WHEN web.item_sk IS NOT NULL THEN web.item_sk + ELSE store.item_sk + END item_sk, + CASE + WHEN web.d_date IS NOT NULL THEN web.d_date + ELSE store.d_date + END d_date, + web.cume_sales web_sales, + store.cume_sales store_sales + FROM web_v1 web + FULL OUTER JOIN store_v1 store ON (web.item_sk = store.item_sk + AND web.d_date = store.d_date))x)y +WHERE web_cumulative > store_cumulative +ORDER BY item_sk NULLS FIRST, + d_date NULLS FIRST +LIMIT 100; + diff --git a/testdata/tpcds/queries/q52.sql b/testdata/tpcds/queries/q52.sql new file mode 100644 index 00000000..4636c8f4 --- /dev/null +++ b/testdata/tpcds/queries/q52.sql @@ -0,0 +1,20 @@ +SELECT dt.d_year, + item.i_brand_id brand_id, + item.i_brand brand, + sum(ss_ext_sales_price) ext_price +FROM date_dim dt, + store_sales, + item +WHERE dt.d_date_sk = store_sales.ss_sold_date_sk + AND store_sales.ss_item_sk = item.i_item_sk + AND item.i_manager_id = 1 + AND dt.d_moy=11 + AND dt.d_year=2000 +GROUP BY dt.d_year, + item.i_brand, + item.i_brand_id +ORDER BY dt.d_year, + ext_price DESC, + brand_id +LIMIT 100 ; + diff --git a/testdata/tpcds/queries/q53.sql b/testdata/tpcds/queries/q53.sql new file mode 100644 index 00000000..6a80e479 --- /dev/null +++ b/testdata/tpcds/queries/q53.sql @@ -0,0 +1,48 @@ +SELECT * +FROM + (SELECT i_manufact_id, + sum(ss_sales_price) sum_sales, + avg(sum(ss_sales_price)) OVER (PARTITION BY i_manufact_id) avg_quarterly_sales + FROM item, + store_sales, + date_dim, + store + WHERE ss_item_sk = i_item_sk + AND ss_sold_date_sk = d_date_sk + AND ss_store_sk = s_store_sk + AND d_month_seq IN (1200, + 1200+1, + 1200+2, + 1200+3, + 1200+4, + 1200+5, + 1200+6, + 1200+7, + 1200+8, + 1200+9, + 1200+10, + 1200+11) + AND ((i_category IN ('Books', + 'Children', + 'Electronics') + AND i_class IN ('personal', + 'portable', + 'reference', + 'self-help') + AND i_brand IN ('scholaramalgamalg #14', + 'scholaramalgamalg #7', + 'exportiunivamalg #9', + 'scholaramalgamalg #9')) or(i_category IN ('Women','Music','Men') + AND i_class IN ('accessories','classical','fragrances','pants') + AND i_brand IN ('amalgimporto #1','edu packscholar #1','exportiimporto #1', 'importoamalg #1'))) + GROUP BY i_manufact_id, + d_qoy) tmp1 +WHERE CASE + WHEN avg_quarterly_sales > 0 THEN ABS (sum_sales - avg_quarterly_sales)/ avg_quarterly_sales + ELSE NULL + END > 0.1 +ORDER BY avg_quarterly_sales, + sum_sales, + i_manufact_id +LIMIT 100; + diff --git a/testdata/tpcds/queries/q54.sql b/testdata/tpcds/queries/q54.sql new file mode 100644 index 00000000..f045abc6 --- /dev/null +++ b/testdata/tpcds/queries/q54.sql @@ -0,0 +1,58 @@ +WITH my_customers AS + (SELECT DISTINCT c_customer_sk, + c_current_addr_sk + FROM + (SELECT cs_sold_date_sk sold_date_sk, + cs_bill_customer_sk customer_sk, + cs_item_sk item_sk + FROM catalog_sales + UNION ALL SELECT ws_sold_date_sk sold_date_sk, + ws_bill_customer_sk customer_sk, + ws_item_sk item_sk + FROM web_sales) cs_or_ws_sales, + item, + date_dim, + customer + WHERE sold_date_sk = d_date_sk + AND item_sk = i_item_sk + AND i_category = 'Women' + AND i_class = 'maternity' + AND c_customer_sk = cs_or_ws_sales.customer_sk + AND d_moy = 12 + AND d_year = 1998 ), + my_revenue AS + (SELECT c_customer_sk, + sum(ss_ext_sales_price) AS revenue + FROM my_customers, + store_sales, + customer_address, + store, + date_dim + WHERE c_current_addr_sk = ca_address_sk + AND ca_county = s_county + AND ca_state = s_state + AND ss_sold_date_sk = d_date_sk + AND c_customer_sk = ss_customer_sk + AND d_month_seq BETWEEN + (SELECT DISTINCT d_month_seq+1 + FROM date_dim + WHERE d_year = 1998 + AND d_moy = 12) AND + (SELECT DISTINCT d_month_seq+3 + FROM date_dim + WHERE d_year = 1998 + AND d_moy = 12) + GROUP BY c_customer_sk), + segments AS + (SELECT cast(round(revenue/50) AS int) AS SEGMENT + FROM my_revenue) +SELECT SEGMENT, + count(*) AS num_customers, + SEGMENT*50 AS segment_base +FROM segments +GROUP BY SEGMENT +ORDER BY SEGMENT NULLS FIRST, + num_customers NULLS FIRST, + segment_base +LIMIT 100; + diff --git a/testdata/tpcds/queries/q55.sql b/testdata/tpcds/queries/q55.sql new file mode 100644 index 00000000..159f1e61 --- /dev/null +++ b/testdata/tpcds/queries/q55.sql @@ -0,0 +1,17 @@ +SELECT i_brand_id brand_id, + i_brand brand, + sum(ss_ext_sales_price) ext_price +FROM date_dim, + store_sales, + item +WHERE d_date_sk = ss_sold_date_sk + AND ss_item_sk = i_item_sk + AND i_manager_id=28 + AND d_moy=11 + AND d_year=1999 +GROUP BY i_brand, + i_brand_id +ORDER BY ext_price DESC, + i_brand_id +LIMIT 100 ; + diff --git a/testdata/tpcds/queries/q56.sql b/testdata/tpcds/queries/q56.sql new file mode 100644 index 00000000..830ad222 --- /dev/null +++ b/testdata/tpcds/queries/q56.sql @@ -0,0 +1,74 @@ +WITH ss AS + (SELECT i_item_id, + sum(ss_ext_sales_price) total_sales + FROM store_sales, + date_dim, + customer_address, + item + WHERE i_item_id IN + (SELECT i_item_id + FROM item + WHERE i_color IN ('slate', + 'blanched', + 'burnished')) + AND ss_item_sk = i_item_sk + AND ss_sold_date_sk = d_date_sk + AND d_year = 2001 + AND d_moy = 2 + AND ss_addr_sk = ca_address_sk + AND ca_gmt_offset = -5 + GROUP BY i_item_id), + cs AS + (SELECT i_item_id, + sum(cs_ext_sales_price) total_sales + FROM catalog_sales, + date_dim, + customer_address, + item + WHERE i_item_id IN + (SELECT i_item_id + FROM item + WHERE i_color IN ('slate', + 'blanched', + 'burnished')) + AND cs_item_sk = i_item_sk + AND cs_sold_date_sk = d_date_sk + AND d_year = 2001 + AND d_moy = 2 + AND cs_bill_addr_sk = ca_address_sk + AND ca_gmt_offset = -5 + GROUP BY i_item_id), + ws AS + (SELECT i_item_id, + sum(ws_ext_sales_price) total_sales + FROM web_sales, + date_dim, + customer_address, + item + WHERE i_item_id IN + (SELECT i_item_id + FROM item + WHERE i_color IN ('slate', + 'blanched', + 'burnished')) + AND ws_item_sk = i_item_sk + AND ws_sold_date_sk = d_date_sk + AND d_year = 2001 + AND d_moy = 2 + AND ws_bill_addr_sk = ca_address_sk + AND ca_gmt_offset = -5 + GROUP BY i_item_id) +SELECT i_item_id, + sum(total_sales) total_sales +FROM + (SELECT * + FROM ss + UNION ALL SELECT * + FROM cs + UNION ALL SELECT * + FROM ws) tmp1 +GROUP BY i_item_id +ORDER BY total_sales NULLS FIRST, + i_item_id NULLS FIRST +LIMIT 100; + diff --git a/testdata/tpcds/queries/q57.sql b/testdata/tpcds/queries/q57.sql new file mode 100644 index 00000000..32eff5c6 --- /dev/null +++ b/testdata/tpcds/queries/q57.sql @@ -0,0 +1,66 @@ +WITH v1 AS + (SELECT i_category, + i_brand, + cc_name, + d_year, + d_moy, + sum(cs_sales_price) sum_sales, + avg(sum(cs_sales_price)) OVER (PARTITION BY i_category, + i_brand, + cc_name, + d_year + ORDER BY d_moy) avg_monthly_sales, -- Modified: Added ORDER BY d_moy to avg() window function for DataFusion compatibility DataFusion requires explicit ordering PARTITION BY + rank() OVER (PARTITION BY i_category, + i_brand, + cc_name + ORDER BY d_year, + d_moy) rn + FROM item, + catalog_sales, + date_dim, + call_center + WHERE cs_item_sk = i_item_sk + AND cs_sold_date_sk = d_date_sk + AND cc_call_center_sk= cs_call_center_sk + AND (d_year = 1999 + OR (d_year = 1999-1 + AND d_moy =12) + OR (d_year = 1999+1 + AND d_moy =1)) + GROUP BY i_category, + i_brand, + cc_name, + d_year, + d_moy), + v2 AS + (SELECT v1.i_category, + v1.i_brand, + v1.cc_name, + v1.d_year, + v1.d_moy, + v1.avg_monthly_sales, + v1.sum_sales, + v1_lag.sum_sales psum, + v1_lead.sum_sales nsum + FROM v1, + v1 v1_lag, + v1 v1_lead + WHERE v1.i_category = v1_lag.i_category + AND v1.i_category = v1_lead.i_category + AND v1.i_brand = v1_lag.i_brand + AND v1.i_brand = v1_lead.i_brand + AND v1. cc_name = v1_lag. cc_name + AND v1. cc_name = v1_lead. cc_name + AND v1.rn = v1_lag.rn + 1 + AND v1.rn = v1_lead.rn - 1) +SELECT * +FROM v2 +WHERE d_year = 1999 + AND avg_monthly_sales > 0 + AND CASE + WHEN avg_monthly_sales > 0 THEN abs(sum_sales - avg_monthly_sales) / avg_monthly_sales + ELSE NULL + END > 0.1 +ORDER BY sum_sales - avg_monthly_sales NULLS FIRST, 1, 2, 3, 4, 5, 6, 7, 8, 9 +LIMIT 100; + diff --git a/testdata/tpcds/queries/q58.sql b/testdata/tpcds/queries/q58.sql new file mode 100644 index 00000000..e1b4be8a --- /dev/null +++ b/testdata/tpcds/queries/q58.sql @@ -0,0 +1,71 @@ +WITH ss_items AS + (SELECT i_item_id item_id, + sum(ss_ext_sales_price) ss_item_rev + FROM store_sales, + item, + date_dim + WHERE ss_item_sk = i_item_sk + AND d_date IN + (SELECT d_date + FROM date_dim + WHERE d_week_seq = + (SELECT d_week_seq + FROM date_dim + WHERE d_date = '2000-01-03')) + AND ss_sold_date_sk = d_date_sk + GROUP BY i_item_id), + cs_items AS + (SELECT i_item_id item_id, + sum(cs_ext_sales_price) cs_item_rev + FROM catalog_sales, + item, + date_dim + WHERE cs_item_sk = i_item_sk + AND d_date IN + (SELECT d_date + FROM date_dim + WHERE d_week_seq = + (SELECT d_week_seq + FROM date_dim + WHERE d_date = '2000-01-03')) + AND cs_sold_date_sk = d_date_sk + GROUP BY i_item_id), + ws_items AS + (SELECT i_item_id item_id, + sum(ws_ext_sales_price) ws_item_rev + FROM web_sales, + item, + date_dim + WHERE ws_item_sk = i_item_sk + AND d_date IN + (SELECT d_date + FROM date_dim + WHERE d_week_seq = + (SELECT d_week_seq + FROM date_dim + WHERE d_date = '2000-01-03')) + AND ws_sold_date_sk = d_date_sk + GROUP BY i_item_id) +SELECT ss_items.item_id, + ss_item_rev, + ss_item_rev/((ss_item_rev+cs_item_rev+ws_item_rev)/3) * 100 ss_dev, + cs_item_rev, + cs_item_rev/((ss_item_rev+cs_item_rev+ws_item_rev)/3) * 100 cs_dev, + ws_item_rev, + ws_item_rev/((ss_item_rev+cs_item_rev+ws_item_rev)/3) * 100 ws_dev, + (ss_item_rev+cs_item_rev+ws_item_rev)/3 average +FROM ss_items, + cs_items, + ws_items +WHERE ss_items.item_id=cs_items.item_id + AND ss_items.item_id=ws_items.item_id + AND ss_item_rev BETWEEN 0.9 * cs_item_rev AND 1.1 * cs_item_rev + AND ss_item_rev BETWEEN 0.9 * ws_item_rev AND 1.1 * ws_item_rev + AND cs_item_rev BETWEEN 0.9 * ss_item_rev AND 1.1 * ss_item_rev + AND cs_item_rev BETWEEN 0.9 * ws_item_rev AND 1.1 * ws_item_rev + AND ws_item_rev BETWEEN 0.9 * ss_item_rev AND 1.1 * ss_item_rev + AND ws_item_rev BETWEEN 0.9 * cs_item_rev AND 1.1 * cs_item_rev +ORDER BY ss_items.item_id NULLS FIRST, + ss_item_rev NULLS FIRST +LIMIT 100; + diff --git a/testdata/tpcds/queries/q59.sql b/testdata/tpcds/queries/q59.sql new file mode 100644 index 00000000..b04cfe68 --- /dev/null +++ b/testdata/tpcds/queries/q59.sql @@ -0,0 +1,86 @@ +WITH wss AS + (SELECT d_week_seq, + ss_store_sk, + sum(CASE + WHEN (d_day_name='Sunday') THEN ss_sales_price + ELSE NULL + END) sun_sales, + sum(CASE + WHEN (d_day_name='Monday') THEN ss_sales_price + ELSE NULL + END) mon_sales, + sum(CASE + WHEN (d_day_name='Tuesday') THEN ss_sales_price + ELSE NULL + END) tue_sales, + sum(CASE + WHEN (d_day_name='Wednesday') THEN ss_sales_price + ELSE NULL + END) wed_sales, + sum(CASE + WHEN (d_day_name='Thursday') THEN ss_sales_price + ELSE NULL + END) thu_sales, + sum(CASE + WHEN (d_day_name='Friday') THEN ss_sales_price + ELSE NULL + END) fri_sales, + sum(CASE + WHEN (d_day_name='Saturday') THEN ss_sales_price + ELSE NULL + END) sat_sales + FROM store_sales, + date_dim + WHERE d_date_sk = ss_sold_date_sk + GROUP BY d_week_seq, + ss_store_sk) +SELECT s_store_name1, + s_store_id1, + d_week_seq1, + sun_sales1/sun_sales2 AS sun_sales_ratio, + mon_sales1/mon_sales2 AS mon_sales_ratio, + tue_sales1/tue_sales2 AS tue_sales_ratio, + wed_sales1/wed_sales2 AS wed_sales_ratio, + thu_sales1/thu_sales2 AS thu_sales_ratio, + fri_sales1/fri_sales2 AS fri_sales_ratio, + sat_sales1/sat_sales2 AS sat_sales_ratio +FROM + (SELECT s_store_name s_store_name1, + wss.d_week_seq d_week_seq1, + s_store_id s_store_id1, + sun_sales sun_sales1, + mon_sales mon_sales1, + tue_sales tue_sales1, + wed_sales wed_sales1, + thu_sales thu_sales1, + fri_sales fri_sales1, + sat_sales sat_sales1 + FROM wss, + store, + date_dim d + WHERE d.d_week_seq = wss.d_week_seq + AND ss_store_sk = s_store_sk + AND d_month_seq BETWEEN 1212 AND 1212 + 11) y, + (SELECT s_store_name s_store_name2, + wss.d_week_seq d_week_seq2, + s_store_id s_store_id2, + sun_sales sun_sales2, + mon_sales mon_sales2, + tue_sales tue_sales2, + wed_sales wed_sales2, + thu_sales thu_sales2, + fri_sales fri_sales2, + sat_sales sat_sales2 + FROM wss, + store, + date_dim d + WHERE d.d_week_seq = wss.d_week_seq + AND ss_store_sk = s_store_sk + AND d_month_seq BETWEEN 1212 + 12 AND 1212 + 23) x +WHERE s_store_id1=s_store_id2 + AND d_week_seq1=d_week_seq2-52 +ORDER BY s_store_name1 NULLS FIRST, + s_store_id1 NULLS FIRST, + d_week_seq1 NULLS FIRST +LIMIT 100; + diff --git a/testdata/tpcds/queries/q6.sql b/testdata/tpcds/queries/q6.sql new file mode 100644 index 00000000..172f8fed --- /dev/null +++ b/testdata/tpcds/queries/q6.sql @@ -0,0 +1,26 @@ +SELECT a.ca_state state, + count(*) cnt +FROM customer_address a , + customer c , + store_sales s , + date_dim d , + item i +WHERE a.ca_address_sk = c.c_current_addr_sk + AND c.c_customer_sk = s.ss_customer_sk + AND s.ss_sold_date_sk = d.d_date_sk + AND s.ss_item_sk = i.i_item_sk + AND d.d_month_seq = + (SELECT DISTINCT (d_month_seq) + FROM date_dim + WHERE d_year = 2001 + AND d_moy = 1 ) + AND i.i_current_price > 1.2 * + (SELECT avg(j.i_current_price) + FROM item j + WHERE j.i_category = i.i_category) +GROUP BY a.ca_state +HAVING count(*) >= 10 +ORDER BY cnt NULLS FIRST, + a.ca_state NULLS FIRST +LIMIT 100; + diff --git a/testdata/tpcds/queries/q60.sql b/testdata/tpcds/queries/q60.sql new file mode 100644 index 00000000..62612986 --- /dev/null +++ b/testdata/tpcds/queries/q60.sql @@ -0,0 +1,68 @@ +WITH ss AS + (SELECT i_item_id, + sum(ss_ext_sales_price) total_sales + FROM store_sales, + date_dim, + customer_address, + item + WHERE i_item_id IN + (SELECT i_item_id + FROM item + WHERE i_category ='Music') + AND ss_item_sk = i_item_sk + AND ss_sold_date_sk = d_date_sk + AND d_year = 1998 + AND d_moy = 9 + AND ss_addr_sk = ca_address_sk + AND ca_gmt_offset = -5 + GROUP BY i_item_id), + cs AS + (SELECT i_item_id, + sum(cs_ext_sales_price) total_sales + FROM catalog_sales, + date_dim, + customer_address, + item + WHERE i_item_id IN + (SELECT i_item_id + FROM item + WHERE i_category ='Music') + AND cs_item_sk = i_item_sk + AND cs_sold_date_sk = d_date_sk + AND d_year = 1998 + AND d_moy = 9 + AND cs_bill_addr_sk = ca_address_sk + AND ca_gmt_offset = -5 + GROUP BY i_item_id), + ws AS + (SELECT i_item_id, + sum(ws_ext_sales_price) total_sales + FROM web_sales, + date_dim, + customer_address, + item + WHERE i_item_id IN + (SELECT i_item_id + FROM item + WHERE i_category = 'Music') + AND ws_item_sk = i_item_sk + AND ws_sold_date_sk = d_date_sk + AND d_year = 1998 + AND d_moy = 9 + AND ws_bill_addr_sk = ca_address_sk + AND ca_gmt_offset = -5 + GROUP BY i_item_id) +SELECT i_item_id, + sum(total_sales) total_sales +FROM + (SELECT * + FROM ss + UNION ALL SELECT * + FROM cs + UNION ALL SELECT * + FROM ws) tmp1 +GROUP BY i_item_id +ORDER BY i_item_id, + total_sales +LIMIT 100; + diff --git a/testdata/tpcds/queries/q61.sql b/testdata/tpcds/queries/q61.sql new file mode 100644 index 00000000..5a527f7a --- /dev/null +++ b/testdata/tpcds/queries/q61.sql @@ -0,0 +1,47 @@ +SELECT promotions, + total, + cast(promotions AS decimal(15,4))/cast(total AS decimal(15,4))*100 +FROM + (SELECT sum(ss_ext_sales_price) promotions + FROM store_sales, + store, + promotion, + date_dim, + customer, + customer_address, + item + WHERE ss_sold_date_sk = d_date_sk + AND ss_store_sk = s_store_sk + AND ss_promo_sk = p_promo_sk + AND ss_customer_sk= c_customer_sk + AND ca_address_sk = c_current_addr_sk + AND ss_item_sk = i_item_sk + AND ca_gmt_offset = -5 + AND i_category = 'Jewelry' + AND (p_channel_dmail = 'Y' + OR p_channel_email = 'Y' + OR p_channel_tv = 'Y') + AND s_gmt_offset = -5 + AND d_year = 1998 + AND d_moy = 11) promotional_sales, + (SELECT sum(ss_ext_sales_price) total + FROM store_sales, + store, + date_dim, + customer, + customer_address, + item + WHERE ss_sold_date_sk = d_date_sk + AND ss_store_sk = s_store_sk + AND ss_customer_sk= c_customer_sk + AND ca_address_sk = c_current_addr_sk + AND ss_item_sk = i_item_sk + AND ca_gmt_offset = -5 + AND i_category = 'Jewelry' + AND s_gmt_offset = -5 + AND d_year = 1998 + AND d_moy = 11) all_sales +ORDER BY promotions, + total +LIMIT 100; + diff --git a/testdata/tpcds/queries/q62.sql b/testdata/tpcds/queries/q62.sql new file mode 100644 index 00000000..dc91b58b --- /dev/null +++ b/testdata/tpcds/queries/q62.sql @@ -0,0 +1,46 @@ +SELECT w_substr, + sm_type, + web_name, + sum(CASE + WHEN (ws_ship_date_sk - ws_sold_date_sk <= 30) THEN 1 + ELSE 0 + END) AS "30 days", + sum(CASE + WHEN (ws_ship_date_sk - ws_sold_date_sk > 30) + AND (ws_ship_date_sk - ws_sold_date_sk <= 60) THEN 1 + ELSE 0 + END) AS "31-60 days", + sum(CASE + WHEN (ws_ship_date_sk - ws_sold_date_sk > 60) + AND (ws_ship_date_sk - ws_sold_date_sk <= 90) THEN 1 + ELSE 0 + END) AS "61-90 days", + sum(CASE + WHEN (ws_ship_date_sk - ws_sold_date_sk > 90) + AND (ws_ship_date_sk - ws_sold_date_sk <= 120) THEN 1 + ELSE 0 + END) AS "91-120 days", + sum(CASE + WHEN (ws_ship_date_sk - ws_sold_date_sk > 120) THEN 1 + ELSE 0 + END) AS ">120 days" +FROM web_sales, + (SELECT SUBSTRING(w_warehouse_name,1,20) w_substr, + * + FROM warehouse) sq1, + ship_mode, + web_site, + date_dim +WHERE d_month_seq BETWEEN 1200 AND 1200 + 11 + AND ws_ship_date_sk = d_date_sk + AND ws_warehouse_sk = w_warehouse_sk + AND ws_ship_mode_sk = sm_ship_mode_sk + AND ws_web_site_sk = web_site_sk +GROUP BY w_substr, + sm_type, + web_name +ORDER BY 1 NULLS FIRST, + 2 NULLS FIRST, + 3 NULLS FIRST +LIMIT 100; + diff --git a/testdata/tpcds/queries/q63.sql b/testdata/tpcds/queries/q63.sql new file mode 100644 index 00000000..8ab312dc --- /dev/null +++ b/testdata/tpcds/queries/q63.sql @@ -0,0 +1,49 @@ + +SELECT * +FROM + (SELECT i_manager_id, + sum(ss_sales_price) sum_sales, + avg(sum(ss_sales_price)) OVER (PARTITION BY i_manager_id) avg_monthly_sales + FROM item, + store_sales, + date_dim, + store + WHERE ss_item_sk = i_item_sk + AND ss_sold_date_sk = d_date_sk + AND ss_store_sk = s_store_sk + AND d_month_seq IN (1200, + 1200+1, + 1200+2, + 1200+3, + 1200+4, + 1200+5, + 1200+6, + 1200+7, + 1200+8, + 1200+9, + 1200+10, + 1200+11) + AND ((i_category IN ('Books', + 'Children', + 'Electronics') + AND i_class IN ('personal', + 'portable', + 'reference', + 'self-help') + AND i_brand IN ('scholaramalgamalg #14', + 'scholaramalgamalg #7', + 'exportiunivamalg #9', + 'scholaramalgamalg #9')) or(i_category IN ('Women','Music','Men') + AND i_class IN ('accessories','classical','fragrances','pants') + AND i_brand IN ('amalgimporto #1','edu packscholar #1','exportiimporto #1', 'importoamalg #1'))) + GROUP BY i_manager_id, + d_moy) tmp1 +WHERE CASE + WHEN avg_monthly_sales > 0 THEN ABS (sum_sales - avg_monthly_sales) / avg_monthly_sales + ELSE NULL + END > 0.1 +ORDER BY i_manager_id, + avg_monthly_sales, + sum_sales +LIMIT 100; + diff --git a/testdata/tpcds/queries/q64.sql b/testdata/tpcds/queries/q64.sql new file mode 100644 index 00000000..1cc445a2 --- /dev/null +++ b/testdata/tpcds/queries/q64.sql @@ -0,0 +1,125 @@ +WITH cs_ui AS + (SELECT cs_item_sk, + sum(cs_ext_list_price) AS sale, + sum(cr_refunded_cash+cr_reversed_charge+cr_store_credit) AS refund + FROM catalog_sales, + catalog_returns + WHERE cs_item_sk = cr_item_sk + AND cs_order_number = cr_order_number + GROUP BY cs_item_sk + HAVING sum(cs_ext_list_price)>2*sum(cr_refunded_cash+cr_reversed_charge+cr_store_credit)), + cross_sales AS + (SELECT i_product_name product_name, + i_item_sk item_sk, + s_store_name store_name, + s_zip store_zip, + ad1.ca_street_number b_street_number, + ad1.ca_street_name b_street_name, + ad1.ca_city b_city, + ad1.ca_zip b_zip, + ad2.ca_street_number c_street_number, + ad2.ca_street_name c_street_name, + ad2.ca_city c_city, + ad2.ca_zip c_zip, + d1.d_year AS syear, + d2.d_year AS fsyear, + d3.d_year s2year, + count(*) cnt, + sum(ss_wholesale_cost) s1, + sum(ss_list_price) s2, + sum(ss_coupon_amt) s3 + FROM store_sales, + store_returns, + cs_ui, + date_dim d1, + date_dim d2, + date_dim d3, + store, + customer, + customer_demographics cd1, + customer_demographics cd2, + promotion, + household_demographics hd1, + household_demographics hd2, + customer_address ad1, + customer_address ad2, + income_band ib1, + income_band ib2, + item + WHERE ss_store_sk = s_store_sk + AND ss_sold_date_sk = d1.d_date_sk + AND ss_customer_sk = c_customer_sk + AND ss_cdemo_sk= cd1.cd_demo_sk + AND ss_hdemo_sk = hd1.hd_demo_sk + AND ss_addr_sk = ad1.ca_address_sk + AND ss_item_sk = i_item_sk + AND ss_item_sk = sr_item_sk + AND ss_ticket_number = sr_ticket_number + AND ss_item_sk = cs_ui.cs_item_sk + AND c_current_cdemo_sk = cd2.cd_demo_sk + AND c_current_hdemo_sk = hd2.hd_demo_sk + AND c_current_addr_sk = ad2.ca_address_sk + AND c_first_sales_date_sk = d2.d_date_sk + AND c_first_shipto_date_sk = d3.d_date_sk + AND ss_promo_sk = p_promo_sk + AND hd1.hd_income_band_sk = ib1.ib_income_band_sk + AND hd2.hd_income_band_sk = ib2.ib_income_band_sk + AND cd1.cd_marital_status <> cd2.cd_marital_status + AND i_color IN ('purple', + 'burlywood', + 'indian', + 'spring', + 'floral', + 'medium') + AND i_current_price BETWEEN 64 AND 64 + 10 + AND i_current_price BETWEEN 64 + 1 AND 64 + 15 + GROUP BY i_product_name, + i_item_sk, + s_store_name, + s_zip, + ad1.ca_street_number, + ad1.ca_street_name, + ad1.ca_city, + ad1.ca_zip, + ad2.ca_street_number, + ad2.ca_street_name, + ad2.ca_city, + ad2.ca_zip, + d1.d_year, + d2.d_year, + d3.d_year) +SELECT cs1.product_name, + cs1.store_name, + cs1.store_zip, + cs1.b_street_number, + cs1.b_street_name, + cs1.b_city, + cs1.b_zip, + cs1.c_street_number, + cs1.c_street_name, + cs1.c_city, + cs1.c_zip, + cs1.syear cs1syear, + cs1.cnt cs1cnt, + cs1.s1 AS s11, + cs1.s2 AS s21, + cs1.s3 AS s31, + cs2.s1 AS s12, + cs2.s2 AS s22, + cs2.s3 AS s32, + cs2.syear, + cs2.cnt +FROM cross_sales cs1, + cross_sales cs2 +WHERE cs1.item_sk=cs2.item_sk + AND cs1.syear = 1999 + AND cs2.syear = 1999 + 1 + AND cs2.cnt <= cs1.cnt + AND cs1.store_name = cs2.store_name + AND cs1.store_zip = cs2.store_zip +ORDER BY cs1.product_name, + cs1.store_name, + cs2.cnt, + cs1.s1, + cs2.s1; + diff --git a/testdata/tpcds/queries/q65.sql b/testdata/tpcds/queries/q65.sql new file mode 100644 index 00000000..671ffa87 --- /dev/null +++ b/testdata/tpcds/queries/q65.sql @@ -0,0 +1,38 @@ +SELECT s_store_name, + i_item_desc, + sc.revenue, + i_current_price, + i_wholesale_cost, + i_brand +FROM store, + item, + (SELECT ss_store_sk, + avg(revenue) AS ave + FROM + (SELECT ss_store_sk, + ss_item_sk, + sum(ss_sales_price) AS revenue + FROM store_sales, + date_dim + WHERE ss_sold_date_sk = d_date_sk + AND d_month_seq BETWEEN 1176 AND 1176+11 + GROUP BY ss_store_sk, + ss_item_sk) sa + GROUP BY ss_store_sk) sb, + (SELECT ss_store_sk, + ss_item_sk, + sum(ss_sales_price) AS revenue + FROM store_sales, + date_dim + WHERE ss_sold_date_sk = d_date_sk + AND d_month_seq BETWEEN 1176 AND 1176+11 + GROUP BY ss_store_sk, + ss_item_sk) sc +WHERE sb.ss_store_sk = sc.ss_store_sk + AND sc.revenue <= 0.1 * sb.ave + AND s_store_sk = sc.ss_store_sk + AND i_item_sk = sc.ss_item_sk +ORDER BY s_store_name NULLS FIRST, + i_item_desc NULLS FIRST +LIMIT 100; + diff --git a/testdata/tpcds/queries/q66.sql b/testdata/tpcds/queries/q66.sql new file mode 100644 index 00000000..2ebbd2bb --- /dev/null +++ b/testdata/tpcds/queries/q66.sql @@ -0,0 +1,218 @@ +select + w_warehouse_name + ,w_warehouse_sq_ft + ,w_city + ,w_county + ,w_state + ,w_country + ,ship_carriers + ,year_ + ,sum(jan_sales) as jan_sales + ,sum(feb_sales) as feb_sales + ,sum(mar_sales) as mar_sales + ,sum(apr_sales) as apr_sales + ,sum(may_sales) as may_sales + ,sum(jun_sales) as jun_sales + ,sum(jul_sales) as jul_sales + ,sum(aug_sales) as aug_sales + ,sum(sep_sales) as sep_sales + ,sum(oct_sales) as oct_sales + ,sum(nov_sales) as nov_sales + ,sum(dec_sales) as dec_sales + ,sum(jan_sales/w_warehouse_sq_ft) as jan_sales_per_sq_foot + ,sum(feb_sales/w_warehouse_sq_ft) as feb_sales_per_sq_foot + ,sum(mar_sales/w_warehouse_sq_ft) as mar_sales_per_sq_foot + ,sum(apr_sales/w_warehouse_sq_ft) as apr_sales_per_sq_foot + ,sum(may_sales/w_warehouse_sq_ft) as may_sales_per_sq_foot + ,sum(jun_sales/w_warehouse_sq_ft) as jun_sales_per_sq_foot + ,sum(jul_sales/w_warehouse_sq_ft) as jul_sales_per_sq_foot + ,sum(aug_sales/w_warehouse_sq_ft) as aug_sales_per_sq_foot + ,sum(sep_sales/w_warehouse_sq_ft) as sep_sales_per_sq_foot + ,sum(oct_sales/w_warehouse_sq_ft) as oct_sales_per_sq_foot + ,sum(nov_sales/w_warehouse_sq_ft) as nov_sales_per_sq_foot + ,sum(dec_sales/w_warehouse_sq_ft) as dec_sales_per_sq_foot + ,sum(jan_net) as jan_net + ,sum(feb_net) as feb_net + ,sum(mar_net) as mar_net + ,sum(apr_net) as apr_net + ,sum(may_net) as may_net + ,sum(jun_net) as jun_net + ,sum(jul_net) as jul_net + ,sum(aug_net) as aug_net + ,sum(sep_net) as sep_net + ,sum(oct_net) as oct_net + ,sum(nov_net) as nov_net + ,sum(dec_net) as dec_net + from ( + select + w_warehouse_name + ,w_warehouse_sq_ft + ,w_city + ,w_county + ,w_state + ,w_country + ,'DHL,BARIAN' as ship_carriers + ,d_year as year_ + ,sum(case when d_moy = 1 + then ws_ext_sales_price* ws_quantity else 0 end) as jan_sales + ,sum(case when d_moy = 2 + then ws_ext_sales_price* ws_quantity else 0 end) as feb_sales + ,sum(case when d_moy = 3 + then ws_ext_sales_price* ws_quantity else 0 end) as mar_sales + ,sum(case when d_moy = 4 + then ws_ext_sales_price* ws_quantity else 0 end) as apr_sales + ,sum(case when d_moy = 5 + then ws_ext_sales_price* ws_quantity else 0 end) as may_sales + ,sum(case when d_moy = 6 + then ws_ext_sales_price* ws_quantity else 0 end) as jun_sales + ,sum(case when d_moy = 7 + then ws_ext_sales_price* ws_quantity else 0 end) as jul_sales + ,sum(case when d_moy = 8 + then ws_ext_sales_price* ws_quantity else 0 end) as aug_sales + ,sum(case when d_moy = 9 + then ws_ext_sales_price* ws_quantity else 0 end) as sep_sales + ,sum(case when d_moy = 10 + then ws_ext_sales_price* ws_quantity else 0 end) as oct_sales + ,sum(case when d_moy = 11 + then ws_ext_sales_price* ws_quantity else 0 end) as nov_sales + ,sum(case when d_moy = 12 + then ws_ext_sales_price* ws_quantity else 0 end) as dec_sales + ,sum(case when d_moy = 1 + then ws_net_paid * ws_quantity else 0 end) as jan_net + ,sum(case when d_moy = 2 + then ws_net_paid * ws_quantity else 0 end) as feb_net + ,sum(case when d_moy = 3 + then ws_net_paid * ws_quantity else 0 end) as mar_net + ,sum(case when d_moy = 4 + then ws_net_paid * ws_quantity else 0 end) as apr_net + ,sum(case when d_moy = 5 + then ws_net_paid * ws_quantity else 0 end) as may_net + ,sum(case when d_moy = 6 + then ws_net_paid * ws_quantity else 0 end) as jun_net + ,sum(case when d_moy = 7 + then ws_net_paid * ws_quantity else 0 end) as jul_net + ,sum(case when d_moy = 8 + then ws_net_paid * ws_quantity else 0 end) as aug_net + ,sum(case when d_moy = 9 + then ws_net_paid * ws_quantity else 0 end) as sep_net + ,sum(case when d_moy = 10 + then ws_net_paid * ws_quantity else 0 end) as oct_net + ,sum(case when d_moy = 11 + then ws_net_paid * ws_quantity else 0 end) as nov_net + ,sum(case when d_moy = 12 + then ws_net_paid * ws_quantity else 0 end) as dec_net + from + web_sales + ,warehouse + ,date_dim + ,time_dim + ,ship_mode + where + ws_warehouse_sk = w_warehouse_sk + and ws_sold_date_sk = d_date_sk + and ws_sold_time_sk = t_time_sk + and ws_ship_mode_sk = sm_ship_mode_sk + and d_year = 2001 + and t_time between 30838 and 30838+28800 + and sm_carrier in ('DHL','BARIAN') + group by + w_warehouse_name + ,w_warehouse_sq_ft + ,w_city + ,w_county + ,w_state + ,w_country + ,d_year + union all + select + w_warehouse_name + ,w_warehouse_sq_ft + ,w_city + ,w_county + ,w_state + ,w_country + ,'DHL,BARIAN' as ship_carriers + ,d_year as year_ + ,sum(case when d_moy = 1 + then cs_sales_price* cs_quantity else 0 end) as jan_sales + ,sum(case when d_moy = 2 + then cs_sales_price* cs_quantity else 0 end) as feb_sales + ,sum(case when d_moy = 3 + then cs_sales_price* cs_quantity else 0 end) as mar_sales + ,sum(case when d_moy = 4 + then cs_sales_price* cs_quantity else 0 end) as apr_sales + ,sum(case when d_moy = 5 + then cs_sales_price* cs_quantity else 0 end) as may_sales + ,sum(case when d_moy = 6 + then cs_sales_price* cs_quantity else 0 end) as jun_sales + ,sum(case when d_moy = 7 + then cs_sales_price* cs_quantity else 0 end) as jul_sales + ,sum(case when d_moy = 8 + then cs_sales_price* cs_quantity else 0 end) as aug_sales + ,sum(case when d_moy = 9 + then cs_sales_price* cs_quantity else 0 end) as sep_sales + ,sum(case when d_moy = 10 + then cs_sales_price* cs_quantity else 0 end) as oct_sales + ,sum(case when d_moy = 11 + then cs_sales_price* cs_quantity else 0 end) as nov_sales + ,sum(case when d_moy = 12 + then cs_sales_price* cs_quantity else 0 end) as dec_sales + ,sum(case when d_moy = 1 + then cs_net_paid_inc_tax * cs_quantity else 0 end) as jan_net + ,sum(case when d_moy = 2 + then cs_net_paid_inc_tax * cs_quantity else 0 end) as feb_net + ,sum(case when d_moy = 3 + then cs_net_paid_inc_tax * cs_quantity else 0 end) as mar_net + ,sum(case when d_moy = 4 + then cs_net_paid_inc_tax * cs_quantity else 0 end) as apr_net + ,sum(case when d_moy = 5 + then cs_net_paid_inc_tax * cs_quantity else 0 end) as may_net + ,sum(case when d_moy = 6 + then cs_net_paid_inc_tax * cs_quantity else 0 end) as jun_net + ,sum(case when d_moy = 7 + then cs_net_paid_inc_tax * cs_quantity else 0 end) as jul_net + ,sum(case when d_moy = 8 + then cs_net_paid_inc_tax * cs_quantity else 0 end) as aug_net + ,sum(case when d_moy = 9 + then cs_net_paid_inc_tax * cs_quantity else 0 end) as sep_net + ,sum(case when d_moy = 10 + then cs_net_paid_inc_tax * cs_quantity else 0 end) as oct_net + ,sum(case when d_moy = 11 + then cs_net_paid_inc_tax * cs_quantity else 0 end) as nov_net + ,sum(case when d_moy = 12 + then cs_net_paid_inc_tax * cs_quantity else 0 end) as dec_net + from + catalog_sales + ,warehouse + ,date_dim + ,time_dim + ,ship_mode + where + cs_warehouse_sk = w_warehouse_sk + and cs_sold_date_sk = d_date_sk + and cs_sold_time_sk = t_time_sk + and cs_ship_mode_sk = sm_ship_mode_sk + and d_year = 2001 + and t_time between 30838 AND 30838+28800 + and sm_carrier in ('DHL','BARIAN') + group by + w_warehouse_name + ,w_warehouse_sq_ft + ,w_city + ,w_county + ,w_state + ,w_country + ,d_year + ) x + group by + w_warehouse_name + ,w_warehouse_sq_ft + ,w_city + ,w_county + ,w_state + ,w_country + ,ship_carriers + ,year_ + order by w_warehouse_name NULLS FIRST +LIMIT 100; + diff --git a/testdata/tpcds/queries/q67.sql b/testdata/tpcds/queries/q67.sql new file mode 100644 index 00000000..d27ba658 --- /dev/null +++ b/testdata/tpcds/queries/q67.sql @@ -0,0 +1,44 @@ +SELECT * +FROM + (SELECT i_category, + i_class, + i_brand, + i_product_name, + d_year, + d_qoy, + d_moy, + s_store_id, + sumsales, + rank() OVER (PARTITION BY i_category + ORDER BY sumsales DESC) rk + FROM + (SELECT i_category, + i_class, + i_brand, + i_product_name, + d_year, + d_qoy, + d_moy, + s_store_id, + sum(coalesce(ss_sales_price*ss_quantity,0)) sumsales + FROM store_sales, + date_dim, + store, + item + WHERE ss_sold_date_sk=d_date_sk + AND ss_item_sk=i_item_sk + AND ss_store_sk = s_store_sk + AND d_month_seq BETWEEN 1200 AND 1200+11 + GROUP BY rollup(i_category, i_class, i_brand, i_product_name, d_year, d_qoy, d_moy,s_store_id))dw1) dw2 +WHERE rk <= 100 +ORDER BY i_category NULLS FIRST, + i_class NULLS FIRST, + i_brand NULLS FIRST, + i_product_name NULLS FIRST, + d_year NULLS FIRST, + d_qoy NULLS FIRST, + d_moy NULLS FIRST, + s_store_id NULLS FIRST, + sumsales NULLS FIRST, + rk NULLS FIRST +LIMIT 100; diff --git a/testdata/tpcds/queries/q68.sql b/testdata/tpcds/queries/q68.sql new file mode 100644 index 00000000..8af38028 --- /dev/null +++ b/testdata/tpcds/queries/q68.sql @@ -0,0 +1,45 @@ +SELECT c_last_name, + c_first_name, + ca_city, + bought_city, + ss_ticket_number, + extended_price, + extended_tax, + list_price +FROM + (SELECT ss_ticket_number, + ss_customer_sk, + ca_city bought_city, + sum(ss_ext_sales_price) extended_price, + sum(ss_ext_list_price) list_price, + sum(ss_ext_tax) extended_tax + FROM store_sales, + date_dim, + store, + household_demographics, + customer_address + WHERE store_sales.ss_sold_date_sk = date_dim.d_date_sk + AND store_sales.ss_store_sk = store.s_store_sk + AND store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk + AND store_sales.ss_addr_sk = customer_address.ca_address_sk + AND date_dim.d_dom BETWEEN 1 AND 2 + AND (household_demographics.hd_dep_count = 4 + OR household_demographics.hd_vehicle_count= 3) + AND date_dim.d_year IN (1999, + 1999+1, + 1999+2) + AND store.s_city IN ('Fairview', + 'Midway') + GROUP BY ss_ticket_number, + ss_customer_sk, + ss_addr_sk, + ca_city) dn, + customer, + customer_address current_addr +WHERE ss_customer_sk = c_customer_sk + AND customer.c_current_addr_sk = current_addr.ca_address_sk + AND current_addr.ca_city <> bought_city +ORDER BY c_last_name NULLS FIRST, + ss_ticket_number NULLS FIRST +LIMIT 100; + diff --git a/testdata/tpcds/queries/q69.sql b/testdata/tpcds/queries/q69.sql new file mode 100644 index 00000000..134bbcb3 --- /dev/null +++ b/testdata/tpcds/queries/q69.sql @@ -0,0 +1,52 @@ +SELECT cd_gender, + cd_marital_status, + cd_education_status, + count(*) cnt1, + cd_purchase_estimate, + count(*) cnt2, + cd_credit_rating, + count(*) cnt3 +FROM customer c, + customer_address ca, + customer_demographics +WHERE c.c_current_addr_sk = ca.ca_address_sk + AND ca_state IN ('KY', + 'GA', + 'NM') + AND cd_demo_sk = c.c_current_cdemo_sk + AND EXISTS + (SELECT * + FROM store_sales, + date_dim + WHERE c.c_customer_sk = ss_customer_sk + AND ss_sold_date_sk = d_date_sk + AND d_year = 2001 + AND d_moy BETWEEN 4 AND 4+2) + AND (NOT EXISTS + (SELECT * + FROM web_sales, + date_dim + WHERE c.c_customer_sk = ws_bill_customer_sk + AND ws_sold_date_sk = d_date_sk + AND d_year = 2001 + AND d_moy BETWEEN 4 AND 4+2) + AND NOT EXISTS + (SELECT * + FROM catalog_sales, + date_dim + WHERE c.c_customer_sk = cs_ship_customer_sk + AND cs_sold_date_sk = d_date_sk + AND d_year = 2001 + AND d_moy BETWEEN 4 AND 4+2)) +GROUP BY cd_gender, + cd_marital_status, + cd_education_status, + cd_purchase_estimate, + cd_credit_rating +ORDER BY cd_gender, + cd_marital_status, + cd_education_status, + cd_purchase_estimate, + cd_credit_rating +LIMIT 100; + diff --git a/testdata/tpcds/queries/q7.sql b/testdata/tpcds/queries/q7.sql new file mode 100644 index 00000000..a41a4d41 --- /dev/null +++ b/testdata/tpcds/queries/q7.sql @@ -0,0 +1,24 @@ +SELECT i_item_id, + avg(ss_quantity) agg1, + avg(ss_list_price) agg2, + avg(ss_coupon_amt) agg3, + avg(ss_sales_price) agg4 +FROM store_sales, + customer_demographics, + date_dim, + item, + promotion +WHERE ss_sold_date_sk = d_date_sk + AND ss_item_sk = i_item_sk + AND ss_cdemo_sk = cd_demo_sk + AND ss_promo_sk = p_promo_sk + AND cd_gender = 'M' + AND cd_marital_status = 'S' + AND cd_education_status = 'College' + AND (p_channel_email = 'N' + OR p_channel_event = 'N') + AND d_year = 2000 +GROUP BY i_item_id +ORDER BY i_item_id +LIMIT 100; + diff --git a/testdata/tpcds/queries/q70.sql b/testdata/tpcds/queries/q70.sql new file mode 100644 index 00000000..94b4d81e --- /dev/null +++ b/testdata/tpcds/queries/q70.sql @@ -0,0 +1,36 @@ +SELECT sum(ss_net_profit) AS total_sum, + s_state, + s_county, + grouping(s_state)+grouping(s_county) AS lochierarchy, + rank() OVER (PARTITION BY grouping(s_state)+grouping(s_county), + CASE + WHEN grouping(s_county) = 0 THEN s_state + END + ORDER BY sum(ss_net_profit) DESC) AS rank_within_parent +FROM store_sales, + date_dim d1, + store +WHERE d1.d_month_seq BETWEEN 1200 AND 1200+11 + AND d1.d_date_sk = ss_sold_date_sk + AND s_store_sk = ss_store_sk + AND s_state IN + (SELECT s_state + FROM + (SELECT s_state AS s_state, + rank() OVER (PARTITION BY s_state + ORDER BY sum(ss_net_profit) DESC) AS ranking + FROM store_sales, + store, + date_dim + WHERE d_month_seq BETWEEN 1200 AND 1200+11 + AND d_date_sk = ss_sold_date_sk + AND s_store_sk = ss_store_sk + GROUP BY s_state) tmp1 + WHERE ranking <= 5 ) +GROUP BY rollup(s_state,s_county) +ORDER BY lochierarchy DESC , + CASE + WHEN grouping(s_state)+grouping(s_county) = 0 THEN s_state + END , + rank_within_parent +LIMIT 100; diff --git a/testdata/tpcds/queries/q71.sql b/testdata/tpcds/queries/q71.sql new file mode 100644 index 00000000..817dc827 --- /dev/null +++ b/testdata/tpcds/queries/q71.sql @@ -0,0 +1,47 @@ +SELECT i_brand_id brand_id, + i_brand brand, + t_hour, + t_minute, + sum(ext_price) ext_price +FROM item, + (SELECT ws_ext_sales_price AS ext_price, + ws_sold_date_sk AS sold_date_sk, + ws_item_sk AS sold_item_sk, + ws_sold_time_sk AS time_sk + FROM web_sales, + date_dim + WHERE d_date_sk = ws_sold_date_sk + AND d_moy=11 + AND d_year=1999 + UNION ALL SELECT cs_ext_sales_price AS ext_price, + cs_sold_date_sk AS sold_date_sk, + cs_item_sk AS sold_item_sk, + cs_sold_time_sk AS time_sk + FROM catalog_sales, + date_dim + WHERE d_date_sk = cs_sold_date_sk + AND d_moy=11 + AND d_year=1999 + UNION ALL SELECT ss_ext_sales_price AS ext_price, + ss_sold_date_sk AS sold_date_sk, + ss_item_sk AS sold_item_sk, + ss_sold_time_sk AS time_sk + FROM store_sales, + date_dim + WHERE d_date_sk = ss_sold_date_sk + AND d_moy=11 + AND d_year=1999 ) tmp, + time_dim +WHERE sold_item_sk = i_item_sk + AND i_manager_id=1 + AND time_sk = t_time_sk + AND (t_meal_time = 'breakfast' + OR t_meal_time = 'dinner') +GROUP BY i_brand, + i_brand_id, + t_hour, + t_minute +ORDER BY ext_price DESC NULLS FIRST, + i_brand_id NULLS FIRST, + t_hour NULLS FIRST; + diff --git a/testdata/tpcds/queries/q72.sql b/testdata/tpcds/queries/q72.sql new file mode 100644 index 00000000..1dee37d9 --- /dev/null +++ b/testdata/tpcds/queries/q72.sql @@ -0,0 +1,39 @@ +SELECT i_item_desc, + w_warehouse_name, + d1.d_week_seq, + sum(CASE + WHEN p_promo_sk IS NULL THEN 1 + ELSE 0 + END) no_promo, + sum(CASE + WHEN p_promo_sk IS NOT NULL THEN 1 + ELSE 0 + END) promo, + count(*) total_cnt +FROM catalog_sales +JOIN inventory ON (cs_item_sk = inv_item_sk) +JOIN warehouse ON (w_warehouse_sk=inv_warehouse_sk) +JOIN item ON (i_item_sk = cs_item_sk) +JOIN customer_demographics ON (cs_bill_cdemo_sk = cd_demo_sk) +JOIN household_demographics ON (cs_bill_hdemo_sk = hd_demo_sk) +JOIN date_dim d1 ON (cs_sold_date_sk = d1.d_date_sk) +JOIN date_dim d2 ON (inv_date_sk = d2.d_date_sk) +JOIN date_dim d3 ON (cs_ship_date_sk = d3.d_date_sk) +LEFT OUTER JOIN promotion ON (cs_promo_sk=p_promo_sk) +LEFT OUTER JOIN catalog_returns ON (cr_item_sk = cs_item_sk + AND cr_order_number = cs_order_number) +WHERE d1.d_week_seq = d2.d_week_seq + AND inv_quantity_on_hand < cs_quantity + AND d3.d_date > d1.d_date + INTERVAL '5' DAY -- Modified - Original duckdb syntax is: d1.d_date + 5 + AND hd_buy_potential = '>10000' + AND d1.d_year = 1999 + AND cd_marital_status = 'D' +GROUP BY i_item_desc, + w_warehouse_name, + d1.d_week_seq +ORDER BY total_cnt DESC NULLS FIRST, + i_item_desc NULLS FIRST, + w_warehouse_name NULLS FIRST, + d1.d_week_seq NULLS FIRST +LIMIT 100; + diff --git a/testdata/tpcds/queries/q73.sql b/testdata/tpcds/queries/q73.sql new file mode 100644 index 00000000..22d5a101 --- /dev/null +++ b/testdata/tpcds/queries/q73.sql @@ -0,0 +1,41 @@ + +SELECT c_last_name, + c_first_name, + c_salutation, + c_preferred_cust_flag, + ss_ticket_number, + cnt +FROM + (SELECT ss_ticket_number, + ss_customer_sk, + count(*) cnt + FROM store_sales, + date_dim, + store, + household_demographics + WHERE store_sales.ss_sold_date_sk = date_dim.d_date_sk + AND store_sales.ss_store_sk = store.s_store_sk + AND store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk + AND date_dim.d_dom BETWEEN 1 AND 2 + AND (household_demographics.hd_buy_potential = 'Unknown' + OR household_demographics.hd_buy_potential = '>10000') + AND household_demographics.hd_vehicle_count > 0 + AND CASE + WHEN household_demographics.hd_vehicle_count > 0 THEN (household_demographics.hd_dep_count*1.000)/ household_demographics.hd_vehicle_count + ELSE NULL + END > 1 + AND date_dim.d_year IN (1999, + 1999+1, + 1999+2) + AND store.s_county IN ('Orange County', + 'Bronx County', + 'Franklin Parish', + 'Williamson County') + GROUP BY ss_ticket_number, + ss_customer_sk) dj, + customer +WHERE ss_customer_sk = c_customer_sk + AND cnt BETWEEN 1 AND 5 +ORDER BY cnt DESC, + c_last_name ASC; + diff --git a/testdata/tpcds/queries/q74.sql b/testdata/tpcds/queries/q74.sql new file mode 100644 index 00000000..859daf90 --- /dev/null +++ b/testdata/tpcds/queries/q74.sql @@ -0,0 +1,65 @@ +WITH year_total AS + (SELECT c_customer_id customer_id, + c_first_name customer_first_name, + c_last_name customer_last_name, + d_year AS year_, + sum(ss_net_paid) year_total, + 's' sale_type + FROM customer, + store_sales, + date_dim + WHERE c_customer_sk = ss_customer_sk + AND ss_sold_date_sk = d_date_sk + AND d_year IN (2001, + 2001+1) + GROUP BY c_customer_id, + c_first_name, + c_last_name, + d_year + UNION ALL SELECT c_customer_id customer_id, + c_first_name customer_first_name, + c_last_name customer_last_name, + d_year AS year_, + sum(ws_net_paid) year_total, + 'w' sale_type + FROM customer, + web_sales, + date_dim + WHERE c_customer_sk = ws_bill_customer_sk + AND ws_sold_date_sk = d_date_sk + AND d_year IN (2001, + 2001+1) + GROUP BY c_customer_id, + c_first_name, + c_last_name, + d_year) +SELECT t_s_secyear.customer_id, + t_s_secyear.customer_first_name, + t_s_secyear.customer_last_name +FROM year_total t_s_firstyear, + year_total t_s_secyear, + year_total t_w_firstyear, + year_total t_w_secyear +WHERE t_s_secyear.customer_id = t_s_firstyear.customer_id + AND t_s_firstyear.customer_id = t_w_secyear.customer_id + AND t_s_firstyear.customer_id = t_w_firstyear.customer_id + AND t_s_firstyear.sale_type = 's' + AND t_w_firstyear.sale_type = 'w' + AND t_s_secyear.sale_type = 's' + AND t_w_secyear.sale_type = 'w' + AND t_s_firstyear.year_ = 2001 + AND t_s_secyear.year_ = 2001+1 + AND t_w_firstyear.year_ = 2001 + AND t_w_secyear.year_ = 2001+1 + AND t_s_firstyear.year_total > 0 + AND t_w_firstyear.year_total > 0 + AND CASE + WHEN t_w_firstyear.year_total > 0 THEN t_w_secyear.year_total / t_w_firstyear.year_total + ELSE NULL + END > CASE + WHEN t_s_firstyear.year_total > 0 THEN t_s_secyear.year_total / t_s_firstyear.year_total + ELSE NULL + END +ORDER BY 1 NULLS FIRST +LIMIT 100; + diff --git a/testdata/tpcds/queries/q75.sql b/testdata/tpcds/queries/q75.sql new file mode 100644 index 00000000..00f30051 --- /dev/null +++ b/testdata/tpcds/queries/q75.sql @@ -0,0 +1,76 @@ +WITH all_sales AS + ( SELECT d_year , + i_brand_id , + i_class_id , + i_category_id , + i_manufact_id , + SUM(sales_cnt) AS sales_cnt , + SUM(sales_amt) AS sales_amt + FROM + (SELECT d_year , + i_brand_id , + i_class_id , + i_category_id , + i_manufact_id , + cs_quantity - COALESCE(cr_return_quantity,0) AS sales_cnt , + cs_ext_sales_price - COALESCE(cr_return_amount,0.0) AS sales_amt + FROM catalog_sales + JOIN item ON i_item_sk=cs_item_sk + JOIN date_dim ON d_date_sk=cs_sold_date_sk + LEFT JOIN catalog_returns ON (cs_order_number=cr_order_number + AND cs_item_sk=cr_item_sk) + WHERE i_category='Books' + UNION SELECT d_year , + i_brand_id , + i_class_id , + i_category_id , + i_manufact_id , + ss_quantity - COALESCE(sr_return_quantity,0) AS sales_cnt , + ss_ext_sales_price - COALESCE(sr_return_amt,0.0) AS sales_amt + FROM store_sales + JOIN item ON i_item_sk=ss_item_sk + JOIN date_dim ON d_date_sk=ss_sold_date_sk + LEFT JOIN store_returns ON (ss_ticket_number=sr_ticket_number + AND ss_item_sk=sr_item_sk) + WHERE i_category='Books' + UNION SELECT d_year , + i_brand_id , + i_class_id , + i_category_id , + i_manufact_id , + ws_quantity - COALESCE(wr_return_quantity,0) AS sales_cnt , + ws_ext_sales_price - COALESCE(wr_return_amt,0.0) AS sales_amt + FROM web_sales + JOIN item ON i_item_sk=ws_item_sk + JOIN date_dim ON d_date_sk=ws_sold_date_sk + LEFT JOIN web_returns ON (ws_order_number=wr_order_number + AND ws_item_sk=wr_item_sk) + WHERE i_category='Books') sales_detail + GROUP BY d_year, + i_brand_id, + i_class_id, + i_category_id, + i_manufact_id) +SELECT prev_yr.d_year AS prev_year , + curr_yr.d_year AS year_ , + curr_yr.i_brand_id , + curr_yr.i_class_id , + curr_yr.i_category_id , + curr_yr.i_manufact_id , + prev_yr.sales_cnt AS prev_yr_cnt , + curr_yr.sales_cnt AS curr_yr_cnt , + curr_yr.sales_cnt-prev_yr.sales_cnt AS sales_cnt_diff , + curr_yr.sales_amt-prev_yr.sales_amt AS sales_amt_diff +FROM all_sales curr_yr, + all_sales prev_yr +WHERE curr_yr.i_brand_id=prev_yr.i_brand_id + AND curr_yr.i_class_id=prev_yr.i_class_id + AND curr_yr.i_category_id=prev_yr.i_category_id + AND curr_yr.i_manufact_id=prev_yr.i_manufact_id + AND curr_yr.d_year=2002 + AND prev_yr.d_year=2002-1 + AND CAST(curr_yr.sales_cnt AS DECIMAL(17,2))/CAST(prev_yr.sales_cnt AS DECIMAL(17,2))<0.9 +ORDER BY sales_cnt_diff, + sales_amt_diff +LIMIT 100; + diff --git a/testdata/tpcds/queries/q76.sql b/testdata/tpcds/queries/q76.sql new file mode 100644 index 00000000..a63b6f1d --- /dev/null +++ b/testdata/tpcds/queries/q76.sql @@ -0,0 +1,56 @@ +SELECT channel, + col_name, + d_year, + d_qoy, + i_category, + COUNT(*) sales_cnt, + SUM(ext_sales_price) sales_amt +FROM + ( SELECT 'store' AS channel, + 'ss_store_sk' col_name, + d_year, + d_qoy, + i_category, + ss_ext_sales_price ext_sales_price + FROM store_sales, + item, + date_dim + WHERE ss_store_sk IS NULL + AND ss_sold_date_sk=d_date_sk + AND ss_item_sk=i_item_sk + UNION ALL SELECT 'web' AS channel, + 'ws_ship_customer_sk' col_name, + d_year, + d_qoy, + i_category, + ws_ext_sales_price ext_sales_price + FROM web_sales, + item, + date_dim + WHERE ws_ship_customer_sk IS NULL + AND ws_sold_date_sk=d_date_sk + AND ws_item_sk=i_item_sk + UNION ALL SELECT 'catalog' AS channel, + 'cs_ship_addr_sk' col_name, + d_year, + d_qoy, + i_category, + cs_ext_sales_price ext_sales_price + FROM catalog_sales, + item, + date_dim + WHERE cs_ship_addr_sk IS NULL + AND cs_sold_date_sk=d_date_sk + AND cs_item_sk=i_item_sk) foo +GROUP BY channel, + col_name, + d_year, + d_qoy, + i_category +ORDER BY channel NULLS FIRST, + col_name NULLS FIRST, + d_year NULLS FIRST, + d_qoy NULLS FIRST, + i_category NULLS FIRST +LIMIT 100; + diff --git a/testdata/tpcds/queries/q77.sql b/testdata/tpcds/queries/q77.sql new file mode 100644 index 00000000..733b0585 --- /dev/null +++ b/testdata/tpcds/queries/q77.sql @@ -0,0 +1,95 @@ +WITH ss AS + (SELECT s_store_sk, + sum(ss_ext_sales_price) AS sales, + sum(ss_net_profit) AS profit + FROM store_sales, + date_dim, + store + WHERE ss_sold_date_sk = d_date_sk + AND d_date BETWEEN cast('2000-08-23' AS date) AND cast('2000-09-22' AS date) + AND ss_store_sk = s_store_sk + GROUP BY s_store_sk) , + sr AS + (SELECT s_store_sk, + sum(sr_return_amt) AS returns_, + sum(sr_net_loss) AS profit_loss + FROM store_returns, + date_dim, + store + WHERE sr_returned_date_sk = d_date_sk + AND d_date BETWEEN cast('2000-08-23' AS date) AND cast('2000-09-22' AS date) + AND sr_store_sk = s_store_sk + GROUP BY s_store_sk), + cs AS + (SELECT cs_call_center_sk, + sum(cs_ext_sales_price) AS sales, + sum(cs_net_profit) AS profit + FROM catalog_sales, + date_dim + WHERE cs_sold_date_sk = d_date_sk + AND d_date BETWEEN cast('2000-08-23' AS date) AND cast('2000-09-22' AS date) + GROUP BY cs_call_center_sk), + cr AS + (SELECT cr_call_center_sk, + sum(cr_return_amount) AS returns_, + sum(cr_net_loss) AS profit_loss + FROM catalog_returns, + date_dim + WHERE cr_returned_date_sk = d_date_sk + AND d_date BETWEEN cast('2000-08-23' AS date) AND cast('2000-09-22' AS date) + GROUP BY cr_call_center_sk ), + ws AS + (SELECT wp_web_page_sk, + sum(ws_ext_sales_price) AS sales, + sum(ws_net_profit) AS profit + FROM web_sales, + date_dim, + web_page + WHERE ws_sold_date_sk = d_date_sk + AND d_date BETWEEN cast('2000-08-23' AS date) AND cast('2000-09-22' AS date) + AND ws_web_page_sk = wp_web_page_sk + GROUP BY wp_web_page_sk), + wr AS + (SELECT wp_web_page_sk, + sum(wr_return_amt) AS returns_, + sum(wr_net_loss) AS profit_loss + FROM web_returns, + date_dim, + web_page + WHERE wr_returned_date_sk = d_date_sk + AND d_date BETWEEN cast('2000-08-23' AS date) AND cast('2000-09-22' AS date) + AND wr_web_page_sk = wp_web_page_sk + GROUP BY wp_web_page_sk) +SELECT channel , + id , + sum(sales) AS sales , + sum(returns_) AS returns_ , + sum(profit) AS profit +FROM + (SELECT 'store channel' AS channel , + ss.s_store_sk AS id , + sales , + coalesce(returns_, 0) AS returns_ , + (profit - coalesce(profit_loss,0)) AS profit + FROM ss + LEFT JOIN sr ON ss.s_store_sk = sr.s_store_sk + UNION ALL SELECT 'catalog channel' AS channel , + cs_call_center_sk AS id , + sales , + returns_ , + (profit - profit_loss) AS profit + FROM cs , + cr + UNION ALL SELECT 'web channel' AS channel , + ws.wp_web_page_sk AS id , + sales , + coalesce(returns_, 0) returns_ , + (profit - coalesce(profit_loss,0)) AS profit + FROM ws + LEFT JOIN wr ON ws.wp_web_page_sk = wr.wp_web_page_sk ) x +GROUP BY ROLLUP (channel, + id) +ORDER BY channel NULLS FIRST, + id NULLS FIRST, + returns_ DESC +LIMIT 100; diff --git a/testdata/tpcds/queries/q78.sql b/testdata/tpcds/queries/q78.sql new file mode 100644 index 00000000..51b51bc3 --- /dev/null +++ b/testdata/tpcds/queries/q78.sql @@ -0,0 +1,77 @@ +WITH ws AS + (SELECT d_year AS ws_sold_year, + ws_item_sk, + ws_bill_customer_sk ws_customer_sk, + sum(ws_quantity) ws_qty, + sum(ws_wholesale_cost) ws_wc, + sum(ws_sales_price) ws_sp + FROM web_sales + LEFT JOIN web_returns ON wr_order_number=ws_order_number + AND ws_item_sk=wr_item_sk + JOIN date_dim ON ws_sold_date_sk = d_date_sk + WHERE wr_order_number IS NULL + GROUP BY d_year, + ws_item_sk, + ws_bill_customer_sk ), + cs AS + (SELECT d_year AS cs_sold_year, + cs_item_sk, + cs_bill_customer_sk cs_customer_sk, + sum(cs_quantity) cs_qty, + sum(cs_wholesale_cost) cs_wc, + sum(cs_sales_price) cs_sp + FROM catalog_sales + LEFT JOIN catalog_returns ON cr_order_number=cs_order_number + AND cs_item_sk=cr_item_sk + JOIN date_dim ON cs_sold_date_sk = d_date_sk + WHERE cr_order_number IS NULL + GROUP BY d_year, + cs_item_sk, + cs_bill_customer_sk ), + ss AS + (SELECT d_year AS ss_sold_year, + ss_item_sk, + ss_customer_sk, + sum(ss_quantity) ss_qty, + sum(ss_wholesale_cost) ss_wc, + sum(ss_sales_price) ss_sp + FROM store_sales + LEFT JOIN store_returns ON sr_ticket_number=ss_ticket_number + AND ss_item_sk=sr_item_sk + JOIN date_dim ON ss_sold_date_sk = d_date_sk + WHERE sr_ticket_number IS NULL + GROUP BY d_year, + ss_item_sk, + ss_customer_sk ) +SELECT ss_sold_year, + ss_item_sk, + ss_customer_sk, + round((ss_qty*1.00)/(coalesce(ws_qty,0)+coalesce(cs_qty,0)),2) ratio, + ss_qty store_qty, + ss_wc store_wholesale_cost, + ss_sp store_sales_price, + coalesce(ws_qty,0)+coalesce(cs_qty,0) other_chan_qty, + coalesce(ws_wc,0)+coalesce(cs_wc,0) other_chan_wholesale_cost, + coalesce(ws_sp,0)+coalesce(cs_sp,0) other_chan_sales_price +FROM ss +LEFT JOIN ws ON (ws_sold_year=ss_sold_year + AND ws_item_sk=ss_item_sk + AND ws_customer_sk=ss_customer_sk) +LEFT JOIN cs ON (cs_sold_year=ss_sold_year + AND cs_item_sk=ss_item_sk + AND cs_customer_sk=ss_customer_sk) +WHERE (coalesce(ws_qty,0)>0 + OR coalesce(cs_qty, 0)>0) + AND ss_sold_year=2000 +ORDER BY ss_sold_year, + ss_item_sk, + ss_customer_sk, + ss_qty DESC, + ss_wc DESC, + ss_sp DESC, + other_chan_qty, + other_chan_wholesale_cost, + other_chan_sales_price, + ratio +LIMIT 100; + diff --git a/testdata/tpcds/queries/q79.sql b/testdata/tpcds/queries/q79.sql new file mode 100644 index 00000000..55c85b72 --- /dev/null +++ b/testdata/tpcds/queries/q79.sql @@ -0,0 +1,38 @@ +SELECT c_last_name, + c_first_name, + SUBSTRING(s_city,1,30), + ss_ticket_number, + amt, + profit +FROM + (SELECT ss_ticket_number , + ss_customer_sk , + store.s_city , + sum(ss_coupon_amt) amt , + sum(ss_net_profit) profit + FROM store_sales, + date_dim, + store, + household_demographics + WHERE store_sales.ss_sold_date_sk = date_dim.d_date_sk + AND store_sales.ss_store_sk = store.s_store_sk + AND store_sales.ss_hdemo_sk = household_demographics.hd_demo_sk + AND (household_demographics.hd_dep_count = 6 + OR household_demographics.hd_vehicle_count > 2) + AND date_dim.d_dow = 1 + AND date_dim.d_year IN (1999, + 1999+1, + 1999+2) + AND store.s_number_employees BETWEEN 200 AND 295 + GROUP BY ss_ticket_number, + ss_customer_sk, + ss_addr_sk, + store.s_city) ms, + customer +WHERE ss_customer_sk = c_customer_sk +ORDER BY c_last_name NULLS FIRST, + c_first_name NULLS FIRST, + SUBSTRING(s_city,1,30) NULLS FIRST, + profit NULLS FIRST, + ss_ticket_number +LIMIT 100; diff --git a/testdata/tpcds/queries/q8.sql b/testdata/tpcds/queries/q8.sql new file mode 100644 index 00000000..a9aa99f5 --- /dev/null +++ b/testdata/tpcds/queries/q8.sql @@ -0,0 +1,428 @@ +SELECT s_store_name, + sum(ss_net_profit) +FROM store_sales, + date_dim, + store, + (SELECT ca_zip + FROM + (SELECT SUBSTRING(ca_zip, 1, 5) ca_zip + FROM customer_address + WHERE SUBSTRING(ca_zip, 1, 5) IN ('24128', + '76232', + '65084', + '87816', + '83926', + '77556', + '20548', + '26231', + '43848', + '15126', + '91137', + '61265', + '98294', + '25782', + '17920', + '18426', + '98235', + '40081', + '84093', + '28577', + '55565', + '17183', + '54601', + '67897', + '22752', + '86284', + '18376', + '38607', + '45200', + '21756', + '29741', + '96765', + '23932', + '89360', + '29839', + '25989', + '28898', + '91068', + '72550', + '10390', + '18845', + '47770', + '82636', + '41367', + '76638', + '86198', + '81312', + '37126', + '39192', + '88424', + '72175', + '81426', + '53672', + '10445', + '42666', + '66864', + '66708', + '41248', + '48583', + '82276', + '18842', + '78890', + '49448', + '14089', + '38122', + '34425', + '79077', + '19849', + '43285', + '39861', + '66162', + '77610', + '13695', + '99543', + '83444', + '83041', + '12305', + '57665', + '68341', + '25003', + '57834', + '62878', + '49130', + '81096', + '18840', + '27700', + '23470', + '50412', + '21195', + '16021', + '76107', + '71954', + '68309', + '18119', + '98359', + '64544', + '10336', + '86379', + '27068', + '39736', + '98569', + '28915', + '24206', + '56529', + '57647', + '54917', + '42961', + '91110', + '63981', + '14922', + '36420', + '23006', + '67467', + '32754', + '30903', + '20260', + '31671', + '51798', + '72325', + '85816', + '68621', + '13955', + '36446', + '41766', + '68806', + '16725', + '15146', + '22744', + '35850', + '88086', + '51649', + '18270', + '52867', + '39972', + '96976', + '63792', + '11376', + '94898', + '13595', + '10516', + '90225', + '58943', + '39371', + '94945', + '28587', + '96576', + '57855', + '28488', + '26105', + '83933', + '25858', + '34322', + '44438', + '73171', + '30122', + '34102', + '22685', + '71256', + '78451', + '54364', + '13354', + '45375', + '40558', + '56458', + '28286', + '45266', + '47305', + '69399', + '83921', + '26233', + '11101', + '15371', + '69913', + '35942', + '15882', + '25631', + '24610', + '44165', + '99076', + '33786', + '70738', + '26653', + '14328', + '72305', + '62496', + '22152', + '10144', + '64147', + '48425', + '14663', + '21076', + '18799', + '30450', + '63089', + '81019', + '68893', + '24996', + '51200', + '51211', + '45692', + '92712', + '70466', + '79994', + '22437', + '25280', + '38935', + '71791', + '73134', + '56571', + '14060', + '19505', + '72425', + '56575', + '74351', + '68786', + '51650', + '20004', + '18383', + '76614', + '11634', + '18906', + '15765', + '41368', + '73241', + '76698', + '78567', + '97189', + '28545', + '76231', + '75691', + '22246', + '51061', + '90578', + '56691', + '68014', + '51103', + '94167', + '57047', + '14867', + '73520', + '15734', + '63435', + '25733', + '35474', + '24676', + '94627', + '53535', + '17879', + '15559', + '53268', + '59166', + '11928', + '59402', + '33282', + '45721', + '43933', + '68101', + '33515', + '36634', + '71286', + '19736', + '58058', + '55253', + '67473', + '41918', + '19515', + '36495', + '19430', + '22351', + '77191', + '91393', + '49156', + '50298', + '87501', + '18652', + '53179', + '18767', + '63193', + '23968', + '65164', + '68880', + '21286', + '72823', + '58470', + '67301', + '13394', + '31016', + '70372', + '67030', + '40604', + '24317', + '45748', + '39127', + '26065', + '77721', + '31029', + '31880', + '60576', + '24671', + '45549', + '13376', + '50016', + '33123', + '19769', + '22927', + '97789', + '46081', + '72151', + '15723', + '46136', + '51949', + '68100', + '96888', + '64528', + '14171', + '79777', + '28709', + '11489', + '25103', + '32213', + '78668', + '22245', + '15798', + '27156', + '37930', + '62971', + '21337', + '51622', + '67853', + '10567', + '38415', + '15455', + '58263', + '42029', + '60279', + '37125', + '56240', + '88190', + '50308', + '26859', + '64457', + '89091', + '82136', + '62377', + '36233', + '63837', + '58078', + '17043', + '30010', + '60099', + '28810', + '98025', + '29178', + '87343', + '73273', + '30469', + '64034', + '39516', + '86057', + '21309', + '90257', + '67875', + '40162', + '11356', + '73650', + '61810', + '72013', + '30431', + '22461', + '19512', + '13375', + '55307', + '30625', + '83849', + '68908', + '26689', + '96451', + '38193', + '46820', + '88885', + '84935', + '69035', + '83144', + '47537', + '56616', + '94983', + '48033', + '69952', + '25486', + '61547', + '27385', + '61860', + '58048', + '56910', + '16807', + '17871', + '35258', + '31387', + '35458', + '35576') INTERSECT + SELECT ca_zip + FROM + (SELECT SUBSTRING(ca_zip, 1, 5) ca_zip, + count(*) cnt + FROM customer_address, + customer + WHERE ca_address_sk = c_current_addr_sk + AND c_preferred_cust_flag='Y' + GROUP BY ca_zip + HAVING count(*) > 10)A1)A2) V1 +WHERE ss_store_sk = s_store_sk + AND ss_sold_date_sk = d_date_sk + AND d_qoy = 2 + AND d_year = 1998 + AND (SUBSTRING(s_zip, 1, 2) = SUBSTRING(V1.ca_zip, 1, 2)) +GROUP BY s_store_name +ORDER BY s_store_name +LIMIT 100; + diff --git a/testdata/tpcds/queries/q80.sql b/testdata/tpcds/queries/q80.sql new file mode 100644 index 00000000..f06c35b1 --- /dev/null +++ b/testdata/tpcds/queries/q80.sql @@ -0,0 +1,86 @@ +WITH ssr AS + (SELECT s_store_id AS store_id, + sum(ss_ext_sales_price) AS sales, + sum(coalesce(sr_return_amt, 0)) AS returns_, + sum(ss_net_profit - coalesce(sr_net_loss, 0)) AS profit + FROM store_sales + LEFT OUTER JOIN store_returns ON (ss_item_sk = sr_item_sk + AND ss_ticket_number = sr_ticket_number), date_dim, + store, + item, + promotion + WHERE ss_sold_date_sk = d_date_sk + AND d_date BETWEEN cast('2000-08-23' AS date) AND cast('2000-09-22' AS date) + AND ss_store_sk = s_store_sk + AND ss_item_sk = i_item_sk + AND i_current_price > 50 + AND ss_promo_sk = p_promo_sk + AND p_channel_tv = 'N' + GROUP BY s_store_id) , + csr AS + (SELECT cp_catalog_page_id AS catalog_page_id, + sum(cs_ext_sales_price) AS sales, + sum(coalesce(cr_return_amount, 0)) AS returns_, + sum(cs_net_profit - coalesce(cr_net_loss, 0)) AS profit + FROM catalog_sales + LEFT OUTER JOIN catalog_returns ON (cs_item_sk = cr_item_sk + AND cs_order_number = cr_order_number), date_dim, + catalog_page, + item, + promotion + WHERE cs_sold_date_sk = d_date_sk + AND d_date BETWEEN cast('2000-08-23' AS date) AND cast('2000-09-22' AS date) + AND cs_catalog_page_sk = cp_catalog_page_sk + AND cs_item_sk = i_item_sk + AND i_current_price > 50 + AND cs_promo_sk = p_promo_sk + AND p_channel_tv = 'N' + GROUP BY cp_catalog_page_id) , + wsr AS + (SELECT web_site_id, + sum(ws_ext_sales_price) AS sales, + sum(coalesce(wr_return_amt, 0)) AS returns_, + sum(ws_net_profit - coalesce(wr_net_loss, 0)) AS profit + FROM web_sales + LEFT OUTER JOIN web_returns ON (ws_item_sk = wr_item_sk + AND ws_order_number = wr_order_number), date_dim, + web_site, + item, + promotion + WHERE ws_sold_date_sk = d_date_sk + AND d_date BETWEEN cast('2000-08-23' AS date) AND cast('2000-09-22' AS date) + AND ws_web_site_sk = web_site_sk + AND ws_item_sk = i_item_sk + AND i_current_price > 50 + AND ws_promo_sk = p_promo_sk + AND p_channel_tv = 'N' + GROUP BY web_site_id) +SELECT channel , + id , + sum(sales) AS sales , + sum(returns_) AS returns_ , + sum(profit) AS profit +FROM + (SELECT 'store channel' AS channel , + concat('store', store_id) AS id , + sales , + returns_ , + profit + FROM ssr + UNION ALL SELECT 'catalog channel' AS channel , + concat('catalog_page', catalog_page_id) AS id , + sales , + returns_ , + profit + FROM csr + UNION ALL SELECT 'web channel' AS channel , + concat('web_site', web_site_id) AS id , + sales , + returns_ , + profit + FROM wsr ) x +GROUP BY ROLLUP (channel, + id) +ORDER BY channel NULLS FIRST, + id NULLS FIRST +LIMIT 100; diff --git a/testdata/tpcds/queries/q81.sql b/testdata/tpcds/queries/q81.sql new file mode 100644 index 00000000..483a1118 --- /dev/null +++ b/testdata/tpcds/queries/q81.sql @@ -0,0 +1,56 @@ +WITH customer_total_return AS + (SELECT cr_returning_customer_sk AS ctr_customer_sk , + ca_state AS ctr_state, + sum(cr_return_amt_inc_tax) AS ctr_total_return + FROM catalog_returns , + date_dim , + customer_address + WHERE cr_returned_date_sk = d_date_sk + AND d_year = 2000 + AND cr_returning_addr_sk = ca_address_sk + GROUP BY cr_returning_customer_sk , + ca_state) +SELECT c_customer_id, + c_salutation, + c_first_name, + c_last_name, + ca_street_number, + ca_street_name , + ca_street_type, + ca_suite_number, + ca_city, + ca_county, + ca_state, + ca_zip, + ca_country, + ca_gmt_offset , + ca_location_type, + ctr_total_return +FROM customer_total_return ctr1 , + customer_address , + customer +WHERE ctr1.ctr_total_return > + (SELECT avg(ctr_total_return)*1.2 + FROM customer_total_return ctr2 + WHERE ctr1.ctr_state = ctr2.ctr_state) + AND ca_address_sk = c_current_addr_sk + AND ca_state = 'GA' + AND ctr1.ctr_customer_sk = c_customer_sk +ORDER BY c_customer_id, + c_salutation, + c_first_name, + c_last_name, + ca_street_number, + ca_street_name , + ca_street_type, + ca_suite_number, + ca_city, + ca_county, + ca_state, + ca_zip, + ca_country, + ca_gmt_offset , + ca_location_type, + ctr_total_return +LIMIT 100; + diff --git a/testdata/tpcds/queries/q82.sql b/testdata/tpcds/queries/q82.sql new file mode 100644 index 00000000..99333a2e --- /dev/null +++ b/testdata/tpcds/queries/q82.sql @@ -0,0 +1,24 @@ + +SELECT i_item_id , + i_item_desc , + i_current_price +FROM item, + inventory, + date_dim, + store_sales +WHERE i_current_price BETWEEN 62 AND 62+30 + AND inv_item_sk = i_item_sk + AND d_date_sk=inv_date_sk + AND d_date BETWEEN cast('2000-05-25' AS date) AND cast('2000-07-24' AS date) + AND i_manufact_id IN (129, + 270, + 821, + 423) + AND inv_quantity_on_hand BETWEEN 100 AND 500 + AND ss_item_sk = i_item_sk +GROUP BY i_item_id, + i_item_desc, + i_current_price +ORDER BY i_item_id +LIMIT 100; + diff --git a/testdata/tpcds/queries/q83.sql b/testdata/tpcds/queries/q83.sql new file mode 100644 index 00000000..7f45372b --- /dev/null +++ b/testdata/tpcds/queries/q83.sql @@ -0,0 +1,71 @@ +WITH sr_items AS + (SELECT i_item_id item_id, + sum(sr_return_quantity) sr_item_qty + FROM store_returns, + item, + date_dim + WHERE sr_item_sk = i_item_sk + AND d_date IN + (SELECT d_date + FROM date_dim + WHERE d_week_seq IN + (SELECT d_week_seq + FROM date_dim + WHERE d_date IN ('2000-06-30', + '2000-09-27', + '2000-11-17'))) + AND sr_returned_date_sk = d_date_sk + GROUP BY i_item_id), + cr_items AS + (SELECT i_item_id item_id, + sum(cr_return_quantity) cr_item_qty + FROM catalog_returns, + item, + date_dim + WHERE cr_item_sk = i_item_sk + AND d_date IN + (SELECT d_date + FROM date_dim + WHERE d_week_seq IN + (SELECT d_week_seq + FROM date_dim + WHERE d_date IN ('2000-06-30', + '2000-09-27', + '2000-11-17'))) + AND cr_returned_date_sk = d_date_sk + GROUP BY i_item_id), + wr_items AS + (SELECT i_item_id item_id, + sum(wr_return_quantity) wr_item_qty + FROM web_returns, + item, + date_dim + WHERE wr_item_sk = i_item_sk + AND d_date IN + (SELECT d_date + FROM date_dim + WHERE d_week_seq IN + (SELECT d_week_seq + FROM date_dim + WHERE d_date IN ('2000-06-30', + '2000-09-27', + '2000-11-17'))) + AND wr_returned_date_sk = d_date_sk + GROUP BY i_item_id) +SELECT sr_items.item_id , + sr_item_qty , + (sr_item_qty*1.0000)/(sr_item_qty+cr_item_qty+wr_item_qty)/3.0000 * 100 sr_dev , + cr_item_qty , + (cr_item_qty*1.0000)/(sr_item_qty+cr_item_qty+wr_item_qty)/3.0000 * 100 cr_dev , + wr_item_qty , + (wr_item_qty*1.0000)/(sr_item_qty+cr_item_qty+wr_item_qty)/3.0000 * 100 wr_dev , + (sr_item_qty+cr_item_qty+wr_item_qty)/3.0 average +FROM sr_items , + cr_items , + wr_items +WHERE sr_items.item_id=cr_items.item_id + AND sr_items.item_id=wr_items.item_id +ORDER BY sr_items.item_id NULLS FIRST, + sr_item_qty NULLS FIRST +LIMIT 100; + diff --git a/testdata/tpcds/queries/q84.sql b/testdata/tpcds/queries/q84.sql new file mode 100644 index 00000000..4dd417e9 --- /dev/null +++ b/testdata/tpcds/queries/q84.sql @@ -0,0 +1,18 @@ +SELECT c_customer_id AS customer_id , + concat(concat(coalesce(c_last_name, '') , ', '), coalesce(c_first_name, '')) AS customername +FROM customer , + customer_address , + customer_demographics , + household_demographics , + income_band , + store_returns +WHERE ca_city = 'Edgewood' + AND c_current_addr_sk = ca_address_sk + AND ib_lower_bound >= 38128 + AND ib_upper_bound <= 38128 + 50000 + AND ib_income_band_sk = hd_income_band_sk + AND cd_demo_sk = c_current_cdemo_sk + AND hd_demo_sk = c_current_hdemo_sk + AND sr_cdemo_sk = cd_demo_sk +ORDER BY c_customer_id NULLS FIRST +LIMIT 100; diff --git a/testdata/tpcds/queries/q85.sql b/testdata/tpcds/queries/q85.sql new file mode 100644 index 00000000..492687da --- /dev/null +++ b/testdata/tpcds/queries/q85.sql @@ -0,0 +1,58 @@ + +SELECT SUBSTRING(r_reason_desc,1,20) , + avg(ws_quantity) avg1, + avg(wr_refunded_cash) avg2, + avg(wr_fee) +FROM web_sales, + web_returns, + web_page, + customer_demographics cd1, + customer_demographics cd2, + customer_address, + date_dim, + reason +WHERE ws_web_page_sk = wp_web_page_sk + AND ws_item_sk = wr_item_sk + AND ws_order_number = wr_order_number + AND ws_sold_date_sk = d_date_sk + AND d_year = 2000 + AND cd1.cd_demo_sk = wr_refunded_cdemo_sk + AND cd2.cd_demo_sk = wr_returning_cdemo_sk + AND ca_address_sk = wr_refunded_addr_sk + AND r_reason_sk = wr_reason_sk + AND ( ( cd1.cd_marital_status = 'M' + AND cd1.cd_marital_status = cd2.cd_marital_status + AND cd1.cd_education_status = 'Advanced Degree' + AND cd1.cd_education_status = cd2.cd_education_status + AND ws_sales_price BETWEEN 100.00 AND 150.00 ) + OR ( cd1.cd_marital_status = 'S' + AND cd1.cd_marital_status = cd2.cd_marital_status + AND cd1.cd_education_status = 'College' + AND cd1.cd_education_status = cd2.cd_education_status + AND ws_sales_price BETWEEN 50.00 AND 100.00 ) + OR ( cd1.cd_marital_status = 'W' + AND cd1.cd_marital_status = cd2.cd_marital_status + AND cd1.cd_education_status = '2 yr Degree' + AND cd1.cd_education_status = cd2.cd_education_status + AND ws_sales_price BETWEEN 150.00 AND 200.00 ) ) + AND ( ( ca_country = 'United States' + AND ca_state IN ('IN', + 'OH', + 'NJ') + AND ws_net_profit BETWEEN 100 AND 200) + OR ( ca_country = 'United States' + AND ca_state IN ('WI', + 'CT', + 'KY') + AND ws_net_profit BETWEEN 150 AND 300) + OR ( ca_country = 'United States' + AND ca_state IN ('LA', + 'IA', + 'AR') + AND ws_net_profit BETWEEN 50 AND 250) ) +GROUP BY r_reason_desc +ORDER BY SUBSTRING(r_reason_desc,1,20) , + avg(ws_quantity) , + avg(wr_refunded_cash) , + avg(wr_fee) +LIMIT 100; diff --git a/testdata/tpcds/queries/q86.sql b/testdata/tpcds/queries/q86.sql new file mode 100644 index 00000000..c54b8b2b --- /dev/null +++ b/testdata/tpcds/queries/q86.sql @@ -0,0 +1,22 @@ +SELECT sum(ws_net_paid) AS total_sum , + i_category , + i_class , + grouping(i_category)+grouping(i_class) AS lochierarchy , + rank() OVER ( PARTITION BY grouping(i_category)+grouping(i_class), + CASE + WHEN grouping(i_class) = 0 THEN i_category + END + ORDER BY sum(ws_net_paid) DESC) AS rank_within_parent +FROM web_sales , + date_dim d1 , + item +WHERE d1.d_month_seq BETWEEN 1200 AND 1200+11 + AND d1.d_date_sk = ws_sold_date_sk + AND i_item_sk = ws_item_sk +GROUP BY rollup(i_category,i_class) +ORDER BY lochierarchy DESC NULLS FIRST, + CASE + WHEN grouping(i_category)+grouping(i_class) = 0 THEN i_category + END NULLS FIRST, + rank_within_parent NULLS FIRST +LIMIT 100; diff --git a/testdata/tpcds/queries/q87.sql b/testdata/tpcds/queries/q87.sql new file mode 100644 index 00000000..d08740b8 --- /dev/null +++ b/testdata/tpcds/queries/q87.sql @@ -0,0 +1,30 @@ +SELECT count(*) +FROM ((SELECT DISTINCT c_last_name, + c_first_name, + d_date + FROM store_sales, + date_dim, + customer + WHERE store_sales.ss_sold_date_sk = date_dim.d_date_sk + AND store_sales.ss_customer_sk = customer.c_customer_sk + AND d_month_seq BETWEEN 1200 AND 1200+11) + EXCEPT + (SELECT DISTINCT c_last_name, + c_first_name, + d_date + FROM catalog_sales, + date_dim, + customer + WHERE catalog_sales.cs_sold_date_sk = date_dim.d_date_sk + AND catalog_sales.cs_bill_customer_sk = customer.c_customer_sk + AND d_month_seq BETWEEN 1200 AND 1200+11) + EXCEPT + (SELECT DISTINCT c_last_name, + c_first_name, + d_date + FROM web_sales, + date_dim, + customer + WHERE web_sales.ws_sold_date_sk = date_dim.d_date_sk + AND web_sales.ws_bill_customer_sk = customer.c_customer_sk + AND d_month_seq BETWEEN 1200 AND 1200+11)) cool_cust ; diff --git a/testdata/tpcds/queries/q88.sql b/testdata/tpcds/queries/q88.sql new file mode 100644 index 00000000..6d1b4701 --- /dev/null +++ b/testdata/tpcds/queries/q88.sql @@ -0,0 +1,139 @@ +SELECT * +FROM + (SELECT count(*) h8_30_to_9 + FROM store_sales, + household_demographics, + time_dim, + store + WHERE ss_sold_time_sk = time_dim.t_time_sk + AND ss_hdemo_sk = household_demographics.hd_demo_sk + AND ss_store_sk = s_store_sk + AND time_dim.t_hour = 8 + AND time_dim.t_minute >= 30 + AND ((household_demographics.hd_dep_count = 4 + AND household_demographics.hd_vehicle_count<=4+2) + OR (household_demographics.hd_dep_count = 2 + AND household_demographics.hd_vehicle_count<=2+2) + OR (household_demographics.hd_dep_count = 0 + AND household_demographics.hd_vehicle_count<=0+2)) + AND store.s_store_name = 'ese') s1, + (SELECT count(*) h9_to_9_30 + FROM store_sales, + household_demographics, + time_dim, + store + WHERE ss_sold_time_sk = time_dim.t_time_sk + AND ss_hdemo_sk = household_demographics.hd_demo_sk + AND ss_store_sk = s_store_sk + AND time_dim.t_hour = 9 + AND time_dim.t_minute < 30 + AND ((household_demographics.hd_dep_count = 4 + AND household_demographics.hd_vehicle_count<=4+2) + OR (household_demographics.hd_dep_count = 2 + AND household_demographics.hd_vehicle_count<=2+2) + OR (household_demographics.hd_dep_count = 0 + AND household_demographics.hd_vehicle_count<=0+2)) + AND store.s_store_name = 'ese') s2, + (SELECT count(*) h9_30_to_10 + FROM store_sales, + household_demographics, + time_dim, + store + WHERE ss_sold_time_sk = time_dim.t_time_sk + AND ss_hdemo_sk = household_demographics.hd_demo_sk + AND ss_store_sk = s_store_sk + AND time_dim.t_hour = 9 + AND time_dim.t_minute >= 30 + AND ((household_demographics.hd_dep_count = 4 + AND household_demographics.hd_vehicle_count<=4+2) + OR (household_demographics.hd_dep_count = 2 + AND household_demographics.hd_vehicle_count<=2+2) + OR (household_demographics.hd_dep_count = 0 + AND household_demographics.hd_vehicle_count<=0+2)) + AND store.s_store_name = 'ese') s3, + (SELECT count(*) h10_to_10_30 + FROM store_sales, + household_demographics, + time_dim, + store + WHERE ss_sold_time_sk = time_dim.t_time_sk + AND ss_hdemo_sk = household_demographics.hd_demo_sk + AND ss_store_sk = s_store_sk + AND time_dim.t_hour = 10 + AND time_dim.t_minute < 30 + AND ((household_demographics.hd_dep_count = 4 + AND household_demographics.hd_vehicle_count<=4+2) + OR (household_demographics.hd_dep_count = 2 + AND household_demographics.hd_vehicle_count<=2+2) + OR (household_demographics.hd_dep_count = 0 + AND household_demographics.hd_vehicle_count<=0+2)) + AND store.s_store_name = 'ese') s4, + (SELECT count(*) h10_30_to_11 + FROM store_sales, + household_demographics, + time_dim, + store + WHERE ss_sold_time_sk = time_dim.t_time_sk + AND ss_hdemo_sk = household_demographics.hd_demo_sk + AND ss_store_sk = s_store_sk + AND time_dim.t_hour = 10 + AND time_dim.t_minute >= 30 + AND ((household_demographics.hd_dep_count = 4 + AND household_demographics.hd_vehicle_count<=4+2) + OR (household_demographics.hd_dep_count = 2 + AND household_demographics.hd_vehicle_count<=2+2) + OR (household_demographics.hd_dep_count = 0 + AND household_demographics.hd_vehicle_count<=0+2)) + AND store.s_store_name = 'ese') s5, + (SELECT count(*) h11_to_11_30 + FROM store_sales, + household_demographics, + time_dim, + store + WHERE ss_sold_time_sk = time_dim.t_time_sk + AND ss_hdemo_sk = household_demographics.hd_demo_sk + AND ss_store_sk = s_store_sk + AND time_dim.t_hour = 11 + AND time_dim.t_minute < 30 + AND ((household_demographics.hd_dep_count = 4 + AND household_demographics.hd_vehicle_count<=4+2) + OR (household_demographics.hd_dep_count = 2 + AND household_demographics.hd_vehicle_count<=2+2) + OR (household_demographics.hd_dep_count = 0 + AND household_demographics.hd_vehicle_count<=0+2)) + AND store.s_store_name = 'ese') s6, + (SELECT count(*) h11_30_to_12 + FROM store_sales, + household_demographics, + time_dim, + store + WHERE ss_sold_time_sk = time_dim.t_time_sk + AND ss_hdemo_sk = household_demographics.hd_demo_sk + AND ss_store_sk = s_store_sk + AND time_dim.t_hour = 11 + AND time_dim.t_minute >= 30 + AND ((household_demographics.hd_dep_count = 4 + AND household_demographics.hd_vehicle_count<=4+2) + OR (household_demographics.hd_dep_count = 2 + AND household_demographics.hd_vehicle_count<=2+2) + OR (household_demographics.hd_dep_count = 0 + AND household_demographics.hd_vehicle_count<=0+2)) + AND store.s_store_name = 'ese') s7, + (SELECT count(*) h12_to_12_30 + FROM store_sales, + household_demographics, + time_dim, + store + WHERE ss_sold_time_sk = time_dim.t_time_sk + AND ss_hdemo_sk = household_demographics.hd_demo_sk + AND ss_store_sk = s_store_sk + AND time_dim.t_hour = 12 + AND time_dim.t_minute < 30 + AND ((household_demographics.hd_dep_count = 4 + AND household_demographics.hd_vehicle_count<=4+2) + OR (household_demographics.hd_dep_count = 2 + AND household_demographics.hd_vehicle_count<=2+2) + OR (household_demographics.hd_dep_count = 0 + AND household_demographics.hd_vehicle_count<=0+2)) + AND store.s_store_name = 'ese') s8 ; + diff --git a/testdata/tpcds/queries/q89.sql b/testdata/tpcds/queries/q89.sql new file mode 100644 index 00000000..88e26898 --- /dev/null +++ b/testdata/tpcds/queries/q89.sql @@ -0,0 +1,21 @@ + +SELECT * from + (SELECT i_category, i_class, i_brand, s_store_name, s_company_name, d_moy, sum(ss_sales_price) sum_sales, avg(sum(ss_sales_price)) OVER (PARTITION BY i_category, i_brand, s_store_name, s_company_name) avg_monthly_sales + FROM item, store_sales, date_dim, store + WHERE ss_item_sk = i_item_sk + AND ss_sold_date_sk = d_date_sk + AND ss_store_sk = s_store_sk + AND d_year = 1999 + AND ((i_category IN ('Books','Electronics','Sports') + AND i_class IN ('computers','stereo','football') ) + OR (i_category IN ('Men','Jewelry','Women') + AND i_class IN ('shirts','birdal','dresses'))) + GROUP BY i_category, i_class, i_brand, s_store_name, s_company_name, d_moy) tmp1 +WHERE CASE + WHEN (avg_monthly_sales <> 0) THEN (abs(sum_sales - avg_monthly_sales) / avg_monthly_sales) + ELSE NULL + END > 0.1 +ORDER BY sum_sales - avg_monthly_sales, + s_store_name, 1, 2, 3, 5, 6, 7, 8 +LIMIT 100; + diff --git a/testdata/tpcds/queries/q9.sql b/testdata/tpcds/queries/q9.sql new file mode 100644 index 00000000..f3b4868c --- /dev/null +++ b/testdata/tpcds/queries/q9.sql @@ -0,0 +1,69 @@ + +SELECT CASE + WHEN + (SELECT count(*) + FROM store_sales + WHERE ss_quantity BETWEEN 1 AND 20) > 74129 THEN + (SELECT avg(ss_ext_discount_amt) + FROM store_sales + WHERE ss_quantity BETWEEN 1 AND 20) + ELSE + (SELECT avg(ss_net_paid) + FROM store_sales + WHERE ss_quantity BETWEEN 1 AND 20) + END bucket1, + CASE + WHEN + (SELECT count(*) + FROM store_sales + WHERE ss_quantity BETWEEN 21 AND 40) > 122840 THEN + (SELECT avg(ss_ext_discount_amt) + FROM store_sales + WHERE ss_quantity BETWEEN 21 AND 40) + ELSE + (SELECT avg(ss_net_paid) + FROM store_sales + WHERE ss_quantity BETWEEN 21 AND 40) + END bucket2, + CASE + WHEN + (SELECT count(*) + FROM store_sales + WHERE ss_quantity BETWEEN 41 AND 60) > 56580 THEN + (SELECT avg(ss_ext_discount_amt) + FROM store_sales + WHERE ss_quantity BETWEEN 41 AND 60) + ELSE + (SELECT avg(ss_net_paid) + FROM store_sales + WHERE ss_quantity BETWEEN 41 AND 60) + END bucket3, + CASE + WHEN + (SELECT count(*) + FROM store_sales + WHERE ss_quantity BETWEEN 61 AND 80) > 10097 THEN + (SELECT avg(ss_ext_discount_amt) + FROM store_sales + WHERE ss_quantity BETWEEN 61 AND 80) + ELSE + (SELECT avg(ss_net_paid) + FROM store_sales + WHERE ss_quantity BETWEEN 61 AND 80) + END bucket4, + CASE + WHEN + (SELECT count(*) + FROM store_sales + WHERE ss_quantity BETWEEN 81 AND 100) > 165306 THEN + (SELECT avg(ss_ext_discount_amt) + FROM store_sales + WHERE ss_quantity BETWEEN 81 AND 100) + ELSE + (SELECT avg(ss_net_paid) + FROM store_sales + WHERE ss_quantity BETWEEN 81 AND 100) + END bucket5 +FROM reason +WHERE r_reason_sk = 1 ; + diff --git a/testdata/tpcds/queries/q90.sql b/testdata/tpcds/queries/q90.sql new file mode 100644 index 00000000..ef1cbe76 --- /dev/null +++ b/testdata/tpcds/queries/q90.sql @@ -0,0 +1,27 @@ +SELECT case when pmc=0 then null else cast(amc AS decimal(15,4))/cast(pmc AS decimal(15,4)) end am_pm_ratio +FROM + (SELECT count(*) amc + FROM web_sales, + household_demographics, + time_dim, + web_page + WHERE ws_sold_time_sk = time_dim.t_time_sk + AND ws_ship_hdemo_sk = household_demographics.hd_demo_sk + AND ws_web_page_sk = web_page.wp_web_page_sk + AND time_dim.t_hour BETWEEN 8 AND 8+1 + AND household_demographics.hd_dep_count = 6 + AND web_page.wp_char_count BETWEEN 5000 AND 5200) at_, + (SELECT count(*) pmc + FROM web_sales, + household_demographics, + time_dim, + web_page + WHERE ws_sold_time_sk = time_dim.t_time_sk + AND ws_ship_hdemo_sk = household_demographics.hd_demo_sk + AND ws_web_page_sk = web_page.wp_web_page_sk + AND time_dim.t_hour BETWEEN 19 AND 19+1 + AND household_demographics.hd_dep_count = 6 + AND web_page.wp_char_count BETWEEN 5000 AND 5200) pt +ORDER BY am_pm_ratio +LIMIT 100; + diff --git a/testdata/tpcds/queries/q91.sql b/testdata/tpcds/queries/q91.sql new file mode 100644 index 00000000..8fb6645a --- /dev/null +++ b/testdata/tpcds/queries/q91.sql @@ -0,0 +1,31 @@ +SELECT cc_call_center_id Call_Center, + cc_name Call_Center_Name, + cc_manager Manager, + sum(cr_net_loss) Returns_Loss +FROM call_center, + catalog_returns, + date_dim, + customer, + customer_address, + customer_demographics, + household_demographics +WHERE cr_call_center_sk = cc_call_center_sk + AND cr_returned_date_sk = d_date_sk + AND cr_returning_customer_sk= c_customer_sk + AND cd_demo_sk = c_current_cdemo_sk + AND hd_demo_sk = c_current_hdemo_sk + AND ca_address_sk = c_current_addr_sk + AND d_year = 1998 + AND d_moy = 11 + AND ((cd_marital_status = 'M' + AND cd_education_status = 'Unknown') or(cd_marital_status = 'W' + AND cd_education_status = 'Advanced Degree')) + AND hd_buy_potential LIKE 'Unknown%' + AND ca_gmt_offset = -7 +GROUP BY cc_call_center_id, + cc_name, + cc_manager, + cd_marital_status, + cd_education_status +ORDER BY sum(cr_net_loss) DESC; + diff --git a/testdata/tpcds/queries/q92.sql b/testdata/tpcds/queries/q92.sql new file mode 100644 index 00000000..7e403e44 --- /dev/null +++ b/testdata/tpcds/queries/q92.sql @@ -0,0 +1,18 @@ +SELECT sum(ws_ext_discount_amt) AS "Excess Discount Amount" +FROM web_sales, + item, + date_dim +WHERE i_manufact_id = 350 + AND i_item_sk = ws_item_sk + AND d_date BETWEEN '2000-01-27' AND cast('2000-04-26' AS date) + AND d_date_sk = ws_sold_date_sk + AND ws_ext_discount_amt > + (SELECT 1.3 * avg(ws_ext_discount_amt) + FROM web_sales, + date_dim + WHERE ws_item_sk = i_item_sk + AND d_date BETWEEN '2000-01-27' AND cast('2000-04-26' AS date) + AND d_date_sk = ws_sold_date_sk ) +ORDER BY sum(ws_ext_discount_amt) +LIMIT 100; + diff --git a/testdata/tpcds/queries/q93.sql b/testdata/tpcds/queries/q93.sql new file mode 100644 index 00000000..04ef40fd --- /dev/null +++ b/testdata/tpcds/queries/q93.sql @@ -0,0 +1,20 @@ +SELECT ss_customer_sk, + sum(act_sales) sumsales +FROM + (SELECT ss_item_sk, + ss_ticket_number, + ss_customer_sk, + CASE + WHEN sr_return_quantity IS NOT NULL THEN (ss_quantity-sr_return_quantity)*ss_sales_price + ELSE (ss_quantity*ss_sales_price) + END act_sales + FROM store_sales + LEFT OUTER JOIN store_returns ON (sr_item_sk = ss_item_sk + AND sr_ticket_number = ss_ticket_number) ,reason + WHERE sr_reason_sk = r_reason_sk + AND r_reason_desc = 'reason 28') t +GROUP BY ss_customer_sk +ORDER BY sumsales NULLS FIRST, + ss_customer_sk NULLS FIRST +LIMIT 100; + diff --git a/testdata/tpcds/queries/q94.sql b/testdata/tpcds/queries/q94.sql new file mode 100644 index 00000000..f587092e --- /dev/null +++ b/testdata/tpcds/queries/q94.sql @@ -0,0 +1,26 @@ + +SELECT count(DISTINCT ws_order_number) AS "order count" , + sum(ws_ext_ship_cost) AS "total shipping cost" , + sum(ws_net_profit) AS "total net profit" +FROM web_sales ws1 , + date_dim , + customer_address , + web_site +WHERE d_date BETWEEN '1999-02-01' AND cast('1999-04-02' AS date) + AND ws1.ws_ship_date_sk = d_date_sk + AND ws1.ws_ship_addr_sk = ca_address_sk + AND ca_state = 'IL' + AND ws1.ws_web_site_sk = web_site_sk + AND web_company_name = 'pri' + AND EXISTS + (SELECT * + FROM web_sales ws2 + WHERE ws1.ws_order_number = ws2.ws_order_number + AND ws1.ws_warehouse_sk <> ws2.ws_warehouse_sk) + AND NOT exists + (SELECT * + FROM web_returns wr1 + WHERE ws1.ws_order_number = wr1.wr_order_number) +ORDER BY count(DISTINCT ws_order_number) +LIMIT 100; + diff --git a/testdata/tpcds/queries/q95.sql b/testdata/tpcds/queries/q95.sql new file mode 100644 index 00000000..d9f54046 --- /dev/null +++ b/testdata/tpcds/queries/q95.sql @@ -0,0 +1,32 @@ +WITH ws_wh AS + (SELECT ws1.ws_order_number, + ws1.ws_warehouse_sk wh1, + ws2.ws_warehouse_sk wh2 + FROM web_sales ws1, + web_sales ws2 + WHERE ws1.ws_order_number = ws2.ws_order_number + AND ws1.ws_warehouse_sk <> ws2.ws_warehouse_sk) +SELECT count(DISTINCT ws_order_number) AS "order count" , + sum(ws_ext_ship_cost) AS "total shipping cost" , + sum(ws_net_profit) AS "total net profit" +FROM web_sales ws1 , + date_dim , + customer_address , + web_site +WHERE d_date BETWEEN '1999-02-01' AND cast('1999-04-02' AS date) + AND ws1.ws_ship_date_sk = d_date_sk + AND ws1.ws_ship_addr_sk = ca_address_sk + AND ca_state = 'IL' + AND ws1.ws_web_site_sk = web_site_sk + AND web_company_name = 'pri' + AND ws1.ws_order_number IN + (SELECT ws_order_number + FROM ws_wh) + AND ws1.ws_order_number IN + (SELECT wr_order_number + FROM web_returns, + ws_wh + WHERE wr_order_number = ws_wh.ws_order_number) +ORDER BY count(DISTINCT ws_order_number) +LIMIT 100; + diff --git a/testdata/tpcds/queries/q96.sql b/testdata/tpcds/queries/q96.sql new file mode 100644 index 00000000..5066e61b --- /dev/null +++ b/testdata/tpcds/queries/q96.sql @@ -0,0 +1,15 @@ +SELECT count(*) +FROM store_sales , + household_demographics, + time_dim, + store +WHERE ss_sold_time_sk = time_dim.t_time_sk + AND ss_hdemo_sk = household_demographics.hd_demo_sk + AND ss_store_sk = s_store_sk + AND time_dim.t_hour = 20 + AND time_dim.t_minute >= 30 + AND household_demographics.hd_dep_count = 7 + AND store.s_store_name = 'ese' +ORDER BY count(*) +LIMIT 100; + diff --git a/testdata/tpcds/queries/q97.sql b/testdata/tpcds/queries/q97.sql new file mode 100644 index 00000000..39d5ca0a --- /dev/null +++ b/testdata/tpcds/queries/q97.sql @@ -0,0 +1,35 @@ +WITH ssci AS + (SELECT ss_customer_sk customer_sk , + ss_item_sk item_sk + FROM store_sales, + date_dim + WHERE ss_sold_date_sk = d_date_sk + AND d_month_seq BETWEEN 1200 AND 1200 + 11 + GROUP BY ss_customer_sk , + ss_item_sk), + csci as + ( SELECT cs_bill_customer_sk customer_sk ,cs_item_sk item_sk + FROM catalog_sales,date_dim + WHERE cs_sold_date_sk = d_date_sk + AND d_month_seq BETWEEN 1200 AND 1200 + 11 + GROUP BY cs_bill_customer_sk ,cs_item_sk) +SELECT sum(CASE + WHEN ssci.customer_sk IS NOT NULL + AND csci.customer_sk IS NULL THEN 1 + ELSE 0 + END) store_only , + sum(CASE + WHEN ssci.customer_sk IS NULL + AND csci.customer_sk IS NOT NULL THEN 1 + ELSE 0 + END) catalog_only , + sum(CASE + WHEN ssci.customer_sk IS NOT NULL + AND csci.customer_sk IS NOT NULL THEN 1 + ELSE 0 + END) store_and_catalog +FROM ssci +FULL OUTER JOIN csci ON (ssci.customer_sk=csci.customer_sk + AND ssci.item_sk = csci.item_sk) +LIMIT 100; + diff --git a/testdata/tpcds/queries/q98.sql b/testdata/tpcds/queries/q98.sql new file mode 100644 index 00000000..b9a57db9 --- /dev/null +++ b/testdata/tpcds/queries/q98.sql @@ -0,0 +1,26 @@ +SELECT i_item_id , + i_item_desc, + i_category, + i_class, + i_current_price , + sum(ss_ext_sales_price) AS itemrevenue, + sum(ss_ext_sales_price)*100.0000/sum(sum(ss_ext_sales_price)) OVER (PARTITION BY i_class) AS revenueratio +FROM store_sales , + item, + date_dim +WHERE ss_item_sk = i_item_sk + AND i_category IN ('Sports', + 'Books', + 'Home') + AND ss_sold_date_sk = d_date_sk + AND d_date BETWEEN cast('1999-02-22' AS date) AND cast('1999-03-24' AS date) +GROUP BY i_item_id , + i_item_desc, + i_category , + i_class , + i_current_price +ORDER BY i_category NULLS FIRST, + i_class NULLS FIRST, + i_item_id NULLS FIRST, + i_item_desc NULLS FIRST, + revenueratio NULLS FIRST; diff --git a/testdata/tpcds/queries/q99.sql b/testdata/tpcds/queries/q99.sql new file mode 100644 index 00000000..0501a61c --- /dev/null +++ b/testdata/tpcds/queries/q99.sql @@ -0,0 +1,45 @@ +SELECT w_substr , + sm_type , + LOWER(cc_name) cc_name_lower , + sum(CASE + WHEN (cs_ship_date_sk - cs_sold_date_sk <= 30) THEN 1 + ELSE 0 + END) AS "30 days", + sum(CASE + WHEN (cs_ship_date_sk - cs_sold_date_sk > 30) + AND (cs_ship_date_sk - cs_sold_date_sk <= 60) THEN 1 + ELSE 0 + END) AS "31-60 days", + sum(CASE + WHEN (cs_ship_date_sk - cs_sold_date_sk > 60) + AND (cs_ship_date_sk - cs_sold_date_sk <= 90) THEN 1 + ELSE 0 + END) AS "61-90 days", + sum(CASE + WHEN (cs_ship_date_sk - cs_sold_date_sk > 90) + AND (cs_ship_date_sk - cs_sold_date_sk <= 120) THEN 1 + ELSE 0 + END) AS "91-120 days", + sum(CASE + WHEN (cs_ship_date_sk - cs_sold_date_sk > 120) THEN 1 + ELSE 0 + END) AS ">120 days" +FROM catalog_sales , + (SELECT SUBSTRING(w_warehouse_name,1,20) w_substr, * + FROM warehouse) AS sq1 , + ship_mode , + call_center , + date_dim +WHERE d_month_seq BETWEEN 1200 AND 1200 + 11 + AND cs_ship_date_sk = d_date_sk + AND cs_warehouse_sk = w_warehouse_sk + AND cs_ship_mode_sk = sm_ship_mode_sk + AND cs_call_center_sk = cc_call_center_sk +GROUP BY w_substr , + sm_type , + cc_name +ORDER BY w_substr NULLS FIRST, + sm_type NULLS FIRST, + cc_name_lower NULLS FIRST +LIMIT 100; + diff --git a/tests/clickbench_correctness_test.rs b/tests/clickbench_correctness_test.rs new file mode 100644 index 00000000..fff1bf2e --- /dev/null +++ b/tests/clickbench_correctness_test.rs @@ -0,0 +1,317 @@ +#[cfg(all(feature = "integration", feature = "clickbench", test))] +mod tests { + use datafusion::arrow::array::RecordBatch; + use datafusion::common::plan_err; + use datafusion::error::Result; + use datafusion::physical_plan::{ExecutionPlan, collect}; + use datafusion::prelude::SessionContext; + use datafusion_distributed::test_utils::localhost::start_localhost_context; + use datafusion_distributed::test_utils::property_based::{ + compare_ordering, compare_result_set, + }; + use datafusion_distributed::test_utils::{benchmarks_common, clickbench}; + use datafusion_distributed::{ + DefaultSessionBuilder, DistributedExec, DistributedExt, display_plan_ascii, + }; + use std::ops::Range; + use std::path::Path; + use std::sync::Arc; + use tokio::sync::OnceCell; + + const NUM_WORKERS: usize = 4; + const FILES_PER_TASK: usize = 2; + const CARDINALITY_TASK_COUNT_FACTOR: f64 = 2.0; + const FILE_RANGE: Range = 0..3; + + #[tokio::test] + #[ignore = "Query 0 did not get distributed"] + async fn test_clickbench_0() -> Result<()> { + test_clickbench_query("q0").await + } + + #[tokio::test] + async fn test_clickbench_1() -> Result<()> { + test_clickbench_query("q1").await + } + + #[tokio::test] + async fn test_clickbench_2() -> Result<()> { + test_clickbench_query("q2").await + } + + #[tokio::test] + #[ignore = "result sets were not equal: Internal error: Row content differs between result sets\nLeft set size: 1, Right set size: 1\n\nRows only in left (1 total):\n 2533767602294735360.00\n\nRows only in right (1 total):\n 2533767602294735872.00.\nThis issue was likely caused by a bug in DataFusion's code. Please help us to resolve this by filing a bug report in our issue tracker: https://github.com/apache/datafusion/issues"] + async fn test_clickbench_3() -> Result<()> { + test_clickbench_query("q3").await + } + + #[tokio::test] + async fn test_clickbench_4() -> Result<()> { + test_clickbench_query("q4").await + } + + #[tokio::test] + async fn test_clickbench_5() -> Result<()> { + test_clickbench_query("q5").await + } + + #[tokio::test] + #[ignore = "Query 6 did not get distributed"] + async fn test_clickbench_6() -> Result<()> { + test_clickbench_query("q6").await + } + + #[tokio::test] + async fn test_clickbench_7() -> Result<()> { + test_clickbench_query("q7").await + } + + #[tokio::test] + async fn test_clickbench_8() -> Result<()> { + test_clickbench_query("q8").await + } + + #[tokio::test] + async fn test_clickbench_9() -> Result<()> { + test_clickbench_query("q9").await + } + + #[tokio::test] + async fn test_clickbench_10() -> Result<()> { + test_clickbench_query("q10").await + } + + #[tokio::test] + async fn test_clickbench_11() -> Result<()> { + test_clickbench_query("q11").await + } + + #[tokio::test] + async fn test_clickbench_12() -> Result<()> { + test_clickbench_query("q12").await + } + + #[tokio::test] + async fn test_clickbench_13() -> Result<()> { + test_clickbench_query("q13").await + } + + #[tokio::test] + async fn test_clickbench_14() -> Result<()> { + test_clickbench_query("q14").await + } + + #[tokio::test] + async fn test_clickbench_15() -> Result<()> { + test_clickbench_query("q15").await + } + + #[tokio::test] + async fn test_clickbench_16() -> Result<()> { + test_clickbench_query("q16").await + } + + #[tokio::test] + #[ignore = "result sets were not equal: Internal error: Row content differs between result sets\nLeft set size: 10, Right set size: 10\n\nRows only in left (10 total):\n 3219866204653196665||4\n 3220056705148678697||11\n 3221898002592879542||1\n 3223026783585713477||23\n 3223839745005575457||116\n 3223839745005575457|d0bcd0bed0b6d0bdd0be20d0bbd0b820d0b220d0bad180d0bed0bad0bed0b4d0b8d180d0bed0b2d0b5d0bbd18cd18820d0b1d180d0bed0b4d18b20d0bdd0b020d181d182d0bed0bbd18b20d0b2d0be20d0b2d0bbd0b0d0b4d0b8d0b2d0bed181d182d0bed0ba20d0b2d0b2d0be|1\n 3223949769615485893||1\n 3226415756450197918||24\n 3226664959488084815||62\n 3227160743723019373||71\n\nRows only in right (10 total):\n 700182585509527889||2\n 724127359630680276|d0b8d0b3d180d18b20d0b820d181d0b5d0b3d0bed0b4d0bdd18f3f|1\n 766120398574852544||1\n 766739966065297239||1\n 783205612738304865||3\n 797289180007803204||2\n 804968013253615745||1\n 830548852254311605||1\n 849024737642146119||1\n 849169469997862534||1.\nThis issue was likely caused by a bug in DataFusion's code. Please help us to resolve this by filing a bug report in our issue tracker: https://github.com/apache/datafusion/issues"] + async fn test_clickbench_17() -> Result<()> { + test_clickbench_query("q17").await + } + + #[tokio::test] + async fn test_clickbench_18() -> Result<()> { + test_clickbench_query("q18").await + } + + #[tokio::test] + async fn test_clickbench_19() -> Result<()> { + test_clickbench_query("q19").await + } + + #[tokio::test] + async fn test_clickbench_20() -> Result<()> { + test_clickbench_query("q20").await + } + + #[tokio::test] + #[ignore = "result sets were not equal: Internal error: Row content differs between result sets\nLeft set size: 10, Right set size: 10\n\nRows only in left (5 total):\n d181d0bbd0b0d0b2d0bbd18fd182d18c20d0bfd0bed180d0bed0b4d0b8d182d181d18f20d0bed182d0b5d0bbd0b8203230313320d181d0bcd0bed182d180d0b5d182d18c|687474703a253246253246766b2e636f6d2e75612f676f6f676c652d6a61726b6f76736b6179612d4c697065636b64|1\n d0b1d0b0d0bdd0bad0bed0bcd0b0d182d0b5d180d0b8d0b0d0bbd18b20d181d0bcd0bed182d180d0b5d182d18c|687474703a2f2f6f72656e627572672e6972722e72752532466b7572746b692532462532467777772e676f6f676c652e72752f6d617a64612d332d6b6f6d6e2d6b762d4b617a616e2e74757475746f72736b2f64657461696c|1\n d0bcd0bed0bdd0b8d182d18c20d0bad0b0d0bad0bed0b520d0bed0b7d0b5d180d0b0|687474703a2f2f6175746f2e7269612e75612f6175746f5f69643d30266f726465723d46616c7365266d696e707269782e72752f6b617465676f726979612f767369652d646c69612d647275676f652f6d61746572696e7374766f2f676f6f676c652d706f6c697331343334343532|1\n d181d0bad0b0d187d0b0d182d18c20d0b4d0b5d0bdd0b5d0b320d181d183d180d0b3d183d182|687474703a2f2f7469656e736b6169612d6d6f64612d627269657469656c6b612d6b6f736b6f76736b2f64657461696c2e676f6f676c65|1\n d0b220d0b0d0b2d0b3d183d181d1822032343720d0b3d180d183d181d182d0b8d0bcd0bed188d0bad0b020d0bdd0b020d0bad180d0b8d181d182d180d0b0d182|687474703a2f2f7469656e736b6169612d6d6f64612d627269756b692f676f6f676c652e72752f7e61706f6b2e72752f635f312d755f313138383839352c39373536|1\n\nRows only in right (5 total):\n d0bcd0bed0b4d0b5d0bad18120d183d0bbd0b8d186d0b5d0bdd0b7d0b8d0bdd0bed0b2d0b020d0b3d0bed0b2d18fd0b4d0b8d0bdd0b0|687474703a2f2f73616d6172612e6972722e72752f636174616c6f675f676f6f676c652d636865726e796a2d393233353636363635372f3f64617465|1\n d0bbd0b0d0b2d0bfd0bbd0b0d0bdd188d0b5d182d0bdd0b8d18520d183d181d0bbd0bed0b2d0b0d0bcd0b820d0b2d181d0b520d181d0b5d180d0b8d0b820d0b4d0b0d182d0b020d186d0b5d0bcd0b5d0bdd0b8|687474703a2f2f73616d6172612e6972722e72752f636174616c6f675f676f6f676c654d425225323661642533443930253236707a|1\n d0bad0b0d0ba20d0bfd180d0bed0b4d0b0d0bcd0b820d0b4d0bbd18f20d0b4d0b5d0b2d183d188d0bad0b8|687474703a253246253246777777772e626f6e707269782e7275253235326625323532663737363925323532663131303931392d6c65766f652d676f6f676c652d7368746f72792e72752f666f72756d2f666f72756d2e6d617465722e72752f6461696c792f63616c63756c61746f72|1\n d0b6d0b0d180d0b5d0bdd18cd18f20d0b32ed181d183d180d0bed0b2d0b0d0bdd0b8d0b520d0b2d0bed180d0bed0bdd0b5d0b6d181d0bad0b0d18f20d0bed0b1d0bbd0b0d181d182d0bed0bfd180d0b8d0bbd0b520d0bfd0bed181d0bbd0b5d0b4d0bdd0b8d0b520d0bad0bed181d18b|687474703a2f2f756b7261696e627572672f65636f2d6d6c656b2f65636f6e646172792f73686f77746f7069632e7068703f69643d3436333837362e68746d6c3f69643d32303634313333363631253246676f6f676c652d4170706c655765624b69742532463533372e333620284b48544d4c2c206c696b65|1\n d180d0b8d0be20d0bdd0b020d0bad0b0d180d182d0bed187d0bdd0b8d186d0b020d181d0bcd0bed182d180d0b5d182d18c20d0bed0bdd0bbd0b0d0b9d0bd|687474703a2f2f73616d6172612e6972722e72752f636174616c6f675f676f6f676c654d425225323661642533443930253236707a|1.\nThis issue was likely caused by a bug in DataFusion's code. Please help us to resolve this by filing a bug report in our issue tracker: https://github.com/apache/datafusion/issues"] + async fn test_clickbench_21() -> Result<()> { + test_clickbench_query("q21").await + } + + #[tokio::test] + #[ignore = "result sets were not equal: Internal error: Row content differs between result sets\nLeft set size: 10, Right set size: 10\n\nRows only in left (1 total):\n d0bad0b0d0bad0bed0b920d0bfd0bbd0bed189d0b0d0b4d0bad0b8d0bcd0b820d0b4d0bed181d182d0b0d0b2d0bad0b8|687474703a253246253246766b2e636f6d2f696672616d652d6f77612e68746d6c3f313d31266369643d353737266f6b693d31266f705f63617465676f72795f69645d3d332673656c656374|d092d0b0d0bad0b0d0bdd181d0b8d18f20d091d0a0d090d09ad090d09d20d090d09dd094d0a0d095d0a1202d20d0bfd0bed0bfd0b0d0bbd0b820d0bad183d0bfd0b8d182d18c20d0b4d0bed0bcd0bed0b5d187d0bdd18bd0b520d188d0bad0b0d184d0b020476f6f676c652e636f6d203a3a20d0bad0bed182d182d0b5d0bad181d1822c20d091d183d180d18fd182d0bdd0b8d0bad0b820d0b4d0bbd18f20d0bfd0b5d187d18c20d0bcd0b5d0b1d0b5d0bbd18cd0b520d0b4d0bbd18f20d0b4d0b5d0b2d183d188d0bad0b0|5|1\n\nRows only in right (1 total):\n d0bad0bed0bfd182d0b8d0bcd0b8d0bad0b2d0b8d0b4d18b20d18ed180d0b8d0b920d0bfd0bed181d0bbd0b5d0b4d0bdd18fd18f|68747470733a2f2f70726f64756b747925324670756c6f76652e72752f626f6f6b6c79617474696f6e2d7761722d73696e696a2d393430343139342c3936323435332f666f746f|d09bd0b5d0b3d0bad0be20d0bdd0b020d183d187d0b0d181d182d0bdd18bd0b520d183d187d0b0d181d182d0bdd0b8d0bad0bed0b22e2c20d0a6d0b5d0bdd18b202d20d0a1d182d0b8d0bbd18cd0bdd0b0d18f20d0bfd0b0d180d0bdd0b5d0bc2e20d0a1d0b0d0b3d0b0d0bdd180d0bed0b320d0b4d0bed0b3d0b0d0b4d0b5d0bdd0b8d18f203a20d0a2d183d180d186d0b8d0b82c20d0bad183d0bfd0b8d182d18c20d18320313020d0b4d0bdd0b520d0bad0bed0bbd18cd0bdd18bd0b520d0bcd0b0d188d0b8d0bdd0bad0b820d0bdd0b520d0bfd180d0b5d0b4d181d182d0b0d0b2d0bad0b8202d20d09dd0bed0b2d0b0d18f20d18120d0b8d0b7d0b1d0b8d0b5d0bdd0b8d0b520d181d0bfd180d0bed0b4d0b0d0b6d0b03a20d0bad0bed182d18fd182d0b0203230313420d0b32ed0b22e20d0a6d0b5d0bdd0b03a2034373530302d313045434f30363020e28093202d2d2d2d2d2d2d2d20d0bad183d0bfd0b8d182d18c20d0bad0b2d0b0d180d182d0b8d180d18320d09ed180d0b5d0bdd0b1d183d180d0b32028d0a0d0bed181d181d0b8d0b82047616c616e7472617820466c616d696c6961646120476f6f676c652c204ed0be20313820d184d0bed182d0bed0bad0bed0bdd0b2d0b5d180d0ba20d0a1d183d0bfd0b5d18020d09ad0b0d180d0b4d0b8d0b3d0b0d0bd|5|1.\nThis issue was likely caused by a bug in DataFusion's code. Please help us to resolve this by filing a bug report in our issue tracker: https://github.com/apache/datafusion/issues"] + async fn test_clickbench_22() -> Result<()> { + test_clickbench_query("q22").await + } + + #[tokio::test] + async fn test_clickbench_23() -> Result<()> { + test_clickbench_query("q23").await + } + + #[tokio::test] + #[ignore = "result sets were not equal: Internal error: Row content differs between result sets\nLeft set size: 10, Right set size: 10\n\nRows only in left (1 total):\n d0b2d181d0bfd0bed0bcd0bdd0b8d182d18c20d181d0bed0bbd0bdd0b5d0bdd0b8d0b520d0b1d0b0d0bdd0bad0b020d0bbd0b0d0b420d184d0b8d0bbd18cd0bc\n\nRows only in right (1 total):\n d0bed182d0b2d0bed0b4d0b020d0b4d0bbd18f20d0bfd0b8d180d0bed0b6d0bad0b820d0bbd0b5d187d0b5d0bdd0bdd18b20d0b2d181d0b520d181d0b5d180d196d197.\nThis issue was likely caused by a bug in DataFusion's code. Please help us to resolve this by filing a bug report in our issue tracker: https://github.com/apache/datafusion/issues"] + async fn test_clickbench_24() -> Result<()> { + test_clickbench_query("q24").await + } + + #[tokio::test] + async fn test_clickbench_25() -> Result<()> { + test_clickbench_query("q25").await + } + + #[tokio::test] + async fn test_clickbench_26() -> Result<()> { + test_clickbench_query("q26").await + } + + #[tokio::test] + async fn test_clickbench_27() -> Result<()> { + test_clickbench_query("q27").await + } + + #[tokio::test] + async fn test_clickbench_28() -> Result<()> { + test_clickbench_query("q28").await + } + + #[tokio::test] + async fn test_clickbench_29() -> Result<()> { + test_clickbench_query("q29").await + } + + #[tokio::test] + async fn test_clickbench_30() -> Result<()> { + test_clickbench_query("q30").await + } + + #[tokio::test] + #[ignore = "result sets were not equal: Internal error: Row content differs between result sets\nLeft set size: 10, Right set size: 10\n\nRows only in left (10 total):\n 8673025726158767406|1264438551|1|0|1990.00\n 5320052218057629211|-1703087277|1|0|1996.00\n 6244273852606083750|1554672832|1|0|1638.00\n 8628753750962053665|1215278356|1|0|1087.00\n 7035318163404387241|1326714320|1|0|1638.00\n 8431857775494210873|1237512945|1|0|1996.00\n 5110752526539992124|37611695|1|0|1917.00\n 8986794334343068049|1860752926|1|0|1638.00\n 8044147848299485837|1382122372|1|0|1368.00\n 7936057634954670727|1897481896|1|0|1638.00\n\nRows only in right (10 total):\n 5132615111782210132|-50313020|1|0|1368.00\n 5783789691451717551|-1310327384|1|0|1638.00\n 5756260993772351383|1484317883|1|0|375.00\n 7739310142000732364|991864113|1|0|1368.00\n 7593472904893539271|-151291403|1|0|1087.00\n 6339599967989898410|1543815587|1|0|1638.00\n 7794346560421945218|1645556180|1|0|1368.00\n 6112645108657361792|593586188|1|0|1638.00\n 6675910710751922756|-816379256|1|0|1368.00\n 5802727636196431835|1986422271|1|0|1996.00.\nThis issue was likely caused by a bug in DataFusion's code. Please help us to resolve this by filing a bug report in our issue tracker: https://github.com/apache/datafusion/issues"] + async fn test_clickbench_31() -> Result<()> { + test_clickbench_query("q31").await + } + + #[tokio::test] + #[ignore = "result sets were not equal: Internal error: Row content differs between result sets\nLeft set size: 10, Right set size: 10\n\nRows only in left (10 total):\n 7643059318918524417|1767085700|1|0|0.00\n 5437163248266133938|-1465369615|1|0|0.00\n 9142541582422390102|-1465369615|1|0|0.00\n 8438994503411842126|-1465369615|1|0|0.00\n 7362096505818029859|-565678477|1|0|0.00\n 4928022308880516715|1699955284|1|0|0.00\n 5269769817689282522|1699955284|1|0|0.00\n 9081648050908046886|1699955284|1|0|0.00\n 6824181869275536503|1699955284|1|0|0.00\n 6905712404475757487|1552811156|1|0|0.00\n\nRows only in right (10 total):\n 6967277596165459879|-941091661|1|0|1368.00\n 5796237228224217668|-1310327384|1|0|1638.00\n 7218628137278606666|1511490240|1|0|1638.00\n 8314760197723815280|1566105210|1|0|1996.00\n 7053263954762394007|757778490|1|0|339.00\n 6283334114093174531|1216031795|1|0|1368.00\n 8818295356247036741|83042182|1|0|1638.00\n 6620528864937282562|-862894777|1|0|1996.00\n 8466121050002905379|83042182|1|0|1638.00\n 7554844936512227411|-1746904856|1|0|1368.00.\nThis issue was likely caused by a bug in DataFusion's code. Please help us to resolve this by filing a bug report in our issue tracker: https://github.com/apache/datafusion/issues"] + async fn test_clickbench_32() -> Result<()> { + test_clickbench_query("q32").await + } + + #[tokio::test] + async fn test_clickbench_33() -> Result<()> { + test_clickbench_query("q33").await + } + + #[tokio::test] + async fn test_clickbench_34() -> Result<()> { + test_clickbench_query("q34").await + } + + #[tokio::test] + async fn test_clickbench_35() -> Result<()> { + test_clickbench_query("q35").await + } + + #[tokio::test] + async fn test_clickbench_36() -> Result<()> { + test_clickbench_query("q36").await + } + + #[tokio::test] + async fn test_clickbench_37() -> Result<()> { + test_clickbench_query("q37").await + } + + #[tokio::test] + async fn test_clickbench_38() -> Result<()> { + test_clickbench_query("q38").await + } + + #[tokio::test] + async fn test_clickbench_39() -> Result<()> { + test_clickbench_query("q39").await + } + + #[tokio::test] + async fn test_clickbench_40() -> Result<()> { + test_clickbench_query("q40").await + } + + #[tokio::test] + async fn test_clickbench_41() -> Result<()> { + test_clickbench_query("q41").await + } + + #[tokio::test] + #[ignore = "ordering mismatch: expected ordering: Some(LexOrdering { exprs: [PhysicalSortExpr { expr: ScalarFunctionExpr { fun: \"\", name: \"date_trunc\", args: [Literal { value: Utf8(\"minute\"), field: Field { name: \"lit\", data_type: Utf8 } }, Column { name: \"m\", index: 0 }], return_field: Field { name: \"date_trunc\", data_type: Timestamp(Second, None), nullable: true } }, options: SortOptions { descending: false, nulls_first: false } }], set: {ScalarFunctionExpr { fun: \"\", name: \"date_trunc\", args: [Literal { value: Utf8(\"minute\"), field: Field { name: \"lit\", data_type: Utf8 } }, Column { name: \"m\", index: 0 }], return_field: Field { name: \"date_trunc\", data_type: Timestamp(Second, None), nullable: true } }} }), actual ordering: Some(LexOrdering { exprs: [PhysicalSortExpr { expr: ScalarFunctionExpr { fun: \"\", name: \"date_trunc\", args: [Literal { value: Utf8(\"minute\"), field: Field { name: \"lit\", data_type: Utf8 } }, Column { name: \"m\", index: 0 }], return_field: Field { name: \"date_trunc\", data_type: Timestamp(Second, None), nullable: true } }, options: SortOptions { descending: false, nulls_first: false } }], set: {ScalarFunctionExpr { fun: \"\", name: \"date_trunc\", args: [Literal { value: Utf8(\"minute\"), field: Field { name: \"lit\", data_type: Utf8 } }, Column { name: \"m\", index: 0 }], return_field: Field { name: \"date_trunc\", data_type: Timestamp(Second, None), nullable: true } }} })"] + async fn test_clickbench_42() -> Result<()> { + test_clickbench_query("q42").await + } + + static INIT_TEST_TPCDS_TABLES: OnceCell<()> = OnceCell::const_new(); + + async fn run( + ctx: &SessionContext, + query_sql: &str, + ) -> (Arc, Arc>>) { + let df = ctx.sql(query_sql).await.unwrap(); + let task_ctx = ctx.task_ctx(); + let plan = df.create_physical_plan().await.unwrap(); + (plan.clone(), Arc::new(collect(plan, task_ctx).await)) // Collect execution errors, do not unwrap. + } + + async fn test_clickbench_query(query_id: &str) -> Result<()> { + let data_dir = Path::new(env!("CARGO_MANIFEST_DIR")).join(format!( + "testdata/clickbench/correctness_range{}-{}", + FILE_RANGE.start, FILE_RANGE.end + )); + INIT_TEST_TPCDS_TABLES + .get_or_init(|| async { + clickbench::generate_clickbench_data(&data_dir, FILE_RANGE) + .await + .unwrap(); + }) + .await; + + let query_sql = clickbench::get_query(query_id)?; + // Create a single node context to compare results to. + let s_ctx = SessionContext::new(); + + // Make distributed localhost context to run queries + let (d_ctx, _guard, _) = start_localhost_context(NUM_WORKERS, DefaultSessionBuilder).await; + let d_ctx = d_ctx + .with_distributed_files_per_task(FILES_PER_TASK)? + .with_distributed_cardinality_effect_task_scale_factor(CARDINALITY_TASK_COUNT_FACTOR)? + .with_distributed_broadcast_joins(true)?; + + benchmarks_common::register_tables(&s_ctx, &data_dir).await?; + benchmarks_common::register_tables(&d_ctx, &data_dir).await?; + + let (s_plan, s_results) = run(&s_ctx, &query_sql).await; + let (d_plan, d_results) = run(&d_ctx, &query_sql).await; + + if !d_plan.as_any().is::() { + return plan_err!("Query {query_id} did not get distributed"); + } + let display = display_plan_ascii(d_plan.as_ref(), false); + println!("Query {query_id}:\n{display}"); + + let compare_result_set = { + let d_results = d_results.clone(); + let s_results = s_results.clone(); + tokio::task::spawn_blocking(move || async move { + compare_result_set(&d_results, &s_results) + }) + }; + let compare_ordering = { + let d_results = d_results.clone(); + tokio::task::spawn_blocking(move || async move { + compare_ordering(d_plan, s_plan, &d_results) + }) + }; + compare_result_set.await.unwrap().await?; + compare_ordering.await.unwrap().await?; + + Ok(()) + } +} diff --git a/tests/clickbench_plans_test.rs b/tests/clickbench_plans_test.rs new file mode 100644 index 00000000..e851827f --- /dev/null +++ b/tests/clickbench_plans_test.rs @@ -0,0 +1,934 @@ +#[cfg(all(feature = "integration", feature = "clickbench", test))] +mod tests { + use datafusion::error::Result; + use datafusion_distributed::test_utils::localhost::start_localhost_context; + use datafusion_distributed::test_utils::{benchmarks_common, clickbench}; + use datafusion_distributed::{ + DefaultSessionBuilder, DistributedExec, DistributedExt, assert_snapshot, display_plan_ascii, + }; + use std::ops::Range; + use std::path::Path; + use tokio::sync::OnceCell; + + const NUM_WORKERS: usize = 4; + const FILES_PER_TASK: usize = 2; + const CARDINALITY_TASK_COUNT_FACTOR: f64 = 1.5; + const FILE_RANGE: Range = 0..3; + + #[tokio::test] + #[ignore = "Query 0 did not get distributed"] + async fn test_clickbench_0() -> Result<()> { + let display = test_clickbench_query("q0").await?; + assert_snapshot!(display, @""); + Ok(()) + } + + #[tokio::test] + async fn test_clickbench_1() -> Result<()> { + let display = test_clickbench_query("q1").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ ProjectionExec: expr=[count(Int64(1))@0 as count(*)] + │ AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] + │ AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] + │ ProjectionExec: expr=[] + │ FilterExec: AdvEngineID@0 != 0 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__] t1:[__,__,p0,p1,__] t2:[__,__,__,__,p0] + │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[AdvEngineID], file_type=parquet, predicate=AdvEngineID@40 != 0, pruning_predicate=AdvEngineID_null_count@2 != row_count@3 AND (AdvEngineID_min@0 != 0 OR 0 != AdvEngineID_max@1), required_guarantees=[AdvEngineID not in (0)] + └────────────────────────────────────────────────── + "); + Ok(()) + } + + #[tokio::test] + async fn test_clickbench_2() -> Result<()> { + let display = test_clickbench_query("q2").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ ProjectionExec: expr=[sum(hits.AdvEngineID)@0 as sum(hits.AdvEngineID), count(Int64(1))@1 as count(*), avg(hits.ResolutionWidth)@2 as avg(hits.ResolutionWidth)] + │ AggregateExec: mode=Final, gby=[], aggr=[sum(hits.AdvEngineID), count(Int64(1)), avg(hits.ResolutionWidth)] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] + │ AggregateExec: mode=Partial, gby=[], aggr=[sum(hits.AdvEngineID), count(Int64(1)), avg(hits.ResolutionWidth)] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__] t1:[__,__,p0,p1,__] t2:[__,__,__,__,p0] + │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[ResolutionWidth, AdvEngineID], file_type=parquet + └────────────────────────────────────────────────── + "); + Ok(()) + } + + #[tokio::test] + #[ignore = "result sets were not equal: Internal error: Row content differs between result sets\nLeft set size: 1, Right set size: 1\n\nRows only in left (1 total):\n 2533767602294735360.00\n\nRows only in right (1 total):\n 2533767602294735872.00.\nThis issue was likely caused by a bug in DataFusion's code. Please help us to resolve this by filing a bug report in our issue tracker: https://github.com/apache/datafusion/issues"] + async fn test_clickbench_3() -> Result<()> { + let display = test_clickbench_query("q3").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ AggregateExec: mode=Final, gby=[], aggr=[avg(hits.UserID)] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ AggregateExec: mode=Partial, gby=[], aggr=[avg(hits.UserID)] + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__] t1:[__,__,__,p0,p1] + │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[UserID], file_type=parquet + └────────────────────────────────────────────────── + "); + Ok(()) + } + + #[tokio::test] + async fn test_clickbench_4() -> Result<()> { + let display = test_clickbench_query("q4").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ ProjectionExec: expr=[count(alias1)@0 as count(DISTINCT hits.UserID)] + │ AggregateExec: mode=Final, gby=[], aggr=[count(alias1)] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ AggregateExec: mode=Partial, gby=[], aggr=[count(alias1)] + │ AggregateExec: mode=FinalPartitioned, gby=[alias1@0 as alias1], aggr=[] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([alias1@0], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[UserID@0 as alias1], aggr=[] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__] t1:[__,__,p0,p1,__] t2:[__,__,__,__,p0] + │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[UserID], file_type=parquet + └────────────────────────────────────────────────── + "); + Ok(()) + } + + #[tokio::test] + async fn test_clickbench_5() -> Result<()> { + let display = test_clickbench_query("q5").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ ProjectionExec: expr=[count(alias1)@0 as count(DISTINCT hits.SearchPhrase)] + │ AggregateExec: mode=Final, gby=[], aggr=[count(alias1)] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ AggregateExec: mode=Partial, gby=[], aggr=[count(alias1)] + │ AggregateExec: mode=FinalPartitioned, gby=[alias1@0 as alias1], aggr=[] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([alias1@0], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[SearchPhrase@0 as alias1], aggr=[] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__] t1:[__,__,p0,p1,__] t2:[__,__,__,__,p0] + │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[SearchPhrase], file_type=parquet + └────────────────────────────────────────────────── + "); + Ok(()) + } + + #[tokio::test] + #[ignore = "Query 6 did not get distributed"] + async fn test_clickbench_6() -> Result<()> { + let display = test_clickbench_query("q6").await?; + assert_snapshot!(display, @""); + Ok(()) + } + + #[tokio::test] + async fn test_clickbench_7() -> Result<()> { + let display = test_clickbench_query("q7").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ ProjectionExec: expr=[AdvEngineID@0 as AdvEngineID, count(*)@1 as count(*)] + │ SortPreservingMergeExec: [count(Int64(1))@2 DESC] + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ SortExec: expr=[count(*)@1 DESC], preserve_partitioning=[true] + │ ProjectionExec: expr=[AdvEngineID@0 as AdvEngineID, count(Int64(1))@1 as count(*), count(Int64(1))@1 as count(Int64(1))] + │ AggregateExec: mode=FinalPartitioned, gby=[AdvEngineID@0 as AdvEngineID], aggr=[count(Int64(1))] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([AdvEngineID@0], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[AdvEngineID@0 as AdvEngineID], aggr=[count(Int64(1))] + │ FilterExec: AdvEngineID@0 != 0 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__] t1:[__,__,p0,p1,__] t2:[__,__,__,__,p0] + │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[AdvEngineID], file_type=parquet, predicate=AdvEngineID@40 != 0, pruning_predicate=AdvEngineID_null_count@2 != row_count@3 AND (AdvEngineID_min@0 != 0 OR 0 != AdvEngineID_max@1), required_guarantees=[AdvEngineID not in (0)] + └────────────────────────────────────────────────── + "); + Ok(()) + } + + #[tokio::test] + async fn test_clickbench_8() -> Result<()> { + let display = test_clickbench_query("q8").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [u@1 DESC], fetch=10 + │ SortExec: TopK(fetch=10), expr=[u@1 DESC], preserve_partitioning=[true] + │ ProjectionExec: expr=[RegionID@0 as RegionID, count(alias1)@1 as u] + │ AggregateExec: mode=FinalPartitioned, gby=[RegionID@0 as RegionID], aggr=[count(alias1)] + │ [Stage 2] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ RepartitionExec: partitioning=Hash([RegionID@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[RegionID@0 as RegionID], aggr=[count(alias1)] + │ AggregateExec: mode=FinalPartitioned, gby=[RegionID@0 as RegionID, alias1@1 as alias1], aggr=[] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([RegionID@0, alias1@1], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[RegionID@0 as RegionID, UserID@1 as alias1], aggr=[] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__] t1:[__,__,p0,p1,__] t2:[__,__,__,__,p0] + │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[RegionID, UserID], file_type=parquet + └────────────────────────────────────────────────── + "); + Ok(()) + } + + #[tokio::test] + async fn test_clickbench_9() -> Result<()> { + let display = test_clickbench_query("q9").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [c@2 DESC], fetch=10 + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ SortExec: TopK(fetch=10), expr=[c@2 DESC], preserve_partitioning=[true] + │ ProjectionExec: expr=[RegionID@0 as RegionID, sum(hits.AdvEngineID)@1 as sum(hits.AdvEngineID), count(Int64(1))@2 as c, avg(hits.ResolutionWidth)@3 as avg(hits.ResolutionWidth), count(DISTINCT hits.UserID)@4 as count(DISTINCT hits.UserID)] + │ AggregateExec: mode=FinalPartitioned, gby=[RegionID@0 as RegionID], aggr=[sum(hits.AdvEngineID), count(Int64(1)), avg(hits.ResolutionWidth), count(DISTINCT hits.UserID)] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([RegionID@0], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[RegionID@0 as RegionID], aggr=[sum(hits.AdvEngineID), count(Int64(1)), avg(hits.ResolutionWidth), count(DISTINCT hits.UserID)] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__] t1:[__,__,p0,p1,__] t2:[__,__,__,__,p0] + │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[RegionID, UserID, ResolutionWidth, AdvEngineID], file_type=parquet + └────────────────────────────────────────────────── + "); + Ok(()) + } + + #[tokio::test] + async fn test_clickbench_10() -> Result<()> { + let display = test_clickbench_query("q10").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [u@1 DESC], fetch=10 + │ SortExec: TopK(fetch=10), expr=[u@1 DESC], preserve_partitioning=[true] + │ ProjectionExec: expr=[MobilePhoneModel@0 as MobilePhoneModel, count(alias1)@1 as u] + │ AggregateExec: mode=FinalPartitioned, gby=[MobilePhoneModel@0 as MobilePhoneModel], aggr=[count(alias1)] + │ [Stage 2] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ RepartitionExec: partitioning=Hash([MobilePhoneModel@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[MobilePhoneModel@0 as MobilePhoneModel], aggr=[count(alias1)] + │ AggregateExec: mode=FinalPartitioned, gby=[MobilePhoneModel@0 as MobilePhoneModel, alias1@1 as alias1], aggr=[] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([MobilePhoneModel@0, alias1@1], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[MobilePhoneModel@1 as MobilePhoneModel, UserID@0 as alias1], aggr=[] + │ FilterExec: MobilePhoneModel@1 != + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__] t1:[__,__,p0,p1,__] t2:[__,__,__,__,p0] + │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[UserID, MobilePhoneModel], file_type=parquet, predicate=MobilePhoneModel@34 != , pruning_predicate=MobilePhoneModel_null_count@2 != row_count@3 AND (MobilePhoneModel_min@0 != OR != MobilePhoneModel_max@1), required_guarantees=[MobilePhoneModel not in ()] + └────────────────────────────────────────────────── + "); + Ok(()) + } + + #[tokio::test] + async fn test_clickbench_11() -> Result<()> { + let display = test_clickbench_query("q11").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [u@2 DESC], fetch=10 + │ SortExec: TopK(fetch=10), expr=[u@2 DESC], preserve_partitioning=[true] + │ ProjectionExec: expr=[MobilePhone@0 as MobilePhone, MobilePhoneModel@1 as MobilePhoneModel, count(alias1)@2 as u] + │ AggregateExec: mode=FinalPartitioned, gby=[MobilePhone@0 as MobilePhone, MobilePhoneModel@1 as MobilePhoneModel], aggr=[count(alias1)] + │ [Stage 2] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ RepartitionExec: partitioning=Hash([MobilePhone@0, MobilePhoneModel@1], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[MobilePhone@0 as MobilePhone, MobilePhoneModel@1 as MobilePhoneModel], aggr=[count(alias1)] + │ AggregateExec: mode=FinalPartitioned, gby=[MobilePhone@0 as MobilePhone, MobilePhoneModel@1 as MobilePhoneModel, alias1@2 as alias1], aggr=[] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([MobilePhone@0, MobilePhoneModel@1, alias1@2], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[MobilePhone@1 as MobilePhone, MobilePhoneModel@2 as MobilePhoneModel, UserID@0 as alias1], aggr=[] + │ FilterExec: MobilePhoneModel@2 != + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__] t1:[__,__,p0,p1,__] t2:[__,__,__,__,p0] + │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[UserID, MobilePhone, MobilePhoneModel], file_type=parquet, predicate=MobilePhoneModel@34 != , pruning_predicate=MobilePhoneModel_null_count@2 != row_count@3 AND (MobilePhoneModel_min@0 != OR != MobilePhoneModel_max@1), required_guarantees=[MobilePhoneModel not in ()] + └────────────────────────────────────────────────── + "); + Ok(()) + } + + #[tokio::test] + async fn test_clickbench_12() -> Result<()> { + let display = test_clickbench_query("q12").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [c@1 DESC], fetch=10 + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ SortExec: TopK(fetch=10), expr=[c@1 DESC], preserve_partitioning=[true] + │ ProjectionExec: expr=[SearchPhrase@0 as SearchPhrase, count(Int64(1))@1 as c] + │ AggregateExec: mode=FinalPartitioned, gby=[SearchPhrase@0 as SearchPhrase], aggr=[count(Int64(1))] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([SearchPhrase@0], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[SearchPhrase@0 as SearchPhrase], aggr=[count(Int64(1))] + │ FilterExec: SearchPhrase@0 != + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__] t1:[__,__,p0,p1,__] t2:[__,__,__,__,p0] + │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[SearchPhrase], file_type=parquet, predicate=SearchPhrase@39 != , pruning_predicate=SearchPhrase_null_count@2 != row_count@3 AND (SearchPhrase_min@0 != OR != SearchPhrase_max@1), required_guarantees=[SearchPhrase not in ()] + └────────────────────────────────────────────────── + "); + Ok(()) + } + + #[tokio::test] + async fn test_clickbench_13() -> Result<()> { + let display = test_clickbench_query("q13").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [u@1 DESC], fetch=10 + │ SortExec: TopK(fetch=10), expr=[u@1 DESC], preserve_partitioning=[true] + │ ProjectionExec: expr=[SearchPhrase@0 as SearchPhrase, count(alias1)@1 as u] + │ AggregateExec: mode=FinalPartitioned, gby=[SearchPhrase@0 as SearchPhrase], aggr=[count(alias1)] + │ [Stage 2] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ RepartitionExec: partitioning=Hash([SearchPhrase@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[SearchPhrase@0 as SearchPhrase], aggr=[count(alias1)] + │ AggregateExec: mode=FinalPartitioned, gby=[SearchPhrase@0 as SearchPhrase, alias1@1 as alias1], aggr=[] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([SearchPhrase@0, alias1@1], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[SearchPhrase@1 as SearchPhrase, UserID@0 as alias1], aggr=[] + │ FilterExec: SearchPhrase@1 != + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__] t1:[__,__,p0,p1,__] t2:[__,__,__,__,p0] + │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[UserID, SearchPhrase], file_type=parquet, predicate=SearchPhrase@39 != , pruning_predicate=SearchPhrase_null_count@2 != row_count@3 AND (SearchPhrase_min@0 != OR != SearchPhrase_max@1), required_guarantees=[SearchPhrase not in ()] + └────────────────────────────────────────────────── + "); + Ok(()) + } + + #[tokio::test] + async fn test_clickbench_14() -> Result<()> { + let display = test_clickbench_query("q14").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [c@2 DESC], fetch=10 + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ SortExec: TopK(fetch=10), expr=[c@2 DESC], preserve_partitioning=[true] + │ ProjectionExec: expr=[SearchEngineID@0 as SearchEngineID, SearchPhrase@1 as SearchPhrase, count(Int64(1))@2 as c] + │ AggregateExec: mode=FinalPartitioned, gby=[SearchEngineID@0 as SearchEngineID, SearchPhrase@1 as SearchPhrase], aggr=[count(Int64(1))] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([SearchEngineID@0, SearchPhrase@1], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[SearchEngineID@0 as SearchEngineID, SearchPhrase@1 as SearchPhrase], aggr=[count(Int64(1))] + │ FilterExec: SearchPhrase@1 != + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__] t1:[__,__,p0,p1,__] t2:[__,__,__,__,p0] + │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[SearchEngineID, SearchPhrase], file_type=parquet, predicate=SearchPhrase@39 != , pruning_predicate=SearchPhrase_null_count@2 != row_count@3 AND (SearchPhrase_min@0 != OR != SearchPhrase_max@1), required_guarantees=[SearchPhrase not in ()] + └────────────────────────────────────────────────── + "); + Ok(()) + } + + #[tokio::test] + async fn test_clickbench_15() -> Result<()> { + let display = test_clickbench_query("q15").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ ProjectionExec: expr=[UserID@0 as UserID, count(*)@1 as count(*)] + │ SortPreservingMergeExec: [count(Int64(1))@2 DESC], fetch=10 + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ SortExec: TopK(fetch=10), expr=[count(*)@1 DESC], preserve_partitioning=[true] + │ ProjectionExec: expr=[UserID@0 as UserID, count(Int64(1))@1 as count(*), count(Int64(1))@1 as count(Int64(1))] + │ AggregateExec: mode=FinalPartitioned, gby=[UserID@0 as UserID], aggr=[count(Int64(1))] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([UserID@0], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[UserID@0 as UserID], aggr=[count(Int64(1))] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__] t1:[__,__,p0,p1,__] t2:[__,__,__,__,p0] + │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[UserID], file_type=parquet + └────────────────────────────────────────────────── + "); + Ok(()) + } + + #[tokio::test] + async fn test_clickbench_16() -> Result<()> { + let display = test_clickbench_query("q16").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ ProjectionExec: expr=[UserID@0 as UserID, SearchPhrase@1 as SearchPhrase, count(*)@2 as count(*)] + │ SortPreservingMergeExec: [count(Int64(1))@3 DESC], fetch=10 + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ SortExec: TopK(fetch=10), expr=[count(*)@2 DESC], preserve_partitioning=[true] + │ ProjectionExec: expr=[UserID@0 as UserID, SearchPhrase@1 as SearchPhrase, count(Int64(1))@2 as count(*), count(Int64(1))@2 as count(Int64(1))] + │ AggregateExec: mode=FinalPartitioned, gby=[UserID@0 as UserID, SearchPhrase@1 as SearchPhrase], aggr=[count(Int64(1))] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([UserID@0, SearchPhrase@1], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[UserID@0 as UserID, SearchPhrase@1 as SearchPhrase], aggr=[count(Int64(1))] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__] t1:[__,__,p0,p1,__] t2:[__,__,__,__,p0] + │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[UserID, SearchPhrase], file_type=parquet + └────────────────────────────────────────────────── + "); + Ok(()) + } + + #[tokio::test] + #[ignore = "result sets were not equal: Internal error: Row content differs between result sets\nLeft set size: 10, Right set size: 10\n\nRows only in left (10 total):\n 3219866204653196665||4\n 3220056705148678697||11\n 3221898002592879542||1\n 3223026783585713477||23\n 3223839745005575457||116\n 3223839745005575457|d0bcd0bed0b6d0bdd0be20d0bbd0b820d0b220d0bad180d0bed0bad0bed0b4d0b8d180d0bed0b2d0b5d0bbd18cd18820d0b1d180d0bed0b4d18b20d0bdd0b020d181d182d0bed0bbd18b20d0b2d0be20d0b2d0bbd0b0d0b4d0b8d0b2d0bed181d182d0bed0ba20d0b2d0b2d0be|1\n 3223949769615485893||1\n 3226415756450197918||24\n 3226664959488084815||62\n 3227160743723019373||71\n\nRows only in right (10 total):\n 700182585509527889||2\n 724127359630680276|d0b8d0b3d180d18b20d0b820d181d0b5d0b3d0bed0b4d0bdd18f3f|1\n 766120398574852544||1\n 766739966065297239||1\n 783205612738304865||3\n 797289180007803204||2\n 804968013253615745||1\n 830548852254311605||1\n 849024737642146119||1\n 849169469997862534||1.\nThis issue was likely caused by a bug in DataFusion's code. Please help us to resolve this by filing a bug report in our issue tracker: https://github.com/apache/datafusion/issues"] + async fn test_clickbench_17() -> Result<()> { + let display = test_clickbench_query("q17").await?; + assert_snapshot!(display, @""); + Ok(()) + } + + #[tokio::test] + async fn test_clickbench_18() -> Result<()> { + let display = test_clickbench_query("q18").await?; + assert_snapshot!(display, @r#" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ ProjectionExec: expr=[UserID@0 as UserID, m@1 as m, SearchPhrase@2 as SearchPhrase, count(*)@3 as count(*)] + │ SortPreservingMergeExec: [count(Int64(1))@4 DESC], fetch=10 + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ SortExec: TopK(fetch=10), expr=[count(*)@3 DESC], preserve_partitioning=[true] + │ ProjectionExec: expr=[UserID@0 as UserID, date_part(Utf8("MINUTE"),to_timestamp_seconds(hits.EventTime))@1 as m, SearchPhrase@2 as SearchPhrase, count(Int64(1))@3 as count(*), count(Int64(1))@3 as count(Int64(1))] + │ AggregateExec: mode=FinalPartitioned, gby=[UserID@0 as UserID, date_part(Utf8("MINUTE"),to_timestamp_seconds(hits.EventTime))@1 as date_part(Utf8("MINUTE"),to_timestamp_seconds(hits.EventTime)), SearchPhrase@2 as SearchPhrase], aggr=[count(Int64(1))] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([UserID@0, date_part(Utf8("MINUTE"),to_timestamp_seconds(hits.EventTime))@1, SearchPhrase@2], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[UserID@1 as UserID, date_part(MINUTE, to_timestamp_seconds(EventTime@0)) as date_part(Utf8("MINUTE"),to_timestamp_seconds(hits.EventTime)), SearchPhrase@2 as SearchPhrase], aggr=[count(Int64(1))] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__] t1:[__,__,p0,p1,__] t2:[__,__,__,__,p0] + │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[EventTime, UserID, SearchPhrase], file_type=parquet + └────────────────────────────────────────────────── + "#); + Ok(()) + } + + #[tokio::test] + async fn test_clickbench_19() -> Result<()> { + let display = test_clickbench_query("q19").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] + │ FilterExec: UserID@0 = 435090932899640449 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__] t1:[__,__,p0,p1,__] t2:[__,__,__,__,p0] + │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[UserID], file_type=parquet, predicate=UserID@9 = 435090932899640449, pruning_predicate=UserID_null_count@2 != row_count@3 AND UserID_min@0 <= 435090932899640449 AND 435090932899640449 <= UserID_max@1, required_guarantees=[UserID in (435090932899640449)] + └────────────────────────────────────────────────── + "); + Ok(()) + } + + #[tokio::test] + async fn test_clickbench_20() -> Result<()> { + let display = test_clickbench_query("q20").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ ProjectionExec: expr=[count(Int64(1))@0 as count(*)] + │ AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] + │ AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] + │ ProjectionExec: expr=[] + │ FilterExec: CAST(URL@0 AS Utf8View) LIKE %google% + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__] t1:[__,__,p0,p1,__] t2:[__,__,__,__,p0] + │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[URL], file_type=parquet, predicate=CAST(URL@13 AS Utf8View) LIKE %google% + └────────────────────────────────────────────────── + "); + Ok(()) + } + + #[tokio::test] + #[ignore = "result sets were not equal: Internal error: Row content differs between result sets\nLeft set size: 10, Right set size: 10\n\nRows only in left (5 total):\n d181d0bbd0b0d0b2d0bbd18fd182d18c20d0bfd0bed180d0bed0b4d0b8d182d181d18f20d0bed182d0b5d0bbd0b8203230313320d181d0bcd0bed182d180d0b5d182d18c|687474703a253246253246766b2e636f6d2e75612f676f6f676c652d6a61726b6f76736b6179612d4c697065636b64|1\n d0b1d0b0d0bdd0bad0bed0bcd0b0d182d0b5d180d0b8d0b0d0bbd18b20d181d0bcd0bed182d180d0b5d182d18c|687474703a2f2f6f72656e627572672e6972722e72752532466b7572746b692532462532467777772e676f6f676c652e72752f6d617a64612d332d6b6f6d6e2d6b762d4b617a616e2e74757475746f72736b2f64657461696c|1\n d0bcd0bed0bdd0b8d182d18c20d0bad0b0d0bad0bed0b520d0bed0b7d0b5d180d0b0|687474703a2f2f6175746f2e7269612e75612f6175746f5f69643d30266f726465723d46616c7365266d696e707269782e72752f6b617465676f726979612f767369652d646c69612d647275676f652f6d61746572696e7374766f2f676f6f676c652d706f6c697331343334343532|1\n d181d0bad0b0d187d0b0d182d18c20d0b4d0b5d0bdd0b5d0b320d181d183d180d0b3d183d182|687474703a2f2f7469656e736b6169612d6d6f64612d627269657469656c6b612d6b6f736b6f76736b2f64657461696c2e676f6f676c65|1\n d0b220d0b0d0b2d0b3d183d181d1822032343720d0b3d180d183d181d182d0b8d0bcd0bed188d0bad0b020d0bdd0b020d0bad180d0b8d181d182d180d0b0d182|687474703a2f2f7469656e736b6169612d6d6f64612d627269756b692f676f6f676c652e72752f7e61706f6b2e72752f635f312d755f313138383839352c39373536|1\n\nRows only in right (5 total):\n d0bcd0bed0b4d0b5d0bad18120d183d0bbd0b8d186d0b5d0bdd0b7d0b8d0bdd0bed0b2d0b020d0b3d0bed0b2d18fd0b4d0b8d0bdd0b0|687474703a2f2f73616d6172612e6972722e72752f636174616c6f675f676f6f676c652d636865726e796a2d393233353636363635372f3f64617465|1\n d0bbd0b0d0b2d0bfd0bbd0b0d0bdd188d0b5d182d0bdd0b8d18520d183d181d0bbd0bed0b2d0b0d0bcd0b820d0b2d181d0b520d181d0b5d180d0b8d0b820d0b4d0b0d182d0b020d186d0b5d0bcd0b5d0bdd0b8|687474703a2f2f73616d6172612e6972722e72752f636174616c6f675f676f6f676c654d425225323661642533443930253236707a|1\n d0bad0b0d0ba20d0bfd180d0bed0b4d0b0d0bcd0b820d0b4d0bbd18f20d0b4d0b5d0b2d183d188d0bad0b8|687474703a253246253246777777772e626f6e707269782e7275253235326625323532663737363925323532663131303931392d6c65766f652d676f6f676c652d7368746f72792e72752f666f72756d2f666f72756d2e6d617465722e72752f6461696c792f63616c63756c61746f72|1\n d0b6d0b0d180d0b5d0bdd18cd18f20d0b32ed181d183d180d0bed0b2d0b0d0bdd0b8d0b520d0b2d0bed180d0bed0bdd0b5d0b6d181d0bad0b0d18f20d0bed0b1d0bbd0b0d181d182d0bed0bfd180d0b8d0bbd0b520d0bfd0bed181d0bbd0b5d0b4d0bdd0b8d0b520d0bad0bed181d18b|687474703a2f2f756b7261696e627572672f65636f2d6d6c656b2f65636f6e646172792f73686f77746f7069632e7068703f69643d3436333837362e68746d6c3f69643d32303634313333363631253246676f6f676c652d4170706c655765624b69742532463533372e333620284b48544d4c2c206c696b65|1\n d180d0b8d0be20d0bdd0b020d0bad0b0d180d182d0bed187d0bdd0b8d186d0b020d181d0bcd0bed182d180d0b5d182d18c20d0bed0bdd0bbd0b0d0b9d0bd|687474703a2f2f73616d6172612e6972722e72752f636174616c6f675f676f6f676c654d425225323661642533443930253236707a|1.\nThis issue was likely caused by a bug in DataFusion's code. Please help us to resolve this by filing a bug report in our issue tracker: https://github.com/apache/datafusion/issues"] + async fn test_clickbench_21() -> Result<()> { + let display = test_clickbench_query("q21").await?; + assert_snapshot!(display, @""); + Ok(()) + } + + #[tokio::test] + #[ignore = "result sets were not equal: Internal error: Row content differs between result sets\nLeft set size: 10, Right set size: 10\n\nRows only in left (1 total):\n d0bad0b0d0bad0bed0b920d0bfd0bbd0bed189d0b0d0b4d0bad0b8d0bcd0b820d0b4d0bed181d182d0b0d0b2d0bad0b8|687474703a253246253246766b2e636f6d2f696672616d652d6f77612e68746d6c3f313d31266369643d353737266f6b693d31266f705f63617465676f72795f69645d3d332673656c656374|d092d0b0d0bad0b0d0bdd181d0b8d18f20d091d0a0d090d09ad090d09d20d090d09dd094d0a0d095d0a1202d20d0bfd0bed0bfd0b0d0bbd0b820d0bad183d0bfd0b8d182d18c20d0b4d0bed0bcd0bed0b5d187d0bdd18bd0b520d188d0bad0b0d184d0b020476f6f676c652e636f6d203a3a20d0bad0bed182d182d0b5d0bad181d1822c20d091d183d180d18fd182d0bdd0b8d0bad0b820d0b4d0bbd18f20d0bfd0b5d187d18c20d0bcd0b5d0b1d0b5d0bbd18cd0b520d0b4d0bbd18f20d0b4d0b5d0b2d183d188d0bad0b0|5|1\n\nRows only in right (1 total):\n d0bad0bed0bfd182d0b8d0bcd0b8d0bad0b2d0b8d0b4d18b20d18ed180d0b8d0b920d0bfd0bed181d0bbd0b5d0b4d0bdd18fd18f|68747470733a2f2f70726f64756b747925324670756c6f76652e72752f626f6f6b6c79617474696f6e2d7761722d73696e696a2d393430343139342c3936323435332f666f746f|d09bd0b5d0b3d0bad0be20d0bdd0b020d183d187d0b0d181d182d0bdd18bd0b520d183d187d0b0d181d182d0bdd0b8d0bad0bed0b22e2c20d0a6d0b5d0bdd18b202d20d0a1d182d0b8d0bbd18cd0bdd0b0d18f20d0bfd0b0d180d0bdd0b5d0bc2e20d0a1d0b0d0b3d0b0d0bdd180d0bed0b320d0b4d0bed0b3d0b0d0b4d0b5d0bdd0b8d18f203a20d0a2d183d180d186d0b8d0b82c20d0bad183d0bfd0b8d182d18c20d18320313020d0b4d0bdd0b520d0bad0bed0bbd18cd0bdd18bd0b520d0bcd0b0d188d0b8d0bdd0bad0b820d0bdd0b520d0bfd180d0b5d0b4d181d182d0b0d0b2d0bad0b8202d20d09dd0bed0b2d0b0d18f20d18120d0b8d0b7d0b1d0b8d0b5d0bdd0b8d0b520d181d0bfd180d0bed0b4d0b0d0b6d0b03a20d0bad0bed182d18fd182d0b0203230313420d0b32ed0b22e20d0a6d0b5d0bdd0b03a2034373530302d313045434f30363020e28093202d2d2d2d2d2d2d2d20d0bad183d0bfd0b8d182d18c20d0bad0b2d0b0d180d182d0b8d180d18320d09ed180d0b5d0bdd0b1d183d180d0b32028d0a0d0bed181d181d0b8d0b82047616c616e7472617820466c616d696c6961646120476f6f676c652c204ed0be20313820d184d0bed182d0bed0bad0bed0bdd0b2d0b5d180d0ba20d0a1d183d0bfd0b5d18020d09ad0b0d180d0b4d0b8d0b3d0b0d0bd|5|1.\nThis issue was likely caused by a bug in DataFusion's code. Please help us to resolve this by filing a bug report in our issue tracker: https://github.com/apache/datafusion/issues"] + async fn test_clickbench_22() -> Result<()> { + let display = test_clickbench_query("q22").await?; + assert_snapshot!(display, @""); + Ok(()) + } + + #[tokio::test] + async fn test_clickbench_23() -> Result<()> { + let display = test_clickbench_query("q23").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [EventTime@4 ASC NULLS LAST], fetch=10 + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] + │ SortExec: TopK(fetch=10), expr=[EventTime@4 ASC NULLS LAST], preserve_partitioning=[true] + │ FilterExec: CAST(URL@13 AS Utf8View) LIKE %google% + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__] t1:[__,__,p0,p1,__] t2:[__,__,__,__,p0] + │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[WatchID, JavaEnable, Title, GoodEvent, EventTime, EventDate, CounterID, ClientIP, RegionID, UserID, CounterClass, OS, UserAgent, URL, Referer, IsRefresh, RefererCategoryID, RefererRegionID, URLCategoryID, URLRegionID, ResolutionWidth, ResolutionHeight, ResolutionDepth, FlashMajor, FlashMinor, FlashMinor2, NetMajor, NetMinor, UserAgentMajor, UserAgentMinor, CookieEnable, JavascriptEnable, IsMobile, MobilePhone, MobilePhoneModel, Params, IPNetworkID, TraficSourceID, SearchEngineID, SearchPhrase, AdvEngineID, IsArtifical, WindowClientWidth, WindowClientHeight, ClientTimeZone, ClientEventTime, SilverlightVersion1, SilverlightVersion2, SilverlightVersion3, SilverlightVersion4, PageCharset, CodeVersion, IsLink, IsDownload, IsNotBounce, FUniqID, OriginalURL, HID, IsOldCounter, IsEvent, IsParameter, DontCountHits, WithHash, HitColor, LocalEventTime, Age, Sex, Income, Interests, Robotness, RemoteIP, WindowName, OpenerName, HistoryLength, BrowserLanguage, BrowserCountry, SocialNetwork, SocialAction, HTTPError, SendTiming, DNSTiming, ConnectTiming, ResponseStartTiming, ResponseEndTiming, FetchTiming, SocialSourceNetworkID, SocialSourcePage, ParamPrice, ParamOrderID, ParamCurrency, ParamCurrencyID, OpenstatServiceName, OpenstatCampaignID, OpenstatAdID, OpenstatSourceID, UTMSource, UTMMedium, UTMCampaign, UTMContent, UTMTerm, FromTag, HasGCLID, RefererHash, URLHash, CLID], file_type=parquet, predicate=CAST(URL@13 AS Utf8View) LIKE %google% AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + "); + Ok(()) + } + + #[tokio::test] + #[ignore = "result sets were not equal: Internal error: Row content differs between result sets\nLeft set size: 10, Right set size: 10\n\nRows only in left (1 total):\n d0b2d181d0bfd0bed0bcd0bdd0b8d182d18c20d181d0bed0bbd0bdd0b5d0bdd0b8d0b520d0b1d0b0d0bdd0bad0b020d0bbd0b0d0b420d184d0b8d0bbd18cd0bc\n\nRows only in right (1 total):\n d0bed182d0b2d0bed0b4d0b020d0b4d0bbd18f20d0bfd0b8d180d0bed0b6d0bad0b820d0bbd0b5d187d0b5d0bdd0bdd18b20d0b2d181d0b520d181d0b5d180d196d197.\nThis issue was likely caused by a bug in DataFusion's code. Please help us to resolve this by filing a bug report in our issue tracker: https://github.com/apache/datafusion/issues"] + async fn test_clickbench_24() -> Result<()> { + let display = test_clickbench_query("q24").await?; + assert_snapshot!(display, @""); + Ok(()) + } + + #[tokio::test] + async fn test_clickbench_25() -> Result<()> { + let display = test_clickbench_query("q25").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [SearchPhrase@0 ASC NULLS LAST], fetch=10 + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] + │ SortExec: TopK(fetch=10), expr=[SearchPhrase@0 ASC NULLS LAST], preserve_partitioning=[true] + │ FilterExec: SearchPhrase@0 != + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__] t1:[__,__,p0,p1,__] t2:[__,__,__,__,p0] + │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[SearchPhrase], file_type=parquet, predicate=SearchPhrase@39 != AND DynamicFilter [ empty ], pruning_predicate=SearchPhrase_null_count@2 != row_count@3 AND (SearchPhrase_min@0 != OR != SearchPhrase_max@1), required_guarantees=[SearchPhrase not in ()] + └────────────────────────────────────────────────── + "); + Ok(()) + } + + #[tokio::test] + async fn test_clickbench_26() -> Result<()> { + let display = test_clickbench_query("q26").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ ProjectionExec: expr=[SearchPhrase@0 as SearchPhrase] + │ SortPreservingMergeExec: [EventTime@1 ASC NULLS LAST, SearchPhrase@0 ASC NULLS LAST], fetch=10 + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] + │ SortExec: TopK(fetch=10), expr=[EventTime@1 ASC NULLS LAST, SearchPhrase@0 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[SearchPhrase@1 as SearchPhrase, EventTime@0 as EventTime] + │ FilterExec: SearchPhrase@1 != + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__] t1:[__,__,p0,p1,__] t2:[__,__,__,__,p0] + │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[EventTime, SearchPhrase], file_type=parquet, predicate=SearchPhrase@39 != AND DynamicFilter [ empty ], pruning_predicate=SearchPhrase_null_count@2 != row_count@3 AND (SearchPhrase_min@0 != OR != SearchPhrase_max@1), required_guarantees=[SearchPhrase not in ()] + └────────────────────────────────────────────────── + "); + Ok(()) + } + + #[tokio::test] + async fn test_clickbench_27() -> Result<()> { + let display = test_clickbench_query("q27").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [l@1 DESC], fetch=25 + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ SortExec: TopK(fetch=25), expr=[l@1 DESC], preserve_partitioning=[true] + │ ProjectionExec: expr=[CounterID@0 as CounterID, avg(length(hits.URL))@1 as l, count(Int64(1))@2 as c] + │ FilterExec: count(Int64(1))@2 > 100000 + │ AggregateExec: mode=FinalPartitioned, gby=[CounterID@0 as CounterID], aggr=[avg(length(hits.URL)), count(Int64(1))] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([CounterID@0], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[CounterID@0 as CounterID], aggr=[avg(length(hits.URL)), count(Int64(1))] + │ FilterExec: URL@1 != + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__] t1:[__,__,p0,p1,__] t2:[__,__,__,__,p0] + │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[CounterID, URL], file_type=parquet, predicate=URL@13 != , pruning_predicate=URL_null_count@2 != row_count@3 AND (URL_min@0 != OR != URL_max@1), required_guarantees=[URL not in ()] + └────────────────────────────────────────────────── + "); + Ok(()) + } + + #[tokio::test] + async fn test_clickbench_28() -> Result<()> { + let display = test_clickbench_query("q28").await?; + assert_snapshot!(display, @r#" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [l@1 DESC], fetch=25 + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ SortExec: TopK(fetch=25), expr=[l@1 DESC], preserve_partitioning=[true] + │ ProjectionExec: expr=[regexp_replace(hits.Referer,Utf8("^https?://(?:www\.)?([^/]+)/.*$"),Utf8("\1"))@0 as k, avg(length(hits.Referer))@1 as l, count(Int64(1))@2 as c, min(hits.Referer)@3 as min(hits.Referer)] + │ FilterExec: count(Int64(1))@2 > 100000 + │ AggregateExec: mode=FinalPartitioned, gby=[regexp_replace(hits.Referer,Utf8("^https?://(?:www\.)?([^/]+)/.*$"),Utf8("\1"))@0 as regexp_replace(hits.Referer,Utf8("^https?://(?:www\.)?([^/]+)/.*$"),Utf8("\1"))], aggr=[avg(length(hits.Referer)), count(Int64(1)), min(hits.Referer)] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([regexp_replace(hits.Referer,Utf8("^https?://(?:www\.)?([^/]+)/.*$"),Utf8("\1"))@0], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[regexp_replace(CAST(Referer@0 AS LargeUtf8), ^https?://(?:www\.)?([^/]+)/.*$, \1) as regexp_replace(hits.Referer,Utf8("^https?://(?:www\.)?([^/]+)/.*$"),Utf8("\1"))], aggr=[avg(length(hits.Referer)), count(Int64(1)), min(hits.Referer)] + │ FilterExec: Referer@0 != + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__] t1:[__,__,p0,p1,__] t2:[__,__,__,__,p0] + │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[Referer], file_type=parquet, predicate=Referer@14 != , pruning_predicate=Referer_null_count@2 != row_count@3 AND (Referer_min@0 != OR != Referer_max@1), required_guarantees=[Referer not in ()] + └────────────────────────────────────────────────── + "#); + Ok(()) + } + + #[tokio::test] + async fn test_clickbench_29() -> Result<()> { + let display = test_clickbench_query("q29").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ AggregateExec: mode=Final, gby=[], aggr=[sum(hits.ResolutionWidth), sum(hits.ResolutionWidth + Int64(1)), sum(hits.ResolutionWidth + Int64(2)), sum(hits.ResolutionWidth + Int64(3)), sum(hits.ResolutionWidth + Int64(4)), sum(hits.ResolutionWidth + Int64(5)), sum(hits.ResolutionWidth + Int64(6)), sum(hits.ResolutionWidth + Int64(7)), sum(hits.ResolutionWidth + Int64(8)), sum(hits.ResolutionWidth + Int64(9)), sum(hits.ResolutionWidth + Int64(10)), sum(hits.ResolutionWidth + Int64(11)), sum(hits.ResolutionWidth + Int64(12)), sum(hits.ResolutionWidth + Int64(13)), sum(hits.ResolutionWidth + Int64(14)), sum(hits.ResolutionWidth + Int64(15)), sum(hits.ResolutionWidth + Int64(16)), sum(hits.ResolutionWidth + Int64(17)), sum(hits.ResolutionWidth + Int64(18)), sum(hits.ResolutionWidth + Int64(19)), sum(hits.ResolutionWidth + Int64(20)), sum(hits.ResolutionWidth + Int64(21)), sum(hits.ResolutionWidth + Int64(22)), sum(hits.ResolutionWidth + Int64(23)), sum(hits.ResolutionWidth + Int64(24)), sum(hits.ResolutionWidth + Int64(25)), sum(hits.ResolutionWidth + Int64(26)), sum(hits.ResolutionWidth + Int64(27)), sum(hits.ResolutionWidth + Int64(28)), sum(hits.ResolutionWidth + Int64(29)), sum(hits.ResolutionWidth + Int64(30)), sum(hits.ResolutionWidth + Int64(31)), sum(hits.ResolutionWidth + Int64(32)), sum(hits.ResolutionWidth + Int64(33)), sum(hits.ResolutionWidth + Int64(34)), sum(hits.ResolutionWidth + Int64(35)), sum(hits.ResolutionWidth + Int64(36)), sum(hits.ResolutionWidth + Int64(37)), sum(hits.ResolutionWidth + Int64(38)), sum(hits.ResolutionWidth + Int64(39)), sum(hits.ResolutionWidth + Int64(40)), sum(hits.ResolutionWidth + Int64(41)), sum(hits.ResolutionWidth + Int64(42)), sum(hits.ResolutionWidth + Int64(43)), sum(hits.ResolutionWidth + Int64(44)), sum(hits.ResolutionWidth + Int64(45)), sum(hits.ResolutionWidth + Int64(46)), sum(hits.ResolutionWidth + Int64(47)), sum(hits.ResolutionWidth + Int64(48)), sum(hits.ResolutionWidth + Int64(49)), sum(hits.ResolutionWidth + Int64(50)), sum(hits.ResolutionWidth + Int64(51)), sum(hits.ResolutionWidth + Int64(52)), sum(hits.ResolutionWidth + Int64(53)), sum(hits.ResolutionWidth + Int64(54)), sum(hits.ResolutionWidth + Int64(55)), sum(hits.ResolutionWidth + Int64(56)), sum(hits.ResolutionWidth + Int64(57)), sum(hits.ResolutionWidth + Int64(58)), sum(hits.ResolutionWidth + Int64(59)), sum(hits.ResolutionWidth + Int64(60)), sum(hits.ResolutionWidth + Int64(61)), sum(hits.ResolutionWidth + Int64(62)), sum(hits.ResolutionWidth + Int64(63)), sum(hits.ResolutionWidth + Int64(64)), sum(hits.ResolutionWidth + Int64(65)), sum(hits.ResolutionWidth + Int64(66)), sum(hits.ResolutionWidth + Int64(67)), sum(hits.ResolutionWidth + Int64(68)), sum(hits.ResolutionWidth + Int64(69)), sum(hits.ResolutionWidth + Int64(70)), sum(hits.ResolutionWidth + Int64(71)), sum(hits.ResolutionWidth + Int64(72)), sum(hits.ResolutionWidth + Int64(73)), sum(hits.ResolutionWidth + Int64(74)), sum(hits.ResolutionWidth + Int64(75)), sum(hits.ResolutionWidth + Int64(76)), sum(hits.ResolutionWidth + Int64(77)), sum(hits.ResolutionWidth + Int64(78)), sum(hits.ResolutionWidth + Int64(79)), sum(hits.ResolutionWidth + Int64(80)), sum(hits.ResolutionWidth + Int64(81)), sum(hits.ResolutionWidth + Int64(82)), sum(hits.ResolutionWidth + Int64(83)), sum(hits.ResolutionWidth + Int64(84)), sum(hits.ResolutionWidth + Int64(85)), sum(hits.ResolutionWidth + Int64(86)), sum(hits.ResolutionWidth + Int64(87)), sum(hits.ResolutionWidth + Int64(88)), sum(hits.ResolutionWidth + Int64(89))] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] + │ AggregateExec: mode=Partial, gby=[], aggr=[sum(hits.ResolutionWidth), sum(hits.ResolutionWidth + Int64(1)), sum(hits.ResolutionWidth + Int64(2)), sum(hits.ResolutionWidth + Int64(3)), sum(hits.ResolutionWidth + Int64(4)), sum(hits.ResolutionWidth + Int64(5)), sum(hits.ResolutionWidth + Int64(6)), sum(hits.ResolutionWidth + Int64(7)), sum(hits.ResolutionWidth + Int64(8)), sum(hits.ResolutionWidth + Int64(9)), sum(hits.ResolutionWidth + Int64(10)), sum(hits.ResolutionWidth + Int64(11)), sum(hits.ResolutionWidth + Int64(12)), sum(hits.ResolutionWidth + Int64(13)), sum(hits.ResolutionWidth + Int64(14)), sum(hits.ResolutionWidth + Int64(15)), sum(hits.ResolutionWidth + Int64(16)), sum(hits.ResolutionWidth + Int64(17)), sum(hits.ResolutionWidth + Int64(18)), sum(hits.ResolutionWidth + Int64(19)), sum(hits.ResolutionWidth + Int64(20)), sum(hits.ResolutionWidth + Int64(21)), sum(hits.ResolutionWidth + Int64(22)), sum(hits.ResolutionWidth + Int64(23)), sum(hits.ResolutionWidth + Int64(24)), sum(hits.ResolutionWidth + Int64(25)), sum(hits.ResolutionWidth + Int64(26)), sum(hits.ResolutionWidth + Int64(27)), sum(hits.ResolutionWidth + Int64(28)), sum(hits.ResolutionWidth + Int64(29)), sum(hits.ResolutionWidth + Int64(30)), sum(hits.ResolutionWidth + Int64(31)), sum(hits.ResolutionWidth + Int64(32)), sum(hits.ResolutionWidth + Int64(33)), sum(hits.ResolutionWidth + Int64(34)), sum(hits.ResolutionWidth + Int64(35)), sum(hits.ResolutionWidth + Int64(36)), sum(hits.ResolutionWidth + Int64(37)), sum(hits.ResolutionWidth + Int64(38)), sum(hits.ResolutionWidth + Int64(39)), sum(hits.ResolutionWidth + Int64(40)), sum(hits.ResolutionWidth + Int64(41)), sum(hits.ResolutionWidth + Int64(42)), sum(hits.ResolutionWidth + Int64(43)), sum(hits.ResolutionWidth + Int64(44)), sum(hits.ResolutionWidth + Int64(45)), sum(hits.ResolutionWidth + Int64(46)), sum(hits.ResolutionWidth + Int64(47)), sum(hits.ResolutionWidth + Int64(48)), sum(hits.ResolutionWidth + Int64(49)), sum(hits.ResolutionWidth + Int64(50)), sum(hits.ResolutionWidth + Int64(51)), sum(hits.ResolutionWidth + Int64(52)), sum(hits.ResolutionWidth + Int64(53)), sum(hits.ResolutionWidth + Int64(54)), sum(hits.ResolutionWidth + Int64(55)), sum(hits.ResolutionWidth + Int64(56)), sum(hits.ResolutionWidth + Int64(57)), sum(hits.ResolutionWidth + Int64(58)), sum(hits.ResolutionWidth + Int64(59)), sum(hits.ResolutionWidth + Int64(60)), sum(hits.ResolutionWidth + Int64(61)), sum(hits.ResolutionWidth + Int64(62)), sum(hits.ResolutionWidth + Int64(63)), sum(hits.ResolutionWidth + Int64(64)), sum(hits.ResolutionWidth + Int64(65)), sum(hits.ResolutionWidth + Int64(66)), sum(hits.ResolutionWidth + Int64(67)), sum(hits.ResolutionWidth + Int64(68)), sum(hits.ResolutionWidth + Int64(69)), sum(hits.ResolutionWidth + Int64(70)), sum(hits.ResolutionWidth + Int64(71)), sum(hits.ResolutionWidth + Int64(72)), sum(hits.ResolutionWidth + Int64(73)), sum(hits.ResolutionWidth + Int64(74)), sum(hits.ResolutionWidth + Int64(75)), sum(hits.ResolutionWidth + Int64(76)), sum(hits.ResolutionWidth + Int64(77)), sum(hits.ResolutionWidth + Int64(78)), sum(hits.ResolutionWidth + Int64(79)), sum(hits.ResolutionWidth + Int64(80)), sum(hits.ResolutionWidth + Int64(81)), sum(hits.ResolutionWidth + Int64(82)), sum(hits.ResolutionWidth + Int64(83)), sum(hits.ResolutionWidth + Int64(84)), sum(hits.ResolutionWidth + Int64(85)), sum(hits.ResolutionWidth + Int64(86)), sum(hits.ResolutionWidth + Int64(87)), sum(hits.ResolutionWidth + Int64(88)), sum(hits.ResolutionWidth + Int64(89))] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__] t1:[__,__,p0,p1,__] t2:[__,__,__,__,p0] + │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[CAST(ResolutionWidth@20 AS Int64) as __common_expr_1], file_type=parquet + └────────────────────────────────────────────────── + "); + Ok(()) + } + + #[tokio::test] + async fn test_clickbench_30() -> Result<()> { + let display = test_clickbench_query("q30").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [c@2 DESC], fetch=10 + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ SortExec: TopK(fetch=10), expr=[c@2 DESC], preserve_partitioning=[true] + │ ProjectionExec: expr=[SearchEngineID@0 as SearchEngineID, ClientIP@1 as ClientIP, count(Int64(1))@2 as c, sum(hits.IsRefresh)@3 as sum(hits.IsRefresh), avg(hits.ResolutionWidth)@4 as avg(hits.ResolutionWidth)] + │ AggregateExec: mode=FinalPartitioned, gby=[SearchEngineID@0 as SearchEngineID, ClientIP@1 as ClientIP], aggr=[count(Int64(1)), sum(hits.IsRefresh), avg(hits.ResolutionWidth)] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([SearchEngineID@0, ClientIP@1], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[SearchEngineID@3 as SearchEngineID, ClientIP@0 as ClientIP], aggr=[count(Int64(1)), sum(hits.IsRefresh), avg(hits.ResolutionWidth)] + │ FilterExec: SearchPhrase@4 != , projection=[ClientIP@0, IsRefresh@1, ResolutionWidth@2, SearchEngineID@3] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__] t1:[__,__,p0,p1,__] t2:[__,__,__,__,p0] + │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[ClientIP, IsRefresh, ResolutionWidth, SearchEngineID, SearchPhrase], file_type=parquet, predicate=SearchPhrase@39 != , pruning_predicate=SearchPhrase_null_count@2 != row_count@3 AND (SearchPhrase_min@0 != OR != SearchPhrase_max@1), required_guarantees=[SearchPhrase not in ()] + └────────────────────────────────────────────────── + "); + Ok(()) + } + + #[tokio::test] + #[ignore = "result sets were not equal: Internal error: Row content differs between result sets\nLeft set size: 10, Right set size: 10\n\nRows only in left (10 total):\n 8673025726158767406|1264438551|1|0|1990.00\n 5320052218057629211|-1703087277|1|0|1996.00\n 6244273852606083750|1554672832|1|0|1638.00\n 8628753750962053665|1215278356|1|0|1087.00\n 7035318163404387241|1326714320|1|0|1638.00\n 8431857775494210873|1237512945|1|0|1996.00\n 5110752526539992124|37611695|1|0|1917.00\n 8986794334343068049|1860752926|1|0|1638.00\n 8044147848299485837|1382122372|1|0|1368.00\n 7936057634954670727|1897481896|1|0|1638.00\n\nRows only in right (10 total):\n 5132615111782210132|-50313020|1|0|1368.00\n 5783789691451717551|-1310327384|1|0|1638.00\n 5756260993772351383|1484317883|1|0|375.00\n 7739310142000732364|991864113|1|0|1368.00\n 7593472904893539271|-151291403|1|0|1087.00\n 6339599967989898410|1543815587|1|0|1638.00\n 7794346560421945218|1645556180|1|0|1368.00\n 6112645108657361792|593586188|1|0|1638.00\n 6675910710751922756|-816379256|1|0|1368.00\n 5802727636196431835|1986422271|1|0|1996.00.\nThis issue was likely caused by a bug in DataFusion's code. Please help us to resolve this by filing a bug report in our issue tracker: https://github.com/apache/datafusion/issues"] + async fn test_clickbench_31() -> Result<()> { + let display = test_clickbench_query("q31").await?; + assert_snapshot!(display, @""); + Ok(()) + } + + #[tokio::test] + #[ignore = "result sets were not equal: Internal error: Row content differs between result sets\nLeft set size: 10, Right set size: 10\n\nRows only in left (10 total):\n 7643059318918524417|1767085700|1|0|0.00\n 5437163248266133938|-1465369615|1|0|0.00\n 9142541582422390102|-1465369615|1|0|0.00\n 8438994503411842126|-1465369615|1|0|0.00\n 7362096505818029859|-565678477|1|0|0.00\n 4928022308880516715|1699955284|1|0|0.00\n 5269769817689282522|1699955284|1|0|0.00\n 9081648050908046886|1699955284|1|0|0.00\n 6824181869275536503|1699955284|1|0|0.00\n 6905712404475757487|1552811156|1|0|0.00\n\nRows only in right (10 total):\n 6967277596165459879|-941091661|1|0|1368.00\n 5796237228224217668|-1310327384|1|0|1638.00\n 7218628137278606666|1511490240|1|0|1638.00\n 8314760197723815280|1566105210|1|0|1996.00\n 7053263954762394007|757778490|1|0|339.00\n 6283334114093174531|1216031795|1|0|1368.00\n 8818295356247036741|83042182|1|0|1638.00\n 6620528864937282562|-862894777|1|0|1996.00\n 8466121050002905379|83042182|1|0|1638.00\n 7554844936512227411|-1746904856|1|0|1368.00.\nThis issue was likely caused by a bug in DataFusion's code. Please help us to resolve this by filing a bug report in our issue tracker: https://github.com/apache/datafusion/issues"] + async fn test_clickbench_32() -> Result<()> { + let display = test_clickbench_query("q32").await?; + assert_snapshot!(display, @""); + Ok(()) + } + + #[tokio::test] + async fn test_clickbench_33() -> Result<()> { + let display = test_clickbench_query("q33").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [c@1 DESC], fetch=10 + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ SortExec: TopK(fetch=10), expr=[c@1 DESC], preserve_partitioning=[true] + │ ProjectionExec: expr=[URL@0 as URL, count(Int64(1))@1 as c] + │ AggregateExec: mode=FinalPartitioned, gby=[URL@0 as URL], aggr=[count(Int64(1))] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([URL@0], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[URL@0 as URL], aggr=[count(Int64(1))] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__] t1:[__,__,p0,p1,__] t2:[__,__,__,__,p0] + │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[URL], file_type=parquet + └────────────────────────────────────────────────── + "); + Ok(()) + } + + #[tokio::test] + async fn test_clickbench_34() -> Result<()> { + let display = test_clickbench_query("q34").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [c@2 DESC], fetch=10 + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ SortExec: TopK(fetch=10), expr=[c@2 DESC], preserve_partitioning=[true] + │ ProjectionExec: expr=[1 as Int64(1), URL@0 as URL, count(Int64(1))@1 as c] + │ AggregateExec: mode=FinalPartitioned, gby=[URL@0 as URL], aggr=[count(Int64(1))] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([URL@0], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[URL@0 as URL], aggr=[count(Int64(1))] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__] t1:[__,__,p0,p1,__] t2:[__,__,__,__,p0] + │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[URL], file_type=parquet + └────────────────────────────────────────────────── + "); + Ok(()) + } + + #[tokio::test] + async fn test_clickbench_35() -> Result<()> { + let display = test_clickbench_query("q35").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [c@4 DESC], fetch=10 + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ SortExec: TopK(fetch=10), expr=[c@4 DESC], preserve_partitioning=[true] + │ ProjectionExec: expr=[ClientIP@0 as ClientIP, hits.ClientIP - Int64(1)@1 as hits.ClientIP - Int64(1), hits.ClientIP - Int64(2)@2 as hits.ClientIP - Int64(2), hits.ClientIP - Int64(3)@3 as hits.ClientIP - Int64(3), count(Int64(1))@4 as c] + │ AggregateExec: mode=FinalPartitioned, gby=[ClientIP@0 as ClientIP, hits.ClientIP - Int64(1)@1 as hits.ClientIP - Int64(1), hits.ClientIP - Int64(2)@2 as hits.ClientIP - Int64(2), hits.ClientIP - Int64(3)@3 as hits.ClientIP - Int64(3)], aggr=[count(Int64(1))] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([ClientIP@0, hits.ClientIP - Int64(1)@1, hits.ClientIP - Int64(2)@2, hits.ClientIP - Int64(3)@3], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[ClientIP@1 as ClientIP, __common_expr_1@0 - 1 as hits.ClientIP - Int64(1), __common_expr_1@0 - 2 as hits.ClientIP - Int64(2), __common_expr_1@0 - 3 as hits.ClientIP - Int64(3)], aggr=[count(Int64(1))] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__] t1:[__,__,p0,p1,__] t2:[__,__,__,__,p0] + │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[CAST(ClientIP@7 AS Int64) as __common_expr_1, ClientIP], file_type=parquet + └────────────────────────────────────────────────── + "); + Ok(()) + } + + #[tokio::test] + async fn test_clickbench_36() -> Result<()> { + let display = test_clickbench_query("q36").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [pageviews@1 DESC], fetch=10 + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ SortExec: TopK(fetch=10), expr=[pageviews@1 DESC], preserve_partitioning=[true] + │ ProjectionExec: expr=[URL@0 as URL, count(Int64(1))@1 as pageviews] + │ AggregateExec: mode=FinalPartitioned, gby=[URL@0 as URL], aggr=[count(Int64(1))] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([URL@0], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[URL@0 as URL], aggr=[count(Int64(1))] + │ FilterExec: CounterID@1 = 62 AND CAST(EventDate@0 AS Utf8) >= 2013-07-01 AND CAST(EventDate@0 AS Utf8) <= 2013-07-31 AND DontCountHits@4 = 0 AND IsRefresh@3 = 0 AND URL@2 != , projection=[URL@2] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__] t1:[__,__,p0,p1,__] t2:[__,__,__,__,p0] + │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[EventDate, CounterID, URL, IsRefresh, DontCountHits], file_type=parquet, predicate=CounterID@6 = 62 AND CAST(EventDate@5 AS Utf8) >= 2013-07-01 AND CAST(EventDate@5 AS Utf8) <= 2013-07-31 AND DontCountHits@61 = 0 AND IsRefresh@15 = 0 AND URL@13 != , pruning_predicate=CounterID_null_count@2 != row_count@3 AND CounterID_min@0 <= 62 AND 62 <= CounterID_max@1 AND DontCountHits_null_count@6 != row_count@3 AND DontCountHits_min@4 <= 0 AND 0 <= DontCountHits_max@5 AND IsRefresh_null_count@9 != row_count@3 AND IsRefresh_min@7 <= 0 AND 0 <= IsRefresh_max@8 AND URL_null_count@12 != row_count@3 AND (URL_min@10 != OR != URL_max@11), required_guarantees=[CounterID in (62), DontCountHits in (0), IsRefresh in (0), URL not in ()] + └────────────────────────────────────────────────── + "); + Ok(()) + } + + #[tokio::test] + async fn test_clickbench_37() -> Result<()> { + let display = test_clickbench_query("q37").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [pageviews@1 DESC], fetch=10 + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ SortExec: TopK(fetch=10), expr=[pageviews@1 DESC], preserve_partitioning=[true] + │ ProjectionExec: expr=[Title@0 as Title, count(Int64(1))@1 as pageviews] + │ AggregateExec: mode=FinalPartitioned, gby=[Title@0 as Title], aggr=[count(Int64(1))] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([Title@0], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[Title@0 as Title], aggr=[count(Int64(1))] + │ FilterExec: CounterID@2 = 62 AND CAST(EventDate@1 AS Utf8) >= 2013-07-01 AND CAST(EventDate@1 AS Utf8) <= 2013-07-31 AND DontCountHits@4 = 0 AND IsRefresh@3 = 0 AND Title@0 != , projection=[Title@0] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__] t1:[__,__,p0,p1,__] t2:[__,__,__,__,p0] + │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[Title, EventDate, CounterID, IsRefresh, DontCountHits], file_type=parquet, predicate=CounterID@6 = 62 AND CAST(EventDate@5 AS Utf8) >= 2013-07-01 AND CAST(EventDate@5 AS Utf8) <= 2013-07-31 AND DontCountHits@61 = 0 AND IsRefresh@15 = 0 AND Title@2 != , pruning_predicate=CounterID_null_count@2 != row_count@3 AND CounterID_min@0 <= 62 AND 62 <= CounterID_max@1 AND DontCountHits_null_count@6 != row_count@3 AND DontCountHits_min@4 <= 0 AND 0 <= DontCountHits_max@5 AND IsRefresh_null_count@9 != row_count@3 AND IsRefresh_min@7 <= 0 AND 0 <= IsRefresh_max@8 AND Title_null_count@12 != row_count@3 AND (Title_min@10 != OR != Title_max@11), required_guarantees=[CounterID in (62), DontCountHits in (0), IsRefresh in (0), Title not in ()] + └────────────────────────────────────────────────── + "); + Ok(()) + } + + #[tokio::test] + async fn test_clickbench_38() -> Result<()> { + let display = test_clickbench_query("q38").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ GlobalLimitExec: skip=1000, fetch=10 + │ SortPreservingMergeExec: [pageviews@1 DESC], fetch=1010 + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ SortExec: TopK(fetch=1010), expr=[pageviews@1 DESC], preserve_partitioning=[true] + │ ProjectionExec: expr=[URL@0 as URL, count(Int64(1))@1 as pageviews] + │ AggregateExec: mode=FinalPartitioned, gby=[URL@0 as URL], aggr=[count(Int64(1))] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([URL@0], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[URL@0 as URL], aggr=[count(Int64(1))] + │ FilterExec: CounterID@1 = 62 AND CAST(EventDate@0 AS Utf8) >= 2013-07-01 AND CAST(EventDate@0 AS Utf8) <= 2013-07-31 AND IsRefresh@3 = 0 AND IsLink@4 != 0 AND IsDownload@5 = 0, projection=[URL@2] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__] t1:[__,__,p0,p1,__] t2:[__,__,__,__,p0] + │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[EventDate, CounterID, URL, IsRefresh, IsLink, IsDownload], file_type=parquet, predicate=CounterID@6 = 62 AND CAST(EventDate@5 AS Utf8) >= 2013-07-01 AND CAST(EventDate@5 AS Utf8) <= 2013-07-31 AND IsRefresh@15 = 0 AND IsLink@52 != 0 AND IsDownload@53 = 0, pruning_predicate=CounterID_null_count@2 != row_count@3 AND CounterID_min@0 <= 62 AND 62 <= CounterID_max@1 AND IsRefresh_null_count@6 != row_count@3 AND IsRefresh_min@4 <= 0 AND 0 <= IsRefresh_max@5 AND IsLink_null_count@9 != row_count@3 AND (IsLink_min@7 != 0 OR 0 != IsLink_max@8) AND IsDownload_null_count@12 != row_count@3 AND IsDownload_min@10 <= 0 AND 0 <= IsDownload_max@11, required_guarantees=[CounterID in (62), IsDownload in (0), IsLink not in (0), IsRefresh in (0)] + └────────────────────────────────────────────────── + "); + Ok(()) + } + + #[tokio::test] + async fn test_clickbench_39() -> Result<()> { + let display = test_clickbench_query("q39").await?; + assert_snapshot!(display, @r#" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ GlobalLimitExec: skip=1000, fetch=10 + │ SortPreservingMergeExec: [pageviews@5 DESC], fetch=1010 + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ SortExec: TopK(fetch=1010), expr=[pageviews@5 DESC], preserve_partitioning=[true] + │ ProjectionExec: expr=[TraficSourceID@0 as TraficSourceID, SearchEngineID@1 as SearchEngineID, AdvEngineID@2 as AdvEngineID, CASE WHEN hits.SearchEngineID = Int64(0) AND hits.AdvEngineID = Int64(0) THEN hits.Referer ELSE Utf8("") END@3 as src, URL@4 as dst, count(Int64(1))@5 as pageviews] + │ AggregateExec: mode=FinalPartitioned, gby=[TraficSourceID@0 as TraficSourceID, SearchEngineID@1 as SearchEngineID, AdvEngineID@2 as AdvEngineID, CASE WHEN hits.SearchEngineID = Int64(0) AND hits.AdvEngineID = Int64(0) THEN hits.Referer ELSE Utf8("") END@3 as CASE WHEN hits.SearchEngineID = Int64(0) AND hits.AdvEngineID = Int64(0) THEN hits.Referer ELSE Utf8("") END, URL@4 as URL], aggr=[count(Int64(1))] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([TraficSourceID@0, SearchEngineID@1, AdvEngineID@2, CASE WHEN hits.SearchEngineID = Int64(0) AND hits.AdvEngineID = Int64(0) THEN hits.Referer ELSE Utf8("") END@3, URL@4], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[TraficSourceID@2 as TraficSourceID, SearchEngineID@3 as SearchEngineID, AdvEngineID@4 as AdvEngineID, CASE WHEN SearchEngineID@3 = 0 AND AdvEngineID@4 = 0 THEN Referer@1 ELSE END as CASE WHEN hits.SearchEngineID = Int64(0) AND hits.AdvEngineID = Int64(0) THEN hits.Referer ELSE Utf8("") END, URL@0 as URL], aggr=[count(Int64(1))] + │ FilterExec: CounterID@1 = 62 AND CAST(EventDate@0 AS Utf8) >= 2013-07-01 AND CAST(EventDate@0 AS Utf8) <= 2013-07-31 AND IsRefresh@4 = 0, projection=[URL@2, Referer@3, TraficSourceID@5, SearchEngineID@6, AdvEngineID@7] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__] t1:[__,__,p0,p1,__] t2:[__,__,__,__,p0] + │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[EventDate, CounterID, URL, Referer, IsRefresh, TraficSourceID, SearchEngineID, AdvEngineID], file_type=parquet, predicate=CounterID@6 = 62 AND CAST(EventDate@5 AS Utf8) >= 2013-07-01 AND CAST(EventDate@5 AS Utf8) <= 2013-07-31 AND IsRefresh@15 = 0, pruning_predicate=CounterID_null_count@2 != row_count@3 AND CounterID_min@0 <= 62 AND 62 <= CounterID_max@1 AND IsRefresh_null_count@6 != row_count@3 AND IsRefresh_min@4 <= 0 AND 0 <= IsRefresh_max@5, required_guarantees=[CounterID in (62), IsRefresh in (0)] + └────────────────────────────────────────────────── + "#); + Ok(()) + } + + #[tokio::test] + async fn test_clickbench_40() -> Result<()> { + let display = test_clickbench_query("q40").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ GlobalLimitExec: skip=100, fetch=10 + │ SortPreservingMergeExec: [pageviews@2 DESC], fetch=110 + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ SortExec: TopK(fetch=110), expr=[pageviews@2 DESC], preserve_partitioning=[true] + │ ProjectionExec: expr=[URLHash@0 as URLHash, EventDate@1 as EventDate, count(Int64(1))@2 as pageviews] + │ AggregateExec: mode=FinalPartitioned, gby=[URLHash@0 as URLHash, EventDate@1 as EventDate], aggr=[count(Int64(1))] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([URLHash@0, EventDate@1], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[URLHash@1 as URLHash, EventDate@0 as EventDate], aggr=[count(Int64(1))] + │ FilterExec: CounterID@1 = 62 AND CAST(EventDate@0 AS Utf8) >= 2013-07-01 AND CAST(EventDate@0 AS Utf8) <= 2013-07-31 AND IsRefresh@2 = 0 AND (TraficSourceID@3 = -1 OR TraficSourceID@3 = 6) AND RefererHash@4 = 3594120000172545465, projection=[EventDate@0, URLHash@5] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__] t1:[__,__,p0,p1,__] t2:[__,__,__,__,p0] + │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[EventDate, CounterID, IsRefresh, TraficSourceID, RefererHash, URLHash], file_type=parquet, predicate=CounterID@6 = 62 AND CAST(EventDate@5 AS Utf8) >= 2013-07-01 AND CAST(EventDate@5 AS Utf8) <= 2013-07-31 AND IsRefresh@15 = 0 AND (TraficSourceID@37 = -1 OR TraficSourceID@37 = 6) AND RefererHash@102 = 3594120000172545465, pruning_predicate=CounterID_null_count@2 != row_count@3 AND CounterID_min@0 <= 62 AND 62 <= CounterID_max@1 AND IsRefresh_null_count@6 != row_count@3 AND IsRefresh_min@4 <= 0 AND 0 <= IsRefresh_max@5 AND (TraficSourceID_null_count@9 != row_count@3 AND TraficSourceID_min@7 <= -1 AND -1 <= TraficSourceID_max@8 OR TraficSourceID_null_count@9 != row_count@3 AND TraficSourceID_min@7 <= 6 AND 6 <= TraficSourceID_max@8) AND RefererHash_null_count@12 != row_count@3 AND RefererHash_min@10 <= 3594120000172545465 AND 3594120000172545465 <= RefererHash_max@11, required_guarantees=[CounterID in (62), IsRefresh in (0), RefererHash in (3594120000172545465), TraficSourceID in (-1, 6)] + └────────────────────────────────────────────────── + "); + Ok(()) + } + + #[tokio::test] + async fn test_clickbench_41() -> Result<()> { + let display = test_clickbench_query("q41").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ GlobalLimitExec: skip=10000, fetch=10 + │ SortPreservingMergeExec: [pageviews@2 DESC], fetch=10010 + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ SortExec: TopK(fetch=10010), expr=[pageviews@2 DESC], preserve_partitioning=[true] + │ ProjectionExec: expr=[WindowClientWidth@0 as WindowClientWidth, WindowClientHeight@1 as WindowClientHeight, count(Int64(1))@2 as pageviews] + │ AggregateExec: mode=FinalPartitioned, gby=[WindowClientWidth@0 as WindowClientWidth, WindowClientHeight@1 as WindowClientHeight], aggr=[count(Int64(1))] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([WindowClientWidth@0, WindowClientHeight@1], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[WindowClientWidth@0 as WindowClientWidth, WindowClientHeight@1 as WindowClientHeight], aggr=[count(Int64(1))] + │ FilterExec: CounterID@1 = 62 AND CAST(EventDate@0 AS Utf8) >= 2013-07-01 AND CAST(EventDate@0 AS Utf8) <= 2013-07-31 AND IsRefresh@2 = 0 AND DontCountHits@5 = 0 AND URLHash@6 = 2868770270353813622, projection=[WindowClientWidth@3, WindowClientHeight@4] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__] t1:[__,__,p0,p1,__] t2:[__,__,__,__,p0] + │ DataSourceExec: file_groups={5 groups: [[/testdata/clickbench/plans_range0-3/hits/0.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/1.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..], [/testdata/clickbench/plans_range0-3/hits/2.parquet:..]]}, projection=[EventDate, CounterID, IsRefresh, WindowClientWidth, WindowClientHeight, DontCountHits, URLHash], file_type=parquet, predicate=CounterID@6 = 62 AND CAST(EventDate@5 AS Utf8) >= 2013-07-01 AND CAST(EventDate@5 AS Utf8) <= 2013-07-31 AND IsRefresh@15 = 0 AND DontCountHits@61 = 0 AND URLHash@103 = 2868770270353813622, pruning_predicate=CounterID_null_count@2 != row_count@3 AND CounterID_min@0 <= 62 AND 62 <= CounterID_max@1 AND IsRefresh_null_count@6 != row_count@3 AND IsRefresh_min@4 <= 0 AND 0 <= IsRefresh_max@5 AND DontCountHits_null_count@9 != row_count@3 AND DontCountHits_min@7 <= 0 AND 0 <= DontCountHits_max@8 AND URLHash_null_count@12 != row_count@3 AND URLHash_min@10 <= 2868770270353813622 AND 2868770270353813622 <= URLHash_max@11, required_guarantees=[CounterID in (62), DontCountHits in (0), IsRefresh in (0), URLHash in (2868770270353813622)] + └────────────────────────────────────────────────── + "); + Ok(()) + } + + #[tokio::test] + #[ignore = "ordering mismatch: expected ordering: Some(LexOrdering { exprs: [PhysicalSortExpr { expr: ScalarFunctionExpr { fun: \"\", name: \"date_trunc\", args: [Literal { value: Utf8(\"minute\"), field: Field { name: \"lit\", data_type: Utf8 } }, Column { name: \"m\", index: 0 }], return_field: Field { name: \"date_trunc\", data_type: Timestamp(Second, None), nullable: true } }, options: SortOptions { descending: false, nulls_first: false } }], set: {ScalarFunctionExpr { fun: \"\", name: \"date_trunc\", args: [Literal { value: Utf8(\"minute\"), field: Field { name: \"lit\", data_type: Utf8 } }, Column { name: \"m\", index: 0 }], return_field: Field { name: \"date_trunc\", data_type: Timestamp(Second, None), nullable: true } }} }), actual ordering: Some(LexOrdering { exprs: [PhysicalSortExpr { expr: ScalarFunctionExpr { fun: \"\", name: \"date_trunc\", args: [Literal { value: Utf8(\"minute\"), field: Field { name: \"lit\", data_type: Utf8 } }, Column { name: \"m\", index: 0 }], return_field: Field { name: \"date_trunc\", data_type: Timestamp(Second, None), nullable: true } }, options: SortOptions { descending: false, nulls_first: false } }], set: {ScalarFunctionExpr { fun: \"\", name: \"date_trunc\", args: [Literal { value: Utf8(\"minute\"), field: Field { name: \"lit\", data_type: Utf8 } }, Column { name: \"m\", index: 0 }], return_field: Field { name: \"date_trunc\", data_type: Timestamp(Second, None), nullable: true } }} })"] + async fn test_clickbench_42() -> Result<()> { + let display = test_clickbench_query("q42").await?; + assert_snapshot!(display, @""); + Ok(()) + } + + static INIT_TEST_TPCDS_TABLES: OnceCell<()> = OnceCell::const_new(); + + async fn test_clickbench_query(query_id: &str) -> Result { + let data_dir = Path::new(env!("CARGO_MANIFEST_DIR")).join(format!( + "testdata/clickbench/plans_range{}-{}", + FILE_RANGE.start, FILE_RANGE.end + )); + INIT_TEST_TPCDS_TABLES + .get_or_init(|| async { + clickbench::generate_clickbench_data(&data_dir, FILE_RANGE) + .await + .unwrap(); + }) + .await; + + let query_sql = clickbench::get_query(query_id)?; + + let (d_ctx, _guard, _) = start_localhost_context(NUM_WORKERS, DefaultSessionBuilder).await; + let d_ctx = d_ctx + .with_distributed_files_per_task(FILES_PER_TASK)? + .with_distributed_cardinality_effect_task_scale_factor(CARDINALITY_TASK_COUNT_FACTOR)? + .with_distributed_broadcast_joins(true)?; + + benchmarks_common::register_tables(&d_ctx, &data_dir).await?; + + let df = d_ctx.sql(&query_sql).await?; + let plan = df.create_physical_plan().await?; + + if !plan.as_any().is::() { + Ok("".to_string()) + } else { + Ok(display_plan_ascii(plan.as_ref(), false)) + } + } +} diff --git a/tests/custom_config_extension.rs b/tests/custom_config_extension.rs index f00c43e8..cf493692 100644 --- a/tests/custom_config_extension.rs +++ b/tests/custom_config_extension.rs @@ -1,22 +1,19 @@ #[cfg(all(feature = "integration", test))] mod tests { - use datafusion::arrow::datatypes::{DataType, Field, Schema}; - use datafusion::common::{extensions_options, internal_err}; + use datafusion::common::tree_node::{Transformed, TreeNode}; + use datafusion::common::{extensions_options, internal_datafusion_err, internal_err}; use datafusion::config::ConfigExtension; use datafusion::error::DataFusionError; - use datafusion::execution::{ - FunctionRegistry, SendableRecordBatchStream, SessionState, SessionStateBuilder, TaskContext, - }; - use datafusion::physical_expr::{EquivalenceProperties, Partitioning}; + use datafusion::execution::{SendableRecordBatchStream, SessionState, TaskContext}; + use datafusion::physical_expr::EquivalenceProperties; use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType}; - use datafusion::physical_plan::repartition::RepartitionExec; - use datafusion::physical_plan::stream::RecordBatchStreamAdapter; use datafusion::physical_plan::{ - DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties, execute_stream, + DisplayAs, DisplayFormatType, ExecutionPlan, ExecutionPlanProperties, PlanProperties, + execute_stream, }; use datafusion_distributed::test_utils::localhost::start_localhost_context; - use datafusion_distributed::{DistributedExt, DistributedSessionBuilderContext}; - use datafusion_distributed::{DistributedPhysicalOptimizerRule, NetworkShuffleExec}; + use datafusion_distributed::test_utils::parquet::register_parquet_tables; + use datafusion_distributed::{DistributedExt, WorkerQueryContext}; use datafusion_proto::physical_plan::PhysicalExtensionCodec; use futures::TryStreamExt; use prost::Message; @@ -24,39 +21,85 @@ mod tests { use std::fmt::Formatter; use std::sync::Arc; + async fn build_state(ctx: WorkerQueryContext) -> Result { + Ok(ctx + .builder + .with_distributed_option_extension_from_headers::(&ctx.headers)? + .with_distributed_user_codec(CustomConfigExtensionRequiredExecCodec) + .build()) + } + #[tokio::test] async fn custom_config_extension() -> Result<(), Box> { - async fn build_state( - ctx: DistributedSessionBuilderContext, - ) -> Result { - Ok(SessionStateBuilder::new() - .with_runtime_env(ctx.runtime_env) - .with_default_features() - .with_distributed_option_extension_from_headers::(&ctx.headers)? - .with_distributed_user_codec(CustomConfigExtensionRequiredExecCodec) - .build()) - } - - let (mut ctx, _guard) = start_localhost_context(3, build_state).await; + let (mut ctx, _guard, _) = start_localhost_context(3, build_state).await; + ctx.set_distributed_user_codec(CustomConfigExtensionRequiredExecCodec); ctx.set_distributed_option_extension(CustomExtension { foo: "foo".to_string(), bar: 1, baz: true, + throw_err: false, + }); + + let query = r#"SELECT "MinTemp" FROM weather WHERE "MinTemp" > 20.0"#; + + register_parquet_tables(&ctx).await?; + let df = ctx.sql(query).await?; + let plan = df.create_physical_plan().await?; + + // Wrap leaf nodes with CustomConfigExtensionRequiredExec to test config extension propagation + let transformed = plan.transform_up(|plan| { + if plan.children().is_empty() { + return Ok(Transformed::yes(Arc::new( + CustomConfigExtensionRequiredExec::new(plan), + ))); + } + Ok(Transformed::no(plan)) })?; + let plan = transformed.data; - let mut plan: Arc = Arc::new(CustomConfigExtensionRequiredExec::new()); + let stream = execute_stream(plan, ctx.task_ctx())?; + // It should not fail. + stream.try_collect::>().await?; - for size in [1, 2, 3] { - plan = Arc::new(NetworkShuffleExec::try_new( - Arc::new(RepartitionExec::try_new( - plan, - Partitioning::Hash(vec![], 10), - )?), - size, - )?); - } + Ok(()) + } - let plan = DistributedPhysicalOptimizerRule::distribute_plan(plan)?; + #[tokio::test] + async fn custom_config_extension_runtime_change() -> Result<(), Box> { + let (mut ctx, _guard, _) = start_localhost_context(3, build_state).await; + ctx.set_distributed_user_codec(CustomConfigExtensionRequiredExecCodec); + ctx.set_distributed_option_extension(CustomExtension { + throw_err: true, + ..Default::default() + }); + + let query = r#"SELECT "MinTemp" FROM weather WHERE "MinTemp" > 20.0"#; + + register_parquet_tables(&ctx).await?; + let df = ctx.sql(query).await?; + let plan = df.create_physical_plan().await?; + + // Wrap leaf nodes with CustomConfigExtensionRequiredExec to test config extension propagation + let transformed = plan.transform_up(|plan| { + if plan.children().is_empty() { + return Ok(Transformed::yes(Arc::new( + CustomConfigExtensionRequiredExec::new(plan), + ))); + } + Ok(Transformed::no(plan)) + })?; + let plan = transformed.data; + + // If the value is modified after setting it as a distributed option extension, it should + // propagate the correct headers. + ctx.state_ref() + .write() + .config_mut() + .options_mut() + .extensions + .get_mut::() + .unwrap() + .throw_err = false; let stream = execute_stream(plan, ctx.task_ctx())?; // It should not fail. stream.try_collect::>().await?; @@ -69,6 +112,7 @@ mod tests { pub foo: String, default = "".to_string() pub bar: usize, default = 0 pub baz: bool, default = false + pub throw_err: bool, default = true } } @@ -79,18 +123,20 @@ mod tests { #[derive(Debug)] pub struct CustomConfigExtensionRequiredExec { plan_properties: PlanProperties, + child: Arc, } impl CustomConfigExtensionRequiredExec { - fn new() -> Self { - let schema = Schema::new(vec![Field::new("numbers", DataType::Int64, false)]); + fn new(child: Arc) -> Self { + let plan_properties = PlanProperties::new( + EquivalenceProperties::new(child.schema()), + child.output_partitioning().clone(), + EmissionType::Incremental, + Boundedness::Bounded, + ); Self { - plan_properties: PlanProperties::new( - EquivalenceProperties::new(Arc::new(schema)), - Partitioning::UnknownPartitioning(1), - EmissionType::Incremental, - Boundedness::Bounded, - ), + plan_properties, + child, } } } @@ -115,34 +161,36 @@ mod tests { } fn children(&self) -> Vec<&Arc> { - vec![] + vec![&self.child] } fn with_new_children( self: Arc, - _: Vec>, + children: Vec>, ) -> datafusion::common::Result> { - Ok(self) + Ok(Arc::new(CustomConfigExtensionRequiredExec::new( + children[0].clone(), + ))) } fn execute( &self, - _: usize, + partition: usize, ctx: Arc, ) -> datafusion::common::Result { - if ctx + let Some(ext) = ctx .session_config() .options() .extensions .get::() - .is_none() - { + else { return internal_err!("CustomExtension not found in context"); + }; + if ext.throw_err { + return internal_err!("CustomExtension requested an error to be thrown"); } - Ok(Box::pin(RecordBatchStreamAdapter::new( - self.schema(), - futures::stream::empty(), - ))) + // Pass through to child + self.child.execute(partition, ctx) } } @@ -155,11 +203,23 @@ mod tests { impl PhysicalExtensionCodec for CustomConfigExtensionRequiredExecCodec { fn try_decode( &self, - _buf: &[u8], - _: &[Arc], - _registry: &dyn FunctionRegistry, + buf: &[u8], + inputs: &[Arc], + _ctx: &TaskContext, ) -> datafusion::common::Result> { - Ok(Arc::new(CustomConfigExtensionRequiredExec::new())) + let _node = CustomConfigExtensionRequiredExecProto::decode(buf) + .map_err(|err| internal_datafusion_err!("{err}"))?; + + if inputs.len() != 1 { + return internal_err!( + "CustomConfigExtensionRequiredExec expects exactly one child, got {}", + inputs.len() + ); + } + + Ok(Arc::new(CustomConfigExtensionRequiredExec::new( + inputs[0].clone(), + ))) } fn try_encode( diff --git a/tests/custom_extension_codec.rs b/tests/custom_extension_codec.rs index 402674e1..baaacc4f 100644 --- a/tests/custom_extension_codec.rs +++ b/tests/custom_extension_codec.rs @@ -1,36 +1,21 @@ #[cfg(all(feature = "integration", test))] mod tests { - use datafusion::arrow::array::Int64Array; - use datafusion::arrow::compute::SortOptions; - use datafusion::arrow::datatypes::{DataType, Field, Schema}; - use datafusion::arrow::record_batch::RecordBatch; use datafusion::arrow::util::pretty::pretty_format_batches; + use datafusion::common::tree_node::{Transformed, TreeNode}; use datafusion::error::DataFusionError; - use datafusion::execution::{ - FunctionRegistry, SendableRecordBatchStream, SessionState, SessionStateBuilder, TaskContext, - }; - use datafusion::logical_expr::Operator; - use datafusion::physical_expr::expressions::{BinaryExpr, col, lit}; - use datafusion::physical_expr::{ - EquivalenceProperties, LexOrdering, Partitioning, PhysicalSortExpr, - }; + use datafusion::execution::{SendableRecordBatchStream, SessionState, TaskContext}; + use datafusion::physical_expr::EquivalenceProperties; use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType}; - use datafusion::physical_plan::filter::FilterExec; - use datafusion::physical_plan::repartition::RepartitionExec; - use datafusion::physical_plan::sorts::sort::SortExec; - use datafusion::physical_plan::stream::RecordBatchStreamAdapter; use datafusion::physical_plan::{ - DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties, displayable, execute_stream, + DisplayAs, DisplayFormatType, ExecutionPlan, ExecutionPlanProperties, PlanProperties, + execute_stream, }; use datafusion_distributed::test_utils::localhost::start_localhost_context; - use datafusion_distributed::{ - DistributedExt, DistributedSessionBuilderContext, PartitionIsolatorExec, assert_snapshot, - display_plan_ascii, - }; - use datafusion_distributed::{DistributedPhysicalOptimizerRule, NetworkShuffleExec}; + use datafusion_distributed::test_utils::parquet::register_parquet_tables; + use datafusion_distributed::{DistributedExt, WorkerQueryContext, assert_snapshot}; use datafusion_proto::physical_plan::PhysicalExtensionCodec; use datafusion_proto::protobuf::proto_error; - use futures::{TryStreamExt, stream}; + use futures::TryStreamExt; use prost::Message; use std::any::Any; use std::fmt::Formatter; @@ -38,170 +23,82 @@ mod tests { #[tokio::test] async fn custom_extension_codec() -> Result<(), Box> { - async fn build_state( - ctx: DistributedSessionBuilderContext, - ) -> Result { - Ok(SessionStateBuilder::new() - .with_runtime_env(ctx.runtime_env) - .with_default_features() - .with_distributed_user_codec(Int64ListExecCodec) + async fn build_state(ctx: WorkerQueryContext) -> Result { + Ok(ctx + .builder + .with_distributed_user_codec(CustomPassThroughExecCodec) .build()) } - let (ctx, _guard) = start_localhost_context(3, build_state).await; - - let single_node_plan = build_plan(false)?; - assert_snapshot!(displayable(single_node_plan.as_ref()).indent(true).to_string(), @r" - SortExec: expr=[numbers@0 DESC NULLS LAST], preserve_partitioning=[false] - FilterExec: numbers@0 > 1 - Int64ListExec: length=6 - "); - - let distributed_plan = build_plan(true)?; - let distributed_plan = DistributedPhysicalOptimizerRule::distribute_plan(distributed_plan)?; + let (mut ctx, _guard, _) = start_localhost_context(3, build_state).await; + ctx.set_distributed_user_codec(CustomPassThroughExecCodec); - assert_snapshot!(display_plan_ascii(distributed_plan.as_ref()), @r" - ┌───── DistributedExec ── Tasks: t0:[p0] - │ SortExec: expr=[numbers@0 DESC NULLS LAST], preserve_partitioning=[false] - │ RepartitionExec: partitioning=RoundRobinBatch(1), input_partitions=10 - │ [Stage 2] => NetworkShuffleExec: output_partitions=10, input_tasks=10 - └────────────────────────────────────────────────── - ┌───── Stage 2 ── Tasks: t0:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9] t1:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9] t2:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9] t3:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9] t4:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9] t5:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9] t6:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9] t7:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9] t8:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9] t9:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9] - │ RepartitionExec: partitioning=Hash([], 10), input_partitions=1 - │ SortExec: expr=[numbers@0 DESC NULLS LAST], preserve_partitioning=[false] - │ [Stage 1] => NetworkShuffleExec: output_partitions=1, input_tasks=1 - └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9] - │ RepartitionExec: partitioning=Hash([numbers@0], 10), input_partitions=1 - │ FilterExec: numbers@0 > 1 - │ Int64ListExec: length=6 - └────────────────────────────────────────────────── - "); + let query = r#"SELECT "MinTemp", "RainToday" FROM weather WHERE "MinTemp" > 20.0 ORDER BY "MinTemp" DESC"#; - let stream = execute_stream(single_node_plan, ctx.task_ctx())?; - let batches_single_node = stream.try_collect::>().await?; + register_parquet_tables(&ctx).await?; + let df = ctx.sql(query).await?; + let plan = df.create_physical_plan().await?; - assert_snapshot!(pretty_format_batches(&batches_single_node).unwrap(), @r" - +---------+ - | numbers | - +---------+ - | 6 | - | 5 | - | 4 | - | 3 | - | 2 | - +---------+ + // Wrap leaf nodes with CustomPassThroughExec to test custom codec + let transformed = plan.transform_up(|plan| { + if plan.children().is_empty() { + return Ok(Transformed::yes(Arc::new(CustomPassThroughExec::new(plan)))); + } + Ok(Transformed::no(plan)) + })?; + let plan = transformed.data; + + let batches = pretty_format_batches( + &execute_stream(plan, ctx.task_ctx())? + .try_collect::>() + .await?, + )?; + + // Verify that the custom execution plan completes successfully + assert!(!batches.to_string().is_empty()); + assert_snapshot!(batches, @r" + +---------+-----------+ + | MinTemp | RainToday | + +---------+-----------+ + | 20.9 | No | + +---------+-----------+ "); - let stream = execute_stream(distributed_plan, ctx.task_ctx())?; - let batches_distributed = stream.try_collect::>().await?; - - assert_snapshot!(pretty_format_batches(&batches_distributed).unwrap(), @r" - +---------+ - | numbers | - +---------+ - | 6 | - | 5 | - | 4 | - | 3 | - | 2 | - +---------+ - "); Ok(()) } - fn build_plan(distributed: bool) -> Result, DataFusionError> { - let mut plan: Arc = Arc::new(Int64ListExec::new(vec![1, 2, 3, 4, 5, 6])); - - if distributed { - plan = Arc::new(PartitionIsolatorExec::new(plan)); - } - - plan = Arc::new(FilterExec::try_new( - Arc::new(BinaryExpr::new( - col("numbers", &plan.schema())?, - Operator::Gt, - lit(1i64), - )), - plan, - )?); - - if distributed { - plan = Arc::new(NetworkShuffleExec::try_new( - Arc::new(RepartitionExec::try_new( - Arc::clone(&plan), - Partitioning::Hash(vec![col("numbers", &plan.schema())?], 1), - )?), - 10, - )?); - } - - plan = Arc::new(SortExec::new( - LexOrdering::new(vec![PhysicalSortExpr::new( - col("numbers", &plan.schema())?, - SortOptions::new(true, false), - )]) - .unwrap(), - plan, - )); - - if distributed { - plan = Arc::new(NetworkShuffleExec::try_new( - Arc::new(RepartitionExec::try_new( - plan, - Partitioning::Hash(vec![], 10), - )?), - 10, - )?); - - plan = Arc::new(RepartitionExec::try_new( - plan, - Partitioning::RoundRobinBatch(1), - )?); - - plan = Arc::new(SortExec::new( - LexOrdering::new(vec![PhysicalSortExpr::new( - col("numbers", &plan.schema())?, - SortOptions::new(true, false), - )]) - .unwrap(), - plan, - )); - } - - Ok(plan) - } - + /// A custom execution plan that wraps a child and passes through execution. + /// This tests that custom user codecs work correctly in distributed execution. #[derive(Debug)] - pub struct Int64ListExec { + pub struct CustomPassThroughExec { plan_properties: PlanProperties, - numbers: Vec, + child: Arc, } - impl Int64ListExec { - fn new(numbers: Vec) -> Self { - let schema = Schema::new(vec![Field::new("numbers", DataType::Int64, false)]); + impl CustomPassThroughExec { + fn new(child: Arc) -> Self { + let plan_properties = PlanProperties::new( + EquivalenceProperties::new(child.schema()), + child.output_partitioning().clone(), + EmissionType::Incremental, + Boundedness::Bounded, + ); Self { - numbers, - plan_properties: PlanProperties::new( - EquivalenceProperties::new(Arc::new(schema)), - Partitioning::UnknownPartitioning(1), - EmissionType::Incremental, - Boundedness::Bounded, - ), + plan_properties, + child, } } } - impl DisplayAs for Int64ListExec { + impl DisplayAs for CustomPassThroughExec { fn fmt_as(&self, _: DisplayFormatType, f: &mut Formatter) -> std::fmt::Result { - write!(f, "Int64ListExec: length={:?}", self.numbers.len()) + write!(f, "CustomPassThroughExec") } } - impl ExecutionPlan for Int64ListExec { + impl ExecutionPlan for CustomPassThroughExec { fn name(&self) -> &str { - "Int64ListExec" + "CustomPassThroughExec" } fn as_any(&self) -> &dyn Any { @@ -213,51 +110,52 @@ mod tests { } fn children(&self) -> Vec<&Arc> { - vec![] + vec![&self.child] } fn with_new_children( self: Arc, - _: Vec>, + children: Vec>, ) -> datafusion::common::Result> { - Ok(self) + Ok(Arc::new(CustomPassThroughExec::new(children[0].clone()))) } fn execute( &self, - _: usize, - _: Arc, + partition: usize, + context: Arc, ) -> datafusion::common::Result { - let array = Int64Array::from(self.numbers.clone()); - let batch = RecordBatch::try_new(self.schema(), vec![Arc::new(array)])?; - - let stream = stream::iter(vec![Ok(batch)]); - Ok(Box::pin(RecordBatchStreamAdapter::new( - self.schema(), - stream, - ))) + // Simply pass through to the child + self.child.execute(partition, context) } } #[derive(Debug)] - struct Int64ListExecCodec; + struct CustomPassThroughExecCodec; #[derive(Clone, PartialEq, ::prost::Message)] - struct Int64ListExecProto { - #[prost(message, repeated, tag = "1")] - numbers: Vec, + struct CustomPassThroughExecProto { + // Empty - we'll handle the child through normal codec mechanisms } - impl PhysicalExtensionCodec for Int64ListExecCodec { + impl PhysicalExtensionCodec for CustomPassThroughExecCodec { fn try_decode( &self, buf: &[u8], - _: &[Arc], - _registry: &dyn FunctionRegistry, + inputs: &[Arc], + _ctx: &TaskContext, ) -> datafusion::common::Result> { - let node = - Int64ListExecProto::decode(buf).map_err(|err| proto_error(format!("{err}")))?; - Ok(Arc::new(Int64ListExec::new(node.numbers.clone()))) + let _node = CustomPassThroughExecProto::decode(buf) + .map_err(|err| proto_error(format!("{err}")))?; + + if inputs.len() != 1 { + return Err(proto_error(format!( + "CustomPassThroughExec expects exactly one child, got {}", + inputs.len() + ))); + } + + Ok(Arc::new(CustomPassThroughExec::new(inputs[0].clone()))) } fn try_encode( @@ -265,17 +163,15 @@ mod tests { node: Arc, buf: &mut Vec, ) -> datafusion::common::Result<()> { - let Some(plan) = node.as_any().downcast_ref::() else { + let Some(_plan) = node.as_any().downcast_ref::() else { return Err(proto_error(format!( - "Expected plan to be of type Int64ListExec, but was {}", + "Expected plan to be of type CustomPassThroughExec, but was {}", node.name() ))); }; - Int64ListExecProto { - numbers: plan.numbers.clone(), - } - .encode(buf) - .map_err(|err| proto_error(format!("{err}"))) + CustomPassThroughExecProto {} + .encode(buf) + .map_err(|err| proto_error(format!("{err}"))) } } } diff --git a/tests/distributed_aggregation.rs b/tests/distributed_aggregation.rs index b0ea216f..8ce80cda 100644 --- a/tests/distributed_aggregation.rs +++ b/tests/distributed_aggregation.rs @@ -1,34 +1,38 @@ #[cfg(all(feature = "integration", test))] mod tests { + use datafusion::arrow::array::{Int32Array, StringArray}; + use datafusion::arrow::datatypes::{DataType, Field, Schema}; + use datafusion::arrow::record_batch::RecordBatch; use datafusion::arrow::util::pretty::pretty_format_batches; - use datafusion::physical_optimizer::PhysicalOptimizerRule; use datafusion::physical_plan::{displayable, execute_stream}; + use datafusion::prelude::SessionContext; use datafusion_distributed::test_utils::localhost::start_localhost_context; use datafusion_distributed::test_utils::parquet::register_parquet_tables; - use datafusion_distributed::{ - DefaultSessionBuilder, DistributedPhysicalOptimizerRule, assert_snapshot, - display_plan_ascii, - }; + use datafusion_distributed::test_utils::session_context::register_temp_parquet_table; + use datafusion_distributed::{DefaultSessionBuilder, assert_snapshot, display_plan_ascii}; use futures::TryStreamExt; use std::error::Error; + use std::sync::Arc; + use uuid::Uuid; #[tokio::test] async fn distributed_aggregation() -> Result<(), Box> { - let (ctx, _guard) = start_localhost_context(3, DefaultSessionBuilder).await; - register_parquet_tables(&ctx).await?; + let (ctx_distributed, _guard, _) = start_localhost_context(3, DefaultSessionBuilder).await; - let df = ctx - .sql(r#"SELECT count(*), "RainToday" FROM weather GROUP BY "RainToday" ORDER BY count(*)"#) - .await?; - let physical = df.create_physical_plan().await?; + let query = + r#"SELECT count(*), "RainToday" FROM weather GROUP BY "RainToday" ORDER BY count(*)"#; + let ctx = SessionContext::default(); + *ctx.state_ref().write().config_mut() = ctx_distributed.copied_config(); + register_parquet_tables(&ctx).await?; + let df = ctx.sql(query).await?; + let physical = df.create_physical_plan().await?; let physical_str = displayable(physical.as_ref()).indent(true).to_string(); - let physical_distributed = DistributedPhysicalOptimizerRule::default() - .with_network_shuffle_tasks(2) - .optimize(physical.clone(), &Default::default())?; - - let physical_distributed_str = display_plan_ascii(physical_distributed.as_ref()); + register_parquet_tables(&ctx_distributed).await?; + let df_distributed = ctx_distributed.sql(query).await?; + let physical_distributed = df_distributed.create_physical_plan().await?; + let physical_distributed_str = display_plan_ascii(physical_distributed.as_ref(), false); assert_snapshot!(physical_str, @r" @@ -37,10 +41,9 @@ mod tests { SortExec: expr=[count(*)@0 ASC NULLS LAST], preserve_partitioning=[true] ProjectionExec: expr=[count(Int64(1))@1 as count(*), RainToday@0 as RainToday, count(Int64(1))@1 as count(Int64(1))] AggregateExec: mode=FinalPartitioned, gby=[RainToday@0 as RainToday], aggr=[count(Int64(1))] - CoalesceBatchesExec: target_batch_size=8192 - RepartitionExec: partitioning=Hash([RainToday@0], 3), input_partitions=3 - AggregateExec: mode=Partial, gby=[RainToday@0 as RainToday], aggr=[count(Int64(1))] - DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[RainToday], file_type=parquet + RepartitionExec: partitioning=Hash([RainToday@0], 3), input_partitions=3 + AggregateExec: mode=Partial, gby=[RainToday@0 as RainToday], aggr=[count(Int64(1))] + DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[RainToday], file_type=parquet ", ); @@ -49,18 +52,20 @@ mod tests { ┌───── DistributedExec ── Tasks: t0:[p0] │ ProjectionExec: expr=[count(*)@0 as count(*), RainToday@1 as RainToday] │ SortPreservingMergeExec: [count(Int64(1))@2 ASC NULLS LAST] - │ SortExec: expr=[count(*)@0 ASC NULLS LAST], preserve_partitioning=[true] - │ ProjectionExec: expr=[count(Int64(1))@1 as count(*), RainToday@0 as RainToday, count(Int64(1))@1 as count(Int64(1))] - │ AggregateExec: mode=FinalPartitioned, gby=[RainToday@0 as RainToday], aggr=[count(Int64(1))] - │ CoalesceBatchesExec: target_batch_size=8192 - │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0,p1,p2] t1:[p0,p1,p2] - │ RepartitionExec: partitioning=Hash([RainToday@0], 3), input_partitions=2 - │ AggregateExec: mode=Partial, gby=[RainToday@0 as RainToday], aggr=[count(Int64(1))] - │ PartitionIsolatorExec: t0:[p0,p1,__] t1:[__,__,p0] - │ DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[RainToday], file_type=parquet + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ SortExec: expr=[count(*)@0 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[count(Int64(1))@1 as count(*), RainToday@0 as RainToday, count(Int64(1))@1 as count(Int64(1))] + │ AggregateExec: mode=FinalPartitioned, gby=[RainToday@0 as RainToday], aggr=[count(Int64(1))] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([RainToday@0], 6), input_partitions=1 + │ AggregateExec: mode=Partial, gby=[RainToday@0 as RainToday], aggr=[count(Int64(1))] + │ PartitionIsolatorExec: t0:[p0,__,__] t1:[__,p0,__] t2:[__,__,p0] + │ DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[RainToday], file_type=parquet + └────────────────────────────────────────────────── ", ); @@ -98,31 +103,29 @@ mod tests { #[tokio::test] async fn distributed_aggregation_head_node_partitioned() -> Result<(), Box> { - let (ctx, _guard) = start_localhost_context(6, DefaultSessionBuilder).await; - register_parquet_tables(&ctx).await?; + let (ctx_distributed, _guard, _) = start_localhost_context(6, DefaultSessionBuilder).await; - let df = ctx - .sql(r#"SELECT count(*), "RainToday" FROM weather GROUP BY "RainToday""#) - .await?; - let physical = df.create_physical_plan().await?; + let query = r#"SELECT count(*), "RainToday" FROM weather GROUP BY "RainToday""#; + let ctx = SessionContext::default(); + *ctx.state_ref().write().config_mut() = ctx_distributed.copied_config(); + register_parquet_tables(&ctx).await?; + let df = ctx.sql(query).await?; + let physical = df.create_physical_plan().await?; let physical_str = displayable(physical.as_ref()).indent(true).to_string(); - let physical_distributed = DistributedPhysicalOptimizerRule::default() - .with_network_shuffle_tasks(6) - .with_network_coalesce_tasks(6) - .optimize(physical.clone(), &Default::default())?; - - let physical_distributed_str = display_plan_ascii(physical_distributed.as_ref()); + register_parquet_tables(&ctx_distributed).await?; + let df_distributed = ctx_distributed.sql(query).await?; + let physical_distributed = df_distributed.create_physical_plan().await?; + let physical_distributed_str = display_plan_ascii(physical_distributed.as_ref(), false); assert_snapshot!(physical_str, @r" ProjectionExec: expr=[count(Int64(1))@1 as count(*), RainToday@0 as RainToday] AggregateExec: mode=FinalPartitioned, gby=[RainToday@0 as RainToday], aggr=[count(Int64(1))] - CoalesceBatchesExec: target_batch_size=8192 - RepartitionExec: partitioning=Hash([RainToday@0], 3), input_partitions=3 - AggregateExec: mode=Partial, gby=[RainToday@0 as RainToday], aggr=[count(Int64(1))] - DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[RainToday], file_type=parquet + RepartitionExec: partitioning=Hash([RainToday@0], 3), input_partitions=3 + AggregateExec: mode=Partial, gby=[RainToday@0 as RainToday], aggr=[count(Int64(1))] + DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[RainToday], file_type=parquet ", ); @@ -130,18 +133,17 @@ mod tests { @r" ┌───── DistributedExec ── Tasks: t0:[p0] │ CoalescePartitionsExec - │ [Stage 2] => NetworkCoalesceExec: output_partitions=18, input_tasks=6 + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 └────────────────────────────────────────────────── - ┌───── Stage 2 ── Tasks: t0:[p0,p1,p2] t1:[p0,p1,p2] t2:[p0,p1,p2] t3:[p0,p1,p2] t4:[p0,p1,p2] t5:[p0,p1,p2] + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p0..p2] │ ProjectionExec: expr=[count(Int64(1))@1 as count(*), RainToday@0 as RainToday] │ AggregateExec: mode=FinalPartitioned, gby=[RainToday@0 as RainToday], aggr=[count(Int64(1))] - │ CoalesceBatchesExec: target_batch_size=8192 - │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=3 └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9,p10,p11,p12,p13,p14,p15,p16,p17] t1:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9,p10,p11,p12,p13,p14,p15,p16,p17] t2:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9,p10,p11,p12,p13,p14,p15,p16,p17] - │ RepartitionExec: partitioning=Hash([RainToday@0], 18), input_partitions=1 + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([RainToday@0], 6), input_partitions=1 │ AggregateExec: mode=Partial, gby=[RainToday@0 as RainToday], aggr=[count(Int64(1))] - │ PartitionIsolatorExec: t0:[p0,__,__] t1:[__,p0,__] t2:[__,__,p0] + │ PartitionIsolatorExec: t0:[p0,__,__] t1:[__,p0,__] t2:[__,__,p0] │ DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[RainToday], file_type=parquet └────────────────────────────────────────────────── ", @@ -149,4 +151,94 @@ mod tests { Ok(()) } + + /// Test that multiple first_value() aggregations work correctly in distributed queries. + // TODO: Once https://github.com/apache/datafusion/pull/18303 is merged, this test will lose + // meaning, since the PR above will mask the underlying problem. Different queries or + // a new approach must be used in this case. + #[tokio::test] + async fn test_multiple_first_value_aggregations() -> Result<(), Box> { + let (ctx, _guard, _) = start_localhost_context(3, DefaultSessionBuilder).await; + + let schema = Arc::new(Schema::new(vec![ + Field::new("group_id", DataType::Int32, false), + Field::new("trace_id", DataType::Utf8, false), + Field::new("value", DataType::Int32, false), + ])); + + // Create 2 batches that will be stored as separate parquet files + let batch1 = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from(vec![1, 2])), + Arc::new(StringArray::from(vec!["trace1", "trace2"])), + Arc::new(Int32Array::from(vec![100, 200])), + ], + )?; + + let batch2 = RecordBatch::try_new( + schema.clone(), + vec![ + Arc::new(Int32Array::from(vec![3, 4])), + Arc::new(StringArray::from(vec!["trace3", "trace4"])), + Arc::new(Int32Array::from(vec![300, 400])), + ], + )?; + + let file1 = + register_temp_parquet_table("records_part1", schema.clone(), vec![batch1], &ctx) + .await?; + let file2 = + register_temp_parquet_table("records_part2", schema.clone(), vec![batch2], &ctx) + .await?; + + // Create a partitioned table by registering multiple files + let temp_dir = std::env::temp_dir(); + let table_dir = temp_dir.join(format!("partitioned_table_{}", Uuid::new_v4())); + std::fs::create_dir(&table_dir)?; + std::fs::copy(&file1, table_dir.join("part1.parquet"))?; + std::fs::copy(&file2, table_dir.join("part2.parquet"))?; + + // Register the directory as a partitioned table + ctx.register_parquet( + "records_partitioned", + table_dir.to_str().unwrap(), + datafusion::prelude::ParquetReadOptions::default(), + ) + .await?; + + let query = r#"SELECT group_id, first_value(trace_id) AS fv1, first_value(value) AS fv2 + FROM records_partitioned + GROUP BY group_id + ORDER BY group_id"#; + + let df = ctx.sql(query).await?; + let physical = df.create_physical_plan().await?; + + // Execute distributed query + let batches_distributed = execute_stream(physical, ctx.task_ctx())? + .try_collect::>() + .await?; + + let actual_result = pretty_format_batches(&batches_distributed)?; + let expected_result = "\ ++----------+--------+-----+ +| group_id | fv1 | fv2 | ++----------+--------+-----+ +| 1 | trace1 | 100 | +| 2 | trace2 | 200 | +| 3 | trace3 | 300 | +| 4 | trace4 | 400 | ++----------+--------+-----+"; + + // Print them out, the error message from `assert_eq` is otherwise hard to read. + println!("{expected_result}"); + println!("{actual_result}"); + + // Compare against result. The regression this is testing for would have NULL values in + // the second and third column. + assert_eq!(actual_result.to_string(), expected_result,); + + Ok(()) + } } diff --git a/tests/distributed_unions.rs b/tests/distributed_unions.rs new file mode 100644 index 00000000..db90999b --- /dev/null +++ b/tests/distributed_unions.rs @@ -0,0 +1,201 @@ +#[cfg(all(feature = "integration", test))] +mod tests { + use datafusion::arrow::util::pretty::pretty_format_batches; + use datafusion::execution::TaskContext; + use datafusion::physical_plan::{ExecutionPlan, execute_stream}; + use datafusion::prelude::SessionContext; + use datafusion_distributed::test_utils::localhost::start_localhost_context; + use datafusion_distributed::test_utils::parquet::register_parquet_tables; + use datafusion_distributed::{DefaultSessionBuilder, assert_snapshot, display_plan_ascii}; + use futures::TryStreamExt; + use std::error::Error; + use std::sync::Arc; + + #[tokio::test] + async fn more_tasks_than_children() -> Result<(), Box> { + let (ctx_distributed, _guard, _) = start_localhost_context(3, DefaultSessionBuilder).await; + + let query = r#" + SELECT "MinTemp", "RainToday" FROM weather WHERE "MinTemp" > 10.0 + UNION ALL + SELECT "MaxTemp", "RainToday" FROM weather WHERE "MaxTemp" < 30.0 + ORDER BY "MinTemp", "RainToday" + "#; + + let ctx = SessionContext::default(); + *ctx.state_ref().write().config_mut() = ctx_distributed.copied_config(); + register_parquet_tables(&ctx).await?; + let df = ctx.sql(query).await?; + let physical = df.create_physical_plan().await?; + + register_parquet_tables(&ctx_distributed).await?; + ctx_distributed + .sql("SET distributed.children_isolator_unions=true;") + .await?; + let df_distributed = ctx_distributed.sql(query).await?; + let physical_distributed = df_distributed.create_physical_plan().await?; + let physical_distributed_str = display_plan_ascii(physical_distributed.as_ref(), false); + + assert_snapshot!(physical_distributed_str, + @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [MinTemp@0 ASC NULLS LAST, RainToday@1 ASC NULLS LAST] + │ [Stage 1] => NetworkCoalesceExec: output_partitions=9, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] t2:[p6..p8] + │ DistributedUnionExec: t0:[c0] t1:[c1(0/2)] t2:[c1(1/2)] + │ SortExec: expr=[MinTemp@0 ASC NULLS LAST, RainToday@1 ASC NULLS LAST], preserve_partitioning=[true] + │ FilterExec: MinTemp@0 > 10 + │ DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MinTemp, RainToday], file_type=parquet, predicate=MinTemp@0 > 10, pruning_predicate=MinTemp_null_count@1 != row_count@2 AND MinTemp_max@0 > 10, required_guarantees=[] + │ SortExec: expr=[MinTemp@0 ASC NULLS LAST, RainToday@1 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[MaxTemp@0 as MinTemp, RainToday@1 as RainToday] + │ FilterExec: MaxTemp@0 < 30 + │ PartitionIsolatorExec: t0:[p0,p1,__] t1:[__,__,p0] + │ DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=MaxTemp@1 < 30, pruning_predicate=MaxTemp_null_count@1 != row_count@2 AND MaxTemp_min@0 < 30, required_guarantees=[] + └────────────────────────────────────────────────── + ", + ); + + exact_same_data(ctx.task_ctx(), physical, physical_distributed).await + } + + #[tokio::test] + async fn same_children_than_tasks() -> Result<(), Box> { + let (ctx_distributed, _guard, _) = start_localhost_context(3, DefaultSessionBuilder).await; + + let query = r#" + SELECT "MinTemp", "RainToday" FROM weather WHERE "MinTemp" > 20.0 + UNION ALL + SELECT "MaxTemp", "RainToday" FROM weather WHERE "MaxTemp" < 25.0 + UNION ALL + SELECT "Temp9am", "RainToday" FROM weather WHERE "Temp9am" > 15.0 + ORDER BY "MinTemp", "RainToday" + "#; + + let ctx = SessionContext::default(); + *ctx.state_ref().write().config_mut() = ctx_distributed.copied_config(); + register_parquet_tables(&ctx).await?; + let df = ctx.sql(query).await?; + let physical = df.create_physical_plan().await?; + + register_parquet_tables(&ctx_distributed).await?; + ctx_distributed + .sql("SET distributed.children_isolator_unions=true;") + .await?; + let df_distributed = ctx_distributed.sql(query).await?; + let physical_distributed = df_distributed.create_physical_plan().await?; + let physical_distributed_str = display_plan_ascii(physical_distributed.as_ref(), false); + + assert_snapshot!(physical_distributed_str, + @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [MinTemp@0 ASC NULLS LAST, RainToday@1 ASC NULLS LAST] + │ [Stage 1] => NetworkCoalesceExec: output_partitions=9, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] t2:[p6..p8] + │ DistributedUnionExec: t0:[c0] t1:[c1] t2:[c2] + │ SortExec: expr=[MinTemp@0 ASC NULLS LAST, RainToday@1 ASC NULLS LAST], preserve_partitioning=[true] + │ FilterExec: MinTemp@0 > 20 + │ DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MinTemp, RainToday], file_type=parquet, predicate=MinTemp@0 > 20, pruning_predicate=MinTemp_null_count@1 != row_count@2 AND MinTemp_max@0 > 20, required_guarantees=[] + │ SortExec: expr=[MinTemp@0 ASC NULLS LAST, RainToday@1 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[MaxTemp@0 as MinTemp, RainToday@1 as RainToday] + │ FilterExec: MaxTemp@0 < 25 + │ DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=MaxTemp@1 < 25, pruning_predicate=MaxTemp_null_count@1 != row_count@2 AND MaxTemp_min@0 < 25, required_guarantees=[] + │ SortExec: expr=[MinTemp@0 ASC NULLS LAST, RainToday@1 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[Temp9am@0 as MinTemp, RainToday@1 as RainToday] + │ FilterExec: Temp9am@0 > 15 + │ DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[Temp9am, RainToday], file_type=parquet, predicate=Temp9am@17 > 15, pruning_predicate=Temp9am_null_count@1 != row_count@2 AND Temp9am_max@0 > 15, required_guarantees=[] + └────────────────────────────────────────────────── + ", + ); + + exact_same_data(ctx.task_ctx(), physical, physical_distributed).await + } + + #[tokio::test] + async fn more_children_than_tasks() -> Result<(), Box> { + let (ctx_distributed, _guard, _) = start_localhost_context(3, DefaultSessionBuilder).await; + + let query = r#" + SELECT "MinTemp", "RainToday" FROM weather WHERE "MinTemp" > 10.0 + UNION ALL + SELECT "MaxTemp", "RainToday" FROM weather WHERE "MaxTemp" < 30.0 + UNION ALL + SELECT "Temp9am", "RainToday" FROM weather WHERE "Temp9am" > 15.0 + UNION ALL + SELECT "Temp3pm", "RainToday" FROM weather WHERE "Temp3pm" < 25.0 + UNION ALL + SELECT "Rainfall", "RainToday" FROM weather WHERE "Rainfall" > 5.0 + ORDER BY "MinTemp", "RainToday" + "#; + + let ctx = SessionContext::default(); + *ctx.state_ref().write().config_mut() = ctx_distributed.copied_config(); + register_parquet_tables(&ctx).await?; + let df = ctx.sql(query).await?; + let physical = df.create_physical_plan().await?; + + register_parquet_tables(&ctx_distributed).await?; + ctx_distributed + .sql("SET distributed.children_isolator_unions=true;") + .await?; + let df_distributed = ctx_distributed.sql(query).await?; + let physical_distributed = df_distributed.create_physical_plan().await?; + let physical_distributed_str = display_plan_ascii(physical_distributed.as_ref(), false); + + assert_snapshot!(physical_distributed_str, + @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [MinTemp@0 ASC NULLS LAST, RainToday@1 ASC NULLS LAST] + │ [Stage 1] => NetworkCoalesceExec: output_partitions=18, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p6..p11] t2:[p12..p17] + │ DistributedUnionExec: t0:[c0, c1] t1:[c2, c3] t2:[c4] + │ SortExec: expr=[MinTemp@0 ASC NULLS LAST, RainToday@1 ASC NULLS LAST], preserve_partitioning=[true] + │ FilterExec: MinTemp@0 > 10 + │ DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MinTemp, RainToday], file_type=parquet, predicate=MinTemp@0 > 10, pruning_predicate=MinTemp_null_count@1 != row_count@2 AND MinTemp_max@0 > 10, required_guarantees=[] + │ SortExec: expr=[MinTemp@0 ASC NULLS LAST, RainToday@1 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[MaxTemp@0 as MinTemp, RainToday@1 as RainToday] + │ FilterExec: MaxTemp@0 < 30 + │ DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MaxTemp, RainToday], file_type=parquet, predicate=MaxTemp@1 < 30, pruning_predicate=MaxTemp_null_count@1 != row_count@2 AND MaxTemp_min@0 < 30, required_guarantees=[] + │ SortExec: expr=[MinTemp@0 ASC NULLS LAST, RainToday@1 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[Temp9am@0 as MinTemp, RainToday@1 as RainToday] + │ FilterExec: Temp9am@0 > 15 + │ DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[Temp9am, RainToday], file_type=parquet, predicate=Temp9am@17 > 15, pruning_predicate=Temp9am_null_count@1 != row_count@2 AND Temp9am_max@0 > 15, required_guarantees=[] + │ SortExec: expr=[MinTemp@0 ASC NULLS LAST, RainToday@1 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[Temp3pm@0 as MinTemp, RainToday@1 as RainToday] + │ FilterExec: Temp3pm@0 < 25 + │ DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[Temp3pm, RainToday], file_type=parquet, predicate=Temp3pm@18 < 25, pruning_predicate=Temp3pm_null_count@1 != row_count@2 AND Temp3pm_min@0 < 25, required_guarantees=[] + │ SortExec: expr=[MinTemp@0 ASC NULLS LAST, RainToday@1 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[Rainfall@0 as MinTemp, RainToday@1 as RainToday] + │ FilterExec: Rainfall@0 > 5 + │ DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[Rainfall, RainToday], file_type=parquet, predicate=Rainfall@2 > 5, pruning_predicate=Rainfall_null_count@1 != row_count@2 AND Rainfall_max@0 > 5, required_guarantees=[] + └────────────────────────────────────────────────── + ", + ); + + exact_same_data(ctx.task_ctx(), physical, physical_distributed).await + } + + async fn exact_same_data( + task_ctx: Arc, + one: Arc, + other: Arc, + ) -> Result<(), Box> { + let batches = pretty_format_batches( + &execute_stream(one, task_ctx.clone())? + .try_collect::>() + .await?, + )?; + + let batches_distributed = pretty_format_batches( + &execute_stream(other, task_ctx)? + .try_collect::>() + .await?, + )?; + + // Verify that both plans produce the same results + assert_eq!(batches.to_string(), batches_distributed.to_string()); + Ok(()) + } +} diff --git a/tests/error_propagation.rs b/tests/error_propagation.rs index 5a2a3706..52dc5c43 100644 --- a/tests/error_propagation.rs +++ b/tests/error_propagation.rs @@ -1,22 +1,18 @@ #[cfg(all(feature = "integration", test))] mod tests { - use datafusion::arrow::datatypes::{DataType, Field, Schema}; + use datafusion::common::tree_node::{Transformed, TreeNode}; use datafusion::error::DataFusionError; - use datafusion::execution::{ - FunctionRegistry, SendableRecordBatchStream, SessionState, SessionStateBuilder, TaskContext, - }; - use datafusion::physical_expr::{EquivalenceProperties, Partitioning}; + use datafusion::execution::{SendableRecordBatchStream, SessionState, TaskContext}; + use datafusion::physical_expr::EquivalenceProperties; use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType}; - use datafusion::physical_plan::repartition::RepartitionExec; use datafusion::physical_plan::stream::RecordBatchStreamAdapter; use datafusion::physical_plan::{ - DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties, execute_stream, + DisplayAs, DisplayFormatType, ExecutionPlan, ExecutionPlanProperties, PlanProperties, + execute_stream, }; use datafusion_distributed::test_utils::localhost::start_localhost_context; - use datafusion_distributed::{ - DistributedExt, DistributedPhysicalOptimizerRule, DistributedSessionBuilderContext, - NetworkShuffleExec, - }; + use datafusion_distributed::test_utils::parquet::register_parquet_tables; + use datafusion_distributed::{DistributedExt, WorkerQueryContext}; use datafusion_proto::physical_plan::PhysicalExtensionCodec; use datafusion_proto::protobuf::proto_error; use futures::{TryStreamExt, stream}; @@ -28,30 +24,34 @@ mod tests { #[tokio::test] async fn test_error_propagation() -> Result<(), Box> { - async fn build_state( - ctx: DistributedSessionBuilderContext, - ) -> Result { - Ok(SessionStateBuilder::new() - .with_runtime_env(ctx.runtime_env) - .with_default_features() - .with_distributed_user_codec(ErrorExecCodec) + async fn build_state(ctx: WorkerQueryContext) -> Result { + Ok(ctx + .builder + .with_distributed_user_codec(ErrorThrowingExecCodec) .build()) } - let (ctx, _guard) = start_localhost_context(3, build_state).await; + let (mut ctx, _guard, _) = start_localhost_context(3, build_state).await; + ctx.set_distributed_user_codec(ErrorThrowingExecCodec); - let mut plan: Arc = Arc::new(ErrorExec::new("something failed")); + let query = r#"SELECT "MinTemp" FROM weather WHERE "MinTemp" > 20.0"#; - for size in [1, 2, 3] { - plan = Arc::new(NetworkShuffleExec::try_new( - Arc::new(RepartitionExec::try_new( + register_parquet_tables(&ctx).await?; + let df = ctx.sql(query).await?; + let plan = df.create_physical_plan().await?; + + // Wrap leaf nodes with ErrorThrowingExec to test error propagation + let transformed = plan.transform_up(|plan| { + if plan.children().is_empty() { + return Ok(Transformed::yes(Arc::new(ErrorThrowingExec::new( plan, - Partitioning::Hash(vec![], size), - )?), - size, - )?); - } - let plan = DistributedPhysicalOptimizerRule::distribute_plan(plan)?; + "something failed", + )))); + } + Ok(Transformed::no(plan)) + })?; + let plan = transformed.data; + let stream = execute_stream(plan, ctx.task_ctx())?; let Err(err) = stream.try_collect::>().await else { @@ -65,36 +65,40 @@ mod tests { Ok(()) } + /// A custom execution plan that wraps a child but always throws an error. + /// This tests that errors are properly propagated in distributed execution. #[derive(Debug)] - pub struct ErrorExec { + pub struct ErrorThrowingExec { msg: String, plan_properties: PlanProperties, + child: Arc, } - impl ErrorExec { - fn new(msg: &str) -> Self { - let schema = Schema::new(vec![Field::new("numbers", DataType::Int64, false)]); + impl ErrorThrowingExec { + fn new(child: Arc, msg: &str) -> Self { + let plan_properties = PlanProperties::new( + EquivalenceProperties::new(child.schema()), + child.output_partitioning().clone(), + EmissionType::Incremental, + Boundedness::Bounded, + ); Self { msg: msg.to_string(), - plan_properties: PlanProperties::new( - EquivalenceProperties::new(Arc::new(schema)), - Partitioning::UnknownPartitioning(1), - EmissionType::Incremental, - Boundedness::Bounded, - ), + plan_properties, + child, } } } - impl DisplayAs for ErrorExec { + impl DisplayAs for ErrorThrowingExec { fn fmt_as(&self, _: DisplayFormatType, f: &mut Formatter) -> std::fmt::Result { - write!(f, "ErrorExec") + write!(f, "ErrorThrowingExec") } } - impl ExecutionPlan for ErrorExec { + impl ExecutionPlan for ErrorThrowingExec { fn name(&self) -> &str { - "ErrorExec" + "ErrorThrowingExec" } fn as_any(&self) -> &dyn Any { @@ -106,14 +110,17 @@ mod tests { } fn children(&self) -> Vec<&Arc> { - vec![] + vec![&self.child] } fn with_new_children( self: Arc, - _: Vec>, + children: Vec>, ) -> datafusion::common::Result> { - Ok(self) + Ok(Arc::new(ErrorThrowingExec::new( + children[0].clone(), + &self.msg, + ))) } fn execute( @@ -121,6 +128,7 @@ mod tests { _: usize, _: Arc, ) -> datafusion::common::Result { + // Return a stream that immediately fails with the configured error message Ok(Box::pin(RecordBatchStreamAdapter::new( self.schema(), stream::iter(vec![Err(DataFusionError::Execution(self.msg.clone()))]), @@ -129,23 +137,35 @@ mod tests { } #[derive(Debug)] - struct ErrorExecCodec; + struct ErrorThrowingExecCodec; #[derive(Clone, PartialEq, ::prost::Message)] - struct ErrorExecProto { + struct ErrorThrowingExecProto { #[prost(string, tag = "1")] msg: String, } - impl PhysicalExtensionCodec for ErrorExecCodec { + impl PhysicalExtensionCodec for ErrorThrowingExecCodec { fn try_decode( &self, buf: &[u8], - _: &[Arc], - _registry: &dyn FunctionRegistry, + inputs: &[Arc], + _ctx: &TaskContext, ) -> datafusion::common::Result> { - let node = ErrorExecProto::decode(buf).map_err(|err| proto_error(format!("{err}")))?; - Ok(Arc::new(ErrorExec::new(&node.msg))) + let node = + ErrorThrowingExecProto::decode(buf).map_err(|err| proto_error(format!("{err}")))?; + + if inputs.len() != 1 { + return Err(proto_error(format!( + "ErrorThrowingExec expects exactly one child, got {}", + inputs.len() + ))); + } + + Ok(Arc::new(ErrorThrowingExec::new( + inputs[0].clone(), + &node.msg, + ))) } fn try_encode( @@ -153,13 +173,13 @@ mod tests { node: Arc, buf: &mut Vec, ) -> datafusion::common::Result<()> { - let Some(plan) = node.as_any().downcast_ref::() else { + let Some(plan) = node.as_any().downcast_ref::() else { return Err(proto_error(format!( - "Expected plan to be of type ErrorExec, but was {}", + "Expected plan to be of type ErrorThrowingExec, but was {}", node.name() ))); }; - ErrorExecProto { + ErrorThrowingExecProto { msg: plan.msg.clone(), } .encode(buf) diff --git a/tests/highly_distributed_query.rs b/tests/highly_distributed_query.rs deleted file mode 100644 index 37a9a8a9..00000000 --- a/tests/highly_distributed_query.rs +++ /dev/null @@ -1,88 +0,0 @@ -#[cfg(all(feature = "integration", test))] -mod tests { - use datafusion::physical_expr::Partitioning; - use datafusion::physical_plan::repartition::RepartitionExec; - use datafusion::physical_plan::{displayable, execute_stream}; - use datafusion_distributed::test_utils::localhost::start_localhost_context; - use datafusion_distributed::test_utils::parquet::register_parquet_tables; - use datafusion_distributed::{ - DefaultSessionBuilder, DistributedPhysicalOptimizerRule, NetworkShuffleExec, - assert_snapshot, display_plan_ascii, - }; - use futures::TryStreamExt; - use std::error::Error; - use std::sync::Arc; - - #[tokio::test] - #[ignore] // This test is flaky - async fn highly_distributed_query() -> Result<(), Box> { - let (ctx, _guard) = start_localhost_context(9, DefaultSessionBuilder).await; - register_parquet_tables(&ctx).await?; - - let df = ctx.sql(r#"SELECT * FROM flights_1m"#).await?; - let physical = df.create_physical_plan().await?; - let physical_str = displayable(physical.as_ref()).indent(true).to_string(); - - let mut physical_distributed = physical.clone(); - for size in [1, 10, 5] { - physical_distributed = Arc::new(NetworkShuffleExec::try_new( - Arc::new(RepartitionExec::try_new( - physical_distributed, - Partitioning::Hash(vec![], size), - )?), - size, - )?); - } - - let physical_distributed = - DistributedPhysicalOptimizerRule::distribute_plan(physical_distributed)?; - let physical_distributed_str = display_plan_ascii(physical_distributed.as_ref()); - - assert_snapshot!(physical_str, - @"DataSourceExec: file_groups={1 group: [[/testdata/flights-1m.parquet]]}, projection=[FL_DATE, DEP_DELAY, ARR_DELAY, AIR_TIME, DISTANCE, DEP_TIME, ARR_TIME], file_type=parquet", - ); - - assert_snapshot!(physical_distributed_str, - @r" - ┌───── DistributedExec ── Tasks: t0:[p0] - │ CoalescePartitionsExec - │ [Stage 3] => NetworkShuffleExec: output_partitions=5, input_tasks=5 - └────────────────────────────────────────────────── - ┌───── Stage 3 ── Tasks: t0:[p0,p1,p2,p3,p4] t1:[p5,p6,p7,p8,p9] t2:[p10,p11,p12,p13,p14] t3:[p15,p16,p17,p18,p19] t4:[p20,p21,p22,p23,p24] - │ RepartitionExec: partitioning=RoundRobinBatch(5), input_partitions=10 - │ [Stage 2] => NetworkShuffleExec: output_partitions=10, input_tasks=10 - └────────────────────────────────────────────────── - ┌───── Stage 2 ── Tasks: t0:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9,p10,p11,p12,p13,p14,p15,p16,p17,p18,p19,p20,p21,p22,p23,p24,p25,p26,p27,p28,p29,p30,p31,p32,p33,p34,p35,p36,p37,p38,p39,p40,p41,p42,p43,p44,p45,p46,p47,p48,p49] t1:[p50,p51,p52,p53,p54,p55,p56,p57,p58,p59,p60,p61,p62,p63,p64,p65,p66,p67,p68,p69,p70,p71,p72,p73,p74,p75,p76,p77,p78,p79,p80,p81,p82,p83,p84,p85,p86,p87,p88,p89,p90,p91,p92,p93,p94,p95,p96,p97,p98,p99] t2:[p100,p101,p102,p103,p104,p105,p106,p107,p108,p109,p110,p111,p112,p113,p114,p115,p116,p117,p118,p119,p120,p121,p122,p123,p124,p125,p126,p127,p128,p129,p130,p131,p132,p133,p134,p135,p136,p137,p138,p139,p140,p141,p142,p143,p144,p145,p146,p147,p148,p149] t3:[p150,p151,p152,p153,p154,p155,p156,p157,p158,p159,p160,p161,p162,p163,p164,p165,p166,p167,p168,p169,p170,p171,p172,p173,p174,p175,p176,p177,p178,p179,p180,p181,p182,p183,p184,p185,p186,p187,p188,p189,p190,p191,p192,p193,p194,p195,p196,p197,p198,p199] t4:[p200,p201,p202,p203,p204,p205,p206,p207,p208,p209,p210,p211,p212,p213,p214,p215,p216,p217,p218,p219,p220,p221,p222,p223,p224,p225,p226,p227,p228,p229,p230,p231,p232,p233,p234,p235,p236,p237,p238,p239,p240,p241,p242,p243,p244,p245,p246,p247,p248,p249] t5:[p250,p251,p252,p253,p254,p255,p256,p257,p258,p259,p260,p261,p262,p263,p264,p265,p266,p267,p268,p269,p270,p271,p272,p273,p274,p275,p276,p277,p278,p279,p280,p281,p282,p283,p284,p285,p286,p287,p288,p289,p290,p291,p292,p293,p294,p295,p296,p297,p298,p299] t6:[p300,p301,p302,p303,p304,p305,p306,p307,p308,p309,p310,p311,p312,p313,p314,p315,p316,p317,p318,p319,p320,p321,p322,p323,p324,p325,p326,p327,p328,p329,p330,p331,p332,p333,p334,p335,p336,p337,p338,p339,p340,p341,p342,p343,p344,p345,p346,p347,p348,p349] t7:[p350,p351,p352,p353,p354,p355,p356,p357,p358,p359,p360,p361,p362,p363,p364,p365,p366,p367,p368,p369,p370,p371,p372,p373,p374,p375,p376,p377,p378,p379,p380,p381,p382,p383,p384,p385,p386,p387,p388,p389,p390,p391,p392,p393,p394,p395,p396,p397,p398,p399] t8:[p400,p401,p402,p403,p404,p405,p406,p407,p408,p409,p410,p411,p412,p413,p414,p415,p416,p417,p418,p419,p420,p421,p422,p423,p424,p425,p426,p427,p428,p429,p430,p431,p432,p433,p434,p435,p436,p437,p438,p439,p440,p441,p442,p443,p444,p445,p446,p447,p448,p449] t9:[p450,p451,p452,p453,p454,p455,p456,p457,p458,p459,p460,p461,p462,p463,p464,p465,p466,p467,p468,p469,p470,p471,p472,p473,p474,p475,p476,p477,p478,p479,p480,p481,p482,p483,p484,p485,p486,p487,p488,p489,p490,p491,p492,p493,p494,p495,p496,p497,p498,p499] - │ RepartitionExec: partitioning=RoundRobinBatch(50), input_partitions=1 - │ [Stage 1] => NetworkShuffleExec: output_partitions=1, input_tasks=1 - └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9] - │ RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=1 - │ DataSourceExec: file_groups={1 group: [[/testdata/flights-1m.parquet]]}, projection=[FL_DATE, DEP_DELAY, ARR_DELAY, AIR_TIME, DISTANCE, DEP_TIME, ARR_TIME], file_type=parquet - └────────────────────────────────────────────────── - ", - ); - - let time = std::time::Instant::now(); - let batches = execute_stream(physical, ctx.task_ctx())? - .try_collect::>() - .await?; - println!("time: {:?}", time.elapsed()); - - let time = std::time::Instant::now(); - let batches_distributed = execute_stream(physical_distributed, ctx.task_ctx())? - .try_collect::>() - .await?; - println!("time: {:?}", time.elapsed()); - - assert_eq!( - batches.iter().map(|v| v.num_rows()).sum::(), - batches_distributed - .iter() - .map(|v| v.num_rows()) - .sum::(), - ); - - Ok(()) - } -} diff --git a/tests/introspection.rs b/tests/introspection.rs new file mode 100644 index 00000000..86201df5 --- /dev/null +++ b/tests/introspection.rs @@ -0,0 +1,72 @@ +#[cfg(all(feature = "integration", test))] +mod tests { + use datafusion::arrow::util::pretty::pretty_format_batches; + use datafusion::physical_plan::execute_stream; + use datafusion_distributed::test_utils::localhost::start_localhost_context; + use datafusion_distributed::test_utils::parquet::register_parquet_tables; + use datafusion_distributed::{DefaultSessionBuilder, assert_snapshot, display_plan_ascii}; + use futures::TryStreamExt; + use std::error::Error; + + #[tokio::test] + async fn distributed_show_columns() -> Result<(), Box> { + let (ctx, _guard, _) = start_localhost_context(3, DefaultSessionBuilder).await; + ctx.state_ref() + .write() + .config_mut() + .options_mut() + .catalog + .information_schema = true; + register_parquet_tables(&ctx).await?; + + let df = ctx.sql(r#"SHOW COLUMNS from weather"#).await?; + let physical_distributed = df.create_physical_plan().await?; + + let physical_distributed_str = display_plan_ascii(physical_distributed.as_ref(), false); + + assert_snapshot!(physical_distributed_str, + @r" + ProjectionExec: expr=[table_catalog@0 as table_catalog, table_schema@1 as table_schema, table_name@2 as table_name, column_name@3 as column_name, data_type@5 as data_type, is_nullable@4 as is_nullable] + FilterExec: table_name@2 = weather + RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=1 + StreamingTableExec: partition_sizes=1, projection=[table_catalog, table_schema, table_name, column_name, is_nullable, data_type] + ", + ); + + let batches_distributed = pretty_format_batches( + &execute_stream(physical_distributed, ctx.task_ctx())? + .try_collect::>() + .await?, + )?; + assert_snapshot!(batches_distributed, @r" + +---------------+--------------+------------+---------------+-----------+-------------+ + | table_catalog | table_schema | table_name | column_name | data_type | is_nullable | + +---------------+--------------+------------+---------------+-----------+-------------+ + | datafusion | public | weather | MinTemp | Float64 | YES | + | datafusion | public | weather | MaxTemp | Float64 | YES | + | datafusion | public | weather | Rainfall | Float64 | YES | + | datafusion | public | weather | Evaporation | Float64 | YES | + | datafusion | public | weather | Sunshine | Utf8View | YES | + | datafusion | public | weather | WindGustDir | Utf8View | YES | + | datafusion | public | weather | WindGustSpeed | Utf8View | YES | + | datafusion | public | weather | WindDir9am | Utf8View | YES | + | datafusion | public | weather | WindDir3pm | Utf8View | YES | + | datafusion | public | weather | WindSpeed9am | Utf8View | YES | + | datafusion | public | weather | WindSpeed3pm | Int64 | YES | + | datafusion | public | weather | Humidity9am | Int64 | YES | + | datafusion | public | weather | Humidity3pm | Int64 | YES | + | datafusion | public | weather | Pressure9am | Float64 | YES | + | datafusion | public | weather | Pressure3pm | Float64 | YES | + | datafusion | public | weather | Cloud9am | Int64 | YES | + | datafusion | public | weather | Cloud3pm | Int64 | YES | + | datafusion | public | weather | Temp9am | Float64 | YES | + | datafusion | public | weather | Temp3pm | Float64 | YES | + | datafusion | public | weather | RainToday | Utf8View | YES | + | datafusion | public | weather | RISK_MM | Float64 | YES | + | datafusion | public | weather | RainTomorrow | Utf8View | YES | + +---------------+--------------+------------+---------------+-----------+-------------+ + "); + + Ok(()) + } +} diff --git a/tests/join.rs b/tests/join.rs new file mode 100644 index 00000000..a72c185f --- /dev/null +++ b/tests/join.rs @@ -0,0 +1,304 @@ +#[cfg(all(feature = "integration", test))] +mod tests { + use arrow::{ + array::RecordBatch, + datatypes::DataType, + util::pretty::{self, pretty_format_batches}, + }; + use datafusion::{ + error::Result, + physical_plan::collect, + prelude::{ParquetReadOptions, SessionContext, col}, + }; + use datafusion_distributed::{ + DefaultSessionBuilder, assert_snapshot, display_plan_ascii, + test_utils::localhost::start_localhost_context, + }; + + fn set_configs(ctx: &mut SessionContext) { + // Preserve hive-style file partitions. + ctx.state_ref() + .write() + .config_mut() + .options_mut() + .optimizer + .preserve_file_partitions = 1; + // Read data from 4 hive-style partitions. + ctx.state_ref() + .write() + .config_mut() + .options_mut() + .execution + .target_partitions = 4; + // Ensure that we use a partitioned hash join. + ctx.state_ref() + .write() + .config_mut() + .options_mut() + .optimizer + .hash_join_single_partition_threshold = 0; + ctx.state_ref() + .write() + .config_mut() + .options_mut() + .optimizer + .hash_join_single_partition_threshold_rows = 0; + } + + async fn register_tables(ctx: &SessionContext) -> Result<()> { + // Register hive-style partitioning for the dim table. + let dim_options = ParquetReadOptions::default() + .table_partition_cols(vec![("d_dkey".to_string(), DataType::Utf8)]); + ctx.register_parquet("dim", "testdata/join/parquet/dim", dim_options) + .await?; + + // Register hive-style partitioning for the fact table. + let fact_options = ParquetReadOptions::default() + .table_partition_cols(vec![("f_dkey".to_string(), DataType::Utf8)]) + .file_sort_order(vec![vec![ + col("f_dkey").sort(true, false), + col("timestamp").sort(true, false), + ]]); + ctx.register_parquet("fact", "testdata/join/parquet/fact", fact_options) + .await?; + Ok(()) + } + + async fn execute_query( + ctx: &SessionContext, + query: &'static str, + ) -> Result<(String, Vec)> { + let df = ctx.sql(query).await?; + let (state, logical_plan) = df.into_parts(); + let physical_plan = state.create_physical_plan(&logical_plan).await?; + let distributed_plan = display_plan_ascii(physical_plan.as_ref(), false); + println!("\n——————— DISTRIBUTED PLAN ———————\n\n{distributed_plan}"); + + let distributed_results = collect(physical_plan, state.task_ctx()).await?; + pretty::print_batches(&distributed_results)?; + Ok((distributed_plan, distributed_results)) + } + + #[tokio::test] + async fn test_join_hive() -> Result<(), Box> { + let query = r#" + SELECT + f.f_dkey, + f.timestamp, + f.value, + d.env, + d.service, + d.host + FROM dim d + INNER JOIN fact f ON d.d_dkey = f.f_dkey + WHERE d.service = 'log' + ORDER BY f_dkey, timestamp + "#; + + // Execute the query using distributed datafusion, 2 workers, + // and hive-style partitioned data. + let (mut distributed_ctx, _guard, _) = + start_localhost_context(2, DefaultSessionBuilder).await; + set_configs(&mut distributed_ctx); + register_tables(&distributed_ctx).await?; + let (distributed_plan, distributed_results) = + execute_query(&distributed_ctx, query).await?; + + // Ensure the distributed plan matches our target plan, registering + // hive-style partitioning and avoiding data-shuffling repartitions. + assert_snapshot!(&distributed_plan, + @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [f_dkey@0 ASC NULLS LAST, timestamp@1 ASC NULLS LAST] + │ [Stage 1] => NetworkCoalesceExec: output_partitions=4, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] + │ ProjectionExec: expr=[f_dkey@5 as f_dkey, timestamp@3 as timestamp, value@4 as value, env@0 as env, service@1 as service, host@2 as host] + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(d_dkey@3, f_dkey@2)], projection=[env@0, service@1, host@2, timestamp@4, value@5, f_dkey@6] + │ FilterExec: service@1 = log + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/join/parquet/dim/d_dkey=A/data0.parquet], [/testdata/join/parquet/dim/d_dkey=B/data0.parquet], [/testdata/join/parquet/dim/d_dkey=C/data0.parquet], [/testdata/join/parquet/dim/d_dkey=D/data0.parquet]]}, projection=[env, service, host, d_dkey], file_type=parquet, predicate=service@1 = log, pruning_predicate=service_null_count@2 != row_count@3 AND service_min@0 <= log AND log <= service_max@1, required_guarantees=[service in (log)] + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/join/parquet/fact/f_dkey=A/data0.parquet], [/testdata/join/parquet/fact/f_dkey=B/data0.parquet], [/testdata/join/parquet/fact/f_dkey=C/data0.parquet], [/testdata/join/parquet/fact/f_dkey=D/data0.parquet]]}, projection=[timestamp, value, f_dkey], output_ordering=[f_dkey@2 ASC NULLS LAST, timestamp@0 ASC NULLS LAST], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + "); + + // Ensure distributed results are correct. + let pretty_results = pretty_format_batches(&distributed_results)?; + assert_snapshot!(pretty_results, + @" + +--------+---------------------+-------+------+---------+--------+ + | f_dkey | timestamp | value | env | service | host | + +--------+---------------------+-------+------+---------+--------+ + | A | 2023-01-01T09:00:00 | 95.5 | dev | log | host-y | + | A | 2023-01-01T09:00:10 | 102.3 | dev | log | host-y | + | A | 2023-01-01T09:00:20 | 98.7 | dev | log | host-y | + | A | 2023-01-01T09:12:20 | 105.1 | dev | log | host-y | + | A | 2023-01-01T09:12:30 | 100.0 | dev | log | host-y | + | A | 2023-01-01T09:12:40 | 150.0 | dev | log | host-y | + | A | 2023-01-01T09:12:50 | 120.8 | dev | log | host-y | + | B | 2023-01-01T09:00:00 | 75.2 | prod | log | host-x | + | B | 2023-01-01T09:00:10 | 82.4 | prod | log | host-x | + | B | 2023-01-01T09:00:20 | 78.9 | prod | log | host-x | + | B | 2023-01-01T09:00:30 | 85.6 | prod | log | host-x | + | B | 2023-01-01T09:12:30 | 80.0 | prod | log | host-x | + | B | 2023-01-01T09:12:40 | 120.0 | prod | log | host-x | + | B | 2023-01-01T09:12:50 | 92.3 | prod | log | host-x | + +--------+---------------------+-------+------+---------+--------+ + "); + + Ok(()) + } + + #[tokio::test] + async fn test_join_agg_hive() -> Result<(), Box> { + let query = r#" + SELECT f_dkey, + date_bin(INTERVAL '30 seconds', timestamp) AS time_bin, + env, + MAX(value) AS max_bin_value + FROM + ( + SELECT + f.f_dkey, + d.env, + d.service, + d.host, + f.timestamp, + f.value + FROM dim d + INNER JOIN fact f ON d.d_dkey = f.f_dkey + WHERE service = 'log' + ) AS j + GROUP BY f_dkey, time_bin, env + ORDER BY f_dkey, time_bin + "#; + + // Execute the query using distributed datafusion, 2 workers, + // and hive-style partitioned data. + let (mut distributed_ctx, _guard, _) = + start_localhost_context(2, DefaultSessionBuilder).await; + set_configs(&mut distributed_ctx); + register_tables(&distributed_ctx).await?; + let (distributed_plan, distributed_results) = + execute_query(&distributed_ctx, query).await?; + + // Ensure the distributed plan matches our target plan, registering + // hive-style partitioning and avoiding data-shuffling repartitions. + assert_snapshot!(&distributed_plan, @r#" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [f_dkey@0 ASC NULLS LAST, time_bin@1 ASC NULLS LAST] + │ [Stage 1] => NetworkCoalesceExec: output_partitions=4, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] + │ ProjectionExec: expr=[f_dkey@0 as f_dkey, date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 30000000000 }"),j.timestamp)@1 as time_bin, env@2 as env, max(j.value)@3 as max_bin_value] + │ AggregateExec: mode=SinglePartitioned, gby=[f_dkey@0 as f_dkey, date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 30000000000 }, timestamp@2) as date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 30000000000 }"),j.timestamp), env@1 as env], aggr=[max(j.value)], ordering_mode=PartiallySorted([0, 1]) + │ ProjectionExec: expr=[f_dkey@3 as f_dkey, env@0 as env, timestamp@1 as timestamp, value@2 as value] + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(d_dkey@1, f_dkey@2)], projection=[env@0, timestamp@2, value@3, f_dkey@4] + │ FilterExec: service@1 = log, projection=[env@0, d_dkey@2] + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/join/parquet/dim/d_dkey=A/data0.parquet], [/testdata/join/parquet/dim/d_dkey=B/data0.parquet], [/testdata/join/parquet/dim/d_dkey=C/data0.parquet], [/testdata/join/parquet/dim/d_dkey=D/data0.parquet]]}, projection=[env, service, d_dkey], file_type=parquet, predicate=service@1 = log, pruning_predicate=service_null_count@2 != row_count@3 AND service_min@0 <= log AND log <= service_max@1, required_guarantees=[service in (log)] + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/join/parquet/fact/f_dkey=A/data0.parquet], [/testdata/join/parquet/fact/f_dkey=B/data0.parquet], [/testdata/join/parquet/fact/f_dkey=C/data0.parquet], [/testdata/join/parquet/fact/f_dkey=D/data0.parquet]]}, projection=[timestamp, value, f_dkey], output_ordering=[f_dkey@2 ASC NULLS LAST, timestamp@0 ASC NULLS LAST], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + "#); + + // Ensure distributed results are correct. + let pretty_results = pretty_format_batches(&distributed_results)?; + assert_snapshot!(pretty_results, @" + +--------+---------------------+------+---------------+ + | f_dkey | time_bin | env | max_bin_value | + +--------+---------------------+------+---------------+ + | A | 2023-01-01T09:00:00 | dev | 102.3 | + | A | 2023-01-01T09:12:00 | dev | 105.1 | + | A | 2023-01-01T09:12:30 | dev | 150.0 | + | B | 2023-01-01T09:00:00 | prod | 82.4 | + | B | 2023-01-01T09:00:30 | prod | 85.6 | + | B | 2023-01-01T09:12:30 | prod | 120.0 | + +--------+---------------------+------+---------------+ + "); + + Ok(()) + } + + #[tokio::test] + async fn test_join_time_space_agg_hive() -> Result<(), Box> { + let query = r#" + SELECT env, time_bin, AVG(max_bin_value) AS avg_max_value + FROM + ( + SELECT f_dkey, + date_bin(INTERVAL '30 seconds', timestamp) AS time_bin, + env, + MAX(value) AS max_bin_value + FROM + ( + SELECT + f.f_dkey, + d.env, + d.service, + d.host, + f.timestamp, + f.value + FROM dim d + INNER JOIN fact f ON d.d_dkey = f.f_dkey + WHERE service = 'log' + ) AS j + GROUP BY f_dkey, time_bin, env + ) AS a + GROUP BY env, time_bin + ORDER BY env, time_bin + "#; + + // Execute the query using distributed datafusion, 2 workers, + // and hive-style partitioned data. + let (mut distributed_ctx, _guard, _) = + start_localhost_context(2, DefaultSessionBuilder).await; + set_configs(&mut distributed_ctx); + register_tables(&distributed_ctx).await?; + let (distributed_plan, distributed_results) = + execute_query(&distributed_ctx, query).await?; + + // Ensure the distributed plan matches our target plan, registering + // hive-style partitioning and avoiding data-shuffling repartitions. + assert_snapshot!(&distributed_plan, @r#" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [env@0 ASC NULLS LAST, time_bin@1 ASC NULLS LAST] + │ SortExec: expr=[env@0 ASC NULLS LAST, time_bin@1 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[env@0 as env, time_bin@1 as time_bin, avg(a.max_bin_value)@2 as avg_max_value] + │ AggregateExec: mode=FinalPartitioned, gby=[env@0 as env, time_bin@1 as time_bin], aggr=[avg(a.max_bin_value)] + │ [Stage 1] => NetworkShuffleExec: output_partitions=4, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p3] t1:[p0..p3] + │ RepartitionExec: partitioning=Hash([env@0, time_bin@1], 4), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[env@1 as env, time_bin@0 as time_bin], aggr=[avg(a.max_bin_value)] + │ ProjectionExec: expr=[date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 30000000000 }"),j.timestamp)@1 as time_bin, env@2 as env, max(j.value)@3 as max_bin_value] + │ AggregateExec: mode=SinglePartitioned, gby=[f_dkey@0 as f_dkey, date_bin(IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 30000000000 }, timestamp@2) as date_bin(IntervalMonthDayNano("IntervalMonthDayNano { months: 0, days: 0, nanoseconds: 30000000000 }"),j.timestamp), env@1 as env], aggr=[max(j.value)], ordering_mode=PartiallySorted([0, 1]) + │ ProjectionExec: expr=[f_dkey@3 as f_dkey, env@0 as env, timestamp@1 as timestamp, value@2 as value] + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(d_dkey@1, f_dkey@2)], projection=[env@0, timestamp@2, value@3, f_dkey@4] + │ FilterExec: service@1 = log, projection=[env@0, d_dkey@2] + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/join/parquet/dim/d_dkey=A/data0.parquet], [/testdata/join/parquet/dim/d_dkey=B/data0.parquet], [/testdata/join/parquet/dim/d_dkey=C/data0.parquet], [/testdata/join/parquet/dim/d_dkey=D/data0.parquet]]}, projection=[env, service, d_dkey], file_type=parquet, predicate=service@1 = log, pruning_predicate=service_null_count@2 != row_count@3 AND service_min@0 <= log AND log <= service_max@1, required_guarantees=[service in (log)] + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/join/parquet/fact/f_dkey=A/data0.parquet], [/testdata/join/parquet/fact/f_dkey=B/data0.parquet], [/testdata/join/parquet/fact/f_dkey=C/data0.parquet], [/testdata/join/parquet/fact/f_dkey=D/data0.parquet]]}, projection=[timestamp, value, f_dkey], output_ordering=[f_dkey@2 ASC NULLS LAST, timestamp@0 ASC NULLS LAST], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + "#); + + // Ensure distributed results are correct. + let pretty_results = pretty_format_batches(&distributed_results)?; + assert_snapshot!(pretty_results, @" + +------+---------------------+---------------+ + | env | time_bin | avg_max_value | + +------+---------------------+---------------+ + | dev | 2023-01-01T09:00:00 | 102.3 | + | dev | 2023-01-01T09:12:00 | 105.1 | + | dev | 2023-01-01T09:12:30 | 150.0 | + | prod | 2023-01-01T09:00:00 | 82.4 | + | prod | 2023-01-01T09:00:30 | 85.6 | + | prod | 2023-01-01T09:12:30 | 120.0 | + +------+---------------------+---------------+ + "); + + Ok(()) + } +} diff --git a/tests/metrics_collection.rs b/tests/metrics_collection.rs new file mode 100644 index 00000000..d7694539 --- /dev/null +++ b/tests/metrics_collection.rs @@ -0,0 +1,269 @@ +#[cfg(all(feature = "integration", test))] +mod tests { + use datafusion::catalog::memory::DataSourceExec; + use datafusion::common::assert_not_contains; + use datafusion::common::tree_node::{Transformed, TreeNode, TreeNodeRecursion}; + use datafusion::physical_plan::display::DisplayableExecutionPlan; + use datafusion::physical_plan::{ExecutionPlan, execute_stream}; + use datafusion::prelude::SessionContext; + use datafusion_distributed::test_utils::localhost::start_localhost_context; + use datafusion_distributed::test_utils::parquet::register_parquet_tables; + use datafusion_distributed::{ + DefaultSessionBuilder, DistributedMetricsFormat, NetworkCoalesceExec, NetworkShuffleExec, + display_plan_ascii, rewrite_distributed_plan_with_metrics, + }; + use futures::TryStreamExt; + use std::sync::Arc; + use test_case::test_case; + + #[test_case(DistributedMetricsFormat::Aggregated ; "aggregated_metrics")] + #[test_case(DistributedMetricsFormat::PerTask ; "per_task_metrics")] + #[tokio::test] + async fn test_metrics_collection_in_aggregation( + format: DistributedMetricsFormat, + ) -> Result<(), Box> { + let (d_ctx, _guard, _) = start_localhost_context(3, DefaultSessionBuilder).await; + + let query = + r#"SELECT count(*), "RainToday" FROM weather GROUP BY "RainToday" ORDER BY count(*)"#; + + let s_ctx = SessionContext::default(); + let (s_physical, mut d_physical) = execute(&s_ctx, &d_ctx, query).await?; + d_physical = rewrite_distributed_plan_with_metrics(d_physical.clone(), format)?; + println!("{}", display_plan_ascii(s_physical.as_ref(), true)); + println!("{}", display_plan_ascii(d_physical.as_ref(), true)); + + assert_metrics_equal::( + ["output_rows", "output_bytes"], + &s_physical, + &d_physical, + 0, + ); + + Ok(()) + } + + #[test_case(DistributedMetricsFormat::Aggregated ; "aggregated_metrics")] + #[test_case(DistributedMetricsFormat::PerTask ; "per_task_metrics")] + #[tokio::test] + async fn test_metrics_collection_in_join( + format: DistributedMetricsFormat, + ) -> Result<(), Box> { + let (d_ctx, _guard, _) = start_localhost_context(3, DefaultSessionBuilder).await; + + let query = r#" + WITH a AS ( + SELECT + AVG("MinTemp") as "MinTemp", + "RainTomorrow" + FROM weather + WHERE "RainToday" = 'Yes' + GROUP BY "RainTomorrow" + ), b AS ( + SELECT + AVG("MaxTemp") as "MaxTemp", + "RainTomorrow" + FROM weather + WHERE "RainToday" = 'No' + GROUP BY "RainTomorrow" + ) + SELECT + a."MinTemp", + b."MaxTemp" + FROM a + LEFT JOIN b + ON a."RainTomorrow" = b."RainTomorrow" + "#; + + let s_ctx = SessionContext::default(); + let (s_physical, mut d_physical) = execute(&s_ctx, &d_ctx, query).await?; + d_physical = rewrite_distributed_plan_with_metrics(d_physical.clone(), format)?; + println!("{}", display_plan_ascii(s_physical.as_ref(), true)); + println!("{}", display_plan_ascii(d_physical.as_ref(), true)); + + for data_source_index in 0..2 { + assert_metrics_equal::( + ["output_rows", "output_bytes"], + &s_physical, + &d_physical, + data_source_index, + ); + } + + Ok(()) + } + + #[test_case(DistributedMetricsFormat::Aggregated ; "aggregated_metrics")] + #[test_case(DistributedMetricsFormat::PerTask ; "per_task_metrics")] + #[tokio::test] + async fn test_metrics_collection_in_union( + format: DistributedMetricsFormat, + ) -> Result<(), Box> { + let (d_ctx, _guard, _) = start_localhost_context(3, DefaultSessionBuilder).await; + + let query = r#" + SELECT "MinTemp", "RainToday" FROM weather WHERE "MinTemp" > 10.0 + UNION ALL + SELECT "MaxTemp", "RainToday" FROM weather WHERE "MaxTemp" < 30.0 + UNION ALL + SELECT "Temp9am", "RainToday" FROM weather WHERE "Temp9am" > 15.0 + UNION ALL + SELECT "Temp3pm", "RainToday" FROM weather WHERE "Temp3pm" < 25.0 + UNION ALL + SELECT "Rainfall", "RainToday" FROM weather WHERE "Rainfall" > 5.0 + "#; + + let s_ctx = SessionContext::default(); + let (s_physical, mut d_physical) = execute(&s_ctx, &d_ctx, query).await?; + + d_physical = rewrite_distributed_plan_with_metrics(d_physical.clone(), format)?; + println!("{}", display_plan_ascii(s_physical.as_ref(), true)); + println!("{}", display_plan_ascii(d_physical.as_ref(), true)); + + for data_source_index in 0..5 { + assert_metrics_equal::( + ["output_rows", "output_bytes"], + &s_physical, + &d_physical, + data_source_index, + ); + } + Ok(()) + } + + #[test_case(DistributedMetricsFormat::Aggregated ; "aggregated_metrics")] + #[test_case(DistributedMetricsFormat::PerTask ; "per_task_metrics")] + #[tokio::test] + async fn test_metric_collection_network_boundaries( + format: DistributedMetricsFormat, + ) -> Result<(), Box> { + let (d_ctx, _guard, _) = start_localhost_context(3, DefaultSessionBuilder).await; + + let query = + r#"SELECT count(*), "RainToday" FROM weather GROUP BY "RainToday" ORDER BY count(*)"#; + + let s_ctx = SessionContext::default(); + let (s_physical, mut d_physical) = execute(&s_ctx, &d_ctx, query).await?; + d_physical = rewrite_distributed_plan_with_metrics(d_physical.clone(), format)?; + println!("{}", display_plan_ascii(s_physical.as_ref(), true)); + println!("{}", display_plan_ascii(d_physical.as_ref(), true)); + + let value = node_metrics::(&d_physical, "bytes_transferred", 1); + assert!(value > 100); + let value = node_metrics::(&d_physical, "max_mem_used", 1); + assert!(value > 100); + let value = node_metrics::(&d_physical, "elapsed_compute", 1); + assert!(value > 100); + let value = node_metrics::(&d_physical, "network_latency_avg", 1); + assert!(value > 0); + + let value = node_metrics::(&d_physical, "bytes_transferred", 1); + assert!(value > 100); + let value = node_metrics::(&d_physical, "max_mem_used", 1); + assert!(value > 100); + let value = node_metrics::(&d_physical, "elapsed_compute", 1); + assert!(value > 100); + let value = node_metrics::(&d_physical, "network_latency_avg", 1); + assert!(value > 0); + + Ok(()) + } + + #[tokio::test] + async fn test_metric_collection_display_all_have_metrics() + -> Result<(), Box> { + let format = DistributedMetricsFormat::PerTask; + let (d_ctx, _guard, _) = start_localhost_context(3, DefaultSessionBuilder).await; + + let query = + r#"SELECT count(*), "RainToday" FROM weather GROUP BY "RainToday" ORDER BY count(*)"#; + + let s_ctx = SessionContext::default(); + let (_, mut d_physical) = execute(&s_ctx, &d_ctx, query).await?; + d_physical = rewrite_distributed_plan_with_metrics(d_physical.clone(), format)?; + + let display = + DisplayableExecutionPlan::with_metrics(d_physical.children().swap_remove(0).as_ref()) + .indent(true) + .to_string(); + assert_not_contains!(display, "metrics=[]"); + + let display = display_plan_ascii(d_physical.as_ref(), true); + assert_not_contains!(display, "metrics=[]"); + + Ok(()) + } + + /// Looks for an [ExecutionPlan] that matches the provided type parameter `T` in + /// both root nodes and compares its metrics. + /// There might be more than one, so `index` determines which one is compared. + /// + /// If the two root nodes contain a child T with different metrics, the assertion fails. + fn assert_metrics_equal( + names: impl IntoIterator, + one: &Arc, + other: &Arc, + index: usize, + ) { + for name in names.into_iter() { + let one_metric = node_metrics::(one, name, index); + let other_metric = node_metrics::(other, name, index); + assert_eq!(one_metric, other_metric); + } + } + + async fn execute( + s_ctx: &SessionContext, + d_ctx: &SessionContext, + query: &str, + ) -> Result<(Arc, Arc), Box> { + register_parquet_tables(s_ctx).await?; + register_parquet_tables(d_ctx).await?; + + let s_df = s_ctx.sql(query).await?; + let s_physical = s_df.create_physical_plan().await?; + execute_stream(s_physical.clone(), s_ctx.task_ctx())? + .try_collect::>() + .await?; + + let d_df = d_ctx.sql(query).await?; + let d_physical = d_df.create_physical_plan().await?; + execute_stream(d_physical.clone(), d_ctx.task_ctx())? + .try_collect::>() + .await?; + + Ok((s_physical, d_physical)) + } + + fn node_metrics( + plan: &Arc, + metric_name: &str, + mut index: usize, + ) -> usize { + let mut metrics = None; + plan.clone() + .transform_down(|plan| { + if plan.name() == T::static_name() { + metrics = plan.metrics(); + if index == 0 { + return Ok(Transformed::new(plan, false, TreeNodeRecursion::Stop)); + } + index -= 1; + } + Ok(Transformed::no(plan)) + }) + .unwrap(); + let metrics = metrics + .unwrap_or_else(|| panic!("Could not find metrics for plan {}", T::static_name())); + let summed = metrics + .iter() + .filter(|v| v.value().name().starts_with(metric_name)) + .map(|v| v.value().as_usize()) + .sum(); + assert!( + summed > 0, + "Sum of metric values is 0. Either the metric {metric_name} is not present or the test is too trivial" + ); + summed + } +} diff --git a/tests/passthrough_headers.rs b/tests/passthrough_headers.rs new file mode 100644 index 00000000..926dece3 --- /dev/null +++ b/tests/passthrough_headers.rs @@ -0,0 +1,51 @@ +#[cfg(all(feature = "integration", test))] +mod tests { + use datafusion::common::exec_datafusion_err; + use datafusion::error::DataFusionError; + use datafusion::execution::SessionState; + use datafusion::physical_plan::execute_stream; + use datafusion_distributed::test_utils::localhost::start_localhost_context; + use datafusion_distributed::test_utils::parquet::register_parquet_tables; + use datafusion_distributed::{DistributedExt, WorkerQueryContext, display_plan_ascii}; + use futures::TryStreamExt; + use http::{HeaderMap, HeaderName, HeaderValue}; + + async fn build_state(ctx: WorkerQueryContext) -> Result { + Ok(ctx + .builder + // by erroring here if the header is not present, we make sure that the header gets + // propagated across stages. If the header was not propagated across stages, it would + // error right here before even forming a SessionState inside a Worker. + .with_distributed_passthrough_headers(HeaderMap::from_iter([( + HeaderName::from_static("foo"), + ctx.headers + .get("foo") + .cloned() + .ok_or_else(|| exec_datafusion_err!("Missing header foo"))?, + )]))? + .build()) + } + + #[tokio::test] + async fn custom_header() -> Result<(), Box> { + let (mut ctx, _guard, _) = start_localhost_context(3, build_state).await; + ctx.set_distributed_passthrough_headers(HeaderMap::from_iter([( + HeaderName::from_static("foo"), + HeaderValue::from_static("bar"), + )]))?; + + let query = r#"SELECT DISTINCT "RainToday", "WindGustDir" FROM weather"#; + + register_parquet_tables(&ctx).await?; + let df = ctx.sql(query).await?; + let plan = df.create_physical_plan().await?; + let display = display_plan_ascii(plan.as_ref(), false); + println!("{display}"); + + let stream = execute_stream(plan, ctx.task_ctx())?; + // It should not fail. + stream.try_collect::>().await?; + + Ok(()) + } +} diff --git a/tests/stateful_data_cleanup.rs b/tests/stateful_data_cleanup.rs new file mode 100644 index 00000000..e294e4c3 --- /dev/null +++ b/tests/stateful_data_cleanup.rs @@ -0,0 +1,103 @@ +#[cfg(all(feature = "integration", feature = "tpch", test))] +mod tests { + use datafusion::common::instant::Instant; + use datafusion::error::Result; + use datafusion::physical_plan::execute_stream; + use datafusion::prelude::SessionContext; + use datafusion_distributed::test_utils::localhost::start_localhost_context; + use datafusion_distributed::test_utils::{benchmarks_common, tpch}; + use datafusion_distributed::{DefaultSessionBuilder, DistributedExt}; + use futures::TryStreamExt; + use std::fs; + use std::path::Path; + use std::time::Duration; + use tokio::sync::OnceCell; + use tokio::time::timeout; + + const NUM_WORKERS: usize = 4; + const TPCH_SCALE_FACTOR: f64 = 1.0; + const TPCH_DATA_PARTS: i32 = 16; + const CARDINALITY_TASK_COUNT_FACTOR: f64 = 1.0; + + #[tokio::test] + async fn no_pending_tasks_if_query_completes() -> Result<()> { + let (d_ctx, _guard, workers) = + start_localhost_context(NUM_WORKERS, DefaultSessionBuilder).await; + run_tpch_query(d_ctx, "q1").await?; + + for (i, worker) in workers.iter().enumerate() { + let tasks_running = worker.tasks_running().await; + assert_eq!( + tasks_running, 0, + "Expected Worker {i} to have 0 tasks running, but got {tasks_running}" + ) + } + + Ok(()) + } + + #[tokio::test(flavor = "multi_thread")] + async fn no_pending_tasks_if_query_aborts() -> Result<()> { + let (d_ctx, _guard, workers) = + start_localhost_context(NUM_WORKERS, DefaultSessionBuilder).await; + + let _ = timeout(Duration::from_millis(100), run_tpch_query(d_ctx, "q1")).await; + + let start = Instant::now(); + let mut tasks_running = 0; + while start.elapsed() < Duration::from_secs(5) { + tokio::time::sleep(Duration::from_millis(100)).await; + tasks_running = 0; + for worker in &workers { + tasks_running += worker.tasks_running().await; + } + if tasks_running == 0 { + return Ok(()); + } + } + + assert_eq!( + tasks_running, 0, + "Expected to have 0 tasks running, but got {tasks_running}" + ); + + Ok(()) + } + + async fn run_tpch_query(d_ctx: SessionContext, query_id: &str) -> Result<()> { + let data_dir = ensure_tpch_data(TPCH_SCALE_FACTOR, TPCH_DATA_PARTS).await; + + let query_sql = tpch::get_query(query_id)?; + + let d_ctx = d_ctx + .with_distributed_cardinality_effect_task_scale_factor(CARDINALITY_TASK_COUNT_FACTOR)?; + + benchmarks_common::register_tables(&d_ctx, &data_dir).await?; + + let df = d_ctx.sql(&query_sql).await?; + let task_ctx = d_ctx.task_ctx(); + let plan = df.create_physical_plan().await?; + + let stream = execute_stream(plan.clone(), task_ctx)?; + stream.try_collect::>().await?; + + Ok(()) + } + + // OnceCell to ensure TPCH tables are generated only once for tests + static INIT_TEST_TPCH_TABLES: OnceCell<()> = OnceCell::const_new(); + + // ensure_tpch_data initializes the TPCH data on disk. + pub async fn ensure_tpch_data(sf: f64, parts: i32) -> std::path::PathBuf { + let data_dir = Path::new(env!("CARGO_MANIFEST_DIR")) + .join(format!("testdata/tpch/stateful_data_cleanup_sf{sf}")); + INIT_TEST_TPCH_TABLES + .get_or_init(|| async { + if !fs::exists(&data_dir).unwrap() { + tpch::generate_tpch_data(&data_dir, sf, parts); + } + }) + .await; + data_dir + } +} diff --git a/tests/stateful_execution_plan.rs b/tests/stateful_execution_plan.rs index 22e01bcd..e93763a0 100644 --- a/tests/stateful_execution_plan.rs +++ b/tests/stateful_execution_plan.rs @@ -1,34 +1,21 @@ #[cfg(all(feature = "integration", test))] mod tests { - use datafusion::arrow::array::Int64Array; - use datafusion::arrow::compute::SortOptions; - use datafusion::arrow::datatypes::{DataType, Field, Schema}; - use datafusion::arrow::record_batch::RecordBatch; use datafusion::arrow::util::pretty::pretty_format_batches; - use datafusion::common::runtime::SpawnedTask; + use datafusion::common::tree_node::{Transformed, TreeNode}; use datafusion::error::DataFusionError; - use datafusion::execution::{ - FunctionRegistry, SendableRecordBatchStream, SessionState, SessionStateBuilder, TaskContext, - }; - use datafusion::logical_expr::Operator; - use datafusion::physical_expr::expressions::{BinaryExpr, col, lit}; - use datafusion::physical_expr::{ - EquivalenceProperties, LexOrdering, Partitioning, PhysicalSortExpr, - }; + use datafusion::execution::{SendableRecordBatchStream, SessionState, TaskContext}; + use datafusion::physical_expr::EquivalenceProperties; use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType}; - use datafusion::physical_plan::filter::FilterExec; - use datafusion::physical_plan::repartition::RepartitionExec; - use datafusion::physical_plan::sorts::sort::SortExec; use datafusion::physical_plan::stream::RecordBatchStreamAdapter; use datafusion::physical_plan::{ - DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties, execute_stream, + DisplayAs, DisplayFormatType, ExecutionPlan, ExecutionPlanProperties, PlanProperties, + execute_stream, }; use datafusion_distributed::test_utils::localhost::start_localhost_context; + use datafusion_distributed::test_utils::parquet::register_parquet_tables; use datafusion_distributed::{ - DistributedExt, DistributedSessionBuilderContext, PartitionIsolatorExec, assert_snapshot, - display_plan_ascii, + DistributedExt, WorkerQueryContext, assert_snapshot, display_plan_ascii, }; - use datafusion_distributed::{DistributedPhysicalOptimizerRule, NetworkShuffleExec}; use datafusion_proto::physical_plan::PhysicalExtensionCodec; use datafusion_proto::protobuf::proto_error; use futures::TryStreamExt; @@ -36,163 +23,109 @@ mod tests { use std::any::Any; use std::fmt::Formatter; use std::sync::{Arc, RwLock}; - use std::time::Duration; - use tokio::sync::mpsc; + use tokio::task::JoinHandle; use tokio_stream::StreamExt; use tokio_stream::wrappers::ReceiverStream; - // This test proves that execution nodes do not get early dropped in the ArrowFlightEndpoint - // when all the partitions start being consumed. + // This test proves that execution nodes do not get early dropped in the Worker when all the + // partitions start being consumed. // - // It uses a StatefulInt64ListExec custom node whose execution depends on it not being dropped. - // If for some reason ArrowFlightEndpoint drops the node before the stream ends, this test - // will fail. + // It uses a StatefulPassThroughExec custom node whose execution depends on it not being dropped. + // The node spawns a background task that collects data from its child DataSourceExec. + // If the Worker drops the node before the stream ends, this test will fail. #[tokio::test] async fn stateful_execution_plan() -> Result<(), Box> { - async fn build_state( - ctx: DistributedSessionBuilderContext, - ) -> Result { - Ok(SessionStateBuilder::new() - .with_runtime_env(ctx.runtime_env) - .with_default_features() - .with_distributed_user_codec(Int64ListExecCodec) + async fn build_state(ctx: WorkerQueryContext) -> Result { + Ok(ctx + .builder + .with_distributed_user_codec(PassThroughExecCodec) .build()) } - let (ctx, _guard) = start_localhost_context(3, build_state).await; + let (mut ctx_distributed, _guard, _) = start_localhost_context(3, build_state).await; + ctx_distributed.set_distributed_user_codec(PassThroughExecCodec); + register_parquet_tables(&ctx_distributed).await?; - let distributed_plan = build_plan()?; - let distributed_plan = DistributedPhysicalOptimizerRule::distribute_plan(distributed_plan)?; + let query = r#"SELECT "MinTemp", "RainToday" FROM weather WHERE "MinTemp" > 20.0 ORDER BY "MinTemp" DESC"#; - assert_snapshot!(display_plan_ascii(distributed_plan.as_ref()), @r" - ┌───── DistributedExec ── Tasks: t0:[p0] - │ SortExec: expr=[numbers@0 DESC NULLS LAST], preserve_partitioning=[false] - │ RepartitionExec: partitioning=RoundRobinBatch(1), input_partitions=10 - │ [Stage 2] => NetworkShuffleExec: output_partitions=10, input_tasks=10 - └────────────────────────────────────────────────── - ┌───── Stage 2 ── Tasks: t0:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9] t1:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9] t2:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9] t3:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9] t4:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9] t5:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9] t6:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9] t7:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9] t8:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9] t9:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9] - │ RepartitionExec: partitioning=Hash([], 10), input_partitions=1 - │ SortExec: expr=[numbers@0 DESC NULLS LAST], preserve_partitioning=[false] - │ [Stage 1] => NetworkShuffleExec: output_partitions=1, input_tasks=1 - └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9] - │ RepartitionExec: partitioning=Hash([numbers@0], 10), input_partitions=1 - │ FilterExec: numbers@0 > 1 - │ StatefulInt64ListExec: length=6 - └────────────────────────────────────────────────── - "); - - let stream = execute_stream(distributed_plan, ctx.task_ctx())?; - let batches_distributed = stream.try_collect::>().await?; + let df_distributed = ctx_distributed.sql(query).await?; + let plan = df_distributed.create_physical_plan().await?; - assert_snapshot!(pretty_format_batches(&batches_distributed).unwrap(), @r" - +---------+ - | numbers | - +---------+ - | 6 | - | 5 | - | 4 | - | 3 | - | 2 | - +---------+ - "); - Ok(()) - } - - fn build_plan() -> Result, DataFusionError> { - let mut plan: Arc = - Arc::new(StatefulInt64ListExec::new(vec![1, 2, 3, 4, 5, 6])); - - plan = Arc::new(PartitionIsolatorExec::new(plan)); - - plan = Arc::new(FilterExec::try_new( - Arc::new(BinaryExpr::new( - col("numbers", &plan.schema())?, - Operator::Gt, - lit(1i64), - )), - plan, - )?); - - plan = Arc::new(NetworkShuffleExec::try_new( - Arc::new(RepartitionExec::try_new( - Arc::clone(&plan), - Partitioning::Hash(vec![col("numbers", &plan.schema())?], 1), - )?), - 10, - )?); + let transformed = plan.transform_up(|plan| { + if plan.children().is_empty() { + return Ok(Transformed::yes(Arc::new(StatefulPassThroughExec::new( + plan, + )))); + } + Ok(Transformed::no(plan)) + })?; + let plan = transformed.data; - plan = Arc::new(SortExec::new( - LexOrdering::new(vec![PhysicalSortExpr::new( - col("numbers", &plan.schema())?, - SortOptions::new(true, false), - )]) - .unwrap(), - plan, - )); + let plan_str = display_plan_ascii(plan.as_ref(), false); - plan = Arc::new(NetworkShuffleExec::try_new( - Arc::new(RepartitionExec::try_new( - plan, - Partitioning::Hash(vec![], 10), - )?), - 10, - )?); + assert_snapshot!(plan_str, + @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [MinTemp@0 DESC] + │ [Stage 1] => NetworkCoalesceExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0] t1:[p1] t2:[p2] + │ SortExec: expr=[MinTemp@0 DESC], preserve_partitioning=[true] + │ FilterExec: MinTemp@0 > 20 + │ PartitionIsolatorExec: t0:[p0,__,__] t1:[__,p0,__] t2:[__,__,p0] + │ StatefulPassThroughExec + │ DataSourceExec: file_groups={3 groups: [[/testdata/weather/result-000000.parquet], [/testdata/weather/result-000001.parquet], [/testdata/weather/result-000002.parquet]]}, projection=[MinTemp, RainToday], file_type=parquet, predicate=MinTemp@0 > 20, pruning_predicate=MinTemp_null_count@1 != row_count@2 AND MinTemp_max@0 > 20, required_guarantees=[] + └────────────────────────────────────────────────── + ", + ); - plan = Arc::new(RepartitionExec::try_new( - plan, - Partitioning::RoundRobinBatch(1), - )?); + let batches_distributed = pretty_format_batches( + &execute_stream(plan, ctx_distributed.task_ctx())? + .try_collect::>() + .await?, + )?; - plan = Arc::new(SortExec::new( - LexOrdering::new(vec![PhysicalSortExpr::new( - col("numbers", &plan.schema())?, - SortOptions::new(true, false), - )]) - .unwrap(), - plan, - )); + // Verify that the stateful execution completes successfully + assert!(!batches_distributed.to_string().is_empty()); - Ok(plan) + Ok(()) } + /// A stateful execution plan that wraps a child and spawns a background task + /// to manage the stream collection. This tests that the node doesn't get + /// dropped prematurely during distributed execution. #[derive(Debug)] - pub struct StatefulInt64ListExec { + pub struct StatefulPassThroughExec { plan_properties: PlanProperties, - numbers: Vec, - task: RwLock>>, - tx: RwLock>>, - rx: RwLock>>, + child: Arc, + task: RwLock>>, } - impl StatefulInt64ListExec { - fn new(numbers: Vec) -> Self { - let schema = Schema::new(vec![Field::new("numbers", DataType::Int64, false)]); - let (tx, rx) = mpsc::channel(10); + impl StatefulPassThroughExec { + fn new(child: Arc) -> Self { + let plan_properties = PlanProperties::new( + EquivalenceProperties::new(child.schema()), + child.output_partitioning().clone(), + EmissionType::Incremental, + Boundedness::Bounded, + ); Self { - numbers, - plan_properties: PlanProperties::new( - EquivalenceProperties::new(Arc::new(schema)), - Partitioning::UnknownPartitioning(1), - EmissionType::Incremental, - Boundedness::Bounded, - ), + plan_properties, + child, task: RwLock::new(None), - tx: RwLock::new(Some(tx)), - rx: RwLock::new(Some(rx)), } } } - impl DisplayAs for StatefulInt64ListExec { + impl DisplayAs for StatefulPassThroughExec { fn fmt_as(&self, _: DisplayFormatType, f: &mut Formatter) -> std::fmt::Result { - write!(f, "StatefulInt64ListExec: length={:?}", self.numbers.len()) + write!(f, "StatefulPassThroughExec") } } - impl ExecutionPlan for StatefulInt64ListExec { + impl ExecutionPlan for StatefulPassThroughExec { fn name(&self) -> &str { - "StatefulInt64ListExec" + "StatefulPassThroughExec" } fn as_any(&self) -> &dyn Any { @@ -204,68 +137,68 @@ mod tests { } fn children(&self) -> Vec<&Arc> { - vec![] + vec![&self.child] } fn with_new_children( self: Arc, - _: Vec>, + children: Vec>, ) -> datafusion::common::Result> { - Ok(self) + Ok(Arc::new(StatefulPassThroughExec::new(children[0].clone()))) } fn execute( &self, - _: usize, - _: Arc, + partition: usize, + context: Arc, ) -> datafusion::common::Result { - if let Some(tx) = self.tx.write().unwrap().take() { - let numbers = self.numbers.clone(); - self.task - .write() - .unwrap() - .replace(SpawnedTask::spawn(async move { - for n in numbers { - tx.send(n).await.unwrap(); - tokio::time::sleep(Duration::from_millis(100)).await; - } - })); - } - - let rx = self.rx.write().unwrap().take().unwrap(); - let schema = self.schema(); - - let stream = ReceiverStream::new(rx).map(move |v| { - RecordBatch::try_new(schema.clone(), vec![Arc::new(Int64Array::from(vec![v]))]) - .map_err(DataFusionError::from) + let (tx, rx) = tokio::sync::mpsc::channel(1); + // Spawn a background task to demonstrate stateful behavior + let mut stream = self.child.execute(partition, context)?; + + let handle = tokio::spawn(async move { + // Simulate some background work + while let Some(batch) = stream.next().await { + if tx.send(batch).await.is_err() { + return; + } + } }); + self.task.write().unwrap().replace(handle); Ok(Box::pin(RecordBatchStreamAdapter::new( - self.schema().clone(), - stream, + self.schema(), + ReceiverStream::new(rx), ))) } } #[derive(Debug)] - struct Int64ListExecCodec; + struct PassThroughExecCodec; #[derive(Clone, PartialEq, ::prost::Message)] - struct Int64ListExecProto { - #[prost(message, repeated, tag = "1")] - numbers: Vec, + struct PassThroughExecProto { + // Empty - we'll handle the child through normal codec mechanisms } - impl PhysicalExtensionCodec for Int64ListExecCodec { + impl PhysicalExtensionCodec for PassThroughExecCodec { fn try_decode( &self, buf: &[u8], - _: &[Arc], - _registry: &dyn FunctionRegistry, + inputs: &[Arc], + _ctx: &TaskContext, ) -> datafusion::common::Result> { - let node = - Int64ListExecProto::decode(buf).map_err(|err| proto_error(format!("{err}")))?; - Ok(Arc::new(StatefulInt64ListExec::new(node.numbers.clone()))) + let _node = + PassThroughExecProto::decode(buf).map_err(|err| proto_error(format!("{err}")))?; + + if inputs.len() != 1 { + return Err(proto_error(format!( + "StatefulPassThroughExec expects exactly one child, got {}", + inputs.len() + ))); + } + + Ok(Arc::new(StatefulPassThroughExec::new(inputs[0].clone()))) } fn try_encode( @@ -273,17 +206,15 @@ mod tests { node: Arc, buf: &mut Vec, ) -> datafusion::common::Result<()> { - let Some(plan) = node.as_any().downcast_ref::() else { + let Some(_plan) = node.as_any().downcast_ref::() else { return Err(proto_error(format!( - "Expected plan to be of type Int64ListExec, but was {}", + "Expected plan to be of type StatefulPassThroughExec, but was {}", node.name() ))); }; - Int64ListExecProto { - numbers: plan.numbers.clone(), - } - .encode(buf) - .map_err(|err| proto_error(format!("{err}"))) + PassThroughExecProto {} + .encode(buf) + .map_err(|err| proto_error(format!("{err}"))) } } } diff --git a/tests/tpcds_correctness_test.rs b/tests/tpcds_correctness_test.rs new file mode 100644 index 00000000..2bf8ecc5 --- /dev/null +++ b/tests/tpcds_correctness_test.rs @@ -0,0 +1,608 @@ +#[cfg(all(feature = "integration", feature = "tpcds", test))] +mod tests { + use datafusion::arrow::array::RecordBatch; + use datafusion::common::plan_err; + use datafusion::error::Result; + use datafusion::physical_plan::{ExecutionPlan, collect}; + use datafusion::prelude::SessionContext; + use datafusion_distributed::test_utils::localhost::start_localhost_context; + use datafusion_distributed::test_utils::property_based::{ + compare_ordering, compare_result_set, + }; + use datafusion_distributed::test_utils::{benchmarks_common, tpcds}; + use datafusion_distributed::{ + DefaultSessionBuilder, DistributedExec, DistributedExt, display_plan_ascii, + }; + use std::fs; + use std::path::Path; + use std::sync::Arc; + use tokio::sync::OnceCell; + + const NUM_WORKERS: usize = 4; + const FILES_PER_TASK: usize = 2; + const CARDINALITY_TASK_COUNT_FACTOR: f64 = 2.0; + const SF: f64 = 1.0; + const PARQUET_PARTITIONS: usize = 4; + + #[tokio::test] + async fn test_tpcds_1() -> Result<()> { + test_tpcds_query("q1").await + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_tpcds_2() -> Result<()> { + test_tpcds_query("q2").await + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_tpcds_3() -> Result<()> { + test_tpcds_query("q3").await + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_tpcds_4() -> Result<()> { + test_tpcds_query("q4").await + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_tpcds_5() -> Result<()> { + test_tpcds_query("q5").await + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_tpcds_6() -> Result<()> { + test_tpcds_query("q6").await + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_tpcds_7() -> Result<()> { + test_tpcds_query("q7").await + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_tpcds_8() -> Result<()> { + test_tpcds_query("q8").await + } + + #[tokio::test] + #[ignore = "expected no error but got: Arrow error: Invalid argument error: must either specify a row count or at least one column"] + async fn test_tpcds_9() -> Result<()> { + test_tpcds_query("q9").await + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_tpcds_10() -> Result<()> { + test_tpcds_query("q10").await + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_tpcds_11() -> Result<()> { + test_tpcds_query("q11").await + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_tpcds_12() -> Result<()> { + test_tpcds_query("q12").await + } + + #[tokio::test(flavor = "multi_thread")] + #[ignore = "Query q13 did not get distributed"] + async fn test_tpcds_13() -> Result<()> { + test_tpcds_query("q13").await + } + + #[tokio::test(flavor = "multi_thread")] + #[ignore = "result sets were not equal: Internal error: Row content differs between result sets\nLeft set size: 100, Right set size: 100\n\nRows only in left (71 total):\n NULL|NULL|NULL|NULL|674173362.51|155629\n catalog|NULL|NULL|NULL|237410857.47|46322\n catalog|1001001.00|NULL|NULL|1697729.02|347\n catalog|1001001.00|1.00|NULL|855204.24|167\n catalog|1001001.00|2.00|NULL|125167.22|24\n catalog|1001001.00|3.00|NULL|198685.08|43\n catalog|1001001.00|4.00|NULL|109585.97|31\n catalog|1001001.00|5.00|NULL|59790.61|17\n catalog|1001001.00|8.00|NULL|55768.46|13\n catalog|1001001.00|8.00|7.00|28872.49|7\n catalog|1001001.00|8.00|10.00|26895.97|6\n catalog|1001001.00|9.00|NULL|30944.19|5\n catalog|1001001.00|9.00|6.00|30944.19|5\n catalog|1001001.00|11.00|NULL|82810.87|12\n catalog|1001001.00|11.00|9.00|82810.87|12\n catalog|1001001.00|12.00|NULL|38427.52|9\n catalog|1001001.00|12.00|10.00|38427.52|9\n catalog|1001001.00|15.00|NULL|112838.10|20\n catalog|1001001.00|15.00|9.00|53508.79|7\n catalog|1001001.00|15.00|10.00|59329.31|13\n catalog|1001002.00|NULL|NULL|3527831.33|706\n catalog|1001002.00|1.00|NULL|2673969.89|530\n catalog|1001002.00|1.00|1.00|2673969.89|530\n catalog|1001002.00|2.00|NULL|140831.91|29\n catalog|1001002.00|2.00|1.00|140831.91|29\n catalog|1001002.00|3.00|NULL|320175.87|67\n catalog|1001002.00|3.00|1.00|320175.87|67\n catalog|1001002.00|4.00|NULL|133287.96|21\n catalog|1001002.00|4.00|1.00|133287.96|21\n catalog|1001002.00|5.00|NULL|16606.90|9\n catalog|1001002.00|5.00|1.00|16606.90|9\n catalog|1001002.00|6.00|NULL|15133.01|4\n catalog|1001002.00|6.00|1.00|15133.01|4\n catalog|1001002.00|7.00|NULL|24471.26|10\n catalog|1001002.00|7.00|1.00|24471.26|10\n catalog|1001002.00|8.00|NULL|63773.05|12\n catalog|1001002.00|8.00|1.00|63773.05|12\n catalog|1001002.00|9.00|NULL|9167.19|3\n catalog|1001002.00|9.00|1.00|9167.19|3\n catalog|1001002.00|12.00|NULL|29108.42|7\n catalog|1001002.00|12.00|1.00|29108.42|7\n catalog|1001002.00|15.00|NULL|31143.45|6\n catalog|1001002.00|15.00|1.00|31143.45|6\n catalog|1001002.00|16.00|NULL|70162.42|8\n catalog|1001002.00|16.00|1.00|70162.42|8\n catalog|1002001.00|NULL|NULL|2114110.72|380\n catalog|1002001.00|1.00|NULL|348693.97|55\n catalog|1002001.00|1.00|1.00|76392.13|14\n catalog|1002001.00|1.00|2.00|118394.33|21\n catalog|1002001.00|1.00|4.00|29395.79|5\n catalog|1002001.00|1.00|5.00|35541.97|4\n catalog|1002001.00|1.00|6.00|26104.36|3\n catalog|1002001.00|1.00|9.00|18793.97|4\n catalog|1002001.00|1.00|10.00|44071.42|4\n catalog|1002001.00|2.00|NULL|1233961.70|225\n catalog|1002001.00|2.00|1.00|239511.02|51\n catalog|1002001.00|2.00|2.00|147993.14|26\n catalog|1002001.00|2.00|3.00|100086.93|17\n catalog|1002001.00|2.00|4.00|53524.42|13\n catalog|1002001.00|2.00|5.00|48494.06|10\n catalog|1002001.00|2.00|6.00|142857.04|20\n catalog|1002001.00|2.00|7.00|116557.98|16\n catalog|1002001.00|2.00|8.00|92743.93|24\n catalog|1002001.00|2.00|9.00|203943.99|38\n catalog|1002001.00|2.00|10.00|88249.19|10\n catalog|1002001.00|3.00|NULL|91054.32|17\n catalog|1002001.00|3.00|2.00|25171.13|6\n catalog|1002001.00|3.00|7.00|27766.70|3\n catalog|1002001.00|3.00|8.00|38116.49|8\n catalog|1002001.00|4.00|NULL|182427.69|32\n catalog|1002001.00|4.00|1.00|66896.68|15\n\nRows only in right (71 total):\n NULL|NULL|NULL|NULL|47788579.87|11068\n NULL|NULL|NULL|NULL|46294358.79|10609\n NULL|NULL|NULL|NULL|40499040.27|9321\n NULL|NULL|NULL|NULL|37952602.75|8889\n NULL|NULL|NULL|NULL|50256292.02|11540\n NULL|NULL|NULL|NULL|27943616.98|6397\n NULL|NULL|NULL|NULL|43114338.77|10000\n NULL|NULL|NULL|NULL|56239021.04|13003\n NULL|NULL|NULL|NULL|25682800.66|6012\n NULL|NULL|NULL|NULL|38529122.81|8922\n NULL|NULL|NULL|NULL|59222982.16|13528\n NULL|NULL|NULL|NULL|48322926.86|11228\n NULL|NULL|NULL|NULL|39166012.10|9010\n NULL|NULL|NULL|NULL|32661391.26|7453\n NULL|NULL|NULL|NULL|43315152.10|10008\n NULL|NULL|NULL|NULL|37185124.07|8641\n catalog|NULL|NULL|NULL|16671923.72|3228\n catalog|NULL|NULL|NULL|16630833.01|3143\n catalog|NULL|NULL|NULL|14038550.02|2798\n catalog|NULL|NULL|NULL|13135427.84|2638\n catalog|NULL|NULL|NULL|17604907.44|3399\n catalog|NULL|NULL|NULL|10119873.49|1959\n catalog|NULL|NULL|NULL|14698922.72|2919\n catalog|NULL|NULL|NULL|19534422.18|3931\n catalog|NULL|NULL|NULL|9075046.95|1756\n catalog|NULL|NULL|NULL|13829338.20|2662\n catalog|NULL|NULL|NULL|21769645.88|4087\n catalog|NULL|NULL|NULL|16890254.59|3343\n catalog|NULL|NULL|NULL|13897305.68|2680\n catalog|NULL|NULL|NULL|11719010.15|2217\n catalog|NULL|NULL|NULL|14773719.71|2947\n catalog|NULL|NULL|NULL|13021675.89|2615\n catalog|1001001.00|NULL|NULL|188446.33|41\n catalog|1001001.00|NULL|NULL|53508.79|7\n catalog|1001001.00|NULL|NULL|100105.28|23\n catalog|1001001.00|NULL|NULL|114412.27|25\n catalog|1001001.00|NULL|NULL|77231.70|15\n catalog|1001001.00|NULL|NULL|174489.15|42\n catalog|1001001.00|NULL|NULL|206490.30|38\n catalog|1001001.00|NULL|NULL|45473.85|13\n catalog|1001001.00|NULL|NULL|146344.47|27\n catalog|1001001.00|NULL|NULL|152599.38|28\n catalog|1001001.00|NULL|NULL|206412.37|36\n catalog|1001001.00|NULL|NULL|119368.21|23\n catalog|1001001.00|NULL|NULL|45014.15|12\n catalog|1001001.00|NULL|NULL|50948.80|14\n catalog|1001001.00|NULL|NULL|16883.97|3\n catalog|1001001.00|1.00|NULL|100105.28|23\n catalog|1001001.00|1.00|NULL|99985.35|21\n catalog|1001001.00|1.00|NULL|107555.43|23\n catalog|1001001.00|1.00|NULL|161349.39|29\n catalog|1001001.00|1.00|NULL|146344.47|27\n catalog|1001001.00|1.00|NULL|122521.31|25\n catalog|1001001.00|1.00|NULL|77861.85|13\n catalog|1001001.00|1.00|NULL|22597.19|3\n catalog|1001001.00|1.00|NULL|16883.97|3\n catalog|1001001.00|2.00|NULL|68565.38|14\n catalog|1001001.00|2.00|NULL|43967.97|7\n catalog|1001001.00|2.00|NULL|12633.87|3\n catalog|1001001.00|3.00|NULL|60551.64|14\n catalog|1001001.00|3.00|NULL|14426.92|4\n catalog|1001001.00|3.00|NULL|36821.61|7\n catalog|1001001.00|3.00|NULL|30078.07|3\n catalog|1001001.00|3.00|NULL|28455.23|4\n catalog|1001001.00|3.00|NULL|28351.61|11\n catalog|1001001.00|4.00|NULL|47553.20|10\n catalog|1001001.00|4.00|NULL|45473.85|13\n catalog|1001001.00|4.00|NULL|16558.92|8\n catalog|1001001.00|5.00|NULL|29678.50|5\n catalog|1001001.00|5.00|NULL|30112.11|12\n catalog|1001001.00|8.00|NULL|26895.97|6.\nThis issue was likely caused by a bug in DataFusion's code. Please help us to resolve this by filing a bug report in our issue tracker: https://github.com/apache/datafusion/issues"] + async fn test_tpcds_14() -> Result<()> { + test_tpcds_query("q14").await + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_tpcds_15() -> Result<()> { + test_tpcds_query("q15").await + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_tpcds_16() -> Result<()> { + test_tpcds_query("q16").await + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_tpcds_17() -> Result<()> { + test_tpcds_query("q17").await + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_tpcds_18() -> Result<()> { + test_tpcds_query("q18").await + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_tpcds_19() -> Result<()> { + test_tpcds_query("q19").await + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_tpcds_20() -> Result<()> { + test_tpcds_query("q20").await + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_tpcds_21() -> Result<()> { + test_tpcds_query("q21").await + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_tpcds_22() -> Result<()> { + test_tpcds_query("q22").await + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_tpcds_23() -> Result<()> { + test_tpcds_query("q23").await + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_tpcds_24() -> Result<()> { + test_tpcds_query("q24").await + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_tpcds_25() -> Result<()> { + test_tpcds_query("q25").await + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_tpcds_26() -> Result<()> { + test_tpcds_query("q26").await + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_tpcds_27() -> Result<()> { + test_tpcds_query("q27").await + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_tpcds_28() -> Result<()> { + test_tpcds_query("q28").await + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_tpcds_29() -> Result<()> { + test_tpcds_query("q29").await + } + + #[tokio::test] + #[ignore = "Fails with column 'c_last_review_date_sk' not found"] + async fn test_tpcds_30() -> Result<()> { + test_tpcds_query("q30").await + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_tpcds_31() -> Result<()> { + test_tpcds_query("q31").await + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_tpcds_32() -> Result<()> { + test_tpcds_query("q32").await + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_tpcds_33() -> Result<()> { + test_tpcds_query("q33").await + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_tpcds_34() -> Result<()> { + test_tpcds_query("q34").await + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_tpcds_35() -> Result<()> { + test_tpcds_query("q35").await + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_tpcds_36() -> Result<()> { + test_tpcds_query("q36").await + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_tpcds_37() -> Result<()> { + test_tpcds_query("q37").await + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_tpcds_38() -> Result<()> { + test_tpcds_query("q38").await + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_tpcds_39() -> Result<()> { + test_tpcds_query("q39").await + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_tpcds_40() -> Result<()> { + test_tpcds_query("q40").await + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_tpcds_41() -> Result<()> { + test_tpcds_query("q41").await + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_tpcds_42() -> Result<()> { + test_tpcds_query("q42").await + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_tpcds_43() -> Result<()> { + test_tpcds_query("q43").await + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_tpcds_44() -> Result<()> { + test_tpcds_query("q44").await + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_tpcds_45() -> Result<()> { + test_tpcds_query("q45").await + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_tpcds_46() -> Result<()> { + test_tpcds_query("q46").await + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_tpcds_47() -> Result<()> { + test_tpcds_query("q47").await + } + + #[tokio::test(flavor = "multi_thread")] + #[ignore = "Query q48 did not get distributed"] + async fn test_tpcds_48() -> Result<()> { + test_tpcds_query("q48").await + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_tpcds_49() -> Result<()> { + test_tpcds_query("q49").await + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_tpcds_50() -> Result<()> { + test_tpcds_query("q50").await + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_tpcds_51() -> Result<()> { + test_tpcds_query("q51").await + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_tpcds_52() -> Result<()> { + test_tpcds_query("q52").await + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_tpcds_53() -> Result<()> { + test_tpcds_query("q53").await + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_tpcds_54() -> Result<()> { + test_tpcds_query("q54").await + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_tpcds_55() -> Result<()> { + test_tpcds_query("q55").await + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_tpcds_56() -> Result<()> { + test_tpcds_query("q56").await + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_tpcds_57() -> Result<()> { + test_tpcds_query("q57").await + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_tpcds_58() -> Result<()> { + test_tpcds_query("q58").await + } + + #[tokio::test(flavor = "multi_thread")] + // FIXME: this test succeeds locally, but for some reason it fails on CI + #[ignore = "result sets were not equal: Internal error: Row counts differ: left=100, right=0"] + async fn test_tpcds_59() -> Result<()> { + test_tpcds_query("q59").await + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_tpcds_60() -> Result<()> { + test_tpcds_query("q60").await + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_tpcds_61() -> Result<()> { + test_tpcds_query("q61").await + } + + #[tokio::test(flavor = "multi_thread")] + #[ignore = "Query q62 did not get distributed"] + async fn test_tpcds_62() -> Result<()> { + test_tpcds_query("q62").await + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_tpcds_63() -> Result<()> { + test_tpcds_query("q63").await + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_tpcds_64() -> Result<()> { + test_tpcds_query("q64").await + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_tpcds_65() -> Result<()> { + test_tpcds_query("q65").await + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_tpcds_66() -> Result<()> { + test_tpcds_query("q66").await + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_tpcds_67() -> Result<()> { + test_tpcds_query("q67").await + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_tpcds_68() -> Result<()> { + test_tpcds_query("q68").await + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_tpcds_69() -> Result<()> { + test_tpcds_query("q69").await + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_tpcds_70() -> Result<()> { + test_tpcds_query("q70").await + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_tpcds_71() -> Result<()> { + test_tpcds_query("q71").await + } + + #[tokio::test] + // For some reason this test takes a ridiculous amount of time to execute. There might be + // nothing wrong with it, and it just might be too heavy. The test passes, but it takes so + // long to execute that it's not worth the time. + #[ignore = "Query takes too long to execute"] + async fn test_tpcds_72() -> Result<()> { + test_tpcds_query("q72").await + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_tpcds_73() -> Result<()> { + test_tpcds_query("q73").await + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_tpcds_74() -> Result<()> { + test_tpcds_query("q74").await + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_tpcds_75() -> Result<()> { + test_tpcds_query("q75").await + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_tpcds_76() -> Result<()> { + test_tpcds_query("q76").await + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_tpcds_77() -> Result<()> { + test_tpcds_query("q77").await + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_tpcds_78() -> Result<()> { + test_tpcds_query("q78").await + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_tpcds_79() -> Result<()> { + test_tpcds_query("q79").await + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_tpcds_80() -> Result<()> { + test_tpcds_query("q80").await + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_tpcds_81() -> Result<()> { + test_tpcds_query("q81").await + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_tpcds_82() -> Result<()> { + test_tpcds_query("q82").await + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_tpcds_83() -> Result<()> { + test_tpcds_query("q83").await + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_tpcds_84() -> Result<()> { + test_tpcds_query("q84").await + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_tpcds_85() -> Result<()> { + test_tpcds_query("q85").await + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_tpcds_86() -> Result<()> { + test_tpcds_query("q86").await + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_tpcds_87() -> Result<()> { + test_tpcds_query("q87").await + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_tpcds_88() -> Result<()> { + test_tpcds_query("q88").await + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_tpcds_89() -> Result<()> { + test_tpcds_query("q89").await + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_tpcds_90() -> Result<()> { + test_tpcds_query("q90").await + } + + #[tokio::test(flavor = "multi_thread")] + #[ignore = "Query q91 did not get distributed"] + async fn test_tpcds_91() -> Result<()> { + test_tpcds_query("q91").await + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_tpcds_92() -> Result<()> { + test_tpcds_query("q92").await + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_tpcds_93() -> Result<()> { + test_tpcds_query("q93").await + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_tpcds_94() -> Result<()> { + test_tpcds_query("q94").await + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_tpcds_95() -> Result<()> { + test_tpcds_query("q95").await + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_tpcds_96() -> Result<()> { + test_tpcds_query("q96").await + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_tpcds_97() -> Result<()> { + test_tpcds_query("q97").await + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_tpcds_98() -> Result<()> { + test_tpcds_query("q98").await + } + + #[tokio::test] + // For some reason this test takes a ridiculous amount of time to execute. There might be + // nothing wrong with it, and it just might be too heavy. The test passes, but it takes so + // long to execute that it's not worth the time. + #[ignore = "Query takes too long to execute"] + async fn test_tpcds_99() -> Result<()> { + test_tpcds_query("q99").await + } + + static INIT_TEST_TPCDS_TABLES: OnceCell<()> = OnceCell::const_new(); + + async fn run( + ctx: &SessionContext, + query_sql: &str, + ) -> (Arc, Arc>>) { + let df = ctx.sql(query_sql).await.unwrap(); + let task_ctx = ctx.task_ctx(); + let plan = df.create_physical_plan().await.unwrap(); + (plan.clone(), Arc::new(collect(plan, task_ctx).await)) // Collect execution errors, do not unwrap. + } + + async fn test_tpcds_query(query_id: &str) -> Result<()> { + let data_dir = Path::new(env!("CARGO_MANIFEST_DIR")).join(format!( + "testdata/tpcds/correctness_sf{SF}_partitions{PARQUET_PARTITIONS}" + )); + INIT_TEST_TPCDS_TABLES + .get_or_init(|| async { + if !fs::exists(&data_dir).unwrap_or(false) { + tpcds::generate_data(&data_dir, SF, PARQUET_PARTITIONS) + .await + .unwrap(); + } + }) + .await; + + let query_sql = tpcds::get_query(query_id)?; + // Create a single node context to compare results to. + let s_ctx = SessionContext::new(); + + // Make distributed localhost context to run queries + let (d_ctx, _guard, _) = start_localhost_context(NUM_WORKERS, DefaultSessionBuilder).await; + let d_ctx = d_ctx + .with_distributed_files_per_task(FILES_PER_TASK)? + .with_distributed_cardinality_effect_task_scale_factor(CARDINALITY_TASK_COUNT_FACTOR)? + .with_distributed_broadcast_joins(true)?; + + benchmarks_common::register_tables(&s_ctx, &data_dir).await?; + benchmarks_common::register_tables(&d_ctx, &data_dir).await?; + + let (s_plan, s_results) = run(&s_ctx, &query_sql).await; + let (d_plan, d_results) = run(&d_ctx, &query_sql).await; + + if !d_plan.as_any().is::() { + return plan_err!("Query {query_id} did not get distributed"); + } + let display = display_plan_ascii(d_plan.as_ref(), false); + println!("Query {query_id}:\n{display}"); + + // The comparison functions can be computationally expensive, so we spawn them in tokio + // blocking tasks so that they do not block the tokio runtime. + let compare_result_set = { + let d_results = d_results.clone(); + let s_results = s_results.clone(); + tokio::task::spawn_blocking(move || async move { + compare_result_set(&d_results, &s_results) + }) + }; + let compare_ordering = { + let d_results = d_results.clone(); + tokio::task::spawn_blocking(move || async move { + compare_ordering(d_plan, s_plan, &d_results) + }) + }; + compare_result_set.await.unwrap().await?; + compare_ordering.await.unwrap().await?; + + Ok(()) + } +} diff --git a/tests/tpcds_plans_test.rs b/tests/tpcds_plans_test.rs new file mode 100644 index 00000000..a59892bb --- /dev/null +++ b/tests/tpcds_plans_test.rs @@ -0,0 +1,10615 @@ +#[cfg(all(feature = "integration", feature = "tpcds", test))] +mod tests { + use datafusion::error::Result; + use datafusion_distributed::test_utils::localhost::start_localhost_context; + use datafusion_distributed::test_utils::{benchmarks_common, tpcds}; + use datafusion_distributed::{ + DefaultSessionBuilder, DistributedExec, DistributedExt, assert_snapshot, display_plan_ascii, + }; + use std::env; + use std::fs; + use std::path::Path; + use tokio::sync::OnceCell; + + const NUM_WORKERS: usize = 4; + const FILES_PER_TASK: usize = 2; + const CARDINALITY_TASK_COUNT_FACTOR: f64 = 2.0; + const SF: f64 = 1.0; + const PARQUET_PARTITIONS: usize = 4; + + #[tokio::test] + async fn test_tpcds_1() -> Result<()> { + let display = test_tpcds_query("q1").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [c_customer_id@0 ASC NULLS LAST], fetch=100 + │ [Stage 9] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 9 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ SortExec: TopK(fetch=100), expr=[c_customer_id@0 ASC NULLS LAST], preserve_partitioning=[true] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ctr_store_sk@0, ctr_store_sk@1)], filter=CAST(ctr_total_return@0 AS Decimal128(30, 15)) > avg(ctr2.ctr_total_return) * Float64(1.2)@1, projection=[c_customer_id@2] + │ CoalescePartitionsExec + │ [Stage 5] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[CAST(CAST(avg(ctr2.ctr_total_return)@1 AS Float64) * 1.2 AS Decimal128(30, 15)) as avg(ctr2.ctr_total_return) * Float64(1.2), ctr_store_sk@0 as ctr_store_sk] + │ AggregateExec: mode=FinalPartitioned, gby=[ctr_store_sk@0 as ctr_store_sk], aggr=[avg(ctr2.ctr_total_return)] + │ [Stage 8] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=3, consumer_tasks=2, output_partitions=6 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ctr_customer_sk@0, CAST(customer.c_customer_sk AS Float64)@2)], projection=[ctr_store_sk@1, ctr_total_return@2, c_customer_id@4] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, c_customer_id@1 as c_customer_id, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_customer_id], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=3, consumer_tasks=2, output_partitions=6 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@1, ctr_store_sk@1)], projection=[ctr_customer_sk@2, ctr_store_sk@3, ctr_total_return@4] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[sr_customer_sk@0 as ctr_customer_sk, sr_store_sk@1 as ctr_store_sk, sum(store_returns.sr_return_amt)@2 as ctr_total_return] + │ AggregateExec: mode=FinalPartitioned, gby=[sr_customer_sk@0 as sr_customer_sk, sr_store_sk@1 as sr_store_sk], aggr=[sum(store_returns.sr_return_amt)] + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=3, consumer_tasks=2, output_partitions=6 + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] + │ FilterExec: s_state@1 = TN, projection=[s_store_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_state], file_type=parquet, predicate=s_state@24 = TN, pruning_predicate=s_state_null_count@2 != row_count@3 AND s_state_min@0 <= TN AND TN <= s_state_max@1, required_guarantees=[s_state in (TN)] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([sr_customer_sk@0, sr_store_sk@1], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[sr_customer_sk@0 as sr_customer_sk, sr_store_sk@1 as sr_store_sk], aggr=[sum(store_returns.sr_return_amt)] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, sr_returned_date_sk@0)], projection=[sr_customer_sk@3, sr_store_sk@4, sr_return_amt@5] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], ...]}, projection=[sr_returned_date_sk, sr_customer_sk, sr_store_sk, sr_return_amt], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_year@1 = 2000, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] + └────────────────────────────────────────────────── + ┌───── Stage 8 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ RepartitionExec: partitioning=Hash([ctr_store_sk@0], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[ctr_store_sk@0 as ctr_store_sk], aggr=[avg(ctr2.ctr_total_return)] + │ ProjectionExec: expr=[sr_store_sk@1 as ctr_store_sk, sum(store_returns.sr_return_amt)@2 as ctr_total_return] + │ AggregateExec: mode=FinalPartitioned, gby=[sr_customer_sk@0 as sr_customer_sk, sr_store_sk@1 as sr_store_sk], aggr=[sum(store_returns.sr_return_amt)] + │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([sr_customer_sk@0, sr_store_sk@1], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[sr_customer_sk@0 as sr_customer_sk, sr_store_sk@1 as sr_store_sk], aggr=[sum(store_returns.sr_return_amt)] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, sr_returned_date_sk@0)], projection=[sr_customer_sk@3, sr_store_sk@4, sr_return_amt@5] + │ CoalescePartitionsExec + │ [Stage 6] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], ...]}, projection=[sr_returned_date_sk, sr_customer_sk, sr_store_sk, sr_return_amt], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_year@1 = 2000, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] + └────────────────────────────────────────────────── + "); + Ok(()) + } + + #[tokio::test] + async fn test_tpcds_2() -> Result<()> { + let display = test_tpcds_query("q2").await?; + assert_snapshot!(display, @r#" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [d_week_seq1@0 ASC] + │ [Stage 8] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 8 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ SortExec: expr=[d_week_seq1@0 ASC], preserve_partitioning=[true] + │ ProjectionExec: expr=[d_week_seq1@0 as d_week_seq1, round(sun_sales1@1 / sun_sales2@8, 2) as r1, round(mon_sales1@2 / mon_sales2@9, 2) as r2, round(tue_sales1@3 / tue_sales2@10, 2) as r3, round(wed_sales1@4 / wed_sales2@11, 2) as r4, round(thu_sales1@5 / thu_sales2@12, 2) as r5, round(fri_sales1@6 / fri_sales2@13, 2) as r6, round(sat_sales1@7 / sat_sales2@14, 2) as round(y.sat_sales1 / z.sat_sales2,Int64(2))] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(y.d_week_seq1 AS Int64)@8, z.d_week_seq2 - Int64(53)@8)], projection=[d_week_seq1@0, sun_sales1@1, mon_sales1@2, tue_sales1@3, wed_sales1@4, thu_sales1@5, fri_sales1@6, sat_sales1@7, sun_sales2@10, mon_sales2@11, tue_sales2@12, wed_sales2@13, thu_sales2@14, fri_sales2@15, sat_sales2@16] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[d_week_seq@0 as d_week_seq2, sun_sales@1 as sun_sales2, mon_sales@2 as mon_sales2, tue_sales@3 as tue_sales2, wed_sales@4 as wed_sales2, thu_sales@5 as thu_sales2, fri_sales@6 as fri_sales2, sat_sales@7 as sat_sales2, CAST(d_week_seq@0 AS Int64) - 53 as z.d_week_seq2 - Int64(53)] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_week_seq@0, d_week_seq@0)], projection=[d_week_seq@1, sun_sales@2, mon_sales@3, tue_sales@4, wed_sales@5, thu_sales@6, fri_sales@7, sat_sales@8] + │ CoalescePartitionsExec + │ [Stage 5] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[d_week_seq@0 as d_week_seq, sum(CASE WHEN date_dim.d_day_name = Utf8("Sunday") THEN wscs.sales_price ELSE NULL END)@1 as sun_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Monday") THEN wscs.sales_price ELSE NULL END)@2 as mon_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Tuesday") THEN wscs.sales_price ELSE NULL END)@3 as tue_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Wednesday") THEN wscs.sales_price ELSE NULL END)@4 as wed_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Thursday") THEN wscs.sales_price ELSE NULL END)@5 as thu_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Friday") THEN wscs.sales_price ELSE NULL END)@6 as fri_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Saturday") THEN wscs.sales_price ELSE NULL END)@7 as sat_sales] + │ AggregateExec: mode=FinalPartitioned, gby=[d_week_seq@0 as d_week_seq], aggr=[sum(CASE WHEN date_dim.d_day_name = Utf8("Sunday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Monday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Tuesday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Wednesday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Thursday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Friday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Saturday") THEN wscs.sales_price ELSE NULL END)] + │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=3, consumer_tasks=2, output_partitions=6 + │ ProjectionExec: expr=[d_week_seq@0 as d_week_seq1, sun_sales@1 as sun_sales1, mon_sales@2 as mon_sales1, tue_sales@3 as tue_sales1, wed_sales@4 as wed_sales1, thu_sales@5 as thu_sales1, fri_sales@6 as fri_sales1, sat_sales@7 as sat_sales1, CAST(d_week_seq@0 AS Int64) as CAST(y.d_week_seq1 AS Int64)] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_week_seq@0, d_week_seq@0)], projection=[d_week_seq@1, sun_sales@2, mon_sales@3, tue_sales@4, wed_sales@5, thu_sales@6, fri_sales@7, sat_sales@8] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[d_week_seq@0 as d_week_seq, sum(CASE WHEN date_dim.d_day_name = Utf8("Sunday") THEN wscs.sales_price ELSE NULL END)@1 as sun_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Monday") THEN wscs.sales_price ELSE NULL END)@2 as mon_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Tuesday") THEN wscs.sales_price ELSE NULL END)@3 as tue_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Wednesday") THEN wscs.sales_price ELSE NULL END)@4 as wed_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Thursday") THEN wscs.sales_price ELSE NULL END)@5 as thu_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Friday") THEN wscs.sales_price ELSE NULL END)@6 as fri_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Saturday") THEN wscs.sales_price ELSE NULL END)@7 as sat_sales] + │ AggregateExec: mode=FinalPartitioned, gby=[d_week_seq@0 as d_week_seq], aggr=[sum(CASE WHEN date_dim.d_day_name = Utf8("Sunday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Monday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Tuesday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Wednesday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Thursday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Friday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Saturday") THEN wscs.sales_price ELSE NULL END)] + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=3, consumer_tasks=2, output_partitions=6 + │ FilterExec: d_year@1 = 2001, projection=[d_week_seq@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_week_seq, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] t3:[p0..p5] + │ RepartitionExec: partitioning=Hash([d_week_seq@0], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[d_week_seq@1 as d_week_seq], aggr=[sum(CASE WHEN date_dim.d_day_name = Utf8("Sunday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Monday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Tuesday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Wednesday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Thursday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Friday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Saturday") THEN wscs.sales_price ELSE NULL END)] + │ ProjectionExec: expr=[sales_price@2 as sales_price, d_week_seq@0 as d_week_seq, d_day_name@1 as d_day_name] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@3, sold_date_sk@0)], projection=[d_week_seq@1, d_day_name@2, sales_price@5] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=2 + │ DistributedUnionExec: t0:[c0(0/2)] t1:[c0(1/2)] t2:[c1(0/2)] t3:[c1(1/2)] + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk@0 as sold_date_sk, ws_ext_sales_price@23 as sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk@0 as sold_date_sk, cs_ext_sales_price@23 as sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p11] t1:[p12..p23] + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_week_seq@1 as d_week_seq, d_day_name@2 as d_day_name, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_week_seq, d_day_name], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=3, consumer_tasks=2, output_partitions=6 + │ FilterExec: d_year@1 = 2002, projection=[d_week_seq@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_week_seq, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] t3:[p0..p5] + │ RepartitionExec: partitioning=Hash([d_week_seq@0], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[d_week_seq@1 as d_week_seq], aggr=[sum(CASE WHEN date_dim.d_day_name = Utf8("Sunday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Monday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Tuesday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Wednesday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Thursday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Friday") THEN wscs.sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Saturday") THEN wscs.sales_price ELSE NULL END)] + │ ProjectionExec: expr=[sales_price@2 as sales_price, d_week_seq@0 as d_week_seq, d_day_name@1 as d_day_name] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@3, sold_date_sk@0)], projection=[d_week_seq@1, d_day_name@2, sales_price@5] + │ CoalescePartitionsExec + │ [Stage 6] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=2 + │ DistributedUnionExec: t0:[c0(0/2)] t1:[c0(1/2)] t2:[c1(0/2)] t3:[c1(1/2)] + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk@0 as sold_date_sk, ws_ext_sales_price@23 as sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk@0 as sold_date_sk, cs_ext_sales_price@23 as sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p11] t1:[p12..p23] + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_week_seq@1 as d_week_seq, d_day_name@2 as d_day_name, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_week_seq, d_day_name], file_type=parquet + └────────────────────────────────────────────────── + "#); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_3() -> Result<()> { + let display = test_tpcds_query("q3").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [d_year@0 ASC NULLS LAST, sum_agg@3 DESC, brand_id@1 ASC NULLS LAST], fetch=100 + │ [Stage 4] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ SortExec: TopK(fetch=100), expr=[d_year@0 ASC NULLS LAST, sum_agg@3 DESC, brand_id@1 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[d_year@0 as d_year, i_brand_id@2 as brand_id, i_brand@1 as brand, sum(store_sales.ss_ext_sales_price)@3 as sum_agg] + │ AggregateExec: mode=FinalPartitioned, gby=[d_year@0 as d_year, i_brand@1 as i_brand, i_brand_id@2 as i_brand_id], aggr=[sum(store_sales.ss_ext_sales_price)] + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([d_year@0, i_brand@1, i_brand_id@2], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[d_year@0 as d_year, i_brand@3 as i_brand, i_brand_id@2 as i_brand_id], aggr=[sum(store_sales.ss_ext_sales_price)] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_item_sk@1, i_item_sk@0)], projection=[d_year@0, ss_ext_sales_price@2, i_brand_id@4, i_brand@5] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=3 + │ FilterExec: i_manufact_id@3 = 128, projection=[i_item_sk@0, i_brand_id@1, i_brand@2] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_brand, i_manufact_id], file_type=parquet, predicate=i_manufact_id@13 = 128 AND DynamicFilter [ empty ], pruning_predicate=i_manufact_id_null_count@2 != row_count@3 AND i_manufact_id_min@0 <= 128 AND 128 <= i_manufact_id_max@1, required_guarantees=[i_manufact_id in (128)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p5] t1:[p6..p11] t2:[p12..p17] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(dt.d_date_sk AS Float64)@2, ss_sold_date_sk@0)], projection=[d_year@1, ss_item_sk@4, ss_ext_sales_price@5] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, CAST(d_date_sk@0 AS Float64) as CAST(dt.d_date_sk AS Float64)] + │ FilterExec: d_moy@2 = 11, projection=[d_date_sk@0, d_year@1] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 11, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1, required_guarantees=[d_moy in (11)] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_4() -> Result<()> { + let display = test_tpcds_query("q4").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [customer_id@0 ASC, customer_first_name@1 ASC, customer_last_name@2 ASC, customer_preferred_cust_flag@3 ASC], fetch=100 + │ [Stage 24] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 24 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ SortExec: TopK(fetch=100), expr=[customer_id@0 ASC, customer_first_name@1 ASC, customer_last_name@2 ASC, customer_preferred_cust_flag@3 ASC], preserve_partitioning=[true] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(customer_id@0, customer_id@0)], filter=CASE WHEN year_total@0 > Some(0),24,6 THEN year_total@1 / year_total@0 END > CASE WHEN year_total@2 > Some(0),24,6 THEN year_total@3 / year_total@2 END, projection=[customer_id@1, customer_first_name@2, customer_last_name@3, customer_preferred_cust_flag@4] + │ CoalescePartitionsExec + │ [Stage 20] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[c_customer_id@0 as customer_id, sum(web_sales.ws_ext_list_price - web_sales.ws_ext_wholesale_cost - web_sales.ws_ext_discount_amt + web_sales.ws_ext_sales_price / Int64(2))@8 as year_total] + │ AggregateExec: mode=FinalPartitioned, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, c_preferred_cust_flag@3 as c_preferred_cust_flag, c_birth_country@4 as c_birth_country, c_login@5 as c_login, c_email_address@6 as c_email_address, d_year@7 as d_year], aggr=[sum(web_sales.ws_ext_list_price - web_sales.ws_ext_wholesale_cost - web_sales.ws_ext_discount_amt + web_sales.ws_ext_sales_price / Int64(2))], ordering_mode=PartiallySorted([7]) + │ [Stage 23] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 20 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=3, consumer_tasks=2, output_partitions=6 + │ ProjectionExec: expr=[customer_id@1 as customer_id, customer_id@2 as customer_id, customer_first_name@3 as customer_first_name, customer_last_name@4 as customer_last_name, customer_preferred_cust_flag@5 as customer_preferred_cust_flag, year_total@6 as year_total, year_total@7 as year_total, year_total@0 as year_total] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(customer_id@0, customer_id@0)], projection=[year_total@1, customer_id@2, customer_id@3, customer_first_name@4, customer_last_name@5, customer_preferred_cust_flag@6, year_total@7, year_total@8] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=6, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(customer_id@0, customer_id@0)], filter=CASE WHEN year_total@2 > Some(0),24,6 THEN year_total@3 / year_total@2 END > CASE WHEN year_total@0 > Some(0),24,6 THEN year_total@1 / year_total@0 END, projection=[customer_id@0, customer_id@2, customer_first_name@3, customer_last_name@4, customer_preferred_cust_flag@5, year_total@7, year_total@9] + │ CoalescePartitionsExec + │ [Stage 16] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[c_customer_id@0 as customer_id, sum(catalog_sales.cs_ext_list_price - catalog_sales.cs_ext_wholesale_cost - catalog_sales.cs_ext_discount_amt + catalog_sales.cs_ext_sales_price / Int64(2))@8 as year_total] + │ AggregateExec: mode=FinalPartitioned, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, c_preferred_cust_flag@3 as c_preferred_cust_flag, c_birth_country@4 as c_birth_country, c_login@5 as c_login, c_email_address@6 as c_email_address, d_year@7 as d_year], aggr=[sum(catalog_sales.cs_ext_list_price - catalog_sales.cs_ext_wholesale_cost - catalog_sales.cs_ext_discount_amt + catalog_sales.cs_ext_sales_price / Int64(2))], ordering_mode=PartiallySorted([7]) + │ [Stage 19] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=3, consumer_tasks=2, output_partitions=6 + │ ProjectionExec: expr=[c_customer_id@0 as customer_id, sum(web_sales.ws_ext_list_price - web_sales.ws_ext_wholesale_cost - web_sales.ws_ext_discount_amt + web_sales.ws_ext_sales_price / Int64(2))@1 as year_total] + │ FilterExec: sum(web_sales.ws_ext_list_price - web_sales.ws_ext_wholesale_cost - web_sales.ws_ext_discount_amt + web_sales.ws_ext_sales_price / Int64(2))@1 > Some(0),24,6 + │ ProjectionExec: expr=[c_customer_id@0 as c_customer_id, sum(web_sales.ws_ext_list_price - web_sales.ws_ext_wholesale_cost - web_sales.ws_ext_discount_amt + web_sales.ws_ext_sales_price / Int64(2))@8 as sum(web_sales.ws_ext_list_price - web_sales.ws_ext_wholesale_cost - web_sales.ws_ext_discount_amt + web_sales.ws_ext_sales_price / Int64(2))] + │ AggregateExec: mode=FinalPartitioned, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, c_preferred_cust_flag@3 as c_preferred_cust_flag, c_birth_country@4 as c_birth_country, c_login@5 as c_login, c_email_address@6 as c_email_address, d_year@7 as d_year], aggr=[sum(web_sales.ws_ext_list_price - web_sales.ws_ext_wholesale_cost - web_sales.ws_ext_discount_amt + web_sales.ws_ext_sales_price / Int64(2))], ordering_mode=PartiallySorted([7]) + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([c_customer_id@0, c_first_name@1, c_last_name@2, c_preferred_cust_flag@3, c_birth_country@4, c_login@5, c_email_address@6, d_year@7], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, c_preferred_cust_flag@3 as c_preferred_cust_flag, c_birth_country@4 as c_birth_country, c_login@5 as c_login, c_email_address@6 as c_email_address, d_year@11 as d_year], aggr=[sum(web_sales.ws_ext_list_price - web_sales.ws_ext_wholesale_cost - web_sales.ws_ext_discount_amt + web_sales.ws_ext_sales_price / Int64(2))], ordering_mode=PartiallySorted([7]) + │ ProjectionExec: expr=[c_customer_id@1 as c_customer_id, c_first_name@2 as c_first_name, c_last_name@3 as c_last_name, c_preferred_cust_flag@4 as c_preferred_cust_flag, c_birth_country@5 as c_birth_country, c_login@6 as c_login, c_email_address@7 as c_email_address, ws_ext_discount_amt@8 as ws_ext_discount_amt, ws_ext_sales_price@9 as ws_ext_sales_price, ws_ext_wholesale_cost@10 as ws_ext_wholesale_cost, ws_ext_list_price@11 as ws_ext_list_price, d_year@0 as d_year] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, ws_sold_date_sk@7)], projection=[d_year@1, c_customer_id@3, c_first_name@4, c_last_name@5, c_preferred_cust_flag@6, c_birth_country@7, c_login@8, c_email_address@9, ws_ext_discount_amt@11, ws_ext_sales_price@12, ws_ext_wholesale_cost@13, ws_ext_list_price@14] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(customer.c_customer_sk AS Float64)@8, ws_bill_customer_sk@1)], projection=[c_customer_id@1, c_first_name@2, c_last_name@3, c_preferred_cust_flag@4, c_birth_country@5, c_login@6, c_email_address@7, ws_sold_date_sk@9, ws_ext_discount_amt@11, ws_ext_sales_price@12, ws_ext_wholesale_cost@13, ws_ext_list_price@14] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_bill_customer_sk, ws_ext_discount_amt, ws_ext_sales_price, ws_ext_wholesale_cost, ws_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_year@1 = 2001 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, c_customer_id@1 as c_customer_id, c_first_name@2 as c_first_name, c_last_name@3 as c_last_name, c_preferred_cust_flag@4 as c_preferred_cust_flag, c_birth_country@5 as c_birth_country, c_login@6 as c_login, c_email_address@7 as c_email_address, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 16 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=3, consumer_tasks=2, output_partitions=6 + │ ProjectionExec: expr=[customer_id@1 as customer_id, year_total@2 as year_total, customer_id@3 as customer_id, customer_first_name@4 as customer_first_name, customer_last_name@5 as customer_last_name, customer_preferred_cust_flag@6 as customer_preferred_cust_flag, year_total@7 as year_total, year_total@0 as year_total] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(customer_id@0, customer_id@0)], projection=[year_total@1, customer_id@2, year_total@3, customer_id@4, customer_first_name@5, customer_last_name@6, customer_preferred_cust_flag@7, year_total@8] + │ CoalescePartitionsExec + │ [Stage 8] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=6, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(customer_id@0, customer_id@0)] + │ CoalescePartitionsExec + │ [Stage 12] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[c_customer_id@0 as customer_id, c_first_name@1 as customer_first_name, c_last_name@2 as customer_last_name, c_preferred_cust_flag@3 as customer_preferred_cust_flag, sum(store_sales.ss_ext_list_price - store_sales.ss_ext_wholesale_cost - store_sales.ss_ext_discount_amt + store_sales.ss_ext_sales_price / Int64(2))@8 as year_total] + │ AggregateExec: mode=FinalPartitioned, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, c_preferred_cust_flag@3 as c_preferred_cust_flag, c_birth_country@4 as c_birth_country, c_login@5 as c_login, c_email_address@6 as c_email_address, d_year@7 as d_year], aggr=[sum(store_sales.ss_ext_list_price - store_sales.ss_ext_wholesale_cost - store_sales.ss_ext_discount_amt + store_sales.ss_ext_sales_price / Int64(2))], ordering_mode=PartiallySorted([7]) + │ [Stage 15] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 8 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=3, consumer_tasks=2, output_partitions=6 + │ ProjectionExec: expr=[c_customer_id@0 as customer_id, sum(catalog_sales.cs_ext_list_price - catalog_sales.cs_ext_wholesale_cost - catalog_sales.cs_ext_discount_amt + catalog_sales.cs_ext_sales_price / Int64(2))@1 as year_total] + │ FilterExec: sum(catalog_sales.cs_ext_list_price - catalog_sales.cs_ext_wholesale_cost - catalog_sales.cs_ext_discount_amt + catalog_sales.cs_ext_sales_price / Int64(2))@1 > Some(0),24,6 + │ ProjectionExec: expr=[c_customer_id@0 as c_customer_id, sum(catalog_sales.cs_ext_list_price - catalog_sales.cs_ext_wholesale_cost - catalog_sales.cs_ext_discount_amt + catalog_sales.cs_ext_sales_price / Int64(2))@8 as sum(catalog_sales.cs_ext_list_price - catalog_sales.cs_ext_wholesale_cost - catalog_sales.cs_ext_discount_amt + catalog_sales.cs_ext_sales_price / Int64(2))] + │ AggregateExec: mode=FinalPartitioned, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, c_preferred_cust_flag@3 as c_preferred_cust_flag, c_birth_country@4 as c_birth_country, c_login@5 as c_login, c_email_address@6 as c_email_address, d_year@7 as d_year], aggr=[sum(catalog_sales.cs_ext_list_price - catalog_sales.cs_ext_wholesale_cost - catalog_sales.cs_ext_discount_amt + catalog_sales.cs_ext_sales_price / Int64(2))], ordering_mode=PartiallySorted([7]) + │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([c_customer_id@0, c_first_name@1, c_last_name@2, c_preferred_cust_flag@3, c_birth_country@4, c_login@5, c_email_address@6, d_year@7], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, c_preferred_cust_flag@3 as c_preferred_cust_flag, c_birth_country@4 as c_birth_country, c_login@5 as c_login, c_email_address@6 as c_email_address, d_year@11 as d_year], aggr=[sum(catalog_sales.cs_ext_list_price - catalog_sales.cs_ext_wholesale_cost - catalog_sales.cs_ext_discount_amt + catalog_sales.cs_ext_sales_price / Int64(2))], ordering_mode=PartiallySorted([7]) + │ ProjectionExec: expr=[c_customer_id@1 as c_customer_id, c_first_name@2 as c_first_name, c_last_name@3 as c_last_name, c_preferred_cust_flag@4 as c_preferred_cust_flag, c_birth_country@5 as c_birth_country, c_login@6 as c_login, c_email_address@7 as c_email_address, cs_ext_discount_amt@8 as cs_ext_discount_amt, cs_ext_sales_price@9 as cs_ext_sales_price, cs_ext_wholesale_cost@10 as cs_ext_wholesale_cost, cs_ext_list_price@11 as cs_ext_list_price, d_year@0 as d_year] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, cs_sold_date_sk@7)], projection=[d_year@1, c_customer_id@3, c_first_name@4, c_last_name@5, c_preferred_cust_flag@6, c_birth_country@7, c_login@8, c_email_address@9, cs_ext_discount_amt@11, cs_ext_sales_price@12, cs_ext_wholesale_cost@13, cs_ext_list_price@14] + │ CoalescePartitionsExec + │ [Stage 5] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(customer.c_customer_sk AS Float64)@8, cs_bill_customer_sk@1)], projection=[c_customer_id@1, c_first_name@2, c_last_name@3, c_preferred_cust_flag@4, c_birth_country@5, c_login@6, c_email_address@7, cs_sold_date_sk@9, cs_ext_discount_amt@11, cs_ext_sales_price@12, cs_ext_wholesale_cost@13, cs_ext_list_price@14] + │ CoalescePartitionsExec + │ [Stage 6] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_ext_discount_amt, cs_ext_sales_price, cs_ext_wholesale_cost, cs_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_year@1 = 2001 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, c_customer_id@1 as c_customer_id, c_first_name@2 as c_first_name, c_last_name@3 as c_last_name, c_preferred_cust_flag@4 as c_preferred_cust_flag, c_birth_country@5 as c_birth_country, c_login@6 as c_login, c_email_address@7 as c_email_address, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 12 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=3, consumer_tasks=2, output_partitions=6 + │ ProjectionExec: expr=[c_customer_id@0 as customer_id, sum(store_sales.ss_ext_list_price - store_sales.ss_ext_wholesale_cost - store_sales.ss_ext_discount_amt + store_sales.ss_ext_sales_price / Int64(2))@1 as year_total] + │ FilterExec: sum(store_sales.ss_ext_list_price - store_sales.ss_ext_wholesale_cost - store_sales.ss_ext_discount_amt + store_sales.ss_ext_sales_price / Int64(2))@1 > Some(0),24,6 + │ ProjectionExec: expr=[c_customer_id@0 as c_customer_id, sum(store_sales.ss_ext_list_price - store_sales.ss_ext_wholesale_cost - store_sales.ss_ext_discount_amt + store_sales.ss_ext_sales_price / Int64(2))@8 as sum(store_sales.ss_ext_list_price - store_sales.ss_ext_wholesale_cost - store_sales.ss_ext_discount_amt + store_sales.ss_ext_sales_price / Int64(2))] + │ AggregateExec: mode=FinalPartitioned, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, c_preferred_cust_flag@3 as c_preferred_cust_flag, c_birth_country@4 as c_birth_country, c_login@5 as c_login, c_email_address@6 as c_email_address, d_year@7 as d_year], aggr=[sum(store_sales.ss_ext_list_price - store_sales.ss_ext_wholesale_cost - store_sales.ss_ext_discount_amt + store_sales.ss_ext_sales_price / Int64(2))], ordering_mode=PartiallySorted([7]) + │ [Stage 11] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 11 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([c_customer_id@0, c_first_name@1, c_last_name@2, c_preferred_cust_flag@3, c_birth_country@4, c_login@5, c_email_address@6, d_year@7], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, c_preferred_cust_flag@3 as c_preferred_cust_flag, c_birth_country@4 as c_birth_country, c_login@5 as c_login, c_email_address@6 as c_email_address, d_year@11 as d_year], aggr=[sum(store_sales.ss_ext_list_price - store_sales.ss_ext_wholesale_cost - store_sales.ss_ext_discount_amt + store_sales.ss_ext_sales_price / Int64(2))], ordering_mode=PartiallySorted([7]) + │ ProjectionExec: expr=[c_customer_id@1 as c_customer_id, c_first_name@2 as c_first_name, c_last_name@3 as c_last_name, c_preferred_cust_flag@4 as c_preferred_cust_flag, c_birth_country@5 as c_birth_country, c_login@6 as c_login, c_email_address@7 as c_email_address, ss_ext_discount_amt@8 as ss_ext_discount_amt, ss_ext_sales_price@9 as ss_ext_sales_price, ss_ext_wholesale_cost@10 as ss_ext_wholesale_cost, ss_ext_list_price@11 as ss_ext_list_price, d_year@0 as d_year] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, ss_sold_date_sk@7)], projection=[d_year@1, c_customer_id@3, c_first_name@4, c_last_name@5, c_preferred_cust_flag@6, c_birth_country@7, c_login@8, c_email_address@9, ss_ext_discount_amt@11, ss_ext_sales_price@12, ss_ext_wholesale_cost@13, ss_ext_list_price@14] + │ CoalescePartitionsExec + │ [Stage 9] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(customer.c_customer_sk AS Float64)@8, ss_customer_sk@1)], projection=[c_customer_id@1, c_first_name@2, c_last_name@3, c_preferred_cust_flag@4, c_birth_country@5, c_login@6, c_email_address@7, ss_sold_date_sk@9, ss_ext_discount_amt@11, ss_ext_sales_price@12, ss_ext_wholesale_cost@13, ss_ext_list_price@14] + │ CoalescePartitionsExec + │ [Stage 10] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_ext_discount_amt, ss_ext_sales_price, ss_ext_wholesale_cost, ss_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 9 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_year@1 = 2001 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + └────────────────────────────────────────────────── + ┌───── Stage 10 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, c_customer_id@1 as c_customer_id, c_first_name@2 as c_first_name, c_last_name@3 as c_last_name, c_preferred_cust_flag@4 as c_preferred_cust_flag, c_birth_country@5 as c_birth_country, c_login@6 as c_login, c_email_address@7 as c_email_address, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 15 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([c_customer_id@0, c_first_name@1, c_last_name@2, c_preferred_cust_flag@3, c_birth_country@4, c_login@5, c_email_address@6, d_year@7], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, c_preferred_cust_flag@3 as c_preferred_cust_flag, c_birth_country@4 as c_birth_country, c_login@5 as c_login, c_email_address@6 as c_email_address, d_year@11 as d_year], aggr=[sum(store_sales.ss_ext_list_price - store_sales.ss_ext_wholesale_cost - store_sales.ss_ext_discount_amt + store_sales.ss_ext_sales_price / Int64(2))], ordering_mode=PartiallySorted([7]) + │ ProjectionExec: expr=[c_customer_id@1 as c_customer_id, c_first_name@2 as c_first_name, c_last_name@3 as c_last_name, c_preferred_cust_flag@4 as c_preferred_cust_flag, c_birth_country@5 as c_birth_country, c_login@6 as c_login, c_email_address@7 as c_email_address, ss_ext_discount_amt@8 as ss_ext_discount_amt, ss_ext_sales_price@9 as ss_ext_sales_price, ss_ext_wholesale_cost@10 as ss_ext_wholesale_cost, ss_ext_list_price@11 as ss_ext_list_price, d_year@0 as d_year] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, ss_sold_date_sk@7)], projection=[d_year@1, c_customer_id@3, c_first_name@4, c_last_name@5, c_preferred_cust_flag@6, c_birth_country@7, c_login@8, c_email_address@9, ss_ext_discount_amt@11, ss_ext_sales_price@12, ss_ext_wholesale_cost@13, ss_ext_list_price@14] + │ CoalescePartitionsExec + │ [Stage 13] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(customer.c_customer_sk AS Float64)@8, ss_customer_sk@1)], projection=[c_customer_id@1, c_first_name@2, c_last_name@3, c_preferred_cust_flag@4, c_birth_country@5, c_login@6, c_email_address@7, ss_sold_date_sk@9, ss_ext_discount_amt@11, ss_ext_sales_price@12, ss_ext_wholesale_cost@13, ss_ext_list_price@14] + │ CoalescePartitionsExec + │ [Stage 14] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_ext_discount_amt, ss_ext_sales_price, ss_ext_wholesale_cost, ss_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 13 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_year@1 = 2002 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] + └────────────────────────────────────────────────── + ┌───── Stage 14 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, c_customer_id@1 as c_customer_id, c_first_name@2 as c_first_name, c_last_name@3 as c_last_name, c_preferred_cust_flag@4 as c_preferred_cust_flag, c_birth_country@5 as c_birth_country, c_login@6 as c_login, c_email_address@7 as c_email_address, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 19 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([c_customer_id@0, c_first_name@1, c_last_name@2, c_preferred_cust_flag@3, c_birth_country@4, c_login@5, c_email_address@6, d_year@7], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, c_preferred_cust_flag@3 as c_preferred_cust_flag, c_birth_country@4 as c_birth_country, c_login@5 as c_login, c_email_address@6 as c_email_address, d_year@11 as d_year], aggr=[sum(catalog_sales.cs_ext_list_price - catalog_sales.cs_ext_wholesale_cost - catalog_sales.cs_ext_discount_amt + catalog_sales.cs_ext_sales_price / Int64(2))], ordering_mode=PartiallySorted([7]) + │ ProjectionExec: expr=[c_customer_id@1 as c_customer_id, c_first_name@2 as c_first_name, c_last_name@3 as c_last_name, c_preferred_cust_flag@4 as c_preferred_cust_flag, c_birth_country@5 as c_birth_country, c_login@6 as c_login, c_email_address@7 as c_email_address, cs_ext_discount_amt@8 as cs_ext_discount_amt, cs_ext_sales_price@9 as cs_ext_sales_price, cs_ext_wholesale_cost@10 as cs_ext_wholesale_cost, cs_ext_list_price@11 as cs_ext_list_price, d_year@0 as d_year] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, cs_sold_date_sk@7)], projection=[d_year@1, c_customer_id@3, c_first_name@4, c_last_name@5, c_preferred_cust_flag@6, c_birth_country@7, c_login@8, c_email_address@9, cs_ext_discount_amt@11, cs_ext_sales_price@12, cs_ext_wholesale_cost@13, cs_ext_list_price@14] + │ CoalescePartitionsExec + │ [Stage 17] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(customer.c_customer_sk AS Float64)@8, cs_bill_customer_sk@1)], projection=[c_customer_id@1, c_first_name@2, c_last_name@3, c_preferred_cust_flag@4, c_birth_country@5, c_login@6, c_email_address@7, cs_sold_date_sk@9, cs_ext_discount_amt@11, cs_ext_sales_price@12, cs_ext_wholesale_cost@13, cs_ext_list_price@14] + │ CoalescePartitionsExec + │ [Stage 18] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_ext_discount_amt, cs_ext_sales_price, cs_ext_wholesale_cost, cs_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 17 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_year@1 = 2002 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] + └────────────────────────────────────────────────── + ┌───── Stage 18 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, c_customer_id@1 as c_customer_id, c_first_name@2 as c_first_name, c_last_name@3 as c_last_name, c_preferred_cust_flag@4 as c_preferred_cust_flag, c_birth_country@5 as c_birth_country, c_login@6 as c_login, c_email_address@7 as c_email_address, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 23 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([c_customer_id@0, c_first_name@1, c_last_name@2, c_preferred_cust_flag@3, c_birth_country@4, c_login@5, c_email_address@6, d_year@7], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, c_preferred_cust_flag@3 as c_preferred_cust_flag, c_birth_country@4 as c_birth_country, c_login@5 as c_login, c_email_address@6 as c_email_address, d_year@11 as d_year], aggr=[sum(web_sales.ws_ext_list_price - web_sales.ws_ext_wholesale_cost - web_sales.ws_ext_discount_amt + web_sales.ws_ext_sales_price / Int64(2))], ordering_mode=PartiallySorted([7]) + │ ProjectionExec: expr=[c_customer_id@1 as c_customer_id, c_first_name@2 as c_first_name, c_last_name@3 as c_last_name, c_preferred_cust_flag@4 as c_preferred_cust_flag, c_birth_country@5 as c_birth_country, c_login@6 as c_login, c_email_address@7 as c_email_address, ws_ext_discount_amt@8 as ws_ext_discount_amt, ws_ext_sales_price@9 as ws_ext_sales_price, ws_ext_wholesale_cost@10 as ws_ext_wholesale_cost, ws_ext_list_price@11 as ws_ext_list_price, d_year@0 as d_year] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, ws_sold_date_sk@7)], projection=[d_year@1, c_customer_id@3, c_first_name@4, c_last_name@5, c_preferred_cust_flag@6, c_birth_country@7, c_login@8, c_email_address@9, ws_ext_discount_amt@11, ws_ext_sales_price@12, ws_ext_wholesale_cost@13, ws_ext_list_price@14] + │ CoalescePartitionsExec + │ [Stage 21] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(customer.c_customer_sk AS Float64)@8, ws_bill_customer_sk@1)], projection=[c_customer_id@1, c_first_name@2, c_last_name@3, c_preferred_cust_flag@4, c_birth_country@5, c_login@6, c_email_address@7, ws_sold_date_sk@9, ws_ext_discount_amt@11, ws_ext_sales_price@12, ws_ext_wholesale_cost@13, ws_ext_list_price@14] + │ CoalescePartitionsExec + │ [Stage 22] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_bill_customer_sk, ws_ext_discount_amt, ws_ext_sales_price, ws_ext_wholesale_cost, ws_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 21 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_year@1 = 2002 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] + └────────────────────────────────────────────────── + ┌───── Stage 22 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, c_customer_id@1 as c_customer_id, c_first_name@2 as c_first_name, c_last_name@3 as c_last_name, c_preferred_cust_flag@4 as c_preferred_cust_flag, c_birth_country@5 as c_birth_country, c_login@6 as c_login, c_email_address@7 as c_email_address, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_5() -> Result<()> { + let display = test_tpcds_query("q5").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [channel@0 ASC, id@1 ASC], fetch=100 + │ SortExec: TopK(fetch=100), expr=[channel@0 ASC, id@1 ASC], preserve_partitioning=[true] + │ ProjectionExec: expr=[channel@0 as channel, id@1 as id, sum(x.sales)@3 as sales, sum(x.returns_)@4 as returns_, sum(x.profit)@5 as profit] + │ AggregateExec: mode=FinalPartitioned, gby=[channel@0 as channel, id@1 as id, __grouping_id@2 as __grouping_id], aggr=[sum(x.sales), sum(x.returns_), sum(x.profit)] + │ [Stage 12] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + └────────────────────────────────────────────────── + ┌───── Stage 12 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] t3:[p0..p2] + │ RepartitionExec: partitioning=Hash([channel@0, id@1, __grouping_id@2], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[(NULL as channel, NULL as id), (channel@0 as channel, NULL as id), (channel@0 as channel, id@1 as id)], aggr=[sum(x.sales), sum(x.returns_), sum(x.profit)] + │ DistributedUnionExec: t0:[c0] t1:[c1] t2:[c2(0/2)] t3:[c2(1/2)] + │ ProjectionExec: expr=[store channel as channel, concat(store, s_store_id@0) as id, sum(salesreturns.sales_price)@1 as sales, sum(salesreturns.return_amt)@3 as returns_, sum(salesreturns.profit)@2 - sum(salesreturns.net_loss)@4 as profit] + │ AggregateExec: mode=FinalPartitioned, gby=[s_store_id@0 as s_store_id], aggr=[sum(salesreturns.sales_price), sum(salesreturns.profit), sum(salesreturns.return_amt), sum(salesreturns.net_loss)] + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ ProjectionExec: expr=[catalog channel as channel, concat(catalog_page, cp_catalog_page_id@0) as id, sum(salesreturns.sales_price)@1 as sales, sum(salesreturns.return_amt)@3 as returns_, sum(salesreturns.profit)@2 - sum(salesreturns.net_loss)@4 as profit] + │ AggregateExec: mode=FinalPartitioned, gby=[cp_catalog_page_id@0 as cp_catalog_page_id], aggr=[sum(salesreturns.sales_price), sum(salesreturns.profit), sum(salesreturns.return_amt), sum(salesreturns.net_loss)] + │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + │ ProjectionExec: expr=[web channel as channel, concat(web_site, web_site_id@0) as id, sum(salesreturns.sales_price)@1 as sales, sum(salesreturns.return_amt)@3 as returns_, sum(salesreturns.profit)@2 - sum(salesreturns.net_loss)@4 as profit] + │ AggregateExec: mode=FinalPartitioned, gby=[web_site_id@0 as web_site_id], aggr=[sum(salesreturns.sales_price), sum(salesreturns.profit), sum(salesreturns.return_amt), sum(salesreturns.net_loss)] + │ [Stage 11] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] t3:[p0..p2] + │ RepartitionExec: partitioning=Hash([s_store_id@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[s_store_id@4 as s_store_id], aggr=[sum(salesreturns.sales_price), sum(salesreturns.profit), sum(salesreturns.return_amt), sum(salesreturns.net_loss)] + │ ProjectionExec: expr=[sales_price@1 as sales_price, profit@2 as profit, return_amt@3 as return_amt, net_loss@4 as net_loss, s_store_id@0 as s_store_id] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@2, store_sk@0)], projection=[s_store_id@1, sales_price@4, profit@5, return_amt@6, net_loss@7] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, date_sk@1)], projection=[store_sk@2, sales_price@4, profit@5, return_amt@6, net_loss@7] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=2 + │ DistributedUnionExec: t0:[c0(0/2)] t1:[c0(1/2)] t2:[c1(0/2)] t3:[c1(1/2)] + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_store_sk@7 as store_sk, ss_sold_date_sk@0 as date_sk, ss_ext_sales_price@15 as sales_price, CAST(ss_net_profit@22 AS Decimal128(7, 2)) as profit, Some(0),7,2 as return_amt, Some(0),7,2 as net_loss], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], ...]}, projection=[sr_store_sk@7 as store_sk, sr_returned_date_sk@0 as date_sk, Some(0),7,2 as sales_price, Some(0),7,2 as profit, sr_return_amt@11 as return_amt, CAST(sr_net_loss@19 AS Decimal128(7, 2)) as net_loss], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p7] t1:[p8..p15] + │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_store_id, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p11] t1:[p12..p23] + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-06, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-06, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-06, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] t3:[p0..p2] + │ RepartitionExec: partitioning=Hash([cp_catalog_page_id@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[cp_catalog_page_id@4 as cp_catalog_page_id], aggr=[sum(salesreturns.sales_price), sum(salesreturns.profit), sum(salesreturns.return_amt), sum(salesreturns.net_loss)] + │ ProjectionExec: expr=[sales_price@1 as sales_price, profit@2 as profit, return_amt@3 as return_amt, net_loss@4 as net_loss, cp_catalog_page_id@0 as cp_catalog_page_id] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(catalog_page.cp_catalog_page_sk AS Float64)@2, page_sk@0)], projection=[cp_catalog_page_id@1, sales_price@4, profit@5, return_amt@6, net_loss@7] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, date_sk@1)], projection=[page_sk@2, sales_price@4, profit@5, return_amt@6, net_loss@7] + │ CoalescePartitionsExec + │ [Stage 5] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=2 + │ DistributedUnionExec: t0:[c0(0/2)] t1:[c0(1/2)] t2:[c1(0/2)] t3:[c1(1/2)] + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_catalog_page_sk@12 as page_sk, cs_sold_date_sk@0 as date_sk, cs_ext_sales_price@23 as sales_price, cs_net_profit@33 as profit, Some(0),7,2 as return_amt, Some(0),7,2 as net_loss], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], ...]}, projection=[cr_catalog_page_sk@12 as page_sk, CAST(cr_returned_date_sk@0 AS Float64) as date_sk, Some(0),7,2 as sales_price, Some(0),7,2 as profit, cr_return_amount@18 as return_amt, cr_net_loss@26 as net_loss], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p11] t1:[p12..p23] + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 + │ ProjectionExec: expr=[cp_catalog_page_sk@0 as cp_catalog_page_sk, cp_catalog_page_id@1 as cp_catalog_page_id, CAST(cp_catalog_page_sk@0 AS Float64) as CAST(catalog_page.cp_catalog_page_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_page/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/catalog_page/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/catalog_page/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/catalog_page/part-3.parquet]]}, projection=[cp_catalog_page_sk, cp_catalog_page_id], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p11] t1:[p12..p23] + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-06, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-06, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-06, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 11 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] t3:[p0..p5] + │ RepartitionExec: partitioning=Hash([web_site_id@0], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[web_site_id@4 as web_site_id], aggr=[sum(salesreturns.sales_price), sum(salesreturns.profit), sum(salesreturns.return_amt), sum(salesreturns.net_loss)] + │ ProjectionExec: expr=[sales_price@1 as sales_price, profit@2 as profit, return_amt@3 as return_amt, net_loss@4 as net_loss, web_site_id@0 as web_site_id] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(web_site.web_site_sk AS Float64)@2, wsr_web_site_sk@0)], projection=[web_site_id@1, sales_price@4, profit@5, return_amt@6, net_loss@7] + │ CoalescePartitionsExec + │ [Stage 7] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=8, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, date_sk@1)], projection=[wsr_web_site_sk@2, sales_price@4, profit@5, return_amt@6, net_loss@7] + │ CoalescePartitionsExec + │ [Stage 8] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=2 + │ DistributedUnionExec: t0:[c0(0/2)] t1:[c0(1/2)] t2:[c1(0/2)] t3:[c1(1/2)] + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_web_site_sk@13 as wsr_web_site_sk, ws_sold_date_sk@0 as date_sk, ws_ext_sales_price@23 as sales_price, ws_net_profit@33 as profit, Some(0),7,2 as return_amt, Some(0),7,2 as net_loss], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ ProjectionExec: expr=[ws_web_site_sk@3 as wsr_web_site_sk, wr_returned_date_sk@0 as date_sk, Some(0),7,2 as sales_price, Some(0),7,2 as profit, wr_return_amt@1 as return_amt, wr_net_loss@2 as net_loss] + │ HashJoinExec: mode=Partitioned, join_type=Left, on=[(wr_item_sk@1, ws_item_sk@0), (wr_order_number@2, ws_order_number@2)], projection=[wr_returned_date_sk@0, wr_return_amt@3, wr_net_loss@4, ws_web_site_sk@6] + │ [Stage 9] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 10] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p7] t1:[p8..p15] + │ BroadcastExec: input_partitions=2, consumer_tasks=4, output_partitions=8 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_site/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_site/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_site/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_site/part-3.parquet]]}, projection=[web_site_sk, web_site_id, CAST(web_site_sk@0 AS Float64) as CAST(web_site.web_site_sk AS Float64)], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 8 ── Tasks: t0:[p0..p11] t1:[p12..p23] + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-06, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-06, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-06, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 9 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ RepartitionExec: partitioning=Hash([wr_item_sk@1, wr_order_number@2], 6), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet]]}, projection=[wr_returned_date_sk, wr_item_sk, wr_order_number, wr_return_amt, wr_net_loss], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 10 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([ws_item_sk@0, ws_order_number@2], 6), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_item_sk, ws_web_site_sk, ws_order_number], file_type=parquet + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_6() -> Result<()> { + let display = test_tpcds_query("q6").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [cnt@1 ASC, state@0 ASC], fetch=100 + │ SortExec: TopK(fetch=100), expr=[cnt@1 ASC, state@0 ASC], preserve_partitioning=[true] + │ ProjectionExec: expr=[ca_state@0 as state, count(Int64(1))@1 as cnt] + │ FilterExec: count(Int64(1))@1 >= 10 + │ AggregateExec: mode=FinalPartitioned, gby=[ca_state@0 as ca_state], aggr=[count(Int64(1))] + │ RepartitionExec: partitioning=Hash([ca_state@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[ca_state@0 as ca_state], aggr=[count(Int64(1))] + │ FilterExec: CAST(i_current_price@1 AS Decimal128(30, 15)) > CAST(1.2 * avg(j.i_current_price)@2 AS Decimal128(30, 15)), projection=[ca_state@0] + │ HashJoinExec: mode=CollectLeft, join_type=Left, on=[(i_category@2, i_category@1)], projection=[ca_state@0, i_current_price@1, avg(j.i_current_price)@3] + │ CoalescePartitionsExec + │ [Stage 7] => NetworkCoalesceExec: output_partitions=9, input_tasks=3 + │ ProjectionExec: expr=[CAST(avg(j.i_current_price)@1 AS Float64) as avg(j.i_current_price), i_category@0 as i_category] + │ AggregateExec: mode=FinalPartitioned, gby=[i_category@0 as i_category], aggr=[avg(j.i_current_price)] + │ [Stage 8] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p2] t1:[p3..p5] t2:[p6..p8] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_month_seq@0, d_month_seq@1)], projection=[ca_state@1, i_current_price@3, i_category@4] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=1 + │ ProjectionExec: expr=[ca_state@2 as ca_state, d_month_seq@3 as d_month_seq, i_current_price@0 as i_current_price, i_category@1 as i_category] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ss_item_sk@1)], projection=[i_current_price@1, i_category@2, ca_state@3, d_month_seq@5] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_sold_date_sk@1, CAST(d.d_date_sk AS Float64)@2)], projection=[ca_state@0, ss_item_sk@2, d_month_seq@4] + │ CoalescePartitionsExec + │ [Stage 6] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=3 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_month_seq@1 as d_month_seq, CAST(d_date_sk@0 AS Float64) as CAST(d.d_date_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_month_seq], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p8] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ AggregateExec: mode=FinalPartitioned, gby=[d_month_seq@0 as d_month_seq], aggr=[] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ RepartitionExec: partitioning=Hash([d_month_seq@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[d_month_seq@0 as d_month_seq], aggr=[] + │ FilterExec: d_year@1 = 2001 AND d_moy@2 = 1, projection=[d_month_seq@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_month_seq, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 1, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 1 AND 1 <= d_moy_max@5, required_guarantees=[d_moy in (1), d_year in (2001)] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_current_price, i_category], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p5] t1:[p6..p11] t2:[p12..p17] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(c.c_customer_sk AS Float64)@2, ss_customer_sk@2)], projection=[ca_state@0, ss_sold_date_sk@3, ss_item_sk@4] + │ CoalescePartitionsExec + │ [Stage 5] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[ca_state@0 as ca_state, c_customer_sk@1 as c_customer_sk, CAST(c_customer_sk@1 AS Float64) as CAST(c.c_customer_sk AS Float64)] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ca_address_sk@0, c_current_addr_sk@1)], projection=[ca_state@1, c_customer_sk@2] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=4, input_tasks=2 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p3] t1:[p4..p7] + │ BroadcastExec: input_partitions=2, consumer_tasks=2, output_partitions=4 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_state], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 8 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ RepartitionExec: partitioning=Hash([i_category@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_category@1 as i_category], aggr=[avg(j.i_current_price)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_current_price, i_category], file_type=parquet + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_7() -> Result<()> { + let display = test_tpcds_query("q7").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [i_item_id@0 ASC NULLS LAST], fetch=100 + │ [Stage 7] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ SortExec: TopK(fetch=100), expr=[i_item_id@0 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[i_item_id@0 as i_item_id, avg(store_sales.ss_quantity)@1 as agg1, avg(store_sales.ss_list_price)@2 as agg2, avg(store_sales.ss_coupon_amt)@3 as agg3, avg(store_sales.ss_sales_price)@4 as agg4] + │ AggregateExec: mode=FinalPartitioned, gby=[i_item_id@0 as i_item_id], aggr=[avg(store_sales.ss_quantity), avg(store_sales.ss_list_price), avg(store_sales.ss_coupon_amt), avg(store_sales.ss_sales_price)] + │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([i_item_id@0], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_item_id@4 as i_item_id], aggr=[avg(store_sales.ss_quantity), avg(store_sales.ss_list_price), avg(store_sales.ss_coupon_amt), avg(store_sales.ss_sales_price)] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(promotion.p_promo_sk AS Float64)@1, ss_promo_sk@0)], projection=[ss_quantity@3, ss_list_price@4, ss_sales_price@5, ss_coupon_amt@6, i_item_id@7] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_item_sk@0, i_item_sk@0)], projection=[ss_promo_sk@1, ss_quantity@2, ss_list_price@3, ss_sales_price@4, ss_coupon_amt@5, i_item_id@7] + │ CoalescePartitionsExec + │ [Stage 5] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=3 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[p_promo_sk@0 as p_promo_sk, CAST(p_promo_sk@0 AS Float64) as CAST(promotion.p_promo_sk AS Float64)] + │ FilterExec: p_channel_email@1 = N OR p_channel_event@2 = N, projection=[p_promo_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/promotion/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/promotion/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/promotion/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/promotion/part-3.parquet]]}, projection=[p_promo_sk, p_channel_email, p_channel_event], file_type=parquet, predicate=p_channel_email@9 = N OR p_channel_event@14 = N, pruning_predicate=p_channel_email_null_count@2 != row_count@3 AND p_channel_email_min@0 <= N AND N <= p_channel_email_max@1 OR p_channel_event_null_count@6 != row_count@3 AND p_channel_event_min@4 <= N AND N <= p_channel_event_max@5, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p8] t1:[p9..p17] t2:[p18..p26] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_item_sk@3, ss_promo_sk@4, ss_quantity@5, ss_list_price@6, ss_sales_price@7, ss_coupon_amt@8] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(customer_demographics.cd_demo_sk AS Float64)@1, ss_cdemo_sk@2)], projection=[ss_sold_date_sk@2, ss_item_sk@3, ss_promo_sk@5, ss_quantity@6, ss_list_price@7, ss_sales_price@8, ss_coupon_amt@9] + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_year@1 = 2000, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] + │ RepartitionExec: partitioning=Hash([CAST(customer_demographics.cd_demo_sk AS Float64)@1], 9), input_partitions=2 + │ ProjectionExec: expr=[cd_demo_sk@0 as cd_demo_sk, CAST(cd_demo_sk@0 AS Float64) as CAST(customer_demographics.cd_demo_sk AS Float64)] + │ FilterExec: cd_gender@1 = M AND cd_marital_status@2 = S AND cd_education_status@3 = College, projection=[cd_demo_sk@0] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], ...]}, projection=[cd_demo_sk, cd_gender, cd_marital_status, cd_education_status], file_type=parquet, predicate=cd_gender@1 = M AND cd_marital_status@2 = S AND cd_education_status@3 = College, pruning_predicate=cd_gender_null_count@2 != row_count@3 AND cd_gender_min@0 <= M AND M <= cd_gender_max@1 AND cd_marital_status_null_count@6 != row_count@3 AND cd_marital_status_min@4 <= S AND S <= cd_marital_status_max@5 AND cd_education_status_null_count@9 != row_count@3 AND cd_education_status_min@7 <= College AND College <= cd_education_status_max@8, required_guarantees=[cd_education_status in (College), cd_gender in (M), cd_marital_status in (S)] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] + │ RepartitionExec: partitioning=Hash([ss_cdemo_sk@2], 9), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_cdemo_sk, ss_promo_sk, ss_quantity, ss_list_price, ss_sales_price, ss_coupon_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_8() -> Result<()> { + let display = test_tpcds_query("q8").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [s_store_name@0 ASC NULLS LAST], fetch=100 + │ SortExec: TopK(fetch=100), expr=[s_store_name@0 ASC NULLS LAST], preserve_partitioning=[true] + │ AggregateExec: mode=FinalPartitioned, gby=[s_store_name@0 as s_store_name], aggr=[sum(store_sales.ss_net_profit)] + │ RepartitionExec: partitioning=Hash([s_store_name@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[s_store_name@1 as s_store_name], aggr=[sum(store_sales.ss_net_profit)] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(substr(store.s_zip,Int64(1),Int64(2))@3, substr(v1.ca_zip,Int64(1),Int64(2))@1)], projection=[ss_net_profit@0, s_store_name@1] + │ CoalescePartitionsExec + │ BroadcastExec: input_partitions=3, consumer_tasks=1, output_partitions=3 + │ ProjectionExec: expr=[ss_net_profit@2 as ss_net_profit, s_store_name@0 as s_store_name, s_zip@1 as s_zip, substr(s_zip@1, 1, 2) as substr(store.s_zip,Int64(1),Int64(2))] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@3, ss_store_sk@0)], projection=[s_store_name@1, s_zip@2, ss_net_profit@5] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=2, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_store_sk@3, ss_net_profit@4] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=3, input_tasks=2 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_store_sk, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ ProjectionExec: expr=[ca_zip@0 as ca_zip, substr(ca_zip@0, 1, 2) as substr(v1.ca_zip,Int64(1),Int64(2))] + │ HashJoinExec: mode=CollectLeft, join_type=LeftSemi, on=[(ca_zip@0, ca_zip@0)], NullsEqual: true + │ CoalescePartitionsExec + │ AggregateExec: mode=FinalPartitioned, gby=[ca_zip@0 as ca_zip], aggr=[] + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ ProjectionExec: expr=[substr(ca_zip@0, 1, 5) as ca_zip] + │ FilterExec: count(Int64(1))@1 > 10, projection=[ca_zip@0] + │ AggregateExec: mode=FinalPartitioned, gby=[ca_zip@0 as ca_zip], aggr=[count(Int64(1))] + │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] + │ BroadcastExec: input_partitions=2, consumer_tasks=1, output_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_store_name, s_zip, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ BroadcastExec: input_partitions=3, consumer_tasks=1, output_partitions=3 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_qoy@2 = 2 AND d_year@1 = 1998, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@10 = 2 AND d_year@6 = 1998, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 2 AND 2 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1998 AND 1998 <= d_year_max@5, required_guarantees=[d_qoy in (2), d_year in (1998)] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ RepartitionExec: partitioning=Hash([ca_zip@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[ca_zip@0 as ca_zip], aggr=[] + │ ProjectionExec: expr=[substr(ca_zip@0, 1, 5) as ca_zip] + │ FilterExec: substr(ca_zip@0, 1, 5) IN (SET) ([24128, 76232, 65084, 87816, 83926, 77556, 20548, 26231, 43848, 15126, 91137, 61265, 98294, 25782, 17920, 18426, 98235, 40081, 84093, 28577, 55565, 17183, 54601, 67897, 22752, 86284, 18376, 38607, 45200, 21756, 29741, 96765, 23932, 89360, 29839, 25989, 28898, 91068, 72550, 10390, 18845, 47770, 82636, 41367, 76638, 86198, 81312, 37126, 39192, 88424, 72175, 81426, 53672, 10445, 42666, 66864, 66708, 41248, 48583, 82276, 18842, 78890, 49448, 14089, 38122, 34425, 79077, 19849, 43285, 39861, 66162, 77610, 13695, 99543, 83444, 83041, 12305, 57665, 68341, 25003, 57834, 62878, 49130, 81096, 18840, 27700, 23470, 50412, 21195, 16021, 76107, 71954, 68309, 18119, 98359, 64544, 10336, 86379, 27068, 39736, 98569, 28915, 24206, 56529, 57647, 54917, 42961, 91110, 63981, 14922, 36420, 23006, 67467, 32754, 30903, 20260, 31671, 51798, 72325, 85816, 68621, 13955, 36446, 41766, 68806, 16725, 15146, 22744, 35850, 88086, 51649, 18270, 52867, 39972, 96976, 63792, 11376, 94898, 13595, 10516, 90225, 58943, 39371, 94945, 28587, 96576, 57855, 28488, 26105, 83933, 25858, 34322, 44438, 73171, 30122, 34102, 22685, 71256, 78451, 54364, 13354, 45375, 40558, 56458, 28286, 45266, 47305, 69399, 83921, 26233, 11101, 15371, 69913, 35942, 15882, 25631, 24610, 44165, 99076, 33786, 70738, 26653, 14328, 72305, 62496, 22152, 10144, 64147, 48425, 14663, 21076, 18799, 30450, 63089, 81019, 68893, 24996, 51200, 51211, 45692, 92712, 70466, 79994, 22437, 25280, 38935, 71791, 73134, 56571, 14060, 19505, 72425, 56575, 74351, 68786, 51650, 20004, 18383, 76614, 11634, 18906, 15765, 41368, 73241, 76698, 78567, 97189, 28545, 76231, 75691, 22246, 51061, 90578, 56691, 68014, 51103, 94167, 57047, 14867, 73520, 15734, 63435, 25733, 35474, 24676, 94627, 53535, 17879, 15559, 53268, 59166, 11928, 59402, 33282, 45721, 43933, 68101, 33515, 36634, 71286, 19736, 58058, 55253, 67473, 41918, 19515, 36495, 19430, 22351, 77191, 91393, 49156, 50298, 87501, 18652, 53179, 18767, 63193, 23968, 65164, 68880, 21286, 72823, 58470, 67301, 13394, 31016, 70372, 67030, 40604, 24317, 45748, 39127, 26065, 77721, 31029, 31880, 60576, 24671, 45549, 13376, 50016, 33123, 19769, 22927, 97789, 46081, 72151, 15723, 46136, 51949, 68100, 96888, 64528, 14171, 79777, 28709, 11489, 25103, 32213, 78668, 22245, 15798, 27156, 37930, 62971, 21337, 51622, 67853, 10567, 38415, 15455, 58263, 42029, 60279, 37125, 56240, 88190, 50308, 26859, 64457, 89091, 82136, 62377, 36233, 63837, 58078, 17043, 30010, 60099, 28810, 98025, 29178, 87343, 73273, 30469, 64034, 39516, 86057, 21309, 90257, 67875, 40162, 11356, 73650, 61810, 72013, 30431, 22461, 19512, 13375, 55307, 30625, 83849, 68908, 26689, 96451, 38193, 46820, 88885, 84935, 69035, 83144, 47537, 56616, 94983, 48033, 69952, 25486, 61547, 27385, 61860, 58048, 56910, 16807, 17871, 35258, 31387, 35458, 35576]) + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_zip], file_type=parquet, predicate=substr(ca_zip@9, 1, 5) IN (SET) ([24128, 76232, 65084, 87816, 83926, 77556, 20548, 26231, 43848, 15126, 91137, 61265, 98294, 25782, 17920, 18426, 98235, 40081, 84093, 28577, 55565, 17183, 54601, 67897, 22752, 86284, 18376, 38607, 45200, 21756, 29741, 96765, 23932, 89360, 29839, 25989, 28898, 91068, 72550, 10390, 18845, 47770, 82636, 41367, 76638, 86198, 81312, 37126, 39192, 88424, 72175, 81426, 53672, 10445, 42666, 66864, 66708, 41248, 48583, 82276, 18842, 78890, 49448, 14089, 38122, 34425, 79077, 19849, 43285, 39861, 66162, 77610, 13695, 99543, 83444, 83041, 12305, 57665, 68341, 25003, 57834, 62878, 49130, 81096, 18840, 27700, 23470, 50412, 21195, 16021, 76107, 71954, 68309, 18119, 98359, 64544, 10336, 86379, 27068, 39736, 98569, 28915, 24206, 56529, 57647, 54917, 42961, 91110, 63981, 14922, 36420, 23006, 67467, 32754, 30903, 20260, 31671, 51798, 72325, 85816, 68621, 13955, 36446, 41766, 68806, 16725, 15146, 22744, 35850, 88086, 51649, 18270, 52867, 39972, 96976, 63792, 11376, 94898, 13595, 10516, 90225, 58943, 39371, 94945, 28587, 96576, 57855, 28488, 26105, 83933, 25858, 34322, 44438, 73171, 30122, 34102, 22685, 71256, 78451, 54364, 13354, 45375, 40558, 56458, 28286, 45266, 47305, 69399, 83921, 26233, 11101, 15371, 69913, 35942, 15882, 25631, 24610, 44165, 99076, 33786, 70738, 26653, 14328, 72305, 62496, 22152, 10144, 64147, 48425, 14663, 21076, 18799, 30450, 63089, 81019, 68893, 24996, 51200, 51211, 45692, 92712, 70466, 79994, 22437, 25280, 38935, 71791, 73134, 56571, 14060, 19505, 72425, 56575, 74351, 68786, 51650, 20004, 18383, 76614, 11634, 18906, 15765, 41368, 73241, 76698, 78567, 97189, 28545, 76231, 75691, 22246, 51061, 90578, 56691, 68014, 51103, 94167, 57047, 14867, 73520, 15734, 63435, 25733, 35474, 24676, 94627, 53535, 17879, 15559, 53268, 59166, 11928, 59402, 33282, 45721, 43933, 68101, 33515, 36634, 71286, 19736, 58058, 55253, 67473, 41918, 19515, 36495, 19430, 22351, 77191, 91393, 49156, 50298, 87501, 18652, 53179, 18767, 63193, 23968, 65164, 68880, 21286, 72823, 58470, 67301, 13394, 31016, 70372, 67030, 40604, 24317, 45748, 39127, 26065, 77721, 31029, 31880, 60576, 24671, 45549, 13376, 50016, 33123, 19769, 22927, 97789, 46081, 72151, 15723, 46136, 51949, 68100, 96888, 64528, 14171, 79777, 28709, 11489, 25103, 32213, 78668, 22245, 15798, 27156, 37930, 62971, 21337, 51622, 67853, 10567, 38415, 15455, 58263, 42029, 60279, 37125, 56240, 88190, 50308, 26859, 64457, 89091, 82136, 62377, 36233, 63837, 58078, 17043, 30010, 60099, 28810, 98025, 29178, 87343, 73273, 30469, 64034, 39516, 86057, 21309, 90257, 67875, 40162, 11356, 73650, 61810, 72013, 30431, 22461, 19512, 13375, 55307, 30625, 83849, 68908, 26689, 96451, 38193, 46820, 88885, 84935, 69035, 83144, 47537, 56616, 94983, 48033, 69952, 25486, 61547, 27385, 61860, 58048, 56910, 16807, 17871, 35258, 31387, 35458, 35576]) + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ RepartitionExec: partitioning=Hash([ca_zip@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[ca_zip@0 as ca_zip], aggr=[count(Int64(1))] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_current_addr_sk@0, ca_address_sk@0)], projection=[ca_zip@2] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=6, input_tasks=2 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_zip], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=3, consumer_tasks=2, output_partitions=6 + │ FilterExec: c_preferred_cust_flag@1 = Y, projection=[c_current_addr_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_current_addr_sk, c_preferred_cust_flag], file_type=parquet, predicate=c_preferred_cust_flag@10 = Y, pruning_predicate=c_preferred_cust_flag_null_count@2 != row_count@3 AND c_preferred_cust_flag_min@0 <= Y AND Y <= c_preferred_cust_flag_max@1, required_guarantees=[c_preferred_cust_flag in (Y)] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_9() -> Result<()> { + let display = test_tpcds_query("q9").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[CASE WHEN count(*)@0 > 74129 THEN avg(store_sales.ss_ext_discount_amt)@1 ELSE avg(store_sales.ss_net_paid)@2 END as bucket1, CASE WHEN count(*)@3 > 122840 THEN avg(store_sales.ss_ext_discount_amt)@4 ELSE avg(store_sales.ss_net_paid)@5 END as bucket2, CASE WHEN count(*)@6 > 56580 THEN avg(store_sales.ss_ext_discount_amt)@7 ELSE avg(store_sales.ss_net_paid)@8 END as bucket3, CASE WHEN count(*)@9 > 10097 THEN avg(store_sales.ss_ext_discount_amt)@10 ELSE avg(store_sales.ss_net_paid)@11 END as bucket4, CASE WHEN count(*)@12 > 165306 THEN avg(store_sales.ss_ext_discount_amt)@13 ELSE avg(store_sales.ss_net_paid)@14 END as bucket5] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=1 + │ NestedLoopJoinExec: join_type=Left + │ NestedLoopJoinExec: join_type=Left + │ NestedLoopJoinExec: join_type=Left + │ NestedLoopJoinExec: join_type=Left + │ NestedLoopJoinExec: join_type=Left + │ NestedLoopJoinExec: join_type=Left + │ NestedLoopJoinExec: join_type=Left + │ NestedLoopJoinExec: join_type=Left + │ NestedLoopJoinExec: join_type=Left + │ NestedLoopJoinExec: join_type=Left + │ NestedLoopJoinExec: join_type=Left + │ NestedLoopJoinExec: join_type=Left + │ NestedLoopJoinExec: join_type=Left + │ NestedLoopJoinExec: join_type=Left + │ NestedLoopJoinExec: join_type=Left + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[count(Int64(1))@0 as count(*)] + │ AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=3 + │ AggregateExec: mode=Final, gby=[], aggr=[avg(store_sales.ss_ext_discount_amt)] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkCoalesceExec: output_partitions=6, input_tasks=3 + │ AggregateExec: mode=Final, gby=[], aggr=[avg(store_sales.ss_net_paid)] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkCoalesceExec: output_partitions=6, input_tasks=3 + │ ProjectionExec: expr=[count(Int64(1))@0 as count(*)] + │ AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))] + │ CoalescePartitionsExec + │ [Stage 5] => NetworkCoalesceExec: output_partitions=6, input_tasks=3 + │ AggregateExec: mode=Final, gby=[], aggr=[avg(store_sales.ss_ext_discount_amt)] + │ CoalescePartitionsExec + │ [Stage 6] => NetworkCoalesceExec: output_partitions=6, input_tasks=3 + │ AggregateExec: mode=Final, gby=[], aggr=[avg(store_sales.ss_net_paid)] + │ CoalescePartitionsExec + │ [Stage 7] => NetworkCoalesceExec: output_partitions=6, input_tasks=3 + │ ProjectionExec: expr=[count(Int64(1))@0 as count(*)] + │ AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))] + │ CoalescePartitionsExec + │ [Stage 8] => NetworkCoalesceExec: output_partitions=6, input_tasks=3 + │ AggregateExec: mode=Final, gby=[], aggr=[avg(store_sales.ss_ext_discount_amt)] + │ CoalescePartitionsExec + │ [Stage 9] => NetworkCoalesceExec: output_partitions=6, input_tasks=3 + │ AggregateExec: mode=Final, gby=[], aggr=[avg(store_sales.ss_net_paid)] + │ CoalescePartitionsExec + │ [Stage 10] => NetworkCoalesceExec: output_partitions=6, input_tasks=3 + │ ProjectionExec: expr=[count(Int64(1))@0 as count(*)] + │ AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))] + │ CoalescePartitionsExec + │ [Stage 11] => NetworkCoalesceExec: output_partitions=6, input_tasks=3 + │ AggregateExec: mode=Final, gby=[], aggr=[avg(store_sales.ss_ext_discount_amt)] + │ CoalescePartitionsExec + │ [Stage 12] => NetworkCoalesceExec: output_partitions=6, input_tasks=3 + │ AggregateExec: mode=Final, gby=[], aggr=[avg(store_sales.ss_net_paid)] + │ CoalescePartitionsExec + │ [Stage 13] => NetworkCoalesceExec: output_partitions=6, input_tasks=3 + │ ProjectionExec: expr=[count(Int64(1))@0 as count(*)] + │ AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))] + │ CoalescePartitionsExec + │ [Stage 14] => NetworkCoalesceExec: output_partitions=6, input_tasks=3 + │ AggregateExec: mode=Final, gby=[], aggr=[avg(store_sales.ss_ext_discount_amt)] + │ CoalescePartitionsExec + │ [Stage 15] => NetworkCoalesceExec: output_partitions=6, input_tasks=3 + │ AggregateExec: mode=Final, gby=[], aggr=[avg(store_sales.ss_net_paid)] + │ CoalescePartitionsExec + │ [Stage 16] => NetworkCoalesceExec: output_partitions=6, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ ProjectionExec: expr=[] + │ FilterExec: r_reason_sk@0 = 1 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/reason/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/reason/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/reason/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/reason/part-3.parquet]]}, projection=[r_reason_sk], file_type=parquet, predicate=r_reason_sk@0 = 1, pruning_predicate=r_reason_sk_null_count@2 != row_count@3 AND r_reason_sk_min@0 <= 1 AND 1 <= r_reason_sk_max@1, required_guarantees=[r_reason_sk in (1)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] + │ AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] + │ ProjectionExec: expr=[] + │ FilterExec: ss_quantity@0 >= 1 AND ss_quantity@0 <= 20 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_quantity], file_type=parquet, predicate=ss_quantity@10 >= 1 AND ss_quantity@10 <= 20, pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 1 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 20, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] + │ AggregateExec: mode=Partial, gby=[], aggr=[avg(store_sales.ss_ext_discount_amt)] + │ FilterExec: ss_quantity@0 >= 1 AND ss_quantity@0 <= 20, projection=[ss_ext_discount_amt@1] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_quantity, ss_ext_discount_amt], file_type=parquet, predicate=ss_quantity@10 >= 1 AND ss_quantity@10 <= 20, pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 1 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 20, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] + │ AggregateExec: mode=Partial, gby=[], aggr=[avg(store_sales.ss_net_paid)] + │ FilterExec: ss_quantity@0 >= 1 AND ss_quantity@0 <= 20, projection=[ss_net_paid@1] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_quantity, ss_net_paid], file_type=parquet, predicate=ss_quantity@10 >= 1 AND ss_quantity@10 <= 20, pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 1 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 20, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] + │ AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] + │ ProjectionExec: expr=[] + │ FilterExec: ss_quantity@0 >= 21 AND ss_quantity@0 <= 40 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_quantity], file_type=parquet, predicate=ss_quantity@10 >= 21 AND ss_quantity@10 <= 40, pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 21 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 40, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] + │ AggregateExec: mode=Partial, gby=[], aggr=[avg(store_sales.ss_ext_discount_amt)] + │ FilterExec: ss_quantity@0 >= 21 AND ss_quantity@0 <= 40, projection=[ss_ext_discount_amt@1] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_quantity, ss_ext_discount_amt], file_type=parquet, predicate=ss_quantity@10 >= 21 AND ss_quantity@10 <= 40, pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 21 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 40, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] + │ AggregateExec: mode=Partial, gby=[], aggr=[avg(store_sales.ss_net_paid)] + │ FilterExec: ss_quantity@0 >= 21 AND ss_quantity@0 <= 40, projection=[ss_net_paid@1] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_quantity, ss_net_paid], file_type=parquet, predicate=ss_quantity@10 >= 21 AND ss_quantity@10 <= 40, pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 21 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 40, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 8 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] + │ AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] + │ ProjectionExec: expr=[] + │ FilterExec: ss_quantity@0 >= 41 AND ss_quantity@0 <= 60 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_quantity], file_type=parquet, predicate=ss_quantity@10 >= 41 AND ss_quantity@10 <= 60, pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 41 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 60, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 9 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] + │ AggregateExec: mode=Partial, gby=[], aggr=[avg(store_sales.ss_ext_discount_amt)] + │ FilterExec: ss_quantity@0 >= 41 AND ss_quantity@0 <= 60, projection=[ss_ext_discount_amt@1] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_quantity, ss_ext_discount_amt], file_type=parquet, predicate=ss_quantity@10 >= 41 AND ss_quantity@10 <= 60, pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 41 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 60, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 10 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] + │ AggregateExec: mode=Partial, gby=[], aggr=[avg(store_sales.ss_net_paid)] + │ FilterExec: ss_quantity@0 >= 41 AND ss_quantity@0 <= 60, projection=[ss_net_paid@1] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_quantity, ss_net_paid], file_type=parquet, predicate=ss_quantity@10 >= 41 AND ss_quantity@10 <= 60, pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 41 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 60, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 11 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] + │ AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] + │ ProjectionExec: expr=[] + │ FilterExec: ss_quantity@0 >= 61 AND ss_quantity@0 <= 80 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_quantity], file_type=parquet, predicate=ss_quantity@10 >= 61 AND ss_quantity@10 <= 80, pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 61 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 80, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 12 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] + │ AggregateExec: mode=Partial, gby=[], aggr=[avg(store_sales.ss_ext_discount_amt)] + │ FilterExec: ss_quantity@0 >= 61 AND ss_quantity@0 <= 80, projection=[ss_ext_discount_amt@1] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_quantity, ss_ext_discount_amt], file_type=parquet, predicate=ss_quantity@10 >= 61 AND ss_quantity@10 <= 80, pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 61 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 80, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 13 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] + │ AggregateExec: mode=Partial, gby=[], aggr=[avg(store_sales.ss_net_paid)] + │ FilterExec: ss_quantity@0 >= 61 AND ss_quantity@0 <= 80, projection=[ss_net_paid@1] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_quantity, ss_net_paid], file_type=parquet, predicate=ss_quantity@10 >= 61 AND ss_quantity@10 <= 80, pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 61 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 80, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 14 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] + │ AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] + │ ProjectionExec: expr=[] + │ FilterExec: ss_quantity@0 >= 81 AND ss_quantity@0 <= 100 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_quantity], file_type=parquet, predicate=ss_quantity@10 >= 81 AND ss_quantity@10 <= 100, pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 81 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 100, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 15 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] + │ AggregateExec: mode=Partial, gby=[], aggr=[avg(store_sales.ss_ext_discount_amt)] + │ FilterExec: ss_quantity@0 >= 81 AND ss_quantity@0 <= 100, projection=[ss_ext_discount_amt@1] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_quantity, ss_ext_discount_amt], file_type=parquet, predicate=ss_quantity@10 >= 81 AND ss_quantity@10 <= 100, pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 81 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 100, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 16 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] + │ AggregateExec: mode=Partial, gby=[], aggr=[avg(store_sales.ss_net_paid)] + │ FilterExec: ss_quantity@0 >= 81 AND ss_quantity@0 <= 100, projection=[ss_net_paid@1] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_quantity, ss_net_paid], file_type=parquet, predicate=ss_quantity@10 >= 81 AND ss_quantity@10 <= 100, pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 81 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 100, required_guarantees=[] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_10() -> Result<()> { + let display = test_tpcds_query("q10").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [cd_gender@0 ASC NULLS LAST, cd_marital_status@1 ASC NULLS LAST, cd_education_status@2 ASC NULLS LAST, cd_purchase_estimate@4 ASC NULLS LAST, cd_credit_rating@6 ASC NULLS LAST, cd_dep_count@8 ASC NULLS LAST, cd_dep_employed_count@10 ASC NULLS LAST, cd_dep_college_count@12 ASC NULLS LAST], fetch=100 + │ SortExec: TopK(fetch=100), expr=[cd_gender@0 ASC NULLS LAST, cd_marital_status@1 ASC NULLS LAST, cd_education_status@2 ASC NULLS LAST, cd_purchase_estimate@4 ASC NULLS LAST, cd_credit_rating@6 ASC NULLS LAST, cd_dep_count@8 ASC NULLS LAST, cd_dep_employed_count@10 ASC NULLS LAST, cd_dep_college_count@12 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[cd_gender@0 as cd_gender, cd_marital_status@1 as cd_marital_status, cd_education_status@2 as cd_education_status, count(Int64(1))@8 as cnt1, cd_purchase_estimate@3 as cd_purchase_estimate, count(Int64(1))@8 as cnt2, cd_credit_rating@4 as cd_credit_rating, count(Int64(1))@8 as cnt3, cd_dep_count@5 as cd_dep_count, count(Int64(1))@8 as cnt4, cd_dep_employed_count@6 as cd_dep_employed_count, count(Int64(1))@8 as cnt5, cd_dep_college_count@7 as cd_dep_college_count, count(Int64(1))@8 as cnt6] + │ AggregateExec: mode=FinalPartitioned, gby=[cd_gender@0 as cd_gender, cd_marital_status@1 as cd_marital_status, cd_education_status@2 as cd_education_status, cd_purchase_estimate@3 as cd_purchase_estimate, cd_credit_rating@4 as cd_credit_rating, cd_dep_count@5 as cd_dep_count, cd_dep_employed_count@6 as cd_dep_employed_count, cd_dep_college_count@7 as cd_dep_college_count], aggr=[count(Int64(1))] + │ [Stage 9] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 9 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] + │ RepartitionExec: partitioning=Hash([cd_gender@0, cd_marital_status@1, cd_education_status@2, cd_purchase_estimate@3, cd_credit_rating@4, cd_dep_count@5, cd_dep_employed_count@6, cd_dep_college_count@7], 3), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[cd_gender@0 as cd_gender, cd_marital_status@1 as cd_marital_status, cd_education_status@2 as cd_education_status, cd_purchase_estimate@3 as cd_purchase_estimate, cd_credit_rating@4 as cd_credit_rating, cd_dep_count@5 as cd_dep_count, cd_dep_employed_count@6 as cd_dep_employed_count, cd_dep_college_count@7 as cd_dep_college_count], aggr=[count(Int64(1))] + │ FilterExec: mark@8 OR mark@9, projection=[cd_gender@0, cd_marital_status@1, cd_education_status@2, cd_purchase_estimate@3, cd_credit_rating@4, cd_dep_count@5, cd_dep_employed_count@6, cd_dep_college_count@7] + │ HashJoinExec: mode=CollectLeft, join_type=RightMark, on=[(cs_ship_customer_sk@0, CAST(c.c_customer_sk AS Float64)@10)], projection=[cd_gender@1, cd_marital_status@2, cd_education_status@3, cd_purchase_estimate@4, cd_credit_rating@5, cd_dep_count@6, cd_dep_employed_count@7, cd_dep_college_count@8, mark@9, mark@11] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=3 + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, cd_gender@1 as cd_gender, cd_marital_status@2 as cd_marital_status, cd_education_status@3 as cd_education_status, cd_purchase_estimate@4 as cd_purchase_estimate, cd_credit_rating@5 as cd_credit_rating, cd_dep_count@6 as cd_dep_count, cd_dep_employed_count@7 as cd_dep_employed_count, cd_dep_college_count@8 as cd_dep_college_count, mark@9 as mark, CAST(c_customer_sk@0 AS Float64) as CAST(c.c_customer_sk AS Float64)] + │ HashJoinExec: mode=CollectLeft, join_type=RightMark, on=[(ws_bill_customer_sk@0, CAST(c.c_customer_sk AS Float64)@9)], projection=[c_customer_sk@0, cd_gender@1, cd_marital_status@2, cd_education_status@3, cd_purchase_estimate@4, cd_credit_rating@5, cd_dep_count@6, cd_dep_employed_count@7, cd_dep_college_count@8, mark@10] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=3 + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, cd_gender@1 as cd_gender, cd_marital_status@2 as cd_marital_status, cd_education_status@3 as cd_education_status, cd_purchase_estimate@4 as cd_purchase_estimate, cd_credit_rating@5 as cd_credit_rating, cd_dep_count@6 as cd_dep_count, cd_dep_employed_count@7 as cd_dep_employed_count, cd_dep_college_count@8 as cd_dep_college_count, CAST(c_customer_sk@0 AS Float64) as CAST(c.c_customer_sk AS Float64)] + │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(ss_customer_sk@0, CAST(c.c_customer_sk AS Float64)@9)], projection=[c_customer_sk@0, cd_gender@1, cd_marital_status@2, cd_education_status@3, cd_purchase_estimate@4, cd_credit_rating@5, cd_dep_count@6, cd_dep_employed_count@7, cd_dep_college_count@8] + │ CoalescePartitionsExec + │ [Stage 6] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=3 + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, cd_gender@1 as cd_gender, cd_marital_status@2 as cd_marital_status, cd_education_status@3 as cd_education_status, cd_purchase_estimate@4 as cd_purchase_estimate, cd_credit_rating@5 as cd_credit_rating, cd_dep_count@6 as cd_dep_count, cd_dep_employed_count@7 as cd_dep_employed_count, cd_dep_college_count@8 as cd_dep_college_count, CAST(c_customer_sk@0 AS Float64) as CAST(c.c_customer_sk AS Float64)] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_current_cdemo_sk@1, CAST(customer_demographics.cd_demo_sk AS Float64)@9)], projection=[c_customer_sk@0, cd_gender@3, cd_marital_status@4, cd_education_status@5, cd_purchase_estimate@6, cd_credit_rating@7, cd_dep_count@8, cd_dep_employed_count@9, cd_dep_college_count@10] + │ CoalescePartitionsExec + │ [Stage 8] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], ...]}, projection=[cd_demo_sk, cd_gender, cd_marital_status, cd_education_status, cd_purchase_estimate, cd_credit_rating, cd_dep_count, cd_dep_employed_count, cd_dep_college_count, CAST(cd_demo_sk@0 AS Float64) as CAST(customer_demographics.cd_demo_sk AS Float64)], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p5] t1:[p6..p11] t2:[p12..p17] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, cs_sold_date_sk@0)], projection=[cs_ship_customer_sk@3] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_ship_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_year@1 = 2002 AND d_moy@2 >= 1 AND d_moy@2 <= 4, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2002 AND d_moy@8 >= 1 AND d_moy@8 <= 4, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 AND d_moy_null_count@5 != row_count@3 AND d_moy_max@4 >= 1 AND d_moy_null_count@5 != row_count@3 AND d_moy_min@6 <= 4, required_guarantees=[d_year in (2002)] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p5] t1:[p6..p11] t2:[p12..p17] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ws_sold_date_sk@0)], projection=[ws_bill_customer_sk@3] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_bill_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_year@1 = 2002 AND d_moy@2 >= 1 AND d_moy@2 <= 4, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2002 AND d_moy@8 >= 1 AND d_moy@8 <= 4, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 AND d_moy_null_count@5 != row_count@3 AND d_moy_max@4 >= 1 AND d_moy_null_count@5 != row_count@3 AND d_moy_min@6 <= 4, required_guarantees=[d_year in (2002)] + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p5] t1:[p6..p11] t2:[p12..p17] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_customer_sk@3] + │ CoalescePartitionsExec + │ [Stage 5] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_year@1 = 2002 AND d_moy@2 >= 1 AND d_moy@2 <= 4, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2002 AND d_moy@8 >= 1 AND d_moy@8 <= 4, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 AND d_moy_null_count@5 != row_count@3 AND d_moy_max@4 >= 1 AND d_moy_null_count@5 != row_count@3 AND d_moy_min@6 <= 4, required_guarantees=[d_year in (2002)] + └────────────────────────────────────────────────── + ┌───── Stage 8 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ca_address_sk@0, c_current_addr_sk@2)], projection=[c_customer_sk@1, c_current_cdemo_sk@2] + │ CoalescePartitionsExec + │ [Stage 7] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=6, input_tasks=2 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_current_cdemo_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=3, consumer_tasks=2, output_partitions=6 + │ FilterExec: ca_county@1 IN (SET) ([Rush County, Toole County, Jefferson County, Dona Ana County, La Porte County]), projection=[ca_address_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_county], file_type=parquet, predicate=ca_county@7 IN (SET) ([Rush County, Toole County, Jefferson County, Dona Ana County, La Porte County]), pruning_predicate=ca_county_null_count@2 != row_count@3 AND ca_county_min@0 <= Rush County AND Rush County <= ca_county_max@1 OR ca_county_null_count@2 != row_count@3 AND ca_county_min@0 <= Toole County AND Toole County <= ca_county_max@1 OR ca_county_null_count@2 != row_count@3 AND ca_county_min@0 <= Jefferson County AND Jefferson County <= ca_county_max@1 OR ca_county_null_count@2 != row_count@3 AND ca_county_min@0 <= Dona Ana County AND Dona Ana County <= ca_county_max@1 OR ca_county_null_count@2 != row_count@3 AND ca_county_min@0 <= La Porte County AND La Porte County <= ca_county_max@1, required_guarantees=[ca_county in (Dona Ana County, Jefferson County, La Porte County, Rush County, Toole County)] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_11() -> Result<()> { + let display = test_tpcds_query("q11").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [customer_id@0 ASC, customer_first_name@1 ASC, customer_last_name@2 ASC, customer_preferred_cust_flag@3 ASC], fetch=100 + │ [Stage 16] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 16 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ SortExec: TopK(fetch=100), expr=[customer_id@0 ASC, customer_first_name@1 ASC, customer_last_name@2 ASC, customer_preferred_cust_flag@3 ASC], preserve_partitioning=[true] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(customer_id@0, customer_id@0)], filter=CASE WHEN year_total@2 > Some(0),18,2 THEN CAST(year_total@3 AS Float64) / CAST(year_total@2 AS Float64) ELSE 0 END > CASE WHEN year_total@0 > Some(0),18,2 THEN CAST(year_total@1 AS Float64) / CAST(year_total@0 AS Float64) ELSE 0 END, projection=[customer_id@2, customer_first_name@3, customer_last_name@4, customer_preferred_cust_flag@5] + │ CoalescePartitionsExec + │ [Stage 12] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[c_customer_id@0 as customer_id, sum(web_sales.ws_ext_list_price - web_sales.ws_ext_discount_amt)@8 as year_total] + │ AggregateExec: mode=FinalPartitioned, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, c_preferred_cust_flag@3 as c_preferred_cust_flag, c_birth_country@4 as c_birth_country, c_login@5 as c_login, c_email_address@6 as c_email_address, d_year@7 as d_year], aggr=[sum(web_sales.ws_ext_list_price - web_sales.ws_ext_discount_amt)], ordering_mode=PartiallySorted([7]) + │ [Stage 15] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 12 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=3, consumer_tasks=2, output_partitions=6 + │ ProjectionExec: expr=[customer_id@1 as customer_id, year_total@2 as year_total, customer_id@3 as customer_id, customer_first_name@4 as customer_first_name, customer_last_name@5 as customer_last_name, customer_preferred_cust_flag@6 as customer_preferred_cust_flag, year_total@7 as year_total, year_total@0 as year_total] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(customer_id@0, customer_id@0)], projection=[year_total@1, customer_id@2, year_total@3, customer_id@4, customer_first_name@5, customer_last_name@6, customer_preferred_cust_flag@7, year_total@8] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=6, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(customer_id@0, customer_id@0)] + │ CoalescePartitionsExec + │ [Stage 8] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[c_customer_id@0 as customer_id, c_first_name@1 as customer_first_name, c_last_name@2 as customer_last_name, c_preferred_cust_flag@3 as customer_preferred_cust_flag, sum(store_sales.ss_ext_list_price - store_sales.ss_ext_discount_amt)@8 as year_total] + │ AggregateExec: mode=FinalPartitioned, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, c_preferred_cust_flag@3 as c_preferred_cust_flag, c_birth_country@4 as c_birth_country, c_login@5 as c_login, c_email_address@6 as c_email_address, d_year@7 as d_year], aggr=[sum(store_sales.ss_ext_list_price - store_sales.ss_ext_discount_amt)], ordering_mode=PartiallySorted([7]) + │ [Stage 11] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=3, consumer_tasks=2, output_partitions=6 + │ ProjectionExec: expr=[c_customer_id@0 as customer_id, sum(web_sales.ws_ext_list_price - web_sales.ws_ext_discount_amt)@1 as year_total] + │ FilterExec: sum(web_sales.ws_ext_list_price - web_sales.ws_ext_discount_amt)@1 > Some(0),18,2 + │ ProjectionExec: expr=[c_customer_id@0 as c_customer_id, sum(web_sales.ws_ext_list_price - web_sales.ws_ext_discount_amt)@8 as sum(web_sales.ws_ext_list_price - web_sales.ws_ext_discount_amt)] + │ AggregateExec: mode=FinalPartitioned, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, c_preferred_cust_flag@3 as c_preferred_cust_flag, c_birth_country@4 as c_birth_country, c_login@5 as c_login, c_email_address@6 as c_email_address, d_year@7 as d_year], aggr=[sum(web_sales.ws_ext_list_price - web_sales.ws_ext_discount_amt)], ordering_mode=PartiallySorted([7]) + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([c_customer_id@0, c_first_name@1, c_last_name@2, c_preferred_cust_flag@3, c_birth_country@4, c_login@5, c_email_address@6, d_year@7], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, c_preferred_cust_flag@3 as c_preferred_cust_flag, c_birth_country@4 as c_birth_country, c_login@5 as c_login, c_email_address@6 as c_email_address, d_year@9 as d_year], aggr=[sum(web_sales.ws_ext_list_price - web_sales.ws_ext_discount_amt)], ordering_mode=PartiallySorted([7]) + │ ProjectionExec: expr=[c_customer_id@1 as c_customer_id, c_first_name@2 as c_first_name, c_last_name@3 as c_last_name, c_preferred_cust_flag@4 as c_preferred_cust_flag, c_birth_country@5 as c_birth_country, c_login@6 as c_login, c_email_address@7 as c_email_address, ws_ext_discount_amt@8 as ws_ext_discount_amt, ws_ext_list_price@9 as ws_ext_list_price, d_year@0 as d_year] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, ws_sold_date_sk@7)], projection=[d_year@1, c_customer_id@3, c_first_name@4, c_last_name@5, c_preferred_cust_flag@6, c_birth_country@7, c_login@8, c_email_address@9, ws_ext_discount_amt@11, ws_ext_list_price@12] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(customer.c_customer_sk AS Float64)@8, ws_bill_customer_sk@1)], projection=[c_customer_id@1, c_first_name@2, c_last_name@3, c_preferred_cust_flag@4, c_birth_country@5, c_login@6, c_email_address@7, ws_sold_date_sk@9, ws_ext_discount_amt@11, ws_ext_list_price@12] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_bill_customer_sk, ws_ext_discount_amt, ws_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_year@1 = 2001 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, c_customer_id@1 as c_customer_id, c_first_name@2 as c_first_name, c_last_name@3 as c_last_name, c_preferred_cust_flag@4 as c_preferred_cust_flag, c_birth_country@5 as c_birth_country, c_login@6 as c_login, c_email_address@7 as c_email_address, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 8 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=3, consumer_tasks=2, output_partitions=6 + │ ProjectionExec: expr=[c_customer_id@0 as customer_id, sum(store_sales.ss_ext_list_price - store_sales.ss_ext_discount_amt)@1 as year_total] + │ FilterExec: sum(store_sales.ss_ext_list_price - store_sales.ss_ext_discount_amt)@1 > Some(0),18,2 + │ ProjectionExec: expr=[c_customer_id@0 as c_customer_id, sum(store_sales.ss_ext_list_price - store_sales.ss_ext_discount_amt)@8 as sum(store_sales.ss_ext_list_price - store_sales.ss_ext_discount_amt)] + │ AggregateExec: mode=FinalPartitioned, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, c_preferred_cust_flag@3 as c_preferred_cust_flag, c_birth_country@4 as c_birth_country, c_login@5 as c_login, c_email_address@6 as c_email_address, d_year@7 as d_year], aggr=[sum(store_sales.ss_ext_list_price - store_sales.ss_ext_discount_amt)], ordering_mode=PartiallySorted([7]) + │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([c_customer_id@0, c_first_name@1, c_last_name@2, c_preferred_cust_flag@3, c_birth_country@4, c_login@5, c_email_address@6, d_year@7], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, c_preferred_cust_flag@3 as c_preferred_cust_flag, c_birth_country@4 as c_birth_country, c_login@5 as c_login, c_email_address@6 as c_email_address, d_year@9 as d_year], aggr=[sum(store_sales.ss_ext_list_price - store_sales.ss_ext_discount_amt)], ordering_mode=PartiallySorted([7]) + │ ProjectionExec: expr=[c_customer_id@1 as c_customer_id, c_first_name@2 as c_first_name, c_last_name@3 as c_last_name, c_preferred_cust_flag@4 as c_preferred_cust_flag, c_birth_country@5 as c_birth_country, c_login@6 as c_login, c_email_address@7 as c_email_address, ss_ext_discount_amt@8 as ss_ext_discount_amt, ss_ext_list_price@9 as ss_ext_list_price, d_year@0 as d_year] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, ss_sold_date_sk@7)], projection=[d_year@1, c_customer_id@3, c_first_name@4, c_last_name@5, c_preferred_cust_flag@6, c_birth_country@7, c_login@8, c_email_address@9, ss_ext_discount_amt@11, ss_ext_list_price@12] + │ CoalescePartitionsExec + │ [Stage 5] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(customer.c_customer_sk AS Float64)@8, ss_customer_sk@1)], projection=[c_customer_id@1, c_first_name@2, c_last_name@3, c_preferred_cust_flag@4, c_birth_country@5, c_login@6, c_email_address@7, ss_sold_date_sk@9, ss_ext_discount_amt@11, ss_ext_list_price@12] + │ CoalescePartitionsExec + │ [Stage 6] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_ext_discount_amt, ss_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_year@1 = 2001 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, c_customer_id@1 as c_customer_id, c_first_name@2 as c_first_name, c_last_name@3 as c_last_name, c_preferred_cust_flag@4 as c_preferred_cust_flag, c_birth_country@5 as c_birth_country, c_login@6 as c_login, c_email_address@7 as c_email_address, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 11 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([c_customer_id@0, c_first_name@1, c_last_name@2, c_preferred_cust_flag@3, c_birth_country@4, c_login@5, c_email_address@6, d_year@7], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, c_preferred_cust_flag@3 as c_preferred_cust_flag, c_birth_country@4 as c_birth_country, c_login@5 as c_login, c_email_address@6 as c_email_address, d_year@9 as d_year], aggr=[sum(store_sales.ss_ext_list_price - store_sales.ss_ext_discount_amt)], ordering_mode=PartiallySorted([7]) + │ ProjectionExec: expr=[c_customer_id@1 as c_customer_id, c_first_name@2 as c_first_name, c_last_name@3 as c_last_name, c_preferred_cust_flag@4 as c_preferred_cust_flag, c_birth_country@5 as c_birth_country, c_login@6 as c_login, c_email_address@7 as c_email_address, ss_ext_discount_amt@8 as ss_ext_discount_amt, ss_ext_list_price@9 as ss_ext_list_price, d_year@0 as d_year] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, ss_sold_date_sk@7)], projection=[d_year@1, c_customer_id@3, c_first_name@4, c_last_name@5, c_preferred_cust_flag@6, c_birth_country@7, c_login@8, c_email_address@9, ss_ext_discount_amt@11, ss_ext_list_price@12] + │ CoalescePartitionsExec + │ [Stage 9] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(customer.c_customer_sk AS Float64)@8, ss_customer_sk@1)], projection=[c_customer_id@1, c_first_name@2, c_last_name@3, c_preferred_cust_flag@4, c_birth_country@5, c_login@6, c_email_address@7, ss_sold_date_sk@9, ss_ext_discount_amt@11, ss_ext_list_price@12] + │ CoalescePartitionsExec + │ [Stage 10] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_ext_discount_amt, ss_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 9 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_year@1 = 2002 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] + └────────────────────────────────────────────────── + ┌───── Stage 10 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, c_customer_id@1 as c_customer_id, c_first_name@2 as c_first_name, c_last_name@3 as c_last_name, c_preferred_cust_flag@4 as c_preferred_cust_flag, c_birth_country@5 as c_birth_country, c_login@6 as c_login, c_email_address@7 as c_email_address, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 15 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([c_customer_id@0, c_first_name@1, c_last_name@2, c_preferred_cust_flag@3, c_birth_country@4, c_login@5, c_email_address@6, d_year@7], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, c_preferred_cust_flag@3 as c_preferred_cust_flag, c_birth_country@4 as c_birth_country, c_login@5 as c_login, c_email_address@6 as c_email_address, d_year@9 as d_year], aggr=[sum(web_sales.ws_ext_list_price - web_sales.ws_ext_discount_amt)], ordering_mode=PartiallySorted([7]) + │ ProjectionExec: expr=[c_customer_id@1 as c_customer_id, c_first_name@2 as c_first_name, c_last_name@3 as c_last_name, c_preferred_cust_flag@4 as c_preferred_cust_flag, c_birth_country@5 as c_birth_country, c_login@6 as c_login, c_email_address@7 as c_email_address, ws_ext_discount_amt@8 as ws_ext_discount_amt, ws_ext_list_price@9 as ws_ext_list_price, d_year@0 as d_year] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, ws_sold_date_sk@7)], projection=[d_year@1, c_customer_id@3, c_first_name@4, c_last_name@5, c_preferred_cust_flag@6, c_birth_country@7, c_login@8, c_email_address@9, ws_ext_discount_amt@11, ws_ext_list_price@12] + │ CoalescePartitionsExec + │ [Stage 13] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(customer.c_customer_sk AS Float64)@8, ws_bill_customer_sk@1)], projection=[c_customer_id@1, c_first_name@2, c_last_name@3, c_preferred_cust_flag@4, c_birth_country@5, c_login@6, c_email_address@7, ws_sold_date_sk@9, ws_ext_discount_amt@11, ws_ext_list_price@12] + │ CoalescePartitionsExec + │ [Stage 14] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_bill_customer_sk, ws_ext_discount_amt, ws_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 13 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_year@1 = 2002 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] + └────────────────────────────────────────────────── + ┌───── Stage 14 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, c_customer_id@1 as c_customer_id, c_first_name@2 as c_first_name, c_last_name@3 as c_last_name, c_preferred_cust_flag@4 as c_preferred_cust_flag, c_birth_country@5 as c_birth_country, c_login@6 as c_login, c_email_address@7 as c_email_address, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name, c_preferred_cust_flag, c_birth_country, c_login, c_email_address], file_type=parquet + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_12() -> Result<()> { + let display = test_tpcds_query("q12").await?; + assert_snapshot!(display, @r#" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [i_category@2 ASC NULLS LAST, i_class@3 ASC NULLS LAST, i_item_id@0 ASC NULLS LAST, i_item_desc@1 ASC NULLS LAST, revenueratio@6 ASC NULLS LAST], fetch=100 + │ SortExec: TopK(fetch=100), expr=[i_category@2 ASC NULLS LAST, i_class@3 ASC NULLS LAST, i_item_id@0 ASC NULLS LAST, i_item_desc@1 ASC NULLS LAST, revenueratio@6 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[i_item_id@0 as i_item_id, i_item_desc@1 as i_item_desc, i_category@2 as i_category, i_class@3 as i_class, i_current_price@4 as i_current_price, sum(web_sales.ws_ext_sales_price)@5 as itemrevenue, CAST(sum(web_sales.ws_ext_sales_price)@5 AS Float64) * 100 / CAST(sum(sum(web_sales.ws_ext_sales_price)) PARTITION BY [item.i_class] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@6 AS Float64) as revenueratio] + │ WindowAggExec: wdw=[sum(sum(web_sales.ws_ext_sales_price)) PARTITION BY [item.i_class] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING: Ok(Field { name: "sum(sum(web_sales.ws_ext_sales_price)) PARTITION BY [item.i_class] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING", data_type: Decimal128(27, 2), nullable: true }), frame: WindowFrame { units: Rows, start_bound: Preceding(UInt64(NULL)), end_bound: Following(UInt64(NULL)), is_causal: false }] + │ SortExec: expr=[i_class@3 ASC NULLS LAST], preserve_partitioning=[true] + │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ RepartitionExec: partitioning=Hash([i_class@3], 3), input_partitions=3 + │ AggregateExec: mode=FinalPartitioned, gby=[i_item_id@0 as i_item_id, i_item_desc@1 as i_item_desc, i_category@2 as i_category, i_class@3 as i_class, i_current_price@4 as i_current_price], aggr=[sum(web_sales.ws_ext_sales_price)] + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([i_item_id@0, i_item_desc@1, i_category@2, i_class@3, i_current_price@4], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[i_item_id@1 as i_item_id, i_item_desc@2 as i_item_desc, i_category@5 as i_category, i_class@4 as i_class, i_current_price@3 as i_current_price], aggr=[sum(web_sales.ws_ext_sales_price)] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ws_sold_date_sk@0)], projection=[ws_ext_sales_price@3, i_item_id@4, i_item_desc@5, i_current_price@6, i_class@7, i_category@8] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ ProjectionExec: expr=[ws_sold_date_sk@5 as ws_sold_date_sk, ws_ext_sales_price@6 as ws_ext_sales_price, i_item_id@0 as i_item_id, i_item_desc@1 as i_item_desc, i_current_price@2 as i_current_price, i_class@3 as i_class, i_category@4 as i_category] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ws_item_sk@1)], projection=[i_item_id@1, i_item_desc@2, i_current_price@3, i_class@4, i_category@5, ws_sold_date_sk@6, ws_ext_sales_price@8] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_item_sk, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_date@1 >= 1999-02-22 AND d_date@1 <= 1999-03-24, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 1999-02-22 AND d_date@2 <= 1999-03-24, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 1999-02-22 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 1999-03-24, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ FilterExec: i_category@5 = Sports OR i_category@5 = Books OR i_category@5 = Home + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_item_id, i_item_desc, i_current_price, i_class, i_category], file_type=parquet, predicate=i_category@12 = Sports OR i_category@12 = Books OR i_category@12 = Home, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Sports AND Sports <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Home AND Home <= i_category_max@1, required_guarantees=[i_category in (Books, Home, Sports)] + └────────────────────────────────────────────────── + "#); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_13() -> Result<()> { + let display = test_tpcds_query("q13").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ ProjectionExec: expr=[avg(store_sales.ss_quantity)@0 as avg1, avg(store_sales.ss_ext_sales_price)@1 as avg2, avg(store_sales.ss_ext_wholesale_cost)@2 as avg3, sum(store_sales.ss_ext_wholesale_cost)@3 as sum(store_sales.ss_ext_wholesale_cost)] + │ AggregateExec: mode=Final, gby=[], aggr=[avg(store_sales.ss_quantity), avg(store_sales.ss_ext_sales_price), avg(store_sales.ss_ext_wholesale_cost), sum(store_sales.ss_ext_wholesale_cost)] + │ CoalescePartitionsExec + │ [Stage 6] => NetworkCoalesceExec: output_partitions=9, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p2] t1:[p3..p5] t2:[p6..p8] + │ AggregateExec: mode=Partial, gby=[], aggr=[avg(store_sales.ss_quantity), avg(store_sales.ss_ext_sales_price), avg(store_sales.ss_ext_wholesale_cost), sum(store_sales.ss_ext_wholesale_cost)] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_sold_date_sk@0, CAST(date_dim.d_date_sk AS Float64)@1)], projection=[ss_quantity@1, ss_ext_sales_price@2, ss_ext_wholesale_cost@3] + │ CoalescePartitionsExec + │ [Stage 5] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=3 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_year@1 = 2001, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p8] t1:[p9..p17] t2:[p18..p26] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_addr_sk@1, CAST(customer_address.ca_address_sk AS Float64)@2)], filter=(ca_state@1 = TX OR ca_state@1 = OH) AND ss_net_profit@0 >= Some(10000),6,2 AND ss_net_profit@0 <= Some(20000),6,2 OR (ca_state@1 = OR OR ca_state@1 = NM OR ca_state@1 = KY) AND ss_net_profit@0 >= Some(15000),6,2 AND ss_net_profit@0 <= Some(30000),6,2 OR (ca_state@1 = VA OR ca_state@1 = TX OR ca_state@1 = MS) AND ss_net_profit@0 >= Some(5000),6,2 AND ss_net_profit@0 <= Some(25000),6,2, projection=[ss_sold_date_sk@0, ss_quantity@2, ss_ext_sales_price@3, ss_ext_wholesale_cost@4] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=3 + │ ProjectionExec: expr=[ca_address_sk@0 as ca_address_sk, ca_state@1 as ca_state, CAST(ca_address_sk@0 AS Float64) as CAST(customer_address.ca_address_sk AS Float64)] + │ FilterExec: __common_expr_4@0 AND __common_expr_4@0 AND ca_country@3 = United States AND ca_state@2 IN (SET) ([TX, OH, OR, NM, KY, VA, MS]), projection=[ca_address_sk@1, ca_state@2] + │ ProjectionExec: expr=[__common_expr_5@0 OR ca_state@2 = OH OR ca_state@2 = OR OR ca_state@2 = NM OR ca_state@2 = KY OR ca_state@2 = VA OR __common_expr_5@0 OR ca_state@2 = MS as __common_expr_4, ca_address_sk@1 as ca_address_sk, ca_state@2 as ca_state, ca_country@3 as ca_country] + │ ProjectionExec: expr=[ca_state@1 = TX as __common_expr_5, ca_address_sk@0 as ca_address_sk, ca_state@1 as ca_state, ca_country@2 as ca_country] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_state, ca_country], file_type=parquet, predicate=ca_country@10 = United States AND ca_state@8 IN (SET) ([TX, OH, OR, NM, KY, VA, MS]), pruning_predicate=ca_country_null_count@2 != row_count@3 AND ca_country_min@0 <= United States AND United States <= ca_country_max@1 AND (ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= TX AND TX <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= OH AND OH <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= OR AND OR <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= NM AND NM <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= KY AND KY <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= VA AND VA <= ca_state_max@5 OR ca_state_null_count@6 != row_count@3 AND ca_state_min@4 <= MS AND MS <= ca_state_max@5), required_guarantees=[ca_country in (United States), ca_state in (KY, MS, NM, OH, OR, TX, VA)] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p8] t1:[p9..p17] t2:[p18..p26] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_hdemo_sk@1, CAST(household_demographics.hd_demo_sk AS Float64)@2)], filter=cd_marital_status@1 = M AND cd_education_status@2 = Advanced Degree AND ss_sales_price@0 >= Some(10000),5,2 AND ss_sales_price@0 <= Some(15000),5,2 AND hd_dep_count@3 = 3 OR cd_marital_status@1 = S AND cd_education_status@2 = College AND ss_sales_price@0 >= Some(5000),5,2 AND ss_sales_price@0 <= Some(10000),5,2 AND hd_dep_count@3 = 1 OR cd_marital_status@1 = W AND cd_education_status@2 = 2 yr Degree AND ss_sales_price@0 >= Some(15000),5,2 AND ss_sales_price@0 <= Some(20000),5,2 AND hd_dep_count@3 = 1, projection=[ss_sold_date_sk@0, ss_addr_sk@2, ss_quantity@3, ss_ext_sales_price@5, ss_ext_wholesale_cost@6, ss_net_profit@7] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=3 + │ ProjectionExec: expr=[hd_demo_sk@0 as hd_demo_sk, hd_dep_count@1 as hd_dep_count, CAST(hd_demo_sk@0 AS Float64) as CAST(household_demographics.hd_demo_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ FilterExec: (__common_expr_3@0 OR hd_dep_count@2 = 1) AND __common_expr_3@0, projection=[hd_demo_sk@1, hd_dep_count@2] + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-3.parquet]]}, projection=[hd_dep_count@3 = 3 OR hd_dep_count@3 = 1 as __common_expr_3, hd_demo_sk, hd_dep_count], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p5] t1:[p6..p11] t2:[p12..p17] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_cdemo_sk@1, CAST(customer_demographics.cd_demo_sk AS Float64)@3)], filter=cd_marital_status@1 = M AND cd_education_status@2 = Advanced Degree AND ss_sales_price@0 >= Some(10000),5,2 AND ss_sales_price@0 <= Some(15000),5,2 OR cd_marital_status@1 = S AND cd_education_status@2 = College AND ss_sales_price@0 >= Some(5000),5,2 AND ss_sales_price@0 <= Some(10000),5,2 OR cd_marital_status@1 = W AND cd_education_status@2 = 2 yr Degree AND ss_sales_price@0 >= Some(15000),5,2 AND ss_sales_price@0 <= Some(20000),5,2, projection=[ss_sold_date_sk@0, ss_hdemo_sk@2, ss_addr_sk@3, ss_quantity@4, ss_sales_price@5, ss_ext_sales_price@6, ss_ext_wholesale_cost@7, ss_net_profit@8, cd_marital_status@10, cd_education_status@11] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=3 + │ ProjectionExec: expr=[cd_demo_sk@0 as cd_demo_sk, cd_marital_status@1 as cd_marital_status, cd_education_status@2 as cd_education_status, CAST(cd_demo_sk@0 AS Float64) as CAST(customer_demographics.cd_demo_sk AS Float64)] + │ FilterExec: cd_marital_status@1 = M AND cd_education_status@2 = Advanced Degree OR cd_marital_status@1 = S AND cd_education_status@2 = College OR cd_marital_status@1 = W AND cd_education_status@2 = 2 yr Degree + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], ...]}, projection=[cd_demo_sk, cd_marital_status, cd_education_status], file_type=parquet, predicate=cd_marital_status@2 = M AND cd_education_status@3 = Advanced Degree OR cd_marital_status@2 = S AND cd_education_status@3 = College OR cd_marital_status@2 = W AND cd_education_status@3 = 2 yr Degree, pruning_predicate=cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= M AND M <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= Advanced Degree AND Advanced Degree <= cd_education_status_max@5 OR cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= S AND S <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= College AND College <= cd_education_status_max@5 OR cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= W AND W <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= 2 yr Degree AND 2 yr Degree <= cd_education_status_max@5, required_guarantees=[cd_education_status in (2 yr Degree, Advanced Degree, College), cd_marital_status in (M, S, W)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p5] t1:[p6..p11] t2:[p12..p17] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@1, ss_store_sk@4)], projection=[ss_sold_date_sk@2, ss_cdemo_sk@3, ss_hdemo_sk@4, ss_addr_sk@5, ss_quantity@7, ss_sales_price@8, ss_ext_sales_price@9, ss_ext_wholesale_cost@10, ss_net_profit@11] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=2 + │ FilterExec: (ss_net_profit@9 >= Some(10000),6,2 AND ss_net_profit@9 <= Some(20000),6,2 OR ss_net_profit@9 >= Some(15000),6,2 AND ss_net_profit@9 <= Some(30000),6,2 OR ss_net_profit@9 >= Some(5000),6,2 AND ss_net_profit@9 <= Some(25000),6,2) AND (ss_sales_price@6 >= Some(10000),5,2 AND ss_sales_price@6 <= Some(15000),5,2 OR ss_sales_price@6 >= Some(5000),5,2 AND ss_sales_price@6 <= Some(10000),5,2 OR ss_sales_price@6 >= Some(15000),5,2 AND ss_sales_price@6 <= Some(20000),5,2) + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_cdemo_sk, ss_hdemo_sk, ss_addr_sk, ss_store_sk, ss_quantity, ss_sales_price, ss_ext_sales_price, ss_ext_wholesale_cost, ss_net_profit], file_type=parquet, predicate=(ss_net_profit@22 >= Some(10000),6,2 AND ss_net_profit@22 <= Some(20000),6,2 OR ss_net_profit@22 >= Some(15000),6,2 AND ss_net_profit@22 <= Some(30000),6,2 OR ss_net_profit@22 >= Some(5000),6,2 AND ss_net_profit@22 <= Some(25000),6,2) AND (ss_sales_price@13 >= Some(10000),5,2 AND ss_sales_price@13 <= Some(15000),5,2 OR ss_sales_price@13 >= Some(5000),5,2 AND ss_sales_price@13 <= Some(10000),5,2 OR ss_sales_price@13 >= Some(15000),5,2 AND ss_sales_price@13 <= Some(20000),5,2) AND DynamicFilter [ empty ], pruning_predicate=(ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_max@0 >= Some(10000),6,2 AND ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_min@3 <= Some(20000),6,2 OR ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_max@0 >= Some(15000),6,2 AND ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_min@3 <= Some(30000),6,2 OR ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_max@0 >= Some(5000),6,2 AND ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_min@3 <= Some(25000),6,2) AND (ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_max@4 >= Some(10000),5,2 AND ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_min@6 <= Some(15000),5,2 OR ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_max@4 >= Some(5000),5,2 AND ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_min@6 <= Some(10000),5,2 OR ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_max@4 >= Some(15000),5,2 AND ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_min@6 <= Some(20000),5,2), required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)], file_type=parquet + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_14() -> Result<()> { + let display = test_tpcds_query("q14").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [channel@0 ASC, i_brand_id@1 ASC, i_class_id@2 ASC, i_category_id@3 ASC], fetch=100 + │ SortExec: TopK(fetch=100), expr=[channel@0 ASC, i_brand_id@1 ASC, i_class_id@2 ASC, i_category_id@3 ASC], preserve_partitioning=[true] + │ ProjectionExec: expr=[channel@0 as channel, i_brand_id@1 as i_brand_id, i_class_id@2 as i_class_id, i_category_id@3 as i_category_id, sum(y.sales)@5 as sum_sales, sum(y.number_sales)@6 as sum_number_sales] + │ AggregateExec: mode=FinalPartitioned, gby=[channel@0 as channel, i_brand_id@1 as i_brand_id, i_class_id@2 as i_class_id, i_category_id@3 as i_category_id, __grouping_id@4 as __grouping_id], aggr=[sum(y.sales), sum(y.number_sales)] + │ RepartitionExec: partitioning=Hash([channel@0, i_brand_id@1, i_class_id@2, i_category_id@3, __grouping_id@4], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[(NULL as channel, NULL as i_brand_id, NULL as i_class_id, NULL as i_category_id), (channel@0 as channel, NULL as i_brand_id, NULL as i_class_id, NULL as i_category_id), (channel@0 as channel, i_brand_id@1 as i_brand_id, NULL as i_class_id, NULL as i_category_id), (channel@0 as channel, i_brand_id@1 as i_brand_id, i_class_id@2 as i_class_id, NULL as i_category_id), (channel@0 as channel, i_brand_id@1 as i_brand_id, i_class_id@2 as i_class_id, i_category_id@3 as i_category_id)], aggr=[sum(y.sales), sum(y.number_sales)] + │ InterleaveExec + │ ProjectionExec: expr=[store as channel, i_brand_id@0 as i_brand_id, i_class_id@1 as i_class_id, i_category_id@2 as i_category_id, sum(store_sales.ss_quantity * store_sales.ss_list_price)@3 as sales, count(Int64(1))@4 as number_sales] + │ NestedLoopJoinExec: join_type=Inner, filter=sum(store_sales.ss_quantity * store_sales.ss_list_price)@0 > average_sales@1, projection=[i_brand_id@1, i_class_id@2, i_category_id@3, sum(store_sales.ss_quantity * store_sales.ss_list_price)@4, count(Int64(1))@5] + │ ProjectionExec: expr=[avg(sq2.quantity * sq2.list_price)@0 as average_sales] + │ AggregateExec: mode=Final, gby=[], aggr=[avg(sq2.quantity * sq2.list_price)] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=12, input_tasks=4 + │ AggregateExec: mode=FinalPartitioned, gby=[i_brand_id@0 as i_brand_id, i_class_id@1 as i_class_id, i_category_id@2 as i_category_id], aggr=[sum(store_sales.ss_quantity * store_sales.ss_list_price), count(Int64(1))] + │ RepartitionExec: partitioning=Hash([i_brand_id@0, i_class_id@1, i_category_id@2], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_brand_id@2 as i_brand_id, i_class_id@3 as i_class_id, i_category_id@4 as i_category_id], aggr=[sum(store_sales.ss_quantity * store_sales.ss_list_price), count(Int64(1))] + │ HashJoinExec: mode=CollectLeft, join_type=LeftSemi, on=[(ss_item_sk@0, ss_item_sk@0)], projection=[ss_quantity@1, ss_list_price@2, i_brand_id@3, i_class_id@4, i_category_id@5] + │ CoalescePartitionsExec + │ [Stage 5] => NetworkCoalesceExec: output_partitions=6, input_tasks=3 + │ ProjectionExec: expr=[i_item_sk@0 as ss_item_sk] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(brand_id@0, i_brand_id@1), (class_id@1, i_class_id@2), (category_id@2, i_category_id@3)], projection=[i_item_sk@3] + │ CoalescePartitionsExec + │ BroadcastExec: input_partitions=3, consumer_tasks=1, output_partitions=3 + │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(i_brand_id@0, brand_id@0), (i_class_id@1, class_id@1), (i_category_id@2, category_id@2)], NullsEqual: true + │ CoalescePartitionsExec + │ [Stage 8] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=2, input_tasks=3 + │ AggregateExec: mode=SinglePartitioned, gby=[brand_id@0 as brand_id, class_id@1 as class_id, category_id@2 as category_id], aggr=[] + │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(i_brand_id@0, brand_id@0), (i_class_id@1, class_id@1), (i_category_id@2, category_id@2)], NullsEqual: true + │ CoalescePartitionsExec + │ [Stage 11] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=2, input_tasks=3 + │ AggregateExec: mode=FinalPartitioned, gby=[brand_id@0 as brand_id, class_id@1 as class_id, category_id@2 as category_id], aggr=[] + │ [Stage 14] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[catalog as channel, i_brand_id@0 as i_brand_id, i_class_id@1 as i_class_id, i_category_id@2 as i_category_id, sum(catalog_sales.cs_quantity * catalog_sales.cs_list_price)@3 as sales, count(Int64(1))@4 as number_sales] + │ NestedLoopJoinExec: join_type=Inner, filter=sum(catalog_sales.cs_quantity * catalog_sales.cs_list_price)@0 > average_sales@1, projection=[i_brand_id@1, i_class_id@2, i_category_id@3, sum(catalog_sales.cs_quantity * catalog_sales.cs_list_price)@4, count(Int64(1))@5] + │ ProjectionExec: expr=[avg(sq2.quantity * sq2.list_price)@0 as average_sales] + │ AggregateExec: mode=Final, gby=[], aggr=[avg(sq2.quantity * sq2.list_price)] + │ CoalescePartitionsExec + │ [Stage 16] => NetworkCoalesceExec: output_partitions=12, input_tasks=4 + │ AggregateExec: mode=FinalPartitioned, gby=[i_brand_id@0 as i_brand_id, i_class_id@1 as i_class_id, i_category_id@2 as i_category_id], aggr=[sum(catalog_sales.cs_quantity * catalog_sales.cs_list_price), count(Int64(1))] + │ RepartitionExec: partitioning=Hash([i_brand_id@0, i_class_id@1, i_category_id@2], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_brand_id@2 as i_brand_id, i_class_id@3 as i_class_id, i_category_id@4 as i_category_id], aggr=[sum(catalog_sales.cs_quantity * catalog_sales.cs_list_price), count(Int64(1))] + │ HashJoinExec: mode=CollectLeft, join_type=LeftSemi, on=[(cs_item_sk@0, ss_item_sk@0)], projection=[cs_quantity@1, cs_list_price@2, i_brand_id@3, i_class_id@4, i_category_id@5] + │ CoalescePartitionsExec + │ [Stage 19] => NetworkCoalesceExec: output_partitions=6, input_tasks=3 + │ ProjectionExec: expr=[i_item_sk@0 as ss_item_sk] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(brand_id@0, i_brand_id@1), (class_id@1, i_class_id@2), (category_id@2, i_category_id@3)], projection=[i_item_sk@3] + │ CoalescePartitionsExec + │ BroadcastExec: input_partitions=3, consumer_tasks=1, output_partitions=3 + │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(i_brand_id@0, brand_id@0), (i_class_id@1, class_id@1), (i_category_id@2, category_id@2)], NullsEqual: true + │ CoalescePartitionsExec + │ [Stage 22] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=2, input_tasks=3 + │ AggregateExec: mode=SinglePartitioned, gby=[brand_id@0 as brand_id, class_id@1 as class_id, category_id@2 as category_id], aggr=[] + │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(i_brand_id@0, brand_id@0), (i_class_id@1, class_id@1), (i_category_id@2, category_id@2)], NullsEqual: true + │ CoalescePartitionsExec + │ [Stage 25] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=2, input_tasks=3 + │ AggregateExec: mode=FinalPartitioned, gby=[brand_id@0 as brand_id, class_id@1 as class_id, category_id@2 as category_id], aggr=[] + │ [Stage 28] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[web as channel, i_brand_id@0 as i_brand_id, i_class_id@1 as i_class_id, i_category_id@2 as i_category_id, sum(web_sales.ws_quantity * web_sales.ws_list_price)@3 as sales, count(Int64(1))@4 as number_sales] + │ NestedLoopJoinExec: join_type=Inner, filter=sum(web_sales.ws_quantity * web_sales.ws_list_price)@0 > average_sales@1, projection=[i_brand_id@1, i_class_id@2, i_category_id@3, sum(web_sales.ws_quantity * web_sales.ws_list_price)@4, count(Int64(1))@5] + │ ProjectionExec: expr=[avg(sq2.quantity * sq2.list_price)@0 as average_sales] + │ AggregateExec: mode=Final, gby=[], aggr=[avg(sq2.quantity * sq2.list_price)] + │ CoalescePartitionsExec + │ [Stage 30] => NetworkCoalesceExec: output_partitions=12, input_tasks=4 + │ AggregateExec: mode=FinalPartitioned, gby=[i_brand_id@0 as i_brand_id, i_class_id@1 as i_class_id, i_category_id@2 as i_category_id], aggr=[sum(web_sales.ws_quantity * web_sales.ws_list_price), count(Int64(1))] + │ RepartitionExec: partitioning=Hash([i_brand_id@0, i_class_id@1, i_category_id@2], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_brand_id@2 as i_brand_id, i_class_id@3 as i_class_id, i_category_id@4 as i_category_id], aggr=[sum(web_sales.ws_quantity * web_sales.ws_list_price), count(Int64(1))] + │ HashJoinExec: mode=CollectLeft, join_type=LeftSemi, on=[(ws_item_sk@0, ss_item_sk@0)], projection=[ws_quantity@1, ws_list_price@2, i_brand_id@3, i_class_id@4, i_category_id@5] + │ CoalescePartitionsExec + │ [Stage 33] => NetworkCoalesceExec: output_partitions=6, input_tasks=3 + │ ProjectionExec: expr=[i_item_sk@0 as ss_item_sk] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(brand_id@0, i_brand_id@1), (class_id@1, i_class_id@2), (category_id@2, i_category_id@3)], projection=[i_item_sk@3] + │ CoalescePartitionsExec + │ BroadcastExec: input_partitions=3, consumer_tasks=1, output_partitions=3 + │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(i_brand_id@0, brand_id@0), (i_class_id@1, class_id@1), (i_category_id@2, category_id@2)], NullsEqual: true + │ CoalescePartitionsExec + │ [Stage 36] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=2, input_tasks=3 + │ AggregateExec: mode=SinglePartitioned, gby=[brand_id@0 as brand_id, class_id@1 as class_id, category_id@2 as category_id], aggr=[] + │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(i_brand_id@0, brand_id@0), (i_class_id@1, class_id@1), (i_category_id@2, category_id@2)], NullsEqual: true + │ CoalescePartitionsExec + │ [Stage 39] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=2, input_tasks=3 + │ AggregateExec: mode=FinalPartitioned, gby=[brand_id@0 as brand_id, class_id@1 as class_id, category_id@2 as category_id], aggr=[] + │ [Stage 42] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p2] t1:[p3..p5] t2:[p6..p8] t3:[p9..p11] + │ AggregateExec: mode=Partial, gby=[], aggr=[avg(sq2.quantity * sq2.list_price)] + │ DistributedUnionExec: t0:[c0] t1:[c1] t2:[c2(0/2)] t3:[c2(1/2)] + │ ProjectionExec: expr=[ss_quantity@0 as quantity, ss_list_price@1 as list_price] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_quantity@3, ss_list_price@4] + │ CoalescePartitionsExec + │ BroadcastExec: input_partitions=3, consumer_tasks=1, output_partitions=3 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_quantity, ss_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[cs_quantity@0 as quantity, cs_list_price@1 as list_price] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, cs_sold_date_sk@0)], projection=[cs_quantity@3, cs_list_price@4] + │ CoalescePartitionsExec + │ BroadcastExec: input_partitions=3, consumer_tasks=1, output_partitions=3 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_quantity, cs_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[ws_quantity@0 as quantity, ws_list_price@1 as list_price] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ws_sold_date_sk@0)], projection=[ws_quantity@3, ws_list_price@4] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=6, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_quantity, ws_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=3, consumer_tasks=2, output_partitions=6 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_item_sk@3, ss_quantity@4, ss_list_price@5, i_brand_id@6, i_class_id@7, i_category_id@8] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ ProjectionExec: expr=[ss_sold_date_sk@3 as ss_sold_date_sk, ss_item_sk@4 as ss_item_sk, ss_quantity@5 as ss_quantity, ss_list_price@6 as ss_list_price, i_brand_id@0 as i_brand_id, i_class_id@1 as i_class_id, i_category_id@2 as i_category_id] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ss_item_sk@1)], projection=[i_brand_id@1, i_class_id@2, i_category_id@3, ss_sold_date_sk@4, ss_item_sk@5, ss_quantity@6, ss_list_price@7] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_quantity, ss_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_year@1 = 2001 AND d_moy@2 = 11, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 11, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 11 AND 11 <= d_moy_max@5, required_guarantees=[d_moy in (11), d_year in (2001)] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 8 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] + │ BroadcastExec: input_partitions=2, consumer_tasks=1, output_partitions=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(d3.d_date_sk AS Float64)@1, ws_sold_date_sk@0)], projection=[i_brand_id@3, i_class_id@4, i_category_id@5] + │ CoalescePartitionsExec + │ [Stage 6] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ ProjectionExec: expr=[ws_sold_date_sk@3 as ws_sold_date_sk, i_brand_id@0 as i_brand_id, i_class_id@1 as i_class_id, i_category_id@2 as i_category_id] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ws_item_sk@1)], projection=[i_brand_id@1, i_class_id@2, i_category_id@3, ws_sold_date_sk@4] + │ CoalescePartitionsExec + │ [Stage 7] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(d3.d_date_sk AS Float64)] + │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 11 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] + │ BroadcastExec: input_partitions=2, consumer_tasks=1, output_partitions=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(d2.d_date_sk AS Float64)@1, cs_sold_date_sk@0)], projection=[i_brand_id@3, i_class_id@4, i_category_id@5] + │ CoalescePartitionsExec + │ [Stage 9] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ ProjectionExec: expr=[cs_sold_date_sk@3 as cs_sold_date_sk, i_brand_id@0 as i_brand_id, i_class_id@1 as i_class_id, i_category_id@2 as i_category_id] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, cs_item_sk@1)], projection=[i_brand_id@1, i_class_id@2, i_category_id@3, cs_sold_date_sk@4] + │ CoalescePartitionsExec + │ [Stage 10] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 9 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(d2.d_date_sk AS Float64)] + │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 10 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 14 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] + │ RepartitionExec: partitioning=Hash([brand_id@0, class_id@1, category_id@2], 3), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[brand_id@0 as brand_id, class_id@1 as class_id, category_id@2 as category_id], aggr=[] + │ ProjectionExec: expr=[i_brand_id@0 as brand_id, i_class_id@1 as class_id, i_category_id@2 as category_id] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(d1.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[i_brand_id@3, i_class_id@4, i_category_id@5] + │ CoalescePartitionsExec + │ [Stage 12] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ ProjectionExec: expr=[ss_sold_date_sk@3 as ss_sold_date_sk, i_brand_id@0 as i_brand_id, i_class_id@1 as i_class_id, i_category_id@2 as i_category_id] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ss_item_sk@1)], projection=[i_brand_id@1, i_class_id@2, i_category_id@3, ss_sold_date_sk@4] + │ CoalescePartitionsExec + │ [Stage 13] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 12 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(d1.d_date_sk AS Float64)] + │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 13 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 16 ── Tasks: t0:[p0..p2] t1:[p3..p5] t2:[p6..p8] t3:[p9..p11] + │ AggregateExec: mode=Partial, gby=[], aggr=[avg(sq2.quantity * sq2.list_price)] + │ DistributedUnionExec: t0:[c0] t1:[c1] t2:[c2(0/2)] t3:[c2(1/2)] + │ ProjectionExec: expr=[ss_quantity@0 as quantity, ss_list_price@1 as list_price] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_quantity@3, ss_list_price@4] + │ CoalescePartitionsExec + │ BroadcastExec: input_partitions=3, consumer_tasks=1, output_partitions=3 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_quantity, ss_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[cs_quantity@0 as quantity, cs_list_price@1 as list_price] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, cs_sold_date_sk@0)], projection=[cs_quantity@3, cs_list_price@4] + │ CoalescePartitionsExec + │ BroadcastExec: input_partitions=3, consumer_tasks=1, output_partitions=3 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_quantity, cs_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[ws_quantity@0 as quantity, ws_list_price@1 as list_price] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ws_sold_date_sk@0)], projection=[ws_quantity@3, ws_list_price@4] + │ CoalescePartitionsExec + │ [Stage 15] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=6, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_quantity, ws_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 15 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=3, consumer_tasks=2, output_partitions=6 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 19 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, cs_sold_date_sk@0)], projection=[cs_item_sk@3, cs_quantity@4, cs_list_price@5, i_brand_id@6, i_class_id@7, i_category_id@8] + │ CoalescePartitionsExec + │ [Stage 17] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ ProjectionExec: expr=[cs_sold_date_sk@3 as cs_sold_date_sk, cs_item_sk@4 as cs_item_sk, cs_quantity@5 as cs_quantity, cs_list_price@6 as cs_list_price, i_brand_id@0 as i_brand_id, i_class_id@1 as i_class_id, i_category_id@2 as i_category_id] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, cs_item_sk@1)], projection=[i_brand_id@1, i_class_id@2, i_category_id@3, cs_sold_date_sk@4, cs_item_sk@5, cs_quantity@6, cs_list_price@7] + │ CoalescePartitionsExec + │ [Stage 18] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_item_sk, cs_quantity, cs_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 17 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_year@1 = 2001 AND d_moy@2 = 11, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 11, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 11 AND 11 <= d_moy_max@5, required_guarantees=[d_moy in (11), d_year in (2001)] + └────────────────────────────────────────────────── + ┌───── Stage 18 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 22 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] + │ BroadcastExec: input_partitions=2, consumer_tasks=1, output_partitions=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(d3.d_date_sk AS Float64)@1, ws_sold_date_sk@0)], projection=[i_brand_id@3, i_class_id@4, i_category_id@5] + │ CoalescePartitionsExec + │ [Stage 20] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ ProjectionExec: expr=[ws_sold_date_sk@3 as ws_sold_date_sk, i_brand_id@0 as i_brand_id, i_class_id@1 as i_class_id, i_category_id@2 as i_category_id] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ws_item_sk@1)], projection=[i_brand_id@1, i_class_id@2, i_category_id@3, ws_sold_date_sk@4] + │ CoalescePartitionsExec + │ [Stage 21] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 20 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(d3.d_date_sk AS Float64)] + │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 21 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 25 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] + │ BroadcastExec: input_partitions=2, consumer_tasks=1, output_partitions=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(d2.d_date_sk AS Float64)@1, cs_sold_date_sk@0)], projection=[i_brand_id@3, i_class_id@4, i_category_id@5] + │ CoalescePartitionsExec + │ [Stage 23] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ ProjectionExec: expr=[cs_sold_date_sk@3 as cs_sold_date_sk, i_brand_id@0 as i_brand_id, i_class_id@1 as i_class_id, i_category_id@2 as i_category_id] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, cs_item_sk@1)], projection=[i_brand_id@1, i_class_id@2, i_category_id@3, cs_sold_date_sk@4] + │ CoalescePartitionsExec + │ [Stage 24] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 23 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(d2.d_date_sk AS Float64)] + │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 24 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 28 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] + │ RepartitionExec: partitioning=Hash([brand_id@0, class_id@1, category_id@2], 3), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[brand_id@0 as brand_id, class_id@1 as class_id, category_id@2 as category_id], aggr=[] + │ ProjectionExec: expr=[i_brand_id@0 as brand_id, i_class_id@1 as class_id, i_category_id@2 as category_id] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(d1.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[i_brand_id@3, i_class_id@4, i_category_id@5] + │ CoalescePartitionsExec + │ [Stage 26] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ ProjectionExec: expr=[ss_sold_date_sk@3 as ss_sold_date_sk, i_brand_id@0 as i_brand_id, i_class_id@1 as i_class_id, i_category_id@2 as i_category_id] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ss_item_sk@1)], projection=[i_brand_id@1, i_class_id@2, i_category_id@3, ss_sold_date_sk@4] + │ CoalescePartitionsExec + │ [Stage 27] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 26 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(d1.d_date_sk AS Float64)] + │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 27 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 30 ── Tasks: t0:[p0..p2] t1:[p3..p5] t2:[p6..p8] t3:[p9..p11] + │ AggregateExec: mode=Partial, gby=[], aggr=[avg(sq2.quantity * sq2.list_price)] + │ DistributedUnionExec: t0:[c0] t1:[c1] t2:[c2(0/2)] t3:[c2(1/2)] + │ ProjectionExec: expr=[ss_quantity@0 as quantity, ss_list_price@1 as list_price] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_quantity@3, ss_list_price@4] + │ CoalescePartitionsExec + │ BroadcastExec: input_partitions=3, consumer_tasks=1, output_partitions=3 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_sales/part-3.parquet:..]]}, projection=[ss_sold_date_sk, ss_quantity, ss_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[cs_quantity@0 as quantity, cs_list_price@1 as list_price] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, cs_sold_date_sk@0)], projection=[cs_quantity@3, cs_list_price@4] + │ CoalescePartitionsExec + │ BroadcastExec: input_partitions=3, consumer_tasks=1, output_partitions=3 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_quantity, cs_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[ws_quantity@0 as quantity, ws_list_price@1 as list_price] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ws_sold_date_sk@0)], projection=[ws_quantity@3, ws_list_price@4] + │ CoalescePartitionsExec + │ [Stage 29] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=6, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_quantity, ws_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 29 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=3, consumer_tasks=2, output_partitions=6 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 33 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ws_sold_date_sk@0)], projection=[ws_item_sk@3, ws_quantity@4, ws_list_price@5, i_brand_id@6, i_class_id@7, i_category_id@8] + │ CoalescePartitionsExec + │ [Stage 31] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ ProjectionExec: expr=[ws_sold_date_sk@3 as ws_sold_date_sk, ws_item_sk@4 as ws_item_sk, ws_quantity@5 as ws_quantity, ws_list_price@6 as ws_list_price, i_brand_id@0 as i_brand_id, i_class_id@1 as i_class_id, i_category_id@2 as i_category_id] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ws_item_sk@1)], projection=[i_brand_id@1, i_class_id@2, i_category_id@3, ws_sold_date_sk@4, ws_item_sk@5, ws_quantity@6, ws_list_price@7] + │ CoalescePartitionsExec + │ [Stage 32] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_item_sk, ws_quantity, ws_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 31 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_year@1 = 2001 AND d_moy@2 = 11, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 11, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 11 AND 11 <= d_moy_max@5, required_guarantees=[d_moy in (11), d_year in (2001)] + └────────────────────────────────────────────────── + ┌───── Stage 32 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 36 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] + │ BroadcastExec: input_partitions=2, consumer_tasks=1, output_partitions=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(d3.d_date_sk AS Float64)@1, ws_sold_date_sk@0)], projection=[i_brand_id@3, i_class_id@4, i_category_id@5] + │ CoalescePartitionsExec + │ [Stage 34] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ ProjectionExec: expr=[ws_sold_date_sk@3 as ws_sold_date_sk, i_brand_id@0 as i_brand_id, i_class_id@1 as i_class_id, i_category_id@2 as i_category_id] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ws_item_sk@1)], projection=[i_brand_id@1, i_class_id@2, i_category_id@3, ws_sold_date_sk@4] + │ CoalescePartitionsExec + │ [Stage 35] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 34 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(d3.d_date_sk AS Float64)] + │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 35 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 39 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] + │ BroadcastExec: input_partitions=2, consumer_tasks=1, output_partitions=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(d2.d_date_sk AS Float64)@1, cs_sold_date_sk@0)], projection=[i_brand_id@3, i_class_id@4, i_category_id@5] + │ CoalescePartitionsExec + │ [Stage 37] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ ProjectionExec: expr=[cs_sold_date_sk@3 as cs_sold_date_sk, i_brand_id@0 as i_brand_id, i_class_id@1 as i_class_id, i_category_id@2 as i_category_id] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, cs_item_sk@1)], projection=[i_brand_id@1, i_class_id@2, i_category_id@3, cs_sold_date_sk@4] + │ CoalescePartitionsExec + │ [Stage 38] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 37 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(d2.d_date_sk AS Float64)] + │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 38 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 42 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] + │ RepartitionExec: partitioning=Hash([brand_id@0, class_id@1, category_id@2], 3), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[brand_id@0 as brand_id, class_id@1 as class_id, category_id@2 as category_id], aggr=[] + │ ProjectionExec: expr=[i_brand_id@0 as brand_id, i_class_id@1 as class_id, i_category_id@2 as category_id] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(d1.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[i_brand_id@3, i_class_id@4, i_category_id@5] + │ CoalescePartitionsExec + │ [Stage 40] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ ProjectionExec: expr=[ss_sold_date_sk@3 as ss_sold_date_sk, i_brand_id@0 as i_brand_id, i_class_id@1 as i_class_id, i_category_id@2 as i_category_id] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ss_item_sk@1)], projection=[i_brand_id@1, i_class_id@2, i_category_id@3, ss_sold_date_sk@4] + │ CoalescePartitionsExec + │ [Stage 41] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 40 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(d1.d_date_sk AS Float64)] + │ FilterExec: d_year@1 >= 1999 AND d_year@1 <= 2001, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 >= 1999 AND d_year@6 <= 2001, pruning_predicate=d_year_null_count@1 != row_count@2 AND d_year_max@0 >= 1999 AND d_year_null_count@1 != row_count@2 AND d_year_min@3 <= 2001, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 41 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id], file_type=parquet + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_15() -> Result<()> { + let display = test_tpcds_query("q15").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [ca_zip@0 ASC], fetch=100 + │ [Stage 6] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ SortExec: TopK(fetch=100), expr=[ca_zip@0 ASC], preserve_partitioning=[true] + │ AggregateExec: mode=FinalPartitioned, gby=[ca_zip@0 as ca_zip], aggr=[sum(catalog_sales.cs_sales_price)] + │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([ca_zip@0], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[ca_zip@1 as ca_zip], aggr=[sum(catalog_sales.cs_sales_price)] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, cs_sold_date_sk@0)], projection=[cs_sales_price@3, ca_zip@4] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ ProjectionExec: expr=[cs_sold_date_sk@1 as cs_sold_date_sk, cs_sales_price@2 as cs_sales_price, ca_zip@0 as ca_zip] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ca_address_sk@0, c_current_addr_sk@2)], filter=substr(ca_zip@2, 1, 5) IN (SET) ([85669, 86197, 88274, 83405, 86475, 85392, 85460, 80348, 81792]) OR ca_state@1 = CA OR ca_state@1 = WA OR ca_state@1 = GA OR cs_sales_price@0 > Some(50000),5,2, projection=[ca_zip@2, cs_sold_date_sk@3, cs_sales_price@4] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[cs_sold_date_sk@1 as cs_sold_date_sk, cs_sales_price@2 as cs_sales_price, c_current_addr_sk@0 as c_current_addr_sk] + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(customer.c_customer_sk AS Float64)@2, cs_bill_customer_sk@1)], projection=[c_current_addr_sk@1, cs_sold_date_sk@3, cs_sales_price@5] + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_qoy@2 = 2 AND d_year@1 = 2001, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@10 = 2 AND d_year@6 = 2001, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 2 AND 2 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5, required_guarantees=[d_qoy in (2), d_year in (2001)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_state, ca_zip], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p8] t1:[p0..p8] + │ RepartitionExec: partitioning=Hash([CAST(customer.c_customer_sk AS Float64)@2], 9), input_partitions=3 + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, c_current_addr_sk@1 as c_current_addr_sk, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] + │ RepartitionExec: partitioning=Hash([cs_bill_customer_sk@1], 9), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_16() -> Result<()> { + let display = test_tpcds_query("q16").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ ProjectionExec: expr=[count(alias1)@0 as order count, sum(alias2)@1 as total shipping cost, sum(alias3)@2 as total net profit] + │ GlobalLimitExec: skip=0, fetch=100 + │ AggregateExec: mode=Final, gby=[], aggr=[count(alias1), sum(alias2), sum(alias3)] + │ CoalescePartitionsExec + │ AggregateExec: mode=Partial, gby=[], aggr=[count(alias1), sum(alias2), sum(alias3)] + │ AggregateExec: mode=FinalPartitioned, gby=[alias1@0 as alias1], aggr=[alias2, alias3] + │ RepartitionExec: partitioning=Hash([alias1@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[cs_order_number@0 as alias1], aggr=[alias2, alias3] + │ HashJoinExec: mode=CollectLeft, join_type=LeftAnti, on=[(cs_order_number@0, cr_order_number@0)] + │ CoalescePartitionsExec + │ HashJoinExec: mode=CollectLeft, join_type=LeftSemi, on=[(cs_order_number@1, cs_order_number@1)], filter=cs_warehouse_sk@0 != cs_warehouse_sk@1, projection=[cs_order_number@1, cs_ext_ship_cost@2, cs_net_profit@3] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkCoalesceExec: output_partitions=6, input_tasks=3 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_warehouse_sk, cs_order_number], file_type=parquet + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-3.parquet:..]]}, projection=[cr_order_number], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(call_center.cc_call_center_sk AS Float64)@1, cs_call_center_sk@0)], projection=[cs_warehouse_sk@3, cs_order_number@4, cs_ext_ship_cost@5, cs_net_profit@6] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(customer_address.ca_address_sk AS Float64)@1, cs_ship_addr_sk@0)], projection=[cs_call_center_sk@3, cs_warehouse_sk@4, cs_order_number@5, cs_ext_ship_cost@6, cs_net_profit@7] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, cs_ship_date_sk@0)], projection=[cs_ship_addr_sk@3, cs_call_center_sk@4, cs_warehouse_sk@5, cs_order_number@6, cs_ext_ship_cost@7, cs_net_profit@8] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_ship_date_sk, cs_ship_addr_sk, cs_call_center_sk, cs_warehouse_sk, cs_order_number, cs_ext_ship_cost, cs_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[cc_call_center_sk@0 as cc_call_center_sk, CAST(cc_call_center_sk@0 AS Float64) as CAST(call_center.cc_call_center_sk AS Float64)] + │ FilterExec: cc_county@1 = Williamson County, projection=[cc_call_center_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/call_center/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/call_center/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/call_center/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/call_center/part-3.parquet]]}, projection=[cc_call_center_sk, cc_county], file_type=parquet, predicate=cc_county@25 = Williamson County, pruning_predicate=cc_county_null_count@2 != row_count@3 AND cc_county_min@0 <= Williamson County AND Williamson County <= cc_county_max@1, required_guarantees=[cc_county in (Williamson County)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[ca_address_sk@0 as ca_address_sk, CAST(ca_address_sk@0 AS Float64) as CAST(customer_address.ca_address_sk AS Float64)] + │ FilterExec: ca_state@1 = GA, projection=[ca_address_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_state], file_type=parquet, predicate=ca_state@8 = GA, pruning_predicate=ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= GA AND GA <= ca_state_max@1, required_guarantees=[ca_state in (GA)] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_date@1 >= 2002-02-01 AND d_date@1 <= 2002-04-02, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2002-02-01 AND d_date@2 <= 2002-04-02, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2002-02-01 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2002-04-02, required_guarantees=[] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_17() -> Result<()> { + let display = test_tpcds_query("q17").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [i_item_id@0 ASC, i_item_desc@1 ASC, s_state@2 ASC], fetch=100 + │ [Stage 11] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 11 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ SortExec: TopK(fetch=100), expr=[i_item_id@0 ASC, i_item_desc@1 ASC, s_state@2 ASC], preserve_partitioning=[true] + │ ProjectionExec: expr=[i_item_id@0 as i_item_id, i_item_desc@1 as i_item_desc, s_state@2 as s_state, count(store_sales.ss_quantity)@3 as store_sales_quantitycount, avg(store_sales.ss_quantity)@4 as store_sales_quantityave, stddev(store_sales.ss_quantity)@5 as store_sales_quantitystdev, stddev(store_sales.ss_quantity)@5 / avg(store_sales.ss_quantity)@4 as store_sales_quantitycov, count(store_returns.sr_return_quantity)@6 as store_returns_quantitycount, avg(store_returns.sr_return_quantity)@7 as store_returns_quantityave, stddev(store_returns.sr_return_quantity)@8 as store_returns_quantitystdev, stddev(store_returns.sr_return_quantity)@8 / avg(store_returns.sr_return_quantity)@7 as store_returns_quantitycov, count(catalog_sales.cs_quantity)@9 as catalog_sales_quantitycount, avg(catalog_sales.cs_quantity)@10 as catalog_sales_quantityave, stddev(catalog_sales.cs_quantity)@11 as catalog_sales_quantitystdev, stddev(catalog_sales.cs_quantity)@11 / avg(catalog_sales.cs_quantity)@10 as catalog_sales_quantitycov] + │ AggregateExec: mode=FinalPartitioned, gby=[i_item_id@0 as i_item_id, i_item_desc@1 as i_item_desc, s_state@2 as s_state], aggr=[count(store_sales.ss_quantity), avg(store_sales.ss_quantity), stddev(store_sales.ss_quantity), count(store_returns.sr_return_quantity), avg(store_returns.sr_return_quantity), stddev(store_returns.sr_return_quantity), count(catalog_sales.cs_quantity), avg(catalog_sales.cs_quantity), stddev(catalog_sales.cs_quantity)] + │ [Stage 10] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 10 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([i_item_id@0, i_item_desc@1, s_state@2], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_item_id@4 as i_item_id, i_item_desc@5 as i_item_desc, s_state@3 as s_state], aggr=[count(store_sales.ss_quantity), avg(store_sales.ss_quantity), stddev(store_sales.ss_quantity), count(store_returns.sr_return_quantity), avg(store_returns.sr_return_quantity), stddev(store_returns.sr_return_quantity), count(catalog_sales.cs_quantity), avg(catalog_sales.cs_quantity), stddev(catalog_sales.cs_quantity)] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_item_sk@0, i_item_sk@0)], projection=[ss_quantity@1, sr_return_quantity@2, cs_quantity@3, s_state@4, i_item_id@6, i_item_desc@7] + │ CoalescePartitionsExec + │ [Stage 9] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=3 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_item_id, i_item_desc], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 9 ── Tasks: t0:[p0..p8] t1:[p9..p17] t2:[p18..p26] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[ss_item_sk@1 as ss_item_sk, ss_quantity@2 as ss_quantity, sr_return_quantity@3 as sr_return_quantity, cs_quantity@4 as cs_quantity, s_state@0 as s_state] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@2, ss_store_sk@1)], projection=[s_state@1, ss_item_sk@3, ss_quantity@5, sr_return_quantity@6, cs_quantity@7] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(d3.d_date_sk AS Float64)@1, cs_sold_date_sk@4)], projection=[ss_item_sk@2, ss_store_sk@3, ss_quantity@4, sr_return_quantity@5, cs_quantity@7] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(d2.d_date_sk AS Float64)@1, sr_returned_date_sk@3)], projection=[ss_item_sk@2, ss_store_sk@3, ss_quantity@4, sr_return_quantity@6, cs_sold_date_sk@7, cs_quantity@8] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(d1.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_item_sk@3, ss_store_sk@4, ss_quantity@5, sr_returned_date_sk@6, sr_return_quantity@7, cs_sold_date_sk@8, cs_quantity@9] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ ProjectionExec: expr=[ss_sold_date_sk@2 as ss_sold_date_sk, ss_item_sk@3 as ss_item_sk, ss_store_sk@4 as ss_store_sk, ss_quantity@5 as ss_quantity, sr_returned_date_sk@6 as sr_returned_date_sk, sr_return_quantity@7 as sr_return_quantity, cs_sold_date_sk@0 as cs_sold_date_sk, cs_quantity@1 as cs_quantity] + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(cs_bill_customer_sk@1, sr_customer_sk@6), (cs_item_sk@2, sr_item_sk@5)], projection=[cs_sold_date_sk@0, cs_quantity@3, ss_sold_date_sk@4, ss_item_sk@5, ss_store_sk@6, ss_quantity@7, sr_returned_date_sk@8, sr_return_quantity@11] + │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + │ [Stage 8] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_state, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(d3.d_date_sk AS Float64)] + │ FilterExec: d_quarter_name@1 = 2001Q1 OR d_quarter_name@1 = 2001Q2 OR d_quarter_name@1 = 2001Q3, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_quarter_name], file_type=parquet, predicate=d_quarter_name@15 = 2001Q1 OR d_quarter_name@15 = 2001Q2 OR d_quarter_name@15 = 2001Q3, pruning_predicate=d_quarter_name_null_count@2 != row_count@3 AND d_quarter_name_min@0 <= 2001Q1 AND 2001Q1 <= d_quarter_name_max@1 OR d_quarter_name_null_count@2 != row_count@3 AND d_quarter_name_min@0 <= 2001Q2 AND 2001Q2 <= d_quarter_name_max@1 OR d_quarter_name_null_count@2 != row_count@3 AND d_quarter_name_min@0 <= 2001Q3 AND 2001Q3 <= d_quarter_name_max@1, required_guarantees=[d_quarter_name in (2001Q1, 2001Q2, 2001Q3)] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(d2.d_date_sk AS Float64)] + │ FilterExec: d_quarter_name@1 = 2001Q1 OR d_quarter_name@1 = 2001Q2 OR d_quarter_name@1 = 2001Q3, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_quarter_name], file_type=parquet, predicate=d_quarter_name@15 = 2001Q1 OR d_quarter_name@15 = 2001Q2 OR d_quarter_name@15 = 2001Q3, pruning_predicate=d_quarter_name_null_count@2 != row_count@3 AND d_quarter_name_min@0 <= 2001Q1 AND 2001Q1 <= d_quarter_name_max@1 OR d_quarter_name_null_count@2 != row_count@3 AND d_quarter_name_min@0 <= 2001Q2 AND 2001Q2 <= d_quarter_name_max@1 OR d_quarter_name_null_count@2 != row_count@3 AND d_quarter_name_min@0 <= 2001Q3 AND 2001Q3 <= d_quarter_name_max@1, required_guarantees=[d_quarter_name in (2001Q1, 2001Q2, 2001Q3)] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(d1.d_date_sk AS Float64)] + │ FilterExec: d_quarter_name@1 = 2001Q1, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_quarter_name], file_type=parquet, predicate=d_quarter_name@15 = 2001Q1, pruning_predicate=d_quarter_name_null_count@2 != row_count@3 AND d_quarter_name_min@0 <= 2001Q1 AND 2001Q1 <= d_quarter_name_max@1, required_guarantees=[d_quarter_name in (2001Q1)] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] + │ RepartitionExec: partitioning=Hash([cs_bill_customer_sk@1, cs_item_sk@2], 9), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_item_sk, cs_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 8 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] + │ RepartitionExec: partitioning=Hash([sr_customer_sk@6, sr_item_sk@5], 9), input_partitions=3 + │ ProjectionExec: expr=[ss_sold_date_sk@4 as ss_sold_date_sk, ss_item_sk@5 as ss_item_sk, ss_store_sk@6 as ss_store_sk, ss_quantity@7 as ss_quantity, sr_returned_date_sk@0 as sr_returned_date_sk, sr_item_sk@1 as sr_item_sk, sr_customer_sk@2 as sr_customer_sk, sr_return_quantity@3 as sr_return_quantity] + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(sr_customer_sk@2, ss_customer_sk@2), (sr_item_sk@1, ss_item_sk@1), (sr_ticket_number@3, ss_ticket_number@4)], projection=[sr_returned_date_sk@0, sr_item_sk@1, sr_customer_sk@2, sr_return_quantity@4, ss_sold_date_sk@5, ss_item_sk@6, ss_store_sk@8, ss_quantity@10] + │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] + │ RepartitionExec: partitioning=Hash([sr_customer_sk@2, sr_item_sk@1, sr_ticket_number@3], 9), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], ...]}, projection=[sr_returned_date_sk, sr_item_sk, sr_customer_sk, sr_ticket_number, sr_return_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] + │ RepartitionExec: partitioning=Hash([ss_customer_sk@2, ss_item_sk@1, ss_ticket_number@4], 9), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_store_sk, ss_ticket_number, ss_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_18() -> Result<()> { + let display = test_tpcds_query("q18").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [ca_country@1 ASC, ca_state@2 ASC, ca_county@3 ASC, i_item_id@0 ASC], fetch=100 + │ [Stage 10] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 10 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ SortExec: TopK(fetch=100), expr=[ca_country@1 ASC, ca_state@2 ASC, ca_county@3 ASC, i_item_id@0 ASC], preserve_partitioning=[true] + │ ProjectionExec: expr=[i_item_id@0 as i_item_id, ca_country@1 as ca_country, ca_state@2 as ca_state, ca_county@3 as ca_county, avg(catalog_sales.cs_quantity)@5 as agg1, avg(catalog_sales.cs_list_price)@6 as agg2, avg(catalog_sales.cs_coupon_amt)@7 as agg3, avg(catalog_sales.cs_sales_price)@8 as agg4, avg(catalog_sales.cs_net_profit)@9 as agg5, avg(customer.c_birth_year)@10 as agg6, avg(cd1.cd_dep_count)@11 as agg7] + │ AggregateExec: mode=FinalPartitioned, gby=[i_item_id@0 as i_item_id, ca_country@1 as ca_country, ca_state@2 as ca_state, ca_county@3 as ca_county, __grouping_id@4 as __grouping_id], aggr=[avg(catalog_sales.cs_quantity), avg(catalog_sales.cs_list_price), avg(catalog_sales.cs_coupon_amt), avg(catalog_sales.cs_sales_price), avg(catalog_sales.cs_net_profit), avg(customer.c_birth_year), avg(cd1.cd_dep_count)] + │ [Stage 9] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 9 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([i_item_id@0, ca_country@1, ca_state@2, ca_county@3, __grouping_id@4], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[(NULL as i_item_id, NULL as ca_country, NULL as ca_state, NULL as ca_county), (i_item_id@10 as i_item_id, NULL as ca_country, NULL as ca_state, NULL as ca_county), (i_item_id@10 as i_item_id, ca_country@9 as ca_country, NULL as ca_state, NULL as ca_county), (i_item_id@10 as i_item_id, ca_country@9 as ca_country, ca_state@8 as ca_state, NULL as ca_county), (i_item_id@10 as i_item_id, ca_country@9 as ca_country, ca_state@8 as ca_state, ca_county@7 as ca_county)], aggr=[avg(catalog_sales.cs_quantity), avg(catalog_sales.cs_list_price), avg(catalog_sales.cs_coupon_amt), avg(catalog_sales.cs_sales_price), avg(catalog_sales.cs_net_profit), avg(customer.c_birth_year), avg(cd1.cd_dep_count)] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(cs_item_sk@0, i_item_sk@0)], projection=[cs_quantity@1, cs_list_price@2, cs_sales_price@3, cs_coupon_amt@4, cs_net_profit@5, cd_dep_count@6, c_birth_year@7, ca_county@8, ca_state@9, ca_country@10, i_item_id@12] + │ CoalescePartitionsExec + │ [Stage 8] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=3 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 8 ── Tasks: t0:[p0..p8] t1:[p9..p17] t2:[p18..p26] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, cs_sold_date_sk@0)], projection=[cs_item_sk@3, cs_quantity@4, cs_list_price@5, cs_sales_price@6, cs_coupon_amt@7, cs_net_profit@8, cd_dep_count@9, c_birth_year@10, ca_county@11, ca_state@12, ca_country@13] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ ProjectionExec: expr=[cs_sold_date_sk@3 as cs_sold_date_sk, cs_item_sk@4 as cs_item_sk, cs_quantity@5 as cs_quantity, cs_list_price@6 as cs_list_price, cs_sales_price@7 as cs_sales_price, cs_coupon_amt@8 as cs_coupon_amt, cs_net_profit@9 as cs_net_profit, cd_dep_count@10 as cd_dep_count, c_birth_year@11 as c_birth_year, ca_county@0 as ca_county, ca_state@1 as ca_state, ca_country@2 as ca_country] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ca_address_sk@0, c_current_addr_sk@8)], projection=[ca_county@1, ca_state@2, ca_country@3, cs_sold_date_sk@4, cs_item_sk@5, cs_quantity@6, cs_list_price@7, cs_sales_price@8, cs_coupon_amt@9, cs_net_profit@10, cd_dep_count@11, c_birth_year@13] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(c_current_cdemo_sk@8, CAST(cd2.cd_demo_sk AS Float64)@1)], projection=[cs_sold_date_sk@0, cs_item_sk@1, cs_quantity@2, cs_list_price@3, cs_sales_price@4, cs_coupon_amt@5, cs_net_profit@6, cd_dep_count@7, c_current_addr_sk@9, c_birth_year@10] + │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_year@1 = 1998, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 1998, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1, required_guarantees=[d_year in (1998)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ FilterExec: ca_state@2 IN (SET) ([MS, IN, ND, OK, NM, VA, MS]) + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_county, ca_state, ca_country], file_type=parquet, predicate=ca_state@8 IN (SET) ([MS, IN, ND, OK, NM, VA, MS]), pruning_predicate=ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= MS AND MS <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= IN AND IN <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= ND AND ND <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= OK AND OK <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= NM AND NM <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= VA AND VA <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= MS AND MS <= ca_state_max@1, required_guarantees=[ca_state in (IN, MS, ND, NM, OK, VA)] + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] + │ RepartitionExec: partitioning=Hash([c_current_cdemo_sk@8], 9), input_partitions=3 + │ ProjectionExec: expr=[cs_sold_date_sk@3 as cs_sold_date_sk, cs_item_sk@4 as cs_item_sk, cs_quantity@5 as cs_quantity, cs_list_price@6 as cs_list_price, cs_sales_price@7 as cs_sales_price, cs_coupon_amt@8 as cs_coupon_amt, cs_net_profit@9 as cs_net_profit, cd_dep_count@10 as cd_dep_count, c_current_cdemo_sk@0 as c_current_cdemo_sk, c_current_addr_sk@1 as c_current_addr_sk, c_birth_year@2 as c_birth_year] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(customer.c_customer_sk AS Float64)@4, cs_bill_customer_sk@1)], projection=[c_current_cdemo_sk@1, c_current_addr_sk@2, c_birth_year@3, cs_sold_date_sk@5, cs_item_sk@7, cs_quantity@8, cs_list_price@9, cs_sales_price@10, cs_coupon_amt@11, cs_net_profit@12, cd_dep_count@13] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ ProjectionExec: expr=[cs_sold_date_sk@1 as cs_sold_date_sk, cs_bill_customer_sk@2 as cs_bill_customer_sk, cs_item_sk@3 as cs_item_sk, cs_quantity@4 as cs_quantity, cs_list_price@5 as cs_list_price, cs_sales_price@6 as cs_sales_price, cs_coupon_amt@7 as cs_coupon_amt, cs_net_profit@8 as cs_net_profit, cd_dep_count@0 as cd_dep_count] + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(cd1.cd_demo_sk AS Float64)@2, cs_bill_cdemo_sk@2)], projection=[cd_dep_count@1, cs_sold_date_sk@3, cs_bill_customer_sk@4, cs_item_sk@6, cs_quantity@7, cs_list_price@8, cs_sales_price@9, cs_coupon_amt@10, cs_net_profit@11] + │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, c_current_cdemo_sk@1 as c_current_cdemo_sk, c_current_addr_sk@2 as c_current_addr_sk, c_birth_year@3 as c_birth_year, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ FilterExec: c_birth_month@3 IN (SET) ([1, 6, 8, 9, 12, 2]), projection=[c_customer_sk@0, c_current_cdemo_sk@1, c_current_addr_sk@2, c_birth_year@4] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_current_cdemo_sk, c_current_addr_sk, c_birth_month, c_birth_year], file_type=parquet, predicate=c_birth_month@12 IN (SET) ([1, 6, 8, 9, 12, 2]) AND DynamicFilter [ empty ], pruning_predicate=c_birth_month_null_count@2 != row_count@3 AND c_birth_month_min@0 <= 1 AND 1 <= c_birth_month_max@1 OR c_birth_month_null_count@2 != row_count@3 AND c_birth_month_min@0 <= 6 AND 6 <= c_birth_month_max@1 OR c_birth_month_null_count@2 != row_count@3 AND c_birth_month_min@0 <= 8 AND 8 <= c_birth_month_max@1 OR c_birth_month_null_count@2 != row_count@3 AND c_birth_month_min@0 <= 9 AND 9 <= c_birth_month_max@1 OR c_birth_month_null_count@2 != row_count@3 AND c_birth_month_min@0 <= 12 AND 12 <= c_birth_month_max@1 OR c_birth_month_null_count@2 != row_count@3 AND c_birth_month_min@0 <= 2 AND 2 <= c_birth_month_max@1, required_guarantees=[c_birth_month in (1, 12, 2, 6, 8, 9)] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] + │ RepartitionExec: partitioning=Hash([CAST(cd1.cd_demo_sk AS Float64)@2], 9), input_partitions=2 + │ ProjectionExec: expr=[cd_demo_sk@0 as cd_demo_sk, cd_dep_count@1 as cd_dep_count, CAST(cd_demo_sk@0 AS Float64) as CAST(cd1.cd_demo_sk AS Float64)] + │ FilterExec: cd_gender@1 = F AND cd_education_status@2 = Unknown, projection=[cd_demo_sk@0, cd_dep_count@3] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], ...]}, projection=[cd_demo_sk, cd_gender, cd_education_status, cd_dep_count], file_type=parquet, predicate=cd_gender@1 = F AND cd_education_status@3 = Unknown, pruning_predicate=cd_gender_null_count@2 != row_count@3 AND cd_gender_min@0 <= F AND F <= cd_gender_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= Unknown AND Unknown <= cd_education_status_max@5, required_guarantees=[cd_education_status in (Unknown), cd_gender in (F)] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] + │ RepartitionExec: partitioning=Hash([cs_bill_cdemo_sk@2], 9), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_bill_cdemo_sk, cs_item_sk, cs_quantity, cs_list_price, cs_sales_price, cs_coupon_amt, cs_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] + │ RepartitionExec: partitioning=Hash([CAST(cd2.cd_demo_sk AS Float64)@1], 9), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], ...]}, projection=[cd_demo_sk, CAST(cd_demo_sk@0 AS Float64) as CAST(cd2.cd_demo_sk AS Float64)], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_19() -> Result<()> { + let display = test_tpcds_query("q19").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [ext_price@4 DESC, brand@1 ASC NULLS LAST, brand_id@0 ASC NULLS LAST, i_manufact_id@2 ASC NULLS LAST, i_manufact@3 ASC NULLS LAST], fetch=100 + │ [Stage 8] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 8 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ SortExec: TopK(fetch=100), expr=[ext_price@4 DESC, brand@1 ASC NULLS LAST, brand_id@0 ASC NULLS LAST, i_manufact_id@2 ASC NULLS LAST, i_manufact@3 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[i_brand_id@1 as brand_id, i_brand@0 as brand, i_manufact_id@2 as i_manufact_id, i_manufact@3 as i_manufact, sum(store_sales.ss_ext_sales_price)@4 as ext_price] + │ AggregateExec: mode=FinalPartitioned, gby=[i_brand@0 as i_brand, i_brand_id@1 as i_brand_id, i_manufact_id@2 as i_manufact_id, i_manufact@3 as i_manufact], aggr=[sum(store_sales.ss_ext_sales_price)] + │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([i_brand@0, i_brand_id@1, i_manufact_id@2, i_manufact@3], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_brand@2 as i_brand, i_brand_id@1 as i_brand_id, i_manufact_id@3 as i_manufact_id, i_manufact@4 as i_manufact], aggr=[sum(store_sales.ss_ext_sales_price)] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@2, ss_store_sk@0)], filter=substr(ca_zip@0, 1, 5) != substr(s_zip@1, 1, 5), projection=[ss_ext_sales_price@4, i_brand_id@5, i_brand@6, i_manufact_id@7, i_manufact@8] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[ss_store_sk@1 as ss_store_sk, ss_ext_sales_price@2 as ss_ext_sales_price, i_brand_id@3 as i_brand_id, i_brand@4 as i_brand, i_manufact_id@5 as i_manufact_id, i_manufact@6 as i_manufact, ca_zip@0 as ca_zip] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ca_address_sk@0, c_current_addr_sk@6)], projection=[ca_zip@1, ss_store_sk@2, ss_ext_sales_price@3, i_brand_id@4, i_brand@5, i_manufact_id@6, i_manufact@7] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=2 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(ss_customer_sk@0, CAST(customer.c_customer_sk AS Float64)@2)], projection=[ss_store_sk@1, ss_ext_sales_price@2, i_brand_id@3, i_brand@4, i_manufact_id@5, i_manufact@6, c_current_addr_sk@8] + │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_zip, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_zip], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] + │ RepartitionExec: partitioning=Hash([ss_customer_sk@0], 9), input_partitions=3 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_item_sk@0, i_item_sk@0)], projection=[ss_customer_sk@1, ss_store_sk@2, ss_ext_sales_price@3, i_brand_id@5, i_brand@6, i_manufact_id@7, i_manufact@8] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=3 + │ FilterExec: i_manager_id@5 = 8, projection=[i_item_sk@0, i_brand_id@1, i_brand@2, i_manufact_id@3, i_manufact@4] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_brand, i_manufact_id, i_manufact, i_manager_id], file_type=parquet, predicate=i_manager_id@20 = 8 AND DynamicFilter [ empty ], pruning_predicate=i_manager_id_null_count@2 != row_count@3 AND i_manager_id_min@0 <= 8 AND 8 <= i_manager_id_max@1, required_guarantees=[i_manager_id in (8)] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p5] t1:[p6..p11] t2:[p12..p17] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_item_sk@3, ss_customer_sk@4, ss_store_sk@5, ss_ext_sales_price@6] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_store_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_moy@2 = 11 AND d_year@1 = 1998, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 11 AND d_year@6 = 1998, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1998 AND 1998 <= d_year_max@5, required_guarantees=[d_moy in (11), d_year in (1998)] + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p8] t1:[p0..p8] + │ RepartitionExec: partitioning=Hash([CAST(customer.c_customer_sk AS Float64)@2], 9), input_partitions=3 + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, c_current_addr_sk@1 as c_current_addr_sk, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_20() -> Result<()> { + let display = test_tpcds_query("q20").await?; + assert_snapshot!(display, @r#" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [i_category@2 ASC, i_class@3 ASC, i_item_id@0 ASC, i_item_desc@1 ASC, revenueratio@6 ASC], fetch=100 + │ SortExec: TopK(fetch=100), expr=[i_category@2 ASC, i_class@3 ASC, i_item_id@0 ASC, i_item_desc@1 ASC, revenueratio@6 ASC], preserve_partitioning=[true] + │ ProjectionExec: expr=[i_item_id@0 as i_item_id, i_item_desc@1 as i_item_desc, i_category@2 as i_category, i_class@3 as i_class, i_current_price@4 as i_current_price, sum(catalog_sales.cs_ext_sales_price)@5 as itemrevenue, CAST(sum(catalog_sales.cs_ext_sales_price)@5 AS Float64) * 100 / CAST(sum(sum(catalog_sales.cs_ext_sales_price)) PARTITION BY [item.i_class] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@6 AS Float64) as revenueratio] + │ WindowAggExec: wdw=[sum(sum(catalog_sales.cs_ext_sales_price)) PARTITION BY [item.i_class] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING: Ok(Field { name: "sum(sum(catalog_sales.cs_ext_sales_price)) PARTITION BY [item.i_class] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING", data_type: Decimal128(27, 2), nullable: true }), frame: WindowFrame { units: Rows, start_bound: Preceding(UInt64(NULL)), end_bound: Following(UInt64(NULL)), is_causal: false }] + │ SortExec: expr=[i_class@3 ASC NULLS LAST], preserve_partitioning=[true] + │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ RepartitionExec: partitioning=Hash([i_class@3], 3), input_partitions=3 + │ AggregateExec: mode=FinalPartitioned, gby=[i_item_id@0 as i_item_id, i_item_desc@1 as i_item_desc, i_category@2 as i_category, i_class@3 as i_class, i_current_price@4 as i_current_price], aggr=[sum(catalog_sales.cs_ext_sales_price)] + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([i_item_id@0, i_item_desc@1, i_category@2, i_class@3, i_current_price@4], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[i_item_id@1 as i_item_id, i_item_desc@2 as i_item_desc, i_category@5 as i_category, i_class@4 as i_class, i_current_price@3 as i_current_price], aggr=[sum(catalog_sales.cs_ext_sales_price)] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, cs_sold_date_sk@0)], projection=[cs_ext_sales_price@3, i_item_id@4, i_item_desc@5, i_current_price@6, i_class@7, i_category@8] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ ProjectionExec: expr=[cs_sold_date_sk@5 as cs_sold_date_sk, cs_ext_sales_price@6 as cs_ext_sales_price, i_item_id@0 as i_item_id, i_item_desc@1 as i_item_desc, i_current_price@2 as i_current_price, i_class@3 as i_class, i_category@4 as i_category] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, cs_item_sk@1)], projection=[i_item_id@1, i_item_desc@2, i_current_price@3, i_class@4, i_category@5, cs_sold_date_sk@6, cs_ext_sales_price@8] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_item_sk, cs_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_date@1 >= 1999-02-22 AND d_date@1 <= 1999-03-24, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 1999-02-22 AND d_date@2 <= 1999-03-24, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 1999-02-22 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 1999-03-24, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ FilterExec: i_category@5 = Sports OR i_category@5 = Books OR i_category@5 = Home + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_item_id, i_item_desc, i_current_price, i_class, i_category], file_type=parquet, predicate=i_category@12 = Sports OR i_category@12 = Books OR i_category@12 = Home, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Sports AND Sports <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Home AND Home <= i_category_max@1, required_guarantees=[i_category in (Books, Home, Sports)] + └────────────────────────────────────────────────── + "#); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_21() -> Result<()> { + let display = test_tpcds_query("q21").await?; + assert_snapshot!(display, @r#" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [w_warehouse_name@0 ASC, i_item_id@1 ASC], fetch=100 + │ [Stage 5] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ SortExec: TopK(fetch=100), expr=[w_warehouse_name@0 ASC, i_item_id@1 ASC], preserve_partitioning=[true] + │ ProjectionExec: expr=[w_warehouse_name@0 as w_warehouse_name, i_item_id@1 as i_item_id, sum(CASE WHEN date_dim.d_date < Utf8("2000-03-11") THEN inventory.inv_quantity_on_hand ELSE Int64(0) END)@2 as inv_before, sum(CASE WHEN date_dim.d_date >= Utf8("2000-03-11") THEN inventory.inv_quantity_on_hand ELSE Int64(0) END)@3 as inv_after] + │ FilterExec: __common_expr_4@0 >= 0.6666666666666666 AND __common_expr_4@0 <= 1.5, projection=[w_warehouse_name@1, i_item_id@2, sum(CASE WHEN date_dim.d_date < Utf8("2000-03-11") THEN inventory.inv_quantity_on_hand ELSE Int64(0) END)@3, sum(CASE WHEN date_dim.d_date >= Utf8("2000-03-11") THEN inventory.inv_quantity_on_hand ELSE Int64(0) END)@4] + │ ProjectionExec: expr=[CASE WHEN sum(CASE WHEN date_dim.d_date < Utf8("2000-03-11") THEN inventory.inv_quantity_on_hand ELSE Int64(0) END)@2 > 0 THEN sum(CASE WHEN date_dim.d_date >= Utf8("2000-03-11") THEN inventory.inv_quantity_on_hand ELSE Int64(0) END)@3 / sum(CASE WHEN date_dim.d_date < Utf8("2000-03-11") THEN inventory.inv_quantity_on_hand ELSE Int64(0) END)@2 END as __common_expr_4, w_warehouse_name@0 as w_warehouse_name, i_item_id@1 as i_item_id, sum(CASE WHEN date_dim.d_date < Utf8("2000-03-11") THEN inventory.inv_quantity_on_hand ELSE Int64(0) END)@2 as sum(CASE WHEN date_dim.d_date < Utf8("2000-03-11") THEN inventory.inv_quantity_on_hand ELSE Int64(0) END), sum(CASE WHEN date_dim.d_date >= Utf8("2000-03-11") THEN inventory.inv_quantity_on_hand ELSE Int64(0) END)@3 as sum(CASE WHEN date_dim.d_date >= Utf8("2000-03-11") THEN inventory.inv_quantity_on_hand ELSE Int64(0) END)] + │ AggregateExec: mode=FinalPartitioned, gby=[w_warehouse_name@0 as w_warehouse_name, i_item_id@1 as i_item_id], aggr=[sum(CASE WHEN date_dim.d_date < Utf8("2000-03-11") THEN inventory.inv_quantity_on_hand ELSE Int64(0) END), sum(CASE WHEN date_dim.d_date >= Utf8("2000-03-11") THEN inventory.inv_quantity_on_hand ELSE Int64(0) END)] + │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([w_warehouse_name@0, i_item_id@1], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[w_warehouse_name@2 as w_warehouse_name, i_item_id@3 as i_item_id], aggr=[sum(CASE WHEN date_dim.d_date < Utf8("2000-03-11") THEN inventory.inv_quantity_on_hand ELSE Int64(0) END), sum(CASE WHEN date_dim.d_date >= Utf8("2000-03-11") THEN inventory.inv_quantity_on_hand ELSE Int64(0) END)] + │ ProjectionExec: expr=[d_date@0 as __common_expr_2, inv_quantity_on_hand@1 as inv_quantity_on_hand, w_warehouse_name@2 as w_warehouse_name, i_item_id@3 as i_item_id] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, inv_date_sk@0)], projection=[d_date@1, inv_quantity_on_hand@3, w_warehouse_name@4, i_item_id@5] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ ProjectionExec: expr=[inv_date_sk@1 as inv_date_sk, inv_quantity_on_hand@2 as inv_quantity_on_hand, w_warehouse_name@3 as w_warehouse_name, i_item_id@0 as i_item_id] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, inv_item_sk@1)], projection=[i_item_id@1, inv_date_sk@2, inv_quantity_on_hand@4, w_warehouse_name@5] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ ProjectionExec: expr=[inv_date_sk@1 as inv_date_sk, inv_item_sk@2 as inv_item_sk, inv_quantity_on_hand@3 as inv_quantity_on_hand, w_warehouse_name@0 as w_warehouse_name] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(w_warehouse_sk@0, inv_warehouse_sk@2)], projection=[w_warehouse_name@1, inv_date_sk@2, inv_item_sk@3, inv_quantity_on_hand@5] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/inventory/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:..], ...]}, projection=[inv_date_sk, inv_item_sk, inv_warehouse_sk, inv_quantity_on_hand], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ FilterExec: d_date@1 >= 2000-02-10 AND d_date@1 <= 2000-04-10 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-02-10 AND d_date@2 <= 2000-04-10, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-02-10 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-04-10, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ FilterExec: i_current_price@2 >= Some(99),4,2 AND i_current_price@2 <= Some(149),4,2, projection=[i_item_sk@0, i_item_id@1] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_item_id, i_current_price], file_type=parquet, predicate=i_current_price@5 >= Some(99),4,2 AND i_current_price@5 <= Some(149),4,2, pruning_predicate=i_current_price_null_count@1 != row_count@2 AND i_current_price_max@0 >= Some(99),4,2 AND i_current_price_null_count@1 != row_count@2 AND i_current_price_min@3 <= Some(149),4,2, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/warehouse/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/warehouse/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/warehouse/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/warehouse/part-3.parquet]]}, projection=[w_warehouse_sk, w_warehouse_name], file_type=parquet + └────────────────────────────────────────────────── + "#); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_22() -> Result<()> { + let display = test_tpcds_query("q22").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [qoh@4 ASC, i_product_name@0 ASC, i_brand@1 ASC, i_class@2 ASC, i_category@3 ASC], fetch=100 + │ [Stage 4] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ SortExec: TopK(fetch=100), expr=[qoh@4 ASC, i_product_name@0 ASC, i_brand@1 ASC, i_class@2 ASC, i_category@3 ASC], preserve_partitioning=[true] + │ ProjectionExec: expr=[i_product_name@0 as i_product_name, i_brand@1 as i_brand, i_class@2 as i_class, i_category@3 as i_category, avg(inventory.inv_quantity_on_hand)@5 as qoh] + │ AggregateExec: mode=FinalPartitioned, gby=[i_product_name@0 as i_product_name, i_brand@1 as i_brand, i_class@2 as i_class, i_category@3 as i_category, __grouping_id@4 as __grouping_id], aggr=[avg(inventory.inv_quantity_on_hand)] + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([i_product_name@0, i_brand@1, i_class@2, i_category@3, __grouping_id@4], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[(NULL as i_product_name, NULL as i_brand, NULL as i_class, NULL as i_category), (i_product_name@4 as i_product_name, NULL as i_brand, NULL as i_class, NULL as i_category), (i_product_name@4 as i_product_name, i_brand@1 as i_brand, NULL as i_class, NULL as i_category), (i_product_name@4 as i_product_name, i_brand@1 as i_brand, i_class@2 as i_class, NULL as i_category), (i_product_name@4 as i_product_name, i_brand@1 as i_brand, i_class@2 as i_class, i_category@3 as i_category)], aggr=[avg(inventory.inv_quantity_on_hand)] + │ ProjectionExec: expr=[inv_quantity_on_hand@4 as inv_quantity_on_hand, i_brand@0 as i_brand, i_class@1 as i_class, i_category@2 as i_category, i_product_name@3 as i_product_name] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, inv_item_sk@0)], projection=[i_brand@1, i_class@2, i_category@3, i_product_name@4, inv_quantity_on_hand@6] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, inv_date_sk@0)], projection=[inv_item_sk@2, inv_quantity_on_hand@3] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/inventory/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:..], ...]}, projection=[inv_date_sk, inv_item_sk, inv_quantity_on_hand], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand, i_class, i_category, i_product_name], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ FilterExec: d_month_seq@1 >= 1200 AND d_month_seq@1 <= 1211, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_23() -> Result<()> { + let display = test_tpcds_query("q23").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [c_last_name@0 ASC, c_first_name@1 ASC, sales@2 ASC], fetch=100 + │ [Stage 33] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 33 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ SortExec: TopK(fetch=100), expr=[c_last_name@0 ASC, c_first_name@1 ASC, sales@2 ASC], preserve_partitioning=[true] + │ InterleaveExec + │ ProjectionExec: expr=[c_last_name@0 as c_last_name, c_first_name@1 as c_first_name, sum(catalog_sales.cs_quantity * catalog_sales.cs_list_price)@2 as sales] + │ AggregateExec: mode=FinalPartitioned, gby=[c_last_name@0 as c_last_name, c_first_name@1 as c_first_name], aggr=[sum(catalog_sales.cs_quantity * catalog_sales.cs_list_price)] + │ [Stage 16] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + │ ProjectionExec: expr=[c_last_name@0 as c_last_name, c_first_name@1 as c_first_name, sum(web_sales.ws_quantity * web_sales.ws_list_price)@2 as sales] + │ AggregateExec: mode=FinalPartitioned, gby=[c_last_name@0 as c_last_name, c_first_name@1 as c_first_name], aggr=[sum(web_sales.ws_quantity * web_sales.ws_list_price)] + │ [Stage 32] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 16 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([c_last_name@0, c_first_name@1], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[c_last_name@3 as c_last_name, c_first_name@2 as c_first_name], aggr=[sum(catalog_sales.cs_quantity * catalog_sales.cs_list_price)] + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(cs_bill_customer_sk@0, CAST(best_ss_customer.c_customer_sk AS Float64)@1)], projection=[cs_quantity@1, cs_list_price@2, c_first_name@3, c_last_name@4] + │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + │ [Stage 15] => NetworkShuffleExec: output_partitions=3, input_tasks=1 + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] + │ RepartitionExec: partitioning=Hash([cs_bill_customer_sk@0], 9), input_partitions=3 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(cs_item_sk@1, item_sk@0)], projection=[cs_bill_customer_sk@0, cs_quantity@2, cs_list_price@3, c_first_name@4, c_last_name@5] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=3 + │ ProjectionExec: expr=[i_item_sk@0 as item_sk] + │ FilterExec: count(Int64(1))@1 > 4, projection=[i_item_sk@0] + │ ProjectionExec: expr=[i_item_sk@1 as i_item_sk, count(Int64(1))@3 as count(Int64(1))] + │ AggregateExec: mode=FinalPartitioned, gby=[itemdesc@0 as itemdesc, i_item_sk@1 as i_item_sk, d_date@2 as d_date], aggr=[count(Int64(1))] + │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p5] t1:[p6..p11] t2:[p12..p17] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, cs_sold_date_sk@0)], projection=[cs_bill_customer_sk@3, cs_item_sk@4, cs_quantity@5, cs_list_price@6, c_first_name@7, c_last_name@8] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ ProjectionExec: expr=[cs_sold_date_sk@2 as cs_sold_date_sk, cs_bill_customer_sk@3 as cs_bill_customer_sk, cs_item_sk@4 as cs_item_sk, cs_quantity@5 as cs_quantity, cs_list_price@6 as cs_list_price, c_first_name@0 as c_first_name, c_last_name@1 as c_last_name] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(customer.c_customer_sk AS Float64)@3, cs_bill_customer_sk@1)], projection=[c_first_name@1, c_last_name@2, cs_sold_date_sk@4, cs_bill_customer_sk@5, cs_item_sk@6, cs_quantity@7, cs_list_price@8] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_item_sk, cs_quantity, cs_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_year@1 = 2000 AND d_moy@2 = 2, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2000 AND d_moy@8 = 2, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 2 AND 2 <= d_moy_max@5, required_guarantees=[d_moy in (2), d_year in (2000)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] + │ RepartitionExec: partitioning=Hash([itemdesc@0, i_item_sk@1, d_date@2], 9), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[itemdesc@1 as itemdesc, i_item_sk@2 as i_item_sk, d_date@0 as d_date], aggr=[count(Int64(1))] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_item_sk@0, i_item_sk@1)], projection=[d_date@1, itemdesc@2, i_item_sk@3] + │ CoalescePartitionsExec + │ [Stage 5] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=3 + │ ProjectionExec: expr=[substr(i_item_desc@1, 1, 30) as itemdesc, i_item_sk@0 as i_item_sk] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_item_desc], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p5] t1:[p6..p11] t2:[p12..p17] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ ProjectionExec: expr=[ss_item_sk@1 as ss_item_sk, d_date@0 as d_date] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, ss_sold_date_sk@0)], projection=[d_date@1, ss_item_sk@4] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_date@1 as d_date, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_year@2 IN (SET) ([2000, 2001, 2002, 2003]), projection=[d_date_sk@0, d_date@1] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date, d_year], file_type=parquet, predicate=d_year@6 IN (SET) ([2000, 2001, 2002, 2003]), pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2003 AND 2003 <= d_year_max@1, required_guarantees=[d_year in (2000, 2001, 2002, 2003)] + └────────────────────────────────────────────────── + ┌───── Stage 15 ── Tasks: t0:[p0..p8] + │ RepartitionExec: partitioning=Hash([CAST(best_ss_customer.c_customer_sk AS Float64)@1], 9), input_partitions=3 + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, CAST(c_customer_sk@0 AS Float64) as CAST(best_ss_customer.c_customer_sk AS Float64)] + │ FilterExec: sum(store_sales.ss_quantity * store_sales.ss_sales_price)@1 > 0.5 * max(max_store_sales.tpcds_cmax)@2, projection=[c_customer_sk@0] + │ AggregateExec: mode=FinalPartitioned, gby=[c_customer_sk@0 as c_customer_sk], aggr=[sum(store_sales.ss_quantity * store_sales.ss_sales_price), max(max_store_sales.tpcds_cmax)] + │ RepartitionExec: partitioning=Hash([c_customer_sk@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[c_customer_sk@2 as c_customer_sk], aggr=[sum(store_sales.ss_quantity * store_sales.ss_sales_price), max(max_store_sales.tpcds_cmax)] + │ ProjectionExec: expr=[ss_quantity@1 as ss_quantity, ss_sales_price@2 as ss_sales_price, c_customer_sk@3 as c_customer_sk, tpcds_cmax@0 as tpcds_cmax] + │ CrossJoinExec + │ ProjectionExec: expr=[max(sq2.csales)@0 as tpcds_cmax] + │ AggregateExec: mode=Final, gby=[], aggr=[max(sq2.csales)] + │ CoalescePartitionsExec + │ [Stage 12] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[ss_quantity@1 as ss_quantity, ss_sales_price@2 as ss_sales_price, c_customer_sk@0 as c_customer_sk] + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(customer.c_customer_sk AS Float64)@1, ss_customer_sk@0)], projection=[c_customer_sk@0, ss_quantity@3, ss_sales_price@4] + │ [Stage 13] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 14] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 12 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ AggregateExec: mode=Partial, gby=[], aggr=[max(sq2.csales)] + │ ProjectionExec: expr=[sum(store_sales.ss_quantity * store_sales.ss_sales_price)@1 as csales] + │ AggregateExec: mode=FinalPartitioned, gby=[c_customer_sk@0 as c_customer_sk], aggr=[sum(store_sales.ss_quantity * store_sales.ss_sales_price)] + │ [Stage 11] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 11 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([c_customer_sk@0], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[c_customer_sk@2 as c_customer_sk], aggr=[sum(store_sales.ss_quantity * store_sales.ss_sales_price)] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_quantity@3, ss_sales_price@4, c_customer_sk@5] + │ CoalescePartitionsExec + │ [Stage 8] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ ProjectionExec: expr=[ss_sold_date_sk@1 as ss_sold_date_sk, ss_quantity@2 as ss_quantity, ss_sales_price@3 as ss_sales_price, c_customer_sk@0 as c_customer_sk] + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(customer.c_customer_sk AS Float64)@1, ss_customer_sk@1)], projection=[c_customer_sk@0, ss_sold_date_sk@2, ss_quantity@4, ss_sales_price@5] + │ [Stage 9] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 10] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 8 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_year@1 IN (SET) ([2000, 2001, 2002, 2003]), projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 IN (SET) ([2000, 2001, 2002, 2003]), pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2003 AND 2003 <= d_year_max@1, required_guarantees=[d_year in (2000, 2001, 2002, 2003)] + └────────────────────────────────────────────────── + ┌───── Stage 9 ── Tasks: t0:[p0..p8] t1:[p0..p8] + │ RepartitionExec: partitioning=Hash([CAST(customer.c_customer_sk AS Float64)@1], 9), input_partitions=3 + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 10 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] + │ RepartitionExec: partitioning=Hash([ss_customer_sk@1], 9), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_quantity, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 13 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ RepartitionExec: partitioning=Hash([CAST(customer.c_customer_sk AS Float64)@1], 3), input_partitions=3 + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 14 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] + │ RepartitionExec: partitioning=Hash([ss_customer_sk@0], 3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_customer_sk, ss_quantity, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 32 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([c_last_name@0, c_first_name@1], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[c_last_name@3 as c_last_name, c_first_name@2 as c_first_name], aggr=[sum(web_sales.ws_quantity * web_sales.ws_list_price)] + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(ws_bill_customer_sk@0, CAST(best_ss_customer.c_customer_sk AS Float64)@1)], projection=[ws_quantity@1, ws_list_price@2, c_first_name@3, c_last_name@4] + │ [Stage 23] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + │ [Stage 31] => NetworkShuffleExec: output_partitions=3, input_tasks=1 + └────────────────────────────────────────────────── + ┌───── Stage 23 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] + │ RepartitionExec: partitioning=Hash([ws_bill_customer_sk@0], 9), input_partitions=3 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ws_item_sk@0, item_sk@0)], projection=[ws_bill_customer_sk@1, ws_quantity@2, ws_list_price@3, c_first_name@4, c_last_name@5] + │ CoalescePartitionsExec + │ [Stage 19] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=3 + │ ProjectionExec: expr=[i_item_sk@0 as item_sk] + │ FilterExec: count(Int64(1))@1 > 4, projection=[i_item_sk@0] + │ ProjectionExec: expr=[i_item_sk@1 as i_item_sk, count(Int64(1))@3 as count(Int64(1))] + │ AggregateExec: mode=FinalPartitioned, gby=[itemdesc@0 as itemdesc, i_item_sk@1 as i_item_sk, d_date@2 as d_date], aggr=[count(Int64(1))] + │ [Stage 22] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 19 ── Tasks: t0:[p0..p5] t1:[p6..p11] t2:[p12..p17] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ws_sold_date_sk@0)], projection=[ws_item_sk@3, ws_bill_customer_sk@4, ws_quantity@5, ws_list_price@6, c_first_name@7, c_last_name@8] + │ CoalescePartitionsExec + │ [Stage 17] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ ProjectionExec: expr=[ws_sold_date_sk@2 as ws_sold_date_sk, ws_item_sk@3 as ws_item_sk, ws_bill_customer_sk@4 as ws_bill_customer_sk, ws_quantity@5 as ws_quantity, ws_list_price@6 as ws_list_price, c_first_name@0 as c_first_name, c_last_name@1 as c_last_name] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(customer.c_customer_sk AS Float64)@3, ws_bill_customer_sk@2)], projection=[c_first_name@1, c_last_name@2, ws_sold_date_sk@4, ws_item_sk@5, ws_bill_customer_sk@6, ws_quantity@7, ws_list_price@8] + │ CoalescePartitionsExec + │ [Stage 18] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_item_sk, ws_bill_customer_sk, ws_quantity, ws_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 17 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_year@1 = 2000 AND d_moy@2 = 2, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2000 AND d_moy@8 = 2, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 2 AND 2 <= d_moy_max@5, required_guarantees=[d_moy in (2), d_year in (2000)] + └────────────────────────────────────────────────── + ┌───── Stage 18 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 22 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] + │ RepartitionExec: partitioning=Hash([itemdesc@0, i_item_sk@1, d_date@2], 9), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[itemdesc@1 as itemdesc, i_item_sk@2 as i_item_sk, d_date@0 as d_date], aggr=[count(Int64(1))] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_item_sk@0, i_item_sk@1)], projection=[d_date@1, itemdesc@2, i_item_sk@3] + │ CoalescePartitionsExec + │ [Stage 21] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=3 + │ ProjectionExec: expr=[substr(i_item_desc@1, 1, 30) as itemdesc, i_item_sk@0 as i_item_sk] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_item_desc], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 21 ── Tasks: t0:[p0..p5] t1:[p6..p11] t2:[p12..p17] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ ProjectionExec: expr=[ss_item_sk@1 as ss_item_sk, d_date@0 as d_date] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, ss_sold_date_sk@0)], projection=[d_date@1, ss_item_sk@4] + │ CoalescePartitionsExec + │ [Stage 20] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 20 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_date@1 as d_date, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_year@2 IN (SET) ([2000, 2001, 2002, 2003]), projection=[d_date_sk@0, d_date@1] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date, d_year], file_type=parquet, predicate=d_year@6 IN (SET) ([2000, 2001, 2002, 2003]), pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2003 AND 2003 <= d_year_max@1, required_guarantees=[d_year in (2000, 2001, 2002, 2003)] + └────────────────────────────────────────────────── + ┌───── Stage 31 ── Tasks: t0:[p0..p8] + │ RepartitionExec: partitioning=Hash([CAST(best_ss_customer.c_customer_sk AS Float64)@1], 9), input_partitions=3 + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, CAST(c_customer_sk@0 AS Float64) as CAST(best_ss_customer.c_customer_sk AS Float64)] + │ FilterExec: sum(store_sales.ss_quantity * store_sales.ss_sales_price)@1 > 0.5 * max(max_store_sales.tpcds_cmax)@2, projection=[c_customer_sk@0] + │ AggregateExec: mode=FinalPartitioned, gby=[c_customer_sk@0 as c_customer_sk], aggr=[sum(store_sales.ss_quantity * store_sales.ss_sales_price), max(max_store_sales.tpcds_cmax)] + │ RepartitionExec: partitioning=Hash([c_customer_sk@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[c_customer_sk@2 as c_customer_sk], aggr=[sum(store_sales.ss_quantity * store_sales.ss_sales_price), max(max_store_sales.tpcds_cmax)] + │ ProjectionExec: expr=[ss_quantity@1 as ss_quantity, ss_sales_price@2 as ss_sales_price, c_customer_sk@3 as c_customer_sk, tpcds_cmax@0 as tpcds_cmax] + │ CrossJoinExec + │ ProjectionExec: expr=[max(sq2.csales)@0 as tpcds_cmax] + │ AggregateExec: mode=Final, gby=[], aggr=[max(sq2.csales)] + │ CoalescePartitionsExec + │ [Stage 28] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[ss_quantity@1 as ss_quantity, ss_sales_price@2 as ss_sales_price, c_customer_sk@0 as c_customer_sk] + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(customer.c_customer_sk AS Float64)@1, ss_customer_sk@0)], projection=[c_customer_sk@0, ss_quantity@3, ss_sales_price@4] + │ [Stage 29] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 30] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 28 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ AggregateExec: mode=Partial, gby=[], aggr=[max(sq2.csales)] + │ ProjectionExec: expr=[sum(store_sales.ss_quantity * store_sales.ss_sales_price)@1 as csales] + │ AggregateExec: mode=FinalPartitioned, gby=[c_customer_sk@0 as c_customer_sk], aggr=[sum(store_sales.ss_quantity * store_sales.ss_sales_price)] + │ [Stage 27] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 27 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([c_customer_sk@0], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[c_customer_sk@2 as c_customer_sk], aggr=[sum(store_sales.ss_quantity * store_sales.ss_sales_price)] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_quantity@3, ss_sales_price@4, c_customer_sk@5] + │ CoalescePartitionsExec + │ [Stage 24] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ ProjectionExec: expr=[ss_sold_date_sk@1 as ss_sold_date_sk, ss_quantity@2 as ss_quantity, ss_sales_price@3 as ss_sales_price, c_customer_sk@0 as c_customer_sk] + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(customer.c_customer_sk AS Float64)@1, ss_customer_sk@1)], projection=[c_customer_sk@0, ss_sold_date_sk@2, ss_quantity@4, ss_sales_price@5] + │ [Stage 25] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 26] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 24 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_year@1 IN (SET) ([2000, 2001, 2002, 2003]), projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 IN (SET) ([2000, 2001, 2002, 2003]), pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2003 AND 2003 <= d_year_max@1, required_guarantees=[d_year in (2000, 2001, 2002, 2003)] + └────────────────────────────────────────────────── + ┌───── Stage 25 ── Tasks: t0:[p0..p8] t1:[p0..p8] + │ RepartitionExec: partitioning=Hash([CAST(customer.c_customer_sk AS Float64)@1], 9), input_partitions=3 + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 26 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] + │ RepartitionExec: partitioning=Hash([ss_customer_sk@1], 9), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_quantity, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 29 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ RepartitionExec: partitioning=Hash([CAST(customer.c_customer_sk AS Float64)@1], 3), input_partitions=3 + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 30 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] + │ RepartitionExec: partitioning=Hash([ss_customer_sk@0], 3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_customer_sk, ss_quantity, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_24() -> Result<()> { + let display = test_tpcds_query("q24").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortExec: expr=[c_last_name@0 ASC NULLS LAST, c_first_name@1 ASC NULLS LAST, s_store_name@2 ASC NULLS LAST], preserve_partitioning=[false] + │ ProjectionExec: expr=[c_last_name@0 as c_last_name, c_first_name@1 as c_first_name, s_store_name@2 as s_store_name, sum(ssales.netpaid)@3 as paid] + │ NestedLoopJoinExec: join_type=Inner, filter=join_proj_push_down_1@1 > Float64(0.05) * avg(ssales.netpaid)@0, projection=[c_last_name@0, c_first_name@1, s_store_name@2, sum(ssales.netpaid)@3, Float64(0.05) * avg(ssales.netpaid)@5] + │ ProjectionExec: expr=[c_last_name@0 as c_last_name, c_first_name@1 as c_first_name, s_store_name@2 as s_store_name, sum(ssales.netpaid)@3 as sum(ssales.netpaid), CAST(sum(ssales.netpaid)@3 AS Decimal128(38, 15)) as join_proj_push_down_1] + │ CoalescePartitionsExec + │ AggregateExec: mode=FinalPartitioned, gby=[c_last_name@0 as c_last_name, c_first_name@1 as c_first_name, s_store_name@2 as s_store_name], aggr=[sum(ssales.netpaid)] + │ [Stage 8] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ ProjectionExec: expr=[CAST(0.05 * CAST(avg(ssales.netpaid)@0 AS Float64) AS Decimal128(38, 15)) as Float64(0.05) * avg(ssales.netpaid)] + │ AggregateExec: mode=Final, gby=[], aggr=[avg(ssales.netpaid)] + │ CoalescePartitionsExec + │ [Stage 16] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 8 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ RepartitionExec: partitioning=Hash([c_last_name@0, c_first_name@1, s_store_name@2], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[c_last_name@0 as c_last_name, c_first_name@1 as c_first_name, s_store_name@2 as s_store_name], aggr=[sum(ssales.netpaid)] + │ ProjectionExec: expr=[c_last_name@0 as c_last_name, c_first_name@1 as c_first_name, s_store_name@2 as s_store_name, sum(store_sales.ss_net_paid)@10 as netpaid] + │ AggregateExec: mode=FinalPartitioned, gby=[c_last_name@0 as c_last_name, c_first_name@1 as c_first_name, s_store_name@2 as s_store_name, ca_state@3 as ca_state, s_state@4 as s_state, i_color@5 as i_color, i_current_price@6 as i_current_price, i_manager_id@7 as i_manager_id, i_units@8 as i_units, i_size@9 as i_size], aggr=[sum(store_sales.ss_net_paid)], ordering_mode=PartiallySorted([5]) + │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([c_last_name@0, c_first_name@1, s_store_name@2, ca_state@3, s_state@4, i_color@5, i_current_price@6, i_manager_id@7, i_units@8, i_size@9], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[c_last_name@9 as c_last_name, c_first_name@8 as c_first_name, s_store_name@1 as s_store_name, ca_state@10 as ca_state, s_state@2 as s_state, i_color@5 as i_color, i_current_price@3 as i_current_price, i_manager_id@7 as i_manager_id, i_units@6 as i_units, i_size@4 as i_size], aggr=[sum(store_sales.ss_net_paid)], ordering_mode=PartiallySorted([5]) + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_current_addr_sk@9, ca_address_sk@0), (s_zip@3, ca_zip@2)], filter=c_birth_country@0 != CAST(upper(ca_country@1) AS Utf8View), projection=[ss_net_paid@0, s_store_name@1, s_state@2, i_current_price@4, i_size@5, i_color@6, i_units@7, i_manager_id@8, c_first_name@10, c_last_name@11, ca_state@14] + │ CoalescePartitionsExec + │ [Stage 6] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=3 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_state, ca_zip, ca_country], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p8] t1:[p9..p17] t2:[p18..p26] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_customer_sk@0, CAST(customer.c_customer_sk AS Float64)@5)], projection=[ss_net_paid@1, s_store_name@2, s_state@3, s_zip@4, i_current_price@5, i_size@6, i_color@7, i_units@8, i_manager_id@9, c_current_addr_sk@11, c_first_name@12, c_last_name@13, c_birth_country@14] + │ CoalescePartitionsExec + │ [Stage 5] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=3 + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, c_current_addr_sk@1 as c_current_addr_sk, c_first_name@2 as c_first_name, c_last_name@3 as c_last_name, c_birth_country@4 as c_birth_country, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_current_addr_sk, c_first_name, c_last_name, c_birth_country], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p8] t1:[p9..p17] t2:[p18..p26] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_item_sk@0, i_item_sk@0)], projection=[ss_customer_sk@1, ss_net_paid@2, s_store_name@3, s_state@4, s_zip@5, i_current_price@7, i_size@8, i_color@9, i_units@10, i_manager_id@11] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=3 + │ FilterExec: i_color@3 = peach + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_current_price, i_size, i_color, i_units, i_manager_id], file_type=parquet, predicate=i_color@17 = peach AND DynamicFilter [ empty ], pruning_predicate=i_color_null_count@2 != row_count@3 AND i_color_min@0 <= peach AND peach <= i_color_max@1, required_guarantees=[i_color in (peach)] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p8] t1:[p9..p17] t2:[p18..p26] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[ss_item_sk@3 as ss_item_sk, ss_customer_sk@4 as ss_customer_sk, ss_net_paid@5 as ss_net_paid, s_store_name@0 as s_store_name, s_state@1 as s_state, s_zip@2 as s_zip] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@4, ss_store_sk@2)], projection=[s_store_name@1, s_state@2, s_zip@3, ss_item_sk@5, ss_customer_sk@6, ss_net_paid@8] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(sr_ticket_number@1, ss_ticket_number@3), (sr_item_sk@0, ss_item_sk@0)], projection=[ss_item_sk@2, ss_customer_sk@3, ss_store_sk@4, ss_net_paid@6] + │ [Stage 2] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, s_store_name@1 as s_store_name, s_state@2 as s_state, s_zip@3 as s_zip, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] + │ FilterExec: s_market_id@2 = 8, projection=[s_store_sk@0, s_store_name@1, s_state@3, s_zip@4] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_store_name, s_market_id, s_state, s_zip], file_type=parquet, predicate=s_market_id@10 = 8, pruning_predicate=s_market_id_null_count@2 != row_count@3 AND s_market_id_min@0 <= 8 AND 8 <= s_market_id_max@1, required_guarantees=[s_market_id in (8)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] + │ RepartitionExec: partitioning=Hash([sr_ticket_number@1, sr_item_sk@0], 9), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], ...]}, projection=[sr_item_sk, sr_ticket_number], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] + │ RepartitionExec: partitioning=Hash([ss_ticket_number@3, ss_item_sk@0], 9), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_item_sk, ss_customer_sk, ss_store_sk, ss_ticket_number, ss_net_paid], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 16 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ AggregateExec: mode=Partial, gby=[], aggr=[avg(ssales.netpaid)] + │ ProjectionExec: expr=[sum(store_sales.ss_net_paid)@10 as netpaid] + │ AggregateExec: mode=FinalPartitioned, gby=[c_last_name@0 as c_last_name, c_first_name@1 as c_first_name, s_store_name@2 as s_store_name, ca_state@3 as ca_state, s_state@4 as s_state, i_color@5 as i_color, i_current_price@6 as i_current_price, i_manager_id@7 as i_manager_id, i_units@8 as i_units, i_size@9 as i_size], aggr=[sum(store_sales.ss_net_paid)] + │ [Stage 15] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 15 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([c_last_name@0, c_first_name@1, s_store_name@2, ca_state@3, s_state@4, i_color@5, i_current_price@6, i_manager_id@7, i_units@8, i_size@9], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[c_last_name@9 as c_last_name, c_first_name@8 as c_first_name, s_store_name@1 as s_store_name, ca_state@10 as ca_state, s_state@2 as s_state, i_color@5 as i_color, i_current_price@3 as i_current_price, i_manager_id@7 as i_manager_id, i_units@6 as i_units, i_size@4 as i_size], aggr=[sum(store_sales.ss_net_paid)] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_current_addr_sk@9, ca_address_sk@0), (s_zip@3, ca_zip@2)], filter=c_birth_country@0 != CAST(upper(ca_country@1) AS Utf8View), projection=[ss_net_paid@0, s_store_name@1, s_state@2, i_current_price@4, i_size@5, i_color@6, i_units@7, i_manager_id@8, c_first_name@10, c_last_name@11, ca_state@14] + │ CoalescePartitionsExec + │ [Stage 14] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=3 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_state, ca_zip, ca_country], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 14 ── Tasks: t0:[p0..p8] t1:[p9..p17] t2:[p18..p26] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_customer_sk@0, CAST(customer.c_customer_sk AS Float64)@5)], projection=[ss_net_paid@1, s_store_name@2, s_state@3, s_zip@4, i_current_price@5, i_size@6, i_color@7, i_units@8, i_manager_id@9, c_current_addr_sk@11, c_first_name@12, c_last_name@13, c_birth_country@14] + │ CoalescePartitionsExec + │ [Stage 13] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=3 + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, c_current_addr_sk@1 as c_current_addr_sk, c_first_name@2 as c_first_name, c_last_name@3 as c_last_name, c_birth_country@4 as c_birth_country, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_current_addr_sk, c_first_name, c_last_name, c_birth_country], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 13 ── Tasks: t0:[p0..p8] t1:[p9..p17] t2:[p18..p26] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_item_sk@0, i_item_sk@0)], projection=[ss_customer_sk@1, ss_net_paid@2, s_store_name@3, s_state@4, s_zip@5, i_current_price@7, i_size@8, i_color@9, i_units@10, i_manager_id@11] + │ CoalescePartitionsExec + │ [Stage 12] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=3 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_current_price, i_size, i_color, i_units, i_manager_id], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 12 ── Tasks: t0:[p0..p8] t1:[p9..p17] t2:[p18..p26] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[ss_item_sk@3 as ss_item_sk, ss_customer_sk@4 as ss_customer_sk, ss_net_paid@5 as ss_net_paid, s_store_name@0 as s_store_name, s_state@1 as s_state, s_zip@2 as s_zip] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@4, ss_store_sk@2)], projection=[s_store_name@1, s_state@2, s_zip@3, ss_item_sk@5, ss_customer_sk@6, ss_net_paid@8] + │ CoalescePartitionsExec + │ [Stage 9] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(sr_ticket_number@1, ss_ticket_number@3), (sr_item_sk@0, ss_item_sk@0)], projection=[ss_item_sk@2, ss_customer_sk@3, ss_store_sk@4, ss_net_paid@6] + │ [Stage 10] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + │ [Stage 11] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 9 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, s_store_name@1 as s_store_name, s_state@2 as s_state, s_zip@3 as s_zip, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] + │ FilterExec: s_market_id@2 = 8, projection=[s_store_sk@0, s_store_name@1, s_state@3, s_zip@4] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_store_name, s_market_id, s_state, s_zip], file_type=parquet, predicate=s_market_id@10 = 8, pruning_predicate=s_market_id_null_count@2 != row_count@3 AND s_market_id_min@0 <= 8 AND 8 <= s_market_id_max@1, required_guarantees=[s_market_id in (8)] + └────────────────────────────────────────────────── + ┌───── Stage 10 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] + │ RepartitionExec: partitioning=Hash([sr_ticket_number@1, sr_item_sk@0], 9), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], ...]}, projection=[sr_item_sk, sr_ticket_number], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 11 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] + │ RepartitionExec: partitioning=Hash([ss_ticket_number@3, ss_item_sk@0], 9), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_item_sk, ss_customer_sk, ss_store_sk, ss_ticket_number, ss_net_paid], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_25() -> Result<()> { + let display = test_tpcds_query("q25").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [i_item_id@0 ASC NULLS LAST, i_item_desc@1 ASC NULLS LAST, s_store_id@2 ASC NULLS LAST, s_store_name@3 ASC NULLS LAST], fetch=100 + │ [Stage 11] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 11 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ SortExec: TopK(fetch=100), expr=[i_item_id@0 ASC NULLS LAST, i_item_desc@1 ASC NULLS LAST, s_store_id@2 ASC NULLS LAST, s_store_name@3 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[i_item_id@0 as i_item_id, i_item_desc@1 as i_item_desc, s_store_id@2 as s_store_id, s_store_name@3 as s_store_name, sum(store_sales.ss_net_profit)@4 as store_sales_profit, sum(store_returns.sr_net_loss)@5 as store_returns_loss, sum(catalog_sales.cs_net_profit)@6 as catalog_sales_profit] + │ AggregateExec: mode=FinalPartitioned, gby=[i_item_id@0 as i_item_id, i_item_desc@1 as i_item_desc, s_store_id@2 as s_store_id, s_store_name@3 as s_store_name], aggr=[sum(store_sales.ss_net_profit), sum(store_returns.sr_net_loss), sum(catalog_sales.cs_net_profit)] + │ [Stage 10] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 10 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([i_item_id@0, i_item_desc@1, s_store_id@2, s_store_name@3], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_item_id@5 as i_item_id, i_item_desc@6 as i_item_desc, s_store_id@3 as s_store_id, s_store_name@4 as s_store_name], aggr=[sum(store_sales.ss_net_profit), sum(store_returns.sr_net_loss), sum(catalog_sales.cs_net_profit)] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_item_sk@0, i_item_sk@0)], projection=[ss_net_profit@1, sr_net_loss@2, cs_net_profit@3, s_store_id@4, s_store_name@5, i_item_id@7, i_item_desc@8] + │ CoalescePartitionsExec + │ [Stage 9] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=3 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_item_id, i_item_desc], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 9 ── Tasks: t0:[p0..p8] t1:[p9..p17] t2:[p18..p26] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[ss_item_sk@2 as ss_item_sk, ss_net_profit@3 as ss_net_profit, sr_net_loss@4 as sr_net_loss, cs_net_profit@5 as cs_net_profit, s_store_id@0 as s_store_id, s_store_name@1 as s_store_name] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@3, ss_store_sk@1)], projection=[s_store_id@1, s_store_name@2, ss_item_sk@4, ss_net_profit@6, sr_net_loss@7, cs_net_profit@8] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(cs_sold_date_sk@4, CAST(d3.d_date_sk AS Float64)@1)], projection=[ss_item_sk@0, ss_store_sk@1, ss_net_profit@2, sr_net_loss@3, cs_net_profit@5] + │ CoalescePartitionsExec + │ [Stage 8] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=3 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(d3.d_date_sk AS Float64)] + │ FilterExec: d_moy@2 >= 4 AND d_moy@2 <= 10 AND d_year@1 = 2001, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 >= 4 AND d_moy@8 <= 10 AND d_year@6 = 2001, pruning_predicate=d_moy_null_count@1 != row_count@2 AND d_moy_max@0 >= 4 AND d_moy_null_count@1 != row_count@2 AND d_moy_min@3 <= 10 AND d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5, required_guarantees=[d_year in (2001)] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_store_id, s_store_name, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 8 ── Tasks: t0:[p0..p8] t1:[p9..p17] t2:[p18..p26] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(sr_returned_date_sk@3, CAST(d2.d_date_sk AS Float64)@1)], projection=[ss_item_sk@0, ss_store_sk@1, ss_net_profit@2, sr_net_loss@4, cs_sold_date_sk@5, cs_net_profit@6] + │ CoalescePartitionsExec + │ [Stage 7] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=3 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(d2.d_date_sk AS Float64)] + │ FilterExec: d_moy@2 >= 4 AND d_moy@2 <= 10 AND d_year@1 = 2001, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 >= 4 AND d_moy@8 <= 10 AND d_year@6 = 2001, pruning_predicate=d_moy_null_count@1 != row_count@2 AND d_moy_max@0 >= 4 AND d_moy_null_count@1 != row_count@2 AND d_moy_min@3 <= 10 AND d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5, required_guarantees=[d_year in (2001)] + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p8] t1:[p9..p17] t2:[p18..p26] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(d1.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_item_sk@3, ss_store_sk@4, ss_net_profit@5, sr_returned_date_sk@6, sr_net_loss@7, cs_sold_date_sk@8, cs_net_profit@9] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ ProjectionExec: expr=[ss_sold_date_sk@2 as ss_sold_date_sk, ss_item_sk@3 as ss_item_sk, ss_store_sk@4 as ss_store_sk, ss_net_profit@5 as ss_net_profit, sr_returned_date_sk@6 as sr_returned_date_sk, sr_net_loss@7 as sr_net_loss, cs_sold_date_sk@0 as cs_sold_date_sk, cs_net_profit@1 as cs_net_profit] + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(cs_bill_customer_sk@1, sr_customer_sk@6), (cs_item_sk@2, sr_item_sk@5)], projection=[cs_sold_date_sk@0, cs_net_profit@3, ss_sold_date_sk@4, ss_item_sk@5, ss_store_sk@6, ss_net_profit@7, sr_returned_date_sk@8, sr_net_loss@11] + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(d1.d_date_sk AS Float64)] + │ FilterExec: d_moy@2 = 4 AND d_year@1 = 2001, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 4 AND d_year@6 = 2001, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 4 AND 4 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5, required_guarantees=[d_moy in (4), d_year in (2001)] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] + │ RepartitionExec: partitioning=Hash([cs_bill_customer_sk@1, cs_item_sk@2], 9), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_item_sk, cs_net_profit], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] + │ RepartitionExec: partitioning=Hash([sr_customer_sk@6, sr_item_sk@5], 9), input_partitions=3 + │ ProjectionExec: expr=[ss_sold_date_sk@4 as ss_sold_date_sk, ss_item_sk@5 as ss_item_sk, ss_store_sk@6 as ss_store_sk, ss_net_profit@7 as ss_net_profit, sr_returned_date_sk@0 as sr_returned_date_sk, sr_item_sk@1 as sr_item_sk, sr_customer_sk@2 as sr_customer_sk, sr_net_loss@3 as sr_net_loss] + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(sr_customer_sk@2, ss_customer_sk@2), (sr_item_sk@1, ss_item_sk@1), (sr_ticket_number@3, ss_ticket_number@4)], projection=[sr_returned_date_sk@0, sr_item_sk@1, sr_customer_sk@2, sr_net_loss@4, ss_sold_date_sk@5, ss_item_sk@6, ss_store_sk@8, ss_net_profit@10] + │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] + │ RepartitionExec: partitioning=Hash([sr_customer_sk@2, sr_item_sk@1, sr_ticket_number@3], 9), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], ...]}, projection=[sr_returned_date_sk, sr_item_sk, sr_customer_sk, sr_ticket_number, sr_net_loss], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] + │ RepartitionExec: partitioning=Hash([ss_customer_sk@2, ss_item_sk@1, ss_ticket_number@4], 9), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_store_sk, ss_ticket_number, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_26() -> Result<()> { + let display = test_tpcds_query("q26").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [i_item_id@0 ASC NULLS LAST], fetch=100 + │ [Stage 7] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ SortExec: TopK(fetch=100), expr=[i_item_id@0 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[i_item_id@0 as i_item_id, avg(catalog_sales.cs_quantity)@1 as agg1, avg(catalog_sales.cs_list_price)@2 as agg2, avg(catalog_sales.cs_coupon_amt)@3 as agg3, avg(catalog_sales.cs_sales_price)@4 as agg4] + │ AggregateExec: mode=FinalPartitioned, gby=[i_item_id@0 as i_item_id], aggr=[avg(catalog_sales.cs_quantity), avg(catalog_sales.cs_list_price), avg(catalog_sales.cs_coupon_amt), avg(catalog_sales.cs_sales_price)] + │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([i_item_id@0], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_item_id@4 as i_item_id], aggr=[avg(catalog_sales.cs_quantity), avg(catalog_sales.cs_list_price), avg(catalog_sales.cs_coupon_amt), avg(catalog_sales.cs_sales_price)] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(promotion.p_promo_sk AS Float64)@1, cs_promo_sk@0)], projection=[cs_quantity@3, cs_list_price@4, cs_sales_price@5, cs_coupon_amt@6, i_item_id@7] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(cs_item_sk@0, i_item_sk@0)], projection=[cs_promo_sk@1, cs_quantity@2, cs_list_price@3, cs_sales_price@4, cs_coupon_amt@5, i_item_id@7] + │ CoalescePartitionsExec + │ [Stage 5] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=3 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[p_promo_sk@0 as p_promo_sk, CAST(p_promo_sk@0 AS Float64) as CAST(promotion.p_promo_sk AS Float64)] + │ FilterExec: p_channel_email@1 = N OR p_channel_event@2 = N, projection=[p_promo_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/promotion/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/promotion/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/promotion/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/promotion/part-3.parquet]]}, projection=[p_promo_sk, p_channel_email, p_channel_event], file_type=parquet, predicate=p_channel_email@9 = N OR p_channel_event@14 = N, pruning_predicate=p_channel_email_null_count@2 != row_count@3 AND p_channel_email_min@0 <= N AND N <= p_channel_email_max@1 OR p_channel_event_null_count@6 != row_count@3 AND p_channel_event_min@4 <= N AND N <= p_channel_event_max@5, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p8] t1:[p9..p17] t2:[p18..p26] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, cs_sold_date_sk@0)], projection=[cs_item_sk@3, cs_promo_sk@4, cs_quantity@5, cs_list_price@6, cs_sales_price@7, cs_coupon_amt@8] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(customer_demographics.cd_demo_sk AS Float64)@1, cs_bill_cdemo_sk@1)], projection=[cs_sold_date_sk@2, cs_item_sk@4, cs_promo_sk@5, cs_quantity@6, cs_list_price@7, cs_sales_price@8, cs_coupon_amt@9] + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_year@1 = 2000, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] + │ RepartitionExec: partitioning=Hash([CAST(customer_demographics.cd_demo_sk AS Float64)@1], 9), input_partitions=2 + │ ProjectionExec: expr=[cd_demo_sk@0 as cd_demo_sk, CAST(cd_demo_sk@0 AS Float64) as CAST(customer_demographics.cd_demo_sk AS Float64)] + │ FilterExec: cd_gender@1 = M AND cd_marital_status@2 = S AND cd_education_status@3 = College, projection=[cd_demo_sk@0] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], ...]}, projection=[cd_demo_sk, cd_gender, cd_marital_status, cd_education_status], file_type=parquet, predicate=cd_gender@1 = M AND cd_marital_status@2 = S AND cd_education_status@3 = College, pruning_predicate=cd_gender_null_count@2 != row_count@3 AND cd_gender_min@0 <= M AND M <= cd_gender_max@1 AND cd_marital_status_null_count@6 != row_count@3 AND cd_marital_status_min@4 <= S AND S <= cd_marital_status_max@5 AND cd_education_status_null_count@9 != row_count@3 AND cd_education_status_min@7 <= College AND College <= cd_education_status_max@8, required_guarantees=[cd_education_status in (College), cd_gender in (M), cd_marital_status in (S)] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] + │ RepartitionExec: partitioning=Hash([cs_bill_cdemo_sk@1], 9), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_bill_cdemo_sk, cs_item_sk, cs_promo_sk, cs_quantity, cs_list_price, cs_sales_price, cs_coupon_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_27() -> Result<()> { + let display = test_tpcds_query("q27").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [i_item_id@0 ASC, s_state@1 ASC], fetch=100 + │ [Stage 19] => NetworkCoalesceExec: output_partitions=12, input_tasks=4 + └────────────────────────────────────────────────── + ┌───── Stage 19 ── Tasks: t0:[p0..p2] t1:[p3..p5] t2:[p6..p8] t3:[p9..p11] + │ SortExec: TopK(fetch=100), expr=[i_item_id@0 ASC], preserve_partitioning=[true] + │ DistributedUnionExec: t0:[c0] t1:[c1(0/2)] t2:[c1(1/2)] t3:[c2] + │ ProjectionExec: expr=[i_item_id@0 as i_item_id, CAST(s_state@1 AS Utf8) as s_state, 0 as g_state, avg(results.agg1)@2 as agg1, avg(results.agg2)@3 as agg2, avg(results.agg3)@4 as agg3, avg(results.agg4)@5 as agg4] + │ AggregateExec: mode=FinalPartitioned, gby=[i_item_id@0 as i_item_id, s_state@1 as s_state], aggr=[avg(results.agg1), avg(results.agg2), avg(results.agg3), avg(results.agg4)], ordering_mode=PartiallySorted([1]) + │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + │ ProjectionExec: expr=[i_item_id@0 as i_item_id, NULL as s_state, 1 as g_state, avg(results.agg1)@1 as agg1, avg(results.agg2)@2 as agg2, avg(results.agg3)@3 as agg3, avg(results.agg4)@4 as agg4] + │ AggregateExec: mode=FinalPartitioned, gby=[i_item_id@0 as i_item_id], aggr=[avg(results.agg1), avg(results.agg2), avg(results.agg3), avg(results.agg4)] + │ [Stage 12] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + │ ProjectionExec: expr=[NULL as i_item_id, NULL as s_state, 1 as g_state, avg(results.agg1)@0 as agg1, avg(results.agg2)@1 as agg2, avg(results.agg3)@2 as agg3, avg(results.agg4)@3 as agg4] + │ AggregateExec: mode=Final, gby=[], aggr=[avg(results.agg1), avg(results.agg2), avg(results.agg3), avg(results.agg4)] + │ CoalescePartitionsExec + │ [Stage 18] => NetworkCoalesceExec: output_partitions=9, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] + │ RepartitionExec: partitioning=Hash([i_item_id@0, s_state@1], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_item_id@0 as i_item_id, s_state@1 as s_state], aggr=[avg(results.agg1), avg(results.agg2), avg(results.agg3), avg(results.agg4)], ordering_mode=PartiallySorted([1]) + │ ProjectionExec: expr=[i_item_id@5 as i_item_id, s_state@4 as s_state, ss_quantity@0 as agg1, ss_list_price@1 as agg2, ss_coupon_amt@3 as agg3, ss_sales_price@2 as agg4] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_item_sk@0, i_item_sk@0)], projection=[ss_quantity@1, ss_list_price@2, ss_sales_price@3, ss_coupon_amt@4, s_state@5, i_item_id@7] + │ CoalescePartitionsExec + │ [Stage 5] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=3 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p8] t1:[p9..p17] t2:[p18..p26] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[ss_item_sk@1 as ss_item_sk, ss_quantity@2 as ss_quantity, ss_list_price@3 as ss_list_price, ss_sales_price@4 as ss_sales_price, ss_coupon_amt@5 as ss_coupon_amt, s_state@0 as s_state] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@2, ss_store_sk@1)], projection=[s_state@1, ss_item_sk@3, ss_quantity@5, ss_list_price@6, ss_sales_price@7, ss_coupon_amt@8] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_item_sk@3, ss_store_sk@4, ss_quantity@5, ss_list_price@6, ss_sales_price@7, ss_coupon_amt@8] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(customer_demographics.cd_demo_sk AS Float64)@1, ss_cdemo_sk@2)], projection=[ss_sold_date_sk@2, ss_item_sk@3, ss_store_sk@5, ss_quantity@6, ss_list_price@7, ss_sales_price@8, ss_coupon_amt@9] + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, s_state@1 as s_state, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] + │ FilterExec: s_state@1 = TN + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_state], file_type=parquet, predicate=s_state@24 = TN, pruning_predicate=s_state_null_count@2 != row_count@3 AND s_state_min@0 <= TN AND TN <= s_state_max@1, required_guarantees=[s_state in (TN)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_year@1 = 2002, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] + │ RepartitionExec: partitioning=Hash([CAST(customer_demographics.cd_demo_sk AS Float64)@1], 9), input_partitions=2 + │ ProjectionExec: expr=[cd_demo_sk@0 as cd_demo_sk, CAST(cd_demo_sk@0 AS Float64) as CAST(customer_demographics.cd_demo_sk AS Float64)] + │ FilterExec: cd_gender@1 = M AND cd_marital_status@2 = S AND cd_education_status@3 = College, projection=[cd_demo_sk@0] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], ...]}, projection=[cd_demo_sk, cd_gender, cd_marital_status, cd_education_status], file_type=parquet, predicate=cd_gender@1 = M AND cd_marital_status@2 = S AND cd_education_status@3 = College, pruning_predicate=cd_gender_null_count@2 != row_count@3 AND cd_gender_min@0 <= M AND M <= cd_gender_max@1 AND cd_marital_status_null_count@6 != row_count@3 AND cd_marital_status_min@4 <= S AND S <= cd_marital_status_max@5 AND cd_education_status_null_count@9 != row_count@3 AND cd_education_status_min@7 <= College AND College <= cd_education_status_max@8, required_guarantees=[cd_education_status in (College), cd_gender in (M), cd_marital_status in (S)] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] + │ RepartitionExec: partitioning=Hash([ss_cdemo_sk@2], 9), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_cdemo_sk, ss_store_sk, ss_quantity, ss_list_price, ss_sales_price, ss_coupon_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 12 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([i_item_id@0], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_item_id@0 as i_item_id], aggr=[avg(results.agg1), avg(results.agg2), avg(results.agg3), avg(results.agg4)] + │ ProjectionExec: expr=[i_item_id@4 as i_item_id, ss_quantity@0 as agg1, ss_list_price@1 as agg2, ss_coupon_amt@3 as agg3, ss_sales_price@2 as agg4] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_item_sk@0, i_item_sk@0)], projection=[ss_quantity@1, ss_list_price@2, ss_sales_price@3, ss_coupon_amt@4, i_item_id@6] + │ CoalescePartitionsExec + │ [Stage 11] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=3 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 11 ── Tasks: t0:[p0..p8] t1:[p9..p17] t2:[p18..p26] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@1, ss_store_sk@1)], projection=[ss_item_sk@2, ss_quantity@4, ss_list_price@5, ss_sales_price@6, ss_coupon_amt@7] + │ CoalescePartitionsExec + │ [Stage 7] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_item_sk@3, ss_store_sk@4, ss_quantity@5, ss_list_price@6, ss_sales_price@7, ss_coupon_amt@8] + │ CoalescePartitionsExec + │ [Stage 8] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(customer_demographics.cd_demo_sk AS Float64)@1, ss_cdemo_sk@2)], projection=[ss_sold_date_sk@2, ss_item_sk@3, ss_store_sk@5, ss_quantity@6, ss_list_price@7, ss_sales_price@8, ss_coupon_amt@9] + │ [Stage 9] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + │ [Stage 10] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] + │ FilterExec: s_state@1 = TN, projection=[s_store_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_state], file_type=parquet, predicate=s_state@24 = TN, pruning_predicate=s_state_null_count@2 != row_count@3 AND s_state_min@0 <= TN AND TN <= s_state_max@1, required_guarantees=[s_state in (TN)] + └────────────────────────────────────────────────── + ┌───── Stage 8 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_year@1 = 2002, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] + └────────────────────────────────────────────────── + ┌───── Stage 9 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] + │ RepartitionExec: partitioning=Hash([CAST(customer_demographics.cd_demo_sk AS Float64)@1], 9), input_partitions=2 + │ ProjectionExec: expr=[cd_demo_sk@0 as cd_demo_sk, CAST(cd_demo_sk@0 AS Float64) as CAST(customer_demographics.cd_demo_sk AS Float64)] + │ FilterExec: cd_gender@1 = M AND cd_marital_status@2 = S AND cd_education_status@3 = College, projection=[cd_demo_sk@0] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], ...]}, projection=[cd_demo_sk, cd_gender, cd_marital_status, cd_education_status], file_type=parquet, predicate=cd_gender@1 = M AND cd_marital_status@2 = S AND cd_education_status@3 = College, pruning_predicate=cd_gender_null_count@2 != row_count@3 AND cd_gender_min@0 <= M AND M <= cd_gender_max@1 AND cd_marital_status_null_count@6 != row_count@3 AND cd_marital_status_min@4 <= S AND S <= cd_marital_status_max@5 AND cd_education_status_null_count@9 != row_count@3 AND cd_education_status_min@7 <= College AND College <= cd_education_status_max@8, required_guarantees=[cd_education_status in (College), cd_gender in (M), cd_marital_status in (S)] + └────────────────────────────────────────────────── + ┌───── Stage 10 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] + │ RepartitionExec: partitioning=Hash([ss_cdemo_sk@2], 9), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_cdemo_sk, ss_store_sk, ss_quantity, ss_list_price, ss_sales_price, ss_coupon_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 18 ── Tasks: t0:[p0..p2] t1:[p3..p5] t2:[p6..p8] + │ AggregateExec: mode=Partial, gby=[], aggr=[avg(results.agg1), avg(results.agg2), avg(results.agg3), avg(results.agg4)] + │ ProjectionExec: expr=[ss_quantity@0 as agg1, ss_list_price@1 as agg2, ss_coupon_amt@3 as agg3, ss_sales_price@2 as agg4] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_item_sk@0, i_item_sk@0)], projection=[ss_quantity@1, ss_list_price@2, ss_sales_price@3, ss_coupon_amt@4] + │ CoalescePartitionsExec + │ [Stage 17] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=3 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 17 ── Tasks: t0:[p0..p8] t1:[p9..p17] t2:[p18..p26] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@1, ss_store_sk@1)], projection=[ss_item_sk@2, ss_quantity@4, ss_list_price@5, ss_sales_price@6, ss_coupon_amt@7] + │ CoalescePartitionsExec + │ [Stage 13] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_item_sk@3, ss_store_sk@4, ss_quantity@5, ss_list_price@6, ss_sales_price@7, ss_coupon_amt@8] + │ CoalescePartitionsExec + │ [Stage 14] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(customer_demographics.cd_demo_sk AS Float64)@1, ss_cdemo_sk@2)], projection=[ss_sold_date_sk@2, ss_item_sk@3, ss_store_sk@5, ss_quantity@6, ss_list_price@7, ss_sales_price@8, ss_coupon_amt@9] + │ [Stage 15] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + │ [Stage 16] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 13 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] + │ FilterExec: s_state@1 = TN, projection=[s_store_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_state], file_type=parquet, predicate=s_state@24 = TN, pruning_predicate=s_state_null_count@2 != row_count@3 AND s_state_min@0 <= TN AND TN <= s_state_max@1, required_guarantees=[s_state in (TN)] + └────────────────────────────────────────────────── + ┌───── Stage 14 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_year@1 = 2002, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] + └────────────────────────────────────────────────── + ┌───── Stage 15 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] + │ RepartitionExec: partitioning=Hash([CAST(customer_demographics.cd_demo_sk AS Float64)@1], 9), input_partitions=2 + │ ProjectionExec: expr=[cd_demo_sk@0 as cd_demo_sk, CAST(cd_demo_sk@0 AS Float64) as CAST(customer_demographics.cd_demo_sk AS Float64)] + │ FilterExec: cd_gender@1 = M AND cd_marital_status@2 = S AND cd_education_status@3 = College, projection=[cd_demo_sk@0] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], ...]}, projection=[cd_demo_sk, cd_gender, cd_marital_status, cd_education_status], file_type=parquet, predicate=cd_gender@1 = M AND cd_marital_status@2 = S AND cd_education_status@3 = College, pruning_predicate=cd_gender_null_count@2 != row_count@3 AND cd_gender_min@0 <= M AND M <= cd_gender_max@1 AND cd_marital_status_null_count@6 != row_count@3 AND cd_marital_status_min@4 <= S AND S <= cd_marital_status_max@5 AND cd_education_status_null_count@9 != row_count@3 AND cd_education_status_min@7 <= College AND College <= cd_education_status_max@8, required_guarantees=[cd_education_status in (College), cd_gender in (M), cd_marital_status in (S)] + └────────────────────────────────────────────────── + ┌───── Stage 16 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] + │ RepartitionExec: partitioning=Hash([ss_cdemo_sk@2], 9), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_cdemo_sk, ss_store_sk, ss_quantity, ss_list_price, ss_sales_price, ss_coupon_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_28() -> Result<()> { + let display = test_tpcds_query("q28").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ ProjectionExec: expr=[b1_lp@3 as b1_lp, b1_cnt@4 as b1_cnt, b1_cntd@5 as b1_cntd, b2_lp@6 as b2_lp, b2_cnt@7 as b2_cnt, b2_cntd@8 as b2_cntd, b3_lp@9 as b3_lp, b3_cnt@10 as b3_cnt, b3_cntd@11 as b3_cntd, b4_lp@12 as b4_lp, b4_cnt@13 as b4_cnt, b4_cntd@14 as b4_cntd, b5_lp@15 as b5_lp, b5_cnt@16 as b5_cnt, b5_cntd@17 as b5_cntd, b6_lp@0 as b6_lp, b6_cnt@1 as b6_cnt, b6_cntd@2 as b6_cntd] + │ GlobalLimitExec: skip=0, fetch=100 + │ CrossJoinExec + │ ProjectionExec: expr=[avg(store_sales.ss_list_price)@0 as b6_lp, count(store_sales.ss_list_price)@1 as b6_cnt, count(DISTINCT store_sales.ss_list_price)@2 as b6_cntd] + │ AggregateExec: mode=Final, gby=[], aggr=[avg(store_sales.ss_list_price), count(store_sales.ss_list_price), count(DISTINCT store_sales.ss_list_price)] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=6, input_tasks=3 + │ ProjectionExec: expr=[b1_lp@3 as b1_lp, b1_cnt@4 as b1_cnt, b1_cntd@5 as b1_cntd, b2_lp@6 as b2_lp, b2_cnt@7 as b2_cnt, b2_cntd@8 as b2_cntd, b3_lp@9 as b3_lp, b3_cnt@10 as b3_cnt, b3_cntd@11 as b3_cntd, b4_lp@12 as b4_lp, b4_cnt@13 as b4_cnt, b4_cntd@14 as b4_cntd, b5_lp@0 as b5_lp, b5_cnt@1 as b5_cnt, b5_cntd@2 as b5_cntd] + │ CrossJoinExec + │ ProjectionExec: expr=[avg(store_sales.ss_list_price)@0 as b5_lp, count(store_sales.ss_list_price)@1 as b5_cnt, count(DISTINCT store_sales.ss_list_price)@2 as b5_cntd] + │ AggregateExec: mode=Final, gby=[], aggr=[avg(store_sales.ss_list_price), count(store_sales.ss_list_price), count(DISTINCT store_sales.ss_list_price)] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=6, input_tasks=3 + │ ProjectionExec: expr=[b1_lp@3 as b1_lp, b1_cnt@4 as b1_cnt, b1_cntd@5 as b1_cntd, b2_lp@6 as b2_lp, b2_cnt@7 as b2_cnt, b2_cntd@8 as b2_cntd, b3_lp@9 as b3_lp, b3_cnt@10 as b3_cnt, b3_cntd@11 as b3_cntd, b4_lp@0 as b4_lp, b4_cnt@1 as b4_cnt, b4_cntd@2 as b4_cntd] + │ CrossJoinExec + │ ProjectionExec: expr=[avg(store_sales.ss_list_price)@0 as b4_lp, count(store_sales.ss_list_price)@1 as b4_cnt, count(DISTINCT store_sales.ss_list_price)@2 as b4_cntd] + │ AggregateExec: mode=Final, gby=[], aggr=[avg(store_sales.ss_list_price), count(store_sales.ss_list_price), count(DISTINCT store_sales.ss_list_price)] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkCoalesceExec: output_partitions=6, input_tasks=3 + │ ProjectionExec: expr=[b1_lp@3 as b1_lp, b1_cnt@4 as b1_cnt, b1_cntd@5 as b1_cntd, b2_lp@6 as b2_lp, b2_cnt@7 as b2_cnt, b2_cntd@8 as b2_cntd, b3_lp@0 as b3_lp, b3_cnt@1 as b3_cnt, b3_cntd@2 as b3_cntd] + │ CrossJoinExec + │ ProjectionExec: expr=[avg(store_sales.ss_list_price)@0 as b3_lp, count(store_sales.ss_list_price)@1 as b3_cnt, count(DISTINCT store_sales.ss_list_price)@2 as b3_cntd] + │ AggregateExec: mode=Final, gby=[], aggr=[avg(store_sales.ss_list_price), count(store_sales.ss_list_price), count(DISTINCT store_sales.ss_list_price)] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkCoalesceExec: output_partitions=6, input_tasks=3 + │ CrossJoinExec + │ ProjectionExec: expr=[avg(store_sales.ss_list_price)@0 as b1_lp, count(store_sales.ss_list_price)@1 as b1_cnt, count(DISTINCT store_sales.ss_list_price)@2 as b1_cntd] + │ AggregateExec: mode=Final, gby=[], aggr=[avg(store_sales.ss_list_price), count(store_sales.ss_list_price), count(DISTINCT store_sales.ss_list_price)] + │ CoalescePartitionsExec + │ [Stage 5] => NetworkCoalesceExec: output_partitions=6, input_tasks=3 + │ ProjectionExec: expr=[avg(store_sales.ss_list_price)@0 as b2_lp, count(store_sales.ss_list_price)@1 as b2_cnt, count(DISTINCT store_sales.ss_list_price)@2 as b2_cntd] + │ AggregateExec: mode=Final, gby=[], aggr=[avg(store_sales.ss_list_price), count(store_sales.ss_list_price), count(DISTINCT store_sales.ss_list_price)] + │ CoalescePartitionsExec + │ [Stage 6] => NetworkCoalesceExec: output_partitions=6, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] + │ AggregateExec: mode=Partial, gby=[], aggr=[avg(store_sales.ss_list_price), count(store_sales.ss_list_price), count(DISTINCT store_sales.ss_list_price)] + │ FilterExec: ss_quantity@0 >= 26 AND ss_quantity@0 <= 30 AND (ss_list_price@2 >= Some(15400),5,2 AND ss_list_price@2 <= Some(16400),5,2 OR ss_coupon_amt@3 >= Some(732600),7,2 AND ss_coupon_amt@3 <= Some(832600),7,2 OR ss_wholesale_cost@1 >= Some(700),5,2 AND ss_wholesale_cost@1 <= Some(2700),5,2), projection=[ss_list_price@2] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_quantity, ss_wholesale_cost, ss_list_price, ss_coupon_amt], file_type=parquet, predicate=ss_quantity@10 >= 26 AND ss_quantity@10 <= 30 AND (ss_list_price@12 >= Some(15400),5,2 AND ss_list_price@12 <= Some(16400),5,2 OR ss_coupon_amt@19 >= Some(732600),7,2 AND ss_coupon_amt@19 <= Some(832600),7,2 OR ss_wholesale_cost@11 >= Some(700),5,2 AND ss_wholesale_cost@11 <= Some(2700),5,2), pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 26 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 30 AND (ss_list_price_null_count@5 != row_count@2 AND ss_list_price_max@4 >= Some(15400),5,2 AND ss_list_price_null_count@5 != row_count@2 AND ss_list_price_min@6 <= Some(16400),5,2 OR ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_max@7 >= Some(732600),7,2 AND ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_min@9 <= Some(832600),7,2 OR ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_max@10 >= Some(700),5,2 AND ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_min@12 <= Some(2700),5,2), required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] + │ AggregateExec: mode=Partial, gby=[], aggr=[avg(store_sales.ss_list_price), count(store_sales.ss_list_price), count(DISTINCT store_sales.ss_list_price)] + │ FilterExec: ss_quantity@0 >= 21 AND ss_quantity@0 <= 25 AND (ss_list_price@2 >= Some(12200),5,2 AND ss_list_price@2 <= Some(13200),5,2 OR ss_coupon_amt@3 >= Some(83600),7,2 AND ss_coupon_amt@3 <= Some(183600),7,2 OR ss_wholesale_cost@1 >= Some(1700),5,2 AND ss_wholesale_cost@1 <= Some(3700),5,2), projection=[ss_list_price@2] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_quantity, ss_wholesale_cost, ss_list_price, ss_coupon_amt], file_type=parquet, predicate=ss_quantity@10 >= 21 AND ss_quantity@10 <= 25 AND (ss_list_price@12 >= Some(12200),5,2 AND ss_list_price@12 <= Some(13200),5,2 OR ss_coupon_amt@19 >= Some(83600),7,2 AND ss_coupon_amt@19 <= Some(183600),7,2 OR ss_wholesale_cost@11 >= Some(1700),5,2 AND ss_wholesale_cost@11 <= Some(3700),5,2), pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 21 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 25 AND (ss_list_price_null_count@5 != row_count@2 AND ss_list_price_max@4 >= Some(12200),5,2 AND ss_list_price_null_count@5 != row_count@2 AND ss_list_price_min@6 <= Some(13200),5,2 OR ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_max@7 >= Some(83600),7,2 AND ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_min@9 <= Some(183600),7,2 OR ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_max@10 >= Some(1700),5,2 AND ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_min@12 <= Some(3700),5,2), required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] + │ AggregateExec: mode=Partial, gby=[], aggr=[avg(store_sales.ss_list_price), count(store_sales.ss_list_price), count(DISTINCT store_sales.ss_list_price)] + │ FilterExec: ss_quantity@0 >= 16 AND ss_quantity@0 <= 20 AND (ss_list_price@2 >= Some(13500),5,2 AND ss_list_price@2 <= Some(14500),5,2 OR ss_coupon_amt@3 >= Some(607100),7,2 AND ss_coupon_amt@3 <= Some(707100),7,2 OR ss_wholesale_cost@1 >= Some(3800),5,2 AND ss_wholesale_cost@1 <= Some(5800),5,2), projection=[ss_list_price@2] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_quantity, ss_wholesale_cost, ss_list_price, ss_coupon_amt], file_type=parquet, predicate=ss_quantity@10 >= 16 AND ss_quantity@10 <= 20 AND (ss_list_price@12 >= Some(13500),5,2 AND ss_list_price@12 <= Some(14500),5,2 OR ss_coupon_amt@19 >= Some(607100),7,2 AND ss_coupon_amt@19 <= Some(707100),7,2 OR ss_wholesale_cost@11 >= Some(3800),5,2 AND ss_wholesale_cost@11 <= Some(5800),5,2), pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 16 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 20 AND (ss_list_price_null_count@5 != row_count@2 AND ss_list_price_max@4 >= Some(13500),5,2 AND ss_list_price_null_count@5 != row_count@2 AND ss_list_price_min@6 <= Some(14500),5,2 OR ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_max@7 >= Some(607100),7,2 AND ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_min@9 <= Some(707100),7,2 OR ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_max@10 >= Some(3800),5,2 AND ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_min@12 <= Some(5800),5,2), required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] + │ AggregateExec: mode=Partial, gby=[], aggr=[avg(store_sales.ss_list_price), count(store_sales.ss_list_price), count(DISTINCT store_sales.ss_list_price)] + │ FilterExec: ss_quantity@0 >= 11 AND ss_quantity@0 <= 15 AND (ss_list_price@2 >= Some(14200),5,2 AND ss_list_price@2 <= Some(15200),5,2 OR ss_coupon_amt@3 >= Some(1221400),7,2 AND ss_coupon_amt@3 <= Some(1321400),7,2 OR ss_wholesale_cost@1 >= Some(7900),5,2 AND ss_wholesale_cost@1 <= Some(9900),5,2), projection=[ss_list_price@2] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_quantity, ss_wholesale_cost, ss_list_price, ss_coupon_amt], file_type=parquet, predicate=ss_quantity@10 >= 11 AND ss_quantity@10 <= 15 AND (ss_list_price@12 >= Some(14200),5,2 AND ss_list_price@12 <= Some(15200),5,2 OR ss_coupon_amt@19 >= Some(1221400),7,2 AND ss_coupon_amt@19 <= Some(1321400),7,2 OR ss_wholesale_cost@11 >= Some(7900),5,2 AND ss_wholesale_cost@11 <= Some(9900),5,2), pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 11 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 15 AND (ss_list_price_null_count@5 != row_count@2 AND ss_list_price_max@4 >= Some(14200),5,2 AND ss_list_price_null_count@5 != row_count@2 AND ss_list_price_min@6 <= Some(15200),5,2 OR ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_max@7 >= Some(1221400),7,2 AND ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_min@9 <= Some(1321400),7,2 OR ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_max@10 >= Some(7900),5,2 AND ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_min@12 <= Some(9900),5,2), required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] + │ AggregateExec: mode=Partial, gby=[], aggr=[avg(store_sales.ss_list_price), count(store_sales.ss_list_price), count(DISTINCT store_sales.ss_list_price)] + │ FilterExec: ss_quantity@0 >= 0 AND ss_quantity@0 <= 5 AND (ss_list_price@2 >= Some(800),5,2 AND ss_list_price@2 <= Some(1800),5,2 OR ss_coupon_amt@3 >= Some(45900),7,2 AND ss_coupon_amt@3 <= Some(145900),7,2 OR ss_wholesale_cost@1 >= Some(5700),5,2 AND ss_wholesale_cost@1 <= Some(7700),5,2), projection=[ss_list_price@2] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_quantity, ss_wholesale_cost, ss_list_price, ss_coupon_amt], file_type=parquet, predicate=ss_quantity@10 >= 0 AND ss_quantity@10 <= 5 AND (ss_list_price@12 >= Some(800),5,2 AND ss_list_price@12 <= Some(1800),5,2 OR ss_coupon_amt@19 >= Some(45900),7,2 AND ss_coupon_amt@19 <= Some(145900),7,2 OR ss_wholesale_cost@11 >= Some(5700),5,2 AND ss_wholesale_cost@11 <= Some(7700),5,2), pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 0 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 5 AND (ss_list_price_null_count@5 != row_count@2 AND ss_list_price_max@4 >= Some(800),5,2 AND ss_list_price_null_count@5 != row_count@2 AND ss_list_price_min@6 <= Some(1800),5,2 OR ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_max@7 >= Some(45900),7,2 AND ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_min@9 <= Some(145900),7,2 OR ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_max@10 >= Some(5700),5,2 AND ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_min@12 <= Some(7700),5,2), required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] + │ AggregateExec: mode=Partial, gby=[], aggr=[avg(store_sales.ss_list_price), count(store_sales.ss_list_price), count(DISTINCT store_sales.ss_list_price)] + │ FilterExec: ss_quantity@0 >= 6 AND ss_quantity@0 <= 10 AND (ss_list_price@2 >= Some(9000),5,2 AND ss_list_price@2 <= Some(10000),5,2 OR ss_coupon_amt@3 >= Some(232300),7,2 AND ss_coupon_amt@3 <= Some(332300),7,2 OR ss_wholesale_cost@1 >= Some(3100),5,2 AND ss_wholesale_cost@1 <= Some(5100),5,2), projection=[ss_list_price@2] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_quantity, ss_wholesale_cost, ss_list_price, ss_coupon_amt], file_type=parquet, predicate=ss_quantity@10 >= 6 AND ss_quantity@10 <= 10 AND (ss_list_price@12 >= Some(9000),5,2 AND ss_list_price@12 <= Some(10000),5,2 OR ss_coupon_amt@19 >= Some(232300),7,2 AND ss_coupon_amt@19 <= Some(332300),7,2 OR ss_wholesale_cost@11 >= Some(3100),5,2 AND ss_wholesale_cost@11 <= Some(5100),5,2), pruning_predicate=ss_quantity_null_count@1 != row_count@2 AND ss_quantity_max@0 >= 6 AND ss_quantity_null_count@1 != row_count@2 AND ss_quantity_min@3 <= 10 AND (ss_list_price_null_count@5 != row_count@2 AND ss_list_price_max@4 >= Some(9000),5,2 AND ss_list_price_null_count@5 != row_count@2 AND ss_list_price_min@6 <= Some(10000),5,2 OR ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_max@7 >= Some(232300),7,2 AND ss_coupon_amt_null_count@8 != row_count@2 AND ss_coupon_amt_min@9 <= Some(332300),7,2 OR ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_max@10 >= Some(3100),5,2 AND ss_wholesale_cost_null_count@11 != row_count@2 AND ss_wholesale_cost_min@12 <= Some(5100),5,2), required_guarantees=[] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_29() -> Result<()> { + let display = test_tpcds_query("q29").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [i_item_id@0 ASC NULLS LAST, i_item_desc@1 ASC NULLS LAST, s_store_id@2 ASC NULLS LAST, s_store_name@3 ASC NULLS LAST], fetch=100 + │ [Stage 11] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 11 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ SortExec: TopK(fetch=100), expr=[i_item_id@0 ASC NULLS LAST, i_item_desc@1 ASC NULLS LAST, s_store_id@2 ASC NULLS LAST, s_store_name@3 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[i_item_id@0 as i_item_id, i_item_desc@1 as i_item_desc, s_store_id@2 as s_store_id, s_store_name@3 as s_store_name, sum(store_sales.ss_quantity)@4 as store_sales_quantity, sum(store_returns.sr_return_quantity)@5 as store_returns_quantity, sum(catalog_sales.cs_quantity)@6 as catalog_sales_quantity] + │ AggregateExec: mode=FinalPartitioned, gby=[i_item_id@0 as i_item_id, i_item_desc@1 as i_item_desc, s_store_id@2 as s_store_id, s_store_name@3 as s_store_name], aggr=[sum(store_sales.ss_quantity), sum(store_returns.sr_return_quantity), sum(catalog_sales.cs_quantity)] + │ [Stage 10] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 10 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([i_item_id@0, i_item_desc@1, s_store_id@2, s_store_name@3], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_item_id@5 as i_item_id, i_item_desc@6 as i_item_desc, s_store_id@3 as s_store_id, s_store_name@4 as s_store_name], aggr=[sum(store_sales.ss_quantity), sum(store_returns.sr_return_quantity), sum(catalog_sales.cs_quantity)] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_item_sk@0, i_item_sk@0)], projection=[ss_quantity@1, sr_return_quantity@2, cs_quantity@3, s_store_id@4, s_store_name@5, i_item_id@7, i_item_desc@8] + │ CoalescePartitionsExec + │ [Stage 9] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=3 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_item_id, i_item_desc], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 9 ── Tasks: t0:[p0..p8] t1:[p9..p17] t2:[p18..p26] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[ss_item_sk@2 as ss_item_sk, ss_quantity@3 as ss_quantity, sr_return_quantity@4 as sr_return_quantity, cs_quantity@5 as cs_quantity, s_store_id@0 as s_store_id, s_store_name@1 as s_store_name] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@3, ss_store_sk@1)], projection=[s_store_id@1, s_store_name@2, ss_item_sk@4, ss_quantity@6, sr_return_quantity@7, cs_quantity@8] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(cs_sold_date_sk@4, CAST(d3.d_date_sk AS Float64)@1)], projection=[ss_item_sk@0, ss_store_sk@1, ss_quantity@2, sr_return_quantity@3, cs_quantity@5] + │ CoalescePartitionsExec + │ [Stage 8] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=3 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(d3.d_date_sk AS Float64)] + │ FilterExec: d_year@1 = 1999 OR d_year@1 = 2000 OR d_year@1 = 2001, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 1999 OR d_year@6 = 2000 OR d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (1999, 2000, 2001)] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_store_id, s_store_name, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 8 ── Tasks: t0:[p0..p8] t1:[p9..p17] t2:[p18..p26] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(d2.d_date_sk AS Float64)@1, sr_returned_date_sk@3)], projection=[ss_item_sk@2, ss_store_sk@3, ss_quantity@4, sr_return_quantity@6, cs_sold_date_sk@7, cs_quantity@8] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(d1.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_item_sk@3, ss_store_sk@4, ss_quantity@5, sr_returned_date_sk@6, sr_return_quantity@7, cs_sold_date_sk@8, cs_quantity@9] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ ProjectionExec: expr=[ss_sold_date_sk@2 as ss_sold_date_sk, ss_item_sk@3 as ss_item_sk, ss_store_sk@4 as ss_store_sk, ss_quantity@5 as ss_quantity, sr_returned_date_sk@6 as sr_returned_date_sk, sr_return_quantity@7 as sr_return_quantity, cs_sold_date_sk@0 as cs_sold_date_sk, cs_quantity@1 as cs_quantity] + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(cs_bill_customer_sk@1, sr_customer_sk@6), (cs_item_sk@2, sr_item_sk@5)], projection=[cs_sold_date_sk@0, cs_quantity@3, ss_sold_date_sk@4, ss_item_sk@5, ss_store_sk@6, ss_quantity@7, sr_returned_date_sk@8, sr_return_quantity@11] + │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(d2.d_date_sk AS Float64)] + │ FilterExec: d_moy@2 >= 9 AND d_moy@2 <= 12 AND d_year@1 = 1999, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 >= 9 AND d_moy@8 <= 12 AND d_year@6 = 1999, pruning_predicate=d_moy_null_count@1 != row_count@2 AND d_moy_max@0 >= 9 AND d_moy_null_count@1 != row_count@2 AND d_moy_min@3 <= 12 AND d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5, required_guarantees=[d_year in (1999)] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(d1.d_date_sk AS Float64)] + │ FilterExec: d_moy@2 = 9 AND d_year@1 = 1999, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 9 AND d_year@6 = 1999, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 9 AND 9 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5, required_guarantees=[d_moy in (9), d_year in (1999)] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] + │ RepartitionExec: partitioning=Hash([cs_bill_customer_sk@1, cs_item_sk@2], 9), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_item_sk, cs_quantity], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] + │ RepartitionExec: partitioning=Hash([sr_customer_sk@6, sr_item_sk@5], 9), input_partitions=3 + │ ProjectionExec: expr=[ss_sold_date_sk@4 as ss_sold_date_sk, ss_item_sk@5 as ss_item_sk, ss_store_sk@6 as ss_store_sk, ss_quantity@7 as ss_quantity, sr_returned_date_sk@0 as sr_returned_date_sk, sr_item_sk@1 as sr_item_sk, sr_customer_sk@2 as sr_customer_sk, sr_return_quantity@3 as sr_return_quantity] + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(sr_customer_sk@2, ss_customer_sk@2), (sr_item_sk@1, ss_item_sk@1), (sr_ticket_number@3, ss_ticket_number@4)], projection=[sr_returned_date_sk@0, sr_item_sk@1, sr_customer_sk@2, sr_return_quantity@4, ss_sold_date_sk@5, ss_item_sk@6, ss_store_sk@8, ss_quantity@10] + │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] + │ RepartitionExec: partitioning=Hash([sr_customer_sk@2, sr_item_sk@1, sr_ticket_number@3], 9), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], ...]}, projection=[sr_returned_date_sk, sr_item_sk, sr_customer_sk, sr_ticket_number, sr_return_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] + │ RepartitionExec: partitioning=Hash([ss_customer_sk@2, ss_item_sk@1, ss_ticket_number@4], 9), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_store_sk, ss_ticket_number, ss_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + #[ignore = "Fails with column 'c_last_review_date_sk' not found"] + async fn test_tpcds_30() -> Result<()> { + let display = test_tpcds_query("q30").await?; + assert_snapshot!(display, @r""); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_31() -> Result<()> { + let display = test_tpcds_query("q31").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [ca_county@0 ASC NULLS LAST] + │ [Stage 24] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 24 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ SortExec: expr=[ca_county@0 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[ca_county@2 as ca_county, d_year@3 as d_year, __common_expr_1@0 / CAST(web_sales@6 AS Float64) as web_q1_q2_increase, __common_expr_2@1 / CAST(store_sales@4 AS Float64) as store_q1_q2_increase, CAST(web_sales@7 AS Float64) / __common_expr_1@0 as web_q2_q3_increase, CAST(store_sales@5 AS Float64) / __common_expr_2@1 as store_q2_q3_increase] + │ ProjectionExec: expr=[CAST(web_sales@6 AS Float64) as __common_expr_1, CAST(store_sales@3 AS Float64) as __common_expr_2, ca_county@0 as ca_county, d_year@1 as d_year, store_sales@2 as store_sales, store_sales@4 as store_sales, web_sales@5 as web_sales, web_sales@7 as web_sales] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ca_county@5, ca_county@0)], filter=CASE WHEN web_sales@2 > Some(0),17,2 THEN CAST(web_sales@3 AS Float64) / CAST(web_sales@2 AS Float64) END > CASE WHEN store_sales@0 > Some(0),17,2 THEN CAST(store_sales@1 AS Float64) / CAST(store_sales@0 AS Float64) END, projection=[ca_county@0, d_year@1, store_sales@2, store_sales@3, store_sales@4, web_sales@6, web_sales@7, web_sales@9] + │ CoalescePartitionsExec + │ [Stage 20] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[ca_county@0 as ca_county, sum(web_sales.ws_ext_sales_price)@3 as web_sales] + │ AggregateExec: mode=FinalPartitioned, gby=[ca_county@0 as ca_county, d_qoy@1 as d_qoy, d_year@2 as d_year], aggr=[sum(web_sales.ws_ext_sales_price)], ordering_mode=PartiallySorted([1, 2]) + │ [Stage 23] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 20 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=3, consumer_tasks=2, output_partitions=6 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ca_county@5, ca_county@0)], filter=CASE WHEN web_sales@2 > Some(0),17,2 THEN CAST(web_sales@3 AS Float64) / CAST(web_sales@2 AS Float64) END > CASE WHEN store_sales@0 > Some(0),17,2 THEN CAST(store_sales@1 AS Float64) / CAST(store_sales@0 AS Float64) END, projection=[ca_county@0, d_year@1, store_sales@2, store_sales@3, store_sales@4, ca_county@5, web_sales@6, web_sales@8] + │ CoalescePartitionsExec + │ [Stage 16] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[ca_county@0 as ca_county, sum(web_sales.ws_ext_sales_price)@3 as web_sales] + │ AggregateExec: mode=FinalPartitioned, gby=[ca_county@0 as ca_county, d_qoy@1 as d_qoy, d_year@2 as d_year], aggr=[sum(web_sales.ws_ext_sales_price)], ordering_mode=PartiallySorted([1, 2]) + │ [Stage 19] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 16 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=3, consumer_tasks=2, output_partitions=6 + │ ProjectionExec: expr=[ca_county@2 as ca_county, d_year@3 as d_year, store_sales@4 as store_sales, store_sales@5 as store_sales, store_sales@6 as store_sales, ca_county@0 as ca_county, web_sales@1 as web_sales] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ca_county@0, ca_county@0)] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=6, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ca_county@3, ca_county@0)], projection=[ca_county@0, d_year@1, store_sales@2, store_sales@4, store_sales@6] + │ CoalescePartitionsExec + │ [Stage 12] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[ca_county@0 as ca_county, sum(store_sales.ss_ext_sales_price)@3 as store_sales] + │ AggregateExec: mode=FinalPartitioned, gby=[ca_county@0 as ca_county, d_qoy@1 as d_qoy, d_year@2 as d_year], aggr=[sum(store_sales.ss_ext_sales_price)], ordering_mode=PartiallySorted([1, 2]) + │ [Stage 15] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=3, consumer_tasks=2, output_partitions=6 + │ ProjectionExec: expr=[ca_county@0 as ca_county, sum(web_sales.ws_ext_sales_price)@3 as web_sales] + │ AggregateExec: mode=FinalPartitioned, gby=[ca_county@0 as ca_county, d_qoy@1 as d_qoy, d_year@2 as d_year], aggr=[sum(web_sales.ws_ext_sales_price)], ordering_mode=PartiallySorted([1, 2]) + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([ca_county@0, d_qoy@1, d_year@2], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[ca_county@3 as ca_county, d_qoy@2 as d_qoy, d_year@1 as d_year], aggr=[sum(web_sales.ws_ext_sales_price)], ordering_mode=PartiallySorted([1, 2]) + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ws_bill_addr_sk@0, CAST(customer_address.ca_address_sk AS Float64)@2)], projection=[ws_ext_sales_price@1, d_year@2, d_qoy@3, ca_county@5] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=3 + │ ProjectionExec: expr=[ca_address_sk@0 as ca_address_sk, ca_county@1 as ca_county, CAST(ca_address_sk@0 AS Float64) as CAST(customer_address.ca_address_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_county], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p5] t1:[p6..p11] t2:[p12..p17] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ ProjectionExec: expr=[ws_bill_addr_sk@2 as ws_bill_addr_sk, ws_ext_sales_price@3 as ws_ext_sales_price, d_year@0 as d_year, d_qoy@1 as d_qoy] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@3, ws_sold_date_sk@0)], projection=[d_year@1, d_qoy@2, ws_bill_addr_sk@5, ws_ext_sales_price@6] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_bill_addr_sk, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, d_qoy@2 as d_qoy, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_qoy@2 = 1 AND d_year@1 = 2000 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@10 = 1 AND d_year@6 = 2000, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 1 AND 1 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5, required_guarantees=[d_qoy in (1), d_year in (2000)] + └────────────────────────────────────────────────── + ┌───── Stage 12 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=3, consumer_tasks=2, output_partitions=6 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ca_county@0, ca_county@0)] + │ CoalescePartitionsExec + │ [Stage 8] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[ca_county@0 as ca_county, sum(store_sales.ss_ext_sales_price)@3 as store_sales] + │ AggregateExec: mode=FinalPartitioned, gby=[ca_county@0 as ca_county, d_qoy@1 as d_qoy, d_year@2 as d_year], aggr=[sum(store_sales.ss_ext_sales_price)], ordering_mode=PartiallySorted([1, 2]) + │ [Stage 11] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 8 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=3, consumer_tasks=2, output_partitions=6 + │ ProjectionExec: expr=[ca_county@0 as ca_county, d_year@2 as d_year, sum(store_sales.ss_ext_sales_price)@3 as store_sales] + │ AggregateExec: mode=FinalPartitioned, gby=[ca_county@0 as ca_county, d_qoy@1 as d_qoy, d_year@2 as d_year], aggr=[sum(store_sales.ss_ext_sales_price)], ordering_mode=PartiallySorted([1, 2]) + │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([ca_county@0, d_qoy@1, d_year@2], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[ca_county@3 as ca_county, d_qoy@2 as d_qoy, d_year@1 as d_year], aggr=[sum(store_sales.ss_ext_sales_price)], ordering_mode=PartiallySorted([1, 2]) + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_addr_sk@0, CAST(customer_address.ca_address_sk AS Float64)@2)], projection=[ss_ext_sales_price@1, d_year@2, d_qoy@3, ca_county@5] + │ CoalescePartitionsExec + │ [Stage 6] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=3 + │ ProjectionExec: expr=[ca_address_sk@0 as ca_address_sk, ca_county@1 as ca_county, CAST(ca_address_sk@0 AS Float64) as CAST(customer_address.ca_address_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_county], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p5] t1:[p6..p11] t2:[p12..p17] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ ProjectionExec: expr=[ss_addr_sk@2 as ss_addr_sk, ss_ext_sales_price@3 as ss_ext_sales_price, d_year@0 as d_year, d_qoy@1 as d_qoy] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@3, ss_sold_date_sk@0)], projection=[d_year@1, d_qoy@2, ss_addr_sk@5, ss_ext_sales_price@6] + │ CoalescePartitionsExec + │ [Stage 5] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_addr_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, d_qoy@2 as d_qoy, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_qoy@2 = 1 AND d_year@1 = 2000 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@10 = 1 AND d_year@6 = 2000, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 1 AND 1 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5, required_guarantees=[d_qoy in (1), d_year in (2000)] + └────────────────────────────────────────────────── + ┌───── Stage 11 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([ca_county@0, d_qoy@1, d_year@2], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[ca_county@3 as ca_county, d_qoy@2 as d_qoy, d_year@1 as d_year], aggr=[sum(store_sales.ss_ext_sales_price)], ordering_mode=PartiallySorted([1, 2]) + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_addr_sk@0, CAST(customer_address.ca_address_sk AS Float64)@2)], projection=[ss_ext_sales_price@1, d_year@2, d_qoy@3, ca_county@5] + │ CoalescePartitionsExec + │ [Stage 10] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=3 + │ ProjectionExec: expr=[ca_address_sk@0 as ca_address_sk, ca_county@1 as ca_county, CAST(ca_address_sk@0 AS Float64) as CAST(customer_address.ca_address_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_county], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 10 ── Tasks: t0:[p0..p5] t1:[p6..p11] t2:[p12..p17] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ ProjectionExec: expr=[ss_addr_sk@2 as ss_addr_sk, ss_ext_sales_price@3 as ss_ext_sales_price, d_year@0 as d_year, d_qoy@1 as d_qoy] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@3, ss_sold_date_sk@0)], projection=[d_year@1, d_qoy@2, ss_addr_sk@5, ss_ext_sales_price@6] + │ CoalescePartitionsExec + │ [Stage 9] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_addr_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 9 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, d_qoy@2 as d_qoy, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_qoy@2 = 2 AND d_year@1 = 2000 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@10 = 2 AND d_year@6 = 2000, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 2 AND 2 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5, required_guarantees=[d_qoy in (2), d_year in (2000)] + └────────────────────────────────────────────────── + ┌───── Stage 15 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([ca_county@0, d_qoy@1, d_year@2], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[ca_county@3 as ca_county, d_qoy@2 as d_qoy, d_year@1 as d_year], aggr=[sum(store_sales.ss_ext_sales_price)], ordering_mode=PartiallySorted([1, 2]) + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_addr_sk@0, CAST(customer_address.ca_address_sk AS Float64)@2)], projection=[ss_ext_sales_price@1, d_year@2, d_qoy@3, ca_county@5] + │ CoalescePartitionsExec + │ [Stage 14] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=3 + │ ProjectionExec: expr=[ca_address_sk@0 as ca_address_sk, ca_county@1 as ca_county, CAST(ca_address_sk@0 AS Float64) as CAST(customer_address.ca_address_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_county], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 14 ── Tasks: t0:[p0..p5] t1:[p6..p11] t2:[p12..p17] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ ProjectionExec: expr=[ss_addr_sk@2 as ss_addr_sk, ss_ext_sales_price@3 as ss_ext_sales_price, d_year@0 as d_year, d_qoy@1 as d_qoy] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@3, ss_sold_date_sk@0)], projection=[d_year@1, d_qoy@2, ss_addr_sk@5, ss_ext_sales_price@6] + │ CoalescePartitionsExec + │ [Stage 13] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_addr_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 13 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, d_qoy@2 as d_qoy, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_qoy@2 = 3 AND d_year@1 = 2000 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@10 = 3 AND d_year@6 = 2000, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 3 AND 3 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5, required_guarantees=[d_qoy in (3), d_year in (2000)] + └────────────────────────────────────────────────── + ┌───── Stage 19 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([ca_county@0, d_qoy@1, d_year@2], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[ca_county@3 as ca_county, d_qoy@2 as d_qoy, d_year@1 as d_year], aggr=[sum(web_sales.ws_ext_sales_price)], ordering_mode=PartiallySorted([1, 2]) + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ws_bill_addr_sk@0, CAST(customer_address.ca_address_sk AS Float64)@2)], projection=[ws_ext_sales_price@1, d_year@2, d_qoy@3, ca_county@5] + │ CoalescePartitionsExec + │ [Stage 18] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=3 + │ ProjectionExec: expr=[ca_address_sk@0 as ca_address_sk, ca_county@1 as ca_county, CAST(ca_address_sk@0 AS Float64) as CAST(customer_address.ca_address_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_county], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 18 ── Tasks: t0:[p0..p5] t1:[p6..p11] t2:[p12..p17] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ ProjectionExec: expr=[ws_bill_addr_sk@2 as ws_bill_addr_sk, ws_ext_sales_price@3 as ws_ext_sales_price, d_year@0 as d_year, d_qoy@1 as d_qoy] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@3, ws_sold_date_sk@0)], projection=[d_year@1, d_qoy@2, ws_bill_addr_sk@5, ws_ext_sales_price@6] + │ CoalescePartitionsExec + │ [Stage 17] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_bill_addr_sk, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 17 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, d_qoy@2 as d_qoy, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_qoy@2 = 2 AND d_year@1 = 2000 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@10 = 2 AND d_year@6 = 2000, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 2 AND 2 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5, required_guarantees=[d_qoy in (2), d_year in (2000)] + └────────────────────────────────────────────────── + ┌───── Stage 23 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([ca_county@0, d_qoy@1, d_year@2], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[ca_county@3 as ca_county, d_qoy@2 as d_qoy, d_year@1 as d_year], aggr=[sum(web_sales.ws_ext_sales_price)], ordering_mode=PartiallySorted([1, 2]) + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ws_bill_addr_sk@0, CAST(customer_address.ca_address_sk AS Float64)@2)], projection=[ws_ext_sales_price@1, d_year@2, d_qoy@3, ca_county@5] + │ CoalescePartitionsExec + │ [Stage 22] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=3 + │ ProjectionExec: expr=[ca_address_sk@0 as ca_address_sk, ca_county@1 as ca_county, CAST(ca_address_sk@0 AS Float64) as CAST(customer_address.ca_address_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_county], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 22 ── Tasks: t0:[p0..p5] t1:[p6..p11] t2:[p12..p17] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ ProjectionExec: expr=[ws_bill_addr_sk@2 as ws_bill_addr_sk, ws_ext_sales_price@3 as ws_ext_sales_price, d_year@0 as d_year, d_qoy@1 as d_qoy] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@3, ws_sold_date_sk@0)], projection=[d_year@1, d_qoy@2, ws_bill_addr_sk@5, ws_ext_sales_price@6] + │ CoalescePartitionsExec + │ [Stage 21] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_bill_addr_sk, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 21 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, d_qoy@2 as d_qoy, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_qoy@2 = 3 AND d_year@1 = 2000 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@10 = 3 AND d_year@6 = 2000, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 3 AND 3 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5, required_guarantees=[d_qoy in (3), d_year in (2000)] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_32() -> Result<()> { + let display = test_tpcds_query("q32").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ ProjectionExec: expr=[sum(catalog_sales.cs_ext_discount_amt)@0 as excess discount amount] + │ GlobalLimitExec: skip=0, fetch=100 + │ AggregateExec: mode=Final, gby=[], aggr=[sum(catalog_sales.cs_ext_discount_amt)] + │ CoalescePartitionsExec + │ [Stage 6] => NetworkCoalesceExec: output_partitions=6, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] + │ AggregateExec: mode=Partial, gby=[], aggr=[sum(catalog_sales.cs_ext_discount_amt)] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(cs_item_sk@1, i_item_sk@1)], filter=CAST(cs_ext_discount_amt@0 AS Decimal128(30, 15)) > Float64(1.3) * avg(catalog_sales.cs_ext_discount_amt)@1, projection=[cs_ext_discount_amt@2] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, cs_sold_date_sk@0)], projection=[cs_ext_discount_amt@3, i_item_sk@4] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ ProjectionExec: expr=[cs_sold_date_sk@1 as cs_sold_date_sk, cs_ext_discount_amt@2 as cs_ext_discount_amt, i_item_sk@0 as i_item_sk] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, cs_item_sk@1)], projection=[i_item_sk@0, cs_sold_date_sk@1, cs_ext_discount_amt@3] + │ CoalescePartitionsExec + │ [Stage 5] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_item_sk, cs_ext_discount_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[CAST(1.3 * CAST(avg(catalog_sales.cs_ext_discount_amt)@1 AS Float64) AS Decimal128(30, 15)) as Float64(1.3) * avg(catalog_sales.cs_ext_discount_amt), cs_item_sk@0 as cs_item_sk] + │ AggregateExec: mode=FinalPartitioned, gby=[cs_item_sk@0 as cs_item_sk], aggr=[avg(catalog_sales.cs_ext_discount_amt)] + │ [Stage 2] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([cs_item_sk@0], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[cs_item_sk@0 as cs_item_sk], aggr=[avg(catalog_sales.cs_ext_discount_amt)] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, cs_sold_date_sk@0)], projection=[cs_item_sk@3, cs_ext_discount_amt@4] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_item_sk, cs_ext_discount_amt], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_date@1 >= 2000-01-27 AND d_date@1 <= 2000-04-26, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-01-27 AND d_date@2 <= 2000-04-26, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-01-27 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-04-26, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_date@1 >= 2000-01-27 AND d_date@1 <= 2000-04-26, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-01-27 AND d_date@2 <= 2000-04-26, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-01-27 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-04-26, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ FilterExec: i_manufact_id@1 = 977, projection=[i_item_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_manufact_id], file_type=parquet, predicate=i_manufact_id@13 = 977 AND DynamicFilter [ empty ], pruning_predicate=i_manufact_id_null_count@2 != row_count@3 AND i_manufact_id_min@0 <= 977 AND 977 <= i_manufact_id_max@1, required_guarantees=[i_manufact_id in (977)] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_33() -> Result<()> { + let display = test_tpcds_query("q33").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [total_sales@1 ASC NULLS LAST], fetch=100 + │ SortExec: TopK(fetch=100), expr=[total_sales@1 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[i_manufact_id@0 as i_manufact_id, sum(tmp1.total_sales)@1 as total_sales] + │ AggregateExec: mode=SinglePartitioned, gby=[i_manufact_id@0 as i_manufact_id], aggr=[sum(tmp1.total_sales)] + │ InterleaveExec + │ ProjectionExec: expr=[i_manufact_id@0 as i_manufact_id, sum(store_sales.ss_ext_sales_price)@1 as total_sales] + │ AggregateExec: mode=FinalPartitioned, gby=[i_manufact_id@0 as i_manufact_id], aggr=[sum(store_sales.ss_ext_sales_price)] + │ RepartitionExec: partitioning=Hash([i_manufact_id@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_manufact_id@1 as i_manufact_id], aggr=[sum(store_sales.ss_ext_sales_price)] + │ HashJoinExec: mode=CollectLeft, join_type=LeftSemi, on=[(i_manufact_id@1, i_manufact_id@0)] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkCoalesceExec: output_partitions=9, input_tasks=3 + │ FilterExec: i_category@0 = Electronics, projection=[i_manufact_id@1] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_category, i_manufact_id], file_type=parquet, predicate=i_category@12 = Electronics, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Electronics AND Electronics <= i_category_max@1, required_guarantees=[i_category in (Electronics)] + │ ProjectionExec: expr=[i_manufact_id@0 as i_manufact_id, sum(catalog_sales.cs_ext_sales_price)@1 as total_sales] + │ AggregateExec: mode=FinalPartitioned, gby=[i_manufact_id@0 as i_manufact_id], aggr=[sum(catalog_sales.cs_ext_sales_price)] + │ RepartitionExec: partitioning=Hash([i_manufact_id@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_manufact_id@1 as i_manufact_id], aggr=[sum(catalog_sales.cs_ext_sales_price)] + │ HashJoinExec: mode=CollectLeft, join_type=LeftSemi, on=[(i_manufact_id@1, i_manufact_id@0)] + │ CoalescePartitionsExec + │ [Stage 8] => NetworkCoalesceExec: output_partitions=9, input_tasks=3 + │ FilterExec: i_category@0 = Electronics, projection=[i_manufact_id@1] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_category, i_manufact_id], file_type=parquet, predicate=i_category@12 = Electronics, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Electronics AND Electronics <= i_category_max@1, required_guarantees=[i_category in (Electronics)] + │ ProjectionExec: expr=[i_manufact_id@0 as i_manufact_id, sum(web_sales.ws_ext_sales_price)@1 as total_sales] + │ AggregateExec: mode=FinalPartitioned, gby=[i_manufact_id@0 as i_manufact_id], aggr=[sum(web_sales.ws_ext_sales_price)] + │ RepartitionExec: partitioning=Hash([i_manufact_id@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_manufact_id@1 as i_manufact_id], aggr=[sum(web_sales.ws_ext_sales_price)] + │ HashJoinExec: mode=CollectLeft, join_type=LeftSemi, on=[(i_manufact_id@1, i_manufact_id@0)] + │ CoalescePartitionsExec + │ [Stage 12] => NetworkCoalesceExec: output_partitions=9, input_tasks=3 + │ FilterExec: i_category@0 = Electronics, projection=[i_manufact_id@1] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_category, i_manufact_id], file_type=parquet, predicate=i_category@12 = Electronics, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Electronics AND Electronics <= i_category_max@1, required_guarantees=[i_category in (Electronics)] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p3..p5] t2:[p6..p8] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_item_sk@0, i_item_sk@0)], projection=[ss_ext_sales_price@1, i_manufact_id@3] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=3 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_manufact_id], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p8] t1:[p9..p17] t2:[p18..p26] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_addr_sk@1, CAST(customer_address.ca_address_sk AS Float64)@1)], projection=[ss_item_sk@0, ss_ext_sales_price@2] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=3 + │ ProjectionExec: expr=[ca_address_sk@0 as ca_address_sk, CAST(ca_address_sk@0 AS Float64) as CAST(customer_address.ca_address_sk AS Float64)] + │ FilterExec: ca_gmt_offset@1 = Some(-500),4,2, projection=[ca_address_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = Some(-500),4,2, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= Some(-500),4,2 AND Some(-500),4,2 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (Some(-500),4,2)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p5] t1:[p6..p11] t2:[p12..p17] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_item_sk@3, ss_addr_sk@4, ss_ext_sales_price@5] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_addr_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_year@1 = 1998 AND d_moy@2 = 5, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 5, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 5 AND 5 <= d_moy_max@5, required_guarantees=[d_moy in (5), d_year in (1998)] + └────────────────────────────────────────────────── + ┌───── Stage 8 ── Tasks: t0:[p0..p2] t1:[p3..p5] t2:[p6..p8] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(cs_item_sk@0, i_item_sk@0)], projection=[cs_ext_sales_price@1, i_manufact_id@3] + │ CoalescePartitionsExec + │ [Stage 7] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=3 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_manufact_id], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p8] t1:[p9..p17] t2:[p18..p26] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(cs_bill_addr_sk@0, CAST(customer_address.ca_address_sk AS Float64)@1)], projection=[cs_item_sk@1, cs_ext_sales_price@2] + │ CoalescePartitionsExec + │ [Stage 6] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=3 + │ ProjectionExec: expr=[ca_address_sk@0 as ca_address_sk, CAST(ca_address_sk@0 AS Float64) as CAST(customer_address.ca_address_sk AS Float64)] + │ FilterExec: ca_gmt_offset@1 = Some(-500),4,2, projection=[ca_address_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = Some(-500),4,2, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= Some(-500),4,2 AND Some(-500),4,2 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (Some(-500),4,2)] + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p5] t1:[p6..p11] t2:[p12..p17] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, cs_sold_date_sk@0)], projection=[cs_bill_addr_sk@3, cs_item_sk@4, cs_ext_sales_price@5] + │ CoalescePartitionsExec + │ [Stage 5] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_bill_addr_sk, cs_item_sk, cs_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_year@1 = 1998 AND d_moy@2 = 5, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 5, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 5 AND 5 <= d_moy_max@5, required_guarantees=[d_moy in (5), d_year in (1998)] + └────────────────────────────────────────────────── + ┌───── Stage 12 ── Tasks: t0:[p0..p2] t1:[p3..p5] t2:[p6..p8] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ws_item_sk@0, i_item_sk@0)], projection=[ws_ext_sales_price@1, i_manufact_id@3] + │ CoalescePartitionsExec + │ [Stage 11] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=3 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_manufact_id], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 11 ── Tasks: t0:[p0..p8] t1:[p9..p17] t2:[p18..p26] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ws_bill_addr_sk@1, CAST(customer_address.ca_address_sk AS Float64)@1)], projection=[ws_item_sk@0, ws_ext_sales_price@2] + │ CoalescePartitionsExec + │ [Stage 10] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=3 + │ ProjectionExec: expr=[ca_address_sk@0 as ca_address_sk, CAST(ca_address_sk@0 AS Float64) as CAST(customer_address.ca_address_sk AS Float64)] + │ FilterExec: ca_gmt_offset@1 = Some(-500),4,2, projection=[ca_address_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = Some(-500),4,2, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= Some(-500),4,2 AND Some(-500),4,2 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (Some(-500),4,2)] + └────────────────────────────────────────────────── + ┌───── Stage 10 ── Tasks: t0:[p0..p5] t1:[p6..p11] t2:[p12..p17] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ws_sold_date_sk@0)], projection=[ws_item_sk@3, ws_bill_addr_sk@4, ws_ext_sales_price@5] + │ CoalescePartitionsExec + │ [Stage 9] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_item_sk, ws_bill_addr_sk, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 9 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_year@1 = 1998 AND d_moy@2 = 5, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 5, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 5 AND 5 <= d_moy_max@5, required_guarantees=[d_moy in (5), d_year in (1998)] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_34() -> Result<()> { + let display = test_tpcds_query("q34").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [c_last_name@0 ASC, c_first_name@1 ASC, c_salutation@2 ASC, c_preferred_cust_flag@3 DESC, ss_ticket_number@4 ASC] + │ [Stage 6] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ SortExec: expr=[c_last_name@0 ASC, c_first_name@1 ASC, c_salutation@2 ASC, c_preferred_cust_flag@3 DESC, ss_ticket_number@4 ASC], preserve_partitioning=[true] + │ ProjectionExec: expr=[c_last_name@4 as c_last_name, c_first_name@3 as c_first_name, c_salutation@2 as c_salutation, c_preferred_cust_flag@5 as c_preferred_cust_flag, ss_ticket_number@0 as ss_ticket_number, cnt@1 as cnt] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_customer_sk@1, CAST(customer.c_customer_sk AS Float64)@5)], projection=[ss_ticket_number@0, cnt@2, c_salutation@4, c_first_name@5, c_last_name@6, c_preferred_cust_flag@7] + │ CoalescePartitionsExec + │ [Stage 5] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, c_salutation@1 as c_salutation, c_first_name@2 as c_first_name, c_last_name@3 as c_last_name, c_preferred_cust_flag@4 as c_preferred_cust_flag, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_salutation, c_first_name, c_last_name, c_preferred_cust_flag], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=3, consumer_tasks=2, output_partitions=6 + │ ProjectionExec: expr=[ss_ticket_number@0 as ss_ticket_number, ss_customer_sk@1 as ss_customer_sk, count(Int64(1))@2 as cnt] + │ FilterExec: count(Int64(1))@2 >= 15 AND count(Int64(1))@2 <= 20 + │ AggregateExec: mode=FinalPartitioned, gby=[ss_ticket_number@0 as ss_ticket_number, ss_customer_sk@1 as ss_customer_sk], aggr=[count(Int64(1))] + │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([ss_ticket_number@0, ss_customer_sk@1], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[ss_ticket_number@1 as ss_ticket_number, ss_customer_sk@0 as ss_customer_sk], aggr=[count(Int64(1))] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(household_demographics.hd_demo_sk AS Float64)@1, ss_hdemo_sk@1)], projection=[ss_customer_sk@2, ss_ticket_number@4] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@1, ss_store_sk@2)], projection=[ss_customer_sk@2, ss_hdemo_sk@3, ss_ticket_number@5] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_customer_sk@3, ss_hdemo_sk@4, ss_store_sk@5, ss_ticket_number@6] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_hdemo_sk, ss_store_sk, ss_ticket_number], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[hd_demo_sk@0 as hd_demo_sk, CAST(hd_demo_sk@0 AS Float64) as CAST(household_demographics.hd_demo_sk AS Float64)] + │ FilterExec: (hd_buy_potential@1 = >10000 OR hd_buy_potential@1 = Unknown) AND hd_vehicle_count@3 > 0 AND CASE WHEN hd_vehicle_count@3 > 0 THEN CAST(hd_dep_count@2 AS Float64) / CAST(hd_vehicle_count@3 AS Float64) END > 1.2, projection=[hd_demo_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-3.parquet]]}, projection=[hd_demo_sk, hd_buy_potential, hd_dep_count, hd_vehicle_count], file_type=parquet, predicate=(hd_buy_potential@2 = >10000 OR hd_buy_potential@2 = Unknown) AND hd_vehicle_count@4 > 0 AND CASE WHEN hd_vehicle_count@4 > 0 THEN CAST(hd_dep_count@3 AS Float64) / CAST(hd_vehicle_count@4 AS Float64) END > 1.2, pruning_predicate=(hd_buy_potential_null_count@2 != row_count@3 AND hd_buy_potential_min@0 <= >10000 AND >10000 <= hd_buy_potential_max@1 OR hd_buy_potential_null_count@2 != row_count@3 AND hd_buy_potential_min@0 <= Unknown AND Unknown <= hd_buy_potential_max@1) AND hd_vehicle_count_null_count@5 != row_count@3 AND hd_vehicle_count_max@4 > 0, required_guarantees=[hd_buy_potential in (>10000, Unknown)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] + │ FilterExec: s_county@1 = Williamson County, projection=[s_store_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_county], file_type=parquet, predicate=s_county@23 = Williamson County, pruning_predicate=s_county_null_count@2 != row_count@3 AND s_county_min@0 <= Williamson County AND Williamson County <= s_county_max@1, required_guarantees=[s_county in (Williamson County)] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: (d_dom@2 >= 1 AND d_dom@2 <= 3 OR d_dom@2 >= 25 AND d_dom@2 <= 28) AND (d_year@1 = 1999 OR d_year@1 = 2000 OR d_year@1 = 2001), projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_dom], file_type=parquet, predicate=(d_dom@9 >= 1 AND d_dom@9 <= 3 OR d_dom@9 >= 25 AND d_dom@9 <= 28) AND (d_year@6 = 1999 OR d_year@6 = 2000 OR d_year@6 = 2001), pruning_predicate=(d_dom_null_count@1 != row_count@2 AND d_dom_max@0 >= 1 AND d_dom_null_count@1 != row_count@2 AND d_dom_min@3 <= 3 OR d_dom_null_count@1 != row_count@2 AND d_dom_max@0 >= 25 AND d_dom_null_count@1 != row_count@2 AND d_dom_min@3 <= 28) AND (d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5 OR d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5 OR d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5), required_guarantees=[d_year in (1999, 2000, 2001)] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_35() -> Result<()> { + let display = test_tpcds_query("q35").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [ca_state@0 ASC, cd_gender@1 ASC, cd_marital_status@2 ASC, cd_dep_count@3 ASC, cd_dep_employed_count@8 ASC, cd_dep_college_count@13 ASC], fetch=100 + │ SortExec: TopK(fetch=100), expr=[ca_state@0 ASC, cd_gender@1 ASC, cd_marital_status@2 ASC, cd_dep_count@3 ASC, cd_dep_employed_count@8 ASC, cd_dep_college_count@13 ASC], preserve_partitioning=[true] + │ ProjectionExec: expr=[ca_state@0 as ca_state, cd_gender@1 as cd_gender, cd_marital_status@2 as cd_marital_status, cd_dep_count@3 as cd_dep_count, count(Int64(1))@6 as cnt1, min(customer_demographics.cd_dep_count)@7 as min1, max(customer_demographics.cd_dep_count)@8 as max1, avg(customer_demographics.cd_dep_count)@9 as avg1, cd_dep_employed_count@4 as cd_dep_employed_count, count(Int64(1))@6 as cnt2, min(customer_demographics.cd_dep_employed_count)@10 as min2, max(customer_demographics.cd_dep_employed_count)@11 as max2, avg(customer_demographics.cd_dep_employed_count)@12 as avg2, cd_dep_college_count@5 as cd_dep_college_count, count(Int64(1))@6 as cnt3, min(customer_demographics.cd_dep_college_count)@13 as min(customer_demographics.cd_dep_college_count), max(customer_demographics.cd_dep_college_count)@14 as max(customer_demographics.cd_dep_college_count), avg(customer_demographics.cd_dep_college_count)@15 as avg(customer_demographics.cd_dep_college_count)] + │ AggregateExec: mode=FinalPartitioned, gby=[ca_state@0 as ca_state, cd_gender@1 as cd_gender, cd_marital_status@2 as cd_marital_status, cd_dep_count@3 as cd_dep_count, cd_dep_employed_count@4 as cd_dep_employed_count, cd_dep_college_count@5 as cd_dep_college_count], aggr=[count(Int64(1)), min(customer_demographics.cd_dep_count), max(customer_demographics.cd_dep_count), avg(customer_demographics.cd_dep_count), min(customer_demographics.cd_dep_employed_count), max(customer_demographics.cd_dep_employed_count), avg(customer_demographics.cd_dep_employed_count), min(customer_demographics.cd_dep_college_count), max(customer_demographics.cd_dep_college_count), avg(customer_demographics.cd_dep_college_count)] + │ [Stage 9] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 9 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] + │ RepartitionExec: partitioning=Hash([ca_state@0, cd_gender@1, cd_marital_status@2, cd_dep_count@3, cd_dep_employed_count@4, cd_dep_college_count@5], 3), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[ca_state@0 as ca_state, cd_gender@1 as cd_gender, cd_marital_status@2 as cd_marital_status, cd_dep_count@3 as cd_dep_count, cd_dep_employed_count@4 as cd_dep_employed_count, cd_dep_college_count@5 as cd_dep_college_count], aggr=[count(Int64(1)), min(customer_demographics.cd_dep_count), max(customer_demographics.cd_dep_count), avg(customer_demographics.cd_dep_count), min(customer_demographics.cd_dep_employed_count), max(customer_demographics.cd_dep_employed_count), avg(customer_demographics.cd_dep_employed_count), min(customer_demographics.cd_dep_college_count), max(customer_demographics.cd_dep_college_count), avg(customer_demographics.cd_dep_college_count)] + │ FilterExec: mark@6 OR mark@7, projection=[ca_state@0, cd_gender@1, cd_marital_status@2, cd_dep_count@3, cd_dep_employed_count@4, cd_dep_college_count@5] + │ HashJoinExec: mode=CollectLeft, join_type=RightMark, on=[(cs_ship_customer_sk@0, CAST(c.c_customer_sk AS Float64)@8)], projection=[ca_state@1, cd_gender@2, cd_marital_status@3, cd_dep_count@4, cd_dep_employed_count@5, cd_dep_college_count@6, mark@7, mark@9] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=3 + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, ca_state@1 as ca_state, cd_gender@2 as cd_gender, cd_marital_status@3 as cd_marital_status, cd_dep_count@4 as cd_dep_count, cd_dep_employed_count@5 as cd_dep_employed_count, cd_dep_college_count@6 as cd_dep_college_count, mark@7 as mark, CAST(c_customer_sk@0 AS Float64) as CAST(c.c_customer_sk AS Float64)] + │ HashJoinExec: mode=CollectLeft, join_type=RightMark, on=[(ws_bill_customer_sk@0, CAST(c.c_customer_sk AS Float64)@7)], projection=[c_customer_sk@0, ca_state@1, cd_gender@2, cd_marital_status@3, cd_dep_count@4, cd_dep_employed_count@5, cd_dep_college_count@6, mark@8] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=3 + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, ca_state@1 as ca_state, cd_gender@2 as cd_gender, cd_marital_status@3 as cd_marital_status, cd_dep_count@4 as cd_dep_count, cd_dep_employed_count@5 as cd_dep_employed_count, cd_dep_college_count@6 as cd_dep_college_count, CAST(c_customer_sk@0 AS Float64) as CAST(c.c_customer_sk AS Float64)] + │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(ss_customer_sk@0, CAST(c.c_customer_sk AS Float64)@7)], projection=[c_customer_sk@0, ca_state@1, cd_gender@2, cd_marital_status@3, cd_dep_count@4, cd_dep_employed_count@5, cd_dep_college_count@6] + │ CoalescePartitionsExec + │ [Stage 6] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=3 + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, ca_state@1 as ca_state, cd_gender@2 as cd_gender, cd_marital_status@3 as cd_marital_status, cd_dep_count@4 as cd_dep_count, cd_dep_employed_count@5 as cd_dep_employed_count, cd_dep_college_count@6 as cd_dep_college_count, CAST(c_customer_sk@0 AS Float64) as CAST(c.c_customer_sk AS Float64)] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_current_cdemo_sk@1, CAST(customer_demographics.cd_demo_sk AS Float64)@6)], projection=[c_customer_sk@0, ca_state@2, cd_gender@4, cd_marital_status@5, cd_dep_count@6, cd_dep_employed_count@7, cd_dep_college_count@8] + │ CoalescePartitionsExec + │ [Stage 8] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], ...]}, projection=[cd_demo_sk, cd_gender, cd_marital_status, cd_dep_count, cd_dep_employed_count, cd_dep_college_count, CAST(cd_demo_sk@0 AS Float64) as CAST(customer_demographics.cd_demo_sk AS Float64)], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p5] t1:[p6..p11] t2:[p12..p17] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, cs_sold_date_sk@0)], projection=[cs_ship_customer_sk@3] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_ship_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_year@1 = 2002 AND d_qoy@2 < 4, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_year@6 = 2002 AND d_qoy@10 < 4, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 AND d_qoy_null_count@5 != row_count@3 AND d_qoy_min@4 < 4, required_guarantees=[d_year in (2002)] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p5] t1:[p6..p11] t2:[p12..p17] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ws_sold_date_sk@0)], projection=[ws_bill_customer_sk@3] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_bill_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_year@1 = 2002 AND d_qoy@2 < 4, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_year@6 = 2002 AND d_qoy@10 < 4, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 AND d_qoy_null_count@5 != row_count@3 AND d_qoy_min@4 < 4, required_guarantees=[d_year in (2002)] + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p5] t1:[p6..p11] t2:[p12..p17] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_customer_sk@3] + │ CoalescePartitionsExec + │ [Stage 5] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_year@1 = 2002 AND d_qoy@2 < 4, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_year@6 = 2002 AND d_qoy@10 < 4, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1 AND d_qoy_null_count@5 != row_count@3 AND d_qoy_min@4 < 4, required_guarantees=[d_year in (2002)] + └────────────────────────────────────────────────── + ┌───── Stage 8 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[c_customer_sk@1 as c_customer_sk, c_current_cdemo_sk@2 as c_current_cdemo_sk, ca_state@0 as ca_state] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ca_address_sk@0, c_current_addr_sk@2)], projection=[ca_state@1, c_customer_sk@2, c_current_cdemo_sk@3] + │ CoalescePartitionsExec + │ [Stage 7] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=4, input_tasks=2 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_current_cdemo_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p3] t1:[p4..p7] + │ BroadcastExec: input_partitions=2, consumer_tasks=2, output_partitions=4 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_state], file_type=parquet + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_36() -> Result<()> { + let display = test_tpcds_query("q36").await?; + assert_snapshot!(display, @r#" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [lochierarchy@3 DESC, CASE WHEN lochierarchy@3 = 0 THEN i_category@1 END ASC, rank_within_parent@4 ASC], fetch=100 + │ SortExec: TopK(fetch=100), expr=[lochierarchy@3 DESC, CASE WHEN lochierarchy@3 = 0 THEN i_category@1 END ASC, rank_within_parent@4 ASC], preserve_partitioning=[true] + │ ProjectionExec: expr=[gross_margin@0 as gross_margin, i_category@1 as i_category, i_class@2 as i_class, lochierarchy@4 as lochierarchy, rank() PARTITION BY [results_rollup.lochierarchy, CASE WHEN results_rollup.t_class = Int64(0) THEN results_rollup.i_category END] ORDER BY [results_rollup.gross_margin ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@5 as rank_within_parent] + │ BoundedWindowAggExec: wdw=[rank() PARTITION BY [results_rollup.lochierarchy, CASE WHEN results_rollup.t_class = Int64(0) THEN results_rollup.i_category END] ORDER BY [results_rollup.gross_margin ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "rank() PARTITION BY [results_rollup.lochierarchy, CASE WHEN results_rollup.t_class = Int64(0) THEN results_rollup.i_category END] ORDER BY [results_rollup.gross_margin ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] + │ SortExec: expr=[lochierarchy@4 ASC NULLS LAST, CASE WHEN t_class@3 = 0 THEN i_category@1 END ASC NULLS LAST, gross_margin@0 ASC NULLS LAST], preserve_partitioning=[true] + │ RepartitionExec: partitioning=Hash([lochierarchy@4, CASE WHEN t_class@3 = 0 THEN i_category@1 END], 3), input_partitions=3 + │ ProjectionExec: expr=[gross_margin@0 as gross_margin, i_category@1 as i_category, i_class@2 as i_class, t_class@4 as t_class, lochierarchy@5 as lochierarchy] + │ AggregateExec: mode=FinalPartitioned, gby=[gross_margin@0 as gross_margin, i_category@1 as i_category, i_class@2 as i_class, t_category@3 as t_category, t_class@4 as t_class, lochierarchy@5 as lochierarchy], aggr=[] + │ [Stage 16] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 16 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ RepartitionExec: partitioning=Hash([gross_margin@0, i_category@1, i_class@2, t_category@3, t_class@4, lochierarchy@5], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[gross_margin@0 as gross_margin, i_category@1 as i_category, i_class@2 as i_class, t_category@3 as t_category, t_class@4 as t_class, lochierarchy@5 as lochierarchy], aggr=[], ordering_mode=PartiallySorted([3]) + │ DistributedUnionExec: t0:[c0] t1:[c1] + │ ProjectionExec: expr=[gross_margin@0 as gross_margin, CAST(i_category@1 AS Utf8) as i_category, i_class@2 as i_class, t_category@3 as t_category, t_class@4 as t_class, lochierarchy@5 as lochierarchy] + │ AggregateExec: mode=FinalPartitioned, gby=[gross_margin@0 as gross_margin, i_category@1 as i_category, i_class@2 as i_class, t_category@3 as t_category, t_class@4 as t_class, lochierarchy@5 as lochierarchy], aggr=[], ordering_mode=PartiallySorted([3]) + │ [Stage 10] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + │ ProjectionExec: expr=[CAST(sum(results.ss_net_profit)@0 AS Float64) / CAST(sum(results.ss_ext_sales_price)@1 AS Float64) as gross_margin, NULL as i_category, NULL as i_class, 1 as t_category, 1 as t_class, 2 as lochierarchy] + │ AggregateExec: mode=Final, gby=[], aggr=[sum(results.ss_net_profit), sum(results.ss_ext_sales_price)] + │ CoalescePartitionsExec + │ [Stage 15] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 10 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] + │ RepartitionExec: partitioning=Hash([gross_margin@0, i_category@1, i_class@2, t_category@3, t_class@4, lochierarchy@5], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[gross_margin@0 as gross_margin, i_category@1 as i_category, i_class@2 as i_class, t_category@3 as t_category, t_class@4 as t_class, lochierarchy@5 as lochierarchy], aggr=[], ordering_mode=PartiallySorted([3, 4, 5]) + │ DistributedUnionExec: t0:[c0(0/2)] t1:[c0(1/2)] t2:[c1] + │ ProjectionExec: expr=[CAST(sum(store_sales.ss_net_profit)@2 AS Float64) / CAST(sum(store_sales.ss_ext_sales_price)@3 AS Float64) as gross_margin, i_category@0 as i_category, CAST(i_class@1 AS Utf8) as i_class, 0 as t_category, 0 as t_class, 0 as lochierarchy] + │ AggregateExec: mode=FinalPartitioned, gby=[i_category@0 as i_category, i_class@1 as i_class], aggr=[sum(store_sales.ss_net_profit), sum(store_sales.ss_ext_sales_price)] + │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + │ ProjectionExec: expr=[CAST(sum(results.ss_net_profit)@1 AS Float64) / CAST(sum(results.ss_ext_sales_price)@2 AS Float64) as gross_margin, i_category@0 as i_category, NULL as i_class, 0 as t_category, 1 as t_class, 1 as lochierarchy] + │ AggregateExec: mode=FinalPartitioned, gby=[i_category@0 as i_category], aggr=[sum(results.ss_net_profit), sum(results.ss_ext_sales_price)] + │ [Stage 9] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([i_category@0, i_class@1], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_category@3 as i_category, i_class@2 as i_class], aggr=[sum(store_sales.ss_net_profit), sum(store_sales.ss_ext_sales_price)] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@1, ss_store_sk@0)], projection=[ss_ext_sales_price@3, ss_net_profit@4, i_class@5, i_category@6] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_item_sk@0, i_item_sk@0)], projection=[ss_store_sk@1, ss_ext_sales_price@2, ss_net_profit@3, i_class@5, i_category@6] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=3 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_class, i_category], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] + │ FilterExec: s_state@1 = TN, projection=[s_store_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_state], file_type=parquet, predicate=s_state@24 = TN, pruning_predicate=s_state_null_count@2 != row_count@3 AND s_state_min@0 <= TN AND TN <= s_state_max@1, required_guarantees=[s_state in (TN)] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p5] t1:[p6..p11] t2:[p12..p17] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(d1.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_item_sk@3, ss_store_sk@4, ss_ext_sales_price@5, ss_net_profit@6] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_ext_sales_price, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(d1.d_date_sk AS Float64)] + │ FilterExec: d_year@1 = 2001, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + └────────────────────────────────────────────────── + ┌───── Stage 9 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ RepartitionExec: partitioning=Hash([i_category@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_category@2 as i_category], aggr=[sum(results.ss_net_profit), sum(results.ss_ext_sales_price)] + │ ProjectionExec: expr=[sum(store_sales.ss_net_profit)@2 as ss_net_profit, sum(store_sales.ss_ext_sales_price)@3 as ss_ext_sales_price, i_category@0 as i_category] + │ AggregateExec: mode=FinalPartitioned, gby=[i_category@0 as i_category, i_class@1 as i_class], aggr=[sum(store_sales.ss_net_profit), sum(store_sales.ss_ext_sales_price)] + │ [Stage 8] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 8 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([i_category@0, i_class@1], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_category@3 as i_category, i_class@2 as i_class], aggr=[sum(store_sales.ss_net_profit), sum(store_sales.ss_ext_sales_price)] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@1, ss_store_sk@0)], projection=[ss_ext_sales_price@3, ss_net_profit@4, i_class@5, i_category@6] + │ CoalescePartitionsExec + │ [Stage 5] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_item_sk@0, i_item_sk@0)], projection=[ss_store_sk@1, ss_ext_sales_price@2, ss_net_profit@3, i_class@5, i_category@6] + │ CoalescePartitionsExec + │ [Stage 7] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=3 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_class, i_category], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] + │ FilterExec: s_state@1 = TN, projection=[s_store_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_state], file_type=parquet, predicate=s_state@24 = TN, pruning_predicate=s_state_null_count@2 != row_count@3 AND s_state_min@0 <= TN AND TN <= s_state_max@1, required_guarantees=[s_state in (TN)] + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p5] t1:[p6..p11] t2:[p12..p17] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(d1.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_item_sk@3, ss_store_sk@4, ss_ext_sales_price@5, ss_net_profit@6] + │ CoalescePartitionsExec + │ [Stage 6] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_ext_sales_price, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(d1.d_date_sk AS Float64)] + │ FilterExec: d_year@1 = 2001, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + └────────────────────────────────────────────────── + ┌───── Stage 15 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ AggregateExec: mode=Partial, gby=[], aggr=[sum(results.ss_net_profit), sum(results.ss_ext_sales_price)] + │ ProjectionExec: expr=[sum(store_sales.ss_net_profit)@2 as ss_net_profit, sum(store_sales.ss_ext_sales_price)@3 as ss_ext_sales_price] + │ AggregateExec: mode=FinalPartitioned, gby=[i_category@0 as i_category, i_class@1 as i_class], aggr=[sum(store_sales.ss_net_profit), sum(store_sales.ss_ext_sales_price)] + │ [Stage 14] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 14 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([i_category@0, i_class@1], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_category@3 as i_category, i_class@2 as i_class], aggr=[sum(store_sales.ss_net_profit), sum(store_sales.ss_ext_sales_price)] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@1, ss_store_sk@0)], projection=[ss_ext_sales_price@3, ss_net_profit@4, i_class@5, i_category@6] + │ CoalescePartitionsExec + │ [Stage 11] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_item_sk@0, i_item_sk@0)], projection=[ss_store_sk@1, ss_ext_sales_price@2, ss_net_profit@3, i_class@5, i_category@6] + │ CoalescePartitionsExec + │ [Stage 13] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=3 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_class, i_category], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 11 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] + │ FilterExec: s_state@1 = TN, projection=[s_store_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_state], file_type=parquet, predicate=s_state@24 = TN, pruning_predicate=s_state_null_count@2 != row_count@3 AND s_state_min@0 <= TN AND TN <= s_state_max@1, required_guarantees=[s_state in (TN)] + └────────────────────────────────────────────────── + ┌───── Stage 13 ── Tasks: t0:[p0..p5] t1:[p6..p11] t2:[p12..p17] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(d1.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_item_sk@3, ss_store_sk@4, ss_ext_sales_price@5, ss_net_profit@6] + │ CoalescePartitionsExec + │ [Stage 12] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_ext_sales_price, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 12 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(d1.d_date_sk AS Float64)] + │ FilterExec: d_year@1 = 2001, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + └────────────────────────────────────────────────── + "#); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_37() -> Result<()> { + let display = test_tpcds_query("q37").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [i_item_id@0 ASC NULLS LAST], fetch=100 + │ [Stage 5] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ SortExec: TopK(fetch=100), expr=[i_item_id@0 ASC NULLS LAST], preserve_partitioning=[true] + │ AggregateExec: mode=FinalPartitioned, gby=[i_item_id@0 as i_item_id, i_item_desc@1 as i_item_desc, i_current_price@2 as i_current_price], aggr=[] + │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([i_item_id@0, i_item_desc@1, i_current_price@2], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[i_item_id@0 as i_item_id, i_item_desc@1 as i_item_desc, i_current_price@2 as i_current_price], aggr=[] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, cs_item_sk@0)], projection=[i_item_id@1, i_item_desc@2, i_current_price@3] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=3 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p8] t1:[p9..p17] t2:[p18..p26] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(inv_date_sk@4, d_date_sk@0)], projection=[i_item_sk@0, i_item_id@1, i_item_desc@2, i_current_price@3] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=3 + │ FilterExec: d_date@1 >= 2000-02-01 AND d_date@1 <= 2000-04-01, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-02-01 AND d_date@2 <= 2000-04-01 AND DynamicFilter [ empty ], pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-02-01 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-04-01, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p5] t1:[p6..p11] t2:[p12..p17] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, inv_item_sk@1)], projection=[i_item_sk@0, i_item_id@1, i_item_desc@2, i_current_price@3, inv_date_sk@4] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ FilterExec: inv_quantity_on_hand@2 >= 100 AND inv_quantity_on_hand@2 <= 500, projection=[inv_date_sk@0, inv_item_sk@1] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/inventory/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:..], ...]}, projection=[inv_date_sk, inv_item_sk, inv_quantity_on_hand], file_type=parquet, predicate=inv_quantity_on_hand@3 >= 100 AND inv_quantity_on_hand@3 <= 500 AND DynamicFilter [ empty ], pruning_predicate=inv_quantity_on_hand_null_count@1 != row_count@2 AND inv_quantity_on_hand_max@0 >= 100 AND inv_quantity_on_hand_null_count@1 != row_count@2 AND inv_quantity_on_hand_min@3 <= 500, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ FilterExec: i_current_price@3 >= Some(6800),4,2 AND i_current_price@3 <= Some(9800),4,2 AND i_manufact_id@4 IN (SET) ([677, 940, 694, 808]), projection=[i_item_sk@0, i_item_id@1, i_item_desc@2, i_current_price@3] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_item_id, i_item_desc, i_current_price, i_manufact_id], file_type=parquet, predicate=i_current_price@5 >= Some(6800),4,2 AND i_current_price@5 <= Some(9800),4,2 AND i_manufact_id@13 IN (SET) ([677, 940, 694, 808]) AND DynamicFilter [ empty ], pruning_predicate=i_current_price_null_count@1 != row_count@2 AND i_current_price_max@0 >= Some(6800),4,2 AND i_current_price_null_count@1 != row_count@2 AND i_current_price_min@3 <= Some(9800),4,2 AND (i_manufact_id_null_count@6 != row_count@2 AND i_manufact_id_min@4 <= 677 AND 677 <= i_manufact_id_max@5 OR i_manufact_id_null_count@6 != row_count@2 AND i_manufact_id_min@4 <= 940 AND 940 <= i_manufact_id_max@5 OR i_manufact_id_null_count@6 != row_count@2 AND i_manufact_id_min@4 <= 694 AND 694 <= i_manufact_id_max@5 OR i_manufact_id_null_count@6 != row_count@2 AND i_manufact_id_min@4 <= 808 AND 808 <= i_manufact_id_max@5), required_guarantees=[i_manufact_id in (677, 694, 808, 940)] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_38() -> Result<()> { + let display = test_tpcds_query("q38").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ ProjectionExec: expr=[count(Int64(1))@0 as count(*)] + │ GlobalLimitExec: skip=0, fetch=100 + │ AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))] + │ CoalescePartitionsExec + │ [Stage 12] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 12 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] + │ ProjectionExec: expr=[] + │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(c_last_name@0, c_last_name@0), (c_first_name@1, c_first_name@1), (d_date@2, d_date@2)], NullsEqual: true + │ CoalescePartitionsExec + │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=6, input_tasks=2 + │ AggregateExec: mode=SinglePartitioned, gby=[c_last_name@0 as c_last_name, c_first_name@1 as c_first_name, d_date@2 as d_date], aggr=[] + │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(c_last_name@0, c_last_name@0), (c_first_name@1, c_first_name@1), (d_date@2, d_date@2)], NullsEqual: true + │ CoalescePartitionsExec + │ [Stage 8] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=6, input_tasks=2 + │ AggregateExec: mode=SinglePartitioned, gby=[c_last_name@0 as c_last_name, c_first_name@1 as c_first_name, d_date@2 as d_date], aggr=[] + │ AggregateExec: mode=FinalPartitioned, gby=[c_last_name@0 as c_last_name, c_first_name@1 as c_first_name, d_date@2 as d_date], aggr=[] + │ [Stage 11] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=3, consumer_tasks=2, output_partitions=6 + │ AggregateExec: mode=FinalPartitioned, gby=[c_last_name@0 as c_last_name, c_first_name@1 as c_first_name, d_date@2 as d_date], aggr=[] + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([c_last_name@0, c_first_name@1, d_date@2], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[c_last_name@0 as c_last_name, c_first_name@1 as c_first_name, d_date@2 as d_date], aggr=[] + │ ProjectionExec: expr=[c_last_name@2 as c_last_name, c_first_name@1 as c_first_name, d_date@0 as d_date] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ws_bill_customer_sk@0, CAST(customer.c_customer_sk AS Float64)@3)], projection=[d_date@1, c_first_name@3, c_last_name@4] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=3 + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p5] t1:[p6..p11] t2:[p12..p17] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ ProjectionExec: expr=[ws_bill_customer_sk@1 as ws_bill_customer_sk, d_date@0 as d_date] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, ws_sold_date_sk@0)], projection=[d_date@1, ws_bill_customer_sk@4] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_bill_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_date@1 as d_date, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_month_seq@2 >= 1200 AND d_month_seq@2 <= 1211, projection=[d_date_sk@0, d_date@1] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 8 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=3, consumer_tasks=2, output_partitions=6 + │ AggregateExec: mode=FinalPartitioned, gby=[c_last_name@0 as c_last_name, c_first_name@1 as c_first_name, d_date@2 as d_date], aggr=[] + │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([c_last_name@0, c_first_name@1, d_date@2], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[c_last_name@0 as c_last_name, c_first_name@1 as c_first_name, d_date@2 as d_date], aggr=[] + │ ProjectionExec: expr=[c_last_name@2 as c_last_name, c_first_name@1 as c_first_name, d_date@0 as d_date] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(cs_bill_customer_sk@0, CAST(customer.c_customer_sk AS Float64)@3)], projection=[d_date@1, c_first_name@3, c_last_name@4] + │ CoalescePartitionsExec + │ [Stage 6] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=3 + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p5] t1:[p6..p11] t2:[p12..p17] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ ProjectionExec: expr=[cs_bill_customer_sk@1 as cs_bill_customer_sk, d_date@0 as d_date] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, cs_sold_date_sk@0)], projection=[d_date@1, cs_bill_customer_sk@4] + │ CoalescePartitionsExec + │ [Stage 5] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_bill_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_date@1 as d_date, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_month_seq@2 >= 1200 AND d_month_seq@2 <= 1211, projection=[d_date_sk@0, d_date@1] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 11 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([c_last_name@0, c_first_name@1, d_date@2], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[c_last_name@0 as c_last_name, c_first_name@1 as c_first_name, d_date@2 as d_date], aggr=[] + │ ProjectionExec: expr=[c_last_name@2 as c_last_name, c_first_name@1 as c_first_name, d_date@0 as d_date] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_customer_sk@0, CAST(customer.c_customer_sk AS Float64)@3)], projection=[d_date@1, c_first_name@3, c_last_name@4] + │ CoalescePartitionsExec + │ [Stage 10] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=3 + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 10 ── Tasks: t0:[p0..p5] t1:[p6..p11] t2:[p12..p17] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ ProjectionExec: expr=[ss_customer_sk@1 as ss_customer_sk, d_date@0 as d_date] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, ss_sold_date_sk@0)], projection=[d_date@1, ss_customer_sk@4] + │ CoalescePartitionsExec + │ [Stage 9] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 9 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_date@1 as d_date, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_month_seq@2 >= 1200 AND d_month_seq@2 <= 1211, projection=[d_date_sk@0, d_date@1] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_39() -> Result<()> { + let display = test_tpcds_query("q39").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ ProjectionExec: expr=[wsk1@0 as wsk1, isk1@1 as isk1, dmoy1@2 as dmoy1, mean1@3 as mean1, cov1@4 as cov1, w_warehouse_sk@5 as w_warehouse_sk, i_item_sk@6 as i_item_sk, d_moy@7 as d_moy, mean@8 as mean, cov@9 as cov] + │ SortPreservingMergeExec: [w_warehouse_sk@10 ASC, i_item_sk@11 ASC, d_moy@12 ASC, mean@13 ASC, cov@14 ASC, d_moy@7 ASC, mean@8 ASC, cov@9 ASC] + │ SortExec: expr=[wsk1@0 ASC, isk1@1 ASC, mean1@3 ASC, cov1@4 ASC, mean@8 ASC, cov@9 ASC], preserve_partitioning=[true] + │ ProjectionExec: expr=[w_warehouse_sk@0 as wsk1, i_item_sk@1 as isk1, d_moy@2 as dmoy1, mean@3 as mean1, cov@4 as cov1, w_warehouse_sk@5 as w_warehouse_sk, i_item_sk@6 as i_item_sk, d_moy@7 as d_moy, mean@8 as mean, cov@9 as cov, w_warehouse_sk@0 as w_warehouse_sk, i_item_sk@1 as i_item_sk, d_moy@2 as d_moy, mean@3 as mean, cov@4 as cov] + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(i_item_sk@1, i_item_sk@1), (w_warehouse_sk@0, w_warehouse_sk@0)] + │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 10] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ RepartitionExec: partitioning=Hash([i_item_sk@1, w_warehouse_sk@0], 3), input_partitions=3 + │ ProjectionExec: expr=[w_warehouse_sk@0 as w_warehouse_sk, i_item_sk@1 as i_item_sk, d_moy@2 as d_moy, avg(inventory.inv_quantity_on_hand)@4 as mean, CASE avg(inventory.inv_quantity_on_hand)@4 WHEN 0 THEN NULL ELSE stddev(inventory.inv_quantity_on_hand)@3 / avg(inventory.inv_quantity_on_hand)@4 END as cov] + │ FilterExec: CASE avg(inventory.inv_quantity_on_hand)@4 WHEN 0 THEN 0 ELSE stddev(inventory.inv_quantity_on_hand)@3 / avg(inventory.inv_quantity_on_hand)@4 END > 1 + │ ProjectionExec: expr=[w_warehouse_sk@1 as w_warehouse_sk, i_item_sk@2 as i_item_sk, d_moy@3 as d_moy, stddev(inventory.inv_quantity_on_hand)@4 as stddev(inventory.inv_quantity_on_hand), avg(inventory.inv_quantity_on_hand)@5 as avg(inventory.inv_quantity_on_hand)] + │ AggregateExec: mode=FinalPartitioned, gby=[w_warehouse_name@0 as w_warehouse_name, w_warehouse_sk@1 as w_warehouse_sk, i_item_sk@2 as i_item_sk, d_moy@3 as d_moy], aggr=[stddev(inventory.inv_quantity_on_hand), avg(inventory.inv_quantity_on_hand)], ordering_mode=PartiallySorted([3]) + │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([w_warehouse_name@0, w_warehouse_sk@1, i_item_sk@2, d_moy@3], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[w_warehouse_name@3 as w_warehouse_name, w_warehouse_sk@2 as w_warehouse_sk, i_item_sk@1 as i_item_sk, d_moy@4 as d_moy], aggr=[stddev(inventory.inv_quantity_on_hand), avg(inventory.inv_quantity_on_hand)], ordering_mode=PartiallySorted([3]) + │ ProjectionExec: expr=[inv_quantity_on_hand@1 as inv_quantity_on_hand, i_item_sk@2 as i_item_sk, w_warehouse_sk@3 as w_warehouse_sk, w_warehouse_name@4 as w_warehouse_name, d_moy@0 as d_moy] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, inv_date_sk@0)], projection=[d_moy@1, inv_quantity_on_hand@3, i_item_sk@4, w_warehouse_sk@5, w_warehouse_name@6] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ ProjectionExec: expr=[inv_date_sk@2 as inv_date_sk, inv_quantity_on_hand@3 as inv_quantity_on_hand, i_item_sk@4 as i_item_sk, w_warehouse_sk@0 as w_warehouse_sk, w_warehouse_name@1 as w_warehouse_name] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(w_warehouse_sk@0, inv_warehouse_sk@1)], projection=[w_warehouse_sk@0, w_warehouse_name@1, inv_date_sk@2, inv_quantity_on_hand@4, i_item_sk@5] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[inv_date_sk@1 as inv_date_sk, inv_warehouse_sk@2 as inv_warehouse_sk, inv_quantity_on_hand@3 as inv_quantity_on_hand, i_item_sk@0 as i_item_sk] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, inv_item_sk@1)], projection=[i_item_sk@0, inv_date_sk@1, inv_warehouse_sk@3, inv_quantity_on_hand@4] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/inventory/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:..], ...]}, projection=[inv_date_sk, inv_item_sk, inv_warehouse_sk, inv_quantity_on_hand], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ FilterExec: d_moy@2 = 1 AND d_year@1 = 2001, projection=[d_date_sk@0, d_moy@2] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 1 AND d_year@6 = 2001, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 1 AND 1 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5, required_guarantees=[d_moy in (1), d_year in (2001)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/warehouse/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/warehouse/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/warehouse/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/warehouse/part-3.parquet]]}, projection=[w_warehouse_sk, w_warehouse_name], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 10 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ RepartitionExec: partitioning=Hash([i_item_sk@1, w_warehouse_sk@0], 3), input_partitions=3 + │ ProjectionExec: expr=[w_warehouse_sk@0 as w_warehouse_sk, i_item_sk@1 as i_item_sk, d_moy@2 as d_moy, avg(inventory.inv_quantity_on_hand)@4 as mean, CASE avg(inventory.inv_quantity_on_hand)@4 WHEN 0 THEN NULL ELSE stddev(inventory.inv_quantity_on_hand)@3 / avg(inventory.inv_quantity_on_hand)@4 END as cov] + │ FilterExec: CASE avg(inventory.inv_quantity_on_hand)@4 WHEN 0 THEN 0 ELSE stddev(inventory.inv_quantity_on_hand)@3 / avg(inventory.inv_quantity_on_hand)@4 END > 1 + │ ProjectionExec: expr=[w_warehouse_sk@1 as w_warehouse_sk, i_item_sk@2 as i_item_sk, d_moy@3 as d_moy, stddev(inventory.inv_quantity_on_hand)@4 as stddev(inventory.inv_quantity_on_hand), avg(inventory.inv_quantity_on_hand)@5 as avg(inventory.inv_quantity_on_hand)] + │ AggregateExec: mode=FinalPartitioned, gby=[w_warehouse_name@0 as w_warehouse_name, w_warehouse_sk@1 as w_warehouse_sk, i_item_sk@2 as i_item_sk, d_moy@3 as d_moy], aggr=[stddev(inventory.inv_quantity_on_hand), avg(inventory.inv_quantity_on_hand)], ordering_mode=PartiallySorted([3]) + │ [Stage 9] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 9 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([w_warehouse_name@0, w_warehouse_sk@1, i_item_sk@2, d_moy@3], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[w_warehouse_name@3 as w_warehouse_name, w_warehouse_sk@2 as w_warehouse_sk, i_item_sk@1 as i_item_sk, d_moy@4 as d_moy], aggr=[stddev(inventory.inv_quantity_on_hand), avg(inventory.inv_quantity_on_hand)], ordering_mode=PartiallySorted([3]) + │ ProjectionExec: expr=[inv_quantity_on_hand@1 as inv_quantity_on_hand, i_item_sk@2 as i_item_sk, w_warehouse_sk@3 as w_warehouse_sk, w_warehouse_name@4 as w_warehouse_name, d_moy@0 as d_moy] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, inv_date_sk@0)], projection=[d_moy@1, inv_quantity_on_hand@3, i_item_sk@4, w_warehouse_sk@5, w_warehouse_name@6] + │ CoalescePartitionsExec + │ [Stage 6] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ ProjectionExec: expr=[inv_date_sk@2 as inv_date_sk, inv_quantity_on_hand@3 as inv_quantity_on_hand, i_item_sk@4 as i_item_sk, w_warehouse_sk@0 as w_warehouse_sk, w_warehouse_name@1 as w_warehouse_name] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(w_warehouse_sk@0, inv_warehouse_sk@1)], projection=[w_warehouse_sk@0, w_warehouse_name@1, inv_date_sk@2, inv_quantity_on_hand@4, i_item_sk@5] + │ CoalescePartitionsExec + │ [Stage 7] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[inv_date_sk@1 as inv_date_sk, inv_warehouse_sk@2 as inv_warehouse_sk, inv_quantity_on_hand@3 as inv_quantity_on_hand, i_item_sk@0 as i_item_sk] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, inv_item_sk@1)], projection=[i_item_sk@0, inv_date_sk@1, inv_warehouse_sk@3, inv_quantity_on_hand@4] + │ CoalescePartitionsExec + │ [Stage 8] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/inventory/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:..], ...]}, projection=[inv_date_sk, inv_item_sk, inv_warehouse_sk, inv_quantity_on_hand], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ FilterExec: d_moy@2 = 2 AND d_year@1 = 2001, projection=[d_date_sk@0, d_moy@2] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 2 AND d_year@6 = 2001, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 2 AND 2 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5, required_guarantees=[d_moy in (2), d_year in (2001)] + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/warehouse/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/warehouse/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/warehouse/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/warehouse/part-3.parquet]]}, projection=[w_warehouse_sk, w_warehouse_name], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 8 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk], file_type=parquet + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_40() -> Result<()> { + let display = test_tpcds_query("q40").await?; + assert_snapshot!(display, @r#" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [w_state@0 ASC NULLS LAST, i_item_id@1 ASC NULLS LAST], fetch=100 + │ [Stage 7] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ SortExec: TopK(fetch=100), expr=[w_state@0 ASC NULLS LAST, i_item_id@1 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[w_state@0 as w_state, i_item_id@1 as i_item_id, sum(CASE WHEN date_dim.d_date < Utf8("2000-03-11") THEN catalog_sales.cs_sales_price - coalesce(catalog_returns.cr_refunded_cash,Int64(0)) ELSE Int64(0) END)@2 as sales_before, sum(CASE WHEN date_dim.d_date >= Utf8("2000-03-11") THEN catalog_sales.cs_sales_price - coalesce(catalog_returns.cr_refunded_cash,Int64(0)) ELSE Int64(0) END)@3 as sales_after] + │ AggregateExec: mode=FinalPartitioned, gby=[w_state@0 as w_state, i_item_id@1 as i_item_id], aggr=[sum(CASE WHEN date_dim.d_date < Utf8("2000-03-11") THEN catalog_sales.cs_sales_price - coalesce(catalog_returns.cr_refunded_cash,Int64(0)) ELSE Int64(0) END), sum(CASE WHEN date_dim.d_date >= Utf8("2000-03-11") THEN catalog_sales.cs_sales_price - coalesce(catalog_returns.cr_refunded_cash,Int64(0)) ELSE Int64(0) END)] + │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([w_state@0, i_item_id@1], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[w_state@3 as w_state, i_item_id@4 as i_item_id], aggr=[sum(CASE WHEN date_dim.d_date < Utf8("2000-03-11") THEN catalog_sales.cs_sales_price - coalesce(catalog_returns.cr_refunded_cash,Int64(0)) ELSE Int64(0) END), sum(CASE WHEN date_dim.d_date >= Utf8("2000-03-11") THEN catalog_sales.cs_sales_price - coalesce(catalog_returns.cr_refunded_cash,Int64(0)) ELSE Int64(0) END)] + │ ProjectionExec: expr=[d_date@4 as __common_expr_1, cs_sales_price@0 as cs_sales_price, cr_refunded_cash@1 as cr_refunded_cash, w_state@2 as w_state, i_item_id@3 as i_item_id] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(cs_sold_date_sk@0, CAST(date_dim.d_date_sk AS Float64)@2)], projection=[cs_sales_price@1, cr_refunded_cash@2, w_state@3, i_item_id@4, d_date@6] + │ CoalescePartitionsExec + │ [Stage 5] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=3 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_date@1 as d_date, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_date@1 >= 2000-02-10 AND d_date@1 <= 2000-04-10 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-02-10 AND d_date@2 <= 2000-04-10, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-02-10 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-04-10, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p8] t1:[p9..p17] t2:[p18..p26] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(cs_item_sk@1, i_item_sk@0)], projection=[cs_sold_date_sk@0, cs_sales_price@2, cr_refunded_cash@3, w_state@4, i_item_id@6] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=3 + │ FilterExec: i_current_price@2 >= Some(99),4,2 AND i_current_price@2 <= Some(149),4,2, projection=[i_item_sk@0, i_item_id@1] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_item_id, i_current_price], file_type=parquet, predicate=i_current_price@5 >= Some(99),4,2 AND i_current_price@5 <= Some(149),4,2 AND DynamicFilter [ empty ], pruning_predicate=i_current_price_null_count@1 != row_count@2 AND i_current_price_max@0 >= Some(99),4,2 AND i_current_price_null_count@1 != row_count@2 AND i_current_price_min@3 <= Some(149),4,2, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p8] t1:[p9..p17] t2:[p18..p26] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[cs_sold_date_sk@1 as cs_sold_date_sk, cs_item_sk@2 as cs_item_sk, cs_sales_price@3 as cs_sales_price, cr_refunded_cash@4 as cr_refunded_cash, w_state@0 as w_state] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(warehouse.w_warehouse_sk AS Float64)@2, cs_warehouse_sk@1)], projection=[w_state@1, cs_sold_date_sk@3, cs_item_sk@5, cs_sales_price@6, cr_refunded_cash@7] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[cs_sold_date_sk@1 as cs_sold_date_sk, cs_warehouse_sk@2 as cs_warehouse_sk, cs_item_sk@3 as cs_item_sk, cs_sales_price@4 as cs_sales_price, cr_refunded_cash@0 as cr_refunded_cash] + │ HashJoinExec: mode=Partitioned, join_type=Right, on=[(cr_order_number@1, cs_order_number@3), (cr_item_sk@0, cs_item_sk@2)], projection=[cr_refunded_cash@2, cs_sold_date_sk@3, cs_warehouse_sk@4, cs_item_sk@5, cs_sales_price@7] + │ [Stage 2] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/warehouse/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/warehouse/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/warehouse/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/warehouse/part-3.parquet]]}, projection=[w_warehouse_sk, w_state, CAST(w_warehouse_sk@0 AS Float64) as CAST(warehouse.w_warehouse_sk AS Float64)], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] + │ RepartitionExec: partitioning=Hash([cr_order_number@1, cr_item_sk@0], 9), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], ...]}, projection=[cr_item_sk, cr_order_number, cr_refunded_cash], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] + │ RepartitionExec: partitioning=Hash([cs_order_number@3, cs_item_sk@2], 9), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_warehouse_sk, cs_item_sk, cs_order_number, cs_sales_price], file_type=parquet + └────────────────────────────────────────────────── + "#); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_41() -> Result<()> { + let display = test_tpcds_query("q41").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [i_product_name@0 ASC NULLS LAST], fetch=100 + │ SortExec: TopK(fetch=100), expr=[i_product_name@0 ASC NULLS LAST], preserve_partitioning=[true] + │ AggregateExec: mode=FinalPartitioned, gby=[i_product_name@0 as i_product_name], aggr=[] + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ RepartitionExec: partitioning=Hash([i_product_name@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_product_name@0 as i_product_name], aggr=[] + │ FilterExec: CASE WHEN __always_true@2 IS NULL THEN 0 ELSE item_cnt@1 END > 0, projection=[i_product_name@0] + │ ProjectionExec: expr=[i_product_name@2 as i_product_name, item_cnt@0 as item_cnt, __always_true@1 as __always_true] + │ HashJoinExec: mode=CollectLeft, join_type=Right, on=[(i_manufact@1, i_manufact@0)], projection=[item_cnt@0, __always_true@2, i_product_name@4] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=6, input_tasks=1 + │ FilterExec: i_manufact_id@0 >= 738 AND i_manufact_id@0 <= 778, projection=[i_manufact@1, i_product_name@2] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_manufact_id, i_manufact, i_product_name], file_type=parquet, predicate=i_manufact_id@13 >= 738 AND i_manufact_id@13 <= 778, pruning_predicate=i_manufact_id_null_count@1 != row_count@2 AND i_manufact_id_max@0 >= 738 AND i_manufact_id_null_count@1 != row_count@2 AND i_manufact_id_min@3 <= 778, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p5] + │ BroadcastExec: input_partitions=3, consumer_tasks=2, output_partitions=6 + │ ProjectionExec: expr=[count(Int64(1))@1 as item_cnt, i_manufact@0 as i_manufact, true as __always_true] + │ AggregateExec: mode=FinalPartitioned, gby=[i_manufact@0 as i_manufact], aggr=[count(Int64(1))] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ RepartitionExec: partitioning=Hash([i_manufact@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_manufact@0 as i_manufact], aggr=[count(Int64(1))] + │ FilterExec: __common_expr_4@0 AND ((i_color@4 = powder OR i_color@4 = khaki) AND (i_units@5 = Ounce OR i_units@5 = Oz) AND (i_size@3 = medium OR i_size@3 = extra large) OR (i_color@4 = brown OR i_color@4 = honeydew) AND (i_units@5 = Bunch OR i_units@5 = Ton) AND (i_size@3 = N/A OR i_size@3 = small)) OR i_category@1 = Men AND (i_color@4 = floral OR i_color@4 = deep) AND (i_units@5 = N/A OR i_units@5 = Dozen) AND i_size@3 = petite OR i_category@1 = Men AND (i_color@4 = light OR i_color@4 = cornflower) AND (i_units@5 = Box OR i_units@5 = Pound) AND (i_size@3 = medium OR i_size@3 = extra large) OR __common_expr_4@0 AND ((i_color@4 = midnight OR i_color@4 = snow) AND (i_units@5 = Pallet OR i_units@5 = Gross) AND (i_size@3 = medium OR i_size@3 = extra large) OR (i_color@4 = cyan OR i_color@4 = papaya) AND (i_units@5 = Cup OR i_units@5 = Dram) AND (i_size@3 = N/A OR i_size@3 = small)) OR i_category@1 = Men AND (i_color@4 = orange OR i_color@4 = frosted) AND (i_units@5 = Each OR i_units@5 = Tbl) AND i_size@3 = petite OR i_category@1 = Men AND (i_color@4 = forest OR i_color@4 = ghost) AND (i_units@5 = Lb OR i_units@5 = Bundle) AND (i_size@3 = medium OR i_size@3 = extra large), projection=[i_manufact@2] + │ ProjectionExec: expr=[i_category@0 = Women as __common_expr_4, i_category@0 as i_category, i_manufact@1 as i_manufact, i_size@2 as i_size, i_color@3 as i_color, i_units@4 as i_units] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_category, i_manufact, i_size, i_color, i_units], file_type=parquet + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_42() -> Result<()> { + let display = test_tpcds_query("q42").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [sum(store_sales.ss_ext_sales_price)@3 DESC, d_year@0 ASC NULLS LAST, i_category_id@1 ASC NULLS LAST, i_category@2 ASC NULLS LAST], fetch=100 + │ [Stage 4] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ SortExec: TopK(fetch=100), expr=[sum(store_sales.ss_ext_sales_price)@3 DESC, i_category_id@1 ASC NULLS LAST, i_category@2 ASC NULLS LAST], preserve_partitioning=[true] + │ AggregateExec: mode=FinalPartitioned, gby=[d_year@0 as d_year, i_category_id@1 as i_category_id, i_category@2 as i_category], aggr=[sum(store_sales.ss_ext_sales_price)], ordering_mode=PartiallySorted([0]) + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([d_year@0, i_category_id@1, i_category@2], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[d_year@0 as d_year, i_category_id@2 as i_category_id, i_category@3 as i_category], aggr=[sum(store_sales.ss_ext_sales_price)], ordering_mode=PartiallySorted([0]) + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_item_sk@1, i_item_sk@0)], projection=[d_year@0, ss_ext_sales_price@2, i_category_id@4, i_category@5] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=3 + │ FilterExec: i_manager_id@3 = 1, projection=[i_item_sk@0, i_category_id@1, i_category@2] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_category_id, i_category, i_manager_id], file_type=parquet, predicate=i_manager_id@20 = 1 AND DynamicFilter [ empty ], pruning_predicate=i_manager_id_null_count@2 != row_count@3 AND i_manager_id_min@0 <= 1 AND 1 <= i_manager_id_max@1, required_guarantees=[i_manager_id in (1)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p5] t1:[p6..p11] t2:[p12..p17] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(dt.d_date_sk AS Float64)@2, ss_sold_date_sk@0)], projection=[d_year@1, ss_item_sk@4, ss_ext_sales_price@5] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, CAST(d_date_sk@0 AS Float64) as CAST(dt.d_date_sk AS Float64)] + │ FilterExec: d_moy@2 = 11 AND d_year@1 = 2000, projection=[d_date_sk@0, d_year@1] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 11 AND d_year@6 = 2000, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5, required_guarantees=[d_moy in (11), d_year in (2000)] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_43() -> Result<()> { + let display = test_tpcds_query("q43").await?; + assert_snapshot!(display, @r#" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [s_store_name@0 ASC NULLS LAST, s_store_id@1 ASC NULLS LAST, sun_sales@2 ASC NULLS LAST, mon_sales@3 ASC NULLS LAST, tue_sales@4 ASC NULLS LAST, wed_sales@5 ASC NULLS LAST, thu_sales@6 ASC NULLS LAST, fri_sales@7 ASC NULLS LAST, sat_sales@8 ASC NULLS LAST], fetch=100 + │ [Stage 4] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ SortExec: TopK(fetch=100), expr=[s_store_name@0 ASC NULLS LAST, s_store_id@1 ASC NULLS LAST, sun_sales@2 ASC NULLS LAST, mon_sales@3 ASC NULLS LAST, tue_sales@4 ASC NULLS LAST, wed_sales@5 ASC NULLS LAST, thu_sales@6 ASC NULLS LAST, fri_sales@7 ASC NULLS LAST, sat_sales@8 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[s_store_name@0 as s_store_name, s_store_id@1 as s_store_id, sum(CASE WHEN date_dim.d_day_name = Utf8("Sunday") THEN store_sales.ss_sales_price ELSE NULL END)@2 as sun_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Monday") THEN store_sales.ss_sales_price ELSE NULL END)@3 as mon_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Tuesday") THEN store_sales.ss_sales_price ELSE NULL END)@4 as tue_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Wednesday") THEN store_sales.ss_sales_price ELSE NULL END)@5 as wed_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Thursday") THEN store_sales.ss_sales_price ELSE NULL END)@6 as thu_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Friday") THEN store_sales.ss_sales_price ELSE NULL END)@7 as fri_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Saturday") THEN store_sales.ss_sales_price ELSE NULL END)@8 as sat_sales] + │ AggregateExec: mode=FinalPartitioned, gby=[s_store_name@0 as s_store_name, s_store_id@1 as s_store_id], aggr=[sum(CASE WHEN date_dim.d_day_name = Utf8("Sunday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Monday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Tuesday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Wednesday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Thursday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Friday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Saturday") THEN store_sales.ss_sales_price ELSE NULL END)] + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([s_store_name@0, s_store_id@1], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[s_store_name@3 as s_store_name, s_store_id@2 as s_store_id], aggr=[sum(CASE WHEN date_dim.d_day_name = Utf8("Sunday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Monday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Tuesday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Wednesday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Thursday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Friday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Saturday") THEN store_sales.ss_sales_price ELSE NULL END)] + │ ProjectionExec: expr=[d_day_name@2 as d_day_name, ss_sales_price@3 as ss_sales_price, s_store_id@0 as s_store_id, s_store_name@1 as s_store_name] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@3, ss_store_sk@1)], projection=[s_store_id@1, s_store_name@2, d_day_name@4, ss_sales_price@6] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, ss_sold_date_sk@0)], projection=[d_day_name@1, ss_store_sk@4, ss_sales_price@5] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, s_store_id@1 as s_store_id, s_store_name@2 as s_store_name, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] + │ FilterExec: s_gmt_offset@3 = Some(-500),3,2, projection=[s_store_sk@0, s_store_id@1, s_store_name@2] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_store_id, s_store_name, s_gmt_offset], file_type=parquet, predicate=s_gmt_offset@27 = Some(-500),3,2, pruning_predicate=s_gmt_offset_null_count@2 != row_count@3 AND s_gmt_offset_min@0 <= Some(-500),3,2 AND Some(-500),3,2 <= s_gmt_offset_max@1, required_guarantees=[s_gmt_offset in (Some(-500),3,2)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_day_name@1 as d_day_name, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_year@1 = 2000, projection=[d_date_sk@0, d_day_name@2] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_day_name], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] + └────────────────────────────────────────────────── + "#); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_44() -> Result<()> { + let display = test_tpcds_query("q44").await?; + assert_snapshot!(display, @r#" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [rnk@0 ASC NULLS LAST], fetch=100 + │ [Stage 9] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 9 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ SortExec: TopK(fetch=100), expr=[rnk@0 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[rnk@1 as rnk, i_product_name@2 as best_performing, i_product_name@0 as worst_performing] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, item_sk@1)], projection=[i_product_name@1, rnk@2, i_product_name@4] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=4, input_tasks=2 + │ ProjectionExec: expr=[rnk@1 as rnk, item_sk@2 as item_sk, i_product_name@0 as i_product_name] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, item_sk@0)], projection=[i_product_name@1, rnk@3, item_sk@4] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=4, input_tasks=2 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(rnk@1, rnk@1)], projection=[item_sk@0, rnk@1, item_sk@2] + │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=1 + │ [Stage 8] => NetworkShuffleExec: output_partitions=3, input_tasks=1 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p3] t1:[p4..p7] + │ BroadcastExec: input_partitions=2, consumer_tasks=2, output_partitions=4 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_product_name], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p3] t1:[p4..p7] + │ BroadcastExec: input_partitions=2, consumer_tasks=2, output_partitions=4 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_product_name], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p5] + │ RepartitionExec: partitioning=Hash([rnk@1], 6), input_partitions=3 + │ ProjectionExec: expr=[item_sk@0 as item_sk, rank() ORDER BY [v1.rank_col ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@1 as rnk] + │ FilterExec: rank() ORDER BY [v1.rank_col ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@1 < 11 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=1, maintains_sort_order=true + │ ProjectionExec: expr=[item_sk@0 as item_sk, rank() ORDER BY [v1.rank_col ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@2 as rank() ORDER BY [v1.rank_col ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW] + │ BoundedWindowAggExec: wdw=[rank() ORDER BY [v1.rank_col ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "rank() ORDER BY [v1.rank_col ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] + │ SortPreservingMergeExec: [rank_col@1 ASC NULLS LAST] + │ SortExec: expr=[rank_col@1 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[ss_item_sk@0 as item_sk, avg(ss1.ss_net_profit)@1 as rank_col] + │ FilterExec: CAST(avg(ss1.ss_net_profit)@1 AS Decimal128(30, 15)) > CAST(0.9 * rank_col@2 AS Decimal128(30, 15)), projection=[ss_item_sk@0, avg(ss1.ss_net_profit)@1] + │ NestedLoopJoinExec: join_type=Left + │ CoalescePartitionsExec + │ AggregateExec: mode=FinalPartitioned, gby=[ss_item_sk@0 as ss_item_sk], aggr=[avg(ss1.ss_net_profit)] + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + │ ProjectionExec: expr=[CAST(avg(store_sales.ss_net_profit)@1 AS Float64) as rank_col] + │ AggregateExec: mode=FinalPartitioned, gby=[ss_store_sk@0 as ss_store_sk], aggr=[avg(store_sales.ss_net_profit)], ordering_mode=Sorted + │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] + │ RepartitionExec: partitioning=Hash([ss_item_sk@0], 3), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[ss_item_sk@0 as ss_item_sk], aggr=[avg(ss1.ss_net_profit)] + │ FilterExec: ss_store_sk@1 = 4, projection=[ss_item_sk@0, ss_net_profit@2] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_item_sk, ss_store_sk, ss_net_profit], file_type=parquet, predicate=ss_store_sk@7 = 4, pruning_predicate=ss_store_sk_null_count@2 != row_count@3 AND ss_store_sk_min@0 <= 4 AND 4 <= ss_store_sk_max@1, required_guarantees=[ss_store_sk in (4)] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] + │ RepartitionExec: partitioning=Hash([ss_store_sk@0], 3), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[ss_store_sk@0 as ss_store_sk], aggr=[avg(store_sales.ss_net_profit)], ordering_mode=Sorted + │ FilterExec: ss_store_sk@1 = 4 AND ss_addr_sk@0 IS NULL, projection=[ss_store_sk@1, ss_net_profit@2] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_addr_sk, ss_store_sk, ss_net_profit], file_type=parquet, predicate=ss_store_sk@7 = 4 AND ss_addr_sk@6 IS NULL, pruning_predicate=ss_store_sk_null_count@2 != row_count@3 AND ss_store_sk_min@0 <= 4 AND 4 <= ss_store_sk_max@1 AND ss_addr_sk_null_count@4 > 0, required_guarantees=[ss_store_sk in (4)] + └────────────────────────────────────────────────── + ┌───── Stage 8 ── Tasks: t0:[p0..p5] + │ RepartitionExec: partitioning=Hash([rnk@1], 6), input_partitions=3 + │ ProjectionExec: expr=[item_sk@0 as item_sk, rank() ORDER BY [v2.rank_col DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@1 as rnk] + │ FilterExec: rank() ORDER BY [v2.rank_col DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@1 < 11 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=1, maintains_sort_order=true + │ ProjectionExec: expr=[item_sk@0 as item_sk, rank() ORDER BY [v2.rank_col DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@2 as rank() ORDER BY [v2.rank_col DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW] + │ BoundedWindowAggExec: wdw=[rank() ORDER BY [v2.rank_col DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "rank() ORDER BY [v2.rank_col DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] + │ SortPreservingMergeExec: [rank_col@1 DESC] + │ SortExec: expr=[rank_col@1 DESC], preserve_partitioning=[true] + │ ProjectionExec: expr=[ss_item_sk@0 as item_sk, avg(ss1.ss_net_profit)@1 as rank_col] + │ FilterExec: CAST(avg(ss1.ss_net_profit)@1 AS Decimal128(30, 15)) > CAST(0.9 * rank_col@2 AS Decimal128(30, 15)), projection=[ss_item_sk@0, avg(ss1.ss_net_profit)@1] + │ NestedLoopJoinExec: join_type=Left + │ CoalescePartitionsExec + │ AggregateExec: mode=FinalPartitioned, gby=[ss_item_sk@0 as ss_item_sk], aggr=[avg(ss1.ss_net_profit)] + │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + │ ProjectionExec: expr=[CAST(avg(store_sales.ss_net_profit)@1 AS Float64) as rank_col] + │ AggregateExec: mode=FinalPartitioned, gby=[ss_store_sk@0 as ss_store_sk], aggr=[avg(store_sales.ss_net_profit)], ordering_mode=Sorted + │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] + │ RepartitionExec: partitioning=Hash([ss_item_sk@0], 3), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[ss_item_sk@0 as ss_item_sk], aggr=[avg(ss1.ss_net_profit)] + │ FilterExec: ss_store_sk@1 = 4, projection=[ss_item_sk@0, ss_net_profit@2] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_item_sk, ss_store_sk, ss_net_profit], file_type=parquet, predicate=ss_store_sk@7 = 4, pruning_predicate=ss_store_sk_null_count@2 != row_count@3 AND ss_store_sk_min@0 <= 4 AND 4 <= ss_store_sk_max@1, required_guarantees=[ss_store_sk in (4)] + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] + │ RepartitionExec: partitioning=Hash([ss_store_sk@0], 3), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[ss_store_sk@0 as ss_store_sk], aggr=[avg(store_sales.ss_net_profit)], ordering_mode=Sorted + │ FilterExec: ss_store_sk@1 = 4 AND ss_addr_sk@0 IS NULL, projection=[ss_store_sk@1, ss_net_profit@2] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_addr_sk, ss_store_sk, ss_net_profit], file_type=parquet, predicate=ss_store_sk@7 = 4 AND ss_addr_sk@6 IS NULL, pruning_predicate=ss_store_sk_null_count@2 != row_count@3 AND ss_store_sk_min@0 <= 4 AND 4 <= ss_store_sk_max@1 AND ss_addr_sk_null_count@4 > 0, required_guarantees=[ss_store_sk in (4)] + └────────────────────────────────────────────────── + "#); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_45() -> Result<()> { + let display = test_tpcds_query("q45").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [ca_zip@0 ASC NULLS LAST, ca_city@1 ASC NULLS LAST], fetch=100 + │ SortExec: TopK(fetch=100), expr=[ca_zip@0 ASC NULLS LAST, ca_city@1 ASC NULLS LAST], preserve_partitioning=[true] + │ AggregateExec: mode=FinalPartitioned, gby=[ca_zip@0 as ca_zip, ca_city@1 as ca_city], aggr=[sum(web_sales.ws_sales_price)] + │ RepartitionExec: partitioning=Hash([ca_zip@0, ca_city@1], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[ca_zip@2 as ca_zip, ca_city@1 as ca_city], aggr=[sum(web_sales.ws_sales_price)] + │ FilterExec: substr(ca_zip@2, 1, 5) IN (SET) ([85669, 86197, 88274, 83405, 86475, 85392, 85460, 80348, 81792]) OR mark@3, projection=[ws_sales_price@0, ca_city@1, ca_zip@2] + │ HashJoinExec: mode=CollectLeft, join_type=LeftMark, on=[(i_item_id@3, i_item_id@0)], projection=[ws_sales_price@0, ca_city@1, ca_zip@2, mark@4] + │ CoalescePartitionsExec + │ [Stage 6] => NetworkCoalesceExec: output_partitions=9, input_tasks=3 + │ FilterExec: i_item_sk@0 IN (SET) ([2, 3, 5, 7, 11, 13, 17, 19, 23, 29]), projection=[i_item_id@1] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=i_item_sk@0 IN (SET) ([2, 3, 5, 7, 11, 13, 17, 19, 23, 29]), pruning_predicate=i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 2 AND 2 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 3 AND 3 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 5 AND 5 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 7 AND 7 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 11 AND 11 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 13 AND 13 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 17 AND 17 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 19 AND 19 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 23 AND 23 <= i_item_sk_max@1 OR i_item_sk_null_count@2 != row_count@3 AND i_item_sk_min@0 <= 29 AND 29 <= i_item_sk_max@1, required_guarantees=[i_item_sk in (11, 13, 17, 19, 2, 23, 29, 3, 5, 7)] + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p2] t1:[p3..p5] t2:[p6..p8] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ws_item_sk@0, i_item_sk@0)], projection=[ws_sales_price@1, ca_city@2, ca_zip@3, i_item_id@5] + │ CoalescePartitionsExec + │ [Stage 5] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=3 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p8] t1:[p9..p17] t2:[p18..p26] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ws_sold_date_sk@0)], projection=[ws_item_sk@3, ws_sales_price@4, ca_city@5, ca_zip@6] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ ProjectionExec: expr=[ws_sold_date_sk@2 as ws_sold_date_sk, ws_item_sk@3 as ws_item_sk, ws_sales_price@4 as ws_sales_price, ca_city@0 as ca_city, ca_zip@1 as ca_zip] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ca_address_sk@0, c_current_addr_sk@3)], projection=[ca_city@1, ca_zip@2, ws_sold_date_sk@3, ws_item_sk@4, ws_sales_price@5] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[ws_sold_date_sk@1 as ws_sold_date_sk, ws_item_sk@2 as ws_item_sk, ws_sales_price@3 as ws_sales_price, c_current_addr_sk@0 as c_current_addr_sk] + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(customer.c_customer_sk AS Float64)@2, ws_bill_customer_sk@2)], projection=[c_current_addr_sk@1, ws_sold_date_sk@3, ws_item_sk@4, ws_sales_price@6] + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_qoy@2 = 2 AND d_year@1 = 2001, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet, predicate=d_qoy@10 = 2 AND d_year@6 = 2001, pruning_predicate=d_qoy_null_count@2 != row_count@3 AND d_qoy_min@0 <= 2 AND 2 <= d_qoy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5, required_guarantees=[d_qoy in (2), d_year in (2001)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_city, ca_zip], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p8] t1:[p0..p8] + │ RepartitionExec: partitioning=Hash([CAST(customer.c_customer_sk AS Float64)@2], 9), input_partitions=3 + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, c_current_addr_sk@1 as c_current_addr_sk, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] + │ RepartitionExec: partitioning=Hash([ws_bill_customer_sk@2], 9), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_item_sk, ws_bill_customer_sk, ws_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_46() -> Result<()> { + let display = test_tpcds_query("q46").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [c_last_name@0 ASC, c_first_name@1 ASC, ca_city@2 ASC, bought_city@3 ASC, ss_ticket_number@4 ASC], fetch=100 + │ [Stage 8] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 8 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ SortExec: TopK(fetch=100), expr=[c_last_name@0 ASC, c_first_name@1 ASC, ca_city@2 ASC, bought_city@3 ASC, ss_ticket_number@4 ASC], preserve_partitioning=[true] + │ ProjectionExec: expr=[c_last_name@5 as c_last_name, c_first_name@4 as c_first_name, ca_city@6 as ca_city, bought_city@1 as bought_city, ss_ticket_number@0 as ss_ticket_number, amt@2 as amt, profit@3 as profit] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_current_addr_sk@4, ca_address_sk@0)], filter=bought_city@0 != ca_city@1, projection=[ss_ticket_number@0, bought_city@1, amt@2, profit@3, c_first_name@5, c_last_name@6, ca_city@8] + │ CoalescePartitionsExec + │ [Stage 7] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=6, input_tasks=2 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_city], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=3, consumer_tasks=2, output_partitions=6 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_customer_sk@1, CAST(customer.c_customer_sk AS Float64)@4)], projection=[ss_ticket_number@0, bought_city@2, amt@3, profit@4, c_current_addr_sk@6, c_first_name@7, c_last_name@8] + │ CoalescePartitionsExec + │ [Stage 6] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, c_current_addr_sk@1 as c_current_addr_sk, c_first_name@2 as c_first_name, c_last_name@3 as c_last_name, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_current_addr_sk, c_first_name, c_last_name], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=3, consumer_tasks=2, output_partitions=6 + │ ProjectionExec: expr=[ss_ticket_number@0 as ss_ticket_number, ss_customer_sk@1 as ss_customer_sk, ca_city@3 as bought_city, sum(store_sales.ss_coupon_amt)@4 as amt, sum(store_sales.ss_net_profit)@5 as profit] + │ AggregateExec: mode=FinalPartitioned, gby=[ss_ticket_number@0 as ss_ticket_number, ss_customer_sk@1 as ss_customer_sk, ss_addr_sk@2 as ss_addr_sk, ca_city@3 as ca_city], aggr=[sum(store_sales.ss_coupon_amt), sum(store_sales.ss_net_profit)] + │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([ss_ticket_number@0, ss_customer_sk@1, ss_addr_sk@2, ca_city@3], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[ss_ticket_number@2 as ss_ticket_number, ss_customer_sk@0 as ss_customer_sk, ss_addr_sk@1 as ss_addr_sk, ca_city@5 as ca_city], aggr=[sum(store_sales.ss_coupon_amt), sum(store_sales.ss_net_profit)] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_addr_sk@1, CAST(customer_address.ca_address_sk AS Float64)@2)], projection=[ss_customer_sk@0, ss_addr_sk@1, ss_ticket_number@2, ss_coupon_amt@3, ss_net_profit@4, ca_city@6] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=3 + │ ProjectionExec: expr=[ca_address_sk@0 as ca_address_sk, ca_city@1 as ca_city, CAST(ca_address_sk@0 AS Float64) as CAST(customer_address.ca_address_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_city], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p5] t1:[p6..p11] t2:[p12..p17] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(household_demographics.hd_demo_sk AS Float64)@1, ss_hdemo_sk@1)], projection=[ss_customer_sk@2, ss_addr_sk@4, ss_ticket_number@5, ss_coupon_amt@6, ss_net_profit@7] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@1, ss_store_sk@3)], projection=[ss_customer_sk@2, ss_hdemo_sk@3, ss_addr_sk@4, ss_ticket_number@6, ss_coupon_amt@7, ss_net_profit@8] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_customer_sk@3, ss_hdemo_sk@4, ss_addr_sk@5, ss_store_sk@6, ss_ticket_number@7, ss_coupon_amt@8, ss_net_profit@9] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_hdemo_sk, ss_addr_sk, ss_store_sk, ss_ticket_number, ss_coupon_amt, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[hd_demo_sk@0 as hd_demo_sk, CAST(hd_demo_sk@0 AS Float64) as CAST(household_demographics.hd_demo_sk AS Float64)] + │ FilterExec: hd_dep_count@1 = 4 OR hd_vehicle_count@2 = 3, projection=[hd_demo_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-3.parquet]]}, projection=[hd_demo_sk, hd_dep_count, hd_vehicle_count], file_type=parquet, predicate=hd_dep_count@3 = 4 OR hd_vehicle_count@4 = 3, pruning_predicate=hd_dep_count_null_count@2 != row_count@3 AND hd_dep_count_min@0 <= 4 AND 4 <= hd_dep_count_max@1 OR hd_vehicle_count_null_count@6 != row_count@3 AND hd_vehicle_count_min@4 <= 3 AND 3 <= hd_vehicle_count_max@5, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] + │ FilterExec: s_city@1 = Fairview OR s_city@1 = Midway, projection=[s_store_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_city], file_type=parquet, predicate=s_city@22 = Fairview OR s_city@22 = Midway, pruning_predicate=s_city_null_count@2 != row_count@3 AND s_city_min@0 <= Fairview AND Fairview <= s_city_max@1 OR s_city_null_count@2 != row_count@3 AND s_city_min@0 <= Midway AND Midway <= s_city_max@1, required_guarantees=[s_city in (Fairview, Midway)] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: (d_dow@2 = 6 OR d_dow@2 = 0) AND (d_year@1 = 1999 OR d_year@1 = 2000 OR d_year@1 = 2001), projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_dow], file_type=parquet, predicate=(d_dow@7 = 6 OR d_dow@7 = 0) AND (d_year@6 = 1999 OR d_year@6 = 2000 OR d_year@6 = 2001), pruning_predicate=(d_dow_null_count@2 != row_count@3 AND d_dow_min@0 <= 6 AND 6 <= d_dow_max@1 OR d_dow_null_count@2 != row_count@3 AND d_dow_min@0 <= 0 AND 0 <= d_dow_max@1) AND (d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5 OR d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5 OR d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5), required_guarantees=[d_dow in (0, 6), d_year in (1999, 2000, 2001)] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_47() -> Result<()> { + let display = test_tpcds_query("q47").await?; + assert_snapshot!(display, @r#" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [sum_sales@7 - avg_monthly_sales@6 ASC NULLS LAST, i_category@0 ASC NULLS LAST, i_brand@1 ASC NULLS LAST, s_store_name@2 ASC NULLS LAST, s_company_name@3 ASC NULLS LAST, d_year@4 ASC NULLS LAST, d_moy@5 ASC NULLS LAST, avg_monthly_sales@6 ASC NULLS LAST, sum_sales@7 ASC NULLS LAST, psum@8 ASC NULLS LAST, nsum@9 ASC NULLS LAST], fetch=100 + │ SortExec: TopK(fetch=100), expr=[sum_sales@7 - avg_monthly_sales@6 ASC NULLS LAST, i_category@0 ASC NULLS LAST, i_brand@1 ASC NULLS LAST, s_store_name@2 ASC NULLS LAST, s_company_name@3 ASC NULLS LAST, d_moy@5 ASC NULLS LAST, avg_monthly_sales@6 ASC NULLS LAST, sum_sales@7 ASC NULLS LAST, psum@8 ASC NULLS LAST, nsum@9 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[i_category@0 as i_category, i_brand@1 as i_brand, s_store_name@2 as s_store_name, s_company_name@3 as s_company_name, d_year@4 as d_year, d_moy@5 as d_moy, avg_monthly_sales@7 as avg_monthly_sales, sum_sales@6 as sum_sales, sum_sales@8 as psum, sum_sales@9 as nsum] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_category@0, i_category@0), (i_brand@1, i_brand@1), (s_store_name@2, s_store_name@2), (s_company_name@3, s_company_name@3), (CAST(v1.rn AS Decimal128(21, 0))@10, v1_lead.rn - Decimal128(Some(1),20,0)@6)], projection=[i_category@0, i_brand@1, s_store_name@2, s_company_name@3, d_year@4, d_moy@5, sum_sales@6, avg_monthly_sales@7, sum_sales@9, sum_sales@15] + │ CoalescePartitionsExec + │ BroadcastExec: input_partitions=3, consumer_tasks=1, output_partitions=3 + │ ProjectionExec: expr=[i_category@0 as i_category, i_brand@1 as i_brand, s_store_name@2 as s_store_name, s_company_name@3 as s_company_name, d_year@4 as d_year, d_moy@5 as d_moy, sum_sales@6 as sum_sales, avg_monthly_sales@7 as avg_monthly_sales, rn@8 as rn, sum_sales@9 as sum_sales, CAST(rn@8 AS Decimal128(21, 0)) as CAST(v1.rn AS Decimal128(21, 0))] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_category@0, i_category@0), (i_brand@1, i_brand@1), (s_store_name@2, s_store_name@2), (s_company_name@3, s_company_name@3), (CAST(v1.rn AS Decimal128(21, 0))@9, v1_lag.rn + Decimal128(Some(1),20,0)@6)], projection=[i_category@0, i_brand@1, s_store_name@2, s_company_name@3, d_year@4, d_moy@5, sum_sales@6, avg_monthly_sales@7, rn@8, sum_sales@14] + │ CoalescePartitionsExec + │ BroadcastExec: input_partitions=3, consumer_tasks=1, output_partitions=3 + │ ProjectionExec: expr=[i_category@0 as i_category, i_brand@1 as i_brand, s_store_name@2 as s_store_name, s_company_name@3 as s_company_name, d_year@4 as d_year, d_moy@5 as d_moy, sum(store_sales.ss_sales_price)@6 as sum_sales, avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name, date_dim.d_year] ORDER BY [date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@7 as avg_monthly_sales, rank() PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@8 as rn, CAST(rank() PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@8 AS Decimal128(21, 0)) as CAST(v1.rn AS Decimal128(21, 0))] + │ FilterExec: d_year@4 = 1999 AND avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name, date_dim.d_year] ORDER BY [date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@7 > Some(0),19,6 AND CASE WHEN avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name, date_dim.d_year] ORDER BY [date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@7 > Some(0),19,6 THEN abs(sum(store_sales.ss_sales_price)@6 - avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name, date_dim.d_year] ORDER BY [date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@7) / avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name, date_dim.d_year] ORDER BY [date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@7 END > Some(1000000000),30,10 + │ BoundedWindowAggExec: wdw=[rank() PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "rank() PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] + │ SortExec: expr=[i_category@0 ASC NULLS LAST, i_brand@1 ASC NULLS LAST, s_store_name@2 ASC NULLS LAST, s_company_name@3 ASC NULLS LAST, d_year@4 ASC NULLS LAST, d_moy@5 ASC NULLS LAST], preserve_partitioning=[true] + │ RepartitionExec: partitioning=Hash([i_category@0, i_brand@1, s_store_name@2, s_company_name@3], 3), input_partitions=3 + │ BoundedWindowAggExec: wdw=[avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name, date_dim.d_year] ORDER BY [date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name, date_dim.d_year] ORDER BY [date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": nullable Decimal128(19, 6) }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] + │ SortExec: expr=[i_category@0 ASC NULLS LAST, i_brand@1 ASC NULLS LAST, s_store_name@2 ASC NULLS LAST, s_company_name@3 ASC NULLS LAST, d_year@4 ASC NULLS LAST, d_moy@5 ASC NULLS LAST], preserve_partitioning=[true] + │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ ProjectionExec: expr=[i_category@0 as i_category, i_brand@1 as i_brand, s_store_name@2 as s_store_name, s_company_name@3 as s_company_name, sum(store_sales.ss_sales_price)@6 as sum_sales, rank() PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@7 as rn, CAST(rank() PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@7 AS Decimal128(20, 0)) + Some(1),20,0 as v1_lag.rn + Decimal128(Some(1),20,0)] + │ BoundedWindowAggExec: wdw=[rank() PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "rank() PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] + │ SortExec: expr=[i_category@0 ASC NULLS LAST, i_brand@1 ASC NULLS LAST, s_store_name@2 ASC NULLS LAST, s_company_name@3 ASC NULLS LAST, d_year@4 ASC NULLS LAST, d_moy@5 ASC NULLS LAST], preserve_partitioning=[true] + │ [Stage 10] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ ProjectionExec: expr=[i_category@0 as i_category, i_brand@1 as i_brand, s_store_name@2 as s_store_name, s_company_name@3 as s_company_name, sum(store_sales.ss_sales_price)@6 as sum_sales, rank() PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@7 as rn, CAST(rank() PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@7 AS Decimal128(20, 0)) - Some(1),20,0 as v1_lead.rn - Decimal128(Some(1),20,0)] + │ BoundedWindowAggExec: wdw=[rank() PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "rank() PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] + │ SortExec: expr=[i_category@0 ASC NULLS LAST, i_brand@1 ASC NULLS LAST, s_store_name@2 ASC NULLS LAST, s_company_name@3 ASC NULLS LAST, d_year@4 ASC NULLS LAST, d_moy@5 ASC NULLS LAST], preserve_partitioning=[true] + │ [Stage 15] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ RepartitionExec: partitioning=Hash([i_category@0, i_brand@1, s_store_name@2, s_company_name@3, d_year@4], 3), input_partitions=3 + │ AggregateExec: mode=FinalPartitioned, gby=[i_category@0 as i_category, i_brand@1 as i_brand, s_store_name@2 as s_store_name, s_company_name@3 as s_company_name, d_year@4 as d_year, d_moy@5 as d_moy], aggr=[sum(store_sales.ss_sales_price)] + │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([i_category@0, i_brand@1, s_store_name@2, s_company_name@3, d_year@4, d_moy@5], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[i_category@1 as i_category, i_brand@0 as i_brand, s_store_name@5 as s_store_name, s_company_name@6 as s_company_name, d_year@3 as d_year, d_moy@4 as d_moy], aggr=[sum(store_sales.ss_sales_price)] + │ ProjectionExec: expr=[i_brand@2 as i_brand, i_category@3 as i_category, ss_sales_price@4 as ss_sales_price, d_year@5 as d_year, d_moy@6 as d_moy, s_store_name@0 as s_store_name, s_company_name@1 as s_company_name] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@3, ss_store_sk@2)], projection=[s_store_name@1, s_company_name@2, i_brand@4, i_category@5, ss_sales_price@7, d_year@8, d_moy@9] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[i_brand@2 as i_brand, i_category@3 as i_category, ss_store_sk@4 as ss_store_sk, ss_sales_price@5 as ss_sales_price, d_year@0 as d_year, d_moy@1 as d_moy] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@3, ss_sold_date_sk@2)], projection=[d_year@1, d_moy@2, i_brand@4, i_category@5, ss_store_sk@7, ss_sales_price@8] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ss_item_sk@1)], projection=[i_brand@1, i_category@2, ss_sold_date_sk@3, ss_store_sk@5, ss_sales_price@6] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_store_name, s_company_name, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, d_moy@2 as d_moy, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_year@1 = 1999 OR d_year@1 = 1998 AND d_moy@2 = 12 OR d_year@1 = 2000 AND d_moy@2 = 1 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1999 OR d_year@6 = 1998 AND d_moy@8 = 12 OR d_year@6 = 2000 AND d_moy@8 = 1, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 1 AND 1 <= d_moy_max@5, required_guarantees=[d_year in (1998, 1999, 2000)] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand, i_category], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 10 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ RepartitionExec: partitioning=Hash([i_category@0, i_brand@1, s_store_name@2, s_company_name@3], 3), input_partitions=3 + │ AggregateExec: mode=FinalPartitioned, gby=[i_category@0 as i_category, i_brand@1 as i_brand, s_store_name@2 as s_store_name, s_company_name@3 as s_company_name, d_year@4 as d_year, d_moy@5 as d_moy], aggr=[sum(store_sales.ss_sales_price)] + │ [Stage 9] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 9 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([i_category@0, i_brand@1, s_store_name@2, s_company_name@3, d_year@4, d_moy@5], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[i_category@1 as i_category, i_brand@0 as i_brand, s_store_name@5 as s_store_name, s_company_name@6 as s_company_name, d_year@3 as d_year, d_moy@4 as d_moy], aggr=[sum(store_sales.ss_sales_price)] + │ ProjectionExec: expr=[i_brand@2 as i_brand, i_category@3 as i_category, ss_sales_price@4 as ss_sales_price, d_year@5 as d_year, d_moy@6 as d_moy, s_store_name@0 as s_store_name, s_company_name@1 as s_company_name] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@3, ss_store_sk@2)], projection=[s_store_name@1, s_company_name@2, i_brand@4, i_category@5, ss_sales_price@7, d_year@8, d_moy@9] + │ CoalescePartitionsExec + │ [Stage 6] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[i_brand@2 as i_brand, i_category@3 as i_category, ss_store_sk@4 as ss_store_sk, ss_sales_price@5 as ss_sales_price, d_year@0 as d_year, d_moy@1 as d_moy] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@3, ss_sold_date_sk@2)], projection=[d_year@1, d_moy@2, i_brand@4, i_category@5, ss_store_sk@7, ss_sales_price@8] + │ CoalescePartitionsExec + │ [Stage 7] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ss_item_sk@1)], projection=[i_brand@1, i_category@2, ss_sold_date_sk@3, ss_store_sk@5, ss_sales_price@6] + │ CoalescePartitionsExec + │ [Stage 8] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_store_name, s_company_name, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, d_moy@2 as d_moy, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_year@1 = 1999 OR d_year@1 = 1998 AND d_moy@2 = 12 OR d_year@1 = 2000 AND d_moy@2 = 1 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1999 OR d_year@6 = 1998 AND d_moy@8 = 12 OR d_year@6 = 2000 AND d_moy@8 = 1, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 1 AND 1 <= d_moy_max@5, required_guarantees=[d_year in (1998, 1999, 2000)] + └────────────────────────────────────────────────── + ┌───── Stage 8 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand, i_category], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 15 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ RepartitionExec: partitioning=Hash([i_category@0, i_brand@1, s_store_name@2, s_company_name@3], 3), input_partitions=3 + │ AggregateExec: mode=FinalPartitioned, gby=[i_category@0 as i_category, i_brand@1 as i_brand, s_store_name@2 as s_store_name, s_company_name@3 as s_company_name, d_year@4 as d_year, d_moy@5 as d_moy], aggr=[sum(store_sales.ss_sales_price)] + │ [Stage 14] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 14 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([i_category@0, i_brand@1, s_store_name@2, s_company_name@3, d_year@4, d_moy@5], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[i_category@1 as i_category, i_brand@0 as i_brand, s_store_name@5 as s_store_name, s_company_name@6 as s_company_name, d_year@3 as d_year, d_moy@4 as d_moy], aggr=[sum(store_sales.ss_sales_price)] + │ ProjectionExec: expr=[i_brand@2 as i_brand, i_category@3 as i_category, ss_sales_price@4 as ss_sales_price, d_year@5 as d_year, d_moy@6 as d_moy, s_store_name@0 as s_store_name, s_company_name@1 as s_company_name] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@3, ss_store_sk@2)], projection=[s_store_name@1, s_company_name@2, i_brand@4, i_category@5, ss_sales_price@7, d_year@8, d_moy@9] + │ CoalescePartitionsExec + │ [Stage 11] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[i_brand@2 as i_brand, i_category@3 as i_category, ss_store_sk@4 as ss_store_sk, ss_sales_price@5 as ss_sales_price, d_year@0 as d_year, d_moy@1 as d_moy] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@3, ss_sold_date_sk@2)], projection=[d_year@1, d_moy@2, i_brand@4, i_category@5, ss_store_sk@7, ss_sales_price@8] + │ CoalescePartitionsExec + │ [Stage 12] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ss_item_sk@1)], projection=[i_brand@1, i_category@2, ss_sold_date_sk@3, ss_store_sk@5, ss_sales_price@6] + │ CoalescePartitionsExec + │ [Stage 13] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 11 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_store_name, s_company_name, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 12 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, d_moy@2 as d_moy, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_year@1 = 1999 OR d_year@1 = 1998 AND d_moy@2 = 12 OR d_year@1 = 2000 AND d_moy@2 = 1 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1999 OR d_year@6 = 1998 AND d_moy@8 = 12 OR d_year@6 = 2000 AND d_moy@8 = 1, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 1 AND 1 <= d_moy_max@5, required_guarantees=[d_year in (1998, 1999, 2000)] + └────────────────────────────────────────────────── + ┌───── Stage 13 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand, i_category], file_type=parquet + └────────────────────────────────────────────────── + "#); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_48() -> Result<()> { + let display = test_tpcds_query("q48").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ AggregateExec: mode=Final, gby=[], aggr=[sum(store_sales.ss_quantity)] + │ CoalescePartitionsExec + │ [Stage 5] => NetworkCoalesceExec: output_partitions=9, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p2] t1:[p3..p5] t2:[p6..p8] + │ AggregateExec: mode=Partial, gby=[], aggr=[sum(store_sales.ss_quantity)] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_sold_date_sk@0, CAST(date_dim.d_date_sk AS Float64)@1)], projection=[ss_quantity@1] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=3 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_year@1 = 2000, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p8] t1:[p9..p17] t2:[p18..p26] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_addr_sk@1, CAST(customer_address.ca_address_sk AS Float64)@2)], filter=(ca_state@1 = CO OR ca_state@1 = OH OR ca_state@1 = TX) AND ss_net_profit@0 >= Some(0),6,2 AND ss_net_profit@0 <= Some(200000),6,2 OR (ca_state@1 = OR OR ca_state@1 = MN OR ca_state@1 = KY) AND ss_net_profit@0 >= Some(15000),6,2 AND ss_net_profit@0 <= Some(300000),6,2 OR (ca_state@1 = VA OR ca_state@1 = CA OR ca_state@1 = MS) AND ss_net_profit@0 >= Some(5000),6,2 AND CAST(ss_net_profit@0 AS Decimal128(22, 2)) <= Some(2500000),22,2, projection=[ss_sold_date_sk@0, ss_quantity@2] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=3 + │ ProjectionExec: expr=[ca_address_sk@0 as ca_address_sk, ca_state@1 as ca_state, CAST(ca_address_sk@0 AS Float64) as CAST(customer_address.ca_address_sk AS Float64)] + │ FilterExec: (ca_state@1 = CO OR ca_state@1 = OH OR ca_state@1 = TX OR ca_state@1 = OR OR ca_state@1 = MN OR ca_state@1 = KY OR ca_state@1 = VA OR ca_state@1 = CA OR ca_state@1 = MS) AND ca_state@1 IN (SET) ([CO, OH, TX, OR, MN, KY, VA, CA, MS]) AND ca_country@2 = United States, projection=[ca_address_sk@0, ca_state@1] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_state, ca_country], file_type=parquet, predicate=(ca_state@8 = CO OR ca_state@8 = OH OR ca_state@8 = TX OR ca_state@8 = OR OR ca_state@8 = MN OR ca_state@8 = KY OR ca_state@8 = VA OR ca_state@8 = CA OR ca_state@8 = MS) AND ca_state@8 IN (SET) ([CO, OH, TX, OR, MN, KY, VA, CA, MS]) AND ca_country@10 = United States, pruning_predicate=(ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= CO AND CO <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= OH AND OH <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= TX AND TX <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= OR AND OR <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= MN AND MN <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= KY AND KY <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= VA AND VA <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= CA AND CA <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= MS AND MS <= ca_state_max@1) AND (ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= CO AND CO <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= OH AND OH <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= TX AND TX <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= OR AND OR <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= MN AND MN <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= KY AND KY <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= VA AND VA <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= CA AND CA <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= MS AND MS <= ca_state_max@1) AND ca_country_null_count@6 != row_count@3 AND ca_country_min@4 <= United States AND United States <= ca_country_max@5, required_guarantees=[ca_country in (United States), ca_state in (CA, CO, KY, MN, MS, OH, OR, TX, VA)] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p5] t1:[p6..p11] t2:[p12..p17] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_cdemo_sk@1, CAST(customer_demographics.cd_demo_sk AS Float64)@3)], filter=cd_marital_status@1 = M AND cd_education_status@2 = 4 yr Degree AND ss_sales_price@0 >= Some(10000),5,2 AND ss_sales_price@0 <= Some(15000),5,2 OR cd_marital_status@1 = D AND cd_education_status@2 = 2 yr Degree AND ss_sales_price@0 >= Some(5000),5,2 AND ss_sales_price@0 <= Some(10000),5,2 OR cd_marital_status@1 = S AND cd_education_status@2 = College AND ss_sales_price@0 >= Some(15000),5,2 AND ss_sales_price@0 <= Some(20000),5,2, projection=[ss_sold_date_sk@0, ss_addr_sk@2, ss_quantity@3, ss_net_profit@5] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=3 + │ ProjectionExec: expr=[cd_demo_sk@0 as cd_demo_sk, cd_marital_status@1 as cd_marital_status, cd_education_status@2 as cd_education_status, CAST(cd_demo_sk@0 AS Float64) as CAST(customer_demographics.cd_demo_sk AS Float64)] + │ FilterExec: cd_marital_status@1 = M AND cd_education_status@2 = 4 yr Degree OR cd_marital_status@1 = D AND cd_education_status@2 = 2 yr Degree OR cd_marital_status@1 = S AND cd_education_status@2 = College + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], ...]}, projection=[cd_demo_sk, cd_marital_status, cd_education_status], file_type=parquet, predicate=cd_marital_status@2 = M AND cd_education_status@3 = 4 yr Degree OR cd_marital_status@2 = D AND cd_education_status@3 = 2 yr Degree OR cd_marital_status@2 = S AND cd_education_status@3 = College, pruning_predicate=cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= M AND M <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= 4 yr Degree AND 4 yr Degree <= cd_education_status_max@5 OR cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= D AND D <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= 2 yr Degree AND 2 yr Degree <= cd_education_status_max@5 OR cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= S AND S <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= College AND College <= cd_education_status_max@5, required_guarantees=[cd_education_status in (2 yr Degree, 4 yr Degree, College), cd_marital_status in (D, M, S)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p5] t1:[p6..p11] t2:[p12..p17] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@1, ss_store_sk@3)], projection=[ss_sold_date_sk@2, ss_cdemo_sk@3, ss_addr_sk@4, ss_quantity@6, ss_sales_price@7, ss_net_profit@8] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=2 + │ FilterExec: (ss_net_profit@6 >= Some(0),6,2 AND ss_net_profit@6 <= Some(200000),6,2 OR ss_net_profit@6 >= Some(15000),6,2 AND ss_net_profit@6 <= Some(300000),6,2 OR ss_net_profit@6 >= Some(5000),6,2 AND CAST(ss_net_profit@6 AS Decimal128(22, 2)) <= Some(2500000),22,2) AND (ss_sales_price@5 >= Some(10000),5,2 AND ss_sales_price@5 <= Some(15000),5,2 OR ss_sales_price@5 >= Some(5000),5,2 AND ss_sales_price@5 <= Some(10000),5,2 OR ss_sales_price@5 >= Some(15000),5,2 AND ss_sales_price@5 <= Some(20000),5,2) + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_cdemo_sk, ss_addr_sk, ss_store_sk, ss_quantity, ss_sales_price, ss_net_profit], file_type=parquet, predicate=(ss_net_profit@22 >= Some(0),6,2 AND ss_net_profit@22 <= Some(200000),6,2 OR ss_net_profit@22 >= Some(15000),6,2 AND ss_net_profit@22 <= Some(300000),6,2 OR ss_net_profit@22 >= Some(5000),6,2 AND CAST(ss_net_profit@22 AS Decimal128(22, 2)) <= Some(2500000),22,2) AND (ss_sales_price@13 >= Some(10000),5,2 AND ss_sales_price@13 <= Some(15000),5,2 OR ss_sales_price@13 >= Some(5000),5,2 AND ss_sales_price@13 <= Some(10000),5,2 OR ss_sales_price@13 >= Some(15000),5,2 AND ss_sales_price@13 <= Some(20000),5,2) AND DynamicFilter [ empty ], pruning_predicate=(ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_max@0 >= Some(0),6,2 AND ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_min@3 <= Some(200000),6,2 OR ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_max@0 >= Some(15000),6,2 AND ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_min@3 <= Some(300000),6,2 OR ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_max@0 >= Some(5000),6,2 AND ss_net_profit_null_count@1 != row_count@2 AND CAST(ss_net_profit_min@3 AS Decimal128(22, 2)) <= Some(2500000),22,2) AND (ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_max@4 >= Some(10000),5,2 AND ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_min@6 <= Some(15000),5,2 OR ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_max@4 >= Some(5000),5,2 AND ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_min@6 <= Some(10000),5,2 OR ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_max@4 >= Some(15000),5,2 AND ss_sales_price_null_count@5 != row_count@2 AND ss_sales_price_min@6 <= Some(20000),5,2), required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)], file_type=parquet + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_49() -> Result<()> { + let display = test_tpcds_query("q49").await?; + assert_snapshot!(display, @r#" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [channel@0 ASC, return_rank@3 ASC, currency_rank@4 ASC, item@1 ASC], fetch=100 + │ SortExec: TopK(fetch=100), expr=[channel@0 ASC, return_rank@3 ASC, currency_rank@4 ASC, item@1 ASC], preserve_partitioning=[true] + │ AggregateExec: mode=FinalPartitioned, gby=[channel@0 as channel, item@1 as item, return_ratio@2 as return_ratio, return_rank@3 as return_rank, currency_rank@4 as currency_rank], aggr=[] + │ [Stage 15] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 15 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] + │ RepartitionExec: partitioning=Hash([channel@0, item@1, return_ratio@2, return_rank@3, currency_rank@4], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[channel@0 as channel, item@1 as item, return_ratio@2 as return_ratio, return_rank@3 as return_rank, currency_rank@4 as currency_rank], aggr=[], ordering_mode=PartiallySorted([0, 4]) + │ DistributedUnionExec: t0:[c0] t1:[c1] t2:[c2] + │ ProjectionExec: expr=[web as channel, item@0 as item, return_ratio@1 as return_ratio, rank() ORDER BY [in_web.return_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@2 as return_rank, rank() ORDER BY [in_web.currency_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@3 as currency_rank] + │ FilterExec: rank() ORDER BY [in_web.return_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@2 <= 10 OR rank() ORDER BY [in_web.currency_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@3 <= 10 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=1, maintains_sort_order=true + │ ProjectionExec: expr=[item@0 as item, return_ratio@1 as return_ratio, rank() ORDER BY [in_web.return_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@3 as rank() ORDER BY [in_web.return_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW, rank() ORDER BY [in_web.currency_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@4 as rank() ORDER BY [in_web.currency_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW] + │ BoundedWindowAggExec: wdw=[rank() ORDER BY [in_web.currency_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "rank() ORDER BY [in_web.currency_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] + │ SortExec: expr=[currency_ratio@2 ASC NULLS LAST], preserve_partitioning=[false] + │ BoundedWindowAggExec: wdw=[rank() ORDER BY [in_web.return_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "rank() ORDER BY [in_web.return_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] + │ SortPreservingMergeExec: [return_ratio@1 ASC NULLS LAST] + │ SortExec: expr=[return_ratio@1 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[ws_item_sk@0 as item, CAST(sum(coalesce(wr.wr_return_quantity,Int64(0)))@1 AS Decimal128(15, 4)) / CAST(sum(coalesce(ws.ws_quantity,Int64(0)))@2 AS Decimal128(15, 4)) as return_ratio, CAST(sum(coalesce(wr.wr_return_amt,Int64(0)))@3 AS Decimal128(15, 4)) / CAST(sum(coalesce(ws.ws_net_paid,Int64(0)))@4 AS Decimal128(15, 4)) as currency_ratio] + │ AggregateExec: mode=FinalPartitioned, gby=[ws_item_sk@0 as ws_item_sk], aggr=[sum(coalesce(wr.wr_return_quantity,Int64(0))), sum(coalesce(ws.ws_quantity,Int64(0))), sum(coalesce(wr.wr_return_amt,Int64(0))), sum(coalesce(ws.ws_net_paid,Int64(0)))] + │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ ProjectionExec: expr=[catalog as channel, item@0 as item, return_ratio@1 as return_ratio, rank() ORDER BY [in_cat.return_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@2 as return_rank, rank() ORDER BY [in_cat.currency_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@3 as currency_rank] + │ FilterExec: rank() ORDER BY [in_cat.return_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@2 <= 10 OR rank() ORDER BY [in_cat.currency_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@3 <= 10 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=1, maintains_sort_order=true + │ ProjectionExec: expr=[item@0 as item, return_ratio@1 as return_ratio, rank() ORDER BY [in_cat.return_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@3 as rank() ORDER BY [in_cat.return_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW, rank() ORDER BY [in_cat.currency_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@4 as rank() ORDER BY [in_cat.currency_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW] + │ BoundedWindowAggExec: wdw=[rank() ORDER BY [in_cat.currency_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "rank() ORDER BY [in_cat.currency_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] + │ SortExec: expr=[currency_ratio@2 ASC NULLS LAST], preserve_partitioning=[false] + │ BoundedWindowAggExec: wdw=[rank() ORDER BY [in_cat.return_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "rank() ORDER BY [in_cat.return_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] + │ SortPreservingMergeExec: [return_ratio@1 ASC NULLS LAST] + │ [Stage 9] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[store as channel, item@0 as item, return_ratio@1 as return_ratio, rank() ORDER BY [in_store.return_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@2 as return_rank, rank() ORDER BY [in_store.currency_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@3 as currency_rank] + │ FilterExec: rank() ORDER BY [in_store.return_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@2 <= 10 OR rank() ORDER BY [in_store.currency_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@3 <= 10 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=1, maintains_sort_order=true + │ ProjectionExec: expr=[item@0 as item, return_ratio@1 as return_ratio, rank() ORDER BY [in_store.return_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@3 as rank() ORDER BY [in_store.return_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW, rank() ORDER BY [in_store.currency_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@4 as rank() ORDER BY [in_store.currency_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW] + │ BoundedWindowAggExec: wdw=[rank() ORDER BY [in_store.currency_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "rank() ORDER BY [in_store.currency_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] + │ SortExec: expr=[currency_ratio@2 ASC NULLS LAST], preserve_partitioning=[false] + │ BoundedWindowAggExec: wdw=[rank() ORDER BY [in_store.return_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "rank() ORDER BY [in_store.return_ratio ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] + │ SortPreservingMergeExec: [return_ratio@1 ASC NULLS LAST] + │ [Stage 14] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ RepartitionExec: partitioning=Hash([ws_item_sk@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[ws_item_sk@2 as ws_item_sk], aggr=[sum(coalesce(wr.wr_return_quantity,Int64(0))), sum(coalesce(ws.ws_quantity,Int64(0))), sum(coalesce(wr.wr_return_amt,Int64(0))), sum(coalesce(ws.ws_net_paid,Int64(0)))] + │ ProjectionExec: expr=[CAST(wr_return_amt@4 AS Decimal128(22, 2)) as __common_expr_1, CAST(ws_net_paid@2 AS Decimal128(22, 2)) as __common_expr_2, ws_item_sk@0 as ws_item_sk, ws_quantity@1 as ws_quantity, wr_return_quantity@3 as wr_return_quantity] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ws_sold_date_sk@0)], projection=[ws_item_sk@3, ws_quantity@4, ws_net_paid@5, wr_return_quantity@6, wr_return_amt@7] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=6, input_tasks=2 + │ FilterExec: wr_return_amt@5 > Some(1000000),7,2 + │ ProjectionExec: expr=[ws_sold_date_sk@2 as ws_sold_date_sk, ws_item_sk@3 as ws_item_sk, ws_quantity@4 as ws_quantity, ws_net_paid@5 as ws_net_paid, wr_return_quantity@0 as wr_return_quantity, wr_return_amt@1 as wr_return_amt] + │ HashJoinExec: mode=Partitioned, join_type=Right, on=[(wr_order_number@1, ws_order_number@2), (wr_item_sk@0, ws_item_sk@1)], projection=[wr_return_quantity@2, wr_return_amt@3, ws_sold_date_sk@4, ws_item_sk@5, ws_quantity@7, ws_net_paid@8] + │ [Stage 2] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=3, consumer_tasks=2, output_partitions=6 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_year@1 = 2001 AND d_moy@2 = 12, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 12, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5, required_guarantees=[d_moy in (12), d_year in (2001)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ RepartitionExec: partitioning=Hash([wr_order_number@1, wr_item_sk@0], 6), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet]]}, projection=[wr_item_sk, wr_order_number, wr_return_quantity, wr_return_amt], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([ws_order_number@2, ws_item_sk@1], 6), input_partitions=2 + │ FilterExec: ws_net_profit@5 > Some(100),7,2 AND ws_net_paid@4 > Some(0),7,2 AND ws_quantity@3 > 0, projection=[ws_sold_date_sk@0, ws_item_sk@1, ws_order_number@2, ws_quantity@3, ws_net_paid@4] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_item_sk, ws_order_number, ws_quantity, ws_net_paid, ws_net_profit], file_type=parquet, predicate=ws_net_profit@33 > Some(100),7,2 AND ws_net_paid@29 > Some(0),7,2 AND ws_quantity@18 > 0, pruning_predicate=ws_net_profit_null_count@1 != row_count@2 AND ws_net_profit_max@0 > Some(100),7,2 AND ws_net_paid_null_count@4 != row_count@2 AND ws_net_paid_max@3 > Some(0),7,2 AND ws_quantity_null_count@6 != row_count@2 AND ws_quantity_max@5 > 0, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 9 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ SortExec: expr=[return_ratio@1 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[cs_item_sk@0 as item, CAST(sum(coalesce(cr.cr_return_quantity,Int64(0)))@1 AS Decimal128(15, 4)) / CAST(sum(coalesce(cs.cs_quantity,Int64(0)))@2 AS Decimal128(15, 4)) as return_ratio, CAST(sum(coalesce(cr.cr_return_amount,Int64(0)))@3 AS Decimal128(15, 4)) / CAST(sum(coalesce(cs.cs_net_paid,Int64(0)))@4 AS Decimal128(15, 4)) as currency_ratio] + │ AggregateExec: mode=FinalPartitioned, gby=[cs_item_sk@0 as cs_item_sk], aggr=[sum(coalesce(cr.cr_return_quantity,Int64(0))), sum(coalesce(cs.cs_quantity,Int64(0))), sum(coalesce(cr.cr_return_amount,Int64(0))), sum(coalesce(cs.cs_net_paid,Int64(0)))] + │ [Stage 8] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 8 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([cs_item_sk@0], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[cs_item_sk@2 as cs_item_sk], aggr=[sum(coalesce(cr.cr_return_quantity,Int64(0))), sum(coalesce(cs.cs_quantity,Int64(0))), sum(coalesce(cr.cr_return_amount,Int64(0))), sum(coalesce(cs.cs_net_paid,Int64(0)))] + │ ProjectionExec: expr=[CAST(cr_return_amount@4 AS Decimal128(22, 2)) as __common_expr_3, CAST(cs_net_paid@2 AS Decimal128(22, 2)) as __common_expr_4, cs_item_sk@0 as cs_item_sk, cs_quantity@1 as cs_quantity, cr_return_quantity@3 as cr_return_quantity] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, cs_sold_date_sk@0)], projection=[cs_item_sk@3, cs_quantity@4, cs_net_paid@5, cr_return_quantity@6, cr_return_amount@7] + │ CoalescePartitionsExec + │ [Stage 5] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ FilterExec: cr_return_amount@5 > Some(1000000),7,2 + │ ProjectionExec: expr=[cs_sold_date_sk@2 as cs_sold_date_sk, cs_item_sk@3 as cs_item_sk, cs_quantity@4 as cs_quantity, cs_net_paid@5 as cs_net_paid, cr_return_quantity@0 as cr_return_quantity, cr_return_amount@1 as cr_return_amount] + │ HashJoinExec: mode=Partitioned, join_type=Right, on=[(cr_order_number@1, cs_order_number@2), (cr_item_sk@0, cs_item_sk@1)], projection=[cr_return_quantity@2, cr_return_amount@3, cs_sold_date_sk@4, cs_item_sk@5, cs_quantity@7, cs_net_paid@8] + │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_year@1 = 2001 AND d_moy@2 = 12, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 12, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5, required_guarantees=[d_moy in (12), d_year in (2001)] + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] + │ RepartitionExec: partitioning=Hash([cr_order_number@1, cr_item_sk@0], 9), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], ...]}, projection=[cr_item_sk, cr_order_number, cr_return_quantity, cr_return_amount], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] + │ RepartitionExec: partitioning=Hash([cs_order_number@2, cs_item_sk@1], 9), input_partitions=2 + │ FilterExec: cs_net_profit@5 > Some(100),7,2 AND cs_net_paid@4 > Some(0),7,2 AND cs_quantity@3 > 0, projection=[cs_sold_date_sk@0, cs_item_sk@1, cs_order_number@2, cs_quantity@3, cs_net_paid@4] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_item_sk, cs_order_number, cs_quantity, cs_net_paid, cs_net_profit], file_type=parquet, predicate=cs_net_profit@33 > Some(100),7,2 AND cs_net_paid@29 > Some(0),7,2 AND cs_quantity@18 > 0, pruning_predicate=cs_net_profit_null_count@1 != row_count@2 AND cs_net_profit_max@0 > Some(100),7,2 AND cs_net_paid_null_count@4 != row_count@2 AND cs_net_paid_max@3 > Some(0),7,2 AND cs_quantity_null_count@6 != row_count@2 AND cs_quantity_max@5 > 0, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 14 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ SortExec: expr=[return_ratio@1 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[ss_item_sk@0 as item, CAST(sum(coalesce(sr.sr_return_quantity,Int64(0)))@1 AS Decimal128(15, 4)) / CAST(sum(coalesce(sts.ss_quantity,Int64(0)))@2 AS Decimal128(15, 4)) as return_ratio, CAST(sum(coalesce(sr.sr_return_amt,Int64(0)))@3 AS Decimal128(15, 4)) / CAST(sum(coalesce(sts.ss_net_paid,Int64(0)))@4 AS Decimal128(15, 4)) as currency_ratio] + │ AggregateExec: mode=FinalPartitioned, gby=[ss_item_sk@0 as ss_item_sk], aggr=[sum(coalesce(sr.sr_return_quantity,Int64(0))), sum(coalesce(sts.ss_quantity,Int64(0))), sum(coalesce(sr.sr_return_amt,Int64(0))), sum(coalesce(sts.ss_net_paid,Int64(0)))] + │ [Stage 13] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 13 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([ss_item_sk@0], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[ss_item_sk@2 as ss_item_sk], aggr=[sum(coalesce(sr.sr_return_quantity,Int64(0))), sum(coalesce(sts.ss_quantity,Int64(0))), sum(coalesce(sr.sr_return_amt,Int64(0))), sum(coalesce(sts.ss_net_paid,Int64(0)))] + │ ProjectionExec: expr=[CAST(sr_return_amt@4 AS Decimal128(22, 2)) as __common_expr_5, CAST(ss_net_paid@2 AS Decimal128(22, 2)) as __common_expr_6, ss_item_sk@0 as ss_item_sk, ss_quantity@1 as ss_quantity, sr_return_quantity@3 as sr_return_quantity] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_item_sk@3, ss_quantity@4, ss_net_paid@5, sr_return_quantity@6, sr_return_amt@7] + │ CoalescePartitionsExec + │ [Stage 10] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ FilterExec: sr_return_amt@5 > Some(1000000),7,2 + │ ProjectionExec: expr=[ss_sold_date_sk@2 as ss_sold_date_sk, ss_item_sk@3 as ss_item_sk, ss_quantity@4 as ss_quantity, ss_net_paid@5 as ss_net_paid, sr_return_quantity@0 as sr_return_quantity, sr_return_amt@1 as sr_return_amt] + │ HashJoinExec: mode=Partitioned, join_type=Right, on=[(sr_ticket_number@1, ss_ticket_number@2), (sr_item_sk@0, ss_item_sk@1)], projection=[sr_return_quantity@2, sr_return_amt@3, ss_sold_date_sk@4, ss_item_sk@5, ss_quantity@7, ss_net_paid@8] + │ [Stage 11] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + │ [Stage 12] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 10 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_year@1 = 2001 AND d_moy@2 = 12, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 12, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5, required_guarantees=[d_moy in (12), d_year in (2001)] + └────────────────────────────────────────────────── + ┌───── Stage 11 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] + │ RepartitionExec: partitioning=Hash([sr_ticket_number@1, sr_item_sk@0], 9), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], ...]}, projection=[sr_item_sk, sr_ticket_number, sr_return_quantity, sr_return_amt], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 12 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] + │ RepartitionExec: partitioning=Hash([ss_ticket_number@2, ss_item_sk@1], 9), input_partitions=2 + │ FilterExec: ss_net_profit@5 > Some(100),6,2 AND ss_net_paid@4 > Some(0),7,2 AND ss_quantity@3 > 0, projection=[ss_sold_date_sk@0, ss_item_sk@1, ss_ticket_number@2, ss_quantity@3, ss_net_paid@4] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ticket_number, ss_quantity, ss_net_paid, ss_net_profit], file_type=parquet, predicate=ss_net_profit@22 > Some(100),6,2 AND ss_net_paid@20 > Some(0),7,2 AND ss_quantity@10 > 0, pruning_predicate=ss_net_profit_null_count@1 != row_count@2 AND ss_net_profit_max@0 > Some(100),6,2 AND ss_net_paid_null_count@4 != row_count@2 AND ss_net_paid_max@3 > Some(0),7,2 AND ss_quantity_null_count@6 != row_count@2 AND ss_quantity_max@5 > 0, required_guarantees=[] + └────────────────────────────────────────────────── + "#); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_50() -> Result<()> { + let display = test_tpcds_query("q50").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [s_store_name@0 ASC NULLS LAST, s_company_id@1 ASC NULLS LAST, s_street_number@2 ASC NULLS LAST, s_street_name@3 ASC NULLS LAST, s_street_type@4 ASC NULLS LAST, s_suite_number@5 ASC NULLS LAST, s_city@6 ASC NULLS LAST, s_county@7 ASC NULLS LAST, s_state@8 ASC NULLS LAST, s_zip@9 ASC NULLS LAST], fetch=100 + │ [Stage 7] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ SortExec: TopK(fetch=100), expr=[s_store_name@0 ASC NULLS LAST, s_company_id@1 ASC NULLS LAST, s_street_number@2 ASC NULLS LAST, s_street_name@3 ASC NULLS LAST, s_street_type@4 ASC NULLS LAST, s_suite_number@5 ASC NULLS LAST, s_city@6 ASC NULLS LAST, s_county@7 ASC NULLS LAST, s_state@8 ASC NULLS LAST, s_zip@9 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[s_store_name@0 as s_store_name, s_company_id@1 as s_company_id, s_street_number@2 as s_street_number, s_street_name@3 as s_street_name, s_street_type@4 as s_street_type, s_suite_number@5 as s_suite_number, s_city@6 as s_city, s_county@7 as s_county, s_state@8 as s_state, s_zip@9 as s_zip, sum(CASE WHEN store_returns.sr_returned_date_sk - store_sales.ss_sold_date_sk <= Int64(30) THEN Int64(1) ELSE Int64(0) END)@10 as 30 days, sum(CASE WHEN store_returns.sr_returned_date_sk - store_sales.ss_sold_date_sk > Int64(30) AND store_returns.sr_returned_date_sk - store_sales.ss_sold_date_sk <= Int64(60) THEN Int64(1) ELSE Int64(0) END)@11 as 31-60 days, sum(CASE WHEN store_returns.sr_returned_date_sk - store_sales.ss_sold_date_sk > Int64(60) AND store_returns.sr_returned_date_sk - store_sales.ss_sold_date_sk <= Int64(90) THEN Int64(1) ELSE Int64(0) END)@12 as 61-90 days, sum(CASE WHEN store_returns.sr_returned_date_sk - store_sales.ss_sold_date_sk > Int64(90) AND store_returns.sr_returned_date_sk - store_sales.ss_sold_date_sk <= Int64(120) THEN Int64(1) ELSE Int64(0) END)@13 as 91-120 days, sum(CASE WHEN store_returns.sr_returned_date_sk - store_sales.ss_sold_date_sk > Int64(120) THEN Int64(1) ELSE Int64(0) END)@14 as >120 days] + │ AggregateExec: mode=FinalPartitioned, gby=[s_store_name@0 as s_store_name, s_company_id@1 as s_company_id, s_street_number@2 as s_street_number, s_street_name@3 as s_street_name, s_street_type@4 as s_street_type, s_suite_number@5 as s_suite_number, s_city@6 as s_city, s_county@7 as s_county, s_state@8 as s_state, s_zip@9 as s_zip], aggr=[sum(CASE WHEN store_returns.sr_returned_date_sk - store_sales.ss_sold_date_sk <= Int64(30) THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN store_returns.sr_returned_date_sk - store_sales.ss_sold_date_sk > Int64(30) AND store_returns.sr_returned_date_sk - store_sales.ss_sold_date_sk <= Int64(60) THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN store_returns.sr_returned_date_sk - store_sales.ss_sold_date_sk > Int64(60) AND store_returns.sr_returned_date_sk - store_sales.ss_sold_date_sk <= Int64(90) THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN store_returns.sr_returned_date_sk - store_sales.ss_sold_date_sk > Int64(90) AND store_returns.sr_returned_date_sk - store_sales.ss_sold_date_sk <= Int64(120) THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN store_returns.sr_returned_date_sk - store_sales.ss_sold_date_sk > Int64(120) THEN Int64(1) ELSE Int64(0) END)] + │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([s_store_name@0, s_company_id@1, s_street_number@2, s_street_name@3, s_street_type@4, s_suite_number@5, s_city@6, s_county@7, s_state@8, s_zip@9], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[s_store_name@1 as s_store_name, s_company_id@2 as s_company_id, s_street_number@3 as s_street_number, s_street_name@4 as s_street_name, s_street_type@5 as s_street_type, s_suite_number@6 as s_suite_number, s_city@7 as s_city, s_county@8 as s_county, s_state@9 as s_state, s_zip@10 as s_zip], aggr=[sum(CASE WHEN store_returns.sr_returned_date_sk - store_sales.ss_sold_date_sk <= Int64(30) THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN store_returns.sr_returned_date_sk - store_sales.ss_sold_date_sk > Int64(30) AND store_returns.sr_returned_date_sk - store_sales.ss_sold_date_sk <= Int64(60) THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN store_returns.sr_returned_date_sk - store_sales.ss_sold_date_sk > Int64(60) AND store_returns.sr_returned_date_sk - store_sales.ss_sold_date_sk <= Int64(90) THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN store_returns.sr_returned_date_sk - store_sales.ss_sold_date_sk > Int64(90) AND store_returns.sr_returned_date_sk - store_sales.ss_sold_date_sk <= Int64(120) THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN store_returns.sr_returned_date_sk - store_sales.ss_sold_date_sk > Int64(120) THEN Int64(1) ELSE Int64(0) END)] + │ ProjectionExec: expr=[sr_returned_date_sk@1 - ss_sold_date_sk@0 as __common_expr_1, s_store_name@2 as s_store_name, s_company_id@3 as s_company_id, s_street_number@4 as s_street_number, s_street_name@5 as s_street_name, s_street_type@6 as s_street_type, s_suite_number@7 as s_suite_number, s_city@8 as s_city, s_county@9 as s_county, s_state@10 as s_state, s_zip@11 as s_zip] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(sr_returned_date_sk@1, CAST(d2.d_date_sk AS Float64)@1)], projection=[ss_sold_date_sk@0, sr_returned_date_sk@1, s_store_name@2, s_company_id@3, s_street_number@4, s_street_name@5, s_street_type@6, s_suite_number@7, s_city@8, s_county@9, s_state@10, s_zip@11] + │ CoalescePartitionsExec + │ [Stage 5] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=3 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(d2.d_date_sk AS Float64)] + │ FilterExec: d_year@1 = 2001 AND d_moy@2 = 8, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 8, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 8 AND 8 <= d_moy_max@5, required_guarantees=[d_moy in (8), d_year in (2001)] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p8] t1:[p9..p17] t2:[p18..p26] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_sold_date_sk@0, CAST(d1.d_date_sk AS Float64)@1)], projection=[ss_sold_date_sk@0, sr_returned_date_sk@1, s_store_name@2, s_company_id@3, s_street_number@4, s_street_name@5, s_street_type@6, s_suite_number@7, s_city@8, s_county@9, s_state@10, s_zip@11] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=3 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(d1.d_date_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p8] t1:[p9..p17] t2:[p18..p26] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[ss_sold_date_sk@10 as ss_sold_date_sk, sr_returned_date_sk@11 as sr_returned_date_sk, s_store_name@0 as s_store_name, s_company_id@1 as s_company_id, s_street_number@2 as s_street_number, s_street_name@3 as s_street_name, s_street_type@4 as s_street_type, s_suite_number@5 as s_suite_number, s_city@6 as s_city, s_county@7 as s_county, s_state@8 as s_state, s_zip@9 as s_zip] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@11, ss_store_sk@1)], projection=[s_store_name@1, s_company_id@2, s_street_number@3, s_street_name@4, s_street_type@5, s_suite_number@6, s_city@7, s_county@8, s_state@9, s_zip@10, ss_sold_date_sk@12, sr_returned_date_sk@14] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[ss_sold_date_sk@1 as ss_sold_date_sk, ss_store_sk@2 as ss_store_sk, sr_returned_date_sk@0 as sr_returned_date_sk] + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(sr_ticket_number@3, ss_ticket_number@4), (sr_item_sk@1, ss_item_sk@1), (sr_customer_sk@2, ss_customer_sk@2)], projection=[sr_returned_date_sk@0, ss_sold_date_sk@4, ss_store_sk@7] + │ [Stage 2] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_store_name, s_company_id, s_street_number, s_street_name, s_street_type, s_suite_number, s_city, s_county, s_state, s_zip, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] + │ RepartitionExec: partitioning=Hash([sr_ticket_number@3, sr_item_sk@1, sr_customer_sk@2], 9), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], ...]}, projection=[sr_returned_date_sk, sr_item_sk, sr_customer_sk, sr_ticket_number], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] + │ RepartitionExec: partitioning=Hash([ss_ticket_number@4, ss_item_sk@1, ss_customer_sk@2], 9), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_store_sk, ss_ticket_number], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_51() -> Result<()> { + let display = test_tpcds_query("q51").await?; + assert_snapshot!(display, @r#" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [item_sk@0 ASC, d_date@1 ASC], fetch=100 + │ SortExec: TopK(fetch=100), expr=[item_sk@0 ASC, d_date@1 ASC], preserve_partitioning=[true] + │ ProjectionExec: expr=[item_sk@0 as item_sk, d_date@1 as d_date, web_sales@2 as web_sales, store_sales@3 as store_sales, max(x.web_sales) PARTITION BY [x.item_sk] ORDER BY [x.d_date ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@4 as web_cumulative, max(x.store_sales) PARTITION BY [x.item_sk] ORDER BY [x.d_date ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@5 as store_cumulative] + │ FilterExec: max(x.web_sales) PARTITION BY [x.item_sk] ORDER BY [x.d_date ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@4 > max(x.store_sales) PARTITION BY [x.item_sk] ORDER BY [x.d_date ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@5 + │ BoundedWindowAggExec: wdw=[max(x.web_sales) PARTITION BY [x.item_sk] ORDER BY [x.d_date ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "max(x.web_sales) PARTITION BY [x.item_sk] ORDER BY [x.d_date ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": nullable Decimal128(25, 2) }, frame: ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW, max(x.store_sales) PARTITION BY [x.item_sk] ORDER BY [x.d_date ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "max(x.store_sales) PARTITION BY [x.item_sk] ORDER BY [x.d_date ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": nullable Decimal128(25, 2) }, frame: ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] + │ SortExec: expr=[item_sk@0 ASC NULLS LAST, d_date@1 ASC NULLS LAST], preserve_partitioning=[true] + │ RepartitionExec: partitioning=Hash([item_sk@0], 3), input_partitions=3 + │ ProjectionExec: expr=[CASE WHEN item_sk@0 IS NOT NULL THEN item_sk@0 ELSE item_sk@3 END as item_sk, CASE WHEN d_date@1 IS NOT NULL THEN d_date@1 ELSE d_date@4 END as d_date, cume_sales@2 as web_sales, cume_sales@5 as store_sales] + │ HashJoinExec: mode=CollectLeft, join_type=Full, on=[(item_sk@0, item_sk@0), (d_date@1, d_date@1)] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[ws_item_sk@0 as item_sk, d_date@1 as d_date, sum(sum(web_sales.ws_sales_price)) PARTITION BY [web_sales.ws_item_sk] ORDER BY [date_dim.d_date ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@3 as cume_sales] + │ BoundedWindowAggExec: wdw=[sum(sum(web_sales.ws_sales_price)) PARTITION BY [web_sales.ws_item_sk] ORDER BY [date_dim.d_date ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "sum(sum(web_sales.ws_sales_price)) PARTITION BY [web_sales.ws_item_sk] ORDER BY [date_dim.d_date ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": nullable Decimal128(25, 2) }, frame: ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] + │ SortExec: expr=[ws_item_sk@0 ASC NULLS LAST, d_date@1 ASC NULLS LAST], preserve_partitioning=[true] + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ ProjectionExec: expr=[ss_item_sk@0 as item_sk, d_date@1 as d_date, sum(sum(store_sales.ss_sales_price)) PARTITION BY [store_sales.ss_item_sk] ORDER BY [date_dim.d_date ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@3 as cume_sales] + │ BoundedWindowAggExec: wdw=[sum(sum(store_sales.ss_sales_price)) PARTITION BY [store_sales.ss_item_sk] ORDER BY [date_dim.d_date ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "sum(sum(store_sales.ss_sales_price)) PARTITION BY [store_sales.ss_item_sk] ORDER BY [date_dim.d_date ASC NULLS LAST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": nullable Decimal128(25, 2) }, frame: ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] + │ SortExec: expr=[ss_item_sk@0 ASC NULLS LAST, d_date@1 ASC NULLS LAST], preserve_partitioning=[true] + │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ RepartitionExec: partitioning=Hash([ws_item_sk@0], 3), input_partitions=3 + │ AggregateExec: mode=FinalPartitioned, gby=[ws_item_sk@0 as ws_item_sk, d_date@1 as d_date], aggr=[sum(web_sales.ws_sales_price)] + │ [Stage 2] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([ws_item_sk@0, d_date@1], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[ws_item_sk@0 as ws_item_sk, d_date@2 as d_date], aggr=[sum(web_sales.ws_sales_price)] + │ ProjectionExec: expr=[ws_item_sk@1 as ws_item_sk, ws_sales_price@2 as ws_sales_price, d_date@0 as d_date] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, ws_sold_date_sk@0)], projection=[d_date@1, ws_item_sk@4, ws_sales_price@5] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ FilterExec: ws_item_sk@1 IS NOT NULL + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_item_sk, ws_sales_price], file_type=parquet, predicate=ws_item_sk@3 IS NOT NULL AND DynamicFilter [ empty ], pruning_predicate=ws_item_sk_null_count@1 != row_count@0, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_date@1 as d_date, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_month_seq@2 >= 1200 AND d_month_seq@2 <= 1211, projection=[d_date_sk@0, d_date@1] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ RepartitionExec: partitioning=Hash([ss_item_sk@0], 3), input_partitions=3 + │ AggregateExec: mode=FinalPartitioned, gby=[ss_item_sk@0 as ss_item_sk, d_date@1 as d_date], aggr=[sum(store_sales.ss_sales_price)] + │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([ss_item_sk@0, d_date@1], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[ss_item_sk@0 as ss_item_sk, d_date@2 as d_date], aggr=[sum(store_sales.ss_sales_price)] + │ ProjectionExec: expr=[ss_item_sk@1 as ss_item_sk, ss_sales_price@2 as ss_sales_price, d_date@0 as d_date] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, ss_sold_date_sk@0)], projection=[d_date@1, ss_item_sk@4, ss_sales_price@5] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ FilterExec: ss_item_sk@1 IS NOT NULL + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_sales_price], file_type=parquet, predicate=ss_item_sk@2 IS NOT NULL AND DynamicFilter [ empty ], pruning_predicate=ss_item_sk_null_count@1 != row_count@0, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_date@1 as d_date, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_month_seq@2 >= 1200 AND d_month_seq@2 <= 1211, projection=[d_date_sk@0, d_date@1] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] + └────────────────────────────────────────────────── + "#); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_52() -> Result<()> { + let display = test_tpcds_query("q52").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [d_year@0 ASC NULLS LAST, ext_price@3 DESC, brand_id@1 ASC NULLS LAST], fetch=100 + │ [Stage 4] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ SortExec: TopK(fetch=100), expr=[ext_price@3 DESC, brand_id@1 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[d_year@0 as d_year, i_brand_id@2 as brand_id, i_brand@1 as brand, sum(store_sales.ss_ext_sales_price)@3 as ext_price] + │ AggregateExec: mode=FinalPartitioned, gby=[d_year@0 as d_year, i_brand@1 as i_brand, i_brand_id@2 as i_brand_id], aggr=[sum(store_sales.ss_ext_sales_price)], ordering_mode=PartiallySorted([0]) + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([d_year@0, i_brand@1, i_brand_id@2], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[d_year@0 as d_year, i_brand@3 as i_brand, i_brand_id@2 as i_brand_id], aggr=[sum(store_sales.ss_ext_sales_price)], ordering_mode=PartiallySorted([0]) + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_item_sk@1, i_item_sk@0)], projection=[d_year@0, ss_ext_sales_price@2, i_brand_id@4, i_brand@5] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=3 + │ FilterExec: i_manager_id@3 = 1, projection=[i_item_sk@0, i_brand_id@1, i_brand@2] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_brand, i_manager_id], file_type=parquet, predicate=i_manager_id@20 = 1 AND DynamicFilter [ empty ], pruning_predicate=i_manager_id_null_count@2 != row_count@3 AND i_manager_id_min@0 <= 1 AND 1 <= i_manager_id_max@1, required_guarantees=[i_manager_id in (1)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p5] t1:[p6..p11] t2:[p12..p17] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(dt.d_date_sk AS Float64)@2, ss_sold_date_sk@0)], projection=[d_year@1, ss_item_sk@4, ss_ext_sales_price@5] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, CAST(d_date_sk@0 AS Float64) as CAST(dt.d_date_sk AS Float64)] + │ FilterExec: d_moy@2 = 11 AND d_year@1 = 2000, projection=[d_date_sk@0, d_year@1] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 11 AND d_year@6 = 2000, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5, required_guarantees=[d_moy in (11), d_year in (2000)] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_53() -> Result<()> { + let display = test_tpcds_query("q53").await?; + assert_snapshot!(display, @r#" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [avg_quarterly_sales@2 ASC NULLS LAST, sum_sales@1 ASC NULLS LAST, i_manufact_id@0 ASC NULLS LAST], fetch=100 + │ SortExec: TopK(fetch=100), expr=[avg_quarterly_sales@2 ASC NULLS LAST, sum_sales@1 ASC NULLS LAST, i_manufact_id@0 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[i_manufact_id@0 as i_manufact_id, sum(store_sales.ss_sales_price)@1 as sum_sales, avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_manufact_id] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@2 as avg_quarterly_sales] + │ FilterExec: CASE WHEN avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_manufact_id] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@2 > Some(0),19,6 THEN abs(sum(store_sales.ss_sales_price)@1 - avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_manufact_id] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@2) / avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_manufact_id] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@2 END > Some(1000000000),30,10 + │ WindowAggExec: wdw=[avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_manufact_id] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING: Ok(Field { name: "avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_manufact_id] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING", data_type: Decimal128(19, 6), nullable: true }), frame: WindowFrame { units: Rows, start_bound: Preceding(UInt64(NULL)), end_bound: Following(UInt64(NULL)), is_causal: false }] + │ SortExec: expr=[i_manufact_id@0 ASC NULLS LAST], preserve_partitioning=[true] + │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ RepartitionExec: partitioning=Hash([i_manufact_id@0], 3), input_partitions=3 + │ ProjectionExec: expr=[i_manufact_id@0 as i_manufact_id, sum(store_sales.ss_sales_price)@2 as sum(store_sales.ss_sales_price)] + │ AggregateExec: mode=FinalPartitioned, gby=[i_manufact_id@0 as i_manufact_id, d_qoy@1 as d_qoy], aggr=[sum(store_sales.ss_sales_price)] + │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([i_manufact_id@0, d_qoy@1], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[i_manufact_id@0 as i_manufact_id, d_qoy@2 as d_qoy], aggr=[sum(store_sales.ss_sales_price)] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@1, ss_store_sk@1)], projection=[i_manufact_id@2, ss_sales_price@4, d_qoy@5] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[i_manufact_id@1 as i_manufact_id, ss_store_sk@2 as ss_store_sk, ss_sales_price@3 as ss_sales_price, d_qoy@0 as d_qoy] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, ss_sold_date_sk@1)], projection=[d_qoy@1, i_manufact_id@3, ss_store_sk@5, ss_sales_price@6] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ss_item_sk@1)], projection=[i_manufact_id@1, ss_sold_date_sk@2, ss_store_sk@4, ss_sales_price@5] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_qoy@1 as d_qoy, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_month_seq@1 IN (SET) ([1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211]), projection=[d_date_sk@0, d_qoy@2] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_month_seq, d_qoy], file_type=parquet, predicate=d_month_seq@3 IN (SET) ([1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211]), pruning_predicate=d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1200 AND 1200 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1201 AND 1201 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1202 AND 1202 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1203 AND 1203 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1204 AND 1204 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1205 AND 1205 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1206 AND 1206 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1207 AND 1207 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1208 AND 1208 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1209 AND 1209 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1210 AND 1210 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1211 AND 1211 <= d_month_seq_max@1, required_guarantees=[d_month_seq in (1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211)] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ FilterExec: (i_category@3 = Books OR i_category@3 = Children OR i_category@3 = Electronics) AND i_class@2 IN (SET) ([personal, portable, reference, self-help]) AND i_brand@1 IN (SET) ([scholaramalgamalg #14, scholaramalgamalg #7, exportiunivamalg #9, scholaramalgamalg #9]) OR (i_category@3 = Women OR i_category@3 = Music OR i_category@3 = Men) AND i_class@2 IN (SET) ([accessories, classical, fragrances, pants]) AND i_brand@1 IN (SET) ([amalgimporto #1, edu packscholar #1, exportiimporto #1, importoamalg #1]), projection=[i_item_sk@0, i_manufact_id@4] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand, i_class, i_category, i_manufact_id], file_type=parquet, predicate=(i_category@12 = Books OR i_category@12 = Children OR i_category@12 = Electronics) AND i_class@10 IN (SET) ([personal, portable, reference, self-help]) AND i_brand@8 IN (SET) ([scholaramalgamalg #14, scholaramalgamalg #7, exportiunivamalg #9, scholaramalgamalg #9]) OR (i_category@12 = Women OR i_category@12 = Music OR i_category@12 = Men) AND i_class@10 IN (SET) ([accessories, classical, fragrances, pants]) AND i_brand@8 IN (SET) ([amalgimporto #1, edu packscholar #1, exportiimporto #1, importoamalg #1]), pruning_predicate=(i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Children AND Children <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Electronics AND Electronics <= i_category_max@1) AND (i_class_null_count@6 != row_count@3 AND i_class_min@4 <= personal AND personal <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= portable AND portable <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= reference AND reference <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= self-help AND self-help <= i_class_max@5) AND (i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= scholaramalgamalg #14 AND scholaramalgamalg #14 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= scholaramalgamalg #7 AND scholaramalgamalg #7 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= exportiunivamalg #9 AND exportiunivamalg #9 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= scholaramalgamalg #9 AND scholaramalgamalg #9 <= i_brand_max@8) OR (i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Women AND Women <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Music AND Music <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Men AND Men <= i_category_max@1) AND (i_class_null_count@6 != row_count@3 AND i_class_min@4 <= accessories AND accessories <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= classical AND classical <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= fragrances AND fragrances <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= pants AND pants <= i_class_max@5) AND (i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= amalgimporto #1 AND amalgimporto #1 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= edu packscholar #1 AND edu packscholar #1 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= exportiimporto #1 AND exportiimporto #1 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= importoamalg #1 AND importoamalg #1 <= i_brand_max@8), required_guarantees=[i_brand in (amalgimporto #1, edu packscholar #1, exportiimporto #1, exportiunivamalg #9, importoamalg #1, scholaramalgamalg #14, scholaramalgamalg #7, scholaramalgamalg #9), i_class in (accessories, classical, fragrances, pants, personal, portable, reference, self-help)] + └────────────────────────────────────────────────── + "#); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_54() -> Result<()> { + let display = test_tpcds_query("q54").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [segment@0 ASC, num_customers@1 ASC, segment_base@2 ASC NULLS LAST], fetch=100 + │ SortExec: TopK(fetch=100), expr=[segment@0 ASC, num_customers@1 ASC, segment_base@2 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[segment@0 as segment, count(Int64(1))@1 as num_customers, CAST(segment@0 AS Int64) * 50 as segment_base] + │ AggregateExec: mode=FinalPartitioned, gby=[segment@0 as segment], aggr=[count(Int64(1))] + │ RepartitionExec: partitioning=Hash([segment@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[segment@0 as segment], aggr=[count(Int64(1))] + │ ProjectionExec: expr=[CAST(round(sum(store_sales.ss_ext_sales_price)@1 / Some(50),20,0) AS Int32) as segment] + │ AggregateExec: mode=FinalPartitioned, gby=[c_customer_sk@0 as c_customer_sk], aggr=[sum(store_sales.ss_ext_sales_price)] + │ RepartitionExec: partitioning=Hash([c_customer_sk@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[c_customer_sk@0 as c_customer_sk], aggr=[sum(store_sales.ss_ext_sales_price)] + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, ss_ext_sales_price@1 as ss_ext_sales_price] + │ NestedLoopJoinExec: join_type=Inner, filter=join_proj_push_down_2@1 <= date_dim.d_month_seq + Int64(3)@0, projection=[c_customer_sk@0, ss_ext_sales_price@1, d_month_seq@2, date_dim.d_month_seq + Int64(3)@4] + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, ss_ext_sales_price@1 as ss_ext_sales_price, d_month_seq@2 as d_month_seq, CAST(d_month_seq@2 AS Int64) as join_proj_push_down_2] + │ CoalescePartitionsExec + │ ProjectionExec: expr=[c_customer_sk@1 as c_customer_sk, ss_ext_sales_price@2 as ss_ext_sales_price, d_month_seq@3 as d_month_seq] + │ NestedLoopJoinExec: join_type=Inner, filter=join_proj_push_down_1@1 >= date_dim.d_month_seq + Int64(1)@0, projection=[date_dim.d_month_seq + Int64(1)@0, c_customer_sk@1, ss_ext_sales_price@2, d_month_seq@3] + │ CoalescePartitionsExec + │ AggregateExec: mode=FinalPartitioned, gby=[date_dim.d_month_seq + Int64(1)@0 as date_dim.d_month_seq + Int64(1)], aggr=[] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, ss_ext_sales_price@1 as ss_ext_sales_price, d_month_seq@2 as d_month_seq, CAST(d_month_seq@2 AS Int64) as join_proj_push_down_1] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_sold_date_sk@1, CAST(date_dim.d_date_sk AS Float64)@2)], projection=[c_customer_sk@0, ss_ext_sales_price@2, d_month_seq@4] + │ CoalescePartitionsExec + │ BroadcastExec: input_partitions=3, consumer_tasks=1, output_partitions=3 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_county@0, ca_county@3), (CAST(store.s_state AS Utf8View)@2, ca_state@4)], projection=[c_customer_sk@3, ss_sold_date_sk@4, ss_ext_sales_price@5] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=2, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_current_addr_sk@1, ca_address_sk@0)], projection=[c_customer_sk@0, ss_sold_date_sk@2, ss_ext_sales_price@3, ca_county@5, ca_state@6] + │ CoalescePartitionsExec + │ [Stage 8] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=2, input_tasks=3 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_county, ca_state], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_month_seq@1 as d_month_seq, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_month_seq], file_type=parquet + │ AggregateExec: mode=FinalPartitioned, gby=[date_dim.d_month_seq + Int64(3)@0 as date_dim.d_month_seq + Int64(3)], aggr=[] + │ [Stage 9] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ RepartitionExec: partitioning=Hash([date_dim.d_month_seq + Int64(1)@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[date_dim.d_month_seq + Int64(1)@0 as date_dim.d_month_seq + Int64(1)], aggr=[] + │ ProjectionExec: expr=[CAST(d_month_seq@0 AS Int64) + 1 as date_dim.d_month_seq + Int64(1)] + │ FilterExec: d_year@1 = 1998 AND d_moy@2 = 12, projection=[d_month_seq@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_month_seq, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 12, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5, required_guarantees=[d_moy in (12), d_year in (1998)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p1] t1:[p2..p3] + │ BroadcastExec: input_partitions=2, consumer_tasks=1, output_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_county, s_state, CAST(s_state@24 AS Utf8View) as CAST(store.s_state AS Utf8View)], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 8 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] + │ BroadcastExec: input_partitions=2, consumer_tasks=1, output_partitions=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(my_customers.c_customer_sk AS Float64)@2, ss_customer_sk@1)], projection=[c_customer_sk@0, c_current_addr_sk@1, ss_sold_date_sk@3, ss_ext_sales_price@5] + │ CoalescePartitionsExec + │ [Stage 7] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, c_current_addr_sk@1 as c_current_addr_sk, CAST(c_customer_sk@0 AS Float64) as CAST(my_customers.c_customer_sk AS Float64)] + │ AggregateExec: mode=FinalPartitioned, gby=[c_customer_sk@0 as c_customer_sk, c_current_addr_sk@1 as c_current_addr_sk], aggr=[] + │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] t3:[p0..p5] + │ RepartitionExec: partitioning=Hash([c_customer_sk@0, c_current_addr_sk@1], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[c_customer_sk@0 as c_customer_sk, c_current_addr_sk@1 as c_current_addr_sk], aggr=[] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(customer_sk@0, CAST(customer.c_customer_sk AS Float64)@2)], projection=[c_customer_sk@1, c_current_addr_sk@2] + │ CoalescePartitionsExec + │ [Stage 5] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, c_current_addr_sk@1 as c_current_addr_sk, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=1 + │ PartitionIsolatorExec: t0:[p0,__,__,__] t1:[__,p0,__,__] t2:[__,__,p0,__] t3:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_current_addr_sk], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p11] t1:[p12..p23] t2:[p24..p35] t3:[p36..p47] + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, sold_date_sk@0)], projection=[customer_sk@3] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, item_sk@2)], projection=[sold_date_sk@1, customer_sk@2] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=2 + │ DistributedUnionExec: t0:[c0(0/2)] t1:[c0(1/2)] t2:[c1(0/2)] t3:[c1(1/2)] + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk@0 as sold_date_sk, cs_bill_customer_sk@3 as customer_sk, cs_item_sk@15 as item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk@0 as sold_date_sk, ws_bill_customer_sk@4 as customer_sk, ws_item_sk@3 as item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p11] t1:[p12..p23] + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_moy@2 = 12 AND d_year@1 = 1998, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 12 AND d_year@6 = 1998, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 12 AND 12 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1998 AND 1998 <= d_year_max@5, required_guarantees=[d_moy in (12), d_year in (1998)] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p11] t1:[p12..p23] + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 + │ FilterExec: i_category@2 = Women AND i_class@1 = maternity, projection=[i_item_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_class, i_category], file_type=parquet, predicate=i_category@12 = Women AND i_class@10 = maternity, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Women AND Women <= i_category_max@1 AND i_class_null_count@6 != row_count@3 AND i_class_min@4 <= maternity AND maternity <= i_class_max@5, required_guarantees=[i_category in (Women), i_class in (maternity)] + └────────────────────────────────────────────────── + ┌───── Stage 9 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ RepartitionExec: partitioning=Hash([date_dim.d_month_seq + Int64(3)@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[date_dim.d_month_seq + Int64(3)@0 as date_dim.d_month_seq + Int64(3)], aggr=[] + │ ProjectionExec: expr=[CAST(d_month_seq@0 AS Int64) + 3 as date_dim.d_month_seq + Int64(3)] + │ FilterExec: d_year@1 = 1998 AND d_moy@2 = 12, projection=[d_month_seq@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_month_seq, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 12, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5, required_guarantees=[d_moy in (12), d_year in (1998)] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_55() -> Result<()> { + let display = test_tpcds_query("q55").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [ext_price@2 DESC, brand_id@0 ASC NULLS LAST], fetch=100 + │ [Stage 4] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ SortExec: TopK(fetch=100), expr=[ext_price@2 DESC, brand_id@0 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[i_brand_id@1 as brand_id, i_brand@0 as brand, sum(store_sales.ss_ext_sales_price)@2 as ext_price] + │ AggregateExec: mode=FinalPartitioned, gby=[i_brand@0 as i_brand, i_brand_id@1 as i_brand_id], aggr=[sum(store_sales.ss_ext_sales_price)] + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([i_brand@0, i_brand_id@1], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_brand@2 as i_brand, i_brand_id@1 as i_brand_id], aggr=[sum(store_sales.ss_ext_sales_price)] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_item_sk@0, i_item_sk@0)], projection=[ss_ext_sales_price@1, i_brand_id@3, i_brand@4] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=3 + │ FilterExec: i_manager_id@3 = 28, projection=[i_item_sk@0, i_brand_id@1, i_brand@2] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_brand, i_manager_id], file_type=parquet, predicate=i_manager_id@20 = 28 AND DynamicFilter [ empty ], pruning_predicate=i_manager_id_null_count@2 != row_count@3 AND i_manager_id_min@0 <= 28 AND 28 <= i_manager_id_max@1, required_guarantees=[i_manager_id in (28)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p5] t1:[p6..p11] t2:[p12..p17] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_item_sk@3, ss_ext_sales_price@4] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_moy@2 = 11 AND d_year@1 = 1999, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 11 AND d_year@6 = 1999, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5, required_guarantees=[d_moy in (11), d_year in (1999)] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_56() -> Result<()> { + let display = test_tpcds_query("q56").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [total_sales@1 ASC, i_item_id@0 ASC], fetch=100 + │ SortExec: TopK(fetch=100), expr=[total_sales@1 ASC, i_item_id@0 ASC], preserve_partitioning=[true] + │ ProjectionExec: expr=[i_item_id@0 as i_item_id, sum(tmp1.total_sales)@1 as total_sales] + │ AggregateExec: mode=SinglePartitioned, gby=[i_item_id@0 as i_item_id], aggr=[sum(tmp1.total_sales)] + │ InterleaveExec + │ ProjectionExec: expr=[i_item_id@0 as i_item_id, sum(store_sales.ss_ext_sales_price)@1 as total_sales] + │ AggregateExec: mode=FinalPartitioned, gby=[i_item_id@0 as i_item_id], aggr=[sum(store_sales.ss_ext_sales_price)] + │ RepartitionExec: partitioning=Hash([i_item_id@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_item_id@1 as i_item_id], aggr=[sum(store_sales.ss_ext_sales_price)] + │ HashJoinExec: mode=CollectLeft, join_type=LeftSemi, on=[(i_item_id@1, i_item_id@0)] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkCoalesceExec: output_partitions=9, input_tasks=3 + │ FilterExec: i_color@1 = slate OR i_color@1 = blanched OR i_color@1 = burnished, projection=[i_item_id@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_id, i_color], file_type=parquet, predicate=i_color@17 = slate OR i_color@17 = blanched OR i_color@17 = burnished, pruning_predicate=i_color_null_count@2 != row_count@3 AND i_color_min@0 <= slate AND slate <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= blanched AND blanched <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= burnished AND burnished <= i_color_max@1, required_guarantees=[i_color in (blanched, burnished, slate)] + │ ProjectionExec: expr=[i_item_id@0 as i_item_id, sum(catalog_sales.cs_ext_sales_price)@1 as total_sales] + │ AggregateExec: mode=FinalPartitioned, gby=[i_item_id@0 as i_item_id], aggr=[sum(catalog_sales.cs_ext_sales_price)] + │ RepartitionExec: partitioning=Hash([i_item_id@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_item_id@1 as i_item_id], aggr=[sum(catalog_sales.cs_ext_sales_price)] + │ HashJoinExec: mode=CollectLeft, join_type=LeftSemi, on=[(i_item_id@1, i_item_id@0)] + │ CoalescePartitionsExec + │ [Stage 8] => NetworkCoalesceExec: output_partitions=9, input_tasks=3 + │ FilterExec: i_color@1 = slate OR i_color@1 = blanched OR i_color@1 = burnished, projection=[i_item_id@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_id, i_color], file_type=parquet, predicate=i_color@17 = slate OR i_color@17 = blanched OR i_color@17 = burnished, pruning_predicate=i_color_null_count@2 != row_count@3 AND i_color_min@0 <= slate AND slate <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= blanched AND blanched <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= burnished AND burnished <= i_color_max@1, required_guarantees=[i_color in (blanched, burnished, slate)] + │ ProjectionExec: expr=[i_item_id@0 as i_item_id, sum(web_sales.ws_ext_sales_price)@1 as total_sales] + │ AggregateExec: mode=FinalPartitioned, gby=[i_item_id@0 as i_item_id], aggr=[sum(web_sales.ws_ext_sales_price)] + │ RepartitionExec: partitioning=Hash([i_item_id@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_item_id@1 as i_item_id], aggr=[sum(web_sales.ws_ext_sales_price)] + │ HashJoinExec: mode=CollectLeft, join_type=LeftSemi, on=[(i_item_id@1, i_item_id@0)] + │ CoalescePartitionsExec + │ [Stage 12] => NetworkCoalesceExec: output_partitions=9, input_tasks=3 + │ FilterExec: i_color@1 = slate OR i_color@1 = blanched OR i_color@1 = burnished, projection=[i_item_id@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_id, i_color], file_type=parquet, predicate=i_color@17 = slate OR i_color@17 = blanched OR i_color@17 = burnished, pruning_predicate=i_color_null_count@2 != row_count@3 AND i_color_min@0 <= slate AND slate <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= blanched AND blanched <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= burnished AND burnished <= i_color_max@1, required_guarantees=[i_color in (blanched, burnished, slate)] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p3..p5] t2:[p6..p8] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_item_sk@0, i_item_sk@0)], projection=[ss_ext_sales_price@1, i_item_id@3] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=3 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p8] t1:[p9..p17] t2:[p18..p26] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_addr_sk@1, CAST(customer_address.ca_address_sk AS Float64)@1)], projection=[ss_item_sk@0, ss_ext_sales_price@2] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=3 + │ ProjectionExec: expr=[ca_address_sk@0 as ca_address_sk, CAST(ca_address_sk@0 AS Float64) as CAST(customer_address.ca_address_sk AS Float64)] + │ FilterExec: ca_gmt_offset@1 = Some(-500),4,2, projection=[ca_address_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = Some(-500),4,2, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= Some(-500),4,2 AND Some(-500),4,2 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (Some(-500),4,2)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p5] t1:[p6..p11] t2:[p12..p17] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_item_sk@3, ss_addr_sk@4, ss_ext_sales_price@5] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_addr_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_year@1 = 2001 AND d_moy@2 = 2, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 2, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 2 AND 2 <= d_moy_max@5, required_guarantees=[d_moy in (2), d_year in (2001)] + └────────────────────────────────────────────────── + ┌───── Stage 8 ── Tasks: t0:[p0..p2] t1:[p3..p5] t2:[p6..p8] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(cs_item_sk@0, i_item_sk@0)], projection=[cs_ext_sales_price@1, i_item_id@3] + │ CoalescePartitionsExec + │ [Stage 7] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=3 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p8] t1:[p9..p17] t2:[p18..p26] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(cs_bill_addr_sk@0, CAST(customer_address.ca_address_sk AS Float64)@1)], projection=[cs_item_sk@1, cs_ext_sales_price@2] + │ CoalescePartitionsExec + │ [Stage 6] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=3 + │ ProjectionExec: expr=[ca_address_sk@0 as ca_address_sk, CAST(ca_address_sk@0 AS Float64) as CAST(customer_address.ca_address_sk AS Float64)] + │ FilterExec: ca_gmt_offset@1 = Some(-500),4,2, projection=[ca_address_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = Some(-500),4,2, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= Some(-500),4,2 AND Some(-500),4,2 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (Some(-500),4,2)] + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p5] t1:[p6..p11] t2:[p12..p17] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, cs_sold_date_sk@0)], projection=[cs_bill_addr_sk@3, cs_item_sk@4, cs_ext_sales_price@5] + │ CoalescePartitionsExec + │ [Stage 5] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_bill_addr_sk, cs_item_sk, cs_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_year@1 = 2001 AND d_moy@2 = 2, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 2, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 2 AND 2 <= d_moy_max@5, required_guarantees=[d_moy in (2), d_year in (2001)] + └────────────────────────────────────────────────── + ┌───── Stage 12 ── Tasks: t0:[p0..p2] t1:[p3..p5] t2:[p6..p8] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ws_item_sk@0, i_item_sk@0)], projection=[ws_ext_sales_price@1, i_item_id@3] + │ CoalescePartitionsExec + │ [Stage 11] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=3 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 11 ── Tasks: t0:[p0..p8] t1:[p9..p17] t2:[p18..p26] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ws_bill_addr_sk@1, CAST(customer_address.ca_address_sk AS Float64)@1)], projection=[ws_item_sk@0, ws_ext_sales_price@2] + │ CoalescePartitionsExec + │ [Stage 10] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=3 + │ ProjectionExec: expr=[ca_address_sk@0 as ca_address_sk, CAST(ca_address_sk@0 AS Float64) as CAST(customer_address.ca_address_sk AS Float64)] + │ FilterExec: ca_gmt_offset@1 = Some(-500),4,2, projection=[ca_address_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = Some(-500),4,2, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= Some(-500),4,2 AND Some(-500),4,2 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (Some(-500),4,2)] + └────────────────────────────────────────────────── + ┌───── Stage 10 ── Tasks: t0:[p0..p5] t1:[p6..p11] t2:[p12..p17] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ws_sold_date_sk@0)], projection=[ws_item_sk@3, ws_bill_addr_sk@4, ws_ext_sales_price@5] + │ CoalescePartitionsExec + │ [Stage 9] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_item_sk, ws_bill_addr_sk, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 9 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_year@1 = 2001 AND d_moy@2 = 2, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 = 2, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 2 AND 2 <= d_moy_max@5, required_guarantees=[d_moy in (2), d_year in (2001)] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_57() -> Result<()> { + let display = test_tpcds_query("q57").await?; + assert_snapshot!(display, @r#" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [sum_sales@6 - avg_monthly_sales@5 ASC, i_category@0 ASC NULLS LAST, i_brand@1 ASC NULLS LAST, cc_name@2 ASC NULLS LAST, d_year@3 ASC NULLS LAST, d_moy@4 ASC NULLS LAST, avg_monthly_sales@5 ASC NULLS LAST, sum_sales@6 ASC NULLS LAST, psum@7 ASC NULLS LAST, nsum@8 ASC NULLS LAST], fetch=100 + │ SortExec: TopK(fetch=100), expr=[sum_sales@6 - avg_monthly_sales@5 ASC, i_category@0 ASC NULLS LAST, i_brand@1 ASC NULLS LAST, cc_name@2 ASC NULLS LAST, d_moy@4 ASC NULLS LAST, avg_monthly_sales@5 ASC NULLS LAST, sum_sales@6 ASC NULLS LAST, psum@7 ASC NULLS LAST, nsum@8 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[i_category@0 as i_category, i_brand@1 as i_brand, cc_name@2 as cc_name, d_year@3 as d_year, d_moy@4 as d_moy, avg_monthly_sales@6 as avg_monthly_sales, sum_sales@5 as sum_sales, sum_sales@7 as psum, sum_sales@8 as nsum] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_category@0, i_category@0), (i_brand@1, i_brand@1), (cc_name@2, cc_name@2), (CAST(v1.rn AS Decimal128(21, 0))@9, v1_lead.rn - Decimal128(Some(1),20,0)@5)], projection=[i_category@0, i_brand@1, cc_name@2, d_year@3, d_moy@4, sum_sales@5, avg_monthly_sales@6, sum_sales@8, sum_sales@13] + │ CoalescePartitionsExec + │ BroadcastExec: input_partitions=3, consumer_tasks=1, output_partitions=3 + │ ProjectionExec: expr=[i_category@0 as i_category, i_brand@1 as i_brand, cc_name@2 as cc_name, d_year@3 as d_year, d_moy@4 as d_moy, sum_sales@5 as sum_sales, avg_monthly_sales@6 as avg_monthly_sales, rn@7 as rn, sum_sales@8 as sum_sales, CAST(rn@7 AS Decimal128(21, 0)) as CAST(v1.rn AS Decimal128(21, 0))] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_category@0, i_category@0), (i_brand@1, i_brand@1), (cc_name@2, cc_name@2), (CAST(v1.rn AS Decimal128(21, 0))@8, v1_lag.rn + Decimal128(Some(1),20,0)@5)], projection=[i_category@0, i_brand@1, cc_name@2, d_year@3, d_moy@4, sum_sales@5, avg_monthly_sales@6, rn@7, sum_sales@12] + │ CoalescePartitionsExec + │ BroadcastExec: input_partitions=3, consumer_tasks=1, output_partitions=3 + │ ProjectionExec: expr=[i_category@0 as i_category, i_brand@1 as i_brand, cc_name@2 as cc_name, d_year@3 as d_year, d_moy@4 as d_moy, sum(catalog_sales.cs_sales_price)@5 as sum_sales, avg(sum(catalog_sales.cs_sales_price)) PARTITION BY [item.i_category, item.i_brand, call_center.cc_name, date_dim.d_year] ORDER BY [date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@6 as avg_monthly_sales, rank() PARTITION BY [item.i_category, item.i_brand, call_center.cc_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@7 as rn, CAST(rank() PARTITION BY [item.i_category, item.i_brand, call_center.cc_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@7 AS Decimal128(21, 0)) as CAST(v1.rn AS Decimal128(21, 0))] + │ FilterExec: d_year@3 = 1999 AND avg(sum(catalog_sales.cs_sales_price)) PARTITION BY [item.i_category, item.i_brand, call_center.cc_name, date_dim.d_year] ORDER BY [date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@6 > Some(0),19,6 AND CASE WHEN avg(sum(catalog_sales.cs_sales_price)) PARTITION BY [item.i_category, item.i_brand, call_center.cc_name, date_dim.d_year] ORDER BY [date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@6 > Some(0),19,6 THEN abs(sum(catalog_sales.cs_sales_price)@5 - avg(sum(catalog_sales.cs_sales_price)) PARTITION BY [item.i_category, item.i_brand, call_center.cc_name, date_dim.d_year] ORDER BY [date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@6) / avg(sum(catalog_sales.cs_sales_price)) PARTITION BY [item.i_category, item.i_brand, call_center.cc_name, date_dim.d_year] ORDER BY [date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@6 END > Some(1000000000),30,10 + │ BoundedWindowAggExec: wdw=[rank() PARTITION BY [item.i_category, item.i_brand, call_center.cc_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "rank() PARTITION BY [item.i_category, item.i_brand, call_center.cc_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] + │ SortExec: expr=[i_category@0 ASC NULLS LAST, i_brand@1 ASC NULLS LAST, cc_name@2 ASC NULLS LAST, d_year@3 ASC NULLS LAST, d_moy@4 ASC NULLS LAST], preserve_partitioning=[true] + │ RepartitionExec: partitioning=Hash([i_category@0, i_brand@1, cc_name@2], 3), input_partitions=3 + │ BoundedWindowAggExec: wdw=[avg(sum(catalog_sales.cs_sales_price)) PARTITION BY [item.i_category, item.i_brand, call_center.cc_name, date_dim.d_year] ORDER BY [date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "avg(sum(catalog_sales.cs_sales_price)) PARTITION BY [item.i_category, item.i_brand, call_center.cc_name, date_dim.d_year] ORDER BY [date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": nullable Decimal128(19, 6) }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] + │ SortExec: expr=[i_category@0 ASC NULLS LAST, i_brand@1 ASC NULLS LAST, cc_name@2 ASC NULLS LAST, d_year@3 ASC NULLS LAST, d_moy@4 ASC NULLS LAST], preserve_partitioning=[true] + │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ ProjectionExec: expr=[i_category@0 as i_category, i_brand@1 as i_brand, cc_name@2 as cc_name, sum(catalog_sales.cs_sales_price)@5 as sum_sales, rank() PARTITION BY [item.i_category, item.i_brand, call_center.cc_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@6 as rn, CAST(rank() PARTITION BY [item.i_category, item.i_brand, call_center.cc_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@6 AS Decimal128(20, 0)) + Some(1),20,0 as v1_lag.rn + Decimal128(Some(1),20,0)] + │ BoundedWindowAggExec: wdw=[rank() PARTITION BY [item.i_category, item.i_brand, call_center.cc_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "rank() PARTITION BY [item.i_category, item.i_brand, call_center.cc_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] + │ SortExec: expr=[i_category@0 ASC NULLS LAST, i_brand@1 ASC NULLS LAST, cc_name@2 ASC NULLS LAST, d_year@3 ASC NULLS LAST, d_moy@4 ASC NULLS LAST], preserve_partitioning=[true] + │ [Stage 10] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ ProjectionExec: expr=[i_category@0 as i_category, i_brand@1 as i_brand, cc_name@2 as cc_name, sum(catalog_sales.cs_sales_price)@5 as sum_sales, rank() PARTITION BY [item.i_category, item.i_brand, call_center.cc_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@6 as rn, CAST(rank() PARTITION BY [item.i_category, item.i_brand, call_center.cc_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@6 AS Decimal128(20, 0)) - Some(1),20,0 as v1_lead.rn - Decimal128(Some(1),20,0)] + │ BoundedWindowAggExec: wdw=[rank() PARTITION BY [item.i_category, item.i_brand, call_center.cc_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "rank() PARTITION BY [item.i_category, item.i_brand, call_center.cc_name] ORDER BY [date_dim.d_year ASC NULLS LAST, date_dim.d_moy ASC NULLS LAST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] + │ SortExec: expr=[i_category@0 ASC NULLS LAST, i_brand@1 ASC NULLS LAST, cc_name@2 ASC NULLS LAST, d_year@3 ASC NULLS LAST, d_moy@4 ASC NULLS LAST], preserve_partitioning=[true] + │ [Stage 15] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ RepartitionExec: partitioning=Hash([i_category@0, i_brand@1, cc_name@2, d_year@3], 3), input_partitions=3 + │ AggregateExec: mode=FinalPartitioned, gby=[i_category@0 as i_category, i_brand@1 as i_brand, cc_name@2 as cc_name, d_year@3 as d_year, d_moy@4 as d_moy], aggr=[sum(catalog_sales.cs_sales_price)] + │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([i_category@0, i_brand@1, cc_name@2, d_year@3, d_moy@4], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[i_category@1 as i_category, i_brand@0 as i_brand, cc_name@5 as cc_name, d_year@3 as d_year, d_moy@4 as d_moy], aggr=[sum(catalog_sales.cs_sales_price)] + │ ProjectionExec: expr=[i_brand@1 as i_brand, i_category@2 as i_category, cs_sales_price@3 as cs_sales_price, d_year@4 as d_year, d_moy@5 as d_moy, cc_name@0 as cc_name] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(call_center.cc_call_center_sk AS Float64)@2, cs_call_center_sk@2)], projection=[cc_name@1, i_brand@3, i_category@4, cs_sales_price@6, d_year@7, d_moy@8] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[i_brand@2 as i_brand, i_category@3 as i_category, cs_call_center_sk@4 as cs_call_center_sk, cs_sales_price@5 as cs_sales_price, d_year@0 as d_year, d_moy@1 as d_moy] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@3, cs_sold_date_sk@2)], projection=[d_year@1, d_moy@2, i_brand@4, i_category@5, cs_call_center_sk@7, cs_sales_price@8] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, cs_item_sk@2)], projection=[i_brand@1, i_category@2, cs_sold_date_sk@3, cs_call_center_sk@4, cs_sales_price@6] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_call_center_sk, cs_item_sk, cs_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/call_center/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/call_center/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/call_center/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/call_center/part-3.parquet]]}, projection=[cc_call_center_sk, cc_name, CAST(cc_call_center_sk@0 AS Float64) as CAST(call_center.cc_call_center_sk AS Float64)], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, d_moy@2 as d_moy, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_year@1 = 1999 OR d_year@1 = 1998 AND d_moy@2 = 12 OR d_year@1 = 2000 AND d_moy@2 = 1 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1999 OR d_year@6 = 1998 AND d_moy@8 = 12 OR d_year@6 = 2000 AND d_moy@8 = 1, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 1 AND 1 <= d_moy_max@5, required_guarantees=[d_year in (1998, 1999, 2000)] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand, i_category], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 10 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ RepartitionExec: partitioning=Hash([i_category@0, i_brand@1, cc_name@2], 3), input_partitions=3 + │ AggregateExec: mode=FinalPartitioned, gby=[i_category@0 as i_category, i_brand@1 as i_brand, cc_name@2 as cc_name, d_year@3 as d_year, d_moy@4 as d_moy], aggr=[sum(catalog_sales.cs_sales_price)] + │ [Stage 9] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 9 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([i_category@0, i_brand@1, cc_name@2, d_year@3, d_moy@4], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[i_category@1 as i_category, i_brand@0 as i_brand, cc_name@5 as cc_name, d_year@3 as d_year, d_moy@4 as d_moy], aggr=[sum(catalog_sales.cs_sales_price)] + │ ProjectionExec: expr=[i_brand@1 as i_brand, i_category@2 as i_category, cs_sales_price@3 as cs_sales_price, d_year@4 as d_year, d_moy@5 as d_moy, cc_name@0 as cc_name] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(call_center.cc_call_center_sk AS Float64)@2, cs_call_center_sk@2)], projection=[cc_name@1, i_brand@3, i_category@4, cs_sales_price@6, d_year@7, d_moy@8] + │ CoalescePartitionsExec + │ [Stage 6] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[i_brand@2 as i_brand, i_category@3 as i_category, cs_call_center_sk@4 as cs_call_center_sk, cs_sales_price@5 as cs_sales_price, d_year@0 as d_year, d_moy@1 as d_moy] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@3, cs_sold_date_sk@2)], projection=[d_year@1, d_moy@2, i_brand@4, i_category@5, cs_call_center_sk@7, cs_sales_price@8] + │ CoalescePartitionsExec + │ [Stage 7] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, cs_item_sk@2)], projection=[i_brand@1, i_category@2, cs_sold_date_sk@3, cs_call_center_sk@4, cs_sales_price@6] + │ CoalescePartitionsExec + │ [Stage 8] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_call_center_sk, cs_item_sk, cs_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/call_center/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/call_center/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/call_center/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/call_center/part-3.parquet]]}, projection=[cc_call_center_sk, cc_name, CAST(cc_call_center_sk@0 AS Float64) as CAST(call_center.cc_call_center_sk AS Float64)], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, d_moy@2 as d_moy, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_year@1 = 1999 OR d_year@1 = 1998 AND d_moy@2 = 12 OR d_year@1 = 2000 AND d_moy@2 = 1 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1999 OR d_year@6 = 1998 AND d_moy@8 = 12 OR d_year@6 = 2000 AND d_moy@8 = 1, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 1 AND 1 <= d_moy_max@5, required_guarantees=[d_year in (1998, 1999, 2000)] + └────────────────────────────────────────────────── + ┌───── Stage 8 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand, i_category], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 15 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ RepartitionExec: partitioning=Hash([i_category@0, i_brand@1, cc_name@2], 3), input_partitions=3 + │ AggregateExec: mode=FinalPartitioned, gby=[i_category@0 as i_category, i_brand@1 as i_brand, cc_name@2 as cc_name, d_year@3 as d_year, d_moy@4 as d_moy], aggr=[sum(catalog_sales.cs_sales_price)] + │ [Stage 14] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 14 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([i_category@0, i_brand@1, cc_name@2, d_year@3, d_moy@4], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[i_category@1 as i_category, i_brand@0 as i_brand, cc_name@5 as cc_name, d_year@3 as d_year, d_moy@4 as d_moy], aggr=[sum(catalog_sales.cs_sales_price)] + │ ProjectionExec: expr=[i_brand@1 as i_brand, i_category@2 as i_category, cs_sales_price@3 as cs_sales_price, d_year@4 as d_year, d_moy@5 as d_moy, cc_name@0 as cc_name] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(call_center.cc_call_center_sk AS Float64)@2, cs_call_center_sk@2)], projection=[cc_name@1, i_brand@3, i_category@4, cs_sales_price@6, d_year@7, d_moy@8] + │ CoalescePartitionsExec + │ [Stage 11] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[i_brand@2 as i_brand, i_category@3 as i_category, cs_call_center_sk@4 as cs_call_center_sk, cs_sales_price@5 as cs_sales_price, d_year@0 as d_year, d_moy@1 as d_moy] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@3, cs_sold_date_sk@2)], projection=[d_year@1, d_moy@2, i_brand@4, i_category@5, cs_call_center_sk@7, cs_sales_price@8] + │ CoalescePartitionsExec + │ [Stage 12] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, cs_item_sk@2)], projection=[i_brand@1, i_category@2, cs_sold_date_sk@3, cs_call_center_sk@4, cs_sales_price@6] + │ CoalescePartitionsExec + │ [Stage 13] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_call_center_sk, cs_item_sk, cs_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 11 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/call_center/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/call_center/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/call_center/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/call_center/part-3.parquet]]}, projection=[cc_call_center_sk, cc_name, CAST(cc_call_center_sk@0 AS Float64) as CAST(call_center.cc_call_center_sk AS Float64)], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 12 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, d_moy@2 as d_moy, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_year@1 = 1999 OR d_year@1 = 1998 AND d_moy@2 = 12 OR d_year@1 = 2000 AND d_moy@2 = 1 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1999 OR d_year@6 = 1998 AND d_moy@8 = 12 OR d_year@6 = 2000 AND d_moy@8 = 1, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 12 AND 12 <= d_moy_max@5 OR d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 1 AND 1 <= d_moy_max@5, required_guarantees=[d_year in (1998, 1999, 2000)] + └────────────────────────────────────────────────── + ┌───── Stage 13 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand, i_category], file_type=parquet + └────────────────────────────────────────────────── + "#); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_58() -> Result<()> { + let display = test_tpcds_query("q58").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [item_id@0 ASC, ss_item_rev@1 ASC], fetch=100 + │ SortExec: TopK(fetch=100), expr=[item_id@0 ASC, ss_item_rev@1 ASC], preserve_partitioning=[true] + │ ProjectionExec: expr=[item_id@1 as item_id, ss_item_rev@2 as ss_item_rev, ss_item_rev@2 / __common_expr_7@0 * Some(100),20,0 as ss_dev, cs_item_rev@3 as cs_item_rev, cs_item_rev@3 / __common_expr_7@0 * Some(100),20,0 as cs_dev, ws_item_rev@4 as ws_item_rev, ws_item_rev@4 / __common_expr_7@0 * Some(100),20,0 as ws_dev, __common_expr_7@0 as average] + │ ProjectionExec: expr=[(ss_item_rev@2 + cs_item_rev@3 + ws_item_rev@0) / Some(3),20,0 as __common_expr_7, item_id@1 as item_id, ss_item_rev@2 as ss_item_rev, cs_item_rev@3 as cs_item_rev, ws_item_rev@0 as ws_item_rev] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(item_id@0, item_id@0)], filter=CAST(ss_item_rev@0 AS Decimal128(30, 15)) >= CAST(0.9 * CAST(ws_item_rev@2 AS Float64) AS Decimal128(30, 15)) AND CAST(ss_item_rev@0 AS Decimal128(30, 15)) <= CAST(1.1 * CAST(ws_item_rev@2 AS Float64) AS Decimal128(30, 15)) AND CAST(cs_item_rev@1 AS Decimal128(30, 15)) >= CAST(0.9 * CAST(ws_item_rev@2 AS Float64) AS Decimal128(30, 15)) AND CAST(cs_item_rev@1 AS Decimal128(30, 15)) <= CAST(1.1 * CAST(ws_item_rev@2 AS Float64) AS Decimal128(30, 15)) AND CAST(ws_item_rev@2 AS Decimal128(30, 15)) >= CAST(0.9 * CAST(ss_item_rev@0 AS Float64) AS Decimal128(30, 15)) AND CAST(ws_item_rev@2 AS Decimal128(30, 15)) <= CAST(1.1 * CAST(ss_item_rev@0 AS Float64) AS Decimal128(30, 15)) AND CAST(ws_item_rev@2 AS Decimal128(30, 15)) >= CAST(0.9 * CAST(cs_item_rev@1 AS Float64) AS Decimal128(30, 15)) AND CAST(ws_item_rev@2 AS Decimal128(30, 15)) <= CAST(1.1 * CAST(cs_item_rev@1 AS Float64) AS Decimal128(30, 15)), projection=[ws_item_rev@1, item_id@2, ss_item_rev@3, cs_item_rev@4] + │ CoalescePartitionsExec + │ BroadcastExec: input_partitions=3, consumer_tasks=1, output_partitions=3 + │ ProjectionExec: expr=[i_item_id@0 as item_id, sum(web_sales.ws_ext_sales_price)@1 as ws_item_rev] + │ AggregateExec: mode=FinalPartitioned, gby=[i_item_id@0 as i_item_id], aggr=[sum(web_sales.ws_ext_sales_price)] + │ RepartitionExec: partitioning=Hash([i_item_id@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_item_id@1 as i_item_id], aggr=[sum(web_sales.ws_ext_sales_price)] + │ HashJoinExec: mode=CollectLeft, join_type=LeftSemi, on=[(d_date@2, d_date@0)], projection=[ws_ext_sales_price@0, i_item_id@1] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkCoalesceExec: output_partitions=9, input_tasks=3 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_week_seq@0, d_week_seq@1)], projection=[d_date@1] + │ CoalescePartitionsExec + │ BroadcastExec: input_partitions=3, consumer_tasks=1, output_partitions=3 + │ FilterExec: d_date@0 = 2000-01-03, projection=[d_week_seq@1] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=d_date@2 = 2000-01-03, pruning_predicate=d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-01-03 AND 2000-01-03 <= d_date_max@1, required_guarantees=[d_date in (2000-01-03)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[item_id@1 as item_id, ss_item_rev@2 as ss_item_rev, cs_item_rev@0 as cs_item_rev] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(item_id@0, item_id@0)], filter=CAST(ss_item_rev@0 AS Decimal128(30, 15)) >= CAST(0.9 * CAST(cs_item_rev@1 AS Float64) AS Decimal128(30, 15)) AND CAST(ss_item_rev@0 AS Decimal128(30, 15)) <= CAST(1.1 * CAST(cs_item_rev@1 AS Float64) AS Decimal128(30, 15)) AND CAST(cs_item_rev@1 AS Decimal128(30, 15)) >= CAST(0.9 * CAST(ss_item_rev@0 AS Float64) AS Decimal128(30, 15)) AND CAST(cs_item_rev@1 AS Decimal128(30, 15)) <= CAST(1.1 * CAST(ss_item_rev@0 AS Float64) AS Decimal128(30, 15)), projection=[cs_item_rev@1, item_id@2, ss_item_rev@3] + │ CoalescePartitionsExec + │ BroadcastExec: input_partitions=3, consumer_tasks=1, output_partitions=3 + │ ProjectionExec: expr=[i_item_id@0 as item_id, sum(catalog_sales.cs_ext_sales_price)@1 as cs_item_rev] + │ AggregateExec: mode=FinalPartitioned, gby=[i_item_id@0 as i_item_id], aggr=[sum(catalog_sales.cs_ext_sales_price)] + │ RepartitionExec: partitioning=Hash([i_item_id@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_item_id@1 as i_item_id], aggr=[sum(catalog_sales.cs_ext_sales_price)] + │ HashJoinExec: mode=CollectLeft, join_type=LeftSemi, on=[(d_date@2, d_date@0)], projection=[cs_ext_sales_price@0, i_item_id@1] + │ CoalescePartitionsExec + │ [Stage 8] => NetworkCoalesceExec: output_partitions=9, input_tasks=3 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_week_seq@0, d_week_seq@1)], projection=[d_date@1] + │ CoalescePartitionsExec + │ BroadcastExec: input_partitions=3, consumer_tasks=1, output_partitions=3 + │ FilterExec: d_date@0 = 2000-01-03, projection=[d_week_seq@1] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=d_date@2 = 2000-01-03, pruning_predicate=d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-01-03 AND 2000-01-03 <= d_date_max@1, required_guarantees=[d_date in (2000-01-03)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[i_item_id@0 as item_id, sum(store_sales.ss_ext_sales_price)@1 as ss_item_rev] + │ AggregateExec: mode=FinalPartitioned, gby=[i_item_id@0 as i_item_id], aggr=[sum(store_sales.ss_ext_sales_price)] + │ RepartitionExec: partitioning=Hash([i_item_id@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_item_id@1 as i_item_id], aggr=[sum(store_sales.ss_ext_sales_price)] + │ HashJoinExec: mode=CollectLeft, join_type=LeftSemi, on=[(d_date@2, d_date@0)], projection=[ss_ext_sales_price@0, i_item_id@1] + │ CoalescePartitionsExec + │ [Stage 12] => NetworkCoalesceExec: output_partitions=9, input_tasks=3 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_week_seq@0, d_week_seq@1)], projection=[d_date@1] + │ CoalescePartitionsExec + │ BroadcastExec: input_partitions=3, consumer_tasks=1, output_partitions=3 + │ FilterExec: d_date@0 = 2000-01-03, projection=[d_week_seq@1] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=d_date@2 = 2000-01-03, pruning_predicate=d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-01-03 AND 2000-01-03 <= d_date_max@1, required_guarantees=[d_date in (2000-01-03)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] + │ ProjectionExec: expr=[ws_ext_sales_price@1 as ws_ext_sales_price, i_item_id@2 as i_item_id, d_date@0 as d_date] + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, ws_sold_date_sk@0)], projection=[d_date@1, ws_ext_sales_price@4, i_item_id@5] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p8] t1:[p0..p8] + │ RepartitionExec: partitioning=Hash([CAST(date_dim.d_date_sk AS Float64)@2], 9), input_partitions=3 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_date@1 as d_date, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] + │ RepartitionExec: partitioning=Hash([ws_sold_date_sk@0], 9), input_partitions=2 + │ ProjectionExec: expr=[ws_sold_date_sk@1 as ws_sold_date_sk, ws_ext_sales_price@2 as ws_ext_sales_price, i_item_id@0 as i_item_id] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ws_item_sk@1)], projection=[i_item_id@1, ws_sold_date_sk@2, ws_ext_sales_price@4] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_item_sk, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_item_id], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 8 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] + │ ProjectionExec: expr=[cs_ext_sales_price@1 as cs_ext_sales_price, i_item_id@2 as i_item_id, d_date@0 as d_date] + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, cs_sold_date_sk@0)], projection=[d_date@1, cs_ext_sales_price@4, i_item_id@5] + │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p8] t1:[p0..p8] + │ RepartitionExec: partitioning=Hash([CAST(date_dim.d_date_sk AS Float64)@2], 9), input_partitions=3 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_date@1 as d_date, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] + │ RepartitionExec: partitioning=Hash([cs_sold_date_sk@0], 9), input_partitions=2 + │ ProjectionExec: expr=[cs_sold_date_sk@1 as cs_sold_date_sk, cs_ext_sales_price@2 as cs_ext_sales_price, i_item_id@0 as i_item_id] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, cs_item_sk@1)], projection=[i_item_id@1, cs_sold_date_sk@2, cs_ext_sales_price@4] + │ CoalescePartitionsExec + │ [Stage 6] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_item_sk, cs_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_item_id], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 12 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] + │ ProjectionExec: expr=[ss_ext_sales_price@1 as ss_ext_sales_price, i_item_id@2 as i_item_id, d_date@0 as d_date] + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, ss_sold_date_sk@0)], projection=[d_date@1, ss_ext_sales_price@4, i_item_id@5] + │ [Stage 9] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 11] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 9 ── Tasks: t0:[p0..p8] t1:[p0..p8] + │ RepartitionExec: partitioning=Hash([CAST(date_dim.d_date_sk AS Float64)@2], 9), input_partitions=3 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_date@1 as d_date, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 11 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] + │ RepartitionExec: partitioning=Hash([ss_sold_date_sk@0], 9), input_partitions=2 + │ ProjectionExec: expr=[ss_sold_date_sk@1 as ss_sold_date_sk, ss_ext_sales_price@2 as ss_ext_sales_price, i_item_id@0 as i_item_id] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ss_item_sk@1)], projection=[i_item_id@1, ss_sold_date_sk@2, ss_ext_sales_price@4] + │ CoalescePartitionsExec + │ [Stage 10] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 10 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_item_id], file_type=parquet + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_59() -> Result<()> { + let display = test_tpcds_query("q59").await?; + assert_snapshot!(display, @r#" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [s_store_name1@0 ASC, s_store_id1@1 ASC, d_week_seq1@2 ASC], fetch=100 + │ [Stage 10] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 10 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ SortExec: TopK(fetch=100), expr=[s_store_name1@0 ASC, s_store_id1@1 ASC, d_week_seq1@2 ASC], preserve_partitioning=[true] + │ ProjectionExec: expr=[s_store_name1@0 as s_store_name1, s_store_id1@2 as s_store_id1, d_week_seq1@1 as d_week_seq1, sun_sales1@3 / sun_sales2@10 as sun_sales_ratio, mon_sales1@4 / mon_sales2@11 as mon_sales_ratio, tue_sales1@5 / tue_sales2@12 as tue_sales_ratio, wed_sales1@6 / wed_sales2@13 as wed_sales_ratio, thu_sales1@7 / thu_sales2@14 as thu_sales_ratio, fri_sales1@8 / fri_sales2@15 as fri_sales_ratio, sat_sales1@9 / sat_sales2@16 as sat_sales_ratio] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_store_id1@2, s_store_id2@1), (CAST(y.d_week_seq1 AS Int64)@10, x.d_week_seq2 - Int64(52)@9)], projection=[s_store_name1@0, d_week_seq1@1, s_store_id1@2, sun_sales1@3, mon_sales1@4, tue_sales1@5, wed_sales1@6, thu_sales1@7, fri_sales1@8, sat_sales1@9, sun_sales2@13, mon_sales2@14, tue_sales2@15, wed_sales2@16, thu_sales2@17, fri_sales2@18, sat_sales2@19] + │ CoalescePartitionsExec + │ [Stage 5] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[d_week_seq@0 as d_week_seq2, s_store_id@8 as s_store_id2, sun_sales@1 as sun_sales2, mon_sales@2 as mon_sales2, tue_sales@3 as tue_sales2, wed_sales@4 as wed_sales2, thu_sales@5 as thu_sales2, fri_sales@6 as fri_sales2, sat_sales@7 as sat_sales2, CAST(d_week_seq@0 AS Int64) - 52 as x.d_week_seq2 - Int64(52)] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_week_seq@0, d_week_seq@0)], projection=[d_week_seq@0, sun_sales@1, mon_sales@2, tue_sales@3, wed_sales@4, thu_sales@5, fri_sales@6, sat_sales@7, s_store_id@8] + │ CoalescePartitionsExec + │ [Stage 9] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=6, input_tasks=2 + │ FilterExec: d_month_seq@0 >= 1224 AND d_month_seq@0 <= 1235, projection=[d_week_seq@1] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_month_seq, d_week_seq], file_type=parquet, predicate=d_month_seq@3 >= 1224 AND d_month_seq@3 <= 1235 AND DynamicFilter [ empty ], pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1224 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1235, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=3, consumer_tasks=2, output_partitions=6 + │ ProjectionExec: expr=[s_store_name@9 as s_store_name1, d_week_seq@0 as d_week_seq1, s_store_id@8 as s_store_id1, sun_sales@1 as sun_sales1, mon_sales@2 as mon_sales1, tue_sales@3 as tue_sales1, wed_sales@4 as wed_sales1, thu_sales@5 as thu_sales1, fri_sales@6 as fri_sales1, sat_sales@7 as sat_sales1, CAST(d_week_seq@0 AS Int64) as CAST(y.d_week_seq1 AS Int64)] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_week_seq@0, d_week_seq@0)], projection=[d_week_seq@0, sun_sales@1, mon_sales@2, tue_sales@3, wed_sales@4, thu_sales@5, fri_sales@6, sat_sales@7, s_store_id@8, s_store_name@9] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=6, input_tasks=2 + │ FilterExec: d_month_seq@0 >= 1212 AND d_month_seq@0 <= 1223, projection=[d_week_seq@1] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_month_seq, d_week_seq], file_type=parquet, predicate=d_month_seq@3 >= 1212 AND d_month_seq@3 <= 1223 AND DynamicFilter [ empty ], pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1212 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1223, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=3, consumer_tasks=2, output_partitions=6 + │ ProjectionExec: expr=[d_week_seq@2 as d_week_seq, sun_sales@3 as sun_sales, mon_sales@4 as mon_sales, tue_sales@5 as tue_sales, wed_sales@6 as wed_sales, thu_sales@7 as thu_sales, fri_sales@8 as fri_sales, sat_sales@9 as sat_sales, s_store_id@0 as s_store_id, s_store_name@1 as s_store_name] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@3, ss_store_sk@1)], projection=[s_store_id@1, s_store_name@2, d_week_seq@4, sun_sales@6, mon_sales@7, tue_sales@8, wed_sales@9, thu_sales@10, fri_sales@11, sat_sales@12] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=4, input_tasks=2 + │ ProjectionExec: expr=[d_week_seq@0 as d_week_seq, ss_store_sk@1 as ss_store_sk, sum(CASE WHEN date_dim.d_day_name = Utf8("Sunday") THEN store_sales.ss_sales_price ELSE NULL END)@2 as sun_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Monday") THEN store_sales.ss_sales_price ELSE NULL END)@3 as mon_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Tuesday") THEN store_sales.ss_sales_price ELSE NULL END)@4 as tue_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Wednesday") THEN store_sales.ss_sales_price ELSE NULL END)@5 as wed_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Thursday") THEN store_sales.ss_sales_price ELSE NULL END)@6 as thu_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Friday") THEN store_sales.ss_sales_price ELSE NULL END)@7 as fri_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Saturday") THEN store_sales.ss_sales_price ELSE NULL END)@8 as sat_sales] + │ AggregateExec: mode=FinalPartitioned, gby=[d_week_seq@0 as d_week_seq, ss_store_sk@1 as ss_store_sk], aggr=[sum(CASE WHEN date_dim.d_day_name = Utf8("Sunday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Monday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Tuesday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Wednesday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Thursday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Friday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Saturday") THEN store_sales.ss_sales_price ELSE NULL END)] + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p3] t1:[p4..p7] + │ BroadcastExec: input_partitions=2, consumer_tasks=2, output_partitions=4 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_store_id, s_store_name, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([d_week_seq@0, ss_store_sk@1], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[d_week_seq@2 as d_week_seq, ss_store_sk@0 as ss_store_sk], aggr=[sum(CASE WHEN date_dim.d_day_name = Utf8("Sunday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Monday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Tuesday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Wednesday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Thursday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Friday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Saturday") THEN store_sales.ss_sales_price ELSE NULL END)] + │ ProjectionExec: expr=[ss_store_sk@2 as ss_store_sk, ss_sales_price@3 as ss_sales_price, d_week_seq@0 as d_week_seq, d_day_name@1 as d_day_name] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@3, ss_sold_date_sk@0)], projection=[d_week_seq@1, d_day_name@2, ss_store_sk@5, ss_sales_price@6] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_week_seq@1 as d_week_seq, d_day_name@2 as d_day_name, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_week_seq, d_day_name], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 9 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=3, consumer_tasks=2, output_partitions=6 + │ ProjectionExec: expr=[d_week_seq@1 as d_week_seq, sun_sales@2 as sun_sales, mon_sales@3 as mon_sales, tue_sales@4 as tue_sales, wed_sales@5 as wed_sales, thu_sales@6 as thu_sales, fri_sales@7 as fri_sales, sat_sales@8 as sat_sales, s_store_id@0 as s_store_id] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@2, ss_store_sk@1)], projection=[s_store_id@1, d_week_seq@3, sun_sales@5, mon_sales@6, tue_sales@7, wed_sales@8, thu_sales@9, fri_sales@10, sat_sales@11] + │ CoalescePartitionsExec + │ [Stage 6] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=4, input_tasks=2 + │ ProjectionExec: expr=[d_week_seq@0 as d_week_seq, ss_store_sk@1 as ss_store_sk, sum(CASE WHEN date_dim.d_day_name = Utf8("Sunday") THEN store_sales.ss_sales_price ELSE NULL END)@2 as sun_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Monday") THEN store_sales.ss_sales_price ELSE NULL END)@3 as mon_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Tuesday") THEN store_sales.ss_sales_price ELSE NULL END)@4 as tue_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Wednesday") THEN store_sales.ss_sales_price ELSE NULL END)@5 as wed_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Thursday") THEN store_sales.ss_sales_price ELSE NULL END)@6 as thu_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Friday") THEN store_sales.ss_sales_price ELSE NULL END)@7 as fri_sales, sum(CASE WHEN date_dim.d_day_name = Utf8("Saturday") THEN store_sales.ss_sales_price ELSE NULL END)@8 as sat_sales] + │ AggregateExec: mode=FinalPartitioned, gby=[d_week_seq@0 as d_week_seq, ss_store_sk@1 as ss_store_sk], aggr=[sum(CASE WHEN date_dim.d_day_name = Utf8("Sunday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Monday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Tuesday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Wednesday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Thursday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Friday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Saturday") THEN store_sales.ss_sales_price ELSE NULL END)] + │ [Stage 8] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p3] t1:[p4..p7] + │ BroadcastExec: input_partitions=2, consumer_tasks=2, output_partitions=4 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_store_id, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 8 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([d_week_seq@0, ss_store_sk@1], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[d_week_seq@2 as d_week_seq, ss_store_sk@0 as ss_store_sk], aggr=[sum(CASE WHEN date_dim.d_day_name = Utf8("Sunday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Monday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Tuesday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Wednesday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Thursday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Friday") THEN store_sales.ss_sales_price ELSE NULL END), sum(CASE WHEN date_dim.d_day_name = Utf8("Saturday") THEN store_sales.ss_sales_price ELSE NULL END)] + │ ProjectionExec: expr=[ss_store_sk@2 as ss_store_sk, ss_sales_price@3 as ss_sales_price, d_week_seq@0 as d_week_seq, d_day_name@1 as d_day_name] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@3, ss_sold_date_sk@0)], projection=[d_week_seq@1, d_day_name@2, ss_store_sk@5, ss_sales_price@6] + │ CoalescePartitionsExec + │ [Stage 7] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_week_seq@1 as d_week_seq, d_day_name@2 as d_day_name, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_week_seq, d_day_name], file_type=parquet + └────────────────────────────────────────────────── + "#); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_60() -> Result<()> { + let display = test_tpcds_query("q60").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [i_item_id@0 ASC NULLS LAST, total_sales@1 ASC NULLS LAST], fetch=100 + │ SortExec: TopK(fetch=100), expr=[i_item_id@0 ASC NULLS LAST, total_sales@1 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[i_item_id@0 as i_item_id, sum(tmp1.total_sales)@1 as total_sales] + │ AggregateExec: mode=SinglePartitioned, gby=[i_item_id@0 as i_item_id], aggr=[sum(tmp1.total_sales)] + │ InterleaveExec + │ ProjectionExec: expr=[i_item_id@0 as i_item_id, sum(store_sales.ss_ext_sales_price)@1 as total_sales] + │ AggregateExec: mode=FinalPartitioned, gby=[i_item_id@0 as i_item_id], aggr=[sum(store_sales.ss_ext_sales_price)] + │ RepartitionExec: partitioning=Hash([i_item_id@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_item_id@1 as i_item_id], aggr=[sum(store_sales.ss_ext_sales_price)] + │ HashJoinExec: mode=CollectLeft, join_type=LeftSemi, on=[(i_item_id@1, i_item_id@0)] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkCoalesceExec: output_partitions=9, input_tasks=3 + │ FilterExec: i_category@1 = Music, projection=[i_item_id@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_id, i_category], file_type=parquet, predicate=i_category@12 = Music, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Music AND Music <= i_category_max@1, required_guarantees=[i_category in (Music)] + │ ProjectionExec: expr=[i_item_id@0 as i_item_id, sum(catalog_sales.cs_ext_sales_price)@1 as total_sales] + │ AggregateExec: mode=FinalPartitioned, gby=[i_item_id@0 as i_item_id], aggr=[sum(catalog_sales.cs_ext_sales_price)] + │ RepartitionExec: partitioning=Hash([i_item_id@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_item_id@1 as i_item_id], aggr=[sum(catalog_sales.cs_ext_sales_price)] + │ HashJoinExec: mode=CollectLeft, join_type=LeftSemi, on=[(i_item_id@1, i_item_id@0)] + │ CoalescePartitionsExec + │ [Stage 8] => NetworkCoalesceExec: output_partitions=9, input_tasks=3 + │ FilterExec: i_category@1 = Music, projection=[i_item_id@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_id, i_category], file_type=parquet, predicate=i_category@12 = Music, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Music AND Music <= i_category_max@1, required_guarantees=[i_category in (Music)] + │ ProjectionExec: expr=[i_item_id@0 as i_item_id, sum(web_sales.ws_ext_sales_price)@1 as total_sales] + │ AggregateExec: mode=FinalPartitioned, gby=[i_item_id@0 as i_item_id], aggr=[sum(web_sales.ws_ext_sales_price)] + │ RepartitionExec: partitioning=Hash([i_item_id@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_item_id@1 as i_item_id], aggr=[sum(web_sales.ws_ext_sales_price)] + │ HashJoinExec: mode=CollectLeft, join_type=LeftSemi, on=[(i_item_id@1, i_item_id@0)] + │ CoalescePartitionsExec + │ [Stage 12] => NetworkCoalesceExec: output_partitions=9, input_tasks=3 + │ FilterExec: i_category@1 = Music, projection=[i_item_id@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_id, i_category], file_type=parquet, predicate=i_category@12 = Music, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Music AND Music <= i_category_max@1, required_guarantees=[i_category in (Music)] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p3..p5] t2:[p6..p8] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_item_sk@0, i_item_sk@0)], projection=[ss_ext_sales_price@1, i_item_id@3] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=3 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p8] t1:[p9..p17] t2:[p18..p26] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_addr_sk@1, CAST(customer_address.ca_address_sk AS Float64)@1)], projection=[ss_item_sk@0, ss_ext_sales_price@2] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=3 + │ ProjectionExec: expr=[ca_address_sk@0 as ca_address_sk, CAST(ca_address_sk@0 AS Float64) as CAST(customer_address.ca_address_sk AS Float64)] + │ FilterExec: ca_gmt_offset@1 = Some(-500),4,2, projection=[ca_address_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = Some(-500),4,2, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= Some(-500),4,2 AND Some(-500),4,2 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (Some(-500),4,2)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p5] t1:[p6..p11] t2:[p12..p17] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_item_sk@3, ss_addr_sk@4, ss_ext_sales_price@5] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_addr_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_year@1 = 1998 AND d_moy@2 = 9, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 9, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 9 AND 9 <= d_moy_max@5, required_guarantees=[d_moy in (9), d_year in (1998)] + └────────────────────────────────────────────────── + ┌───── Stage 8 ── Tasks: t0:[p0..p2] t1:[p3..p5] t2:[p6..p8] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(cs_item_sk@0, i_item_sk@0)], projection=[cs_ext_sales_price@1, i_item_id@3] + │ CoalescePartitionsExec + │ [Stage 7] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=3 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p8] t1:[p9..p17] t2:[p18..p26] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(cs_bill_addr_sk@0, CAST(customer_address.ca_address_sk AS Float64)@1)], projection=[cs_item_sk@1, cs_ext_sales_price@2] + │ CoalescePartitionsExec + │ [Stage 6] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=3 + │ ProjectionExec: expr=[ca_address_sk@0 as ca_address_sk, CAST(ca_address_sk@0 AS Float64) as CAST(customer_address.ca_address_sk AS Float64)] + │ FilterExec: ca_gmt_offset@1 = Some(-500),4,2, projection=[ca_address_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = Some(-500),4,2, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= Some(-500),4,2 AND Some(-500),4,2 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (Some(-500),4,2)] + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p5] t1:[p6..p11] t2:[p12..p17] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, cs_sold_date_sk@0)], projection=[cs_bill_addr_sk@3, cs_item_sk@4, cs_ext_sales_price@5] + │ CoalescePartitionsExec + │ [Stage 5] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_bill_addr_sk, cs_item_sk, cs_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_year@1 = 1998 AND d_moy@2 = 9, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 9, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 9 AND 9 <= d_moy_max@5, required_guarantees=[d_moy in (9), d_year in (1998)] + └────────────────────────────────────────────────── + ┌───── Stage 12 ── Tasks: t0:[p0..p2] t1:[p3..p5] t2:[p6..p8] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ws_item_sk@0, i_item_sk@0)], projection=[ws_ext_sales_price@1, i_item_id@3] + │ CoalescePartitionsExec + │ [Stage 11] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=3 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_item_id], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 11 ── Tasks: t0:[p0..p8] t1:[p9..p17] t2:[p18..p26] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ws_bill_addr_sk@1, CAST(customer_address.ca_address_sk AS Float64)@1)], projection=[ws_item_sk@0, ws_ext_sales_price@2] + │ CoalescePartitionsExec + │ [Stage 10] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=3 + │ ProjectionExec: expr=[ca_address_sk@0 as ca_address_sk, CAST(ca_address_sk@0 AS Float64) as CAST(customer_address.ca_address_sk AS Float64)] + │ FilterExec: ca_gmt_offset@1 = Some(-500),4,2, projection=[ca_address_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = Some(-500),4,2, pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= Some(-500),4,2 AND Some(-500),4,2 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (Some(-500),4,2)] + └────────────────────────────────────────────────── + ┌───── Stage 10 ── Tasks: t0:[p0..p5] t1:[p6..p11] t2:[p12..p17] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ws_sold_date_sk@0)], projection=[ws_item_sk@3, ws_bill_addr_sk@4, ws_ext_sales_price@5] + │ CoalescePartitionsExec + │ [Stage 9] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_item_sk, ws_bill_addr_sk, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 9 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_year@1 = 1998 AND d_moy@2 = 9, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 9, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 9 AND 9 <= d_moy_max@5, required_guarantees=[d_moy in (9), d_year in (1998)] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_61() -> Result<()> { + let display = test_tpcds_query("q61").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortExec: TopK(fetch=100), expr=[total@1 ASC NULLS LAST], preserve_partitioning=[false] + │ ProjectionExec: expr=[promotions@0 as promotions, total@1 as total, CAST(promotions@0 AS Decimal128(15, 4)) / CAST(total@1 AS Decimal128(15, 4)) * Some(100),20,0 as promotional_sales.promotions / all_sales.total * Int64(100)] + │ CrossJoinExec + │ ProjectionExec: expr=[sum(store_sales.ss_ext_sales_price)@0 as promotions] + │ AggregateExec: mode=Final, gby=[], aggr=[sum(store_sales.ss_ext_sales_price)] + │ CoalescePartitionsExec + │ [Stage 7] => NetworkCoalesceExec: output_partitions=9, input_tasks=3 + │ ProjectionExec: expr=[sum(store_sales.ss_ext_sales_price)@0 as total] + │ AggregateExec: mode=Final, gby=[], aggr=[sum(store_sales.ss_ext_sales_price)] + │ CoalescePartitionsExec + │ [Stage 13] => NetworkCoalesceExec: output_partitions=9, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p2] t1:[p3..p5] t2:[p6..p8] + │ AggregateExec: mode=Partial, gby=[], aggr=[sum(store_sales.ss_ext_sales_price)] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_item_sk@0, i_item_sk@0)], projection=[ss_ext_sales_price@1] + │ CoalescePartitionsExec + │ [Stage 6] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=3 + │ FilterExec: i_category@1 = Jewelry, projection=[i_item_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_category], file_type=parquet, predicate=i_category@12 = Jewelry AND DynamicFilter [ empty ], pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Jewelry AND Jewelry <= i_category_max@1, required_guarantees=[i_category in (Jewelry)] + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p8] t1:[p9..p17] t2:[p18..p26] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_current_addr_sk@2, ca_address_sk@0)], projection=[ss_item_sk@0, ss_ext_sales_price@1] + │ CoalescePartitionsExec + │ [Stage 5] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=3 + │ FilterExec: ca_gmt_offset@1 = Some(-500),4,2, projection=[ca_address_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = Some(-500),4,2 AND DynamicFilter [ empty ], pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= Some(-500),4,2 AND Some(-500),4,2 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (Some(-500),4,2)] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p8] t1:[p9..p17] t2:[p18..p26] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_customer_sk@1, CAST(customer.c_customer_sk AS Float64)@2)], projection=[ss_item_sk@0, ss_ext_sales_price@2, c_current_addr_sk@4] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=3 + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, c_current_addr_sk@1 as c_current_addr_sk, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_current_addr_sk], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p8] t1:[p9..p17] t2:[p18..p26] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_sold_date_sk@0, CAST(date_dim.d_date_sk AS Float64)@1)], projection=[ss_item_sk@1, ss_customer_sk@2, ss_ext_sales_price@3] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=3 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_year@1 = 1998 AND d_moy@2 = 11, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 11, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 11 AND 11 <= d_moy_max@5, required_guarantees=[d_moy in (11), d_year in (1998)] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p8] t1:[p9..p17] t2:[p18..p26] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_promo_sk@3, CAST(promotion.p_promo_sk AS Float64)@1)], projection=[ss_sold_date_sk@0, ss_item_sk@1, ss_customer_sk@2, ss_ext_sales_price@4] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=3 + │ ProjectionExec: expr=[p_promo_sk@0 as p_promo_sk, CAST(p_promo_sk@0 AS Float64) as CAST(promotion.p_promo_sk AS Float64)] + │ FilterExec: p_channel_dmail@1 = Y OR p_channel_email@2 = Y OR p_channel_tv@3 = Y, projection=[p_promo_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/promotion/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/promotion/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/promotion/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/promotion/part-3.parquet]]}, projection=[p_promo_sk, p_channel_dmail, p_channel_email, p_channel_tv], file_type=parquet, predicate=p_channel_dmail@8 = Y OR p_channel_email@9 = Y OR p_channel_tv@11 = Y, pruning_predicate=p_channel_dmail_null_count@2 != row_count@3 AND p_channel_dmail_min@0 <= Y AND Y <= p_channel_dmail_max@1 OR p_channel_email_null_count@6 != row_count@3 AND p_channel_email_min@4 <= Y AND Y <= p_channel_email_max@5 OR p_channel_tv_null_count@9 != row_count@3 AND p_channel_tv_min@7 <= Y AND Y <= p_channel_tv_max@8, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p5] t1:[p6..p11] t2:[p12..p17] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@1, ss_store_sk@3)], projection=[ss_sold_date_sk@2, ss_item_sk@3, ss_customer_sk@4, ss_promo_sk@6, ss_ext_sales_price@7] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_store_sk, ss_promo_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] + │ FilterExec: s_gmt_offset@1 = Some(-500),3,2, projection=[s_store_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_gmt_offset], file_type=parquet, predicate=s_gmt_offset@27 = Some(-500),3,2, pruning_predicate=s_gmt_offset_null_count@2 != row_count@3 AND s_gmt_offset_min@0 <= Some(-500),3,2 AND Some(-500),3,2 <= s_gmt_offset_max@1, required_guarantees=[s_gmt_offset in (Some(-500),3,2)] + └────────────────────────────────────────────────── + ┌───── Stage 13 ── Tasks: t0:[p0..p2] t1:[p3..p5] t2:[p6..p8] + │ AggregateExec: mode=Partial, gby=[], aggr=[sum(store_sales.ss_ext_sales_price)] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_item_sk@0, i_item_sk@0)], projection=[ss_ext_sales_price@1] + │ CoalescePartitionsExec + │ [Stage 12] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=3 + │ FilterExec: i_category@1 = Jewelry, projection=[i_item_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_category], file_type=parquet, predicate=i_category@12 = Jewelry AND DynamicFilter [ empty ], pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Jewelry AND Jewelry <= i_category_max@1, required_guarantees=[i_category in (Jewelry)] + └────────────────────────────────────────────────── + ┌───── Stage 12 ── Tasks: t0:[p0..p8] t1:[p9..p17] t2:[p18..p26] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_current_addr_sk@2, ca_address_sk@0)], projection=[ss_item_sk@0, ss_ext_sales_price@1] + │ CoalescePartitionsExec + │ [Stage 11] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=3 + │ FilterExec: ca_gmt_offset@1 = Some(-500),4,2, projection=[ca_address_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = Some(-500),4,2 AND DynamicFilter [ empty ], pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= Some(-500),4,2 AND Some(-500),4,2 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (Some(-500),4,2)] + └────────────────────────────────────────────────── + ┌───── Stage 11 ── Tasks: t0:[p0..p8] t1:[p9..p17] t2:[p18..p26] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_customer_sk@1, CAST(customer.c_customer_sk AS Float64)@2)], projection=[ss_item_sk@0, ss_ext_sales_price@2, c_current_addr_sk@4] + │ CoalescePartitionsExec + │ [Stage 10] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=3 + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, c_current_addr_sk@1 as c_current_addr_sk, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_current_addr_sk], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 10 ── Tasks: t0:[p0..p8] t1:[p9..p17] t2:[p18..p26] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_sold_date_sk@0, CAST(date_dim.d_date_sk AS Float64)@1)], projection=[ss_item_sk@1, ss_customer_sk@2, ss_ext_sales_price@3] + │ CoalescePartitionsExec + │ [Stage 9] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=3 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_year@1 = 1998 AND d_moy@2 = 11, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 11, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 11 AND 11 <= d_moy_max@5, required_guarantees=[d_moy in (11), d_year in (1998)] + └────────────────────────────────────────────────── + ┌───── Stage 9 ── Tasks: t0:[p0..p5] t1:[p6..p11] t2:[p12..p17] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@1, ss_store_sk@3)], projection=[ss_sold_date_sk@2, ss_item_sk@3, ss_customer_sk@4, ss_ext_sales_price@6] + │ CoalescePartitionsExec + │ [Stage 8] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_store_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 8 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] + │ FilterExec: s_gmt_offset@1 = Some(-500),3,2, projection=[s_store_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_gmt_offset], file_type=parquet, predicate=s_gmt_offset@27 = Some(-500),3,2, pruning_predicate=s_gmt_offset_null_count@2 != row_count@3 AND s_gmt_offset_min@0 <= Some(-500),3,2 AND Some(-500),3,2 <= s_gmt_offset_max@1, required_guarantees=[s_gmt_offset in (Some(-500),3,2)] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_62() -> Result<()> { + let display = test_tpcds_query("q62").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [w_substr@0 ASC, sm_type@1 ASC, web_name@2 ASC], fetch=100 + │ [Stage 6] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ SortExec: TopK(fetch=100), expr=[w_substr@0 ASC, sm_type@1 ASC, web_name@2 ASC], preserve_partitioning=[true] + │ ProjectionExec: expr=[w_substr@0 as w_substr, sm_type@1 as sm_type, web_name@2 as web_name, sum(CASE WHEN web_sales.ws_ship_date_sk - web_sales.ws_sold_date_sk <= Int64(30) THEN Int64(1) ELSE Int64(0) END)@3 as 30 days, sum(CASE WHEN web_sales.ws_ship_date_sk - web_sales.ws_sold_date_sk > Int64(30) AND web_sales.ws_ship_date_sk - web_sales.ws_sold_date_sk <= Int64(60) THEN Int64(1) ELSE Int64(0) END)@4 as 31-60 days, sum(CASE WHEN web_sales.ws_ship_date_sk - web_sales.ws_sold_date_sk > Int64(60) AND web_sales.ws_ship_date_sk - web_sales.ws_sold_date_sk <= Int64(90) THEN Int64(1) ELSE Int64(0) END)@5 as 61-90 days, sum(CASE WHEN web_sales.ws_ship_date_sk - web_sales.ws_sold_date_sk > Int64(90) AND web_sales.ws_ship_date_sk - web_sales.ws_sold_date_sk <= Int64(120) THEN Int64(1) ELSE Int64(0) END)@6 as 91-120 days, sum(CASE WHEN web_sales.ws_ship_date_sk - web_sales.ws_sold_date_sk > Int64(120) THEN Int64(1) ELSE Int64(0) END)@7 as >120 days] + │ AggregateExec: mode=FinalPartitioned, gby=[w_substr@0 as w_substr, sm_type@1 as sm_type, web_name@2 as web_name], aggr=[sum(CASE WHEN web_sales.ws_ship_date_sk - web_sales.ws_sold_date_sk <= Int64(30) THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN web_sales.ws_ship_date_sk - web_sales.ws_sold_date_sk > Int64(30) AND web_sales.ws_ship_date_sk - web_sales.ws_sold_date_sk <= Int64(60) THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN web_sales.ws_ship_date_sk - web_sales.ws_sold_date_sk > Int64(60) AND web_sales.ws_ship_date_sk - web_sales.ws_sold_date_sk <= Int64(90) THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN web_sales.ws_ship_date_sk - web_sales.ws_sold_date_sk > Int64(90) AND web_sales.ws_ship_date_sk - web_sales.ws_sold_date_sk <= Int64(120) THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN web_sales.ws_ship_date_sk - web_sales.ws_sold_date_sk > Int64(120) THEN Int64(1) ELSE Int64(0) END)] + │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([w_substr@0, sm_type@1, web_name@2], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[w_substr@1 as w_substr, sm_type@2 as sm_type, web_name@3 as web_name], aggr=[sum(CASE WHEN web_sales.ws_ship_date_sk - web_sales.ws_sold_date_sk <= Int64(30) THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN web_sales.ws_ship_date_sk - web_sales.ws_sold_date_sk > Int64(30) AND web_sales.ws_ship_date_sk - web_sales.ws_sold_date_sk <= Int64(60) THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN web_sales.ws_ship_date_sk - web_sales.ws_sold_date_sk > Int64(60) AND web_sales.ws_ship_date_sk - web_sales.ws_sold_date_sk <= Int64(90) THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN web_sales.ws_ship_date_sk - web_sales.ws_sold_date_sk > Int64(90) AND web_sales.ws_ship_date_sk - web_sales.ws_sold_date_sk <= Int64(120) THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN web_sales.ws_ship_date_sk - web_sales.ws_sold_date_sk > Int64(120) THEN Int64(1) ELSE Int64(0) END)] + │ ProjectionExec: expr=[ws_ship_date_sk@1 - ws_sold_date_sk@0 as __common_expr_1, w_substr@2 as w_substr, sm_type@3 as sm_type, web_name@4 as web_name] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ws_ship_date_sk@1, CAST(date_dim.d_date_sk AS Float64)@1)], projection=[ws_sold_date_sk@0, ws_ship_date_sk@1, w_substr@2, sm_type@3, web_name@4] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=3 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_month_seq@1 >= 1200 AND d_month_seq@1 <= 1211, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p5] t1:[p6..p11] t2:[p12..p17] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ws_web_site_sk@2, CAST(web_site.web_site_sk AS Float64)@2)], projection=[ws_sold_date_sk@0, ws_ship_date_sk@1, w_substr@3, sm_type@4, web_name@6] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=3 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_site/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_site/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_site/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_site/part-3.parquet]]}, projection=[web_site_sk, web_name, CAST(web_site_sk@0 AS Float64) as CAST(web_site.web_site_sk AS Float64)], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p5] t1:[p6..p11] t2:[p12..p17] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ws_ship_mode_sk@3, CAST(ship_mode.sm_ship_mode_sk AS Float64)@2)], projection=[ws_sold_date_sk@0, ws_ship_date_sk@1, ws_web_site_sk@2, w_substr@4, sm_type@6] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=3 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/ship_mode/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/ship_mode/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/ship_mode/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/ship_mode/part-3.parquet]]}, projection=[sm_ship_mode_sk, sm_type, CAST(sm_ship_mode_sk@0 AS Float64) as CAST(ship_mode.sm_ship_mode_sk AS Float64)], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p5] t1:[p6..p11] t2:[p12..p17] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ ProjectionExec: expr=[ws_sold_date_sk@1 as ws_sold_date_sk, ws_ship_date_sk@2 as ws_ship_date_sk, ws_web_site_sk@3 as ws_web_site_sk, ws_ship_mode_sk@4 as ws_ship_mode_sk, w_substr@0 as w_substr] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(sq1.w_warehouse_sk AS Float64)@2, ws_warehouse_sk@4)], projection=[w_substr@0, ws_sold_date_sk@3, ws_ship_date_sk@4, ws_web_site_sk@5, ws_ship_mode_sk@6] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_ship_date_sk, ws_web_site_sk, ws_ship_mode_sk, ws_warehouse_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/warehouse/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/warehouse/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/warehouse/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/warehouse/part-3.parquet]]}, projection=[substr(w_warehouse_name@2, 1, 20) as w_substr, w_warehouse_sk, CAST(w_warehouse_sk@0 AS Float64) as CAST(sq1.w_warehouse_sk AS Float64)], file_type=parquet + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_63() -> Result<()> { + let display = test_tpcds_query("q63").await?; + assert_snapshot!(display, @r#" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [i_manager_id@0 ASC NULLS LAST, avg_monthly_sales@2 ASC NULLS LAST, sum_sales@1 ASC NULLS LAST], fetch=100 + │ SortExec: TopK(fetch=100), expr=[i_manager_id@0 ASC NULLS LAST, avg_monthly_sales@2 ASC NULLS LAST, sum_sales@1 ASC NULLS LAST], preserve_partitioning=[true], sort_prefix=[i_manager_id@0 ASC NULLS LAST, avg_monthly_sales@2 ASC NULLS LAST] + │ ProjectionExec: expr=[i_manager_id@0 as i_manager_id, sum(store_sales.ss_sales_price)@1 as sum_sales, avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_manager_id] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@2 as avg_monthly_sales] + │ FilterExec: CASE WHEN avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_manager_id] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@2 > Some(0),19,6 THEN abs(sum(store_sales.ss_sales_price)@1 - avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_manager_id] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@2) / avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_manager_id] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@2 END > Some(1000000000),30,10 + │ WindowAggExec: wdw=[avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_manager_id] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING: Ok(Field { name: "avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_manager_id] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING", data_type: Decimal128(19, 6), nullable: true }), frame: WindowFrame { units: Rows, start_bound: Preceding(UInt64(NULL)), end_bound: Following(UInt64(NULL)), is_causal: false }] + │ SortExec: expr=[i_manager_id@0 ASC NULLS LAST], preserve_partitioning=[true] + │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ RepartitionExec: partitioning=Hash([i_manager_id@0], 3), input_partitions=3 + │ ProjectionExec: expr=[i_manager_id@0 as i_manager_id, sum(store_sales.ss_sales_price)@2 as sum(store_sales.ss_sales_price)] + │ AggregateExec: mode=FinalPartitioned, gby=[i_manager_id@0 as i_manager_id, d_moy@1 as d_moy], aggr=[sum(store_sales.ss_sales_price)] + │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([i_manager_id@0, d_moy@1], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[i_manager_id@0 as i_manager_id, d_moy@2 as d_moy], aggr=[sum(store_sales.ss_sales_price)] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@1, ss_store_sk@1)], projection=[i_manager_id@2, ss_sales_price@4, d_moy@5] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[i_manager_id@1 as i_manager_id, ss_store_sk@2 as ss_store_sk, ss_sales_price@3 as ss_sales_price, d_moy@0 as d_moy] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, ss_sold_date_sk@1)], projection=[d_moy@1, i_manager_id@3, ss_store_sk@5, ss_sales_price@6] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ss_item_sk@1)], projection=[i_manager_id@1, ss_sold_date_sk@2, ss_store_sk@4, ss_sales_price@5] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_moy@1 as d_moy, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_month_seq@1 IN (SET) ([1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211]), projection=[d_date_sk@0, d_moy@2] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_month_seq, d_moy], file_type=parquet, predicate=d_month_seq@3 IN (SET) ([1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211]), pruning_predicate=d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1200 AND 1200 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1201 AND 1201 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1202 AND 1202 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1203 AND 1203 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1204 AND 1204 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1205 AND 1205 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1206 AND 1206 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1207 AND 1207 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1208 AND 1208 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1209 AND 1209 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1210 AND 1210 <= d_month_seq_max@1 OR d_month_seq_null_count@2 != row_count@3 AND d_month_seq_min@0 <= 1211 AND 1211 <= d_month_seq_max@1, required_guarantees=[d_month_seq in (1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211)] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ FilterExec: (i_category@3 = Books OR i_category@3 = Children OR i_category@3 = Electronics) AND i_class@2 IN (SET) ([personal, portable, reference, self-help]) AND i_brand@1 IN (SET) ([scholaramalgamalg #14, scholaramalgamalg #7, exportiunivamalg #9, scholaramalgamalg #9]) OR (i_category@3 = Women OR i_category@3 = Music OR i_category@3 = Men) AND i_class@2 IN (SET) ([accessories, classical, fragrances, pants]) AND i_brand@1 IN (SET) ([amalgimporto #1, edu packscholar #1, exportiimporto #1, importoamalg #1]), projection=[i_item_sk@0, i_manager_id@4] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand, i_class, i_category, i_manager_id], file_type=parquet, predicate=(i_category@12 = Books OR i_category@12 = Children OR i_category@12 = Electronics) AND i_class@10 IN (SET) ([personal, portable, reference, self-help]) AND i_brand@8 IN (SET) ([scholaramalgamalg #14, scholaramalgamalg #7, exportiunivamalg #9, scholaramalgamalg #9]) OR (i_category@12 = Women OR i_category@12 = Music OR i_category@12 = Men) AND i_class@10 IN (SET) ([accessories, classical, fragrances, pants]) AND i_brand@8 IN (SET) ([amalgimporto #1, edu packscholar #1, exportiimporto #1, importoamalg #1]), pruning_predicate=(i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Children AND Children <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Electronics AND Electronics <= i_category_max@1) AND (i_class_null_count@6 != row_count@3 AND i_class_min@4 <= personal AND personal <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= portable AND portable <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= reference AND reference <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= self-help AND self-help <= i_class_max@5) AND (i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= scholaramalgamalg #14 AND scholaramalgamalg #14 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= scholaramalgamalg #7 AND scholaramalgamalg #7 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= exportiunivamalg #9 AND exportiunivamalg #9 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= scholaramalgamalg #9 AND scholaramalgamalg #9 <= i_brand_max@8) OR (i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Women AND Women <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Music AND Music <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Men AND Men <= i_category_max@1) AND (i_class_null_count@6 != row_count@3 AND i_class_min@4 <= accessories AND accessories <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= classical AND classical <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= fragrances AND fragrances <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= pants AND pants <= i_class_max@5) AND (i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= amalgimporto #1 AND amalgimporto #1 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= edu packscholar #1 AND edu packscholar #1 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= exportiimporto #1 AND exportiimporto #1 <= i_brand_max@8 OR i_brand_null_count@9 != row_count@3 AND i_brand_min@7 <= importoamalg #1 AND importoamalg #1 <= i_brand_max@8), required_guarantees=[i_brand in (amalgimporto #1, edu packscholar #1, exportiimporto #1, exportiunivamalg #9, importoamalg #1, scholaramalgamalg #14, scholaramalgamalg #7, scholaramalgamalg #9), i_class in (accessories, classical, fragrances, pants, personal, portable, reference, self-help)] + └────────────────────────────────────────────────── + "#); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_64() -> Result<()> { + let display = test_tpcds_query("q64").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ ProjectionExec: expr=[product_name@0 as product_name, store_name@1 as store_name, store_zip@2 as store_zip, b_street_number@3 as b_street_number, b_street_name@4 as b_street_name, b_city@5 as b_city, b_zip@6 as b_zip, c_street_number@7 as c_street_number, c_street_name@8 as c_street_name, c_city@9 as c_city, c_zip@10 as c_zip, cs1syear@11 as cs1syear, cs1cnt@12 as cs1cnt, s11@13 as s11, s21@14 as s21, s31@15 as s31, s12@16 as s12, s22@17 as s22, s32@18 as s32, syear@19 as syear, cnt@20 as cnt] + │ SortPreservingMergeExec: [product_name@0 ASC NULLS LAST, store_name@1 ASC NULLS LAST, cnt@20 ASC NULLS LAST, s1@21 ASC NULLS LAST, s1@22 ASC NULLS LAST] + │ [Stage 46] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 46 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ SortExec: expr=[product_name@0 ASC NULLS LAST, store_name@1 ASC NULLS LAST, cnt@20 ASC NULLS LAST, s11@13 ASC NULLS LAST, s12@16 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[product_name@0 as product_name, store_name@1 as store_name, store_zip@2 as store_zip, b_street_number@3 as b_street_number, b_street_name@4 as b_street_name, b_city@5 as b_city, b_zip@6 as b_zip, c_street_number@7 as c_street_number, c_street_name@8 as c_street_name, c_city@9 as c_city, c_zip@10 as c_zip, syear@11 as cs1syear, cnt@12 as cs1cnt, s1@13 as s11, s2@14 as s21, s3@15 as s31, s1@18 as s12, s2@19 as s22, s3@20 as s32, syear@16 as syear, cnt@17 as cnt, s1@13 as s1, s1@18 as s1] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(item_sk@1, item_sk@0), (store_name@2, store_name@1), (store_zip@3, store_zip@2)], filter=cnt@1 <= cnt@0, projection=[product_name@0, store_name@2, store_zip@3, b_street_number@4, b_street_name@5, b_city@6, b_zip@7, c_street_number@8, c_street_name@9, c_city@10, c_zip@11, syear@12, cnt@13, s1@14, s2@15, s3@16, syear@20, cnt@21, s1@22, s2@23, s3@24] + │ CoalescePartitionsExec + │ [Stage 23] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[i_item_sk@1 as item_sk, s_store_name@2 as store_name, s_zip@3 as store_zip, d_year@12 as syear, count(Int64(1))@15 as cnt, sum(store_sales.ss_wholesale_cost)@16 as s1, sum(store_sales.ss_list_price)@17 as s2, sum(store_sales.ss_coupon_amt)@18 as s3] + │ AggregateExec: mode=FinalPartitioned, gby=[i_product_name@0 as i_product_name, i_item_sk@1 as i_item_sk, s_store_name@2 as s_store_name, s_zip@3 as s_zip, ca_street_number@4 as ca_street_number, ca_street_name@5 as ca_street_name, ca_city@6 as ca_city, ca_zip@7 as ca_zip, ca_street_number@8 as ca_street_number, ca_street_name@9 as ca_street_name, ca_city@10 as ca_city, ca_zip@11 as ca_zip, d_year@12 as d_year, d_year@13 as d_year, d_year@14 as d_year], aggr=[count(Int64(1)), sum(store_sales.ss_wholesale_cost), sum(store_sales.ss_list_price), sum(store_sales.ss_coupon_amt)], ordering_mode=PartiallySorted([12]) + │ [Stage 45] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 23 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=3, consumer_tasks=2, output_partitions=6 + │ ProjectionExec: expr=[i_product_name@0 as product_name, i_item_sk@1 as item_sk, s_store_name@2 as store_name, s_zip@3 as store_zip, ca_street_number@4 as b_street_number, ca_street_name@5 as b_street_name, ca_city@6 as b_city, ca_zip@7 as b_zip, ca_street_number@8 as c_street_number, ca_street_name@9 as c_street_name, ca_city@10 as c_city, ca_zip@11 as c_zip, d_year@12 as syear, count(Int64(1))@15 as cnt, sum(store_sales.ss_wholesale_cost)@16 as s1, sum(store_sales.ss_list_price)@17 as s2, sum(store_sales.ss_coupon_amt)@18 as s3] + │ AggregateExec: mode=FinalPartitioned, gby=[i_product_name@0 as i_product_name, i_item_sk@1 as i_item_sk, s_store_name@2 as s_store_name, s_zip@3 as s_zip, ca_street_number@4 as ca_street_number, ca_street_name@5 as ca_street_name, ca_city@6 as ca_city, ca_zip@7 as ca_zip, ca_street_number@8 as ca_street_number, ca_street_name@9 as ca_street_name, ca_city@10 as ca_city, ca_zip@11 as ca_zip, d_year@12 as d_year, d_year@13 as d_year, d_year@14 as d_year], aggr=[count(Int64(1)), sum(store_sales.ss_wholesale_cost), sum(store_sales.ss_list_price), sum(store_sales.ss_coupon_amt)], ordering_mode=PartiallySorted([12]) + │ [Stage 22] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 22 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([i_product_name@0, i_item_sk@1, s_store_name@2, s_zip@3, ca_street_number@4, ca_street_name@5, ca_city@6, ca_zip@7, ca_street_number@8, ca_street_name@9, ca_city@10, ca_zip@11, d_year@12, d_year@13, d_year@14], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_product_name@17 as i_product_name, i_item_sk@16 as i_item_sk, s_store_name@6 as s_store_name, s_zip@7 as s_zip, ca_street_number@8 as ca_street_number, ca_street_name@9 as ca_street_name, ca_city@10 as ca_city, ca_zip@11 as ca_zip, ca_street_number@12 as ca_street_number, ca_street_name@13 as ca_street_name, ca_city@14 as ca_city, ca_zip@15 as ca_zip, d_year@3 as d_year, d_year@4 as d_year, d_year@5 as d_year], aggr=[count(Int64(1)), sum(store_sales.ss_wholesale_cost), sum(store_sales.ss_list_price), sum(store_sales.ss_coupon_amt)], ordering_mode=PartiallySorted([12]) + │ ProjectionExec: expr=[ss_wholesale_cost@0 as ss_wholesale_cost, ss_list_price@1 as ss_list_price, ss_coupon_amt@2 as ss_coupon_amt, d_year@3 as d_year, d_year@6 as d_year, d_year@7 as d_year, s_store_name@4 as s_store_name, s_zip@5 as s_zip, ca_street_number@8 as ca_street_number, ca_street_name@9 as ca_street_name, ca_city@10 as ca_city, ca_zip@11 as ca_zip, ca_street_number@12 as ca_street_number, ca_street_name@13 as ca_street_name, ca_city@14 as ca_city, ca_zip@15 as ca_zip, i_item_sk@16 as i_item_sk, i_product_name@17 as i_product_name] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_item_sk@0, i_item_sk@0)], projection=[ss_wholesale_cost@1, ss_list_price@2, ss_coupon_amt@3, d_year@4, s_store_name@5, s_zip@6, d_year@7, d_year@8, ca_street_number@9, ca_street_name@10, ca_city@11, ca_zip@12, ca_street_number@13, ca_street_name@14, ca_city@15, ca_zip@16, i_item_sk@17, i_product_name@18] + │ CoalescePartitionsExec + │ [Stage 21] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=3 + │ FilterExec: i_color@2 IN (SET) ([purple, burlywood, indian, spring, floral, medium]) AND i_current_price@1 >= Some(6500),4,2 AND i_current_price@1 <= Some(7400),4,2, projection=[i_item_sk@0, i_product_name@3] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_current_price, i_color, i_product_name], file_type=parquet, predicate=i_color@17 IN (SET) ([purple, burlywood, indian, spring, floral, medium]) AND i_current_price@5 >= Some(6500),4,2 AND i_current_price@5 <= Some(7400),4,2 AND DynamicFilter [ empty ], pruning_predicate=(i_color_null_count@2 != row_count@3 AND i_color_min@0 <= purple AND purple <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= burlywood AND burlywood <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= indian AND indian <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= spring AND spring <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= floral AND floral <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= medium AND medium <= i_color_max@1) AND i_current_price_null_count@5 != row_count@3 AND i_current_price_max@4 >= Some(6500),4,2 AND i_current_price_null_count@5 != row_count@3 AND i_current_price_min@6 <= Some(7400),4,2, required_guarantees=[i_color in (burlywood, floral, indian, medium, purple, spring)] + └────────────────────────────────────────────────── + ┌───── Stage 21 ── Tasks: t0:[p0..p5] t1:[p6..p11] t2:[p12..p17] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(hd_income_band_sk@9, ib_income_band_sk@0)], projection=[ss_item_sk@0, ss_wholesale_cost@1, ss_list_price@2, ss_coupon_amt@3, d_year@4, s_store_name@5, s_zip@6, d_year@7, d_year@8, ca_street_number@10, ca_street_name@11, ca_city@12, ca_zip@13, ca_street_number@14, ca_street_name@15, ca_city@16, ca_zip@17] + │ CoalescePartitionsExec + │ [Stage 20] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=3 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/income_band/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/income_band/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/income_band/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/income_band/part-3.parquet]]}, projection=[ib_income_band_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 20 ── Tasks: t0:[p0..p8] t1:[p9..p17] t2:[p18..p26] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ib_income_band_sk@0, hd_income_band_sk@9)], projection=[ss_item_sk@1, ss_wholesale_cost@2, ss_list_price@3, ss_coupon_amt@4, d_year@5, s_store_name@6, s_zip@7, d_year@8, d_year@9, hd_income_band_sk@11, ca_street_number@12, ca_street_name@13, ca_city@14, ca_zip@15, ca_street_number@16, ca_street_name@17, ca_city@18, ca_zip@19] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_current_addr_sk@7, ca_address_sk@0)], projection=[ss_item_sk@0, ss_wholesale_cost@1, ss_list_price@2, ss_coupon_amt@3, d_year@4, s_store_name@5, s_zip@6, d_year@8, d_year@9, hd_income_band_sk@10, hd_income_band_sk@11, ca_street_number@12, ca_street_name@13, ca_city@14, ca_zip@15, ca_street_number@17, ca_street_name@18, ca_city@19, ca_zip@20] + │ CoalescePartitionsExec + │ [Stage 19] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=3 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_street_number, ca_street_name, ca_city, ca_zip], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/income_band/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/income_band/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/income_band/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/income_band/part-3.parquet]]}, projection=[ib_income_band_sk], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 19 ── Tasks: t0:[p0..p8] t1:[p9..p17] t2:[p18..p26] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_addr_sk@1, CAST(ad1.ca_address_sk AS Float64)@5)], projection=[ss_item_sk@0, ss_wholesale_cost@2, ss_list_price@3, ss_coupon_amt@4, d_year@5, s_store_name@6, s_zip@7, c_current_addr_sk@8, d_year@9, d_year@10, hd_income_band_sk@11, hd_income_band_sk@12, ca_street_number@14, ca_street_name@15, ca_city@16, ca_zip@17] + │ CoalescePartitionsExec + │ [Stage 18] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=3 + │ ProjectionExec: expr=[ca_address_sk@0 as ca_address_sk, ca_street_number@1 as ca_street_number, ca_street_name@2 as ca_street_name, ca_city@3 as ca_city, ca_zip@4 as ca_zip, CAST(ca_address_sk@0 AS Float64) as CAST(ad1.ca_address_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_street_number, ca_street_name, ca_city, ca_zip], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 18 ── Tasks: t0:[p0..p5] t1:[p6..p11] t2:[p12..p17] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_current_hdemo_sk@8, CAST(hd2.hd_demo_sk AS Float64)@2)], projection=[ss_item_sk@0, ss_addr_sk@1, ss_wholesale_cost@2, ss_list_price@3, ss_coupon_amt@4, d_year@5, s_store_name@6, s_zip@7, c_current_addr_sk@9, d_year@10, d_year@11, hd_income_band_sk@12, hd_income_band_sk@14] + │ CoalescePartitionsExec + │ [Stage 17] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=3 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-3.parquet]]}, projection=[hd_demo_sk, hd_income_band_sk, CAST(hd_demo_sk@0 AS Float64) as CAST(hd2.hd_demo_sk AS Float64)], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 17 ── Tasks: t0:[p0..p5] t1:[p6..p11] t2:[p12..p17] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_hdemo_sk@1, CAST(hd1.hd_demo_sk AS Float64)@2)], projection=[ss_item_sk@0, ss_addr_sk@2, ss_wholesale_cost@3, ss_list_price@4, ss_coupon_amt@5, d_year@6, s_store_name@7, s_zip@8, c_current_hdemo_sk@9, c_current_addr_sk@10, d_year@11, d_year@12, hd_income_band_sk@14] + │ CoalescePartitionsExec + │ [Stage 16] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=3 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-3.parquet]]}, projection=[hd_demo_sk, hd_income_band_sk, CAST(hd_demo_sk@0 AS Float64) as CAST(hd1.hd_demo_sk AS Float64)], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 16 ── Tasks: t0:[p0..p5] t1:[p6..p11] t2:[p12..p17] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(promotion.p_promo_sk AS Float64)@1, ss_promo_sk@3)], projection=[ss_item_sk@2, ss_hdemo_sk@3, ss_addr_sk@4, ss_wholesale_cost@6, ss_list_price@7, ss_coupon_amt@8, d_year@9, s_store_name@10, s_zip@11, c_current_hdemo_sk@12, c_current_addr_sk@13, d_year@14, d_year@15] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_current_cdemo_sk@10, CAST(cd2.cd_demo_sk AS Float64)@2)], filter=cd_marital_status@1 != cd_marital_status@0, projection=[ss_item_sk@0, ss_hdemo_sk@1, ss_addr_sk@2, ss_promo_sk@3, ss_wholesale_cost@4, ss_list_price@5, ss_coupon_amt@6, d_year@7, s_store_name@8, s_zip@9, c_current_hdemo_sk@11, c_current_addr_sk@12, d_year@13, d_year@14] + │ CoalescePartitionsExec + │ [Stage 15] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=3 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], ...]}, projection=[cd_demo_sk, cd_marital_status, CAST(cd_demo_sk@0 AS Float64) as CAST(cd2.cd_demo_sk AS Float64)], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/promotion/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/promotion/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/promotion/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/promotion/part-3.parquet]]}, projection=[p_promo_sk, CAST(p_promo_sk@0 AS Float64) as CAST(promotion.p_promo_sk AS Float64)], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 15 ── Tasks: t0:[p0..p5] t1:[p6..p11] t2:[p12..p17] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_cdemo_sk@1, CAST(cd1.cd_demo_sk AS Float64)@2)], projection=[ss_item_sk@0, ss_hdemo_sk@2, ss_addr_sk@3, ss_promo_sk@4, ss_wholesale_cost@5, ss_list_price@6, ss_coupon_amt@7, d_year@8, s_store_name@9, s_zip@10, c_current_cdemo_sk@11, c_current_hdemo_sk@12, c_current_addr_sk@13, d_year@14, d_year@15, cd_marital_status@17] + │ CoalescePartitionsExec + │ [Stage 14] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=3 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], ...]}, projection=[cd_demo_sk, cd_marital_status, CAST(cd_demo_sk@0 AS Float64) as CAST(cd1.cd_demo_sk AS Float64)], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 14 ── Tasks: t0:[p0..p8] t1:[p9..p17] t2:[p18..p26] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_first_shipto_date_sk@14, CAST(d3.d_date_sk AS Float64)@2)], projection=[ss_item_sk@0, ss_cdemo_sk@1, ss_hdemo_sk@2, ss_addr_sk@3, ss_promo_sk@4, ss_wholesale_cost@5, ss_list_price@6, ss_coupon_amt@7, d_year@8, s_store_name@9, s_zip@10, c_current_cdemo_sk@11, c_current_hdemo_sk@12, c_current_addr_sk@13, d_year@15, d_year@17] + │ CoalescePartitionsExec + │ [Stage 13] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=3 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, CAST(d_date_sk@0 AS Float64) as CAST(d3.d_date_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 13 ── Tasks: t0:[p0..p8] t1:[p9..p17] t2:[p18..p26] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_first_sales_date_sk@15, CAST(d2.d_date_sk AS Float64)@2)], projection=[ss_item_sk@0, ss_cdemo_sk@1, ss_hdemo_sk@2, ss_addr_sk@3, ss_promo_sk@4, ss_wholesale_cost@5, ss_list_price@6, ss_coupon_amt@7, d_year@8, s_store_name@9, s_zip@10, c_current_cdemo_sk@11, c_current_hdemo_sk@12, c_current_addr_sk@13, c_first_shipto_date_sk@14, d_year@17] + │ CoalescePartitionsExec + │ [Stage 12] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=3 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, CAST(d_date_sk@0 AS Float64) as CAST(d2.d_date_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 12 ── Tasks: t0:[p0..p8] t1:[p9..p17] t2:[p18..p26] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_customer_sk@1, CAST(customer.c_customer_sk AS Float64)@6)], projection=[ss_item_sk@0, ss_cdemo_sk@2, ss_hdemo_sk@3, ss_addr_sk@4, ss_promo_sk@5, ss_wholesale_cost@6, ss_list_price@7, ss_coupon_amt@8, d_year@9, s_store_name@10, s_zip@11, c_current_cdemo_sk@13, c_current_hdemo_sk@14, c_current_addr_sk@15, c_first_shipto_date_sk@16, c_first_sales_date_sk@17] + │ CoalescePartitionsExec + │ [Stage 11] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=3 + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, c_current_cdemo_sk@1 as c_current_cdemo_sk, c_current_hdemo_sk@2 as c_current_hdemo_sk, c_current_addr_sk@3 as c_current_addr_sk, c_first_shipto_date_sk@4 as c_first_shipto_date_sk, c_first_sales_date_sk@5 as c_first_sales_date_sk, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_current_cdemo_sk, c_current_hdemo_sk, c_current_addr_sk, c_first_shipto_date_sk, c_first_sales_date_sk], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 11 ── Tasks: t0:[p0..p8] t1:[p9..p17] t2:[p18..p26] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[ss_item_sk@2 as ss_item_sk, ss_customer_sk@3 as ss_customer_sk, ss_cdemo_sk@4 as ss_cdemo_sk, ss_hdemo_sk@5 as ss_hdemo_sk, ss_addr_sk@6 as ss_addr_sk, ss_promo_sk@7 as ss_promo_sk, ss_wholesale_cost@8 as ss_wholesale_cost, ss_list_price@9 as ss_list_price, ss_coupon_amt@10 as ss_coupon_amt, d_year@11 as d_year, s_store_name@0 as s_store_name, s_zip@1 as s_zip] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@3, ss_store_sk@5)], projection=[s_store_name@1, s_zip@2, ss_item_sk@4, ss_customer_sk@5, ss_cdemo_sk@6, ss_hdemo_sk@7, ss_addr_sk@8, ss_promo_sk@10, ss_wholesale_cost@11, ss_list_price@12, ss_coupon_amt@13, d_year@14] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[ss_item_sk@1 as ss_item_sk, ss_customer_sk@2 as ss_customer_sk, ss_cdemo_sk@3 as ss_cdemo_sk, ss_hdemo_sk@4 as ss_hdemo_sk, ss_addr_sk@5 as ss_addr_sk, ss_store_sk@6 as ss_store_sk, ss_promo_sk@7 as ss_promo_sk, ss_wholesale_cost@8 as ss_wholesale_cost, ss_list_price@9 as ss_list_price, ss_coupon_amt@10 as ss_coupon_amt, d_year@0 as d_year] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(d1.d_date_sk AS Float64)@2, ss_sold_date_sk@0)], projection=[d_year@1, ss_item_sk@4, ss_customer_sk@5, ss_cdemo_sk@6, ss_hdemo_sk@7, ss_addr_sk@8, ss_store_sk@9, ss_promo_sk@10, ss_wholesale_cost@11, ss_list_price@12, ss_coupon_amt@13] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(cs_item_sk@0, ss_item_sk@1)], projection=[ss_sold_date_sk@1, ss_item_sk@2, ss_customer_sk@3, ss_cdemo_sk@4, ss_hdemo_sk@5, ss_addr_sk@6, ss_store_sk@7, ss_promo_sk@8, ss_wholesale_cost@9, ss_list_price@10, ss_coupon_amt@11] + │ CoalescePartitionsExec + │ [Stage 8] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(sr_item_sk@0, ss_item_sk@1), (sr_ticket_number@1, ss_ticket_number@8)], projection=[ss_sold_date_sk@2, ss_item_sk@3, ss_customer_sk@4, ss_cdemo_sk@5, ss_hdemo_sk@6, ss_addr_sk@7, ss_store_sk@8, ss_promo_sk@9, ss_wholesale_cost@11, ss_list_price@12, ss_coupon_amt@13] + │ [Stage 9] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + │ [Stage 10] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_store_name, s_zip, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, CAST(d_date_sk@0 AS Float64) as CAST(d1.d_date_sk AS Float64)] + │ FilterExec: d_year@1 = 1999 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 1999, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1, required_guarantees=[d_year in (1999)] + └────────────────────────────────────────────────── + ┌───── Stage 8 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ FilterExec: CAST(sum(catalog_sales.cs_ext_list_price)@1 AS Decimal128(38, 2)) > Some(2),20,0 * sum(catalog_returns.cr_refunded_cash + catalog_returns.cr_reversed_charge + catalog_returns.cr_store_credit)@2, projection=[cs_item_sk@0] + │ AggregateExec: mode=FinalPartitioned, gby=[cs_item_sk@0 as cs_item_sk], aggr=[sum(catalog_sales.cs_ext_list_price), sum(catalog_returns.cr_refunded_cash + catalog_returns.cr_reversed_charge + catalog_returns.cr_store_credit)] + │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([cs_item_sk@0], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[cs_item_sk@0 as cs_item_sk], aggr=[sum(catalog_sales.cs_ext_list_price), sum(catalog_returns.cr_refunded_cash + catalog_returns.cr_reversed_charge + catalog_returns.cr_store_credit)] + │ ProjectionExec: expr=[cs_item_sk@3 as cs_item_sk, cs_ext_list_price@4 as cs_ext_list_price, cr_refunded_cash@0 as cr_refunded_cash, cr_reversed_charge@1 as cr_reversed_charge, cr_store_credit@2 as cr_store_credit] + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(cr_item_sk@0, cs_item_sk@0), (cr_order_number@1, cs_order_number@1)], projection=[cr_refunded_cash@2, cr_reversed_charge@3, cr_store_credit@4, cs_item_sk@5, cs_ext_list_price@7] + │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] + │ RepartitionExec: partitioning=Hash([cr_item_sk@0, cr_order_number@1], 9), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], ...]}, projection=[cr_item_sk, cr_order_number, cr_refunded_cash, cr_reversed_charge, cr_store_credit], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] + │ RepartitionExec: partitioning=Hash([cs_item_sk@0, cs_order_number@1], 9), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_item_sk, cs_order_number, cs_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 9 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] + │ RepartitionExec: partitioning=Hash([sr_item_sk@0, sr_ticket_number@1], 9), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], ...]}, projection=[sr_item_sk, sr_ticket_number], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 10 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] + │ RepartitionExec: partitioning=Hash([ss_item_sk@1, ss_ticket_number@8], 9), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_cdemo_sk, ss_hdemo_sk, ss_addr_sk, ss_store_sk, ss_promo_sk, ss_ticket_number, ss_wholesale_cost, ss_list_price, ss_coupon_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 45 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([i_product_name@0, i_item_sk@1, s_store_name@2, s_zip@3, ca_street_number@4, ca_street_name@5, ca_city@6, ca_zip@7, ca_street_number@8, ca_street_name@9, ca_city@10, ca_zip@11, d_year@12, d_year@13, d_year@14], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_product_name@17 as i_product_name, i_item_sk@16 as i_item_sk, s_store_name@6 as s_store_name, s_zip@7 as s_zip, ca_street_number@8 as ca_street_number, ca_street_name@9 as ca_street_name, ca_city@10 as ca_city, ca_zip@11 as ca_zip, ca_street_number@12 as ca_street_number, ca_street_name@13 as ca_street_name, ca_city@14 as ca_city, ca_zip@15 as ca_zip, d_year@3 as d_year, d_year@4 as d_year, d_year@5 as d_year], aggr=[count(Int64(1)), sum(store_sales.ss_wholesale_cost), sum(store_sales.ss_list_price), sum(store_sales.ss_coupon_amt)], ordering_mode=PartiallySorted([12]) + │ ProjectionExec: expr=[ss_wholesale_cost@0 as ss_wholesale_cost, ss_list_price@1 as ss_list_price, ss_coupon_amt@2 as ss_coupon_amt, d_year@3 as d_year, d_year@6 as d_year, d_year@7 as d_year, s_store_name@4 as s_store_name, s_zip@5 as s_zip, ca_street_number@8 as ca_street_number, ca_street_name@9 as ca_street_name, ca_city@10 as ca_city, ca_zip@11 as ca_zip, ca_street_number@12 as ca_street_number, ca_street_name@13 as ca_street_name, ca_city@14 as ca_city, ca_zip@15 as ca_zip, i_item_sk@16 as i_item_sk, i_product_name@17 as i_product_name] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_item_sk@0, i_item_sk@0)], projection=[ss_wholesale_cost@1, ss_list_price@2, ss_coupon_amt@3, d_year@4, s_store_name@5, s_zip@6, d_year@7, d_year@8, ca_street_number@9, ca_street_name@10, ca_city@11, ca_zip@12, ca_street_number@13, ca_street_name@14, ca_city@15, ca_zip@16, i_item_sk@17, i_product_name@18] + │ CoalescePartitionsExec + │ [Stage 44] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=3 + │ FilterExec: i_color@2 IN (SET) ([purple, burlywood, indian, spring, floral, medium]) AND i_current_price@1 >= Some(6500),4,2 AND i_current_price@1 <= Some(7400),4,2, projection=[i_item_sk@0, i_product_name@3] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_current_price, i_color, i_product_name], file_type=parquet, predicate=i_color@17 IN (SET) ([purple, burlywood, indian, spring, floral, medium]) AND i_current_price@5 >= Some(6500),4,2 AND i_current_price@5 <= Some(7400),4,2 AND DynamicFilter [ empty ], pruning_predicate=(i_color_null_count@2 != row_count@3 AND i_color_min@0 <= purple AND purple <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= burlywood AND burlywood <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= indian AND indian <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= spring AND spring <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= floral AND floral <= i_color_max@1 OR i_color_null_count@2 != row_count@3 AND i_color_min@0 <= medium AND medium <= i_color_max@1) AND i_current_price_null_count@5 != row_count@3 AND i_current_price_max@4 >= Some(6500),4,2 AND i_current_price_null_count@5 != row_count@3 AND i_current_price_min@6 <= Some(7400),4,2, required_guarantees=[i_color in (burlywood, floral, indian, medium, purple, spring)] + └────────────────────────────────────────────────── + ┌───── Stage 44 ── Tasks: t0:[p0..p5] t1:[p6..p11] t2:[p12..p17] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(hd_income_band_sk@9, ib_income_band_sk@0)], projection=[ss_item_sk@0, ss_wholesale_cost@1, ss_list_price@2, ss_coupon_amt@3, d_year@4, s_store_name@5, s_zip@6, d_year@7, d_year@8, ca_street_number@10, ca_street_name@11, ca_city@12, ca_zip@13, ca_street_number@14, ca_street_name@15, ca_city@16, ca_zip@17] + │ CoalescePartitionsExec + │ [Stage 43] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=3 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/income_band/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/income_band/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/income_band/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/income_band/part-3.parquet]]}, projection=[ib_income_band_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 43 ── Tasks: t0:[p0..p8] t1:[p9..p17] t2:[p18..p26] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ib_income_band_sk@0, hd_income_band_sk@9)], projection=[ss_item_sk@1, ss_wholesale_cost@2, ss_list_price@3, ss_coupon_amt@4, d_year@5, s_store_name@6, s_zip@7, d_year@8, d_year@9, hd_income_band_sk@11, ca_street_number@12, ca_street_name@13, ca_city@14, ca_zip@15, ca_street_number@16, ca_street_name@17, ca_city@18, ca_zip@19] + │ CoalescePartitionsExec + │ [Stage 24] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_current_addr_sk@7, ca_address_sk@0)], projection=[ss_item_sk@0, ss_wholesale_cost@1, ss_list_price@2, ss_coupon_amt@3, d_year@4, s_store_name@5, s_zip@6, d_year@8, d_year@9, hd_income_band_sk@10, hd_income_band_sk@11, ca_street_number@12, ca_street_name@13, ca_city@14, ca_zip@15, ca_street_number@17, ca_street_name@18, ca_city@19, ca_zip@20] + │ CoalescePartitionsExec + │ [Stage 42] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=3 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_street_number, ca_street_name, ca_city, ca_zip], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 24 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/income_band/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/income_band/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/income_band/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/income_band/part-3.parquet]]}, projection=[ib_income_band_sk], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 42 ── Tasks: t0:[p0..p8] t1:[p9..p17] t2:[p18..p26] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_addr_sk@1, CAST(ad1.ca_address_sk AS Float64)@5)], projection=[ss_item_sk@0, ss_wholesale_cost@2, ss_list_price@3, ss_coupon_amt@4, d_year@5, s_store_name@6, s_zip@7, c_current_addr_sk@8, d_year@9, d_year@10, hd_income_band_sk@11, hd_income_band_sk@12, ca_street_number@14, ca_street_name@15, ca_city@16, ca_zip@17] + │ CoalescePartitionsExec + │ [Stage 41] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=3 + │ ProjectionExec: expr=[ca_address_sk@0 as ca_address_sk, ca_street_number@1 as ca_street_number, ca_street_name@2 as ca_street_name, ca_city@3 as ca_city, ca_zip@4 as ca_zip, CAST(ca_address_sk@0 AS Float64) as CAST(ad1.ca_address_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_street_number, ca_street_name, ca_city, ca_zip], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 41 ── Tasks: t0:[p0..p5] t1:[p6..p11] t2:[p12..p17] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_current_hdemo_sk@8, CAST(hd2.hd_demo_sk AS Float64)@2)], projection=[ss_item_sk@0, ss_addr_sk@1, ss_wholesale_cost@2, ss_list_price@3, ss_coupon_amt@4, d_year@5, s_store_name@6, s_zip@7, c_current_addr_sk@9, d_year@10, d_year@11, hd_income_band_sk@12, hd_income_band_sk@14] + │ CoalescePartitionsExec + │ [Stage 40] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=3 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-3.parquet]]}, projection=[hd_demo_sk, hd_income_band_sk, CAST(hd_demo_sk@0 AS Float64) as CAST(hd2.hd_demo_sk AS Float64)], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 40 ── Tasks: t0:[p0..p5] t1:[p6..p11] t2:[p12..p17] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_hdemo_sk@1, CAST(hd1.hd_demo_sk AS Float64)@2)], projection=[ss_item_sk@0, ss_addr_sk@2, ss_wholesale_cost@3, ss_list_price@4, ss_coupon_amt@5, d_year@6, s_store_name@7, s_zip@8, c_current_hdemo_sk@9, c_current_addr_sk@10, d_year@11, d_year@12, hd_income_band_sk@14] + │ CoalescePartitionsExec + │ [Stage 39] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=3 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-3.parquet]]}, projection=[hd_demo_sk, hd_income_band_sk, CAST(hd_demo_sk@0 AS Float64) as CAST(hd1.hd_demo_sk AS Float64)], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 39 ── Tasks: t0:[p0..p5] t1:[p6..p11] t2:[p12..p17] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(promotion.p_promo_sk AS Float64)@1, ss_promo_sk@3)], projection=[ss_item_sk@2, ss_hdemo_sk@3, ss_addr_sk@4, ss_wholesale_cost@6, ss_list_price@7, ss_coupon_amt@8, d_year@9, s_store_name@10, s_zip@11, c_current_hdemo_sk@12, c_current_addr_sk@13, d_year@14, d_year@15] + │ CoalescePartitionsExec + │ [Stage 25] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_current_cdemo_sk@10, CAST(cd2.cd_demo_sk AS Float64)@2)], filter=cd_marital_status@1 != cd_marital_status@0, projection=[ss_item_sk@0, ss_hdemo_sk@1, ss_addr_sk@2, ss_promo_sk@3, ss_wholesale_cost@4, ss_list_price@5, ss_coupon_amt@6, d_year@7, s_store_name@8, s_zip@9, c_current_hdemo_sk@11, c_current_addr_sk@12, d_year@13, d_year@14] + │ CoalescePartitionsExec + │ [Stage 38] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=3 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], ...]}, projection=[cd_demo_sk, cd_marital_status, CAST(cd_demo_sk@0 AS Float64) as CAST(cd2.cd_demo_sk AS Float64)], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 25 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/promotion/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/promotion/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/promotion/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/promotion/part-3.parquet]]}, projection=[p_promo_sk, CAST(p_promo_sk@0 AS Float64) as CAST(promotion.p_promo_sk AS Float64)], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 38 ── Tasks: t0:[p0..p5] t1:[p6..p11] t2:[p12..p17] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_cdemo_sk@1, CAST(cd1.cd_demo_sk AS Float64)@2)], projection=[ss_item_sk@0, ss_hdemo_sk@2, ss_addr_sk@3, ss_promo_sk@4, ss_wholesale_cost@5, ss_list_price@6, ss_coupon_amt@7, d_year@8, s_store_name@9, s_zip@10, c_current_cdemo_sk@11, c_current_hdemo_sk@12, c_current_addr_sk@13, d_year@14, d_year@15, cd_marital_status@17] + │ CoalescePartitionsExec + │ [Stage 37] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=3 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], ...]}, projection=[cd_demo_sk, cd_marital_status, CAST(cd_demo_sk@0 AS Float64) as CAST(cd1.cd_demo_sk AS Float64)], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 37 ── Tasks: t0:[p0..p8] t1:[p9..p17] t2:[p18..p26] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_first_shipto_date_sk@14, CAST(d3.d_date_sk AS Float64)@2)], projection=[ss_item_sk@0, ss_cdemo_sk@1, ss_hdemo_sk@2, ss_addr_sk@3, ss_promo_sk@4, ss_wholesale_cost@5, ss_list_price@6, ss_coupon_amt@7, d_year@8, s_store_name@9, s_zip@10, c_current_cdemo_sk@11, c_current_hdemo_sk@12, c_current_addr_sk@13, d_year@15, d_year@17] + │ CoalescePartitionsExec + │ [Stage 36] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=3 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, CAST(d_date_sk@0 AS Float64) as CAST(d3.d_date_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 36 ── Tasks: t0:[p0..p8] t1:[p9..p17] t2:[p18..p26] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_first_sales_date_sk@15, CAST(d2.d_date_sk AS Float64)@2)], projection=[ss_item_sk@0, ss_cdemo_sk@1, ss_hdemo_sk@2, ss_addr_sk@3, ss_promo_sk@4, ss_wholesale_cost@5, ss_list_price@6, ss_coupon_amt@7, d_year@8, s_store_name@9, s_zip@10, c_current_cdemo_sk@11, c_current_hdemo_sk@12, c_current_addr_sk@13, c_first_shipto_date_sk@14, d_year@17] + │ CoalescePartitionsExec + │ [Stage 35] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=3 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, CAST(d_date_sk@0 AS Float64) as CAST(d2.d_date_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 35 ── Tasks: t0:[p0..p8] t1:[p9..p17] t2:[p18..p26] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_customer_sk@1, CAST(customer.c_customer_sk AS Float64)@6)], projection=[ss_item_sk@0, ss_cdemo_sk@2, ss_hdemo_sk@3, ss_addr_sk@4, ss_promo_sk@5, ss_wholesale_cost@6, ss_list_price@7, ss_coupon_amt@8, d_year@9, s_store_name@10, s_zip@11, c_current_cdemo_sk@13, c_current_hdemo_sk@14, c_current_addr_sk@15, c_first_shipto_date_sk@16, c_first_sales_date_sk@17] + │ CoalescePartitionsExec + │ [Stage 34] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=3 + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, c_current_cdemo_sk@1 as c_current_cdemo_sk, c_current_hdemo_sk@2 as c_current_hdemo_sk, c_current_addr_sk@3 as c_current_addr_sk, c_first_shipto_date_sk@4 as c_first_shipto_date_sk, c_first_sales_date_sk@5 as c_first_sales_date_sk, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_current_cdemo_sk, c_current_hdemo_sk, c_current_addr_sk, c_first_shipto_date_sk, c_first_sales_date_sk], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 34 ── Tasks: t0:[p0..p8] t1:[p9..p17] t2:[p18..p26] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[ss_item_sk@2 as ss_item_sk, ss_customer_sk@3 as ss_customer_sk, ss_cdemo_sk@4 as ss_cdemo_sk, ss_hdemo_sk@5 as ss_hdemo_sk, ss_addr_sk@6 as ss_addr_sk, ss_promo_sk@7 as ss_promo_sk, ss_wholesale_cost@8 as ss_wholesale_cost, ss_list_price@9 as ss_list_price, ss_coupon_amt@10 as ss_coupon_amt, d_year@11 as d_year, s_store_name@0 as s_store_name, s_zip@1 as s_zip] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@3, ss_store_sk@5)], projection=[s_store_name@1, s_zip@2, ss_item_sk@4, ss_customer_sk@5, ss_cdemo_sk@6, ss_hdemo_sk@7, ss_addr_sk@8, ss_promo_sk@10, ss_wholesale_cost@11, ss_list_price@12, ss_coupon_amt@13, d_year@14] + │ CoalescePartitionsExec + │ [Stage 26] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[ss_item_sk@1 as ss_item_sk, ss_customer_sk@2 as ss_customer_sk, ss_cdemo_sk@3 as ss_cdemo_sk, ss_hdemo_sk@4 as ss_hdemo_sk, ss_addr_sk@5 as ss_addr_sk, ss_store_sk@6 as ss_store_sk, ss_promo_sk@7 as ss_promo_sk, ss_wholesale_cost@8 as ss_wholesale_cost, ss_list_price@9 as ss_list_price, ss_coupon_amt@10 as ss_coupon_amt, d_year@0 as d_year] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(d1.d_date_sk AS Float64)@2, ss_sold_date_sk@0)], projection=[d_year@1, ss_item_sk@4, ss_customer_sk@5, ss_cdemo_sk@6, ss_hdemo_sk@7, ss_addr_sk@8, ss_store_sk@9, ss_promo_sk@10, ss_wholesale_cost@11, ss_list_price@12, ss_coupon_amt@13] + │ CoalescePartitionsExec + │ [Stage 27] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(cs_item_sk@0, ss_item_sk@1)], projection=[ss_sold_date_sk@1, ss_item_sk@2, ss_customer_sk@3, ss_cdemo_sk@4, ss_hdemo_sk@5, ss_addr_sk@6, ss_store_sk@7, ss_promo_sk@8, ss_wholesale_cost@9, ss_list_price@10, ss_coupon_amt@11] + │ CoalescePartitionsExec + │ [Stage 31] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(sr_item_sk@0, ss_item_sk@1), (sr_ticket_number@1, ss_ticket_number@8)], projection=[ss_sold_date_sk@2, ss_item_sk@3, ss_customer_sk@4, ss_cdemo_sk@5, ss_hdemo_sk@6, ss_addr_sk@7, ss_store_sk@8, ss_promo_sk@9, ss_wholesale_cost@11, ss_list_price@12, ss_coupon_amt@13] + │ [Stage 32] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + │ [Stage 33] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 26 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_store_name, s_zip, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 27 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, CAST(d_date_sk@0 AS Float64) as CAST(d1.d_date_sk AS Float64)] + │ FilterExec: d_year@1 = 2000 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] + └────────────────────────────────────────────────── + ┌───── Stage 31 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ FilterExec: CAST(sum(catalog_sales.cs_ext_list_price)@1 AS Decimal128(38, 2)) > Some(2),20,0 * sum(catalog_returns.cr_refunded_cash + catalog_returns.cr_reversed_charge + catalog_returns.cr_store_credit)@2, projection=[cs_item_sk@0] + │ AggregateExec: mode=FinalPartitioned, gby=[cs_item_sk@0 as cs_item_sk], aggr=[sum(catalog_sales.cs_ext_list_price), sum(catalog_returns.cr_refunded_cash + catalog_returns.cr_reversed_charge + catalog_returns.cr_store_credit)] + │ [Stage 30] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 30 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([cs_item_sk@0], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[cs_item_sk@0 as cs_item_sk], aggr=[sum(catalog_sales.cs_ext_list_price), sum(catalog_returns.cr_refunded_cash + catalog_returns.cr_reversed_charge + catalog_returns.cr_store_credit)] + │ ProjectionExec: expr=[cs_item_sk@3 as cs_item_sk, cs_ext_list_price@4 as cs_ext_list_price, cr_refunded_cash@0 as cr_refunded_cash, cr_reversed_charge@1 as cr_reversed_charge, cr_store_credit@2 as cr_store_credit] + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(cr_item_sk@0, cs_item_sk@0), (cr_order_number@1, cs_order_number@1)], projection=[cr_refunded_cash@2, cr_reversed_charge@3, cr_store_credit@4, cs_item_sk@5, cs_ext_list_price@7] + │ [Stage 28] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + │ [Stage 29] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 28 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] + │ RepartitionExec: partitioning=Hash([cr_item_sk@0, cr_order_number@1], 9), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], ...]}, projection=[cr_item_sk, cr_order_number, cr_refunded_cash, cr_reversed_charge, cr_store_credit], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 29 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] + │ RepartitionExec: partitioning=Hash([cs_item_sk@0, cs_order_number@1], 9), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_item_sk, cs_order_number, cs_ext_list_price], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 32 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] + │ RepartitionExec: partitioning=Hash([sr_item_sk@0, sr_ticket_number@1], 9), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], ...]}, projection=[sr_item_sk, sr_ticket_number], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 33 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] + │ RepartitionExec: partitioning=Hash([ss_item_sk@1, ss_ticket_number@8], 9), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_cdemo_sk, ss_hdemo_sk, ss_addr_sk, ss_store_sk, ss_promo_sk, ss_ticket_number, ss_wholesale_cost, ss_list_price, ss_coupon_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_65() -> Result<()> { + let display = test_tpcds_query("q65").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [s_store_name@0 ASC, i_item_desc@1 ASC], fetch=100 + │ [Stage 9] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 9 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ SortExec: TopK(fetch=100), expr=[s_store_name@0 ASC, i_item_desc@1 ASC], preserve_partitioning=[true] + │ ProjectionExec: expr=[s_store_name@0 as s_store_name, i_item_desc@2 as i_item_desc, revenue@1 as revenue, i_current_price@3 as i_current_price, i_wholesale_cost@4 as i_wholesale_cost, i_brand@5 as i_brand] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_store_sk@1, ss_store_sk@0)], filter=CAST(revenue@0 AS Decimal128(30, 15)) <= CAST(0.1 * CAST(ave@1 AS Float64) AS Decimal128(30, 15)), projection=[s_store_name@0, revenue@2, i_item_desc@3, i_current_price@4, i_wholesale_cost@5, i_brand@6] + │ CoalescePartitionsExec + │ [Stage 5] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[ss_store_sk@0 as ss_store_sk, avg(sa.revenue)@1 as ave] + │ AggregateExec: mode=FinalPartitioned, gby=[ss_store_sk@0 as ss_store_sk], aggr=[avg(sa.revenue)] + │ [Stage 8] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=3, consumer_tasks=2, output_partitions=6 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_item_sk@2, i_item_sk@0)], projection=[s_store_name@0, ss_store_sk@1, revenue@3, i_item_desc@5, i_current_price@6, i_wholesale_cost@7, i_brand@8] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=6, input_tasks=2 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_item_desc, i_current_price, i_wholesale_cost, i_brand], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=3, consumer_tasks=2, output_partitions=6 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@2, ss_store_sk@0)], projection=[s_store_name@1, ss_store_sk@3, ss_item_sk@4, revenue@5] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=4, input_tasks=2 + │ ProjectionExec: expr=[ss_store_sk@0 as ss_store_sk, ss_item_sk@1 as ss_item_sk, sum(store_sales.ss_sales_price)@2 as revenue] + │ AggregateExec: mode=FinalPartitioned, gby=[ss_store_sk@0 as ss_store_sk, ss_item_sk@1 as ss_item_sk], aggr=[sum(store_sales.ss_sales_price)] + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p3] t1:[p4..p7] + │ BroadcastExec: input_partitions=2, consumer_tasks=2, output_partitions=4 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_store_name, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([ss_store_sk@0, ss_item_sk@1], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[ss_store_sk@1 as ss_store_sk, ss_item_sk@0 as ss_item_sk], aggr=[sum(store_sales.ss_sales_price)] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_item_sk@3, ss_store_sk@4, ss_sales_price@5] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_month_seq@1 >= 1176 AND d_month_seq@1 <= 1187, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1176 AND d_month_seq@3 <= 1187, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1176 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1187, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 8 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ RepartitionExec: partitioning=Hash([ss_store_sk@0], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[ss_store_sk@0 as ss_store_sk], aggr=[avg(sa.revenue)] + │ ProjectionExec: expr=[ss_store_sk@0 as ss_store_sk, sum(store_sales.ss_sales_price)@2 as revenue] + │ AggregateExec: mode=FinalPartitioned, gby=[ss_store_sk@0 as ss_store_sk, ss_item_sk@1 as ss_item_sk], aggr=[sum(store_sales.ss_sales_price)] + │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([ss_store_sk@0, ss_item_sk@1], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[ss_store_sk@1 as ss_store_sk, ss_item_sk@0 as ss_item_sk], aggr=[sum(store_sales.ss_sales_price)] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_item_sk@3, ss_store_sk@4, ss_sales_price@5] + │ CoalescePartitionsExec + │ [Stage 6] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_month_seq@1 >= 1176 AND d_month_seq@1 <= 1187, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1176 AND d_month_seq@3 <= 1187, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1176 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1187, required_guarantees=[] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_66() -> Result<()> { + let display = test_tpcds_query("q66").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [w_warehouse_name@0 ASC], fetch=100 + │ SortExec: TopK(fetch=100), expr=[w_warehouse_name@0 ASC], preserve_partitioning=[true] + │ ProjectionExec: expr=[w_warehouse_name@0 as w_warehouse_name, w_warehouse_sq_ft@1 as w_warehouse_sq_ft, w_city@2 as w_city, w_county@3 as w_county, w_state@4 as w_state, w_country@5 as w_country, ship_carriers@6 as ship_carriers, year_@7 as year_, sum(x.jan_sales)@8 as jan_sales, sum(x.feb_sales)@9 as feb_sales, sum(x.mar_sales)@10 as mar_sales, sum(x.apr_sales)@11 as apr_sales, sum(x.may_sales)@12 as may_sales, sum(x.jun_sales)@13 as jun_sales, sum(x.jul_sales)@14 as jul_sales, sum(x.aug_sales)@15 as aug_sales, sum(x.sep_sales)@16 as sep_sales, sum(x.oct_sales)@17 as oct_sales, sum(x.nov_sales)@18 as nov_sales, sum(x.dec_sales)@19 as dec_sales, sum(x.jan_sales / x.w_warehouse_sq_ft)@20 as jan_sales_per_sq_foot, sum(x.feb_sales / x.w_warehouse_sq_ft)@21 as feb_sales_per_sq_foot, sum(x.mar_sales / x.w_warehouse_sq_ft)@22 as mar_sales_per_sq_foot, sum(x.apr_sales / x.w_warehouse_sq_ft)@23 as apr_sales_per_sq_foot, sum(x.may_sales / x.w_warehouse_sq_ft)@24 as may_sales_per_sq_foot, sum(x.jun_sales / x.w_warehouse_sq_ft)@25 as jun_sales_per_sq_foot, sum(x.jul_sales / x.w_warehouse_sq_ft)@26 as jul_sales_per_sq_foot, sum(x.aug_sales / x.w_warehouse_sq_ft)@27 as aug_sales_per_sq_foot, sum(x.sep_sales / x.w_warehouse_sq_ft)@28 as sep_sales_per_sq_foot, sum(x.oct_sales / x.w_warehouse_sq_ft)@29 as oct_sales_per_sq_foot, sum(x.nov_sales / x.w_warehouse_sq_ft)@30 as nov_sales_per_sq_foot, sum(x.dec_sales / x.w_warehouse_sq_ft)@31 as dec_sales_per_sq_foot, sum(x.jan_net)@32 as jan_net, sum(x.feb_net)@33 as feb_net, sum(x.mar_net)@34 as mar_net, sum(x.apr_net)@35 as apr_net, sum(x.may_net)@36 as may_net, sum(x.jun_net)@37 as jun_net, sum(x.jul_net)@38 as jul_net, sum(x.aug_net)@39 as aug_net, sum(x.sep_net)@40 as sep_net, sum(x.oct_net)@41 as oct_net, sum(x.nov_net)@42 as nov_net, sum(x.dec_net)@43 as dec_net] + │ AggregateExec: mode=FinalPartitioned, gby=[w_warehouse_name@0 as w_warehouse_name, w_warehouse_sq_ft@1 as w_warehouse_sq_ft, w_city@2 as w_city, w_county@3 as w_county, w_state@4 as w_state, w_country@5 as w_country, ship_carriers@6 as ship_carriers, year_@7 as year_], aggr=[sum(x.jan_sales), sum(x.feb_sales), sum(x.mar_sales), sum(x.apr_sales), sum(x.may_sales), sum(x.jun_sales), sum(x.jul_sales), sum(x.aug_sales), sum(x.sep_sales), sum(x.oct_sales), sum(x.nov_sales), sum(x.dec_sales), sum(x.jan_sales / x.w_warehouse_sq_ft), sum(x.feb_sales / x.w_warehouse_sq_ft), sum(x.mar_sales / x.w_warehouse_sq_ft), sum(x.apr_sales / x.w_warehouse_sq_ft), sum(x.may_sales / x.w_warehouse_sq_ft), sum(x.jun_sales / x.w_warehouse_sq_ft), sum(x.jul_sales / x.w_warehouse_sq_ft), sum(x.aug_sales / x.w_warehouse_sq_ft), sum(x.sep_sales / x.w_warehouse_sq_ft), sum(x.oct_sales / x.w_warehouse_sq_ft), sum(x.nov_sales / x.w_warehouse_sq_ft), sum(x.dec_sales / x.w_warehouse_sq_ft), sum(x.jan_net), sum(x.feb_net), sum(x.mar_net), sum(x.apr_net), sum(x.may_net), sum(x.jun_net), sum(x.jul_net), sum(x.aug_net), sum(x.sep_net), sum(x.oct_net), sum(x.nov_net), sum(x.dec_net)] + │ [Stage 11] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 11 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ RepartitionExec: partitioning=Hash([w_warehouse_name@0, w_warehouse_sq_ft@1, w_city@2, w_county@3, w_state@4, w_country@5, ship_carriers@6, year_@7], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[w_warehouse_name@1 as w_warehouse_name, w_warehouse_sq_ft@2 as w_warehouse_sq_ft, w_city@3 as w_city, w_county@4 as w_county, w_state@5 as w_state, w_country@6 as w_country, ship_carriers@7 as ship_carriers, year_@8 as year_], aggr=[sum(x.jan_sales), sum(x.feb_sales), sum(x.mar_sales), sum(x.apr_sales), sum(x.may_sales), sum(x.jun_sales), sum(x.jul_sales), sum(x.aug_sales), sum(x.sep_sales), sum(x.oct_sales), sum(x.nov_sales), sum(x.dec_sales), sum(x.jan_sales / x.w_warehouse_sq_ft), sum(x.feb_sales / x.w_warehouse_sq_ft), sum(x.mar_sales / x.w_warehouse_sq_ft), sum(x.apr_sales / x.w_warehouse_sq_ft), sum(x.may_sales / x.w_warehouse_sq_ft), sum(x.jun_sales / x.w_warehouse_sq_ft), sum(x.jul_sales / x.w_warehouse_sq_ft), sum(x.aug_sales / x.w_warehouse_sq_ft), sum(x.sep_sales / x.w_warehouse_sq_ft), sum(x.oct_sales / x.w_warehouse_sq_ft), sum(x.nov_sales / x.w_warehouse_sq_ft), sum(x.dec_sales / x.w_warehouse_sq_ft), sum(x.jan_net), sum(x.feb_net), sum(x.mar_net), sum(x.apr_net), sum(x.may_net), sum(x.jun_net), sum(x.jul_net), sum(x.aug_net), sum(x.sep_net), sum(x.oct_net), sum(x.nov_net), sum(x.dec_net)] + │ ProjectionExec: expr=[CAST(w_warehouse_sq_ft@1 AS Float64) as __common_expr_1, w_warehouse_name@0 as w_warehouse_name, w_warehouse_sq_ft@1 as w_warehouse_sq_ft, w_city@2 as w_city, w_county@3 as w_county, w_state@4 as w_state, w_country@5 as w_country, ship_carriers@6 as ship_carriers, year_@7 as year_, jan_sales@8 as jan_sales, feb_sales@9 as feb_sales, mar_sales@10 as mar_sales, apr_sales@11 as apr_sales, may_sales@12 as may_sales, jun_sales@13 as jun_sales, jul_sales@14 as jul_sales, aug_sales@15 as aug_sales, sep_sales@16 as sep_sales, oct_sales@17 as oct_sales, nov_sales@18 as nov_sales, dec_sales@19 as dec_sales, jan_net@20 as jan_net, feb_net@21 as feb_net, mar_net@22 as mar_net, apr_net@23 as apr_net, may_net@24 as may_net, jun_net@25 as jun_net, jul_net@26 as jul_net, aug_net@27 as aug_net, sep_net@28 as sep_net, oct_net@29 as oct_net, nov_net@30 as nov_net, dec_net@31 as dec_net] + │ InterleaveExec + │ ProjectionExec: expr=[w_warehouse_name@0 as w_warehouse_name, w_warehouse_sq_ft@1 as w_warehouse_sq_ft, w_city@2 as w_city, w_county@3 as w_county, w_state@4 as w_state, w_country@5 as w_country, DHL,BARIAN as ship_carriers, d_year@6 as year_, sum(CASE WHEN date_dim.d_moy = Int64(1) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END)@7 as jan_sales, sum(CASE WHEN date_dim.d_moy = Int64(2) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END)@8 as feb_sales, sum(CASE WHEN date_dim.d_moy = Int64(3) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END)@9 as mar_sales, sum(CASE WHEN date_dim.d_moy = Int64(4) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END)@10 as apr_sales, sum(CASE WHEN date_dim.d_moy = Int64(5) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END)@11 as may_sales, sum(CASE WHEN date_dim.d_moy = Int64(6) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END)@12 as jun_sales, sum(CASE WHEN date_dim.d_moy = Int64(7) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END)@13 as jul_sales, sum(CASE WHEN date_dim.d_moy = Int64(8) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END)@14 as aug_sales, sum(CASE WHEN date_dim.d_moy = Int64(9) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END)@15 as sep_sales, sum(CASE WHEN date_dim.d_moy = Int64(10) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END)@16 as oct_sales, sum(CASE WHEN date_dim.d_moy = Int64(11) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END)@17 as nov_sales, sum(CASE WHEN date_dim.d_moy = Int64(12) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END)@18 as dec_sales, sum(CASE WHEN date_dim.d_moy = Int64(1) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END)@19 as jan_net, sum(CASE WHEN date_dim.d_moy = Int64(2) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END)@20 as feb_net, sum(CASE WHEN date_dim.d_moy = Int64(3) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END)@21 as mar_net, sum(CASE WHEN date_dim.d_moy = Int64(4) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END)@22 as apr_net, sum(CASE WHEN date_dim.d_moy = Int64(5) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END)@23 as may_net, sum(CASE WHEN date_dim.d_moy = Int64(6) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END)@24 as jun_net, sum(CASE WHEN date_dim.d_moy = Int64(7) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END)@25 as jul_net, sum(CASE WHEN date_dim.d_moy = Int64(8) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END)@26 as aug_net, sum(CASE WHEN date_dim.d_moy = Int64(9) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END)@27 as sep_net, sum(CASE WHEN date_dim.d_moy = Int64(10) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END)@28 as oct_net, sum(CASE WHEN date_dim.d_moy = Int64(11) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END)@29 as nov_net, sum(CASE WHEN date_dim.d_moy = Int64(12) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END)@30 as dec_net] + │ AggregateExec: mode=FinalPartitioned, gby=[w_warehouse_name@0 as w_warehouse_name, w_warehouse_sq_ft@1 as w_warehouse_sq_ft, w_city@2 as w_city, w_county@3 as w_county, w_state@4 as w_state, w_country@5 as w_country, d_year@6 as d_year], aggr=[sum(CASE WHEN date_dim.d_moy = Int64(1) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(2) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(3) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(4) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(5) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(6) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(7) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(8) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(9) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(10) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(11) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(12) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(1) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(2) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(3) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(4) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(5) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(6) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(7) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(8) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(9) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(10) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(11) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(12) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END)], ordering_mode=PartiallySorted([6]) + │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + │ ProjectionExec: expr=[w_warehouse_name@0 as w_warehouse_name, w_warehouse_sq_ft@1 as w_warehouse_sq_ft, w_city@2 as w_city, w_county@3 as w_county, w_state@4 as w_state, w_country@5 as w_country, DHL,BARIAN as ship_carriers, d_year@6 as year_, sum(CASE WHEN date_dim.d_moy = Int64(1) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END)@7 as jan_sales, sum(CASE WHEN date_dim.d_moy = Int64(2) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END)@8 as feb_sales, sum(CASE WHEN date_dim.d_moy = Int64(3) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END)@9 as mar_sales, sum(CASE WHEN date_dim.d_moy = Int64(4) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END)@10 as apr_sales, sum(CASE WHEN date_dim.d_moy = Int64(5) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END)@11 as may_sales, sum(CASE WHEN date_dim.d_moy = Int64(6) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END)@12 as jun_sales, sum(CASE WHEN date_dim.d_moy = Int64(7) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END)@13 as jul_sales, sum(CASE WHEN date_dim.d_moy = Int64(8) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END)@14 as aug_sales, sum(CASE WHEN date_dim.d_moy = Int64(9) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END)@15 as sep_sales, sum(CASE WHEN date_dim.d_moy = Int64(10) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END)@16 as oct_sales, sum(CASE WHEN date_dim.d_moy = Int64(11) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END)@17 as nov_sales, sum(CASE WHEN date_dim.d_moy = Int64(12) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END)@18 as dec_sales, sum(CASE WHEN date_dim.d_moy = Int64(1) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END)@19 as jan_net, sum(CASE WHEN date_dim.d_moy = Int64(2) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END)@20 as feb_net, sum(CASE WHEN date_dim.d_moy = Int64(3) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END)@21 as mar_net, sum(CASE WHEN date_dim.d_moy = Int64(4) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END)@22 as apr_net, sum(CASE WHEN date_dim.d_moy = Int64(5) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END)@23 as may_net, sum(CASE WHEN date_dim.d_moy = Int64(6) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END)@24 as jun_net, sum(CASE WHEN date_dim.d_moy = Int64(7) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END)@25 as jul_net, sum(CASE WHEN date_dim.d_moy = Int64(8) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END)@26 as aug_net, sum(CASE WHEN date_dim.d_moy = Int64(9) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END)@27 as sep_net, sum(CASE WHEN date_dim.d_moy = Int64(10) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END)@28 as oct_net, sum(CASE WHEN date_dim.d_moy = Int64(11) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END)@29 as nov_net, sum(CASE WHEN date_dim.d_moy = Int64(12) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END)@30 as dec_net] + │ AggregateExec: mode=FinalPartitioned, gby=[w_warehouse_name@0 as w_warehouse_name, w_warehouse_sq_ft@1 as w_warehouse_sq_ft, w_city@2 as w_city, w_county@3 as w_county, w_state@4 as w_state, w_country@5 as w_country, d_year@6 as d_year], aggr=[sum(CASE WHEN date_dim.d_moy = Int64(1) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(2) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(3) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(4) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(5) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(6) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(7) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(8) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(9) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(10) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(11) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(12) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(1) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(2) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(3) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(4) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(5) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(6) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(7) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(8) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(9) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(10) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(11) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(12) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END)], ordering_mode=PartiallySorted([6]) + │ [Stage 10] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([w_warehouse_name@0, w_warehouse_sq_ft@1, w_city@2, w_county@3, w_state@4, w_country@5, d_year@6], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[w_warehouse_name@15 as w_warehouse_name, w_warehouse_sq_ft@16 as w_warehouse_sq_ft, w_city@17 as w_city, w_county@18 as w_county, w_state@19 as w_state, w_country@20 as w_country, d_year@21 as d_year], aggr=[sum(CASE WHEN date_dim.d_moy = Int64(1) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(2) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(3) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(4) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(5) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(6) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(7) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(8) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(9) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(10) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(11) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(12) THEN web_sales.ws_ext_sales_price * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(1) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(2) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(3) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(4) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(5) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(6) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(7) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(8) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(9) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(10) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(11) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(12) THEN web_sales.ws_net_paid * web_sales.ws_quantity ELSE Int64(0) END)], ordering_mode=PartiallySorted([6]) + │ ProjectionExec: expr=[d_moy@10 = 1 as __common_expr_2, d_moy@10 = 2 as __common_expr_3, d_moy@10 = 3 as __common_expr_4, d_moy@10 = 4 as __common_expr_5, d_moy@10 = 5 as __common_expr_6, d_moy@10 = 6 as __common_expr_7, d_moy@10 = 7 as __common_expr_8, d_moy@10 = 8 as __common_expr_9, d_moy@10 = 9 as __common_expr_10, d_moy@10 = 10 as __common_expr_11, d_moy@10 = 11 as __common_expr_12, d_moy@10 = 12 as __common_expr_13, ws_quantity@0 as ws_quantity, ws_ext_sales_price@1 as ws_ext_sales_price, ws_net_paid@2 as ws_net_paid, w_warehouse_name@3 as w_warehouse_name, w_warehouse_sq_ft@4 as w_warehouse_sq_ft, w_city@5 as w_city, w_county@6 as w_county, w_state@7 as w_state, w_country@8 as w_country, d_year@9 as d_year] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(ship_mode.sm_ship_mode_sk AS Float64)@1, ws_ship_mode_sk@0)], projection=[ws_quantity@3, ws_ext_sales_price@4, ws_net_paid@5, w_warehouse_name@6, w_warehouse_sq_ft@7, w_city@8, w_county@9, w_state@10, w_country@11, d_year@12, d_moy@13] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ws_sold_time_sk@0, CAST(time_dim.t_time_sk AS Float64)@1)], projection=[ws_ship_mode_sk@1, ws_quantity@2, ws_ext_sales_price@3, ws_net_paid@4, w_warehouse_name@5, w_warehouse_sq_ft@6, w_city@7, w_county@8, w_state@9, w_country@10, d_year@11, d_moy@12] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=3 + │ ProjectionExec: expr=[t_time_sk@0 as t_time_sk, CAST(t_time_sk@0 AS Float64) as CAST(time_dim.t_time_sk AS Float64)] + │ FilterExec: t_time@1 >= 30838 AND t_time@1 <= 59638, projection=[t_time_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet]]}, projection=[t_time_sk, t_time], file_type=parquet, predicate=t_time@2 >= 30838 AND t_time@2 <= 59638, pruning_predicate=t_time_null_count@1 != row_count@2 AND t_time_max@0 >= 30838 AND t_time_null_count@1 != row_count@2 AND t_time_min@3 <= 59638, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[sm_ship_mode_sk@0 as sm_ship_mode_sk, CAST(sm_ship_mode_sk@0 AS Float64) as CAST(ship_mode.sm_ship_mode_sk AS Float64)] + │ FilterExec: sm_carrier@1 = DHL OR sm_carrier@1 = BARIAN, projection=[sm_ship_mode_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/ship_mode/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/ship_mode/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/ship_mode/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/ship_mode/part-3.parquet]]}, projection=[sm_ship_mode_sk, sm_carrier], file_type=parquet, predicate=sm_carrier@4 = DHL OR sm_carrier@4 = BARIAN, pruning_predicate=sm_carrier_null_count@2 != row_count@3 AND sm_carrier_min@0 <= DHL AND DHL <= sm_carrier_max@1 OR sm_carrier_null_count@2 != row_count@3 AND sm_carrier_min@0 <= BARIAN AND BARIAN <= sm_carrier_max@1, required_guarantees=[sm_carrier in (BARIAN, DHL)] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p8] t1:[p9..p17] t2:[p18..p26] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ws_sold_date_sk@0, CAST(date_dim.d_date_sk AS Float64)@3)], projection=[ws_sold_time_sk@1, ws_ship_mode_sk@2, ws_quantity@3, ws_ext_sales_price@4, ws_net_paid@5, w_warehouse_name@6, w_warehouse_sq_ft@7, w_city@8, w_county@9, w_state@10, w_country@11, d_year@13, d_moy@14] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=3 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, d_moy@2 as d_moy, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_year@1 = 2001 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p5] t1:[p6..p11] t2:[p12..p17] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ ProjectionExec: expr=[ws_sold_date_sk@6 as ws_sold_date_sk, ws_sold_time_sk@7 as ws_sold_time_sk, ws_ship_mode_sk@8 as ws_ship_mode_sk, ws_quantity@9 as ws_quantity, ws_ext_sales_price@10 as ws_ext_sales_price, ws_net_paid@11 as ws_net_paid, w_warehouse_name@0 as w_warehouse_name, w_warehouse_sq_ft@1 as w_warehouse_sq_ft, w_city@2 as w_city, w_county@3 as w_county, w_state@4 as w_state, w_country@5 as w_country] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(warehouse.w_warehouse_sk AS Float64)@7, ws_warehouse_sk@3)], projection=[w_warehouse_name@1, w_warehouse_sq_ft@2, w_city@3, w_county@4, w_state@5, w_country@6, ws_sold_date_sk@8, ws_sold_time_sk@9, ws_ship_mode_sk@10, ws_quantity@12, ws_ext_sales_price@13, ws_net_paid@14] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_sold_time_sk, ws_ship_mode_sk, ws_warehouse_sk, ws_quantity, ws_ext_sales_price, ws_net_paid], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/warehouse/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/warehouse/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/warehouse/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/warehouse/part-3.parquet]]}, projection=[w_warehouse_sk, w_warehouse_name, w_warehouse_sq_ft, w_city, w_county, w_state, w_country, CAST(w_warehouse_sk@0 AS Float64) as CAST(warehouse.w_warehouse_sk AS Float64)], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 10 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([w_warehouse_name@0, w_warehouse_sq_ft@1, w_city@2, w_county@3, w_state@4, w_country@5, d_year@6], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[w_warehouse_name@15 as w_warehouse_name, w_warehouse_sq_ft@16 as w_warehouse_sq_ft, w_city@17 as w_city, w_county@18 as w_county, w_state@19 as w_state, w_country@20 as w_country, d_year@21 as d_year], aggr=[sum(CASE WHEN date_dim.d_moy = Int64(1) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(2) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(3) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(4) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(5) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(6) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(7) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(8) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(9) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(10) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(11) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(12) THEN catalog_sales.cs_sales_price * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(1) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(2) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(3) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(4) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(5) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(6) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(7) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(8) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(9) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(10) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(11) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END), sum(CASE WHEN date_dim.d_moy = Int64(12) THEN catalog_sales.cs_net_paid_inc_tax * catalog_sales.cs_quantity ELSE Int64(0) END)], ordering_mode=PartiallySorted([6]) + │ ProjectionExec: expr=[d_moy@10 = 1 as __common_expr_14, d_moy@10 = 2 as __common_expr_15, d_moy@10 = 3 as __common_expr_16, d_moy@10 = 4 as __common_expr_17, d_moy@10 = 5 as __common_expr_18, d_moy@10 = 6 as __common_expr_19, d_moy@10 = 7 as __common_expr_20, d_moy@10 = 8 as __common_expr_21, d_moy@10 = 9 as __common_expr_22, d_moy@10 = 10 as __common_expr_23, d_moy@10 = 11 as __common_expr_24, d_moy@10 = 12 as __common_expr_25, cs_quantity@0 as cs_quantity, cs_sales_price@1 as cs_sales_price, cs_net_paid_inc_tax@2 as cs_net_paid_inc_tax, w_warehouse_name@3 as w_warehouse_name, w_warehouse_sq_ft@4 as w_warehouse_sq_ft, w_city@5 as w_city, w_county@6 as w_county, w_state@7 as w_state, w_country@8 as w_country, d_year@9 as d_year] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(ship_mode.sm_ship_mode_sk AS Float64)@1, cs_ship_mode_sk@0)], projection=[cs_quantity@3, cs_sales_price@4, cs_net_paid_inc_tax@5, w_warehouse_name@6, w_warehouse_sq_ft@7, w_city@8, w_county@9, w_state@10, w_country@11, d_year@12, d_moy@13] + │ CoalescePartitionsExec + │ [Stage 6] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(cs_sold_time_sk@0, CAST(time_dim.t_time_sk AS Float64)@1)], projection=[cs_ship_mode_sk@1, cs_quantity@2, cs_sales_price@3, cs_net_paid_inc_tax@4, w_warehouse_name@5, w_warehouse_sq_ft@6, w_city@7, w_county@8, w_state@9, w_country@10, d_year@11, d_moy@12] + │ CoalescePartitionsExec + │ [Stage 9] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=3 + │ ProjectionExec: expr=[t_time_sk@0 as t_time_sk, CAST(t_time_sk@0 AS Float64) as CAST(time_dim.t_time_sk AS Float64)] + │ FilterExec: t_time@1 >= 30838 AND t_time@1 <= 59638, projection=[t_time_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet]]}, projection=[t_time_sk, t_time], file_type=parquet, predicate=t_time@2 >= 30838 AND t_time@2 <= 59638, pruning_predicate=t_time_null_count@1 != row_count@2 AND t_time_max@0 >= 30838 AND t_time_null_count@1 != row_count@2 AND t_time_min@3 <= 59638, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[sm_ship_mode_sk@0 as sm_ship_mode_sk, CAST(sm_ship_mode_sk@0 AS Float64) as CAST(ship_mode.sm_ship_mode_sk AS Float64)] + │ FilterExec: sm_carrier@1 = DHL OR sm_carrier@1 = BARIAN, projection=[sm_ship_mode_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/ship_mode/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/ship_mode/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/ship_mode/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/ship_mode/part-3.parquet]]}, projection=[sm_ship_mode_sk, sm_carrier], file_type=parquet, predicate=sm_carrier@4 = DHL OR sm_carrier@4 = BARIAN, pruning_predicate=sm_carrier_null_count@2 != row_count@3 AND sm_carrier_min@0 <= DHL AND DHL <= sm_carrier_max@1 OR sm_carrier_null_count@2 != row_count@3 AND sm_carrier_min@0 <= BARIAN AND BARIAN <= sm_carrier_max@1, required_guarantees=[sm_carrier in (BARIAN, DHL)] + └────────────────────────────────────────────────── + ┌───── Stage 9 ── Tasks: t0:[p0..p8] t1:[p9..p17] t2:[p18..p26] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(cs_sold_date_sk@0, CAST(date_dim.d_date_sk AS Float64)@3)], projection=[cs_sold_time_sk@1, cs_ship_mode_sk@2, cs_quantity@3, cs_sales_price@4, cs_net_paid_inc_tax@5, w_warehouse_name@6, w_warehouse_sq_ft@7, w_city@8, w_county@9, w_state@10, w_country@11, d_year@13, d_moy@14] + │ CoalescePartitionsExec + │ [Stage 8] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=3 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, d_moy@2 as d_moy, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_year@1 = 2001 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + └────────────────────────────────────────────────── + ┌───── Stage 8 ── Tasks: t0:[p0..p5] t1:[p6..p11] t2:[p12..p17] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ ProjectionExec: expr=[cs_sold_date_sk@6 as cs_sold_date_sk, cs_sold_time_sk@7 as cs_sold_time_sk, cs_ship_mode_sk@8 as cs_ship_mode_sk, cs_quantity@9 as cs_quantity, cs_sales_price@10 as cs_sales_price, cs_net_paid_inc_tax@11 as cs_net_paid_inc_tax, w_warehouse_name@0 as w_warehouse_name, w_warehouse_sq_ft@1 as w_warehouse_sq_ft, w_city@2 as w_city, w_county@3 as w_county, w_state@4 as w_state, w_country@5 as w_country] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(warehouse.w_warehouse_sk AS Float64)@7, cs_warehouse_sk@3)], projection=[w_warehouse_name@1, w_warehouse_sq_ft@2, w_city@3, w_county@4, w_state@5, w_country@6, cs_sold_date_sk@8, cs_sold_time_sk@9, cs_ship_mode_sk@10, cs_quantity@12, cs_sales_price@13, cs_net_paid_inc_tax@14] + │ CoalescePartitionsExec + │ [Stage 7] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_sold_time_sk, cs_ship_mode_sk, cs_warehouse_sk, cs_quantity, cs_sales_price, cs_net_paid_inc_tax], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/warehouse/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/warehouse/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/warehouse/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/warehouse/part-3.parquet]]}, projection=[w_warehouse_sk, w_warehouse_name, w_warehouse_sq_ft, w_city, w_county, w_state, w_country, CAST(w_warehouse_sk@0 AS Float64) as CAST(warehouse.w_warehouse_sk AS Float64)], file_type=parquet + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_67() -> Result<()> { + let display = test_tpcds_query("q67").await?; + assert_snapshot!(display, @r#" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [i_category@0 ASC, i_class@1 ASC, i_brand@2 ASC, i_product_name@3 ASC, d_year@4 ASC, d_qoy@5 ASC, d_moy@6 ASC, s_store_id@7 ASC, sumsales@8 ASC, rk@9 ASC], fetch=100 + │ SortExec: TopK(fetch=100), expr=[i_category@0 ASC, i_class@1 ASC, i_brand@2 ASC, i_product_name@3 ASC, d_year@4 ASC, d_qoy@5 ASC, d_moy@6 ASC, s_store_id@7 ASC, sumsales@8 ASC, rk@9 ASC], preserve_partitioning=[true] + │ ProjectionExec: expr=[i_category@0 as i_category, i_class@1 as i_class, i_brand@2 as i_brand, i_product_name@3 as i_product_name, d_year@4 as d_year, d_qoy@5 as d_qoy, d_moy@6 as d_moy, s_store_id@7 as s_store_id, sumsales@8 as sumsales, rank() PARTITION BY [dw1.i_category] ORDER BY [dw1.sumsales DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@9 as rk] + │ FilterExec: rank() PARTITION BY [dw1.i_category] ORDER BY [dw1.sumsales DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@9 <= 100 + │ BoundedWindowAggExec: wdw=[rank() PARTITION BY [dw1.i_category] ORDER BY [dw1.sumsales DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "rank() PARTITION BY [dw1.i_category] ORDER BY [dw1.sumsales DESC NULLS FIRST] RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] + │ SortExec: expr=[i_category@0 ASC NULLS LAST, sumsales@8 DESC], preserve_partitioning=[true] + │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ RepartitionExec: partitioning=Hash([i_category@0], 3), input_partitions=3 + │ ProjectionExec: expr=[i_category@0 as i_category, i_class@1 as i_class, i_brand@2 as i_brand, i_product_name@3 as i_product_name, d_year@4 as d_year, d_qoy@5 as d_qoy, d_moy@6 as d_moy, s_store_id@7 as s_store_id, sum(coalesce(store_sales.ss_sales_price * store_sales.ss_quantity,Int64(0)))@9 as sumsales] + │ AggregateExec: mode=FinalPartitioned, gby=[i_category@0 as i_category, i_class@1 as i_class, i_brand@2 as i_brand, i_product_name@3 as i_product_name, d_year@4 as d_year, d_qoy@5 as d_qoy, d_moy@6 as d_moy, s_store_id@7 as s_store_id, __grouping_id@8 as __grouping_id], aggr=[sum(coalesce(store_sales.ss_sales_price * store_sales.ss_quantity,Int64(0)))] + │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([i_category@0, i_class@1, i_brand@2, i_product_name@3, d_year@4, d_qoy@5, d_moy@6, s_store_id@7, __grouping_id@8], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[(NULL as i_category, NULL as i_class, NULL as i_brand, NULL as i_product_name, NULL as d_year, NULL as d_qoy, NULL as d_moy, NULL as s_store_id), (i_category@7 as i_category, NULL as i_class, NULL as i_brand, NULL as i_product_name, NULL as d_year, NULL as d_qoy, NULL as d_moy, NULL as s_store_id), (i_category@7 as i_category, i_class@6 as i_class, NULL as i_brand, NULL as i_product_name, NULL as d_year, NULL as d_qoy, NULL as d_moy, NULL as s_store_id), (i_category@7 as i_category, i_class@6 as i_class, i_brand@5 as i_brand, NULL as i_product_name, NULL as d_year, NULL as d_qoy, NULL as d_moy, NULL as s_store_id), (i_category@7 as i_category, i_class@6 as i_class, i_brand@5 as i_brand, i_product_name@8 as i_product_name, NULL as d_year, NULL as d_qoy, NULL as d_moy, NULL as s_store_id), (i_category@7 as i_category, i_class@6 as i_class, i_brand@5 as i_brand, i_product_name@8 as i_product_name, d_year@1 as d_year, NULL as d_qoy, NULL as d_moy, NULL as s_store_id), (i_category@7 as i_category, i_class@6 as i_class, i_brand@5 as i_brand, i_product_name@8 as i_product_name, d_year@1 as d_year, d_qoy@3 as d_qoy, NULL as d_moy, NULL as s_store_id), (i_category@7 as i_category, i_class@6 as i_class, i_brand@5 as i_brand, i_product_name@8 as i_product_name, d_year@1 as d_year, d_qoy@3 as d_qoy, d_moy@2 as d_moy, NULL as s_store_id), (i_category@7 as i_category, i_class@6 as i_class, i_brand@5 as i_brand, i_product_name@8 as i_product_name, d_year@1 as d_year, d_qoy@3 as d_qoy, d_moy@2 as d_moy, s_store_id@4 as s_store_id)], aggr=[sum(coalesce(store_sales.ss_sales_price * store_sales.ss_quantity,Int64(0)))] + │ ProjectionExec: expr=[CAST(ss_sales_price@1 AS Float64) * ss_quantity@0 as __common_expr_1, d_year@2 as d_year, d_moy@3 as d_moy, d_qoy@4 as d_qoy, s_store_id@5 as s_store_id, i_brand@6 as i_brand, i_class@7 as i_class, i_category@8 as i_category, i_product_name@9 as i_product_name] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_item_sk@0, i_item_sk@0)], projection=[ss_quantity@1, ss_sales_price@2, d_year@3, d_moy@4, d_qoy@5, s_store_id@6, i_brand@8, i_class@9, i_category@10, i_product_name@11] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=3 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand, i_class, i_category, i_product_name], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p5] t1:[p6..p11] t2:[p12..p17] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ ProjectionExec: expr=[ss_item_sk@1 as ss_item_sk, ss_quantity@2 as ss_quantity, ss_sales_price@3 as ss_sales_price, d_year@4 as d_year, d_moy@5 as d_moy, d_qoy@6 as d_qoy, s_store_id@0 as s_store_id] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@2, ss_store_sk@1)], projection=[s_store_id@1, ss_item_sk@3, ss_quantity@5, ss_sales_price@6, d_year@7, d_moy@8, d_qoy@9] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[ss_item_sk@3 as ss_item_sk, ss_store_sk@4 as ss_store_sk, ss_quantity@5 as ss_quantity, ss_sales_price@6 as ss_sales_price, d_year@0 as d_year, d_moy@1 as d_moy, d_qoy@2 as d_qoy] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@4, ss_sold_date_sk@0)], projection=[d_year@1, d_moy@2, d_qoy@3, ss_item_sk@6, ss_store_sk@7, ss_quantity@8, ss_sales_price@9] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_quantity, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_store_id, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, d_moy@2 as d_moy, d_qoy@3 as d_qoy, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_month_seq@1 >= 1200 AND d_month_seq@1 <= 1211, projection=[d_date_sk@0, d_year@2, d_moy@3, d_qoy@4] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_month_seq, d_year, d_moy, d_qoy], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] + └────────────────────────────────────────────────── + "#); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_68() -> Result<()> { + let display = test_tpcds_query("q68").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [c_last_name@0 ASC, ss_ticket_number@4 ASC], fetch=100 + │ [Stage 8] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 8 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ SortExec: TopK(fetch=100), expr=[c_last_name@0 ASC, ss_ticket_number@4 ASC], preserve_partitioning=[true] + │ ProjectionExec: expr=[c_last_name@6 as c_last_name, c_first_name@5 as c_first_name, ca_city@7 as ca_city, bought_city@1 as bought_city, ss_ticket_number@0 as ss_ticket_number, extended_price@2 as extended_price, extended_tax@4 as extended_tax, list_price@3 as list_price] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_current_addr_sk@5, ca_address_sk@0)], filter=bought_city@0 != ca_city@1, projection=[ss_ticket_number@0, bought_city@1, extended_price@2, list_price@3, extended_tax@4, c_first_name@6, c_last_name@7, ca_city@9] + │ CoalescePartitionsExec + │ [Stage 7] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=6, input_tasks=2 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_city], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=3, consumer_tasks=2, output_partitions=6 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_customer_sk@1, CAST(customer.c_customer_sk AS Float64)@4)], projection=[ss_ticket_number@0, bought_city@2, extended_price@3, list_price@4, extended_tax@5, c_current_addr_sk@7, c_first_name@8, c_last_name@9] + │ CoalescePartitionsExec + │ [Stage 6] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, c_current_addr_sk@1 as c_current_addr_sk, c_first_name@2 as c_first_name, c_last_name@3 as c_last_name, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_current_addr_sk, c_first_name, c_last_name], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=3, consumer_tasks=2, output_partitions=6 + │ ProjectionExec: expr=[ss_ticket_number@0 as ss_ticket_number, ss_customer_sk@1 as ss_customer_sk, ca_city@3 as bought_city, sum(store_sales.ss_ext_sales_price)@4 as extended_price, sum(store_sales.ss_ext_list_price)@5 as list_price, sum(store_sales.ss_ext_tax)@6 as extended_tax] + │ AggregateExec: mode=FinalPartitioned, gby=[ss_ticket_number@0 as ss_ticket_number, ss_customer_sk@1 as ss_customer_sk, ss_addr_sk@2 as ss_addr_sk, ca_city@3 as ca_city], aggr=[sum(store_sales.ss_ext_sales_price), sum(store_sales.ss_ext_list_price), sum(store_sales.ss_ext_tax)] + │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([ss_ticket_number@0, ss_customer_sk@1, ss_addr_sk@2, ca_city@3], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[ss_ticket_number@2 as ss_ticket_number, ss_customer_sk@0 as ss_customer_sk, ss_addr_sk@1 as ss_addr_sk, ca_city@6 as ca_city], aggr=[sum(store_sales.ss_ext_sales_price), sum(store_sales.ss_ext_list_price), sum(store_sales.ss_ext_tax)] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_addr_sk@1, CAST(customer_address.ca_address_sk AS Float64)@2)], projection=[ss_customer_sk@0, ss_addr_sk@1, ss_ticket_number@2, ss_ext_sales_price@3, ss_ext_list_price@4, ss_ext_tax@5, ca_city@7] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=3 + │ ProjectionExec: expr=[ca_address_sk@0 as ca_address_sk, ca_city@1 as ca_city, CAST(ca_address_sk@0 AS Float64) as CAST(customer_address.ca_address_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_city], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p5] t1:[p6..p11] t2:[p12..p17] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(household_demographics.hd_demo_sk AS Float64)@1, ss_hdemo_sk@1)], projection=[ss_customer_sk@2, ss_addr_sk@4, ss_ticket_number@5, ss_ext_sales_price@6, ss_ext_list_price@7, ss_ext_tax@8] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@1, ss_store_sk@3)], projection=[ss_customer_sk@2, ss_hdemo_sk@3, ss_addr_sk@4, ss_ticket_number@6, ss_ext_sales_price@7, ss_ext_list_price@8, ss_ext_tax@9] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_customer_sk@3, ss_hdemo_sk@4, ss_addr_sk@5, ss_store_sk@6, ss_ticket_number@7, ss_ext_sales_price@8, ss_ext_list_price@9, ss_ext_tax@10] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_hdemo_sk, ss_addr_sk, ss_store_sk, ss_ticket_number, ss_ext_sales_price, ss_ext_list_price, ss_ext_tax], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[hd_demo_sk@0 as hd_demo_sk, CAST(hd_demo_sk@0 AS Float64) as CAST(household_demographics.hd_demo_sk AS Float64)] + │ FilterExec: hd_dep_count@1 = 4 OR hd_vehicle_count@2 = 3, projection=[hd_demo_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-3.parquet]]}, projection=[hd_demo_sk, hd_dep_count, hd_vehicle_count], file_type=parquet, predicate=hd_dep_count@3 = 4 OR hd_vehicle_count@4 = 3, pruning_predicate=hd_dep_count_null_count@2 != row_count@3 AND hd_dep_count_min@0 <= 4 AND 4 <= hd_dep_count_max@1 OR hd_vehicle_count_null_count@6 != row_count@3 AND hd_vehicle_count_min@4 <= 3 AND 3 <= hd_vehicle_count_max@5, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] + │ FilterExec: s_city@1 = Fairview OR s_city@1 = Midway, projection=[s_store_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_city], file_type=parquet, predicate=s_city@22 = Fairview OR s_city@22 = Midway, pruning_predicate=s_city_null_count@2 != row_count@3 AND s_city_min@0 <= Fairview AND Fairview <= s_city_max@1 OR s_city_null_count@2 != row_count@3 AND s_city_min@0 <= Midway AND Midway <= s_city_max@1, required_guarantees=[s_city in (Fairview, Midway)] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_dom@2 >= 1 AND d_dom@2 <= 2 AND (d_year@1 = 1999 OR d_year@1 = 2000 OR d_year@1 = 2001), projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_dom], file_type=parquet, predicate=d_dom@9 >= 1 AND d_dom@9 <= 2 AND (d_year@6 = 1999 OR d_year@6 = 2000 OR d_year@6 = 2001), pruning_predicate=d_dom_null_count@1 != row_count@2 AND d_dom_max@0 >= 1 AND d_dom_null_count@1 != row_count@2 AND d_dom_min@3 <= 2 AND (d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5 OR d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5 OR d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5), required_guarantees=[d_year in (1999, 2000, 2001)] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_69() -> Result<()> { + let display = test_tpcds_query("q69").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [cd_gender@0 ASC NULLS LAST, cd_marital_status@1 ASC NULLS LAST, cd_education_status@2 ASC NULLS LAST, cd_purchase_estimate@4 ASC NULLS LAST, cd_credit_rating@6 ASC NULLS LAST], fetch=100 + │ [Stage 10] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 10 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ SortExec: TopK(fetch=100), expr=[cd_gender@0 ASC NULLS LAST, cd_marital_status@1 ASC NULLS LAST, cd_education_status@2 ASC NULLS LAST, cd_purchase_estimate@4 ASC NULLS LAST, cd_credit_rating@6 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[cd_gender@0 as cd_gender, cd_marital_status@1 as cd_marital_status, cd_education_status@2 as cd_education_status, count(Int64(1))@5 as cnt1, cd_purchase_estimate@3 as cd_purchase_estimate, count(Int64(1))@5 as cnt2, cd_credit_rating@4 as cd_credit_rating, count(Int64(1))@5 as cnt3] + │ AggregateExec: mode=FinalPartitioned, gby=[cd_gender@0 as cd_gender, cd_marital_status@1 as cd_marital_status, cd_education_status@2 as cd_education_status, cd_purchase_estimate@3 as cd_purchase_estimate, cd_credit_rating@4 as cd_credit_rating], aggr=[count(Int64(1))] + │ [Stage 9] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 9 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([cd_gender@0, cd_marital_status@1, cd_education_status@2, cd_purchase_estimate@3, cd_credit_rating@4], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[cd_gender@0 as cd_gender, cd_marital_status@1 as cd_marital_status, cd_education_status@2 as cd_education_status, cd_purchase_estimate@3 as cd_purchase_estimate, cd_credit_rating@4 as cd_credit_rating], aggr=[count(Int64(1))] + │ HashJoinExec: mode=CollectLeft, join_type=RightAnti, on=[(cs_ship_customer_sk@0, CAST(c.c_customer_sk AS Float64)@6)], projection=[cd_gender@1, cd_marital_status@2, cd_education_status@3, cd_purchase_estimate@4, cd_credit_rating@5] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=3 + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, cd_gender@1 as cd_gender, cd_marital_status@2 as cd_marital_status, cd_education_status@3 as cd_education_status, cd_purchase_estimate@4 as cd_purchase_estimate, cd_credit_rating@5 as cd_credit_rating, CAST(c_customer_sk@0 AS Float64) as CAST(c.c_customer_sk AS Float64)] + │ HashJoinExec: mode=CollectLeft, join_type=RightAnti, on=[(ws_bill_customer_sk@0, CAST(c.c_customer_sk AS Float64)@6)], projection=[c_customer_sk@0, cd_gender@1, cd_marital_status@2, cd_education_status@3, cd_purchase_estimate@4, cd_credit_rating@5] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=3 + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, cd_gender@1 as cd_gender, cd_marital_status@2 as cd_marital_status, cd_education_status@3 as cd_education_status, cd_purchase_estimate@4 as cd_purchase_estimate, cd_credit_rating@5 as cd_credit_rating, CAST(c_customer_sk@0 AS Float64) as CAST(c.c_customer_sk AS Float64)] + │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(ss_customer_sk@0, CAST(c.c_customer_sk AS Float64)@6)], projection=[c_customer_sk@0, cd_gender@1, cd_marital_status@2, cd_education_status@3, cd_purchase_estimate@4, cd_credit_rating@5] + │ CoalescePartitionsExec + │ [Stage 6] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=3 + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, cd_gender@1 as cd_gender, cd_marital_status@2 as cd_marital_status, cd_education_status@3 as cd_education_status, cd_purchase_estimate@4 as cd_purchase_estimate, cd_credit_rating@5 as cd_credit_rating, CAST(c_customer_sk@0 AS Float64) as CAST(c.c_customer_sk AS Float64)] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_current_cdemo_sk@1, CAST(customer_demographics.cd_demo_sk AS Float64)@6)], projection=[c_customer_sk@0, cd_gender@3, cd_marital_status@4, cd_education_status@5, cd_purchase_estimate@6, cd_credit_rating@7] + │ CoalescePartitionsExec + │ [Stage 8] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], ...]}, projection=[cd_demo_sk, cd_gender, cd_marital_status, cd_education_status, cd_purchase_estimate, cd_credit_rating, CAST(cd_demo_sk@0 AS Float64) as CAST(customer_demographics.cd_demo_sk AS Float64)], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p5] t1:[p6..p11] t2:[p12..p17] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, cs_sold_date_sk@0)], projection=[cs_ship_customer_sk@3] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_ship_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_year@1 = 2001 AND d_moy@2 >= 4 AND d_moy@2 <= 6, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 >= 4 AND d_moy@8 <= 6, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@5 != row_count@3 AND d_moy_max@4 >= 4 AND d_moy_null_count@5 != row_count@3 AND d_moy_min@6 <= 6, required_guarantees=[d_year in (2001)] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p5] t1:[p6..p11] t2:[p12..p17] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ws_sold_date_sk@0)], projection=[ws_bill_customer_sk@3] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_bill_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_year@1 = 2001 AND d_moy@2 >= 4 AND d_moy@2 <= 6, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 >= 4 AND d_moy@8 <= 6, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@5 != row_count@3 AND d_moy_max@4 >= 4 AND d_moy_null_count@5 != row_count@3 AND d_moy_min@6 <= 6, required_guarantees=[d_year in (2001)] + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p5] t1:[p6..p11] t2:[p12..p17] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_customer_sk@3] + │ CoalescePartitionsExec + │ [Stage 5] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_year@1 = 2001 AND d_moy@2 >= 4 AND d_moy@2 <= 6, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 2001 AND d_moy@8 >= 4 AND d_moy@8 <= 6, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1 AND d_moy_null_count@5 != row_count@3 AND d_moy_max@4 >= 4 AND d_moy_null_count@5 != row_count@3 AND d_moy_min@6 <= 6, required_guarantees=[d_year in (2001)] + └────────────────────────────────────────────────── + ┌───── Stage 8 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ca_address_sk@0, c_current_addr_sk@2)], projection=[c_customer_sk@1, c_current_cdemo_sk@2] + │ CoalescePartitionsExec + │ [Stage 7] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=6, input_tasks=2 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_current_cdemo_sk, c_current_addr_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=3, consumer_tasks=2, output_partitions=6 + │ FilterExec: ca_state@1 = KY OR ca_state@1 = GA OR ca_state@1 = NM, projection=[ca_address_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_state], file_type=parquet, predicate=ca_state@8 = KY OR ca_state@8 = GA OR ca_state@8 = NM, pruning_predicate=ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= KY AND KY <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= GA AND GA <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= NM AND NM <= ca_state_max@1, required_guarantees=[ca_state in (GA, KY, NM)] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + #[ignore = "The ordering of the column names in the first nodes is non deterministickI"] + async fn test_tpcds_70() -> Result<()> { + let display = test_tpcds_query("q70").await?; + assert_snapshot!(display, @r#""#); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_71() -> Result<()> { + let display = test_tpcds_query("q71").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [ext_price@4 DESC, brand_id@0 ASC, t_hour@2 ASC] + │ [Stage 5] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ SortExec: expr=[ext_price@4 DESC, brand_id@0 ASC, t_hour@2 ASC], preserve_partitioning=[true] + │ ProjectionExec: expr=[i_brand_id@1 as brand_id, i_brand@0 as brand, t_hour@2 as t_hour, t_minute@3 as t_minute, sum(tmp.ext_price)@4 as ext_price] + │ AggregateExec: mode=FinalPartitioned, gby=[i_brand@0 as i_brand, i_brand_id@1 as i_brand_id, t_hour@2 as t_hour, t_minute@3 as t_minute], aggr=[sum(tmp.ext_price)] + │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] t3:[p0..p5] + │ RepartitionExec: partitioning=Hash([i_brand@0, i_brand_id@1, t_hour@2, t_minute@3], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_brand@1 as i_brand, i_brand_id@0 as i_brand_id, t_hour@3 as t_hour, t_minute@4 as t_minute], aggr=[sum(tmp.ext_price)] + │ ProjectionExec: expr=[i_brand_id@2 as i_brand_id, i_brand@3 as i_brand, ext_price@4 as ext_price, t_hour@0 as t_hour, t_minute@1 as t_minute] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(time_dim.t_time_sk AS Float64)@3, time_sk@3)], projection=[t_hour@1, t_minute@2, i_brand_id@4, i_brand@5, ext_price@6] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=2 + │ ProjectionExec: expr=[i_brand_id@2 as i_brand_id, i_brand@3 as i_brand, ext_price@0 as ext_price, time_sk@1 as time_sk] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(sold_item_sk@1, i_item_sk@0)], projection=[ext_price@0, time_sk@2, i_brand_id@4, i_brand@5] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=12, input_tasks=4 + │ FilterExec: i_manager_id@3 = 1, projection=[i_item_sk@0, i_brand_id@1, i_brand@2] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=1 + │ PartitionIsolatorExec: t0:[p0,__,__,__] t1:[__,p0,__,__] t2:[__,__,p0,__] t3:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_brand, i_manager_id], file_type=parquet, predicate=i_manager_id@20 = 1 AND DynamicFilter [ empty ], pruning_predicate=i_manager_id_null_count@2 != row_count@3 AND i_manager_id_min@0 <= 1 AND 1 <= i_manager_id_max@1, required_guarantees=[i_manager_id in (1)] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p11] t1:[p12..p23] + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 + │ ProjectionExec: expr=[t_time_sk@0 as t_time_sk, t_hour@1 as t_hour, t_minute@2 as t_minute, CAST(t_time_sk@0 AS Float64) as CAST(time_dim.t_time_sk AS Float64)] + │ FilterExec: t_meal_time@3 = breakfast OR t_meal_time@3 = dinner, projection=[t_time_sk@0, t_hour@1, t_minute@2] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet]]}, projection=[t_time_sk, t_hour, t_minute, t_meal_time], file_type=parquet, predicate=t_meal_time@9 = breakfast OR t_meal_time@9 = dinner, pruning_predicate=t_meal_time_null_count@2 != row_count@3 AND t_meal_time_min@0 <= breakfast AND breakfast <= t_meal_time_max@1 OR t_meal_time_null_count@2 != row_count@3 AND t_meal_time_min@0 <= dinner AND dinner <= t_meal_time_max@1, required_guarantees=[t_meal_time in (breakfast, dinner)] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p11] t1:[p12..p23] t2:[p24..p35] t3:[p36..p47] + │ BroadcastExec: input_partitions=3, consumer_tasks=4, output_partitions=12 + │ DistributedUnionExec: t0:[c0] t1:[c1] t2:[c2(0/2)] t3:[c2(1/2)] + │ ProjectionExec: expr=[ws_ext_sales_price@2 as ext_price, ws_item_sk@1 as sold_item_sk, ws_sold_time_sk@0 as time_sk] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ws_sold_date_sk@0)], projection=[ws_sold_time_sk@3, ws_item_sk@4, ws_ext_sales_price@5] + │ CoalescePartitionsExec + │ BroadcastExec: input_partitions=3, consumer_tasks=1, output_partitions=3 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_moy@2 = 11 AND d_year@1 = 1999, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 11 AND d_year@6 = 1999, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5, required_guarantees=[d_moy in (11), d_year in (1999)] + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_sold_date_sk, ws_sold_time_sk, ws_item_sk, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[cs_ext_sales_price@2 as ext_price, cs_item_sk@1 as sold_item_sk, cs_sold_time_sk@0 as time_sk] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, cs_sold_date_sk@0)], projection=[cs_sold_time_sk@3, cs_item_sk@4, cs_ext_sales_price@5] + │ CoalescePartitionsExec + │ BroadcastExec: input_partitions=3, consumer_tasks=1, output_partitions=3 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_moy@2 = 11 AND d_year@1 = 1999, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 11 AND d_year@6 = 1999, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5, required_guarantees=[d_moy in (11), d_year in (1999)] + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-3.parquet:..]]}, projection=[cs_sold_date_sk, cs_sold_time_sk, cs_item_sk, cs_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] + │ ProjectionExec: expr=[ss_ext_sales_price@2 as ext_price, ss_item_sk@1 as sold_item_sk, ss_sold_time_sk@0 as time_sk] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_sold_time_sk@3, ss_item_sk@4, ss_ext_sales_price@5] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=6, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,p2,__,__,__] t1:[__,__,__,p0,p1,p2] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_sold_time_sk, ss_item_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=3, consumer_tasks=2, output_partitions=6 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_moy@2 = 11 AND d_year@1 = 1999, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_moy@8 = 11 AND d_year@6 = 1999, pruning_predicate=d_moy_null_count@2 != row_count@3 AND d_moy_min@0 <= 11 AND 11 <= d_moy_max@1 AND d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5, required_guarantees=[d_moy in (11), d_year in (1999)] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_72() -> Result<()> { + let display = test_tpcds_query("q72").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [total_cnt@5 DESC, i_item_desc@0 ASC, w_warehouse_name@1 ASC, d_week_seq@2 ASC], fetch=100 + │ SortExec: TopK(fetch=100), expr=[total_cnt@5 DESC, i_item_desc@0 ASC, w_warehouse_name@1 ASC, d_week_seq@2 ASC], preserve_partitioning=[true] + │ ProjectionExec: expr=[i_item_desc@0 as i_item_desc, w_warehouse_name@1 as w_warehouse_name, d_week_seq@2 as d_week_seq, sum(CASE WHEN promotion.p_promo_sk IS NULL THEN Int64(1) ELSE Int64(0) END)@3 as no_promo, sum(CASE WHEN promotion.p_promo_sk IS NOT NULL THEN Int64(1) ELSE Int64(0) END)@4 as promo, count(Int64(1))@5 as total_cnt] + │ AggregateExec: mode=FinalPartitioned, gby=[i_item_desc@0 as i_item_desc, w_warehouse_name@1 as w_warehouse_name, d_week_seq@2 as d_week_seq], aggr=[sum(CASE WHEN promotion.p_promo_sk IS NULL THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN promotion.p_promo_sk IS NOT NULL THEN Int64(1) ELSE Int64(0) END), count(Int64(1))] + │ RepartitionExec: partitioning=Hash([i_item_desc@0, w_warehouse_name@1, d_week_seq@2], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_item_desc@1 as i_item_desc, w_warehouse_name@0 as w_warehouse_name, d_week_seq@2 as d_week_seq], aggr=[sum(CASE WHEN promotion.p_promo_sk IS NULL THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN promotion.p_promo_sk IS NOT NULL THEN Int64(1) ELSE Int64(0) END), count(Int64(1))] + │ HashJoinExec: mode=CollectLeft, join_type=Left, on=[(cs_item_sk@0, cr_item_sk@0), (cs_order_number@1, cr_order_number@1)], projection=[w_warehouse_name@2, i_item_desc@3, d_week_seq@4, p_promo_sk@5] + │ CoalescePartitionsExec + │ [Stage 12] => NetworkCoalesceExec: output_partitions=9, input_tasks=3 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-3.parquet:..]]}, projection=[cr_item_sk, cr_order_number], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 12 ── Tasks: t0:[p0..p2] t1:[p3..p5] t2:[p6..p8] + │ ProjectionExec: expr=[cs_item_sk@1 as cs_item_sk, cs_order_number@2 as cs_order_number, w_warehouse_name@3 as w_warehouse_name, i_item_desc@4 as i_item_desc, d_week_seq@5 as d_week_seq, p_promo_sk@0 as p_promo_sk] + │ HashJoinExec: mode=CollectLeft, join_type=Right, on=[(CAST(promotion.p_promo_sk AS Float64)@1, cs_promo_sk@1)], projection=[p_promo_sk@0, cs_item_sk@2, cs_order_number@4, w_warehouse_name@5, i_item_desc@6, d_week_seq@7] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(cs_ship_date_sk@0, CAST(d3.d_date_sk AS Float64)@2)], filter=d_date@1 > d_date@0 + IntervalMonthDayNano { months: 0, days: 5, nanoseconds: 0 }, projection=[cs_item_sk@1, cs_promo_sk@2, cs_order_number@3, w_warehouse_name@4, i_item_desc@5, d_week_seq@7] + │ CoalescePartitionsExec + │ [Stage 11] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=3 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_date@1 as d_date, CAST(d_date_sk@0 AS Float64) as CAST(d3.d_date_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/promotion/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/promotion/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/promotion/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/promotion/part-3.parquet]]}, projection=[p_promo_sk, CAST(p_promo_sk@0 AS Float64) as CAST(promotion.p_promo_sk AS Float64)], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 11 ── Tasks: t0:[p0..p8] t1:[p9..p17] t2:[p18..p26] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(inv_date_sk@4, d_date_sk@0), (d_week_seq@8, d_week_seq@1)], projection=[cs_ship_date_sk@0, cs_item_sk@1, cs_promo_sk@2, cs_order_number@3, w_warehouse_name@5, i_item_desc@6, d_date@7, d_week_seq@8] + │ CoalescePartitionsExec + │ [Stage 10] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=3 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_week_seq], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 10 ── Tasks: t0:[p0..p8] t1:[p9..p17] t2:[p18..p26] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[cs_ship_date_sk@2 as cs_ship_date_sk, cs_item_sk@3 as cs_item_sk, cs_promo_sk@4 as cs_promo_sk, cs_order_number@5 as cs_order_number, inv_date_sk@6 as inv_date_sk, w_warehouse_name@7 as w_warehouse_name, i_item_desc@8 as i_item_desc, d_date@0 as d_date, d_week_seq@1 as d_week_seq] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(d1.d_date_sk AS Float64)@3, cs_sold_date_sk@0)], projection=[d_date@1, d_week_seq@2, cs_ship_date_sk@5, cs_item_sk@6, cs_promo_sk@7, cs_order_number@8, inv_date_sk@9, w_warehouse_name@10, i_item_desc@11] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(household_demographics.hd_demo_sk AS Float64)@1, cs_bill_hdemo_sk@2)], projection=[cs_sold_date_sk@2, cs_ship_date_sk@3, cs_item_sk@5, cs_promo_sk@6, cs_order_number@7, inv_date_sk@8, w_warehouse_name@9, i_item_desc@10] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(customer_demographics.cd_demo_sk AS Float64)@1, cs_bill_cdemo_sk@2)], projection=[cs_sold_date_sk@2, cs_ship_date_sk@3, cs_bill_hdemo_sk@5, cs_item_sk@6, cs_promo_sk@7, cs_order_number@8, inv_date_sk@9, w_warehouse_name@10, i_item_desc@11] + │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + │ [Stage 9] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_date@1 as d_date, d_week_seq@2 as d_week_seq, CAST(d_date_sk@0 AS Float64) as CAST(d1.d_date_sk AS Float64)] + │ FilterExec: d_year@3 = 1999, projection=[d_date_sk@0, d_date@1, d_week_seq@2] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date, d_week_seq, d_year], file_type=parquet, predicate=d_year@6 = 1999, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1, required_guarantees=[d_year in (1999)] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[hd_demo_sk@0 as hd_demo_sk, CAST(hd_demo_sk@0 AS Float64) as CAST(household_demographics.hd_demo_sk AS Float64)] + │ FilterExec: hd_buy_potential@1 = >10000, projection=[hd_demo_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-3.parquet]]}, projection=[hd_demo_sk, hd_buy_potential], file_type=parquet, predicate=hd_buy_potential@2 = >10000, pruning_predicate=hd_buy_potential_null_count@2 != row_count@3 AND hd_buy_potential_min@0 <= >10000 AND >10000 <= hd_buy_potential_max@1, required_guarantees=[hd_buy_potential in (>10000)] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] + │ RepartitionExec: partitioning=Hash([CAST(customer_demographics.cd_demo_sk AS Float64)@1], 9), input_partitions=2 + │ ProjectionExec: expr=[cd_demo_sk@0 as cd_demo_sk, CAST(cd_demo_sk@0 AS Float64) as CAST(customer_demographics.cd_demo_sk AS Float64)] + │ FilterExec: cd_marital_status@1 = D, projection=[cd_demo_sk@0] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], ...]}, projection=[cd_demo_sk, cd_marital_status], file_type=parquet, predicate=cd_marital_status@2 = D, pruning_predicate=cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= D AND D <= cd_marital_status_max@1, required_guarantees=[cd_marital_status in (D)] + └────────────────────────────────────────────────── + ┌───── Stage 9 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] + │ RepartitionExec: partitioning=Hash([cs_bill_cdemo_sk@2], 9), input_partitions=3 + │ ProjectionExec: expr=[cs_sold_date_sk@1 as cs_sold_date_sk, cs_ship_date_sk@2 as cs_ship_date_sk, cs_bill_cdemo_sk@3 as cs_bill_cdemo_sk, cs_bill_hdemo_sk@4 as cs_bill_hdemo_sk, cs_item_sk@5 as cs_item_sk, cs_promo_sk@6 as cs_promo_sk, cs_order_number@7 as cs_order_number, inv_date_sk@8 as inv_date_sk, w_warehouse_name@9 as w_warehouse_name, i_item_desc@0 as i_item_desc] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, cs_item_sk@4)], projection=[i_item_desc@1, cs_sold_date_sk@2, cs_ship_date_sk@3, cs_bill_cdemo_sk@4, cs_bill_hdemo_sk@5, cs_item_sk@6, cs_promo_sk@7, cs_order_number@8, inv_date_sk@9, w_warehouse_name@10] + │ CoalescePartitionsExec + │ [Stage 5] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[cs_sold_date_sk@1 as cs_sold_date_sk, cs_ship_date_sk@2 as cs_ship_date_sk, cs_bill_cdemo_sk@3 as cs_bill_cdemo_sk, cs_bill_hdemo_sk@4 as cs_bill_hdemo_sk, cs_item_sk@5 as cs_item_sk, cs_promo_sk@6 as cs_promo_sk, cs_order_number@7 as cs_order_number, inv_date_sk@8 as inv_date_sk, w_warehouse_name@0 as w_warehouse_name] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(w_warehouse_sk@0, inv_warehouse_sk@8)], projection=[w_warehouse_name@1, cs_sold_date_sk@2, cs_ship_date_sk@3, cs_bill_cdemo_sk@4, cs_bill_hdemo_sk@5, cs_item_sk@6, cs_promo_sk@7, cs_order_number@8, inv_date_sk@9] + │ CoalescePartitionsExec + │ [Stage 6] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=2 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(cs_item_sk@4, inv_item_sk@1)], filter=inv_quantity_on_hand@1 < cs_quantity@0, projection=[cs_sold_date_sk@0, cs_ship_date_sk@1, cs_bill_cdemo_sk@2, cs_bill_hdemo_sk@3, cs_item_sk@4, cs_promo_sk@5, cs_order_number@6, inv_date_sk@8, inv_warehouse_sk@10] + │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + │ [Stage 8] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_item_desc], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/warehouse/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/warehouse/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/warehouse/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/warehouse/part-3.parquet]]}, projection=[w_warehouse_sk, w_warehouse_name], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] + │ RepartitionExec: partitioning=Hash([cs_item_sk@4], 9), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_ship_date_sk, cs_bill_cdemo_sk, cs_bill_hdemo_sk, cs_item_sk, cs_promo_sk, cs_order_number, cs_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 8 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] + │ RepartitionExec: partitioning=Hash([inv_item_sk@1], 9), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/inventory/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:..], ...]}, projection=[inv_date_sk, inv_item_sk, inv_warehouse_sk, inv_quantity_on_hand], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_73() -> Result<()> { + let display = test_tpcds_query("q73").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [cnt@5 DESC, c_last_name@0 ASC NULLS LAST] + │ [Stage 6] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ SortExec: expr=[cnt@5 DESC, c_last_name@0 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[c_last_name@4 as c_last_name, c_first_name@3 as c_first_name, c_salutation@2 as c_salutation, c_preferred_cust_flag@5 as c_preferred_cust_flag, ss_ticket_number@0 as ss_ticket_number, cnt@1 as cnt] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_customer_sk@1, CAST(customer.c_customer_sk AS Float64)@5)], projection=[ss_ticket_number@0, cnt@2, c_salutation@4, c_first_name@5, c_last_name@6, c_preferred_cust_flag@7] + │ CoalescePartitionsExec + │ [Stage 5] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, c_salutation@1 as c_salutation, c_first_name@2 as c_first_name, c_last_name@3 as c_last_name, c_preferred_cust_flag@4 as c_preferred_cust_flag, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_salutation, c_first_name, c_last_name, c_preferred_cust_flag], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=3, consumer_tasks=2, output_partitions=6 + │ ProjectionExec: expr=[ss_ticket_number@0 as ss_ticket_number, ss_customer_sk@1 as ss_customer_sk, count(Int64(1))@2 as cnt] + │ FilterExec: count(Int64(1))@2 >= 1 AND count(Int64(1))@2 <= 5 + │ AggregateExec: mode=FinalPartitioned, gby=[ss_ticket_number@0 as ss_ticket_number, ss_customer_sk@1 as ss_customer_sk], aggr=[count(Int64(1))] + │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([ss_ticket_number@0, ss_customer_sk@1], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[ss_ticket_number@1 as ss_ticket_number, ss_customer_sk@0 as ss_customer_sk], aggr=[count(Int64(1))] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(household_demographics.hd_demo_sk AS Float64)@1, ss_hdemo_sk@1)], projection=[ss_customer_sk@2, ss_ticket_number@4] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@1, ss_store_sk@2)], projection=[ss_customer_sk@2, ss_hdemo_sk@3, ss_ticket_number@5] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_customer_sk@3, ss_hdemo_sk@4, ss_store_sk@5, ss_ticket_number@6] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_hdemo_sk, ss_store_sk, ss_ticket_number], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[hd_demo_sk@0 as hd_demo_sk, CAST(hd_demo_sk@0 AS Float64) as CAST(household_demographics.hd_demo_sk AS Float64)] + │ FilterExec: (hd_buy_potential@1 = Unknown OR hd_buy_potential@1 = >10000) AND hd_vehicle_count@3 > 0 AND CASE WHEN hd_vehicle_count@3 > 0 THEN CAST(hd_dep_count@2 AS Float64) / CAST(hd_vehicle_count@3 AS Float64) END > 1, projection=[hd_demo_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-3.parquet]]}, projection=[hd_demo_sk, hd_buy_potential, hd_dep_count, hd_vehicle_count], file_type=parquet, predicate=(hd_buy_potential@2 = Unknown OR hd_buy_potential@2 = >10000) AND hd_vehicle_count@4 > 0 AND CASE WHEN hd_vehicle_count@4 > 0 THEN CAST(hd_dep_count@3 AS Float64) / CAST(hd_vehicle_count@4 AS Float64) END > 1, pruning_predicate=(hd_buy_potential_null_count@2 != row_count@3 AND hd_buy_potential_min@0 <= Unknown AND Unknown <= hd_buy_potential_max@1 OR hd_buy_potential_null_count@2 != row_count@3 AND hd_buy_potential_min@0 <= >10000 AND >10000 <= hd_buy_potential_max@1) AND hd_vehicle_count_null_count@5 != row_count@3 AND hd_vehicle_count_max@4 > 0, required_guarantees=[hd_buy_potential in (>10000, Unknown)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] + │ FilterExec: s_county@1 IN (SET) ([Orange County, Bronx County, Franklin Parish, Williamson County]), projection=[s_store_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_county], file_type=parquet, predicate=s_county@23 IN (SET) ([Orange County, Bronx County, Franklin Parish, Williamson County]), pruning_predicate=s_county_null_count@2 != row_count@3 AND s_county_min@0 <= Orange County AND Orange County <= s_county_max@1 OR s_county_null_count@2 != row_count@3 AND s_county_min@0 <= Bronx County AND Bronx County <= s_county_max@1 OR s_county_null_count@2 != row_count@3 AND s_county_min@0 <= Franklin Parish AND Franklin Parish <= s_county_max@1 OR s_county_null_count@2 != row_count@3 AND s_county_min@0 <= Williamson County AND Williamson County <= s_county_max@1, required_guarantees=[s_county in (Bronx County, Franklin Parish, Orange County, Williamson County)] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_dom@2 >= 1 AND d_dom@2 <= 2 AND (d_year@1 = 1999 OR d_year@1 = 2000 OR d_year@1 = 2001), projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_dom], file_type=parquet, predicate=d_dom@9 >= 1 AND d_dom@9 <= 2 AND (d_year@6 = 1999 OR d_year@6 = 2000 OR d_year@6 = 2001), pruning_predicate=d_dom_null_count@1 != row_count@2 AND d_dom_max@0 >= 1 AND d_dom_null_count@1 != row_count@2 AND d_dom_min@3 <= 2 AND (d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5 OR d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5 OR d_year_null_count@6 != row_count@2 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5), required_guarantees=[d_year in (1999, 2000, 2001)] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_74() -> Result<()> { + let display = test_tpcds_query("q74").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [customer_id@0 ASC], fetch=100 + │ [Stage 16] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 16 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ SortExec: TopK(fetch=100), expr=[customer_id@0 ASC], preserve_partitioning=[true] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(customer_id@0, customer_id@0)], filter=CASE WHEN year_total@2 > Some(0),17,2 THEN year_total@3 / year_total@2 END > CASE WHEN year_total@0 > Some(0),17,2 THEN year_total@1 / year_total@0 END, projection=[customer_id@2, customer_first_name@3, customer_last_name@4] + │ CoalescePartitionsExec + │ [Stage 12] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[c_customer_id@0 as customer_id, sum(web_sales.ws_net_paid)@4 as year_total] + │ AggregateExec: mode=FinalPartitioned, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, d_year@3 as d_year], aggr=[sum(web_sales.ws_net_paid)], ordering_mode=PartiallySorted([3]) + │ [Stage 15] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 12 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=3, consumer_tasks=2, output_partitions=6 + │ ProjectionExec: expr=[customer_id@1 as customer_id, year_total@2 as year_total, customer_id@3 as customer_id, customer_first_name@4 as customer_first_name, customer_last_name@5 as customer_last_name, year_total@6 as year_total, year_total@0 as year_total] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(customer_id@0, customer_id@0)], projection=[year_total@1, customer_id@2, year_total@3, customer_id@4, customer_first_name@5, customer_last_name@6, year_total@7] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=6, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(customer_id@0, customer_id@0)] + │ CoalescePartitionsExec + │ [Stage 8] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[c_customer_id@0 as customer_id, c_first_name@1 as customer_first_name, c_last_name@2 as customer_last_name, sum(store_sales.ss_net_paid)@4 as year_total] + │ AggregateExec: mode=FinalPartitioned, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, d_year@3 as d_year], aggr=[sum(store_sales.ss_net_paid)], ordering_mode=PartiallySorted([3]) + │ [Stage 11] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=3, consumer_tasks=2, output_partitions=6 + │ ProjectionExec: expr=[c_customer_id@0 as customer_id, sum(web_sales.ws_net_paid)@1 as year_total] + │ FilterExec: sum(web_sales.ws_net_paid)@1 > Some(0),17,2 + │ ProjectionExec: expr=[c_customer_id@0 as c_customer_id, sum(web_sales.ws_net_paid)@4 as sum(web_sales.ws_net_paid)] + │ AggregateExec: mode=FinalPartitioned, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, d_year@3 as d_year], aggr=[sum(web_sales.ws_net_paid)], ordering_mode=PartiallySorted([3]) + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([c_customer_id@0, c_first_name@1, c_last_name@2, d_year@3], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, d_year@4 as d_year], aggr=[sum(web_sales.ws_net_paid)], ordering_mode=PartiallySorted([3]) + │ ProjectionExec: expr=[c_customer_id@1 as c_customer_id, c_first_name@2 as c_first_name, c_last_name@3 as c_last_name, ws_net_paid@4 as ws_net_paid, d_year@0 as d_year] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, ws_sold_date_sk@3)], projection=[d_year@1, c_customer_id@3, c_first_name@4, c_last_name@5, ws_net_paid@7] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(customer.c_customer_sk AS Float64)@4, ws_bill_customer_sk@1)], projection=[c_customer_id@1, c_first_name@2, c_last_name@3, ws_sold_date_sk@5, ws_net_paid@7] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_bill_customer_sk, ws_net_paid], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_year@1 = 2001 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, c_customer_id@1 as c_customer_id, c_first_name@2 as c_first_name, c_last_name@3 as c_last_name, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 8 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=3, consumer_tasks=2, output_partitions=6 + │ ProjectionExec: expr=[c_customer_id@0 as customer_id, sum(store_sales.ss_net_paid)@1 as year_total] + │ FilterExec: sum(store_sales.ss_net_paid)@1 > Some(0),17,2 + │ ProjectionExec: expr=[c_customer_id@0 as c_customer_id, sum(store_sales.ss_net_paid)@4 as sum(store_sales.ss_net_paid)] + │ AggregateExec: mode=FinalPartitioned, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, d_year@3 as d_year], aggr=[sum(store_sales.ss_net_paid)], ordering_mode=PartiallySorted([3]) + │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([c_customer_id@0, c_first_name@1, c_last_name@2, d_year@3], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, d_year@4 as d_year], aggr=[sum(store_sales.ss_net_paid)], ordering_mode=PartiallySorted([3]) + │ ProjectionExec: expr=[c_customer_id@1 as c_customer_id, c_first_name@2 as c_first_name, c_last_name@3 as c_last_name, ss_net_paid@4 as ss_net_paid, d_year@0 as d_year] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, ss_sold_date_sk@3)], projection=[d_year@1, c_customer_id@3, c_first_name@4, c_last_name@5, ss_net_paid@7] + │ CoalescePartitionsExec + │ [Stage 5] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(customer.c_customer_sk AS Float64)@4, ss_customer_sk@1)], projection=[c_customer_id@1, c_first_name@2, c_last_name@3, ss_sold_date_sk@5, ss_net_paid@7] + │ CoalescePartitionsExec + │ [Stage 6] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_net_paid], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_year@1 = 2001 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, c_customer_id@1 as c_customer_id, c_first_name@2 as c_first_name, c_last_name@3 as c_last_name, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 11 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([c_customer_id@0, c_first_name@1, c_last_name@2, d_year@3], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, d_year@4 as d_year], aggr=[sum(store_sales.ss_net_paid)], ordering_mode=PartiallySorted([3]) + │ ProjectionExec: expr=[c_customer_id@1 as c_customer_id, c_first_name@2 as c_first_name, c_last_name@3 as c_last_name, ss_net_paid@4 as ss_net_paid, d_year@0 as d_year] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, ss_sold_date_sk@3)], projection=[d_year@1, c_customer_id@3, c_first_name@4, c_last_name@5, ss_net_paid@7] + │ CoalescePartitionsExec + │ [Stage 9] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(customer.c_customer_sk AS Float64)@4, ss_customer_sk@1)], projection=[c_customer_id@1, c_first_name@2, c_last_name@3, ss_sold_date_sk@5, ss_net_paid@7] + │ CoalescePartitionsExec + │ [Stage 10] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_net_paid], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 9 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_year@1 = 2002 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] + └────────────────────────────────────────────────── + ┌───── Stage 10 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, c_customer_id@1 as c_customer_id, c_first_name@2 as c_first_name, c_last_name@3 as c_last_name, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 15 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([c_customer_id@0, c_first_name@1, c_last_name@2, d_year@3], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, d_year@4 as d_year], aggr=[sum(web_sales.ws_net_paid)], ordering_mode=PartiallySorted([3]) + │ ProjectionExec: expr=[c_customer_id@1 as c_customer_id, c_first_name@2 as c_first_name, c_last_name@3 as c_last_name, ws_net_paid@4 as ws_net_paid, d_year@0 as d_year] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, ws_sold_date_sk@3)], projection=[d_year@1, c_customer_id@3, c_first_name@4, c_last_name@5, ws_net_paid@7] + │ CoalescePartitionsExec + │ [Stage 13] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(customer.c_customer_sk AS Float64)@4, ws_bill_customer_sk@1)], projection=[c_customer_id@1, c_first_name@2, c_last_name@3, ws_sold_date_sk@5, ws_net_paid@7] + │ CoalescePartitionsExec + │ [Stage 14] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_bill_customer_sk, ws_net_paid], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 13 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_year@1 = 2002 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] + └────────────────────────────────────────────────── + ┌───── Stage 14 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, c_customer_id@1 as c_customer_id, c_first_name@2 as c_first_name, c_last_name@3 as c_last_name, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_customer_id, c_first_name, c_last_name], file_type=parquet + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_75() -> Result<()> { + let display = test_tpcds_query("q75").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [sales_cnt_diff@8 ASC NULLS LAST, sales_amt_diff@9 ASC NULLS LAST], fetch=100 + │ SortExec: TopK(fetch=100), expr=[sales_cnt_diff@8 ASC NULLS LAST, sales_amt_diff@9 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[d_year@7 as prev_year, d_year@0 as year_, i_brand_id@1 as i_brand_id, i_class_id@2 as i_class_id, i_category_id@3 as i_category_id, i_manufact_id@4 as i_manufact_id, sales_cnt@8 as prev_yr_cnt, sales_cnt@5 as curr_yr_cnt, sales_cnt@5 - sales_cnt@8 as sales_cnt_diff, sales_amt@6 - sales_amt@9 as sales_amt_diff] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_brand_id@1, i_brand_id@1), (i_class_id@2, i_class_id@2), (i_category_id@3, i_category_id@3), (i_manufact_id@4, i_manufact_id@4)], filter=CAST(sales_cnt@0 AS Decimal128(17, 2)) / CAST(sales_cnt@1 AS Decimal128(17, 2)) < Some(900000),23,6, projection=[d_year@0, i_brand_id@1, i_class_id@2, i_category_id@3, i_manufact_id@4, sales_cnt@5, sales_amt@6, d_year@7, sales_cnt@12, sales_amt@13] + │ CoalescePartitionsExec + │ BroadcastExec: input_partitions=3, consumer_tasks=1, output_partitions=3 + │ ProjectionExec: expr=[d_year@0 as d_year, i_brand_id@1 as i_brand_id, i_class_id@2 as i_class_id, i_category_id@3 as i_category_id, i_manufact_id@4 as i_manufact_id, sum(sales_detail.sales_cnt)@5 as sales_cnt, sum(sales_detail.sales_amt)@6 as sales_amt] + │ AggregateExec: mode=FinalPartitioned, gby=[d_year@0 as d_year, i_brand_id@1 as i_brand_id, i_class_id@2 as i_class_id, i_category_id@3 as i_category_id, i_manufact_id@4 as i_manufact_id], aggr=[sum(sales_detail.sales_cnt), sum(sales_detail.sales_amt)], ordering_mode=PartiallySorted([0]) + │ [Stage 11] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ ProjectionExec: expr=[d_year@0 as d_year, i_brand_id@1 as i_brand_id, i_class_id@2 as i_class_id, i_category_id@3 as i_category_id, i_manufact_id@4 as i_manufact_id, sum(sales_detail.sales_cnt)@5 as sales_cnt, sum(sales_detail.sales_amt)@6 as sales_amt] + │ AggregateExec: mode=FinalPartitioned, gby=[d_year@0 as d_year, i_brand_id@1 as i_brand_id, i_class_id@2 as i_class_id, i_category_id@3 as i_category_id, i_manufact_id@4 as i_manufact_id], aggr=[sum(sales_detail.sales_cnt), sum(sales_detail.sales_amt)], ordering_mode=PartiallySorted([0]) + │ [Stage 22] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 11 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ RepartitionExec: partitioning=Hash([d_year@0, i_brand_id@1, i_class_id@2, i_category_id@3, i_manufact_id@4], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[d_year@0 as d_year, i_brand_id@1 as i_brand_id, i_class_id@2 as i_class_id, i_category_id@3 as i_category_id, i_manufact_id@4 as i_manufact_id], aggr=[sum(sales_detail.sales_cnt), sum(sales_detail.sales_amt)], ordering_mode=PartiallySorted([0]) + │ AggregateExec: mode=FinalPartitioned, gby=[d_year@0 as d_year, i_brand_id@1 as i_brand_id, i_class_id@2 as i_class_id, i_category_id@3 as i_category_id, i_manufact_id@4 as i_manufact_id, sales_cnt@5 as sales_cnt, sales_amt@6 as sales_amt], aggr=[], ordering_mode=PartiallySorted([0]) + │ [Stage 10] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 10 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([d_year@0, i_brand_id@1, i_class_id@2, i_category_id@3, i_manufact_id@4, sales_cnt@5, sales_amt@6], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[d_year@0 as d_year, i_brand_id@1 as i_brand_id, i_class_id@2 as i_class_id, i_category_id@3 as i_category_id, i_manufact_id@4 as i_manufact_id, sales_cnt@5 as sales_cnt, sales_amt@6 as sales_amt], aggr=[], ordering_mode=PartiallySorted([0]) + │ DistributedUnionExec: t0:[c0] t1:[c1] t2:[c2] + │ ProjectionExec: expr=[d_year@7 as d_year, i_brand_id@3 as i_brand_id, i_class_id@4 as i_class_id, i_category_id@5 as i_category_id, i_manufact_id@6 as i_manufact_id, cs_quantity@1 - CASE WHEN cr_return_quantity@8 IS NOT NULL THEN cr_return_quantity@8 ELSE 0 END as sales_cnt, cs_ext_sales_price@2 - CASE WHEN __common_expr_1@0 IS NOT NULL THEN __common_expr_1@0 ELSE Some(0),30,15 END as sales_amt] + │ ProjectionExec: expr=[CAST(cr_return_amount@8 AS Decimal128(30, 15)) as __common_expr_1, cs_quantity@0 as cs_quantity, cs_ext_sales_price@1 as cs_ext_sales_price, i_brand_id@2 as i_brand_id, i_class_id@3 as i_class_id, i_category_id@4 as i_category_id, i_manufact_id@5 as i_manufact_id, d_year@6 as d_year, cr_return_quantity@7 as cr_return_quantity] + │ HashJoinExec: mode=CollectLeft, join_type=Left, on=[(cs_order_number@1, cr_order_number@1), (cs_item_sk@0, cr_item_sk@0)], projection=[cs_quantity@2, cs_ext_sales_price@3, i_brand_id@4, i_class_id@5, i_category_id@6, i_manufact_id@7, d_year@8, cr_return_quantity@11, cr_return_amount@12] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkCoalesceExec: output_partitions=6, input_tasks=3 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-3.parquet:..]]}, projection=[cr_item_sk, cr_order_number, cr_return_quantity, cr_return_amount], file_type=parquet + │ ProjectionExec: expr=[d_year@7 as d_year, i_brand_id@3 as i_brand_id, i_class_id@4 as i_class_id, i_category_id@5 as i_category_id, i_manufact_id@6 as i_manufact_id, ss_quantity@1 - CASE WHEN sr_return_quantity@8 IS NOT NULL THEN sr_return_quantity@8 ELSE 0 END as sales_cnt, ss_ext_sales_price@2 - CASE WHEN __common_expr_2@0 IS NOT NULL THEN __common_expr_2@0 ELSE Some(0),30,15 END as sales_amt] + │ ProjectionExec: expr=[CAST(sr_return_amt@8 AS Decimal128(30, 15)) as __common_expr_2, ss_quantity@0 as ss_quantity, ss_ext_sales_price@1 as ss_ext_sales_price, i_brand_id@2 as i_brand_id, i_class_id@3 as i_class_id, i_category_id@4 as i_category_id, i_manufact_id@5 as i_manufact_id, d_year@6 as d_year, sr_return_quantity@7 as sr_return_quantity] + │ HashJoinExec: mode=CollectLeft, join_type=Left, on=[(ss_ticket_number@1, sr_ticket_number@1), (ss_item_sk@0, sr_item_sk@0)], projection=[ss_quantity@2, ss_ext_sales_price@3, i_brand_id@4, i_class_id@5, i_category_id@6, i_manufact_id@7, d_year@8, sr_return_quantity@11, sr_return_amt@12] + │ CoalescePartitionsExec + │ [Stage 6] => NetworkCoalesceExec: output_partitions=6, input_tasks=3 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_item_sk, sr_ticket_number, sr_return_quantity, sr_return_amt], file_type=parquet + │ ProjectionExec: expr=[d_year@7 as d_year, i_brand_id@3 as i_brand_id, i_class_id@4 as i_class_id, i_category_id@5 as i_category_id, i_manufact_id@6 as i_manufact_id, ws_quantity@1 - CASE WHEN wr_return_quantity@8 IS NOT NULL THEN wr_return_quantity@8 ELSE 0 END as sales_cnt, ws_ext_sales_price@2 - CASE WHEN __common_expr_3@0 IS NOT NULL THEN __common_expr_3@0 ELSE Some(0),30,15 END as sales_amt] + │ ProjectionExec: expr=[CAST(wr_return_amt@8 AS Decimal128(30, 15)) as __common_expr_3, ws_quantity@0 as ws_quantity, ws_ext_sales_price@1 as ws_ext_sales_price, i_brand_id@2 as i_brand_id, i_class_id@3 as i_class_id, i_category_id@4 as i_category_id, i_manufact_id@5 as i_manufact_id, d_year@6 as d_year, wr_return_quantity@7 as wr_return_quantity] + │ HashJoinExec: mode=CollectLeft, join_type=Left, on=[(ws_order_number@1, wr_order_number@1), (ws_item_sk@0, wr_item_sk@0)], projection=[ws_quantity@2, ws_ext_sales_price@3, i_brand_id@4, i_class_id@5, i_category_id@6, i_manufact_id@7, d_year@8, wr_return_quantity@11, wr_return_amt@12] + │ CoalescePartitionsExec + │ [Stage 9] => NetworkCoalesceExec: output_partitions=6, input_tasks=3 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet]]}, projection=[wr_item_sk, wr_order_number, wr_return_quantity, wr_return_amt], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] + │ ProjectionExec: expr=[cs_item_sk@1 as cs_item_sk, cs_order_number@2 as cs_order_number, cs_quantity@3 as cs_quantity, cs_ext_sales_price@4 as cs_ext_sales_price, i_brand_id@5 as i_brand_id, i_class_id@6 as i_class_id, i_category_id@7 as i_category_id, i_manufact_id@8 as i_manufact_id, d_year@0 as d_year] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, cs_sold_date_sk@0)], projection=[d_year@1, cs_item_sk@4, cs_order_number@5, cs_quantity@6, cs_ext_sales_price@7, i_brand_id@8, i_class_id@9, i_category_id@10, i_manufact_id@11] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ ProjectionExec: expr=[cs_sold_date_sk@4 as cs_sold_date_sk, cs_item_sk@5 as cs_item_sk, cs_order_number@6 as cs_order_number, cs_quantity@7 as cs_quantity, cs_ext_sales_price@8 as cs_ext_sales_price, i_brand_id@0 as i_brand_id, i_class_id@1 as i_class_id, i_category_id@2 as i_category_id, i_manufact_id@3 as i_manufact_id] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, cs_item_sk@1)], projection=[i_brand_id@1, i_class_id@2, i_category_id@3, i_manufact_id@4, cs_sold_date_sk@5, cs_item_sk@6, cs_order_number@7, cs_quantity@8, cs_ext_sales_price@9] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_item_sk, cs_order_number, cs_quantity, cs_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_year@1 = 2002 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ FilterExec: i_category@4 = Books, projection=[i_item_sk@0, i_brand_id@1, i_class_id@2, i_category_id@3, i_manufact_id@5] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id, i_category, i_manufact_id], file_type=parquet, predicate=i_category@12 = Books, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1, required_guarantees=[i_category in (Books)] + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] + │ ProjectionExec: expr=[ss_item_sk@1 as ss_item_sk, ss_ticket_number@2 as ss_ticket_number, ss_quantity@3 as ss_quantity, ss_ext_sales_price@4 as ss_ext_sales_price, i_brand_id@5 as i_brand_id, i_class_id@6 as i_class_id, i_category_id@7 as i_category_id, i_manufact_id@8 as i_manufact_id, d_year@0 as d_year] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, ss_sold_date_sk@0)], projection=[d_year@1, ss_item_sk@4, ss_ticket_number@5, ss_quantity@6, ss_ext_sales_price@7, i_brand_id@8, i_class_id@9, i_category_id@10, i_manufact_id@11] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ ProjectionExec: expr=[ss_sold_date_sk@4 as ss_sold_date_sk, ss_item_sk@5 as ss_item_sk, ss_ticket_number@6 as ss_ticket_number, ss_quantity@7 as ss_quantity, ss_ext_sales_price@8 as ss_ext_sales_price, i_brand_id@0 as i_brand_id, i_class_id@1 as i_class_id, i_category_id@2 as i_category_id, i_manufact_id@3 as i_manufact_id] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ss_item_sk@1)], projection=[i_brand_id@1, i_class_id@2, i_category_id@3, i_manufact_id@4, ss_sold_date_sk@5, ss_item_sk@6, ss_ticket_number@7, ss_quantity@8, ss_ext_sales_price@9] + │ CoalescePartitionsExec + │ [Stage 5] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ticket_number, ss_quantity, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_year@1 = 2002 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ FilterExec: i_category@4 = Books, projection=[i_item_sk@0, i_brand_id@1, i_class_id@2, i_category_id@3, i_manufact_id@5] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id, i_category, i_manufact_id], file_type=parquet, predicate=i_category@12 = Books, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1, required_guarantees=[i_category in (Books)] + └────────────────────────────────────────────────── + ┌───── Stage 9 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] + │ ProjectionExec: expr=[ws_item_sk@1 as ws_item_sk, ws_order_number@2 as ws_order_number, ws_quantity@3 as ws_quantity, ws_ext_sales_price@4 as ws_ext_sales_price, i_brand_id@5 as i_brand_id, i_class_id@6 as i_class_id, i_category_id@7 as i_category_id, i_manufact_id@8 as i_manufact_id, d_year@0 as d_year] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, ws_sold_date_sk@0)], projection=[d_year@1, ws_item_sk@4, ws_order_number@5, ws_quantity@6, ws_ext_sales_price@7, i_brand_id@8, i_class_id@9, i_category_id@10, i_manufact_id@11] + │ CoalescePartitionsExec + │ [Stage 7] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ ProjectionExec: expr=[ws_sold_date_sk@4 as ws_sold_date_sk, ws_item_sk@5 as ws_item_sk, ws_order_number@6 as ws_order_number, ws_quantity@7 as ws_quantity, ws_ext_sales_price@8 as ws_ext_sales_price, i_brand_id@0 as i_brand_id, i_class_id@1 as i_class_id, i_category_id@2 as i_category_id, i_manufact_id@3 as i_manufact_id] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ws_item_sk@1)], projection=[i_brand_id@1, i_class_id@2, i_category_id@3, i_manufact_id@4, ws_sold_date_sk@5, ws_item_sk@6, ws_order_number@7, ws_quantity@8, ws_ext_sales_price@9] + │ CoalescePartitionsExec + │ [Stage 8] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_item_sk, ws_order_number, ws_quantity, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_year@1 = 2002 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2002, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2002 AND 2002 <= d_year_max@1, required_guarantees=[d_year in (2002)] + └────────────────────────────────────────────────── + ┌───── Stage 8 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ FilterExec: i_category@4 = Books, projection=[i_item_sk@0, i_brand_id@1, i_class_id@2, i_category_id@3, i_manufact_id@5] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id, i_category, i_manufact_id], file_type=parquet, predicate=i_category@12 = Books, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1, required_guarantees=[i_category in (Books)] + └────────────────────────────────────────────────── + ┌───── Stage 22 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ RepartitionExec: partitioning=Hash([d_year@0, i_brand_id@1, i_class_id@2, i_category_id@3, i_manufact_id@4], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[d_year@0 as d_year, i_brand_id@1 as i_brand_id, i_class_id@2 as i_class_id, i_category_id@3 as i_category_id, i_manufact_id@4 as i_manufact_id], aggr=[sum(sales_detail.sales_cnt), sum(sales_detail.sales_amt)], ordering_mode=PartiallySorted([0]) + │ AggregateExec: mode=FinalPartitioned, gby=[d_year@0 as d_year, i_brand_id@1 as i_brand_id, i_class_id@2 as i_class_id, i_category_id@3 as i_category_id, i_manufact_id@4 as i_manufact_id, sales_cnt@5 as sales_cnt, sales_amt@6 as sales_amt], aggr=[], ordering_mode=PartiallySorted([0]) + │ [Stage 21] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 21 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([d_year@0, i_brand_id@1, i_class_id@2, i_category_id@3, i_manufact_id@4, sales_cnt@5, sales_amt@6], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[d_year@0 as d_year, i_brand_id@1 as i_brand_id, i_class_id@2 as i_class_id, i_category_id@3 as i_category_id, i_manufact_id@4 as i_manufact_id, sales_cnt@5 as sales_cnt, sales_amt@6 as sales_amt], aggr=[], ordering_mode=PartiallySorted([0]) + │ DistributedUnionExec: t0:[c0] t1:[c1] t2:[c2] + │ ProjectionExec: expr=[d_year@7 as d_year, i_brand_id@3 as i_brand_id, i_class_id@4 as i_class_id, i_category_id@5 as i_category_id, i_manufact_id@6 as i_manufact_id, cs_quantity@1 - CASE WHEN cr_return_quantity@8 IS NOT NULL THEN cr_return_quantity@8 ELSE 0 END as sales_cnt, cs_ext_sales_price@2 - CASE WHEN __common_expr_4@0 IS NOT NULL THEN __common_expr_4@0 ELSE Some(0),30,15 END as sales_amt] + │ ProjectionExec: expr=[CAST(cr_return_amount@8 AS Decimal128(30, 15)) as __common_expr_4, cs_quantity@0 as cs_quantity, cs_ext_sales_price@1 as cs_ext_sales_price, i_brand_id@2 as i_brand_id, i_class_id@3 as i_class_id, i_category_id@4 as i_category_id, i_manufact_id@5 as i_manufact_id, d_year@6 as d_year, cr_return_quantity@7 as cr_return_quantity] + │ HashJoinExec: mode=CollectLeft, join_type=Left, on=[(cs_order_number@1, cr_order_number@1), (cs_item_sk@0, cr_item_sk@0)], projection=[cs_quantity@2, cs_ext_sales_price@3, i_brand_id@4, i_class_id@5, i_category_id@6, i_manufact_id@7, d_year@8, cr_return_quantity@11, cr_return_amount@12] + │ CoalescePartitionsExec + │ [Stage 14] => NetworkCoalesceExec: output_partitions=6, input_tasks=3 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-3.parquet:..]]}, projection=[cr_item_sk, cr_order_number, cr_return_quantity, cr_return_amount], file_type=parquet + │ ProjectionExec: expr=[d_year@7 as d_year, i_brand_id@3 as i_brand_id, i_class_id@4 as i_class_id, i_category_id@5 as i_category_id, i_manufact_id@6 as i_manufact_id, ss_quantity@1 - CASE WHEN sr_return_quantity@8 IS NOT NULL THEN sr_return_quantity@8 ELSE 0 END as sales_cnt, ss_ext_sales_price@2 - CASE WHEN __common_expr_5@0 IS NOT NULL THEN __common_expr_5@0 ELSE Some(0),30,15 END as sales_amt] + │ ProjectionExec: expr=[CAST(sr_return_amt@8 AS Decimal128(30, 15)) as __common_expr_5, ss_quantity@0 as ss_quantity, ss_ext_sales_price@1 as ss_ext_sales_price, i_brand_id@2 as i_brand_id, i_class_id@3 as i_class_id, i_category_id@4 as i_category_id, i_manufact_id@5 as i_manufact_id, d_year@6 as d_year, sr_return_quantity@7 as sr_return_quantity] + │ HashJoinExec: mode=CollectLeft, join_type=Left, on=[(ss_ticket_number@1, sr_ticket_number@1), (ss_item_sk@0, sr_item_sk@0)], projection=[ss_quantity@2, ss_ext_sales_price@3, i_brand_id@4, i_class_id@5, i_category_id@6, i_manufact_id@7, d_year@8, sr_return_quantity@11, sr_return_amt@12] + │ CoalescePartitionsExec + │ [Stage 17] => NetworkCoalesceExec: output_partitions=6, input_tasks=3 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/store_returns/part-3.parquet:..]]}, projection=[sr_item_sk, sr_ticket_number, sr_return_quantity, sr_return_amt], file_type=parquet + │ ProjectionExec: expr=[d_year@7 as d_year, i_brand_id@3 as i_brand_id, i_class_id@4 as i_class_id, i_category_id@5 as i_category_id, i_manufact_id@6 as i_manufact_id, ws_quantity@1 - CASE WHEN wr_return_quantity@8 IS NOT NULL THEN wr_return_quantity@8 ELSE 0 END as sales_cnt, ws_ext_sales_price@2 - CASE WHEN __common_expr_6@0 IS NOT NULL THEN __common_expr_6@0 ELSE Some(0),30,15 END as sales_amt] + │ ProjectionExec: expr=[CAST(wr_return_amt@8 AS Decimal128(30, 15)) as __common_expr_6, ws_quantity@0 as ws_quantity, ws_ext_sales_price@1 as ws_ext_sales_price, i_brand_id@2 as i_brand_id, i_class_id@3 as i_class_id, i_category_id@4 as i_category_id, i_manufact_id@5 as i_manufact_id, d_year@6 as d_year, wr_return_quantity@7 as wr_return_quantity] + │ HashJoinExec: mode=CollectLeft, join_type=Left, on=[(ws_order_number@1, wr_order_number@1), (ws_item_sk@0, wr_item_sk@0)], projection=[ws_quantity@2, ws_ext_sales_price@3, i_brand_id@4, i_class_id@5, i_category_id@6, i_manufact_id@7, d_year@8, wr_return_quantity@11, wr_return_amt@12] + │ CoalescePartitionsExec + │ [Stage 20] => NetworkCoalesceExec: output_partitions=6, input_tasks=3 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet]]}, projection=[wr_item_sk, wr_order_number, wr_return_quantity, wr_return_amt], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 14 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] + │ ProjectionExec: expr=[cs_item_sk@1 as cs_item_sk, cs_order_number@2 as cs_order_number, cs_quantity@3 as cs_quantity, cs_ext_sales_price@4 as cs_ext_sales_price, i_brand_id@5 as i_brand_id, i_class_id@6 as i_class_id, i_category_id@7 as i_category_id, i_manufact_id@8 as i_manufact_id, d_year@0 as d_year] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, cs_sold_date_sk@0)], projection=[d_year@1, cs_item_sk@4, cs_order_number@5, cs_quantity@6, cs_ext_sales_price@7, i_brand_id@8, i_class_id@9, i_category_id@10, i_manufact_id@11] + │ CoalescePartitionsExec + │ [Stage 12] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ ProjectionExec: expr=[cs_sold_date_sk@4 as cs_sold_date_sk, cs_item_sk@5 as cs_item_sk, cs_order_number@6 as cs_order_number, cs_quantity@7 as cs_quantity, cs_ext_sales_price@8 as cs_ext_sales_price, i_brand_id@0 as i_brand_id, i_class_id@1 as i_class_id, i_category_id@2 as i_category_id, i_manufact_id@3 as i_manufact_id] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, cs_item_sk@1)], projection=[i_brand_id@1, i_class_id@2, i_category_id@3, i_manufact_id@4, cs_sold_date_sk@5, cs_item_sk@6, cs_order_number@7, cs_quantity@8, cs_ext_sales_price@9] + │ CoalescePartitionsExec + │ [Stage 13] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_item_sk, cs_order_number, cs_quantity, cs_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 12 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_year@1 = 2001 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + └────────────────────────────────────────────────── + ┌───── Stage 13 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ FilterExec: i_category@4 = Books, projection=[i_item_sk@0, i_brand_id@1, i_class_id@2, i_category_id@3, i_manufact_id@5] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id, i_category, i_manufact_id], file_type=parquet, predicate=i_category@12 = Books, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1, required_guarantees=[i_category in (Books)] + └────────────────────────────────────────────────── + ┌───── Stage 17 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] + │ ProjectionExec: expr=[ss_item_sk@1 as ss_item_sk, ss_ticket_number@2 as ss_ticket_number, ss_quantity@3 as ss_quantity, ss_ext_sales_price@4 as ss_ext_sales_price, i_brand_id@5 as i_brand_id, i_class_id@6 as i_class_id, i_category_id@7 as i_category_id, i_manufact_id@8 as i_manufact_id, d_year@0 as d_year] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, ss_sold_date_sk@0)], projection=[d_year@1, ss_item_sk@4, ss_ticket_number@5, ss_quantity@6, ss_ext_sales_price@7, i_brand_id@8, i_class_id@9, i_category_id@10, i_manufact_id@11] + │ CoalescePartitionsExec + │ [Stage 15] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ ProjectionExec: expr=[ss_sold_date_sk@4 as ss_sold_date_sk, ss_item_sk@5 as ss_item_sk, ss_ticket_number@6 as ss_ticket_number, ss_quantity@7 as ss_quantity, ss_ext_sales_price@8 as ss_ext_sales_price, i_brand_id@0 as i_brand_id, i_class_id@1 as i_class_id, i_category_id@2 as i_category_id, i_manufact_id@3 as i_manufact_id] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ss_item_sk@1)], projection=[i_brand_id@1, i_class_id@2, i_category_id@3, i_manufact_id@4, ss_sold_date_sk@5, ss_item_sk@6, ss_ticket_number@7, ss_quantity@8, ss_ext_sales_price@9] + │ CoalescePartitionsExec + │ [Stage 16] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ticket_number, ss_quantity, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 15 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_year@1 = 2001 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + └────────────────────────────────────────────────── + ┌───── Stage 16 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ FilterExec: i_category@4 = Books, projection=[i_item_sk@0, i_brand_id@1, i_class_id@2, i_category_id@3, i_manufact_id@5] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id, i_category, i_manufact_id], file_type=parquet, predicate=i_category@12 = Books, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1, required_guarantees=[i_category in (Books)] + └────────────────────────────────────────────────── + ┌───── Stage 20 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] + │ ProjectionExec: expr=[ws_item_sk@1 as ws_item_sk, ws_order_number@2 as ws_order_number, ws_quantity@3 as ws_quantity, ws_ext_sales_price@4 as ws_ext_sales_price, i_brand_id@5 as i_brand_id, i_class_id@6 as i_class_id, i_category_id@7 as i_category_id, i_manufact_id@8 as i_manufact_id, d_year@0 as d_year] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, ws_sold_date_sk@0)], projection=[d_year@1, ws_item_sk@4, ws_order_number@5, ws_quantity@6, ws_ext_sales_price@7, i_brand_id@8, i_class_id@9, i_category_id@10, i_manufact_id@11] + │ CoalescePartitionsExec + │ [Stage 18] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ ProjectionExec: expr=[ws_sold_date_sk@4 as ws_sold_date_sk, ws_item_sk@5 as ws_item_sk, ws_order_number@6 as ws_order_number, ws_quantity@7 as ws_quantity, ws_ext_sales_price@8 as ws_ext_sales_price, i_brand_id@0 as i_brand_id, i_class_id@1 as i_class_id, i_category_id@2 as i_category_id, i_manufact_id@3 as i_manufact_id] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ws_item_sk@1)], projection=[i_brand_id@1, i_class_id@2, i_category_id@3, i_manufact_id@4, ws_sold_date_sk@5, ws_item_sk@6, ws_order_number@7, ws_quantity@8, ws_ext_sales_price@9] + │ CoalescePartitionsExec + │ [Stage 19] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_item_sk, ws_order_number, ws_quantity, ws_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 18 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_year@1 = 2001 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2001, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2001 AND 2001 <= d_year_max@1, required_guarantees=[d_year in (2001)] + └────────────────────────────────────────────────── + ┌───── Stage 19 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ FilterExec: i_category@4 = Books, projection=[i_item_sk@0, i_brand_id@1, i_class_id@2, i_category_id@3, i_manufact_id@5] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand_id, i_class_id, i_category_id, i_category, i_manufact_id], file_type=parquet, predicate=i_category@12 = Books, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1, required_guarantees=[i_category in (Books)] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_76() -> Result<()> { + let display = test_tpcds_query("q76").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [channel@0 ASC, col_name@1 ASC, d_year@2 ASC, d_qoy@3 ASC, i_category@4 ASC], fetch=100 + │ [Stage 11] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 11 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ SortExec: TopK(fetch=100), expr=[channel@0 ASC, col_name@1 ASC, d_year@2 ASC, d_qoy@3 ASC, i_category@4 ASC], preserve_partitioning=[true] + │ ProjectionExec: expr=[channel@0 as channel, col_name@1 as col_name, d_year@2 as d_year, d_qoy@3 as d_qoy, i_category@4 as i_category, count(Int64(1))@5 as sales_cnt, sum(foo.ext_sales_price)@6 as sales_amt] + │ AggregateExec: mode=FinalPartitioned, gby=[channel@0 as channel, col_name@1 as col_name, d_year@2 as d_year, d_qoy@3 as d_qoy, i_category@4 as i_category], aggr=[count(Int64(1)), sum(foo.ext_sales_price)] + │ [Stage 10] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + └────────────────────────────────────────────────── + ┌───── Stage 10 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] t3:[p0..p5] + │ RepartitionExec: partitioning=Hash([channel@0, col_name@1, d_year@2, d_qoy@3, i_category@4], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[channel@0 as channel, col_name@1 as col_name, d_year@2 as d_year, d_qoy@3 as d_qoy, i_category@4 as i_category], aggr=[count(Int64(1)), sum(foo.ext_sales_price)], ordering_mode=PartiallySorted([0, 1]) + │ DistributedUnionExec: t0:[c0] t1:[c1] t2:[c2(0/2)] t3:[c2(1/2)] + │ ProjectionExec: expr=[store as channel, ss_store_sk as col_name, d_year@0 as d_year, d_qoy@1 as d_qoy, i_category@3 as i_category, ss_ext_sales_price@2 as ext_sales_price] + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@3, ss_sold_date_sk@0)], projection=[d_year@1, d_qoy@2, ss_ext_sales_price@5, i_category@6] + │ [Stage 1] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + │ ProjectionExec: expr=[web as channel, ws_ship_customer_sk as col_name, d_year@0 as d_year, d_qoy@1 as d_qoy, i_category@3 as i_category, ws_ext_sales_price@2 as ext_sales_price] + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@3, ws_sold_date_sk@0)], projection=[d_year@1, d_qoy@2, ws_ext_sales_price@5, i_category@6] + │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + │ ProjectionExec: expr=[catalog as channel, cs_ship_addr_sk as col_name, d_year@0 as d_year, d_qoy@1 as d_qoy, i_category@3 as i_category, cs_ext_sales_price@2 as ext_sales_price] + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@3, cs_sold_date_sk@0)], projection=[d_year@1, d_qoy@2, cs_ext_sales_price@5, i_category@6] + │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 9] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ RepartitionExec: partitioning=Hash([CAST(date_dim.d_date_sk AS Float64)@3], 3), input_partitions=3 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, d_qoy@2 as d_qoy, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] + │ RepartitionExec: partitioning=Hash([ss_sold_date_sk@0], 3), input_partitions=2 + │ ProjectionExec: expr=[ss_sold_date_sk@1 as ss_sold_date_sk, ss_ext_sales_price@2 as ss_ext_sales_price, i_category@0 as i_category] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ss_item_sk@1)], projection=[i_category@1, ss_sold_date_sk@2, ss_ext_sales_price@4] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=2 + │ FilterExec: ss_store_sk@2 IS NULL, projection=[ss_sold_date_sk@0, ss_item_sk@1, ss_ext_sales_price@3] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_ext_sales_price], file_type=parquet, predicate=ss_store_sk@7 IS NULL AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], pruning_predicate=ss_store_sk_null_count@0 > 0, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_category], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ RepartitionExec: partitioning=Hash([CAST(date_dim.d_date_sk AS Float64)@3], 3), input_partitions=3 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, d_qoy@2 as d_qoy, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] + │ RepartitionExec: partitioning=Hash([ws_sold_date_sk@0], 3), input_partitions=2 + │ ProjectionExec: expr=[ws_sold_date_sk@1 as ws_sold_date_sk, ws_ext_sales_price@2 as ws_ext_sales_price, i_category@0 as i_category] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ws_item_sk@1)], projection=[i_category@1, ws_sold_date_sk@2, ws_ext_sales_price@4] + │ CoalescePartitionsExec + │ [Stage 5] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=2 + │ FilterExec: ws_ship_customer_sk@2 IS NULL, projection=[ws_sold_date_sk@0, ws_item_sk@1, ws_ext_sales_price@3] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_item_sk, ws_ship_customer_sk, ws_ext_sales_price], file_type=parquet, predicate=ws_ship_customer_sk@8 IS NULL AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], pruning_predicate=ws_ship_customer_sk_null_count@0 > 0, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_category], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ RepartitionExec: partitioning=Hash([CAST(date_dim.d_date_sk AS Float64)@3], 6), input_partitions=3 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, d_qoy@2 as d_qoy, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_qoy], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 9 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([cs_sold_date_sk@0], 6), input_partitions=2 + │ ProjectionExec: expr=[cs_sold_date_sk@1 as cs_sold_date_sk, cs_ext_sales_price@2 as cs_ext_sales_price, i_category@0 as i_category] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, cs_item_sk@1)], projection=[i_category@1, cs_sold_date_sk@2, cs_ext_sales_price@4] + │ CoalescePartitionsExec + │ [Stage 8] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=2 + │ FilterExec: cs_ship_addr_sk@1 IS NULL, projection=[cs_sold_date_sk@0, cs_item_sk@2, cs_ext_sales_price@3] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_ship_addr_sk, cs_item_sk, cs_ext_sales_price], file_type=parquet, predicate=cs_ship_addr_sk@10 IS NULL AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], pruning_predicate=cs_ship_addr_sk_null_count@0 > 0, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 8 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_category], file_type=parquet + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_77() -> Result<()> { + let display = test_tpcds_query("q77").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [channel@0 ASC, id@1 ASC, returns_@3 DESC], fetch=100 + │ [Stage 21] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 21 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ SortExec: TopK(fetch=100), expr=[channel@0 ASC, id@1 ASC, returns_@3 DESC], preserve_partitioning=[true] + │ ProjectionExec: expr=[channel@0 as channel, id@1 as id, sum(x.sales)@3 as sales, sum(x.returns_)@4 as returns_, sum(x.profit)@5 as profit] + │ AggregateExec: mode=FinalPartitioned, gby=[channel@0 as channel, id@1 as id, __grouping_id@2 as __grouping_id], aggr=[sum(x.sales), sum(x.returns_), sum(x.profit)] + │ [Stage 20] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + └────────────────────────────────────────────────── + ┌───── Stage 20 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] t3:[p0..p5] + │ RepartitionExec: partitioning=Hash([channel@0, id@1, __grouping_id@2], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[(NULL as channel, NULL as id), (channel@0 as channel, NULL as id), (channel@0 as channel, id@1 as id)], aggr=[sum(x.sales), sum(x.returns_), sum(x.profit)] + │ DistributedUnionExec: t0:[c0(0/2)] t1:[c0(1/2)] t2:[c1] t3:[c2] + │ ProjectionExec: expr=[store channel as channel, CAST(s_store_sk@2 AS Float64) as id, sales@3 as sales, CASE WHEN __common_expr_1@0 IS NOT NULL THEN __common_expr_1@0 ELSE Some(0),22,2 END as returns_, profit@4 - CASE WHEN __common_expr_2@1 IS NOT NULL THEN __common_expr_2@1 ELSE Some(0),22,2 END as profit] + │ ProjectionExec: expr=[CAST(returns_@0 AS Decimal128(22, 2)) as __common_expr_1, CAST(profit_loss@1 AS Decimal128(22, 2)) as __common_expr_2, s_store_sk@2 as s_store_sk, sales@3 as sales, profit@4 as profit] + │ HashJoinExec: mode=CollectLeft, join_type=Right, on=[(s_store_sk@0, s_store_sk@0)], projection=[returns_@1, profit_loss@2, s_store_sk@3, sales@4, profit@5] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, sum(store_sales.ss_ext_sales_price)@1 as sales, sum(store_sales.ss_net_profit)@2 as profit] + │ AggregateExec: mode=FinalPartitioned, gby=[s_store_sk@0 as s_store_sk], aggr=[sum(store_sales.ss_ext_sales_price), sum(store_sales.ss_net_profit)] + │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + │ ProjectionExec: expr=[catalog channel as channel, cs_call_center_sk@0 as id, sales@1 as sales, CAST(returns_@3 AS Decimal128(22, 2)) as returns_, CAST(profit@2 - profit_loss@4 AS Decimal128(23, 2)) as profit] + │ CrossJoinExec + │ CoalescePartitionsExec + │ [Stage 10] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[sum(catalog_returns.cr_return_amount)@1 as returns_, sum(catalog_returns.cr_net_loss)@2 as profit_loss] + │ AggregateExec: mode=FinalPartitioned, gby=[cr_call_center_sk@0 as cr_call_center_sk], aggr=[sum(catalog_returns.cr_return_amount), sum(catalog_returns.cr_net_loss)] + │ [Stage 12] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + │ ProjectionExec: expr=[web channel as channel, CAST(wp_web_page_sk@2 AS Float64) as id, sales@3 as sales, CASE WHEN __common_expr_3@0 IS NOT NULL THEN __common_expr_3@0 ELSE Some(0),22,2 END as returns_, profit@4 - CASE WHEN __common_expr_4@1 IS NOT NULL THEN __common_expr_4@1 ELSE Some(0),22,2 END as profit] + │ ProjectionExec: expr=[CAST(returns_@3 AS Decimal128(22, 2)) as __common_expr_3, CAST(profit_loss@4 AS Decimal128(22, 2)) as __common_expr_4, wp_web_page_sk@0 as wp_web_page_sk, sales@1 as sales, profit@2 as profit] + │ HashJoinExec: mode=CollectLeft, join_type=Left, on=[(wp_web_page_sk@0, wp_web_page_sk@0)], projection=[wp_web_page_sk@0, sales@1, profit@2, returns_@4, profit_loss@5] + │ CoalescePartitionsExec + │ [Stage 16] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[wp_web_page_sk@0 as wp_web_page_sk, sum(web_returns.wr_return_amt)@1 as returns_, sum(web_returns.wr_net_loss)@2 as profit_loss] + │ AggregateExec: mode=FinalPartitioned, gby=[wp_web_page_sk@0 as wp_web_page_sk], aggr=[sum(web_returns.wr_return_amt), sum(web_returns.wr_net_loss)] + │ [Stage 19] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=3, consumer_tasks=2, output_partitions=6 + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, sum(store_returns.sr_return_amt)@1 as returns_, sum(store_returns.sr_net_loss)@2 as profit_loss] + │ AggregateExec: mode=FinalPartitioned, gby=[s_store_sk@0 as s_store_sk], aggr=[sum(store_returns.sr_return_amt), sum(store_returns.sr_net_loss)] + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([s_store_sk@0], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[s_store_sk@2 as s_store_sk], aggr=[sum(store_returns.sr_return_amt), sum(store_returns.sr_net_loss)] + │ ProjectionExec: expr=[sr_return_amt@1 as sr_return_amt, sr_net_loss@2 as sr_net_loss, s_store_sk@0 as s_store_sk] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@1, sr_store_sk@0)], projection=[s_store_sk@0, sr_return_amt@3, sr_net_loss@4] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, sr_returned_date_sk@0)], projection=[sr_store_sk@3, sr_return_amt@4, sr_net_loss@5] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], ...]}, projection=[sr_returned_date_sk, sr_store_sk, sr_return_amt, sr_net_loss], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-22, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([s_store_sk@0], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[s_store_sk@2 as s_store_sk], aggr=[sum(store_sales.ss_ext_sales_price), sum(store_sales.ss_net_profit)] + │ ProjectionExec: expr=[ss_ext_sales_price@1 as ss_ext_sales_price, ss_net_profit@2 as ss_net_profit, s_store_sk@0 as s_store_sk] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@1, ss_store_sk@0)], projection=[s_store_sk@0, ss_ext_sales_price@3, ss_net_profit@4] + │ CoalescePartitionsExec + │ [Stage 5] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_store_sk@3, ss_ext_sales_price@4, ss_net_profit@5] + │ CoalescePartitionsExec + │ [Stage 6] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_store_sk, ss_ext_sales_price, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-22, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 10 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ ProjectionExec: expr=[cs_call_center_sk@0 as cs_call_center_sk, sum(catalog_sales.cs_ext_sales_price)@1 as sales, sum(catalog_sales.cs_net_profit)@2 as profit] + │ AggregateExec: mode=FinalPartitioned, gby=[cs_call_center_sk@0 as cs_call_center_sk], aggr=[sum(catalog_sales.cs_ext_sales_price), sum(catalog_sales.cs_net_profit)] + │ [Stage 9] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 9 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([cs_call_center_sk@0], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[cs_call_center_sk@0 as cs_call_center_sk], aggr=[sum(catalog_sales.cs_ext_sales_price), sum(catalog_sales.cs_net_profit)] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, cs_sold_date_sk@0)], projection=[cs_call_center_sk@3, cs_ext_sales_price@4, cs_net_profit@5] + │ CoalescePartitionsExec + │ [Stage 8] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_call_center_sk, cs_ext_sales_price, cs_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 8 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-22, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 12 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] + │ RepartitionExec: partitioning=Hash([cr_call_center_sk@0], 3), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[cr_call_center_sk@0 as cr_call_center_sk], aggr=[sum(catalog_returns.cr_return_amount), sum(catalog_returns.cr_net_loss)] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, cr_returned_date_sk@0)], projection=[cr_call_center_sk@2, cr_return_amount@3, cr_net_loss@4] + │ CoalescePartitionsExec + │ [Stage 11] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], ...]}, projection=[cr_returned_date_sk, cr_call_center_sk, cr_return_amount, cr_net_loss], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 11 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ FilterExec: d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-22, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 16 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ ProjectionExec: expr=[wp_web_page_sk@0 as wp_web_page_sk, sum(web_sales.ws_ext_sales_price)@1 as sales, sum(web_sales.ws_net_profit)@2 as profit] + │ AggregateExec: mode=FinalPartitioned, gby=[wp_web_page_sk@0 as wp_web_page_sk], aggr=[sum(web_sales.ws_ext_sales_price), sum(web_sales.ws_net_profit)] + │ [Stage 15] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 15 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([wp_web_page_sk@0], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[wp_web_page_sk@2 as wp_web_page_sk], aggr=[sum(web_sales.ws_ext_sales_price), sum(web_sales.ws_net_profit)] + │ ProjectionExec: expr=[ws_ext_sales_price@1 as ws_ext_sales_price, ws_net_profit@2 as ws_net_profit, wp_web_page_sk@0 as wp_web_page_sk] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(web_page.wp_web_page_sk AS Float64)@1, ws_web_page_sk@0)], projection=[wp_web_page_sk@0, ws_ext_sales_price@3, ws_net_profit@4] + │ CoalescePartitionsExec + │ [Stage 13] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ws_sold_date_sk@0)], projection=[ws_web_page_sk@3, ws_ext_sales_price@4, ws_net_profit@5] + │ CoalescePartitionsExec + │ [Stage 14] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_web_page_sk, ws_ext_sales_price, ws_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 13 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_page/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_page/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_page/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_page/part-3.parquet]]}, projection=[wp_web_page_sk, CAST(wp_web_page_sk@0 AS Float64) as CAST(web_page.wp_web_page_sk AS Float64)], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 14 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-22, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 19 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ RepartitionExec: partitioning=Hash([wp_web_page_sk@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[wp_web_page_sk@2 as wp_web_page_sk], aggr=[sum(web_returns.wr_return_amt), sum(web_returns.wr_net_loss)] + │ ProjectionExec: expr=[wr_return_amt@1 as wr_return_amt, wr_net_loss@2 as wr_net_loss, wp_web_page_sk@0 as wp_web_page_sk] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(web_page.wp_web_page_sk AS Float64)@1, wr_web_page_sk@0)], projection=[wp_web_page_sk@0, wr_return_amt@3, wr_net_loss@4] + │ CoalescePartitionsExec + │ [Stage 17] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=4, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, wr_returned_date_sk@0)], projection=[wr_web_page_sk@3, wr_return_amt@4, wr_net_loss@5] + │ CoalescePartitionsExec + │ [Stage 18] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=6, input_tasks=2 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet]]}, projection=[wr_returned_date_sk, wr_web_page_sk, wr_return_amt, wr_net_loss], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 17 ── Tasks: t0:[p0..p3] t1:[p4..p7] + │ BroadcastExec: input_partitions=2, consumer_tasks=2, output_partitions=4 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_page/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_page/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_page/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_page/part-3.parquet]]}, projection=[wp_web_page_sk, CAST(wp_web_page_sk@0 AS Float64) as CAST(web_page.wp_web_page_sk AS Float64)], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 18 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=3, consumer_tasks=2, output_partitions=6 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-22, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_78() -> Result<()> { + let display = test_tpcds_query("q78").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ ProjectionExec: expr=[ss_sold_year@0 as ss_sold_year, ss_item_sk@1 as ss_item_sk, ss_customer_sk@2 as ss_customer_sk, ratio@3 as ratio, store_qty@4 as store_qty, store_wholesale_cost@5 as store_wholesale_cost, store_sales_price@6 as store_sales_price, other_chan_qty@7 as other_chan_qty, other_chan_wholesale_cost@8 as other_chan_wholesale_cost, other_chan_sales_price@9 as other_chan_sales_price] + │ SortPreservingMergeExec: [ss_sold_year@0 ASC NULLS LAST, ss_item_sk@1 ASC NULLS LAST, ss_customer_sk@2 ASC NULLS LAST, ss_qty@10 DESC, ss_wc@11 DESC, ss_sp@12 DESC, other_chan_qty@7 ASC NULLS LAST, other_chan_wholesale_cost@8 ASC NULLS LAST, other_chan_sales_price@9 ASC NULLS LAST, ratio@3 ASC NULLS LAST], fetch=100 + │ SortExec: TopK(fetch=100), expr=[ss_item_sk@1 ASC NULLS LAST, ss_customer_sk@2 ASC NULLS LAST, store_qty@4 DESC, store_wholesale_cost@5 DESC, store_sales_price@6 DESC, other_chan_qty@7 ASC NULLS LAST, other_chan_wholesale_cost@8 ASC NULLS LAST, other_chan_sales_price@9 ASC NULLS LAST, ratio@3 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[ss_sold_year@5 as ss_sold_year, ss_item_sk@6 as ss_item_sk, ss_customer_sk@7 as ss_customer_sk, round(ss_qty@8 / __common_expr_1@0, 2) as ratio, ss_qty@8 as store_qty, ss_wc@9 as store_wholesale_cost, ss_sp@10 as store_sales_price, __common_expr_1@0 as other_chan_qty, CASE WHEN __common_expr_2@1 IS NOT NULL THEN __common_expr_2@1 ELSE Some(0),22,2 END + CASE WHEN __common_expr_3@2 IS NOT NULL THEN __common_expr_3@2 ELSE Some(0),22,2 END as other_chan_wholesale_cost, CASE WHEN __common_expr_4@3 IS NOT NULL THEN __common_expr_4@3 ELSE Some(0),22,2 END + CASE WHEN __common_expr_5@4 IS NOT NULL THEN __common_expr_5@4 ELSE Some(0),22,2 END as other_chan_sales_price, ss_qty@8 as ss_qty, ss_wc@9 as ss_wc, ss_sp@10 as ss_sp] + │ ProjectionExec: expr=[CASE WHEN ws_qty@6 IS NOT NULL THEN ws_qty@6 ELSE 0 END + CASE WHEN cs_qty@9 IS NOT NULL THEN cs_qty@9 ELSE 0 END as __common_expr_1, CAST(ws_wc@7 AS Decimal128(22, 2)) as __common_expr_2, CAST(cs_wc@10 AS Decimal128(22, 2)) as __common_expr_3, CAST(ws_sp@8 AS Decimal128(22, 2)) as __common_expr_4, CAST(cs_sp@11 AS Decimal128(22, 2)) as __common_expr_5, ss_sold_year@0 as ss_sold_year, ss_item_sk@1 as ss_item_sk, ss_customer_sk@2 as ss_customer_sk, ss_qty@3 as ss_qty, ss_wc@4 as ss_wc, ss_sp@5 as ss_sp] + │ FilterExec: CASE WHEN ws_qty@6 IS NOT NULL THEN ws_qty@6 ELSE 0 END > 0 OR CASE WHEN cs_qty@9 IS NOT NULL THEN cs_qty@9 ELSE 0 END > 0 + │ HashJoinExec: mode=CollectLeft, join_type=Left, on=[(ss_sold_year@0, cs_sold_year@0), (ss_item_sk@1, cs_item_sk@1), (ss_customer_sk@2, cs_customer_sk@2)], projection=[ss_sold_year@0, ss_item_sk@1, ss_customer_sk@2, ss_qty@3, ss_wc@4, ss_sp@5, ws_qty@6, ws_wc@7, ws_sp@8, cs_qty@12, cs_wc@13, cs_sp@14] + │ CoalescePartitionsExec + │ HashJoinExec: mode=CollectLeft, join_type=Left, on=[(ss_sold_year@0, ws_sold_year@0), (ss_item_sk@1, ws_item_sk@1), (ss_customer_sk@2, ws_customer_sk@2)], projection=[ss_sold_year@0, ss_item_sk@1, ss_customer_sk@2, ss_qty@3, ss_wc@4, ss_sp@5, ws_qty@9, ws_wc@10, ws_sp@11] + │ CoalescePartitionsExec + │ [Stage 5] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[d_year@0 as ws_sold_year, ws_item_sk@1 as ws_item_sk, ws_bill_customer_sk@2 as ws_customer_sk, sum(web_sales.ws_quantity)@3 as ws_qty, sum(web_sales.ws_wholesale_cost)@4 as ws_wc, sum(web_sales.ws_sales_price)@5 as ws_sp] + │ AggregateExec: mode=FinalPartitioned, gby=[d_year@0 as d_year, ws_item_sk@1 as ws_item_sk, ws_bill_customer_sk@2 as ws_bill_customer_sk], aggr=[sum(web_sales.ws_quantity), sum(web_sales.ws_wholesale_cost), sum(web_sales.ws_sales_price)] + │ [Stage 9] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ ProjectionExec: expr=[d_year@0 as cs_sold_year, cs_item_sk@1 as cs_item_sk, cs_bill_customer_sk@2 as cs_customer_sk, sum(catalog_sales.cs_quantity)@3 as cs_qty, sum(catalog_sales.cs_wholesale_cost)@4 as cs_wc, sum(catalog_sales.cs_sales_price)@5 as cs_sp] + │ AggregateExec: mode=FinalPartitioned, gby=[d_year@0 as d_year, cs_item_sk@1 as cs_item_sk, cs_bill_customer_sk@2 as cs_bill_customer_sk], aggr=[sum(catalog_sales.cs_quantity), sum(catalog_sales.cs_wholesale_cost), sum(catalog_sales.cs_sales_price)] + │ [Stage 14] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ ProjectionExec: expr=[d_year@0 as ss_sold_year, ss_item_sk@1 as ss_item_sk, ss_customer_sk@2 as ss_customer_sk, sum(store_sales.ss_quantity)@3 as ss_qty, sum(store_sales.ss_wholesale_cost)@4 as ss_wc, sum(store_sales.ss_sales_price)@5 as ss_sp] + │ AggregateExec: mode=FinalPartitioned, gby=[d_year@0 as d_year, ss_item_sk@1 as ss_item_sk, ss_customer_sk@2 as ss_customer_sk], aggr=[sum(store_sales.ss_quantity), sum(store_sales.ss_wholesale_cost), sum(store_sales.ss_sales_price)], ordering_mode=PartiallySorted([0]) + │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([d_year@0, ss_item_sk@1, ss_customer_sk@2], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[d_year@5 as d_year, ss_item_sk@0 as ss_item_sk, ss_customer_sk@1 as ss_customer_sk], aggr=[sum(store_sales.ss_quantity), sum(store_sales.ss_wholesale_cost), sum(store_sales.ss_sales_price)], ordering_mode=PartiallySorted([0]) + │ ProjectionExec: expr=[ss_item_sk@1 as ss_item_sk, ss_customer_sk@2 as ss_customer_sk, ss_quantity@3 as ss_quantity, ss_wholesale_cost@4 as ss_wholesale_cost, ss_sales_price@5 as ss_sales_price, d_year@0 as d_year] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, ss_sold_date_sk@0)], projection=[d_year@1, ss_item_sk@4, ss_customer_sk@5, ss_quantity@6, ss_wholesale_cost@7, ss_sales_price@8] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ FilterExec: sr_ticket_number@6 IS NULL, projection=[ss_sold_date_sk@0, ss_item_sk@1, ss_customer_sk@2, ss_quantity@3, ss_wholesale_cost@4, ss_sales_price@5] + │ ProjectionExec: expr=[ss_sold_date_sk@1 as ss_sold_date_sk, ss_item_sk@2 as ss_item_sk, ss_customer_sk@3 as ss_customer_sk, ss_quantity@4 as ss_quantity, ss_wholesale_cost@5 as ss_wholesale_cost, ss_sales_price@6 as ss_sales_price, sr_ticket_number@0 as sr_ticket_number] + │ HashJoinExec: mode=Partitioned, join_type=Right, on=[(sr_ticket_number@1, ss_ticket_number@3), (sr_item_sk@0, ss_item_sk@1)], projection=[sr_ticket_number@1, ss_sold_date_sk@2, ss_item_sk@3, ss_customer_sk@4, ss_quantity@6, ss_wholesale_cost@7, ss_sales_price@8] + │ [Stage 2] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_year@1 = 2000 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] + │ RepartitionExec: partitioning=Hash([sr_ticket_number@1, sr_item_sk@0], 9), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], ...]}, projection=[sr_item_sk, sr_ticket_number], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] + │ RepartitionExec: partitioning=Hash([ss_ticket_number@3, ss_item_sk@1], 9), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk, ss_ticket_number, ss_quantity, ss_wholesale_cost, ss_sales_price], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 9 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ RepartitionExec: partitioning=Hash([d_year@0, ws_item_sk@1, ws_bill_customer_sk@2], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[d_year@5 as d_year, ws_item_sk@0 as ws_item_sk, ws_bill_customer_sk@1 as ws_bill_customer_sk], aggr=[sum(web_sales.ws_quantity), sum(web_sales.ws_wholesale_cost), sum(web_sales.ws_sales_price)] + │ ProjectionExec: expr=[ws_item_sk@1 as ws_item_sk, ws_bill_customer_sk@2 as ws_bill_customer_sk, ws_quantity@3 as ws_quantity, ws_wholesale_cost@4 as ws_wholesale_cost, ws_sales_price@5 as ws_sales_price, d_year@0 as d_year] + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, ws_sold_date_sk@0)], projection=[d_year@1, ws_item_sk@4, ws_bill_customer_sk@5, ws_quantity@6, ws_wholesale_cost@7, ws_sales_price@8] + │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 8] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ RepartitionExec: partitioning=Hash([CAST(date_dim.d_date_sk AS Float64)@2], 6), input_partitions=3 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 8 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([ws_sold_date_sk@0], 6), input_partitions=2 + │ FilterExec: wr_order_number@6 IS NULL, projection=[ws_sold_date_sk@0, ws_item_sk@1, ws_bill_customer_sk@2, ws_quantity@3, ws_wholesale_cost@4, ws_sales_price@5] + │ ProjectionExec: expr=[ws_sold_date_sk@1 as ws_sold_date_sk, ws_item_sk@2 as ws_item_sk, ws_bill_customer_sk@3 as ws_bill_customer_sk, ws_quantity@4 as ws_quantity, ws_wholesale_cost@5 as ws_wholesale_cost, ws_sales_price@6 as ws_sales_price, wr_order_number@0 as wr_order_number] + │ HashJoinExec: mode=CollectLeft, join_type=Right, on=[(wr_order_number@1, ws_order_number@3), (wr_item_sk@0, ws_item_sk@1)], projection=[wr_order_number@1, ws_sold_date_sk@2, ws_item_sk@3, ws_bill_customer_sk@4, ws_quantity@6, ws_wholesale_cost@7, ws_sales_price@8] + │ CoalescePartitionsExec + │ [Stage 7] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_item_sk, ws_bill_customer_sk, ws_order_number, ws_quantity, ws_wholesale_cost, ws_sales_price], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet]]}, projection=[wr_item_sk, wr_order_number], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 14 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ RepartitionExec: partitioning=Hash([d_year@0, cs_item_sk@1, cs_bill_customer_sk@2], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[d_year@5 as d_year, cs_item_sk@1 as cs_item_sk, cs_bill_customer_sk@0 as cs_bill_customer_sk], aggr=[sum(catalog_sales.cs_quantity), sum(catalog_sales.cs_wholesale_cost), sum(catalog_sales.cs_sales_price)] + │ ProjectionExec: expr=[cs_bill_customer_sk@1 as cs_bill_customer_sk, cs_item_sk@2 as cs_item_sk, cs_quantity@3 as cs_quantity, cs_wholesale_cost@4 as cs_wholesale_cost, cs_sales_price@5 as cs_sales_price, d_year@0 as d_year] + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, cs_sold_date_sk@0)], projection=[d_year@1, cs_bill_customer_sk@4, cs_item_sk@5, cs_quantity@6, cs_wholesale_cost@7, cs_sales_price@8] + │ [Stage 10] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 13] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 10 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ RepartitionExec: partitioning=Hash([CAST(date_dim.d_date_sk AS Float64)@2], 6), input_partitions=3 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_year@1 as d_year, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 13 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([cs_sold_date_sk@0], 6), input_partitions=3 + │ FilterExec: cr_order_number@6 IS NULL, projection=[cs_sold_date_sk@0, cs_bill_customer_sk@1, cs_item_sk@2, cs_quantity@3, cs_wholesale_cost@4, cs_sales_price@5] + │ ProjectionExec: expr=[cs_sold_date_sk@1 as cs_sold_date_sk, cs_bill_customer_sk@2 as cs_bill_customer_sk, cs_item_sk@3 as cs_item_sk, cs_quantity@4 as cs_quantity, cs_wholesale_cost@5 as cs_wholesale_cost, cs_sales_price@6 as cs_sales_price, cr_order_number@0 as cr_order_number] + │ HashJoinExec: mode=Partitioned, join_type=Right, on=[(cr_order_number@1, cs_order_number@3), (cr_item_sk@0, cs_item_sk@2)], projection=[cr_order_number@1, cs_sold_date_sk@2, cs_bill_customer_sk@3, cs_item_sk@4, cs_quantity@6, cs_wholesale_cost@7, cs_sales_price@8] + │ [Stage 11] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + │ [Stage 12] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 11 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] + │ RepartitionExec: partitioning=Hash([cr_order_number@1, cr_item_sk@0], 9), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], ...]}, projection=[cr_item_sk, cr_order_number], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 12 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] + │ RepartitionExec: partitioning=Hash([cs_order_number@3, cs_item_sk@2], 9), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_item_sk, cs_order_number, cs_quantity, cs_wholesale_cost, cs_sales_price], file_type=parquet + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_79() -> Result<()> { + let display = test_tpcds_query("q79").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [c_last_name@0 ASC, c_first_name@1 ASC, substr(ms.s_city,Int64(1),Int64(30))@2 ASC, profit@5 ASC, ss_ticket_number@3 ASC NULLS LAST], fetch=100 + │ [Stage 6] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ SortExec: TopK(fetch=100), expr=[c_last_name@0 ASC, c_first_name@1 ASC, substr(ms.s_city,Int64(1),Int64(30))@2 ASC, profit@5 ASC, ss_ticket_number@3 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[c_last_name@5 as c_last_name, c_first_name@4 as c_first_name, substr(s_city@1, 1, 30) as substr(ms.s_city,Int64(1),Int64(30)), ss_ticket_number@0 as ss_ticket_number, amt@2 as amt, profit@3 as profit] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_customer_sk@1, CAST(customer.c_customer_sk AS Float64)@3)], projection=[ss_ticket_number@0, s_city@2, amt@3, profit@4, c_first_name@6, c_last_name@7] + │ CoalescePartitionsExec + │ [Stage 5] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=3, consumer_tasks=2, output_partitions=6 + │ ProjectionExec: expr=[ss_ticket_number@0 as ss_ticket_number, ss_customer_sk@1 as ss_customer_sk, s_city@3 as s_city, sum(store_sales.ss_coupon_amt)@4 as amt, sum(store_sales.ss_net_profit)@5 as profit] + │ AggregateExec: mode=FinalPartitioned, gby=[ss_ticket_number@0 as ss_ticket_number, ss_customer_sk@1 as ss_customer_sk, ss_addr_sk@2 as ss_addr_sk, s_city@3 as s_city], aggr=[sum(store_sales.ss_coupon_amt), sum(store_sales.ss_net_profit)] + │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([ss_ticket_number@0, ss_customer_sk@1, ss_addr_sk@2, s_city@3], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[ss_ticket_number@2 as ss_ticket_number, ss_customer_sk@0 as ss_customer_sk, ss_addr_sk@1 as ss_addr_sk, s_city@5 as s_city], aggr=[sum(store_sales.ss_coupon_amt), sum(store_sales.ss_net_profit)] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(household_demographics.hd_demo_sk AS Float64)@1, ss_hdemo_sk@1)], projection=[ss_customer_sk@2, ss_addr_sk@4, ss_ticket_number@5, ss_coupon_amt@6, ss_net_profit@7, s_city@8] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ ProjectionExec: expr=[ss_customer_sk@1 as ss_customer_sk, ss_hdemo_sk@2 as ss_hdemo_sk, ss_addr_sk@3 as ss_addr_sk, ss_ticket_number@4 as ss_ticket_number, ss_coupon_amt@5 as ss_coupon_amt, ss_net_profit@6 as ss_net_profit, s_city@0 as s_city] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@2, ss_store_sk@3)], projection=[s_city@1, ss_customer_sk@3, ss_hdemo_sk@4, ss_addr_sk@5, ss_ticket_number@7, ss_coupon_amt@8, ss_net_profit@9] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_customer_sk@3, ss_hdemo_sk@4, ss_addr_sk@5, ss_store_sk@6, ss_ticket_number@7, ss_coupon_amt@8, ss_net_profit@9] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_customer_sk, ss_hdemo_sk, ss_addr_sk, ss_store_sk, ss_ticket_number, ss_coupon_amt, ss_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[hd_demo_sk@0 as hd_demo_sk, CAST(hd_demo_sk@0 AS Float64) as CAST(household_demographics.hd_demo_sk AS Float64)] + │ FilterExec: hd_dep_count@1 = 6 OR hd_vehicle_count@2 > 2, projection=[hd_demo_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-3.parquet]]}, projection=[hd_demo_sk, hd_dep_count, hd_vehicle_count], file_type=parquet, predicate=hd_dep_count@3 = 6 OR hd_vehicle_count@4 > 2, pruning_predicate=hd_dep_count_null_count@2 != row_count@3 AND hd_dep_count_min@0 <= 6 AND 6 <= hd_dep_count_max@1 OR hd_vehicle_count_null_count@5 != row_count@3 AND hd_vehicle_count_max@4 > 2, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, s_city@1 as s_city, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] + │ FilterExec: s_number_employees@1 >= 200 AND s_number_employees@1 <= 295, projection=[s_store_sk@0, s_city@2] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_number_employees, s_city], file_type=parquet, predicate=s_number_employees@6 >= 200 AND s_number_employees@6 <= 295, pruning_predicate=s_number_employees_null_count@1 != row_count@2 AND s_number_employees_max@0 >= 200 AND s_number_employees_null_count@1 != row_count@2 AND s_number_employees_min@3 <= 295, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_dow@2 = 1 AND (d_year@1 = 1999 OR d_year@1 = 2000 OR d_year@1 = 2001), projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_dow], file_type=parquet, predicate=d_dow@7 = 1 AND (d_year@6 = 1999 OR d_year@6 = 2000 OR d_year@6 = 2001), pruning_predicate=d_dow_null_count@2 != row_count@3 AND d_dow_min@0 <= 1 AND 1 <= d_dow_max@1 AND (d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 1999 AND 1999 <= d_year_max@5 OR d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2000 AND 2000 <= d_year_max@5 OR d_year_null_count@6 != row_count@3 AND d_year_min@4 <= 2001 AND 2001 <= d_year_max@5), required_guarantees=[d_dow in (1), d_year in (1999, 2000, 2001)] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_80() -> Result<()> { + let display = test_tpcds_query("q80").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [channel@0 ASC, id@1 ASC], fetch=100 + │ SortExec: TopK(fetch=100), expr=[channel@0 ASC, id@1 ASC], preserve_partitioning=[true] + │ ProjectionExec: expr=[channel@0 as channel, id@1 as id, sum(x.sales)@3 as sales, sum(x.returns_)@4 as returns_, sum(x.profit)@5 as profit] + │ AggregateExec: mode=FinalPartitioned, gby=[channel@0 as channel, id@1 as id, __grouping_id@2 as __grouping_id], aggr=[sum(x.sales), sum(x.returns_), sum(x.profit)] + │ [Stage 22] => NetworkShuffleExec: output_partitions=3, input_tasks=4 + └────────────────────────────────────────────────── + ┌───── Stage 22 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] t3:[p0..p2] + │ RepartitionExec: partitioning=Hash([channel@0, id@1, __grouping_id@2], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[(NULL as channel, NULL as id), (channel@0 as channel, NULL as id), (channel@0 as channel, id@1 as id)], aggr=[sum(x.sales), sum(x.returns_), sum(x.profit)] + │ DistributedUnionExec: t0:[c0] t1:[c1] t2:[c2(0/2)] t3:[c2(1/2)] + │ ProjectionExec: expr=[store channel as channel, concat(store, s_store_id@0) as id, sum(store_sales.ss_ext_sales_price)@1 as sales, sum(coalesce(store_returns.sr_return_amt,Int64(0)))@2 as returns_, sum(store_sales.ss_net_profit - coalesce(store_returns.sr_net_loss,Int64(0)))@3 as profit] + │ AggregateExec: mode=FinalPartitioned, gby=[s_store_id@0 as s_store_id], aggr=[sum(store_sales.ss_ext_sales_price), sum(coalesce(store_returns.sr_return_amt,Int64(0))), sum(store_sales.ss_net_profit - coalesce(store_returns.sr_net_loss,Int64(0)))] + │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + │ ProjectionExec: expr=[catalog channel as channel, concat(catalog_page, cp_catalog_page_id@0) as id, sum(catalog_sales.cs_ext_sales_price)@1 as sales, sum(coalesce(catalog_returns.cr_return_amount,Int64(0)))@2 as returns_, sum(catalog_sales.cs_net_profit - coalesce(catalog_returns.cr_net_loss,Int64(0)))@3 as profit] + │ AggregateExec: mode=FinalPartitioned, gby=[cp_catalog_page_id@0 as cp_catalog_page_id], aggr=[sum(catalog_sales.cs_ext_sales_price), sum(coalesce(catalog_returns.cr_return_amount,Int64(0))), sum(catalog_sales.cs_net_profit - coalesce(catalog_returns.cr_net_loss,Int64(0)))] + │ [Stage 14] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + │ ProjectionExec: expr=[web channel as channel, concat(web_site, web_site_id@0) as id, sum(web_sales.ws_ext_sales_price)@1 as sales, sum(coalesce(web_returns.wr_return_amt,Int64(0)))@2 as returns_, sum(web_sales.ws_net_profit - coalesce(web_returns.wr_net_loss,Int64(0)))@3 as profit] + │ AggregateExec: mode=FinalPartitioned, gby=[web_site_id@0 as web_site_id], aggr=[sum(web_sales.ws_ext_sales_price), sum(coalesce(web_returns.wr_return_amt,Int64(0))), sum(web_sales.ws_net_profit - coalesce(web_returns.wr_net_loss,Int64(0)))] + │ [Stage 21] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] + │ RepartitionExec: partitioning=Hash([s_store_id@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[s_store_id@4 as s_store_id], aggr=[sum(store_sales.ss_ext_sales_price), sum(coalesce(store_returns.sr_return_amt,Int64(0))), sum(store_sales.ss_net_profit - coalesce(store_returns.sr_net_loss,Int64(0)))] + │ ProjectionExec: expr=[CAST(sr_return_amt@2 AS Decimal128(22, 2)) as __common_expr_1, CAST(sr_net_loss@3 AS Decimal128(22, 2)) as __common_expr_2, ss_ext_sales_price@0 as ss_ext_sales_price, ss_net_profit@1 as ss_net_profit, s_store_id@4 as s_store_id] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(promotion.p_promo_sk AS Float64)@1, ss_promo_sk@0)], projection=[ss_ext_sales_price@3, ss_net_profit@4, sr_return_amt@5, sr_net_loss@6, s_store_id@7] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ss_item_sk@0)], projection=[ss_promo_sk@2, ss_ext_sales_price@3, ss_net_profit@4, sr_return_amt@5, sr_net_loss@6, s_store_id@7] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ ProjectionExec: expr=[ss_item_sk@1 as ss_item_sk, ss_promo_sk@2 as ss_promo_sk, ss_ext_sales_price@3 as ss_ext_sales_price, ss_net_profit@4 as ss_net_profit, sr_return_amt@5 as sr_return_amt, sr_net_loss@6 as sr_net_loss, s_store_id@0 as s_store_id] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@2, ss_store_sk@1)], projection=[s_store_id@1, ss_item_sk@3, ss_promo_sk@5, ss_ext_sales_price@6, ss_net_profit@7, sr_return_amt@8, sr_net_loss@9] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_item_sk@3, ss_store_sk@4, ss_promo_sk@5, ss_ext_sales_price@6, ss_net_profit@7, sr_return_amt@8, sr_net_loss@9] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ ProjectionExec: expr=[ss_sold_date_sk@2 as ss_sold_date_sk, ss_item_sk@3 as ss_item_sk, ss_store_sk@4 as ss_store_sk, ss_promo_sk@5 as ss_promo_sk, ss_ext_sales_price@6 as ss_ext_sales_price, ss_net_profit@7 as ss_net_profit, sr_return_amt@0 as sr_return_amt, sr_net_loss@1 as sr_net_loss] + │ HashJoinExec: mode=Partitioned, join_type=Right, on=[(sr_item_sk@0, ss_item_sk@1), (sr_ticket_number@1, ss_ticket_number@4)], projection=[sr_return_amt@2, sr_net_loss@3, ss_sold_date_sk@4, ss_item_sk@5, ss_store_sk@6, ss_promo_sk@7, ss_ext_sales_price@9, ss_net_profit@10] + │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[p_promo_sk@0 as p_promo_sk, CAST(p_promo_sk@0 AS Float64) as CAST(promotion.p_promo_sk AS Float64)] + │ FilterExec: p_channel_tv@1 = N, projection=[p_promo_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/promotion/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/promotion/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/promotion/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/promotion/part-3.parquet]]}, projection=[p_promo_sk, p_channel_tv], file_type=parquet, predicate=p_channel_tv@11 = N, pruning_predicate=p_channel_tv_null_count@2 != row_count@3 AND p_channel_tv_min@0 <= N AND N <= p_channel_tv_max@1, required_guarantees=[p_channel_tv in (N)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ FilterExec: i_current_price@1 > Some(5000),4,2, projection=[i_item_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_current_price], file_type=parquet, predicate=i_current_price@5 > Some(5000),4,2, pruning_predicate=i_current_price_null_count@1 != row_count@2 AND i_current_price_max@0 > Some(5000),4,2, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_store_id, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-22, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] + │ RepartitionExec: partitioning=Hash([sr_item_sk@0, sr_ticket_number@1], 9), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], ...]}, projection=[sr_item_sk, sr_ticket_number, sr_return_amt, sr_net_loss], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] + │ RepartitionExec: partitioning=Hash([ss_item_sk@1, ss_ticket_number@4], 9), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_promo_sk, ss_ticket_number, ss_ext_sales_price, ss_net_profit], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 14 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] + │ RepartitionExec: partitioning=Hash([cp_catalog_page_id@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[cp_catalog_page_id@4 as cp_catalog_page_id], aggr=[sum(catalog_sales.cs_ext_sales_price), sum(coalesce(catalog_returns.cr_return_amount,Int64(0))), sum(catalog_sales.cs_net_profit - coalesce(catalog_returns.cr_net_loss,Int64(0)))] + │ ProjectionExec: expr=[CAST(cr_return_amount@2 AS Decimal128(22, 2)) as __common_expr_3, CAST(cr_net_loss@3 AS Decimal128(22, 2)) as __common_expr_4, cs_ext_sales_price@0 as cs_ext_sales_price, cs_net_profit@1 as cs_net_profit, cp_catalog_page_id@4 as cp_catalog_page_id] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(promotion.p_promo_sk AS Float64)@1, cs_promo_sk@0)], projection=[cs_ext_sales_price@3, cs_net_profit@4, cr_return_amount@5, cr_net_loss@6, cp_catalog_page_id@7] + │ CoalescePartitionsExec + │ [Stage 8] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, cs_item_sk@0)], projection=[cs_promo_sk@2, cs_ext_sales_price@3, cs_net_profit@4, cr_return_amount@5, cr_net_loss@6, cp_catalog_page_id@7] + │ CoalescePartitionsExec + │ [Stage 9] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ ProjectionExec: expr=[cs_item_sk@1 as cs_item_sk, cs_promo_sk@2 as cs_promo_sk, cs_ext_sales_price@3 as cs_ext_sales_price, cs_net_profit@4 as cs_net_profit, cr_return_amount@5 as cr_return_amount, cr_net_loss@6 as cr_net_loss, cp_catalog_page_id@0 as cp_catalog_page_id] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(catalog_page.cp_catalog_page_sk AS Float64)@2, cs_catalog_page_sk@0)], projection=[cp_catalog_page_id@1, cs_item_sk@4, cs_promo_sk@5, cs_ext_sales_price@6, cs_net_profit@7, cr_return_amount@8, cr_net_loss@9] + │ CoalescePartitionsExec + │ [Stage 10] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, cs_sold_date_sk@0)], projection=[cs_catalog_page_sk@3, cs_item_sk@4, cs_promo_sk@5, cs_ext_sales_price@6, cs_net_profit@7, cr_return_amount@8, cr_net_loss@9] + │ CoalescePartitionsExec + │ [Stage 11] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ ProjectionExec: expr=[cs_sold_date_sk@2 as cs_sold_date_sk, cs_catalog_page_sk@3 as cs_catalog_page_sk, cs_item_sk@4 as cs_item_sk, cs_promo_sk@5 as cs_promo_sk, cs_ext_sales_price@6 as cs_ext_sales_price, cs_net_profit@7 as cs_net_profit, cr_return_amount@0 as cr_return_amount, cr_net_loss@1 as cr_net_loss] + │ HashJoinExec: mode=Partitioned, join_type=Right, on=[(cr_item_sk@0, cs_item_sk@2), (cr_order_number@1, cs_order_number@4)], projection=[cr_return_amount@2, cr_net_loss@3, cs_sold_date_sk@4, cs_catalog_page_sk@5, cs_item_sk@6, cs_promo_sk@7, cs_ext_sales_price@9, cs_net_profit@10] + │ [Stage 12] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + │ [Stage 13] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 8 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[p_promo_sk@0 as p_promo_sk, CAST(p_promo_sk@0 AS Float64) as CAST(promotion.p_promo_sk AS Float64)] + │ FilterExec: p_channel_tv@1 = N, projection=[p_promo_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/promotion/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/promotion/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/promotion/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/promotion/part-3.parquet]]}, projection=[p_promo_sk, p_channel_tv], file_type=parquet, predicate=p_channel_tv@11 = N, pruning_predicate=p_channel_tv_null_count@2 != row_count@3 AND p_channel_tv_min@0 <= N AND N <= p_channel_tv_max@1, required_guarantees=[p_channel_tv in (N)] + └────────────────────────────────────────────────── + ┌───── Stage 9 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ FilterExec: i_current_price@1 > Some(5000),4,2, projection=[i_item_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_current_price], file_type=parquet, predicate=i_current_price@5 > Some(5000),4,2, pruning_predicate=i_current_price_null_count@1 != row_count@2 AND i_current_price_max@0 > Some(5000),4,2, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 10 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[cp_catalog_page_sk@0 as cp_catalog_page_sk, cp_catalog_page_id@1 as cp_catalog_page_id, CAST(cp_catalog_page_sk@0 AS Float64) as CAST(catalog_page.cp_catalog_page_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_page/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/catalog_page/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/catalog_page/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/catalog_page/part-3.parquet]]}, projection=[cp_catalog_page_sk, cp_catalog_page_id], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 11 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-22, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 12 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] + │ RepartitionExec: partitioning=Hash([cr_item_sk@0, cr_order_number@1], 9), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], ...]}, projection=[cr_item_sk, cr_order_number, cr_return_amount, cr_net_loss], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 13 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] + │ RepartitionExec: partitioning=Hash([cs_item_sk@2, cs_order_number@4], 9), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_catalog_page_sk, cs_item_sk, cs_promo_sk, cs_order_number, cs_ext_sales_price, cs_net_profit], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 21 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([web_site_id@0], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[web_site_id@4 as web_site_id], aggr=[sum(web_sales.ws_ext_sales_price), sum(coalesce(web_returns.wr_return_amt,Int64(0))), sum(web_sales.ws_net_profit - coalesce(web_returns.wr_net_loss,Int64(0)))] + │ ProjectionExec: expr=[CAST(wr_return_amt@2 AS Decimal128(22, 2)) as __common_expr_5, CAST(wr_net_loss@3 AS Decimal128(22, 2)) as __common_expr_6, ws_ext_sales_price@0 as ws_ext_sales_price, ws_net_profit@1 as ws_net_profit, web_site_id@4 as web_site_id] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ws_promo_sk@0, CAST(promotion.p_promo_sk AS Float64)@1)], projection=[ws_ext_sales_price@1, ws_net_profit@2, wr_return_amt@3, wr_net_loss@4, web_site_id@5] + │ CoalescePartitionsExec + │ [Stage 20] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=3 + │ ProjectionExec: expr=[p_promo_sk@0 as p_promo_sk, CAST(p_promo_sk@0 AS Float64) as CAST(promotion.p_promo_sk AS Float64)] + │ FilterExec: p_channel_tv@1 = N, projection=[p_promo_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/promotion/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/promotion/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/promotion/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/promotion/part-3.parquet]]}, projection=[p_promo_sk, p_channel_tv], file_type=parquet, predicate=p_channel_tv@11 = N, pruning_predicate=p_channel_tv_null_count@2 != row_count@3 AND p_channel_tv_min@0 <= N AND N <= p_channel_tv_max@1, required_guarantees=[p_channel_tv in (N)] + └────────────────────────────────────────────────── + ┌───── Stage 20 ── Tasks: t0:[p0..p8] t1:[p9..p17] t2:[p18..p26] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ws_item_sk@0, i_item_sk@0)], projection=[ws_promo_sk@1, ws_ext_sales_price@2, ws_net_profit@3, wr_return_amt@4, wr_net_loss@5, web_site_id@6] + │ CoalescePartitionsExec + │ [Stage 19] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=3 + │ FilterExec: i_current_price@1 > Some(5000),4,2, projection=[i_item_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_current_price], file_type=parquet, predicate=i_current_price@5 > Some(5000),4,2 AND DynamicFilter [ empty ], pruning_predicate=i_current_price_null_count@1 != row_count@2 AND i_current_price_max@0 > Some(5000),4,2, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 19 ── Tasks: t0:[p0..p8] t1:[p9..p17] t2:[p18..p26] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[ws_item_sk@1 as ws_item_sk, ws_promo_sk@2 as ws_promo_sk, ws_ext_sales_price@3 as ws_ext_sales_price, ws_net_profit@4 as ws_net_profit, wr_return_amt@5 as wr_return_amt, wr_net_loss@6 as wr_net_loss, web_site_id@0 as web_site_id] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(web_site.web_site_sk AS Float64)@2, ws_web_site_sk@1)], projection=[web_site_id@1, ws_item_sk@3, ws_promo_sk@5, ws_ext_sales_price@6, ws_net_profit@7, wr_return_amt@8, wr_net_loss@9] + │ CoalescePartitionsExec + │ [Stage 15] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ws_sold_date_sk@0)], projection=[ws_item_sk@3, ws_web_site_sk@4, ws_promo_sk@5, ws_ext_sales_price@6, ws_net_profit@7, wr_return_amt@8, wr_net_loss@9] + │ CoalescePartitionsExec + │ [Stage 16] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ ProjectionExec: expr=[ws_sold_date_sk@2 as ws_sold_date_sk, ws_item_sk@3 as ws_item_sk, ws_web_site_sk@4 as ws_web_site_sk, ws_promo_sk@5 as ws_promo_sk, ws_ext_sales_price@6 as ws_ext_sales_price, ws_net_profit@7 as ws_net_profit, wr_return_amt@0 as wr_return_amt, wr_net_loss@1 as wr_net_loss] + │ HashJoinExec: mode=Partitioned, join_type=Right, on=[(wr_item_sk@0, ws_item_sk@1), (wr_order_number@1, ws_order_number@4)], projection=[wr_return_amt@2, wr_net_loss@3, ws_sold_date_sk@4, ws_item_sk@5, ws_web_site_sk@6, ws_promo_sk@7, ws_ext_sales_price@9, ws_net_profit@10] + │ [Stage 17] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 18] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 15 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_site/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_site/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_site/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_site/part-3.parquet]]}, projection=[web_site_sk, web_site_id, CAST(web_site_sk@0 AS Float64) as CAST(web_site.web_site_sk AS Float64)], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 16 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_date@1 >= 2000-08-23 AND d_date@1 <= 2000-09-22, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-08-23 AND d_date@2 <= 2000-09-22, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-08-23 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-09-22, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 17 ── Tasks: t0:[p0..p8] t1:[p0..p8] + │ RepartitionExec: partitioning=Hash([wr_item_sk@0, wr_order_number@1], 9), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet]]}, projection=[wr_item_sk, wr_order_number, wr_return_amt, wr_net_loss], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 18 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] + │ RepartitionExec: partitioning=Hash([ws_item_sk@1, ws_order_number@4], 9), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_item_sk, ws_web_site_sk, ws_promo_sk, ws_order_number, ws_ext_sales_price, ws_net_profit], file_type=parquet + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_81() -> Result<()> { + let display = test_tpcds_query("q81").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [c_customer_id@0 ASC NULLS LAST, c_salutation@1 ASC NULLS LAST, c_first_name@2 ASC NULLS LAST, c_last_name@3 ASC NULLS LAST, ca_street_number@4 ASC NULLS LAST, ca_street_name@5 ASC NULLS LAST, ca_street_type@6 ASC NULLS LAST, ca_suite_number@7 ASC NULLS LAST, ca_city@8 ASC NULLS LAST, ca_county@9 ASC NULLS LAST, ca_state@10 ASC NULLS LAST, ca_zip@11 ASC NULLS LAST, ca_country@12 ASC NULLS LAST, ca_gmt_offset@13 ASC NULLS LAST, ca_location_type@14 ASC NULLS LAST, ctr_total_return@15 ASC NULLS LAST], fetch=100 + │ [Stage 11] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 11 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ SortExec: TopK(fetch=100), expr=[c_customer_id@0 ASC NULLS LAST, c_salutation@1 ASC NULLS LAST, c_first_name@2 ASC NULLS LAST, c_last_name@3 ASC NULLS LAST, ca_street_number@4 ASC NULLS LAST, ca_street_name@5 ASC NULLS LAST, ca_street_type@6 ASC NULLS LAST, ca_suite_number@7 ASC NULLS LAST, ca_city@8 ASC NULLS LAST, ca_county@9 ASC NULLS LAST, ca_zip@11 ASC NULLS LAST, ca_country@12 ASC NULLS LAST, ca_gmt_offset@13 ASC NULLS LAST, ca_location_type@14 ASC NULLS LAST, ctr_total_return@15 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[c_customer_id@12 as c_customer_id, c_salutation@13 as c_salutation, c_first_name@14 as c_first_name, c_last_name@15 as c_last_name, ca_street_number@1 as ca_street_number, ca_street_name@2 as ca_street_name, ca_street_type@3 as ca_street_type, ca_suite_number@4 as ca_suite_number, ca_city@5 as ca_city, ca_county@6 as ca_county, ca_state@7 as ca_state, ca_zip@8 as ca_zip, ca_country@9 as ca_country, ca_gmt_offset@10 as ca_gmt_offset, ca_location_type@11 as ca_location_type, ctr_total_return@0 as ctr_total_return] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ctr_state@0, ctr_state@1)], filter=CAST(ctr_total_return@0 AS Decimal128(30, 15)) > avg(ctr2.ctr_total_return) * Float64(1.2)@1, projection=[ctr_total_return@1, ca_street_number@2, ca_street_name@3, ca_street_type@4, ca_suite_number@5, ca_city@6, ca_county@7, ca_state@8, ca_zip@9, ca_country@10, ca_gmt_offset@11, ca_location_type@12, c_customer_id@13, c_salutation@14, c_first_name@15, c_last_name@16] + │ CoalescePartitionsExec + │ [Stage 6] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[CAST(CAST(avg(ctr2.ctr_total_return)@1 AS Float64) * 1.2 AS Decimal128(30, 15)) as avg(ctr2.ctr_total_return) * Float64(1.2), ctr_state@0 as ctr_state] + │ AggregateExec: mode=FinalPartitioned, gby=[ctr_state@0 as ctr_state], aggr=[avg(ctr2.ctr_total_return)] + │ [Stage 10] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=3, consumer_tasks=2, output_partitions=6 + │ ProjectionExec: expr=[ctr_state@11 as ctr_state, ctr_total_return@12 as ctr_total_return, ca_street_number@0 as ca_street_number, ca_street_name@1 as ca_street_name, ca_street_type@2 as ca_street_type, ca_suite_number@3 as ca_suite_number, ca_city@4 as ca_city, ca_county@5 as ca_county, ca_state@6 as ca_state, ca_zip@7 as ca_zip, ca_country@8 as ca_country, ca_gmt_offset@9 as ca_gmt_offset, ca_location_type@10 as ca_location_type, c_customer_id@13 as c_customer_id, c_salutation@14 as c_salutation, c_first_name@15 as c_first_name, c_last_name@16 as c_last_name] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ca_address_sk@0, c_current_addr_sk@3)], projection=[ca_street_number@1, ca_street_name@2, ca_street_type@3, ca_suite_number@4, ca_city@5, ca_county@6, ca_state@7, ca_zip@8, ca_country@9, ca_gmt_offset@10, ca_location_type@11, ctr_state@12, ctr_total_return@13, c_customer_id@14, c_salutation@16, c_first_name@17, c_last_name@18] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=6, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ctr_customer_sk@0, CAST(customer.c_customer_sk AS Float64)@6)], projection=[ctr_state@1, ctr_total_return@2, c_customer_id@4, c_current_addr_sk@5, c_salutation@6, c_first_name@7, c_last_name@8] + │ CoalescePartitionsExec + │ [Stage 5] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, c_customer_id@1 as c_customer_id, c_current_addr_sk@2 as c_current_addr_sk, c_salutation@3 as c_salutation, c_first_name@4 as c_first_name, c_last_name@5 as c_last_name, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_customer_id, c_current_addr_sk, c_salutation, c_first_name, c_last_name], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=3, consumer_tasks=2, output_partitions=6 + │ FilterExec: ca_state@7 = GA + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_street_number, ca_street_name, ca_street_type, ca_suite_number, ca_city, ca_county, ca_state, ca_zip, ca_country, ca_gmt_offset, ca_location_type], file_type=parquet, predicate=ca_state@8 = GA, pruning_predicate=ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= GA AND GA <= ca_state_max@1, required_guarantees=[ca_state in (GA)] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=3, consumer_tasks=2, output_partitions=6 + │ ProjectionExec: expr=[cr_returning_customer_sk@0 as ctr_customer_sk, ca_state@1 as ctr_state, sum(catalog_returns.cr_return_amt_inc_tax)@2 as ctr_total_return] + │ AggregateExec: mode=FinalPartitioned, gby=[cr_returning_customer_sk@0 as cr_returning_customer_sk, ca_state@1 as ca_state], aggr=[sum(catalog_returns.cr_return_amt_inc_tax)] + │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([cr_returning_customer_sk@0, ca_state@1], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[cr_returning_customer_sk@0 as cr_returning_customer_sk, ca_state@2 as ca_state], aggr=[sum(catalog_returns.cr_return_amt_inc_tax)] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(cr_returning_addr_sk@1, CAST(customer_address.ca_address_sk AS Float64)@2)], projection=[cr_returning_customer_sk@0, cr_return_amt_inc_tax@2, ca_state@4] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=3 + │ ProjectionExec: expr=[ca_address_sk@0 as ca_address_sk, ca_state@1 as ca_state, CAST(ca_address_sk@0 AS Float64) as CAST(customer_address.ca_address_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_state], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p5] t1:[p6..p11] t2:[p12..p17] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, cr_returned_date_sk@0)], projection=[cr_returning_customer_sk@2, cr_returning_addr_sk@3, cr_return_amt_inc_tax@4] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], ...]}, projection=[cr_returned_date_sk, cr_returning_customer_sk, cr_returning_addr_sk, cr_return_amt_inc_tax], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ FilterExec: d_year@1 = 2000, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] + └────────────────────────────────────────────────── + ┌───── Stage 10 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ RepartitionExec: partitioning=Hash([ctr_state@0], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[ctr_state@0 as ctr_state], aggr=[avg(ctr2.ctr_total_return)] + │ ProjectionExec: expr=[ca_state@1 as ctr_state, sum(catalog_returns.cr_return_amt_inc_tax)@2 as ctr_total_return] + │ AggregateExec: mode=FinalPartitioned, gby=[cr_returning_customer_sk@0 as cr_returning_customer_sk, ca_state@1 as ca_state], aggr=[sum(catalog_returns.cr_return_amt_inc_tax)] + │ [Stage 9] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 9 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([cr_returning_customer_sk@0, ca_state@1], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[cr_returning_customer_sk@0 as cr_returning_customer_sk, ca_state@2 as ca_state], aggr=[sum(catalog_returns.cr_return_amt_inc_tax)] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(cr_returning_addr_sk@1, CAST(customer_address.ca_address_sk AS Float64)@2)], projection=[cr_returning_customer_sk@0, cr_return_amt_inc_tax@2, ca_state@4] + │ CoalescePartitionsExec + │ [Stage 8] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=3 + │ ProjectionExec: expr=[ca_address_sk@0 as ca_address_sk, ca_state@1 as ca_state, CAST(ca_address_sk@0 AS Float64) as CAST(customer_address.ca_address_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_state], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 8 ── Tasks: t0:[p0..p5] t1:[p6..p11] t2:[p12..p17] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, cr_returned_date_sk@0)], projection=[cr_returning_customer_sk@2, cr_returning_addr_sk@3, cr_return_amt_inc_tax@4] + │ CoalescePartitionsExec + │ [Stage 7] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], ...]}, projection=[cr_returned_date_sk, cr_returning_customer_sk, cr_returning_addr_sk, cr_return_amt_inc_tax], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ FilterExec: d_year@1 = 2000, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_82() -> Result<()> { + let display = test_tpcds_query("q82").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [i_item_id@0 ASC NULLS LAST], fetch=100 + │ [Stage 5] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ SortExec: TopK(fetch=100), expr=[i_item_id@0 ASC NULLS LAST], preserve_partitioning=[true] + │ AggregateExec: mode=FinalPartitioned, gby=[i_item_id@0 as i_item_id, i_item_desc@1 as i_item_desc, i_current_price@2 as i_current_price], aggr=[] + │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([i_item_id@0, i_item_desc@1, i_current_price@2], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[i_item_id@0 as i_item_id, i_item_desc@1 as i_item_desc, i_current_price@2 as i_current_price], aggr=[] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ss_item_sk@0)], projection=[i_item_id@1, i_item_desc@2, i_current_price@3] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=3 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p8] t1:[p9..p17] t2:[p18..p26] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(inv_date_sk@4, d_date_sk@0)], projection=[i_item_sk@0, i_item_id@1, i_item_desc@2, i_current_price@3] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=3 + │ FilterExec: d_date@1 >= 2000-05-25 AND d_date@1 <= 2000-07-24, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-05-25 AND d_date@2 <= 2000-07-24 AND DynamicFilter [ empty ], pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-05-25 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-07-24, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p5] t1:[p6..p11] t2:[p12..p17] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, inv_item_sk@1)], projection=[i_item_sk@0, i_item_id@1, i_item_desc@2, i_current_price@3, inv_date_sk@4] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ FilterExec: inv_quantity_on_hand@2 >= 100 AND inv_quantity_on_hand@2 <= 500, projection=[inv_date_sk@0, inv_item_sk@1] + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/inventory/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/inventory/part-2.parquet:..], ...]}, projection=[inv_date_sk, inv_item_sk, inv_quantity_on_hand], file_type=parquet, predicate=inv_quantity_on_hand@3 >= 100 AND inv_quantity_on_hand@3 <= 500 AND DynamicFilter [ empty ], pruning_predicate=inv_quantity_on_hand_null_count@1 != row_count@2 AND inv_quantity_on_hand_max@0 >= 100 AND inv_quantity_on_hand_null_count@1 != row_count@2 AND inv_quantity_on_hand_min@3 <= 500, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ FilterExec: i_current_price@3 >= Some(6200),4,2 AND i_current_price@3 <= Some(9200),4,2 AND i_manufact_id@4 IN (SET) ([129, 270, 821, 423]), projection=[i_item_sk@0, i_item_id@1, i_item_desc@2, i_current_price@3] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_item_id, i_item_desc, i_current_price, i_manufact_id], file_type=parquet, predicate=i_current_price@5 >= Some(6200),4,2 AND i_current_price@5 <= Some(9200),4,2 AND i_manufact_id@13 IN (SET) ([129, 270, 821, 423]) AND DynamicFilter [ empty ], pruning_predicate=i_current_price_null_count@1 != row_count@2 AND i_current_price_max@0 >= Some(6200),4,2 AND i_current_price_null_count@1 != row_count@2 AND i_current_price_min@3 <= Some(9200),4,2 AND (i_manufact_id_null_count@6 != row_count@2 AND i_manufact_id_min@4 <= 129 AND 129 <= i_manufact_id_max@5 OR i_manufact_id_null_count@6 != row_count@2 AND i_manufact_id_min@4 <= 270 AND 270 <= i_manufact_id_max@5 OR i_manufact_id_null_count@6 != row_count@2 AND i_manufact_id_min@4 <= 821 AND 821 <= i_manufact_id_max@5 OR i_manufact_id_null_count@6 != row_count@2 AND i_manufact_id_min@4 <= 423 AND 423 <= i_manufact_id_max@5), required_guarantees=[i_manufact_id in (129, 270, 423, 821)] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_83() -> Result<()> { + let display = test_tpcds_query("q83").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [item_id@0 ASC, sr_item_qty@1 ASC], fetch=100 + │ [Stage 17] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 17 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ SortExec: TopK(fetch=100), expr=[item_id@0 ASC, sr_item_qty@1 ASC], preserve_partitioning=[true] + │ ProjectionExec: expr=[item_id@1 as item_id, sr_item_qty@2 as sr_item_qty, sr_item_qty@2 / __common_expr_7@0 / 3 * 100 as sr_dev, cr_item_qty@3 as cr_item_qty, cr_item_qty@3 / __common_expr_7@0 / 3 * 100 as cr_dev, wr_item_qty@4 as wr_item_qty, wr_item_qty@4 / __common_expr_7@0 / 3 * 100 as wr_dev, __common_expr_7@0 / 3 as average] + │ ProjectionExec: expr=[sr_item_qty@2 + cr_item_qty@3 + wr_item_qty@0 as __common_expr_7, item_id@1 as item_id, sr_item_qty@2 as sr_item_qty, cr_item_qty@3 as cr_item_qty, wr_item_qty@0 as wr_item_qty] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(item_id@0, item_id@0)], projection=[wr_item_qty@1, item_id@2, sr_item_qty@3, cr_item_qty@4] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=6, input_tasks=1 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(item_id@0, item_id@0)], projection=[item_id@0, sr_item_qty@1, cr_item_qty@3] + │ CoalescePartitionsExec + │ [Stage 11] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[i_item_id@0 as item_id, sum(catalog_returns.cr_return_quantity)@1 as cr_item_qty] + │ AggregateExec: mode=FinalPartitioned, gby=[i_item_id@0 as i_item_id], aggr=[sum(catalog_returns.cr_return_quantity)] + │ [Stage 16] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p5] + │ BroadcastExec: input_partitions=3, consumer_tasks=2, output_partitions=6 + │ ProjectionExec: expr=[i_item_id@0 as item_id, sum(web_returns.wr_return_quantity)@1 as wr_item_qty] + │ AggregateExec: mode=FinalPartitioned, gby=[i_item_id@0 as i_item_id], aggr=[sum(web_returns.wr_return_quantity)] + │ RepartitionExec: partitioning=Hash([i_item_id@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_item_id@1 as i_item_id], aggr=[sum(web_returns.wr_return_quantity)] + │ HashJoinExec: mode=CollectLeft, join_type=LeftSemi, on=[(d_date@2, d_date@0)], projection=[wr_return_quantity@0, i_item_id@1] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(d_week_seq@0, d_week_seq@1)], projection=[d_date@0] + │ CoalescePartitionsExec + │ BroadcastExec: input_partitions=3, consumer_tasks=1, output_partitions=3 + │ FilterExec: d_date@0 = 2000-06-30 OR d_date@0 = 2000-09-27 OR d_date@0 = 2000-11-17, projection=[d_week_seq@1] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=d_date@2 = 2000-06-30 OR d_date@2 = 2000-09-27 OR d_date@2 = 2000-11-17, pruning_predicate=d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-06-30 AND 2000-06-30 <= d_date_max@1 OR d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-09-27 AND 2000-09-27 <= d_date_max@1 OR d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-11-17 AND 2000-11-17 <= d_date_max@1, required_guarantees=[d_date in (2000-06-30, 2000-09-27, 2000-11-17)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date, d_week_seq], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p3..p5] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(wr_returned_date_sk@0, CAST(date_dim.d_date_sk AS Float64)@2)], projection=[wr_return_quantity@1, i_item_id@2, d_date@4] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_date@1 as d_date, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=3, consumer_tasks=2, output_partitions=6 + │ ProjectionExec: expr=[wr_returned_date_sk@1 as wr_returned_date_sk, wr_return_quantity@2 as wr_return_quantity, i_item_id@0 as i_item_id] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, wr_item_sk@1)], projection=[i_item_id@1, wr_returned_date_sk@2, wr_return_quantity@4] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=4, input_tasks=2 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet]]}, projection=[wr_returned_date_sk, wr_item_sk, wr_return_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p3] t1:[p4..p7] + │ BroadcastExec: input_partitions=2, consumer_tasks=2, output_partitions=4 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_item_id], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 11 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=3, consumer_tasks=2, output_partitions=6 + │ ProjectionExec: expr=[i_item_id@0 as item_id, sum(store_returns.sr_return_quantity)@1 as sr_item_qty] + │ AggregateExec: mode=FinalPartitioned, gby=[i_item_id@0 as i_item_id], aggr=[sum(store_returns.sr_return_quantity)] + │ [Stage 10] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 10 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([i_item_id@0], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[i_item_id@1 as i_item_id], aggr=[sum(store_returns.sr_return_quantity)] + │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(d_date@0, d_date@2)], projection=[sr_return_quantity@0, i_item_id@1] + │ CoalescePartitionsExec + │ [Stage 6] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ ProjectionExec: expr=[sr_return_quantity@1 as sr_return_quantity, i_item_id@2 as i_item_id, d_date@0 as d_date] + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, sr_returned_date_sk@0)], projection=[d_date@1, sr_return_quantity@4, i_item_id@5] + │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 9] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(d_week_seq@0, d_week_seq@1)], projection=[d_date@0] + │ CoalescePartitionsExec + │ [Stage 5] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=6, input_tasks=2 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date, d_week_seq], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=3, consumer_tasks=2, output_partitions=6 + │ FilterExec: d_date@0 = 2000-06-30 OR d_date@0 = 2000-09-27 OR d_date@0 = 2000-11-17, projection=[d_week_seq@1] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=d_date@2 = 2000-06-30 OR d_date@2 = 2000-09-27 OR d_date@2 = 2000-11-17, pruning_predicate=d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-06-30 AND 2000-06-30 <= d_date_max@1 OR d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-09-27 AND 2000-09-27 <= d_date_max@1 OR d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-11-17 AND 2000-11-17 <= d_date_max@1, required_guarantees=[d_date in (2000-06-30, 2000-09-27, 2000-11-17)] + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p8] t1:[p0..p8] + │ RepartitionExec: partitioning=Hash([CAST(date_dim.d_date_sk AS Float64)@2], 9), input_partitions=3 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_date@1 as d_date, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 9 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] + │ RepartitionExec: partitioning=Hash([sr_returned_date_sk@0], 9), input_partitions=2 + │ ProjectionExec: expr=[sr_returned_date_sk@1 as sr_returned_date_sk, sr_return_quantity@2 as sr_return_quantity, i_item_id@0 as i_item_id] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, sr_item_sk@1)], projection=[i_item_id@1, sr_returned_date_sk@2, sr_return_quantity@4] + │ CoalescePartitionsExec + │ [Stage 8] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], ...]}, projection=[sr_returned_date_sk, sr_item_sk, sr_return_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 8 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_item_id], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 16 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([i_item_id@0], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[i_item_id@1 as i_item_id], aggr=[sum(catalog_returns.cr_return_quantity)] + │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(d_date@0, d_date@2)], projection=[cr_return_quantity@0, i_item_id@1] + │ CoalescePartitionsExec + │ [Stage 13] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ ProjectionExec: expr=[cr_return_quantity@1 as cr_return_quantity, i_item_id@2 as i_item_id, d_date@0 as d_date] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(d_date_sk@0, cr_returned_date_sk@0)], projection=[d_date@1, cr_return_quantity@3, i_item_id@4] + │ CoalescePartitionsExec + │ [Stage 14] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[cr_returned_date_sk@1 as cr_returned_date_sk, cr_return_quantity@2 as cr_return_quantity, i_item_id@0 as i_item_id] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, cr_item_sk@1)], projection=[i_item_id@1, cr_returned_date_sk@2, cr_return_quantity@4] + │ CoalescePartitionsExec + │ [Stage 15] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], ...]}, projection=[cr_returned_date_sk, cr_item_sk, cr_return_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 13 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(d_week_seq@0, d_week_seq@1)], projection=[d_date@0] + │ CoalescePartitionsExec + │ [Stage 12] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=6, input_tasks=2 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date, d_week_seq], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 12 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=3, consumer_tasks=2, output_partitions=6 + │ FilterExec: d_date@0 = 2000-06-30 OR d_date@0 = 2000-09-27 OR d_date@0 = 2000-11-17, projection=[d_week_seq@1] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date, d_week_seq], file_type=parquet, predicate=d_date@2 = 2000-06-30 OR d_date@2 = 2000-09-27 OR d_date@2 = 2000-11-17, pruning_predicate=d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-06-30 AND 2000-06-30 <= d_date_max@1 OR d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-09-27 AND 2000-09-27 <= d_date_max@1 OR d_date_null_count@2 != row_count@3 AND d_date_min@0 <= 2000-11-17 AND 2000-11-17 <= d_date_max@1, required_guarantees=[d_date in (2000-06-30, 2000-09-27, 2000-11-17)] + └────────────────────────────────────────────────── + ┌───── Stage 14 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 15 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_item_id], file_type=parquet + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_84() -> Result<()> { + let display = test_tpcds_query("q84").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ ProjectionExec: expr=[customer_id@0 as customer_id, customername@1 as customername] + │ SortPreservingMergeExec: [c_customer_id@2 ASC], fetch=100 + │ [Stage 6] => NetworkCoalesceExec: output_partitions=6, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] + │ SortExec: TopK(fetch=100), expr=[customer_id@0 ASC], preserve_partitioning=[true] + │ ProjectionExec: expr=[c_customer_id@0 as customer_id, concat(concat(CASE WHEN c_last_name@2 IS NOT NULL THEN c_last_name@2 ELSE END, , ), CASE WHEN c_first_name@1 IS NOT NULL THEN c_first_name@1 ELSE END) as customername, c_customer_id@0 as c_customer_id] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(customer_demographics.cd_demo_sk AS Float64)@4, sr_cdemo_sk@0)], projection=[c_customer_id@0, c_first_name@1, c_last_name@2] + │ CoalescePartitionsExec + │ [Stage 5] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=3 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], ...]}, projection=[sr_cdemo_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p5] t1:[p6..p11] t2:[p12..p17] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ ProjectionExec: expr=[c_customer_id@0 as c_customer_id, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, cd_demo_sk@3 as cd_demo_sk, CAST(cd_demo_sk@3 AS Float64) as CAST(customer_demographics.cd_demo_sk AS Float64)] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ib_income_band_sk@0, hd_income_band_sk@4)], projection=[c_customer_id@1, c_first_name@2, c_last_name@3, cd_demo_sk@4] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ ProjectionExec: expr=[c_customer_id@1 as c_customer_id, c_first_name@2 as c_first_name, c_last_name@3 as c_last_name, cd_demo_sk@4 as cd_demo_sk, hd_income_band_sk@0 as hd_income_band_sk] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(household_demographics.hd_demo_sk AS Float64)@2, c_current_hdemo_sk@1)], projection=[hd_income_band_sk@1, c_customer_id@3, c_first_name@5, c_last_name@6, cd_demo_sk@7] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_current_cdemo_sk@1, CAST(customer_demographics.cd_demo_sk AS Float64)@1)], projection=[c_customer_id@0, c_current_hdemo_sk@2, c_first_name@3, c_last_name@4, cd_demo_sk@5] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], ...]}, projection=[cd_demo_sk, CAST(cd_demo_sk@0 AS Float64) as CAST(customer_demographics.cd_demo_sk AS Float64)], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ FilterExec: ib_lower_bound@1 >= 38128 AND ib_upper_bound@2 <= 88128, projection=[ib_income_band_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/income_band/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/income_band/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/income_band/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/income_band/part-3.parquet]]}, projection=[ib_income_band_sk, ib_lower_bound, ib_upper_bound], file_type=parquet, predicate=ib_lower_bound@1 >= 38128 AND ib_upper_bound@2 <= 88128, pruning_predicate=ib_lower_bound_null_count@1 != row_count@2 AND ib_lower_bound_max@0 >= 38128 AND ib_upper_bound_null_count@4 != row_count@2 AND ib_upper_bound_min@3 <= 88128, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-3.parquet]]}, projection=[hd_demo_sk, hd_income_band_sk, CAST(hd_demo_sk@0 AS Float64) as CAST(household_demographics.hd_demo_sk AS Float64)], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ca_address_sk@0, c_current_addr_sk@3)], projection=[c_customer_id@1, c_current_cdemo_sk@2, c_current_hdemo_sk@3, c_first_name@5, c_last_name@6] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=6, input_tasks=2 + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_id, c_current_cdemo_sk, c_current_hdemo_sk, c_current_addr_sk, c_first_name, c_last_name], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=3, consumer_tasks=2, output_partitions=6 + │ FilterExec: ca_city@1 = Edgewood, projection=[ca_address_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_city], file_type=parquet, predicate=ca_city@6 = Edgewood, pruning_predicate=ca_city_null_count@2 != row_count@3 AND ca_city_min@0 <= Edgewood AND Edgewood <= ca_city_max@1, required_guarantees=[ca_city in (Edgewood)] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_85() -> Result<()> { + let display = test_tpcds_query("q85").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [substr(reason.r_reason_desc,Int64(1),Int64(20))@0 ASC NULLS LAST, avg1@1 ASC NULLS LAST, avg2@2 ASC NULLS LAST, avg(web_returns.wr_fee)@3 ASC NULLS LAST], fetch=100 + │ [Stage 10] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 10 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ SortExec: TopK(fetch=100), expr=[substr(reason.r_reason_desc,Int64(1),Int64(20))@0 ASC NULLS LAST, avg1@1 ASC NULLS LAST, avg2@2 ASC NULLS LAST, avg(web_returns.wr_fee)@3 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[substr(r_reason_desc@0, 1, 20) as substr(reason.r_reason_desc,Int64(1),Int64(20)), avg(web_sales.ws_quantity)@1 as avg1, avg(web_returns.wr_refunded_cash)@2 as avg2, avg(web_returns.wr_fee)@3 as avg(web_returns.wr_fee)] + │ AggregateExec: mode=FinalPartitioned, gby=[r_reason_desc@0 as r_reason_desc], aggr=[avg(web_sales.ws_quantity), avg(web_returns.wr_refunded_cash), avg(web_returns.wr_fee)] + │ [Stage 9] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 9 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([r_reason_desc@0], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[r_reason_desc@3 as r_reason_desc], aggr=[avg(web_sales.ws_quantity), avg(web_returns.wr_refunded_cash), avg(web_returns.wr_fee)] + │ ProjectionExec: expr=[ws_quantity@1 as ws_quantity, wr_fee@2 as wr_fee, wr_refunded_cash@3 as wr_refunded_cash, r_reason_desc@0 as r_reason_desc] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(reason.r_reason_sk AS Float64)@2, wr_reason_sk@1)], projection=[r_reason_desc@1, ws_quantity@3, wr_fee@5, wr_refunded_cash@6] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ws_sold_date_sk@0, CAST(date_dim.d_date_sk AS Float64)@1)], projection=[ws_quantity@1, wr_reason_sk@2, wr_fee@3, wr_refunded_cash@4] + │ CoalescePartitionsExec + │ [Stage 8] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=3 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_year@1 = 2000, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year], file_type=parquet, predicate=d_year@6 = 2000, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 2000 AND 2000 <= d_year_max@1, required_guarantees=[d_year in (2000)] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/reason/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/reason/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/reason/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/reason/part-3.parquet]]}, projection=[r_reason_sk, r_reason_desc, CAST(r_reason_sk@0 AS Float64) as CAST(reason.r_reason_sk AS Float64)], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 8 ── Tasks: t0:[p0..p8] t1:[p9..p17] t2:[p18..p26] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(wr_refunded_addr_sk@3, CAST(customer_address.ca_address_sk AS Float64)@2)], filter=(ca_state@1 = IN OR ca_state@1 = OH OR ca_state@1 = NJ) AND ws_net_profit@0 >= Some(10000),7,2 AND ws_net_profit@0 <= Some(20000),7,2 OR (ca_state@1 = WI OR ca_state@1 = CT OR ca_state@1 = KY) AND ws_net_profit@0 >= Some(15000),7,2 AND ws_net_profit@0 <= Some(30000),7,2 OR (ca_state@1 = LA OR ca_state@1 = IA OR ca_state@1 = AR) AND ws_net_profit@0 >= Some(5000),7,2 AND ws_net_profit@0 <= Some(25000),7,2, projection=[ws_sold_date_sk@0, ws_quantity@1, wr_reason_sk@4, wr_fee@5, wr_refunded_cash@6] + │ CoalescePartitionsExec + │ [Stage 7] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=3 + │ ProjectionExec: expr=[ca_address_sk@0 as ca_address_sk, ca_state@1 as ca_state, CAST(ca_address_sk@0 AS Float64) as CAST(customer_address.ca_address_sk AS Float64)] + │ FilterExec: (ca_state@1 = IN OR ca_state@1 = OH OR ca_state@1 = NJ OR ca_state@1 = WI OR ca_state@1 = CT OR ca_state@1 = KY OR ca_state@1 = LA OR ca_state@1 = IA OR ca_state@1 = AR) AND ca_state@1 IN (SET) ([IN, OH, NJ, WI, CT, KY, LA, IA, AR]) AND ca_country@2 = United States, projection=[ca_address_sk@0, ca_state@1] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_state, ca_country], file_type=parquet, predicate=(ca_state@8 = IN OR ca_state@8 = OH OR ca_state@8 = NJ OR ca_state@8 = WI OR ca_state@8 = CT OR ca_state@8 = KY OR ca_state@8 = LA OR ca_state@8 = IA OR ca_state@8 = AR) AND ca_state@8 IN (SET) ([IN, OH, NJ, WI, CT, KY, LA, IA, AR]) AND ca_country@10 = United States, pruning_predicate=(ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= IN AND IN <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= OH AND OH <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= NJ AND NJ <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= WI AND WI <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= CT AND CT <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= KY AND KY <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= LA AND LA <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= IA AND IA <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= AR AND AR <= ca_state_max@1) AND (ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= IN AND IN <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= OH AND OH <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= NJ AND NJ <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= WI AND WI <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= CT AND CT <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= KY AND KY <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= LA AND LA <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= IA AND IA <= ca_state_max@1 OR ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= AR AND AR <= ca_state_max@1) AND ca_country_null_count@6 != row_count@3 AND ca_country_min@4 <= United States AND United States <= ca_country_max@5, required_guarantees=[ca_country in (United States), ca_state in (AR, CT, IA, IN, KY, LA, NJ, OH, WI)] + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p5] t1:[p6..p11] t2:[p12..p17] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(wr_returning_cdemo_sk@4, CAST(cd2.cd_demo_sk AS Float64)@3), (cd_marital_status@8, cd_marital_status@1), (cd_education_status@9, cd_education_status@2)], projection=[ws_sold_date_sk@0, ws_quantity@1, ws_net_profit@2, wr_refunded_addr_sk@3, wr_reason_sk@5, wr_fee@6, wr_refunded_cash@7] + │ CoalescePartitionsExec + │ [Stage 6] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=3 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], ...]}, projection=[cd_demo_sk, cd_marital_status, cd_education_status, CAST(cd_demo_sk@0 AS Float64) as CAST(cd2.cd_demo_sk AS Float64)], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p5] t1:[p6..p11] t2:[p12..p17] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(wr_refunded_cdemo_sk@4, CAST(cd1.cd_demo_sk AS Float64)@3)], filter=cd_marital_status@1 = M AND cd_education_status@2 = Advanced Degree AND ws_sales_price@0 >= Some(10000),5,2 AND ws_sales_price@0 <= Some(15000),5,2 OR cd_marital_status@1 = S AND cd_education_status@2 = College AND ws_sales_price@0 >= Some(5000),5,2 AND ws_sales_price@0 <= Some(10000),5,2 OR cd_marital_status@1 = W AND cd_education_status@2 = 2 yr Degree AND ws_sales_price@0 >= Some(15000),5,2 AND ws_sales_price@0 <= Some(20000),5,2, projection=[ws_sold_date_sk@0, ws_quantity@1, ws_net_profit@3, wr_refunded_addr_sk@5, wr_returning_cdemo_sk@6, wr_reason_sk@7, wr_fee@8, wr_refunded_cash@9, cd_marital_status@11, cd_education_status@12] + │ CoalescePartitionsExec + │ [Stage 5] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ ProjectionExec: expr=[cd_demo_sk@0 as cd_demo_sk, cd_marital_status@1 as cd_marital_status, cd_education_status@2 as cd_education_status, CAST(cd_demo_sk@0 AS Float64) as CAST(cd1.cd_demo_sk AS Float64)] + │ FilterExec: cd_marital_status@1 = M AND cd_education_status@2 = Advanced Degree OR cd_marital_status@1 = S AND cd_education_status@2 = College OR cd_marital_status@1 = W AND cd_education_status@2 = 2 yr Degree + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], ...]}, projection=[cd_demo_sk, cd_marital_status, cd_education_status], file_type=parquet, predicate=cd_marital_status@2 = M AND cd_education_status@3 = Advanced Degree OR cd_marital_status@2 = S AND cd_education_status@3 = College OR cd_marital_status@2 = W AND cd_education_status@3 = 2 yr Degree, pruning_predicate=cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= M AND M <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= Advanced Degree AND Advanced Degree <= cd_education_status_max@5 OR cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= S AND S <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= College AND College <= cd_education_status_max@5 OR cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= W AND W <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= 2 yr Degree AND 2 yr Degree <= cd_education_status_max@5, required_guarantees=[cd_education_status in (2 yr Degree, Advanced Degree, College), cd_marital_status in (M, S, W)] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(web_page.wp_web_page_sk AS Float64)@1, ws_web_page_sk@1)], projection=[ws_sold_date_sk@2, ws_quantity@4, ws_sales_price@5, ws_net_profit@6, wr_refunded_cdemo_sk@7, wr_refunded_addr_sk@8, wr_returning_cdemo_sk@9, wr_reason_sk@10, wr_fee@11, wr_refunded_cash@12] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=4, input_tasks=2 + │ ProjectionExec: expr=[ws_sold_date_sk@6 as ws_sold_date_sk, ws_web_page_sk@7 as ws_web_page_sk, ws_quantity@8 as ws_quantity, ws_sales_price@9 as ws_sales_price, ws_net_profit@10 as ws_net_profit, wr_refunded_cdemo_sk@0 as wr_refunded_cdemo_sk, wr_refunded_addr_sk@1 as wr_refunded_addr_sk, wr_returning_cdemo_sk@2 as wr_returning_cdemo_sk, wr_reason_sk@3 as wr_reason_sk, wr_fee@4 as wr_fee, wr_refunded_cash@5 as wr_refunded_cash] + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(wr_item_sk@0, ws_item_sk@1), (wr_order_number@5, ws_order_number@3)], projection=[wr_refunded_cdemo_sk@1, wr_refunded_addr_sk@2, wr_returning_cdemo_sk@3, wr_reason_sk@4, wr_fee@6, wr_refunded_cash@7, ws_sold_date_sk@8, ws_web_page_sk@10, ws_quantity@12, ws_sales_price@13, ws_net_profit@14] + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p3] t1:[p4..p7] + │ BroadcastExec: input_partitions=2, consumer_tasks=2, output_partitions=4 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_page/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_page/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_page/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_page/part-3.parquet]]}, projection=[wp_web_page_sk, CAST(wp_web_page_sk@0 AS Float64) as CAST(web_page.wp_web_page_sk AS Float64)], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ RepartitionExec: partitioning=Hash([wr_item_sk@0, wr_order_number@5], 6), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet]]}, projection=[wr_item_sk, wr_refunded_cdemo_sk, wr_refunded_addr_sk, wr_returning_cdemo_sk, wr_reason_sk, wr_order_number, wr_fee, wr_refunded_cash], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([ws_item_sk@1, ws_order_number@3], 6), input_partitions=2 + │ FilterExec: (ws_net_profit@6 >= Some(10000),7,2 AND ws_net_profit@6 <= Some(20000),7,2 OR ws_net_profit@6 >= Some(15000),7,2 AND ws_net_profit@6 <= Some(30000),7,2 OR ws_net_profit@6 >= Some(5000),7,2 AND ws_net_profit@6 <= Some(25000),7,2) AND (ws_sales_price@5 >= Some(10000),5,2 AND ws_sales_price@5 <= Some(15000),5,2 OR ws_sales_price@5 >= Some(5000),5,2 AND ws_sales_price@5 <= Some(10000),5,2 OR ws_sales_price@5 >= Some(15000),5,2 AND ws_sales_price@5 <= Some(20000),5,2) + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_item_sk, ws_web_page_sk, ws_order_number, ws_quantity, ws_sales_price, ws_net_profit], file_type=parquet, predicate=(ws_net_profit@33 >= Some(10000),7,2 AND ws_net_profit@33 <= Some(20000),7,2 OR ws_net_profit@33 >= Some(15000),7,2 AND ws_net_profit@33 <= Some(30000),7,2 OR ws_net_profit@33 >= Some(5000),7,2 AND ws_net_profit@33 <= Some(25000),7,2) AND (ws_sales_price@21 >= Some(10000),5,2 AND ws_sales_price@21 <= Some(15000),5,2 OR ws_sales_price@21 >= Some(5000),5,2 AND ws_sales_price@21 <= Some(10000),5,2 OR ws_sales_price@21 >= Some(15000),5,2 AND ws_sales_price@21 <= Some(20000),5,2) AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], pruning_predicate=(ws_net_profit_null_count@1 != row_count@2 AND ws_net_profit_max@0 >= Some(10000),7,2 AND ws_net_profit_null_count@1 != row_count@2 AND ws_net_profit_min@3 <= Some(20000),7,2 OR ws_net_profit_null_count@1 != row_count@2 AND ws_net_profit_max@0 >= Some(15000),7,2 AND ws_net_profit_null_count@1 != row_count@2 AND ws_net_profit_min@3 <= Some(30000),7,2 OR ws_net_profit_null_count@1 != row_count@2 AND ws_net_profit_max@0 >= Some(5000),7,2 AND ws_net_profit_null_count@1 != row_count@2 AND ws_net_profit_min@3 <= Some(25000),7,2) AND (ws_sales_price_null_count@5 != row_count@2 AND ws_sales_price_max@4 >= Some(10000),5,2 AND ws_sales_price_null_count@5 != row_count@2 AND ws_sales_price_min@6 <= Some(15000),5,2 OR ws_sales_price_null_count@5 != row_count@2 AND ws_sales_price_max@4 >= Some(5000),5,2 AND ws_sales_price_null_count@5 != row_count@2 AND ws_sales_price_min@6 <= Some(10000),5,2 OR ws_sales_price_null_count@5 != row_count@2 AND ws_sales_price_max@4 >= Some(15000),5,2 AND ws_sales_price_null_count@5 != row_count@2 AND ws_sales_price_min@6 <= Some(20000),5,2), required_guarantees=[] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + #[ignore = "The ordering of the column names in the first nodes is non deterministickI"] + async fn test_tpcds_86() -> Result<()> { + let display = test_tpcds_query("q86").await?; + assert_snapshot!(display, @r#""#); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_87() -> Result<()> { + let display = test_tpcds_query("q87").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ ProjectionExec: expr=[count(Int64(1))@0 as count(*)] + │ AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))] + │ CoalescePartitionsExec + │ [Stage 12] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 12 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] + │ ProjectionExec: expr=[] + │ HashJoinExec: mode=CollectLeft, join_type=RightAnti, on=[(c_last_name@0, c_last_name@0), (c_first_name@1, c_first_name@1), (d_date@2, d_date@2)], NullsEqual: true + │ CoalescePartitionsExec + │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=6, input_tasks=2 + │ AggregateExec: mode=SinglePartitioned, gby=[c_last_name@0 as c_last_name, c_first_name@1 as c_first_name, d_date@2 as d_date], aggr=[] + │ HashJoinExec: mode=CollectLeft, join_type=RightAnti, on=[(c_last_name@0, c_last_name@0), (c_first_name@1, c_first_name@1), (d_date@2, d_date@2)], NullsEqual: true + │ CoalescePartitionsExec + │ [Stage 8] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=6, input_tasks=2 + │ AggregateExec: mode=SinglePartitioned, gby=[c_last_name@0 as c_last_name, c_first_name@1 as c_first_name, d_date@2 as d_date], aggr=[] + │ AggregateExec: mode=FinalPartitioned, gby=[c_last_name@0 as c_last_name, c_first_name@1 as c_first_name, d_date@2 as d_date], aggr=[] + │ [Stage 11] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=3, consumer_tasks=2, output_partitions=6 + │ AggregateExec: mode=FinalPartitioned, gby=[c_last_name@0 as c_last_name, c_first_name@1 as c_first_name, d_date@2 as d_date], aggr=[] + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([c_last_name@0, c_first_name@1, d_date@2], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[c_last_name@0 as c_last_name, c_first_name@1 as c_first_name, d_date@2 as d_date], aggr=[] + │ ProjectionExec: expr=[c_last_name@2 as c_last_name, c_first_name@1 as c_first_name, d_date@0 as d_date] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ws_bill_customer_sk@0, CAST(customer.c_customer_sk AS Float64)@3)], projection=[d_date@1, c_first_name@3, c_last_name@4] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=3 + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p5] t1:[p6..p11] t2:[p12..p17] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ ProjectionExec: expr=[ws_bill_customer_sk@1 as ws_bill_customer_sk, d_date@0 as d_date] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, ws_sold_date_sk@0)], projection=[d_date@1, ws_bill_customer_sk@4] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_bill_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_date@1 as d_date, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_month_seq@2 >= 1200 AND d_month_seq@2 <= 1211, projection=[d_date_sk@0, d_date@1] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 8 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=3, consumer_tasks=2, output_partitions=6 + │ AggregateExec: mode=FinalPartitioned, gby=[c_last_name@0 as c_last_name, c_first_name@1 as c_first_name, d_date@2 as d_date], aggr=[] + │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([c_last_name@0, c_first_name@1, d_date@2], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[c_last_name@0 as c_last_name, c_first_name@1 as c_first_name, d_date@2 as d_date], aggr=[] + │ ProjectionExec: expr=[c_last_name@2 as c_last_name, c_first_name@1 as c_first_name, d_date@0 as d_date] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(cs_bill_customer_sk@0, CAST(customer.c_customer_sk AS Float64)@3)], projection=[d_date@1, c_first_name@3, c_last_name@4] + │ CoalescePartitionsExec + │ [Stage 6] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=3 + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p5] t1:[p6..p11] t2:[p12..p17] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ ProjectionExec: expr=[cs_bill_customer_sk@1 as cs_bill_customer_sk, d_date@0 as d_date] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, cs_sold_date_sk@0)], projection=[d_date@1, cs_bill_customer_sk@4] + │ CoalescePartitionsExec + │ [Stage 5] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_bill_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_date@1 as d_date, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_month_seq@2 >= 1200 AND d_month_seq@2 <= 1211, projection=[d_date_sk@0, d_date@1] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 11 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([c_last_name@0, c_first_name@1, d_date@2], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[c_last_name@0 as c_last_name, c_first_name@1 as c_first_name, d_date@2 as d_date], aggr=[] + │ ProjectionExec: expr=[c_last_name@2 as c_last_name, c_first_name@1 as c_first_name, d_date@0 as d_date] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ss_customer_sk@0, CAST(customer.c_customer_sk AS Float64)@3)], projection=[d_date@1, c_first_name@3, c_last_name@4] + │ CoalescePartitionsExec + │ [Stage 10] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=3 + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, c_first_name@1 as c_first_name, c_last_name@2 as c_last_name, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_first_name, c_last_name], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 10 ── Tasks: t0:[p0..p5] t1:[p6..p11] t2:[p12..p17] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ ProjectionExec: expr=[ss_customer_sk@1 as ss_customer_sk, d_date@0 as d_date] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, ss_sold_date_sk@0)], projection=[d_date@1, ss_customer_sk@4] + │ CoalescePartitionsExec + │ [Stage 9] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 9 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_date@1 as d_date, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_month_seq@2 >= 1200 AND d_month_seq@2 <= 1211, projection=[d_date_sk@0, d_date@1] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_88() -> Result<()> { + let display = test_tpcds_query("q88").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ ProjectionExec: expr=[h8_30_to_9@1 as h8_30_to_9, h9_to_9_30@2 as h9_to_9_30, h9_30_to_10@3 as h9_30_to_10, h10_to_10_30@4 as h10_to_10_30, h10_30_to_11@5 as h10_30_to_11, h11_to_11_30@6 as h11_to_11_30, h11_30_to_12@7 as h11_30_to_12, h12_to_12_30@0 as h12_to_12_30] + │ CrossJoinExec + │ ProjectionExec: expr=[count(Int64(1))@0 as h12_to_12_30] + │ AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkCoalesceExec: output_partitions=6, input_tasks=3 + │ ProjectionExec: expr=[h8_30_to_9@1 as h8_30_to_9, h9_to_9_30@2 as h9_to_9_30, h9_30_to_10@3 as h9_30_to_10, h10_to_10_30@4 as h10_to_10_30, h10_30_to_11@5 as h10_30_to_11, h11_to_11_30@6 as h11_to_11_30, h11_30_to_12@0 as h11_30_to_12] + │ CrossJoinExec + │ ProjectionExec: expr=[count(Int64(1))@0 as h11_30_to_12] + │ AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))] + │ CoalescePartitionsExec + │ [Stage 8] => NetworkCoalesceExec: output_partitions=6, input_tasks=3 + │ ProjectionExec: expr=[h8_30_to_9@1 as h8_30_to_9, h9_to_9_30@2 as h9_to_9_30, h9_30_to_10@3 as h9_30_to_10, h10_to_10_30@4 as h10_to_10_30, h10_30_to_11@5 as h10_30_to_11, h11_to_11_30@0 as h11_to_11_30] + │ CrossJoinExec + │ ProjectionExec: expr=[count(Int64(1))@0 as h11_to_11_30] + │ AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))] + │ CoalescePartitionsExec + │ [Stage 12] => NetworkCoalesceExec: output_partitions=6, input_tasks=3 + │ ProjectionExec: expr=[h8_30_to_9@1 as h8_30_to_9, h9_to_9_30@2 as h9_to_9_30, h9_30_to_10@3 as h9_30_to_10, h10_to_10_30@4 as h10_to_10_30, h10_30_to_11@0 as h10_30_to_11] + │ CrossJoinExec + │ ProjectionExec: expr=[count(Int64(1))@0 as h10_30_to_11] + │ AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))] + │ CoalescePartitionsExec + │ [Stage 16] => NetworkCoalesceExec: output_partitions=6, input_tasks=3 + │ ProjectionExec: expr=[h8_30_to_9@1 as h8_30_to_9, h9_to_9_30@2 as h9_to_9_30, h9_30_to_10@3 as h9_30_to_10, h10_to_10_30@0 as h10_to_10_30] + │ CrossJoinExec + │ ProjectionExec: expr=[count(Int64(1))@0 as h10_to_10_30] + │ AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))] + │ CoalescePartitionsExec + │ [Stage 20] => NetworkCoalesceExec: output_partitions=6, input_tasks=3 + │ ProjectionExec: expr=[h8_30_to_9@1 as h8_30_to_9, h9_to_9_30@2 as h9_to_9_30, h9_30_to_10@0 as h9_30_to_10] + │ CrossJoinExec + │ ProjectionExec: expr=[count(Int64(1))@0 as h9_30_to_10] + │ AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))] + │ CoalescePartitionsExec + │ [Stage 24] => NetworkCoalesceExec: output_partitions=6, input_tasks=3 + │ CrossJoinExec + │ ProjectionExec: expr=[count(Int64(1))@0 as h8_30_to_9] + │ AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))] + │ CoalescePartitionsExec + │ [Stage 28] => NetworkCoalesceExec: output_partitions=6, input_tasks=3 + │ ProjectionExec: expr=[count(Int64(1))@0 as h9_to_9_30] + │ AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))] + │ CoalescePartitionsExec + │ [Stage 32] => NetworkCoalesceExec: output_partitions=6, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] + │ AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] + │ ProjectionExec: expr=[] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@1, ss_store_sk@0)] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(time_dim.t_time_sk AS Float64)@1, ss_sold_time_sk@0)], projection=[ss_store_sk@3] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(household_demographics.hd_demo_sk AS Float64)@1, ss_hdemo_sk@1)], projection=[ss_sold_time_sk@2, ss_store_sk@4] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] + │ FilterExec: s_store_name@1 = ese, projection=[s_store_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_store_name], file_type=parquet, predicate=s_store_name@5 = ese, pruning_predicate=s_store_name_null_count@2 != row_count@3 AND s_store_name_min@0 <= ese AND ese <= s_store_name_max@1, required_guarantees=[s_store_name in (ese)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[t_time_sk@0 as t_time_sk, CAST(t_time_sk@0 AS Float64) as CAST(time_dim.t_time_sk AS Float64)] + │ FilterExec: t_hour@1 = 12 AND t_minute@2 < 30, projection=[t_time_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@3 = 12 AND t_minute@4 < 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 12 AND 12 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_min@4 < 30, required_guarantees=[t_hour in (12)] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[hd_demo_sk@0 as hd_demo_sk, CAST(hd_demo_sk@0 AS Float64) as CAST(household_demographics.hd_demo_sk AS Float64)] + │ FilterExec: hd_dep_count@1 = 4 AND hd_vehicle_count@2 <= 6 OR hd_dep_count@1 = 2 AND hd_vehicle_count@2 <= 4 OR hd_dep_count@1 = 0 AND hd_vehicle_count@2 <= 2, projection=[hd_demo_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-3.parquet]]}, projection=[hd_demo_sk, hd_dep_count, hd_vehicle_count], file_type=parquet, predicate=hd_dep_count@3 = 4 AND hd_vehicle_count@4 <= 6 OR hd_dep_count@3 = 2 AND hd_vehicle_count@4 <= 4 OR hd_dep_count@3 = 0 AND hd_vehicle_count@4 <= 2, pruning_predicate=hd_dep_count_null_count@2 != row_count@3 AND hd_dep_count_min@0 <= 4 AND 4 <= hd_dep_count_max@1 AND hd_vehicle_count_null_count@5 != row_count@3 AND hd_vehicle_count_min@4 <= 6 OR hd_dep_count_null_count@2 != row_count@3 AND hd_dep_count_min@0 <= 2 AND 2 <= hd_dep_count_max@1 AND hd_vehicle_count_null_count@5 != row_count@3 AND hd_vehicle_count_min@4 <= 4 OR hd_dep_count_null_count@2 != row_count@3 AND hd_dep_count_min@0 <= 0 AND 0 <= hd_dep_count_max@1 AND hd_vehicle_count_null_count@5 != row_count@3 AND hd_vehicle_count_min@4 <= 2, required_guarantees=[hd_dep_count in (0, 2, 4)] + └────────────────────────────────────────────────── + ┌───── Stage 8 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] + │ AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] + │ ProjectionExec: expr=[] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@1, ss_store_sk@0)] + │ CoalescePartitionsExec + │ [Stage 5] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(time_dim.t_time_sk AS Float64)@1, ss_sold_time_sk@0)], projection=[ss_store_sk@3] + │ CoalescePartitionsExec + │ [Stage 6] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(household_demographics.hd_demo_sk AS Float64)@1, ss_hdemo_sk@1)], projection=[ss_sold_time_sk@2, ss_store_sk@4] + │ CoalescePartitionsExec + │ [Stage 7] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] + │ FilterExec: s_store_name@1 = ese, projection=[s_store_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_store_name], file_type=parquet, predicate=s_store_name@5 = ese, pruning_predicate=s_store_name_null_count@2 != row_count@3 AND s_store_name_min@0 <= ese AND ese <= s_store_name_max@1, required_guarantees=[s_store_name in (ese)] + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[t_time_sk@0 as t_time_sk, CAST(t_time_sk@0 AS Float64) as CAST(time_dim.t_time_sk AS Float64)] + │ FilterExec: t_hour@1 = 11 AND t_minute@2 >= 30, projection=[t_time_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@3 = 11 AND t_minute@4 >= 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 11 AND 11 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_max@4 >= 30, required_guarantees=[t_hour in (11)] + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[hd_demo_sk@0 as hd_demo_sk, CAST(hd_demo_sk@0 AS Float64) as CAST(household_demographics.hd_demo_sk AS Float64)] + │ FilterExec: hd_dep_count@1 = 4 AND hd_vehicle_count@2 <= 6 OR hd_dep_count@1 = 2 AND hd_vehicle_count@2 <= 4 OR hd_dep_count@1 = 0 AND hd_vehicle_count@2 <= 2, projection=[hd_demo_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-3.parquet]]}, projection=[hd_demo_sk, hd_dep_count, hd_vehicle_count], file_type=parquet, predicate=hd_dep_count@3 = 4 AND hd_vehicle_count@4 <= 6 OR hd_dep_count@3 = 2 AND hd_vehicle_count@4 <= 4 OR hd_dep_count@3 = 0 AND hd_vehicle_count@4 <= 2, pruning_predicate=hd_dep_count_null_count@2 != row_count@3 AND hd_dep_count_min@0 <= 4 AND 4 <= hd_dep_count_max@1 AND hd_vehicle_count_null_count@5 != row_count@3 AND hd_vehicle_count_min@4 <= 6 OR hd_dep_count_null_count@2 != row_count@3 AND hd_dep_count_min@0 <= 2 AND 2 <= hd_dep_count_max@1 AND hd_vehicle_count_null_count@5 != row_count@3 AND hd_vehicle_count_min@4 <= 4 OR hd_dep_count_null_count@2 != row_count@3 AND hd_dep_count_min@0 <= 0 AND 0 <= hd_dep_count_max@1 AND hd_vehicle_count_null_count@5 != row_count@3 AND hd_vehicle_count_min@4 <= 2, required_guarantees=[hd_dep_count in (0, 2, 4)] + └────────────────────────────────────────────────── + ┌───── Stage 12 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] + │ AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] + │ ProjectionExec: expr=[] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@1, ss_store_sk@0)] + │ CoalescePartitionsExec + │ [Stage 9] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(time_dim.t_time_sk AS Float64)@1, ss_sold_time_sk@0)], projection=[ss_store_sk@3] + │ CoalescePartitionsExec + │ [Stage 10] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(household_demographics.hd_demo_sk AS Float64)@1, ss_hdemo_sk@1)], projection=[ss_sold_time_sk@2, ss_store_sk@4] + │ CoalescePartitionsExec + │ [Stage 11] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 9 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] + │ FilterExec: s_store_name@1 = ese, projection=[s_store_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_store_name], file_type=parquet, predicate=s_store_name@5 = ese, pruning_predicate=s_store_name_null_count@2 != row_count@3 AND s_store_name_min@0 <= ese AND ese <= s_store_name_max@1, required_guarantees=[s_store_name in (ese)] + └────────────────────────────────────────────────── + ┌───── Stage 10 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[t_time_sk@0 as t_time_sk, CAST(t_time_sk@0 AS Float64) as CAST(time_dim.t_time_sk AS Float64)] + │ FilterExec: t_hour@1 = 11 AND t_minute@2 < 30, projection=[t_time_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@3 = 11 AND t_minute@4 < 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 11 AND 11 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_min@4 < 30, required_guarantees=[t_hour in (11)] + └────────────────────────────────────────────────── + ┌───── Stage 11 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[hd_demo_sk@0 as hd_demo_sk, CAST(hd_demo_sk@0 AS Float64) as CAST(household_demographics.hd_demo_sk AS Float64)] + │ FilterExec: hd_dep_count@1 = 4 AND hd_vehicle_count@2 <= 6 OR hd_dep_count@1 = 2 AND hd_vehicle_count@2 <= 4 OR hd_dep_count@1 = 0 AND hd_vehicle_count@2 <= 2, projection=[hd_demo_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-3.parquet]]}, projection=[hd_demo_sk, hd_dep_count, hd_vehicle_count], file_type=parquet, predicate=hd_dep_count@3 = 4 AND hd_vehicle_count@4 <= 6 OR hd_dep_count@3 = 2 AND hd_vehicle_count@4 <= 4 OR hd_dep_count@3 = 0 AND hd_vehicle_count@4 <= 2, pruning_predicate=hd_dep_count_null_count@2 != row_count@3 AND hd_dep_count_min@0 <= 4 AND 4 <= hd_dep_count_max@1 AND hd_vehicle_count_null_count@5 != row_count@3 AND hd_vehicle_count_min@4 <= 6 OR hd_dep_count_null_count@2 != row_count@3 AND hd_dep_count_min@0 <= 2 AND 2 <= hd_dep_count_max@1 AND hd_vehicle_count_null_count@5 != row_count@3 AND hd_vehicle_count_min@4 <= 4 OR hd_dep_count_null_count@2 != row_count@3 AND hd_dep_count_min@0 <= 0 AND 0 <= hd_dep_count_max@1 AND hd_vehicle_count_null_count@5 != row_count@3 AND hd_vehicle_count_min@4 <= 2, required_guarantees=[hd_dep_count in (0, 2, 4)] + └────────────────────────────────────────────────── + ┌───── Stage 16 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] + │ AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] + │ ProjectionExec: expr=[] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@1, ss_store_sk@0)] + │ CoalescePartitionsExec + │ [Stage 13] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(time_dim.t_time_sk AS Float64)@1, ss_sold_time_sk@0)], projection=[ss_store_sk@3] + │ CoalescePartitionsExec + │ [Stage 14] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(household_demographics.hd_demo_sk AS Float64)@1, ss_hdemo_sk@1)], projection=[ss_sold_time_sk@2, ss_store_sk@4] + │ CoalescePartitionsExec + │ [Stage 15] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 13 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] + │ FilterExec: s_store_name@1 = ese, projection=[s_store_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_store_name], file_type=parquet, predicate=s_store_name@5 = ese, pruning_predicate=s_store_name_null_count@2 != row_count@3 AND s_store_name_min@0 <= ese AND ese <= s_store_name_max@1, required_guarantees=[s_store_name in (ese)] + └────────────────────────────────────────────────── + ┌───── Stage 14 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[t_time_sk@0 as t_time_sk, CAST(t_time_sk@0 AS Float64) as CAST(time_dim.t_time_sk AS Float64)] + │ FilterExec: t_hour@1 = 10 AND t_minute@2 >= 30, projection=[t_time_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@3 = 10 AND t_minute@4 >= 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 10 AND 10 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_max@4 >= 30, required_guarantees=[t_hour in (10)] + └────────────────────────────────────────────────── + ┌───── Stage 15 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[hd_demo_sk@0 as hd_demo_sk, CAST(hd_demo_sk@0 AS Float64) as CAST(household_demographics.hd_demo_sk AS Float64)] + │ FilterExec: hd_dep_count@1 = 4 AND hd_vehicle_count@2 <= 6 OR hd_dep_count@1 = 2 AND hd_vehicle_count@2 <= 4 OR hd_dep_count@1 = 0 AND hd_vehicle_count@2 <= 2, projection=[hd_demo_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-3.parquet]]}, projection=[hd_demo_sk, hd_dep_count, hd_vehicle_count], file_type=parquet, predicate=hd_dep_count@3 = 4 AND hd_vehicle_count@4 <= 6 OR hd_dep_count@3 = 2 AND hd_vehicle_count@4 <= 4 OR hd_dep_count@3 = 0 AND hd_vehicle_count@4 <= 2, pruning_predicate=hd_dep_count_null_count@2 != row_count@3 AND hd_dep_count_min@0 <= 4 AND 4 <= hd_dep_count_max@1 AND hd_vehicle_count_null_count@5 != row_count@3 AND hd_vehicle_count_min@4 <= 6 OR hd_dep_count_null_count@2 != row_count@3 AND hd_dep_count_min@0 <= 2 AND 2 <= hd_dep_count_max@1 AND hd_vehicle_count_null_count@5 != row_count@3 AND hd_vehicle_count_min@4 <= 4 OR hd_dep_count_null_count@2 != row_count@3 AND hd_dep_count_min@0 <= 0 AND 0 <= hd_dep_count_max@1 AND hd_vehicle_count_null_count@5 != row_count@3 AND hd_vehicle_count_min@4 <= 2, required_guarantees=[hd_dep_count in (0, 2, 4)] + └────────────────────────────────────────────────── + ┌───── Stage 20 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] + │ AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] + │ ProjectionExec: expr=[] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@1, ss_store_sk@0)] + │ CoalescePartitionsExec + │ [Stage 17] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(time_dim.t_time_sk AS Float64)@1, ss_sold_time_sk@0)], projection=[ss_store_sk@3] + │ CoalescePartitionsExec + │ [Stage 18] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(household_demographics.hd_demo_sk AS Float64)@1, ss_hdemo_sk@1)], projection=[ss_sold_time_sk@2, ss_store_sk@4] + │ CoalescePartitionsExec + │ [Stage 19] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 17 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] + │ FilterExec: s_store_name@1 = ese, projection=[s_store_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_store_name], file_type=parquet, predicate=s_store_name@5 = ese, pruning_predicate=s_store_name_null_count@2 != row_count@3 AND s_store_name_min@0 <= ese AND ese <= s_store_name_max@1, required_guarantees=[s_store_name in (ese)] + └────────────────────────────────────────────────── + ┌───── Stage 18 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[t_time_sk@0 as t_time_sk, CAST(t_time_sk@0 AS Float64) as CAST(time_dim.t_time_sk AS Float64)] + │ FilterExec: t_hour@1 = 10 AND t_minute@2 < 30, projection=[t_time_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@3 = 10 AND t_minute@4 < 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 10 AND 10 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_min@4 < 30, required_guarantees=[t_hour in (10)] + └────────────────────────────────────────────────── + ┌───── Stage 19 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[hd_demo_sk@0 as hd_demo_sk, CAST(hd_demo_sk@0 AS Float64) as CAST(household_demographics.hd_demo_sk AS Float64)] + │ FilterExec: hd_dep_count@1 = 4 AND hd_vehicle_count@2 <= 6 OR hd_dep_count@1 = 2 AND hd_vehicle_count@2 <= 4 OR hd_dep_count@1 = 0 AND hd_vehicle_count@2 <= 2, projection=[hd_demo_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-3.parquet]]}, projection=[hd_demo_sk, hd_dep_count, hd_vehicle_count], file_type=parquet, predicate=hd_dep_count@3 = 4 AND hd_vehicle_count@4 <= 6 OR hd_dep_count@3 = 2 AND hd_vehicle_count@4 <= 4 OR hd_dep_count@3 = 0 AND hd_vehicle_count@4 <= 2, pruning_predicate=hd_dep_count_null_count@2 != row_count@3 AND hd_dep_count_min@0 <= 4 AND 4 <= hd_dep_count_max@1 AND hd_vehicle_count_null_count@5 != row_count@3 AND hd_vehicle_count_min@4 <= 6 OR hd_dep_count_null_count@2 != row_count@3 AND hd_dep_count_min@0 <= 2 AND 2 <= hd_dep_count_max@1 AND hd_vehicle_count_null_count@5 != row_count@3 AND hd_vehicle_count_min@4 <= 4 OR hd_dep_count_null_count@2 != row_count@3 AND hd_dep_count_min@0 <= 0 AND 0 <= hd_dep_count_max@1 AND hd_vehicle_count_null_count@5 != row_count@3 AND hd_vehicle_count_min@4 <= 2, required_guarantees=[hd_dep_count in (0, 2, 4)] + └────────────────────────────────────────────────── + ┌───── Stage 24 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] + │ AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] + │ ProjectionExec: expr=[] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@1, ss_store_sk@0)] + │ CoalescePartitionsExec + │ [Stage 21] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(time_dim.t_time_sk AS Float64)@1, ss_sold_time_sk@0)], projection=[ss_store_sk@3] + │ CoalescePartitionsExec + │ [Stage 22] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(household_demographics.hd_demo_sk AS Float64)@1, ss_hdemo_sk@1)], projection=[ss_sold_time_sk@2, ss_store_sk@4] + │ CoalescePartitionsExec + │ [Stage 23] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 21 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] + │ FilterExec: s_store_name@1 = ese, projection=[s_store_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_store_name], file_type=parquet, predicate=s_store_name@5 = ese, pruning_predicate=s_store_name_null_count@2 != row_count@3 AND s_store_name_min@0 <= ese AND ese <= s_store_name_max@1, required_guarantees=[s_store_name in (ese)] + └────────────────────────────────────────────────── + ┌───── Stage 22 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[t_time_sk@0 as t_time_sk, CAST(t_time_sk@0 AS Float64) as CAST(time_dim.t_time_sk AS Float64)] + │ FilterExec: t_hour@1 = 9 AND t_minute@2 >= 30, projection=[t_time_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@3 = 9 AND t_minute@4 >= 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 9 AND 9 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_max@4 >= 30, required_guarantees=[t_hour in (9)] + └────────────────────────────────────────────────── + ┌───── Stage 23 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[hd_demo_sk@0 as hd_demo_sk, CAST(hd_demo_sk@0 AS Float64) as CAST(household_demographics.hd_demo_sk AS Float64)] + │ FilterExec: hd_dep_count@1 = 4 AND hd_vehicle_count@2 <= 6 OR hd_dep_count@1 = 2 AND hd_vehicle_count@2 <= 4 OR hd_dep_count@1 = 0 AND hd_vehicle_count@2 <= 2, projection=[hd_demo_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-3.parquet]]}, projection=[hd_demo_sk, hd_dep_count, hd_vehicle_count], file_type=parquet, predicate=hd_dep_count@3 = 4 AND hd_vehicle_count@4 <= 6 OR hd_dep_count@3 = 2 AND hd_vehicle_count@4 <= 4 OR hd_dep_count@3 = 0 AND hd_vehicle_count@4 <= 2, pruning_predicate=hd_dep_count_null_count@2 != row_count@3 AND hd_dep_count_min@0 <= 4 AND 4 <= hd_dep_count_max@1 AND hd_vehicle_count_null_count@5 != row_count@3 AND hd_vehicle_count_min@4 <= 6 OR hd_dep_count_null_count@2 != row_count@3 AND hd_dep_count_min@0 <= 2 AND 2 <= hd_dep_count_max@1 AND hd_vehicle_count_null_count@5 != row_count@3 AND hd_vehicle_count_min@4 <= 4 OR hd_dep_count_null_count@2 != row_count@3 AND hd_dep_count_min@0 <= 0 AND 0 <= hd_dep_count_max@1 AND hd_vehicle_count_null_count@5 != row_count@3 AND hd_vehicle_count_min@4 <= 2, required_guarantees=[hd_dep_count in (0, 2, 4)] + └────────────────────────────────────────────────── + ┌───── Stage 28 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] + │ AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] + │ ProjectionExec: expr=[] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@1, ss_store_sk@0)] + │ CoalescePartitionsExec + │ [Stage 25] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(time_dim.t_time_sk AS Float64)@1, ss_sold_time_sk@0)], projection=[ss_store_sk@3] + │ CoalescePartitionsExec + │ [Stage 26] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(household_demographics.hd_demo_sk AS Float64)@1, ss_hdemo_sk@1)], projection=[ss_sold_time_sk@2, ss_store_sk@4] + │ CoalescePartitionsExec + │ [Stage 27] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 25 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] + │ FilterExec: s_store_name@1 = ese, projection=[s_store_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_store_name], file_type=parquet, predicate=s_store_name@5 = ese, pruning_predicate=s_store_name_null_count@2 != row_count@3 AND s_store_name_min@0 <= ese AND ese <= s_store_name_max@1, required_guarantees=[s_store_name in (ese)] + └────────────────────────────────────────────────── + ┌───── Stage 26 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[t_time_sk@0 as t_time_sk, CAST(t_time_sk@0 AS Float64) as CAST(time_dim.t_time_sk AS Float64)] + │ FilterExec: t_hour@1 = 8 AND t_minute@2 >= 30, projection=[t_time_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@3 = 8 AND t_minute@4 >= 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 8 AND 8 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_max@4 >= 30, required_guarantees=[t_hour in (8)] + └────────────────────────────────────────────────── + ┌───── Stage 27 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[hd_demo_sk@0 as hd_demo_sk, CAST(hd_demo_sk@0 AS Float64) as CAST(household_demographics.hd_demo_sk AS Float64)] + │ FilterExec: hd_dep_count@1 = 4 AND hd_vehicle_count@2 <= 6 OR hd_dep_count@1 = 2 AND hd_vehicle_count@2 <= 4 OR hd_dep_count@1 = 0 AND hd_vehicle_count@2 <= 2, projection=[hd_demo_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-3.parquet]]}, projection=[hd_demo_sk, hd_dep_count, hd_vehicle_count], file_type=parquet, predicate=hd_dep_count@3 = 4 AND hd_vehicle_count@4 <= 6 OR hd_dep_count@3 = 2 AND hd_vehicle_count@4 <= 4 OR hd_dep_count@3 = 0 AND hd_vehicle_count@4 <= 2, pruning_predicate=hd_dep_count_null_count@2 != row_count@3 AND hd_dep_count_min@0 <= 4 AND 4 <= hd_dep_count_max@1 AND hd_vehicle_count_null_count@5 != row_count@3 AND hd_vehicle_count_min@4 <= 6 OR hd_dep_count_null_count@2 != row_count@3 AND hd_dep_count_min@0 <= 2 AND 2 <= hd_dep_count_max@1 AND hd_vehicle_count_null_count@5 != row_count@3 AND hd_vehicle_count_min@4 <= 4 OR hd_dep_count_null_count@2 != row_count@3 AND hd_dep_count_min@0 <= 0 AND 0 <= hd_dep_count_max@1 AND hd_vehicle_count_null_count@5 != row_count@3 AND hd_vehicle_count_min@4 <= 2, required_guarantees=[hd_dep_count in (0, 2, 4)] + └────────────────────────────────────────────────── + ┌───── Stage 32 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] + │ AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] + │ ProjectionExec: expr=[] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@1, ss_store_sk@0)] + │ CoalescePartitionsExec + │ [Stage 29] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(time_dim.t_time_sk AS Float64)@1, ss_sold_time_sk@0)], projection=[ss_store_sk@3] + │ CoalescePartitionsExec + │ [Stage 30] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(household_demographics.hd_demo_sk AS Float64)@1, ss_hdemo_sk@1)], projection=[ss_sold_time_sk@2, ss_store_sk@4] + │ CoalescePartitionsExec + │ [Stage 31] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 29 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] + │ FilterExec: s_store_name@1 = ese, projection=[s_store_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_store_name], file_type=parquet, predicate=s_store_name@5 = ese, pruning_predicate=s_store_name_null_count@2 != row_count@3 AND s_store_name_min@0 <= ese AND ese <= s_store_name_max@1, required_guarantees=[s_store_name in (ese)] + └────────────────────────────────────────────────── + ┌───── Stage 30 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[t_time_sk@0 as t_time_sk, CAST(t_time_sk@0 AS Float64) as CAST(time_dim.t_time_sk AS Float64)] + │ FilterExec: t_hour@1 = 9 AND t_minute@2 < 30, projection=[t_time_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@3 = 9 AND t_minute@4 < 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 9 AND 9 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_min@4 < 30, required_guarantees=[t_hour in (9)] + └────────────────────────────────────────────────── + ┌───── Stage 31 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[hd_demo_sk@0 as hd_demo_sk, CAST(hd_demo_sk@0 AS Float64) as CAST(household_demographics.hd_demo_sk AS Float64)] + │ FilterExec: hd_dep_count@1 = 4 AND hd_vehicle_count@2 <= 6 OR hd_dep_count@1 = 2 AND hd_vehicle_count@2 <= 4 OR hd_dep_count@1 = 0 AND hd_vehicle_count@2 <= 2, projection=[hd_demo_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-3.parquet]]}, projection=[hd_demo_sk, hd_dep_count, hd_vehicle_count], file_type=parquet, predicate=hd_dep_count@3 = 4 AND hd_vehicle_count@4 <= 6 OR hd_dep_count@3 = 2 AND hd_vehicle_count@4 <= 4 OR hd_dep_count@3 = 0 AND hd_vehicle_count@4 <= 2, pruning_predicate=hd_dep_count_null_count@2 != row_count@3 AND hd_dep_count_min@0 <= 4 AND 4 <= hd_dep_count_max@1 AND hd_vehicle_count_null_count@5 != row_count@3 AND hd_vehicle_count_min@4 <= 6 OR hd_dep_count_null_count@2 != row_count@3 AND hd_dep_count_min@0 <= 2 AND 2 <= hd_dep_count_max@1 AND hd_vehicle_count_null_count@5 != row_count@3 AND hd_vehicle_count_min@4 <= 4 OR hd_dep_count_null_count@2 != row_count@3 AND hd_dep_count_min@0 <= 0 AND 0 <= hd_dep_count_max@1 AND hd_vehicle_count_null_count@5 != row_count@3 AND hd_vehicle_count_min@4 <= 2, required_guarantees=[hd_dep_count in (0, 2, 4)] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_89() -> Result<()> { + let display = test_tpcds_query("q89").await?; + assert_snapshot!(display, @r#" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [sum_sales@6 - avg_monthly_sales@7 ASC NULLS LAST, s_store_name@3 ASC NULLS LAST, i_category@0 ASC NULLS LAST, i_class@1 ASC NULLS LAST, i_brand@2 ASC NULLS LAST, s_company_name@4 ASC NULLS LAST, d_moy@5 ASC NULLS LAST, sum_sales@6 ASC NULLS LAST, avg_monthly_sales@7 ASC NULLS LAST], fetch=100 + │ SortExec: TopK(fetch=100), expr=[sum_sales@6 - avg_monthly_sales@7 ASC NULLS LAST, s_store_name@3 ASC NULLS LAST, i_category@0 ASC NULLS LAST, i_class@1 ASC NULLS LAST, i_brand@2 ASC NULLS LAST, s_company_name@4 ASC NULLS LAST, d_moy@5 ASC NULLS LAST, sum_sales@6 ASC NULLS LAST, avg_monthly_sales@7 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[i_category@0 as i_category, i_class@1 as i_class, i_brand@2 as i_brand, s_store_name@3 as s_store_name, s_company_name@4 as s_company_name, d_moy@5 as d_moy, sum(store_sales.ss_sales_price)@6 as sum_sales, avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@7 as avg_monthly_sales] + │ FilterExec: CASE WHEN avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@7 != Some(0),19,6 THEN abs(sum(store_sales.ss_sales_price)@6 - avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@7) / avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@7 END > Some(1000000000),30,10 + │ WindowAggExec: wdw=[avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING: Ok(Field { name: "avg(sum(store_sales.ss_sales_price)) PARTITION BY [item.i_category, item.i_brand, store.s_store_name, store.s_company_name] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING", data_type: Decimal128(19, 6), nullable: true }), frame: WindowFrame { units: Rows, start_bound: Preceding(UInt64(NULL)), end_bound: Following(UInt64(NULL)), is_causal: false }] + │ SortExec: expr=[i_category@0 ASC NULLS LAST, i_brand@2 ASC NULLS LAST, s_store_name@3 ASC NULLS LAST, s_company_name@4 ASC NULLS LAST], preserve_partitioning=[true] + │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ RepartitionExec: partitioning=Hash([i_category@0, i_brand@2, s_store_name@3, s_company_name@4], 3), input_partitions=3 + │ AggregateExec: mode=FinalPartitioned, gby=[i_category@0 as i_category, i_class@1 as i_class, i_brand@2 as i_brand, s_store_name@3 as s_store_name, s_company_name@4 as s_company_name, d_moy@5 as d_moy], aggr=[sum(store_sales.ss_sales_price)] + │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([i_category@0, i_class@1, i_brand@2, s_store_name@3, s_company_name@4, d_moy@5], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[i_category@2 as i_category, i_class@1 as i_class, i_brand@0 as i_brand, s_store_name@5 as s_store_name, s_company_name@6 as s_company_name, d_moy@4 as d_moy], aggr=[sum(store_sales.ss_sales_price)] + │ ProjectionExec: expr=[i_brand@2 as i_brand, i_class@3 as i_class, i_category@4 as i_category, ss_sales_price@5 as ss_sales_price, d_moy@6 as d_moy, s_store_name@0 as s_store_name, s_company_name@1 as s_company_name] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@3, ss_store_sk@3)], projection=[s_store_name@1, s_company_name@2, i_brand@4, i_class@5, i_category@6, ss_sales_price@8, d_moy@9] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[i_brand@1 as i_brand, i_class@2 as i_class, i_category@3 as i_category, ss_store_sk@4 as ss_store_sk, ss_sales_price@5 as ss_sales_price, d_moy@0 as d_moy] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@2, ss_sold_date_sk@3)], projection=[d_moy@1, i_brand@3, i_class@4, i_category@5, ss_store_sk@7, ss_sales_price@8] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ss_item_sk@1)], projection=[i_brand@1, i_class@2, i_category@3, ss_sold_date_sk@4, ss_store_sk@6, ss_sales_price@7] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_store_sk, ss_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_store_name, s_company_name, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, d_moy@1 as d_moy, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_year@1 = 1999, projection=[d_date_sk@0, d_moy@2] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1999, pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1999 AND 1999 <= d_year_max@1, required_guarantees=[d_year in (1999)] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ FilterExec: (i_category@3 = Books OR i_category@3 = Electronics OR i_category@3 = Sports) AND (i_class@2 = computers OR i_class@2 = stereo OR i_class@2 = football) OR (i_category@3 = Men OR i_category@3 = Jewelry OR i_category@3 = Women) AND (i_class@2 = shirts OR i_class@2 = birdal OR i_class@2 = dresses) + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_brand, i_class, i_category], file_type=parquet, predicate=(i_category@12 = Books OR i_category@12 = Electronics OR i_category@12 = Sports) AND (i_class@10 = computers OR i_class@10 = stereo OR i_class@10 = football) OR (i_category@12 = Men OR i_category@12 = Jewelry OR i_category@12 = Women) AND (i_class@10 = shirts OR i_class@10 = birdal OR i_class@10 = dresses), pruning_predicate=(i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Electronics AND Electronics <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Sports AND Sports <= i_category_max@1) AND (i_class_null_count@6 != row_count@3 AND i_class_min@4 <= computers AND computers <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= stereo AND stereo <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= football AND football <= i_class_max@5) OR (i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Men AND Men <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Jewelry AND Jewelry <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Women AND Women <= i_category_max@1) AND (i_class_null_count@6 != row_count@3 AND i_class_min@4 <= shirts AND shirts <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= birdal AND birdal <= i_class_max@5 OR i_class_null_count@6 != row_count@3 AND i_class_min@4 <= dresses AND dresses <= i_class_max@5), required_guarantees=[] + └────────────────────────────────────────────────── + "#); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_90() -> Result<()> { + let display = test_tpcds_query("q90").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortExec: TopK(fetch=100), expr=[am_pm_ratio@0 ASC NULLS LAST], preserve_partitioning=[false] + │ ProjectionExec: expr=[CASE WHEN pmc@1 = 0 THEN None,23,8 ELSE CAST(amc@0 AS Decimal128(15, 4)) / CAST(pmc@1 AS Decimal128(15, 4)) END as am_pm_ratio] + │ CrossJoinExec + │ ProjectionExec: expr=[count(Int64(1))@0 as amc] + │ AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkCoalesceExec: output_partitions=6, input_tasks=3 + │ ProjectionExec: expr=[count(Int64(1))@0 as pmc] + │ AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))] + │ CoalescePartitionsExec + │ [Stage 8] => NetworkCoalesceExec: output_partitions=6, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] + │ AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] + │ ProjectionExec: expr=[] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(web_page.wp_web_page_sk AS Float64)@1, ws_web_page_sk@0)] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(time_dim.t_time_sk AS Float64)@1, ws_sold_time_sk@0)], projection=[ws_web_page_sk@3] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(household_demographics.hd_demo_sk AS Float64)@1, ws_ship_hdemo_sk@1)], projection=[ws_sold_time_sk@2, ws_web_page_sk@4] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_time_sk, ws_ship_hdemo_sk, ws_web_page_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[wp_web_page_sk@0 as wp_web_page_sk, CAST(wp_web_page_sk@0 AS Float64) as CAST(web_page.wp_web_page_sk AS Float64)] + │ FilterExec: wp_char_count@1 >= 5000 AND wp_char_count@1 <= 5200, projection=[wp_web_page_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_page/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_page/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_page/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_page/part-3.parquet]]}, projection=[wp_web_page_sk, wp_char_count], file_type=parquet, predicate=wp_char_count@10 >= 5000 AND wp_char_count@10 <= 5200, pruning_predicate=wp_char_count_null_count@1 != row_count@2 AND wp_char_count_max@0 >= 5000 AND wp_char_count_null_count@1 != row_count@2 AND wp_char_count_min@3 <= 5200, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[t_time_sk@0 as t_time_sk, CAST(t_time_sk@0 AS Float64) as CAST(time_dim.t_time_sk AS Float64)] + │ FilterExec: t_hour@1 >= 8 AND t_hour@1 <= 9, projection=[t_time_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet]]}, projection=[t_time_sk, t_hour], file_type=parquet, predicate=t_hour@3 >= 8 AND t_hour@3 <= 9, pruning_predicate=t_hour_null_count@1 != row_count@2 AND t_hour_max@0 >= 8 AND t_hour_null_count@1 != row_count@2 AND t_hour_min@3 <= 9, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[hd_demo_sk@0 as hd_demo_sk, CAST(hd_demo_sk@0 AS Float64) as CAST(household_demographics.hd_demo_sk AS Float64)] + │ FilterExec: hd_dep_count@1 = 6, projection=[hd_demo_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-3.parquet]]}, projection=[hd_demo_sk, hd_dep_count], file_type=parquet, predicate=hd_dep_count@3 = 6, pruning_predicate=hd_dep_count_null_count@2 != row_count@3 AND hd_dep_count_min@0 <= 6 AND 6 <= hd_dep_count_max@1, required_guarantees=[hd_dep_count in (6)] + └────────────────────────────────────────────────── + ┌───── Stage 8 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] + │ AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] + │ ProjectionExec: expr=[] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(web_page.wp_web_page_sk AS Float64)@1, ws_web_page_sk@0)] + │ CoalescePartitionsExec + │ [Stage 5] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(time_dim.t_time_sk AS Float64)@1, ws_sold_time_sk@0)], projection=[ws_web_page_sk@3] + │ CoalescePartitionsExec + │ [Stage 6] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(household_demographics.hd_demo_sk AS Float64)@1, ws_ship_hdemo_sk@1)], projection=[ws_sold_time_sk@2, ws_web_page_sk@4] + │ CoalescePartitionsExec + │ [Stage 7] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_time_sk, ws_ship_hdemo_sk, ws_web_page_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[wp_web_page_sk@0 as wp_web_page_sk, CAST(wp_web_page_sk@0 AS Float64) as CAST(web_page.wp_web_page_sk AS Float64)] + │ FilterExec: wp_char_count@1 >= 5000 AND wp_char_count@1 <= 5200, projection=[wp_web_page_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_page/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_page/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_page/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_page/part-3.parquet]]}, projection=[wp_web_page_sk, wp_char_count], file_type=parquet, predicate=wp_char_count@10 >= 5000 AND wp_char_count@10 <= 5200, pruning_predicate=wp_char_count_null_count@1 != row_count@2 AND wp_char_count_max@0 >= 5000 AND wp_char_count_null_count@1 != row_count@2 AND wp_char_count_min@3 <= 5200, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[t_time_sk@0 as t_time_sk, CAST(t_time_sk@0 AS Float64) as CAST(time_dim.t_time_sk AS Float64)] + │ FilterExec: t_hour@1 >= 19 AND t_hour@1 <= 20, projection=[t_time_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet]]}, projection=[t_time_sk, t_hour], file_type=parquet, predicate=t_hour@3 >= 19 AND t_hour@3 <= 20, pruning_predicate=t_hour_null_count@1 != row_count@2 AND t_hour_max@0 >= 19 AND t_hour_null_count@1 != row_count@2 AND t_hour_min@3 <= 20, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[hd_demo_sk@0 as hd_demo_sk, CAST(hd_demo_sk@0 AS Float64) as CAST(household_demographics.hd_demo_sk AS Float64)] + │ FilterExec: hd_dep_count@1 = 6, projection=[hd_demo_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-3.parquet]]}, projection=[hd_demo_sk, hd_dep_count], file_type=parquet, predicate=hd_dep_count@3 = 6, pruning_predicate=hd_dep_count_null_count@2 != row_count@3 AND hd_dep_count_min@0 <= 6 AND 6 <= hd_dep_count_max@1, required_guarantees=[hd_dep_count in (6)] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_91() -> Result<()> { + let display = test_tpcds_query("q91").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [returns_loss@3 DESC] + │ [Stage 8] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 8 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ SortExec: expr=[returns_loss@3 DESC], preserve_partitioning=[true] + │ ProjectionExec: expr=[cc_call_center_id@0 as call_center, cc_name@1 as call_center_name, cc_manager@2 as manager, sum(catalog_returns.cr_net_loss)@5 as returns_loss] + │ AggregateExec: mode=FinalPartitioned, gby=[cc_call_center_id@0 as cc_call_center_id, cc_name@1 as cc_name, cc_manager@2 as cc_manager, cd_marital_status@3 as cd_marital_status, cd_education_status@4 as cd_education_status], aggr=[sum(catalog_returns.cr_net_loss)] + │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([cc_call_center_id@0, cc_name@1, cc_manager@2, cd_marital_status@3, cd_education_status@4], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[cc_call_center_id@0 as cc_call_center_id, cc_name@1 as cc_name, cc_manager@2 as cc_manager, cd_marital_status@4 as cd_marital_status, cd_education_status@5 as cd_education_status], aggr=[sum(catalog_returns.cr_net_loss)] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_current_hdemo_sk@4, CAST(household_demographics.hd_demo_sk AS Float64)@1)], projection=[cc_call_center_id@0, cc_name@1, cc_manager@2, cr_net_loss@3, cd_marital_status@5, cd_education_status@6] + │ CoalescePartitionsExec + │ [Stage 6] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=3 + │ ProjectionExec: expr=[hd_demo_sk@0 as hd_demo_sk, CAST(hd_demo_sk@0 AS Float64) as CAST(household_demographics.hd_demo_sk AS Float64)] + │ FilterExec: hd_buy_potential@1 LIKE Unknown%, projection=[hd_demo_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-3.parquet]]}, projection=[hd_demo_sk, hd_buy_potential], file_type=parquet, predicate=hd_buy_potential@2 LIKE Unknown%, pruning_predicate=hd_buy_potential_null_count@2 != row_count@3 AND hd_buy_potential_min@0 <= Unknowo AND Unknown <= hd_buy_potential_max@1, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p5] t1:[p6..p11] t2:[p12..p17] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_current_cdemo_sk@4, CAST(customer_demographics.cd_demo_sk AS Float64)@3)], projection=[cc_call_center_id@0, cc_name@1, cc_manager@2, cr_net_loss@3, c_current_hdemo_sk@5, cd_marital_status@7, cd_education_status@8] + │ CoalescePartitionsExec + │ [Stage 5] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=3 + │ ProjectionExec: expr=[cd_demo_sk@0 as cd_demo_sk, cd_marital_status@1 as cd_marital_status, cd_education_status@2 as cd_education_status, CAST(cd_demo_sk@0 AS Float64) as CAST(customer_demographics.cd_demo_sk AS Float64)] + │ FilterExec: cd_marital_status@1 = M AND cd_education_status@2 = Unknown OR cd_marital_status@1 = W AND cd_education_status@2 = Advanced Degree + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/customer_demographics/part-2.parquet:..], ...]}, projection=[cd_demo_sk, cd_marital_status, cd_education_status], file_type=parquet, predicate=cd_marital_status@2 = M AND cd_education_status@3 = Unknown OR cd_marital_status@2 = W AND cd_education_status@3 = Advanced Degree, pruning_predicate=cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= M AND M <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= Unknown AND Unknown <= cd_education_status_max@5 OR cd_marital_status_null_count@2 != row_count@3 AND cd_marital_status_min@0 <= W AND W <= cd_marital_status_max@1 AND cd_education_status_null_count@6 != row_count@3 AND cd_education_status_min@4 <= Advanced Degree AND Advanced Degree <= cd_education_status_max@5, required_guarantees=[cd_education_status in (Advanced Degree, Unknown), cd_marital_status in (M, W)] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p8] t1:[p9..p17] t2:[p18..p26] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_current_addr_sk@6, ca_address_sk@0)], projection=[cc_call_center_id@0, cc_name@1, cc_manager@2, cr_net_loss@3, c_current_cdemo_sk@4, c_current_hdemo_sk@5] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=3 + │ FilterExec: ca_gmt_offset@1 = Some(-700),4,2, projection=[ca_address_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_gmt_offset], file_type=parquet, predicate=ca_gmt_offset@11 = Some(-700),4,2 AND DynamicFilter [ empty ], pruning_predicate=ca_gmt_offset_null_count@2 != row_count@3 AND ca_gmt_offset_min@0 <= Some(-700),4,2 AND Some(-700),4,2 <= ca_gmt_offset_max@1, required_guarantees=[ca_gmt_offset in (Some(-700),4,2)] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p8] t1:[p9..p17] t2:[p18..p26] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(cr_returning_customer_sk@3, CAST(customer.c_customer_sk AS Float64)@4)], projection=[cc_call_center_id@0, cc_name@1, cc_manager@2, cr_net_loss@4, c_current_cdemo_sk@6, c_current_hdemo_sk@7, c_current_addr_sk@8] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=3 + │ ProjectionExec: expr=[c_customer_sk@0 as c_customer_sk, c_current_cdemo_sk@1 as c_current_cdemo_sk, c_current_hdemo_sk@2 as c_current_hdemo_sk, c_current_addr_sk@3 as c_current_addr_sk, CAST(c_customer_sk@0 AS Float64) as CAST(customer.c_customer_sk AS Float64)] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer/part-3.parquet]]}, projection=[c_customer_sk, c_current_cdemo_sk, c_current_hdemo_sk, c_current_addr_sk], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p8] t1:[p9..p17] t2:[p18..p26] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(cr_returned_date_sk@3, d_date_sk@0)], projection=[cc_call_center_id@0, cc_name@1, cc_manager@2, cr_returning_customer_sk@4, cr_net_loss@5] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=3 + │ FilterExec: d_year@1 = 1998 AND d_moy@2 = 11, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_year, d_moy], file_type=parquet, predicate=d_year@6 = 1998 AND d_moy@8 = 11 AND DynamicFilter [ empty ], pruning_predicate=d_year_null_count@2 != row_count@3 AND d_year_min@0 <= 1998 AND 1998 <= d_year_max@1 AND d_moy_null_count@6 != row_count@3 AND d_moy_min@4 <= 11 AND 11 <= d_moy_max@5, required_guarantees=[d_moy in (11), d_year in (1998)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p5] t1:[p6..p11] t2:[p12..p17] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(call_center.cc_call_center_sk AS Float64)@4, cr_call_center_sk@2)], projection=[cc_call_center_id@1, cc_name@2, cc_manager@3, cr_returned_date_sk@5, cr_returning_customer_sk@6, cr_net_loss@8] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_returns/part-2.parquet:..], ...]}, projection=[cr_returned_date_sk, cr_returning_customer_sk, cr_call_center_sk, cr_net_loss], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/call_center/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/call_center/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/call_center/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/call_center/part-3.parquet]]}, projection=[cc_call_center_sk, cc_call_center_id, cc_name, cc_manager, CAST(cc_call_center_sk@0 AS Float64) as CAST(call_center.cc_call_center_sk AS Float64)], file_type=parquet + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_92() -> Result<()> { + let display = test_tpcds_query("q92").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ ProjectionExec: expr=[sum(web_sales.ws_ext_discount_amt)@0 as Excess Discount Amount] + │ GlobalLimitExec: skip=0, fetch=100 + │ AggregateExec: mode=Final, gby=[], aggr=[sum(web_sales.ws_ext_discount_amt)] + │ CoalescePartitionsExec + │ [Stage 6] => NetworkCoalesceExec: output_partitions=6, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] + │ AggregateExec: mode=Partial, gby=[], aggr=[sum(web_sales.ws_ext_discount_amt)] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ws_item_sk@1, i_item_sk@1)], filter=CAST(ws_ext_discount_amt@0 AS Decimal128(30, 15)) > Float64(1.3) * avg(web_sales.ws_ext_discount_amt)@1, projection=[ws_ext_discount_amt@2] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ws_sold_date_sk@0)], projection=[ws_ext_discount_amt@3, i_item_sk@4] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ ProjectionExec: expr=[ws_sold_date_sk@1 as ws_sold_date_sk, ws_ext_discount_amt@2 as ws_ext_discount_amt, i_item_sk@0 as i_item_sk] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ws_item_sk@1)], projection=[i_item_sk@0, ws_sold_date_sk@1, ws_ext_discount_amt@3] + │ CoalescePartitionsExec + │ [Stage 5] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_item_sk, ws_ext_discount_amt], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[CAST(1.3 * CAST(avg(web_sales.ws_ext_discount_amt)@1 AS Float64) AS Decimal128(30, 15)) as Float64(1.3) * avg(web_sales.ws_ext_discount_amt), ws_item_sk@0 as ws_item_sk] + │ AggregateExec: mode=FinalPartitioned, gby=[ws_item_sk@0 as ws_item_sk], aggr=[avg(web_sales.ws_ext_discount_amt)] + │ [Stage 2] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([ws_item_sk@0], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[ws_item_sk@0 as ws_item_sk], aggr=[avg(web_sales.ws_ext_discount_amt)] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ws_sold_date_sk@0)], projection=[ws_item_sk@3, ws_ext_discount_amt@4] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_sold_date_sk, ws_item_sk, ws_ext_discount_amt], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_date@1 >= 2000-01-27 AND d_date@1 <= 2000-04-26, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-01-27 AND d_date@2 <= 2000-04-26, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-01-27 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-04-26, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_date@1 >= 2000-01-27 AND d_date@1 <= 2000-04-26, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 2000-01-27 AND d_date@2 <= 2000-04-26, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 2000-01-27 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 2000-04-26, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ FilterExec: i_manufact_id@1 = 350, projection=[i_item_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_manufact_id], file_type=parquet, predicate=i_manufact_id@13 = 350 AND DynamicFilter [ empty ], pruning_predicate=i_manufact_id_null_count@2 != row_count@3 AND i_manufact_id_min@0 <= 350 AND 350 <= i_manufact_id_max@1, required_guarantees=[i_manufact_id in (350)] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_93() -> Result<()> { + let display = test_tpcds_query("q93").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [sumsales@1 ASC, ss_customer_sk@0 ASC], fetch=100 + │ [Stage 5] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ SortExec: TopK(fetch=100), expr=[sumsales@1 ASC, ss_customer_sk@0 ASC], preserve_partitioning=[true] + │ ProjectionExec: expr=[ss_customer_sk@0 as ss_customer_sk, sum(t.act_sales)@1 as sumsales] + │ AggregateExec: mode=FinalPartitioned, gby=[ss_customer_sk@0 as ss_customer_sk], aggr=[sum(t.act_sales)] + │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([ss_customer_sk@0], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[ss_customer_sk@0 as ss_customer_sk], aggr=[sum(t.act_sales)] + │ ProjectionExec: expr=[ss_customer_sk@0 as ss_customer_sk, CASE WHEN sr_return_quantity@3 IS NOT NULL THEN (ss_quantity@1 - sr_return_quantity@3) * CAST(ss_sales_price@2 AS Float64) ELSE ss_quantity@1 * CAST(ss_sales_price@2 AS Float64) END as act_sales] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(reason.r_reason_sk AS Float64)@1, sr_reason_sk@3)], projection=[ss_customer_sk@2, ss_quantity@3, ss_sales_price@4, sr_return_quantity@6] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ ProjectionExec: expr=[ss_customer_sk@2 as ss_customer_sk, ss_quantity@3 as ss_quantity, ss_sales_price@4 as ss_sales_price, sr_reason_sk@0 as sr_reason_sk, sr_return_quantity@1 as sr_return_quantity] + │ HashJoinExec: mode=Partitioned, join_type=Right, on=[(sr_item_sk@0, ss_item_sk@0), (sr_ticket_number@2, ss_ticket_number@2)], projection=[sr_reason_sk@1, sr_return_quantity@3, ss_customer_sk@5, ss_quantity@7, ss_sales_price@8] + │ [Stage 2] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[r_reason_sk@0 as r_reason_sk, CAST(r_reason_sk@0 AS Float64) as CAST(reason.r_reason_sk AS Float64)] + │ FilterExec: r_reason_desc@1 = reason 28, projection=[r_reason_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/reason/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/reason/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/reason/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/reason/part-3.parquet]]}, projection=[r_reason_sk, r_reason_desc], file_type=parquet, predicate=r_reason_desc@2 = reason 28, pruning_predicate=r_reason_desc_null_count@2 != row_count@3 AND r_reason_desc_min@0 <= reason 28 AND reason 28 <= r_reason_desc_max@1, required_guarantees=[r_reason_desc in (reason 28)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] + │ RepartitionExec: partitioning=Hash([sr_item_sk@0, sr_ticket_number@2], 9), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_returns/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_returns/part-2.parquet:..], ...]}, projection=[sr_item_sk, sr_reason_sk, sr_ticket_number, sr_return_quantity], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p8] t1:[p0..p8] t2:[p0..p8] + │ RepartitionExec: partitioning=Hash([ss_item_sk@0, ss_ticket_number@2], 9), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_item_sk, ss_customer_sk, ss_ticket_number, ss_quantity, ss_sales_price], file_type=parquet + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_94() -> Result<()> { + let display = test_tpcds_query("q94").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ ProjectionExec: expr=[count(alias1)@0 as order count, sum(alias2)@1 as total shipping cost, sum(alias3)@2 as total net profit] + │ GlobalLimitExec: skip=0, fetch=100 + │ AggregateExec: mode=Final, gby=[], aggr=[count(alias1), sum(alias2), sum(alias3)] + │ CoalescePartitionsExec + │ AggregateExec: mode=Partial, gby=[], aggr=[count(alias1), sum(alias2), sum(alias3)] + │ AggregateExec: mode=FinalPartitioned, gby=[alias1@0 as alias1], aggr=[alias2, alias3] + │ RepartitionExec: partitioning=Hash([alias1@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[ws_order_number@0 as alias1], aggr=[alias2, alias3] + │ HashJoinExec: mode=CollectLeft, join_type=LeftAnti, on=[(ws_order_number@0, wr_order_number@0)] + │ CoalescePartitionsExec + │ HashJoinExec: mode=CollectLeft, join_type=LeftSemi, on=[(ws_order_number@1, ws_order_number@1)], filter=ws_warehouse_sk@0 != ws_warehouse_sk@1, projection=[ws_order_number@1, ws_ext_ship_cost@2, ws_net_profit@3] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkCoalesceExec: output_partitions=6, input_tasks=3 + │ DataSourceExec: file_groups={3 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:.., /testdata/tpcds/plans_sf1_partitions4/web_sales/part-3.parquet:..]]}, projection=[ws_warehouse_sk, ws_order_number], file_type=parquet + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet]]}, projection=[wr_order_number], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(web_site.web_site_sk AS Float64)@1, ws_web_site_sk@0)], projection=[ws_warehouse_sk@3, ws_order_number@4, ws_ext_ship_cost@5, ws_net_profit@6] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(customer_address.ca_address_sk AS Float64)@1, ws_ship_addr_sk@0)], projection=[ws_web_site_sk@3, ws_warehouse_sk@4, ws_order_number@5, ws_ext_ship_cost@6, ws_net_profit@7] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ws_ship_date_sk@0)], projection=[ws_ship_addr_sk@3, ws_web_site_sk@4, ws_warehouse_sk@5, ws_order_number@6, ws_ext_ship_cost@7, ws_net_profit@8] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_ship_date_sk, ws_ship_addr_sk, ws_web_site_sk, ws_warehouse_sk, ws_order_number, ws_ext_ship_cost, ws_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[web_site_sk@0 as web_site_sk, CAST(web_site_sk@0 AS Float64) as CAST(web_site.web_site_sk AS Float64)] + │ FilterExec: web_company_name@1 = pri, projection=[web_site_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_site/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_site/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_site/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_site/part-3.parquet]]}, projection=[web_site_sk, web_company_name], file_type=parquet, predicate=web_company_name@14 = pri, pruning_predicate=web_company_name_null_count@2 != row_count@3 AND web_company_name_min@0 <= pri AND pri <= web_company_name_max@1, required_guarantees=[web_company_name in (pri)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[ca_address_sk@0 as ca_address_sk, CAST(ca_address_sk@0 AS Float64) as CAST(customer_address.ca_address_sk AS Float64)] + │ FilterExec: ca_state@1 = IL, projection=[ca_address_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_state], file_type=parquet, predicate=ca_state@8 = IL, pruning_predicate=ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= IL AND IL <= ca_state_max@1, required_guarantees=[ca_state in (IL)] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_date@1 >= 1999-02-01 AND d_date@1 <= 1999-04-02, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 1999-02-01 AND d_date@2 <= 1999-04-02, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 1999-02-01 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 1999-04-02, required_guarantees=[] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_95() -> Result<()> { + let display = test_tpcds_query("q95").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ ProjectionExec: expr=[count(alias1)@0 as order count, sum(alias2)@1 as total shipping cost, sum(alias3)@2 as total net profit] + │ GlobalLimitExec: skip=0, fetch=100 + │ AggregateExec: mode=Final, gby=[], aggr=[count(alias1), sum(alias2), sum(alias3)] + │ CoalescePartitionsExec + │ AggregateExec: mode=Partial, gby=[], aggr=[count(alias1), sum(alias2), sum(alias3)] + │ AggregateExec: mode=FinalPartitioned, gby=[alias1@0 as alias1], aggr=[alias2, alias3] + │ RepartitionExec: partitioning=Hash([alias1@0], 3), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[ws_order_number@0 as alias1], aggr=[alias2, alias3] + │ HashJoinExec: mode=CollectLeft, join_type=LeftSemi, on=[(ws_order_number@0, wr_order_number@0)] + │ CoalescePartitionsExec + │ HashJoinExec: mode=CollectLeft, join_type=LeftSemi, on=[(ws_order_number@0, ws_order_number@0)] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkCoalesceExec: output_partitions=6, input_tasks=3 + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(ws_order_number@1, ws_order_number@1)], filter=ws_warehouse_sk@1 != ws_warehouse_sk@0, projection=[ws_order_number@1] + │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + │ [Stage 6] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(wr_order_number@0, ws_order_number@0)], projection=[wr_order_number@0] + │ CoalescePartitionsExec + │ BroadcastExec: input_partitions=2, consumer_tasks=1, output_partitions=2 + │ DataSourceExec: file_groups={2 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_returns/part-0.parquet, /testdata/tpcds/plans_sf1_partitions4/web_returns/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_returns/part-2.parquet, /testdata/tpcds/plans_sf1_partitions4/web_returns/part-3.parquet]]}, projection=[wr_order_number], file_type=parquet + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(ws_order_number@1, ws_order_number@1)], filter=ws_warehouse_sk@1 != ws_warehouse_sk@0, projection=[ws_order_number@1] + │ [Stage 7] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + │ [Stage 8] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(web_site.web_site_sk AS Float64)@1, ws_web_site_sk@0)], projection=[ws_order_number@3, ws_ext_ship_cost@4, ws_net_profit@5] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(customer_address.ca_address_sk AS Float64)@1, ws_ship_addr_sk@0)], projection=[ws_web_site_sk@3, ws_order_number@4, ws_ext_ship_cost@5, ws_net_profit@6] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ws_ship_date_sk@0)], projection=[ws_ship_addr_sk@3, ws_web_site_sk@4, ws_order_number@5, ws_ext_ship_cost@6, ws_net_profit@7] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_ship_date_sk, ws_ship_addr_sk, ws_web_site_sk, ws_order_number, ws_ext_ship_cost, ws_net_profit], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[web_site_sk@0 as web_site_sk, CAST(web_site_sk@0 AS Float64) as CAST(web_site.web_site_sk AS Float64)] + │ FilterExec: web_company_name@1 = pri, projection=[web_site_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_site/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_site/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_site/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/web_site/part-3.parquet]]}, projection=[web_site_sk, web_company_name], file_type=parquet, predicate=web_company_name@14 = pri, pruning_predicate=web_company_name_null_count@2 != row_count@3 AND web_company_name_min@0 <= pri AND pri <= web_company_name_max@1, required_guarantees=[web_company_name in (pri)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[ca_address_sk@0 as ca_address_sk, CAST(ca_address_sk@0 AS Float64) as CAST(customer_address.ca_address_sk AS Float64)] + │ FilterExec: ca_state@1 = IL, projection=[ca_address_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/customer_address/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/customer_address/part-3.parquet]]}, projection=[ca_address_sk, ca_state], file_type=parquet, predicate=ca_state@8 = IL, pruning_predicate=ca_state_null_count@2 != row_count@3 AND ca_state_min@0 <= IL AND IL <= ca_state_max@1, required_guarantees=[ca_state in (IL)] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_date@1 >= 1999-02-01 AND d_date@1 <= 1999-04-02, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 1999-02-01 AND d_date@2 <= 1999-04-02, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 1999-02-01 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 1999-04-02, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] + │ RepartitionExec: partitioning=Hash([ws_order_number@1], 3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_warehouse_sk, ws_order_number], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] + │ RepartitionExec: partitioning=Hash([ws_order_number@1], 3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_warehouse_sk, ws_order_number], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] + │ RepartitionExec: partitioning=Hash([ws_order_number@1], 3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_warehouse_sk, ws_order_number], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 8 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] + │ RepartitionExec: partitioning=Hash([ws_order_number@1], 3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/web_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/web_sales/part-2.parquet:..], ...]}, projection=[ws_warehouse_sk, ws_order_number], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_96() -> Result<()> { + let display = test_tpcds_query("q96").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ ProjectionExec: expr=[count(Int64(1))@0 as count(*)] + │ GlobalLimitExec: skip=0, fetch=100 + │ AggregateExec: mode=Final, gby=[], aggr=[count(Int64(1))] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkCoalesceExec: output_partitions=6, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p1] t1:[p2..p3] t2:[p4..p5] + │ AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] + │ ProjectionExec: expr=[] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(store.s_store_sk AS Float64)@1, ss_store_sk@0)] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(time_dim.t_time_sk AS Float64)@1, ss_sold_time_sk@0)], projection=[ss_store_sk@3] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(household_demographics.hd_demo_sk AS Float64)@1, ss_hdemo_sk@1)], projection=[ss_sold_time_sk@2, ss_store_sk@4] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_time_sk, ss_hdemo_sk, ss_store_sk], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[s_store_sk@0 as s_store_sk, CAST(s_store_sk@0 AS Float64) as CAST(store.s_store_sk AS Float64)] + │ FilterExec: s_store_name@1 = ese, projection=[s_store_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/store/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/store/part-3.parquet]]}, projection=[s_store_sk, s_store_name], file_type=parquet, predicate=s_store_name@5 = ese, pruning_predicate=s_store_name_null_count@2 != row_count@3 AND s_store_name_min@0 <= ese AND ese <= s_store_name_max@1, required_guarantees=[s_store_name in (ese)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[t_time_sk@0 as t_time_sk, CAST(t_time_sk@0 AS Float64) as CAST(time_dim.t_time_sk AS Float64)] + │ FilterExec: t_hour@1 = 20 AND t_minute@2 >= 30, projection=[t_time_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/time_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/time_dim/part-3.parquet]]}, projection=[t_time_sk, t_hour, t_minute], file_type=parquet, predicate=t_hour@3 = 20 AND t_minute@4 >= 30, pruning_predicate=t_hour_null_count@2 != row_count@3 AND t_hour_min@0 <= 20 AND 20 <= t_hour_max@1 AND t_minute_null_count@5 != row_count@3 AND t_minute_max@4 >= 30, required_guarantees=[t_hour in (20)] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[hd_demo_sk@0 as hd_demo_sk, CAST(hd_demo_sk@0 AS Float64) as CAST(household_demographics.hd_demo_sk AS Float64)] + │ FilterExec: hd_dep_count@1 = 7, projection=[hd_demo_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/household_demographics/part-3.parquet]]}, projection=[hd_demo_sk, hd_dep_count], file_type=parquet, predicate=hd_dep_count@3 = 7, pruning_predicate=hd_dep_count_null_count@2 != row_count@3 AND hd_dep_count_min@0 <= 7 AND 7 <= hd_dep_count_max@1, required_guarantees=[hd_dep_count in (7)] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_97() -> Result<()> { + let display = test_tpcds_query("q97").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ ProjectionExec: expr=[sum(CASE WHEN ssci.customer_sk IS NOT NULL AND csci.customer_sk IS NULL THEN Int64(1) ELSE Int64(0) END)@0 as store_only, sum(CASE WHEN ssci.customer_sk IS NULL AND csci.customer_sk IS NOT NULL THEN Int64(1) ELSE Int64(0) END)@1 as catalog_only, sum(CASE WHEN ssci.customer_sk IS NOT NULL AND csci.customer_sk IS NOT NULL THEN Int64(1) ELSE Int64(0) END)@2 as store_and_catalog] + │ GlobalLimitExec: skip=0, fetch=100 + │ AggregateExec: mode=Final, gby=[], aggr=[sum(CASE WHEN ssci.customer_sk IS NOT NULL AND csci.customer_sk IS NULL THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN ssci.customer_sk IS NULL AND csci.customer_sk IS NOT NULL THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN ssci.customer_sk IS NOT NULL AND csci.customer_sk IS NOT NULL THEN Int64(1) ELSE Int64(0) END)] + │ CoalescePartitionsExec + │ AggregateExec: mode=Partial, gby=[], aggr=[sum(CASE WHEN ssci.customer_sk IS NOT NULL AND csci.customer_sk IS NULL THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN ssci.customer_sk IS NULL AND csci.customer_sk IS NOT NULL THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN ssci.customer_sk IS NOT NULL AND csci.customer_sk IS NOT NULL THEN Int64(1) ELSE Int64(0) END)] + │ ProjectionExec: expr=[customer_sk@1 IS NOT NULL as __common_expr_1, customer_sk@1 as customer_sk, customer_sk@0 as customer_sk] + │ HashJoinExec: mode=CollectLeft, join_type=Full, on=[(customer_sk@0, customer_sk@0), (item_sk@1, item_sk@1)], projection=[customer_sk@0, customer_sk@2] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + │ ProjectionExec: expr=[ss_customer_sk@0 as customer_sk, ss_item_sk@1 as item_sk] + │ AggregateExec: mode=FinalPartitioned, gby=[ss_customer_sk@0 as ss_customer_sk, ss_item_sk@1 as ss_item_sk], aggr=[] + │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ ProjectionExec: expr=[cs_bill_customer_sk@0 as customer_sk, cs_item_sk@1 as item_sk] + │ AggregateExec: mode=FinalPartitioned, gby=[cs_bill_customer_sk@0 as cs_bill_customer_sk, cs_item_sk@1 as cs_item_sk], aggr=[] + │ [Stage 2] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([cs_bill_customer_sk@0, cs_item_sk@1], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[cs_bill_customer_sk@0 as cs_bill_customer_sk, cs_item_sk@1 as cs_item_sk], aggr=[] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, cs_sold_date_sk@0)], projection=[cs_bill_customer_sk@3, cs_item_sk@4] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_bill_customer_sk, cs_item_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_month_seq@1 >= 1200 AND d_month_seq@1 <= 1211, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p2] t1:[p0..p2] t2:[p0..p2] + │ RepartitionExec: partitioning=Hash([ss_customer_sk@0, ss_item_sk@1], 3), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[ss_customer_sk@1 as ss_customer_sk, ss_item_sk@0 as ss_item_sk], aggr=[] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_item_sk@3, ss_customer_sk@4] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_customer_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_month_seq@1 >= 1200 AND d_month_seq@1 <= 1211, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] + └────────────────────────────────────────────────── + "); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_98() -> Result<()> { + let display = test_tpcds_query("q98").await?; + assert_snapshot!(display, @r#" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [i_category@2 ASC, i_class@3 ASC, i_item_id@0 ASC, i_item_desc@1 ASC, revenueratio@6 ASC] + │ SortExec: expr=[i_category@2 ASC, i_class@3 ASC, i_item_id@0 ASC, i_item_desc@1 ASC, revenueratio@6 ASC], preserve_partitioning=[true] + │ ProjectionExec: expr=[i_item_id@0 as i_item_id, i_item_desc@1 as i_item_desc, i_category@2 as i_category, i_class@3 as i_class, i_current_price@4 as i_current_price, sum(store_sales.ss_ext_sales_price)@5 as itemrevenue, CAST(sum(store_sales.ss_ext_sales_price)@5 AS Float64) * 100 / CAST(sum(sum(store_sales.ss_ext_sales_price)) PARTITION BY [item.i_class] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING@6 AS Float64) as revenueratio] + │ WindowAggExec: wdw=[sum(sum(store_sales.ss_ext_sales_price)) PARTITION BY [item.i_class] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING: Ok(Field { name: "sum(sum(store_sales.ss_ext_sales_price)) PARTITION BY [item.i_class] ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING", data_type: Decimal128(27, 2), nullable: true }), frame: WindowFrame { units: Rows, start_bound: Preceding(UInt64(NULL)), end_bound: Following(UInt64(NULL)), is_causal: false }] + │ SortExec: expr=[i_class@3 ASC NULLS LAST], preserve_partitioning=[true] + │ [Stage 4] => NetworkShuffleExec: output_partitions=3, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ RepartitionExec: partitioning=Hash([i_class@3], 3), input_partitions=3 + │ AggregateExec: mode=FinalPartitioned, gby=[i_item_id@0 as i_item_id, i_item_desc@1 as i_item_desc, i_category@2 as i_category, i_class@3 as i_class, i_current_price@4 as i_current_price], aggr=[sum(store_sales.ss_ext_sales_price)] + │ [Stage 3] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([i_item_id@0, i_item_desc@1, i_category@2, i_class@3, i_current_price@4], 6), input_partitions=2 + │ AggregateExec: mode=Partial, gby=[i_item_id@1 as i_item_id, i_item_desc@2 as i_item_desc, i_category@5 as i_category, i_class@4 as i_class, i_current_price@3 as i_current_price], aggr=[sum(store_sales.ss_ext_sales_price)] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(date_dim.d_date_sk AS Float64)@1, ss_sold_date_sk@0)], projection=[ss_ext_sales_price@3, i_item_id@4, i_item_desc@5, i_current_price@6, i_class@7, i_category@8] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ ProjectionExec: expr=[ss_sold_date_sk@5 as ss_sold_date_sk, ss_ext_sales_price@6 as ss_ext_sales_price, i_item_id@0 as i_item_id, i_item_desc@1 as i_item_desc, i_current_price@2 as i_current_price, i_class@3 as i_class, i_category@4 as i_category] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(i_item_sk@0, ss_item_sk@1)], projection=[i_item_id@1, i_item_desc@2, i_current_price@3, i_class@4, i_category@5, ss_sold_date_sk@6, ss_ext_sales_price@8] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=3, stage_partitions=9, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/store_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/store_sales/part-2.parquet:..], ...]}, projection=[ss_sold_date_sk, ss_item_sk, ss_ext_sales_price], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_date@1 >= 1999-02-22 AND d_date@1 <= 1999-03-24, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_date], file_type=parquet, predicate=d_date@2 >= 1999-02-22 AND d_date@2 <= 1999-03-24, pruning_predicate=d_date_null_count@1 != row_count@2 AND d_date_max@0 >= 1999-02-22 AND d_date_null_count@1 != row_count@2 AND d_date_min@3 <= 1999-03-24, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p8] t1:[p9..p17] + │ BroadcastExec: input_partitions=3, consumer_tasks=3, output_partitions=9 + │ FilterExec: i_category@5 = Sports OR i_category@5 = Books OR i_category@5 = Home + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/item/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/item/part-3.parquet]]}, projection=[i_item_sk, i_item_id, i_item_desc, i_current_price, i_class, i_category], file_type=parquet, predicate=i_category@12 = Sports OR i_category@12 = Books OR i_category@12 = Home, pruning_predicate=i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Sports AND Sports <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Books AND Books <= i_category_max@1 OR i_category_null_count@2 != row_count@3 AND i_category_min@0 <= Home AND Home <= i_category_max@1, required_guarantees=[i_category in (Books, Home, Sports)] + └────────────────────────────────────────────────── + "#); + Ok(()) + } + #[tokio::test] + async fn test_tpcds_99() -> Result<()> { + let display = test_tpcds_query("q99").await?; + assert_snapshot!(display, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [w_substr@0 ASC, sm_type@1 ASC, cc_name_lower@2 ASC], fetch=100 + │ [Stage 6] => NetworkCoalesceExec: output_partitions=6, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p2] t1:[p0..p2] + │ SortExec: TopK(fetch=100), expr=[w_substr@0 ASC, sm_type@1 ASC, cc_name_lower@2 ASC], preserve_partitioning=[true] + │ ProjectionExec: expr=[w_substr@0 as w_substr, sm_type@1 as sm_type, lower(cc_name@2) as cc_name_lower, sum(CASE WHEN catalog_sales.cs_ship_date_sk - catalog_sales.cs_sold_date_sk <= Int64(30) THEN Int64(1) ELSE Int64(0) END)@3 as 30 days, sum(CASE WHEN catalog_sales.cs_ship_date_sk - catalog_sales.cs_sold_date_sk > Int64(30) AND catalog_sales.cs_ship_date_sk - catalog_sales.cs_sold_date_sk <= Int64(60) THEN Int64(1) ELSE Int64(0) END)@4 as 31-60 days, sum(CASE WHEN catalog_sales.cs_ship_date_sk - catalog_sales.cs_sold_date_sk > Int64(60) AND catalog_sales.cs_ship_date_sk - catalog_sales.cs_sold_date_sk <= Int64(90) THEN Int64(1) ELSE Int64(0) END)@5 as 61-90 days, sum(CASE WHEN catalog_sales.cs_ship_date_sk - catalog_sales.cs_sold_date_sk > Int64(90) AND catalog_sales.cs_ship_date_sk - catalog_sales.cs_sold_date_sk <= Int64(120) THEN Int64(1) ELSE Int64(0) END)@6 as 91-120 days, sum(CASE WHEN catalog_sales.cs_ship_date_sk - catalog_sales.cs_sold_date_sk > Int64(120) THEN Int64(1) ELSE Int64(0) END)@7 as >120 days] + │ AggregateExec: mode=FinalPartitioned, gby=[w_substr@0 as w_substr, sm_type@1 as sm_type, cc_name@2 as cc_name], aggr=[sum(CASE WHEN catalog_sales.cs_ship_date_sk - catalog_sales.cs_sold_date_sk <= Int64(30) THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN catalog_sales.cs_ship_date_sk - catalog_sales.cs_sold_date_sk > Int64(30) AND catalog_sales.cs_ship_date_sk - catalog_sales.cs_sold_date_sk <= Int64(60) THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN catalog_sales.cs_ship_date_sk - catalog_sales.cs_sold_date_sk > Int64(60) AND catalog_sales.cs_ship_date_sk - catalog_sales.cs_sold_date_sk <= Int64(90) THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN catalog_sales.cs_ship_date_sk - catalog_sales.cs_sold_date_sk > Int64(90) AND catalog_sales.cs_ship_date_sk - catalog_sales.cs_sold_date_sk <= Int64(120) THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN catalog_sales.cs_ship_date_sk - catalog_sales.cs_sold_date_sk > Int64(120) THEN Int64(1) ELSE Int64(0) END)] + │ [Stage 5] => NetworkShuffleExec: output_partitions=3, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ RepartitionExec: partitioning=Hash([w_substr@0, sm_type@1, cc_name@2], 6), input_partitions=3 + │ AggregateExec: mode=Partial, gby=[w_substr@1 as w_substr, sm_type@2 as sm_type, cc_name@3 as cc_name], aggr=[sum(CASE WHEN catalog_sales.cs_ship_date_sk - catalog_sales.cs_sold_date_sk <= Int64(30) THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN catalog_sales.cs_ship_date_sk - catalog_sales.cs_sold_date_sk > Int64(30) AND catalog_sales.cs_ship_date_sk - catalog_sales.cs_sold_date_sk <= Int64(60) THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN catalog_sales.cs_ship_date_sk - catalog_sales.cs_sold_date_sk > Int64(60) AND catalog_sales.cs_ship_date_sk - catalog_sales.cs_sold_date_sk <= Int64(90) THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN catalog_sales.cs_ship_date_sk - catalog_sales.cs_sold_date_sk > Int64(90) AND catalog_sales.cs_ship_date_sk - catalog_sales.cs_sold_date_sk <= Int64(120) THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN catalog_sales.cs_ship_date_sk - catalog_sales.cs_sold_date_sk > Int64(120) THEN Int64(1) ELSE Int64(0) END)] + │ ProjectionExec: expr=[cs_ship_date_sk@1 - cs_sold_date_sk@0 as __common_expr_1, w_substr@2 as w_substr, sm_type@3 as sm_type, cc_name@4 as cc_name] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(cs_ship_date_sk@1, CAST(date_dim.d_date_sk AS Float64)@1)], projection=[cs_sold_date_sk@0, cs_ship_date_sk@1, w_substr@2, sm_type@3, cc_name@4] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=3 + │ ProjectionExec: expr=[d_date_sk@0 as d_date_sk, CAST(d_date_sk@0 AS Float64) as CAST(date_dim.d_date_sk AS Float64)] + │ FilterExec: d_month_seq@1 >= 1200 AND d_month_seq@1 <= 1211, projection=[d_date_sk@0] + │ RepartitionExec: partitioning=RoundRobinBatch(3), input_partitions=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/date_dim/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/date_dim/part-3.parquet]]}, projection=[d_date_sk, d_month_seq], file_type=parquet, predicate=d_month_seq@3 >= 1200 AND d_month_seq@3 <= 1211, pruning_predicate=d_month_seq_null_count@1 != row_count@2 AND d_month_seq_max@0 >= 1200 AND d_month_seq_null_count@1 != row_count@2 AND d_month_seq_min@3 <= 1211, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p5] t1:[p6..p11] t2:[p12..p17] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(cs_call_center_sk@2, CAST(call_center.cc_call_center_sk AS Float64)@2)], projection=[cs_sold_date_sk@0, cs_ship_date_sk@1, w_substr@3, sm_type@4, cc_name@6] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=3 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/call_center/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/call_center/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/call_center/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/call_center/part-3.parquet]]}, projection=[cc_call_center_sk, cc_name, CAST(cc_call_center_sk@0 AS Float64) as CAST(call_center.cc_call_center_sk AS Float64)], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p5] t1:[p6..p11] t2:[p12..p17] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(cs_ship_mode_sk@3, CAST(ship_mode.sm_ship_mode_sk AS Float64)@2)], projection=[cs_sold_date_sk@0, cs_ship_date_sk@1, cs_call_center_sk@2, w_substr@4, sm_type@6] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=3 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,__] t2:[__,__,__,p0] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/ship_mode/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/ship_mode/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/ship_mode/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/ship_mode/part-3.parquet]]}, projection=[sm_ship_mode_sk, sm_type, CAST(sm_ship_mode_sk@0 AS Float64) as CAST(ship_mode.sm_ship_mode_sk AS Float64)], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p5] t1:[p6..p11] t2:[p12..p17] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ ProjectionExec: expr=[cs_sold_date_sk@1 as cs_sold_date_sk, cs_ship_date_sk@2 as cs_ship_date_sk, cs_call_center_sk@3 as cs_call_center_sk, cs_ship_mode_sk@4 as cs_ship_mode_sk, w_substr@0 as w_substr] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(CAST(sq1.w_warehouse_sk AS Float64)@2, cs_warehouse_sk@4)], projection=[w_substr@0, cs_sold_date_sk@3, cs_ship_date_sk@4, cs_call_center_sk@5, cs_ship_mode_sk@6] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=2, stage_partitions=6, input_tasks=2 + │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-0.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-1.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], [/testdata/tpcds/plans_sf1_partitions4/catalog_sales/part-2.parquet:..], ...]}, projection=[cs_sold_date_sk, cs_ship_date_sk, cs_call_center_sk, cs_ship_mode_sk, cs_warehouse_sk], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p5] t1:[p6..p11] + │ BroadcastExec: input_partitions=2, consumer_tasks=3, output_partitions=6 + │ PartitionIsolatorExec: t0:[p0,p1,__,__] t1:[__,__,p0,p1] + │ DataSourceExec: file_groups={4 groups: [[/testdata/tpcds/plans_sf1_partitions4/warehouse/part-0.parquet], [/testdata/tpcds/plans_sf1_partitions4/warehouse/part-1.parquet], [/testdata/tpcds/plans_sf1_partitions4/warehouse/part-2.parquet], [/testdata/tpcds/plans_sf1_partitions4/warehouse/part-3.parquet]]}, projection=[substr(w_warehouse_name@2, 1, 20) as w_substr, w_warehouse_sk, CAST(w_warehouse_sk@0 AS Float64) as CAST(sq1.w_warehouse_sk AS Float64)], file_type=parquet + └────────────────────────────────────────────────── + "); + Ok(()) + } + + static INIT_TEST_TPCDS_TABLES: OnceCell<()> = OnceCell::const_new(); + + async fn test_tpcds_query(query_id: &str) -> Result { + let data_dir = Path::new(env!("CARGO_MANIFEST_DIR")).join(format!( + "testdata/tpcds/plans_sf{SF}_partitions{PARQUET_PARTITIONS}" + )); + INIT_TEST_TPCDS_TABLES + .get_or_init(|| async { + if !fs::exists(&data_dir).unwrap_or(false) { + tpcds::generate_data(&data_dir, SF, PARQUET_PARTITIONS) + .await + .unwrap(); + } + }) + .await; + + let query_sql = tpcds::get_query(query_id)?; + // Make distributed localhost context to run queries + let (d_ctx, _guard, _) = start_localhost_context(NUM_WORKERS, DefaultSessionBuilder).await; + let d_ctx = d_ctx + .with_distributed_files_per_task(FILES_PER_TASK)? + .with_distributed_cardinality_effect_task_scale_factor(CARDINALITY_TASK_COUNT_FACTOR)? + .with_distributed_broadcast_joins(true)?; + + benchmarks_common::register_tables(&d_ctx, &data_dir).await?; + + let df = d_ctx.sql(&query_sql).await?; + let plan = df.create_physical_plan().await?; + if plan.as_any().is::() { + Ok(display_plan_ascii(plan.as_ref(), false)) + } else { + Ok("".to_string()) + } + } +} diff --git a/tests/tpch_correctness_test.rs b/tests/tpch_correctness_test.rs new file mode 100644 index 00000000..114dc2e1 --- /dev/null +++ b/tests/tpch_correctness_test.rs @@ -0,0 +1,206 @@ +#[cfg(all(feature = "integration", feature = "tpch", test))] +mod tests { + use datafusion::physical_plan::execute_stream; + use datafusion::prelude::SessionContext; + use datafusion_distributed::test_utils::localhost::start_localhost_context; + use datafusion_distributed::test_utils::{benchmarks_common, tpch}; + use datafusion_distributed::{DefaultSessionBuilder, DistributedExt}; + use futures::TryStreamExt; + use std::error::Error; + use std::fmt::Display; + use std::fs; + use std::path::Path; + use tokio::sync::OnceCell; + + const PARTITIONS: usize = 6; + const TPCH_SCALE_FACTOR: f64 = 1.0; + const TPCH_DATA_PARTS: i32 = 16; + + #[tokio::test] + async fn test_tpch_1() -> Result<(), Box> { + test_tpch_query(tpch::get_query("q1")?).await + } + + #[tokio::test] + async fn test_tpch_2() -> Result<(), Box> { + test_tpch_query(tpch::get_query("q2")?).await + } + + #[tokio::test] + async fn test_tpch_3() -> Result<(), Box> { + test_tpch_query(tpch::get_query("q3")?).await + } + + #[tokio::test] + async fn test_tpch_4() -> Result<(), Box> { + test_tpch_query(tpch::get_query("q4")?).await + } + + #[tokio::test] + async fn test_tpch_5() -> Result<(), Box> { + test_tpch_query(tpch::get_query("q5")?).await + } + + #[tokio::test] + async fn test_tpch_6() -> Result<(), Box> { + test_tpch_query(tpch::get_query("q6")?).await + } + + #[tokio::test] + async fn test_tpch_7() -> Result<(), Box> { + test_tpch_query(tpch::get_query("q7")?).await + } + + #[tokio::test] + async fn test_tpch_8() -> Result<(), Box> { + test_tpch_query(tpch::get_query("q8")?).await + } + + #[tokio::test] + async fn test_tpch_9() -> Result<(), Box> { + test_tpch_query(tpch::get_query("q9")?).await + } + + #[tokio::test] + async fn test_tpch_10() -> Result<(), Box> { + let sql = tpch::get_query("q10")?; + // There is a chance that this query returns non-deterministic results if two entries + // happen to have the exact same revenue. With small scales, this never happens, but with + // bigger scales, this is very likely to happen. + // This extra ordering accounts for it. + let sql = sql.replace("revenue desc", "revenue, c_acctbal desc"); + test_tpch_query(sql).await + } + + #[tokio::test] + async fn test_tpch_11() -> Result<(), Box> { + test_tpch_query(tpch::get_query("q11")?).await + } + + #[tokio::test] + async fn test_tpch_12() -> Result<(), Box> { + test_tpch_query(tpch::get_query("q12")?).await + } + + #[tokio::test] + async fn test_tpch_13() -> Result<(), Box> { + test_tpch_query(tpch::get_query("q13")?).await + } + + #[tokio::test] + async fn test_tpch_14() -> Result<(), Box> { + test_tpch_query(tpch::get_query("q14")?).await + } + + #[tokio::test] + async fn test_tpch_15() -> Result<(), Box> { + test_tpch_query(tpch::get_query("q15")?).await + } + + #[tokio::test] + async fn test_tpch_16() -> Result<(), Box> { + test_tpch_query(tpch::get_query("q16")?).await + } + + #[tokio::test] + async fn test_tpch_17() -> Result<(), Box> { + test_tpch_query(tpch::get_query("q17")?).await + } + + #[tokio::test] + async fn test_tpch_18() -> Result<(), Box> { + test_tpch_query(tpch::get_query("q18")?).await + } + + #[tokio::test] + async fn test_tpch_19() -> Result<(), Box> { + test_tpch_query(tpch::get_query("q19")?).await + } + + #[tokio::test] + async fn test_tpch_20() -> Result<(), Box> { + test_tpch_query(tpch::get_query("q20")?).await + } + + #[tokio::test] + async fn test_tpch_21() -> Result<(), Box> { + test_tpch_query(tpch::get_query("q21")?).await + } + + #[tokio::test] + async fn test_tpch_22() -> Result<(), Box> { + test_tpch_query(tpch::get_query("q22")?).await + } + + // test_tpch_query runs each TPC-H query twice - once in a distributed manner and once + // in a non-distributed manner. For each query, it asserts that the results are identical. + async fn test_tpch_query(sql: String) -> Result<(), Box> { + let (d_ctx, _guard, _) = start_localhost_context(4, DefaultSessionBuilder).await; + let d_ctx = d_ctx.with_distributed_broadcast_joins(true)?; + + let results_d = run_tpch_query(d_ctx, sql.clone()).await?; + let results_s = run_tpch_query(SessionContext::new(), sql).await?; + + pretty_assertions::assert_eq!(results_d.to_string(), results_s.to_string(),); + Ok(()) + } + + async fn run_tpch_query( + ctx: SessionContext, + sql: String, + ) -> Result> { + let data_dir = ensure_tpch_data(TPCH_SCALE_FACTOR, TPCH_DATA_PARTS).await; + ctx.state_ref() + .write() + .config_mut() + .options_mut() + .execution + .target_partitions = PARTITIONS; + + benchmarks_common::register_tables(&ctx, &data_dir).await?; + + // Query 15 has three queries in it, one creating the view, the second + // executing, which we want to capture the output of, and the third + // tearing down the view + let stream = if sql.starts_with("create view") { + let queries: Vec<&str> = sql + .split(';') + .map(str::trim) + .filter(|s| !s.is_empty()) + .collect(); + + ctx.sql(queries[0]).await?.collect().await?; + let df = ctx.sql(queries[1]).await?; + let plan = df.create_physical_plan().await?; + let stream = execute_stream(plan.clone(), ctx.task_ctx())?; + ctx.sql(queries[2]).await?.collect().await?; + + stream + } else { + let df = ctx.sql(&sql).await?; + let plan = df.create_physical_plan().await?; + execute_stream(plan.clone(), ctx.task_ctx())? + }; + + let batches = stream.try_collect::>().await?; + + Ok(arrow::util::pretty::pretty_format_batches(&batches)?) + } + + // OnceCell to ensure TPCH tables are generated only once for tests + static INIT_TEST_TPCH_TABLES: OnceCell<()> = OnceCell::const_new(); + + // ensure_tpch_data initializes the TPCH data on disk. + pub async fn ensure_tpch_data(sf: f64, parts: i32) -> std::path::PathBuf { + let data_dir = + Path::new(env!("CARGO_MANIFEST_DIR")).join(format!("testdata/tpch/correctness_sf{sf}")); + INIT_TEST_TPCH_TABLES + .get_or_init(|| async { + if !fs::exists(&data_dir).unwrap() { + tpch::generate_tpch_data(&data_dir, sf, parts); + } + }) + .await; + data_dir + } +} diff --git a/tests/tpch_plans_test.rs b/tests/tpch_plans_test.rs new file mode 100644 index 00000000..2d03fbfb --- /dev/null +++ b/tests/tpch_plans_test.rs @@ -0,0 +1,1174 @@ +#[cfg(all(feature = "integration", feature = "tpch", test))] +mod tests { + use datafusion_distributed::test_utils::localhost::start_localhost_context; + use datafusion_distributed::test_utils::{benchmarks_common, tpch}; + use datafusion_distributed::{ + DefaultSessionBuilder, DistributedExt, assert_snapshot, display_plan_ascii, + }; + use std::error::Error; + use std::fs; + use std::path::Path; + use tokio::sync::OnceCell; + + const PARTITIONS: usize = 6; + const TPCH_SCALE_FACTOR: f64 = 0.02; + const TPCH_DATA_PARTS: i32 = 16; + + #[tokio::test] + async fn test_tpch_1() -> Result<(), Box> { + let plan = test_tpch_query("q1").await?; + assert_snapshot!(plan, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [l_returnflag@0 ASC NULLS LAST, l_linestatus@1 ASC NULLS LAST] + │ [Stage 2] => NetworkCoalesceExec: output_partitions=12, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ SortExec: expr=[l_returnflag@0 ASC NULLS LAST, l_linestatus@1 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[l_returnflag@0 as l_returnflag, l_linestatus@1 as l_linestatus, sum(lineitem.l_quantity)@2 as sum_qty, sum(lineitem.l_extendedprice)@3 as sum_base_price, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@4 as sum_disc_price, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount * Int64(1) + lineitem.l_tax)@5 as sum_charge, avg(lineitem.l_quantity)@6 as avg_qty, avg(lineitem.l_extendedprice)@7 as avg_price, avg(lineitem.l_discount)@8 as avg_disc, count(Int64(1))@9 as count_order] + │ AggregateExec: mode=FinalPartitioned, gby=[l_returnflag@0 as l_returnflag, l_linestatus@1 as l_linestatus], aggr=[sum(lineitem.l_quantity), sum(lineitem.l_extendedprice), sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount), sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount * Int64(1) + lineitem.l_tax), avg(lineitem.l_quantity), avg(lineitem.l_extendedprice), avg(lineitem.l_discount), count(Int64(1))] + │ [Stage 1] => NetworkShuffleExec: output_partitions=6, input_tasks=4 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p11] t1:[p0..p11] t2:[p0..p11] t3:[p0..p11] + │ RepartitionExec: partitioning=Hash([l_returnflag@0, l_linestatus@1], 12), input_partitions=4 + │ AggregateExec: mode=Partial, gby=[l_returnflag@5 as l_returnflag, l_linestatus@6 as l_linestatus], aggr=[sum(lineitem.l_quantity), sum(lineitem.l_extendedprice), sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount), sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount * Int64(1) + lineitem.l_tax), avg(lineitem.l_quantity), avg(lineitem.l_extendedprice), avg(lineitem.l_discount), count(Int64(1))] + │ ProjectionExec: expr=[l_extendedprice@1 * (Some(1),20,0 - l_discount@2) as __common_expr_1, l_quantity@0 as l_quantity, l_extendedprice@1 as l_extendedprice, l_discount@2 as l_discount, l_tax@3 as l_tax, l_returnflag@4 as l_returnflag, l_linestatus@5 as l_linestatus] + │ FilterExec: l_shipdate@6 <= 1998-09-02, projection=[l_quantity@0, l_extendedprice@1, l_discount@2, l_tax@3, l_returnflag@4, l_linestatus@5] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet], [/testdata/tpch/plan_sf0.02/lineitem/10.parquet], [/testdata/tpch/plan_sf0.02/lineitem/11.parquet], [/testdata/tpch/plan_sf0.02/lineitem/12.parquet], [/testdata/tpch/plan_sf0.02/lineitem/13.parquet], ...]}, projection=[l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus, l_shipdate], file_type=parquet, predicate=l_shipdate@10 <= 1998-09-02, pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@0 <= 1998-09-02, required_guarantees=[] + └────────────────────────────────────────────────── + "); + Ok(()) + } + + #[tokio::test] + async fn test_tpch_2() -> Result<(), Box> { + let plan = test_tpch_query("q2").await?; + assert_snapshot!(plan, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [s_acctbal@0 DESC, n_name@2 ASC NULLS LAST, s_name@1 ASC NULLS LAST, p_partkey@3 ASC NULLS LAST] + │ [Stage 11] => NetworkCoalesceExec: output_partitions=24, input_tasks=4 + └────────────────────────────────────────────────── + ┌───── Stage 11 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] t3:[p0..p5] + │ SortExec: expr=[s_acctbal@0 DESC, n_name@2 ASC NULLS LAST, s_name@1 ASC NULLS LAST, p_partkey@3 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[s_acctbal@5 as s_acctbal, s_name@2 as s_name, n_name@7 as n_name, p_partkey@0 as p_partkey, p_mfgr@1 as p_mfgr, s_address@3 as s_address, s_phone@4 as s_phone, s_comment@6 as s_comment] + │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(p_partkey@0, ps_partkey@1), (ps_supplycost@7, min(partsupp.ps_supplycost)@0)], projection=[p_partkey@0, p_mfgr@1, s_name@2, s_address@3, s_phone@4, s_acctbal@5, s_comment@6, n_name@8] + │ [Stage 5] => NetworkShuffleExec: output_partitions=6, input_tasks=4 + │ [Stage 10] => NetworkShuffleExec: output_partitions=6, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p23] t1:[p0..p23] t2:[p0..p23] t3:[p0..p23] + │ RepartitionExec: partitioning=Hash([p_partkey@0, ps_supplycost@7], 24), input_partitions=4 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(r_regionkey@0, n_regionkey@9)], projection=[p_partkey@1, p_mfgr@2, s_name@3, s_address@4, s_phone@5, s_acctbal@6, s_comment@7, ps_supplycost@8, n_name@9] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=4, stage_partitions=16, input_tasks=4 + │ ProjectionExec: expr=[p_partkey@2 as p_partkey, p_mfgr@3 as p_mfgr, s_name@4 as s_name, s_address@5 as s_address, s_phone@6 as s_phone, s_acctbal@7 as s_acctbal, s_comment@8 as s_comment, ps_supplycost@9 as ps_supplycost, n_name@0 as n_name, n_regionkey@1 as n_regionkey] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@4)], projection=[n_name@1, n_regionkey@2, p_partkey@3, p_mfgr@4, s_name@5, s_address@6, s_phone@8, s_acctbal@9, s_comment@10, ps_supplycost@11] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=4, stage_partitions=16, input_tasks=4 + │ ProjectionExec: expr=[p_partkey@6 as p_partkey, p_mfgr@7 as p_mfgr, s_name@0 as s_name, s_address@1 as s_address, s_nationkey@2 as s_nationkey, s_phone@3 as s_phone, s_acctbal@4 as s_acctbal, s_comment@5 as s_comment, ps_supplycost@8 as ps_supplycost] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_suppkey@0, ps_suppkey@2)], projection=[s_name@1, s_address@2, s_nationkey@3, s_phone@4, s_acctbal@5, s_comment@6, p_partkey@7, p_mfgr@8, ps_supplycost@10] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=4, stage_partitions=16, input_tasks=4 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(p_partkey@0, ps_partkey@0)], projection=[p_partkey@0, p_mfgr@1, ps_suppkey@3, ps_supplycost@4] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=4, stage_partitions=16, input_tasks=4 + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/partsupp/1.parquet], [/testdata/tpch/plan_sf0.02/partsupp/10.parquet], [/testdata/tpch/plan_sf0.02/partsupp/11.parquet], [/testdata/tpch/plan_sf0.02/partsupp/12.parquet], [/testdata/tpch/plan_sf0.02/partsupp/13.parquet], ...]}, projection=[ps_partkey, ps_suppkey, ps_supplycost], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p15] t1:[p16..p31] t2:[p32..p47] t3:[p48..p63] + │ BroadcastExec: input_partitions=4, consumer_tasks=4, output_partitions=16 + │ FilterExec: r_name@1 = EUROPE, projection=[r_regionkey@0] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/region/1.parquet], [/testdata/tpch/plan_sf0.02/region/10.parquet], [/testdata/tpch/plan_sf0.02/region/11.parquet], [/testdata/tpch/plan_sf0.02/region/12.parquet], [/testdata/tpch/plan_sf0.02/region/13.parquet], ...]}, projection=[r_regionkey, r_name], file_type=parquet, predicate=r_name@1 = EUROPE, pruning_predicate=r_name_null_count@2 != row_count@3 AND r_name_min@0 <= EUROPE AND EUROPE <= r_name_max@1, required_guarantees=[r_name in (EUROPE)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p15] t1:[p16..p31] t2:[p32..p47] t3:[p48..p63] + │ BroadcastExec: input_partitions=4, consumer_tasks=4, output_partitions=16 + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/nation/1.parquet], [/testdata/tpch/plan_sf0.02/nation/10.parquet], [/testdata/tpch/plan_sf0.02/nation/11.parquet], [/testdata/tpch/plan_sf0.02/nation/12.parquet], [/testdata/tpch/plan_sf0.02/nation/13.parquet], ...]}, projection=[n_nationkey, n_name, n_regionkey], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p15] t1:[p16..p31] t2:[p32..p47] t3:[p48..p63] + │ BroadcastExec: input_partitions=4, consumer_tasks=4, output_partitions=16 + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/supplier/1.parquet], [/testdata/tpch/plan_sf0.02/supplier/10.parquet], [/testdata/tpch/plan_sf0.02/supplier/11.parquet], [/testdata/tpch/plan_sf0.02/supplier/12.parquet], [/testdata/tpch/plan_sf0.02/supplier/13.parquet], ...]}, projection=[s_suppkey, s_name, s_address, s_nationkey, s_phone, s_acctbal, s_comment], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p15] t1:[p16..p31] t2:[p32..p47] t3:[p48..p63] + │ BroadcastExec: input_partitions=4, consumer_tasks=4, output_partitions=16 + │ FilterExec: p_size@3 = 15 AND p_type@2 LIKE %BRASS, projection=[p_partkey@0, p_mfgr@1] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/part/1.parquet], [/testdata/tpch/plan_sf0.02/part/10.parquet], [/testdata/tpch/plan_sf0.02/part/11.parquet], [/testdata/tpch/plan_sf0.02/part/12.parquet], [/testdata/tpch/plan_sf0.02/part/13.parquet], ...]}, projection=[p_partkey, p_mfgr, p_type, p_size], file_type=parquet, predicate=p_size@5 = 15 AND p_type@4 LIKE %BRASS, pruning_predicate=p_size_null_count@2 != row_count@3 AND p_size_min@0 <= 15 AND 15 <= p_size_max@1, required_guarantees=[p_size in (15)] + └────────────────────────────────────────────────── + ┌───── Stage 10 ── Tasks: t0:[p0..p23] t1:[p0..p23] t2:[p0..p23] + │ RepartitionExec: partitioning=Hash([ps_partkey@1, min(partsupp.ps_supplycost)@0], 24), input_partitions=6 + │ ProjectionExec: expr=[min(partsupp.ps_supplycost)@1 as min(partsupp.ps_supplycost), ps_partkey@0 as ps_partkey] + │ AggregateExec: mode=FinalPartitioned, gby=[ps_partkey@0 as ps_partkey], aggr=[min(partsupp.ps_supplycost)] + │ [Stage 9] => NetworkShuffleExec: output_partitions=6, input_tasks=4 + └────────────────────────────────────────────────── + ┌───── Stage 9 ── Tasks: t0:[p0..p17] t1:[p0..p17] t2:[p0..p17] t3:[p0..p17] + │ RepartitionExec: partitioning=Hash([ps_partkey@0], 18), input_partitions=4 + │ AggregateExec: mode=Partial, gby=[ps_partkey@0 as ps_partkey], aggr=[min(partsupp.ps_supplycost)] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(r_regionkey@0, n_regionkey@2)], projection=[ps_partkey@1, ps_supplycost@2] + │ CoalescePartitionsExec + │ [Stage 6] => NetworkBroadcastExec: partitions_per_consumer=4, stage_partitions=16, input_tasks=4 + │ ProjectionExec: expr=[ps_partkey@1 as ps_partkey, ps_supplycost@2 as ps_supplycost, n_regionkey@0 as n_regionkey] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@2)], projection=[n_regionkey@1, ps_partkey@2, ps_supplycost@3] + │ CoalescePartitionsExec + │ [Stage 7] => NetworkBroadcastExec: partitions_per_consumer=4, stage_partitions=16, input_tasks=4 + │ ProjectionExec: expr=[ps_partkey@1 as ps_partkey, ps_supplycost@2 as ps_supplycost, s_nationkey@0 as s_nationkey] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_suppkey@0, ps_suppkey@1)], projection=[s_nationkey@1, ps_partkey@2, ps_supplycost@4] + │ CoalescePartitionsExec + │ [Stage 8] => NetworkBroadcastExec: partitions_per_consumer=4, stage_partitions=16, input_tasks=4 + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/partsupp/1.parquet], [/testdata/tpch/plan_sf0.02/partsupp/10.parquet], [/testdata/tpch/plan_sf0.02/partsupp/11.parquet], [/testdata/tpch/plan_sf0.02/partsupp/12.parquet], [/testdata/tpch/plan_sf0.02/partsupp/13.parquet], ...]}, projection=[ps_partkey, ps_suppkey, ps_supplycost], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p15] t1:[p16..p31] t2:[p32..p47] t3:[p48..p63] + │ BroadcastExec: input_partitions=4, consumer_tasks=4, output_partitions=16 + │ FilterExec: r_name@1 = EUROPE, projection=[r_regionkey@0] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/region/1.parquet], [/testdata/tpch/plan_sf0.02/region/10.parquet], [/testdata/tpch/plan_sf0.02/region/11.parquet], [/testdata/tpch/plan_sf0.02/region/12.parquet], [/testdata/tpch/plan_sf0.02/region/13.parquet], ...]}, projection=[r_regionkey, r_name], file_type=parquet, predicate=r_name@1 = EUROPE, pruning_predicate=r_name_null_count@2 != row_count@3 AND r_name_min@0 <= EUROPE AND EUROPE <= r_name_max@1, required_guarantees=[r_name in (EUROPE)] + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p15] t1:[p16..p31] t2:[p32..p47] t3:[p48..p63] + │ BroadcastExec: input_partitions=4, consumer_tasks=4, output_partitions=16 + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/nation/1.parquet], [/testdata/tpch/plan_sf0.02/nation/10.parquet], [/testdata/tpch/plan_sf0.02/nation/11.parquet], [/testdata/tpch/plan_sf0.02/nation/12.parquet], [/testdata/tpch/plan_sf0.02/nation/13.parquet], ...]}, projection=[n_nationkey, n_regionkey], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 8 ── Tasks: t0:[p0..p15] t1:[p16..p31] t2:[p32..p47] t3:[p48..p63] + │ BroadcastExec: input_partitions=4, consumer_tasks=4, output_partitions=16 + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/supplier/1.parquet], [/testdata/tpch/plan_sf0.02/supplier/10.parquet], [/testdata/tpch/plan_sf0.02/supplier/11.parquet], [/testdata/tpch/plan_sf0.02/supplier/12.parquet], [/testdata/tpch/plan_sf0.02/supplier/13.parquet], ...]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + "); + Ok(()) + } + + #[tokio::test] + async fn test_tpch_3() -> Result<(), Box> { + let plan = test_tpch_query("q3").await?; + assert_snapshot!(plan, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [revenue@1 DESC, o_orderdate@2 ASC NULLS LAST] + │ [Stage 4] => NetworkCoalesceExec: output_partitions=18, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ SortExec: expr=[revenue@1 DESC, o_orderdate@2 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[l_orderkey@0 as l_orderkey, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@3 as revenue, o_orderdate@1 as o_orderdate, o_shippriority@2 as o_shippriority] + │ AggregateExec: mode=FinalPartitioned, gby=[l_orderkey@0 as l_orderkey, o_orderdate@1 as o_orderdate, o_shippriority@2 as o_shippriority], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] + │ [Stage 3] => NetworkShuffleExec: output_partitions=6, input_tasks=4 + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p17] t1:[p0..p17] t2:[p0..p17] t3:[p0..p17] + │ RepartitionExec: partitioning=Hash([l_orderkey@0, o_orderdate@1, o_shippriority@2], 18), input_partitions=4 + │ AggregateExec: mode=Partial, gby=[l_orderkey@2 as l_orderkey, o_orderdate@0 as o_orderdate, o_shippriority@1 as o_shippriority], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(o_orderkey@0, l_orderkey@0)], projection=[o_orderdate@1, o_shippriority@2, l_orderkey@3, l_extendedprice@4, l_discount@5] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=4, stage_partitions=16, input_tasks=4 + │ FilterExec: l_shipdate@3 > 1995-03-15, projection=[l_orderkey@0, l_extendedprice@1, l_discount@2] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet], [/testdata/tpch/plan_sf0.02/lineitem/10.parquet], [/testdata/tpch/plan_sf0.02/lineitem/11.parquet], [/testdata/tpch/plan_sf0.02/lineitem/12.parquet], [/testdata/tpch/plan_sf0.02/lineitem/13.parquet], ...]}, projection=[l_orderkey, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@10 > 1995-03-15 AND DynamicFilter [ empty ], pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 > 1995-03-15, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p15] t1:[p16..p31] t2:[p32..p47] t3:[p48..p63] + │ BroadcastExec: input_partitions=4, consumer_tasks=4, output_partitions=16 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_custkey@0, o_custkey@1)], projection=[o_orderkey@1, o_orderdate@3, o_shippriority@4] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=4, stage_partitions=16, input_tasks=4 + │ FilterExec: o_orderdate@2 < 1995-03-15 + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/orders/1.parquet], [/testdata/tpch/plan_sf0.02/orders/10.parquet], [/testdata/tpch/plan_sf0.02/orders/11.parquet], [/testdata/tpch/plan_sf0.02/orders/12.parquet], [/testdata/tpch/plan_sf0.02/orders/13.parquet], ...]}, projection=[o_orderkey, o_custkey, o_orderdate, o_shippriority], file_type=parquet, predicate=o_orderdate@4 < 1995-03-15 AND DynamicFilter [ empty ], pruning_predicate=o_orderdate_null_count@1 != row_count@2 AND o_orderdate_min@0 < 1995-03-15, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p15] t1:[p16..p31] t2:[p32..p47] t3:[p48..p63] + │ BroadcastExec: input_partitions=4, consumer_tasks=4, output_partitions=16 + │ FilterExec: c_mktsegment@1 = BUILDING, projection=[c_custkey@0] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/customer/1.parquet], [/testdata/tpch/plan_sf0.02/customer/10.parquet], [/testdata/tpch/plan_sf0.02/customer/11.parquet], [/testdata/tpch/plan_sf0.02/customer/12.parquet], [/testdata/tpch/plan_sf0.02/customer/13.parquet], ...]}, projection=[c_custkey, c_mktsegment], file_type=parquet, predicate=c_mktsegment@6 = BUILDING, pruning_predicate=c_mktsegment_null_count@2 != row_count@3 AND c_mktsegment_min@0 <= BUILDING AND BUILDING <= c_mktsegment_max@1, required_guarantees=[c_mktsegment in (BUILDING)] + └────────────────────────────────────────────────── + "); + Ok(()) + } + + #[tokio::test] + async fn test_tpch_4() -> Result<(), Box> { + let plan = test_tpch_query("q4").await?; + assert_snapshot!(plan, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [o_orderpriority@0 ASC NULLS LAST] + │ SortExec: expr=[o_orderpriority@0 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[o_orderpriority@0 as o_orderpriority, count(Int64(1))@1 as order_count] + │ AggregateExec: mode=FinalPartitioned, gby=[o_orderpriority@0 as o_orderpriority], aggr=[count(Int64(1))] + │ RepartitionExec: partitioning=Hash([o_orderpriority@0], 6), input_partitions=6 + │ AggregateExec: mode=Partial, gby=[o_orderpriority@0 as o_orderpriority], aggr=[count(Int64(1))] + │ HashJoinExec: mode=CollectLeft, join_type=LeftSemi, on=[(o_orderkey@0, l_orderkey@0)], projection=[o_orderpriority@1] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=16, input_tasks=4 + │ FilterExec: l_receiptdate@2 > l_commitdate@1, projection=[l_orderkey@0] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet, /testdata/tpch/plan_sf0.02/lineitem/10.parquet, /testdata/tpch/plan_sf0.02/lineitem/11.parquet], [/testdata/tpch/plan_sf0.02/lineitem/12.parquet, /testdata/tpch/plan_sf0.02/lineitem/13.parquet, /testdata/tpch/plan_sf0.02/lineitem/14.parquet], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet, /testdata/tpch/plan_sf0.02/lineitem/16.parquet, /testdata/tpch/plan_sf0.02/lineitem/2.parquet], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet, /testdata/tpch/plan_sf0.02/lineitem/4.parquet, /testdata/tpch/plan_sf0.02/lineitem/5.parquet], [/testdata/tpch/plan_sf0.02/lineitem/6.parquet, /testdata/tpch/plan_sf0.02/lineitem/7.parquet, /testdata/tpch/plan_sf0.02/lineitem/8.parquet], ...]}, projection=[l_orderkey, l_commitdate, l_receiptdate], file_type=parquet, predicate=l_receiptdate@12 > l_commitdate@11 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p3] t1:[p4..p7] t2:[p8..p11] t3:[p12..p15] + │ FilterExec: o_orderdate@1 >= 1993-07-01 AND o_orderdate@1 < 1993-10-01, projection=[o_orderkey@0, o_orderpriority@2] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/orders/1.parquet], [/testdata/tpch/plan_sf0.02/orders/10.parquet], [/testdata/tpch/plan_sf0.02/orders/11.parquet], [/testdata/tpch/plan_sf0.02/orders/12.parquet], [/testdata/tpch/plan_sf0.02/orders/13.parquet], ...]}, projection=[o_orderkey, o_orderdate, o_orderpriority], file_type=parquet, predicate=o_orderdate@4 >= 1993-07-01 AND o_orderdate@4 < 1993-10-01, pruning_predicate=o_orderdate_null_count@1 != row_count@2 AND o_orderdate_max@0 >= 1993-07-01 AND o_orderdate_null_count@1 != row_count@2 AND o_orderdate_min@3 < 1993-10-01, required_guarantees=[] + └────────────────────────────────────────────────── + "); + Ok(()) + } + + #[tokio::test] + async fn test_tpch_5() -> Result<(), Box> { + let plan = test_tpch_query("q5").await?; + assert_snapshot!(plan, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [revenue@1 DESC] + │ [Stage 7] => NetworkCoalesceExec: output_partitions=18, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ SortExec: expr=[revenue@1 DESC], preserve_partitioning=[true] + │ ProjectionExec: expr=[n_name@0 as n_name, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@1 as revenue] + │ AggregateExec: mode=FinalPartitioned, gby=[n_name@0 as n_name], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] + │ [Stage 6] => NetworkShuffleExec: output_partitions=6, input_tasks=4 + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p17] t1:[p0..p17] t2:[p0..p17] t3:[p0..p17] + │ RepartitionExec: partitioning=Hash([n_name@0], 18), input_partitions=4 + │ AggregateExec: mode=Partial, gby=[n_name@2 as n_name], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(r_regionkey@0, n_regionkey@3)], projection=[l_extendedprice@1, l_discount@2, n_name@3] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=4, stage_partitions=16, input_tasks=4 + │ ProjectionExec: expr=[l_extendedprice@2 as l_extendedprice, l_discount@3 as l_discount, n_name@0 as n_name, n_regionkey@1 as n_regionkey] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@2)], projection=[n_name@1, n_regionkey@2, l_extendedprice@3, l_discount@4] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=4, stage_partitions=16, input_tasks=4 + │ ProjectionExec: expr=[l_extendedprice@1 as l_extendedprice, l_discount@2 as l_discount, s_nationkey@0 as s_nationkey] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_suppkey@0, l_suppkey@1), (s_nationkey@1, c_nationkey@0)], projection=[s_nationkey@1, l_extendedprice@4, l_discount@5] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=4, stage_partitions=16, input_tasks=4 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(o_orderkey@1, l_orderkey@0)], projection=[c_nationkey@0, l_suppkey@3, l_extendedprice@4, l_discount@5] + │ CoalescePartitionsExec + │ [Stage 5] => NetworkBroadcastExec: partitions_per_consumer=4, stage_partitions=16, input_tasks=4 + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet], [/testdata/tpch/plan_sf0.02/lineitem/10.parquet], [/testdata/tpch/plan_sf0.02/lineitem/11.parquet], [/testdata/tpch/plan_sf0.02/lineitem/12.parquet], [/testdata/tpch/plan_sf0.02/lineitem/13.parquet], ...]}, projection=[l_orderkey, l_suppkey, l_extendedprice, l_discount], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p15] t1:[p16..p31] t2:[p32..p47] t3:[p48..p63] + │ BroadcastExec: input_partitions=4, consumer_tasks=4, output_partitions=16 + │ FilterExec: r_name@1 = ASIA, projection=[r_regionkey@0] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/region/1.parquet], [/testdata/tpch/plan_sf0.02/region/10.parquet], [/testdata/tpch/plan_sf0.02/region/11.parquet], [/testdata/tpch/plan_sf0.02/region/12.parquet], [/testdata/tpch/plan_sf0.02/region/13.parquet], ...]}, projection=[r_regionkey, r_name], file_type=parquet, predicate=r_name@1 = ASIA, pruning_predicate=r_name_null_count@2 != row_count@3 AND r_name_min@0 <= ASIA AND ASIA <= r_name_max@1, required_guarantees=[r_name in (ASIA)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p15] t1:[p16..p31] t2:[p32..p47] t3:[p48..p63] + │ BroadcastExec: input_partitions=4, consumer_tasks=4, output_partitions=16 + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/nation/1.parquet], [/testdata/tpch/plan_sf0.02/nation/10.parquet], [/testdata/tpch/plan_sf0.02/nation/11.parquet], [/testdata/tpch/plan_sf0.02/nation/12.parquet], [/testdata/tpch/plan_sf0.02/nation/13.parquet], ...]}, projection=[n_nationkey, n_name, n_regionkey], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p15] t1:[p16..p31] t2:[p32..p47] t3:[p48..p63] + │ BroadcastExec: input_partitions=4, consumer_tasks=4, output_partitions=16 + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/supplier/1.parquet], [/testdata/tpch/plan_sf0.02/supplier/10.parquet], [/testdata/tpch/plan_sf0.02/supplier/11.parquet], [/testdata/tpch/plan_sf0.02/supplier/12.parquet], [/testdata/tpch/plan_sf0.02/supplier/13.parquet], ...]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p15] t1:[p16..p31] t2:[p32..p47] t3:[p48..p63] + │ BroadcastExec: input_partitions=4, consumer_tasks=4, output_partitions=16 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_custkey@0, o_custkey@1)], projection=[c_nationkey@1, o_orderkey@2] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=4, stage_partitions=16, input_tasks=4 + │ FilterExec: o_orderdate@2 >= 1994-01-01 AND o_orderdate@2 < 1995-01-01, projection=[o_orderkey@0, o_custkey@1] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/orders/1.parquet], [/testdata/tpch/plan_sf0.02/orders/10.parquet], [/testdata/tpch/plan_sf0.02/orders/11.parquet], [/testdata/tpch/plan_sf0.02/orders/12.parquet], [/testdata/tpch/plan_sf0.02/orders/13.parquet], ...]}, projection=[o_orderkey, o_custkey, o_orderdate], file_type=parquet, predicate=o_orderdate@4 >= 1994-01-01 AND o_orderdate@4 < 1995-01-01 AND DynamicFilter [ empty ], pruning_predicate=o_orderdate_null_count@1 != row_count@2 AND o_orderdate_max@0 >= 1994-01-01 AND o_orderdate_null_count@1 != row_count@2 AND o_orderdate_min@3 < 1995-01-01, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p15] t1:[p16..p31] t2:[p32..p47] t3:[p48..p63] + │ BroadcastExec: input_partitions=4, consumer_tasks=4, output_partitions=16 + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/customer/1.parquet], [/testdata/tpch/plan_sf0.02/customer/10.parquet], [/testdata/tpch/plan_sf0.02/customer/11.parquet], [/testdata/tpch/plan_sf0.02/customer/12.parquet], [/testdata/tpch/plan_sf0.02/customer/13.parquet], ...]}, projection=[c_custkey, c_nationkey], file_type=parquet + └────────────────────────────────────────────────── + "); + Ok(()) + } + + #[tokio::test] + async fn test_tpch_6() -> Result<(), Box> { + let plan = test_tpch_query("q6").await?; + assert_snapshot!(plan, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ ProjectionExec: expr=[sum(lineitem.l_extendedprice * lineitem.l_discount)@0 as revenue] + │ AggregateExec: mode=Final, gby=[], aggr=[sum(lineitem.l_extendedprice * lineitem.l_discount)] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=16, input_tasks=4 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p3] t1:[p4..p7] t2:[p8..p11] t3:[p12..p15] + │ AggregateExec: mode=Partial, gby=[], aggr=[sum(lineitem.l_extendedprice * lineitem.l_discount)] + │ FilterExec: l_shipdate@3 >= 1994-01-01 AND l_shipdate@3 < 1995-01-01 AND l_discount@2 >= Some(5),15,2 AND l_discount@2 <= Some(7),15,2 AND l_quantity@0 < Some(2400),15,2, projection=[l_extendedprice@1, l_discount@2] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet], [/testdata/tpch/plan_sf0.02/lineitem/10.parquet], [/testdata/tpch/plan_sf0.02/lineitem/11.parquet], [/testdata/tpch/plan_sf0.02/lineitem/12.parquet], [/testdata/tpch/plan_sf0.02/lineitem/13.parquet], ...]}, projection=[l_quantity, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@10 >= 1994-01-01 AND l_shipdate@10 < 1995-01-01 AND l_discount@6 >= Some(5),15,2 AND l_discount@6 <= Some(7),15,2 AND l_quantity@4 < Some(2400),15,2, pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1994-01-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 < 1995-01-01 AND l_discount_null_count@5 != row_count@2 AND l_discount_max@4 >= Some(5),15,2 AND l_discount_null_count@5 != row_count@2 AND l_discount_min@6 <= Some(7),15,2 AND l_quantity_null_count@8 != row_count@2 AND l_quantity_min@7 < Some(2400),15,2, required_guarantees=[] + └────────────────────────────────────────────────── + "); + Ok(()) + } + + #[tokio::test] + async fn test_tpch_7() -> Result<(), Box> { + let plan = test_tpch_query("q7").await?; + assert_snapshot!(plan, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [supp_nation@0 ASC NULLS LAST, cust_nation@1 ASC NULLS LAST, l_year@2 ASC NULLS LAST] + │ [Stage 7] => NetworkCoalesceExec: output_partitions=18, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ SortExec: expr=[supp_nation@0 ASC NULLS LAST, cust_nation@1 ASC NULLS LAST, l_year@2 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[supp_nation@0 as supp_nation, cust_nation@1 as cust_nation, l_year@2 as l_year, sum(shipping.volume)@3 as revenue] + │ AggregateExec: mode=FinalPartitioned, gby=[supp_nation@0 as supp_nation, cust_nation@1 as cust_nation, l_year@2 as l_year], aggr=[sum(shipping.volume)] + │ [Stage 6] => NetworkShuffleExec: output_partitions=6, input_tasks=4 + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p17] t1:[p0..p17] t2:[p0..p17] t3:[p0..p17] + │ RepartitionExec: partitioning=Hash([supp_nation@0, cust_nation@1, l_year@2], 18), input_partitions=4 + │ AggregateExec: mode=Partial, gby=[supp_nation@0 as supp_nation, cust_nation@1 as cust_nation, l_year@2 as l_year], aggr=[sum(shipping.volume)] + │ ProjectionExec: expr=[n_name@4 as supp_nation, n_name@0 as cust_nation, date_part(YEAR, l_shipdate@3) as l_year, l_extendedprice@1 * (Some(1),20,0 - l_discount@2) as volume] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, c_nationkey@3)], filter=n_name@0 = FRANCE AND n_name@1 = GERMANY OR n_name@0 = GERMANY AND n_name@1 = FRANCE, projection=[n_name@1, l_extendedprice@2, l_discount@3, l_shipdate@4, n_name@6] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=4, stage_partitions=16, input_tasks=4 + │ ProjectionExec: expr=[l_extendedprice@1 as l_extendedprice, l_discount@2 as l_discount, l_shipdate@3 as l_shipdate, c_nationkey@4 as c_nationkey, n_name@0 as n_name] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@0)], projection=[n_name@1, l_extendedprice@3, l_discount@4, l_shipdate@5, c_nationkey@6] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=4, stage_partitions=16, input_tasks=4 + │ ProjectionExec: expr=[s_nationkey@1 as s_nationkey, l_extendedprice@2 as l_extendedprice, l_discount@3 as l_discount, l_shipdate@4 as l_shipdate, c_nationkey@0 as c_nationkey] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_custkey@0, o_custkey@4)], projection=[c_nationkey@1, s_nationkey@2, l_extendedprice@3, l_discount@4, l_shipdate@5] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=4, stage_partitions=16, input_tasks=4 + │ ProjectionExec: expr=[s_nationkey@1 as s_nationkey, l_extendedprice@2 as l_extendedprice, l_discount@3 as l_discount, l_shipdate@4 as l_shipdate, o_custkey@0 as o_custkey] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(o_orderkey@0, l_orderkey@1)], projection=[o_custkey@1, s_nationkey@2, l_extendedprice@4, l_discount@5, l_shipdate@6] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=4, stage_partitions=16, input_tasks=4 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_suppkey@0, l_suppkey@1)], projection=[s_nationkey@1, l_orderkey@2, l_extendedprice@4, l_discount@5, l_shipdate@6] + │ CoalescePartitionsExec + │ [Stage 5] => NetworkBroadcastExec: partitions_per_consumer=4, stage_partitions=16, input_tasks=4 + │ FilterExec: l_shipdate@4 >= 1995-01-01 AND l_shipdate@4 <= 1996-12-31 + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet], [/testdata/tpch/plan_sf0.02/lineitem/10.parquet], [/testdata/tpch/plan_sf0.02/lineitem/11.parquet], [/testdata/tpch/plan_sf0.02/lineitem/12.parquet], [/testdata/tpch/plan_sf0.02/lineitem/13.parquet], ...]}, projection=[l_orderkey, l_suppkey, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@10 >= 1995-01-01 AND l_shipdate@10 <= 1996-12-31 AND DynamicFilter [ empty ] AND DynamicFilter [ empty ], pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1995-01-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 <= 1996-12-31, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p15] t1:[p16..p31] t2:[p32..p47] t3:[p48..p63] + │ BroadcastExec: input_partitions=4, consumer_tasks=4, output_partitions=16 + │ FilterExec: n_name@1 = GERMANY OR n_name@1 = FRANCE + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/nation/1.parquet], [/testdata/tpch/plan_sf0.02/nation/10.parquet], [/testdata/tpch/plan_sf0.02/nation/11.parquet], [/testdata/tpch/plan_sf0.02/nation/12.parquet], [/testdata/tpch/plan_sf0.02/nation/13.parquet], ...]}, projection=[n_nationkey, n_name], file_type=parquet, predicate=n_name@1 = GERMANY OR n_name@1 = FRANCE, pruning_predicate=n_name_null_count@2 != row_count@3 AND n_name_min@0 <= GERMANY AND GERMANY <= n_name_max@1 OR n_name_null_count@2 != row_count@3 AND n_name_min@0 <= FRANCE AND FRANCE <= n_name_max@1, required_guarantees=[n_name in (FRANCE, GERMANY)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p15] t1:[p16..p31] t2:[p32..p47] t3:[p48..p63] + │ BroadcastExec: input_partitions=4, consumer_tasks=4, output_partitions=16 + │ FilterExec: n_name@1 = FRANCE OR n_name@1 = GERMANY + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/nation/1.parquet], [/testdata/tpch/plan_sf0.02/nation/10.parquet], [/testdata/tpch/plan_sf0.02/nation/11.parquet], [/testdata/tpch/plan_sf0.02/nation/12.parquet], [/testdata/tpch/plan_sf0.02/nation/13.parquet], ...]}, projection=[n_nationkey, n_name], file_type=parquet, predicate=n_name@1 = FRANCE OR n_name@1 = GERMANY, pruning_predicate=n_name_null_count@2 != row_count@3 AND n_name_min@0 <= FRANCE AND FRANCE <= n_name_max@1 OR n_name_null_count@2 != row_count@3 AND n_name_min@0 <= GERMANY AND GERMANY <= n_name_max@1, required_guarantees=[n_name in (FRANCE, GERMANY)] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p15] t1:[p16..p31] t2:[p32..p47] t3:[p48..p63] + │ BroadcastExec: input_partitions=4, consumer_tasks=4, output_partitions=16 + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/customer/1.parquet], [/testdata/tpch/plan_sf0.02/customer/10.parquet], [/testdata/tpch/plan_sf0.02/customer/11.parquet], [/testdata/tpch/plan_sf0.02/customer/12.parquet], [/testdata/tpch/plan_sf0.02/customer/13.parquet], ...]}, projection=[c_custkey, c_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p15] t1:[p16..p31] t2:[p32..p47] t3:[p48..p63] + │ BroadcastExec: input_partitions=4, consumer_tasks=4, output_partitions=16 + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/orders/1.parquet], [/testdata/tpch/plan_sf0.02/orders/10.parquet], [/testdata/tpch/plan_sf0.02/orders/11.parquet], [/testdata/tpch/plan_sf0.02/orders/12.parquet], [/testdata/tpch/plan_sf0.02/orders/13.parquet], ...]}, projection=[o_orderkey, o_custkey], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p15] t1:[p16..p31] t2:[p32..p47] t3:[p48..p63] + │ BroadcastExec: input_partitions=4, consumer_tasks=4, output_partitions=16 + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/supplier/1.parquet], [/testdata/tpch/plan_sf0.02/supplier/10.parquet], [/testdata/tpch/plan_sf0.02/supplier/11.parquet], [/testdata/tpch/plan_sf0.02/supplier/12.parquet], [/testdata/tpch/plan_sf0.02/supplier/13.parquet], ...]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + "); + Ok(()) + } + + #[tokio::test] + async fn test_tpch_8() -> Result<(), Box> { + let plan = test_tpch_query("q8").await?; + assert_snapshot!(plan, @r#" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [o_year@0 ASC NULLS LAST] + │ [Stage 9] => NetworkCoalesceExec: output_partitions=18, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 9 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ SortExec: expr=[o_year@0 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[o_year@0 as o_year, sum(CASE WHEN all_nations.nation = Utf8("BRAZIL") THEN all_nations.volume ELSE Int64(0) END)@1 / sum(all_nations.volume)@2 as mkt_share] + │ AggregateExec: mode=FinalPartitioned, gby=[o_year@0 as o_year], aggr=[sum(CASE WHEN all_nations.nation = Utf8("BRAZIL") THEN all_nations.volume ELSE Int64(0) END), sum(all_nations.volume)] + │ [Stage 8] => NetworkShuffleExec: output_partitions=6, input_tasks=4 + └────────────────────────────────────────────────── + ┌───── Stage 8 ── Tasks: t0:[p0..p17] t1:[p0..p17] t2:[p0..p17] t3:[p0..p17] + │ RepartitionExec: partitioning=Hash([o_year@0], 18), input_partitions=4 + │ AggregateExec: mode=Partial, gby=[o_year@0 as o_year], aggr=[sum(CASE WHEN all_nations.nation = Utf8("BRAZIL") THEN all_nations.volume ELSE Int64(0) END), sum(all_nations.volume)] + │ ProjectionExec: expr=[date_part(YEAR, o_orderdate@2) as o_year, l_extendedprice@0 * (Some(1),20,0 - l_discount@1) as volume, n_name@3 as nation] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(r_regionkey@0, n_regionkey@3)], projection=[l_extendedprice@1, l_discount@2, o_orderdate@3, n_name@5] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=4, stage_partitions=16, input_tasks=4 + │ ProjectionExec: expr=[l_extendedprice@1 as l_extendedprice, l_discount@2 as l_discount, o_orderdate@3 as o_orderdate, n_regionkey@4 as n_regionkey, n_name@0 as n_name] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@2)], projection=[n_name@1, l_extendedprice@2, l_discount@3, o_orderdate@5, n_regionkey@6] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=4, stage_partitions=16, input_tasks=4 + │ ProjectionExec: expr=[l_extendedprice@1 as l_extendedprice, l_discount@2 as l_discount, s_nationkey@3 as s_nationkey, o_orderdate@4 as o_orderdate, n_regionkey@0 as n_regionkey] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, c_nationkey@4)], projection=[n_regionkey@1, l_extendedprice@2, l_discount@3, s_nationkey@4, o_orderdate@5] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=4, stage_partitions=16, input_tasks=4 + │ ProjectionExec: expr=[l_extendedprice@1 as l_extendedprice, l_discount@2 as l_discount, s_nationkey@3 as s_nationkey, o_orderdate@4 as o_orderdate, c_nationkey@0 as c_nationkey] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_custkey@0, o_custkey@3)], projection=[c_nationkey@1, l_extendedprice@2, l_discount@3, s_nationkey@4, o_orderdate@6] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=4, stage_partitions=16, input_tasks=4 + │ ProjectionExec: expr=[l_extendedprice@2 as l_extendedprice, l_discount@3 as l_discount, s_nationkey@4 as s_nationkey, o_custkey@0 as o_custkey, o_orderdate@1 as o_orderdate] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(o_orderkey@0, l_orderkey@0)], projection=[o_custkey@1, o_orderdate@2, l_extendedprice@4, l_discount@5, s_nationkey@6] + │ CoalescePartitionsExec + │ [Stage 5] => NetworkBroadcastExec: partitions_per_consumer=4, stage_partitions=16, input_tasks=4 + │ ProjectionExec: expr=[l_orderkey@1 as l_orderkey, l_extendedprice@2 as l_extendedprice, l_discount@3 as l_discount, s_nationkey@0 as s_nationkey] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_suppkey@0, l_suppkey@1)], projection=[s_nationkey@1, l_orderkey@2, l_extendedprice@4, l_discount@5] + │ CoalescePartitionsExec + │ [Stage 6] => NetworkBroadcastExec: partitions_per_consumer=4, stage_partitions=16, input_tasks=4 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(p_partkey@0, l_partkey@1)], projection=[l_orderkey@1, l_suppkey@3, l_extendedprice@4, l_discount@5] + │ CoalescePartitionsExec + │ [Stage 7] => NetworkBroadcastExec: partitions_per_consumer=4, stage_partitions=16, input_tasks=4 + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet], [/testdata/tpch/plan_sf0.02/lineitem/10.parquet], [/testdata/tpch/plan_sf0.02/lineitem/11.parquet], [/testdata/tpch/plan_sf0.02/lineitem/12.parquet], [/testdata/tpch/plan_sf0.02/lineitem/13.parquet], ...]}, projection=[l_orderkey, l_partkey, l_suppkey, l_extendedprice, l_discount], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p15] t1:[p16..p31] t2:[p32..p47] t3:[p48..p63] + │ BroadcastExec: input_partitions=4, consumer_tasks=4, output_partitions=16 + │ FilterExec: r_name@1 = AMERICA, projection=[r_regionkey@0] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/region/1.parquet], [/testdata/tpch/plan_sf0.02/region/10.parquet], [/testdata/tpch/plan_sf0.02/region/11.parquet], [/testdata/tpch/plan_sf0.02/region/12.parquet], [/testdata/tpch/plan_sf0.02/region/13.parquet], ...]}, projection=[r_regionkey, r_name], file_type=parquet, predicate=r_name@1 = AMERICA, pruning_predicate=r_name_null_count@2 != row_count@3 AND r_name_min@0 <= AMERICA AND AMERICA <= r_name_max@1, required_guarantees=[r_name in (AMERICA)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p15] t1:[p16..p31] t2:[p32..p47] t3:[p48..p63] + │ BroadcastExec: input_partitions=4, consumer_tasks=4, output_partitions=16 + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/nation/1.parquet], [/testdata/tpch/plan_sf0.02/nation/10.parquet], [/testdata/tpch/plan_sf0.02/nation/11.parquet], [/testdata/tpch/plan_sf0.02/nation/12.parquet], [/testdata/tpch/plan_sf0.02/nation/13.parquet], ...]}, projection=[n_nationkey, n_name], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p15] t1:[p16..p31] t2:[p32..p47] t3:[p48..p63] + │ BroadcastExec: input_partitions=4, consumer_tasks=4, output_partitions=16 + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/nation/1.parquet], [/testdata/tpch/plan_sf0.02/nation/10.parquet], [/testdata/tpch/plan_sf0.02/nation/11.parquet], [/testdata/tpch/plan_sf0.02/nation/12.parquet], [/testdata/tpch/plan_sf0.02/nation/13.parquet], ...]}, projection=[n_nationkey, n_regionkey], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p15] t1:[p16..p31] t2:[p32..p47] t3:[p48..p63] + │ BroadcastExec: input_partitions=4, consumer_tasks=4, output_partitions=16 + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/customer/1.parquet], [/testdata/tpch/plan_sf0.02/customer/10.parquet], [/testdata/tpch/plan_sf0.02/customer/11.parquet], [/testdata/tpch/plan_sf0.02/customer/12.parquet], [/testdata/tpch/plan_sf0.02/customer/13.parquet], ...]}, projection=[c_custkey, c_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p15] t1:[p16..p31] t2:[p32..p47] t3:[p48..p63] + │ BroadcastExec: input_partitions=4, consumer_tasks=4, output_partitions=16 + │ FilterExec: o_orderdate@2 >= 1995-01-01 AND o_orderdate@2 <= 1996-12-31 + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/orders/1.parquet], [/testdata/tpch/plan_sf0.02/orders/10.parquet], [/testdata/tpch/plan_sf0.02/orders/11.parquet], [/testdata/tpch/plan_sf0.02/orders/12.parquet], [/testdata/tpch/plan_sf0.02/orders/13.parquet], ...]}, projection=[o_orderkey, o_custkey, o_orderdate], file_type=parquet, predicate=o_orderdate@4 >= 1995-01-01 AND o_orderdate@4 <= 1996-12-31 AND DynamicFilter [ empty ], pruning_predicate=o_orderdate_null_count@1 != row_count@2 AND o_orderdate_max@0 >= 1995-01-01 AND o_orderdate_null_count@1 != row_count@2 AND o_orderdate_min@3 <= 1996-12-31, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p15] t1:[p16..p31] t2:[p32..p47] t3:[p48..p63] + │ BroadcastExec: input_partitions=4, consumer_tasks=4, output_partitions=16 + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/supplier/1.parquet], [/testdata/tpch/plan_sf0.02/supplier/10.parquet], [/testdata/tpch/plan_sf0.02/supplier/11.parquet], [/testdata/tpch/plan_sf0.02/supplier/12.parquet], [/testdata/tpch/plan_sf0.02/supplier/13.parquet], ...]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p15] t1:[p16..p31] t2:[p32..p47] t3:[p48..p63] + │ BroadcastExec: input_partitions=4, consumer_tasks=4, output_partitions=16 + │ FilterExec: p_type@1 = ECONOMY ANODIZED STEEL, projection=[p_partkey@0] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/part/1.parquet], [/testdata/tpch/plan_sf0.02/part/10.parquet], [/testdata/tpch/plan_sf0.02/part/11.parquet], [/testdata/tpch/plan_sf0.02/part/12.parquet], [/testdata/tpch/plan_sf0.02/part/13.parquet], ...]}, projection=[p_partkey, p_type], file_type=parquet, predicate=p_type@4 = ECONOMY ANODIZED STEEL, pruning_predicate=p_type_null_count@2 != row_count@3 AND p_type_min@0 <= ECONOMY ANODIZED STEEL AND ECONOMY ANODIZED STEEL <= p_type_max@1, required_guarantees=[p_type in (ECONOMY ANODIZED STEEL)] + └────────────────────────────────────────────────── + "#); + Ok(()) + } + + #[tokio::test] + async fn test_tpch_9() -> Result<(), Box> { + let plan = test_tpch_query("q9").await?; + assert_snapshot!(plan, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [nation@0 ASC NULLS LAST, o_year@1 DESC] + │ [Stage 7] => NetworkCoalesceExec: output_partitions=18, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 7 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ SortExec: expr=[nation@0 ASC NULLS LAST, o_year@1 DESC], preserve_partitioning=[true] + │ ProjectionExec: expr=[nation@0 as nation, o_year@1 as o_year, sum(profit.amount)@2 as sum_profit] + │ AggregateExec: mode=FinalPartitioned, gby=[nation@0 as nation, o_year@1 as o_year], aggr=[sum(profit.amount)] + │ [Stage 6] => NetworkShuffleExec: output_partitions=6, input_tasks=4 + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p17] t1:[p0..p17] t2:[p0..p17] t3:[p0..p17] + │ RepartitionExec: partitioning=Hash([nation@0, o_year@1], 18), input_partitions=4 + │ AggregateExec: mode=Partial, gby=[nation@0 as nation, o_year@1 as o_year], aggr=[sum(profit.amount)] + │ ProjectionExec: expr=[n_name@0 as nation, date_part(YEAR, o_orderdate@5) as o_year, l_extendedprice@2 * (Some(1),20,0 - l_discount@3) - ps_supplycost@4 * l_quantity@1 as amount] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@3)], projection=[n_name@1, l_quantity@2, l_extendedprice@3, l_discount@4, ps_supplycost@6, o_orderdate@7] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=4, stage_partitions=16, input_tasks=4 + │ ProjectionExec: expr=[l_quantity@1 as l_quantity, l_extendedprice@2 as l_extendedprice, l_discount@3 as l_discount, s_nationkey@4 as s_nationkey, ps_supplycost@5 as ps_supplycost, o_orderdate@0 as o_orderdate] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(o_orderkey@0, l_orderkey@0)], projection=[o_orderdate@1, l_quantity@3, l_extendedprice@4, l_discount@5, s_nationkey@6, ps_supplycost@7] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=4, stage_partitions=16, input_tasks=4 + │ ProjectionExec: expr=[l_orderkey@1 as l_orderkey, l_quantity@2 as l_quantity, l_extendedprice@3 as l_extendedprice, l_discount@4 as l_discount, s_nationkey@5 as s_nationkey, ps_supplycost@0 as ps_supplycost] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ps_suppkey@1, l_suppkey@2), (ps_partkey@0, l_partkey@1)], projection=[ps_supplycost@2, l_orderkey@3, l_quantity@6, l_extendedprice@7, l_discount@8, s_nationkey@9] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=4, stage_partitions=16, input_tasks=4 + │ ProjectionExec: expr=[l_orderkey@1 as l_orderkey, l_partkey@2 as l_partkey, l_suppkey@3 as l_suppkey, l_quantity@4 as l_quantity, l_extendedprice@5 as l_extendedprice, l_discount@6 as l_discount, s_nationkey@0 as s_nationkey] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_suppkey@0, l_suppkey@2)], projection=[s_nationkey@1, l_orderkey@2, l_partkey@3, l_suppkey@4, l_quantity@5, l_extendedprice@6, l_discount@7] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=4, stage_partitions=16, input_tasks=4 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(p_partkey@0, l_partkey@1)], projection=[l_orderkey@1, l_partkey@2, l_suppkey@3, l_quantity@4, l_extendedprice@5, l_discount@6] + │ CoalescePartitionsExec + │ [Stage 5] => NetworkBroadcastExec: partitions_per_consumer=4, stage_partitions=16, input_tasks=4 + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet], [/testdata/tpch/plan_sf0.02/lineitem/10.parquet], [/testdata/tpch/plan_sf0.02/lineitem/11.parquet], [/testdata/tpch/plan_sf0.02/lineitem/12.parquet], [/testdata/tpch/plan_sf0.02/lineitem/13.parquet], ...]}, projection=[l_orderkey, l_partkey, l_suppkey, l_quantity, l_extendedprice, l_discount], file_type=parquet, predicate=DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p15] t1:[p16..p31] t2:[p32..p47] t3:[p48..p63] + │ BroadcastExec: input_partitions=4, consumer_tasks=4, output_partitions=16 + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/nation/1.parquet], [/testdata/tpch/plan_sf0.02/nation/10.parquet], [/testdata/tpch/plan_sf0.02/nation/11.parquet], [/testdata/tpch/plan_sf0.02/nation/12.parquet], [/testdata/tpch/plan_sf0.02/nation/13.parquet], ...]}, projection=[n_nationkey, n_name], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p15] t1:[p16..p31] t2:[p32..p47] t3:[p48..p63] + │ BroadcastExec: input_partitions=4, consumer_tasks=4, output_partitions=16 + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/orders/1.parquet], [/testdata/tpch/plan_sf0.02/orders/10.parquet], [/testdata/tpch/plan_sf0.02/orders/11.parquet], [/testdata/tpch/plan_sf0.02/orders/12.parquet], [/testdata/tpch/plan_sf0.02/orders/13.parquet], ...]}, projection=[o_orderkey, o_orderdate], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p15] t1:[p16..p31] t2:[p32..p47] t3:[p48..p63] + │ BroadcastExec: input_partitions=4, consumer_tasks=4, output_partitions=16 + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/partsupp/1.parquet], [/testdata/tpch/plan_sf0.02/partsupp/10.parquet], [/testdata/tpch/plan_sf0.02/partsupp/11.parquet], [/testdata/tpch/plan_sf0.02/partsupp/12.parquet], [/testdata/tpch/plan_sf0.02/partsupp/13.parquet], ...]}, projection=[ps_partkey, ps_suppkey, ps_supplycost], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p15] t1:[p16..p31] t2:[p32..p47] t3:[p48..p63] + │ BroadcastExec: input_partitions=4, consumer_tasks=4, output_partitions=16 + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/supplier/1.parquet], [/testdata/tpch/plan_sf0.02/supplier/10.parquet], [/testdata/tpch/plan_sf0.02/supplier/11.parquet], [/testdata/tpch/plan_sf0.02/supplier/12.parquet], [/testdata/tpch/plan_sf0.02/supplier/13.parquet], ...]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p15] t1:[p16..p31] t2:[p32..p47] t3:[p48..p63] + │ BroadcastExec: input_partitions=4, consumer_tasks=4, output_partitions=16 + │ FilterExec: p_name@1 LIKE %green%, projection=[p_partkey@0] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/part/1.parquet], [/testdata/tpch/plan_sf0.02/part/10.parquet], [/testdata/tpch/plan_sf0.02/part/11.parquet], [/testdata/tpch/plan_sf0.02/part/12.parquet], [/testdata/tpch/plan_sf0.02/part/13.parquet], ...]}, projection=[p_partkey, p_name], file_type=parquet, predicate=p_name@1 LIKE %green% + └────────────────────────────────────────────────── + "); + Ok(()) + } + + #[tokio::test] + async fn test_tpch_10() -> Result<(), Box> { + let plan = test_tpch_query("q10").await?; + assert_snapshot!(plan, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [revenue@2 DESC] + │ [Stage 5] => NetworkCoalesceExec: output_partitions=18, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ SortExec: expr=[revenue@2 DESC], preserve_partitioning=[true] + │ ProjectionExec: expr=[c_custkey@0 as c_custkey, c_name@1 as c_name, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@7 as revenue, c_acctbal@2 as c_acctbal, n_name@4 as n_name, c_address@5 as c_address, c_phone@3 as c_phone, c_comment@6 as c_comment] + │ AggregateExec: mode=FinalPartitioned, gby=[c_custkey@0 as c_custkey, c_name@1 as c_name, c_acctbal@2 as c_acctbal, c_phone@3 as c_phone, n_name@4 as n_name, c_address@5 as c_address, c_comment@6 as c_comment], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] + │ [Stage 4] => NetworkShuffleExec: output_partitions=6, input_tasks=4 + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p17] t1:[p0..p17] t2:[p0..p17] t3:[p0..p17] + │ RepartitionExec: partitioning=Hash([c_custkey@0, c_name@1, c_acctbal@2, c_phone@3, n_name@4, c_address@5, c_comment@6], 18), input_partitions=4 + │ AggregateExec: mode=Partial, gby=[c_custkey@0 as c_custkey, c_name@1 as c_name, c_acctbal@4 as c_acctbal, c_phone@3 as c_phone, n_name@8 as n_name, c_address@2 as c_address, c_comment@5 as c_comment], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] + │ ProjectionExec: expr=[c_custkey@1 as c_custkey, c_name@2 as c_name, c_address@3 as c_address, c_phone@4 as c_phone, c_acctbal@5 as c_acctbal, c_comment@6 as c_comment, l_extendedprice@7 as l_extendedprice, l_discount@8 as l_discount, n_name@0 as n_name] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, c_nationkey@3)], projection=[n_name@1, c_custkey@2, c_name@3, c_address@4, c_phone@6, c_acctbal@7, c_comment@8, l_extendedprice@9, l_discount@10] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=4, stage_partitions=16, input_tasks=4 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(o_orderkey@7, l_orderkey@0)], projection=[c_custkey@0, c_name@1, c_address@2, c_nationkey@3, c_phone@4, c_acctbal@5, c_comment@6, l_extendedprice@9, l_discount@10] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=4, stage_partitions=16, input_tasks=4 + │ FilterExec: l_returnflag@3 = R, projection=[l_orderkey@0, l_extendedprice@1, l_discount@2] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet], [/testdata/tpch/plan_sf0.02/lineitem/10.parquet], [/testdata/tpch/plan_sf0.02/lineitem/11.parquet], [/testdata/tpch/plan_sf0.02/lineitem/12.parquet], [/testdata/tpch/plan_sf0.02/lineitem/13.parquet], ...]}, projection=[l_orderkey, l_extendedprice, l_discount, l_returnflag], file_type=parquet, predicate=l_returnflag@8 = R AND DynamicFilter [ empty ], pruning_predicate=l_returnflag_null_count@2 != row_count@3 AND l_returnflag_min@0 <= R AND R <= l_returnflag_max@1, required_guarantees=[l_returnflag in (R)] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p15] t1:[p16..p31] t2:[p32..p47] t3:[p48..p63] + │ BroadcastExec: input_partitions=4, consumer_tasks=4, output_partitions=16 + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/nation/1.parquet], [/testdata/tpch/plan_sf0.02/nation/10.parquet], [/testdata/tpch/plan_sf0.02/nation/11.parquet], [/testdata/tpch/plan_sf0.02/nation/12.parquet], [/testdata/tpch/plan_sf0.02/nation/13.parquet], ...]}, projection=[n_nationkey, n_name], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p15] t1:[p16..p31] t2:[p32..p47] t3:[p48..p63] + │ BroadcastExec: input_partitions=4, consumer_tasks=4, output_partitions=16 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_custkey@0, o_custkey@1)], projection=[c_custkey@0, c_name@1, c_address@2, c_nationkey@3, c_phone@4, c_acctbal@5, c_comment@6, o_orderkey@7] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=4, stage_partitions=16, input_tasks=4 + │ FilterExec: o_orderdate@2 >= 1993-10-01 AND o_orderdate@2 < 1994-01-01, projection=[o_orderkey@0, o_custkey@1] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/orders/1.parquet], [/testdata/tpch/plan_sf0.02/orders/10.parquet], [/testdata/tpch/plan_sf0.02/orders/11.parquet], [/testdata/tpch/plan_sf0.02/orders/12.parquet], [/testdata/tpch/plan_sf0.02/orders/13.parquet], ...]}, projection=[o_orderkey, o_custkey, o_orderdate], file_type=parquet, predicate=o_orderdate@4 >= 1993-10-01 AND o_orderdate@4 < 1994-01-01 AND DynamicFilter [ empty ], pruning_predicate=o_orderdate_null_count@1 != row_count@2 AND o_orderdate_max@0 >= 1993-10-01 AND o_orderdate_null_count@1 != row_count@2 AND o_orderdate_min@3 < 1994-01-01, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p15] t1:[p16..p31] t2:[p32..p47] t3:[p48..p63] + │ BroadcastExec: input_partitions=4, consumer_tasks=4, output_partitions=16 + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/customer/1.parquet], [/testdata/tpch/plan_sf0.02/customer/10.parquet], [/testdata/tpch/plan_sf0.02/customer/11.parquet], [/testdata/tpch/plan_sf0.02/customer/12.parquet], [/testdata/tpch/plan_sf0.02/customer/13.parquet], ...]}, projection=[c_custkey, c_name, c_address, c_nationkey, c_phone, c_acctbal, c_comment], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + "); + Ok(()) + } + + #[tokio::test] + async fn test_tpch_11() -> Result<(), Box> { + let plan = test_tpch_query("q11").await?; + assert_snapshot!(plan, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [value@1 DESC] + │ SortExec: expr=[value@1 DESC], preserve_partitioning=[true] + │ ProjectionExec: expr=[ps_partkey@1 as ps_partkey, sum(partsupp.ps_supplycost * partsupp.ps_availqty)@2 as value] + │ NestedLoopJoinExec: join_type=Inner, filter=join_proj_push_down_1@1 > sum(partsupp.ps_supplycost * partsupp.ps_availqty) * Float64(0.0001)@0, projection=[sum(partsupp.ps_supplycost * partsupp.ps_availqty) * Float64(0.0001)@0, ps_partkey@1, sum(partsupp.ps_supplycost * partsupp.ps_availqty)@2] + │ ProjectionExec: expr=[CAST(CAST(sum(partsupp.ps_supplycost * partsupp.ps_availqty)@0 AS Float64) * 0.0001 AS Decimal128(38, 15)) as sum(partsupp.ps_supplycost * partsupp.ps_availqty) * Float64(0.0001)] + │ AggregateExec: mode=Final, gby=[], aggr=[sum(partsupp.ps_supplycost * partsupp.ps_availqty)] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkCoalesceExec: output_partitions=16, input_tasks=4 + │ ProjectionExec: expr=[ps_partkey@0 as ps_partkey, sum(partsupp.ps_supplycost * partsupp.ps_availqty)@1 as sum(partsupp.ps_supplycost * partsupp.ps_availqty), CAST(sum(partsupp.ps_supplycost * partsupp.ps_availqty)@1 AS Decimal128(38, 15)) as join_proj_push_down_1] + │ AggregateExec: mode=FinalPartitioned, gby=[ps_partkey@0 as ps_partkey], aggr=[sum(partsupp.ps_supplycost * partsupp.ps_availqty)] + │ [Stage 6] => NetworkShuffleExec: output_partitions=6, input_tasks=4 + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p3] t1:[p4..p7] t2:[p8..p11] t3:[p12..p15] + │ AggregateExec: mode=Partial, gby=[], aggr=[sum(partsupp.ps_supplycost * partsupp.ps_availqty)] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@2)], projection=[ps_availqty@1, ps_supplycost@2] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=4, stage_partitions=16, input_tasks=4 + │ ProjectionExec: expr=[ps_availqty@1 as ps_availqty, ps_supplycost@2 as ps_supplycost, s_nationkey@0 as s_nationkey] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_suppkey@0, ps_suppkey@0)], projection=[s_nationkey@1, ps_availqty@3, ps_supplycost@4] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=4, stage_partitions=16, input_tasks=4 + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/partsupp/1.parquet], [/testdata/tpch/plan_sf0.02/partsupp/10.parquet], [/testdata/tpch/plan_sf0.02/partsupp/11.parquet], [/testdata/tpch/plan_sf0.02/partsupp/12.parquet], [/testdata/tpch/plan_sf0.02/partsupp/13.parquet], ...]}, projection=[ps_suppkey, ps_availqty, ps_supplycost], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p15] t1:[p16..p31] t2:[p32..p47] t3:[p48..p63] + │ BroadcastExec: input_partitions=4, consumer_tasks=4, output_partitions=16 + │ FilterExec: n_name@1 = GERMANY, projection=[n_nationkey@0] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/nation/1.parquet], [/testdata/tpch/plan_sf0.02/nation/10.parquet], [/testdata/tpch/plan_sf0.02/nation/11.parquet], [/testdata/tpch/plan_sf0.02/nation/12.parquet], [/testdata/tpch/plan_sf0.02/nation/13.parquet], ...]}, projection=[n_nationkey, n_name], file_type=parquet, predicate=n_name@1 = GERMANY, pruning_predicate=n_name_null_count@2 != row_count@3 AND n_name_min@0 <= GERMANY AND GERMANY <= n_name_max@1, required_guarantees=[n_name in (GERMANY)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p15] t1:[p16..p31] t2:[p32..p47] t3:[p48..p63] + │ BroadcastExec: input_partitions=4, consumer_tasks=4, output_partitions=16 + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/supplier/1.parquet], [/testdata/tpch/plan_sf0.02/supplier/10.parquet], [/testdata/tpch/plan_sf0.02/supplier/11.parquet], [/testdata/tpch/plan_sf0.02/supplier/12.parquet], [/testdata/tpch/plan_sf0.02/supplier/13.parquet], ...]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] t3:[p0..p5] + │ RepartitionExec: partitioning=Hash([ps_partkey@0], 6), input_partitions=4 + │ AggregateExec: mode=Partial, gby=[ps_partkey@0 as ps_partkey], aggr=[sum(partsupp.ps_supplycost * partsupp.ps_availqty)] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@3)], projection=[ps_partkey@1, ps_availqty@2, ps_supplycost@3] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=4, stage_partitions=16, input_tasks=4 + │ ProjectionExec: expr=[ps_partkey@1 as ps_partkey, ps_availqty@2 as ps_availqty, ps_supplycost@3 as ps_supplycost, s_nationkey@0 as s_nationkey] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_suppkey@0, ps_suppkey@1)], projection=[s_nationkey@1, ps_partkey@2, ps_availqty@4, ps_supplycost@5] + │ CoalescePartitionsExec + │ [Stage 5] => NetworkBroadcastExec: partitions_per_consumer=4, stage_partitions=16, input_tasks=4 + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/partsupp/1.parquet], [/testdata/tpch/plan_sf0.02/partsupp/10.parquet], [/testdata/tpch/plan_sf0.02/partsupp/11.parquet], [/testdata/tpch/plan_sf0.02/partsupp/12.parquet], [/testdata/tpch/plan_sf0.02/partsupp/13.parquet], ...]}, projection=[ps_partkey, ps_suppkey, ps_availqty, ps_supplycost], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p15] t1:[p16..p31] t2:[p32..p47] t3:[p48..p63] + │ BroadcastExec: input_partitions=4, consumer_tasks=4, output_partitions=16 + │ FilterExec: n_name@1 = GERMANY, projection=[n_nationkey@0] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/nation/1.parquet], [/testdata/tpch/plan_sf0.02/nation/10.parquet], [/testdata/tpch/plan_sf0.02/nation/11.parquet], [/testdata/tpch/plan_sf0.02/nation/12.parquet], [/testdata/tpch/plan_sf0.02/nation/13.parquet], ...]}, projection=[n_nationkey, n_name], file_type=parquet, predicate=n_name@1 = GERMANY, pruning_predicate=n_name_null_count@2 != row_count@3 AND n_name_min@0 <= GERMANY AND GERMANY <= n_name_max@1, required_guarantees=[n_name in (GERMANY)] + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p15] t1:[p16..p31] t2:[p32..p47] t3:[p48..p63] + │ BroadcastExec: input_partitions=4, consumer_tasks=4, output_partitions=16 + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/supplier/1.parquet], [/testdata/tpch/plan_sf0.02/supplier/10.parquet], [/testdata/tpch/plan_sf0.02/supplier/11.parquet], [/testdata/tpch/plan_sf0.02/supplier/12.parquet], [/testdata/tpch/plan_sf0.02/supplier/13.parquet], ...]}, projection=[s_suppkey, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + "); + Ok(()) + } + + #[tokio::test] + async fn test_tpch_12() -> Result<(), Box> { + let plan = test_tpch_query("q12").await?; + assert_snapshot!(plan, @r#" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [l_shipmode@0 ASC NULLS LAST] + │ [Stage 3] => NetworkCoalesceExec: output_partitions=18, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ SortExec: expr=[l_shipmode@0 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[l_shipmode@0 as l_shipmode, sum(CASE WHEN orders.o_orderpriority = Utf8("1-URGENT") OR orders.o_orderpriority = Utf8("2-HIGH") THEN Int64(1) ELSE Int64(0) END)@1 as high_line_count, sum(CASE WHEN orders.o_orderpriority != Utf8("1-URGENT") AND orders.o_orderpriority != Utf8("2-HIGH") THEN Int64(1) ELSE Int64(0) END)@2 as low_line_count] + │ AggregateExec: mode=FinalPartitioned, gby=[l_shipmode@0 as l_shipmode], aggr=[sum(CASE WHEN orders.o_orderpriority = Utf8("1-URGENT") OR orders.o_orderpriority = Utf8("2-HIGH") THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN orders.o_orderpriority != Utf8("1-URGENT") AND orders.o_orderpriority != Utf8("2-HIGH") THEN Int64(1) ELSE Int64(0) END)] + │ [Stage 2] => NetworkShuffleExec: output_partitions=6, input_tasks=4 + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p17] t1:[p0..p17] t2:[p0..p17] t3:[p0..p17] + │ RepartitionExec: partitioning=Hash([l_shipmode@0], 18), input_partitions=4 + │ AggregateExec: mode=Partial, gby=[l_shipmode@0 as l_shipmode], aggr=[sum(CASE WHEN orders.o_orderpriority = Utf8("1-URGENT") OR orders.o_orderpriority = Utf8("2-HIGH") THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN orders.o_orderpriority != Utf8("1-URGENT") AND orders.o_orderpriority != Utf8("2-HIGH") THEN Int64(1) ELSE Int64(0) END)] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(l_orderkey@0, o_orderkey@0)], projection=[l_shipmode@1, o_orderpriority@3] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=4, stage_partitions=16, input_tasks=4 + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/orders/1.parquet], [/testdata/tpch/plan_sf0.02/orders/10.parquet], [/testdata/tpch/plan_sf0.02/orders/11.parquet], [/testdata/tpch/plan_sf0.02/orders/12.parquet], [/testdata/tpch/plan_sf0.02/orders/13.parquet], ...]}, projection=[o_orderkey, o_orderpriority], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p15] t1:[p16..p31] t2:[p32..p47] t3:[p48..p63] + │ BroadcastExec: input_partitions=4, consumer_tasks=4, output_partitions=16 + │ FilterExec: (l_shipmode@4 = MAIL OR l_shipmode@4 = SHIP) AND l_receiptdate@3 > l_commitdate@2 AND l_shipdate@1 < l_commitdate@2 AND l_receiptdate@3 >= 1994-01-01 AND l_receiptdate@3 < 1995-01-01, projection=[l_orderkey@0, l_shipmode@4] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet], [/testdata/tpch/plan_sf0.02/lineitem/10.parquet], [/testdata/tpch/plan_sf0.02/lineitem/11.parquet], [/testdata/tpch/plan_sf0.02/lineitem/12.parquet], [/testdata/tpch/plan_sf0.02/lineitem/13.parquet], ...]}, projection=[l_orderkey, l_shipdate, l_commitdate, l_receiptdate, l_shipmode], file_type=parquet, predicate=(l_shipmode@14 = MAIL OR l_shipmode@14 = SHIP) AND l_receiptdate@12 > l_commitdate@11 AND l_shipdate@10 < l_commitdate@11 AND l_receiptdate@12 >= 1994-01-01 AND l_receiptdate@12 < 1995-01-01, pruning_predicate=(l_shipmode_null_count@2 != row_count@3 AND l_shipmode_min@0 <= MAIL AND MAIL <= l_shipmode_max@1 OR l_shipmode_null_count@2 != row_count@3 AND l_shipmode_min@0 <= SHIP AND SHIP <= l_shipmode_max@1) AND l_receiptdate_null_count@5 != row_count@3 AND l_receiptdate_max@4 >= 1994-01-01 AND l_receiptdate_null_count@5 != row_count@3 AND l_receiptdate_min@6 < 1995-01-01, required_guarantees=[l_shipmode in (MAIL, SHIP)] + └────────────────────────────────────────────────── + "#); + Ok(()) + } + + #[tokio::test] + async fn test_tpch_13() -> Result<(), Box> { + let plan = test_tpch_query("q13").await?; + assert_snapshot!(plan, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [custdist@1 DESC, c_count@0 DESC] + │ [Stage 3] => NetworkCoalesceExec: output_partitions=12, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ SortExec: expr=[custdist@1 DESC, c_count@0 DESC], preserve_partitioning=[true] + │ ProjectionExec: expr=[c_count@0 as c_count, count(Int64(1))@1 as custdist] + │ AggregateExec: mode=FinalPartitioned, gby=[c_count@0 as c_count], aggr=[count(Int64(1))] + │ [Stage 2] => NetworkShuffleExec: output_partitions=6, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p11] t1:[p0..p11] t2:[p0..p11] + │ RepartitionExec: partitioning=Hash([c_count@0], 12), input_partitions=6 + │ AggregateExec: mode=Partial, gby=[c_count@0 as c_count], aggr=[count(Int64(1))] + │ ProjectionExec: expr=[count(orders.o_orderkey)@1 as c_count] + │ AggregateExec: mode=FinalPartitioned, gby=[c_custkey@0 as c_custkey], aggr=[count(orders.o_orderkey)] + │ [Stage 1] => NetworkShuffleExec: output_partitions=6, input_tasks=4 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p17] t1:[p0..p17] t2:[p0..p17] t3:[p0..p17] + │ RepartitionExec: partitioning=Hash([c_custkey@0], 18), input_partitions=4 + │ AggregateExec: mode=Partial, gby=[c_custkey@0 as c_custkey], aggr=[count(orders.o_orderkey)] + │ HashJoinExec: mode=CollectLeft, join_type=Left, on=[(c_custkey@0, o_custkey@1)], projection=[c_custkey@0, o_orderkey@1] + │ CoalescePartitionsExec + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/customer/1.parquet], [/testdata/tpch/plan_sf0.02/customer/10.parquet], [/testdata/tpch/plan_sf0.02/customer/11.parquet], [/testdata/tpch/plan_sf0.02/customer/12.parquet], [/testdata/tpch/plan_sf0.02/customer/13.parquet], ...]}, projection=[c_custkey], file_type=parquet + │ FilterExec: o_comment@2 NOT LIKE %special%requests%, projection=[o_orderkey@0, o_custkey@1] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/orders/1.parquet], [/testdata/tpch/plan_sf0.02/orders/10.parquet], [/testdata/tpch/plan_sf0.02/orders/11.parquet], [/testdata/tpch/plan_sf0.02/orders/12.parquet], [/testdata/tpch/plan_sf0.02/orders/13.parquet], ...]}, projection=[o_orderkey, o_custkey, o_comment], file_type=parquet, predicate=o_comment@8 NOT LIKE %special%requests% + └────────────────────────────────────────────────── + "); + Ok(()) + } + + #[tokio::test] + async fn test_tpch_14() -> Result<(), Box> { + let plan = test_tpch_query("q14").await?; + assert_snapshot!(plan, @r#" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ ProjectionExec: expr=[100 * CAST(sum(CASE WHEN part.p_type LIKE Utf8("PROMO%") THEN lineitem.l_extendedprice * Int64(1) - lineitem.l_discount ELSE Int64(0) END)@0 AS Float64) / CAST(sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@1 AS Float64) as promo_revenue] + │ AggregateExec: mode=Final, gby=[], aggr=[sum(CASE WHEN part.p_type LIKE Utf8("PROMO%") THEN lineitem.l_extendedprice * Int64(1) - lineitem.l_discount ELSE Int64(0) END), sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=16, input_tasks=4 + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p3] t1:[p4..p7] t2:[p8..p11] t3:[p12..p15] + │ AggregateExec: mode=Partial, gby=[], aggr=[sum(CASE WHEN part.p_type LIKE Utf8("PROMO%") THEN lineitem.l_extendedprice * Int64(1) - lineitem.l_discount ELSE Int64(0) END), sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] + │ ProjectionExec: expr=[l_extendedprice@1 * (Some(1),20,0 - l_discount@2) as __common_expr_1, p_type@0 as p_type] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(p_partkey@0, l_partkey@0)], projection=[p_type@1, l_extendedprice@3, l_discount@4] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=4, stage_partitions=16, input_tasks=4 + │ FilterExec: l_shipdate@3 >= 1995-09-01 AND l_shipdate@3 < 1995-10-01, projection=[l_partkey@0, l_extendedprice@1, l_discount@2] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet], [/testdata/tpch/plan_sf0.02/lineitem/10.parquet], [/testdata/tpch/plan_sf0.02/lineitem/11.parquet], [/testdata/tpch/plan_sf0.02/lineitem/12.parquet], [/testdata/tpch/plan_sf0.02/lineitem/13.parquet], ...]}, projection=[l_partkey, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@10 >= 1995-09-01 AND l_shipdate@10 < 1995-10-01 AND DynamicFilter [ empty ], pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1995-09-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 < 1995-10-01, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p15] t1:[p16..p31] t2:[p32..p47] t3:[p48..p63] + │ BroadcastExec: input_partitions=4, consumer_tasks=4, output_partitions=16 + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/part/1.parquet], [/testdata/tpch/plan_sf0.02/part/10.parquet], [/testdata/tpch/plan_sf0.02/part/11.parquet], [/testdata/tpch/plan_sf0.02/part/12.parquet], [/testdata/tpch/plan_sf0.02/part/13.parquet], ...]}, projection=[p_partkey, p_type], file_type=parquet + └────────────────────────────────────────────────── + "#); + Ok(()) + } + + #[tokio::test] + async fn test_tpch_15() -> Result<(), Box> { + let plan = test_tpch_query("q15").await?; + assert_snapshot!(plan, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [s_suppkey@0 ASC NULLS LAST] + │ [Stage 6] => NetworkCoalesceExec: output_partitions=24, input_tasks=4 + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] t3:[p0..p5] + │ SortExec: expr=[s_suppkey@0 ASC NULLS LAST], preserve_partitioning=[true] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(max(revenue0.total_revenue)@0, total_revenue@4)], projection=[s_suppkey@1, s_name@2, s_address@3, s_phone@4, total_revenue@5] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=1, stage_partitions=4, input_tasks=1 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_suppkey@0, supplier_no@0)], projection=[s_suppkey@0, s_name@1, s_address@2, s_phone@3, total_revenue@5] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=4, stage_partitions=16, input_tasks=4 + │ ProjectionExec: expr=[l_suppkey@0 as supplier_no, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@1 as total_revenue] + │ AggregateExec: mode=FinalPartitioned, gby=[l_suppkey@0 as l_suppkey], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] + │ [Stage 5] => NetworkShuffleExec: output_partitions=6, input_tasks=4 + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p3] + │ BroadcastExec: input_partitions=1, consumer_tasks=4, output_partitions=4 + │ AggregateExec: mode=Final, gby=[], aggr=[max(revenue0.total_revenue)] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=12, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ AggregateExec: mode=Partial, gby=[], aggr=[max(revenue0.total_revenue)] + │ ProjectionExec: expr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@1 as total_revenue] + │ AggregateExec: mode=FinalPartitioned, gby=[l_suppkey@0 as l_suppkey], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] + │ [Stage 1] => NetworkShuffleExec: output_partitions=6, input_tasks=4 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p11] t1:[p0..p11] t2:[p0..p11] t3:[p0..p11] + │ RepartitionExec: partitioning=Hash([l_suppkey@0], 12), input_partitions=4 + │ AggregateExec: mode=Partial, gby=[l_suppkey@0 as l_suppkey], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] + │ FilterExec: l_shipdate@3 >= 1996-01-01 AND l_shipdate@3 < 1996-04-01, projection=[l_suppkey@0, l_extendedprice@1, l_discount@2] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet], [/testdata/tpch/plan_sf0.02/lineitem/10.parquet], [/testdata/tpch/plan_sf0.02/lineitem/11.parquet], [/testdata/tpch/plan_sf0.02/lineitem/12.parquet], [/testdata/tpch/plan_sf0.02/lineitem/13.parquet], ...]}, projection=[l_suppkey, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@10 >= 1996-01-01 AND l_shipdate@10 < 1996-04-01, pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1996-01-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 < 1996-04-01, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p15] t1:[p16..p31] t2:[p32..p47] t3:[p48..p63] + │ BroadcastExec: input_partitions=4, consumer_tasks=4, output_partitions=16 + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/supplier/1.parquet], [/testdata/tpch/plan_sf0.02/supplier/10.parquet], [/testdata/tpch/plan_sf0.02/supplier/11.parquet], [/testdata/tpch/plan_sf0.02/supplier/12.parquet], [/testdata/tpch/plan_sf0.02/supplier/13.parquet], ...]}, projection=[s_suppkey, s_name, s_address, s_phone], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p23] t1:[p0..p23] t2:[p0..p23] t3:[p0..p23] + │ RepartitionExec: partitioning=Hash([l_suppkey@0], 24), input_partitions=4 + │ AggregateExec: mode=Partial, gby=[l_suppkey@0 as l_suppkey], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] + │ FilterExec: l_shipdate@3 >= 1996-01-01 AND l_shipdate@3 < 1996-04-01, projection=[l_suppkey@0, l_extendedprice@1, l_discount@2] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet], [/testdata/tpch/plan_sf0.02/lineitem/10.parquet], [/testdata/tpch/plan_sf0.02/lineitem/11.parquet], [/testdata/tpch/plan_sf0.02/lineitem/12.parquet], [/testdata/tpch/plan_sf0.02/lineitem/13.parquet], ...]}, projection=[l_suppkey, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@10 >= 1996-01-01 AND l_shipdate@10 < 1996-04-01, pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1996-01-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 < 1996-04-01, required_guarantees=[] + └────────────────────────────────────────────────── + "); + Ok(()) + } + + #[tokio::test] + async fn test_tpch_16() -> Result<(), Box> { + let plan = test_tpch_query("q16").await?; + assert_snapshot!(plan, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [supplier_cnt@3 DESC, p_brand@0 ASC NULLS LAST, p_type@1 ASC NULLS LAST, p_size@2 ASC NULLS LAST] + │ [Stage 5] => NetworkCoalesceExec: output_partitions=12, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p5] t1:[p0..p5] + │ SortExec: expr=[supplier_cnt@3 DESC, p_brand@0 ASC NULLS LAST, p_type@1 ASC NULLS LAST, p_size@2 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[p_brand@0 as p_brand, p_type@1 as p_type, p_size@2 as p_size, count(alias1)@3 as supplier_cnt] + │ AggregateExec: mode=FinalPartitioned, gby=[p_brand@0 as p_brand, p_type@1 as p_type, p_size@2 as p_size], aggr=[count(alias1)] + │ [Stage 4] => NetworkShuffleExec: output_partitions=6, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p11] t1:[p0..p11] t2:[p0..p11] + │ RepartitionExec: partitioning=Hash([p_brand@0, p_type@1, p_size@2], 12), input_partitions=6 + │ AggregateExec: mode=Partial, gby=[p_brand@0 as p_brand, p_type@1 as p_type, p_size@2 as p_size], aggr=[count(alias1)] + │ AggregateExec: mode=FinalPartitioned, gby=[p_brand@0 as p_brand, p_type@1 as p_type, p_size@2 as p_size, alias1@3 as alias1], aggr=[] + │ [Stage 3] => NetworkShuffleExec: output_partitions=6, input_tasks=4 + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p17] t1:[p0..p17] t2:[p0..p17] t3:[p0..p17] + │ RepartitionExec: partitioning=Hash([p_brand@0, p_type@1, p_size@2, alias1@3], 18), input_partitions=4 + │ AggregateExec: mode=Partial, gby=[p_brand@1 as p_brand, p_type@2 as p_type, p_size@3 as p_size, ps_suppkey@0 as alias1], aggr=[] + │ HashJoinExec: mode=CollectLeft, join_type=RightAnti, on=[(s_suppkey@0, ps_suppkey@0)] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=4, stage_partitions=16, input_tasks=4 + │ ProjectionExec: expr=[ps_suppkey@3 as ps_suppkey, p_brand@0 as p_brand, p_type@1 as p_type, p_size@2 as p_size] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(p_partkey@0, ps_partkey@0)], projection=[p_brand@1, p_type@2, p_size@3, ps_suppkey@5] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=4, stage_partitions=16, input_tasks=4 + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/partsupp/1.parquet], [/testdata/tpch/plan_sf0.02/partsupp/10.parquet], [/testdata/tpch/plan_sf0.02/partsupp/11.parquet], [/testdata/tpch/plan_sf0.02/partsupp/12.parquet], [/testdata/tpch/plan_sf0.02/partsupp/13.parquet], ...]}, projection=[ps_partkey, ps_suppkey], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p15] t1:[p16..p31] t2:[p32..p47] t3:[p48..p63] + │ BroadcastExec: input_partitions=4, consumer_tasks=4, output_partitions=16 + │ FilterExec: s_comment@1 LIKE %Customer%Complaints%, projection=[s_suppkey@0] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/supplier/1.parquet], [/testdata/tpch/plan_sf0.02/supplier/10.parquet], [/testdata/tpch/plan_sf0.02/supplier/11.parquet], [/testdata/tpch/plan_sf0.02/supplier/12.parquet], [/testdata/tpch/plan_sf0.02/supplier/13.parquet], ...]}, projection=[s_suppkey, s_comment], file_type=parquet, predicate=s_comment@6 LIKE %Customer%Complaints% + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p15] t1:[p16..p31] t2:[p32..p47] t3:[p48..p63] + │ BroadcastExec: input_partitions=4, consumer_tasks=4, output_partitions=16 + │ FilterExec: p_brand@1 != Brand#45 AND p_type@2 NOT LIKE MEDIUM POLISHED% AND p_size@3 IN (SET) ([49, 14, 23, 45, 19, 3, 36, 9]) + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/part/1.parquet], [/testdata/tpch/plan_sf0.02/part/10.parquet], [/testdata/tpch/plan_sf0.02/part/11.parquet], [/testdata/tpch/plan_sf0.02/part/12.parquet], [/testdata/tpch/plan_sf0.02/part/13.parquet], ...]}, projection=[p_partkey, p_brand, p_type, p_size], file_type=parquet, predicate=p_brand@3 != Brand#45 AND p_type@4 NOT LIKE MEDIUM POLISHED% AND p_size@5 IN (SET) ([49, 14, 23, 45, 19, 3, 36, 9]), pruning_predicate=p_brand_null_count@2 != row_count@3 AND (p_brand_min@0 != Brand#45 OR Brand#45 != p_brand_max@1) AND p_type_null_count@6 != row_count@3 AND (p_type_min@4 NOT LIKE MEDIUM POLISHED% OR p_type_max@5 NOT LIKE MEDIUM POLISHED%) AND (p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 49 AND 49 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 14 AND 14 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 23 AND 23 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 45 AND 45 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 19 AND 19 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 3 AND 3 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 36 AND 36 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 9 AND 9 <= p_size_max@8), required_guarantees=[p_brand not in (Brand#45), p_size in (14, 19, 23, 3, 36, 45, 49, 9)] + └────────────────────────────────────────────────── + "); + Ok(()) + } + + #[tokio::test] + async fn test_tpch_17() -> Result<(), Box> { + let plan = test_tpch_query("q17").await?; + assert_snapshot!(plan, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ ProjectionExec: expr=[CAST(sum(lineitem.l_extendedprice)@0 AS Float64) / 7 as avg_yearly] + │ AggregateExec: mode=Final, gby=[], aggr=[sum(lineitem.l_extendedprice)] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkCoalesceExec: output_partitions=24, input_tasks=4 + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] t3:[p0..p5] + │ AggregateExec: mode=Partial, gby=[], aggr=[sum(lineitem.l_extendedprice)] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(p_partkey@2, l_partkey@1)], filter=CAST(l_quantity@0 AS Decimal128(30, 15)) < Float64(0.2) * avg(lineitem.l_quantity)@1, projection=[l_extendedprice@1] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=4, stage_partitions=16, input_tasks=4 + │ ProjectionExec: expr=[CAST(0.2 * CAST(avg(lineitem.l_quantity)@1 AS Float64) AS Decimal128(30, 15)) as Float64(0.2) * avg(lineitem.l_quantity), l_partkey@0 as l_partkey] + │ AggregateExec: mode=FinalPartitioned, gby=[l_partkey@0 as l_partkey], aggr=[avg(lineitem.l_quantity)] + │ [Stage 3] => NetworkShuffleExec: output_partitions=6, input_tasks=4 + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p15] t1:[p16..p31] t2:[p32..p47] t3:[p48..p63] + │ BroadcastExec: input_partitions=4, consumer_tasks=4, output_partitions=16 + │ ProjectionExec: expr=[l_quantity@1 as l_quantity, l_extendedprice@2 as l_extendedprice, p_partkey@0 as p_partkey] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(p_partkey@0, l_partkey@0)], projection=[p_partkey@0, l_quantity@2, l_extendedprice@3] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=4, stage_partitions=16, input_tasks=4 + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet], [/testdata/tpch/plan_sf0.02/lineitem/10.parquet], [/testdata/tpch/plan_sf0.02/lineitem/11.parquet], [/testdata/tpch/plan_sf0.02/lineitem/12.parquet], [/testdata/tpch/plan_sf0.02/lineitem/13.parquet], ...]}, projection=[l_partkey, l_quantity, l_extendedprice], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p15] t1:[p16..p31] t2:[p32..p47] t3:[p48..p63] + │ BroadcastExec: input_partitions=4, consumer_tasks=4, output_partitions=16 + │ FilterExec: p_brand@1 = Brand#23 AND p_container@2 = MED BOX, projection=[p_partkey@0] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/part/1.parquet], [/testdata/tpch/plan_sf0.02/part/10.parquet], [/testdata/tpch/plan_sf0.02/part/11.parquet], [/testdata/tpch/plan_sf0.02/part/12.parquet], [/testdata/tpch/plan_sf0.02/part/13.parquet], ...]}, projection=[p_partkey, p_brand, p_container], file_type=parquet, predicate=p_brand@3 = Brand#23 AND p_container@6 = MED BOX, pruning_predicate=p_brand_null_count@2 != row_count@3 AND p_brand_min@0 <= Brand#23 AND Brand#23 <= p_brand_max@1 AND p_container_null_count@6 != row_count@3 AND p_container_min@4 <= MED BOX AND MED BOX <= p_container_max@5, required_guarantees=[p_brand in (Brand#23), p_container in (MED BOX)] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p23] t1:[p0..p23] t2:[p0..p23] t3:[p0..p23] + │ RepartitionExec: partitioning=Hash([l_partkey@0], 24), input_partitions=4 + │ AggregateExec: mode=Partial, gby=[l_partkey@0 as l_partkey], aggr=[avg(lineitem.l_quantity)] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet], [/testdata/tpch/plan_sf0.02/lineitem/10.parquet], [/testdata/tpch/plan_sf0.02/lineitem/11.parquet], [/testdata/tpch/plan_sf0.02/lineitem/12.parquet], [/testdata/tpch/plan_sf0.02/lineitem/13.parquet], ...]}, projection=[l_partkey, l_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + "); + Ok(()) + } + + #[tokio::test] + async fn test_tpch_18() -> Result<(), Box> { + let plan = test_tpch_query("q18").await?; + assert_snapshot!(plan, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [o_totalprice@4 DESC, o_orderdate@3 ASC NULLS LAST] + │ [Stage 6] => NetworkCoalesceExec: output_partitions=18, input_tasks=3 + └────────────────────────────────────────────────── + ┌───── Stage 6 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] + │ SortExec: expr=[o_totalprice@4 DESC, o_orderdate@3 ASC NULLS LAST], preserve_partitioning=[true] + │ AggregateExec: mode=FinalPartitioned, gby=[c_name@0 as c_name, c_custkey@1 as c_custkey, o_orderkey@2 as o_orderkey, o_orderdate@3 as o_orderdate, o_totalprice@4 as o_totalprice], aggr=[sum(lineitem.l_quantity)] + │ [Stage 5] => NetworkShuffleExec: output_partitions=6, input_tasks=4 + └────────────────────────────────────────────────── + ┌───── Stage 5 ── Tasks: t0:[p0..p17] t1:[p0..p17] t2:[p0..p17] t3:[p0..p17] + │ RepartitionExec: partitioning=Hash([c_name@0, c_custkey@1, o_orderkey@2, o_orderdate@3, o_totalprice@4], 18), input_partitions=4 + │ AggregateExec: mode=Partial, gby=[c_name@1 as c_name, c_custkey@0 as c_custkey, o_orderkey@2 as o_orderkey, o_orderdate@4 as o_orderdate, o_totalprice@3 as o_totalprice], aggr=[sum(lineitem.l_quantity)] + │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(l_orderkey@0, o_orderkey@2)] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=6, stage_partitions=24, input_tasks=3 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(o_orderkey@2, l_orderkey@0)], projection=[c_custkey@0, c_name@1, o_orderkey@2, o_totalprice@3, o_orderdate@4, l_quantity@6] + │ CoalescePartitionsExec + │ [Stage 4] => NetworkBroadcastExec: partitions_per_consumer=4, stage_partitions=16, input_tasks=4 + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet], [/testdata/tpch/plan_sf0.02/lineitem/10.parquet], [/testdata/tpch/plan_sf0.02/lineitem/11.parquet], [/testdata/tpch/plan_sf0.02/lineitem/12.parquet], [/testdata/tpch/plan_sf0.02/lineitem/13.parquet], ...]}, projection=[l_orderkey, l_quantity], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p23] t1:[p24..p47] t2:[p48..p71] + │ BroadcastExec: input_partitions=6, consumer_tasks=4, output_partitions=24 + │ FilterExec: sum(lineitem.l_quantity)@1 > Some(30000),25,2, projection=[l_orderkey@0] + │ AggregateExec: mode=FinalPartitioned, gby=[l_orderkey@0 as l_orderkey], aggr=[sum(lineitem.l_quantity)] + │ [Stage 1] => NetworkShuffleExec: output_partitions=6, input_tasks=4 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p17] t1:[p0..p17] t2:[p0..p17] t3:[p0..p17] + │ RepartitionExec: partitioning=Hash([l_orderkey@0], 18), input_partitions=4 + │ AggregateExec: mode=Partial, gby=[l_orderkey@0 as l_orderkey], aggr=[sum(lineitem.l_quantity)] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet], [/testdata/tpch/plan_sf0.02/lineitem/10.parquet], [/testdata/tpch/plan_sf0.02/lineitem/11.parquet], [/testdata/tpch/plan_sf0.02/lineitem/12.parquet], [/testdata/tpch/plan_sf0.02/lineitem/13.parquet], ...]}, projection=[l_orderkey, l_quantity], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p15] t1:[p16..p31] t2:[p32..p47] t3:[p48..p63] + │ BroadcastExec: input_partitions=4, consumer_tasks=4, output_partitions=16 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_custkey@0, o_custkey@1)], projection=[c_custkey@0, c_name@1, o_orderkey@2, o_totalprice@4, o_orderdate@5] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=4, stage_partitions=16, input_tasks=4 + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/orders/1.parquet], [/testdata/tpch/plan_sf0.02/orders/10.parquet], [/testdata/tpch/plan_sf0.02/orders/11.parquet], [/testdata/tpch/plan_sf0.02/orders/12.parquet], [/testdata/tpch/plan_sf0.02/orders/13.parquet], ...]}, projection=[o_orderkey, o_custkey, o_totalprice, o_orderdate], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p15] t1:[p16..p31] t2:[p32..p47] t3:[p48..p63] + │ BroadcastExec: input_partitions=4, consumer_tasks=4, output_partitions=16 + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/customer/1.parquet], [/testdata/tpch/plan_sf0.02/customer/10.parquet], [/testdata/tpch/plan_sf0.02/customer/11.parquet], [/testdata/tpch/plan_sf0.02/customer/12.parquet], [/testdata/tpch/plan_sf0.02/customer/13.parquet], ...]}, projection=[c_custkey, c_name], file_type=parquet + └────────────────────────────────────────────────── + "); + Ok(()) + } + + #[tokio::test] + async fn test_tpch_19() -> Result<(), Box> { + let plan = test_tpch_query("q19").await?; + assert_snapshot!(plan, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ ProjectionExec: expr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@0 as revenue] + │ AggregateExec: mode=Final, gby=[], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=16, input_tasks=4 + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p3] t1:[p4..p7] t2:[p8..p11] t3:[p12..p15] + │ AggregateExec: mode=Partial, gby=[], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(p_partkey@0, l_partkey@0)], filter=p_brand@1 = Brand#12 AND p_container@3 IN (SET) ([SM CASE, SM BOX, SM PACK, SM PKG]) AND l_quantity@0 >= Some(100),15,2 AND l_quantity@0 <= Some(1100),15,2 AND p_size@2 <= 5 OR p_brand@1 = Brand#23 AND p_container@3 IN (SET) ([MED BAG, MED BOX, MED PKG, MED PACK]) AND l_quantity@0 >= Some(1000),15,2 AND l_quantity@0 <= Some(2000),15,2 AND p_size@2 <= 10 OR p_brand@1 = Brand#34 AND p_container@3 IN (SET) ([LG CASE, LG BOX, LG PACK, LG PKG]) AND l_quantity@0 >= Some(2000),15,2 AND l_quantity@0 <= Some(3000),15,2 AND p_size@2 <= 15, projection=[l_extendedprice@6, l_discount@7] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=4, stage_partitions=16, input_tasks=4 + │ FilterExec: (l_quantity@1 >= Some(100),15,2 AND l_quantity@1 <= Some(1100),15,2 OR l_quantity@1 >= Some(1000),15,2 AND l_quantity@1 <= Some(2000),15,2 OR l_quantity@1 >= Some(2000),15,2 AND l_quantity@1 <= Some(3000),15,2) AND (l_shipmode@5 = AIR OR l_shipmode@5 = AIR REG) AND l_shipinstruct@4 = DELIVER IN PERSON, projection=[l_partkey@0, l_quantity@1, l_extendedprice@2, l_discount@3] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet], [/testdata/tpch/plan_sf0.02/lineitem/10.parquet], [/testdata/tpch/plan_sf0.02/lineitem/11.parquet], [/testdata/tpch/plan_sf0.02/lineitem/12.parquet], [/testdata/tpch/plan_sf0.02/lineitem/13.parquet], ...]}, projection=[l_partkey, l_quantity, l_extendedprice, l_discount, l_shipinstruct, l_shipmode], file_type=parquet, predicate=(l_quantity@4 >= Some(100),15,2 AND l_quantity@4 <= Some(1100),15,2 OR l_quantity@4 >= Some(1000),15,2 AND l_quantity@4 <= Some(2000),15,2 OR l_quantity@4 >= Some(2000),15,2 AND l_quantity@4 <= Some(3000),15,2) AND (l_shipmode@14 = AIR OR l_shipmode@14 = AIR REG) AND l_shipinstruct@13 = DELIVER IN PERSON AND DynamicFilter [ empty ], pruning_predicate=(l_quantity_null_count@1 != row_count@2 AND l_quantity_max@0 >= Some(100),15,2 AND l_quantity_null_count@1 != row_count@2 AND l_quantity_min@3 <= Some(1100),15,2 OR l_quantity_null_count@1 != row_count@2 AND l_quantity_max@0 >= Some(1000),15,2 AND l_quantity_null_count@1 != row_count@2 AND l_quantity_min@3 <= Some(2000),15,2 OR l_quantity_null_count@1 != row_count@2 AND l_quantity_max@0 >= Some(2000),15,2 AND l_quantity_null_count@1 != row_count@2 AND l_quantity_min@3 <= Some(3000),15,2) AND (l_shipmode_null_count@6 != row_count@2 AND l_shipmode_min@4 <= AIR AND AIR <= l_shipmode_max@5 OR l_shipmode_null_count@6 != row_count@2 AND l_shipmode_min@4 <= AIR REG AND AIR REG <= l_shipmode_max@5) AND l_shipinstruct_null_count@9 != row_count@2 AND l_shipinstruct_min@7 <= DELIVER IN PERSON AND DELIVER IN PERSON <= l_shipinstruct_max@8, required_guarantees=[l_shipinstruct in (DELIVER IN PERSON), l_shipmode in (AIR, AIR REG)] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p15] t1:[p16..p31] t2:[p32..p47] t3:[p48..p63] + │ BroadcastExec: input_partitions=4, consumer_tasks=4, output_partitions=16 + │ FilterExec: (p_brand@1 = Brand#12 AND p_container@3 IN (SET) ([SM CASE, SM BOX, SM PACK, SM PKG]) AND p_size@2 <= 5 OR p_brand@1 = Brand#23 AND p_container@3 IN (SET) ([MED BAG, MED BOX, MED PKG, MED PACK]) AND p_size@2 <= 10 OR p_brand@1 = Brand#34 AND p_container@3 IN (SET) ([LG CASE, LG BOX, LG PACK, LG PKG]) AND p_size@2 <= 15) AND p_size@2 >= 1 + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/part/1.parquet], [/testdata/tpch/plan_sf0.02/part/10.parquet], [/testdata/tpch/plan_sf0.02/part/11.parquet], [/testdata/tpch/plan_sf0.02/part/12.parquet], [/testdata/tpch/plan_sf0.02/part/13.parquet], ...]}, projection=[p_partkey, p_brand, p_size, p_container], file_type=parquet, predicate=(p_brand@3 = Brand#12 AND p_container@6 IN (SET) ([SM CASE, SM BOX, SM PACK, SM PKG]) AND p_size@5 <= 5 OR p_brand@3 = Brand#23 AND p_container@6 IN (SET) ([MED BAG, MED BOX, MED PKG, MED PACK]) AND p_size@5 <= 10 OR p_brand@3 = Brand#34 AND p_container@6 IN (SET) ([LG CASE, LG BOX, LG PACK, LG PKG]) AND p_size@5 <= 15) AND p_size@5 >= 1, pruning_predicate=(p_brand_null_count@2 != row_count@3 AND p_brand_min@0 <= Brand#12 AND Brand#12 <= p_brand_max@1 AND (p_container_null_count@6 != row_count@3 AND p_container_min@4 <= SM CASE AND SM CASE <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= SM BOX AND SM BOX <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= SM PACK AND SM PACK <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= SM PKG AND SM PKG <= p_container_max@5) AND p_size_null_count@8 != row_count@3 AND p_size_min@7 <= 5 OR p_brand_null_count@2 != row_count@3 AND p_brand_min@0 <= Brand#23 AND Brand#23 <= p_brand_max@1 AND (p_container_null_count@6 != row_count@3 AND p_container_min@4 <= MED BAG AND MED BAG <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= MED BOX AND MED BOX <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= MED PKG AND MED PKG <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= MED PACK AND MED PACK <= p_container_max@5) AND p_size_null_count@8 != row_count@3 AND p_size_min@7 <= 10 OR p_brand_null_count@2 != row_count@3 AND p_brand_min@0 <= Brand#34 AND Brand#34 <= p_brand_max@1 AND (p_container_null_count@6 != row_count@3 AND p_container_min@4 <= LG CASE AND LG CASE <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= LG BOX AND LG BOX <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= LG PACK AND LG PACK <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= LG PKG AND LG PKG <= p_container_max@5) AND p_size_null_count@8 != row_count@3 AND p_size_min@7 <= 15) AND p_size_null_count@8 != row_count@3 AND p_size_max@9 >= 1, required_guarantees=[p_brand in (Brand#12, Brand#23, Brand#34), p_container in (LG BOX, LG CASE, LG PACK, LG PKG, MED BAG, MED BOX, MED PACK, MED PKG, SM BOX, SM CASE, SM PACK, SM PKG)] + └────────────────────────────────────────────────── + "); + Ok(()) + } + + #[tokio::test] + async fn test_tpch_20() -> Result<(), Box> { + let plan = test_tpch_query("q20").await?; + assert_snapshot!(plan, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [s_name@0 ASC NULLS LAST] + │ SortExec: expr=[s_name@0 ASC NULLS LAST], preserve_partitioning=[true] + │ HashJoinExec: mode=CollectLeft, join_type=LeftSemi, on=[(s_suppkey@0, ps_suppkey@0)], projection=[s_name@1, s_address@2] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=16, input_tasks=4 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ps_partkey@0, l_partkey@1), (ps_suppkey@1, l_suppkey@2)], filter=CAST(ps_availqty@0 AS Float64) > Float64(0.5) * sum(lineitem.l_quantity)@1, projection=[ps_suppkey@1] + │ CoalescePartitionsExec + │ BroadcastExec: input_partitions=6, consumer_tasks=1, output_partitions=6 + │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(p_partkey@0, ps_partkey@0)] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=4, stage_partitions=4, input_tasks=4 + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/partsupp/1.parquet, /testdata/tpch/plan_sf0.02/partsupp/10.parquet, /testdata/tpch/plan_sf0.02/partsupp/11.parquet], [/testdata/tpch/plan_sf0.02/partsupp/12.parquet, /testdata/tpch/plan_sf0.02/partsupp/13.parquet, /testdata/tpch/plan_sf0.02/partsupp/14.parquet], [/testdata/tpch/plan_sf0.02/partsupp/15.parquet, /testdata/tpch/plan_sf0.02/partsupp/16.parquet, /testdata/tpch/plan_sf0.02/partsupp/2.parquet], [/testdata/tpch/plan_sf0.02/partsupp/3.parquet, /testdata/tpch/plan_sf0.02/partsupp/4.parquet, /testdata/tpch/plan_sf0.02/partsupp/5.parquet], [/testdata/tpch/plan_sf0.02/partsupp/6.parquet, /testdata/tpch/plan_sf0.02/partsupp/7.parquet, /testdata/tpch/plan_sf0.02/partsupp/8.parquet], ...]}, projection=[ps_partkey, ps_suppkey, ps_availqty], file_type=parquet + │ ProjectionExec: expr=[0.5 * CAST(sum(lineitem.l_quantity)@2 AS Float64) as Float64(0.5) * sum(lineitem.l_quantity), l_partkey@0 as l_partkey, l_suppkey@1 as l_suppkey] + │ AggregateExec: mode=FinalPartitioned, gby=[l_partkey@0 as l_partkey, l_suppkey@1 as l_suppkey], aggr=[sum(lineitem.l_quantity)] + │ [Stage 4] => NetworkShuffleExec: output_partitions=6, input_tasks=4 + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p3] t1:[p4..p7] t2:[p8..p11] t3:[p12..p15] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@3)], projection=[s_suppkey@1, s_name@2, s_address@3] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=4, stage_partitions=16, input_tasks=4 + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/supplier/1.parquet], [/testdata/tpch/plan_sf0.02/supplier/10.parquet], [/testdata/tpch/plan_sf0.02/supplier/11.parquet], [/testdata/tpch/plan_sf0.02/supplier/12.parquet], [/testdata/tpch/plan_sf0.02/supplier/13.parquet], ...]}, projection=[s_suppkey, s_name, s_address, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p15] t1:[p16..p31] t2:[p32..p47] t3:[p48..p63] + │ BroadcastExec: input_partitions=4, consumer_tasks=4, output_partitions=16 + │ FilterExec: n_name@1 = CANADA, projection=[n_nationkey@0] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/nation/1.parquet], [/testdata/tpch/plan_sf0.02/nation/10.parquet], [/testdata/tpch/plan_sf0.02/nation/11.parquet], [/testdata/tpch/plan_sf0.02/nation/12.parquet], [/testdata/tpch/plan_sf0.02/nation/13.parquet], ...]}, projection=[n_nationkey, n_name], file_type=parquet, predicate=n_name@1 = CANADA, pruning_predicate=n_name_null_count@2 != row_count@3 AND n_name_min@0 <= CANADA AND CANADA <= n_name_max@1, required_guarantees=[n_name in (CANADA)] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p3] t1:[p4..p7] t2:[p8..p11] t3:[p12..p15] + │ BroadcastExec: input_partitions=4, consumer_tasks=1, output_partitions=4 + │ FilterExec: p_name@1 LIKE forest%, projection=[p_partkey@0] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/part/1.parquet], [/testdata/tpch/plan_sf0.02/part/10.parquet], [/testdata/tpch/plan_sf0.02/part/11.parquet], [/testdata/tpch/plan_sf0.02/part/12.parquet], [/testdata/tpch/plan_sf0.02/part/13.parquet], ...]}, projection=[p_partkey, p_name], file_type=parquet, predicate=p_name@1 LIKE forest%, pruning_predicate=p_name_null_count@2 != row_count@3 AND p_name_min@0 <= foresu AND forest <= p_name_max@1, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p5] t1:[p0..p5] t2:[p0..p5] t3:[p0..p5] + │ RepartitionExec: partitioning=Hash([l_partkey@0, l_suppkey@1], 6), input_partitions=4 + │ AggregateExec: mode=Partial, gby=[l_partkey@0 as l_partkey, l_suppkey@1 as l_suppkey], aggr=[sum(lineitem.l_quantity)] + │ FilterExec: l_shipdate@3 >= 1994-01-01 AND l_shipdate@3 < 1995-01-01, projection=[l_partkey@0, l_suppkey@1, l_quantity@2] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet], [/testdata/tpch/plan_sf0.02/lineitem/10.parquet], [/testdata/tpch/plan_sf0.02/lineitem/11.parquet], [/testdata/tpch/plan_sf0.02/lineitem/12.parquet], [/testdata/tpch/plan_sf0.02/lineitem/13.parquet], ...]}, projection=[l_partkey, l_suppkey, l_quantity, l_shipdate], file_type=parquet, predicate=l_shipdate@10 >= 1994-01-01 AND l_shipdate@10 < 1995-01-01 AND DynamicFilter [ empty ], pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1994-01-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 < 1995-01-01, required_guarantees=[] + └────────────────────────────────────────────────── + "); + Ok(()) + } + + #[tokio::test] + async fn test_tpch_21() -> Result<(), Box> { + let plan = test_tpch_query("q21").await?; + assert_snapshot!(plan, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [numwait@1 DESC, s_name@0 ASC NULLS LAST] + │ SortExec: expr=[numwait@1 DESC, s_name@0 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[s_name@0 as s_name, count(Int64(1))@1 as numwait] + │ AggregateExec: mode=FinalPartitioned, gby=[s_name@0 as s_name], aggr=[count(Int64(1))] + │ RepartitionExec: partitioning=Hash([s_name@0], 6), input_partitions=6 + │ AggregateExec: mode=Partial, gby=[s_name@0 as s_name], aggr=[count(Int64(1))] + │ HashJoinExec: mode=CollectLeft, join_type=LeftAnti, on=[(l_orderkey@1, l_orderkey@0)], filter=l_suppkey@1 != l_suppkey@0, projection=[s_name@0] + │ CoalescePartitionsExec + │ HashJoinExec: mode=CollectLeft, join_type=LeftSemi, on=[(l_orderkey@1, l_orderkey@0)], filter=l_suppkey@1 != l_suppkey@0 + │ CoalescePartitionsExec + │ [Stage 4] => NetworkCoalesceExec: output_partitions=16, input_tasks=4 + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet, /testdata/tpch/plan_sf0.02/lineitem/10.parquet, /testdata/tpch/plan_sf0.02/lineitem/11.parquet], [/testdata/tpch/plan_sf0.02/lineitem/12.parquet, /testdata/tpch/plan_sf0.02/lineitem/13.parquet, /testdata/tpch/plan_sf0.02/lineitem/14.parquet], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet, /testdata/tpch/plan_sf0.02/lineitem/16.parquet, /testdata/tpch/plan_sf0.02/lineitem/2.parquet], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet, /testdata/tpch/plan_sf0.02/lineitem/4.parquet, /testdata/tpch/plan_sf0.02/lineitem/5.parquet], [/testdata/tpch/plan_sf0.02/lineitem/6.parquet, /testdata/tpch/plan_sf0.02/lineitem/7.parquet, /testdata/tpch/plan_sf0.02/lineitem/8.parquet], ...]}, projection=[l_orderkey, l_suppkey], file_type=parquet + │ FilterExec: l_receiptdate@3 > l_commitdate@2, projection=[l_orderkey@0, l_suppkey@1] + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet, /testdata/tpch/plan_sf0.02/lineitem/10.parquet, /testdata/tpch/plan_sf0.02/lineitem/11.parquet], [/testdata/tpch/plan_sf0.02/lineitem/12.parquet, /testdata/tpch/plan_sf0.02/lineitem/13.parquet, /testdata/tpch/plan_sf0.02/lineitem/14.parquet], [/testdata/tpch/plan_sf0.02/lineitem/15.parquet, /testdata/tpch/plan_sf0.02/lineitem/16.parquet, /testdata/tpch/plan_sf0.02/lineitem/2.parquet], [/testdata/tpch/plan_sf0.02/lineitem/3.parquet, /testdata/tpch/plan_sf0.02/lineitem/4.parquet, /testdata/tpch/plan_sf0.02/lineitem/5.parquet], [/testdata/tpch/plan_sf0.02/lineitem/6.parquet, /testdata/tpch/plan_sf0.02/lineitem/7.parquet, /testdata/tpch/plan_sf0.02/lineitem/8.parquet], ...]}, projection=[l_orderkey, l_suppkey, l_commitdate, l_receiptdate], file_type=parquet, predicate=l_receiptdate@12 > l_commitdate@11 + └────────────────────────────────────────────────── + ┌───── Stage 4 ── Tasks: t0:[p0..p3] t1:[p4..p7] t2:[p8..p11] t3:[p12..p15] + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@1)], projection=[s_name@1, l_orderkey@3, l_suppkey@4] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkBroadcastExec: partitions_per_consumer=4, stage_partitions=16, input_tasks=4 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(o_orderkey@0, l_orderkey@2)], projection=[s_name@1, s_nationkey@2, l_orderkey@3, l_suppkey@4] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkBroadcastExec: partitions_per_consumer=4, stage_partitions=16, input_tasks=4 + │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_suppkey@0, l_suppkey@1)], projection=[s_name@1, s_nationkey@2, l_orderkey@3, l_suppkey@4] + │ CoalescePartitionsExec + │ [Stage 3] => NetworkBroadcastExec: partitions_per_consumer=4, stage_partitions=16, input_tasks=4 + │ FilterExec: l_receiptdate@3 > l_commitdate@2, projection=[l_orderkey@0, l_suppkey@1] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/lineitem/1.parquet], [/testdata/tpch/plan_sf0.02/lineitem/10.parquet], [/testdata/tpch/plan_sf0.02/lineitem/11.parquet], [/testdata/tpch/plan_sf0.02/lineitem/12.parquet], [/testdata/tpch/plan_sf0.02/lineitem/13.parquet], ...]}, projection=[l_orderkey, l_suppkey, l_commitdate, l_receiptdate], file_type=parquet, predicate=l_receiptdate@12 > l_commitdate@11 AND DynamicFilter [ empty ] AND DynamicFilter [ empty ] + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p15] t1:[p16..p31] t2:[p32..p47] t3:[p48..p63] + │ BroadcastExec: input_partitions=4, consumer_tasks=4, output_partitions=16 + │ FilterExec: n_name@1 = SAUDI ARABIA, projection=[n_nationkey@0] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/nation/1.parquet], [/testdata/tpch/plan_sf0.02/nation/10.parquet], [/testdata/tpch/plan_sf0.02/nation/11.parquet], [/testdata/tpch/plan_sf0.02/nation/12.parquet], [/testdata/tpch/plan_sf0.02/nation/13.parquet], ...]}, projection=[n_nationkey, n_name], file_type=parquet, predicate=n_name@1 = SAUDI ARABIA, pruning_predicate=n_name_null_count@2 != row_count@3 AND n_name_min@0 <= SAUDI ARABIA AND SAUDI ARABIA <= n_name_max@1, required_guarantees=[n_name in (SAUDI ARABIA)] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p15] t1:[p16..p31] t2:[p32..p47] t3:[p48..p63] + │ BroadcastExec: input_partitions=4, consumer_tasks=4, output_partitions=16 + │ FilterExec: o_orderstatus@1 = F, projection=[o_orderkey@0] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/orders/1.parquet], [/testdata/tpch/plan_sf0.02/orders/10.parquet], [/testdata/tpch/plan_sf0.02/orders/11.parquet], [/testdata/tpch/plan_sf0.02/orders/12.parquet], [/testdata/tpch/plan_sf0.02/orders/13.parquet], ...]}, projection=[o_orderkey, o_orderstatus], file_type=parquet, predicate=o_orderstatus@2 = F, pruning_predicate=o_orderstatus_null_count@2 != row_count@3 AND o_orderstatus_min@0 <= F AND F <= o_orderstatus_max@1, required_guarantees=[o_orderstatus in (F)] + └────────────────────────────────────────────────── + ┌───── Stage 3 ── Tasks: t0:[p0..p15] t1:[p16..p31] t2:[p32..p47] t3:[p48..p63] + │ BroadcastExec: input_partitions=4, consumer_tasks=4, output_partitions=16 + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/supplier/1.parquet], [/testdata/tpch/plan_sf0.02/supplier/10.parquet], [/testdata/tpch/plan_sf0.02/supplier/11.parquet], [/testdata/tpch/plan_sf0.02/supplier/12.parquet], [/testdata/tpch/plan_sf0.02/supplier/13.parquet], ...]}, projection=[s_suppkey, s_name, s_nationkey], file_type=parquet, predicate=DynamicFilter [ empty ] + └────────────────────────────────────────────────── + "); + Ok(()) + } + + #[tokio::test] + async fn test_tpch_22() -> Result<(), Box> { + let plan = test_tpch_query("q22").await?; + assert_snapshot!(plan, @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ SortPreservingMergeExec: [cntrycode@0 ASC NULLS LAST] + │ SortExec: expr=[cntrycode@0 ASC NULLS LAST], preserve_partitioning=[true] + │ ProjectionExec: expr=[cntrycode@0 as cntrycode, count(Int64(1))@1 as numcust, sum(custsale.c_acctbal)@2 as totacctbal] + │ AggregateExec: mode=FinalPartitioned, gby=[cntrycode@0 as cntrycode], aggr=[count(Int64(1)), sum(custsale.c_acctbal)] + │ RepartitionExec: partitioning=Hash([cntrycode@0], 6), input_partitions=6 + │ AggregateExec: mode=Partial, gby=[cntrycode@0 as cntrycode], aggr=[count(Int64(1)), sum(custsale.c_acctbal)] + │ ProjectionExec: expr=[substr(c_phone@1, 1, 2) as cntrycode, c_acctbal@2 as c_acctbal] + │ NestedLoopJoinExec: join_type=Inner, filter=join_proj_push_down_1@1 > avg(customer.c_acctbal)@0, projection=[avg(customer.c_acctbal)@0, c_phone@1, c_acctbal@2] + │ AggregateExec: mode=Final, gby=[], aggr=[avg(customer.c_acctbal)] + │ CoalescePartitionsExec + │ [Stage 1] => NetworkCoalesceExec: output_partitions=16, input_tasks=4 + │ ProjectionExec: expr=[c_phone@0 as c_phone, c_acctbal@1 as c_acctbal, CAST(c_acctbal@1 AS Decimal128(19, 6)) as join_proj_push_down_1] + │ HashJoinExec: mode=CollectLeft, join_type=LeftAnti, on=[(c_custkey@0, o_custkey@0)], projection=[c_phone@1, c_acctbal@2] + │ CoalescePartitionsExec + │ [Stage 2] => NetworkCoalesceExec: output_partitions=16, input_tasks=4 + │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/plan_sf0.02/orders/1.parquet, /testdata/tpch/plan_sf0.02/orders/10.parquet, /testdata/tpch/plan_sf0.02/orders/11.parquet], [/testdata/tpch/plan_sf0.02/orders/12.parquet, /testdata/tpch/plan_sf0.02/orders/13.parquet, /testdata/tpch/plan_sf0.02/orders/14.parquet], [/testdata/tpch/plan_sf0.02/orders/15.parquet, /testdata/tpch/plan_sf0.02/orders/16.parquet, /testdata/tpch/plan_sf0.02/orders/2.parquet], [/testdata/tpch/plan_sf0.02/orders/3.parquet, /testdata/tpch/plan_sf0.02/orders/4.parquet, /testdata/tpch/plan_sf0.02/orders/5.parquet], [/testdata/tpch/plan_sf0.02/orders/6.parquet, /testdata/tpch/plan_sf0.02/orders/7.parquet, /testdata/tpch/plan_sf0.02/orders/8.parquet], ...]}, projection=[o_custkey], file_type=parquet + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p3] t1:[p4..p7] t2:[p8..p11] t3:[p12..p15] + │ AggregateExec: mode=Partial, gby=[], aggr=[avg(customer.c_acctbal)] + │ FilterExec: c_acctbal@1 > Some(0),15,2 AND substr(c_phone@0, 1, 2) IN (SET) ([13, 31, 23, 29, 30, 18, 17]), projection=[c_acctbal@1] + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/customer/1.parquet], [/testdata/tpch/plan_sf0.02/customer/10.parquet], [/testdata/tpch/plan_sf0.02/customer/11.parquet], [/testdata/tpch/plan_sf0.02/customer/12.parquet], [/testdata/tpch/plan_sf0.02/customer/13.parquet], ...]}, projection=[c_phone, c_acctbal], file_type=parquet, predicate=c_acctbal@5 > Some(0),15,2 AND substr(c_phone@4, 1, 2) IN (SET) ([13, 31, 23, 29, 30, 18, 17]), pruning_predicate=c_acctbal_null_count@1 != row_count@2 AND c_acctbal_max@0 > Some(0),15,2, required_guarantees=[] + └────────────────────────────────────────────────── + ┌───── Stage 2 ── Tasks: t0:[p0..p3] t1:[p4..p7] t2:[p8..p11] t3:[p12..p15] + │ FilterExec: substr(c_phone@1, 1, 2) IN (SET) ([13, 31, 23, 29, 30, 18, 17]) + │ PartitionIsolatorExec: t0:[p0,p1,p2,p3,__,__,__,__,__,__,__,__,__,__,__,__] t1:[__,__,__,__,p0,p1,p2,p3,__,__,__,__,__,__,__,__] t2:[__,__,__,__,__,__,__,__,p0,p1,p2,p3,__,__,__,__] t3:[__,__,__,__,__,__,__,__,__,__,__,__,p0,p1,p2,p3] + │ DataSourceExec: file_groups={16 groups: [[/testdata/tpch/plan_sf0.02/customer/1.parquet], [/testdata/tpch/plan_sf0.02/customer/10.parquet], [/testdata/tpch/plan_sf0.02/customer/11.parquet], [/testdata/tpch/plan_sf0.02/customer/12.parquet], [/testdata/tpch/plan_sf0.02/customer/13.parquet], ...]}, projection=[c_custkey, c_phone, c_acctbal], file_type=parquet, predicate=substr(c_phone@4, 1, 2) IN (SET) ([13, 31, 23, 29, 30, 18, 17]) + └────────────────────────────────────────────────── + "); + Ok(()) + } + + // test_tpch_query generates and displays a distributed plan for each TPC-H query. + async fn test_tpch_query(query_id: &str) -> Result> { + let (d_ctx, _guard, _) = start_localhost_context(4, DefaultSessionBuilder).await; + let d_ctx = d_ctx.with_distributed_broadcast_joins(true)?; + let data_dir = ensure_tpch_data(TPCH_SCALE_FACTOR, TPCH_DATA_PARTS).await; + let sql = tpch::get_query(query_id)?; + d_ctx + .state_ref() + .write() + .config_mut() + .options_mut() + .execution + .target_partitions = PARTITIONS; + + benchmarks_common::register_tables(&d_ctx, &data_dir).await?; + + // Query 15 has three queries in it, one creating the view, the second + // executing, which we want to capture the output of, and the third + // tearing down the view + let plan = if query_id == "q15" { + let queries: Vec<&str> = sql + .split(';') + .map(str::trim) + .filter(|s| !s.is_empty()) + .collect(); + + d_ctx.sql(queries[0]).await?.collect().await?; + let df = d_ctx.sql(queries[1]).await?; + let plan = df.create_physical_plan().await?; + d_ctx.sql(queries[2]).await?.collect().await?; + plan + } else { + let df = d_ctx.sql(&sql).await?; + df.create_physical_plan().await? + }; + + Ok(display_plan_ascii(plan.as_ref(), false)) + } + + // OnceCell to ensure TPCH tables are generated only once for tests + static INIT_TEST_TPCH_TABLES: OnceCell<()> = OnceCell::const_new(); + + // ensure_tpch_data initializes the TPCH data on disk. + pub async fn ensure_tpch_data(sf: f64, parts: i32) -> std::path::PathBuf { + let data_dir = + Path::new(env!("CARGO_MANIFEST_DIR")).join(format!("testdata/tpch/plan_sf{sf}")); + INIT_TEST_TPCH_TABLES + .get_or_init(|| async { + if !fs::exists(&data_dir).unwrap() { + tpch::generate_tpch_data(&data_dir, sf, parts); + } + }) + .await; + data_dir + } +} diff --git a/tests/tpch_validation_test.rs b/tests/tpch_validation_test.rs deleted file mode 100644 index 273320d0..00000000 --- a/tests/tpch_validation_test.rs +++ /dev/null @@ -1,1301 +0,0 @@ -#[cfg(all(feature = "integration", feature = "tpch", test))] -mod tests { - use datafusion::error::DataFusionError; - use datafusion::execution::{SessionState, SessionStateBuilder}; - use datafusion::physical_plan::execute_stream; - use datafusion::prelude::{SessionConfig, SessionContext}; - use datafusion_distributed::test_utils::localhost::start_localhost_context; - use datafusion_distributed::test_utils::tpch; - use datafusion_distributed::{ - DistributedPhysicalOptimizerRule, DistributedSessionBuilderContext, assert_snapshot, - display_plan_ascii, - }; - use futures::TryStreamExt; - use std::error::Error; - use std::fs; - use std::sync::Arc; - use tokio::sync::OnceCell; - - const PARTITIONS: usize = 6; - const SHUFFLE_TASKS: usize = 3; - const COALESCE_TASKS: usize = 4; - - const TPCH_SCALE_FACTOR: f64 = 0.1; - const TPCH_DATA_PARTS: i32 = 16; - - #[tokio::test] - async fn test_tpch_1() -> Result<(), Box> { - let plan = test_tpch_query(1).await?; - assert_snapshot!(plan, @r" - ┌───── DistributedExec ── Tasks: t0:[p0] - │ SortPreservingMergeExec: [l_returnflag@0 ASC NULLS LAST, l_linestatus@1 ASC NULLS LAST] - │ [Stage 2] => NetworkCoalesceExec: output_partitions=24, input_tasks=4 - └────────────────────────────────────────────────── - ┌───── Stage 2 ── Tasks: t0:[p0,p1,p2,p3,p4,p5] t1:[p0,p1,p2,p3,p4,p5] t2:[p0,p1,p2,p3,p4,p5] t3:[p0,p1,p2,p3,p4,p5] - │ SortExec: expr=[l_returnflag@0 ASC NULLS LAST, l_linestatus@1 ASC NULLS LAST], preserve_partitioning=[true] - │ ProjectionExec: expr=[l_returnflag@0 as l_returnflag, l_linestatus@1 as l_linestatus, sum(lineitem.l_quantity)@2 as sum_qty, sum(lineitem.l_extendedprice)@3 as sum_base_price, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@4 as sum_disc_price, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount * Int64(1) + lineitem.l_tax)@5 as sum_charge, avg(lineitem.l_quantity)@6 as avg_qty, avg(lineitem.l_extendedprice)@7 as avg_price, avg(lineitem.l_discount)@8 as avg_disc, count(Int64(1))@9 as count_order] - │ AggregateExec: mode=FinalPartitioned, gby=[l_returnflag@0 as l_returnflag, l_linestatus@1 as l_linestatus], aggr=[sum(lineitem.l_quantity), sum(lineitem.l_extendedprice), sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount), sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount * Int64(1) + lineitem.l_tax), avg(lineitem.l_quantity), avg(lineitem.l_extendedprice), avg(lineitem.l_discount), count(Int64(1))] - │ CoalesceBatchesExec: target_batch_size=8192 - │ [Stage 1] => NetworkShuffleExec: output_partitions=6, input_tasks=3 - └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9,p10,p11,p12,p13,p14,p15,p16,p17,p18,p19,p20,p21,p22,p23] t1:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9,p10,p11,p12,p13,p14,p15,p16,p17,p18,p19,p20,p21,p22,p23] t2:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9,p10,p11,p12,p13,p14,p15,p16,p17,p18,p19,p20,p21,p22,p23] - │ RepartitionExec: partitioning=Hash([l_returnflag@0, l_linestatus@1], 24), input_partitions=2 - │ AggregateExec: mode=Partial, gby=[l_returnflag@5 as l_returnflag, l_linestatus@6 as l_linestatus], aggr=[sum(lineitem.l_quantity), sum(lineitem.l_extendedprice), sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount), sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount * Int64(1) + lineitem.l_tax), avg(lineitem.l_quantity), avg(lineitem.l_extendedprice), avg(lineitem.l_discount), count(Int64(1))] - │ ProjectionExec: expr=[l_extendedprice@1 * (Some(1),20,0 - l_discount@2) as __common_expr_1, l_quantity@0 as l_quantity, l_extendedprice@1 as l_extendedprice, l_discount@2 as l_discount, l_tax@3 as l_tax, l_returnflag@4 as l_returnflag, l_linestatus@5 as l_linestatus] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: l_shipdate@6 <= 1998-09-02, projection=[l_quantity@0, l_extendedprice@1, l_discount@2, l_tax@3, l_returnflag@4, l_linestatus@5] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/data/lineitem/1.parquet:.., /testdata/tpch/data/lineitem/10.parquet:.., /testdata/tpch/data/lineitem/11.parquet:..], [/testdata/tpch/data/lineitem/11.parquet:.., /testdata/tpch/data/lineitem/12.parquet:.., /testdata/tpch/data/lineitem/13.parquet:.., /testdata/tpch/data/lineitem/14.parquet:..], [/testdata/tpch/data/lineitem/14.parquet:.., /testdata/tpch/data/lineitem/15.parquet:.., /testdata/tpch/data/lineitem/16.parquet:..], [/testdata/tpch/data/lineitem/16.parquet:.., /testdata/tpch/data/lineitem/2.parquet:.., /testdata/tpch/data/lineitem/3.parquet:.., /testdata/tpch/data/lineitem/4.parquet:..], [/testdata/tpch/data/lineitem/4.parquet:.., /testdata/tpch/data/lineitem/5.parquet:.., /testdata/tpch/data/lineitem/6.parquet:.., /testdata/tpch/data/lineitem/7.parquet:..], ...]}, projection=[l_quantity, l_extendedprice, l_discount, l_tax, l_returnflag, l_linestatus, l_shipdate], file_type=parquet, predicate=l_shipdate@6 <= 1998-09-02, pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@0 <= 1998-09-02, required_guarantees=[] - └────────────────────────────────────────────────── - "); - Ok(()) - } - - #[tokio::test] - async fn test_tpch_2() -> Result<(), Box> { - let plan = test_tpch_query(2).await?; - assert_snapshot!(plan, @r" - ┌───── DistributedExec ── Tasks: t0:[p0] - │ SortPreservingMergeExec: [s_acctbal@0 DESC, n_name@2 ASC NULLS LAST, s_name@1 ASC NULLS LAST, p_partkey@3 ASC NULLS LAST] - │ [Stage 7] => NetworkCoalesceExec: output_partitions=24, input_tasks=4 - └────────────────────────────────────────────────── - ┌───── Stage 7 ── Tasks: t0:[p0,p1,p2,p3,p4,p5] t1:[p0,p1,p2,p3,p4,p5] t2:[p0,p1,p2,p3,p4,p5] t3:[p0,p1,p2,p3,p4,p5] - │ SortExec: expr=[s_acctbal@0 DESC, n_name@2 ASC NULLS LAST, s_name@1 ASC NULLS LAST, p_partkey@3 ASC NULLS LAST], preserve_partitioning=[true] - │ ProjectionExec: expr=[s_acctbal@5 as s_acctbal, s_name@2 as s_name, n_name@7 as n_name, p_partkey@0 as p_partkey, p_mfgr@1 as p_mfgr, s_address@3 as s_address, s_phone@4 as s_phone, s_comment@6 as s_comment] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(p_partkey@0, ps_partkey@1), (ps_supplycost@7, min(partsupp.ps_supplycost)@0)], projection=[p_partkey@0, p_mfgr@1, s_name@2, s_address@3, s_phone@4, s_acctbal@5, s_comment@6, n_name@8] - │ CoalesceBatchesExec: target_batch_size=8192 - │ [Stage 3] => NetworkShuffleExec: output_partitions=6, input_tasks=1 - │ CoalesceBatchesExec: target_batch_size=8192 - │ [Stage 6] => NetworkShuffleExec: output_partitions=6, input_tasks=3 - └────────────────────────────────────────────────── - ┌───── Stage 3 ── Tasks: t0:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9,p10,p11,p12,p13,p14,p15,p16,p17,p18,p19,p20,p21,p22,p23] - │ RepartitionExec: partitioning=Hash([p_partkey@0, ps_supplycost@7], 24), input_partitions=6 - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(r_regionkey@0, n_regionkey@9)], projection=[p_partkey@1, p_mfgr@2, s_name@3, s_address@4, s_phone@5, s_acctbal@6, s_comment@7, ps_supplycost@8, n_name@9] - │ CoalescePartitionsExec - │ [Stage 1] => NetworkCoalesceExec: output_partitions=8, input_tasks=4 - │ ProjectionExec: expr=[p_partkey@2 as p_partkey, p_mfgr@3 as p_mfgr, s_name@4 as s_name, s_address@5 as s_address, s_phone@6 as s_phone, s_acctbal@7 as s_acctbal, s_comment@8 as s_comment, ps_supplycost@9 as ps_supplycost, n_name@0 as n_name, n_regionkey@1 as n_regionkey] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@4)], projection=[n_name@1, n_regionkey@2, p_partkey@3, p_mfgr@4, s_name@5, s_address@6, s_phone@8, s_acctbal@9, s_comment@10, ps_supplycost@11] - │ CoalescePartitionsExec - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/data/nation/1.parquet, /testdata/tpch/data/nation/10.parquet, /testdata/tpch/data/nation/11.parquet], [/testdata/tpch/data/nation/12.parquet, /testdata/tpch/data/nation/13.parquet, /testdata/tpch/data/nation/14.parquet], [/testdata/tpch/data/nation/15.parquet, /testdata/tpch/data/nation/16.parquet, /testdata/tpch/data/nation/2.parquet], [/testdata/tpch/data/nation/3.parquet, /testdata/tpch/data/nation/4.parquet, /testdata/tpch/data/nation/5.parquet], [/testdata/tpch/data/nation/6.parquet, /testdata/tpch/data/nation/7.parquet, /testdata/tpch/data/nation/8.parquet], ...]}, projection=[n_nationkey, n_name, n_regionkey], file_type=parquet - │ ProjectionExec: expr=[p_partkey@6 as p_partkey, p_mfgr@7 as p_mfgr, s_name@0 as s_name, s_address@1 as s_address, s_nationkey@2 as s_nationkey, s_phone@3 as s_phone, s_acctbal@4 as s_acctbal, s_comment@5 as s_comment, ps_supplycost@8 as ps_supplycost] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_suppkey@0, ps_suppkey@2)], projection=[s_name@1, s_address@2, s_nationkey@3, s_phone@4, s_acctbal@5, s_comment@6, p_partkey@7, p_mfgr@8, ps_supplycost@10] - │ CoalescePartitionsExec - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/data/supplier/1.parquet, /testdata/tpch/data/supplier/10.parquet, /testdata/tpch/data/supplier/11.parquet], [/testdata/tpch/data/supplier/12.parquet, /testdata/tpch/data/supplier/13.parquet, /testdata/tpch/data/supplier/14.parquet], [/testdata/tpch/data/supplier/15.parquet, /testdata/tpch/data/supplier/16.parquet, /testdata/tpch/data/supplier/2.parquet], [/testdata/tpch/data/supplier/3.parquet, /testdata/tpch/data/supplier/4.parquet, /testdata/tpch/data/supplier/5.parquet], [/testdata/tpch/data/supplier/6.parquet, /testdata/tpch/data/supplier/7.parquet, /testdata/tpch/data/supplier/8.parquet], ...]}, projection=[s_suppkey, s_name, s_address, s_nationkey, s_phone, s_acctbal, s_comment], file_type=parquet - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(p_partkey@0, ps_partkey@0)], projection=[p_partkey@0, p_mfgr@1, ps_suppkey@3, ps_supplycost@4] - │ CoalescePartitionsExec - │ [Stage 2] => NetworkCoalesceExec: output_partitions=8, input_tasks=4 - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/data/partsupp/1.parquet:.., /testdata/tpch/data/partsupp/10.parquet:.., /testdata/tpch/data/partsupp/11.parquet:..], [/testdata/tpch/data/partsupp/11.parquet:.., /testdata/tpch/data/partsupp/12.parquet:.., /testdata/tpch/data/partsupp/13.parquet:.., /testdata/tpch/data/partsupp/14.parquet:..], [/testdata/tpch/data/partsupp/14.parquet:.., /testdata/tpch/data/partsupp/15.parquet:.., /testdata/tpch/data/partsupp/16.parquet:.., /testdata/tpch/data/partsupp/2.parquet:..], [/testdata/tpch/data/partsupp/2.parquet:.., /testdata/tpch/data/partsupp/3.parquet:.., /testdata/tpch/data/partsupp/4.parquet:..], [/testdata/tpch/data/partsupp/4.parquet:.., /testdata/tpch/data/partsupp/5.parquet:.., /testdata/tpch/data/partsupp/6.parquet:.., /testdata/tpch/data/partsupp/7.parquet:..], ...]}, projection=[ps_partkey, ps_suppkey, ps_supplycost], file_type=parquet, predicate=DynamicFilterPhysicalExpr [ true ] AND DynamicFilterPhysicalExpr [ true ] - └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0,p1] t1:[p2,p3] t2:[p4,p5] t3:[p6,p7] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: r_name@1 = EUROPE, projection=[r_regionkey@0] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/data/region/1.parquet, /testdata/tpch/data/region/10.parquet, /testdata/tpch/data/region/11.parquet], [/testdata/tpch/data/region/12.parquet, /testdata/tpch/data/region/13.parquet, /testdata/tpch/data/region/14.parquet], [/testdata/tpch/data/region/15.parquet, /testdata/tpch/data/region/16.parquet, /testdata/tpch/data/region/2.parquet], [/testdata/tpch/data/region/3.parquet, /testdata/tpch/data/region/4.parquet, /testdata/tpch/data/region/5.parquet], [/testdata/tpch/data/region/6.parquet, /testdata/tpch/data/region/7.parquet, /testdata/tpch/data/region/8.parquet], ...]}, projection=[r_regionkey, r_name], file_type=parquet, predicate=r_name@1 = EUROPE, pruning_predicate=r_name_null_count@2 != row_count@3 AND r_name_min@0 <= EUROPE AND EUROPE <= r_name_max@1, required_guarantees=[r_name in (EUROPE)] - └────────────────────────────────────────────────── - ┌───── Stage 2 ── Tasks: t0:[p0,p1] t1:[p2,p3] t2:[p4,p5] t3:[p6,p7] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: p_size@3 = 15 AND p_type@2 LIKE %BRASS, projection=[p_partkey@0, p_mfgr@1] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/data/part/1.parquet, /testdata/tpch/data/part/10.parquet, /testdata/tpch/data/part/11.parquet], [/testdata/tpch/data/part/12.parquet, /testdata/tpch/data/part/13.parquet, /testdata/tpch/data/part/14.parquet], [/testdata/tpch/data/part/15.parquet, /testdata/tpch/data/part/16.parquet, /testdata/tpch/data/part/2.parquet], [/testdata/tpch/data/part/3.parquet, /testdata/tpch/data/part/4.parquet, /testdata/tpch/data/part/5.parquet], [/testdata/tpch/data/part/6.parquet, /testdata/tpch/data/part/7.parquet, /testdata/tpch/data/part/8.parquet], ...]}, projection=[p_partkey, p_mfgr, p_type, p_size], file_type=parquet, predicate=p_size@3 = 15 AND p_type@2 LIKE %BRASS, pruning_predicate=p_size_null_count@2 != row_count@3 AND p_size_min@0 <= 15 AND 15 <= p_size_max@1, required_guarantees=[p_size in (15)] - └────────────────────────────────────────────────── - ┌───── Stage 6 ── Tasks: t0:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9,p10,p11,p12,p13,p14,p15,p16,p17,p18,p19,p20,p21,p22,p23] t1:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9,p10,p11,p12,p13,p14,p15,p16,p17,p18,p19,p20,p21,p22,p23] t2:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9,p10,p11,p12,p13,p14,p15,p16,p17,p18,p19,p20,p21,p22,p23] - │ RepartitionExec: partitioning=Hash([ps_partkey@1, min(partsupp.ps_supplycost)@0], 24), input_partitions=6 - │ ProjectionExec: expr=[min(partsupp.ps_supplycost)@1 as min(partsupp.ps_supplycost), ps_partkey@0 as ps_partkey] - │ AggregateExec: mode=FinalPartitioned, gby=[ps_partkey@0 as ps_partkey], aggr=[min(partsupp.ps_supplycost)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ [Stage 5] => NetworkShuffleExec: output_partitions=6, input_tasks=1 - └────────────────────────────────────────────────── - ┌───── Stage 5 ── Tasks: t0:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9,p10,p11,p12,p13,p14,p15,p16,p17] - │ RepartitionExec: partitioning=Hash([ps_partkey@0], 18), input_partitions=6 - │ AggregateExec: mode=Partial, gby=[ps_partkey@0 as ps_partkey], aggr=[min(partsupp.ps_supplycost)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(r_regionkey@0, n_regionkey@2)], projection=[ps_partkey@1, ps_supplycost@2] - │ CoalescePartitionsExec - │ [Stage 4] => NetworkCoalesceExec: output_partitions=8, input_tasks=4 - │ ProjectionExec: expr=[ps_partkey@1 as ps_partkey, ps_supplycost@2 as ps_supplycost, n_regionkey@0 as n_regionkey] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@2)], projection=[n_regionkey@1, ps_partkey@2, ps_supplycost@3] - │ CoalescePartitionsExec - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/data/nation/1.parquet, /testdata/tpch/data/nation/10.parquet, /testdata/tpch/data/nation/11.parquet], [/testdata/tpch/data/nation/12.parquet, /testdata/tpch/data/nation/13.parquet, /testdata/tpch/data/nation/14.parquet], [/testdata/tpch/data/nation/15.parquet, /testdata/tpch/data/nation/16.parquet, /testdata/tpch/data/nation/2.parquet], [/testdata/tpch/data/nation/3.parquet, /testdata/tpch/data/nation/4.parquet, /testdata/tpch/data/nation/5.parquet], [/testdata/tpch/data/nation/6.parquet, /testdata/tpch/data/nation/7.parquet, /testdata/tpch/data/nation/8.parquet], ...]}, projection=[n_nationkey, n_regionkey], file_type=parquet - │ ProjectionExec: expr=[ps_partkey@1 as ps_partkey, ps_supplycost@2 as ps_supplycost, s_nationkey@0 as s_nationkey] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_suppkey@0, ps_suppkey@1)], projection=[s_nationkey@1, ps_partkey@2, ps_supplycost@4] - │ CoalescePartitionsExec - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/data/supplier/1.parquet, /testdata/tpch/data/supplier/10.parquet, /testdata/tpch/data/supplier/11.parquet], [/testdata/tpch/data/supplier/12.parquet, /testdata/tpch/data/supplier/13.parquet, /testdata/tpch/data/supplier/14.parquet], [/testdata/tpch/data/supplier/15.parquet, /testdata/tpch/data/supplier/16.parquet, /testdata/tpch/data/supplier/2.parquet], [/testdata/tpch/data/supplier/3.parquet, /testdata/tpch/data/supplier/4.parquet, /testdata/tpch/data/supplier/5.parquet], [/testdata/tpch/data/supplier/6.parquet, /testdata/tpch/data/supplier/7.parquet, /testdata/tpch/data/supplier/8.parquet], ...]}, projection=[s_suppkey, s_nationkey], file_type=parquet - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/data/partsupp/1.parquet:.., /testdata/tpch/data/partsupp/10.parquet:.., /testdata/tpch/data/partsupp/11.parquet:..], [/testdata/tpch/data/partsupp/11.parquet:.., /testdata/tpch/data/partsupp/12.parquet:.., /testdata/tpch/data/partsupp/13.parquet:.., /testdata/tpch/data/partsupp/14.parquet:..], [/testdata/tpch/data/partsupp/14.parquet:.., /testdata/tpch/data/partsupp/15.parquet:.., /testdata/tpch/data/partsupp/16.parquet:.., /testdata/tpch/data/partsupp/2.parquet:..], [/testdata/tpch/data/partsupp/2.parquet:.., /testdata/tpch/data/partsupp/3.parquet:.., /testdata/tpch/data/partsupp/4.parquet:..], [/testdata/tpch/data/partsupp/4.parquet:.., /testdata/tpch/data/partsupp/5.parquet:.., /testdata/tpch/data/partsupp/6.parquet:.., /testdata/tpch/data/partsupp/7.parquet:..], ...]}, projection=[ps_partkey, ps_suppkey, ps_supplycost], file_type=parquet, predicate=DynamicFilterPhysicalExpr [ true ] - └────────────────────────────────────────────────── - ┌───── Stage 4 ── Tasks: t0:[p0,p1] t1:[p2,p3] t2:[p4,p5] t3:[p6,p7] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: r_name@1 = EUROPE, projection=[r_regionkey@0] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/data/region/1.parquet, /testdata/tpch/data/region/10.parquet, /testdata/tpch/data/region/11.parquet], [/testdata/tpch/data/region/12.parquet, /testdata/tpch/data/region/13.parquet, /testdata/tpch/data/region/14.parquet], [/testdata/tpch/data/region/15.parquet, /testdata/tpch/data/region/16.parquet, /testdata/tpch/data/region/2.parquet], [/testdata/tpch/data/region/3.parquet, /testdata/tpch/data/region/4.parquet, /testdata/tpch/data/region/5.parquet], [/testdata/tpch/data/region/6.parquet, /testdata/tpch/data/region/7.parquet, /testdata/tpch/data/region/8.parquet], ...]}, projection=[r_regionkey, r_name], file_type=parquet, predicate=r_name@1 = EUROPE, pruning_predicate=r_name_null_count@2 != row_count@3 AND r_name_min@0 <= EUROPE AND EUROPE <= r_name_max@1, required_guarantees=[r_name in (EUROPE)] - └────────────────────────────────────────────────── - "); - Ok(()) - } - - #[tokio::test] - async fn test_tpch_3() -> Result<(), Box> { - let plan = test_tpch_query(3).await?; - assert_snapshot!(plan, @r" - ┌───── DistributedExec ── Tasks: t0:[p0] - │ SortPreservingMergeExec: [revenue@1 DESC, o_orderdate@2 ASC NULLS LAST] - │ [Stage 3] => NetworkCoalesceExec: output_partitions=24, input_tasks=4 - └────────────────────────────────────────────────── - ┌───── Stage 3 ── Tasks: t0:[p0,p1,p2,p3,p4,p5] t1:[p0,p1,p2,p3,p4,p5] t2:[p0,p1,p2,p3,p4,p5] t3:[p0,p1,p2,p3,p4,p5] - │ SortExec: expr=[revenue@1 DESC, o_orderdate@2 ASC NULLS LAST], preserve_partitioning=[true] - │ ProjectionExec: expr=[l_orderkey@0 as l_orderkey, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@3 as revenue, o_orderdate@1 as o_orderdate, o_shippriority@2 as o_shippriority] - │ AggregateExec: mode=FinalPartitioned, gby=[l_orderkey@0 as l_orderkey, o_orderdate@1 as o_orderdate, o_shippriority@2 as o_shippriority], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ [Stage 2] => NetworkShuffleExec: output_partitions=6, input_tasks=1 - └────────────────────────────────────────────────── - ┌───── Stage 2 ── Tasks: t0:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9,p10,p11,p12,p13,p14,p15,p16,p17,p18,p19,p20,p21,p22,p23] - │ RepartitionExec: partitioning=Hash([l_orderkey@0, o_orderdate@1, o_shippriority@2], 24), input_partitions=6 - │ AggregateExec: mode=Partial, gby=[l_orderkey@2 as l_orderkey, o_orderdate@0 as o_orderdate, o_shippriority@1 as o_shippriority], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(o_orderkey@0, l_orderkey@0)], projection=[o_orderdate@1, o_shippriority@2, l_orderkey@3, l_extendedprice@4, l_discount@5] - │ CoalescePartitionsExec - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(c_custkey@0, o_custkey@1)], projection=[o_orderkey@1, o_orderdate@3, o_shippriority@4] - │ CoalescePartitionsExec - │ [Stage 1] => NetworkCoalesceExec: output_partitions=8, input_tasks=4 - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: o_orderdate@2 < 1995-03-15 - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/data/orders/1.parquet:.., /testdata/tpch/data/orders/10.parquet:.., /testdata/tpch/data/orders/11.parquet:..], [/testdata/tpch/data/orders/11.parquet:.., /testdata/tpch/data/orders/12.parquet:.., /testdata/tpch/data/orders/13.parquet:.., /testdata/tpch/data/orders/14.parquet:..], [/testdata/tpch/data/orders/14.parquet:.., /testdata/tpch/data/orders/15.parquet:.., /testdata/tpch/data/orders/16.parquet:..], [/testdata/tpch/data/orders/16.parquet:.., /testdata/tpch/data/orders/2.parquet:.., /testdata/tpch/data/orders/3.parquet:.., /testdata/tpch/data/orders/4.parquet:..], [/testdata/tpch/data/orders/4.parquet:.., /testdata/tpch/data/orders/5.parquet:.., /testdata/tpch/data/orders/6.parquet:.., /testdata/tpch/data/orders/7.parquet:..], ...]}, projection=[o_orderkey, o_custkey, o_orderdate, o_shippriority], file_type=parquet, predicate=o_orderdate@2 < 1995-03-15 AND DynamicFilterPhysicalExpr [ true ], pruning_predicate=o_orderdate_null_count@1 != row_count@2 AND o_orderdate_min@0 < 1995-03-15, required_guarantees=[] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: l_shipdate@3 > 1995-03-15, projection=[l_orderkey@0, l_extendedprice@1, l_discount@2] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/data/lineitem/1.parquet:.., /testdata/tpch/data/lineitem/10.parquet:.., /testdata/tpch/data/lineitem/11.parquet:..], [/testdata/tpch/data/lineitem/11.parquet:.., /testdata/tpch/data/lineitem/12.parquet:.., /testdata/tpch/data/lineitem/13.parquet:.., /testdata/tpch/data/lineitem/14.parquet:..], [/testdata/tpch/data/lineitem/14.parquet:.., /testdata/tpch/data/lineitem/15.parquet:.., /testdata/tpch/data/lineitem/16.parquet:..], [/testdata/tpch/data/lineitem/16.parquet:.., /testdata/tpch/data/lineitem/2.parquet:.., /testdata/tpch/data/lineitem/3.parquet:.., /testdata/tpch/data/lineitem/4.parquet:..], [/testdata/tpch/data/lineitem/4.parquet:.., /testdata/tpch/data/lineitem/5.parquet:.., /testdata/tpch/data/lineitem/6.parquet:.., /testdata/tpch/data/lineitem/7.parquet:..], ...]}, projection=[l_orderkey, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@3 > 1995-03-15 AND DynamicFilterPhysicalExpr [ true ], pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 > 1995-03-15, required_guarantees=[] - └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0,p1] t1:[p2,p3] t2:[p4,p5] t3:[p6,p7] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: c_mktsegment@1 = BUILDING, projection=[c_custkey@0] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/data/customer/1.parquet, /testdata/tpch/data/customer/10.parquet, /testdata/tpch/data/customer/11.parquet], [/testdata/tpch/data/customer/12.parquet, /testdata/tpch/data/customer/13.parquet, /testdata/tpch/data/customer/14.parquet], [/testdata/tpch/data/customer/15.parquet, /testdata/tpch/data/customer/16.parquet, /testdata/tpch/data/customer/2.parquet], [/testdata/tpch/data/customer/3.parquet, /testdata/tpch/data/customer/4.parquet, /testdata/tpch/data/customer/5.parquet], [/testdata/tpch/data/customer/6.parquet, /testdata/tpch/data/customer/7.parquet, /testdata/tpch/data/customer/8.parquet], ...]}, projection=[c_custkey, c_mktsegment], file_type=parquet, predicate=c_mktsegment@1 = BUILDING, pruning_predicate=c_mktsegment_null_count@2 != row_count@3 AND c_mktsegment_min@0 <= BUILDING AND BUILDING <= c_mktsegment_max@1, required_guarantees=[c_mktsegment in (BUILDING)] - └────────────────────────────────────────────────── - "); - Ok(()) - } - - #[tokio::test] - async fn test_tpch_4() -> Result<(), Box> { - let plan = test_tpch_query(4).await?; - assert_snapshot!(plan, @r" - ┌───── DistributedExec ── Tasks: t0:[p0] - │ SortPreservingMergeExec: [o_orderpriority@0 ASC NULLS LAST] - │ [Stage 3] => NetworkCoalesceExec: output_partitions=24, input_tasks=4 - └────────────────────────────────────────────────── - ┌───── Stage 3 ── Tasks: t0:[p0,p1,p2,p3,p4,p5] t1:[p0,p1,p2,p3,p4,p5] t2:[p0,p1,p2,p3,p4,p5] t3:[p0,p1,p2,p3,p4,p5] - │ SortExec: expr=[o_orderpriority@0 ASC NULLS LAST], preserve_partitioning=[true] - │ ProjectionExec: expr=[o_orderpriority@0 as o_orderpriority, count(Int64(1))@1 as order_count] - │ AggregateExec: mode=FinalPartitioned, gby=[o_orderpriority@0 as o_orderpriority], aggr=[count(Int64(1))] - │ CoalesceBatchesExec: target_batch_size=8192 - │ [Stage 2] => NetworkShuffleExec: output_partitions=6, input_tasks=1 - └────────────────────────────────────────────────── - ┌───── Stage 2 ── Tasks: t0:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9,p10,p11,p12,p13,p14,p15,p16,p17,p18,p19,p20,p21,p22,p23] - │ RepartitionExec: partitioning=Hash([o_orderpriority@0], 24), input_partitions=6 - │ AggregateExec: mode=Partial, gby=[o_orderpriority@0 as o_orderpriority], aggr=[count(Int64(1))] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(l_orderkey@0, o_orderkey@0)], projection=[o_orderpriority@1] - │ CoalescePartitionsExec - │ [Stage 1] => NetworkCoalesceExec: output_partitions=8, input_tasks=4 - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: o_orderdate@1 >= 1993-07-01 AND o_orderdate@1 < 1993-10-01, projection=[o_orderkey@0, o_orderpriority@2] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/data/orders/1.parquet:.., /testdata/tpch/data/orders/10.parquet:.., /testdata/tpch/data/orders/11.parquet:..], [/testdata/tpch/data/orders/11.parquet:.., /testdata/tpch/data/orders/12.parquet:.., /testdata/tpch/data/orders/13.parquet:.., /testdata/tpch/data/orders/14.parquet:..], [/testdata/tpch/data/orders/14.parquet:.., /testdata/tpch/data/orders/15.parquet:.., /testdata/tpch/data/orders/16.parquet:..], [/testdata/tpch/data/orders/16.parquet:.., /testdata/tpch/data/orders/2.parquet:.., /testdata/tpch/data/orders/3.parquet:.., /testdata/tpch/data/orders/4.parquet:..], [/testdata/tpch/data/orders/4.parquet:.., /testdata/tpch/data/orders/5.parquet:.., /testdata/tpch/data/orders/6.parquet:.., /testdata/tpch/data/orders/7.parquet:..], ...]}, projection=[o_orderkey, o_orderdate, o_orderpriority], file_type=parquet, predicate=o_orderdate@1 >= 1993-07-01 AND o_orderdate@1 < 1993-10-01, pruning_predicate=o_orderdate_null_count@1 != row_count@2 AND o_orderdate_max@0 >= 1993-07-01 AND o_orderdate_null_count@1 != row_count@2 AND o_orderdate_min@3 < 1993-10-01, required_guarantees=[] - └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0,p1] t1:[p2,p3] t2:[p4,p5] t3:[p6,p7] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: l_receiptdate@2 > l_commitdate@1, projection=[l_orderkey@0] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/data/lineitem/1.parquet:.., /testdata/tpch/data/lineitem/10.parquet:.., /testdata/tpch/data/lineitem/11.parquet:..], [/testdata/tpch/data/lineitem/11.parquet:.., /testdata/tpch/data/lineitem/12.parquet:.., /testdata/tpch/data/lineitem/13.parquet:.., /testdata/tpch/data/lineitem/14.parquet:..], [/testdata/tpch/data/lineitem/14.parquet:.., /testdata/tpch/data/lineitem/15.parquet:.., /testdata/tpch/data/lineitem/16.parquet:..], [/testdata/tpch/data/lineitem/16.parquet:.., /testdata/tpch/data/lineitem/2.parquet:.., /testdata/tpch/data/lineitem/3.parquet:.., /testdata/tpch/data/lineitem/4.parquet:..], [/testdata/tpch/data/lineitem/4.parquet:.., /testdata/tpch/data/lineitem/5.parquet:.., /testdata/tpch/data/lineitem/6.parquet:.., /testdata/tpch/data/lineitem/7.parquet:..], ...]}, projection=[l_orderkey, l_commitdate, l_receiptdate], file_type=parquet, predicate=l_receiptdate@2 > l_commitdate@1 - └────────────────────────────────────────────────── - "); - Ok(()) - } - - #[tokio::test] - async fn test_tpch_5() -> Result<(), Box> { - let plan = test_tpch_query(5).await?; - assert_snapshot!(plan, @r" - ┌───── DistributedExec ── Tasks: t0:[p0] - │ SortPreservingMergeExec: [revenue@1 DESC] - │ [Stage 4] => NetworkCoalesceExec: output_partitions=24, input_tasks=4 - └────────────────────────────────────────────────── - ┌───── Stage 4 ── Tasks: t0:[p0,p1,p2,p3,p4,p5] t1:[p0,p1,p2,p3,p4,p5] t2:[p0,p1,p2,p3,p4,p5] t3:[p0,p1,p2,p3,p4,p5] - │ SortExec: expr=[revenue@1 DESC], preserve_partitioning=[true] - │ ProjectionExec: expr=[n_name@0 as n_name, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@1 as revenue] - │ AggregateExec: mode=FinalPartitioned, gby=[n_name@0 as n_name], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ [Stage 3] => NetworkShuffleExec: output_partitions=6, input_tasks=1 - └────────────────────────────────────────────────── - ┌───── Stage 3 ── Tasks: t0:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9,p10,p11,p12,p13,p14,p15,p16,p17,p18,p19,p20,p21,p22,p23] - │ RepartitionExec: partitioning=Hash([n_name@0], 24), input_partitions=6 - │ AggregateExec: mode=Partial, gby=[n_name@2 as n_name], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(r_regionkey@0, n_regionkey@3)], projection=[l_extendedprice@1, l_discount@2, n_name@3] - │ CoalescePartitionsExec - │ [Stage 1] => NetworkCoalesceExec: output_partitions=8, input_tasks=4 - │ ProjectionExec: expr=[l_extendedprice@2 as l_extendedprice, l_discount@3 as l_discount, n_name@0 as n_name, n_regionkey@1 as n_regionkey] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@2)], projection=[n_name@1, n_regionkey@2, l_extendedprice@3, l_discount@4] - │ CoalescePartitionsExec - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/data/nation/1.parquet, /testdata/tpch/data/nation/10.parquet, /testdata/tpch/data/nation/11.parquet], [/testdata/tpch/data/nation/12.parquet, /testdata/tpch/data/nation/13.parquet, /testdata/tpch/data/nation/14.parquet], [/testdata/tpch/data/nation/15.parquet, /testdata/tpch/data/nation/16.parquet, /testdata/tpch/data/nation/2.parquet], [/testdata/tpch/data/nation/3.parquet, /testdata/tpch/data/nation/4.parquet, /testdata/tpch/data/nation/5.parquet], [/testdata/tpch/data/nation/6.parquet, /testdata/tpch/data/nation/7.parquet, /testdata/tpch/data/nation/8.parquet], ...]}, projection=[n_nationkey, n_name, n_regionkey], file_type=parquet - │ ProjectionExec: expr=[l_extendedprice@1 as l_extendedprice, l_discount@2 as l_discount, s_nationkey@0 as s_nationkey] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_suppkey@0, l_suppkey@1), (s_nationkey@1, c_nationkey@0)], projection=[s_nationkey@1, l_extendedprice@4, l_discount@5] - │ CoalescePartitionsExec - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/data/supplier/1.parquet, /testdata/tpch/data/supplier/10.parquet, /testdata/tpch/data/supplier/11.parquet], [/testdata/tpch/data/supplier/12.parquet, /testdata/tpch/data/supplier/13.parquet, /testdata/tpch/data/supplier/14.parquet], [/testdata/tpch/data/supplier/15.parquet, /testdata/tpch/data/supplier/16.parquet, /testdata/tpch/data/supplier/2.parquet], [/testdata/tpch/data/supplier/3.parquet, /testdata/tpch/data/supplier/4.parquet, /testdata/tpch/data/supplier/5.parquet], [/testdata/tpch/data/supplier/6.parquet, /testdata/tpch/data/supplier/7.parquet, /testdata/tpch/data/supplier/8.parquet], ...]}, projection=[s_suppkey, s_nationkey], file_type=parquet - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(o_orderkey@1, l_orderkey@0)], projection=[c_nationkey@0, l_suppkey@3, l_extendedprice@4, l_discount@5] - │ CoalescePartitionsExec - │ ProjectionExec: expr=[c_nationkey@1 as c_nationkey, o_orderkey@0 as o_orderkey] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(o_custkey@1, c_custkey@0)], projection=[o_orderkey@0, c_nationkey@3] - │ CoalescePartitionsExec - │ [Stage 2] => NetworkCoalesceExec: output_partitions=8, input_tasks=4 - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/data/customer/1.parquet, /testdata/tpch/data/customer/10.parquet, /testdata/tpch/data/customer/11.parquet], [/testdata/tpch/data/customer/12.parquet, /testdata/tpch/data/customer/13.parquet, /testdata/tpch/data/customer/14.parquet], [/testdata/tpch/data/customer/15.parquet, /testdata/tpch/data/customer/16.parquet, /testdata/tpch/data/customer/2.parquet], [/testdata/tpch/data/customer/3.parquet, /testdata/tpch/data/customer/4.parquet, /testdata/tpch/data/customer/5.parquet], [/testdata/tpch/data/customer/6.parquet, /testdata/tpch/data/customer/7.parquet, /testdata/tpch/data/customer/8.parquet], ...]}, projection=[c_custkey, c_nationkey], file_type=parquet, predicate=DynamicFilterPhysicalExpr [ true ] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/data/lineitem/1.parquet:.., /testdata/tpch/data/lineitem/10.parquet:.., /testdata/tpch/data/lineitem/11.parquet:..], [/testdata/tpch/data/lineitem/11.parquet:.., /testdata/tpch/data/lineitem/12.parquet:.., /testdata/tpch/data/lineitem/13.parquet:.., /testdata/tpch/data/lineitem/14.parquet:..], [/testdata/tpch/data/lineitem/14.parquet:.., /testdata/tpch/data/lineitem/15.parquet:.., /testdata/tpch/data/lineitem/16.parquet:..], [/testdata/tpch/data/lineitem/16.parquet:.., /testdata/tpch/data/lineitem/2.parquet:.., /testdata/tpch/data/lineitem/3.parquet:.., /testdata/tpch/data/lineitem/4.parquet:..], [/testdata/tpch/data/lineitem/4.parquet:.., /testdata/tpch/data/lineitem/5.parquet:.., /testdata/tpch/data/lineitem/6.parquet:.., /testdata/tpch/data/lineitem/7.parquet:..], ...]}, projection=[l_orderkey, l_suppkey, l_extendedprice, l_discount], file_type=parquet, predicate=DynamicFilterPhysicalExpr [ true ] - └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0,p1] t1:[p2,p3] t2:[p4,p5] t3:[p6,p7] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: r_name@1 = ASIA, projection=[r_regionkey@0] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/data/region/1.parquet, /testdata/tpch/data/region/10.parquet, /testdata/tpch/data/region/11.parquet], [/testdata/tpch/data/region/12.parquet, /testdata/tpch/data/region/13.parquet, /testdata/tpch/data/region/14.parquet], [/testdata/tpch/data/region/15.parquet, /testdata/tpch/data/region/16.parquet, /testdata/tpch/data/region/2.parquet], [/testdata/tpch/data/region/3.parquet, /testdata/tpch/data/region/4.parquet, /testdata/tpch/data/region/5.parquet], [/testdata/tpch/data/region/6.parquet, /testdata/tpch/data/region/7.parquet, /testdata/tpch/data/region/8.parquet], ...]}, projection=[r_regionkey, r_name], file_type=parquet, predicate=r_name@1 = ASIA, pruning_predicate=r_name_null_count@2 != row_count@3 AND r_name_min@0 <= ASIA AND ASIA <= r_name_max@1, required_guarantees=[r_name in (ASIA)] - └────────────────────────────────────────────────── - ┌───── Stage 2 ── Tasks: t0:[p0,p1] t1:[p2,p3] t2:[p4,p5] t3:[p6,p7] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: o_orderdate@2 >= 1994-01-01 AND o_orderdate@2 < 1995-01-01, projection=[o_orderkey@0, o_custkey@1] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/data/orders/1.parquet:.., /testdata/tpch/data/orders/10.parquet:.., /testdata/tpch/data/orders/11.parquet:..], [/testdata/tpch/data/orders/11.parquet:.., /testdata/tpch/data/orders/12.parquet:.., /testdata/tpch/data/orders/13.parquet:.., /testdata/tpch/data/orders/14.parquet:..], [/testdata/tpch/data/orders/14.parquet:.., /testdata/tpch/data/orders/15.parquet:.., /testdata/tpch/data/orders/16.parquet:..], [/testdata/tpch/data/orders/16.parquet:.., /testdata/tpch/data/orders/2.parquet:.., /testdata/tpch/data/orders/3.parquet:.., /testdata/tpch/data/orders/4.parquet:..], [/testdata/tpch/data/orders/4.parquet:.., /testdata/tpch/data/orders/5.parquet:.., /testdata/tpch/data/orders/6.parquet:.., /testdata/tpch/data/orders/7.parquet:..], ...]}, projection=[o_orderkey, o_custkey, o_orderdate], file_type=parquet, predicate=o_orderdate@2 >= 1994-01-01 AND o_orderdate@2 < 1995-01-01, pruning_predicate=o_orderdate_null_count@1 != row_count@2 AND o_orderdate_max@0 >= 1994-01-01 AND o_orderdate_null_count@1 != row_count@2 AND o_orderdate_min@3 < 1995-01-01, required_guarantees=[] - └────────────────────────────────────────────────── - "); - Ok(()) - } - - #[tokio::test] - async fn test_tpch_6() -> Result<(), Box> { - let plan = test_tpch_query(6).await?; - assert_snapshot!(plan, @r" - ┌───── DistributedExec ── Tasks: t0:[p0] - │ ProjectionExec: expr=[sum(lineitem.l_extendedprice * lineitem.l_discount)@0 as revenue] - │ AggregateExec: mode=Final, gby=[], aggr=[sum(lineitem.l_extendedprice * lineitem.l_discount)] - │ CoalescePartitionsExec - │ [Stage 1] => NetworkCoalesceExec: output_partitions=8, input_tasks=4 - └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0,p1] t1:[p2,p3] t2:[p4,p5] t3:[p6,p7] - │ AggregateExec: mode=Partial, gby=[], aggr=[sum(lineitem.l_extendedprice * lineitem.l_discount)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: l_shipdate@3 >= 1994-01-01 AND l_shipdate@3 < 1995-01-01 AND l_discount@2 >= Some(5),15,2 AND l_discount@2 <= Some(7),15,2 AND l_quantity@0 < Some(2400),15,2, projection=[l_extendedprice@1, l_discount@2] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/data/lineitem/1.parquet:.., /testdata/tpch/data/lineitem/10.parquet:.., /testdata/tpch/data/lineitem/11.parquet:..], [/testdata/tpch/data/lineitem/11.parquet:.., /testdata/tpch/data/lineitem/12.parquet:.., /testdata/tpch/data/lineitem/13.parquet:.., /testdata/tpch/data/lineitem/14.parquet:..], [/testdata/tpch/data/lineitem/14.parquet:.., /testdata/tpch/data/lineitem/15.parquet:.., /testdata/tpch/data/lineitem/16.parquet:..], [/testdata/tpch/data/lineitem/16.parquet:.., /testdata/tpch/data/lineitem/2.parquet:.., /testdata/tpch/data/lineitem/3.parquet:.., /testdata/tpch/data/lineitem/4.parquet:..], [/testdata/tpch/data/lineitem/4.parquet:.., /testdata/tpch/data/lineitem/5.parquet:.., /testdata/tpch/data/lineitem/6.parquet:.., /testdata/tpch/data/lineitem/7.parquet:..], ...]}, projection=[l_quantity, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@3 >= 1994-01-01 AND l_shipdate@3 < 1995-01-01 AND l_discount@2 >= Some(5),15,2 AND l_discount@2 <= Some(7),15,2 AND l_quantity@0 < Some(2400),15,2, pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1994-01-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 < 1995-01-01 AND l_discount_null_count@5 != row_count@2 AND l_discount_max@4 >= Some(5),15,2 AND l_discount_null_count@5 != row_count@2 AND l_discount_min@6 <= Some(7),15,2 AND l_quantity_null_count@8 != row_count@2 AND l_quantity_min@7 < Some(2400),15,2, required_guarantees=[] - └────────────────────────────────────────────────── - "); - Ok(()) - } - - #[tokio::test] - async fn test_tpch_7() -> Result<(), Box> { - let plan = test_tpch_query(7).await?; - assert_snapshot!(plan, @r" - ┌───── DistributedExec ── Tasks: t0:[p0] - │ SortPreservingMergeExec: [supp_nation@0 ASC NULLS LAST, cust_nation@1 ASC NULLS LAST, l_year@2 ASC NULLS LAST] - │ [Stage 8] => NetworkCoalesceExec: output_partitions=24, input_tasks=4 - └────────────────────────────────────────────────── - ┌───── Stage 8 ── Tasks: t0:[p0,p1,p2,p3,p4,p5] t1:[p0,p1,p2,p3,p4,p5] t2:[p0,p1,p2,p3,p4,p5] t3:[p0,p1,p2,p3,p4,p5] - │ SortExec: expr=[supp_nation@0 ASC NULLS LAST, cust_nation@1 ASC NULLS LAST, l_year@2 ASC NULLS LAST], preserve_partitioning=[true] - │ ProjectionExec: expr=[supp_nation@0 as supp_nation, cust_nation@1 as cust_nation, l_year@2 as l_year, sum(shipping.volume)@3 as revenue] - │ AggregateExec: mode=FinalPartitioned, gby=[supp_nation@0 as supp_nation, cust_nation@1 as cust_nation, l_year@2 as l_year], aggr=[sum(shipping.volume)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ [Stage 7] => NetworkShuffleExec: output_partitions=6, input_tasks=1 - └────────────────────────────────────────────────── - ┌───── Stage 7 ── Tasks: t0:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9,p10,p11,p12,p13,p14,p15,p16,p17,p18,p19,p20,p21,p22,p23] - │ RepartitionExec: partitioning=Hash([supp_nation@0, cust_nation@1, l_year@2], 24), input_partitions=6 - │ AggregateExec: mode=Partial, gby=[supp_nation@0 as supp_nation, cust_nation@1 as cust_nation, l_year@2 as l_year], aggr=[sum(shipping.volume)] - │ ProjectionExec: expr=[n_name@4 as supp_nation, n_name@0 as cust_nation, date_part(YEAR, l_shipdate@3) as l_year, l_extendedprice@1 * (Some(1),20,0 - l_discount@2) as volume] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, c_nationkey@3)], filter=n_name@0 = FRANCE AND n_name@1 = GERMANY OR n_name@0 = GERMANY AND n_name@1 = FRANCE, projection=[n_name@1, l_extendedprice@2, l_discount@3, l_shipdate@4, n_name@6] - │ CoalescePartitionsExec - │ [Stage 1] => NetworkCoalesceExec: output_partitions=8, input_tasks=4 - │ ProjectionExec: expr=[l_extendedprice@1 as l_extendedprice, l_discount@2 as l_discount, l_shipdate@3 as l_shipdate, c_nationkey@4 as c_nationkey, n_name@0 as n_name] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@0)], projection=[n_name@1, l_extendedprice@3, l_discount@4, l_shipdate@5, c_nationkey@6] - │ CoalescePartitionsExec - │ [Stage 2] => NetworkCoalesceExec: output_partitions=8, input_tasks=4 - │ ProjectionExec: expr=[s_nationkey@1 as s_nationkey, l_extendedprice@2 as l_extendedprice, l_discount@3 as l_discount, l_shipdate@4 as l_shipdate, c_nationkey@0 as c_nationkey] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(c_custkey@0, o_custkey@4)], projection=[c_nationkey@1, s_nationkey@2, l_extendedprice@3, l_discount@4, l_shipdate@5] - │ CoalesceBatchesExec: target_batch_size=8192 - │ [Stage 3] => NetworkShuffleExec: output_partitions=6, input_tasks=3 - │ CoalesceBatchesExec: target_batch_size=8192 - │ [Stage 6] => NetworkShuffleExec: output_partitions=6, input_tasks=3 - └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0,p1] t1:[p2,p3] t2:[p4,p5] t3:[p6,p7] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: n_name@1 = GERMANY OR n_name@1 = FRANCE - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/data/nation/1.parquet, /testdata/tpch/data/nation/10.parquet, /testdata/tpch/data/nation/11.parquet], [/testdata/tpch/data/nation/12.parquet, /testdata/tpch/data/nation/13.parquet, /testdata/tpch/data/nation/14.parquet], [/testdata/tpch/data/nation/15.parquet, /testdata/tpch/data/nation/16.parquet, /testdata/tpch/data/nation/2.parquet], [/testdata/tpch/data/nation/3.parquet, /testdata/tpch/data/nation/4.parquet, /testdata/tpch/data/nation/5.parquet], [/testdata/tpch/data/nation/6.parquet, /testdata/tpch/data/nation/7.parquet, /testdata/tpch/data/nation/8.parquet], ...]}, projection=[n_nationkey, n_name], file_type=parquet, predicate=n_name@1 = GERMANY OR n_name@1 = FRANCE, pruning_predicate=n_name_null_count@2 != row_count@3 AND n_name_min@0 <= GERMANY AND GERMANY <= n_name_max@1 OR n_name_null_count@2 != row_count@3 AND n_name_min@0 <= FRANCE AND FRANCE <= n_name_max@1, required_guarantees=[n_name in (FRANCE, GERMANY)] - └────────────────────────────────────────────────── - ┌───── Stage 2 ── Tasks: t0:[p0,p1] t1:[p2,p3] t2:[p4,p5] t3:[p6,p7] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: n_name@1 = FRANCE OR n_name@1 = GERMANY - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/data/nation/1.parquet, /testdata/tpch/data/nation/10.parquet, /testdata/tpch/data/nation/11.parquet], [/testdata/tpch/data/nation/12.parquet, /testdata/tpch/data/nation/13.parquet, /testdata/tpch/data/nation/14.parquet], [/testdata/tpch/data/nation/15.parquet, /testdata/tpch/data/nation/16.parquet, /testdata/tpch/data/nation/2.parquet], [/testdata/tpch/data/nation/3.parquet, /testdata/tpch/data/nation/4.parquet, /testdata/tpch/data/nation/5.parquet], [/testdata/tpch/data/nation/6.parquet, /testdata/tpch/data/nation/7.parquet, /testdata/tpch/data/nation/8.parquet], ...]}, projection=[n_nationkey, n_name], file_type=parquet, predicate=n_name@1 = FRANCE OR n_name@1 = GERMANY, pruning_predicate=n_name_null_count@2 != row_count@3 AND n_name_min@0 <= FRANCE AND FRANCE <= n_name_max@1 OR n_name_null_count@2 != row_count@3 AND n_name_min@0 <= GERMANY AND GERMANY <= n_name_max@1, required_guarantees=[n_name in (FRANCE, GERMANY)] - └────────────────────────────────────────────────── - ┌───── Stage 3 ── Tasks: t0:[p0,p1,p2,p3,p4,p5] t1:[p0,p1,p2,p3,p4,p5] t2:[p0,p1,p2,p3,p4,p5] - │ RepartitionExec: partitioning=Hash([c_custkey@0], 6), input_partitions=2 - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/data/customer/1.parquet, /testdata/tpch/data/customer/10.parquet, /testdata/tpch/data/customer/11.parquet], [/testdata/tpch/data/customer/12.parquet, /testdata/tpch/data/customer/13.parquet, /testdata/tpch/data/customer/14.parquet], [/testdata/tpch/data/customer/15.parquet, /testdata/tpch/data/customer/16.parquet, /testdata/tpch/data/customer/2.parquet], [/testdata/tpch/data/customer/3.parquet, /testdata/tpch/data/customer/4.parquet, /testdata/tpch/data/customer/5.parquet], [/testdata/tpch/data/customer/6.parquet, /testdata/tpch/data/customer/7.parquet, /testdata/tpch/data/customer/8.parquet], ...]}, projection=[c_custkey, c_nationkey], file_type=parquet, predicate=DynamicFilterPhysicalExpr [ true ] - └────────────────────────────────────────────────── - ┌───── Stage 6 ── Tasks: t0:[p0,p1,p2,p3,p4,p5] t1:[p0,p1,p2,p3,p4,p5] t2:[p0,p1,p2,p3,p4,p5] - │ RepartitionExec: partitioning=Hash([o_custkey@4], 6), input_partitions=6 - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(l_orderkey@1, o_orderkey@0)], projection=[s_nationkey@0, l_extendedprice@2, l_discount@3, l_shipdate@4, o_custkey@6] - │ CoalesceBatchesExec: target_batch_size=8192 - │ [Stage 4] => NetworkShuffleExec: output_partitions=6, input_tasks=1 - │ CoalesceBatchesExec: target_batch_size=8192 - │ [Stage 5] => NetworkShuffleExec: output_partitions=6, input_tasks=3 - └────────────────────────────────────────────────── - ┌───── Stage 4 ── Tasks: t0:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9,p10,p11,p12,p13,p14,p15,p16,p17] - │ RepartitionExec: partitioning=Hash([l_orderkey@1], 18), input_partitions=6 - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_suppkey@0, l_suppkey@1)], projection=[s_nationkey@1, l_orderkey@2, l_extendedprice@4, l_discount@5, l_shipdate@6] - │ CoalescePartitionsExec - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/data/supplier/1.parquet, /testdata/tpch/data/supplier/10.parquet, /testdata/tpch/data/supplier/11.parquet], [/testdata/tpch/data/supplier/12.parquet, /testdata/tpch/data/supplier/13.parquet, /testdata/tpch/data/supplier/14.parquet], [/testdata/tpch/data/supplier/15.parquet, /testdata/tpch/data/supplier/16.parquet, /testdata/tpch/data/supplier/2.parquet], [/testdata/tpch/data/supplier/3.parquet, /testdata/tpch/data/supplier/4.parquet, /testdata/tpch/data/supplier/5.parquet], [/testdata/tpch/data/supplier/6.parquet, /testdata/tpch/data/supplier/7.parquet, /testdata/tpch/data/supplier/8.parquet], ...]}, projection=[s_suppkey, s_nationkey], file_type=parquet - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: l_shipdate@4 >= 1995-01-01 AND l_shipdate@4 <= 1996-12-31 - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/data/lineitem/1.parquet:.., /testdata/tpch/data/lineitem/10.parquet:.., /testdata/tpch/data/lineitem/11.parquet:..], [/testdata/tpch/data/lineitem/11.parquet:.., /testdata/tpch/data/lineitem/12.parquet:.., /testdata/tpch/data/lineitem/13.parquet:.., /testdata/tpch/data/lineitem/14.parquet:..], [/testdata/tpch/data/lineitem/14.parquet:.., /testdata/tpch/data/lineitem/15.parquet:.., /testdata/tpch/data/lineitem/16.parquet:..], [/testdata/tpch/data/lineitem/16.parquet:.., /testdata/tpch/data/lineitem/2.parquet:.., /testdata/tpch/data/lineitem/3.parquet:.., /testdata/tpch/data/lineitem/4.parquet:..], [/testdata/tpch/data/lineitem/4.parquet:.., /testdata/tpch/data/lineitem/5.parquet:.., /testdata/tpch/data/lineitem/6.parquet:.., /testdata/tpch/data/lineitem/7.parquet:..], ...]}, projection=[l_orderkey, l_suppkey, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@4 >= 1995-01-01 AND l_shipdate@4 <= 1996-12-31 AND DynamicFilterPhysicalExpr [ true ], pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1995-01-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 <= 1996-12-31, required_guarantees=[] - └────────────────────────────────────────────────── - ┌───── Stage 5 ── Tasks: t0:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9,p10,p11,p12,p13,p14,p15,p16,p17] t1:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9,p10,p11,p12,p13,p14,p15,p16,p17] t2:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9,p10,p11,p12,p13,p14,p15,p16,p17] - │ RepartitionExec: partitioning=Hash([o_orderkey@0], 18), input_partitions=2 - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/data/orders/1.parquet:.., /testdata/tpch/data/orders/10.parquet:.., /testdata/tpch/data/orders/11.parquet:..], [/testdata/tpch/data/orders/11.parquet:.., /testdata/tpch/data/orders/12.parquet:.., /testdata/tpch/data/orders/13.parquet:.., /testdata/tpch/data/orders/14.parquet:..], [/testdata/tpch/data/orders/14.parquet:.., /testdata/tpch/data/orders/15.parquet:.., /testdata/tpch/data/orders/16.parquet:..], [/testdata/tpch/data/orders/16.parquet:.., /testdata/tpch/data/orders/2.parquet:.., /testdata/tpch/data/orders/3.parquet:.., /testdata/tpch/data/orders/4.parquet:..], [/testdata/tpch/data/orders/4.parquet:.., /testdata/tpch/data/orders/5.parquet:.., /testdata/tpch/data/orders/6.parquet:.., /testdata/tpch/data/orders/7.parquet:..], ...]}, projection=[o_orderkey, o_custkey], file_type=parquet, predicate=DynamicFilterPhysicalExpr [ true ] AND DynamicFilterPhysicalExpr [ true ] - └────────────────────────────────────────────────── - "); - Ok(()) - } - - #[tokio::test] - async fn test_tpch_8() -> Result<(), Box> { - let plan = test_tpch_query(8).await?; - assert_snapshot!(plan, @r#" - ┌───── DistributedExec ── Tasks: t0:[p0] - │ SortPreservingMergeExec: [o_year@0 ASC NULLS LAST] - │ [Stage 8] => NetworkCoalesceExec: output_partitions=24, input_tasks=4 - └────────────────────────────────────────────────── - ┌───── Stage 8 ── Tasks: t0:[p0,p1,p2,p3,p4,p5] t1:[p0,p1,p2,p3,p4,p5] t2:[p0,p1,p2,p3,p4,p5] t3:[p0,p1,p2,p3,p4,p5] - │ SortExec: expr=[o_year@0 ASC NULLS LAST], preserve_partitioning=[true] - │ ProjectionExec: expr=[o_year@0 as o_year, sum(CASE WHEN all_nations.nation = Utf8("BRAZIL") THEN all_nations.volume ELSE Int64(0) END)@1 / sum(all_nations.volume)@2 as mkt_share] - │ AggregateExec: mode=FinalPartitioned, gby=[o_year@0 as o_year], aggr=[sum(CASE WHEN all_nations.nation = Utf8("BRAZIL") THEN all_nations.volume ELSE Int64(0) END), sum(all_nations.volume)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ [Stage 7] => NetworkShuffleExec: output_partitions=6, input_tasks=1 - └────────────────────────────────────────────────── - ┌───── Stage 7 ── Tasks: t0:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9,p10,p11,p12,p13,p14,p15,p16,p17,p18,p19,p20,p21,p22,p23] - │ RepartitionExec: partitioning=Hash([o_year@0], 24), input_partitions=6 - │ AggregateExec: mode=Partial, gby=[o_year@0 as o_year], aggr=[sum(CASE WHEN all_nations.nation = Utf8("BRAZIL") THEN all_nations.volume ELSE Int64(0) END), sum(all_nations.volume)] - │ ProjectionExec: expr=[date_part(YEAR, o_orderdate@2) as o_year, l_extendedprice@0 * (Some(1),20,0 - l_discount@1) as volume, n_name@3 as nation] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(r_regionkey@0, n_regionkey@3)], projection=[l_extendedprice@1, l_discount@2, o_orderdate@3, n_name@5] - │ CoalescePartitionsExec - │ [Stage 1] => NetworkCoalesceExec: output_partitions=8, input_tasks=4 - │ ProjectionExec: expr=[l_extendedprice@1 as l_extendedprice, l_discount@2 as l_discount, o_orderdate@3 as o_orderdate, n_regionkey@4 as n_regionkey, n_name@0 as n_name] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@2)], projection=[n_name@1, l_extendedprice@2, l_discount@3, o_orderdate@5, n_regionkey@6] - │ CoalescePartitionsExec - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/data/nation/1.parquet, /testdata/tpch/data/nation/10.parquet, /testdata/tpch/data/nation/11.parquet], [/testdata/tpch/data/nation/12.parquet, /testdata/tpch/data/nation/13.parquet, /testdata/tpch/data/nation/14.parquet], [/testdata/tpch/data/nation/15.parquet, /testdata/tpch/data/nation/16.parquet, /testdata/tpch/data/nation/2.parquet], [/testdata/tpch/data/nation/3.parquet, /testdata/tpch/data/nation/4.parquet, /testdata/tpch/data/nation/5.parquet], [/testdata/tpch/data/nation/6.parquet, /testdata/tpch/data/nation/7.parquet, /testdata/tpch/data/nation/8.parquet], ...]}, projection=[n_nationkey, n_name], file_type=parquet - │ ProjectionExec: expr=[l_extendedprice@1 as l_extendedprice, l_discount@2 as l_discount, s_nationkey@3 as s_nationkey, o_orderdate@4 as o_orderdate, n_regionkey@0 as n_regionkey] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, c_nationkey@4)], projection=[n_regionkey@1, l_extendedprice@2, l_discount@3, s_nationkey@4, o_orderdate@5] - │ CoalescePartitionsExec - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/data/nation/1.parquet, /testdata/tpch/data/nation/10.parquet, /testdata/tpch/data/nation/11.parquet], [/testdata/tpch/data/nation/12.parquet, /testdata/tpch/data/nation/13.parquet, /testdata/tpch/data/nation/14.parquet], [/testdata/tpch/data/nation/15.parquet, /testdata/tpch/data/nation/16.parquet, /testdata/tpch/data/nation/2.parquet], [/testdata/tpch/data/nation/3.parquet, /testdata/tpch/data/nation/4.parquet, /testdata/tpch/data/nation/5.parquet], [/testdata/tpch/data/nation/6.parquet, /testdata/tpch/data/nation/7.parquet, /testdata/tpch/data/nation/8.parquet], ...]}, projection=[n_nationkey, n_regionkey], file_type=parquet - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(o_custkey@3, c_custkey@0)], projection=[l_extendedprice@0, l_discount@1, s_nationkey@2, o_orderdate@4, c_nationkey@6] - │ CoalesceBatchesExec: target_batch_size=8192 - │ [Stage 5] => NetworkShuffleExec: output_partitions=6, input_tasks=3 - │ CoalesceBatchesExec: target_batch_size=8192 - │ [Stage 6] => NetworkShuffleExec: output_partitions=6, input_tasks=3 - └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0,p1] t1:[p2,p3] t2:[p4,p5] t3:[p6,p7] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: r_name@1 = AMERICA, projection=[r_regionkey@0] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/data/region/1.parquet, /testdata/tpch/data/region/10.parquet, /testdata/tpch/data/region/11.parquet], [/testdata/tpch/data/region/12.parquet, /testdata/tpch/data/region/13.parquet, /testdata/tpch/data/region/14.parquet], [/testdata/tpch/data/region/15.parquet, /testdata/tpch/data/region/16.parquet, /testdata/tpch/data/region/2.parquet], [/testdata/tpch/data/region/3.parquet, /testdata/tpch/data/region/4.parquet, /testdata/tpch/data/region/5.parquet], [/testdata/tpch/data/region/6.parquet, /testdata/tpch/data/region/7.parquet, /testdata/tpch/data/region/8.parquet], ...]}, projection=[r_regionkey, r_name], file_type=parquet, predicate=r_name@1 = AMERICA, pruning_predicate=r_name_null_count@2 != row_count@3 AND r_name_min@0 <= AMERICA AND AMERICA <= r_name_max@1, required_guarantees=[r_name in (AMERICA)] - └────────────────────────────────────────────────── - ┌───── Stage 5 ── Tasks: t0:[p0,p1,p2,p3,p4,p5] t1:[p0,p1,p2,p3,p4,p5] t2:[p0,p1,p2,p3,p4,p5] - │ RepartitionExec: partitioning=Hash([o_custkey@3], 6), input_partitions=6 - │ ProjectionExec: expr=[l_extendedprice@2 as l_extendedprice, l_discount@3 as l_discount, s_nationkey@4 as s_nationkey, o_custkey@0 as o_custkey, o_orderdate@1 as o_orderdate] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(o_orderkey@0, l_orderkey@0)], projection=[o_custkey@1, o_orderdate@2, l_extendedprice@4, l_discount@5, s_nationkey@6] - │ CoalesceBatchesExec: target_batch_size=8192 - │ [Stage 2] => NetworkShuffleExec: output_partitions=6, input_tasks=3 - │ CoalesceBatchesExec: target_batch_size=8192 - │ [Stage 4] => NetworkShuffleExec: output_partitions=6, input_tasks=1 - └────────────────────────────────────────────────── - ┌───── Stage 2 ── Tasks: t0:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9,p10,p11,p12,p13,p14,p15,p16,p17] t1:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9,p10,p11,p12,p13,p14,p15,p16,p17] t2:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9,p10,p11,p12,p13,p14,p15,p16,p17] - │ RepartitionExec: partitioning=Hash([o_orderkey@0], 18), input_partitions=2 - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: o_orderdate@2 >= 1995-01-01 AND o_orderdate@2 <= 1996-12-31 - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/data/orders/1.parquet:.., /testdata/tpch/data/orders/10.parquet:.., /testdata/tpch/data/orders/11.parquet:..], [/testdata/tpch/data/orders/11.parquet:.., /testdata/tpch/data/orders/12.parquet:.., /testdata/tpch/data/orders/13.parquet:.., /testdata/tpch/data/orders/14.parquet:..], [/testdata/tpch/data/orders/14.parquet:.., /testdata/tpch/data/orders/15.parquet:.., /testdata/tpch/data/orders/16.parquet:..], [/testdata/tpch/data/orders/16.parquet:.., /testdata/tpch/data/orders/2.parquet:.., /testdata/tpch/data/orders/3.parquet:.., /testdata/tpch/data/orders/4.parquet:..], [/testdata/tpch/data/orders/4.parquet:.., /testdata/tpch/data/orders/5.parquet:.., /testdata/tpch/data/orders/6.parquet:.., /testdata/tpch/data/orders/7.parquet:..], ...]}, projection=[o_orderkey, o_custkey, o_orderdate], file_type=parquet, predicate=o_orderdate@2 >= 1995-01-01 AND o_orderdate@2 <= 1996-12-31, pruning_predicate=o_orderdate_null_count@1 != row_count@2 AND o_orderdate_max@0 >= 1995-01-01 AND o_orderdate_null_count@1 != row_count@2 AND o_orderdate_min@3 <= 1996-12-31, required_guarantees=[] - └────────────────────────────────────────────────── - ┌───── Stage 4 ── Tasks: t0:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9,p10,p11,p12,p13,p14,p15,p16,p17] - │ RepartitionExec: partitioning=Hash([l_orderkey@0], 18), input_partitions=6 - │ ProjectionExec: expr=[l_orderkey@1 as l_orderkey, l_extendedprice@2 as l_extendedprice, l_discount@3 as l_discount, s_nationkey@0 as s_nationkey] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_suppkey@0, l_suppkey@1)], projection=[s_nationkey@1, l_orderkey@2, l_extendedprice@4, l_discount@5] - │ CoalescePartitionsExec - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/data/supplier/1.parquet, /testdata/tpch/data/supplier/10.parquet, /testdata/tpch/data/supplier/11.parquet], [/testdata/tpch/data/supplier/12.parquet, /testdata/tpch/data/supplier/13.parquet, /testdata/tpch/data/supplier/14.parquet], [/testdata/tpch/data/supplier/15.parquet, /testdata/tpch/data/supplier/16.parquet, /testdata/tpch/data/supplier/2.parquet], [/testdata/tpch/data/supplier/3.parquet, /testdata/tpch/data/supplier/4.parquet, /testdata/tpch/data/supplier/5.parquet], [/testdata/tpch/data/supplier/6.parquet, /testdata/tpch/data/supplier/7.parquet, /testdata/tpch/data/supplier/8.parquet], ...]}, projection=[s_suppkey, s_nationkey], file_type=parquet - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(p_partkey@0, l_partkey@1)], projection=[l_orderkey@1, l_suppkey@3, l_extendedprice@4, l_discount@5] - │ CoalescePartitionsExec - │ [Stage 3] => NetworkCoalesceExec: output_partitions=8, input_tasks=4 - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/data/lineitem/1.parquet:.., /testdata/tpch/data/lineitem/10.parquet:.., /testdata/tpch/data/lineitem/11.parquet:..], [/testdata/tpch/data/lineitem/11.parquet:.., /testdata/tpch/data/lineitem/12.parquet:.., /testdata/tpch/data/lineitem/13.parquet:.., /testdata/tpch/data/lineitem/14.parquet:..], [/testdata/tpch/data/lineitem/14.parquet:.., /testdata/tpch/data/lineitem/15.parquet:.., /testdata/tpch/data/lineitem/16.parquet:..], [/testdata/tpch/data/lineitem/16.parquet:.., /testdata/tpch/data/lineitem/2.parquet:.., /testdata/tpch/data/lineitem/3.parquet:.., /testdata/tpch/data/lineitem/4.parquet:..], [/testdata/tpch/data/lineitem/4.parquet:.., /testdata/tpch/data/lineitem/5.parquet:.., /testdata/tpch/data/lineitem/6.parquet:.., /testdata/tpch/data/lineitem/7.parquet:..], ...]}, projection=[l_orderkey, l_partkey, l_suppkey, l_extendedprice, l_discount], file_type=parquet, predicate=DynamicFilterPhysicalExpr [ true ] AND DynamicFilterPhysicalExpr [ true ] AND DynamicFilterPhysicalExpr [ true ] - └────────────────────────────────────────────────── - ┌───── Stage 3 ── Tasks: t0:[p0,p1] t1:[p2,p3] t2:[p4,p5] t3:[p6,p7] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: p_type@1 = ECONOMY ANODIZED STEEL, projection=[p_partkey@0] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/data/part/1.parquet, /testdata/tpch/data/part/10.parquet, /testdata/tpch/data/part/11.parquet], [/testdata/tpch/data/part/12.parquet, /testdata/tpch/data/part/13.parquet, /testdata/tpch/data/part/14.parquet], [/testdata/tpch/data/part/15.parquet, /testdata/tpch/data/part/16.parquet, /testdata/tpch/data/part/2.parquet], [/testdata/tpch/data/part/3.parquet, /testdata/tpch/data/part/4.parquet, /testdata/tpch/data/part/5.parquet], [/testdata/tpch/data/part/6.parquet, /testdata/tpch/data/part/7.parquet, /testdata/tpch/data/part/8.parquet], ...]}, projection=[p_partkey, p_type], file_type=parquet, predicate=p_type@1 = ECONOMY ANODIZED STEEL, pruning_predicate=p_type_null_count@2 != row_count@3 AND p_type_min@0 <= ECONOMY ANODIZED STEEL AND ECONOMY ANODIZED STEEL <= p_type_max@1, required_guarantees=[p_type in (ECONOMY ANODIZED STEEL)] - └────────────────────────────────────────────────── - ┌───── Stage 6 ── Tasks: t0:[p0,p1,p2,p3,p4,p5] t1:[p0,p1,p2,p3,p4,p5] t2:[p0,p1,p2,p3,p4,p5] - │ RepartitionExec: partitioning=Hash([c_custkey@0], 6), input_partitions=2 - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/data/customer/1.parquet, /testdata/tpch/data/customer/10.parquet, /testdata/tpch/data/customer/11.parquet], [/testdata/tpch/data/customer/12.parquet, /testdata/tpch/data/customer/13.parquet, /testdata/tpch/data/customer/14.parquet], [/testdata/tpch/data/customer/15.parquet, /testdata/tpch/data/customer/16.parquet, /testdata/tpch/data/customer/2.parquet], [/testdata/tpch/data/customer/3.parquet, /testdata/tpch/data/customer/4.parquet, /testdata/tpch/data/customer/5.parquet], [/testdata/tpch/data/customer/6.parquet, /testdata/tpch/data/customer/7.parquet, /testdata/tpch/data/customer/8.parquet], ...]}, projection=[c_custkey, c_nationkey], file_type=parquet, predicate=DynamicFilterPhysicalExpr [ true ] AND DynamicFilterPhysicalExpr [ true ] - └────────────────────────────────────────────────── - "#); - Ok(()) - } - - #[tokio::test] - async fn test_tpch_9() -> Result<(), Box> { - let plan = test_tpch_query(9).await?; - assert_snapshot!(plan, @r" - ┌───── DistributedExec ── Tasks: t0:[p0] - │ SortPreservingMergeExec: [nation@0 ASC NULLS LAST, o_year@1 DESC] - │ [Stage 7] => NetworkCoalesceExec: output_partitions=24, input_tasks=4 - └────────────────────────────────────────────────── - ┌───── Stage 7 ── Tasks: t0:[p0,p1,p2,p3,p4,p5] t1:[p0,p1,p2,p3,p4,p5] t2:[p0,p1,p2,p3,p4,p5] t3:[p0,p1,p2,p3,p4,p5] - │ SortExec: expr=[nation@0 ASC NULLS LAST, o_year@1 DESC], preserve_partitioning=[true] - │ ProjectionExec: expr=[nation@0 as nation, o_year@1 as o_year, sum(profit.amount)@2 as sum_profit] - │ AggregateExec: mode=FinalPartitioned, gby=[nation@0 as nation, o_year@1 as o_year], aggr=[sum(profit.amount)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ [Stage 6] => NetworkShuffleExec: output_partitions=6, input_tasks=1 - └────────────────────────────────────────────────── - ┌───── Stage 6 ── Tasks: t0:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9,p10,p11,p12,p13,p14,p15,p16,p17,p18,p19,p20,p21,p22,p23] - │ RepartitionExec: partitioning=Hash([nation@0, o_year@1], 24), input_partitions=6 - │ AggregateExec: mode=Partial, gby=[nation@0 as nation, o_year@1 as o_year], aggr=[sum(profit.amount)] - │ ProjectionExec: expr=[n_name@0 as nation, date_part(YEAR, o_orderdate@5) as o_year, l_extendedprice@2 * (Some(1),20,0 - l_discount@3) - ps_supplycost@4 * l_quantity@1 as amount] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@3)], projection=[n_name@1, l_quantity@2, l_extendedprice@3, l_discount@4, ps_supplycost@6, o_orderdate@7] - │ CoalescePartitionsExec - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/data/nation/1.parquet, /testdata/tpch/data/nation/10.parquet, /testdata/tpch/data/nation/11.parquet], [/testdata/tpch/data/nation/12.parquet, /testdata/tpch/data/nation/13.parquet, /testdata/tpch/data/nation/14.parquet], [/testdata/tpch/data/nation/15.parquet, /testdata/tpch/data/nation/16.parquet, /testdata/tpch/data/nation/2.parquet], [/testdata/tpch/data/nation/3.parquet, /testdata/tpch/data/nation/4.parquet, /testdata/tpch/data/nation/5.parquet], [/testdata/tpch/data/nation/6.parquet, /testdata/tpch/data/nation/7.parquet, /testdata/tpch/data/nation/8.parquet], ...]}, projection=[n_nationkey, n_name], file_type=parquet - │ ProjectionExec: expr=[l_quantity@1 as l_quantity, l_extendedprice@2 as l_extendedprice, l_discount@3 as l_discount, s_nationkey@4 as s_nationkey, ps_supplycost@5 as ps_supplycost, o_orderdate@0 as o_orderdate] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(o_orderkey@0, l_orderkey@0)], projection=[o_orderdate@1, l_quantity@3, l_extendedprice@4, l_discount@5, s_nationkey@6, ps_supplycost@7] - │ CoalesceBatchesExec: target_batch_size=8192 - │ [Stage 1] => NetworkShuffleExec: output_partitions=6, input_tasks=3 - │ CoalesceBatchesExec: target_batch_size=8192 - │ [Stage 5] => NetworkShuffleExec: output_partitions=6, input_tasks=3 - └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0,p1,p2,p3,p4,p5] t1:[p0,p1,p2,p3,p4,p5] t2:[p0,p1,p2,p3,p4,p5] - │ RepartitionExec: partitioning=Hash([o_orderkey@0], 6), input_partitions=2 - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/data/orders/1.parquet:.., /testdata/tpch/data/orders/10.parquet:.., /testdata/tpch/data/orders/11.parquet:..], [/testdata/tpch/data/orders/11.parquet:.., /testdata/tpch/data/orders/12.parquet:.., /testdata/tpch/data/orders/13.parquet:.., /testdata/tpch/data/orders/14.parquet:..], [/testdata/tpch/data/orders/14.parquet:.., /testdata/tpch/data/orders/15.parquet:.., /testdata/tpch/data/orders/16.parquet:..], [/testdata/tpch/data/orders/16.parquet:.., /testdata/tpch/data/orders/2.parquet:.., /testdata/tpch/data/orders/3.parquet:.., /testdata/tpch/data/orders/4.parquet:..], [/testdata/tpch/data/orders/4.parquet:.., /testdata/tpch/data/orders/5.parquet:.., /testdata/tpch/data/orders/6.parquet:.., /testdata/tpch/data/orders/7.parquet:..], ...]}, projection=[o_orderkey, o_orderdate], file_type=parquet - └────────────────────────────────────────────────── - ┌───── Stage 5 ── Tasks: t0:[p0,p1,p2,p3,p4,p5] t1:[p0,p1,p2,p3,p4,p5] t2:[p0,p1,p2,p3,p4,p5] - │ RepartitionExec: partitioning=Hash([l_orderkey@0], 6), input_partitions=6 - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(l_suppkey@2, ps_suppkey@1), (l_partkey@1, ps_partkey@0)], projection=[l_orderkey@0, l_quantity@3, l_extendedprice@4, l_discount@5, s_nationkey@6, ps_supplycost@9] - │ CoalesceBatchesExec: target_batch_size=8192 - │ [Stage 3] => NetworkShuffleExec: output_partitions=6, input_tasks=1 - │ CoalesceBatchesExec: target_batch_size=8192 - │ [Stage 4] => NetworkShuffleExec: output_partitions=6, input_tasks=3 - └────────────────────────────────────────────────── - ┌───── Stage 3 ── Tasks: t0:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9,p10,p11,p12,p13,p14,p15,p16,p17] - │ RepartitionExec: partitioning=Hash([l_suppkey@2, l_partkey@1], 18), input_partitions=6 - │ ProjectionExec: expr=[l_orderkey@1 as l_orderkey, l_partkey@2 as l_partkey, l_suppkey@3 as l_suppkey, l_quantity@4 as l_quantity, l_extendedprice@5 as l_extendedprice, l_discount@6 as l_discount, s_nationkey@0 as s_nationkey] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_suppkey@0, l_suppkey@2)], projection=[s_nationkey@1, l_orderkey@2, l_partkey@3, l_suppkey@4, l_quantity@5, l_extendedprice@6, l_discount@7] - │ CoalescePartitionsExec - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/data/supplier/1.parquet, /testdata/tpch/data/supplier/10.parquet, /testdata/tpch/data/supplier/11.parquet], [/testdata/tpch/data/supplier/12.parquet, /testdata/tpch/data/supplier/13.parquet, /testdata/tpch/data/supplier/14.parquet], [/testdata/tpch/data/supplier/15.parquet, /testdata/tpch/data/supplier/16.parquet, /testdata/tpch/data/supplier/2.parquet], [/testdata/tpch/data/supplier/3.parquet, /testdata/tpch/data/supplier/4.parquet, /testdata/tpch/data/supplier/5.parquet], [/testdata/tpch/data/supplier/6.parquet, /testdata/tpch/data/supplier/7.parquet, /testdata/tpch/data/supplier/8.parquet], ...]}, projection=[s_suppkey, s_nationkey], file_type=parquet - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(p_partkey@0, l_partkey@1)], projection=[l_orderkey@1, l_partkey@2, l_suppkey@3, l_quantity@4, l_extendedprice@5, l_discount@6] - │ CoalescePartitionsExec - │ [Stage 2] => NetworkCoalesceExec: output_partitions=8, input_tasks=4 - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/data/lineitem/1.parquet:.., /testdata/tpch/data/lineitem/10.parquet:.., /testdata/tpch/data/lineitem/11.parquet:..], [/testdata/tpch/data/lineitem/11.parquet:.., /testdata/tpch/data/lineitem/12.parquet:.., /testdata/tpch/data/lineitem/13.parquet:.., /testdata/tpch/data/lineitem/14.parquet:..], [/testdata/tpch/data/lineitem/14.parquet:.., /testdata/tpch/data/lineitem/15.parquet:.., /testdata/tpch/data/lineitem/16.parquet:..], [/testdata/tpch/data/lineitem/16.parquet:.., /testdata/tpch/data/lineitem/2.parquet:.., /testdata/tpch/data/lineitem/3.parquet:.., /testdata/tpch/data/lineitem/4.parquet:..], [/testdata/tpch/data/lineitem/4.parquet:.., /testdata/tpch/data/lineitem/5.parquet:.., /testdata/tpch/data/lineitem/6.parquet:.., /testdata/tpch/data/lineitem/7.parquet:..], ...]}, projection=[l_orderkey, l_partkey, l_suppkey, l_quantity, l_extendedprice, l_discount], file_type=parquet, predicate=DynamicFilterPhysicalExpr [ true ] AND DynamicFilterPhysicalExpr [ true ] AND DynamicFilterPhysicalExpr [ true ] - └────────────────────────────────────────────────── - ┌───── Stage 2 ── Tasks: t0:[p0,p1] t1:[p2,p3] t2:[p4,p5] t3:[p6,p7] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: p_name@1 LIKE %green%, projection=[p_partkey@0] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/data/part/1.parquet, /testdata/tpch/data/part/10.parquet, /testdata/tpch/data/part/11.parquet], [/testdata/tpch/data/part/12.parquet, /testdata/tpch/data/part/13.parquet, /testdata/tpch/data/part/14.parquet], [/testdata/tpch/data/part/15.parquet, /testdata/tpch/data/part/16.parquet, /testdata/tpch/data/part/2.parquet], [/testdata/tpch/data/part/3.parquet, /testdata/tpch/data/part/4.parquet, /testdata/tpch/data/part/5.parquet], [/testdata/tpch/data/part/6.parquet, /testdata/tpch/data/part/7.parquet, /testdata/tpch/data/part/8.parquet], ...]}, projection=[p_partkey, p_name], file_type=parquet, predicate=p_name@1 LIKE %green% - └────────────────────────────────────────────────── - ┌───── Stage 4 ── Tasks: t0:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9,p10,p11,p12,p13,p14,p15,p16,p17] t1:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9,p10,p11,p12,p13,p14,p15,p16,p17] t2:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9,p10,p11,p12,p13,p14,p15,p16,p17] - │ RepartitionExec: partitioning=Hash([ps_suppkey@1, ps_partkey@0], 18), input_partitions=2 - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/data/partsupp/1.parquet:.., /testdata/tpch/data/partsupp/10.parquet:.., /testdata/tpch/data/partsupp/11.parquet:..], [/testdata/tpch/data/partsupp/11.parquet:.., /testdata/tpch/data/partsupp/12.parquet:.., /testdata/tpch/data/partsupp/13.parquet:.., /testdata/tpch/data/partsupp/14.parquet:..], [/testdata/tpch/data/partsupp/14.parquet:.., /testdata/tpch/data/partsupp/15.parquet:.., /testdata/tpch/data/partsupp/16.parquet:.., /testdata/tpch/data/partsupp/2.parquet:..], [/testdata/tpch/data/partsupp/2.parquet:.., /testdata/tpch/data/partsupp/3.parquet:.., /testdata/tpch/data/partsupp/4.parquet:..], [/testdata/tpch/data/partsupp/4.parquet:.., /testdata/tpch/data/partsupp/5.parquet:.., /testdata/tpch/data/partsupp/6.parquet:.., /testdata/tpch/data/partsupp/7.parquet:..], ...]}, projection=[ps_partkey, ps_suppkey, ps_supplycost], file_type=parquet, predicate=DynamicFilterPhysicalExpr [ true ] - └────────────────────────────────────────────────── - "); - Ok(()) - } - - #[tokio::test] - async fn test_tpch_10() -> Result<(), Box> { - let plan = test_tpch_query(10).await?; - assert_snapshot!(plan, @r" - ┌───── DistributedExec ── Tasks: t0:[p0] - │ SortPreservingMergeExec: [revenue@2 DESC] - │ [Stage 3] => NetworkCoalesceExec: output_partitions=24, input_tasks=4 - └────────────────────────────────────────────────── - ┌───── Stage 3 ── Tasks: t0:[p0,p1,p2,p3,p4,p5] t1:[p0,p1,p2,p3,p4,p5] t2:[p0,p1,p2,p3,p4,p5] t3:[p0,p1,p2,p3,p4,p5] - │ SortExec: expr=[revenue@2 DESC], preserve_partitioning=[true] - │ ProjectionExec: expr=[c_custkey@0 as c_custkey, c_name@1 as c_name, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@7 as revenue, c_acctbal@2 as c_acctbal, n_name@4 as n_name, c_address@5 as c_address, c_phone@3 as c_phone, c_comment@6 as c_comment] - │ AggregateExec: mode=FinalPartitioned, gby=[c_custkey@0 as c_custkey, c_name@1 as c_name, c_acctbal@2 as c_acctbal, c_phone@3 as c_phone, n_name@4 as n_name, c_address@5 as c_address, c_comment@6 as c_comment], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ [Stage 2] => NetworkShuffleExec: output_partitions=6, input_tasks=1 - └────────────────────────────────────────────────── - ┌───── Stage 2 ── Tasks: t0:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9,p10,p11,p12,p13,p14,p15,p16,p17,p18,p19,p20,p21,p22,p23] - │ RepartitionExec: partitioning=Hash([c_custkey@0, c_name@1, c_acctbal@2, c_phone@3, n_name@4, c_address@5, c_comment@6], 24), input_partitions=6 - │ AggregateExec: mode=Partial, gby=[c_custkey@0 as c_custkey, c_name@1 as c_name, c_acctbal@4 as c_acctbal, c_phone@3 as c_phone, n_name@8 as n_name, c_address@2 as c_address, c_comment@5 as c_comment], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] - │ ProjectionExec: expr=[c_custkey@1 as c_custkey, c_name@2 as c_name, c_address@3 as c_address, c_phone@4 as c_phone, c_acctbal@5 as c_acctbal, c_comment@6 as c_comment, l_extendedprice@7 as l_extendedprice, l_discount@8 as l_discount, n_name@0 as n_name] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, c_nationkey@3)], projection=[n_name@1, c_custkey@2, c_name@3, c_address@4, c_phone@6, c_acctbal@7, c_comment@8, l_extendedprice@9, l_discount@10] - │ CoalescePartitionsExec - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/data/nation/1.parquet, /testdata/tpch/data/nation/10.parquet, /testdata/tpch/data/nation/11.parquet], [/testdata/tpch/data/nation/12.parquet, /testdata/tpch/data/nation/13.parquet, /testdata/tpch/data/nation/14.parquet], [/testdata/tpch/data/nation/15.parquet, /testdata/tpch/data/nation/16.parquet, /testdata/tpch/data/nation/2.parquet], [/testdata/tpch/data/nation/3.parquet, /testdata/tpch/data/nation/4.parquet, /testdata/tpch/data/nation/5.parquet], [/testdata/tpch/data/nation/6.parquet, /testdata/tpch/data/nation/7.parquet, /testdata/tpch/data/nation/8.parquet], ...]}, projection=[n_nationkey, n_name], file_type=parquet - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(o_orderkey@7, l_orderkey@0)], projection=[c_custkey@0, c_name@1, c_address@2, c_nationkey@3, c_phone@4, c_acctbal@5, c_comment@6, l_extendedprice@9, l_discount@10] - │ CoalescePartitionsExec - │ ProjectionExec: expr=[c_custkey@1 as c_custkey, c_name@2 as c_name, c_address@3 as c_address, c_nationkey@4 as c_nationkey, c_phone@5 as c_phone, c_acctbal@6 as c_acctbal, c_comment@7 as c_comment, o_orderkey@0 as o_orderkey] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(o_custkey@1, c_custkey@0)], projection=[o_orderkey@0, c_custkey@2, c_name@3, c_address@4, c_nationkey@5, c_phone@6, c_acctbal@7, c_comment@8] - │ CoalescePartitionsExec - │ [Stage 1] => NetworkCoalesceExec: output_partitions=8, input_tasks=4 - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/data/customer/1.parquet, /testdata/tpch/data/customer/10.parquet, /testdata/tpch/data/customer/11.parquet], [/testdata/tpch/data/customer/12.parquet, /testdata/tpch/data/customer/13.parquet, /testdata/tpch/data/customer/14.parquet], [/testdata/tpch/data/customer/15.parquet, /testdata/tpch/data/customer/16.parquet, /testdata/tpch/data/customer/2.parquet], [/testdata/tpch/data/customer/3.parquet, /testdata/tpch/data/customer/4.parquet, /testdata/tpch/data/customer/5.parquet], [/testdata/tpch/data/customer/6.parquet, /testdata/tpch/data/customer/7.parquet, /testdata/tpch/data/customer/8.parquet], ...]}, projection=[c_custkey, c_name, c_address, c_nationkey, c_phone, c_acctbal, c_comment], file_type=parquet, predicate=DynamicFilterPhysicalExpr [ true ] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: l_returnflag@3 = R, projection=[l_orderkey@0, l_extendedprice@1, l_discount@2] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/data/lineitem/1.parquet:.., /testdata/tpch/data/lineitem/10.parquet:.., /testdata/tpch/data/lineitem/11.parquet:..], [/testdata/tpch/data/lineitem/11.parquet:.., /testdata/tpch/data/lineitem/12.parquet:.., /testdata/tpch/data/lineitem/13.parquet:.., /testdata/tpch/data/lineitem/14.parquet:..], [/testdata/tpch/data/lineitem/14.parquet:.., /testdata/tpch/data/lineitem/15.parquet:.., /testdata/tpch/data/lineitem/16.parquet:..], [/testdata/tpch/data/lineitem/16.parquet:.., /testdata/tpch/data/lineitem/2.parquet:.., /testdata/tpch/data/lineitem/3.parquet:.., /testdata/tpch/data/lineitem/4.parquet:..], [/testdata/tpch/data/lineitem/4.parquet:.., /testdata/tpch/data/lineitem/5.parquet:.., /testdata/tpch/data/lineitem/6.parquet:.., /testdata/tpch/data/lineitem/7.parquet:..], ...]}, projection=[l_orderkey, l_extendedprice, l_discount, l_returnflag], file_type=parquet, predicate=l_returnflag@3 = R AND DynamicFilterPhysicalExpr [ true ], pruning_predicate=l_returnflag_null_count@2 != row_count@3 AND l_returnflag_min@0 <= R AND R <= l_returnflag_max@1, required_guarantees=[l_returnflag in (R)] - └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0,p1] t1:[p2,p3] t2:[p4,p5] t3:[p6,p7] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: o_orderdate@2 >= 1993-10-01 AND o_orderdate@2 < 1994-01-01, projection=[o_orderkey@0, o_custkey@1] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/data/orders/1.parquet:.., /testdata/tpch/data/orders/10.parquet:.., /testdata/tpch/data/orders/11.parquet:..], [/testdata/tpch/data/orders/11.parquet:.., /testdata/tpch/data/orders/12.parquet:.., /testdata/tpch/data/orders/13.parquet:.., /testdata/tpch/data/orders/14.parquet:..], [/testdata/tpch/data/orders/14.parquet:.., /testdata/tpch/data/orders/15.parquet:.., /testdata/tpch/data/orders/16.parquet:..], [/testdata/tpch/data/orders/16.parquet:.., /testdata/tpch/data/orders/2.parquet:.., /testdata/tpch/data/orders/3.parquet:.., /testdata/tpch/data/orders/4.parquet:..], [/testdata/tpch/data/orders/4.parquet:.., /testdata/tpch/data/orders/5.parquet:.., /testdata/tpch/data/orders/6.parquet:.., /testdata/tpch/data/orders/7.parquet:..], ...]}, projection=[o_orderkey, o_custkey, o_orderdate], file_type=parquet, predicate=o_orderdate@2 >= 1993-10-01 AND o_orderdate@2 < 1994-01-01, pruning_predicate=o_orderdate_null_count@1 != row_count@2 AND o_orderdate_max@0 >= 1993-10-01 AND o_orderdate_null_count@1 != row_count@2 AND o_orderdate_min@3 < 1994-01-01, required_guarantees=[] - └────────────────────────────────────────────────── - "); - Ok(()) - } - - #[tokio::test] - async fn test_tpch_11() -> Result<(), Box> { - let plan = test_tpch_query(11).await?; - assert_snapshot!(plan, @r" - ┌───── DistributedExec ── Tasks: t0:[p0] - │ SortPreservingMergeExec: [value@1 DESC] - │ SortExec: expr=[value@1 DESC], preserve_partitioning=[true] - │ ProjectionExec: expr=[ps_partkey@0 as ps_partkey, sum(partsupp.ps_supplycost * partsupp.ps_availqty)@1 as value] - │ NestedLoopJoinExec: join_type=Inner, filter=CAST(sum(partsupp.ps_supplycost * partsupp.ps_availqty)@0 AS Decimal128(38, 15)) > sum(partsupp.ps_supplycost * partsupp.ps_availqty) * Float64(0.0001)@1, projection=[ps_partkey@1, sum(partsupp.ps_supplycost * partsupp.ps_availqty)@2] - │ ProjectionExec: expr=[CAST(CAST(sum(partsupp.ps_supplycost * partsupp.ps_availqty)@0 AS Float64) * 0.0001 AS Decimal128(38, 15)) as sum(partsupp.ps_supplycost * partsupp.ps_availqty) * Float64(0.0001)] - │ AggregateExec: mode=Final, gby=[], aggr=[sum(partsupp.ps_supplycost * partsupp.ps_availqty)] - │ CoalescePartitionsExec - │ AggregateExec: mode=Partial, gby=[], aggr=[sum(partsupp.ps_supplycost * partsupp.ps_availqty)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@2)], projection=[ps_availqty@1, ps_supplycost@2] - │ CoalescePartitionsExec - │ [Stage 1] => NetworkCoalesceExec: output_partitions=8, input_tasks=4 - │ ProjectionExec: expr=[ps_availqty@1 as ps_availqty, ps_supplycost@2 as ps_supplycost, s_nationkey@0 as s_nationkey] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_suppkey@0, ps_suppkey@0)], projection=[s_nationkey@1, ps_availqty@3, ps_supplycost@4] - │ CoalescePartitionsExec - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/data/supplier/1.parquet, /testdata/tpch/data/supplier/10.parquet, /testdata/tpch/data/supplier/11.parquet], [/testdata/tpch/data/supplier/12.parquet, /testdata/tpch/data/supplier/13.parquet, /testdata/tpch/data/supplier/14.parquet], [/testdata/tpch/data/supplier/15.parquet, /testdata/tpch/data/supplier/16.parquet, /testdata/tpch/data/supplier/2.parquet], [/testdata/tpch/data/supplier/3.parquet, /testdata/tpch/data/supplier/4.parquet, /testdata/tpch/data/supplier/5.parquet], [/testdata/tpch/data/supplier/6.parquet, /testdata/tpch/data/supplier/7.parquet, /testdata/tpch/data/supplier/8.parquet], ...]}, projection=[s_suppkey, s_nationkey], file_type=parquet - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/data/partsupp/1.parquet:.., /testdata/tpch/data/partsupp/10.parquet:.., /testdata/tpch/data/partsupp/11.parquet:..], [/testdata/tpch/data/partsupp/11.parquet:.., /testdata/tpch/data/partsupp/12.parquet:.., /testdata/tpch/data/partsupp/13.parquet:.., /testdata/tpch/data/partsupp/14.parquet:..], [/testdata/tpch/data/partsupp/14.parquet:.., /testdata/tpch/data/partsupp/15.parquet:.., /testdata/tpch/data/partsupp/16.parquet:.., /testdata/tpch/data/partsupp/2.parquet:..], [/testdata/tpch/data/partsupp/2.parquet:.., /testdata/tpch/data/partsupp/3.parquet:.., /testdata/tpch/data/partsupp/4.parquet:..], [/testdata/tpch/data/partsupp/4.parquet:.., /testdata/tpch/data/partsupp/5.parquet:.., /testdata/tpch/data/partsupp/6.parquet:.., /testdata/tpch/data/partsupp/7.parquet:..], ...]}, projection=[ps_suppkey, ps_availqty, ps_supplycost], file_type=parquet, predicate=DynamicFilterPhysicalExpr [ ps_suppkey@0 >= 1 AND ps_suppkey@0 <= 1000 OR ps_suppkey@0 >= 1 AND ps_suppkey@0 <= 1000 OR ps_suppkey@0 >= 1 AND ps_suppkey@0 <= 1000 OR ps_suppkey@0 >= 1 AND ps_suppkey@0 <= 1000 OR ps_suppkey@0 >= 1 AND ps_suppkey@0 <= 1000 OR ps_suppkey@0 >= 1 AND ps_suppkey@0 <= 1000 ], pruning_predicate=ps_suppkey_null_count@1 != row_count@2 AND ps_suppkey_max@0 >= 1 AND ps_suppkey_null_count@1 != row_count@2 AND ps_suppkey_min@3 <= 1000 OR ps_suppkey_null_count@1 != row_count@2 AND ps_suppkey_max@0 >= 1 AND ps_suppkey_null_count@1 != row_count@2 AND ps_suppkey_min@3 <= 1000 OR ps_suppkey_null_count@1 != row_count@2 AND ps_suppkey_max@0 >= 1 AND ps_suppkey_null_count@1 != row_count@2 AND ps_suppkey_min@3 <= 1000 OR ps_suppkey_null_count@1 != row_count@2 AND ps_suppkey_max@0 >= 1 AND ps_suppkey_null_count@1 != row_count@2 AND ps_suppkey_min@3 <= 1000 OR ps_suppkey_null_count@1 != row_count@2 AND ps_suppkey_max@0 >= 1 AND ps_suppkey_null_count@1 != row_count@2 AND ps_suppkey_min@3 <= 1000 OR ps_suppkey_null_count@1 != row_count@2 AND ps_suppkey_max@0 >= 1 AND ps_suppkey_null_count@1 != row_count@2 AND ps_suppkey_min@3 <= 1000, required_guarantees=[] - │ AggregateExec: mode=FinalPartitioned, gby=[ps_partkey@0 as ps_partkey], aggr=[sum(partsupp.ps_supplycost * partsupp.ps_availqty)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ RepartitionExec: partitioning=Hash([ps_partkey@0], 6), input_partitions=6 - │ AggregateExec: mode=Partial, gby=[ps_partkey@0 as ps_partkey], aggr=[sum(partsupp.ps_supplycost * partsupp.ps_availqty)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@3)], projection=[ps_partkey@1, ps_availqty@2, ps_supplycost@3] - │ CoalescePartitionsExec - │ [Stage 2] => NetworkCoalesceExec: output_partitions=8, input_tasks=4 - │ ProjectionExec: expr=[ps_partkey@1 as ps_partkey, ps_availqty@2 as ps_availqty, ps_supplycost@3 as ps_supplycost, s_nationkey@0 as s_nationkey] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_suppkey@0, ps_suppkey@1)], projection=[s_nationkey@1, ps_partkey@2, ps_availqty@4, ps_supplycost@5] - │ CoalescePartitionsExec - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/data/supplier/1.parquet, /testdata/tpch/data/supplier/10.parquet, /testdata/tpch/data/supplier/11.parquet], [/testdata/tpch/data/supplier/12.parquet, /testdata/tpch/data/supplier/13.parquet, /testdata/tpch/data/supplier/14.parquet], [/testdata/tpch/data/supplier/15.parquet, /testdata/tpch/data/supplier/16.parquet, /testdata/tpch/data/supplier/2.parquet], [/testdata/tpch/data/supplier/3.parquet, /testdata/tpch/data/supplier/4.parquet, /testdata/tpch/data/supplier/5.parquet], [/testdata/tpch/data/supplier/6.parquet, /testdata/tpch/data/supplier/7.parquet, /testdata/tpch/data/supplier/8.parquet], ...]}, projection=[s_suppkey, s_nationkey], file_type=parquet - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/data/partsupp/1.parquet:.., /testdata/tpch/data/partsupp/10.parquet:.., /testdata/tpch/data/partsupp/11.parquet:..], [/testdata/tpch/data/partsupp/11.parquet:.., /testdata/tpch/data/partsupp/12.parquet:.., /testdata/tpch/data/partsupp/13.parquet:.., /testdata/tpch/data/partsupp/14.parquet:..], [/testdata/tpch/data/partsupp/14.parquet:.., /testdata/tpch/data/partsupp/15.parquet:.., /testdata/tpch/data/partsupp/16.parquet:.., /testdata/tpch/data/partsupp/2.parquet:..], [/testdata/tpch/data/partsupp/2.parquet:.., /testdata/tpch/data/partsupp/3.parquet:.., /testdata/tpch/data/partsupp/4.parquet:..], [/testdata/tpch/data/partsupp/4.parquet:.., /testdata/tpch/data/partsupp/5.parquet:.., /testdata/tpch/data/partsupp/6.parquet:.., /testdata/tpch/data/partsupp/7.parquet:..], ...]}, projection=[ps_partkey, ps_suppkey, ps_availqty, ps_supplycost], file_type=parquet, predicate=DynamicFilterPhysicalExpr [ ps_suppkey@1 >= 1 AND ps_suppkey@1 <= 1000 OR ps_suppkey@1 >= 1 AND ps_suppkey@1 <= 1000 OR ps_suppkey@1 >= 1 AND ps_suppkey@1 <= 1000 OR ps_suppkey@1 >= 1 AND ps_suppkey@1 <= 1000 OR ps_suppkey@1 >= 1 AND ps_suppkey@1 <= 1000 OR ps_suppkey@1 >= 1 AND ps_suppkey@1 <= 1000 ], pruning_predicate=ps_suppkey_null_count@1 != row_count@2 AND ps_suppkey_max@0 >= 1 AND ps_suppkey_null_count@1 != row_count@2 AND ps_suppkey_min@3 <= 1000 OR ps_suppkey_null_count@1 != row_count@2 AND ps_suppkey_max@0 >= 1 AND ps_suppkey_null_count@1 != row_count@2 AND ps_suppkey_min@3 <= 1000 OR ps_suppkey_null_count@1 != row_count@2 AND ps_suppkey_max@0 >= 1 AND ps_suppkey_null_count@1 != row_count@2 AND ps_suppkey_min@3 <= 1000 OR ps_suppkey_null_count@1 != row_count@2 AND ps_suppkey_max@0 >= 1 AND ps_suppkey_null_count@1 != row_count@2 AND ps_suppkey_min@3 <= 1000 OR ps_suppkey_null_count@1 != row_count@2 AND ps_suppkey_max@0 >= 1 AND ps_suppkey_null_count@1 != row_count@2 AND ps_suppkey_min@3 <= 1000 OR ps_suppkey_null_count@1 != row_count@2 AND ps_suppkey_max@0 >= 1 AND ps_suppkey_null_count@1 != row_count@2 AND ps_suppkey_min@3 <= 1000, required_guarantees=[] - └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0,p1] t1:[p2,p3] t2:[p4,p5] t3:[p6,p7] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: n_name@1 = GERMANY, projection=[n_nationkey@0] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/data/nation/1.parquet, /testdata/tpch/data/nation/10.parquet, /testdata/tpch/data/nation/11.parquet], [/testdata/tpch/data/nation/12.parquet, /testdata/tpch/data/nation/13.parquet, /testdata/tpch/data/nation/14.parquet], [/testdata/tpch/data/nation/15.parquet, /testdata/tpch/data/nation/16.parquet, /testdata/tpch/data/nation/2.parquet], [/testdata/tpch/data/nation/3.parquet, /testdata/tpch/data/nation/4.parquet, /testdata/tpch/data/nation/5.parquet], [/testdata/tpch/data/nation/6.parquet, /testdata/tpch/data/nation/7.parquet, /testdata/tpch/data/nation/8.parquet], ...]}, projection=[n_nationkey, n_name], file_type=parquet, predicate=n_name@1 = GERMANY, pruning_predicate=n_name_null_count@2 != row_count@3 AND n_name_min@0 <= GERMANY AND GERMANY <= n_name_max@1, required_guarantees=[n_name in (GERMANY)] - └────────────────────────────────────────────────── - ┌───── Stage 2 ── Tasks: t0:[p0,p1] t1:[p2,p3] t2:[p4,p5] t3:[p6,p7] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: n_name@1 = GERMANY, projection=[n_nationkey@0] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/data/nation/1.parquet, /testdata/tpch/data/nation/10.parquet, /testdata/tpch/data/nation/11.parquet], [/testdata/tpch/data/nation/12.parquet, /testdata/tpch/data/nation/13.parquet, /testdata/tpch/data/nation/14.parquet], [/testdata/tpch/data/nation/15.parquet, /testdata/tpch/data/nation/16.parquet, /testdata/tpch/data/nation/2.parquet], [/testdata/tpch/data/nation/3.parquet, /testdata/tpch/data/nation/4.parquet, /testdata/tpch/data/nation/5.parquet], [/testdata/tpch/data/nation/6.parquet, /testdata/tpch/data/nation/7.parquet, /testdata/tpch/data/nation/8.parquet], ...]}, projection=[n_nationkey, n_name], file_type=parquet, predicate=n_name@1 = GERMANY, pruning_predicate=n_name_null_count@2 != row_count@3 AND n_name_min@0 <= GERMANY AND GERMANY <= n_name_max@1, required_guarantees=[n_name in (GERMANY)] - └────────────────────────────────────────────────── - "); - Ok(()) - } - - #[tokio::test] - async fn test_tpch_12() -> Result<(), Box> { - let plan = test_tpch_query(12).await?; - assert_snapshot!(plan, @r#" - ┌───── DistributedExec ── Tasks: t0:[p0] - │ SortPreservingMergeExec: [l_shipmode@0 ASC NULLS LAST] - │ [Stage 4] => NetworkCoalesceExec: output_partitions=24, input_tasks=4 - └────────────────────────────────────────────────── - ┌───── Stage 4 ── Tasks: t0:[p0,p1,p2,p3,p4,p5] t1:[p0,p1,p2,p3,p4,p5] t2:[p0,p1,p2,p3,p4,p5] t3:[p0,p1,p2,p3,p4,p5] - │ SortExec: expr=[l_shipmode@0 ASC NULLS LAST], preserve_partitioning=[true] - │ ProjectionExec: expr=[l_shipmode@0 as l_shipmode, sum(CASE WHEN orders.o_orderpriority = Utf8("1-URGENT") OR orders.o_orderpriority = Utf8("2-HIGH") THEN Int64(1) ELSE Int64(0) END)@1 as high_line_count, sum(CASE WHEN orders.o_orderpriority != Utf8("1-URGENT") AND orders.o_orderpriority != Utf8("2-HIGH") THEN Int64(1) ELSE Int64(0) END)@2 as low_line_count] - │ AggregateExec: mode=FinalPartitioned, gby=[l_shipmode@0 as l_shipmode], aggr=[sum(CASE WHEN orders.o_orderpriority = Utf8("1-URGENT") OR orders.o_orderpriority = Utf8("2-HIGH") THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN orders.o_orderpriority != Utf8("1-URGENT") AND orders.o_orderpriority != Utf8("2-HIGH") THEN Int64(1) ELSE Int64(0) END)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ [Stage 3] => NetworkShuffleExec: output_partitions=6, input_tasks=3 - └────────────────────────────────────────────────── - ┌───── Stage 3 ── Tasks: t0:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9,p10,p11,p12,p13,p14,p15,p16,p17,p18,p19,p20,p21,p22,p23] t1:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9,p10,p11,p12,p13,p14,p15,p16,p17,p18,p19,p20,p21,p22,p23] t2:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9,p10,p11,p12,p13,p14,p15,p16,p17,p18,p19,p20,p21,p22,p23] - │ RepartitionExec: partitioning=Hash([l_shipmode@0], 24), input_partitions=6 - │ AggregateExec: mode=Partial, gby=[l_shipmode@0 as l_shipmode], aggr=[sum(CASE WHEN orders.o_orderpriority = Utf8("1-URGENT") OR orders.o_orderpriority = Utf8("2-HIGH") THEN Int64(1) ELSE Int64(0) END), sum(CASE WHEN orders.o_orderpriority != Utf8("1-URGENT") AND orders.o_orderpriority != Utf8("2-HIGH") THEN Int64(1) ELSE Int64(0) END)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(l_orderkey@0, o_orderkey@0)], projection=[l_shipmode@1, o_orderpriority@3] - │ CoalesceBatchesExec: target_batch_size=8192 - │ [Stage 1] => NetworkShuffleExec: output_partitions=6, input_tasks=3 - │ CoalesceBatchesExec: target_batch_size=8192 - │ [Stage 2] => NetworkShuffleExec: output_partitions=6, input_tasks=3 - └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9,p10,p11,p12,p13,p14,p15,p16,p17] t1:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9,p10,p11,p12,p13,p14,p15,p16,p17] t2:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9,p10,p11,p12,p13,p14,p15,p16,p17] - │ RepartitionExec: partitioning=Hash([l_orderkey@0], 18), input_partitions=2 - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: (l_shipmode@4 = MAIL OR l_shipmode@4 = SHIP) AND l_receiptdate@3 > l_commitdate@2 AND l_shipdate@1 < l_commitdate@2 AND l_receiptdate@3 >= 1994-01-01 AND l_receiptdate@3 < 1995-01-01, projection=[l_orderkey@0, l_shipmode@4] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/data/lineitem/1.parquet:.., /testdata/tpch/data/lineitem/10.parquet:.., /testdata/tpch/data/lineitem/11.parquet:..], [/testdata/tpch/data/lineitem/11.parquet:.., /testdata/tpch/data/lineitem/12.parquet:.., /testdata/tpch/data/lineitem/13.parquet:.., /testdata/tpch/data/lineitem/14.parquet:..], [/testdata/tpch/data/lineitem/14.parquet:.., /testdata/tpch/data/lineitem/15.parquet:.., /testdata/tpch/data/lineitem/16.parquet:..], [/testdata/tpch/data/lineitem/16.parquet:.., /testdata/tpch/data/lineitem/2.parquet:.., /testdata/tpch/data/lineitem/3.parquet:.., /testdata/tpch/data/lineitem/4.parquet:..], [/testdata/tpch/data/lineitem/4.parquet:.., /testdata/tpch/data/lineitem/5.parquet:.., /testdata/tpch/data/lineitem/6.parquet:.., /testdata/tpch/data/lineitem/7.parquet:..], ...]}, projection=[l_orderkey, l_shipdate, l_commitdate, l_receiptdate, l_shipmode], file_type=parquet, predicate=(l_shipmode@4 = MAIL OR l_shipmode@4 = SHIP) AND l_receiptdate@3 > l_commitdate@2 AND l_shipdate@1 < l_commitdate@2 AND l_receiptdate@3 >= 1994-01-01 AND l_receiptdate@3 < 1995-01-01, pruning_predicate=(l_shipmode_null_count@2 != row_count@3 AND l_shipmode_min@0 <= MAIL AND MAIL <= l_shipmode_max@1 OR l_shipmode_null_count@2 != row_count@3 AND l_shipmode_min@0 <= SHIP AND SHIP <= l_shipmode_max@1) AND l_receiptdate_null_count@5 != row_count@3 AND l_receiptdate_max@4 >= 1994-01-01 AND l_receiptdate_null_count@5 != row_count@3 AND l_receiptdate_min@6 < 1995-01-01, required_guarantees=[l_shipmode in (MAIL, SHIP)] - └────────────────────────────────────────────────── - ┌───── Stage 2 ── Tasks: t0:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9,p10,p11,p12,p13,p14,p15,p16,p17] t1:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9,p10,p11,p12,p13,p14,p15,p16,p17] t2:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9,p10,p11,p12,p13,p14,p15,p16,p17] - │ RepartitionExec: partitioning=Hash([o_orderkey@0], 18), input_partitions=2 - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/data/orders/1.parquet:.., /testdata/tpch/data/orders/10.parquet:.., /testdata/tpch/data/orders/11.parquet:..], [/testdata/tpch/data/orders/11.parquet:.., /testdata/tpch/data/orders/12.parquet:.., /testdata/tpch/data/orders/13.parquet:.., /testdata/tpch/data/orders/14.parquet:..], [/testdata/tpch/data/orders/14.parquet:.., /testdata/tpch/data/orders/15.parquet:.., /testdata/tpch/data/orders/16.parquet:..], [/testdata/tpch/data/orders/16.parquet:.., /testdata/tpch/data/orders/2.parquet:.., /testdata/tpch/data/orders/3.parquet:.., /testdata/tpch/data/orders/4.parquet:..], [/testdata/tpch/data/orders/4.parquet:.., /testdata/tpch/data/orders/5.parquet:.., /testdata/tpch/data/orders/6.parquet:.., /testdata/tpch/data/orders/7.parquet:..], ...]}, projection=[o_orderkey, o_orderpriority], file_type=parquet, predicate=DynamicFilterPhysicalExpr [ true ] - └────────────────────────────────────────────────── - "#); - Ok(()) - } - - #[tokio::test] - async fn test_tpch_13() -> Result<(), Box> { - let plan = test_tpch_query(13).await?; - assert_snapshot!(plan, @r" - ┌───── DistributedExec ── Tasks: t0:[p0] - │ SortPreservingMergeExec: [custdist@1 DESC, c_count@0 DESC] - │ [Stage 4] => NetworkCoalesceExec: output_partitions=24, input_tasks=4 - └────────────────────────────────────────────────── - ┌───── Stage 4 ── Tasks: t0:[p0,p1,p2,p3,p4,p5] t1:[p0,p1,p2,p3,p4,p5] t2:[p0,p1,p2,p3,p4,p5] t3:[p0,p1,p2,p3,p4,p5] - │ SortExec: expr=[custdist@1 DESC, c_count@0 DESC], preserve_partitioning=[true] - │ ProjectionExec: expr=[c_count@0 as c_count, count(Int64(1))@1 as custdist] - │ AggregateExec: mode=FinalPartitioned, gby=[c_count@0 as c_count], aggr=[count(Int64(1))] - │ CoalesceBatchesExec: target_batch_size=8192 - │ [Stage 3] => NetworkShuffleExec: output_partitions=6, input_tasks=3 - └────────────────────────────────────────────────── - ┌───── Stage 3 ── Tasks: t0:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9,p10,p11,p12,p13,p14,p15,p16,p17,p18,p19,p20,p21,p22,p23] t1:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9,p10,p11,p12,p13,p14,p15,p16,p17,p18,p19,p20,p21,p22,p23] t2:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9,p10,p11,p12,p13,p14,p15,p16,p17,p18,p19,p20,p21,p22,p23] - │ RepartitionExec: partitioning=Hash([c_count@0], 24), input_partitions=6 - │ AggregateExec: mode=Partial, gby=[c_count@0 as c_count], aggr=[count(Int64(1))] - │ ProjectionExec: expr=[count(orders.o_orderkey)@1 as c_count] - │ AggregateExec: mode=FinalPartitioned, gby=[c_custkey@0 as c_custkey], aggr=[count(orders.o_orderkey)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ [Stage 2] => NetworkShuffleExec: output_partitions=6, input_tasks=1 - └────────────────────────────────────────────────── - ┌───── Stage 2 ── Tasks: t0:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9,p10,p11,p12,p13,p14,p15,p16,p17] - │ RepartitionExec: partitioning=Hash([c_custkey@0], 18), input_partitions=6 - │ AggregateExec: mode=Partial, gby=[c_custkey@0 as c_custkey], aggr=[count(orders.o_orderkey)] - │ ProjectionExec: expr=[c_custkey@1 as c_custkey, o_orderkey@0 as o_orderkey] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Right, on=[(o_custkey@1, c_custkey@0)], projection=[o_orderkey@0, c_custkey@2] - │ CoalescePartitionsExec - │ [Stage 1] => NetworkCoalesceExec: output_partitions=8, input_tasks=4 - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/data/customer/1.parquet, /testdata/tpch/data/customer/10.parquet, /testdata/tpch/data/customer/11.parquet], [/testdata/tpch/data/customer/12.parquet, /testdata/tpch/data/customer/13.parquet, /testdata/tpch/data/customer/14.parquet], [/testdata/tpch/data/customer/15.parquet, /testdata/tpch/data/customer/16.parquet, /testdata/tpch/data/customer/2.parquet], [/testdata/tpch/data/customer/3.parquet, /testdata/tpch/data/customer/4.parquet, /testdata/tpch/data/customer/5.parquet], [/testdata/tpch/data/customer/6.parquet, /testdata/tpch/data/customer/7.parquet, /testdata/tpch/data/customer/8.parquet], ...]}, projection=[c_custkey], file_type=parquet - └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0,p1] t1:[p2,p3] t2:[p4,p5] t3:[p6,p7] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: o_comment@2 NOT LIKE %special%requests%, projection=[o_orderkey@0, o_custkey@1] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/data/orders/1.parquet:.., /testdata/tpch/data/orders/10.parquet:.., /testdata/tpch/data/orders/11.parquet:..], [/testdata/tpch/data/orders/11.parquet:.., /testdata/tpch/data/orders/12.parquet:.., /testdata/tpch/data/orders/13.parquet:.., /testdata/tpch/data/orders/14.parquet:..], [/testdata/tpch/data/orders/14.parquet:.., /testdata/tpch/data/orders/15.parquet:.., /testdata/tpch/data/orders/16.parquet:..], [/testdata/tpch/data/orders/16.parquet:.., /testdata/tpch/data/orders/2.parquet:.., /testdata/tpch/data/orders/3.parquet:.., /testdata/tpch/data/orders/4.parquet:..], [/testdata/tpch/data/orders/4.parquet:.., /testdata/tpch/data/orders/5.parquet:.., /testdata/tpch/data/orders/6.parquet:.., /testdata/tpch/data/orders/7.parquet:..], ...]}, projection=[o_orderkey, o_custkey, o_comment], file_type=parquet, predicate=o_comment@2 NOT LIKE %special%requests% - └────────────────────────────────────────────────── - "); - Ok(()) - } - - #[tokio::test] - async fn test_tpch_14() -> Result<(), Box> { - let plan = test_tpch_query(14).await?; - assert_snapshot!(plan, @r#" - ┌───── DistributedExec ── Tasks: t0:[p0] - │ ProjectionExec: expr=[100 * CAST(sum(CASE WHEN part.p_type LIKE Utf8("PROMO%") THEN lineitem.l_extendedprice * Int64(1) - lineitem.l_discount ELSE Int64(0) END)@0 AS Float64) / CAST(sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@1 AS Float64) as promo_revenue] - │ AggregateExec: mode=Final, gby=[], aggr=[sum(CASE WHEN part.p_type LIKE Utf8("PROMO%") THEN lineitem.l_extendedprice * Int64(1) - lineitem.l_discount ELSE Int64(0) END), sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] - │ CoalescePartitionsExec - │ [Stage 3] => NetworkCoalesceExec: output_partitions=24, input_tasks=4 - └────────────────────────────────────────────────── - ┌───── Stage 3 ── Tasks: t0:[p0,p1,p2,p3,p4,p5] t1:[p0,p1,p2,p3,p4,p5] t2:[p0,p1,p2,p3,p4,p5] t3:[p0,p1,p2,p3,p4,p5] - │ AggregateExec: mode=Partial, gby=[], aggr=[sum(CASE WHEN part.p_type LIKE Utf8("PROMO%") THEN lineitem.l_extendedprice * Int64(1) - lineitem.l_discount ELSE Int64(0) END), sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] - │ ProjectionExec: expr=[l_extendedprice@1 * (Some(1),20,0 - l_discount@2) as __common_expr_1, p_type@0 as p_type] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(p_partkey@0, l_partkey@0)], projection=[p_type@1, l_extendedprice@3, l_discount@4] - │ CoalesceBatchesExec: target_batch_size=8192 - │ [Stage 1] => NetworkShuffleExec: output_partitions=6, input_tasks=3 - │ CoalesceBatchesExec: target_batch_size=8192 - │ [Stage 2] => NetworkShuffleExec: output_partitions=6, input_tasks=3 - └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9,p10,p11,p12,p13,p14,p15,p16,p17,p18,p19,p20,p21,p22,p23] t1:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9,p10,p11,p12,p13,p14,p15,p16,p17,p18,p19,p20,p21,p22,p23] t2:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9,p10,p11,p12,p13,p14,p15,p16,p17,p18,p19,p20,p21,p22,p23] - │ RepartitionExec: partitioning=Hash([p_partkey@0], 24), input_partitions=2 - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/data/part/1.parquet, /testdata/tpch/data/part/10.parquet, /testdata/tpch/data/part/11.parquet], [/testdata/tpch/data/part/12.parquet, /testdata/tpch/data/part/13.parquet, /testdata/tpch/data/part/14.parquet], [/testdata/tpch/data/part/15.parquet, /testdata/tpch/data/part/16.parquet, /testdata/tpch/data/part/2.parquet], [/testdata/tpch/data/part/3.parquet, /testdata/tpch/data/part/4.parquet, /testdata/tpch/data/part/5.parquet], [/testdata/tpch/data/part/6.parquet, /testdata/tpch/data/part/7.parquet, /testdata/tpch/data/part/8.parquet], ...]}, projection=[p_partkey, p_type], file_type=parquet - └────────────────────────────────────────────────── - ┌───── Stage 2 ── Tasks: t0:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9,p10,p11,p12,p13,p14,p15,p16,p17,p18,p19,p20,p21,p22,p23] t1:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9,p10,p11,p12,p13,p14,p15,p16,p17,p18,p19,p20,p21,p22,p23] t2:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9,p10,p11,p12,p13,p14,p15,p16,p17,p18,p19,p20,p21,p22,p23] - │ RepartitionExec: partitioning=Hash([l_partkey@0], 24), input_partitions=2 - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: l_shipdate@3 >= 1995-09-01 AND l_shipdate@3 < 1995-10-01, projection=[l_partkey@0, l_extendedprice@1, l_discount@2] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/data/lineitem/1.parquet:.., /testdata/tpch/data/lineitem/10.parquet:.., /testdata/tpch/data/lineitem/11.parquet:..], [/testdata/tpch/data/lineitem/11.parquet:.., /testdata/tpch/data/lineitem/12.parquet:.., /testdata/tpch/data/lineitem/13.parquet:.., /testdata/tpch/data/lineitem/14.parquet:..], [/testdata/tpch/data/lineitem/14.parquet:.., /testdata/tpch/data/lineitem/15.parquet:.., /testdata/tpch/data/lineitem/16.parquet:..], [/testdata/tpch/data/lineitem/16.parquet:.., /testdata/tpch/data/lineitem/2.parquet:.., /testdata/tpch/data/lineitem/3.parquet:.., /testdata/tpch/data/lineitem/4.parquet:..], [/testdata/tpch/data/lineitem/4.parquet:.., /testdata/tpch/data/lineitem/5.parquet:.., /testdata/tpch/data/lineitem/6.parquet:.., /testdata/tpch/data/lineitem/7.parquet:..], ...]}, projection=[l_partkey, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@3 >= 1995-09-01 AND l_shipdate@3 < 1995-10-01 AND DynamicFilterPhysicalExpr [ true ], pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1995-09-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 < 1995-10-01, required_guarantees=[] - └────────────────────────────────────────────────── - "#); - Ok(()) - } - - #[tokio::test] - async fn test_tpch_15() -> Result<(), Box> { - let plan = test_tpch_query(15).await?; - assert_snapshot!(plan, @r" - ┌───── DistributedExec ── Tasks: t0:[p0] - │ SortPreservingMergeExec: [s_suppkey@0 ASC NULLS LAST] - │ SortExec: expr=[s_suppkey@0 ASC NULLS LAST], preserve_partitioning=[true] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(max(revenue0.total_revenue)@0, total_revenue@4)], projection=[s_suppkey@1, s_name@2, s_address@3, s_phone@4, total_revenue@5] - │ AggregateExec: mode=Final, gby=[], aggr=[max(revenue0.total_revenue)] - │ CoalescePartitionsExec - │ [Stage 2] => NetworkCoalesceExec: output_partitions=24, input_tasks=4 - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_suppkey@0, supplier_no@0)], projection=[s_suppkey@0, s_name@1, s_address@2, s_phone@3, total_revenue@5] - │ CoalescePartitionsExec - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/data/supplier/1.parquet, /testdata/tpch/data/supplier/10.parquet, /testdata/tpch/data/supplier/11.parquet], [/testdata/tpch/data/supplier/12.parquet, /testdata/tpch/data/supplier/13.parquet, /testdata/tpch/data/supplier/14.parquet], [/testdata/tpch/data/supplier/15.parquet, /testdata/tpch/data/supplier/16.parquet, /testdata/tpch/data/supplier/2.parquet], [/testdata/tpch/data/supplier/3.parquet, /testdata/tpch/data/supplier/4.parquet, /testdata/tpch/data/supplier/5.parquet], [/testdata/tpch/data/supplier/6.parquet, /testdata/tpch/data/supplier/7.parquet, /testdata/tpch/data/supplier/8.parquet], ...]}, projection=[s_suppkey, s_name, s_address, s_phone], file_type=parquet - │ ProjectionExec: expr=[l_suppkey@0 as supplier_no, sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@1 as total_revenue] - │ AggregateExec: mode=FinalPartitioned, gby=[l_suppkey@0 as l_suppkey], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ [Stage 3] => NetworkShuffleExec: output_partitions=6, input_tasks=3 - └────────────────────────────────────────────────── - ┌───── Stage 2 ── Tasks: t0:[p0,p1,p2,p3,p4,p5] t1:[p0,p1,p2,p3,p4,p5] t2:[p0,p1,p2,p3,p4,p5] t3:[p0,p1,p2,p3,p4,p5] - │ AggregateExec: mode=Partial, gby=[], aggr=[max(revenue0.total_revenue)] - │ ProjectionExec: expr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@1 as total_revenue] - │ AggregateExec: mode=FinalPartitioned, gby=[l_suppkey@0 as l_suppkey], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ [Stage 1] => NetworkShuffleExec: output_partitions=6, input_tasks=3 - └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9,p10,p11,p12,p13,p14,p15,p16,p17,p18,p19,p20,p21,p22,p23] t1:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9,p10,p11,p12,p13,p14,p15,p16,p17,p18,p19,p20,p21,p22,p23] t2:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9,p10,p11,p12,p13,p14,p15,p16,p17,p18,p19,p20,p21,p22,p23] - │ RepartitionExec: partitioning=Hash([l_suppkey@0], 24), input_partitions=2 - │ AggregateExec: mode=Partial, gby=[l_suppkey@0 as l_suppkey], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: l_shipdate@3 >= 1996-01-01 AND l_shipdate@3 < 1996-04-01, projection=[l_suppkey@0, l_extendedprice@1, l_discount@2] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/data/lineitem/1.parquet:.., /testdata/tpch/data/lineitem/10.parquet:.., /testdata/tpch/data/lineitem/11.parquet:..], [/testdata/tpch/data/lineitem/11.parquet:.., /testdata/tpch/data/lineitem/12.parquet:.., /testdata/tpch/data/lineitem/13.parquet:.., /testdata/tpch/data/lineitem/14.parquet:..], [/testdata/tpch/data/lineitem/14.parquet:.., /testdata/tpch/data/lineitem/15.parquet:.., /testdata/tpch/data/lineitem/16.parquet:..], [/testdata/tpch/data/lineitem/16.parquet:.., /testdata/tpch/data/lineitem/2.parquet:.., /testdata/tpch/data/lineitem/3.parquet:.., /testdata/tpch/data/lineitem/4.parquet:..], [/testdata/tpch/data/lineitem/4.parquet:.., /testdata/tpch/data/lineitem/5.parquet:.., /testdata/tpch/data/lineitem/6.parquet:.., /testdata/tpch/data/lineitem/7.parquet:..], ...]}, projection=[l_suppkey, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@3 >= 1996-01-01 AND l_shipdate@3 < 1996-04-01, pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1996-01-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 < 1996-04-01, required_guarantees=[] - └────────────────────────────────────────────────── - ┌───── Stage 3 ── Tasks: t0:[p0,p1,p2,p3,p4,p5] t1:[p0,p1,p2,p3,p4,p5] t2:[p0,p1,p2,p3,p4,p5] - │ RepartitionExec: partitioning=Hash([l_suppkey@0], 6), input_partitions=2 - │ AggregateExec: mode=Partial, gby=[l_suppkey@0 as l_suppkey], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: l_shipdate@3 >= 1996-01-01 AND l_shipdate@3 < 1996-04-01, projection=[l_suppkey@0, l_extendedprice@1, l_discount@2] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/data/lineitem/1.parquet:.., /testdata/tpch/data/lineitem/10.parquet:.., /testdata/tpch/data/lineitem/11.parquet:..], [/testdata/tpch/data/lineitem/11.parquet:.., /testdata/tpch/data/lineitem/12.parquet:.., /testdata/tpch/data/lineitem/13.parquet:.., /testdata/tpch/data/lineitem/14.parquet:..], [/testdata/tpch/data/lineitem/14.parquet:.., /testdata/tpch/data/lineitem/15.parquet:.., /testdata/tpch/data/lineitem/16.parquet:..], [/testdata/tpch/data/lineitem/16.parquet:.., /testdata/tpch/data/lineitem/2.parquet:.., /testdata/tpch/data/lineitem/3.parquet:.., /testdata/tpch/data/lineitem/4.parquet:..], [/testdata/tpch/data/lineitem/4.parquet:.., /testdata/tpch/data/lineitem/5.parquet:.., /testdata/tpch/data/lineitem/6.parquet:.., /testdata/tpch/data/lineitem/7.parquet:..], ...]}, projection=[l_suppkey, l_extendedprice, l_discount, l_shipdate], file_type=parquet, predicate=l_shipdate@3 >= 1996-01-01 AND l_shipdate@3 < 1996-04-01, pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1996-01-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 < 1996-04-01, required_guarantees=[] - └────────────────────────────────────────────────── - "); - Ok(()) - } - - #[tokio::test] - async fn test_tpch_16() -> Result<(), Box> { - let plan = test_tpch_query(16).await?; - assert_snapshot!(plan, @r#" - ┌───── DistributedExec ── Tasks: t0:[p0] - │ SortPreservingMergeExec: [supplier_cnt@3 DESC, p_brand@0 ASC NULLS LAST, p_type@1 ASC NULLS LAST, p_size@2 ASC NULLS LAST] - │ [Stage 5] => NetworkCoalesceExec: output_partitions=24, input_tasks=4 - └────────────────────────────────────────────────── - ┌───── Stage 5 ── Tasks: t0:[p0,p1,p2,p3,p4,p5] t1:[p0,p1,p2,p3,p4,p5] t2:[p0,p1,p2,p3,p4,p5] t3:[p0,p1,p2,p3,p4,p5] - │ SortExec: expr=[supplier_cnt@3 DESC, p_brand@0 ASC NULLS LAST, p_type@1 ASC NULLS LAST, p_size@2 ASC NULLS LAST], preserve_partitioning=[true] - │ ProjectionExec: expr=[p_brand@0 as p_brand, p_type@1 as p_type, p_size@2 as p_size, count(alias1)@3 as supplier_cnt] - │ AggregateExec: mode=FinalPartitioned, gby=[p_brand@0 as p_brand, p_type@1 as p_type, p_size@2 as p_size], aggr=[count(alias1)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ [Stage 4] => NetworkShuffleExec: output_partitions=6, input_tasks=3 - └────────────────────────────────────────────────── - ┌───── Stage 4 ── Tasks: t0:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9,p10,p11,p12,p13,p14,p15,p16,p17,p18,p19,p20,p21,p22,p23] t1:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9,p10,p11,p12,p13,p14,p15,p16,p17,p18,p19,p20,p21,p22,p23] t2:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9,p10,p11,p12,p13,p14,p15,p16,p17,p18,p19,p20,p21,p22,p23] - │ RepartitionExec: partitioning=Hash([p_brand@0, p_type@1, p_size@2], 24), input_partitions=6 - │ AggregateExec: mode=Partial, gby=[p_brand@0 as p_brand, p_type@1 as p_type, p_size@2 as p_size], aggr=[count(alias1)] - │ AggregateExec: mode=FinalPartitioned, gby=[p_brand@0 as p_brand, p_type@1 as p_type, p_size@2 as p_size, alias1@3 as alias1], aggr=[] - │ CoalesceBatchesExec: target_batch_size=8192 - │ [Stage 3] => NetworkShuffleExec: output_partitions=6, input_tasks=1 - └────────────────────────────────────────────────── - ┌───── Stage 3 ── Tasks: t0:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9,p10,p11,p12,p13,p14,p15,p16,p17] - │ RepartitionExec: partitioning=Hash([p_brand@0, p_type@1, p_size@2, alias1@3], 18), input_partitions=6 - │ AggregateExec: mode=Partial, gby=[p_brand@1 as p_brand, p_type@2 as p_type, p_size@3 as p_size, ps_suppkey@0 as alias1], aggr=[] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=RightAnti, on=[(s_suppkey@0, ps_suppkey@0)] - │ CoalescePartitionsExec - │ [Stage 1] => NetworkCoalesceExec: output_partitions=8, input_tasks=4 - │ ProjectionExec: expr=[ps_suppkey@3 as ps_suppkey, p_brand@0 as p_brand, p_type@1 as p_type, p_size@2 as p_size] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(p_partkey@0, ps_partkey@0)], projection=[p_brand@1, p_type@2, p_size@3, ps_suppkey@5] - │ CoalescePartitionsExec - │ [Stage 2] => NetworkCoalesceExec: output_partitions=8, input_tasks=4 - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/data/partsupp/1.parquet:.., /testdata/tpch/data/partsupp/10.parquet:.., /testdata/tpch/data/partsupp/11.parquet:..], [/testdata/tpch/data/partsupp/11.parquet:.., /testdata/tpch/data/partsupp/12.parquet:.., /testdata/tpch/data/partsupp/13.parquet:.., /testdata/tpch/data/partsupp/14.parquet:..], [/testdata/tpch/data/partsupp/14.parquet:.., /testdata/tpch/data/partsupp/15.parquet:.., /testdata/tpch/data/partsupp/16.parquet:.., /testdata/tpch/data/partsupp/2.parquet:..], [/testdata/tpch/data/partsupp/2.parquet:.., /testdata/tpch/data/partsupp/3.parquet:.., /testdata/tpch/data/partsupp/4.parquet:..], [/testdata/tpch/data/partsupp/4.parquet:.., /testdata/tpch/data/partsupp/5.parquet:.., /testdata/tpch/data/partsupp/6.parquet:.., /testdata/tpch/data/partsupp/7.parquet:..], ...]}, projection=[ps_partkey, ps_suppkey], file_type=parquet, predicate=DynamicFilterPhysicalExpr [ true ] - └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0,p1] t1:[p2,p3] t2:[p4,p5] t3:[p6,p7] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: s_comment@1 LIKE %Customer%Complaints%, projection=[s_suppkey@0] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/data/supplier/1.parquet, /testdata/tpch/data/supplier/10.parquet, /testdata/tpch/data/supplier/11.parquet], [/testdata/tpch/data/supplier/12.parquet, /testdata/tpch/data/supplier/13.parquet, /testdata/tpch/data/supplier/14.parquet], [/testdata/tpch/data/supplier/15.parquet, /testdata/tpch/data/supplier/16.parquet, /testdata/tpch/data/supplier/2.parquet], [/testdata/tpch/data/supplier/3.parquet, /testdata/tpch/data/supplier/4.parquet, /testdata/tpch/data/supplier/5.parquet], [/testdata/tpch/data/supplier/6.parquet, /testdata/tpch/data/supplier/7.parquet, /testdata/tpch/data/supplier/8.parquet], ...]}, projection=[s_suppkey, s_comment], file_type=parquet, predicate=s_comment@1 LIKE %Customer%Complaints% - └────────────────────────────────────────────────── - ┌───── Stage 2 ── Tasks: t0:[p0,p1] t1:[p2,p3] t2:[p4,p5] t3:[p6,p7] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: p_brand@1 != Brand#45 AND p_type@2 NOT LIKE MEDIUM POLISHED% AND Use p_size@3 IN (SET) ([Literal { value: Int32(49), field: Field { name: "lit", data_type: Int32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Int32(14), field: Field { name: "lit", data_type: Int32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Int32(23), field: Field { name: "lit", data_type: Int32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Int32(45), field: Field { name: "lit", data_type: Int32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Int32(19), field: Field { name: "lit", data_type: Int32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Int32(3), field: Field { name: "lit", data_type: Int32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Int32(36), field: Field { name: "lit", data_type: Int32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Int32(9), field: Field { name: "lit", data_type: Int32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }]) - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/data/part/1.parquet, /testdata/tpch/data/part/10.parquet, /testdata/tpch/data/part/11.parquet], [/testdata/tpch/data/part/12.parquet, /testdata/tpch/data/part/13.parquet, /testdata/tpch/data/part/14.parquet], [/testdata/tpch/data/part/15.parquet, /testdata/tpch/data/part/16.parquet, /testdata/tpch/data/part/2.parquet], [/testdata/tpch/data/part/3.parquet, /testdata/tpch/data/part/4.parquet, /testdata/tpch/data/part/5.parquet], [/testdata/tpch/data/part/6.parquet, /testdata/tpch/data/part/7.parquet, /testdata/tpch/data/part/8.parquet], ...]}, projection=[p_partkey, p_brand, p_type, p_size], file_type=parquet, predicate=p_brand@1 != Brand#45 AND p_type@2 NOT LIKE MEDIUM POLISHED% AND Use p_size@3 IN (SET) ([Literal { value: Int32(49), field: Field { name: "lit", data_type: Int32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Int32(14), field: Field { name: "lit", data_type: Int32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Int32(23), field: Field { name: "lit", data_type: Int32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Int32(45), field: Field { name: "lit", data_type: Int32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Int32(19), field: Field { name: "lit", data_type: Int32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Int32(3), field: Field { name: "lit", data_type: Int32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Int32(36), field: Field { name: "lit", data_type: Int32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Int32(9), field: Field { name: "lit", data_type: Int32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }]), pruning_predicate=p_brand_null_count@2 != row_count@3 AND (p_brand_min@0 != Brand#45 OR Brand#45 != p_brand_max@1) AND p_type_null_count@6 != row_count@3 AND (p_type_min@4 NOT LIKE MEDIUM POLISHED% OR p_type_max@5 NOT LIKE MEDIUM POLISHED%) AND (p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 49 AND 49 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 14 AND 14 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 23 AND 23 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 45 AND 45 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 19 AND 19 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 3 AND 3 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 36 AND 36 <= p_size_max@8 OR p_size_null_count@9 != row_count@3 AND p_size_min@7 <= 9 AND 9 <= p_size_max@8), required_guarantees=[p_brand not in (Brand#45), p_size in (14, 19, 23, 3, 36, 45, 49, 9)] - └────────────────────────────────────────────────── - "#); - Ok(()) - } - - #[tokio::test] - async fn test_tpch_17() -> Result<(), Box> { - let plan = test_tpch_query(17).await?; - assert_snapshot!(plan, @r" - ┌───── DistributedExec ── Tasks: t0:[p0] - │ ProjectionExec: expr=[CAST(sum(lineitem.l_extendedprice)@0 AS Float64) / 7 as avg_yearly] - │ AggregateExec: mode=Final, gby=[], aggr=[sum(lineitem.l_extendedprice)] - │ CoalescePartitionsExec - │ [Stage 4] => NetworkCoalesceExec: output_partitions=24, input_tasks=4 - └────────────────────────────────────────────────── - ┌───── Stage 4 ── Tasks: t0:[p0,p1,p2,p3,p4,p5] t1:[p0,p1,p2,p3,p4,p5] t2:[p0,p1,p2,p3,p4,p5] t3:[p0,p1,p2,p3,p4,p5] - │ AggregateExec: mode=Partial, gby=[], aggr=[sum(lineitem.l_extendedprice)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(p_partkey@2, l_partkey@1)], filter=CAST(l_quantity@0 AS Decimal128(30, 15)) < Float64(0.2) * avg(lineitem.l_quantity)@1, projection=[l_extendedprice@1] - │ CoalesceBatchesExec: target_batch_size=8192 - │ [Stage 2] => NetworkShuffleExec: output_partitions=6, input_tasks=1 - │ ProjectionExec: expr=[CAST(0.2 * CAST(avg(lineitem.l_quantity)@1 AS Float64) AS Decimal128(30, 15)) as Float64(0.2) * avg(lineitem.l_quantity), l_partkey@0 as l_partkey] - │ AggregateExec: mode=FinalPartitioned, gby=[l_partkey@0 as l_partkey], aggr=[avg(lineitem.l_quantity)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ [Stage 3] => NetworkShuffleExec: output_partitions=6, input_tasks=3 - └────────────────────────────────────────────────── - ┌───── Stage 2 ── Tasks: t0:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9,p10,p11,p12,p13,p14,p15,p16,p17,p18,p19,p20,p21,p22,p23] - │ RepartitionExec: partitioning=Hash([p_partkey@2], 24), input_partitions=6 - │ ProjectionExec: expr=[l_quantity@1 as l_quantity, l_extendedprice@2 as l_extendedprice, p_partkey@0 as p_partkey] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(p_partkey@0, l_partkey@0)], projection=[p_partkey@0, l_quantity@2, l_extendedprice@3] - │ CoalescePartitionsExec - │ [Stage 1] => NetworkCoalesceExec: output_partitions=8, input_tasks=4 - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/data/lineitem/1.parquet:.., /testdata/tpch/data/lineitem/10.parquet:.., /testdata/tpch/data/lineitem/11.parquet:..], [/testdata/tpch/data/lineitem/11.parquet:.., /testdata/tpch/data/lineitem/12.parquet:.., /testdata/tpch/data/lineitem/13.parquet:.., /testdata/tpch/data/lineitem/14.parquet:..], [/testdata/tpch/data/lineitem/14.parquet:.., /testdata/tpch/data/lineitem/15.parquet:.., /testdata/tpch/data/lineitem/16.parquet:..], [/testdata/tpch/data/lineitem/16.parquet:.., /testdata/tpch/data/lineitem/2.parquet:.., /testdata/tpch/data/lineitem/3.parquet:.., /testdata/tpch/data/lineitem/4.parquet:..], [/testdata/tpch/data/lineitem/4.parquet:.., /testdata/tpch/data/lineitem/5.parquet:.., /testdata/tpch/data/lineitem/6.parquet:.., /testdata/tpch/data/lineitem/7.parquet:..], ...]}, projection=[l_partkey, l_quantity, l_extendedprice], file_type=parquet, predicate=DynamicFilterPhysicalExpr [ true ] - └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0,p1] t1:[p2,p3] t2:[p4,p5] t3:[p6,p7] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: p_brand@1 = Brand#23 AND p_container@2 = MED BOX, projection=[p_partkey@0] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/data/part/1.parquet, /testdata/tpch/data/part/10.parquet, /testdata/tpch/data/part/11.parquet], [/testdata/tpch/data/part/12.parquet, /testdata/tpch/data/part/13.parquet, /testdata/tpch/data/part/14.parquet], [/testdata/tpch/data/part/15.parquet, /testdata/tpch/data/part/16.parquet, /testdata/tpch/data/part/2.parquet], [/testdata/tpch/data/part/3.parquet, /testdata/tpch/data/part/4.parquet, /testdata/tpch/data/part/5.parquet], [/testdata/tpch/data/part/6.parquet, /testdata/tpch/data/part/7.parquet, /testdata/tpch/data/part/8.parquet], ...]}, projection=[p_partkey, p_brand, p_container], file_type=parquet, predicate=p_brand@1 = Brand#23 AND p_container@2 = MED BOX, pruning_predicate=p_brand_null_count@2 != row_count@3 AND p_brand_min@0 <= Brand#23 AND Brand#23 <= p_brand_max@1 AND p_container_null_count@6 != row_count@3 AND p_container_min@4 <= MED BOX AND MED BOX <= p_container_max@5, required_guarantees=[p_brand in (Brand#23), p_container in (MED BOX)] - └────────────────────────────────────────────────── - ┌───── Stage 3 ── Tasks: t0:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9,p10,p11,p12,p13,p14,p15,p16,p17,p18,p19,p20,p21,p22,p23] t1:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9,p10,p11,p12,p13,p14,p15,p16,p17,p18,p19,p20,p21,p22,p23] t2:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9,p10,p11,p12,p13,p14,p15,p16,p17,p18,p19,p20,p21,p22,p23] - │ RepartitionExec: partitioning=Hash([l_partkey@0], 24), input_partitions=2 - │ AggregateExec: mode=Partial, gby=[l_partkey@0 as l_partkey], aggr=[avg(lineitem.l_quantity)] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/data/lineitem/1.parquet:.., /testdata/tpch/data/lineitem/10.parquet:.., /testdata/tpch/data/lineitem/11.parquet:..], [/testdata/tpch/data/lineitem/11.parquet:.., /testdata/tpch/data/lineitem/12.parquet:.., /testdata/tpch/data/lineitem/13.parquet:.., /testdata/tpch/data/lineitem/14.parquet:..], [/testdata/tpch/data/lineitem/14.parquet:.., /testdata/tpch/data/lineitem/15.parquet:.., /testdata/tpch/data/lineitem/16.parquet:..], [/testdata/tpch/data/lineitem/16.parquet:.., /testdata/tpch/data/lineitem/2.parquet:.., /testdata/tpch/data/lineitem/3.parquet:.., /testdata/tpch/data/lineitem/4.parquet:..], [/testdata/tpch/data/lineitem/4.parquet:.., /testdata/tpch/data/lineitem/5.parquet:.., /testdata/tpch/data/lineitem/6.parquet:.., /testdata/tpch/data/lineitem/7.parquet:..], ...]}, projection=[l_partkey, l_quantity], file_type=parquet - └────────────────────────────────────────────────── - "); - Ok(()) - } - - #[tokio::test] - async fn test_tpch_18() -> Result<(), Box> { - let plan = test_tpch_query(18).await?; - assert_snapshot!(plan, @r" - ┌───── DistributedExec ── Tasks: t0:[p0] - │ SortPreservingMergeExec: [o_totalprice@4 DESC, o_orderdate@3 ASC NULLS LAST] - │ [Stage 8] => NetworkCoalesceExec: output_partitions=24, input_tasks=4 - └────────────────────────────────────────────────── - ┌───── Stage 8 ── Tasks: t0:[p0,p1,p2,p3,p4,p5] t1:[p0,p1,p2,p3,p4,p5] t2:[p0,p1,p2,p3,p4,p5] t3:[p0,p1,p2,p3,p4,p5] - │ SortExec: expr=[o_totalprice@4 DESC, o_orderdate@3 ASC NULLS LAST], preserve_partitioning=[true] - │ AggregateExec: mode=FinalPartitioned, gby=[c_name@0 as c_name, c_custkey@1 as c_custkey, o_orderkey@2 as o_orderkey, o_orderdate@3 as o_orderdate, o_totalprice@4 as o_totalprice], aggr=[sum(lineitem.l_quantity)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ [Stage 7] => NetworkShuffleExec: output_partitions=6, input_tasks=1 - └────────────────────────────────────────────────── - ┌───── Stage 7 ── Tasks: t0:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9,p10,p11,p12,p13,p14,p15,p16,p17,p18,p19,p20,p21,p22,p23] - │ RepartitionExec: partitioning=Hash([c_name@0, c_custkey@1, o_orderkey@2, o_orderdate@3, o_totalprice@4], 24), input_partitions=6 - │ AggregateExec: mode=Partial, gby=[c_name@1 as c_name, c_custkey@0 as c_custkey, o_orderkey@2 as o_orderkey, o_orderdate@4 as o_orderdate, o_totalprice@3 as o_totalprice], aggr=[sum(lineitem.l_quantity)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(l_orderkey@0, o_orderkey@2)] - │ CoalescePartitionsExec - │ [Stage 2] => NetworkCoalesceExec: output_partitions=24, input_tasks=4 - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(o_orderkey@2, l_orderkey@0)], projection=[c_custkey@0, c_name@1, o_orderkey@2, o_totalprice@3, o_orderdate@4, l_quantity@6] - │ CoalesceBatchesExec: target_batch_size=8192 - │ [Stage 5] => NetworkShuffleExec: output_partitions=6, input_tasks=3 - │ CoalesceBatchesExec: target_batch_size=8192 - │ [Stage 6] => NetworkShuffleExec: output_partitions=6, input_tasks=3 - └────────────────────────────────────────────────── - ┌───── Stage 2 ── Tasks: t0:[p0,p1,p2,p3,p4,p5] t1:[p0,p1,p2,p3,p4,p5] t2:[p0,p1,p2,p3,p4,p5] t3:[p0,p1,p2,p3,p4,p5] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: sum(lineitem.l_quantity)@1 > Some(30000),25,2, projection=[l_orderkey@0] - │ AggregateExec: mode=FinalPartitioned, gby=[l_orderkey@0 as l_orderkey], aggr=[sum(lineitem.l_quantity)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ [Stage 1] => NetworkShuffleExec: output_partitions=6, input_tasks=3 - └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9,p10,p11,p12,p13,p14,p15,p16,p17,p18,p19,p20,p21,p22,p23] t1:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9,p10,p11,p12,p13,p14,p15,p16,p17,p18,p19,p20,p21,p22,p23] t2:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9,p10,p11,p12,p13,p14,p15,p16,p17,p18,p19,p20,p21,p22,p23] - │ RepartitionExec: partitioning=Hash([l_orderkey@0], 24), input_partitions=2 - │ AggregateExec: mode=Partial, gby=[l_orderkey@0 as l_orderkey], aggr=[sum(lineitem.l_quantity)] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/data/lineitem/1.parquet:.., /testdata/tpch/data/lineitem/10.parquet:.., /testdata/tpch/data/lineitem/11.parquet:..], [/testdata/tpch/data/lineitem/11.parquet:.., /testdata/tpch/data/lineitem/12.parquet:.., /testdata/tpch/data/lineitem/13.parquet:.., /testdata/tpch/data/lineitem/14.parquet:..], [/testdata/tpch/data/lineitem/14.parquet:.., /testdata/tpch/data/lineitem/15.parquet:.., /testdata/tpch/data/lineitem/16.parquet:..], [/testdata/tpch/data/lineitem/16.parquet:.., /testdata/tpch/data/lineitem/2.parquet:.., /testdata/tpch/data/lineitem/3.parquet:.., /testdata/tpch/data/lineitem/4.parquet:..], [/testdata/tpch/data/lineitem/4.parquet:.., /testdata/tpch/data/lineitem/5.parquet:.., /testdata/tpch/data/lineitem/6.parquet:.., /testdata/tpch/data/lineitem/7.parquet:..], ...]}, projection=[l_orderkey, l_quantity], file_type=parquet - └────────────────────────────────────────────────── - ┌───── Stage 5 ── Tasks: t0:[p0,p1,p2,p3,p4,p5] t1:[p0,p1,p2,p3,p4,p5] t2:[p0,p1,p2,p3,p4,p5] - │ RepartitionExec: partitioning=Hash([o_orderkey@2], 6), input_partitions=6 - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=Partitioned, join_type=Inner, on=[(c_custkey@0, o_custkey@1)], projection=[c_custkey@0, c_name@1, o_orderkey@2, o_totalprice@4, o_orderdate@5] - │ CoalesceBatchesExec: target_batch_size=8192 - │ [Stage 3] => NetworkShuffleExec: output_partitions=6, input_tasks=3 - │ CoalesceBatchesExec: target_batch_size=8192 - │ [Stage 4] => NetworkShuffleExec: output_partitions=6, input_tasks=3 - └────────────────────────────────────────────────── - ┌───── Stage 3 ── Tasks: t0:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9,p10,p11,p12,p13,p14,p15,p16,p17] t1:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9,p10,p11,p12,p13,p14,p15,p16,p17] t2:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9,p10,p11,p12,p13,p14,p15,p16,p17] - │ RepartitionExec: partitioning=Hash([c_custkey@0], 18), input_partitions=2 - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/data/customer/1.parquet, /testdata/tpch/data/customer/10.parquet, /testdata/tpch/data/customer/11.parquet], [/testdata/tpch/data/customer/12.parquet, /testdata/tpch/data/customer/13.parquet, /testdata/tpch/data/customer/14.parquet], [/testdata/tpch/data/customer/15.parquet, /testdata/tpch/data/customer/16.parquet, /testdata/tpch/data/customer/2.parquet], [/testdata/tpch/data/customer/3.parquet, /testdata/tpch/data/customer/4.parquet, /testdata/tpch/data/customer/5.parquet], [/testdata/tpch/data/customer/6.parquet, /testdata/tpch/data/customer/7.parquet, /testdata/tpch/data/customer/8.parquet], ...]}, projection=[c_custkey, c_name], file_type=parquet - └────────────────────────────────────────────────── - ┌───── Stage 4 ── Tasks: t0:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9,p10,p11,p12,p13,p14,p15,p16,p17] t1:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9,p10,p11,p12,p13,p14,p15,p16,p17] t2:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9,p10,p11,p12,p13,p14,p15,p16,p17] - │ RepartitionExec: partitioning=Hash([o_custkey@1], 18), input_partitions=2 - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/data/orders/1.parquet:.., /testdata/tpch/data/orders/10.parquet:.., /testdata/tpch/data/orders/11.parquet:..], [/testdata/tpch/data/orders/11.parquet:.., /testdata/tpch/data/orders/12.parquet:.., /testdata/tpch/data/orders/13.parquet:.., /testdata/tpch/data/orders/14.parquet:..], [/testdata/tpch/data/orders/14.parquet:.., /testdata/tpch/data/orders/15.parquet:.., /testdata/tpch/data/orders/16.parquet:..], [/testdata/tpch/data/orders/16.parquet:.., /testdata/tpch/data/orders/2.parquet:.., /testdata/tpch/data/orders/3.parquet:.., /testdata/tpch/data/orders/4.parquet:..], [/testdata/tpch/data/orders/4.parquet:.., /testdata/tpch/data/orders/5.parquet:.., /testdata/tpch/data/orders/6.parquet:.., /testdata/tpch/data/orders/7.parquet:..], ...]}, projection=[o_orderkey, o_custkey, o_totalprice, o_orderdate], file_type=parquet, predicate=DynamicFilterPhysicalExpr [ true ] - └────────────────────────────────────────────────── - ┌───── Stage 6 ── Tasks: t0:[p0,p1,p2,p3,p4,p5] t1:[p0,p1,p2,p3,p4,p5] t2:[p0,p1,p2,p3,p4,p5] - │ RepartitionExec: partitioning=Hash([l_orderkey@0], 6), input_partitions=2 - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/data/lineitem/1.parquet:.., /testdata/tpch/data/lineitem/10.parquet:.., /testdata/tpch/data/lineitem/11.parquet:..], [/testdata/tpch/data/lineitem/11.parquet:.., /testdata/tpch/data/lineitem/12.parquet:.., /testdata/tpch/data/lineitem/13.parquet:.., /testdata/tpch/data/lineitem/14.parquet:..], [/testdata/tpch/data/lineitem/14.parquet:.., /testdata/tpch/data/lineitem/15.parquet:.., /testdata/tpch/data/lineitem/16.parquet:..], [/testdata/tpch/data/lineitem/16.parquet:.., /testdata/tpch/data/lineitem/2.parquet:.., /testdata/tpch/data/lineitem/3.parquet:.., /testdata/tpch/data/lineitem/4.parquet:..], [/testdata/tpch/data/lineitem/4.parquet:.., /testdata/tpch/data/lineitem/5.parquet:.., /testdata/tpch/data/lineitem/6.parquet:.., /testdata/tpch/data/lineitem/7.parquet:..], ...]}, projection=[l_orderkey, l_quantity], file_type=parquet, predicate=DynamicFilterPhysicalExpr [ true ] - └────────────────────────────────────────────────── - "); - Ok(()) - } - - #[tokio::test] - async fn test_tpch_19() -> Result<(), Box> { - let plan = test_tpch_query(19).await?; - assert_snapshot!(plan, @r#" - ┌───── DistributedExec ── Tasks: t0:[p0] - │ ProjectionExec: expr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)@0 as revenue] - │ AggregateExec: mode=Final, gby=[], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] - │ CoalescePartitionsExec - │ AggregateExec: mode=Partial, gby=[], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(p_partkey@0, l_partkey@0)], filter=p_brand@1 = Brand#12 AND p_container@3 IN ([Literal { value: Utf8View("SM CASE"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("SM BOX"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("SM PACK"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("SM PKG"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }]) AND l_quantity@0 >= Some(100),15,2 AND l_quantity@0 <= Some(1100),15,2 AND p_size@2 <= 5 OR p_brand@1 = Brand#23 AND p_container@3 IN ([Literal { value: Utf8View("MED BAG"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("MED BOX"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("MED PKG"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("MED PACK"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }]) AND l_quantity@0 >= Some(1000),15,2 AND l_quantity@0 <= Some(2000),15,2 AND p_size@2 <= 10 OR p_brand@1 = Brand#34 AND p_container@3 IN ([Literal { value: Utf8View("LG CASE"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("LG BOX"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("LG PACK"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("LG PKG"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }]) AND l_quantity@0 >= Some(2000),15,2 AND l_quantity@0 <= Some(3000),15,2 AND p_size@2 <= 15, projection=[l_extendedprice@6, l_discount@7] - │ CoalescePartitionsExec - │ [Stage 1] => NetworkCoalesceExec: output_partitions=8, input_tasks=4 - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: (l_quantity@1 >= Some(100),15,2 AND l_quantity@1 <= Some(1100),15,2 OR l_quantity@1 >= Some(1000),15,2 AND l_quantity@1 <= Some(2000),15,2 OR l_quantity@1 >= Some(2000),15,2 AND l_quantity@1 <= Some(3000),15,2) AND (l_shipmode@5 = AIR OR l_shipmode@5 = AIR REG) AND l_shipinstruct@4 = DELIVER IN PERSON, projection=[l_partkey@0, l_quantity@1, l_extendedprice@2, l_discount@3] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/data/lineitem/1.parquet:.., /testdata/tpch/data/lineitem/10.parquet:.., /testdata/tpch/data/lineitem/11.parquet:..], [/testdata/tpch/data/lineitem/11.parquet:.., /testdata/tpch/data/lineitem/12.parquet:.., /testdata/tpch/data/lineitem/13.parquet:.., /testdata/tpch/data/lineitem/14.parquet:..], [/testdata/tpch/data/lineitem/14.parquet:.., /testdata/tpch/data/lineitem/15.parquet:.., /testdata/tpch/data/lineitem/16.parquet:..], [/testdata/tpch/data/lineitem/16.parquet:.., /testdata/tpch/data/lineitem/2.parquet:.., /testdata/tpch/data/lineitem/3.parquet:.., /testdata/tpch/data/lineitem/4.parquet:..], [/testdata/tpch/data/lineitem/4.parquet:.., /testdata/tpch/data/lineitem/5.parquet:.., /testdata/tpch/data/lineitem/6.parquet:.., /testdata/tpch/data/lineitem/7.parquet:..], ...]}, projection=[l_partkey, l_quantity, l_extendedprice, l_discount, l_shipinstruct, l_shipmode], file_type=parquet, predicate=(l_quantity@1 >= Some(100),15,2 AND l_quantity@1 <= Some(1100),15,2 OR l_quantity@1 >= Some(1000),15,2 AND l_quantity@1 <= Some(2000),15,2 OR l_quantity@1 >= Some(2000),15,2 AND l_quantity@1 <= Some(3000),15,2) AND (l_shipmode@5 = AIR OR l_shipmode@5 = AIR REG) AND l_shipinstruct@4 = DELIVER IN PERSON AND DynamicFilterPhysicalExpr [ l_partkey@0 >= 55 AND l_partkey@0 <= 19916 OR l_partkey@0 >= 55 AND l_partkey@0 <= 19916 OR l_partkey@0 >= 55 AND l_partkey@0 <= 19916 OR l_partkey@0 >= 55 AND l_partkey@0 <= 19916 OR l_partkey@0 >= 55 AND l_partkey@0 <= 19916 OR l_partkey@0 >= 55 AND l_partkey@0 <= 19916 ], pruning_predicate=(l_quantity_null_count@1 != row_count@2 AND l_quantity_max@0 >= Some(100),15,2 AND l_quantity_null_count@1 != row_count@2 AND l_quantity_min@3 <= Some(1100),15,2 OR l_quantity_null_count@1 != row_count@2 AND l_quantity_max@0 >= Some(1000),15,2 AND l_quantity_null_count@1 != row_count@2 AND l_quantity_min@3 <= Some(2000),15,2 OR l_quantity_null_count@1 != row_count@2 AND l_quantity_max@0 >= Some(2000),15,2 AND l_quantity_null_count@1 != row_count@2 AND l_quantity_min@3 <= Some(3000),15,2) AND (l_shipmode_null_count@6 != row_count@2 AND l_shipmode_min@4 <= AIR AND AIR <= l_shipmode_max@5 OR l_shipmode_null_count@6 != row_count@2 AND l_shipmode_min@4 <= AIR REG AND AIR REG <= l_shipmode_max@5) AND l_shipinstruct_null_count@9 != row_count@2 AND l_shipinstruct_min@7 <= DELIVER IN PERSON AND DELIVER IN PERSON <= l_shipinstruct_max@8 AND (l_partkey_null_count@11 != row_count@2 AND l_partkey_max@10 >= 55 AND l_partkey_null_count@11 != row_count@2 AND l_partkey_min@12 <= 19916 OR l_partkey_null_count@11 != row_count@2 AND l_partkey_max@10 >= 55 AND l_partkey_null_count@11 != row_count@2 AND l_partkey_min@12 <= 19916 OR l_partkey_null_count@11 != row_count@2 AND l_partkey_max@10 >= 55 AND l_partkey_null_count@11 != row_count@2 AND l_partkey_min@12 <= 19916 OR l_partkey_null_count@11 != row_count@2 AND l_partkey_max@10 >= 55 AND l_partkey_null_count@11 != row_count@2 AND l_partkey_min@12 <= 19916 OR l_partkey_null_count@11 != row_count@2 AND l_partkey_max@10 >= 55 AND l_partkey_null_count@11 != row_count@2 AND l_partkey_min@12 <= 19916 OR l_partkey_null_count@11 != row_count@2 AND l_partkey_max@10 >= 55 AND l_partkey_null_count@11 != row_count@2 AND l_partkey_min@12 <= 19916), required_guarantees=[l_shipinstruct in (DELIVER IN PERSON), l_shipmode in (AIR, AIR REG)] - └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0,p1] t1:[p2,p3] t2:[p4,p5] t3:[p6,p7] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: (p_brand@1 = Brand#12 AND p_container@3 IN ([Literal { value: Utf8View("SM CASE"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("SM BOX"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("SM PACK"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("SM PKG"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }]) AND p_size@2 <= 5 OR p_brand@1 = Brand#23 AND p_container@3 IN ([Literal { value: Utf8View("MED BAG"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("MED BOX"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("MED PKG"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("MED PACK"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }]) AND p_size@2 <= 10 OR p_brand@1 = Brand#34 AND p_container@3 IN ([Literal { value: Utf8View("LG CASE"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("LG BOX"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("LG PACK"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("LG PKG"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }]) AND p_size@2 <= 15) AND p_size@2 >= 1 - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/data/part/1.parquet, /testdata/tpch/data/part/10.parquet, /testdata/tpch/data/part/11.parquet], [/testdata/tpch/data/part/12.parquet, /testdata/tpch/data/part/13.parquet, /testdata/tpch/data/part/14.parquet], [/testdata/tpch/data/part/15.parquet, /testdata/tpch/data/part/16.parquet, /testdata/tpch/data/part/2.parquet], [/testdata/tpch/data/part/3.parquet, /testdata/tpch/data/part/4.parquet, /testdata/tpch/data/part/5.parquet], [/testdata/tpch/data/part/6.parquet, /testdata/tpch/data/part/7.parquet, /testdata/tpch/data/part/8.parquet], ...]}, projection=[p_partkey, p_brand, p_size, p_container], file_type=parquet, predicate=(p_brand@1 = Brand#12 AND p_container@3 IN ([Literal { value: Utf8View("SM CASE"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("SM BOX"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("SM PACK"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("SM PKG"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }]) AND p_size@2 <= 5 OR p_brand@1 = Brand#23 AND p_container@3 IN ([Literal { value: Utf8View("MED BAG"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("MED BOX"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("MED PKG"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("MED PACK"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }]) AND p_size@2 <= 10 OR p_brand@1 = Brand#34 AND p_container@3 IN ([Literal { value: Utf8View("LG CASE"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("LG BOX"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("LG PACK"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("LG PKG"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }]) AND p_size@2 <= 15) AND p_size@2 >= 1, pruning_predicate=(p_brand_null_count@2 != row_count@3 AND p_brand_min@0 <= Brand#12 AND Brand#12 <= p_brand_max@1 AND (p_container_null_count@6 != row_count@3 AND p_container_min@4 <= SM CASE AND SM CASE <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= SM BOX AND SM BOX <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= SM PACK AND SM PACK <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= SM PKG AND SM PKG <= p_container_max@5) AND p_size_null_count@8 != row_count@3 AND p_size_min@7 <= 5 OR p_brand_null_count@2 != row_count@3 AND p_brand_min@0 <= Brand#23 AND Brand#23 <= p_brand_max@1 AND (p_container_null_count@6 != row_count@3 AND p_container_min@4 <= MED BAG AND MED BAG <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= MED BOX AND MED BOX <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= MED PKG AND MED PKG <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= MED PACK AND MED PACK <= p_container_max@5) AND p_size_null_count@8 != row_count@3 AND p_size_min@7 <= 10 OR p_brand_null_count@2 != row_count@3 AND p_brand_min@0 <= Brand#34 AND Brand#34 <= p_brand_max@1 AND (p_container_null_count@6 != row_count@3 AND p_container_min@4 <= LG CASE AND LG CASE <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= LG BOX AND LG BOX <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= LG PACK AND LG PACK <= p_container_max@5 OR p_container_null_count@6 != row_count@3 AND p_container_min@4 <= LG PKG AND LG PKG <= p_container_max@5) AND p_size_null_count@8 != row_count@3 AND p_size_min@7 <= 15) AND p_size_null_count@8 != row_count@3 AND p_size_max@9 >= 1, required_guarantees=[p_brand in (Brand#12, Brand#23, Brand#34), p_container in (LG BOX, LG CASE, LG PACK, LG PKG, MED BAG, MED BOX, MED PACK, MED PKG, SM BOX, SM CASE, SM PACK, SM PKG)] - └────────────────────────────────────────────────── - "#); - Ok(()) - } - - #[tokio::test] - async fn test_tpch_20() -> Result<(), Box> { - let plan = test_tpch_query(20).await?; - assert_snapshot!(plan, @r" - ┌───── DistributedExec ── Tasks: t0:[p0] - │ SortPreservingMergeExec: [s_name@0 ASC NULLS LAST] - │ SortExec: expr=[s_name@0 ASC NULLS LAST], preserve_partitioning=[true] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=LeftSemi, on=[(s_suppkey@0, ps_suppkey@0)], projection=[s_name@1, s_address@2] - │ CoalescePartitionsExec - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@3)], projection=[s_suppkey@1, s_name@2, s_address@3] - │ CoalescePartitionsExec - │ [Stage 1] => NetworkCoalesceExec: output_partitions=8, input_tasks=4 - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/data/supplier/1.parquet, /testdata/tpch/data/supplier/10.parquet, /testdata/tpch/data/supplier/11.parquet], [/testdata/tpch/data/supplier/12.parquet, /testdata/tpch/data/supplier/13.parquet, /testdata/tpch/data/supplier/14.parquet], [/testdata/tpch/data/supplier/15.parquet, /testdata/tpch/data/supplier/16.parquet, /testdata/tpch/data/supplier/2.parquet], [/testdata/tpch/data/supplier/3.parquet, /testdata/tpch/data/supplier/4.parquet, /testdata/tpch/data/supplier/5.parquet], [/testdata/tpch/data/supplier/6.parquet, /testdata/tpch/data/supplier/7.parquet, /testdata/tpch/data/supplier/8.parquet], ...]}, projection=[s_suppkey, s_name, s_address, s_nationkey], file_type=parquet, predicate=DynamicFilterPhysicalExpr [ s_nationkey@3 >= 3 AND s_nationkey@3 <= 3 OR s_nationkey@3 >= 3 AND s_nationkey@3 <= 3 OR s_nationkey@3 >= 3 AND s_nationkey@3 <= 3 OR s_nationkey@3 >= 3 AND s_nationkey@3 <= 3 OR s_nationkey@3 >= 3 AND s_nationkey@3 <= 3 OR s_nationkey@3 >= 3 AND s_nationkey@3 <= 3 ], pruning_predicate=s_nationkey_null_count@1 != row_count@2 AND s_nationkey_max@0 >= 3 AND s_nationkey_null_count@1 != row_count@2 AND s_nationkey_min@3 <= 3 OR s_nationkey_null_count@1 != row_count@2 AND s_nationkey_max@0 >= 3 AND s_nationkey_null_count@1 != row_count@2 AND s_nationkey_min@3 <= 3 OR s_nationkey_null_count@1 != row_count@2 AND s_nationkey_max@0 >= 3 AND s_nationkey_null_count@1 != row_count@2 AND s_nationkey_min@3 <= 3 OR s_nationkey_null_count@1 != row_count@2 AND s_nationkey_max@0 >= 3 AND s_nationkey_null_count@1 != row_count@2 AND s_nationkey_min@3 <= 3 OR s_nationkey_null_count@1 != row_count@2 AND s_nationkey_max@0 >= 3 AND s_nationkey_null_count@1 != row_count@2 AND s_nationkey_min@3 <= 3 OR s_nationkey_null_count@1 != row_count@2 AND s_nationkey_max@0 >= 3 AND s_nationkey_null_count@1 != row_count@2 AND s_nationkey_min@3 <= 3, required_guarantees=[] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(ps_partkey@0, l_partkey@1), (ps_suppkey@1, l_suppkey@2)], filter=CAST(ps_availqty@0 AS Float64) > Float64(0.5) * sum(lineitem.l_quantity)@1, projection=[ps_suppkey@1] - │ CoalescePartitionsExec - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=RightSemi, on=[(p_partkey@0, ps_partkey@0)] - │ CoalescePartitionsExec - │ [Stage 2] => NetworkCoalesceExec: output_partitions=8, input_tasks=4 - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/data/partsupp/1.parquet:.., /testdata/tpch/data/partsupp/10.parquet:.., /testdata/tpch/data/partsupp/11.parquet:..], [/testdata/tpch/data/partsupp/11.parquet:.., /testdata/tpch/data/partsupp/12.parquet:.., /testdata/tpch/data/partsupp/13.parquet:.., /testdata/tpch/data/partsupp/14.parquet:..], [/testdata/tpch/data/partsupp/14.parquet:.., /testdata/tpch/data/partsupp/15.parquet:.., /testdata/tpch/data/partsupp/16.parquet:.., /testdata/tpch/data/partsupp/2.parquet:..], [/testdata/tpch/data/partsupp/2.parquet:.., /testdata/tpch/data/partsupp/3.parquet:.., /testdata/tpch/data/partsupp/4.parquet:..], [/testdata/tpch/data/partsupp/4.parquet:.., /testdata/tpch/data/partsupp/5.parquet:.., /testdata/tpch/data/partsupp/6.parquet:.., /testdata/tpch/data/partsupp/7.parquet:..], ...]}, projection=[ps_partkey, ps_suppkey, ps_availqty], file_type=parquet - │ ProjectionExec: expr=[0.5 * CAST(sum(lineitem.l_quantity)@2 AS Float64) as Float64(0.5) * sum(lineitem.l_quantity), l_partkey@0 as l_partkey, l_suppkey@1 as l_suppkey] - │ AggregateExec: mode=FinalPartitioned, gby=[l_partkey@0 as l_partkey, l_suppkey@1 as l_suppkey], aggr=[sum(lineitem.l_quantity)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ [Stage 3] => NetworkShuffleExec: output_partitions=6, input_tasks=3 - └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0,p1] t1:[p2,p3] t2:[p4,p5] t3:[p6,p7] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: n_name@1 = CANADA, projection=[n_nationkey@0] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/data/nation/1.parquet, /testdata/tpch/data/nation/10.parquet, /testdata/tpch/data/nation/11.parquet], [/testdata/tpch/data/nation/12.parquet, /testdata/tpch/data/nation/13.parquet, /testdata/tpch/data/nation/14.parquet], [/testdata/tpch/data/nation/15.parquet, /testdata/tpch/data/nation/16.parquet, /testdata/tpch/data/nation/2.parquet], [/testdata/tpch/data/nation/3.parquet, /testdata/tpch/data/nation/4.parquet, /testdata/tpch/data/nation/5.parquet], [/testdata/tpch/data/nation/6.parquet, /testdata/tpch/data/nation/7.parquet, /testdata/tpch/data/nation/8.parquet], ...]}, projection=[n_nationkey, n_name], file_type=parquet, predicate=n_name@1 = CANADA, pruning_predicate=n_name_null_count@2 != row_count@3 AND n_name_min@0 <= CANADA AND CANADA <= n_name_max@1, required_guarantees=[n_name in (CANADA)] - └────────────────────────────────────────────────── - ┌───── Stage 2 ── Tasks: t0:[p0,p1] t1:[p2,p3] t2:[p4,p5] t3:[p6,p7] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: p_name@1 LIKE forest%, projection=[p_partkey@0] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/data/part/1.parquet, /testdata/tpch/data/part/10.parquet, /testdata/tpch/data/part/11.parquet], [/testdata/tpch/data/part/12.parquet, /testdata/tpch/data/part/13.parquet, /testdata/tpch/data/part/14.parquet], [/testdata/tpch/data/part/15.parquet, /testdata/tpch/data/part/16.parquet, /testdata/tpch/data/part/2.parquet], [/testdata/tpch/data/part/3.parquet, /testdata/tpch/data/part/4.parquet, /testdata/tpch/data/part/5.parquet], [/testdata/tpch/data/part/6.parquet, /testdata/tpch/data/part/7.parquet, /testdata/tpch/data/part/8.parquet], ...]}, projection=[p_partkey, p_name], file_type=parquet, predicate=p_name@1 LIKE forest%, pruning_predicate=p_name_null_count@2 != row_count@3 AND p_name_min@0 <= foresu AND forest <= p_name_max@1, required_guarantees=[] - └────────────────────────────────────────────────── - ┌───── Stage 3 ── Tasks: t0:[p0,p1,p2,p3,p4,p5] t1:[p0,p1,p2,p3,p4,p5] t2:[p0,p1,p2,p3,p4,p5] - │ RepartitionExec: partitioning=Hash([l_partkey@0, l_suppkey@1], 6), input_partitions=2 - │ AggregateExec: mode=Partial, gby=[l_partkey@0 as l_partkey, l_suppkey@1 as l_suppkey], aggr=[sum(lineitem.l_quantity)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: l_shipdate@3 >= 1994-01-01 AND l_shipdate@3 < 1995-01-01, projection=[l_partkey@0, l_suppkey@1, l_quantity@2] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,p1] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/data/lineitem/1.parquet:.., /testdata/tpch/data/lineitem/10.parquet:.., /testdata/tpch/data/lineitem/11.parquet:..], [/testdata/tpch/data/lineitem/11.parquet:.., /testdata/tpch/data/lineitem/12.parquet:.., /testdata/tpch/data/lineitem/13.parquet:.., /testdata/tpch/data/lineitem/14.parquet:..], [/testdata/tpch/data/lineitem/14.parquet:.., /testdata/tpch/data/lineitem/15.parquet:.., /testdata/tpch/data/lineitem/16.parquet:..], [/testdata/tpch/data/lineitem/16.parquet:.., /testdata/tpch/data/lineitem/2.parquet:.., /testdata/tpch/data/lineitem/3.parquet:.., /testdata/tpch/data/lineitem/4.parquet:..], [/testdata/tpch/data/lineitem/4.parquet:.., /testdata/tpch/data/lineitem/5.parquet:.., /testdata/tpch/data/lineitem/6.parquet:.., /testdata/tpch/data/lineitem/7.parquet:..], ...]}, projection=[l_partkey, l_suppkey, l_quantity, l_shipdate], file_type=parquet, predicate=l_shipdate@3 >= 1994-01-01 AND l_shipdate@3 < 1995-01-01, pruning_predicate=l_shipdate_null_count@1 != row_count@2 AND l_shipdate_max@0 >= 1994-01-01 AND l_shipdate_null_count@1 != row_count@2 AND l_shipdate_min@3 < 1995-01-01, required_guarantees=[] - └────────────────────────────────────────────────── - "); - Ok(()) - } - - #[tokio::test] - async fn test_tpch_21() -> Result<(), Box> { - let plan = test_tpch_query(21).await?; - assert_snapshot!(plan, @r" - ┌───── DistributedExec ── Tasks: t0:[p0] - │ SortPreservingMergeExec: [numwait@1 DESC, s_name@0 ASC NULLS LAST] - │ [Stage 4] => NetworkCoalesceExec: output_partitions=24, input_tasks=4 - └────────────────────────────────────────────────── - ┌───── Stage 4 ── Tasks: t0:[p0,p1,p2,p3,p4,p5] t1:[p0,p1,p2,p3,p4,p5] t2:[p0,p1,p2,p3,p4,p5] t3:[p0,p1,p2,p3,p4,p5] - │ SortExec: expr=[numwait@1 DESC, s_name@0 ASC NULLS LAST], preserve_partitioning=[true] - │ ProjectionExec: expr=[s_name@0 as s_name, count(Int64(1))@1 as numwait] - │ AggregateExec: mode=FinalPartitioned, gby=[s_name@0 as s_name], aggr=[count(Int64(1))] - │ CoalesceBatchesExec: target_batch_size=8192 - │ [Stage 3] => NetworkShuffleExec: output_partitions=6, input_tasks=1 - └────────────────────────────────────────────────── - ┌───── Stage 3 ── Tasks: t0:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9,p10,p11,p12,p13,p14,p15,p16,p17,p18,p19,p20,p21,p22,p23] - │ RepartitionExec: partitioning=Hash([s_name@0], 24), input_partitions=6 - │ AggregateExec: mode=Partial, gby=[s_name@0 as s_name], aggr=[count(Int64(1))] - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=LeftAnti, on=[(l_orderkey@1, l_orderkey@0)], filter=l_suppkey@1 != l_suppkey@0, projection=[s_name@0] - │ CoalescePartitionsExec - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=LeftSemi, on=[(l_orderkey@1, l_orderkey@0)], filter=l_suppkey@1 != l_suppkey@0 - │ CoalescePartitionsExec - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(n_nationkey@0, s_nationkey@1)], projection=[s_name@1, l_orderkey@3, l_suppkey@4] - │ CoalescePartitionsExec - │ [Stage 1] => NetworkCoalesceExec: output_partitions=8, input_tasks=4 - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(o_orderkey@0, l_orderkey@2)], projection=[s_name@1, s_nationkey@2, l_orderkey@3, l_suppkey@4] - │ CoalescePartitionsExec - │ [Stage 2] => NetworkCoalesceExec: output_partitions=8, input_tasks=4 - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(s_suppkey@0, l_suppkey@1)], projection=[s_name@1, s_nationkey@2, l_orderkey@3, l_suppkey@4] - │ CoalescePartitionsExec - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/data/supplier/1.parquet, /testdata/tpch/data/supplier/10.parquet, /testdata/tpch/data/supplier/11.parquet], [/testdata/tpch/data/supplier/12.parquet, /testdata/tpch/data/supplier/13.parquet, /testdata/tpch/data/supplier/14.parquet], [/testdata/tpch/data/supplier/15.parquet, /testdata/tpch/data/supplier/16.parquet, /testdata/tpch/data/supplier/2.parquet], [/testdata/tpch/data/supplier/3.parquet, /testdata/tpch/data/supplier/4.parquet, /testdata/tpch/data/supplier/5.parquet], [/testdata/tpch/data/supplier/6.parquet, /testdata/tpch/data/supplier/7.parquet, /testdata/tpch/data/supplier/8.parquet], ...]}, projection=[s_suppkey, s_name, s_nationkey], file_type=parquet - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: l_receiptdate@3 > l_commitdate@2, projection=[l_orderkey@0, l_suppkey@1] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/data/lineitem/1.parquet:.., /testdata/tpch/data/lineitem/10.parquet:.., /testdata/tpch/data/lineitem/11.parquet:..], [/testdata/tpch/data/lineitem/11.parquet:.., /testdata/tpch/data/lineitem/12.parquet:.., /testdata/tpch/data/lineitem/13.parquet:.., /testdata/tpch/data/lineitem/14.parquet:..], [/testdata/tpch/data/lineitem/14.parquet:.., /testdata/tpch/data/lineitem/15.parquet:.., /testdata/tpch/data/lineitem/16.parquet:..], [/testdata/tpch/data/lineitem/16.parquet:.., /testdata/tpch/data/lineitem/2.parquet:.., /testdata/tpch/data/lineitem/3.parquet:.., /testdata/tpch/data/lineitem/4.parquet:..], [/testdata/tpch/data/lineitem/4.parquet:.., /testdata/tpch/data/lineitem/5.parquet:.., /testdata/tpch/data/lineitem/6.parquet:.., /testdata/tpch/data/lineitem/7.parquet:..], ...]}, projection=[l_orderkey, l_suppkey, l_commitdate, l_receiptdate], file_type=parquet, predicate=l_receiptdate@3 > l_commitdate@2 AND DynamicFilterPhysicalExpr [ true ] AND DynamicFilterPhysicalExpr [ true ] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/data/lineitem/1.parquet:.., /testdata/tpch/data/lineitem/10.parquet:.., /testdata/tpch/data/lineitem/11.parquet:..], [/testdata/tpch/data/lineitem/11.parquet:.., /testdata/tpch/data/lineitem/12.parquet:.., /testdata/tpch/data/lineitem/13.parquet:.., /testdata/tpch/data/lineitem/14.parquet:..], [/testdata/tpch/data/lineitem/14.parquet:.., /testdata/tpch/data/lineitem/15.parquet:.., /testdata/tpch/data/lineitem/16.parquet:..], [/testdata/tpch/data/lineitem/16.parquet:.., /testdata/tpch/data/lineitem/2.parquet:.., /testdata/tpch/data/lineitem/3.parquet:.., /testdata/tpch/data/lineitem/4.parquet:..], [/testdata/tpch/data/lineitem/4.parquet:.., /testdata/tpch/data/lineitem/5.parquet:.., /testdata/tpch/data/lineitem/6.parquet:.., /testdata/tpch/data/lineitem/7.parquet:..], ...]}, projection=[l_orderkey, l_suppkey], file_type=parquet - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: l_receiptdate@3 > l_commitdate@2, projection=[l_orderkey@0, l_suppkey@1] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/data/lineitem/1.parquet:.., /testdata/tpch/data/lineitem/10.parquet:.., /testdata/tpch/data/lineitem/11.parquet:..], [/testdata/tpch/data/lineitem/11.parquet:.., /testdata/tpch/data/lineitem/12.parquet:.., /testdata/tpch/data/lineitem/13.parquet:.., /testdata/tpch/data/lineitem/14.parquet:..], [/testdata/tpch/data/lineitem/14.parquet:.., /testdata/tpch/data/lineitem/15.parquet:.., /testdata/tpch/data/lineitem/16.parquet:..], [/testdata/tpch/data/lineitem/16.parquet:.., /testdata/tpch/data/lineitem/2.parquet:.., /testdata/tpch/data/lineitem/3.parquet:.., /testdata/tpch/data/lineitem/4.parquet:..], [/testdata/tpch/data/lineitem/4.parquet:.., /testdata/tpch/data/lineitem/5.parquet:.., /testdata/tpch/data/lineitem/6.parquet:.., /testdata/tpch/data/lineitem/7.parquet:..], ...]}, projection=[l_orderkey, l_suppkey, l_commitdate, l_receiptdate], file_type=parquet, predicate=l_receiptdate@3 > l_commitdate@2 - └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0,p1] t1:[p2,p3] t2:[p4,p5] t3:[p6,p7] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: n_name@1 = SAUDI ARABIA, projection=[n_nationkey@0] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/data/nation/1.parquet, /testdata/tpch/data/nation/10.parquet, /testdata/tpch/data/nation/11.parquet], [/testdata/tpch/data/nation/12.parquet, /testdata/tpch/data/nation/13.parquet, /testdata/tpch/data/nation/14.parquet], [/testdata/tpch/data/nation/15.parquet, /testdata/tpch/data/nation/16.parquet, /testdata/tpch/data/nation/2.parquet], [/testdata/tpch/data/nation/3.parquet, /testdata/tpch/data/nation/4.parquet, /testdata/tpch/data/nation/5.parquet], [/testdata/tpch/data/nation/6.parquet, /testdata/tpch/data/nation/7.parquet, /testdata/tpch/data/nation/8.parquet], ...]}, projection=[n_nationkey, n_name], file_type=parquet, predicate=n_name@1 = SAUDI ARABIA, pruning_predicate=n_name_null_count@2 != row_count@3 AND n_name_min@0 <= SAUDI ARABIA AND SAUDI ARABIA <= n_name_max@1, required_guarantees=[n_name in (SAUDI ARABIA)] - └────────────────────────────────────────────────── - ┌───── Stage 2 ── Tasks: t0:[p0,p1] t1:[p2,p3] t2:[p4,p5] t3:[p6,p7] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: o_orderstatus@1 = F, projection=[o_orderkey@0] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/data/orders/1.parquet:.., /testdata/tpch/data/orders/10.parquet:.., /testdata/tpch/data/orders/11.parquet:..], [/testdata/tpch/data/orders/11.parquet:.., /testdata/tpch/data/orders/12.parquet:.., /testdata/tpch/data/orders/13.parquet:.., /testdata/tpch/data/orders/14.parquet:..], [/testdata/tpch/data/orders/14.parquet:.., /testdata/tpch/data/orders/15.parquet:.., /testdata/tpch/data/orders/16.parquet:..], [/testdata/tpch/data/orders/16.parquet:.., /testdata/tpch/data/orders/2.parquet:.., /testdata/tpch/data/orders/3.parquet:.., /testdata/tpch/data/orders/4.parquet:..], [/testdata/tpch/data/orders/4.parquet:.., /testdata/tpch/data/orders/5.parquet:.., /testdata/tpch/data/orders/6.parquet:.., /testdata/tpch/data/orders/7.parquet:..], ...]}, projection=[o_orderkey, o_orderstatus], file_type=parquet, predicate=o_orderstatus@1 = F, pruning_predicate=o_orderstatus_null_count@2 != row_count@3 AND o_orderstatus_min@0 <= F AND F <= o_orderstatus_max@1, required_guarantees=[o_orderstatus in (F)] - └────────────────────────────────────────────────── - "); - Ok(()) - } - - #[tokio::test] - async fn test_tpch_22() -> Result<(), Box> { - let plan = test_tpch_query(22).await?; - assert_snapshot!(plan, @r#" - ┌───── DistributedExec ── Tasks: t0:[p0] - │ SortPreservingMergeExec: [cntrycode@0 ASC NULLS LAST] - │ [Stage 4] => NetworkCoalesceExec: output_partitions=24, input_tasks=4 - └────────────────────────────────────────────────── - ┌───── Stage 4 ── Tasks: t0:[p0,p1,p2,p3,p4,p5] t1:[p0,p1,p2,p3,p4,p5] t2:[p0,p1,p2,p3,p4,p5] t3:[p0,p1,p2,p3,p4,p5] - │ SortExec: expr=[cntrycode@0 ASC NULLS LAST], preserve_partitioning=[true] - │ ProjectionExec: expr=[cntrycode@0 as cntrycode, count(Int64(1))@1 as numcust, sum(custsale.c_acctbal)@2 as totacctbal] - │ AggregateExec: mode=FinalPartitioned, gby=[cntrycode@0 as cntrycode], aggr=[count(Int64(1)), sum(custsale.c_acctbal)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ [Stage 3] => NetworkShuffleExec: output_partitions=6, input_tasks=1 - └────────────────────────────────────────────────── - ┌───── Stage 3 ── Tasks: t0:[p0,p1,p2,p3,p4,p5,p6,p7,p8,p9,p10,p11,p12,p13,p14,p15,p16,p17,p18,p19,p20,p21,p22,p23] - │ RepartitionExec: partitioning=Hash([cntrycode@0], 24), input_partitions=6 - │ AggregateExec: mode=Partial, gby=[cntrycode@0 as cntrycode], aggr=[count(Int64(1)), sum(custsale.c_acctbal)] - │ ProjectionExec: expr=[substr(c_phone@0, 1, 2) as cntrycode, c_acctbal@1 as c_acctbal] - │ NestedLoopJoinExec: join_type=Inner, filter=CAST(c_acctbal@0 AS Decimal128(19, 6)) > avg(customer.c_acctbal)@1, projection=[c_phone@1, c_acctbal@2] - │ AggregateExec: mode=Final, gby=[], aggr=[avg(customer.c_acctbal)] - │ CoalescePartitionsExec - │ [Stage 1] => NetworkCoalesceExec: output_partitions=8, input_tasks=4 - │ CoalesceBatchesExec: target_batch_size=8192 - │ HashJoinExec: mode=CollectLeft, join_type=LeftAnti, on=[(c_custkey@0, o_custkey@0)], projection=[c_phone@1, c_acctbal@2] - │ CoalescePartitionsExec - │ [Stage 2] => NetworkCoalesceExec: output_partitions=8, input_tasks=4 - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/data/orders/1.parquet:.., /testdata/tpch/data/orders/10.parquet:.., /testdata/tpch/data/orders/11.parquet:..], [/testdata/tpch/data/orders/11.parquet:.., /testdata/tpch/data/orders/12.parquet:.., /testdata/tpch/data/orders/13.parquet:.., /testdata/tpch/data/orders/14.parquet:..], [/testdata/tpch/data/orders/14.parquet:.., /testdata/tpch/data/orders/15.parquet:.., /testdata/tpch/data/orders/16.parquet:..], [/testdata/tpch/data/orders/16.parquet:.., /testdata/tpch/data/orders/2.parquet:.., /testdata/tpch/data/orders/3.parquet:.., /testdata/tpch/data/orders/4.parquet:..], [/testdata/tpch/data/orders/4.parquet:.., /testdata/tpch/data/orders/5.parquet:.., /testdata/tpch/data/orders/6.parquet:.., /testdata/tpch/data/orders/7.parquet:..], ...]}, projection=[o_custkey], file_type=parquet - └────────────────────────────────────────────────── - ┌───── Stage 1 ── Tasks: t0:[p0,p1] t1:[p2,p3] t2:[p4,p5] t3:[p6,p7] - │ AggregateExec: mode=Partial, gby=[], aggr=[avg(customer.c_acctbal)] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: c_acctbal@1 > Some(0),15,2 AND substr(c_phone@0, 1, 2) IN ([Literal { value: Utf8View("13"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("31"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("23"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("29"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("30"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("18"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("17"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }]), projection=[c_acctbal@1] - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/data/customer/1.parquet, /testdata/tpch/data/customer/10.parquet, /testdata/tpch/data/customer/11.parquet], [/testdata/tpch/data/customer/12.parquet, /testdata/tpch/data/customer/13.parquet, /testdata/tpch/data/customer/14.parquet], [/testdata/tpch/data/customer/15.parquet, /testdata/tpch/data/customer/16.parquet, /testdata/tpch/data/customer/2.parquet], [/testdata/tpch/data/customer/3.parquet, /testdata/tpch/data/customer/4.parquet, /testdata/tpch/data/customer/5.parquet], [/testdata/tpch/data/customer/6.parquet, /testdata/tpch/data/customer/7.parquet, /testdata/tpch/data/customer/8.parquet], ...]}, projection=[c_phone, c_acctbal], file_type=parquet, predicate=c_acctbal@1 > Some(0),15,2 AND substr(c_phone@0, 1, 2) IN ([Literal { value: Utf8View("13"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("31"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("23"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("29"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("30"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("18"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("17"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }]), pruning_predicate=c_acctbal_null_count@1 != row_count@2 AND c_acctbal_max@0 > Some(0),15,2, required_guarantees=[] - └────────────────────────────────────────────────── - ┌───── Stage 2 ── Tasks: t0:[p0,p1] t1:[p2,p3] t2:[p4,p5] t3:[p6,p7] - │ CoalesceBatchesExec: target_batch_size=8192 - │ FilterExec: substr(c_phone@1, 1, 2) IN ([Literal { value: Utf8View("13"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("31"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("23"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("29"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("30"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("18"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("17"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }]) - │ PartitionIsolatorExec: t0:[p0,p1,__,__,__,__] t1:[__,__,p0,p1,__,__] t2:[__,__,__,__,p0,__] t3:[__,__,__,__,__,p0] - │ DataSourceExec: file_groups={6 groups: [[/testdata/tpch/data/customer/1.parquet, /testdata/tpch/data/customer/10.parquet, /testdata/tpch/data/customer/11.parquet], [/testdata/tpch/data/customer/12.parquet, /testdata/tpch/data/customer/13.parquet, /testdata/tpch/data/customer/14.parquet], [/testdata/tpch/data/customer/15.parquet, /testdata/tpch/data/customer/16.parquet, /testdata/tpch/data/customer/2.parquet], [/testdata/tpch/data/customer/3.parquet, /testdata/tpch/data/customer/4.parquet, /testdata/tpch/data/customer/5.parquet], [/testdata/tpch/data/customer/6.parquet, /testdata/tpch/data/customer/7.parquet, /testdata/tpch/data/customer/8.parquet], ...]}, projection=[c_custkey, c_phone, c_acctbal], file_type=parquet, predicate=substr(c_phone@1, 1, 2) IN ([Literal { value: Utf8View("13"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("31"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("23"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("29"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("30"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("18"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("17"), field: Field { name: "lit", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }]) - └────────────────────────────────────────────────── - "#); - Ok(()) - } - - async fn test_tpch_query(query_id: u8) -> Result> { - let (ctx, _guard) = start_localhost_context(2, build_state).await; - run_tpch_query(ctx, query_id).await - } - - async fn build_state( - ctx: DistributedSessionBuilderContext, - ) -> Result { - let rule = DistributedPhysicalOptimizerRule::new() - .with_network_shuffle_tasks(SHUFFLE_TASKS) - .with_network_coalesce_tasks(COALESCE_TASKS); - Ok(SessionStateBuilder::new() - .with_runtime_env(ctx.runtime_env) - .with_default_features() - .with_physical_optimizer_rule(Arc::new(rule)) - .build()) - } - - // test_non_distributed_consistency runs each TPC-H query twice - once in a distributed manner - // and once in a non-distributed manner. For each query, it asserts that the results are identical. - async fn run_tpch_query(ctx_d: SessionContext, query_id: u8) -> Result> { - ensure_tpch_data(TPCH_SCALE_FACTOR, TPCH_DATA_PARTS).await; - let sql = get_test_tpch_query(query_id); - ctx_d - .state_ref() - .write() - .config_mut() - .options_mut() - .execution - .target_partitions = PARTITIONS; - - // Context 1: Non-distributed execution. - let config_s = SessionConfig::new().with_target_partitions(PARTITIONS); - let state_s = SessionStateBuilder::new() - .with_default_features() - .with_config(config_s) - .build(); - let ctx_s = SessionContext::new_with_state(state_s); - - // Register tables for first context - for table_name in [ - "lineitem", "orders", "part", "partsupp", "customer", "nation", "region", "supplier", - ] { - let query_path = get_test_data_dir().join(table_name); - ctx_s - .register_parquet( - table_name, - query_path.to_string_lossy().as_ref(), - datafusion::prelude::ParquetReadOptions::default(), - ) - .await?; - - ctx_d - .register_parquet( - table_name, - query_path.to_string_lossy().as_ref(), - datafusion::prelude::ParquetReadOptions::default(), - ) - .await?; - } - - // Query 15 has three queries in it, one creating the view, the second - // executing, which we want to capture the output of, and the third - // tearing down the view - let (stream_s, stream_d, plan_d) = if query_id == 15 { - let queries: Vec<&str> = sql - .split(';') - .map(str::trim) - .filter(|s| !s.is_empty()) - .collect(); - - ctx_s.sql(queries[0]).await?.collect().await?; - ctx_d.sql(queries[0]).await?.collect().await?; - let df_s = ctx_s.sql(queries[1]).await?; - let df_d = ctx_d.sql(queries[1]).await?; - - let plan_d = df_d.create_physical_plan().await?; - - let stream_s = df_s.execute_stream().await?; - let stream_d = execute_stream(plan_d.clone(), ctx_d.task_ctx())?; - - ctx_s.sql(queries[2]).await?.collect().await?; - ctx_d.sql(queries[2]).await?.collect().await?; - (stream_s, stream_d, plan_d) - } else { - let stream_s = ctx_s.sql(&sql).await?.execute_stream().await?; - let df_d = ctx_d.sql(&sql).await?; - - let plan_d = df_d.create_physical_plan().await?; - - let stream_d = execute_stream(plan_d.clone(), ctx_d.task_ctx())?; - - (stream_s, stream_d, plan_d) - }; - - let batches_s = stream_s.try_collect::>().await?; - let batches_d = stream_d.try_collect::>().await?; - - let formatted_s = arrow::util::pretty::pretty_format_batches(&batches_s)?; - let formatted_d = arrow::util::pretty::pretty_format_batches(&batches_d)?; - - assert_eq!( - formatted_d.to_string(), - formatted_s.to_string(), - "Query {} results differ between executions", - query_id - ); - Ok(display_plan_ascii(plan_d.as_ref())) - } - - pub fn get_test_data_dir() -> std::path::PathBuf { - std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("testdata/tpch/data") - } - - pub fn get_test_queries_dir() -> std::path::PathBuf { - std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("testdata/tpch/queries") - } - - pub fn get_test_tpch_query(num: u8) -> String { - let queries_dir = get_test_queries_dir(); - tpch::tpch_query_from_dir(&queries_dir, num) - } - - // OnceCell to ensure TPCH tables are generated only once for tests - static INIT_TEST_TPCH_TABLES: OnceCell<()> = OnceCell::const_new(); - - // ensure_tpch_data initializes the TPCH data on disk. - pub async fn ensure_tpch_data(sf: f64, parts: i32) { - INIT_TEST_TPCH_TABLES - .get_or_init(|| async { - if !fs::exists(get_test_data_dir()).unwrap() { - tpch::generate_tpch_data(&get_test_data_dir(), sf, parts); - } - }) - .await; - } -} diff --git a/tests/udfs.rs b/tests/udfs.rs new file mode 100644 index 00000000..3df06e75 --- /dev/null +++ b/tests/udfs.rs @@ -0,0 +1,128 @@ +#[cfg(all(feature = "integration", test))] +mod tests { + use arrow::datatypes::{Field, Schema}; + use arrow::util::pretty::pretty_format_batches; + use datafusion::arrow::datatypes::DataType; + use datafusion::error::DataFusionError; + use datafusion::execution::{SessionState, SessionStateBuilder}; + use datafusion::logical_expr::{ + ColumnarValue, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Signature, Volatility, + }; + use datafusion::physical_expr::expressions::lit; + use datafusion::physical_expr::{Partitioning, ScalarFunctionExpr}; + use datafusion::physical_optimizer::PhysicalOptimizerRule; + use datafusion::physical_plan::empty::EmptyExec; + use datafusion::physical_plan::repartition::RepartitionExec; + use datafusion::physical_plan::{ExecutionPlan, execute_stream}; + use datafusion_distributed::test_utils::localhost::start_localhost_context; + use datafusion_distributed::{ + DistributedExt, DistributedPhysicalOptimizerRule, WorkerQueryContext, assert_snapshot, + display_plan_ascii, + }; + use futures::TryStreamExt; + use std::any::Any; + use std::error::Error; + use std::sync::Arc; + + #[tokio::test] + async fn test_udf_in_partitioning_field() -> Result<(), Box> { + async fn build_state(ctx: WorkerQueryContext) -> Result { + Ok(ctx.builder.with_scalar_functions(vec![udf()]).build()) + } + + let (mut ctx, _guard, _) = start_localhost_context(3, build_state).await; + ctx = SessionStateBuilder::from(ctx.state()) + .with_distributed_task_estimator(2) + .with_scalar_functions(vec![udf()]) + .build() + .into(); + + let wrap = |input: Arc| -> Arc { + Arc::new( + RepartitionExec::try_new( + input, + Partitioning::Hash( + vec![Arc::new(ScalarFunctionExpr::new( + "test_udf", + udf(), + vec![lit(1)], + Arc::new(Field::new("return", DataType::Int32, false)), + Default::default(), + ))], + 1, + ), + ) + .unwrap(), + ) + }; + + let node = wrap(wrap(Arc::new(EmptyExec::new(Arc::new(Schema::empty()))))); + + let physical_distributed = + DistributedPhysicalOptimizerRule.optimize(node, ctx.copied_config().options())?; + + let physical_distributed_str = display_plan_ascii(physical_distributed.as_ref(), false); + + assert_snapshot!(physical_distributed_str, + @r" + ┌───── DistributedExec ── Tasks: t0:[p0] + │ [Stage 2] => NetworkShuffleExec: output_partitions=1, input_tasks=2 + └────────────────────────────────────────────────── + ┌───── Stage 1 ── Tasks: t0:[p0..p1] t1:[p0..p1] + │ RepartitionExec: partitioning=Hash([test_udf(1)], 2), input_partitions=1 + │ EmptyExec + └────────────────────────────────────────────────── + ", + ); + + let batches = pretty_format_batches( + &execute_stream(physical_distributed, ctx.task_ctx())? + .try_collect::>() + .await?, + )?; + + assert_snapshot!(batches, @r" + ++ + ++ + "); + Ok(()) + } + + fn udf() -> Arc { + Arc::new(ScalarUDF::new_from_impl(Udf::new())) + } + + #[derive(Debug, PartialEq, Eq, Hash)] + struct Udf(Signature); + + impl Udf { + fn new() -> Self { + Self(Signature::any(1, Volatility::Immutable)) + } + } + + impl ScalarUDFImpl for Udf { + fn as_any(&self) -> &dyn Any { + self + } + + fn name(&self) -> &str { + "test_udf" + } + + fn signature(&self) -> &Signature { + &self.0 + } + + fn return_type(&self, arg_types: &[DataType]) -> datafusion::common::Result { + Ok(arg_types[0].clone()) + } + + fn invoke_with_args( + &self, + mut args: ScalarFunctionArgs, + ) -> datafusion::common::Result { + Ok(args.args.remove(0)) + } + } +}