From ae9b281f91f25ba4a0fe41789451a33d3c671a4b Mon Sep 17 00:00:00 2001 From: Adam Gutglick Date: Wed, 17 Jun 2026 13:40:07 +0100 Subject: [PATCH 1/2] Minimize default sql benchmarks suite Signed-off-by: Adam Gutglick --- .github/workflows/bench-dispatch.yml | 18 +++++++ .github/workflows/sql-benchmarks.yml | 72 +++++++++++++++++++++++----- .github/workflows/sql-pr.yml | 26 ++++++++-- docs/developer-guide/benchmarking.md | 6 ++- 4 files changed, 105 insertions(+), 17 deletions(-) diff --git a/.github/workflows/bench-dispatch.yml b/.github/workflows/bench-dispatch.yml index 48bf6fbe4af..d1116d964b5 100644 --- a/.github/workflows/bench-dispatch.yml +++ b/.github/workflows/bench-dispatch.yml @@ -47,3 +47,21 @@ jobs: needs: remove-sql-label uses: ./.github/workflows/sql-pr.yml secrets: inherit + + remove-sql-full-label: + runs-on: ubuntu-latest + timeout-minutes: 10 + if: github.event.label.name == 'action/benchmark-sql-full' + steps: + - uses: actions-ecosystem/action-remove-labels@2ce5d41b4b6aa8503e285553f75ed56e0a40bae0 # v1 + if: github.event.pull_request.head.repo.full_name == 'vortex-data/vortex' + with: + labels: action/benchmark-sql-full + fail_on_error: true + + sql-full-bench: + needs: remove-sql-full-label + uses: ./.github/workflows/sql-pr.yml + secrets: inherit + with: + benchmark_profile: "full" diff --git a/.github/workflows/sql-benchmarks.yml b/.github/workflows/sql-benchmarks.yml index b46fe455a34..3adeec0f90f 100644 --- a/.github/workflows/sql-benchmarks.yml +++ b/.github/workflows/sql-benchmarks.yml @@ -277,9 +277,54 @@ on: "iterations": "10" } ] + benchmark_profile: + required: false + type: string + description: "Benchmark profile to run: full or base" + default: "full" jobs: + resolve-benchmark-matrix: + runs-on: ubuntu-latest + outputs: + benchmark_matrix: ${{ steps.matrix.outputs.benchmark_matrix }} + steps: + - name: Resolve benchmark matrix + id: matrix + shell: bash + run: | + cat > matrix.json <<'__BENCHMARK_MATRIX__' + ${{ inputs.benchmark_matrix }} + __BENCHMARK_MATRIX__ + + case "${{ inputs.benchmark_profile }}" in + full) + jq -c "." matrix.json > resolved-matrix.json + ;; + base) + jq -c ' + map( + select(.id != "tpch-s3-10" and .subcommand != "appian") + | .data_formats |= map(select(. != "vortex-compact" and . != "duckdb")) + | .pr_targets |= map(select(.format != "vortex-compact" and (.engine != "duckdb" or .format != "duckdb"))) + | .develop_targets |= map(select(.format != "vortex-compact" and (.engine != "duckdb" or .format != "duckdb"))) + ) + ' matrix.json > resolved-matrix.json + ;; + *) + echo "::error::Unknown benchmark_profile '${{ inputs.benchmark_profile }}'. Expected 'full' or 'base'." + exit 1 + ;; + esac + + { + echo "benchmark_matrix<<__BENCHMARK_MATRIX__" + cat resolved-matrix.json + echo "__BENCHMARK_MATRIX__" + } >> "$GITHUB_OUTPUT" + bench: + needs: resolve-benchmark-matrix timeout-minutes: 120 env: VORTEX_EXPERIMENTAL_PATCHED_ARRAY: "1" @@ -289,7 +334,7 @@ jobs: strategy: fail-fast: false matrix: - include: ${{ fromJSON(inputs.benchmark_matrix) }} + include: ${{ fromJSON(needs.resolve-benchmark-matrix.outputs.benchmark_matrix) }} runs-on: >- ${{ github.repository == 'vortex-data/vortex' @@ -321,7 +366,7 @@ jobs: run: | wget -qO- https://github.com/duckdb/duckdb/releases/download/v1.5.3/duckdb_cli-linux-amd64.zip | funzip > duckdb chmod +x duckdb - echo "$PWD" >> $GITHUB_PATH + echo "$PWD" >> "$GITHUB_PATH" - uses: ./.github/actions/system-info @@ -345,11 +390,11 @@ jobs: env: RUSTFLAGS: "-C target-cpu=native" run: | - packages="--bin data-gen --bin datafusion-bench --bin duckdb-bench" + packages=(--bin data-gen --bin datafusion-bench --bin duckdb-bench) if [ "${{ inputs.mode }}" != "pr" ]; then - packages="$packages --bin lance-bench" + packages+=(--bin lance-bench) fi - cargo build $packages --profile release_debug --features unstable_encodings + cargo build "${packages[@]}" --profile release_debug --features unstable_encodings - name: Generate data shell: bash @@ -446,11 +491,16 @@ jobs: python3 scripts/s3-download.py s3://vortex-ci-benchmark-results/data.json.gz data.json.gz --no-sign-request gzip -d -c data.json.gz > base.json - echo '# Benchmarks: ${{ matrix.name }}' > comment.md + benchmark_name="${{ matrix.name }}" + if [ "${{ inputs.benchmark_profile }}" != "full" ]; then + benchmark_name="$benchmark_name (${{ inputs.benchmark_profile }})" + fi + + echo "# Benchmarks: $benchmark_name" > comment.md echo '' >> comment.md - uv run --no-project scripts/compare-benchmark-jsons.py base.json results.json "${{ matrix.name }}" \ + uv run --no-project scripts/compare-benchmark-jsons.py base.json results.json "$benchmark_name" \ >> comment.md - cat comment.md >> $GITHUB_STEP_SUMMARY + cat comment.md >> "$GITHUB_STEP_SUMMARY" - name: Comment PR if: inputs.mode == 'pr' && github.event.pull_request.head.repo.fork == false @@ -460,7 +510,7 @@ jobs: # There is exactly one comment per comment-tag. If a comment with this tag already exists, # this action will *update* the comment instead of posting a new comment. Therefore, each # unique benchmark configuration must have a unique comment-tag. - comment-tag: bench-pr-comment-${{ matrix.id }} + comment-tag: bench-pr-comment-${{ matrix.id }}${{ inputs.benchmark_profile == 'base' && '-base' || '' }} - name: Comment PR on failure if: failure() && inputs.mode == 'pr' && github.event.pull_request.head.repo.fork == false @@ -469,8 +519,8 @@ jobs: message: | # 🚨🚨🚨❌❌❌ SQL BENCHMARK FAILED ❌❌❌🚨🚨🚨 - Benchmark `${{ matrix.name }}` failed! Check the [workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details. - comment-tag: bench-pr-comment-${{ matrix.id }} + Benchmark `${{ matrix.name }}` (${{ inputs.benchmark_profile }}) failed! Check the [workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details. + comment-tag: bench-pr-comment-${{ matrix.id }}${{ inputs.benchmark_profile == 'base' && '-base' || '' }} - name: Upload Benchmark Results if: inputs.mode == 'develop' diff --git a/.github/workflows/sql-pr.yml b/.github/workflows/sql-pr.yml index 420425a4321..45b0ed1d675 100644 --- a/.github/workflows/sql-pr.yml +++ b/.github/workflows/sql-pr.yml @@ -1,15 +1,32 @@ # Runs SQL benchmarks once for a pull request. -# Called from bench-dispatch.yml when the `action/benchmark-sql` label is added. +# Called from bench-dispatch.yml when SQL benchmark labels are added. name: PR SQL Benchmarks concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + group: >- + ${{ github.workflow }}-${{ inputs.benchmark_profile || 'base' }}-${{ + github.head_ref || github.run_id + }} cancel-in-progress: true on: - workflow_call: { } - workflow_dispatch: { } + workflow_call: + inputs: + benchmark_profile: + required: false + type: string + default: "base" + workflow_dispatch: + inputs: + benchmark_profile: + description: "SQL benchmark profile to run" + required: false + type: choice + default: "base" + options: + - "base" + - "full" permissions: contents: read @@ -22,3 +39,4 @@ jobs: secrets: inherit with: mode: "pr" + benchmark_profile: ${{ inputs.benchmark_profile || 'base' }} diff --git a/docs/developer-guide/benchmarking.md b/docs/developer-guide/benchmarking.md index 0034965a056..dfa0fa1cd45 100644 --- a/docs/developer-guide/benchmarking.md +++ b/docs/developer-guide/benchmarking.md @@ -212,8 +212,10 @@ Benchmarks run automatically on all commits to `develop` and can be run on-deman `develop`, with results uploaded for historical tracking. - **PR benchmarks** -- triggered by the `action/benchmark` label. Results are compared against the latest `develop` run and posted as a PR comment. -- **SQL benchmarks** -- triggered by the `action/benchmark-sql` label. Runs a parametric matrix - of suites, engines, formats, and storage backends (NVMe, S3). +- **SQL benchmarks** -- triggered by the `action/benchmark-sql` label. Runs the base SQL matrix, + which excludes Appian, TPC-H SF=10 on S3, `vortex-compact`, and `duckdb:duckdb`. +- **Full SQL benchmarks** -- triggered by the `action/benchmark-sql-full` label. Runs the full + SQL matrix of suites, engines, formats, and storage backends (NVMe, S3). All CI benchmarks run on dedicated instances with the `release_debug` profile and `-C target-cpu=native` to produce representative numbers. From e8c464e31e27f723906dc874ee4a3743e25062ae Mon Sep 17 00:00:00 2001 From: Adam Gutglick Date: Wed, 17 Jun 2026 14:13:15 +0100 Subject: [PATCH 2/2] explicit config Signed-off-by: Adam Gutglick --- .github/workflows/sql-benchmarks.yml | 226 ++++++++++++++++++++++----- 1 file changed, 184 insertions(+), 42 deletions(-) diff --git a/.github/workflows/sql-benchmarks.yml b/.github/workflows/sql-benchmarks.yml index 3adeec0f90f..3686934af39 100644 --- a/.github/workflows/sql-benchmarks.yml +++ b/.github/workflows/sql-benchmarks.yml @@ -13,7 +13,7 @@ on: benchmark_matrix: required: false type: string - description: "JSON string containing the matrix configuration" + description: "JSON string containing the full matrix configuration" default: | [ { @@ -277,6 +277,188 @@ on: "iterations": "10" } ] + base_benchmark_matrix: + required: false + type: string + description: "JSON string containing the base matrix configuration" + default: | + [ + { + "id": "clickbench-nvme", + "subcommand": "clickbench", + "name": "Clickbench on NVME", + "data_formats": ["parquet", "vortex"], + "pr_targets": [ + {"engine": "datafusion", "format": "parquet"}, + {"engine": "datafusion", "format": "vortex"}, + {"engine": "duckdb", "format": "parquet"}, + {"engine": "duckdb", "format": "vortex"} + ], + "develop_targets": [ + {"engine": "datafusion", "format": "parquet"}, + {"engine": "datafusion", "format": "vortex"}, + {"engine": "datafusion", "format": "lance"}, + {"engine": "duckdb", "format": "parquet"}, + {"engine": "duckdb", "format": "vortex"} + ] + }, + { + "id": "tpch-nvme", + "subcommand": "tpch", + "name": "TPC-H SF=1 on NVME", + "data_formats": ["parquet", "vortex"], + "pr_targets": [ + {"engine": "datafusion", "format": "arrow"}, + {"engine": "datafusion", "format": "parquet"}, + {"engine": "datafusion", "format": "vortex"}, + {"engine": "duckdb", "format": "parquet"}, + {"engine": "duckdb", "format": "vortex"} + ], + "develop_targets": [ + {"engine": "datafusion", "format": "arrow"}, + {"engine": "datafusion", "format": "parquet"}, + {"engine": "datafusion", "format": "vortex"}, + {"engine": "datafusion", "format": "lance"}, + {"engine": "duckdb", "format": "parquet"}, + {"engine": "duckdb", "format": "vortex"} + ], + "scale_factor": "1.0", + "iterations": "10" + }, + { + "id": "tpch-s3", + "subcommand": "tpch", + "name": "TPC-H SF=1 on S3", + "local_dir": "vortex-bench/data/tpch/1.0", + "remote_storage": "s3://vortex-ci-benchmark-datasets/${{github.ref_name}}/${{github.run_id}}/tpch/1.0/", + "data_formats": ["parquet", "vortex"], + "pr_targets": [ + {"engine": "datafusion", "format": "parquet"}, + {"engine": "datafusion", "format": "vortex"}, + {"engine": "duckdb", "format": "parquet"}, + {"engine": "duckdb", "format": "vortex"} + ], + "develop_targets": [ + {"engine": "datafusion", "format": "parquet"}, + {"engine": "datafusion", "format": "vortex"}, + {"engine": "duckdb", "format": "parquet"}, + {"engine": "duckdb", "format": "vortex"} + ], + "scale_factor": "1.0", + "iterations": "10" + }, + { + "id": "tpch-nvme-10", + "subcommand": "tpch", + "name": "TPC-H SF=10 on NVME", + "data_formats": ["parquet", "vortex"], + "pr_targets": [ + {"engine": "datafusion", "format": "arrow"}, + {"engine": "datafusion", "format": "parquet"}, + {"engine": "datafusion", "format": "vortex"}, + {"engine": "duckdb", "format": "parquet"}, + {"engine": "duckdb", "format": "vortex"} + ], + "develop_targets": [ + {"engine": "datafusion", "format": "arrow"}, + {"engine": "datafusion", "format": "parquet"}, + {"engine": "datafusion", "format": "vortex"}, + {"engine": "datafusion", "format": "lance"}, + {"engine": "duckdb", "format": "parquet"}, + {"engine": "duckdb", "format": "vortex"} + ], + "scale_factor": "10.0", + "iterations": "10" + }, + { + "id": "tpcds-nvme", + "subcommand": "tpcds", + "name": "TPC-DS SF=1 on NVME", + "data_formats": ["parquet", "vortex"], + "pr_targets": [ + {"engine": "datafusion", "format": "parquet"}, + {"engine": "datafusion", "format": "vortex"}, + {"engine": "duckdb", "format": "parquet"}, + {"engine": "duckdb", "format": "vortex"} + ], + "develop_targets": [ + {"engine": "datafusion", "format": "parquet"}, + {"engine": "datafusion", "format": "vortex"}, + {"engine": "duckdb", "format": "parquet"}, + {"engine": "duckdb", "format": "vortex"} + ], + "scale_factor": "1.0" + }, + { + "id": "statpopgen", + "subcommand": "statpopgen", + "name": "Statistical and Population Genetics", + "local_dir": "vortex-bench/data/statpopgen", + "data_formats": ["parquet", "vortex"], + "pr_targets": [ + {"engine": "duckdb", "format": "parquet"}, + {"engine": "duckdb", "format": "vortex"} + ], + "develop_targets": [ + {"engine": "duckdb", "format": "parquet"}, + {"engine": "duckdb", "format": "vortex"} + ], + "scale_factor": "100" + }, + { + "id": "fineweb", + "subcommand": "fineweb", + "name": "FineWeb NVMe", + "data_formats": ["parquet", "vortex"], + "pr_targets": [ + {"engine": "datafusion", "format": "parquet"}, + {"engine": "datafusion", "format": "vortex"}, + {"engine": "duckdb", "format": "parquet"}, + {"engine": "duckdb", "format": "vortex"} + ], + "develop_targets": [ + {"engine": "datafusion", "format": "parquet"}, + {"engine": "datafusion", "format": "vortex"}, + {"engine": "duckdb", "format": "parquet"}, + {"engine": "duckdb", "format": "vortex"} + ], + "scale_factor": "100" + }, + { + "id": "fineweb-s3", + "subcommand": "fineweb", + "name": "FineWeb S3", + "local_dir": "vortex-bench/data/fineweb", + "remote_storage": "s3://vortex-ci-benchmark-datasets/${{github.ref_name}}/${{github.run_id}}/fineweb/", + "data_formats": ["parquet", "vortex"], + "pr_targets": [ + {"engine": "datafusion", "format": "parquet"}, + {"engine": "datafusion", "format": "vortex"}, + {"engine": "duckdb", "format": "parquet"}, + {"engine": "duckdb", "format": "vortex"} + ], + "develop_targets": [ + {"engine": "datafusion", "format": "parquet"}, + {"engine": "datafusion", "format": "vortex"}, + {"engine": "duckdb", "format": "parquet"}, + {"engine": "duckdb", "format": "vortex"} + ], + "scale_factor": "100" + }, + { + "id": "polarsignals", + "subcommand": "polarsignals", + "name": "PolarSignals Profiling", + "data_formats": ["vortex"], + "pr_targets": [ + {"engine": "datafusion", "format": "vortex"} + ], + "develop_targets": [ + {"engine": "datafusion", "format": "vortex"} + ], + "scale_factor": "1" + } + ] benchmark_profile: required: false type: string @@ -284,47 +466,7 @@ on: default: "full" jobs: - resolve-benchmark-matrix: - runs-on: ubuntu-latest - outputs: - benchmark_matrix: ${{ steps.matrix.outputs.benchmark_matrix }} - steps: - - name: Resolve benchmark matrix - id: matrix - shell: bash - run: | - cat > matrix.json <<'__BENCHMARK_MATRIX__' - ${{ inputs.benchmark_matrix }} - __BENCHMARK_MATRIX__ - - case "${{ inputs.benchmark_profile }}" in - full) - jq -c "." matrix.json > resolved-matrix.json - ;; - base) - jq -c ' - map( - select(.id != "tpch-s3-10" and .subcommand != "appian") - | .data_formats |= map(select(. != "vortex-compact" and . != "duckdb")) - | .pr_targets |= map(select(.format != "vortex-compact" and (.engine != "duckdb" or .format != "duckdb"))) - | .develop_targets |= map(select(.format != "vortex-compact" and (.engine != "duckdb" or .format != "duckdb"))) - ) - ' matrix.json > resolved-matrix.json - ;; - *) - echo "::error::Unknown benchmark_profile '${{ inputs.benchmark_profile }}'. Expected 'full' or 'base'." - exit 1 - ;; - esac - - { - echo "benchmark_matrix<<__BENCHMARK_MATRIX__" - cat resolved-matrix.json - echo "__BENCHMARK_MATRIX__" - } >> "$GITHUB_OUTPUT" - bench: - needs: resolve-benchmark-matrix timeout-minutes: 120 env: VORTEX_EXPERIMENTAL_PATCHED_ARRAY: "1" @@ -334,7 +476,7 @@ jobs: strategy: fail-fast: false matrix: - include: ${{ fromJSON(needs.resolve-benchmark-matrix.outputs.benchmark_matrix) }} + include: ${{ fromJSON(inputs.benchmark_profile == 'base' && inputs.base_benchmark_matrix || inputs.benchmark_matrix) }} runs-on: >- ${{ github.repository == 'vortex-data/vortex'