From 6eb0e9006ab9cd67ee75f339a6c35b95c047a914 Mon Sep 17 00:00:00 2001 From: Connor Tsui Date: Mon, 27 Jul 2026 09:45:18 -0400 Subject: [PATCH] Centralize SQL benchmark matrices in bench-orchestrator Signed-off-by: Connor Tsui --- .github/workflows/bench-dispatch.yml | 2 +- .github/workflows/bench-pr.yml | 1 + .github/workflows/bench.yml | 1 + .github/workflows/nightly-bench.yml | 44 +- .github/workflows/sql-benchmarks.yml | 584 ++---------------- .github/workflows/sql-pr.yml | 18 +- bench-orchestrator/README.md | 28 + .../bench_orchestrator/ci_matrix/__init__.py | 18 + .../bench_orchestrator/ci_matrix/catalog.py | 246 ++++++++ .../bench_orchestrator/ci_matrix/model.py | 54 ++ .../bench_orchestrator/ci_matrix/render.py | 57 ++ .../bench_orchestrator/ci_matrix/targets.py | 56 ++ .../bench_orchestrator/ci_matrix/validate.py | 55 ++ bench-orchestrator/bench_orchestrator/cli.py | 26 + bench-orchestrator/tests/test_matrix.py | 118 ++++ 15 files changed, 717 insertions(+), 591 deletions(-) create mode 100644 bench-orchestrator/bench_orchestrator/ci_matrix/__init__.py create mode 100644 bench-orchestrator/bench_orchestrator/ci_matrix/catalog.py create mode 100644 bench-orchestrator/bench_orchestrator/ci_matrix/model.py create mode 100644 bench-orchestrator/bench_orchestrator/ci_matrix/render.py create mode 100644 bench-orchestrator/bench_orchestrator/ci_matrix/targets.py create mode 100644 bench-orchestrator/bench_orchestrator/ci_matrix/validate.py create mode 100644 bench-orchestrator/tests/test_matrix.py diff --git a/.github/workflows/bench-dispatch.yml b/.github/workflows/bench-dispatch.yml index d1116d964b5..6313e37c64a 100644 --- a/.github/workflows/bench-dispatch.yml +++ b/.github/workflows/bench-dispatch.yml @@ -64,4 +64,4 @@ jobs: uses: ./.github/workflows/sql-pr.yml secrets: inherit with: - benchmark_profile: "full" + matrix_preset: "pr-full" diff --git a/.github/workflows/bench-pr.yml b/.github/workflows/bench-pr.yml index d2a3febf266..c88afe3ff00 100644 --- a/.github/workflows/bench-pr.yml +++ b/.github/workflows/bench-pr.yml @@ -139,3 +139,4 @@ jobs: secrets: inherit with: mode: "pr" + matrix_preset: "pr-full" diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 36001d81ebf..13a29c90228 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -171,3 +171,4 @@ jobs: secrets: inherit with: mode: "develop" + matrix_preset: "develop" diff --git a/.github/workflows/nightly-bench.yml b/.github/workflows/nightly-bench.yml index b89395516be..4564ef681ca 100644 --- a/.github/workflows/nightly-bench.yml +++ b/.github/workflows/nightly-bench.yml @@ -23,49 +23,7 @@ jobs: with: mode: "develop" machine_type: ${{ matrix.machine_type.instance_name }} - benchmark_matrix: | - [ - { - "id": "tpch-nvme", - "subcommand": "tpch", - "name": "TPC-H on NVME", - "data_formats": ["parquet", "vortex"], - "pr_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"} - ], - "develop_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"} - ], - "scale_factor": "100" - }, - { - "id": "tpch-s3", - "subcommand": "tpch", - "name": "TPC-H on S3", - "local_dir": "vortex-bench/data/tpch/100.0", - "remote_storage": "s3://vortex-ci-benchmark-datasets/${{github.ref_name}}/${{github.run_id}}/tpch/100.0/", - "data_formats": ["parquet", "vortex"], - "pr_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"} - ], - "develop_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"} - ], - "scale_factor": "100.0" - } - ] + matrix_preset: "nightly" strategy: # A single run not should kill the others fail-fast: false diff --git a/.github/workflows/sql-benchmarks.yml b/.github/workflows/sql-benchmarks.yml index c00cb38a297..7222ed727ca 100644 --- a/.github/workflows/sql-benchmarks.yml +++ b/.github/workflows/sql-benchmarks.yml @@ -6,520 +6,45 @@ on: mode: required: true type: string + matrix_preset: + required: true + type: string + description: "Named benchmark matrix to run" machine_type: required: false type: string default: i7i.metal-24xl - benchmark_matrix: - required: false - type: string - description: "JSON string containing the full matrix configuration" - default: | - [ - { - "id": "clickbench-nvme", - "subcommand": "clickbench", - "name": "Clickbench on NVME", - "data_formats": ["parquet", "vortex", "vortex-compact", "duckdb"], - "pr_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"}, - {"engine": "duckdb", "format": "duckdb"} - ], - "develop_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "datafusion", "format": "vortex-compact"}, - {"engine": "datafusion", "format": "lance"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"}, - {"engine": "duckdb", "format": "vortex-compact"}, - {"engine": "duckdb", "format": "duckdb"} - ] - }, - { - "id": "clickbench-sorted-nvme", - "subcommand": "clickbench-sorted", - "name": "Clickbench Sorted on NVME", - "data_formats": ["parquet", "vortex", "vortex-compact", "duckdb"], - "pr_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"}, - {"engine": "duckdb", "format": "duckdb"} - ], - "develop_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "datafusion", "format": "vortex-compact"}, - {"engine": "datafusion", "format": "lance"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"}, - {"engine": "duckdb", "format": "vortex-compact"}, - {"engine": "duckdb", "format": "duckdb"} - ] - }, - { - "id": "tpch-nvme", - "subcommand": "tpch", - "name": "TPC-H SF=1 on NVME", - "data_formats": ["parquet", "vortex", "vortex-compact", "duckdb"], - "pr_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "datafusion", "format": "vortex-compact"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"}, - {"engine": "duckdb", "format": "vortex-compact"}, - {"engine": "duckdb", "format": "duckdb"} - ], - "develop_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "datafusion", "format": "vortex-compact"}, - {"engine": "datafusion", "format": "lance"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"}, - {"engine": "duckdb", "format": "vortex-compact"}, - {"engine": "duckdb", "format": "duckdb"} - ], - "scale_factor": "1.0", - "iterations": "10" - }, - { - "id": "tpch-s3", - "subcommand": "tpch", - "name": "TPC-H SF=1 on S3", - "local_dir": "vortex-bench/data/tpch/1.0", - "remote_storage": "s3://vortex-ci-benchmark-datasets/${{github.ref_name}}/${{github.run_id}}/tpch/1.0/", - "data_formats": ["parquet", "vortex", "vortex-compact"], - "pr_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "datafusion", "format": "vortex-compact"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"}, - {"engine": "duckdb", "format": "vortex-compact"} - ], - "develop_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "datafusion", "format": "vortex-compact"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"}, - {"engine": "duckdb", "format": "vortex-compact"} - ], - "scale_factor": "1.0", - "iterations": "10" - }, - { - "id": "tpch-nvme-10", - "subcommand": "tpch", - "name": "TPC-H SF=10 on NVME", - "data_formats": ["parquet", "vortex", "vortex-compact", "duckdb"], - "pr_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "datafusion", "format": "vortex-compact"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"}, - {"engine": "duckdb", "format": "vortex-compact"}, - {"engine": "duckdb", "format": "duckdb"} - ], - "develop_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "datafusion", "format": "vortex-compact"}, - {"engine": "datafusion", "format": "lance"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"}, - {"engine": "duckdb", "format": "vortex-compact"}, - {"engine": "duckdb", "format": "duckdb"} - ], - "scale_factor": "10.0", - "iterations": "10" - }, - { - "id": "tpch-s3-10", - "subcommand": "tpch", - "name": "TPC-H SF=10 on S3", - "local_dir": "vortex-bench/data/tpch/10.0", - "remote_storage": "s3://vortex-ci-benchmark-datasets/${{github.ref_name}}/${{github.run_id}}/tpch/10.0/", - "data_formats": ["parquet", "vortex", "vortex-compact"], - "pr_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "datafusion", "format": "vortex-compact"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"}, - {"engine": "duckdb", "format": "vortex-compact"} - ], - "develop_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "datafusion", "format": "vortex-compact"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"}, - {"engine": "duckdb", "format": "vortex-compact"} - ], - "scale_factor": "10.0", - "iterations": "10" - }, - { - "id": "tpcds-nvme", - "subcommand": "tpcds", - "name": "TPC-DS SF=1 on NVME", - "data_formats": ["parquet", "vortex", "vortex-compact", "duckdb"], - "pr_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "datafusion", "format": "vortex-compact"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"}, - {"engine": "duckdb", "format": "vortex-compact"}, - {"engine": "duckdb", "format": "duckdb"} - ], - "develop_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "datafusion", "format": "vortex-compact"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"}, - {"engine": "duckdb", "format": "vortex-compact"}, - {"engine": "duckdb", "format": "duckdb"} - ], - "scale_factor": "1.0" - }, - { - "id": "statpopgen", - "subcommand": "statpopgen", - "name": "Statistical and Population Genetics", - "local_dir": "vortex-bench/data/statpopgen", - "data_formats": ["parquet", "vortex", "vortex-compact"], - "pr_targets": [ - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"}, - {"engine": "duckdb", "format": "vortex-compact"} - ], - "develop_targets": [ - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"}, - {"engine": "duckdb", "format": "vortex-compact"} - ], - "scale_factor": "100" - }, - { - "id": "fineweb", - "subcommand": "fineweb", - "name": "FineWeb NVMe", - "data_formats": ["parquet", "vortex", "vortex-compact"], - "pr_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "datafusion", "format": "vortex-compact"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"}, - {"engine": "duckdb", "format": "vortex-compact"} - ], - "develop_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "datafusion", "format": "vortex-compact"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"}, - {"engine": "duckdb", "format": "vortex-compact"} - ], - "scale_factor": "100" - }, - { - "id": "fineweb-s3", - "subcommand": "fineweb", - "name": "FineWeb S3", - "local_dir": "vortex-bench/data/fineweb", - "remote_storage": "s3://vortex-ci-benchmark-datasets/${{github.ref_name}}/${{github.run_id}}/fineweb/", - "data_formats": ["parquet", "vortex", "vortex-compact"], - "pr_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "datafusion", "format": "vortex-compact"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"}, - {"engine": "duckdb", "format": "vortex-compact"} - ], - "develop_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "datafusion", "format": "vortex-compact"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"}, - {"engine": "duckdb", "format": "vortex-compact"} - ], - "scale_factor": "100" - }, - { - "id": "polarsignals", - "subcommand": "polarsignals", - "name": "PolarSignals Profiling", - "data_formats": ["vortex"], - "pr_targets": [ - {"engine": "datafusion", "format": "vortex"} - ], - "develop_targets": [ - {"engine": "datafusion", "format": "vortex"} - ], - "scale_factor": "1" - }, - { - "id": "appian-nvme", - "subcommand": "appian", - "name": "Appian on NVME", - "data_formats": ["parquet", "vortex", "vortex-compact", "duckdb"], - "pr_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"}, - {"engine": "duckdb", "format": "duckdb"} - ], - "develop_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "datafusion", "format": "vortex-compact"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"}, - {"engine": "duckdb", "format": "vortex-compact"}, - {"engine": "duckdb", "format": "duckdb"} - ], - "iterations": "10" - }, - { - "id": "vortex-queries", - "subcommand": "vortex", - "name": "Vortex queries", - "data_formats": ["parquet", "vortex"], - "pr_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"} - ], - "develop_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"} - ], - "iterations": "100" - } - ] - base_benchmark_matrix: - required: false - type: string - description: "JSON string containing the base matrix configuration" - default: | - [ - { - "id": "clickbench-nvme", - "subcommand": "clickbench", - "name": "Clickbench on NVME", - "data_formats": ["parquet", "vortex"], - "pr_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"} - ], - "develop_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "datafusion", "format": "lance"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"} - ] - }, - { - "id": "clickbench-sorted-nvme", - "subcommand": "clickbench-sorted", - "name": "Clickbench Sorted on NVME", - "data_formats": ["parquet", "vortex"], - "pr_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"} - ], - "develop_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "datafusion", "format": "lance"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"} - ] - }, - { - "id": "tpch-nvme", - "subcommand": "tpch", - "name": "TPC-H SF=1 on NVME", - "data_formats": ["parquet", "vortex"], - "pr_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"} - ], - "develop_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "datafusion", "format": "lance"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"} - ], - "scale_factor": "1.0", - "iterations": "10" - }, - { - "id": "tpch-s3", - "subcommand": "tpch", - "name": "TPC-H SF=1 on S3", - "local_dir": "vortex-bench/data/tpch/1.0", - "remote_storage": "s3://vortex-ci-benchmark-datasets/${{github.ref_name}}/${{github.run_id}}/tpch/1.0/", - "data_formats": ["parquet", "vortex"], - "pr_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"} - ], - "develop_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"} - ], - "scale_factor": "1.0", - "iterations": "10" - }, - { - "id": "tpch-nvme-10", - "subcommand": "tpch", - "name": "TPC-H SF=10 on NVME", - "data_formats": ["parquet", "vortex"], - "pr_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"} - ], - "develop_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "datafusion", "format": "lance"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"} - ], - "scale_factor": "10.0", - "iterations": "10" - }, - { - "id": "tpcds-nvme", - "subcommand": "tpcds", - "name": "TPC-DS SF=1 on NVME", - "data_formats": ["parquet", "vortex"], - "pr_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"} - ], - "develop_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"} - ], - "scale_factor": "1.0" - }, - { - "id": "statpopgen", - "subcommand": "statpopgen", - "name": "Statistical and Population Genetics", - "local_dir": "vortex-bench/data/statpopgen", - "data_formats": ["parquet", "vortex"], - "pr_targets": [ - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"} - ], - "develop_targets": [ - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"} - ], - "scale_factor": "100" - }, - { - "id": "fineweb", - "subcommand": "fineweb", - "name": "FineWeb NVMe", - "data_formats": ["parquet", "vortex"], - "pr_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"} - ], - "develop_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"} - ], - "scale_factor": "100" - }, - { - "id": "fineweb-s3", - "subcommand": "fineweb", - "name": "FineWeb S3", - "local_dir": "vortex-bench/data/fineweb", - "remote_storage": "s3://vortex-ci-benchmark-datasets/${{github.ref_name}}/${{github.run_id}}/fineweb/", - "data_formats": ["parquet", "vortex"], - "pr_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"} - ], - "develop_targets": [ - {"engine": "datafusion", "format": "parquet"}, - {"engine": "datafusion", "format": "vortex"}, - {"engine": "duckdb", "format": "parquet"}, - {"engine": "duckdb", "format": "vortex"} - ], - "scale_factor": "100" - }, - { - "id": "polarsignals", - "subcommand": "polarsignals", - "name": "PolarSignals Profiling", - "data_formats": ["vortex"], - "pr_targets": [ - {"engine": "datafusion", "format": "vortex"} - ], - "develop_targets": [ - {"engine": "datafusion", "format": "vortex"} - ], - "scale_factor": "1" - } - ] - benchmark_profile: - required: false - type: string - description: "Benchmark profile to run: full or base" - default: "full" jobs: + resolve-matrix: + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + outputs: + benchmark_matrix: ${{ steps.resolve.outputs.benchmark_matrix }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + ref: ${{ inputs.mode == 'pr' && github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + - name: Install uv + uses: spiraldb/actions/.github/actions/setup-uv@a746510eafaa926484c354541cfc49b2ec06cc63 # 0.18.6 + with: + sync: false + - name: Resolve benchmark matrix + id: resolve + shell: bash + env: + MATRIX_PRESET: ${{ inputs.matrix_preset }} + run: | + set -Eeuo pipefail + + matrix="$(uv run --project bench-orchestrator vx-bench matrix "$MATRIX_PRESET")" + echo "benchmark_matrix=$matrix" >> "$GITHUB_OUTPUT" + bench: + needs: resolve-matrix timeout-minutes: 120 env: VORTEX_EXPERIMENTAL_PATCHED_ARRAY: "1" @@ -529,7 +54,7 @@ jobs: strategy: fail-fast: false matrix: - include: ${{ fromJSON(inputs.benchmark_profile == 'base' && inputs.base_benchmark_matrix || inputs.benchmark_matrix) }} + include: ${{ fromJSON(needs.resolve-matrix.outputs.benchmark_matrix) }} runs-on: >- ${{ github.repository == 'vortex-data/vortex' @@ -541,12 +66,8 @@ jobs: with: sccache: s3 - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - if: inputs.mode == 'pr' with: - ref: ${{ github.event.pull_request.head.sha }} - - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - if: inputs.mode != 'pr' + ref: ${{ inputs.mode == 'pr' && github.event.pull_request.head.sha || github.sha }} - name: Setup benchmark environment run: sudo bash scripts/setup-benchmark.sh - uses: ./.github/actions/setup-rust @@ -566,21 +87,6 @@ jobs: - uses: ./.github/actions/system-info - - name: Resolve benchmark targets - id: targets - shell: bash - run: | - if [ "${{ inputs.mode }}" = "pr" ]; then - targets='${{ toJSON(matrix.pr_targets) }}' - else - targets='${{ toJSON(matrix.develop_targets) }}' - fi - { - echo 'targets_json<<__TARGETS_JSON__' - echo "$targets" - echo '__TARGETS_JSON__' - } >> "$GITHUB_OUTPUT" - - name: Build binaries shell: bash env: @@ -611,13 +117,14 @@ jobs: role-duration-seconds: 7200 - name: Upload data - if: matrix.remote_storage != null && (inputs.mode != 'pr' || github.event.pull_request.head.repo.fork == false) + if: matrix.remote_key != null && (inputs.mode != 'pr' || github.event.pull_request.head.repo.fork == false) shell: bash env: AWS_REGION: "us-east-1" + REMOTE_STORAGE: "s3://vortex-ci-benchmark-datasets/${{ github.ref_name }}/${{ github.run_id }}/${{ matrix.remote_key }}/" run: | - aws s3 rm --recursive ${{ matrix.remote_storage }} - aws s3 cp --recursive ${{matrix.local_dir}} ${{ matrix.remote_storage }} + aws s3 rm --recursive "$REMOTE_STORAGE" + aws s3 cp --recursive "${{ matrix.local_dir }}" "$REMOTE_STORAGE" - name: Setup Polar Signals if: inputs.mode != 'pr' || github.event.pull_request.head.repo.fork == false @@ -630,7 +137,7 @@ jobs: extra_args: "--off-cpu-threshold=0.03" # Personally tuned by @brancz - name: Run ${{ matrix.name }} benchmark - if: matrix.remote_storage == null || github.event.pull_request.head.repo.fork == true + if: matrix.remote_key == null || github.event.pull_request.head.repo.fork == true shell: bash env: OTEL_SERVICE_NAME: "vortex-bench" @@ -640,7 +147,7 @@ jobs: OTEL_RESOURCE_ATTRIBUTES: "bench-name=${{ matrix.id }}" run: | bash scripts/bench-taskset.sh uv run --project bench-orchestrator vx-bench run "${{ matrix.subcommand }}" \ - --targets-json '${{ steps.targets.outputs.targets_json }}' \ + --targets-json '${{ toJSON(matrix.targets) }}' \ --output results.json \ --gh-json-v3 results.v3.jsonl \ --no-build \ @@ -649,7 +156,7 @@ jobs: ${{ matrix.scale_factor && format('--opt scale-factor={0}', matrix.scale_factor) || '' }} - name: Run ${{ matrix.name }} benchmark (remote) - if: matrix.remote_storage != null && (inputs.mode != 'pr' || github.event.pull_request.head.repo.fork == false) + if: matrix.remote_key != null && (inputs.mode != 'pr' || github.event.pull_request.head.repo.fork == false) shell: bash env: AWS_REGION: "us-east-1" @@ -658,19 +165,20 @@ jobs: OTEL_EXPORTER_OTLP_ENDPOINT: "${{ (inputs.mode != 'pr' || github.event.pull_request.head.repo.fork == false) && secrets.OTEL_EXPORTER_OTLP_ENDPOINT || '' }}" OTEL_EXPORTER_OTLP_HEADERS: "${{ (inputs.mode != 'pr' || github.event.pull_request.head.repo.fork == false) && secrets.OTEL_EXPORTER_OTLP_HEADERS || '' }}" OTEL_RESOURCE_ATTRIBUTES: "bench-name=${{ matrix.id }}" + REMOTE_STORAGE: "s3://vortex-ci-benchmark-datasets/${{ github.ref_name }}/${{ github.run_id }}/${{ matrix.remote_key }}/" run: | bash scripts/bench-taskset.sh uv run --project bench-orchestrator vx-bench run "${{ matrix.subcommand }}" \ - --targets-json '${{ steps.targets.outputs.targets_json }}' \ + --targets-json '${{ toJSON(matrix.targets) }}' \ --output results.json \ --gh-json-v3 results.v3.jsonl \ --no-build \ --runner "ec2_${{ inputs.machine_type }}" \ ${{ matrix.iterations && format('--iterations {0}', matrix.iterations) || '' }} \ - --opt remote-data-dir=${{ matrix.remote_storage }} \ + --opt remote-data-dir="$REMOTE_STORAGE" \ ${{ matrix.scale_factor && format('--opt scale-factor={0}', matrix.scale_factor) || '' }} - name: Capture file sizes - if: matrix.remote_storage == null + if: matrix.remote_key == null shell: bash run: | uv run --no-project scripts/capture-file-sizes.py \ @@ -709,7 +217,7 @@ jobs: message: | # 🚨🚨🚨❌❌❌ SQL BENCHMARK FAILED ❌❌❌🚨🚨🚨 - Benchmark `${{ matrix.name }}` (${{ inputs.benchmark_profile }}) failed! Check the [workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details. + Benchmark `${{ matrix.name }}` (${{ inputs.matrix_preset }}) failed! Check the [workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details. comment-tag: bench-pr-comment-${{ matrix.id }} - name: Upload Benchmark Results diff --git a/.github/workflows/sql-pr.yml b/.github/workflows/sql-pr.yml index 45b0ed1d675..b14016030c0 100644 --- a/.github/workflows/sql-pr.yml +++ b/.github/workflows/sql-pr.yml @@ -5,7 +5,7 @@ name: PR SQL Benchmarks concurrency: group: >- - ${{ github.workflow }}-${{ inputs.benchmark_profile || 'base' }}-${{ + ${{ github.workflow }}-${{ inputs.matrix_preset || 'pr' }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true @@ -13,20 +13,20 @@ concurrency: on: workflow_call: inputs: - benchmark_profile: + matrix_preset: required: false type: string - default: "base" + default: "pr" workflow_dispatch: inputs: - benchmark_profile: - description: "SQL benchmark profile to run" + matrix_preset: + description: "SQL benchmark matrix to run" required: false type: choice - default: "base" + default: "pr" options: - - "base" - - "full" + - "pr" + - "pr-full" permissions: contents: read @@ -39,4 +39,4 @@ jobs: secrets: inherit with: mode: "pr" - benchmark_profile: ${{ inputs.benchmark_profile || 'base' }} + matrix_preset: ${{ inputs.matrix_preset }} diff --git a/bench-orchestrator/README.md b/bench-orchestrator/README.md index 22b1b1a1249..ccd09ac3598 100644 --- a/bench-orchestrator/README.md +++ b/bench-orchestrator/README.md @@ -70,6 +70,24 @@ vx-bench prepare-data [options] - `--formats-json`: Exact data formats as JSON, e.g. `'["parquet","vortex"]'` - `--opt`: Benchmark-specific options such as `scale-factor=10.0` + +### `matrix` - Render a CI Benchmark Matrix + +Emit the GitHub Actions `include:` array for a named preset. The benchmark workflows use this +to keep benchmark coverage in Python instead of copying large JSON matrices between YAML files. + +```bash +vx-bench matrix # list available presets +vx-bench matrix develop # emit compact JSON +vx-bench matrix pr-full # emit full pull-request coverage +vx-bench matrix nightly --pretty +``` + +**Options:** + +- `--list`: List available presets and exit +- `--pretty`: Pretty-print the JSON output + ### `compare` - Compare Results Compare benchmark results within a run or across multiple runs. Results are displayed in a pivot table format. @@ -137,6 +155,16 @@ vx-bench clean --older-than "30 days" [options] - `--keep-labeled`: Don't delete labeled runs (default: true) - `--dry-run, -n`: Show what would be deleted +## CI Benchmark Matrices + +CI benchmark coverage lives in `bench_orchestrator/ci_matrix/catalog.py`. This is the only file to +edit when changing what runs in the `develop`, `pr`, `pr-full`, or `nightly` presets. The remaining +modules in `ci_matrix` model, validate, and render that catalog as the JSON expected by GitHub +Actions. + +When adding coverage, update the benchmark declarations and the expected preset membership in +`tests/test_matrix.py`. + ## Example Workflows ### 1. Basic Performance Comparison diff --git a/bench-orchestrator/bench_orchestrator/ci_matrix/__init__.py b/bench-orchestrator/bench_orchestrator/ci_matrix/__init__.py new file mode 100644 index 00000000000..ed3a858f69d --- /dev/null +++ b/bench-orchestrator/bench_orchestrator/ci_matrix/__init__.py @@ -0,0 +1,18 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright the Vortex contributors + +"""Declarative CI benchmark matrix resolution.""" + +from .catalog import CATALOG +from .render import render_matrix + +MATRIX_PRESETS = CATALOG.presets +BENCHMARKS = CATALOG.benchmarks + + +def resolve_matrix(preset: str) -> list[dict[str, object]]: + """Render a named preset from the canonical benchmark catalog.""" + return render_matrix(preset, CATALOG) + + +__all__ = ["BENCHMARKS", "MATRIX_PRESETS", "resolve_matrix"] diff --git a/bench-orchestrator/bench_orchestrator/ci_matrix/catalog.py b/bench-orchestrator/bench_orchestrator/ci_matrix/catalog.py new file mode 100644 index 00000000000..55f6ce523f1 --- /dev/null +++ b/bench-orchestrator/bench_orchestrator/ci_matrix/catalog.py @@ -0,0 +1,246 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright the Vortex contributors + +"""Canonical declaration of what SQL benchmarks run in each CI preset. + +Edit this file when changing CI benchmark coverage. The other modules in this package only model, +validate, and render these declarations. +""" + +from ..config import Benchmark, Engine, Format +from .model import BenchmarkCase, Catalog, Coverage, Storage +from .targets import df, duck + +PRESETS = { + "develop": "Every regular SQL benchmark at full target coverage.", + "pr": "The quicker pull-request SQL benchmark matrix.", + "pr-full": "Every regular SQL benchmark at full PR target coverage.", + "nightly": "Large-scale SF=100 TPC-H on NVMe and S3 at default targets.", +} + +# Reusable target coverage + +DEFAULT_TARGETS = df(Format.PARQUET, Format.VORTEX) | duck(Format.PARQUET, Format.VORTEX) +STANDARD_TARGETS = df(Format.PARQUET, Format.VORTEX, Format.VORTEX_COMPACT) | duck( + Format.PARQUET, + Format.VORTEX, + Format.VORTEX_COMPACT, +) +FULL_LOCAL_TARGETS = df( + Format.PARQUET, + Format.VORTEX, + Format.VORTEX_COMPACT, + Format.LANCE, +) | duck( + Format.PARQUET, + Format.VORTEX, + Format.VORTEX_COMPACT, + Format.DUCKDB, +) +FULL_PR_TARGETS = df( + Format.PARQUET, + Format.VORTEX, + Format.VORTEX_COMPACT, +) | duck( + Format.PARQUET, + Format.VORTEX, + Format.VORTEX_COMPACT, + Format.DUCKDB, +) +DEFAULT_WITH_DUCKDB_TARGETS = DEFAULT_TARGETS | duck(Format.DUCKDB) +STANDARD_WITH_DUCKDB_TARGETS = STANDARD_TARGETS | duck(Format.DUCKDB) +DUCKDB_DEFAULT_TARGETS = DEFAULT_TARGETS.only(Engine.DUCKDB) +DUCKDB_STANDARD_TARGETS = STANDARD_TARGETS.only(Engine.DUCKDB) +DATAFUSION_VORTEX_TARGETS = df(Format.VORTEX) + +DEFAULT = Coverage(DEFAULT_TARGETS) +STANDARD = Coverage(STANDARD_TARGETS) +FULL_LOCAL = Coverage(FULL_LOCAL_TARGETS) +FULL_PR = Coverage( + FULL_PR_TARGETS, + data_formats=(Format.PARQUET, Format.VORTEX, Format.VORTEX_COMPACT, Format.DUCKDB), +) +DEFAULT_WITH_DUCKDB_PR_FULL = Coverage( + DEFAULT_WITH_DUCKDB_TARGETS, + data_formats=(Format.PARQUET, Format.VORTEX, Format.VORTEX_COMPACT, Format.DUCKDB), +) +STANDARD_WITH_DUCKDB = Coverage(STANDARD_WITH_DUCKDB_TARGETS) +DUCKDB_DEFAULT = Coverage(DUCKDB_DEFAULT_TARGETS) +DUCKDB_STANDARD = Coverage(DUCKDB_STANDARD_TARGETS) +DATAFUSION_VORTEX = Coverage(DATAFUSION_VORTEX_TARGETS) + +# Concrete benchmark cases + +BENCHMARKS = ( + BenchmarkCase( + id="clickbench-nvme", + benchmark=Benchmark.CLICKBENCH, + name="Clickbench on NVME", + runs={ + "pr": DEFAULT, + "pr-full": DEFAULT_WITH_DUCKDB_PR_FULL, + "develop": FULL_LOCAL, + }, + ), + BenchmarkCase( + id="clickbench-sorted-nvme", + benchmark=Benchmark.CLICKBENCH_SORTED, + name="Clickbench Sorted on NVME", + runs={ + "pr": DEFAULT, + "pr-full": DEFAULT_WITH_DUCKDB_PR_FULL, + "develop": FULL_LOCAL, + }, + ), + BenchmarkCase( + id="tpch-nvme", + benchmark=Benchmark.TPCH, + name="TPC-H SF=1 on NVME", + scale_factor=1.0, + iterations=10, + runs={ + "pr": DEFAULT, + "pr-full": FULL_PR, + "develop": FULL_LOCAL, + }, + ), + BenchmarkCase( + id="tpch-s3", + benchmark=Benchmark.TPCH, + name="TPC-H SF=1 on S3", + storage=Storage.S3, + scale_factor=1.0, + iterations=10, + local_dir="vortex-bench/data/tpch/1.0", + remote_key="tpch/1.0", + runs={ + "pr": DEFAULT, + "pr-full": STANDARD, + "develop": STANDARD, + }, + ), + BenchmarkCase( + id="tpch-nvme-10", + benchmark=Benchmark.TPCH, + name="TPC-H SF=10 on NVME", + scale_factor=10.0, + iterations=10, + runs={ + "pr": DEFAULT, + "pr-full": FULL_PR, + "develop": FULL_LOCAL, + }, + ), + BenchmarkCase( + id="tpch-s3-10", + benchmark=Benchmark.TPCH, + name="TPC-H SF=10 on S3", + storage=Storage.S3, + scale_factor=10.0, + iterations=10, + local_dir="vortex-bench/data/tpch/10.0", + remote_key="tpch/10.0", + runs={ + "pr-full": STANDARD, + "develop": STANDARD, + }, + ), + BenchmarkCase( + id="tpch-nvme", + benchmark=Benchmark.TPCH, + name="TPC-H on NVME", + scale_factor=100, + runs={"nightly": DEFAULT}, + ), + BenchmarkCase( + id="tpch-s3", + benchmark=Benchmark.TPCH, + name="TPC-H on S3", + storage=Storage.S3, + scale_factor=100.0, + local_dir="vortex-bench/data/tpch/100.0", + remote_key="tpch/100.0", + runs={"nightly": DEFAULT}, + ), + BenchmarkCase( + id="tpcds-nvme", + benchmark=Benchmark.TPCDS, + name="TPC-DS SF=1 on NVME", + scale_factor=1.0, + runs={ + "pr": DEFAULT, + "pr-full": STANDARD_WITH_DUCKDB, + "develop": STANDARD_WITH_DUCKDB, + }, + ), + BenchmarkCase( + id="statpopgen", + benchmark=Benchmark.STATPOPGEN, + name="Statistical and Population Genetics", + scale_factor=100, + local_dir="vortex-bench/data/statpopgen", + runs={ + "pr": DUCKDB_DEFAULT, + "pr-full": DUCKDB_STANDARD, + "develop": DUCKDB_STANDARD, + }, + ), + BenchmarkCase( + id="fineweb", + benchmark=Benchmark.FINEWEB, + name="FineWeb NVMe", + scale_factor=100, + runs={ + "pr": DEFAULT, + "pr-full": STANDARD, + "develop": STANDARD, + }, + ), + BenchmarkCase( + id="fineweb-s3", + benchmark=Benchmark.FINEWEB, + name="FineWeb S3", + storage=Storage.S3, + scale_factor=100, + local_dir="vortex-bench/data/fineweb", + remote_key="fineweb", + runs={ + "pr": DEFAULT, + "pr-full": STANDARD, + "develop": STANDARD, + }, + ), + BenchmarkCase( + id="polarsignals", + benchmark=Benchmark.POLARSIGNALS, + name="PolarSignals Profiling", + scale_factor=1, + runs={ + "pr": DATAFUSION_VORTEX, + "pr-full": DATAFUSION_VORTEX, + "develop": DATAFUSION_VORTEX, + }, + ), + BenchmarkCase( + id="appian-nvme", + benchmark=Benchmark.APPIAN, + name="Appian on NVME", + iterations=10, + runs={ + "pr-full": DEFAULT_WITH_DUCKDB_PR_FULL, + "develop": STANDARD_WITH_DUCKDB, + }, + ), + BenchmarkCase( + id="vortex-queries", + benchmark=Benchmark.VORTEX_QUERIES, + name="Vortex queries", + iterations=100, + runs={ + "pr-full": DEFAULT, + "develop": DEFAULT, + }, + ), +) + +CATALOG = Catalog(presets=PRESETS, benchmarks=BENCHMARKS) diff --git a/bench-orchestrator/bench_orchestrator/ci_matrix/model.py b/bench-orchestrator/bench_orchestrator/ci_matrix/model.py new file mode 100644 index 00000000000..587f7f36da4 --- /dev/null +++ b/bench-orchestrator/bench_orchestrator/ci_matrix/model.py @@ -0,0 +1,54 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright the Vortex contributors + +"""Data model for declarative CI benchmark coverage.""" + +from collections.abc import Mapping +from dataclasses import dataclass +from enum import Enum + +from ..config import Benchmark, Format +from .targets import TargetSet + + +class Storage(Enum): + """Where a benchmark's data lives when it runs.""" + + NVME = "nvme" + S3 = "s3" + + +@dataclass(frozen=True) +class Coverage: + """Targets and generated formats for one benchmark in one preset.""" + + targets: TargetSet + data_formats: tuple[Format, ...] | None = None + + +@dataclass(frozen=True) +class BenchmarkCase: + """One concrete benchmark case and the presets that schedule it.""" + + id: str + benchmark: Benchmark + name: str + runs: Mapping[str, Coverage] + storage: Storage = Storage.NVME + scale_factor: float | int | None = None + iterations: int | None = None + local_dir: str | None = None + remote_key: str | None = None + + @property + def subcommand(self) -> str: + """Return the ``vx-bench`` subcommand for this benchmark.""" + return self.benchmark.value + + +@dataclass(frozen=True) +class Catalog: + """Named presets and the benchmark cases available to them.""" + + presets: Mapping[str, str] + benchmarks: tuple[BenchmarkCase, ...] diff --git a/bench-orchestrator/bench_orchestrator/ci_matrix/render.py b/bench-orchestrator/bench_orchestrator/ci_matrix/render.py new file mode 100644 index 00000000000..4a2051cf30a --- /dev/null +++ b/bench-orchestrator/bench_orchestrator/ci_matrix/render.py @@ -0,0 +1,57 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright the Vortex contributors + +"""Render declarative benchmark coverage as GitHub Actions matrices.""" + +from ..config import Format +from .model import BenchmarkCase, Catalog, Coverage +from .validate import validate_catalog + +_NOT_GENERATED = frozenset({Format.LANCE}) +_FORMAT_ORDER = ( + Format.PARQUET, + Format.VORTEX, + Format.VORTEX_COMPACT, + Format.VORTEX_NATIVE, + Format.DUCKDB, + Format.LANCE, +) + + +def _data_formats(coverage: Coverage) -> list[Format]: + if coverage.data_formats is not None: + return list(coverage.data_formats) + + present = {target.format for target in coverage.targets} + return [fmt for fmt in _FORMAT_ORDER if fmt in present and fmt not in _NOT_GENERATED] + + +def _matrix_entry(benchmark: BenchmarkCase, coverage: Coverage) -> dict[str, object]: + entry: dict[str, object] = { + "id": benchmark.id, + "subcommand": benchmark.subcommand, + "name": benchmark.name, + "targets": [target.to_dict() for target in coverage.targets], + "data_formats": [fmt.value for fmt in _data_formats(coverage)], + } + if benchmark.scale_factor is not None: + entry["scale_factor"] = str(benchmark.scale_factor) + if benchmark.iterations is not None: + entry["iterations"] = str(benchmark.iterations) + if benchmark.local_dir is not None: + entry["local_dir"] = benchmark.local_dir + if benchmark.remote_key is not None: + entry["remote_key"] = benchmark.remote_key + return entry + + +def render_matrix(preset: str, catalog: Catalog) -> list[dict[str, object]]: + """Render one named catalog preset as GitHub Actions matrix entries.""" + if preset not in catalog.presets: + known = ", ".join(catalog.presets) + raise ValueError(f"Unknown matrix preset {preset!r}. Available: {known}") + + validate_catalog(catalog) + return [ + _matrix_entry(benchmark, benchmark.runs[preset]) for benchmark in catalog.benchmarks if preset in benchmark.runs + ] diff --git a/bench-orchestrator/bench_orchestrator/ci_matrix/targets.py b/bench-orchestrator/bench_orchestrator/ci_matrix/targets.py new file mode 100644 index 00000000000..cacb25199d8 --- /dev/null +++ b/bench-orchestrator/bench_orchestrator/ci_matrix/targets.py @@ -0,0 +1,56 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright the Vortex contributors + +"""Ordered target sets used by CI benchmark declarations.""" + +from __future__ import annotations + +from collections.abc import Iterable, Iterator +from dataclasses import dataclass + +from ..config import BenchmarkTarget, Engine, Format + + +def _dedupe(targets: Iterable[BenchmarkTarget]) -> tuple[BenchmarkTarget, ...]: + """Normalize and de-duplicate targets, preserving first-seen order.""" + return tuple(dict.fromkeys(target.normalized() for target in targets)) + + +@dataclass(frozen=True) +class TargetSet: + """An ordered set of engine/format targets with small set algebra.""" + + targets: tuple[BenchmarkTarget, ...] = () + + def __post_init__(self) -> None: + object.__setattr__(self, "targets", _dedupe(self.targets)) + + def __or__(self, other: TargetSet) -> TargetSet: + """Return the ordered union of two target sets.""" + return TargetSet(self.targets + other.targets) + + def only(self, *engines: Engine) -> TargetSet: + """Restrict the target set to the given engines.""" + keep = set(engines) + return TargetSet(tuple(target for target in self.targets if target.engine in keep)) + + def __iter__(self) -> Iterator[BenchmarkTarget]: + return iter(self.targets) + + def __len__(self) -> int: + return len(self.targets) + + +def targets(engine: Engine, *formats: Format) -> TargetSet: + """Build targets for one engine across several formats.""" + return TargetSet(tuple(BenchmarkTarget(engine=engine, format=fmt) for fmt in formats)) + + +def df(*formats: Format) -> TargetSet: + """Build DataFusion targets across several formats.""" + return targets(Engine.DATAFUSION, *formats) + + +def duck(*formats: Format) -> TargetSet: + """Build DuckDB targets across several formats.""" + return targets(Engine.DUCKDB, *formats) diff --git a/bench-orchestrator/bench_orchestrator/ci_matrix/validate.py b/bench-orchestrator/bench_orchestrator/ci_matrix/validate.py new file mode 100644 index 00000000000..5f950077a13 --- /dev/null +++ b/bench-orchestrator/bench_orchestrator/ci_matrix/validate.py @@ -0,0 +1,55 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright the Vortex contributors + +"""Validation for declarative CI benchmark catalogs.""" + +from ..config import Format, engines_for_benchmark +from .model import BenchmarkCase, Catalog, Coverage, Storage + + +def _validate_coverage(benchmark: BenchmarkCase, preset: str, coverage: Coverage) -> None: + if len(coverage.targets) == 0: + raise ValueError(f"Benchmark {benchmark.id!r} resolved to no runnable targets") + + for target in coverage.targets: + if not target.is_supported(): + raise ValueError(f"Benchmark {benchmark.id!r} has unsupported target {target} in preset {preset!r}") + if target.engine not in engines_for_benchmark(benchmark.benchmark): + raise ValueError(f"Benchmark {benchmark.id!r} cannot run target {target} in preset {preset!r}") + if benchmark.storage is Storage.S3 and target.format is Format.LANCE: + raise ValueError(f"Benchmark {benchmark.id!r} cannot run Lance from S3 in preset {preset!r}") + + if coverage.data_formats is not None: + if len(coverage.data_formats) == 0: + raise ValueError(f"Benchmark {benchmark.id!r} generates no data formats in preset {preset!r}") + if len(coverage.data_formats) != len(set(coverage.data_formats)): + raise ValueError(f"Benchmark {benchmark.id!r} repeats a data format in preset {preset!r}") + if Format.LANCE in coverage.data_formats: + raise ValueError(f"Benchmark {benchmark.id!r} cannot generate Lance data in preset {preset!r}") + + +def validate_catalog(catalog: Catalog) -> None: + """Validate catalog references and per-preset benchmark invariants.""" + if not catalog.presets: + raise ValueError("Benchmark catalog defines no matrix presets") + + known_presets = set(catalog.presets) + ids_by_preset: dict[str, set[str]] = {preset: set() for preset in catalog.presets} + + for benchmark in catalog.benchmarks: + if not benchmark.runs: + raise ValueError(f"Benchmark {benchmark.id!r} is not scheduled by any matrix preset") + + unknown_presets = set(benchmark.runs) - known_presets + if unknown_presets: + unknown = ", ".join(sorted(unknown_presets)) + raise ValueError(f"Benchmark {benchmark.id!r} references unknown matrix presets: {unknown}") + + if benchmark.storage is Storage.S3 and (benchmark.local_dir is None or benchmark.remote_key is None): + raise ValueError(f"S3 benchmark {benchmark.id!r} must define local_dir and remote_key") + + for preset, coverage in benchmark.runs.items(): + if benchmark.id in ids_by_preset[preset]: + raise ValueError(f"Duplicate benchmark ID {benchmark.id!r} in matrix preset {preset!r}") + ids_by_preset[preset].add(benchmark.id) + _validate_coverage(benchmark, preset, coverage) diff --git a/bench-orchestrator/bench_orchestrator/cli.py b/bench-orchestrator/bench_orchestrator/cli.py index b9d7f9bb2ef..f20eac3162e 100644 --- a/bench-orchestrator/bench_orchestrator/cli.py +++ b/bench-orchestrator/bench_orchestrator/cli.py @@ -3,6 +3,7 @@ """CLI for benchmark orchestration.""" +import json import subprocess from contextlib import contextmanager from datetime import datetime, timedelta @@ -15,6 +16,7 @@ from rich.console import Console from rich.table import Table +from .ci_matrix import MATRIX_PRESETS, resolve_matrix from .comparison import analyzer from .comparison.reporter import pivot_comparison_table from .config import ( @@ -214,6 +216,30 @@ def prepare_data( raise typer.Exit(1) from exc +@app.command("matrix") +def matrix( + preset: Annotated[ + str | None, + typer.Argument(help="Matrix preset to render; omit to list available presets"), + ] = None, + list_presets: Annotated[bool, typer.Option("--list", help="List available presets and exit")] = False, + pretty: Annotated[bool, typer.Option("--pretty", help="Pretty-print the JSON output")] = False, +) -> None: + """Emit a GitHub Actions benchmark matrix.""" + if preset is None or list_presets: + for name, description in MATRIX_PRESETS.items(): + console.print(f"[bold cyan]{name}[/bold cyan]: {description}") + return + + if preset not in MATRIX_PRESETS: + known = ", ".join(MATRIX_PRESETS) + console.print(f"[red]Unknown matrix preset '{preset}'. Available: {known}[/red]") + raise typer.Exit(1) + + entries = resolve_matrix(preset) + typer.echo(json.dumps(entries, indent=2 if pretty else None)) + + @app.command() def run( benchmark: Annotated[Benchmark, typer.Argument(help="Benchmark suite to run")], diff --git a/bench-orchestrator/tests/test_matrix.py b/bench-orchestrator/tests/test_matrix.py new file mode 100644 index 00000000000..dd19d12e07f --- /dev/null +++ b/bench-orchestrator/tests/test_matrix.py @@ -0,0 +1,118 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright the Vortex contributors + +"""Tests for CI benchmark matrices.""" + +import json +from dataclasses import replace +from typing import cast + +import pytest +from bench_orchestrator import cli as cli_module +from bench_orchestrator.ci_matrix import MATRIX_PRESETS, resolve_matrix +from bench_orchestrator.ci_matrix.catalog import CATALOG +from bench_orchestrator.ci_matrix.model import Catalog, Coverage +from bench_orchestrator.ci_matrix.render import render_matrix +from bench_orchestrator.ci_matrix.targets import TargetSet +from typer.testing import CliRunner + +runner = CliRunner() + +REGULAR_IDS = ( + "clickbench-nvme", + "clickbench-sorted-nvme", + "tpch-nvme", + "tpch-s3", + "tpch-nvme-10", + "tpch-s3-10", + "tpcds-nvme", + "statpopgen", + "fineweb", + "fineweb-s3", + "polarsignals", + "appian-nvme", + "vortex-queries", +) +EXPECTED_IDS = { + "develop": REGULAR_IDS, + "pr": tuple( + benchmark_id + for benchmark_id in REGULAR_IDS + if benchmark_id not in {"tpch-s3-10", "appian-nvme", "vortex-queries"} + ), + "pr-full": REGULAR_IDS, + "nightly": ("tpch-nvme", "tpch-s3"), +} + + +def _entries(preset: str) -> list[dict[str, object]]: + return resolve_matrix(preset) + + +def _targets(entry: dict[str, object]) -> set[tuple[str, str]]: + targets = cast("list[dict[str, str]]", entry["targets"]) + return {(target["engine"], target["format"]) for target in targets} + + +@pytest.mark.parametrize(("preset", "expected_ids"), EXPECTED_IDS.items()) +def test_matrix_presets(preset: str, expected_ids: tuple[str, ...]) -> None: + entries = _entries(preset) + ids = [entry["id"] for entry in entries] + + assert tuple(ids) == expected_ids + assert len(ids) == len(set(ids)) + assert all(entry["targets"] for entry in entries) + assert "${{" not in json.dumps(entries) + + +def test_pr_target_selection() -> None: + develop = {entry["id"]: entry for entry in _entries("develop")} + pr = {entry["id"]: entry for entry in _entries("pr")} + pr_full = {entry["id"]: entry for entry in _entries("pr-full")} + + assert _targets(pr["tpch-nvme"]) == { + ("datafusion", "parquet"), + ("datafusion", "vortex"), + ("duckdb", "parquet"), + ("duckdb", "vortex"), + } + assert ("datafusion", "lance") in _targets(develop["tpch-nvme"]) + assert all(("datafusion", "lance") not in _targets(entry) for entry in pr_full.values()) + assert "vortex-compact" in cast("list[str]", pr_full["clickbench-nvme"]["data_formats"]) + + +def test_resolver_rejects_empty_targets() -> None: + benchmark = replace( + CATALOG.benchmarks[0], + id="empty", + runs={"develop": Coverage(TargetSet())}, + ) + catalog = Catalog(presets=CATALOG.presets, benchmarks=(benchmark,)) + + with pytest.raises(ValueError, match="Benchmark 'empty' resolved to no runnable targets"): + _ = render_matrix("develop", catalog) + + +def test_resolver_rejects_duplicate_ids() -> None: + benchmark = CATALOG.benchmarks[0] + catalog = Catalog( + presets=CATALOG.presets, + benchmarks=(benchmark, replace(benchmark, name="Duplicate")), + ) + + with pytest.raises(ValueError, match="Duplicate benchmark ID 'clickbench-nvme'"): + _ = render_matrix("develop", catalog) + + +def test_matrix_command() -> None: + result = runner.invoke(cli_module.app, ["matrix", "develop"]) + assert result.exit_code == 0 + assert json.loads(result.stdout) == _entries("develop") + + result = runner.invoke(cli_module.app, ["matrix", "does-not-exist"]) + assert result.exit_code == 1 + assert "Unknown matrix preset" in result.stdout + + +def test_preset_names_are_stable() -> None: + assert set(MATRIX_PRESETS) == set(EXPECTED_IDS)