From 9c031e27ef15ae8fa99808e6ef1d172c5858e1b8 Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Fri, 17 Jul 2026 10:45:57 +0000 Subject: [PATCH 1/3] Link per-suite explainer docs from CI benchmark PR comments Each "Benchmarks: " PR comment now carries a book icon link in its title pointing at a markdown explainer that lives next to the suite definition (vortex-bench/sql/* for SQL suites, benchmarks/* for the random-access and compression binaries). The doc path is a new "doc" field on each benchmark matrix entry, rendered into the comment title by the Compare results steps. Signed-off-by: Joe Isaacs --- .github/workflows/bench-pr.yml | 4 +++- .github/workflows/bench.yml | 1 + .github/workflows/sql-benchmarks.yml | 24 ++++++++++++++++++- .github/workflows/sql-vortex-pr.yml | 1 + benchmarks/compress-bench/README.md | 16 +++++++++++++ benchmarks/random-access-bench/README.md | 22 +++++++++++++++++ vortex-bench/REUSE.toml | 6 +++++ vortex-bench/sql/appian/README.md | 22 +++++++++++++++++ vortex-bench/sql/clickbench.md | 30 ++++++++++++++++++++++++ vortex-bench/sql/fineweb.md | 21 +++++++++++++++++ vortex-bench/sql/polarsignals.md | 23 ++++++++++++++++++ vortex-bench/sql/statpopgen.md | 25 ++++++++++++++++++++ vortex-bench/sql/tpcds/README.md | 20 ++++++++++++++++ vortex-bench/sql/tpch/README.md | 23 ++++++++++++++++++ vortex-bench/sql/vortex/README.md | 26 ++++++++++++++++++++ 15 files changed, 262 insertions(+), 2 deletions(-) create mode 100644 benchmarks/compress-bench/README.md create mode 100644 benchmarks/random-access-bench/README.md create mode 100644 vortex-bench/sql/appian/README.md create mode 100644 vortex-bench/sql/clickbench.md create mode 100644 vortex-bench/sql/fineweb.md create mode 100644 vortex-bench/sql/polarsignals.md create mode 100644 vortex-bench/sql/statpopgen.md create mode 100644 vortex-bench/sql/tpcds/README.md create mode 100644 vortex-bench/sql/tpch/README.md create mode 100644 vortex-bench/sql/vortex/README.md diff --git a/.github/workflows/bench-pr.yml b/.github/workflows/bench-pr.yml index 12137726122..ab8c439afe0 100644 --- a/.github/workflows/bench-pr.yml +++ b/.github/workflows/bench-pr.yml @@ -30,9 +30,11 @@ jobs: benchmark: - id: random-access-bench name: Random Access + doc: benchmarks/random-access-bench/README.md build_args: "--features lance" - id: compress-bench name: Compression + doc: benchmarks/compress-bench/README.md steps: - uses: runs-on/action@v2 if: github.event.pull_request.head.repo.fork == false @@ -113,7 +115,7 @@ jobs: python3 scripts/s3-download.py s3://vortex-ci-benchmark-results/data.json.gz data.json.gz --no-sign-request gzip -d -c data.json.gz > base.json - echo '# Benchmarks: ${{ matrix.benchmark.name }}' > comment.md + echo '# Benchmarks: ${{ matrix.benchmark.name }}${{ matrix.benchmark.doc && format(' [๐Ÿ“–]({0}/{1}/blob/develop/{2})', github.server_url, github.repository, matrix.benchmark.doc) || '' }}' > comment.md echo '' >> comment.md uv run --no-project scripts/compare-benchmark-jsons.py base.json results.json "${{ matrix.benchmark.name }}" \ >> comment.md diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index b1e29277f00..1bbdaa02939 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -183,6 +183,7 @@ jobs: "id": "vortex-queries", "subcommand": "vortex", "name": "Vortex queries", + "doc": "vortex-bench/sql/vortex/README.md", "data_formats": ["parquet", "vortex"], "pr_targets": [ {"engine": "datafusion", "format": "parquet"}, diff --git a/.github/workflows/sql-benchmarks.yml b/.github/workflows/sql-benchmarks.yml index 243db83aff8..840bfc8e0bf 100644 --- a/.github/workflows/sql-benchmarks.yml +++ b/.github/workflows/sql-benchmarks.yml @@ -20,6 +20,7 @@ on: "id": "clickbench-nvme", "subcommand": "clickbench", "name": "Clickbench on NVME", + "doc": "vortex-bench/sql/clickbench.md", "data_formats": ["parquet", "vortex", "vortex-compact", "duckdb"], "pr_targets": [ {"engine": "datafusion", "format": "parquet"}, @@ -43,6 +44,7 @@ on: "id": "clickbench-sorted-nvme", "subcommand": "clickbench-sorted", "name": "Clickbench Sorted on NVME", + "doc": "vortex-bench/sql/clickbench.md#sorted-variant", "data_formats": ["parquet", "vortex", "vortex-compact", "duckdb"], "pr_targets": [ {"engine": "datafusion", "format": "parquet"}, @@ -66,6 +68,7 @@ on: "id": "tpch-nvme", "subcommand": "tpch", "name": "TPC-H SF=1 on NVME", + "doc": "vortex-bench/sql/tpch/README.md", "data_formats": ["parquet", "vortex", "vortex-compact", "duckdb"], "pr_targets": [ {"engine": "datafusion", "format": "arrow"}, @@ -95,6 +98,7 @@ on: "id": "tpch-s3", "subcommand": "tpch", "name": "TPC-H SF=1 on S3", + "doc": "vortex-bench/sql/tpch/README.md", "local_dir": "vortex-bench/data/tpch/1.0", "remote_storage": "s3://vortex-ci-benchmark-datasets/${{github.ref_name}}/${{github.run_id}}/tpch/1.0/", "data_formats": ["parquet", "vortex", "vortex-compact"], @@ -121,6 +125,7 @@ on: "id": "tpch-nvme-10", "subcommand": "tpch", "name": "TPC-H SF=10 on NVME", + "doc": "vortex-bench/sql/tpch/README.md", "data_formats": ["parquet", "vortex", "vortex-compact", "duckdb"], "pr_targets": [ {"engine": "datafusion", "format": "arrow"}, @@ -150,6 +155,7 @@ on: "id": "tpch-s3-10", "subcommand": "tpch", "name": "TPC-H SF=10 on S3", + "doc": "vortex-bench/sql/tpch/README.md", "local_dir": "vortex-bench/data/tpch/10.0", "remote_storage": "s3://vortex-ci-benchmark-datasets/${{github.ref_name}}/${{github.run_id}}/tpch/10.0/", "data_formats": ["parquet", "vortex", "vortex-compact"], @@ -176,6 +182,7 @@ on: "id": "tpcds-nvme", "subcommand": "tpcds", "name": "TPC-DS SF=1 on NVME", + "doc": "vortex-bench/sql/tpcds/README.md", "data_formats": ["parquet", "vortex", "vortex-compact", "duckdb"], "pr_targets": [ {"engine": "datafusion", "format": "parquet"}, @@ -201,6 +208,7 @@ on: "id": "statpopgen", "subcommand": "statpopgen", "name": "Statistical and Population Genetics", + "doc": "vortex-bench/sql/statpopgen.md", "local_dir": "vortex-bench/data/statpopgen", "data_formats": ["parquet", "vortex", "vortex-compact"], "pr_targets": [ @@ -219,6 +227,7 @@ on: "id": "fineweb", "subcommand": "fineweb", "name": "FineWeb NVMe", + "doc": "vortex-bench/sql/fineweb.md", "data_formats": ["parquet", "vortex", "vortex-compact"], "pr_targets": [ {"engine": "datafusion", "format": "parquet"}, @@ -242,6 +251,7 @@ on: "id": "fineweb-s3", "subcommand": "fineweb", "name": "FineWeb S3", + "doc": "vortex-bench/sql/fineweb.md", "local_dir": "vortex-bench/data/fineweb", "remote_storage": "s3://vortex-ci-benchmark-datasets/${{github.ref_name}}/${{github.run_id}}/fineweb/", "data_formats": ["parquet", "vortex", "vortex-compact"], @@ -267,6 +277,7 @@ on: "id": "polarsignals", "subcommand": "polarsignals", "name": "PolarSignals Profiling", + "doc": "vortex-bench/sql/polarsignals.md", "data_formats": ["vortex"], "pr_targets": [ {"engine": "datafusion", "format": "vortex"} @@ -280,6 +291,7 @@ on: "id": "appian-nvme", "subcommand": "appian", "name": "Appian on NVME", + "doc": "vortex-bench/sql/appian/README.md", "data_formats": ["parquet", "vortex", "vortex-compact", "duckdb"], "pr_targets": [ {"engine": "datafusion", "format": "parquet"}, @@ -310,6 +322,7 @@ on: "id": "clickbench-nvme", "subcommand": "clickbench", "name": "Clickbench on NVME", + "doc": "vortex-bench/sql/clickbench.md", "data_formats": ["parquet", "vortex"], "pr_targets": [ {"engine": "datafusion", "format": "parquet"}, @@ -329,6 +342,7 @@ on: "id": "clickbench-sorted-nvme", "subcommand": "clickbench-sorted", "name": "Clickbench Sorted on NVME", + "doc": "vortex-bench/sql/clickbench.md#sorted-variant", "data_formats": ["parquet", "vortex"], "pr_targets": [ {"engine": "datafusion", "format": "parquet"}, @@ -348,6 +362,7 @@ on: "id": "tpch-nvme", "subcommand": "tpch", "name": "TPC-H SF=1 on NVME", + "doc": "vortex-bench/sql/tpch/README.md", "data_formats": ["parquet", "vortex"], "pr_targets": [ {"engine": "datafusion", "format": "arrow"}, @@ -371,6 +386,7 @@ on: "id": "tpch-s3", "subcommand": "tpch", "name": "TPC-H SF=1 on S3", + "doc": "vortex-bench/sql/tpch/README.md", "local_dir": "vortex-bench/data/tpch/1.0", "remote_storage": "s3://vortex-ci-benchmark-datasets/${{github.ref_name}}/${{github.run_id}}/tpch/1.0/", "data_formats": ["parquet", "vortex"], @@ -393,6 +409,7 @@ on: "id": "tpch-nvme-10", "subcommand": "tpch", "name": "TPC-H SF=10 on NVME", + "doc": "vortex-bench/sql/tpch/README.md", "data_formats": ["parquet", "vortex"], "pr_targets": [ {"engine": "datafusion", "format": "arrow"}, @@ -416,6 +433,7 @@ on: "id": "tpcds-nvme", "subcommand": "tpcds", "name": "TPC-DS SF=1 on NVME", + "doc": "vortex-bench/sql/tpcds/README.md", "data_formats": ["parquet", "vortex"], "pr_targets": [ {"engine": "datafusion", "format": "parquet"}, @@ -435,6 +453,7 @@ on: "id": "statpopgen", "subcommand": "statpopgen", "name": "Statistical and Population Genetics", + "doc": "vortex-bench/sql/statpopgen.md", "local_dir": "vortex-bench/data/statpopgen", "data_formats": ["parquet", "vortex"], "pr_targets": [ @@ -451,6 +470,7 @@ on: "id": "fineweb", "subcommand": "fineweb", "name": "FineWeb NVMe", + "doc": "vortex-bench/sql/fineweb.md", "data_formats": ["parquet", "vortex"], "pr_targets": [ {"engine": "datafusion", "format": "parquet"}, @@ -470,6 +490,7 @@ on: "id": "fineweb-s3", "subcommand": "fineweb", "name": "FineWeb S3", + "doc": "vortex-bench/sql/fineweb.md", "local_dir": "vortex-bench/data/fineweb", "remote_storage": "s3://vortex-ci-benchmark-datasets/${{github.ref_name}}/${{github.run_id}}/fineweb/", "data_formats": ["parquet", "vortex"], @@ -491,6 +512,7 @@ on: "id": "polarsignals", "subcommand": "polarsignals", "name": "PolarSignals Profiling", + "doc": "vortex-bench/sql/polarsignals.md", "data_formats": ["vortex"], "pr_targets": [ {"engine": "datafusion", "format": "vortex"} @@ -678,7 +700,7 @@ jobs: python3 scripts/s3-download.py s3://vortex-ci-benchmark-results/data.json.gz data.json.gz --no-sign-request gzip -d -c data.json.gz > base.json - echo "# Benchmarks: ${{ matrix.name }}" > comment.md + echo "# Benchmarks: ${{ matrix.name }}${{ matrix.doc && format(' [๐Ÿ“–]({0}/{1}/blob/develop/{2})', github.server_url, github.repository, matrix.doc) || '' }}" > comment.md echo '' >> comment.md uv run --no-project scripts/compare-benchmark-jsons.py base.json results.json "${{ matrix.name }}" \ >> comment.md diff --git a/.github/workflows/sql-vortex-pr.yml b/.github/workflows/sql-vortex-pr.yml index 2b99eb0e4d9..b81b7e7d342 100644 --- a/.github/workflows/sql-vortex-pr.yml +++ b/.github/workflows/sql-vortex-pr.yml @@ -29,6 +29,7 @@ jobs: "id": "vortex-queries", "subcommand": "vortex", "name": "Vortex queries", + "doc": "vortex-bench/sql/vortex/README.md", "data_formats": ["parquet", "vortex"], "pr_targets": [ {"engine": "datafusion", "format": "parquet"}, diff --git a/benchmarks/compress-bench/README.md b/benchmarks/compress-bench/README.md new file mode 100644 index 00000000000..df1027e2bd0 --- /dev/null +++ b/benchmarks/compress-bench/README.md @@ -0,0 +1,16 @@ +# Compression benchmark + +Measures compression and decompression throughput, plus resulting file sizes, for Vortex +versus Parquet (and optionally Lance) across a range of datasets: NYC taxi data, several +[Public BI](https://github.com/cwida/public_bi_benchmark) tables (Arade, Bimbo, +CMSprovider, Euro2016, Food, HashTags), TPC-H `l_comment` variants, and synthetic nested +data. This is the workload behind the `Compression` PR comment. + +See [`src/main.rs`](./src/main.rs) for the dataset list and CLI flags (`--formats`, +`--datasets`, `--ops compress,decompress`). + +## Running locally + +```bash +cargo run -p compress-bench --profile release_debug +``` diff --git a/benchmarks/random-access-bench/README.md b/benchmarks/random-access-bench/README.md new file mode 100644 index 00000000000..14949c85fcb --- /dev/null +++ b/benchmarks/random-access-bench/README.md @@ -0,0 +1,22 @@ +# Random Access benchmark + +Measures point-lookup latency: fetching individual rows by index from a file, rather than +scanning it. This is the workload behind the `Random Access` PR comment. + +Two access patterns are generated with a fixed seed (see [`src/main.rs`](./src/main.rs)): + +- **correlated**: several clusters of consecutive indices scattered across the dataset, + simulating lookups with spatial locality; +- **uniform**: indices drawn from a Poisson process spread uniformly across the dataset, + simulating lookups with no locality. + +Each pattern runs over four datasets (`taxi`, `feature-vectors`, `nested-lists`, +`nested-structs`) in Parquet, Lance, and Vortex, both with a cached open file handle and +reopening the file per lookup. CI drives the full matrix via +[`scripts/random-access-split.py`](../../scripts/random-access-split.py). + +## Running locally + +```bash +cargo run -p random-access-bench --profile release_debug --features lance +``` diff --git a/vortex-bench/REUSE.toml b/vortex-bench/REUSE.toml index 5fb9bd33466..a39e10c12b5 100644 --- a/vortex-bench/REUSE.toml +++ b/vortex-bench/REUSE.toml @@ -10,6 +10,12 @@ path = "**/*.csv" SPDX-FileCopyrightText = "Copyright the Vortex contributors" SPDX-License-Identifier = "Apache-2.0" +# Benchmark suite explainer docs linked from CI PR comments. +[[annotations]] +path = "sql/**/*.md" +SPDX-FileCopyrightText = "Copyright the Vortex contributors" +SPDX-License-Identifier = "CC-BY-4.0" + # `insta` snapshot files do not allow leading comment lines. [[annotations]] path = "src/snapshots/**.snap" diff --git a/vortex-bench/sql/appian/README.md b/vortex-bench/sql/appian/README.md new file mode 100644 index 00000000000..9304bcee2a0 --- /dev/null +++ b/vortex-bench/sql/appian/README.md @@ -0,0 +1,22 @@ +# Appian benchmark + +Mirrors the queries from DuckDB's in-tree +[`appian_benchmarks`](https://github.com/duckdb/duckdb/tree/main/benchmark/appian_benchmarks) +suite โ€” a real-world business-application workload (customers, orders, addresses) with +selective filters and joins over camelCase-named columns. + +The eight queries live in this directory (`q1.sql` ... `q8.sql`). Upstream ships the data +as a single `.duckdb` blob (~593 MB); the harness downloads it once and projects each +table into Parquet with lowercased column names so that DataFusion and DuckDB resolve the +verbatim queries identically โ€” see [`src/appian`](../../src/appian) for the details. + +## CI variant + +CI runs this suite from local NVMe as the `Appian on NVME` PR comment, comparing +DataFusion and DuckDB over Parquet and Vortex files (plus a native DuckDB baseline). + +## Running locally + +```bash +vx-bench run appian --engine datafusion,duckdb --format parquet,vortex +``` diff --git a/vortex-bench/sql/clickbench.md b/vortex-bench/sql/clickbench.md new file mode 100644 index 00000000000..eb3db810649 --- /dev/null +++ b/vortex-bench/sql/clickbench.md @@ -0,0 +1,30 @@ +# ClickBench benchmark + +[ClickBench](https://github.com/ClickHouse/ClickBench) is ClickHouse's web-analytics +benchmark: 43 queries over a single wide `hits` table (~100M rows of real-ish traffic +data). It is heavy on aggregations, `GROUP BY`s over high-cardinality columns, and +selective string filters, and is the main "wide table scan" workload in CI. + +The queries live in [`clickbench_queries.sql`](./clickbench_queries.sql) (one query per +line, numbered from Q0 in file order). The harness lives in +[`src/clickbench`](../src/clickbench). + +## CI variant + +CI runs this suite from local NVMe as the `Clickbench on NVME` PR comment, comparing +DataFusion and DuckDB over Parquet, Vortex, and vortex-compact files (plus a native +DuckDB baseline). + +## Sorted variant + +`Clickbench Sorted on NVME` runs the same table sorted by event time, split into 100 +shards whose filenames are shuffled so engines cannot rely on file order, exercising sort +pushdown and zone-map-style pruning on a subset of the queries. See +[`ClickBenchSortedBenchmark`](../src/clickbench/benchmark.rs). + +## Running locally + +```bash +vx-bench run clickbench --engine datafusion,duckdb --format parquet,vortex +vx-bench run clickbench-sorted --engine datafusion,duckdb --format parquet,vortex +``` diff --git a/vortex-bench/sql/fineweb.md b/vortex-bench/sql/fineweb.md new file mode 100644 index 00000000000..13421cc5456 --- /dev/null +++ b/vortex-bench/sql/fineweb.md @@ -0,0 +1,21 @@ +# FineWeb benchmark + +A string-heavy benchmark over a sample shard of the HuggingFace +[FineWeb](https://huggingface.co/datasets/HuggingFaceFW/fineweb) web-crawl dataset +(URLs, dates, and full page text). The dataset exercises dictionary and FSST string +compression heavily, and the hand-crafted queries in [`fineweb.sql`](./fineweb.sql) +(numbered from Q0 in file order) focus on string predicates: equality filters, `LIKE` +prefix and containment patterns, and aggregations over string columns. + +The harness lives in [`src/fineweb`](../src/fineweb). + +## CI variants + +CI runs this suite from local NVMe (`FineWeb NVMe`) and from S3 (`FineWeb S3`), comparing +DataFusion and DuckDB over Parquet, Vortex, and vortex-compact files. + +## Running locally + +```bash +vx-bench run fineweb --engine datafusion,duckdb --format parquet,vortex +``` diff --git a/vortex-bench/sql/polarsignals.md b/vortex-bench/sql/polarsignals.md new file mode 100644 index 00000000000..499466593e7 --- /dev/null +++ b/vortex-bench/sql/polarsignals.md @@ -0,0 +1,23 @@ +# PolarSignals Profiling benchmark + +A benchmark over continuous-profiling stacktrace data from +[Polar Signals](https://www.polarsignals.com/). The schema features a sparse struct (ten +nullable label fields covering five fill-rate tiers), deeply nested locations +(`List>>>`), and several low-cardinality string columns โ€” +making it the main deeply-nested / sparse-data workload in CI. + +The queries in [`polarsignals.sql`](./polarsignals.sql) select stacktrace samples over +increasingly wide time ranges with equality filters on the low-cardinality metadata +columns. The harness lives in [`src/polarsignals`](../src/polarsignals). + +## CI variant + +CI runs this suite from local NVMe as the `PolarSignals Profiling` PR comment. It runs +DataFusion over Vortex only (there is no Parquet control for this suite, so its geomean is +reported without a drift-corrected verdict). + +## Running locally + +```bash +vx-bench run polarsignals --engine datafusion --format vortex +``` diff --git a/vortex-bench/sql/statpopgen.md b/vortex-bench/sql/statpopgen.md new file mode 100644 index 00000000000..c0b90a54f54 --- /dev/null +++ b/vortex-bench/sql/statpopgen.md @@ -0,0 +1,25 @@ +# Statistical and Population Genetics benchmark + +A genomics analytics benchmark using the gnomAD v3.1.2 release of the jointly called One +Thousand Genomes + Human Genome Diversity Project (1kG+HGDP) dataset โ€” a prefix of +chromosome 21. The data source is . + +The distinguishing feature is the `GT` genotype column: a large variable-length list of +small nullable integers per row, which stresses list encodings and list-aware compute. +The queries in [`statpopgen.sql`](./statpopgen.sql) cover common genomics patterns: +allele-frequency calculations, Hardy-Weinberg equilibrium statistics, variant filtering by +frequency and by locus interval, and random access to a specific variant. + +The harness lives in [`src/statpopgen`](../src/statpopgen). + +## CI variant + +CI runs this suite from local NVMe as the `Statistical and Population Genetics` PR +comment. Only DuckDB is exercised (over Parquet, Vortex, and vortex-compact files), +because the queries rely on DuckDB list lambdas. + +## Running locally + +```bash +vx-bench run statpopgen --engine duckdb --format parquet,vortex +``` diff --git a/vortex-bench/sql/tpcds/README.md b/vortex-bench/sql/tpcds/README.md new file mode 100644 index 00000000000..3e4258ce251 --- /dev/null +++ b/vortex-bench/sql/tpcds/README.md @@ -0,0 +1,20 @@ +# TPC-DS benchmark + +The industry-standard [TPC-DS](https://www.tpc.org/tpcds/) decision-support benchmark: 99 +analytical queries (`01.sql` ... `99.sql`) over a retail snowflake schema. Compared to +TPC-H it has many more queries, a wider schema, and heavier use of joins, subqueries, and +window functions, making it a broad regression net for scan and pushdown behavior. + +The benchmark harness lives in [`src/tpcds`](../../src/tpcds). + +## CI variant + +CI runs this suite at scale factor 1 from local NVMe as the `TPC-DS SF=1 on NVME` PR +comment, comparing DataFusion and DuckDB over Parquet, Vortex, and vortex-compact files +(plus a native DuckDB baseline). + +## Running locally + +```bash +vx-bench run tpcds --engine datafusion,duckdb --format parquet,vortex --opt scale-factor=1.0 +``` diff --git a/vortex-bench/sql/tpch/README.md b/vortex-bench/sql/tpch/README.md new file mode 100644 index 00000000000..aa5a16fbab2 --- /dev/null +++ b/vortex-bench/sql/tpch/README.md @@ -0,0 +1,23 @@ +# TPC-H benchmark + +The industry-standard [TPC-H](https://www.tpc.org/tpch/) decision-support benchmark: 22 +analytical queries over an 8-table warehouse schema (`lineitem`, `orders`, `customer`, ...). +It stresses scan throughput, filter pushdown, joins, and aggregations over mostly numeric +and date columns. + +The queries in this directory (`q1.sql` ... `q22.sql`) are the standard TPC-H queries. +Data is generated deterministically by [`tpchgen`](../../src/tpch/tpchgen.rs); the benchmark +harness lives in [`src/tpch`](../../src/tpch). + +## CI variants + +CI runs this suite at scale factor 1 and 10, reading either from local NVMe or from S3 +(the data is uploaded to a bucket first), as the `TPC-H SF={1,10} on {NVME,S3}` PR +comments. Each variant compares DataFusion and DuckDB over Parquet, Vortex, and +vortex-compact files (plus in-memory Arrow and native DuckDB baselines on NVMe). + +## Running locally + +```bash +vx-bench run tpch --engine datafusion,duckdb --format parquet,vortex --opt scale-factor=1.0 +``` diff --git a/vortex-bench/sql/vortex/README.md b/vortex-bench/sql/vortex/README.md new file mode 100644 index 00000000000..793f7fb0f67 --- /dev/null +++ b/vortex-bench/sql/vortex/README.md @@ -0,0 +1,26 @@ +# Vortex queries benchmark + +A small suite of microbenchmark queries targeting Vortex-specific scan paths, run on +**every PR commit** (unlike the label-gated suites) with a high iteration count, so it is +the most sensitive โ€” and most frequently seen โ€” benchmark comment. + +[`init.sql`](./init.sql) generates a 25M-row two-column table; the numbered queries each +pin down one scan behavior: + +- [`0_sum-with-filter.sql`](./0_sum-with-filter.sql): a filtered aggregation that forces a + linear scan today; once statistics are propagated to arrays it should use zone maps + instead of decoding every row. +- [`1_sum.sql`](./1_sum.sql): an unfiltered aggregation that should eventually be answered + from footer statistics without decoding data. + +## CI variant + +CI runs this as the `Vortex queries` PR comment (see +[`.github/workflows/sql-vortex-pr.yml`](../../../.github/workflows/sql-vortex-pr.yml)), +comparing DataFusion and DuckDB over Parquet and Vortex files with 100 iterations. + +## Running locally + +```bash +vx-bench run vortex --engine datafusion,duckdb --format parquet,vortex +``` From 16238b158317e2ae615306812100183c30e4614a Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Fri, 17 Jul 2026 13:13:04 +0000 Subject: [PATCH 2/3] Move benchmark doc links into the Rust benchmark definitions The suite explainer path now lives on the Benchmark trait (doc_path) next to each suite implementation, is stamped onto every gh-json result row as a "doc" field, and compare-benchmark-jsons.py renders the full comment title (including the book-icon link) from it. This removes the per-suite "doc" fields from the workflow matrices and the title echo lines from the Compare results steps, making the Rust benchmark code the single source of truth for where each suite is documented. The random-access and compress binaries carry their doc path as a constant. Signed-off-by: Joe Isaacs --- .github/workflows/bench-pr.yml | 6 +---- .github/workflows/bench.yml | 1 - .github/workflows/sql-benchmarks.yml | 26 +------------------ .github/workflows/sql-vortex-pr.yml | 1 - benchmarks/compress-bench/src/main.rs | 7 +++-- benchmarks/random-access-bench/src/main.rs | 5 +++- scripts/compare-benchmark-jsons.py | 22 ++++++++++++++++ vortex-bench/src/appian/mod.rs | 4 +++ vortex-bench/src/benchmark.rs | 6 +++++ vortex-bench/src/clickbench/benchmark.rs | 8 ++++++ vortex-bench/src/display.rs | 7 ++++- vortex-bench/src/fineweb/mod.rs | 4 +++ vortex-bench/src/polarsignals/benchmark.rs | 4 +++ vortex-bench/src/runner.rs | 10 +++++-- .../src/statpopgen/statpopgen_benchmark.rs | 4 +++ vortex-bench/src/tpcds/tpcds_benchmark.rs | 4 +++ vortex-bench/src/tpch/benchmark.rs | 4 +++ vortex-bench/src/vortex_queries.rs | 4 +++ 18 files changed, 89 insertions(+), 38 deletions(-) diff --git a/.github/workflows/bench-pr.yml b/.github/workflows/bench-pr.yml index ab8c439afe0..b5b011150a6 100644 --- a/.github/workflows/bench-pr.yml +++ b/.github/workflows/bench-pr.yml @@ -30,11 +30,9 @@ jobs: benchmark: - id: random-access-bench name: Random Access - doc: benchmarks/random-access-bench/README.md build_args: "--features lance" - id: compress-bench name: Compression - doc: benchmarks/compress-bench/README.md steps: - uses: runs-on/action@v2 if: github.event.pull_request.head.repo.fork == false @@ -115,10 +113,8 @@ jobs: python3 scripts/s3-download.py s3://vortex-ci-benchmark-results/data.json.gz data.json.gz --no-sign-request gzip -d -c data.json.gz > base.json - echo '# Benchmarks: ${{ matrix.benchmark.name }}${{ matrix.benchmark.doc && format(' [๐Ÿ“–]({0}/{1}/blob/develop/{2})', github.server_url, github.repository, matrix.benchmark.doc) || '' }}' > comment.md - echo '' >> comment.md uv run --no-project scripts/compare-benchmark-jsons.py base.json results.json "${{ matrix.benchmark.name }}" \ - >> comment.md + > comment.md cat comment.md >> $GITHUB_STEP_SUMMARY - name: Comment PR diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 1bbdaa02939..b1e29277f00 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -183,7 +183,6 @@ jobs: "id": "vortex-queries", "subcommand": "vortex", "name": "Vortex queries", - "doc": "vortex-bench/sql/vortex/README.md", "data_formats": ["parquet", "vortex"], "pr_targets": [ {"engine": "datafusion", "format": "parquet"}, diff --git a/.github/workflows/sql-benchmarks.yml b/.github/workflows/sql-benchmarks.yml index 840bfc8e0bf..59dd9571b36 100644 --- a/.github/workflows/sql-benchmarks.yml +++ b/.github/workflows/sql-benchmarks.yml @@ -20,7 +20,6 @@ on: "id": "clickbench-nvme", "subcommand": "clickbench", "name": "Clickbench on NVME", - "doc": "vortex-bench/sql/clickbench.md", "data_formats": ["parquet", "vortex", "vortex-compact", "duckdb"], "pr_targets": [ {"engine": "datafusion", "format": "parquet"}, @@ -44,7 +43,6 @@ on: "id": "clickbench-sorted-nvme", "subcommand": "clickbench-sorted", "name": "Clickbench Sorted on NVME", - "doc": "vortex-bench/sql/clickbench.md#sorted-variant", "data_formats": ["parquet", "vortex", "vortex-compact", "duckdb"], "pr_targets": [ {"engine": "datafusion", "format": "parquet"}, @@ -68,7 +66,6 @@ on: "id": "tpch-nvme", "subcommand": "tpch", "name": "TPC-H SF=1 on NVME", - "doc": "vortex-bench/sql/tpch/README.md", "data_formats": ["parquet", "vortex", "vortex-compact", "duckdb"], "pr_targets": [ {"engine": "datafusion", "format": "arrow"}, @@ -98,7 +95,6 @@ on: "id": "tpch-s3", "subcommand": "tpch", "name": "TPC-H SF=1 on S3", - "doc": "vortex-bench/sql/tpch/README.md", "local_dir": "vortex-bench/data/tpch/1.0", "remote_storage": "s3://vortex-ci-benchmark-datasets/${{github.ref_name}}/${{github.run_id}}/tpch/1.0/", "data_formats": ["parquet", "vortex", "vortex-compact"], @@ -125,7 +121,6 @@ on: "id": "tpch-nvme-10", "subcommand": "tpch", "name": "TPC-H SF=10 on NVME", - "doc": "vortex-bench/sql/tpch/README.md", "data_formats": ["parquet", "vortex", "vortex-compact", "duckdb"], "pr_targets": [ {"engine": "datafusion", "format": "arrow"}, @@ -155,7 +150,6 @@ on: "id": "tpch-s3-10", "subcommand": "tpch", "name": "TPC-H SF=10 on S3", - "doc": "vortex-bench/sql/tpch/README.md", "local_dir": "vortex-bench/data/tpch/10.0", "remote_storage": "s3://vortex-ci-benchmark-datasets/${{github.ref_name}}/${{github.run_id}}/tpch/10.0/", "data_formats": ["parquet", "vortex", "vortex-compact"], @@ -182,7 +176,6 @@ on: "id": "tpcds-nvme", "subcommand": "tpcds", "name": "TPC-DS SF=1 on NVME", - "doc": "vortex-bench/sql/tpcds/README.md", "data_formats": ["parquet", "vortex", "vortex-compact", "duckdb"], "pr_targets": [ {"engine": "datafusion", "format": "parquet"}, @@ -208,7 +201,6 @@ on: "id": "statpopgen", "subcommand": "statpopgen", "name": "Statistical and Population Genetics", - "doc": "vortex-bench/sql/statpopgen.md", "local_dir": "vortex-bench/data/statpopgen", "data_formats": ["parquet", "vortex", "vortex-compact"], "pr_targets": [ @@ -227,7 +219,6 @@ on: "id": "fineweb", "subcommand": "fineweb", "name": "FineWeb NVMe", - "doc": "vortex-bench/sql/fineweb.md", "data_formats": ["parquet", "vortex", "vortex-compact"], "pr_targets": [ {"engine": "datafusion", "format": "parquet"}, @@ -251,7 +242,6 @@ on: "id": "fineweb-s3", "subcommand": "fineweb", "name": "FineWeb S3", - "doc": "vortex-bench/sql/fineweb.md", "local_dir": "vortex-bench/data/fineweb", "remote_storage": "s3://vortex-ci-benchmark-datasets/${{github.ref_name}}/${{github.run_id}}/fineweb/", "data_formats": ["parquet", "vortex", "vortex-compact"], @@ -277,7 +267,6 @@ on: "id": "polarsignals", "subcommand": "polarsignals", "name": "PolarSignals Profiling", - "doc": "vortex-bench/sql/polarsignals.md", "data_formats": ["vortex"], "pr_targets": [ {"engine": "datafusion", "format": "vortex"} @@ -291,7 +280,6 @@ on: "id": "appian-nvme", "subcommand": "appian", "name": "Appian on NVME", - "doc": "vortex-bench/sql/appian/README.md", "data_formats": ["parquet", "vortex", "vortex-compact", "duckdb"], "pr_targets": [ {"engine": "datafusion", "format": "parquet"}, @@ -322,7 +310,6 @@ on: "id": "clickbench-nvme", "subcommand": "clickbench", "name": "Clickbench on NVME", - "doc": "vortex-bench/sql/clickbench.md", "data_formats": ["parquet", "vortex"], "pr_targets": [ {"engine": "datafusion", "format": "parquet"}, @@ -342,7 +329,6 @@ on: "id": "clickbench-sorted-nvme", "subcommand": "clickbench-sorted", "name": "Clickbench Sorted on NVME", - "doc": "vortex-bench/sql/clickbench.md#sorted-variant", "data_formats": ["parquet", "vortex"], "pr_targets": [ {"engine": "datafusion", "format": "parquet"}, @@ -362,7 +348,6 @@ on: "id": "tpch-nvme", "subcommand": "tpch", "name": "TPC-H SF=1 on NVME", - "doc": "vortex-bench/sql/tpch/README.md", "data_formats": ["parquet", "vortex"], "pr_targets": [ {"engine": "datafusion", "format": "arrow"}, @@ -386,7 +371,6 @@ on: "id": "tpch-s3", "subcommand": "tpch", "name": "TPC-H SF=1 on S3", - "doc": "vortex-bench/sql/tpch/README.md", "local_dir": "vortex-bench/data/tpch/1.0", "remote_storage": "s3://vortex-ci-benchmark-datasets/${{github.ref_name}}/${{github.run_id}}/tpch/1.0/", "data_formats": ["parquet", "vortex"], @@ -409,7 +393,6 @@ on: "id": "tpch-nvme-10", "subcommand": "tpch", "name": "TPC-H SF=10 on NVME", - "doc": "vortex-bench/sql/tpch/README.md", "data_formats": ["parquet", "vortex"], "pr_targets": [ {"engine": "datafusion", "format": "arrow"}, @@ -433,7 +416,6 @@ on: "id": "tpcds-nvme", "subcommand": "tpcds", "name": "TPC-DS SF=1 on NVME", - "doc": "vortex-bench/sql/tpcds/README.md", "data_formats": ["parquet", "vortex"], "pr_targets": [ {"engine": "datafusion", "format": "parquet"}, @@ -453,7 +435,6 @@ on: "id": "statpopgen", "subcommand": "statpopgen", "name": "Statistical and Population Genetics", - "doc": "vortex-bench/sql/statpopgen.md", "local_dir": "vortex-bench/data/statpopgen", "data_formats": ["parquet", "vortex"], "pr_targets": [ @@ -470,7 +451,6 @@ on: "id": "fineweb", "subcommand": "fineweb", "name": "FineWeb NVMe", - "doc": "vortex-bench/sql/fineweb.md", "data_formats": ["parquet", "vortex"], "pr_targets": [ {"engine": "datafusion", "format": "parquet"}, @@ -490,7 +470,6 @@ on: "id": "fineweb-s3", "subcommand": "fineweb", "name": "FineWeb S3", - "doc": "vortex-bench/sql/fineweb.md", "local_dir": "vortex-bench/data/fineweb", "remote_storage": "s3://vortex-ci-benchmark-datasets/${{github.ref_name}}/${{github.run_id}}/fineweb/", "data_formats": ["parquet", "vortex"], @@ -512,7 +491,6 @@ on: "id": "polarsignals", "subcommand": "polarsignals", "name": "PolarSignals Profiling", - "doc": "vortex-bench/sql/polarsignals.md", "data_formats": ["vortex"], "pr_targets": [ {"engine": "datafusion", "format": "vortex"} @@ -700,10 +678,8 @@ jobs: python3 scripts/s3-download.py s3://vortex-ci-benchmark-results/data.json.gz data.json.gz --no-sign-request gzip -d -c data.json.gz > base.json - echo "# Benchmarks: ${{ matrix.name }}${{ matrix.doc && format(' [๐Ÿ“–]({0}/{1}/blob/develop/{2})', github.server_url, github.repository, matrix.doc) || '' }}" > comment.md - echo '' >> comment.md uv run --no-project scripts/compare-benchmark-jsons.py base.json results.json "${{ matrix.name }}" \ - >> comment.md + > comment.md cat comment.md >> "$GITHUB_STEP_SUMMARY" - name: Comment PR diff --git a/.github/workflows/sql-vortex-pr.yml b/.github/workflows/sql-vortex-pr.yml index b81b7e7d342..2b99eb0e4d9 100644 --- a/.github/workflows/sql-vortex-pr.yml +++ b/.github/workflows/sql-vortex-pr.yml @@ -29,7 +29,6 @@ jobs: "id": "vortex-queries", "subcommand": "vortex", "name": "Vortex queries", - "doc": "vortex-bench/sql/vortex/README.md", "data_formats": ["parquet", "vortex"], "pr_targets": [ {"engine": "datafusion", "format": "parquet"}, diff --git a/benchmarks/compress-bench/src/main.rs b/benchmarks/compress-bench/src/main.rs index 8d9fa73f915..353d7af7a0b 100644 --- a/benchmarks/compress-bench/src/main.rs +++ b/benchmarks/compress-bench/src/main.rs @@ -113,6 +113,9 @@ fn get_compressor(format: Format) -> Box { /// The benchmark ID used for output path. const BENCHMARK_ID: &str = "compress"; +/// Repo-relative path of the suite explainer linked from CI benchmark PR comments. +const DOC_PATH: &str = "benchmarks/compress-bench/README.md"; + async fn run_compress( iterations: usize, datasets_filter: Option, @@ -212,8 +215,8 @@ async fn run_compress( ) } DisplayFormat::GhJson => { - print_measurements_json(&mut writer, measurements.timings)?; - print_measurements_json(&mut writer, measurements.ratios) + print_measurements_json(&mut writer, measurements.timings, Some(DOC_PATH))?; + print_measurements_json(&mut writer, measurements.ratios, Some(DOC_PATH)) } } } diff --git a/benchmarks/random-access-bench/src/main.rs b/benchmarks/random-access-bench/src/main.rs index 6722265fde6..9a387786321 100644 --- a/benchmarks/random-access-bench/src/main.rs +++ b/benchmarks/random-access-bench/src/main.rs @@ -382,6 +382,9 @@ async fn open_accessor( /// The benchmark ID used for output path. const BENCHMARK_ID: &str = "random-access"; +/// Repo-relative path of the suite explainer linked from CI benchmark PR comments. +const DOC_PATH: &str = "benchmarks/random-access-bench/README.md"; + /// Fixed indices used by the original taxi benchmark (preserved for historical continuity). const FIXED_TAXI_INDICES: [u64; 6] = [10, 11, 12, 13, 100_000, 3_000_000]; @@ -500,7 +503,7 @@ async fn run_random_access(config: RunConfig) -> Result<()> { } DisplayFormat::GhJson => { let timings: Vec = runs.into_iter().map(|r| r.timing).collect(); - print_measurements_json(&mut writer, timings)?; + print_measurements_json(&mut writer, timings, Some(DOC_PATH))?; } } diff --git a/scripts/compare-benchmark-jsons.py b/scripts/compare-benchmark-jsons.py index 7d560c144a3..116247c7b66 100644 --- a/scripts/compare-benchmark-jsons.py +++ b/scripts/compare-benchmark-jsons.py @@ -12,6 +12,7 @@ # SPDX-FileCopyrightText: Copyright the Vortex contributors import math +import os import re import sys from dataclasses import dataclass @@ -763,6 +764,24 @@ def format_within_engine_summary(analyses: dict[str, dict[str, Any]]) -> str | N return " ยท ".join(summaries) +def format_title(benchmark_name: str, pr: pd.DataFrame) -> str: + """Render the comment title, linking the suite explainer doc emitted by the benchmark binary. + + The doc path is a repo-relative markdown path carried on the PR result rows (the `doc` + field, populated from `Benchmark::doc_path` in Rust), so the benchmark code is the single + source of truth for where each suite is documented. + """ + + title = f"# Benchmarks: {benchmark_name}" if benchmark_name else "# Benchmarks" + if "doc" in pr.columns: + docs = pr["doc"].dropna().unique() + if len(docs) > 0: + server_url = os.environ.get("GITHUB_SERVER_URL", "https://github.com") + repository = os.environ.get("GITHUB_REPOSITORY", "vortex-data/vortex") + title += f" [\N{OPEN BOOK}]({server_url}/{repository}/blob/develop/{docs[0]})" + return title + + def format_report_help() -> str: """Render explanatory markdown for the benchmark report headline fields.""" @@ -823,6 +842,7 @@ def main() -> None: benchmark_name = sys.argv[3] if len(sys.argv) > 3 else "" pr = pd.read_json(sys.argv[2], lines=True) + title = format_title(benchmark_name, pr) base = read_latest_baseline_rows(sys.argv[1], pr) base_commit_id = set(base["commit_id"].unique()) @@ -896,6 +916,8 @@ def main() -> None: shifts += f" ยท Median polish {format_ratio_change(float(np.exp(polish.overall)))}" summary_fields.append(f"**Shifts**: {shifts}") + print(title) + print("") print("
".join(summary_fields)) print("") print(format_report_help()) diff --git a/vortex-bench/src/appian/mod.rs b/vortex-bench/src/appian/mod.rs index cdc82ac200a..fe42d2abe30 100644 --- a/vortex-bench/src/appian/mod.rs +++ b/vortex-bench/src/appian/mod.rs @@ -121,6 +121,10 @@ impl AppianBenchmark { #[async_trait::async_trait] impl Benchmark for AppianBenchmark { + fn doc_path(&self) -> Option<&'static str> { + Some("vortex-bench/sql/appian/README.md") + } + fn queries(&self) -> anyhow::Result> { Ok(appian_queries().collect()) } diff --git a/vortex-bench/src/benchmark.rs b/vortex-bench/src/benchmark.rs index d59708886a6..488080af248 100644 --- a/vortex-bench/src/benchmark.rs +++ b/vortex-bench/src/benchmark.rs @@ -64,6 +64,12 @@ pub trait Benchmark: Send + Sync { fn dataset(&self) -> BenchmarkDataset; + /// Repo-relative path of the markdown explainer for this benchmark suite, linked from the + /// title of CI benchmark PR comments. `None` for suites without a doc. + fn doc_path(&self) -> Option<&'static str> { + None + } + /// Get the name of the benchmark dataset fn dataset_name(&self) -> &str; diff --git a/vortex-bench/src/clickbench/benchmark.rs b/vortex-bench/src/clickbench/benchmark.rs index 346496dbda0..2a254d1976c 100644 --- a/vortex-bench/src/clickbench/benchmark.rs +++ b/vortex-bench/src/clickbench/benchmark.rs @@ -76,6 +76,10 @@ fn read_clickbench_queries(queries_file: Option<&str>) -> Result Option<&'static str> { + Some("vortex-bench/sql/clickbench.md") + } + fn queries(&self) -> Result> { read_clickbench_queries(self.queries_file.as_deref()) } @@ -120,6 +124,10 @@ impl Benchmark for ClickBenchBenchmark { #[async_trait::async_trait] impl Benchmark for ClickBenchSortedBenchmark { + fn doc_path(&self) -> Option<&'static str> { + Some("vortex-bench/sql/clickbench.md#sorted-variant") + } + fn queries(&self) -> Result> { Ok(read_clickbench_queries(self.queries_file.as_deref())? .into_iter() diff --git a/vortex-bench/src/display.rs b/vortex-bench/src/display.rs index a8e903df25a..29aebddc40b 100644 --- a/vortex-bench/src/display.rs +++ b/vortex-bench/src/display.rs @@ -105,9 +105,14 @@ pub fn render_table( pub fn print_measurements_json( writer: &mut dyn Write, all_measurements: Vec, + doc: Option<&str>, ) -> anyhow::Result<()> { for measurement in all_measurements { - writeln!(writer, "{}", measurement.to_json())?; + let mut json = measurement.to_json(); + if let (Some(doc), Some(obj)) = (doc, json.as_object_mut()) { + obj.insert("doc".to_string(), doc.into()); + } + writeln!(writer, "{json}")?; } Ok(()) diff --git a/vortex-bench/src/fineweb/mod.rs b/vortex-bench/src/fineweb/mod.rs index b8c86925905..f7dade2de01 100644 --- a/vortex-bench/src/fineweb/mod.rs +++ b/vortex-bench/src/fineweb/mod.rs @@ -53,6 +53,10 @@ impl FinewebBenchmark { #[async_trait::async_trait] impl Benchmark for FinewebBenchmark { + fn doc_path(&self) -> Option<&'static str> { + Some("vortex-bench/sql/fineweb.md") + } + /// Some basic string-focused queries, numbered from Q0 in `sql/fineweb.sql` file order. fn queries(&self) -> anyhow::Result> { // `;`-separated; a `;` must not appear in a comment, or it would split a statement in two. diff --git a/vortex-bench/src/polarsignals/benchmark.rs b/vortex-bench/src/polarsignals/benchmark.rs index fce692ceb68..2f00181ba00 100644 --- a/vortex-bench/src/polarsignals/benchmark.rs +++ b/vortex-bench/src/polarsignals/benchmark.rs @@ -59,6 +59,10 @@ impl PolarSignalsBenchmark { #[async_trait::async_trait] impl Benchmark for PolarSignalsBenchmark { + fn doc_path(&self) -> Option<&'static str> { + Some("vortex-bench/sql/polarsignals.md") + } + fn queries(&self) -> Result> { let queries_file = workspace_root() .join("vortex-bench") diff --git a/vortex-bench/src/runner.rs b/vortex-bench/src/runner.rs index 1983c63ba05..26d0455b69a 100644 --- a/vortex-bench/src/runner.rs +++ b/vortex-bench/src/runner.rs @@ -75,6 +75,7 @@ pub struct SqlBenchmarkRunner { formats: Vec, memory_tracker: Option, hide_progress_bar: bool, + doc: Option<&'static str>, query_measurements: Vec, memory_measurements: Vec, } @@ -106,6 +107,7 @@ impl SqlBenchmarkRunner { formats, memory_tracker, hide_progress_bar, + doc: benchmark.doc_path(), query_measurements: Vec::new(), memory_measurements: Vec::new(), }) @@ -223,6 +225,7 @@ impl SqlBenchmarkRunner { display_format, self.engine, &self.formats, + self.doc, f, ) } @@ -232,6 +235,7 @@ impl SqlBenchmarkRunner { display_format, self.engine, &self.formats, + self.doc, std::io::stdout().lock(), ), } @@ -249,6 +253,7 @@ impl SqlBenchmarkRunner { display_format, self.engine, &self.formats, + self.doc, output, ) } @@ -457,6 +462,7 @@ pub fn export_results( display_format: &DisplayFormat, engine: Engine, formats: &[Format], + doc: Option<&str>, mut output: W, ) -> anyhow::Result<()> { let targets = formats @@ -467,13 +473,13 @@ pub fn export_results( if !memory.is_empty() { match display_format { DisplayFormat::Table => render_table(&mut output, memory, &targets)?, - DisplayFormat::GhJson => print_measurements_json(&mut output, memory)?, + DisplayFormat::GhJson => print_measurements_json(&mut output, memory, doc)?, }; } match display_format { DisplayFormat::Table => render_table(&mut output, queries, &targets)?, - DisplayFormat::GhJson => print_measurements_json(&mut output, queries)?, + DisplayFormat::GhJson => print_measurements_json(&mut output, queries, doc)?, }; Ok(()) diff --git a/vortex-bench/src/statpopgen/statpopgen_benchmark.rs b/vortex-bench/src/statpopgen/statpopgen_benchmark.rs index 0861f7bfc88..bf4092d0843 100644 --- a/vortex-bench/src/statpopgen/statpopgen_benchmark.rs +++ b/vortex-bench/src/statpopgen/statpopgen_benchmark.rs @@ -103,6 +103,10 @@ impl StatPopGenBenchmark { #[async_trait::async_trait] impl Benchmark for StatPopGenBenchmark { + fn doc_path(&self) -> Option<&'static str> { + Some("vortex-bench/sql/statpopgen.md") + } + fn queries(&self) -> Result> { let queries_file = workspace_root() .join("vortex-bench") diff --git a/vortex-bench/src/tpcds/tpcds_benchmark.rs b/vortex-bench/src/tpcds/tpcds_benchmark.rs index 7f8f2d5281e..90d924044ef 100644 --- a/vortex-bench/src/tpcds/tpcds_benchmark.rs +++ b/vortex-bench/src/tpcds/tpcds_benchmark.rs @@ -56,6 +56,10 @@ impl TpcDsBenchmark { #[async_trait::async_trait] impl Benchmark for TpcDsBenchmark { + fn doc_path(&self) -> Option<&'static str> { + Some("vortex-bench/sql/tpcds/README.md") + } + fn queries(&self) -> Result> { Ok(tpcds_queries().collect()) } diff --git a/vortex-bench/src/tpch/benchmark.rs b/vortex-bench/src/tpch/benchmark.rs index 6851e5f5f06..da62b953bc3 100644 --- a/vortex-bench/src/tpch/benchmark.rs +++ b/vortex-bench/src/tpch/benchmark.rs @@ -75,6 +75,10 @@ impl TpcHBenchmark { #[async_trait::async_trait] impl Benchmark for TpcHBenchmark { + fn doc_path(&self) -> Option<&'static str> { + Some("vortex-bench/sql/tpch/README.md") + } + fn queries(&self) -> anyhow::Result> { Ok(tpch_queries().collect()) } diff --git a/vortex-bench/src/vortex_queries.rs b/vortex-bench/src/vortex_queries.rs index f611238bb6b..cff7eea1050 100644 --- a/vortex-bench/src/vortex_queries.rs +++ b/vortex-bench/src/vortex_queries.rs @@ -89,6 +89,10 @@ impl VortexBenchmark { #[async_trait::async_trait] impl Benchmark for VortexBenchmark { + fn doc_path(&self) -> Option<&'static str> { + Some("vortex-bench/sql/vortex/README.md") + } + fn queries(&self) -> Result> { self.query_files()? .iter() From 42b58dc4c12b9c11ea391bface54e1721584e135 Mon Sep 17 00:00:00 2001 From: Joe Isaacs Date: Fri, 17 Jul 2026 13:42:19 +0000 Subject: [PATCH 3/3] Require Benchmark::doc_path and pin comment doc links to the PR commit doc_path is now a required trait method, so every suite must ship an explainer doc; spatialbench, gharchive, and public-bi gain docs to satisfy it, and the doc plumbing through the runner and JSONL export becomes non-optional. The comment title link now pins the results' commit SHA instead of develop, so it resolves before the PR merges and stays valid afterwards. Signed-off-by: Joe Isaacs --- benchmarks/compress-bench/src/main.rs | 4 ++-- benchmarks/random-access-bench/src/main.rs | 2 +- scripts/compare-benchmark-jsons.py | 7 +++++-- vortex-bench/sql/gharchive.md | 14 ++++++++++++++ vortex-bench/sql/public-bi.md | 15 +++++++++++++++ vortex-bench/sql/spatialbench.md | 15 +++++++++++++++ vortex-bench/src/appian/mod.rs | 4 ++-- vortex-bench/src/benchmark.rs | 6 ++---- vortex-bench/src/clickbench/benchmark.rs | 8 ++++---- vortex-bench/src/display.rs | 4 ++-- vortex-bench/src/fineweb/mod.rs | 4 ++-- vortex-bench/src/polarsignals/benchmark.rs | 4 ++-- vortex-bench/src/public_bi.rs | 4 ++++ vortex-bench/src/realnest/gharchive.rs | 4 ++++ vortex-bench/src/runner.rs | 4 ++-- vortex-bench/src/spatialbench/benchmark.rs | 4 ++++ .../src/statpopgen/statpopgen_benchmark.rs | 4 ++-- vortex-bench/src/tpcds/tpcds_benchmark.rs | 4 ++-- vortex-bench/src/tpch/benchmark.rs | 4 ++-- vortex-bench/src/vortex_queries.rs | 4 ++-- 20 files changed, 88 insertions(+), 31 deletions(-) create mode 100644 vortex-bench/sql/gharchive.md create mode 100644 vortex-bench/sql/public-bi.md create mode 100644 vortex-bench/sql/spatialbench.md diff --git a/benchmarks/compress-bench/src/main.rs b/benchmarks/compress-bench/src/main.rs index 353d7af7a0b..1cbf3f47c55 100644 --- a/benchmarks/compress-bench/src/main.rs +++ b/benchmarks/compress-bench/src/main.rs @@ -215,8 +215,8 @@ async fn run_compress( ) } DisplayFormat::GhJson => { - print_measurements_json(&mut writer, measurements.timings, Some(DOC_PATH))?; - print_measurements_json(&mut writer, measurements.ratios, Some(DOC_PATH)) + print_measurements_json(&mut writer, measurements.timings, DOC_PATH)?; + print_measurements_json(&mut writer, measurements.ratios, DOC_PATH) } } } diff --git a/benchmarks/random-access-bench/src/main.rs b/benchmarks/random-access-bench/src/main.rs index 9a387786321..1681a8548de 100644 --- a/benchmarks/random-access-bench/src/main.rs +++ b/benchmarks/random-access-bench/src/main.rs @@ -503,7 +503,7 @@ async fn run_random_access(config: RunConfig) -> Result<()> { } DisplayFormat::GhJson => { let timings: Vec = runs.into_iter().map(|r| r.timing).collect(); - print_measurements_json(&mut writer, timings, Some(DOC_PATH))?; + print_measurements_json(&mut writer, timings, DOC_PATH)?; } } diff --git a/scripts/compare-benchmark-jsons.py b/scripts/compare-benchmark-jsons.py index 116247c7b66..d7cd34793db 100644 --- a/scripts/compare-benchmark-jsons.py +++ b/scripts/compare-benchmark-jsons.py @@ -769,7 +769,8 @@ def format_title(benchmark_name: str, pr: pd.DataFrame) -> str: The doc path is a repo-relative markdown path carried on the PR result rows (the `doc` field, populated from `Benchmark::doc_path` in Rust), so the benchmark code is the single - source of truth for where each suite is documented. + source of truth for where each suite is documented. The link pins the PR's own commit so + it resolves before the PR merges and stays valid afterwards. """ title = f"# Benchmarks: {benchmark_name}" if benchmark_name else "# Benchmarks" @@ -778,7 +779,9 @@ def format_title(benchmark_name: str, pr: pd.DataFrame) -> str: if len(docs) > 0: server_url = os.environ.get("GITHUB_SERVER_URL", "https://github.com") repository = os.environ.get("GITHUB_REPOSITORY", "vortex-data/vortex") - title += f" [\N{OPEN BOOK}]({server_url}/{repository}/blob/develop/{docs[0]})" + commits = pr["commit_id"].dropna().unique() if "commit_id" in pr.columns else [] + ref = commits[0] if len(commits) > 0 else "develop" + title += f" [\N{OPEN BOOK}]({server_url}/{repository}/blob/{ref}/{docs[0]})" return title diff --git a/vortex-bench/sql/gharchive.md b/vortex-bench/sql/gharchive.md new file mode 100644 index 00000000000..16cebd2d1a9 --- /dev/null +++ b/vortex-bench/sql/gharchive.md @@ -0,0 +1,14 @@ +# GitHub Archive benchmark + +A deeply nested real-world dataset from the [GitHub Archive](https://www.gharchive.org/): +GitHub event records whose `payload`, `repo`, `actor`, and `org` fields are nested structs. +The queries in [`gharchive.sql`](./gharchive.sql) (numbered from Q0 in file order) filter +and aggregate on those nested fields, making this the main struct-field-pushdown workload. + +The harness lives in [`src/realnest/gharchive.rs`](../src/realnest/gharchive.rs). + +## Running locally + +```bash +cargo run --bin datafusion-bench --profile release_debug -- --benchmark gharchive +``` diff --git a/vortex-bench/sql/public-bi.md b/vortex-bench/sql/public-bi.md new file mode 100644 index 00000000000..60859677741 --- /dev/null +++ b/vortex-bench/sql/public-bi.md @@ -0,0 +1,15 @@ +# Public BI benchmark + +Real-world BI datasets from the +[Public BI benchmark](https://github.com/cwida/public_bi_benchmark) โ€” anonymized Tableau +workbooks with messy, wide, string-heavy tables. A subset of the datasets (Arade, Bimbo, +CMSprovider, Euro2016, Food, HashTags) also feeds the +[Compression benchmark](../../benchmarks/compress-bench/README.md). + +The datasets, schemas, and queries are defined in [`src/public_bi.rs`](../src/public_bi.rs). + +## Running locally + +```bash +vx-bench run public-bi +``` diff --git a/vortex-bench/sql/spatialbench.md b/vortex-bench/sql/spatialbench.md new file mode 100644 index 00000000000..f959e006887 --- /dev/null +++ b/vortex-bench/sql/spatialbench.md @@ -0,0 +1,15 @@ +# SpatialBench benchmark + +The [Apache Sedona SpatialBench](https://sedona.apache.org/spatialbench/) geospatial +analytics benchmark: twelve queries (Q1 ... Q12 in [`spatialbench.sql`](./spatialbench.sql), +DuckDB dialect) over a trips/zones schema, exercising spatial predicates and functions such +as `ST_DWithin`, `ST_Intersects`, and `ST_Distance`. The query logic matches upstream +`sedona-spatialbench`; only formatting differs. + +The harness lives in [`src/spatialbench`](../src/spatialbench). + +## Running locally + +```bash +vx-bench run spatialbench +``` diff --git a/vortex-bench/src/appian/mod.rs b/vortex-bench/src/appian/mod.rs index fe42d2abe30..20dabb5676a 100644 --- a/vortex-bench/src/appian/mod.rs +++ b/vortex-bench/src/appian/mod.rs @@ -121,8 +121,8 @@ impl AppianBenchmark { #[async_trait::async_trait] impl Benchmark for AppianBenchmark { - fn doc_path(&self) -> Option<&'static str> { - Some("vortex-bench/sql/appian/README.md") + fn doc_path(&self) -> &'static str { + "vortex-bench/sql/appian/README.md" } fn queries(&self) -> anyhow::Result> { diff --git a/vortex-bench/src/benchmark.rs b/vortex-bench/src/benchmark.rs index 488080af248..b7015d242a7 100644 --- a/vortex-bench/src/benchmark.rs +++ b/vortex-bench/src/benchmark.rs @@ -65,10 +65,8 @@ pub trait Benchmark: Send + Sync { fn dataset(&self) -> BenchmarkDataset; /// Repo-relative path of the markdown explainer for this benchmark suite, linked from the - /// title of CI benchmark PR comments. `None` for suites without a doc. - fn doc_path(&self) -> Option<&'static str> { - None - } + /// title of CI benchmark PR comments. Required: every suite must ship a doc. + fn doc_path(&self) -> &'static str; /// Get the name of the benchmark dataset fn dataset_name(&self) -> &str; diff --git a/vortex-bench/src/clickbench/benchmark.rs b/vortex-bench/src/clickbench/benchmark.rs index 2a254d1976c..fcba1b9f9c1 100644 --- a/vortex-bench/src/clickbench/benchmark.rs +++ b/vortex-bench/src/clickbench/benchmark.rs @@ -76,8 +76,8 @@ fn read_clickbench_queries(queries_file: Option<&str>) -> Result Option<&'static str> { - Some("vortex-bench/sql/clickbench.md") + fn doc_path(&self) -> &'static str { + "vortex-bench/sql/clickbench.md" } fn queries(&self) -> Result> { @@ -124,8 +124,8 @@ impl Benchmark for ClickBenchBenchmark { #[async_trait::async_trait] impl Benchmark for ClickBenchSortedBenchmark { - fn doc_path(&self) -> Option<&'static str> { - Some("vortex-bench/sql/clickbench.md#sorted-variant") + fn doc_path(&self) -> &'static str { + "vortex-bench/sql/clickbench.md#sorted-variant" } fn queries(&self) -> Result> { diff --git a/vortex-bench/src/display.rs b/vortex-bench/src/display.rs index 29aebddc40b..343c1a06612 100644 --- a/vortex-bench/src/display.rs +++ b/vortex-bench/src/display.rs @@ -105,11 +105,11 @@ pub fn render_table( pub fn print_measurements_json( writer: &mut dyn Write, all_measurements: Vec, - doc: Option<&str>, + doc: &str, ) -> anyhow::Result<()> { for measurement in all_measurements { let mut json = measurement.to_json(); - if let (Some(doc), Some(obj)) = (doc, json.as_object_mut()) { + if let Some(obj) = json.as_object_mut() { obj.insert("doc".to_string(), doc.into()); } writeln!(writer, "{json}")?; diff --git a/vortex-bench/src/fineweb/mod.rs b/vortex-bench/src/fineweb/mod.rs index f7dade2de01..aefa796af3a 100644 --- a/vortex-bench/src/fineweb/mod.rs +++ b/vortex-bench/src/fineweb/mod.rs @@ -53,8 +53,8 @@ impl FinewebBenchmark { #[async_trait::async_trait] impl Benchmark for FinewebBenchmark { - fn doc_path(&self) -> Option<&'static str> { - Some("vortex-bench/sql/fineweb.md") + fn doc_path(&self) -> &'static str { + "vortex-bench/sql/fineweb.md" } /// Some basic string-focused queries, numbered from Q0 in `sql/fineweb.sql` file order. diff --git a/vortex-bench/src/polarsignals/benchmark.rs b/vortex-bench/src/polarsignals/benchmark.rs index 2f00181ba00..d1097e83bd1 100644 --- a/vortex-bench/src/polarsignals/benchmark.rs +++ b/vortex-bench/src/polarsignals/benchmark.rs @@ -59,8 +59,8 @@ impl PolarSignalsBenchmark { #[async_trait::async_trait] impl Benchmark for PolarSignalsBenchmark { - fn doc_path(&self) -> Option<&'static str> { - Some("vortex-bench/sql/polarsignals.md") + fn doc_path(&self) -> &'static str { + "vortex-bench/sql/polarsignals.md" } fn queries(&self) -> Result> { diff --git a/vortex-bench/src/public_bi.rs b/vortex-bench/src/public_bi.rs index f5c22bdea8f..3e651df5bb7 100644 --- a/vortex-bench/src/public_bi.rs +++ b/vortex-bench/src/public_bi.rs @@ -530,6 +530,10 @@ impl PublicBiBenchmark { #[async_trait] impl Benchmark for PublicBiBenchmark { + fn doc_path(&self) -> &'static str { + "vortex-bench/sql/public-bi.md" + } + fn queries(&self) -> anyhow::Result> { self.pbi_benchmark().queries() } diff --git a/vortex-bench/src/realnest/gharchive.rs b/vortex-bench/src/realnest/gharchive.rs index a84395f4a10..6596598d5ba 100644 --- a/vortex-bench/src/realnest/gharchive.rs +++ b/vortex-bench/src/realnest/gharchive.rs @@ -66,6 +66,10 @@ impl GithubArchiveBenchmark { #[async_trait::async_trait] impl Benchmark for GithubArchiveBenchmark { + fn doc_path(&self) -> &'static str { + "vortex-bench/sql/gharchive.md" + } + /// GitHub Archive queries, numbered from Q0 in `sql/gharchive.sql` file order. fn queries(&self) -> anyhow::Result> { // `;`-separated; a `;` must not appear in a comment, or it would split a statement in two. diff --git a/vortex-bench/src/runner.rs b/vortex-bench/src/runner.rs index 26d0455b69a..9e5c4db3d08 100644 --- a/vortex-bench/src/runner.rs +++ b/vortex-bench/src/runner.rs @@ -75,7 +75,7 @@ pub struct SqlBenchmarkRunner { formats: Vec, memory_tracker: Option, hide_progress_bar: bool, - doc: Option<&'static str>, + doc: &'static str, query_measurements: Vec, memory_measurements: Vec, } @@ -462,7 +462,7 @@ pub fn export_results( display_format: &DisplayFormat, engine: Engine, formats: &[Format], - doc: Option<&str>, + doc: &str, mut output: W, ) -> anyhow::Result<()> { let targets = formats diff --git a/vortex-bench/src/spatialbench/benchmark.rs b/vortex-bench/src/spatialbench/benchmark.rs index 48fa84f8ec1..5df7671bd1f 100644 --- a/vortex-bench/src/spatialbench/benchmark.rs +++ b/vortex-bench/src/spatialbench/benchmark.rs @@ -58,6 +58,10 @@ impl SpatialBenchBenchmark { #[async_trait::async_trait] impl Benchmark for SpatialBenchBenchmark { + fn doc_path(&self) -> &'static str { + "vortex-bench/sql/spatialbench.md" + } + /// All SpatialBench queries, numbered started at Q1 in `spatialbench.sql` file order. fn queries(&self) -> anyhow::Result> { // `;`-separated; a `;` must not appear in a comment, or it would split a statement in two. diff --git a/vortex-bench/src/statpopgen/statpopgen_benchmark.rs b/vortex-bench/src/statpopgen/statpopgen_benchmark.rs index bf4092d0843..2c677fb79a3 100644 --- a/vortex-bench/src/statpopgen/statpopgen_benchmark.rs +++ b/vortex-bench/src/statpopgen/statpopgen_benchmark.rs @@ -103,8 +103,8 @@ impl StatPopGenBenchmark { #[async_trait::async_trait] impl Benchmark for StatPopGenBenchmark { - fn doc_path(&self) -> Option<&'static str> { - Some("vortex-bench/sql/statpopgen.md") + fn doc_path(&self) -> &'static str { + "vortex-bench/sql/statpopgen.md" } fn queries(&self) -> Result> { diff --git a/vortex-bench/src/tpcds/tpcds_benchmark.rs b/vortex-bench/src/tpcds/tpcds_benchmark.rs index 90d924044ef..5bacae73516 100644 --- a/vortex-bench/src/tpcds/tpcds_benchmark.rs +++ b/vortex-bench/src/tpcds/tpcds_benchmark.rs @@ -56,8 +56,8 @@ impl TpcDsBenchmark { #[async_trait::async_trait] impl Benchmark for TpcDsBenchmark { - fn doc_path(&self) -> Option<&'static str> { - Some("vortex-bench/sql/tpcds/README.md") + fn doc_path(&self) -> &'static str { + "vortex-bench/sql/tpcds/README.md" } fn queries(&self) -> Result> { diff --git a/vortex-bench/src/tpch/benchmark.rs b/vortex-bench/src/tpch/benchmark.rs index da62b953bc3..4dfdaf1207d 100644 --- a/vortex-bench/src/tpch/benchmark.rs +++ b/vortex-bench/src/tpch/benchmark.rs @@ -75,8 +75,8 @@ impl TpcHBenchmark { #[async_trait::async_trait] impl Benchmark for TpcHBenchmark { - fn doc_path(&self) -> Option<&'static str> { - Some("vortex-bench/sql/tpch/README.md") + fn doc_path(&self) -> &'static str { + "vortex-bench/sql/tpch/README.md" } fn queries(&self) -> anyhow::Result> { diff --git a/vortex-bench/src/vortex_queries.rs b/vortex-bench/src/vortex_queries.rs index cff7eea1050..63e9a658068 100644 --- a/vortex-bench/src/vortex_queries.rs +++ b/vortex-bench/src/vortex_queries.rs @@ -89,8 +89,8 @@ impl VortexBenchmark { #[async_trait::async_trait] impl Benchmark for VortexBenchmark { - fn doc_path(&self) -> Option<&'static str> { - Some("vortex-bench/sql/vortex/README.md") + fn doc_path(&self) -> &'static str { + "vortex-bench/sql/vortex/README.md" } fn queries(&self) -> Result> {