From a0a5b393f74f9d19d3db9e928b233f08f05b2021 Mon Sep 17 00:00:00 2001
From: Connor Tsui <connor.tsui20@gmail.com>
Date: Mon, 27 Jul 2026 12:40:19 -0400
Subject: [PATCH 1/5] ci: add focused PR benchmark labels

Signed-off-by: Connor Tsui <connor.tsui20@gmail.com>
---
 .github/workflows/commit-metadata.yml         |  4 +-
 .../{bench.yml => develop-bench.yml}          |  4 +-
 .github/workflows/nightly-bench.yml           |  2 +-
 .github/workflows/pr-bench-compress.yml       | 20 +++++
 ...nch-dispatch.yml => pr-bench-dispatch.yml} | 62 +++++++++------
 ...bench-pr.yml => pr-bench-gpu-compress.yml} | 10 +--
 .github/workflows/pr-bench-random-access.yml  | 21 +++++
 .../{bench-pr.yml => pr-bench-runner.yml}     | 76 ++++++++-----------
 .../{sql-pr.yml => pr-bench-sql.yml}          |  5 +-
 ...ql-benchmarks.yml => sql-bench-matrix.yml} | 12 +--
 .../bench_orchestrator/ci_matrix/catalog.py   | 16 ++++
 bench-orchestrator/tests/test_matrix.py       |  9 +++
 docs/developer-guide/benchmarking.md          | 15 ++--
 13 files changed, 166 insertions(+), 90 deletions(-)
 rename .github/workflows/{bench.yml => develop-bench.yml} (99%)
 create mode 100644 .github/workflows/pr-bench-compress.yml
 rename .github/workflows/{bench-dispatch.yml => pr-bench-dispatch.yml} (54%)
 rename .github/workflows/{gpu-compress-bench-pr.yml => pr-bench-gpu-compress.yml} (89%)
 create mode 100644 .github/workflows/pr-bench-random-access.yml
 rename .github/workflows/{bench-pr.yml => pr-bench-runner.yml} (70%)
 rename .github/workflows/{sql-pr.yml => pr-bench-sql.yml} (84%)
 rename .github/workflows/{sql-benchmarks.yml => sql-bench-matrix.yml} (96%)

diff --git a/.github/workflows/commit-metadata.yml b/.github/workflows/commit-metadata.yml
index c30b2194cd1..5c136028984 100644
--- a/.github/workflows/commit-metadata.yml
+++ b/.github/workflows/commit-metadata.yml
@@ -22,13 +22,13 @@ jobs:
         with:
           fetch-depth: 2
 
-      # v4 (Postgres) ingest -- REQUIRED (see bench.yml rationale). Empty records:
+      # v4 (Postgres) ingest -- REQUIRED (see develop-bench.yml rationale). Empty records:
       # post-ingest.py --postgres upserts the commit row only. Gated on the
       # ingest-role ARN var (the assume-role input that MUST exist for OIDC to
       # succeed).
       #
       # `sync: false` -- the ingest runs `uv run --no-project --with`, which needs only
-      # the uv binary, never the synced workspace (see bench.yml rationale).
+      # the uv binary, never the synced workspace (see develop-bench.yml rationale).
       - name: Install uv for v4 ingest
         if: vars.GH_BENCH_INGEST_ROLE_ARN != ''
         uses: spiraldb/actions/.github/actions/setup-uv@a746510eafaa926484c354541cfc49b2ec06cc63  # 0.18.6
diff --git a/.github/workflows/bench.yml b/.github/workflows/develop-bench.yml
similarity index 99%
rename from .github/workflows/bench.yml
rename to .github/workflows/develop-bench.yml
index a046acc8ff4..103583bb2aa 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/develop-bench.yml
@@ -1,6 +1,6 @@
 # Runs after every commit to `develop` (or in other words, _after_ every pull request merges).
 
-name: Benchmarks
+name: Develop Benchmarks
 
 on:
   push:
@@ -188,7 +188,7 @@ jobs:
           deduplication-key: ci-bench-${{ matrix.benchmark.id }}-failure
 
   sql:
-    uses: ./.github/workflows/sql-benchmarks.yml
+    uses: ./.github/workflows/sql-bench-matrix.yml
     secrets: inherit
     with:
       mode: "develop"
diff --git a/.github/workflows/nightly-bench.yml b/.github/workflows/nightly-bench.yml
index 538d567091e..cc90e07316f 100644
--- a/.github/workflows/nightly-bench.yml
+++ b/.github/workflows/nightly-bench.yml
@@ -18,7 +18,7 @@ permissions:
 
 jobs:
   sql:
-    uses: ./.github/workflows/sql-benchmarks.yml
+    uses: ./.github/workflows/sql-bench-matrix.yml
     secrets: inherit
     with:
       mode: "develop"
diff --git a/.github/workflows/pr-bench-compress.yml b/.github/workflows/pr-bench-compress.yml
new file mode 100644
index 00000000000..6d7abd7e86f
--- /dev/null
+++ b/.github/workflows/pr-bench-compress.yml
@@ -0,0 +1,20 @@
+# Runs the compression benchmark for a pull request.
+
+name: PR Compression Benchmark
+
+on:
+  workflow_call: { }
+  workflow_dispatch: { }
+
+permissions:
+  contents: read
+  pull-requests: write  # for commenting on PRs
+  id-token: write  # enables AWS-GitHub OIDC
+
+jobs:
+  bench:
+    uses: ./.github/workflows/pr-bench-runner.yml
+    secrets: inherit
+    with:
+      benchmark_id: compress-bench
+      benchmark_name: Compression
diff --git a/.github/workflows/bench-dispatch.yml b/.github/workflows/pr-bench-dispatch.yml
similarity index 54%
rename from .github/workflows/bench-dispatch.yml
rename to .github/workflows/pr-bench-dispatch.yml
index 92b3fafb96f..4e79c6fc5f5 100644
--- a/.github/workflows/bench-dispatch.yml
+++ b/.github/workflows/pr-bench-dispatch.yml
@@ -2,7 +2,7 @@
 # This is a separate workflow so that non-benchmark label events don't create
 # phantom check suites that obscure in-progress benchmark runs on the PR.
 
-name: Benchmark Dispatch
+name: PR Benchmark Dispatch
 
 on:
   pull_request:
@@ -16,68 +16,86 @@ permissions:
   id-token: write  # enables AWS-GitHub OIDC
 
 jobs:
-  remove-bench-label:
+  remove-random-access-label:
     runs-on: ubuntu-latest
     timeout-minutes: 10
-    if: github.event.label.name == 'action/benchmark'
+    if: github.event.label.name == 'action/bench-random-access'
     steps:
       - uses: actions-ecosystem/action-remove-labels@2ce5d41b4b6aa8503e285553f75ed56e0a40bae0  # v1
         if: github.event.pull_request.head.repo.full_name == 'vortex-data/vortex'
         with:
-          labels: action/benchmark
+          labels: action/bench-random-access
           fail_on_error: true
 
-  bench:
-    needs: remove-bench-label
-    uses: ./.github/workflows/bench-pr.yml
+  random-access-bench:
+    needs: remove-random-access-label
+    uses: ./.github/workflows/pr-bench-random-access.yml
     secrets: inherit
 
-  remove-gpu-compress-bench-label:
+  remove-compress-label:
     runs-on: ubuntu-latest
     timeout-minutes: 10
-    if: github.event.label.name == 'action/benchmark-gpu-compress'
+    if: github.event.label.name == 'action/bench-compress'
     steps:
       - uses: actions-ecosystem/action-remove-labels@2ce5d41b4b6aa8503e285553f75ed56e0a40bae0  # v1
         if: github.event.pull_request.head.repo.full_name == 'vortex-data/vortex'
         with:
-          labels: action/benchmark-gpu-compress
+          labels: action/bench-compress
+          fail_on_error: true
+
+  compression-bench:
+    needs: remove-compress-label
+    uses: ./.github/workflows/pr-bench-compress.yml
+    secrets: inherit
+
+  remove-gpu-compress-label:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    if: github.event.label.name == 'action/bench-gpu-compress'
+    steps:
+      - uses: actions-ecosystem/action-remove-labels@2ce5d41b4b6aa8503e285553f75ed56e0a40bae0  # v1
+        if: github.event.pull_request.head.repo.full_name == 'vortex-data/vortex'
+        with:
+          labels: action/bench-gpu-compress
           fail_on_error: true
 
   gpu-compress-bench:
-    needs: remove-gpu-compress-bench-label
-    uses: ./.github/workflows/gpu-compress-bench-pr.yml
+    needs: remove-gpu-compress-label
+    uses: ./.github/workflows/pr-bench-gpu-compress.yml
     secrets: inherit
 
   remove-sql-label:
     runs-on: ubuntu-latest
     timeout-minutes: 10
-    if: github.event.label.name == 'action/benchmark-sql'
+    if: github.event.label.name == 'action/bench-sql'
     steps:
       - uses: actions-ecosystem/action-remove-labels@2ce5d41b4b6aa8503e285553f75ed56e0a40bae0  # v1
         if: github.event.pull_request.head.repo.full_name == 'vortex-data/vortex'
         with:
-          labels: action/benchmark-sql
+          labels: action/bench-sql
           fail_on_error: true
 
   sql-bench:
     needs: remove-sql-label
-    uses: ./.github/workflows/sql-pr.yml
+    uses: ./.github/workflows/pr-bench-sql.yml
     secrets: inherit
+    with:
+      matrix_preset: "pr"
 
-  remove-sql-full-label:
+  remove-sql-compact-label:
     runs-on: ubuntu-latest
     timeout-minutes: 10
-    if: github.event.label.name == 'action/benchmark-sql-full'
+    if: github.event.label.name == 'action/bench-sql-compact'
     steps:
       - uses: actions-ecosystem/action-remove-labels@2ce5d41b4b6aa8503e285553f75ed56e0a40bae0  # v1
         if: github.event.pull_request.head.repo.full_name == 'vortex-data/vortex'
         with:
-          labels: action/benchmark-sql-full
+          labels: action/bench-sql-compact
           fail_on_error: true
 
-  sql-full-bench:
-    needs: remove-sql-full-label
-    uses: ./.github/workflows/sql-pr.yml
+  sql-compact-bench:
+    needs: remove-sql-compact-label
+    uses: ./.github/workflows/pr-bench-sql.yml
     secrets: inherit
     with:
-      matrix_preset: "pr-full"
+      matrix_preset: "pr-compact"
diff --git a/.github/workflows/gpu-compress-bench-pr.yml b/.github/workflows/pr-bench-gpu-compress.yml
similarity index 89%
rename from .github/workflows/gpu-compress-bench-pr.yml
rename to .github/workflows/pr-bench-gpu-compress.yml
index 3abb5b54533..757cd1aebd5 100644
--- a/.github/workflows/gpu-compress-bench-pr.yml
+++ b/.github/workflows/pr-bench-gpu-compress.yml
@@ -1,10 +1,10 @@
 # Runs the GPU-enabled compression decompression benchmarks for a pull request.
-# Called from bench-dispatch.yml when the `action/benchmark-gpu-compress` label is added.
+# Called from pr-bench-dispatch.yml when the `action/bench-gpu-compress` label is added.
 
-name: GPU Compression Benchmarks
+name: PR GPU Compression Benchmark
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}-gpu-compress
   cancel-in-progress: false
 
 on:
@@ -71,13 +71,13 @@ jobs:
           } > comment.md
           cat comment.md >> "$GITHUB_STEP_SUMMARY"
       - name: Comment PR
-        if: github.event.pull_request.head.repo.fork == false
+        if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false
         uses: thollander/actions-comment-pull-request@24bffb9b452ba05a4f3f77933840a6a841d1b32b  # v3
         with:
           file-path: comment.md
           comment-tag: bench-pr-comment-gpu-compress
       - name: Comment PR on failure
-        if: failure() && github.event.pull_request.head.repo.fork == false
+        if: failure() && github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false
         uses: thollander/actions-comment-pull-request@24bffb9b452ba05a4f3f77933840a6a841d1b32b  # v3
         with:
           message: |
diff --git a/.github/workflows/pr-bench-random-access.yml b/.github/workflows/pr-bench-random-access.yml
new file mode 100644
index 00000000000..112d4d8687d
--- /dev/null
+++ b/.github/workflows/pr-bench-random-access.yml
@@ -0,0 +1,21 @@
+# Runs the random-access benchmark for a pull request.
+
+name: PR Random Access Benchmark
+
+on:
+  workflow_call: { }
+  workflow_dispatch: { }
+
+permissions:
+  contents: read
+  pull-requests: write  # for commenting on PRs
+  id-token: write  # enables AWS-GitHub OIDC
+
+jobs:
+  bench:
+    uses: ./.github/workflows/pr-bench-runner.yml
+    secrets: inherit
+    with:
+      benchmark_id: random-access-bench
+      benchmark_name: Random Access
+      with_lance: true
diff --git a/.github/workflows/bench-pr.yml b/.github/workflows/pr-bench-runner.yml
similarity index 70%
rename from .github/workflows/bench-pr.yml
rename to .github/workflows/pr-bench-runner.yml
index 2a12c0a60b2..b1b5d8c8799 100644
--- a/.github/workflows/bench-pr.yml
+++ b/.github/workflows/pr-bench-runner.yml
@@ -1,17 +1,26 @@
-# Runs all benchmarks once for a pull request.
-# Called from bench-dispatch.yml when the `action/benchmark` label is added.
+# Runs one non-SQL benchmark for a pull request.
 
-name: PR Benchmarks
+name: PR Benchmark Runner
 
 concurrency:
   # The group causes runs to queue instead of running in parallel.
-  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}-${{ inputs.benchmark_id }}
   # Don't cancel benchmarks that are already running, instead just queue them up.
   cancel-in-progress: false
 
 on:
-  workflow_call: { }
-  workflow_dispatch: { }
+  workflow_call:
+    inputs:
+      benchmark_id:
+        required: true
+        type: string
+      benchmark_name:
+        required: true
+        type: string
+      with_lance:
+        required: false
+        type: boolean
+        default: false
 
 permissions:
   contents: read
@@ -23,20 +32,8 @@ jobs:
     timeout-minutes: 120
     runs-on: >-
       ${{ github.repository == 'vortex-data/vortex'
-          && format('runs-on={0}/runner=bench-dedicated/family=c6id.metal/tag={1}{2}', github.run_id, matrix.benchmark.id, github.event.pull_request.head.repo.fork == false && '/extras=s3-cache' || '')
+          && format('runs-on={0}/runner=bench-dedicated/family=c6id.metal/tag={1}{2}', github.run_id, inputs.benchmark_id, github.event.pull_request.head.repo.fork == false && '/extras=s3-cache' || '')
           || 'ubuntu-latest' }}
-    strategy:
-      matrix:
-        benchmark:
-          - id: random-access-bench
-            name: Random Access
-            build_args: "--features lance"
-          - id: compress-bench
-            name: Compression
-          # No run_args: the default suite emits the three tracked metrics
-          # (size, write, read). The codec microbenchmark is a local diagnostic.
-          - id: string-bench
-            name: String Encoding
     steps:
       - uses: runs-on/action@v2
         if: github.event.pull_request.head.repo.fork == false
@@ -44,7 +41,7 @@ jobs:
           sccache: s3
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7
         with:
-          ref: ${{ github.event.pull_request.head.sha }}
+          ref: ${{ github.event.pull_request.head.sha || github.sha }}
       - uses: ./.github/actions/setup-rust
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
@@ -63,7 +60,8 @@ jobs:
         env:
           RUSTFLAGS: "-C target-cpu=native -C force-frame-pointers=yes"
         run: |
-          cargo build --package ${{ matrix.benchmark.id }} --profile release_debug ${{ matrix.benchmark.build_args }} --features unstable_encodings
+          cargo build --package ${{ inputs.benchmark_id }} --profile release_debug \
+            --features ${{ inputs.with_lance && 'lance,' || '' }}unstable_encodings
 
       - name: Pre-upload benchmark debuginfo to Polar Signals
         if: github.event.pull_request.head.repo.fork == false
@@ -71,7 +69,7 @@ jobs:
         continue-on-error: true
         with:
           paths: |
-            target/release_debug/${{ matrix.benchmark.id }}
+            target/release_debug/${{ inputs.benchmark_id }}
           polarsignals-cloud-token: ${{ secrets.POLAR_SIGNALS_API_KEY }}
           project-id: "e5d846e1-b54c-46e7-9174-8bf055a3af56"
 
@@ -80,9 +78,9 @@ jobs:
         uses: polarsignals/gh-actions-ps-profiling@68ae857e375a826606352016e5b90f01a2a7ff7a  # v0.8.1
         with:
           polarsignals_cloud_token: ${{ secrets.POLAR_SIGNALS_API_KEY }}
-          labels: "branch=${{ github.ref_name }};gh_run_id=${{ github.run_id }};commit_sha=${{ github.event.pull_request.head.sha }};benchmark=${{ matrix.benchmark.id }}"
+          labels: "branch=${{ github.ref_name }};gh_run_id=${{ github.run_id }};commit_sha=${{ github.event.pull_request.head.sha }};benchmark=${{ inputs.benchmark_id }}"
           project_uuid: "e5d846e1-b54c-46e7-9174-8bf055a3af56"
-          job_name: "${{ matrix.benchmark.id }}"
+          job_name: "${{ inputs.benchmark_id }}"
           profiling_frequency: 199
           extra_args: "--debuginfo-strip=false"
           parca_agent_version: "0.49.0"
@@ -90,8 +88,8 @@ jobs:
       - name: Setup benchmark environment
         run: sudo bash scripts/setup-benchmark.sh
 
-      - name: Run ${{ matrix.benchmark.name }} benchmark (per-combination)
-        if: matrix.benchmark.id == 'random-access-bench'
+      - name: Run ${{ inputs.benchmark_name }} benchmark (per-combination)
+        if: inputs.benchmark_id == 'random-access-bench'
         shell: bash
         env:
           RUST_BACKTRACE: full
@@ -100,16 +98,15 @@ jobs:
         run: |
           python3 scripts/random-access-split.py
 
-      - name: Run ${{ matrix.benchmark.name }} benchmark
-        if: matrix.benchmark.id != 'random-access-bench'
+      - name: Run ${{ inputs.benchmark_name }} benchmark
+        if: inputs.benchmark_id != 'random-access-bench'
         shell: bash
         env:
           RUST_BACKTRACE: full
           VORTEX_EXPERIMENTAL_PATCHED_ARRAY: "1"
           FLAT_LAYOUT_INLINE_ARRAY_NODE: "1"
         run: |
-          bash scripts/bench-taskset.sh target/release_debug/${{ matrix.benchmark.id }} \
-            ${{ matrix.benchmark.run_args }} -d gh-json -o results.json
+          bash scripts/bench-taskset.sh target/release_debug/${{ inputs.benchmark_id }} -d gh-json -o results.json
 
       - name: Setup AWS CLI
         if: github.event.pull_request.head.repo.fork == false
@@ -131,30 +128,23 @@ jobs:
           python3 scripts/s3-download.py s3://vortex-ci-benchmark-results/data.json.gz data.json.gz --no-sign-request
           gzip -d -c data.json.gz > base.json
 
-          uv run --no-project scripts/compare-benchmark-jsons.py base.json results.json "${{ matrix.benchmark.name }}" \
+          uv run --no-project scripts/compare-benchmark-jsons.py base.json results.json "${{ inputs.benchmark_name }}" \
             > comment.md
           cat comment.md >> "$GITHUB_STEP_SUMMARY"
 
       - name: Comment PR
-        if: github.event.pull_request.head.repo.fork == false
+        if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false
         uses: thollander/actions-comment-pull-request@24bffb9b452ba05a4f3f77933840a6a841d1b32b  # v3
         with:
           file-path: comment.md
-          comment-tag: bench-pr-comment-${{ matrix.benchmark.id }}
+          comment-tag: bench-pr-comment-${{ inputs.benchmark_id }}
 
       - name: Comment PR on failure
-        if: failure() && github.event.pull_request.head.repo.fork == false
+        if: failure() && github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false
         uses: thollander/actions-comment-pull-request@24bffb9b452ba05a4f3f77933840a6a841d1b32b  # v3
         with:
           message: |
             # BENCHMARK FAILED
 
-            Benchmark `${{ matrix.benchmark.name }}` failed! Check the [workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details.
-          comment-tag: bench-pr-comment-${{ matrix.benchmark.id }}
-
-  sql:
-    uses: ./.github/workflows/sql-benchmarks.yml
-    secrets: inherit
-    with:
-      mode: "pr"
-      matrix_preset: "pr-full"
+            Benchmark `${{ inputs.benchmark_name }}` failed! Check the [workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details.
+          comment-tag: bench-pr-comment-${{ inputs.benchmark_id }}
diff --git a/.github/workflows/sql-pr.yml b/.github/workflows/pr-bench-sql.yml
similarity index 84%
rename from .github/workflows/sql-pr.yml
rename to .github/workflows/pr-bench-sql.yml
index b14016030c0..63f1bd8f7f2 100644
--- a/.github/workflows/sql-pr.yml
+++ b/.github/workflows/pr-bench-sql.yml
@@ -1,5 +1,5 @@
 # Runs SQL benchmarks once for a pull request.
-# Called from bench-dispatch.yml when SQL benchmark labels are added.
+# Called from pr-bench-dispatch.yml when SQL benchmark labels are added.
 
 name: PR SQL Benchmarks
 
@@ -26,6 +26,7 @@ on:
         default: "pr"
         options:
           - "pr"
+          - "pr-compact"
           - "pr-full"
 
 permissions:
@@ -35,7 +36,7 @@ permissions:
 
 jobs:
   sql:
-    uses: ./.github/workflows/sql-benchmarks.yml
+    uses: ./.github/workflows/sql-bench-matrix.yml
     secrets: inherit
     with:
       mode: "pr"
diff --git a/.github/workflows/sql-benchmarks.yml b/.github/workflows/sql-bench-matrix.yml
similarity index 96%
rename from .github/workflows/sql-benchmarks.yml
rename to .github/workflows/sql-bench-matrix.yml
index 4f2297a0e8f..bd76a46b20f 100644
--- a/.github/workflows/sql-benchmarks.yml
+++ b/.github/workflows/sql-bench-matrix.yml
@@ -1,4 +1,4 @@
-name: "SQL-related benchmarks"
+name: SQL Benchmark Matrix
 
 on:
   workflow_call:
@@ -259,8 +259,10 @@ jobs:
         with:
           file-path: comment.md
           # There is exactly one comment per comment-tag. If a comment with this tag already exists,
-          # this action will *update* the comment instead of posting a new comment.
-          comment-tag: bench-pr-comment-${{ matrix.id }}
+          # this action will *update* the comment instead of posting a new comment. The preset is
+          # part of the tag because presets overlap on benchmark ids (`pr` and `pr-compact` share
+          # nine) and can run concurrently, so a preset-less tag would let them clobber each other.
+          comment-tag: bench-pr-comment-${{ matrix.id }}-${{ inputs.matrix_preset }}
 
       - name: Comment PR on failure
         if: failure() && inputs.mode == 'pr' && github.event.pull_request.head.repo.fork == false
@@ -270,7 +272,7 @@ jobs:
             # 🚨🚨🚨❌❌❌ SQL BENCHMARK FAILED ❌❌❌🚨🚨🚨
 
             Benchmark `${{ matrix.name }}` (${{ inputs.matrix_preset }}) failed! Check the [workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details.
-          comment-tag: bench-pr-comment-${{ matrix.id }}
+          comment-tag: bench-pr-comment-${{ matrix.id }}-${{ inputs.matrix_preset }}
 
       - name: Upload Benchmark Results
         if: inputs.mode == 'develop'
@@ -279,7 +281,7 @@ jobs:
           bash scripts/cat-s3.sh vortex-ci-benchmark-results data.json.gz results.json
 
       # v4 (Postgres) ingest -- the REQUIRED benchmark-results pipeline feeding the live
-      # benchmarks website (see bench.yml for the full rationale); a failure here fails the
+      # benchmarks website (see develop-bench.yml for the full rationale); a failure here fails the
       # job. Gated on inputs.mode == 'develop' + the ingest-role ARN var (the assume-role
       # input that MUST exist for OIDC to succeed). post-ingest.py mints the RDS IAM token
       # (boto3) from the assumed GitHubBenchmarkIngestRole; sslmode=verify-full validates
diff --git a/bench-orchestrator/bench_orchestrator/ci_matrix/catalog.py b/bench-orchestrator/bench_orchestrator/ci_matrix/catalog.py
index 55f6ce523f1..f7b915f7001 100644
--- a/bench-orchestrator/bench_orchestrator/ci_matrix/catalog.py
+++ b/bench-orchestrator/bench_orchestrator/ci_matrix/catalog.py
@@ -14,6 +14,7 @@
 PRESETS = {
     "develop": "Every regular SQL benchmark at full target coverage.",
     "pr": "The quicker pull-request SQL benchmark matrix.",
+    "pr-compact": "Pull-request SQL benchmarks that only run Vortex Compact.",
     "pr-full": "Every regular SQL benchmark at full PR target coverage.",
     "nightly": "Large-scale SF=100 TPC-H on NVMe and S3 at default targets.",
 }
@@ -52,6 +53,8 @@
 DUCKDB_DEFAULT_TARGETS = DEFAULT_TARGETS.only(Engine.DUCKDB)
 DUCKDB_STANDARD_TARGETS = STANDARD_TARGETS.only(Engine.DUCKDB)
 DATAFUSION_VORTEX_TARGETS = df(Format.VORTEX)
+COMPACT_TARGETS = df(Format.VORTEX_COMPACT) | duck(Format.VORTEX_COMPACT)
+COMPACT_DUCKDB_TARGETS = duck(Format.VORTEX_COMPACT)
 
 DEFAULT = Coverage(DEFAULT_TARGETS)
 STANDARD = Coverage(STANDARD_TARGETS)
@@ -68,6 +71,8 @@
 DUCKDB_DEFAULT = Coverage(DUCKDB_DEFAULT_TARGETS)
 DUCKDB_STANDARD = Coverage(DUCKDB_STANDARD_TARGETS)
 DATAFUSION_VORTEX = Coverage(DATAFUSION_VORTEX_TARGETS)
+COMPACT = Coverage(COMPACT_TARGETS)
+COMPACT_DUCKDB = Coverage(COMPACT_DUCKDB_TARGETS)
 
 # Concrete benchmark cases
 
@@ -78,6 +83,7 @@
         name="Clickbench on NVME",
         runs={
             "pr": DEFAULT,
+            "pr-compact": COMPACT,
             "pr-full": DEFAULT_WITH_DUCKDB_PR_FULL,
             "develop": FULL_LOCAL,
         },
@@ -88,6 +94,7 @@
         name="Clickbench Sorted on NVME",
         runs={
             "pr": DEFAULT,
+            "pr-compact": COMPACT,
             "pr-full": DEFAULT_WITH_DUCKDB_PR_FULL,
             "develop": FULL_LOCAL,
         },
@@ -100,6 +107,7 @@
         iterations=10,
         runs={
             "pr": DEFAULT,
+            "pr-compact": COMPACT,
             "pr-full": FULL_PR,
             "develop": FULL_LOCAL,
         },
@@ -115,6 +123,7 @@
         remote_key="tpch/1.0",
         runs={
             "pr": DEFAULT,
+            "pr-compact": COMPACT,
             "pr-full": STANDARD,
             "develop": STANDARD,
         },
@@ -127,6 +136,7 @@
         iterations=10,
         runs={
             "pr": DEFAULT,
+            "pr-compact": COMPACT,
             "pr-full": FULL_PR,
             "develop": FULL_LOCAL,
         },
@@ -141,6 +151,7 @@
         local_dir="vortex-bench/data/tpch/10.0",
         remote_key="tpch/10.0",
         runs={
+            "pr-compact": COMPACT,
             "pr-full": STANDARD,
             "develop": STANDARD,
         },
@@ -169,6 +180,7 @@
         scale_factor=1.0,
         runs={
             "pr": DEFAULT,
+            "pr-compact": COMPACT,
             "pr-full": STANDARD_WITH_DUCKDB,
             "develop": STANDARD_WITH_DUCKDB,
         },
@@ -181,6 +193,7 @@
         local_dir="vortex-bench/data/statpopgen",
         runs={
             "pr": DUCKDB_DEFAULT,
+            "pr-compact": COMPACT_DUCKDB,
             "pr-full": DUCKDB_STANDARD,
             "develop": DUCKDB_STANDARD,
         },
@@ -192,6 +205,7 @@
         scale_factor=100,
         runs={
             "pr": DEFAULT,
+            "pr-compact": COMPACT,
             "pr-full": STANDARD,
             "develop": STANDARD,
         },
@@ -206,6 +220,7 @@
         remote_key="fineweb",
         runs={
             "pr": DEFAULT,
+            "pr-compact": COMPACT,
             "pr-full": STANDARD,
             "develop": STANDARD,
         },
@@ -227,6 +242,7 @@
         name="Appian on NVME",
         iterations=10,
         runs={
+            "pr-compact": COMPACT,
             "pr-full": DEFAULT_WITH_DUCKDB_PR_FULL,
             "develop": STANDARD_WITH_DUCKDB,
         },
diff --git a/bench-orchestrator/tests/test_matrix.py b/bench-orchestrator/tests/test_matrix.py
index dd19d12e07f..ed94961b3d1 100644
--- a/bench-orchestrator/tests/test_matrix.py
+++ b/bench-orchestrator/tests/test_matrix.py
@@ -33,6 +33,9 @@
     "appian-nvme",
     "vortex-queries",
 )
+COMPACT_IDS = tuple(
+    benchmark_id for benchmark_id in REGULAR_IDS if benchmark_id not in {"polarsignals", "vortex-queries"}
+)
 EXPECTED_IDS = {
     "develop": REGULAR_IDS,
     "pr": tuple(
@@ -40,6 +43,7 @@
         for benchmark_id in REGULAR_IDS
         if benchmark_id not in {"tpch-s3-10", "appian-nvme", "vortex-queries"}
     ),
+    "pr-compact": COMPACT_IDS,
     "pr-full": REGULAR_IDS,
     "nightly": ("tpch-nvme", "tpch-s3"),
 }
@@ -68,6 +72,7 @@ def test_matrix_presets(preset: str, expected_ids: tuple[str, ...]) -> None:
 def test_pr_target_selection() -> None:
     develop = {entry["id"]: entry for entry in _entries("develop")}
     pr = {entry["id"]: entry for entry in _entries("pr")}
+    pr_compact = {entry["id"]: entry for entry in _entries("pr-compact")}
     pr_full = {entry["id"]: entry for entry in _entries("pr-full")}
 
     assert _targets(pr["tpch-nvme"]) == {
@@ -79,6 +84,10 @@ def test_pr_target_selection() -> None:
     assert ("datafusion", "lance") in _targets(develop["tpch-nvme"])
     assert all(("datafusion", "lance") not in _targets(entry) for entry in pr_full.values())
     assert "vortex-compact" in cast("list[str]", pr_full["clickbench-nvme"]["data_formats"])
+    assert all(
+        all(target[1] == "vortex-compact" for target in _targets(entry)) and entry["data_formats"] == ["vortex-compact"]
+        for entry in pr_compact.values()
+    )
 
 
 def test_resolver_rejects_empty_targets() -> None:
diff --git a/docs/developer-guide/benchmarking.md b/docs/developer-guide/benchmarking.md
index 1655d7f8a08..45695a4a720 100644
--- a/docs/developer-guide/benchmarking.md
+++ b/docs/developer-guide/benchmarking.md
@@ -210,14 +210,13 @@ Benchmarks run automatically on all commits to `develop` and can be run on-deman
 
 - **Post-commit** -- compression, random access, and SQL benchmarks run on every commit to
   `develop`, with results uploaded for historical tracking.
-- **PR benchmarks** -- triggered by the `action/benchmark` label. Results are compared against
-  the latest `develop` run and posted as a PR comment.
-- **GPU compression benchmarks** -- triggered by the `action/benchmark-gpu-compress` label. Runs
-  the allow-listed Vortex decompression cases on a GPU runner and posts the timings as a PR comment.
-- **SQL benchmarks** -- triggered by the `action/benchmark-sql` label. Runs the base SQL matrix,
-  which excludes Appian, TPC-H SF=10 on S3, `vortex-compact`, and `duckdb:duckdb`.
-- **Full SQL benchmarks** -- triggered by the `action/benchmark-sql-full` label. Runs the full
-  SQL matrix of suites, engines, formats, and storage backends (NVMe, S3).
+- **Random access** -- `action/bench-random-access` runs only the random-access benchmark.
+- **Compression** -- `action/bench-compress` runs only the compression benchmark.
+- **GPU compression** -- `action/bench-gpu-compress` runs the allow-listed Vortex decompression
+  cases on a GPU runner.
+- **SQL** -- `action/bench-sql` runs the `pr` preset, which excludes `vortex-compact`.
+- **SQL Compact** -- `action/bench-sql-compact` runs the `pr-compact` preset, which generates
+  and benchmarks only `vortex-compact`.
 
 All CI benchmarks run on dedicated instances with the `release_debug` profile and
 `-C target-cpu=native` to produce representative numbers.

From 4f974d3238704b826f460b928f570540bfdb1682 Mon Sep 17 00:00:00 2001
From: Connor Tsui <connor.tsui20@gmail.com>
Date: Wed, 29 Jul 2026 10:42:45 -0400
Subject: [PATCH 2/5] ci: make benchmark comparisons reliable

Signed-off-by: Connor Tsui <connor.tsui20@gmail.com>
---
 .github/workflows/pr-bench-dispatch.yml       |   3 +
 .github/workflows/pr-bench-runner.yml         |  35 ++++--
 .github/workflows/pr-bench-sql.yml            |   1 +
 .github/workflows/sql-bench-matrix.yml        |  94 +++++++++++++--
 .../bench_orchestrator/ci_matrix/catalog.py   |   9 +-
 bench-orchestrator/tests/test_matrix.py       |   8 +-
 docs/developer-guide/benchmarking.md          |   4 +-
 scripts/compare-benchmark-jsons.py            | 114 +++++++++++++++---
 scripts/tests/test_benchmark_reporting.py     |  78 ++++++++++++
 9 files changed, 301 insertions(+), 45 deletions(-)

diff --git a/.github/workflows/pr-bench-dispatch.yml b/.github/workflows/pr-bench-dispatch.yml
index 4e79c6fc5f5..81fdf2d0481 100644
--- a/.github/workflows/pr-bench-dispatch.yml
+++ b/.github/workflows/pr-bench-dispatch.yml
@@ -3,6 +3,9 @@
 # phantom check suites that obscure in-progress benchmark runs on the PR.
 
 name: PR Benchmark Dispatch
+run-name: >-
+  PR #${{ github.event.pull_request.number }} benchmark:
+  ${{ github.event.label.name }}
 
 on:
   pull_request:
diff --git a/.github/workflows/pr-bench-runner.yml b/.github/workflows/pr-bench-runner.yml
index b1b5d8c8799..fda25096528 100644
--- a/.github/workflows/pr-bench-runner.yml
+++ b/.github/workflows/pr-bench-runner.yml
@@ -41,7 +41,9 @@ jobs:
           sccache: s3
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7
         with:
-          ref: ${{ github.event.pull_request.head.sha || github.sha }}
+          # Benchmark the synthetic merge commit so the measured code includes
+          # the exact base revision used by this pull_request event.
+          ref: ${{ github.sha }}
       - uses: ./.github/actions/setup-rust
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
@@ -78,7 +80,7 @@ jobs:
         uses: polarsignals/gh-actions-ps-profiling@68ae857e375a826606352016e5b90f01a2a7ff7a  # v0.8.1
         with:
           polarsignals_cloud_token: ${{ secrets.POLAR_SIGNALS_API_KEY }}
-          labels: "branch=${{ github.ref_name }};gh_run_id=${{ github.run_id }};commit_sha=${{ github.event.pull_request.head.sha }};benchmark=${{ inputs.benchmark_id }}"
+          labels: "branch=${{ github.ref_name }};gh_run_id=${{ github.run_id }};commit_sha=${{ github.sha }};benchmark=${{ inputs.benchmark_id }}"
           project_uuid: "e5d846e1-b54c-46e7-9174-8bf055a3af56"
           job_name: "${{ inputs.benchmark_id }}"
           profiling_frequency: 199
@@ -122,14 +124,33 @@ jobs:
 
       - name: Compare results
         shell: bash
+        env:
+          BASELINE_COMMIT: ${{ github.event.pull_request.base.sha }}
         run: |
           set -Eeu -o pipefail -x
 
-          python3 scripts/s3-download.py s3://vortex-ci-benchmark-results/data.json.gz data.json.gz --no-sign-request
-          gzip -d -c data.json.gz > base.json
-
-          uv run --no-project scripts/compare-benchmark-jsons.py base.json results.json "${{ inputs.benchmark_name }}" \
-            > comment.md
+          baseline_args=()
+          if [[ -n "$BASELINE_COMMIT" ]]; then
+            baseline_args+=(--baseline-commit "$BASELINE_COMMIT")
+          fi
+
+          for attempt in {1..40}; do
+            python3 scripts/s3-download.py \
+              s3://vortex-ci-benchmark-results/data.json.gz data.json.gz --no-sign-request
+            gzip -d -c data.json.gz > base.json
+
+            if uv run --no-project scripts/compare-benchmark-jsons.py \
+              base.json results.json "${{ inputs.benchmark_name }}" "${baseline_args[@]}" \
+              > comment.md 2> compare.err; then
+              break
+            fi
+            if ! grep -q "No baseline rows found" compare.err || (( attempt == 40 )); then
+              cat compare.err >&2
+              exit 1
+            fi
+            echo "Waiting for benchmark results from base commit $BASELINE_COMMIT"
+            sleep 60
+          done
           cat comment.md >> "$GITHUB_STEP_SUMMARY"
 
       - name: Comment PR
diff --git a/.github/workflows/pr-bench-sql.yml b/.github/workflows/pr-bench-sql.yml
index 63f1bd8f7f2..b595a286e86 100644
--- a/.github/workflows/pr-bench-sql.yml
+++ b/.github/workflows/pr-bench-sql.yml
@@ -30,6 +30,7 @@ on:
           - "pr-full"
 
 permissions:
+  actions: read  # for checking whether the base commit's benchmarks succeeded
   contents: read
   pull-requests: write  # for commenting on PRs
   id-token: write  # enables AWS-GitHub OIDC
diff --git a/.github/workflows/sql-bench-matrix.yml b/.github/workflows/sql-bench-matrix.yml
index bd76a46b20f..36cd3d362da 100644
--- a/.github/workflows/sql-bench-matrix.yml
+++ b/.github/workflows/sql-bench-matrix.yml
@@ -18,16 +18,81 @@ on:
 jobs:
   resolve-matrix:
     runs-on: ubuntu-latest
-    timeout-minutes: 10
+    timeout-minutes: ${{ inputs.mode == 'pr' && 45 || 10 }}
     permissions:
+      actions: read
       contents: read
     outputs:
       benchmark_matrix: ${{ steps.resolve.outputs.benchmark_matrix }}
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7
         with:
-          ref: ${{ inputs.mode == 'pr' && github.event.pull_request.head.sha || github.sha }}
+          ref: ${{ github.sha }}
           persist-credentials: false
+      - name: Wait for SQL baseline
+        if: inputs.mode == 'pr' && github.event_name == 'pull_request'
+        shell: bash
+        env:
+          BASELINE_COMMIT: ${{ github.event.pull_request.base.sha }}
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          set -Eeuo pipefail
+
+          for attempt in {1..40}; do
+            runs="$(
+              gh api --method GET "repos/${GITHUB_REPOSITORY}/actions/runs" \
+                -f head_sha="$BASELINE_COMMIT" \
+                -f event=push \
+                -f per_page=100
+            )"
+            run_id="$(
+              jq -r '
+                [.workflow_runs[]
+                  | select(
+                      .path == ".github/workflows/bench.yml"
+                      or .path == ".github/workflows/develop-bench.yml"
+                    )]
+                | max_by(.id)
+                | .id // empty
+              ' <<< "$runs"
+            )"
+
+            if [[ -n "$run_id" ]]; then
+              run_status="$(jq -r --argjson run_id "$run_id" '
+                .workflow_runs[]
+                | select(.id == $run_id)
+                | .status
+              ' <<< "$runs")"
+
+              if [[ "$run_status" == "completed" ]]; then
+                jobs="$(
+                  gh api --method GET \
+                    "repos/${GITHUB_REPOSITORY}/actions/runs/${run_id}/jobs" \
+                    -f filter=latest \
+                    -f per_page=100
+                )"
+                sql_jobs="$(jq '[.jobs[] | select(.name | startswith("sql / bench ("))]' <<< "$jobs")"
+                failed_jobs="$(jq '[.[] | select(.conclusion != "success")] | length' <<< "$sql_jobs")"
+
+                if (( failed_jobs == 0 )) && (( $(jq length <<< "$sql_jobs") > 0 )); then
+                  echo "SQL baselines are ready for $BASELINE_COMMIT"
+                  break
+                fi
+
+                echo "SQL benchmarks failed for base commit $BASELINE_COMMIT:" >&2
+                jq -r '.[] | select(.conclusion != "success") | "  \(.name): \(.conclusion)"' \
+                  <<< "$sql_jobs" >&2
+                exit 1
+              fi
+            fi
+
+            if (( attempt == 40 )); then
+              echo "Timed out waiting for SQL baselines from $BASELINE_COMMIT" >&2
+              exit 1
+            fi
+            echo "Waiting for SQL benchmarks from base commit $BASELINE_COMMIT"
+            sleep 60
+          done
       - name: Install uv
         uses: spiraldb/actions/.github/actions/setup-uv@a746510eafaa926484c354541cfc49b2ec06cc63  # 0.18.6
         with:
@@ -120,7 +185,9 @@ jobs:
           sccache: s3
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7
         with:
-          ref: ${{ inputs.mode == 'pr' && github.event.pull_request.head.sha || github.sha }}
+          # Pull-request events use the synthetic merge commit so the measured
+          # code includes the exact base revision in the event payload.
+          ref: ${{ github.sha }}
       - uses: ./.github/actions/setup-rust
         with:
           repo-token: ${{ secrets.GITHUB_TOKEN }}
@@ -170,7 +237,7 @@ jobs:
         shell: bash
         env:
           AWS_REGION: "us-east-1"
-          REMOTE_STORAGE: "s3://vortex-ci-benchmark-datasets/${{ github.ref_name }}/${{ github.run_id }}/${{ matrix.remote_key }}/"
+          REMOTE_STORAGE: "s3://vortex-ci-benchmark-datasets/${{ github.ref_name }}/${{ github.run_id }}/${{ inputs.matrix_preset }}/${{ matrix.remote_key }}/"
         run: |
           aws s3 rm --recursive "$REMOTE_STORAGE"
           aws s3 cp --recursive "${{ matrix.local_dir }}" "$REMOTE_STORAGE"
@@ -180,7 +247,7 @@ jobs:
         uses: polarsignals/gh-actions-ps-profiling@68ae857e375a826606352016e5b90f01a2a7ff7a  # v0.8.1
         with:
           polarsignals_cloud_token: ${{ secrets.POLAR_SIGNALS_API_KEY }}
-          labels: "branch=${{ github.ref_name }};gh_run_id=${{ github.run_id }};commit_sha=${{ inputs.mode == 'pr' && github.event.pull_request.head.sha || github.sha }};benchmark=${{ matrix.id }}"
+          labels: "branch=${{ github.ref_name }};gh_run_id=${{ github.run_id }};commit_sha=${{ github.sha }};benchmark=${{ matrix.id }}"
           project_uuid: "e5d846e1-b54c-46e7-9174-8bf055a3af56"
           job_name: "${{ matrix.id }}"
           profiling_frequency: 199
@@ -218,7 +285,7 @@ jobs:
           OTEL_EXPORTER_OTLP_ENDPOINT: "${{ (inputs.mode != 'pr' || github.event.pull_request.head.repo.fork == false) && secrets.OTEL_EXPORTER_OTLP_ENDPOINT || '' }}"
           OTEL_EXPORTER_OTLP_HEADERS: "${{ (inputs.mode != 'pr' || github.event.pull_request.head.repo.fork == false) && secrets.OTEL_EXPORTER_OTLP_HEADERS || '' }}"
           OTEL_RESOURCE_ATTRIBUTES: "bench-name=${{ matrix.id }}"
-          REMOTE_STORAGE: "s3://vortex-ci-benchmark-datasets/${{ github.ref_name }}/${{ github.run_id }}/${{ matrix.remote_key }}/"
+          REMOTE_STORAGE: "s3://vortex-ci-benchmark-datasets/${{ github.ref_name }}/${{ github.run_id }}/${{ inputs.matrix_preset }}/${{ matrix.remote_key }}/"
         run: |
           bash scripts/bench-taskset.sh uv run --project bench-orchestrator vx-bench run "${{ matrix.subcommand }}" \
             --targets-json '${{ toJSON(matrix.targets) }}' \
@@ -236,20 +303,29 @@ jobs:
           uv run --no-project scripts/capture-file-sizes.py \
             vortex-bench/data \
             --benchmark ${{ matrix.subcommand }} \
-            --commit ${{ inputs.mode == 'pr' && github.event.pull_request.head.sha || github.sha }} \
+            --commit ${{ github.sha }} \
             -o sizes.json
           cat sizes.json >> results.json
 
       - name: Compare results
         if: inputs.mode == 'pr'
         shell: bash
+        env:
+          BASELINE_COMMIT: ${{ github.event.pull_request.base.sha }}
         run: |
           set -Eeu -o pipefail -x
 
-          python3 scripts/s3-download.py s3://vortex-ci-benchmark-results/data.json.gz data.json.gz --no-sign-request
+          baseline_args=()
+          if [[ -n "$BASELINE_COMMIT" ]]; then
+            baseline_args+=(--baseline-commit "$BASELINE_COMMIT")
+          fi
+
+          python3 scripts/s3-download.py \
+            s3://vortex-ci-benchmark-results/data.json.gz data.json.gz --no-sign-request
           gzip -d -c data.json.gz > base.json
 
-          uv run --no-project scripts/compare-benchmark-jsons.py base.json results.json "${{ matrix.name }}" \
+          uv run --no-project scripts/compare-benchmark-jsons.py \
+            base.json results.json "${{ matrix.name }}" "${baseline_args[@]}" \
             > comment.md
           cat comment.md >> "$GITHUB_STEP_SUMMARY"
 
diff --git a/bench-orchestrator/bench_orchestrator/ci_matrix/catalog.py b/bench-orchestrator/bench_orchestrator/ci_matrix/catalog.py
index f7b915f7001..680d9519627 100644
--- a/bench-orchestrator/bench_orchestrator/ci_matrix/catalog.py
+++ b/bench-orchestrator/bench_orchestrator/ci_matrix/catalog.py
@@ -14,7 +14,7 @@
 PRESETS = {
     "develop": "Every regular SQL benchmark at full target coverage.",
     "pr": "The quicker pull-request SQL benchmark matrix.",
-    "pr-compact": "Pull-request SQL benchmarks that only run Vortex Compact.",
+    "pr-compact": "Pull-request SQL benchmarks for Vortex Compact plus Parquet controls.",
     "pr-full": "Every regular SQL benchmark at full PR target coverage.",
     "nightly": "Large-scale SF=100 TPC-H on NVMe and S3 at default targets.",
 }
@@ -53,8 +53,11 @@
 DUCKDB_DEFAULT_TARGETS = DEFAULT_TARGETS.only(Engine.DUCKDB)
 DUCKDB_STANDARD_TARGETS = STANDARD_TARGETS.only(Engine.DUCKDB)
 DATAFUSION_VORTEX_TARGETS = df(Format.VORTEX)
-COMPACT_TARGETS = df(Format.VORTEX_COMPACT) | duck(Format.VORTEX_COMPACT)
-COMPACT_DUCKDB_TARGETS = duck(Format.VORTEX_COMPACT)
+COMPACT_TARGETS = df(Format.PARQUET, Format.VORTEX_COMPACT) | duck(
+    Format.PARQUET,
+    Format.VORTEX_COMPACT,
+)
+COMPACT_DUCKDB_TARGETS = duck(Format.PARQUET, Format.VORTEX_COMPACT)
 
 DEFAULT = Coverage(DEFAULT_TARGETS)
 STANDARD = Coverage(STANDARD_TARGETS)
diff --git a/bench-orchestrator/tests/test_matrix.py b/bench-orchestrator/tests/test_matrix.py
index ed94961b3d1..a5c18c3dc1d 100644
--- a/bench-orchestrator/tests/test_matrix.py
+++ b/bench-orchestrator/tests/test_matrix.py
@@ -84,10 +84,10 @@ def test_pr_target_selection() -> None:
     assert ("datafusion", "lance") in _targets(develop["tpch-nvme"])
     assert all(("datafusion", "lance") not in _targets(entry) for entry in pr_full.values())
     assert "vortex-compact" in cast("list[str]", pr_full["clickbench-nvme"]["data_formats"])
-    assert all(
-        all(target[1] == "vortex-compact" for target in _targets(entry)) and entry["data_formats"] == ["vortex-compact"]
-        for entry in pr_compact.values()
-    )
+    for entry in pr_compact.values():
+        targets = _targets(entry)
+        assert {file_format for _engine, file_format in targets} == {"parquet", "vortex-compact"}
+        assert set(cast("list[str]", entry["data_formats"])) == {"parquet", "vortex-compact"}
 
 
 def test_resolver_rejects_empty_targets() -> None:
diff --git a/docs/developer-guide/benchmarking.md b/docs/developer-guide/benchmarking.md
index 45695a4a720..c0e900ae205 100644
--- a/docs/developer-guide/benchmarking.md
+++ b/docs/developer-guide/benchmarking.md
@@ -215,8 +215,8 @@ Benchmarks run automatically on all commits to `develop` and can be run on-deman
 - **GPU compression** -- `action/bench-gpu-compress` runs the allow-listed Vortex decompression
   cases on a GPU runner.
 - **SQL** -- `action/bench-sql` runs the `pr` preset, which excludes `vortex-compact`.
-- **SQL Compact** -- `action/bench-sql-compact` runs the `pr-compact` preset, which generates
-  and benchmarks only `vortex-compact`.
+- **SQL Compact** -- `action/bench-sql-compact` runs the `pr-compact` preset, which benchmarks
+  `vortex-compact` plus Parquet control rows used to distinguish code changes from runner drift.
 
 All CI benchmarks run on dedicated instances with the `release_debug` profile and
 `-C target-cpu=native` to produce representative numbers.
diff --git a/scripts/compare-benchmark-jsons.py b/scripts/compare-benchmark-jsons.py
index d226c23cb48..2b6eafc7296 100644
--- a/scripts/compare-benchmark-jsons.py
+++ b/scripts/compare-benchmark-jsons.py
@@ -11,10 +11,10 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright the Vortex contributors
 
+import argparse
 import math
 import os
 import re
-import sys
 from dataclasses import dataclass
 from io import StringIO
 from typing import Any
@@ -106,7 +106,7 @@ def normalize_format_name(value: Any) -> str | None:
     return FORMAT_DISPLAY_NAMES.get(value, value)
 
 
-def comparison_target(name: Any, target: Any = None) -> tuple[str, str, int | None]:
+def comparison_target(name: Any, target: Any = None) -> tuple[str, str, int | str | None]:
     """Return the engine, display format, and optional SQL query number."""
 
     target_engine = None
@@ -120,6 +120,20 @@ def comparison_target(name: Any, target: Any = None) -> tuple[str, str, int | No
     name_format = match.group(3) if match is not None else None
     query = int(match.group(1)) if match is not None else None
 
+    if match is None and isinstance(name, str):
+        random_access = re.match(
+            r"^(?P<prefix>random-access(?:/.*)?)/"
+            r"(?P<file_format>parquet|vortex|lance)-(?P<variant>.+)$",
+            name,
+        )
+        if random_access is not None:
+            file_format = random_access.group("file_format")
+            if file_format == "vortex":
+                file_format = "vortex-file-compressed"
+            return "random-access", file_format, (
+                f"{random_access.group('prefix')}/{random_access.group('variant')}"
+            )
+
     engine = str(target_engine or name_engine or "unknown")
     file_format = str(target_format or normalize_format_name(name_format) or "unknown")
     return engine, file_format, query
@@ -170,25 +184,48 @@ def benchmark_identity_rows(df: pd.DataFrame) -> pd.DataFrame:
 
 
 def read_jsonl_rows_for_commit(path: str, commit_id: str) -> pd.DataFrame:
-    """Read only rows matching a commit from a JSONL benchmark history."""
+    """Read the latest copy of each row matching a history commit.
 
-    rows = []
+    Re-running a develop workflow appends a second result block for the same
+    commit. Keeping the last copy of each logical row prevents a many-to-one
+    merge from weighting that baseline multiple times.
+    """
+
+    rows_by_identity: dict[tuple[Any, ...], dict[str, Any]] = {}
     with open(path, encoding="utf-8") as lines:
         for line in lines:
             if '"commit_id"' not in line or f'"{commit_id}"' not in line:
                 continue
             record = orjson.loads(line)
-            if record.get("commit_id") == commit_id:
-                rows.append(record)
-    return pd.DataFrame(rows)
-
+            if record.get("commit_id") != commit_id:
+                continue
 
-def read_latest_baseline_rows(path: str, pr: pd.DataFrame) -> pd.DataFrame:
+            file_size = record.get("file_size")
+            if isinstance(file_size, dict):
+                identity = (
+                    FILE_SIZE_METRIC,
+                    file_size.get("benchmark"),
+                    file_size.get("scale_factor"),
+                    file_size.get("format"),
+                    file_size.get("file"),
+                )
+            else:
+                identity = ("timing", benchmark_identity(record))
+            rows_by_identity[identity] = record
+    return pd.DataFrame(rows_by_identity.values())
+
+
+def read_latest_baseline_rows(
+    path: str,
+    pr: pd.DataFrame,
+    baseline_commit_order: list[str] | None = None,
+) -> pd.DataFrame:
     """Read rows from the latest history commit matching the PR benchmark.
 
     A benchmark can be new to the PR workflow and therefore have no baseline
     yet. Return an empty frame with the PR schema in that case so the report
-    can show the measurements without comparison.
+    can show the measurements without comparison. When an exact baseline is
+    requested, never substitute a different develop commit.
     """
 
     pr_identities = set(benchmark_identity_rows(pr)["benchmark_identity"])
@@ -196,6 +233,7 @@ def read_latest_baseline_rows(path: str, pr: pd.DataFrame) -> pd.DataFrame:
         return pd.read_json(path, lines=True)
 
     baseline_commit_id = None
+    matching_commit_ids: set[str] = set()
     with open(path, encoding="utf-8") as lines:
         for line in lines:
             if '"name"' not in line or '"commit_id"' not in line:
@@ -205,14 +243,27 @@ def read_latest_baseline_rows(path: str, pr: pd.DataFrame) -> pd.DataFrame:
                 commit_id = record.get("commit_id")
                 if commit_id is not None:
                     baseline_commit_id = commit_id
+                    matching_commit_ids.add(commit_id)
+
+    if baseline_commit_order is not None:
+        baseline_commit_id = next(
+            (commit_id for commit_id in baseline_commit_order if commit_id in matching_commit_ids),
+            None,
+        )
 
     if baseline_commit_id is None:
+        if baseline_commit_order is not None:
+            raise ValueError("No baseline rows found in the requested commit ancestry")
         return pr.iloc[0:0].copy()
 
     return read_jsonl_rows_for_commit(path, baseline_commit_id)
 
 
-def select_latest_baseline_rows(base: pd.DataFrame, pr: pd.DataFrame) -> pd.DataFrame:
+def select_latest_baseline_rows(
+    base: pd.DataFrame,
+    pr: pd.DataFrame,
+    baseline_commit_order: list[str] | None = None,
+) -> pd.DataFrame:
     """Select rows from the latest baseline commit containing this benchmark.
 
     The persisted benchmark history is append-only. A row only appears after
@@ -225,6 +276,10 @@ def select_latest_baseline_rows(base: pd.DataFrame, pr: pd.DataFrame) -> pd.Data
 
     commit_ids = base["commit_id"].dropna().unique()
     if len(commit_ids) <= 1:
+        if baseline_commit_order is not None and any(
+            commit_id not in baseline_commit_order for commit_id in commit_ids
+        ):
+            raise ValueError("No baseline rows found in the requested commit ancestry")
         return base
 
     pr_identities = set(benchmark_identity_rows(pr)["benchmark_identity"])
@@ -237,7 +292,16 @@ def select_latest_baseline_rows(base: pd.DataFrame, pr: pd.DataFrame) -> pd.Data
     if matches.empty:
         return base.iloc[0:0].copy()
 
-    baseline_commit_id = matches["commit_id"].iloc[-1]
+    if baseline_commit_order is None:
+        baseline_commit_id = matches["commit_id"].iloc[-1]
+    else:
+        matching_commit_ids = set(matches["commit_id"])
+        baseline_commit_id = next(
+            (commit_id for commit_id in baseline_commit_order if commit_id in matching_commit_ids),
+            None,
+        )
+        if baseline_commit_id is None:
+            raise ValueError("No baseline rows found in the requested commit ancestry")
     return base[base["commit_id"] == baseline_commit_id].copy()
 
 
@@ -261,7 +325,7 @@ def normalize_measurement_rows(df: pd.DataFrame) -> pd.DataFrame:
         columns=["engine", "file_format", "query"],
         index=df.index,
     )
-    df["query"] = pd.array(df["query"], dtype="Int64")
+    df["query"] = df["query"].astype(object)
     return df
 
 
@@ -422,7 +486,7 @@ def build_statistical_analysis(df: pd.DataFrame, threshold_pct: int) -> dict[str
         rows.append(
             {
                 "name": row["name"],
-                "query": int(row["query"]),
+                "query": row["query"],
                 "engine": row["engine"],
                 "file_format": row["file_format"],
                 "combo": f"{row['engine']}:{row['file_format']}",
@@ -442,7 +506,7 @@ def build_statistical_analysis(df: pd.DataFrame, threshold_pct: int) -> dict[str
         beta_log_ratio = float(group["log_ratio"].mean())
         query_rows.append(
             {
-                "query": int(query),
+                "query": query,
                 "beta_log_ratio": beta_log_ratio,
                 "beta_ratio": float(np.exp(beta_log_ratio)),
                 "beta_log_se": mean_with_standard_error(group, "log_ratio", "log_ratio_se"),
@@ -914,11 +978,21 @@ def group_sort_key(group_key: tuple[str, str, str]) -> tuple[int, int, int, str,
 def main() -> None:
     """Render the benchmark comparison markdown used in CI PR comments."""
 
-    benchmark_name = sys.argv[3] if len(sys.argv) > 3 else ""
+    parser = argparse.ArgumentParser()
+    parser.add_argument("base_path")
+    parser.add_argument("pr_path")
+    parser.add_argument("benchmark_name", nargs="?", default="")
+    parser.add_argument(
+        "--baseline-commit",
+        help="Exact develop commit whose benchmark results must be used",
+    )
+    args = parser.parse_args()
+
+    baseline_commit_order = [args.baseline_commit] if args.baseline_commit else None
 
-    pr = pd.read_json(sys.argv[2], lines=True)
-    title = format_title(benchmark_name, pr)
-    base = read_latest_baseline_rows(sys.argv[1], pr)
+    pr = pd.read_json(args.pr_path, lines=True)
+    title = format_title(args.benchmark_name, pr)
+    base = read_latest_baseline_rows(args.base_path, pr, baseline_commit_order)
 
     base_commit_ids = set(base["commit_id"].unique())
     pr_commit_id = set(pr["commit_id"].unique())
@@ -938,7 +1012,7 @@ def main() -> None:
     df3["unit"] = df3["unit"].fillna("unit")
     df3["ratio"] = df3["value_pr"] / df3["value_base"]
 
-    is_s3_benchmark = "s3" in benchmark_name.lower()
+    is_s3_benchmark = "s3" in args.benchmark_name.lower()
     threshold_pct = 30 if is_s3_benchmark else 10
     improvement_threshold = 1.0 - (threshold_pct / 100.0)
     regression_threshold = 1.0 + (threshold_pct / 100.0)
diff --git a/scripts/tests/test_benchmark_reporting.py b/scripts/tests/test_benchmark_reporting.py
index 4e7bc7c9933..b660a0df5cb 100644
--- a/scripts/tests/test_benchmark_reporting.py
+++ b/scripts/tests/test_benchmark_reporting.py
@@ -8,6 +8,7 @@
 from pathlib import Path
 
 import pandas as pd
+import pytest
 
 REPO_ROOT = Path(__file__).resolve().parents[2]
 COMPARE_SCRIPT = REPO_ROOT / "scripts" / "compare-benchmark-jsons.py"
@@ -202,6 +203,62 @@ def test_read_latest_baseline_rows_streams_latest_matching_benchmark_commit(tmp_
     assert len(selected) == 2
 
 
+def test_read_latest_baseline_rows_uses_requested_base_commit(tmp_path: Path) -> None:
+    compare = load_compare_module()
+    history_path = tmp_path / "history.jsonl"
+    history_rows = [
+        stored_timing_row("base-ancestor", "tpch_q01/datafusion:parquet", 100),
+        stored_timing_row("future-develop", "tpch_q01/datafusion:parquet", 90),
+    ]
+    history_path.write_text(
+        "".join(f"{json.dumps(row)}\n" for row in history_rows),
+        encoding="utf-8",
+    )
+    pr = pd.DataFrame([stored_timing_row("pr-sha", "tpch_q01/datafusion:parquet", 105)])
+
+    selected = compare.read_latest_baseline_rows(
+        history_path,
+        pr,
+        ["base-ancestor"],
+    )
+
+    assert set(selected["commit_id"]) == {"base-ancestor"}
+
+
+def test_read_latest_baseline_rows_does_not_fall_back_from_requested_commit(tmp_path: Path) -> None:
+    compare = load_compare_module()
+    history_path = tmp_path / "history.jsonl"
+    history_path.write_text(
+        f"{json.dumps(stored_timing_row('other-commit', 'tpch_q01/datafusion:parquet', 100))}\n",
+        encoding="utf-8",
+    )
+    pr = pd.DataFrame([stored_timing_row("pr-sha", "tpch_q01/datafusion:parquet", 105)])
+
+    with pytest.raises(ValueError, match="No baseline rows found"):
+        compare.read_latest_baseline_rows(history_path, pr, ["requested-base"])
+
+
+def test_read_latest_baseline_rows_uses_last_result_from_rerun(tmp_path: Path) -> None:
+    compare = load_compare_module()
+    history_path = tmp_path / "history.jsonl"
+    history_path.write_text(
+        "".join(
+            f"{json.dumps(row)}\n"
+            for row in [
+                stored_timing_row("base", "tpch_q01/datafusion:parquet", 100),
+                stored_timing_row("base", "tpch_q01/datafusion:parquet", 110),
+            ]
+        ),
+        encoding="utf-8",
+    )
+    pr = pd.DataFrame([stored_timing_row("pr", "tpch_q01/datafusion:parquet", 105)])
+
+    selected = compare.read_latest_baseline_rows(history_path, pr, ["base"])
+
+    assert len(selected) == 1
+    assert selected.iloc[0]["value"] == 110
+
+
 def test_within_engine_analysis_uses_each_engines_own_parquet_control() -> None:
     compare = load_compare_module()
     rows = [
@@ -220,6 +277,27 @@ def test_within_engine_analysis_uses_each_engines_own_parquet_control() -> None:
     assert compare.build_verdict(analyses["duckdb"])["impact"] == "+20.0%"
 
 
+def test_random_access_rows_use_matching_parquet_controls() -> None:
+    compare = load_compare_module()
+    rows = [
+        timing_row("random-access/taxi/correlated/parquet-tokio-local-disk", 100, 110),
+        timing_row("random-access/taxi/correlated/vortex-tokio-local-disk", 100, 99),
+        timing_row("random-access/taxi/uniform/parquet-tokio-local-disk", 100, 110),
+        timing_row("random-access/taxi/uniform/vortex-tokio-local-disk", 100, 99),
+    ]
+    df = pd.DataFrame(rows)
+    df[["engine", "file_format", "query"]] = df["name"].apply(compare.extract_target_fields)
+
+    analyses = compare.build_within_engine_statistical_analyses(df, threshold_pct=5)
+
+    assert set(analyses) == {"random-access"}
+    assert set(analyses["random-access"]["detail_df"]["file_format"]) == {
+        "parquet",
+        "vortex-file-compressed",
+    }
+    assert compare.build_verdict(analyses["random-access"])["status"] == "Likely improvement"
+
+
 def test_comparison_report_groups_by_target_and_unit(tmp_path: Path) -> None:
     base_rows = [
         stored_custom_row("base-sha", "timing/fixture", "ms", 12.5),

From b79ebea544ab6c9e2f74e63a06bfea66915ce96e Mon Sep 17 00:00:00 2001
From: Connor Tsui <connor.tsui20@gmail.com>
Date: Wed, 29 Jul 2026 11:25:33 -0400
Subject: [PATCH 3/5] ci: add all CPU benchmarks label

Signed-off-by: Connor Tsui <connor.tsui20@gmail.com>
---
 .github/workflows/pr-bench-dispatch.yml       | 28 +++++++++++++++++++
 .github/workflows/pr-bench-sql.yml            |  1 +
 bench-orchestrator/README.md                  |  7 +++--
 .../bench_orchestrator/ci_matrix/catalog.py   | 13 +++++++++
 bench-orchestrator/tests/test_matrix.py       | 20 +++++++++++++
 docs/developer-guide/benchmarking.md          |  3 ++
 6 files changed, 69 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/pr-bench-dispatch.yml b/.github/workflows/pr-bench-dispatch.yml
index 81fdf2d0481..77b1afb3688 100644
--- a/.github/workflows/pr-bench-dispatch.yml
+++ b/.github/workflows/pr-bench-dispatch.yml
@@ -19,6 +19,34 @@ permissions:
   id-token: write  # enables AWS-GitHub OIDC
 
 jobs:
+  remove-all-label:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    if: github.event.label.name == 'action/bench-all'
+    steps:
+      - uses: actions-ecosystem/action-remove-labels@2ce5d41b4b6aa8503e285553f75ed56e0a40bae0  # v1
+        if: github.event.pull_request.head.repo.full_name == 'vortex-data/vortex'
+        with:
+          labels: action/bench-all
+          fail_on_error: true
+
+  all-random-access-bench:
+    needs: remove-all-label
+    uses: ./.github/workflows/pr-bench-random-access.yml
+    secrets: inherit
+
+  all-compression-bench:
+    needs: remove-all-label
+    uses: ./.github/workflows/pr-bench-compress.yml
+    secrets: inherit
+
+  all-sql-bench:
+    needs: remove-all-label
+    uses: ./.github/workflows/pr-bench-sql.yml
+    secrets: inherit
+    with:
+      matrix_preset: "pr-all"
+
   remove-random-access-label:
     runs-on: ubuntu-latest
     timeout-minutes: 10
diff --git a/.github/workflows/pr-bench-sql.yml b/.github/workflows/pr-bench-sql.yml
index b595a286e86..67f7e41359d 100644
--- a/.github/workflows/pr-bench-sql.yml
+++ b/.github/workflows/pr-bench-sql.yml
@@ -27,6 +27,7 @@ on:
         options:
           - "pr"
           - "pr-compact"
+          - "pr-all"
           - "pr-full"
 
 permissions:
diff --git a/bench-orchestrator/README.md b/bench-orchestrator/README.md
index ccd09ac3598..ecd90e140e4 100644
--- a/bench-orchestrator/README.md
+++ b/bench-orchestrator/README.md
@@ -79,6 +79,7 @@ to keep benchmark coverage in Python instead of copying large JSON matrices betw
 ```bash
 vx-bench matrix            # list available presets
 vx-bench matrix develop    # emit compact JSON
+vx-bench matrix pr-all      # emit the combined focused PR coverage
 vx-bench matrix pr-full     # emit full pull-request coverage
 vx-bench matrix nightly --pretty
 ```
@@ -158,9 +159,9 @@ vx-bench clean --older-than "30 days" [options]
 ## CI Benchmark Matrices
 
 CI benchmark coverage lives in `bench_orchestrator/ci_matrix/catalog.py`. This is the only file to
-edit when changing what runs in the `develop`, `pr`, `pr-full`, or `nightly` presets. The remaining
-modules in `ci_matrix` model, validate, and render that catalog as the JSON expected by GitHub
-Actions.
+edit when changing what runs in the `develop`, `pr`, `pr-compact`, `pr-all`, `pr-full`, or
+`nightly` presets. The remaining modules in `ci_matrix` model, validate, and render that catalog as
+the JSON expected by GitHub Actions.
 
 When adding coverage, update the benchmark declarations and the expected preset membership in
 `tests/test_matrix.py`.
diff --git a/bench-orchestrator/bench_orchestrator/ci_matrix/catalog.py b/bench-orchestrator/bench_orchestrator/ci_matrix/catalog.py
index 680d9519627..9f8f0fee915 100644
--- a/bench-orchestrator/bench_orchestrator/ci_matrix/catalog.py
+++ b/bench-orchestrator/bench_orchestrator/ci_matrix/catalog.py
@@ -15,6 +15,7 @@
     "develop": "Every regular SQL benchmark at full target coverage.",
     "pr": "The quicker pull-request SQL benchmark matrix.",
     "pr-compact": "Pull-request SQL benchmarks for Vortex Compact plus Parquet controls.",
+    "pr-all": "The union of the focused PR and PR Compact benchmark matrices.",
     "pr-full": "Every regular SQL benchmark at full PR target coverage.",
     "nightly": "Large-scale SF=100 TPC-H on NVMe and S3 at default targets.",
 }
@@ -87,6 +88,7 @@
         runs={
             "pr": DEFAULT,
             "pr-compact": COMPACT,
+            "pr-all": STANDARD,
             "pr-full": DEFAULT_WITH_DUCKDB_PR_FULL,
             "develop": FULL_LOCAL,
         },
@@ -98,6 +100,7 @@
         runs={
             "pr": DEFAULT,
             "pr-compact": COMPACT,
+            "pr-all": STANDARD,
             "pr-full": DEFAULT_WITH_DUCKDB_PR_FULL,
             "develop": FULL_LOCAL,
         },
@@ -111,6 +114,7 @@
         runs={
             "pr": DEFAULT,
             "pr-compact": COMPACT,
+            "pr-all": STANDARD,
             "pr-full": FULL_PR,
             "develop": FULL_LOCAL,
         },
@@ -127,6 +131,7 @@
         runs={
             "pr": DEFAULT,
             "pr-compact": COMPACT,
+            "pr-all": STANDARD,
             "pr-full": STANDARD,
             "develop": STANDARD,
         },
@@ -140,6 +145,7 @@
         runs={
             "pr": DEFAULT,
             "pr-compact": COMPACT,
+            "pr-all": STANDARD,
             "pr-full": FULL_PR,
             "develop": FULL_LOCAL,
         },
@@ -155,6 +161,7 @@
         remote_key="tpch/10.0",
         runs={
             "pr-compact": COMPACT,
+            "pr-all": COMPACT,
             "pr-full": STANDARD,
             "develop": STANDARD,
         },
@@ -184,6 +191,7 @@
         runs={
             "pr": DEFAULT,
             "pr-compact": COMPACT,
+            "pr-all": STANDARD,
             "pr-full": STANDARD_WITH_DUCKDB,
             "develop": STANDARD_WITH_DUCKDB,
         },
@@ -197,6 +205,7 @@
         runs={
             "pr": DUCKDB_DEFAULT,
             "pr-compact": COMPACT_DUCKDB,
+            "pr-all": DUCKDB_STANDARD,
             "pr-full": DUCKDB_STANDARD,
             "develop": DUCKDB_STANDARD,
         },
@@ -209,6 +218,7 @@
         runs={
             "pr": DEFAULT,
             "pr-compact": COMPACT,
+            "pr-all": STANDARD,
             "pr-full": STANDARD,
             "develop": STANDARD,
         },
@@ -224,6 +234,7 @@
         runs={
             "pr": DEFAULT,
             "pr-compact": COMPACT,
+            "pr-all": STANDARD,
             "pr-full": STANDARD,
             "develop": STANDARD,
         },
@@ -235,6 +246,7 @@
         scale_factor=1,
         runs={
             "pr": DATAFUSION_VORTEX,
+            "pr-all": DATAFUSION_VORTEX,
             "pr-full": DATAFUSION_VORTEX,
             "develop": DATAFUSION_VORTEX,
         },
@@ -246,6 +258,7 @@
         iterations=10,
         runs={
             "pr-compact": COMPACT,
+            "pr-all": COMPACT,
             "pr-full": DEFAULT_WITH_DUCKDB_PR_FULL,
             "develop": STANDARD_WITH_DUCKDB,
         },
diff --git a/bench-orchestrator/tests/test_matrix.py b/bench-orchestrator/tests/test_matrix.py
index a5c18c3dc1d..d0f765be246 100644
--- a/bench-orchestrator/tests/test_matrix.py
+++ b/bench-orchestrator/tests/test_matrix.py
@@ -36,6 +36,7 @@
 COMPACT_IDS = tuple(
     benchmark_id for benchmark_id in REGULAR_IDS if benchmark_id not in {"polarsignals", "vortex-queries"}
 )
+PR_ALL_IDS = tuple(benchmark_id for benchmark_id in REGULAR_IDS if benchmark_id != "vortex-queries")
 EXPECTED_IDS = {
     "develop": REGULAR_IDS,
     "pr": tuple(
@@ -44,6 +45,7 @@
         if benchmark_id not in {"tpch-s3-10", "appian-nvme", "vortex-queries"}
     ),
     "pr-compact": COMPACT_IDS,
+    "pr-all": PR_ALL_IDS,
     "pr-full": REGULAR_IDS,
     "nightly": ("tpch-nvme", "tpch-s3"),
 }
@@ -90,6 +92,24 @@ def test_pr_target_selection() -> None:
         assert set(cast("list[str]", entry["data_formats"])) == {"parquet", "vortex-compact"}
 
 
+def test_pr_all_is_union_of_focused_presets() -> None:
+    pr = {entry["id"]: entry for entry in _entries("pr")}
+    pr_compact = {entry["id"]: entry for entry in _entries("pr-compact")}
+    pr_all = {entry["id"]: entry for entry in _entries("pr-all")}
+
+    assert set(pr_all) == set(pr) | set(pr_compact)
+    for benchmark_id, entry in pr_all.items():
+        expected_targets: set[tuple[str, str]] = set()
+        expected_formats: set[str] = set()
+        for preset in (pr, pr_compact):
+            if source := preset.get(benchmark_id):
+                expected_targets |= _targets(source)
+                expected_formats |= set(cast("list[str]", source["data_formats"]))
+
+        assert _targets(entry) == expected_targets
+        assert set(cast("list[str]", entry["data_formats"])) == expected_formats
+
+
 def test_resolver_rejects_empty_targets() -> None:
     benchmark = replace(
         CATALOG.benchmarks[0],
diff --git a/docs/developer-guide/benchmarking.md b/docs/developer-guide/benchmarking.md
index c0e900ae205..fdd90aa28c5 100644
--- a/docs/developer-guide/benchmarking.md
+++ b/docs/developer-guide/benchmarking.md
@@ -217,6 +217,9 @@ Benchmarks run automatically on all commits to `develop` and can be run on-deman
 - **SQL** -- `action/bench-sql` runs the `pr` preset, which excludes `vortex-compact`.
 - **SQL Compact** -- `action/bench-sql-compact` runs the `pr-compact` preset, which benchmarks
   `vortex-compact` plus Parquet control rows used to distinguish code changes from runner drift.
+- **All CPU benchmarks** -- `action/bench-all` runs random access, compression, and the `pr-all`
+  SQL preset, which combines the `pr` and `pr-compact` coverage without repeating shared jobs. Do
+  not combine it with other benchmark labels; GPU compression is the only exception.
 
 All CI benchmarks run on dedicated instances with the `release_debug` profile and
 `-C target-cpu=native` to produce representative numbers.

From cc653c400c3404df0c9320979ceb6056751cc94d Mon Sep 17 00:00:00 2001
From: Connor Tsui <connor.tsui20@gmail.com>
Date: Thu, 30 Jul 2026 14:26:16 -0400
Subject: [PATCH 4/5] fix: ignore zero-byte benchmark files

Signed-off-by: Connor Tsui <connor.tsui20@gmail.com>
---
 scripts/compare-benchmark-jsons.py        |  7 +++++--
 scripts/tests/test_benchmark_reporting.py | 11 +++++++++++
 2 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/scripts/compare-benchmark-jsons.py b/scripts/compare-benchmark-jsons.py
index 2b6eafc7296..c575adf1eba 100644
--- a/scripts/compare-benchmark-jsons.py
+++ b/scripts/compare-benchmark-jsons.py
@@ -646,7 +646,7 @@ def format_pct_change(pct: float) -> str:
 
 
 def extract_file_size_data(df: pd.DataFrame) -> dict[tuple[str, str, str, str], int]:
-    """Extract file-size rows keyed by benchmark, scale factor, format, and file."""
+    """Extract nonzero file-size rows keyed by benchmark, scale factor, format, and file."""
 
     data = {}
     if df.empty:
@@ -666,7 +666,10 @@ def extract_file_size_data(df: pd.DataFrame) -> dict[tuple[str, str, str, str],
         value = row.get("value")
         if pd.isna(value):
             continue
-        data[key] = int(value)
+        size = int(value)
+        if size == 0:
+            continue
+        data[key] = size
 
     return data
 
diff --git a/scripts/tests/test_benchmark_reporting.py b/scripts/tests/test_benchmark_reporting.py
index b660a0df5cb..c75c9ebb1c0 100644
--- a/scripts/tests/test_benchmark_reporting.py
+++ b/scripts/tests/test_benchmark_reporting.py
@@ -428,6 +428,17 @@ def test_file_size_report_reads_shared_benchmark_rows() -> None:
     assert "| part-0.vortex | 10 | vortex-file-compressed | 100 B | 125 B | +25 B | +25.0% |" in report
 
 
+def test_file_size_report_ignores_zero_byte_files() -> None:
+    compare = load_compare_module()
+
+    report = compare.format_file_size_report(
+        pd.DataFrame([file_size_record("base-sha", 0)]),
+        pd.DataFrame([file_size_record("pr-sha", 0)]),
+    )
+
+    assert report == ""
+
+
 def test_file_size_report_ignores_baseline_rows_outside_pr_scope() -> None:
     compare = load_compare_module()
 

From b1b5ca54990e58c5bd948d3ae207b130727e3fc3 Mon Sep 17 00:00:00 2001
From: Connor Tsui <connor.tsui20@gmail.com>
Date: Fri, 31 Jul 2026 13:23:22 -0400
Subject: [PATCH 5/5] ci: include string benchmark in PR labels

Signed-off-by: Connor Tsui <connor.tsui20@gmail.com>
---
 .github/workflows/pr-bench-dispatch.yml | 21 +++++++++++++++++++++
 .github/workflows/pr-bench-string.yml   | 20 ++++++++++++++++++++
 docs/developer-guide/benchmarking.md    | 12 +++++++-----
 3 files changed, 48 insertions(+), 5 deletions(-)
 create mode 100644 .github/workflows/pr-bench-string.yml

diff --git a/.github/workflows/pr-bench-dispatch.yml b/.github/workflows/pr-bench-dispatch.yml
index 77b1afb3688..3cb60c619bf 100644
--- a/.github/workflows/pr-bench-dispatch.yml
+++ b/.github/workflows/pr-bench-dispatch.yml
@@ -40,6 +40,11 @@ jobs:
     uses: ./.github/workflows/pr-bench-compress.yml
     secrets: inherit
 
+  all-string-bench:
+    needs: remove-all-label
+    uses: ./.github/workflows/pr-bench-string.yml
+    secrets: inherit
+
   all-sql-bench:
     needs: remove-all-label
     uses: ./.github/workflows/pr-bench-sql.yml
@@ -79,6 +84,22 @@ jobs:
     uses: ./.github/workflows/pr-bench-compress.yml
     secrets: inherit
 
+  remove-string-label:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    if: github.event.label.name == 'action/bench-string'
+    steps:
+      - uses: actions-ecosystem/action-remove-labels@2ce5d41b4b6aa8503e285553f75ed56e0a40bae0  # v1
+        if: github.event.pull_request.head.repo.full_name == 'vortex-data/vortex'
+        with:
+          labels: action/bench-string
+          fail_on_error: true
+
+  string-bench:
+    needs: remove-string-label
+    uses: ./.github/workflows/pr-bench-string.yml
+    secrets: inherit
+
   remove-gpu-compress-label:
     runs-on: ubuntu-latest
     timeout-minutes: 10
diff --git a/.github/workflows/pr-bench-string.yml b/.github/workflows/pr-bench-string.yml
new file mode 100644
index 00000000000..e323f84a11a
--- /dev/null
+++ b/.github/workflows/pr-bench-string.yml
@@ -0,0 +1,20 @@
+# Runs the string encoding benchmark for a pull request.
+
+name: PR String Encoding Benchmark
+
+on:
+  workflow_call: { }
+  workflow_dispatch: { }
+
+permissions:
+  contents: read
+  pull-requests: write  # for commenting on PRs
+  id-token: write  # enables AWS-GitHub OIDC
+
+jobs:
+  bench:
+    uses: ./.github/workflows/pr-bench-runner.yml
+    secrets: inherit
+    with:
+      benchmark_id: string-bench
+      benchmark_name: String Encoding
diff --git a/docs/developer-guide/benchmarking.md b/docs/developer-guide/benchmarking.md
index fdd90aa28c5..74ce2c5f4f8 100644
--- a/docs/developer-guide/benchmarking.md
+++ b/docs/developer-guide/benchmarking.md
@@ -208,18 +208,20 @@ uv run --project bench-orchestrator vx-bench run tpch \
 
 Benchmarks run automatically on all commits to `develop` and can be run on-demand for PRs:
 
-- **Post-commit** -- compression, random access, and SQL benchmarks run on every commit to
-  `develop`, with results uploaded for historical tracking.
+- **Post-commit** -- compression, string encoding, random access, and SQL benchmarks run on every
+  commit to `develop`, with results uploaded for historical tracking.
 - **Random access** -- `action/bench-random-access` runs only the random-access benchmark.
 - **Compression** -- `action/bench-compress` runs only the compression benchmark.
+- **String encoding** -- `action/bench-string` runs only the string encoding benchmark.
 - **GPU compression** -- `action/bench-gpu-compress` runs the allow-listed Vortex decompression
   cases on a GPU runner.
 - **SQL** -- `action/bench-sql` runs the `pr` preset, which excludes `vortex-compact`.
 - **SQL Compact** -- `action/bench-sql-compact` runs the `pr-compact` preset, which benchmarks
   `vortex-compact` plus Parquet control rows used to distinguish code changes from runner drift.
-- **All CPU benchmarks** -- `action/bench-all` runs random access, compression, and the `pr-all`
-  SQL preset, which combines the `pr` and `pr-compact` coverage without repeating shared jobs. Do
-  not combine it with other benchmark labels; GPU compression is the only exception.
+- **All CPU benchmarks** -- `action/bench-all` runs random access, compression, string encoding,
+  and the `pr-all` SQL preset, which combines the `pr` and `pr-compact` coverage without
+  repeating shared jobs. Do not combine it with other benchmark labels; GPU compression is the
+  only exception.
 
 All CI benchmarks run on dedicated instances with the `release_debug` profile and
 `-C target-cpu=native` to produce representative numbers.