Add clickbench-sorted benchmark

.github/workflows/sql-benchmarks.yml

@@ -39,6 +39,29 @@ on:
                 {"engine": "duckdb", "format": "duckdb"}
               ]
             },
+            {
+              "id": "clickbench-sorted-nvme",
+              "subcommand": "clickbench-sorted",
+              "name": "Clickbench Sorted on NVME",
+              "data_formats": ["parquet", "vortex", "vortex-compact", "duckdb"],
+              "pr_targets": [
+                {"engine": "datafusion", "format": "parquet"},
+                {"engine": "datafusion", "format": "vortex"},
+                {"engine": "duckdb", "format": "parquet"},
+                {"engine": "duckdb", "format": "vortex"},
+                {"engine": "duckdb", "format": "duckdb"}
+              ],
+              "develop_targets": [
+                {"engine": "datafusion", "format": "parquet"},
+                {"engine": "datafusion", "format": "vortex"},
+                {"engine": "datafusion", "format": "vortex-compact"},
+                {"engine": "datafusion", "format": "lance"},
+                {"engine": "duckdb", "format": "parquet"},
+                {"engine": "duckdb", "format": "vortex"},
+                {"engine": "duckdb", "format": "vortex-compact"},
+                {"engine": "duckdb", "format": "duckdb"}
+              ]
+            },
             {
               "id": "tpch-nvme",
               "subcommand": "tpch",
@@ -302,6 +325,25 @@ on:
                 {"engine": "duckdb", "format": "vortex"}
               ]
             },
+            {
+              "id": "clickbench-sorted-nvme",
+              "subcommand": "clickbench-sorted",
+              "name": "Clickbench Sorted on NVME",
+              "data_formats": ["parquet", "vortex"],
+              "pr_targets": [
+                {"engine": "datafusion", "format": "parquet"},
+                {"engine": "datafusion", "format": "vortex"},
+                {"engine": "duckdb", "format": "parquet"},
+                {"engine": "duckdb", "format": "vortex"}
+              ],
+              "develop_targets": [
+                {"engine": "datafusion", "format": "parquet"},
+                {"engine": "datafusion", "format": "vortex"},
+                {"engine": "datafusion", "format": "lance"},
+                {"engine": "duckdb", "format": "parquet"},
+                {"engine": "duckdb", "format": "vortex"}
+              ]
+            },
             {
               "id": "tpch-nvme",
               "subcommand": "tpch",

bench-orchestrator/bench_orchestrator/config.py

@@ -46,6 +46,7 @@ class Benchmark(Enum):
     TPCH = "tpch"
     TPCDS = "tpcds"
     CLICKBENCH = "clickbench"
+    CLICKBENCH_SORTED = "clickbench-sorted"
     FINEWEB = "fineweb"
     GHARCHIVE = "gh-archive"
     POLARSIGNALS = "polarsignals"

benchmarks-website/src/config.js

@@ -14,6 +14,15 @@ export const QUERY_SUITES = [
     tags: ["Queries (NVMe)"],
     hiddenDatasets: ["datafusion:lance"],
   },
+  {
+    prefix: "clickbench-sorted",
+    displayName: "Clickbench Sorted",
+    queryPrefix: "CLICKBENCH SORTED",
+    description:
+      "ClickBench queries over data globally sorted by event date and event time",
+    tags: ["Queries (NVMe)"],
+    hiddenDatasets: ["datafusion:lance"],
+  },
   {
     prefix: "statpopgen",
     displayName: "Statistical and Population Genetics",

benchmarks-website/src/utils.js

@@ -97,6 +97,7 @@ export function getBenchmarkDescription(categoryName) {
     'Compression': 'Measures encoding and decoding throughput (MB/s) for Vortex and Parquet files',
     'Compression Size': 'Compares compressed file sizes across different encoding strategies',
     'Clickbench': "ClickHouse's analytical benchmark suite on web analytics data",
+    'Clickbench Sorted': 'ClickBench queries over data globally sorted by event date and event time',
     'Statistical and Population Genetics': 'Statistical and population genetics queries on gnomAD dataset',
   };
   return descriptions[categoryName] || '';

vortex-bench/src/clickbench/benchmark.rs

@@ -41,21 +41,41 @@ impl ClickBenchBenchmark {
     }
 }
 
+/// ClickBench sorted by event date and event time.
+pub struct ClickBenchSortedBenchmark {
+    pub queries_file: Option<String>,
+    pub data_url: Url,
+}
+
+impl ClickBenchSortedBenchmark {
+    /// Create the sorted ClickBench benchmark, optionally using a remote data directory.
+    pub fn new(use_remote_data_dir: Option<String>) -> Result<Self> {
+        Ok(Self {
+            queries_file: None,
+            data_url: resolve_data_url(use_remote_data_dir.as_deref(), CLICKBENCH_SORTED_NAME)?,
+        })
+    }
+}
+
+fn read_clickbench_queries(queries_file: Option<&str>) -> Result<Vec<(usize, String)>> {
+    let queries_filepath = match queries_file {
+        Some(file) => file.into(),
+        None => Path::new(env!("CARGO_MANIFEST_DIR")).join("clickbench_queries.sql"),
+    };
+
+    Ok(fs::read_to_string(queries_filepath)?
+        .split(';')
+        .map(|s| s.trim())
+        .filter(|s| !s.is_empty())
+        .map(|s| s.to_string())
+        .enumerate()
+        .collect())
+}
+
 #[async_trait::async_trait]
 impl Benchmark for ClickBenchBenchmark {
     fn queries(&self) -> Result<Vec<(usize, String)>> {
-        let queries_filepath = match &self.queries_file {
-            Some(file) => file.into(),
-            None => Path::new(env!("CARGO_MANIFEST_DIR")).join("clickbench_queries.sql"),
-        };
-
-        Ok(fs::read_to_string(queries_filepath)?
-            .split(';')
-            .map(|s| s.trim())
-            .filter(|s| !s.is_empty())
-            .map(|s| s.to_string())
-            .enumerate()
-            .collect())
+        read_clickbench_queries(self.queries_file.as_deref())
     }
 
     async fn generate_base_data(&self) -> Result<()> {
@@ -70,10 +90,7 @@ impl Benchmark for ClickBenchBenchmark {
     }
 
     fn expected_row_counts(&self) -> Option<Vec<usize>> {
-        Some(vec![
-            1, 1, 1, 1, 1, 1, 1, 18, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 4, 1, 10, 10, 10,
-            10, 10, 10, 25, 25, 1, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10,
-        ])
+        Some(clickbench_expected_row_counts())
     }
 
     fn dataset(&self) -> BenchmarkDataset {
@@ -99,6 +116,48 @@ impl Benchmark for ClickBenchBenchmark {
     }
 }
 
+#[async_trait::async_trait]
+impl Benchmark for ClickBenchSortedBenchmark {
+    fn queries(&self) -> Result<Vec<(usize, String)>> {
+        Ok(read_clickbench_queries(self.queries_file.as_deref())?
+            .into_iter()
+            .filter(|(idx, _)| CLICKBENCH_SORTED_QUERY_IDS.contains(idx))
+            .collect())
+    }
+
+    async fn generate_base_data(&self) -> Result<()> {
+        if self.data_url.scheme() != "file" {
+            return Ok(());
+        }
+
+        generate_sorted_clickbench(CLICKBENCH_SORTED_NAME.to_data_path()).await
+    }
+
+    fn expected_row_counts(&self) -> Option<Vec<usize>> {
+        Some(clickbench_expected_row_counts())
+    }
+
+    fn dataset(&self) -> BenchmarkDataset {
+        BenchmarkDataset::ClickBenchSorted
+    }
+
+    fn dataset_name(&self) -> &str {
+        CLICKBENCH_SORTED_NAME
+    }
+
+    fn dataset_display(&self) -> String {
+        CLICKBENCH_SORTED_NAME.to_string()
+    }
+
+    fn data_url(&self) -> &Url {
+        &self.data_url
+    }
+
+    fn table_specs(&self) -> Vec<TableSpec> {
+        vec![TableSpec::new("hits", Some(HITS_SCHEMA.clone()))]
+    }
+}
+
 fn clickbench_flavor(flavor: Flavor) -> String {
     format!("clickbench_{flavor}")
 }