From 0548331413d0d245419a662e9344180cc30a3ca3 Mon Sep 17 00:00:00 2001 From: Adam Gutglick Date: Fri, 17 Jul 2026 13:54:50 +0100 Subject: [PATCH] Skip building binaries in when there are no tests Signed-off-by: Adam Gutglick --- benchmarks/compress-bench/Cargo.toml | 7 + benchmarks/datafusion-bench/Cargo.toml | 7 + benchmarks/duckdb-bench/Cargo.toml | 7 + benchmarks/lance-bench/Cargo.toml | 7 + benchmarks/random-access-bench/Cargo.toml | 4 + benchmarks/random-access-bench/src/lib.rs | 488 +++++++++++++++++++ benchmarks/random-access-bench/src/main.rs | 518 ++------------------- vortex-bench/Cargo.toml | 4 + vortex-cuda/gpu-scan-cli/Cargo.toml | 4 + vortex-test/compat-gen/Cargo.toml | 1 + vortex-test/e2e-cuda/Cargo.toml | 4 + vortex-tui/Cargo.toml | 1 + xtask/Cargo.toml | 4 + 13 files changed, 565 insertions(+), 491 deletions(-) create mode 100644 benchmarks/random-access-bench/src/lib.rs diff --git a/benchmarks/compress-bench/Cargo.toml b/benchmarks/compress-bench/Cargo.toml index d214af23511..4e8fbc530b2 100644 --- a/benchmarks/compress-bench/Cargo.toml +++ b/benchmarks/compress-bench/Cargo.toml @@ -37,5 +37,12 @@ vortex-bench = { workspace = true } lance = ["dep:lance-bench"] unstable_encodings = ["vortex/unstable_encodings"] +[[bin]] +name = "compress-bench" +test = false + +[lib] +test = false + [lints] workspace = true diff --git a/benchmarks/datafusion-bench/Cargo.toml b/benchmarks/datafusion-bench/Cargo.toml index a195bbaa03f..e7e18e6376e 100644 --- a/benchmarks/datafusion-bench/Cargo.toml +++ b/benchmarks/datafusion-bench/Cargo.toml @@ -17,6 +17,13 @@ publish = false [package.metadata.cargo-shear] ignored = ["vortex-cuda"] +[[bin]] +name = "datafusion-bench" +test = false + +[lib] +test = false + [dependencies] anyhow = { workspace = true } clap = { workspace = true, features = ["derive"] } diff --git a/benchmarks/duckdb-bench/Cargo.toml b/benchmarks/duckdb-bench/Cargo.toml index 77c3d5280fc..609f2fe2150 100644 --- a/benchmarks/duckdb-bench/Cargo.toml +++ b/benchmarks/duckdb-bench/Cargo.toml @@ -17,6 +17,13 @@ publish = false [package.metadata.cargo-shear] ignored = ["vortex-cuda"] +[[bin]] +name = "duckdb-bench" +test = false + +[lib] +test = false + [dependencies] anyhow = { workspace = true } clap = { workspace = true, features = ["derive"] } diff --git a/benchmarks/lance-bench/Cargo.toml b/benchmarks/lance-bench/Cargo.toml index 7cab6af2119..44ae517c16a 100644 --- a/benchmarks/lance-bench/Cargo.toml +++ b/benchmarks/lance-bench/Cargo.toml @@ -32,5 +32,12 @@ vortex-bench = { workspace = true } [features] unstable_encodings = ["vortex-bench/unstable_encodings"] +[lib] +test = false + +[[bin]] +name = "lance-bench" +test = false + [lints] workspace = true diff --git a/benchmarks/random-access-bench/Cargo.toml b/benchmarks/random-access-bench/Cargo.toml index 500db9ec7c4..c45c1ac2bff 100644 --- a/benchmarks/random-access-bench/Cargo.toml +++ b/benchmarks/random-access-bench/Cargo.toml @@ -29,5 +29,9 @@ vortex-bench = { workspace = true } lance = ["dep:lance-bench"] unstable_encodings = ["vortex-bench/unstable_encodings"] +[[bin]] +name = "random-access-bench" +test = false + [lints] workspace = true diff --git a/benchmarks/random-access-bench/src/lib.rs b/benchmarks/random-access-bench/src/lib.rs new file mode 100644 index 00000000000..7d4b7d2c389 --- /dev/null +++ b/benchmarks/random-access-bench/src/lib.rs @@ -0,0 +1,488 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +use std::path::PathBuf; +use std::time::Duration; +use std::time::Instant; + +use anyhow::Result; +use clap::ValueEnum; +use indicatif::ProgressBar; +use rand::RngExt; +use rand::SeedableRng; +use rand::rngs::StdRng; +use rand_distr::Distribution; +use rand_distr::Exp; +use vortex_bench::Engine; +use vortex_bench::Format; +use vortex_bench::Target; +use vortex_bench::create_output_writer; +use vortex_bench::display::DisplayFormat; +use vortex_bench::display::print_measurements_json; +use vortex_bench::measurements::TimingMeasurement; +use vortex_bench::random_access::BenchDataset; +use vortex_bench::random_access::ParquetRandomAccessor; +use vortex_bench::random_access::RandomAccessor; +use vortex_bench::random_access::VortexRandomAccessor; +use vortex_bench::utils::constants::STORAGE_NVME; +use vortex_bench::v3; + +use crate::render::RandomAccessRun; +use crate::render::render_random_access_table; + +mod render; + +// --------------------------------------------------------------------------- +// Access patterns +// --------------------------------------------------------------------------- + +/// Access pattern for random access benchmarks. +#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq, ValueEnum)] +pub enum AccessPattern { + /// Multiple clusters of sequential indices scattered across the dataset, + /// simulating workloads with spatial locality (e.g. scanning nearby records). + Correlated, + /// Indices generated by a Poisson process (exponential inter-arrival times) + /// spread uniformly across the dataset, simulating random lookups with no locality. + Uniform, +} + +impl AccessPattern { + fn name(&self) -> &'static str { + match self { + AccessPattern::Correlated => "correlated", + AccessPattern::Uniform => "uniform", + } + } +} + +/// Number of clusters for the correlated pattern. +const NUM_CLUSTERS: usize = 5; + +/// Number of consecutive indices per cluster. +const CLUSTER_SIZE: usize = 20; + +/// Expected number of indices for the Poisson (uniform) pattern. +const POISSON_EXPECTED_COUNT: usize = 100; + +/// Generate indices for the given dataset and access pattern. +fn generate_indices(dataset: &dyn BenchDataset, pattern: AccessPattern) -> Vec { + let row_count = dataset.row_count(); + let mut rng = StdRng::seed_from_u64(42); + + match pattern { + AccessPattern::Correlated => { + // Pick random cluster starts, then emit CLUSTER_SIZE consecutive indices from each. + let mut indices = Vec::with_capacity(NUM_CLUSTERS * CLUSTER_SIZE); + for _ in 0..NUM_CLUSTERS { + let start = rng.random_range(0..row_count.saturating_sub(CLUSTER_SIZE as u64)); + for offset in 0..CLUSTER_SIZE as u64 { + indices.push(start + offset); + } + } + indices.sort_unstable(); + indices + } + AccessPattern::Uniform => { + // Poisson process: exponential inter-arrival times with rate chosen to yield + // ~POISSON_EXPECTED_COUNT indices across the dataset. + let rate = POISSON_EXPECTED_COUNT as f64 / row_count as f64; + // SAFETY: rate is always positive (POISSON_EXPECTED_COUNT > 0, row_count > 0). + #[expect(clippy::unwrap_used)] + let exp = Exp::new(rate).unwrap(); + let mut indices = Vec::with_capacity(POISSON_EXPECTED_COUNT); + let mut pos = 0.0_f64; + loop { + let gap: f64 = exp.sample(&mut rng); + pos += gap; + #[expect(clippy::cast_possible_truncation)] + let idx = pos as u64; + if idx >= row_count { + break; + } + indices.push(idx); + } + indices + } + } +} + +// --------------------------------------------------------------------------- +// Benchmark configuration +// --------------------------------------------------------------------------- + +/// Controls whether the file handle is reused or reopened each iteration. +#[derive(ValueEnum, Clone, Copy, Debug, PartialEq, Eq)] +pub enum OpenMode { + /// Reuse the file handle across iterations (cached metadata). + #[clap(name = "cached")] + Cached, + /// Reopen the file each iteration (includes footer parsing). + #[clap(name = "reopen")] + Reopen, + /// Run both cached and reopen variants. + #[clap(name = "both")] + Both, +} + +// --------------------------------------------------------------------------- +// Benchmark core +// --------------------------------------------------------------------------- + +/// Run a random access benchmark. +/// +/// Runs the take operation repeatedly until the time limit is reached, +/// collecting timing for each run. When `reopen` is true, the accessor is +/// recreated from scratch before each iteration so that file metadata +/// parsing is included in the timing. +#[expect(clippy::too_many_arguments)] +async fn benchmark_random_access( + dataset: &dyn BenchDataset, + format: Format, + measurement_name: &str, + pattern: Option, + indices: &[u64], + time_limit_secs: u64, + storage: &str, + reopen: bool, +) -> Result { + let time_limit = Duration::from_secs(time_limit_secs); + let overall_start = Instant::now(); + let mut runs = Vec::new(); + let mut accessor = open_accessor(dataset, format).await?; + + loop { + let start = Instant::now(); + let arr = accessor.take(indices).await?; + runs.push(start.elapsed()); + drop(arr); + + if overall_start.elapsed() >= time_limit { + break; + } + + if reopen { + accessor = open_accessor(dataset, format).await?; + } + } + + let timing = TimingMeasurement { + name: measurement_name.to_string(), + storage: storage.to_string(), + target: Target::new(format_to_engine(format), format), + runs, + }; + Ok(RandomAccessRun { + display_name: display_name(dataset.name(), pattern), + dataset: dataset.name().to_string(), + pattern, + reopen, + timing, + }) +} + +/// Row label for the table view. Format is implied by the column header, so +/// it is omitted from the row label even though it stays in the +/// [`TimingMeasurement::name`] used for JSON back-compat. +fn display_name(dataset: &str, pattern: Option) -> String { + match pattern { + Some(p) => format!("random-access/{}/{}", dataset, p.name()), + None => format!("random-access/{}", dataset), + } +} + +/// Build a measurement name for a benchmark run. +/// +/// For taxi (legacy), the name is `random-access/{format}-tokio-local-disk` to preserve +/// historical continuity with existing benchmark data. +/// For other datasets, includes dataset and pattern: +/// `random-access/{dataset}/{pattern}/{format}-tokio-local-disk`. +fn measurement_name(dataset: &str, pattern: Option, format: Format) -> String { + let fmt = format.ext(); + match pattern { + Some(p) => format!( + "random-access/{}/{}/{}-tokio-local-disk", + dataset, + p.name(), + fmt + ), + None => format!("random-access/{}-tokio-local-disk", fmt), + } +} + +fn v3_random_access_dataset_name(dataset: &str, pattern: Option) -> String { + match pattern { + Some(pattern) => format!("{dataset}/{}", pattern.name()), + None => dataset.to_string(), + } +} + +fn push_v3_random_access_record(records: &mut Vec, run: &RandomAccessRun) { + if run.reopen { + return; + } + + let dataset = v3_random_access_dataset_name(&run.dataset, run.pattern); + records.push(v3::random_access_record(&run.timing, &dataset)); +} + +/// Map format to the appropriate engine for random access benchmarks. +fn format_to_engine(format: Format) -> Engine { + match format { + Format::OnDiskVortex | Format::VortexCompact => Engine::Vortex, + Format::Parquet => Engine::Arrow, + #[cfg(feature = "lance")] + Format::Lance => Engine::Arrow, // Is this right here? + _ => Engine::default(), + } +} + +/// Open a random accessor for any supported format. +/// +/// For Vortex and Parquet, the path comes from [`BenchDataset::path`]. +/// For Lance (behind the `lance` feature), the path is resolved from lance-bench helpers. +async fn open_accessor( + dataset: &dyn BenchDataset, + format: Format, +) -> Result> { + let name = format!( + "random-access/{}/{}-tokio-local-disk", + dataset.name(), + format.ext() + ); + match format { + Format::OnDiskVortex | Format::VortexCompact => { + let path = dataset.path(format).await?; + Ok(Box::new( + VortexRandomAccessor::open(path, name, format).await?, + )) + } + Format::Parquet => { + let path = dataset.path(format).await?; + Ok(Box::new(ParquetRandomAccessor::open(path, name).await?)) + } + #[cfg(feature = "lance")] + Format::Lance => { + use lance_bench::random_access; + let path = match dataset.name() { + "taxi" => random_access::taxi_data_lance().await?, + "feature-vectors" => random_access::feature_vectors_lance().await?, + "nested-lists" => random_access::nested_lists_lance().await?, + "nested-structs" => random_access::nested_structs_lance().await?, + other => anyhow::bail!("Unknown dataset for Lance: {other}"), + }; + Ok(Box::new( + random_access::LanceRandomAccessor::open(path, name).await?, + )) + } + other => unimplemented!("open_accessor not implemented for {other}"), + } +} + +/// The benchmark ID used for output path. +const BENCHMARK_ID: &str = "random-access"; + +/// Fixed indices used by the original taxi benchmark (preserved for historical continuity). +const FIXED_TAXI_INDICES: [u64; 6] = [10, 11, 12, 13, 100_000, 3_000_000]; + +/// Resolved configuration for a single random-access benchmark invocation. +pub struct RunConfig { + /// Datasets to benchmark. + pub datasets: Vec>, + /// File formats to measure for each dataset. + pub formats: Vec, + /// Index access patterns to measure. + pub patterns: Vec, + /// Maximum duration of each benchmark target in seconds. + pub time_limit: u64, + /// Whether each benchmark run reuses or reopens its file handle. + pub open_mode: OpenMode, + /// Format used to emit measurements. + pub display_format: DisplayFormat, + /// Optional path for the primary result output. + pub output_path: Option, + /// Optional path for v3 JSONL benchmark records. + pub gh_json_v3: Option, +} + +/// Run random-access benchmarks with `config`. +pub async fn run(config: RunConfig) -> Result<()> { + let RunConfig { + datasets, + formats, + patterns, + time_limit, + open_mode, + display_format, + output_path, + gh_json_v3, + } = config; + + let reopen_variants: &[bool] = match open_mode { + OpenMode::Cached => &[false], + OpenMode::Reopen => &[true], + OpenMode::Both => &[false, true], + }; + + let total_steps: usize = datasets + .iter() + .map(|d| { + let legacy_extra = if d.name() == "taxi" { formats.len() } else { 0 }; + (formats.len() * patterns.len() + legacy_extra) * reopen_variants.len() + }) + .sum(); + let progress = ProgressBar::new(total_steps as u64); + + let mut runs: Vec = Vec::new(); + let mut v3_records: Vec = Vec::new(); + + // Iteration order matters for the table renderer: row order is set by the + // first time each `(dataset, pattern)` pair is observed. + for dataset in &datasets { + for format in &formats { + if dataset.name() == "taxi" { + let name = measurement_name(dataset.name(), None, *format); + for &reopen in reopen_variants { + let bench_name = if reopen { + format!("{name}-footer") + } else { + name.clone() + }; + let run = benchmark_random_access( + dataset.as_ref(), + *format, + &bench_name, + None, + &FIXED_TAXI_INDICES, + time_limit, + STORAGE_NVME, + reopen, + ) + .await?; + + push_v3_random_access_record(&mut v3_records, &run); + runs.push(run); + progress.inc(1); + } + } + + for pattern in &patterns { + let indices = generate_indices(dataset.as_ref(), *pattern); + let name = measurement_name(dataset.name(), Some(*pattern), *format); + for &reopen in reopen_variants { + let bench_name = if reopen { + format!("{name}-footer") + } else { + name.clone() + }; + let run = benchmark_random_access( + dataset.as_ref(), + *format, + &bench_name, + Some(*pattern), + &indices, + time_limit, + STORAGE_NVME, + reopen, + ) + .await?; + + push_v3_random_access_record(&mut v3_records, &run); + runs.push(run); + progress.inc(1); + } + } + } + } + + progress.finish(); + + if let Some(path) = gh_json_v3 { + v3::write_jsonl_to_path(&path, &v3_records)?; + } + + let mut writer = create_output_writer(&display_format, output_path, BENCHMARK_ID)?; + + match display_format { + DisplayFormat::Table => { + render_random_access_table(&mut writer, &runs, &formats, reopen_variants)?; + } + DisplayFormat::GhJson => { + let timings: Vec = runs.into_iter().map(|r| r.timing).collect(); + print_measurements_json(&mut writer, timings)?; + } + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn v3_random_access_dataset_names_match_schema_dims() { + assert_eq!(v3_random_access_dataset_name("taxi", None), "taxi"); + assert_eq!( + v3_random_access_dataset_name("taxi", Some(AccessPattern::Correlated)), + "taxi/correlated" + ); + assert_eq!( + v3_random_access_dataset_name("feature-vectors", Some(AccessPattern::Uniform)), + "feature-vectors/uniform" + ); + } + + fn fake_run(dataset: &str, pattern: Option, reopen: bool) -> RandomAccessRun { + RandomAccessRun { + timing: TimingMeasurement { + name: format!("random-access/{dataset}/parquet-tokio-local-disk"), + target: Target::new(Engine::Arrow, Format::Parquet), + storage: STORAGE_NVME.to_string(), + runs: vec![Duration::from_nanos(10)], + }, + dataset: dataset.to_string(), + pattern, + reopen, + display_name: display_name(dataset, pattern), + } + } + + #[test] + fn v3_random_access_records_skip_reopen_variants() { + let mut records = Vec::new(); + + push_v3_random_access_record(&mut records, &fake_run("taxi", None, false)); + push_v3_random_access_record( + &mut records, + &fake_run("taxi", Some(AccessPattern::Uniform), false), + ); + push_v3_random_access_record( + &mut records, + &fake_run("taxi", Some(AccessPattern::Correlated), true), + ); + + assert_eq!(records.len(), 2); + match &records[0] { + v3::V3Record::RandomAccessTime(record) => assert_eq!(record.dataset, "taxi"), + other => panic!("expected random-access record, got {other:?}"), + } + match &records[1] { + v3::V3Record::RandomAccessTime(record) => assert_eq!(record.dataset, "taxi/uniform"), + other => panic!("expected random-access record, got {other:?}"), + } + } + + #[test] + fn display_name_drops_format_extension() { + assert_eq!(display_name("taxi", None), "random-access/taxi"); + assert_eq!( + display_name("taxi", Some(AccessPattern::Uniform)), + "random-access/taxi/uniform" + ); + assert_eq!( + display_name("feature-vectors", Some(AccessPattern::Correlated)), + "random-access/feature-vectors/correlated" + ); + } +} diff --git a/benchmarks/random-access-bench/src/main.rs b/benchmarks/random-access-bench/src/main.rs index 6722265fde6..f3d46ef2ff6 100644 --- a/benchmarks/random-access-bench/src/main.rs +++ b/benchmarks/random-access-bench/src/main.rs @@ -2,134 +2,21 @@ // SPDX-FileCopyrightText: Copyright the Vortex contributors use std::path::PathBuf; -use std::time::Duration; -use std::time::Instant; use anyhow::Result; use clap::Parser; use clap::ValueEnum; -use indicatif::ProgressBar; -use rand::RngExt; -use rand::SeedableRng; -use rand::rngs::StdRng; -use rand_distr::Distribution; -use rand_distr::Exp; -use vortex_bench::Engine; +use random_access_bench::AccessPattern; +use random_access_bench::OpenMode; +use random_access_bench::RunConfig; use vortex_bench::Format; -use vortex_bench::Target; -use vortex_bench::create_output_writer; use vortex_bench::datasets::feature_vectors::FeatureVectorsData; use vortex_bench::datasets::nested_lists::NestedListsData; use vortex_bench::datasets::nested_structs::NestedStructsData; use vortex_bench::datasets::taxi_data::TaxiData; use vortex_bench::display::DisplayFormat; -use vortex_bench::display::print_measurements_json; -use vortex_bench::measurements::TimingMeasurement; use vortex_bench::random_access::BenchDataset; -use vortex_bench::random_access::ParquetRandomAccessor; -use vortex_bench::random_access::RandomAccessor; -use vortex_bench::random_access::VortexRandomAccessor; use vortex_bench::setup_logging_and_tracing; -use vortex_bench::utils::constants::STORAGE_NVME; -use vortex_bench::v3; - -use crate::render::RandomAccessRun; -use crate::render::render_random_access_table; - -mod render; - -// --------------------------------------------------------------------------- -// Access patterns -// --------------------------------------------------------------------------- - -/// Access pattern for random access benchmarks. -#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq, ValueEnum)] -pub enum AccessPattern { - /// Multiple clusters of sequential indices scattered across the dataset, - /// simulating workloads with spatial locality (e.g. scanning nearby records). - Correlated, - /// Indices generated by a Poisson process (exponential inter-arrival times) - /// spread uniformly across the dataset, simulating random lookups with no locality. - Uniform, -} - -impl AccessPattern { - fn name(&self) -> &'static str { - match self { - AccessPattern::Correlated => "correlated", - AccessPattern::Uniform => "uniform", - } - } -} - -/// Number of clusters for the correlated pattern. -const NUM_CLUSTERS: usize = 5; - -/// Number of consecutive indices per cluster. -const CLUSTER_SIZE: usize = 20; - -/// Expected number of indices for the Poisson (uniform) pattern. -const POISSON_EXPECTED_COUNT: usize = 100; - -/// Generate indices for the given dataset and access pattern. -fn generate_indices(dataset: &dyn BenchDataset, pattern: AccessPattern) -> Vec { - let row_count = dataset.row_count(); - let mut rng = StdRng::seed_from_u64(42); - - match pattern { - AccessPattern::Correlated => { - // Pick random cluster starts, then emit CLUSTER_SIZE consecutive indices from each. - let mut indices = Vec::with_capacity(NUM_CLUSTERS * CLUSTER_SIZE); - for _ in 0..NUM_CLUSTERS { - let start = rng.random_range(0..row_count.saturating_sub(CLUSTER_SIZE as u64)); - for offset in 0..CLUSTER_SIZE as u64 { - indices.push(start + offset); - } - } - indices.sort_unstable(); - indices - } - AccessPattern::Uniform => { - // Poisson process: exponential inter-arrival times with rate chosen to yield - // ~POISSON_EXPECTED_COUNT indices across the dataset. - let rate = POISSON_EXPECTED_COUNT as f64 / row_count as f64; - // SAFETY: rate is always positive (POISSON_EXPECTED_COUNT > 0, row_count > 0). - #[expect(clippy::unwrap_used)] - let exp = Exp::new(rate).unwrap(); - let mut indices = Vec::with_capacity(POISSON_EXPECTED_COUNT); - let mut pos = 0.0_f64; - loop { - let gap: f64 = exp.sample(&mut rng); - pos += gap; - #[expect(clippy::cast_possible_truncation)] - let idx = pos as u64; - if idx >= row_count { - break; - } - indices.push(idx); - } - indices - } - } -} - -// --------------------------------------------------------------------------- -// CLI -// --------------------------------------------------------------------------- - -/// Controls whether the file handle is reused or reopened each iteration. -#[derive(ValueEnum, Clone, Copy, Debug, PartialEq, Eq)] -enum OpenMode { - /// Reuse the file handle across iterations (cached metadata). - #[clap(name = "cached")] - Cached, - /// Reopen the file each iteration (includes footer parsing). - #[clap(name = "reopen")] - Reopen, - /// Run both cached and reopen variants. - #[clap(name = "both")] - Both, -} /// Which synthetic dataset to benchmark. #[derive(ValueEnum, Clone, Copy, Debug)] @@ -147,10 +34,10 @@ enum DatasetArg { impl DatasetArg { fn into_dataset(self) -> Box { match self { - DatasetArg::Taxi => Box::new(TaxiData), - DatasetArg::FeatureVectors => Box::new(FeatureVectorsData), - DatasetArg::NestedLists => Box::new(NestedListsData), - DatasetArg::NestedStructs => Box::new(NestedStructsData), + Self::Taxi => Box::new(TaxiData), + Self::FeatureVectors => Box::new(FeatureVectorsData), + Self::NestedLists => Box::new(NestedListsData), + Self::NestedStructs => Box::new(NestedStructsData), } } } @@ -201,379 +88,28 @@ struct Args { open_mode: OpenMode, } -#[tokio::main] -async fn main() -> Result<()> { - let args = Args::parse(); - - setup_logging_and_tracing(args.verbose, args.tracing)?; - - let config = RunConfig { - datasets: args - .datasets - .into_iter() - .map(|d| d.into_dataset()) - .collect(), - formats: args.formats, - patterns: args.patterns, - time_limit: args.time_limit, - open_mode: args.open_mode, - display_format: args.display_format, - output_path: args.output_path, - gh_json_v3: args.gh_json_v3, - }; - - run_random_access(config).await -} - -// --------------------------------------------------------------------------- -// Benchmark core -// --------------------------------------------------------------------------- - -/// Run a random access benchmark. -/// -/// Runs the take operation repeatedly until the time limit is reached, -/// collecting timing for each run. When `reopen` is true, the accessor is -/// recreated from scratch before each iteration so that file metadata -/// parsing is included in the timing. -#[expect(clippy::too_many_arguments)] -async fn benchmark_random_access( - dataset: &dyn BenchDataset, - format: Format, - measurement_name: &str, - pattern: Option, - indices: &[u64], - time_limit_secs: u64, - storage: &str, - reopen: bool, -) -> Result { - let time_limit = Duration::from_secs(time_limit_secs); - let overall_start = Instant::now(); - let mut runs = Vec::new(); - let mut accessor = open_accessor(dataset, format).await?; - - loop { - let start = Instant::now(); - let arr = accessor.take(indices).await?; - runs.push(start.elapsed()); - drop(arr); - - if overall_start.elapsed() >= time_limit { - break; - } - - if reopen { - accessor = open_accessor(dataset, format).await?; +impl Args { + fn into_run_config(self) -> RunConfig { + RunConfig { + datasets: self + .datasets + .into_iter() + .map(DatasetArg::into_dataset) + .collect(), + formats: self.formats, + patterns: self.patterns, + time_limit: self.time_limit, + open_mode: self.open_mode, + display_format: self.display_format, + output_path: self.output_path, + gh_json_v3: self.gh_json_v3, } } - - let timing = TimingMeasurement { - name: measurement_name.to_string(), - storage: storage.to_string(), - target: Target::new(format_to_engine(format), format), - runs, - }; - Ok(RandomAccessRun { - display_name: display_name(dataset.name(), pattern), - dataset: dataset.name().to_string(), - pattern, - reopen, - timing, - }) } -/// Row label for the table view. Format is implied by the column header, so -/// it is omitted from the row label even though it stays in the -/// [`TimingMeasurement::name`] used for JSON back-compat. -fn display_name(dataset: &str, pattern: Option) -> String { - match pattern { - Some(p) => format!("random-access/{}/{}", dataset, p.name()), - None => format!("random-access/{}", dataset), - } -} - -/// Build a measurement name for a benchmark run. -/// -/// For taxi (legacy), the name is `random-access/{format}-tokio-local-disk` to preserve -/// historical continuity with existing benchmark data. -/// For other datasets, includes dataset and pattern: -/// `random-access/{dataset}/{pattern}/{format}-tokio-local-disk`. -fn measurement_name(dataset: &str, pattern: Option, format: Format) -> String { - let fmt = format.ext(); - match pattern { - Some(p) => format!( - "random-access/{}/{}/{}-tokio-local-disk", - dataset, - p.name(), - fmt - ), - None => format!("random-access/{}-tokio-local-disk", fmt), - } -} - -fn v3_random_access_dataset_name(dataset: &str, pattern: Option) -> String { - match pattern { - Some(pattern) => format!("{dataset}/{}", pattern.name()), - None => dataset.to_string(), - } -} - -fn push_v3_random_access_record(records: &mut Vec, run: &RandomAccessRun) { - if run.reopen { - return; - } - - let dataset = v3_random_access_dataset_name(&run.dataset, run.pattern); - records.push(v3::random_access_record(&run.timing, &dataset)); -} - -/// Map format to the appropriate engine for random access benchmarks. -fn format_to_engine(format: Format) -> Engine { - match format { - Format::OnDiskVortex | Format::VortexCompact => Engine::Vortex, - Format::Parquet => Engine::Arrow, - #[cfg(feature = "lance")] - Format::Lance => Engine::Arrow, // Is this right here? - _ => Engine::default(), - } -} - -/// Open a random accessor for any supported format. -/// -/// For Vortex and Parquet, the path comes from [`BenchDataset::path`]. -/// For Lance (behind the `lance` feature), the path is resolved from lance-bench helpers. -async fn open_accessor( - dataset: &dyn BenchDataset, - format: Format, -) -> Result> { - let name = format!( - "random-access/{}/{}-tokio-local-disk", - dataset.name(), - format.ext() - ); - match format { - Format::OnDiskVortex | Format::VortexCompact => { - let path = dataset.path(format).await?; - Ok(Box::new( - VortexRandomAccessor::open(path, name, format).await?, - )) - } - Format::Parquet => { - let path = dataset.path(format).await?; - Ok(Box::new(ParquetRandomAccessor::open(path, name).await?)) - } - #[cfg(feature = "lance")] - Format::Lance => { - use lance_bench::random_access; - let path = match dataset.name() { - "taxi" => random_access::taxi_data_lance().await?, - "feature-vectors" => random_access::feature_vectors_lance().await?, - "nested-lists" => random_access::nested_lists_lance().await?, - "nested-structs" => random_access::nested_structs_lance().await?, - other => anyhow::bail!("Unknown dataset for Lance: {other}"), - }; - Ok(Box::new( - random_access::LanceRandomAccessor::open(path, name).await?, - )) - } - other => unimplemented!("open_accessor not implemented for {other}"), - } -} - -/// The benchmark ID used for output path. -const BENCHMARK_ID: &str = "random-access"; - -/// Fixed indices used by the original taxi benchmark (preserved for historical continuity). -const FIXED_TAXI_INDICES: [u64; 6] = [10, 11, 12, 13, 100_000, 3_000_000]; - -/// Resolved configuration for a single random-access benchmark invocation. -struct RunConfig { - datasets: Vec>, - formats: Vec, - patterns: Vec, - time_limit: u64, - open_mode: OpenMode, - display_format: DisplayFormat, - output_path: Option, - gh_json_v3: Option, -} - -async fn run_random_access(config: RunConfig) -> Result<()> { - let RunConfig { - datasets, - formats, - patterns, - time_limit, - open_mode, - display_format, - output_path, - gh_json_v3, - } = config; - - let reopen_variants: &[bool] = match open_mode { - OpenMode::Cached => &[false], - OpenMode::Reopen => &[true], - OpenMode::Both => &[false, true], - }; - - let total_steps: usize = datasets - .iter() - .map(|d| { - let legacy_extra = if d.name() == "taxi" { formats.len() } else { 0 }; - (formats.len() * patterns.len() + legacy_extra) * reopen_variants.len() - }) - .sum(); - let progress = ProgressBar::new(total_steps as u64); - - let mut runs: Vec = Vec::new(); - let mut v3_records: Vec = Vec::new(); - - // Iteration order matters for the table renderer: row order is set by the - // first time each `(dataset, pattern)` pair is observed. - for dataset in &datasets { - for format in &formats { - if dataset.name() == "taxi" { - let name = measurement_name(dataset.name(), None, *format); - for &reopen in reopen_variants { - let bench_name = if reopen { - format!("{name}-footer") - } else { - name.clone() - }; - let run = benchmark_random_access( - dataset.as_ref(), - *format, - &bench_name, - None, - &FIXED_TAXI_INDICES, - time_limit, - STORAGE_NVME, - reopen, - ) - .await?; - - push_v3_random_access_record(&mut v3_records, &run); - runs.push(run); - progress.inc(1); - } - } - - for pattern in &patterns { - let indices = generate_indices(dataset.as_ref(), *pattern); - let name = measurement_name(dataset.name(), Some(*pattern), *format); - for &reopen in reopen_variants { - let bench_name = if reopen { - format!("{name}-footer") - } else { - name.clone() - }; - let run = benchmark_random_access( - dataset.as_ref(), - *format, - &bench_name, - Some(*pattern), - &indices, - time_limit, - STORAGE_NVME, - reopen, - ) - .await?; - - push_v3_random_access_record(&mut v3_records, &run); - runs.push(run); - progress.inc(1); - } - } - } - } - - progress.finish(); - - if let Some(path) = gh_json_v3 { - v3::write_jsonl_to_path(&path, &v3_records)?; - } - - let mut writer = create_output_writer(&display_format, output_path, BENCHMARK_ID)?; - - match display_format { - DisplayFormat::Table => { - render_random_access_table(&mut writer, &runs, &formats, reopen_variants)?; - } - DisplayFormat::GhJson => { - let timings: Vec = runs.into_iter().map(|r| r.timing).collect(); - print_measurements_json(&mut writer, timings)?; - } - } - - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn v3_random_access_dataset_names_match_schema_dims() { - assert_eq!(v3_random_access_dataset_name("taxi", None), "taxi"); - assert_eq!( - v3_random_access_dataset_name("taxi", Some(AccessPattern::Correlated)), - "taxi/correlated" - ); - assert_eq!( - v3_random_access_dataset_name("feature-vectors", Some(AccessPattern::Uniform)), - "feature-vectors/uniform" - ); - } - - fn fake_run(dataset: &str, pattern: Option, reopen: bool) -> RandomAccessRun { - RandomAccessRun { - timing: TimingMeasurement { - name: format!("random-access/{dataset}/parquet-tokio-local-disk"), - target: Target::new(Engine::Arrow, Format::Parquet), - storage: STORAGE_NVME.to_string(), - runs: vec![Duration::from_nanos(10)], - }, - dataset: dataset.to_string(), - pattern, - reopen, - display_name: display_name(dataset, pattern), - } - } - - #[test] - fn v3_random_access_records_skip_reopen_variants() { - let mut records = Vec::new(); - - push_v3_random_access_record(&mut records, &fake_run("taxi", None, false)); - push_v3_random_access_record( - &mut records, - &fake_run("taxi", Some(AccessPattern::Uniform), false), - ); - push_v3_random_access_record( - &mut records, - &fake_run("taxi", Some(AccessPattern::Correlated), true), - ); - - assert_eq!(records.len(), 2); - match &records[0] { - v3::V3Record::RandomAccessTime(record) => assert_eq!(record.dataset, "taxi"), - other => panic!("expected random-access record, got {other:?}"), - } - match &records[1] { - v3::V3Record::RandomAccessTime(record) => assert_eq!(record.dataset, "taxi/uniform"), - other => panic!("expected random-access record, got {other:?}"), - } - } - - #[test] - fn display_name_drops_format_extension() { - assert_eq!(display_name("taxi", None), "random-access/taxi"); - assert_eq!( - display_name("taxi", Some(AccessPattern::Uniform)), - "random-access/taxi/uniform" - ); - assert_eq!( - display_name("feature-vectors", Some(AccessPattern::Correlated)), - "random-access/feature-vectors/correlated" - ); - } +#[tokio::main] +async fn main() -> Result<()> { + let args = Args::parse(); + setup_logging_and_tracing(args.verbose, args.tracing)?; + random_access_bench::run(args.into_run_config()).await } diff --git a/vortex-bench/Cargo.toml b/vortex-bench/Cargo.toml index f228e1571d3..02f043b8e68 100644 --- a/vortex-bench/Cargo.toml +++ b/vortex-bench/Cargo.toml @@ -13,6 +13,10 @@ repository = { workspace = true } rust-version = { workspace = true } version = { workspace = true } +[[bin]] +name = "data-gen" +test = false + [lints] workspace = true diff --git a/vortex-cuda/gpu-scan-cli/Cargo.toml b/vortex-cuda/gpu-scan-cli/Cargo.toml index 9234d303952..468158a35ba 100644 --- a/vortex-cuda/gpu-scan-cli/Cargo.toml +++ b/vortex-cuda/gpu-scan-cli/Cargo.toml @@ -12,6 +12,10 @@ repository = { workspace = true } rust-version = { workspace = true } version = { workspace = true } +[[bin]] +name = "gpu-scan-cli" +test = false + [lints] workspace = true diff --git a/vortex-test/compat-gen/Cargo.toml b/vortex-test/compat-gen/Cargo.toml index 4fbd90a6273..4a62aca3671 100644 --- a/vortex-test/compat-gen/Cargo.toml +++ b/vortex-test/compat-gen/Cargo.toml @@ -18,6 +18,7 @@ workspace = true [[bin]] name = "vortex-compat" path = "src/main.rs" +test = false [dependencies] # Vortex crates diff --git a/vortex-test/e2e-cuda/Cargo.toml b/vortex-test/e2e-cuda/Cargo.toml index 4ef30925b11..3985b9d11e5 100644 --- a/vortex-test/e2e-cuda/Cargo.toml +++ b/vortex-test/e2e-cuda/Cargo.toml @@ -15,6 +15,10 @@ version = { workspace = true } [lib] crate-type = ["cdylib"] +[[bin]] +name = "cudf_harness_runner" +test = false + [lints] workspace = true diff --git a/vortex-tui/Cargo.toml b/vortex-tui/Cargo.toml index 34b90ecfc05..09c929796d7 100644 --- a/vortex-tui/Cargo.toml +++ b/vortex-tui/Cargo.toml @@ -39,6 +39,7 @@ crate-type = ["cdylib", "rlib"] name = "vx" path = "src/main.rs" required-features = ["native"] +test = false [dependencies] # Shared dependencies diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml index ebbc6987d46..eae2db43413 100644 --- a/xtask/Cargo.toml +++ b/xtask/Cargo.toml @@ -15,6 +15,10 @@ repository = { workspace = true } rust-version = { workspace = true } version = { workspace = true } +[[bin]] +name = "xtask" +test = false + [dependencies] anyhow = { workspace = true } clap = { workspace = true, features = ["derive"] }