Skip to content

refactor[fsst]: take ArrayRef in fsst_compress, decide offset width upfront #7900

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
robert3005 merged 9 commits into from
Jun 22, 2026
Merged

refactor[fsst]: take ArrayRef in fsst_compress, decide offset width upfront #7900

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
2256c73
refactor[fsst]: take ArrayRef in fsst_compress, decide offset width u…
mprammer Jun 11, 2026
330f68d
fix
robert3005 Jun 11, 2026
cbc3fa6
inline
robert3005 Jun 22, 2026
ffdf752
inline
robert3005 Jun 22, 2026
da1103d
more
robert3005 Jun 22, 2026
a042856
less clone
robert3005 Jun 22, 2026
8024f57
fixes
robert3005 Jun 22, 2026
f5589bb
more
robert3005 Jun 22, 2026
a4b73a5
format
robert3005 Jun 22, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions Cargo.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions encodings/fsst/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ workspace = true

[dependencies]
fsst-rs = { workspace = true }
num-traits = { workspace = true }
prost = { workspace = true }
rand = { workspace = true, optional = true }
vortex-array = { workspace = true }
Expand Down
80 changes: 35 additions & 45 deletions encodings/fsst/benches/fsst_compress.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ use divan::Bencher;
use rand::RngExt;
use rand::SeedableRng;
use rand::rngs::StdRng;
use vortex_array::ArrayRef;
use vortex_array::Canonical;
use vortex_array::IntoArray;
use vortex_array::RecursiveCanonical;
Expand Down Expand Up @@ -56,60 +57,50 @@ const BENCH_ARGS: &[(usize, usize, u8)] = &[
#[divan::bench(args = BENCH_ARGS)]
fn compress_fsst(bencher: Bencher, (string_count, avg_len, unique_chars): (usize, usize, u8)) {
let array = generate_test_data(string_count, avg_len, unique_chars);
let compressor = fsst_train_compressor(&array);
let compressor = fsst_train_compressor(&array, &mut SESSION.create_execution_ctx()).unwrap();
bencher
.with_inputs(|| (&array, &compressor, SESSION.create_execution_ctx()))
.bench_refs(|(array, compressor, ctx)| {
fsst_compress(*array, array.len(), array.dtype(), compressor, ctx)
})
.bench_refs(|(array, compressor, ctx)| fsst_compress(array, compressor, ctx).unwrap())
}

#[divan::bench(args = BENCH_ARGS)]
fn decompress_fsst(bencher: Bencher, (string_count, avg_len, unique_chars): (usize, usize, u8)) {
let array = generate_test_data(string_count, avg_len, unique_chars);
let compressor = fsst_train_compressor(&array);
let len = array.len();
let dtype = array.dtype().clone();
let encoded = fsst_compress(
array,
len,
&dtype,
&compressor,
&mut SESSION.create_execution_ctx(),
);
let mut ctx = SESSION.create_execution_ctx();
let compressor = fsst_train_compressor(&array, &mut ctx).unwrap();
let encoded = fsst_compress(&array, &compressor, &mut ctx)
.unwrap()
.into_array();

bencher
.with_inputs(|| (&encoded, SESSION.create_execution_ctx()))
.bench_refs(|(encoded, ctx)| (**encoded).clone().into_array().execute::<Canonical>(ctx))
.bench_refs(|(encoded, ctx)| (*encoded).clone().execute::<Canonical>(ctx))
}

#[divan::bench(args = BENCH_ARGS)]
fn train_compressor(bencher: Bencher, (string_count, avg_len, unique_chars): (usize, usize, u8)) {
let array = generate_test_data(string_count, avg_len, unique_chars);
bencher
.with_inputs(|| &array)
.bench_refs(|array| fsst_train_compressor(array))
.with_inputs(|| (&array, SESSION.create_execution_ctx()))
.bench_refs(|(array, ctx)| fsst_train_compressor(array, ctx).unwrap())
}

#[divan::bench(args = BENCH_ARGS)]
fn pushdown_compare(bencher: Bencher, (string_count, avg_len, unique_chars): (usize, usize, u8)) {
let array = generate_test_data(string_count, avg_len, unique_chars);
let compressor = fsst_train_compressor(&array);
let fsst_array = fsst_compress(
&array,
array.len(),
array.dtype(),
&compressor,
&mut SESSION.create_execution_ctx(),
);
let constant = ConstantArray::new(Scalar::from(&b"const"[..]), array.len());
let len = array.len();
let mut ctx = SESSION.create_execution_ctx();
let compressor = fsst_train_compressor(&array, &mut ctx).unwrap();
let fsst_array = fsst_compress(&array, &compressor, &mut ctx)
.unwrap()
.into_array();
let constant = ConstantArray::new(Scalar::from(&b"const"[..]), len);

bencher
.with_inputs(|| (&fsst_array, &constant, SESSION.create_execution_ctx()))
.bench_refs(|(fsst_array, constant, ctx)| {
fsst_array
.clone()
.into_array()
.binary(constant.clone().into_array(), Operator::Eq)
.unwrap()
.execute::<RecursiveCanonical>(ctx)
Expand All @@ -123,22 +114,19 @@ fn canonicalize_compare(
(string_count, avg_len, unique_chars): (usize, usize, u8),
) {
let array = generate_test_data(string_count, avg_len, unique_chars);
let compressor = fsst_train_compressor(&array);
let fsst_array = fsst_compress(
&array,
array.len(),
array.dtype(),
&compressor,
&mut SESSION.create_execution_ctx(),
);
let constant = ConstantArray::new(Scalar::from(&b"const"[..]), array.len());
let len = array.len();
let mut ctx = SESSION.create_execution_ctx();
let compressor = fsst_train_compressor(&array, &mut ctx).unwrap();
let fsst_array = fsst_compress(&array, &compressor, &mut ctx)
.unwrap()
.into_array();
let constant = ConstantArray::new(Scalar::from(&b"const"[..]), len);

bencher
.with_inputs(|| (&fsst_array, &constant, SESSION.create_execution_ctx()))
.bench_refs(|(fsst_array, constant, ctx)| {
(*fsst_array)
fsst_array
.clone()
.into_array()
.execute::<Canonical>(ctx)
.unwrap()
.into_array()
Expand Down Expand Up @@ -190,11 +178,11 @@ fn chunked_into_canonical(

bencher
.with_inputs(|| (&array, SESSION.create_execution_ctx()))
.bench_refs(|(array, ctx)| (**array).clone().into_array().execute::<Canonical>(ctx));
.bench_refs(|(array, ctx)| array.clone().execute::<Canonical>(ctx));
}

/// Helper function to generate random string data.
fn generate_test_data(string_count: usize, avg_len: usize, unique_chars: u8) -> VarBinArray {
fn generate_test_data(string_count: usize, avg_len: usize, unique_chars: u8) -> ArrayRef {
let mut rng = StdRng::seed_from_u64(0);
let mut strings = Vec::with_capacity(string_count);

Expand All @@ -216,22 +204,24 @@ fn generate_test_data(string_count: usize, avg_len: usize, unique_chars: u8) ->
.map(|opt_s| opt_s.map(Vec::into_boxed_slice)),
DType::Binary(Nullability::NonNullable),
)
.into_array()
}

fn generate_chunked_test_data(
chunk_size: usize,
string_count: usize,
avg_len: usize,
unique_chars: u8,
) -> ChunkedArray {
) -> ArrayRef {
let mut ctx = SESSION.create_execution_ctx();
(0..chunk_size)
.map(|_| {
let array = generate_test_data(string_count, avg_len, unique_chars);
let compressor = fsst_train_compressor(&array);
let len = array.len();
let dtype = array.dtype().clone();
fsst_compress(array, len, &dtype, &compressor, &mut ctx).into_array()
let compressor = fsst_train_compressor(&array, &mut ctx).unwrap();
fsst_compress(&array, &compressor, &mut ctx)
.unwrap()
.into_array()
})
.collect::<ChunkedArray>()
.into_array()
}
Loading
Loading