diff --git a/Cargo.lock b/Cargo.lock index 28fe6a43a5c5..790c74b0c37d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -106,7 +106,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -117,7 +117,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -1248,7 +1248,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -2132,7 +2132,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -2355,6 +2355,8 @@ dependencies = [ "snap", "sysinfo", "tempfile", + "tikv-jemalloc-ctl", + "tikv-jemallocator", "tokio", "twox-hash", "zstd", @@ -2958,7 +2960,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -3222,7 +3224,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -3331,10 +3333,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.3.4", + "getrandom 0.4.2", "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -3344,7 +3346,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "230a1b821ccbd75b185820a1f1ff7b14d21da1e442e22c0863ea5f08771a8874" dependencies = [ "rustix", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -3402,6 +3404,37 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "tikv-jemalloc-ctl" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a184c43b8ab2f41df2733b55556e3f5f632f4aeaa205b1bb018f574b7f5f142" +dependencies = [ + "libc", + "paste", + "tikv-jemalloc-sys", +] + +[[package]] +name = "tikv-jemalloc-sys" +version = "0.7.1+5.3.1-0-g81034ce1f1373e37dc865038e1bc8eeecf559ce8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a2825c78386b4ae0314074867860ba9577875de945f05992c38815cbec327f0" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "tikv-jemallocator" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "249f09e49ab1609436f34c776e84231bead18d6a955f119f939bdc1d847561bd" +dependencies = [ + "libc", + "tikv-jemalloc-sys", +] + [[package]] name = "tiny-keccak" version = "2.0.2" @@ -4018,7 +4051,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] diff --git a/parquet/Cargo.toml b/parquet/Cargo.toml index dd2c872ede50..34eac0182895 100644 --- a/parquet/Cargo.toml +++ b/parquet/Cargo.toml @@ -95,6 +95,21 @@ rand = { version = "0.9", default-features = false, features = ["std", "std_rng" object_store = { workspace = true, features = ["azure", "fs"] } sysinfo = { version = "0.38.1", default-features = false, features = ["system"] } +# Used as the global allocator in the `arrow_writer` benchmark, with its +# page-decay policy pinned via a compiled-in `malloc_conf` symbol, to keep +# per-iteration buffer allocations on warm pages (see that bench). The +# `unprefixed_*` feature makes jemalloc read the unprefixed `malloc_conf` +# symbol defined there; `tikv-jemalloc-ctl` is used to assert at startup that +# the policy actually took effect (so a silently-ignored config fails loudly). +# +# Gated to Linux: jemalloc does not build on some targets (e.g. wasm, msvc), and +# the unprefixed `malloc_conf` symbol it relies on is not honored on others +# (e.g. apple, android). On every other target the bench falls back to the +# default allocator. Linux is where the canonical benchmark runner runs. +[target.'cfg(target_os = "linux")'.dev-dependencies] +tikv-jemallocator = { version = "0.7", default-features = false, features = ["unprefixed_malloc_on_supported_platforms"] } +tikv-jemalloc-ctl = { version = "0.7", default-features = false } + [package.metadata.docs.rs] all-features = true diff --git a/parquet/benches/arrow_writer.rs b/parquet/benches/arrow_writer.rs index 9b22bb04b3e7..2e54fe62e677 100644 --- a/parquet/benches/arrow_writer.rs +++ b/parquet/benches/arrow_writer.rs @@ -18,6 +18,71 @@ #[macro_use] extern crate criterion; +// Use jemalloc, with page decay disabled, for the writer benchmarks. +// +// Each criterion iteration builds a fresh `ArrowWriter`, so the writer's +// internal encode buffers are allocated and freed every iteration. Whether +// those buffers are served from warm (already-faulted) pages or fresh pages +// depends on the heap state left by previously-run benchmarks in the same +// process. The cold-page case pays a per-page minor fault on every byte +// written, which roughly doubles the measured time for the byte-array writers +// (e.g. `string/parquet_2` swings between ~106ms and ~190ms purely on +// allocation order, with no code change). +// +// The retention policy is pinned, not left to the allocator default, via the +// compiled-in `malloc_conf` symbol below: `dirty_decay_ms:-1,muzzy_decay_ms:-1` +// disables jemalloc's page decay, so freed pages are kept mapped and reused +// warm instead of being returned to the OS. That removes the per-iteration +// fault tax and collapses the order-dependent bimodality, so the numbers +// reflect encoder work. Pinning the setting (rather than relying on an +// allocator's default) keeps the benchmark stable across allocator upgrades. +// +// Gated to Linux (see Cargo.toml): jemalloc does not build on some targets and +// its unprefixed `malloc_conf` symbol is not honored on others; elsewhere the +// bench just uses the default allocator (and `assert_page_decay_disabled` is a +// no-op). Linux is where the canonical benchmark runner runs. +#[cfg(target_os = "linux")] +#[global_allocator] +static GLOBAL: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; + +// jemalloc reads its options from a symbol named `malloc_conf`. The +// `unprefixed_malloc_on_supported_platforms` feature (see Cargo.toml) is what +// makes jemalloc look for the unprefixed name this defines; without it the +// symbol would be silently ignored. `assert_page_decay_disabled` below guards +// against exactly that, so a config that fails to apply fails loudly instead +// of quietly reintroducing the instability. +#[cfg(target_os = "linux")] +#[allow(non_upper_case_globals)] +#[unsafe(export_name = "malloc_conf")] +pub static malloc_conf: &[u8] = b"dirty_decay_ms:-1,muzzy_decay_ms:-1\0"; + +/// Assert the `malloc_conf` above actually took effect. If the symbol is ever +/// silently ignored (feature dropped, unsupported platform, renamed symbol, +/// allocator swapped), the byte-array writer benchmarks would quietly become +/// order-dependent again; failing loudly here prevents that. +#[cfg(target_os = "linux")] +fn assert_page_decay_disabled() { + // SAFETY: reading immutable `opt.*` mallctl values; the type matches + // jemalloc's `ssize_t`. + let (dirty, muzzy): (isize, isize) = unsafe { + ( + tikv_jemalloc_ctl::raw::read(b"opt.dirty_decay_ms\0").unwrap(), + tikv_jemalloc_ctl::raw::read(b"opt.muzzy_decay_ms\0").unwrap(), + ) + }; + assert!( + dirty == -1 && muzzy == -1, + "malloc_conf did not take effect (dirty_decay_ms={dirty}, muzzy_decay_ms={muzzy}, \ + expected -1/-1); the arrow_writer benchmark would be order-dependent. Ensure the \ + `unprefixed_malloc_on_supported_platforms` jemalloc feature is enabled.", + ); +} + +/// On non-Linux targets the benchmark uses the default allocator, so there is +/// no jemalloc page-decay setting to check. +#[cfg(not(target_os = "linux"))] +fn assert_page_decay_disabled() {} + use criterion::{Bencher, Criterion, Throughput}; use parquet::arrow::ArrowWriter; use parquet::basic::{Compression, ZstdLevel}; @@ -466,6 +531,7 @@ fn create_writer_props() -> Vec<(&'static str, WriterProperties)> { } fn bench_all_writers(c: &mut Criterion) { + assert_page_decay_disabled(); let batches = create_batches(); let props = create_writer_props();