From 10ba11583183765ca3eb260f9d5088256137e4c9 Mon Sep 17 00:00:00 2001 From: Joshua MacDonald Date: Tue, 30 Jun 2026 18:06:37 -0700 Subject: [PATCH 01/18] chore(benchmarks): add otap_parquet study of OTAP/IPC vs flattened Parquet Adds a benchmark and unit-tested support library comparing the read and write cost, serialized size, and server-side OTAP-to-Parquet conversion cost of OTAP logs. It measures the current OTAP interleaved Arrow IPC representation against three flattened single-file Parquet layouts named nested, map, and wide, across the zstd, lz4, snappy, and none compressors. The server_cost model quantifies where the OTAP-to-Parquet conversion CPU lands when the data becomes Parquet on the server either way. It compares the server converting received OTAP/IPC against the server accepting Parquet the client already produced. This is dev-only benchmark tooling, so no changelog entry is included. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- rust/otap-dataflow/benchmarks/Cargo.toml | 16 +- .../benchmarks/benches/otap_parquet/README.md | 139 +++++ .../benchmarks/benches/otap_parquet/main.rs | 304 ++++++++++ rust/otap-dataflow/benchmarks/src/lib.rs | 13 + .../benchmarks/src/parquet_study/attrs.rs | 390 ++++++++++++ .../benchmarks/src/parquet_study/datagen.rs | 189 ++++++ .../benchmarks/src/parquet_study/ipc.rs | 84 +++ .../benchmarks/src/parquet_study/map.rs | 234 ++++++++ .../benchmarks/src/parquet_study/mod.rs | 211 +++++++ .../benchmarks/src/parquet_study/nested.rs | 161 +++++ .../src/parquet_study/parquet_io.rs | 47 ++ .../benchmarks/src/parquet_study/server.rs | 92 +++ .../benchmarks/src/parquet_study/wide.rs | 558 ++++++++++++++++++ 13 files changed, 2434 insertions(+), 4 deletions(-) create mode 100644 rust/otap-dataflow/benchmarks/benches/otap_parquet/README.md create mode 100644 rust/otap-dataflow/benchmarks/benches/otap_parquet/main.rs create mode 100644 rust/otap-dataflow/benchmarks/src/lib.rs create mode 100644 rust/otap-dataflow/benchmarks/src/parquet_study/attrs.rs create mode 100644 rust/otap-dataflow/benchmarks/src/parquet_study/datagen.rs create mode 100644 rust/otap-dataflow/benchmarks/src/parquet_study/ipc.rs create mode 100644 rust/otap-dataflow/benchmarks/src/parquet_study/map.rs create mode 100644 rust/otap-dataflow/benchmarks/src/parquet_study/mod.rs create mode 100644 rust/otap-dataflow/benchmarks/src/parquet_study/nested.rs create mode 100644 rust/otap-dataflow/benchmarks/src/parquet_study/parquet_io.rs create mode 100644 rust/otap-dataflow/benchmarks/src/parquet_study/server.rs create mode 100644 rust/otap-dataflow/benchmarks/src/parquet_study/wide.rs diff --git a/rust/otap-dataflow/benchmarks/Cargo.toml b/rust/otap-dataflow/benchmarks/Cargo.toml index b56527df16..2382022ca5 100644 --- a/rust/otap-dataflow/benchmarks/Cargo.toml +++ b/rust/otap-dataflow/benchmarks/Cargo.toml @@ -10,23 +10,28 @@ rust-version.workspace = true [dependencies] arrow.workspace = true +arrow-ipc.workspace = true +# Parquet study (benches/otap_parquet) needs the compression codecs enabled so +# the Arrow writer can emit zstd/snappy/lz4-compressed files. +parquet = { workspace = true, features = ["zstd", "snap", "lz4"] } +bytes.workspace = true +prost = { workspace = true } tokio.workspace = true serde_json.workspace = true otap-df-core-nodes = { workspace = true, features = ["dev-tools", "bench"] } otap-df-otap = { workspace = true } +otap-df-pdata = { workspace = true, features = ["bench", "testing"] } +otap-df-pdata-views = { workspace = true } [dev-dependencies] criterion = { workspace = true, features = ["html_reports", "async_tokio"] } tonic = { workspace = true } -prost = { workspace = true } otap-df-control-channel = { workspace = true } otap-df-config = { workspace = true } otap-df-channel = { workspace = true } otap-df-engine = { workspace = true } otap-df-telemetry = { workspace = true } -otap-df-pdata = { workspace = true, features = ["bench", "testing"] } -otap-df-pdata-views = { workspace = true } tracing.workspace = true tracing-subscriber = { workspace = true, features = ["registry"] } @@ -43,7 +48,6 @@ unsync.workspace = true portpicker.workspace = true tokio-stream.workspace = true weaver_common.workspace = true -bytes.workspace = true [target.'cfg(not(windows))'.dev-dependencies] tikv-jemallocator.workspace = true @@ -51,6 +55,10 @@ tikv-jemallocator.workspace = true [lints] workspace = true +[[bench]] +name = "otap_parquet" +harness = false + [[bench]] name = "channel" harness = false diff --git a/rust/otap-dataflow/benchmarks/benches/otap_parquet/README.md b/rust/otap-dataflow/benchmarks/benches/otap_parquet/README.md new file mode 100644 index 0000000000..e223568a55 --- /dev/null +++ b/rust/otap-dataflow/benchmarks/benches/otap_parquet/README.md @@ -0,0 +1,139 @@ + + +# `otap_parquet` benchmark: OTAP/IPC vs flattened Parquet + +**Status:** *experiment* (branch `jmacd/parquet_study`) + +This benchmark quantifies moving OTAP **logs** from a client to a server that +ultimately stores **flattened, single-file Parquet**. Because the data ends up +as Parquet on the server either way, the central question is where the +`OTAP -> Parquet` conversion CPU is spent: + +- **Option A** — the client sends OTAP as Arrow IPC and the **server** converts + it to Parquet (IPC-decode + flatten + Parquet-encode). +- **Option B** — the **client** precomputes the flattened Parquet and sends + that; the server just persists the bytes (or reparses them into Arrow if it + needs to partition/index/validate). + +It measures **write time**, **read time**, **serialized size**, and the +**server-side conversion cost** for the same logs batch across four +representations and four compressors. + +## Contenders + +- `ipc` — the OTAP representation we have today: interleaved Arrow IPC streams + (`Producer` / `Consumer`), each per-payload stream compressed. +- `parquet-nested` — a single flattened Parquet file where each log row carries + its denormalized resource / scope / log attributes as + `List` columns. +- `parquet-map` — the same, but attributes are `Map`. +- `parquet-wide` — the "analytics-flat" extreme: every distinct attribute key + becomes its own typed top-level column (`resource.`, `scope.`, + `log.`). + +## Compressors + +Compressors are explicit codecs so `zstd` can be compared head-to-head with +`lz4`. This matters for cross-language consumers: some Arrow/Parquet stacks (for +example certain .NET builds) may not support `zstd`, so `lz4` (and `snappy` for +Parquet) need first-class numbers. + +| compressor | Arrow IPC | Parquet | +|------------|---------------|-------------| +| `zstd` | `ZSTD` | `ZSTD` | +| `lz4` | `LZ4_FRAME` | `LZ4_RAW` | +| `snappy` | *unsupported* | `SNAPPY` | +| `none` | uncompressed | uncompressed| + +Arrow IPC only supports `zstd` and `lz4`, so `snappy` is offered for the Parquet +schemes only. Parquet uses `LZ4_RAW` (the cross-language interoperable variant, +not the deprecated Hadoop-framed `LZ4`). + +## Running + +```bash +cargo bench -p benchmarks --bench otap_parquet +``` + +Two tables (serialized size, and the server-side CPU model) are printed to +stdout before the timed benchmarks run. For a quick pass: + +```bash +cargo bench -p benchmarks --bench otap_parquet -- \ + --warm-up-time 0.3 --measurement-time 0.6 --sample-size 10 +``` + +## What each measurement covers + +- **IPC write:** `Producer::produce_bar` + prost-encode `BatchArrowRecords`. +- **IPC read:** prost-decode, `Consumer::consume_bar`, `from_record_messages`, + and `decode_transport_optimized_ids` back to `OtapArrowRecords`. +- **Parquet write:** flatten `OtapArrowRecords` → one Arrow `RecordBatch` → + `ArrowWriter` → `Vec`. +- **Parquet read:** `ParquetRecordBatchReader` → Arrow `RecordBatch` → unflatten + back to `OtapArrowRecords`. +- **server_cost `convert-A`:** IPC bytes → decode → flatten → Parquet bytes + (Option A: the server converts). +- **server_cost `accept-B`:** Parquet bytes → Arrow `RecordBatch`, i.e. reparse + without rebuilding OTAP (Option B when the server must touch the data). If the + server only persists the received bytes, its conversion CPU is ~0. + +The flattened Parquet layouts keep the entire root `Logs` record batch intact +(so decode carries its scalar/struct columns straight back, just like the IPC +path does — not penalized by re-walking the body) and only the attribute tables +are denormalized and rebuilt. Resource/scope attribute sets are re-normalized on +decode using the `resource.id` / `scope.id` join keys the `Logs` batch still +carries. `parquet-wide` is lossless for type-consistent keys; any attribute that +does not fit its single typed column spills into a per-group `List` +overflow column, so the round-trip stays exact. + +## Illustrative results + +From one development machine (WSL, jemalloc); absolute values vary by host, but +the relationships are stable. Shape `r1_s1_l5000` = 5000 log records under a +single resource/scope (OTLP proto = 1,135,223 bytes). + +Serialized size (bytes): + +| contender | zstd | lz4 | snappy | none | +|----------------|--------|--------|--------|-----------| +| ipc | 53,614 | 63,598 | — | 1,236,336 | +| parquet-nested | 35,634 | 44,904 | 52,471 | 223,441 | +| parquet-map | 35,946 | 45,216 | 52,783 | 223,753 | +| parquet-wide | 40,524 | 48,907 | 48,939 | 50,532 | + +Server-side conversion cost (indicative ms), shape `r1_s1_l5000`: + +| flatten / comp | A: server converts | B: reparse | saved if server just stores | +|-----------------------|--------------------|------------|-----------------------------| +| parquet-nested / zstd | 14.46 ms | 2.88 ms | 14.46 ms | +| parquet-nested / lz4 | 13.49 ms | 2.80 ms | 13.49 ms | +| parquet-wide / lz4 | 9.95 ms | 1.19 ms | 9.95 ms | + +## Takeaways + +- **The conversion is the expensive part.** Flatten + Parquet-encode dominates; + IPC decode is a small fraction. Accepting client-precomputed Parquet (Option + B, persist) removes essentially all of that server CPU — here ~10–14 ms per + 5000-log batch, an ~80–100% reduction versus converting server-side. Even if + the server must reparse the Parquet into Arrow, it still avoids the + flatten+encode and saves ~80%. +- **zstd vs lz4.** For IPC, lz4 is ~19% larger than zstd; for Parquet-nested, + lz4 is ~26% larger and snappy ~47% larger than zstd. But lz4 encode/decode is + a touch *cheaper* on CPU, and `parquet-nested/lz4` (44.9 KB) is still smaller + than `ipc/zstd` (53.6 KB) — so a zstd-less client (e.g. some .NET stacks) can + use lz4 (or snappy) and remain competitive on the wire. +- **Layout.** `parquet-nested`/`parquet-map` compress smallest with zstd; + `parquet-wide` is far smaller uncompressed and is the cheapest to write and to + reparse (typed columns), at a modest size premium under zstd. +- **For the debate:** if server CPU is the constraint (it is here), have clients + send precomputed flattened Parquet and let the server persist it. lz4 is a safe + cross-language codec choice with a small size cost relative to zstd. + +## Extending + +Contenders are the `Scheme` enum and its `Codec` impls in +`benchmarks/src/parquet_study`; add a variant to include it everywhere. Input +shapes are in `input_shapes()` in `benches/otap_parquet/main.rs`. The +flatten/unflatten round-trips and the server-cost helpers have unit tests +(`cargo test -p benchmarks --lib parquet_study`). diff --git a/rust/otap-dataflow/benchmarks/benches/otap_parquet/main.rs b/rust/otap-dataflow/benchmarks/benches/otap_parquet/main.rs new file mode 100644 index 0000000000..84d37eb300 --- /dev/null +++ b/rust/otap-dataflow/benchmarks/benches/otap_parquet/main.rs @@ -0,0 +1,304 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +//! Parquet study benchmark. +//! +//! Compares the read/write cost and serialized size of OTAP logs encoded as +//! compressed Arrow IPC (the representation we have today) versus several +//! flattened single-file Parquet layouts, across the `zstd`, `lz4`, `snappy`, +//! and `none` compressors. +//! +//! Two questions are answered: +//! +//! 1. Round-trip cost and size of each representation (the `write` / `read` and +//! size-table output). +//! 2. Where the OTAP -> Parquet conversion CPU lands, given the data ends up as +//! Parquet on the server either way (the `server_cost` output): the server +//! converting received OTAP/IPC (Option A) versus accepting Parquet the +//! client already produced (Option B). +//! +//! Run with: +//! +//! ```bash +//! cargo bench -p benchmarks --bench otap_parquet +//! ``` +//! +//! Two tables are printed to stdout before the timed benchmarks run. + +#![allow(missing_docs)] +// This benchmark intentionally prints comparison tables to stdout before +// running the timed measurements. +#![allow(clippy::print_stdout)] + +use criterion::{BatchSize, BenchmarkId, Criterion, criterion_group, criterion_main}; +use std::hint::black_box; +use std::time::{Duration, Instant}; + +use benchmarks::parquet_study::datagen::{LogsGenParams, gen_logs_otap}; +use benchmarks::parquet_study::{Compressor, Scheme, server}; + +#[cfg(not(windows))] +use tikv_jemallocator::Jemalloc; + +#[cfg(not(windows))] +#[global_allocator] +static GLOBAL: Jemalloc = Jemalloc; + +/// Input shapes covering a few resource/scope/log fan-outs. The first is the +/// "many small groups" shape from the `otap_encoder` bench; the rest scale the +/// number of log records under a single resource/scope (so resource/scope +/// attribute denormalization is amortized over more rows). +fn input_shapes() -> Vec { + vec![ + LogsGenParams { + num_resources: 5, + num_scopes: 10, + num_logs: 5, + }, + LogsGenParams { + num_resources: 50, + num_scopes: 2, + num_logs: 10, + }, + LogsGenParams { + num_resources: 1, + num_scopes: 1, + num_logs: 1000, + }, + LogsGenParams { + num_resources: 1, + num_scopes: 1, + num_logs: 5000, + }, + ] +} + +/// Print a serialized-size comparison table to stdout (not part of the timed +/// measurements, but the most important output for the "on the wire" debate). +fn print_size_table(shapes: &[LogsGenParams]) { + println!("\n=== OTAP logs serialized size (bytes) ==="); + println!( + "{:<16} {:<8} {:>12} {:>10} {:>10} {:>12}", + "contender", "comp", "bytes", "vs-otlp", "b/log", "vs-ipc-zstd" + ); + + for shape in shapes { + let (otap, proto_len) = gen_logs_otap(shape); + let total_logs = shape.total_logs(); + println!( + "-- shape {} : {} log records, OTLP proto = {} bytes --", + shape.label(), + total_logs, + proto_len + ); + + let ipc_zstd = Scheme::Ipc + .codec(Compressor::Zstd) + .write(otap.clone()) + .expect("ipc zstd write") + .len(); + + for scheme in Scheme::ALL { + for &compressor in scheme.compressors() { + let codec = scheme.codec(compressor); + let bytes = codec + .write(otap.clone()) + .unwrap_or_else(|e| panic!("{} write failed: {e}", codec.name())) + .len(); + let vs_otlp = proto_len as f64 / bytes as f64; + let per_log = bytes as f64 / total_logs as f64; + let vs_ipc = bytes as f64 / ipc_zstd as f64; + println!( + "{:<16} {:<8} {:>12} {:>9.2}x {:>10.1} {:>11.2}x", + codec.name(), + compressor.label(), + bytes, + vs_otlp, + per_log, + vs_ipc + ); + } + } + println!(); + } +} + +/// Median wall-clock milliseconds of `f` over a few iterations (with warm-up). +/// Indicative only; the `server_cost` Criterion group provides rigorous numbers. +fn median_ms(mut f: impl FnMut()) -> f64 { + for _ in 0..2 { + f(); + } + let iters = 11; + let mut samples = Vec::with_capacity(iters); + for _ in 0..iters { + let start = Instant::now(); + f(); + samples.push(start.elapsed().as_secs_f64() * 1e3); + } + samples.sort_by(|a, b| a.partial_cmp(b).expect("no NaN")); + samples[samples.len() / 2] +} + +/// Print the server-side CPU model: where the OTAP -> Parquet conversion lands. +fn print_server_cost_table(shapes: &[LogsGenParams]) { + println!( + "\n=== Server-side CPU: where does OTAP->Parquet conversion land? (indicative ms) ===" + ); + println!("Option A (client sends OTAP/IPC): server = IPC-decode + flatten + Parquet-encode"); + println!( + "Option B (client sends Parquet): server = persist (~0 CPU) or reparse Parquet->Arrow" + ); + println!("IPC input fixed at zstd; 'comp' is the Parquet output compressor."); + println!( + "{:<16} {:<8} {:>12} {:>12} {:>13} {:>15}", + "flatten", "comp", "A:convert", "B:reparse", "save(store)", "save(reparse)" + ); + + for shape in shapes { + let (otap, _) = gen_logs_otap(shape); + println!( + "-- shape {} : {} log records --", + shape.label(), + shape.total_logs() + ); + let ipc_bytes = Scheme::Ipc + .codec(Compressor::Zstd) + .write(otap.clone()) + .expect("ipc write"); + + for scheme in Scheme::PARQUET { + for compressor in Compressor::ALL { + let pcodec = scheme.codec(compressor); + let parquet_bytes = pcodec.write(otap.clone()).expect("parquet write"); + + let convert_a = median_ms(|| { + let _ = server::convert_ipc_to_parquet(&ipc_bytes, &*pcodec).expect("convert"); + }); + let reparse_b = median_ms(|| { + let _ = server::reparse_parquet(&parquet_bytes).expect("reparse"); + }); + + println!( + "{:<16} {:<8} {:>10.3}ms {:>10.3}ms {:>11.3}ms {:>13.3}ms", + scheme.name(), + compressor.label(), + convert_a, + reparse_b, + convert_a, + convert_a - reparse_b + ); + } + } + println!(); + } +} + +fn bench_round_trip(c: &mut Criterion) { + let shapes = input_shapes(); + print_size_table(&shapes); + print_server_cost_table(&shapes); + + let mut write_group = c.benchmark_group("parquet_study/write"); + let _ = write_group.sample_size(20); + let _ = write_group.warm_up_time(Duration::from_millis(500)); + let _ = write_group.measurement_time(Duration::from_secs(2)); + for shape in &shapes { + let (otap, _) = gen_logs_otap(shape); + for scheme in Scheme::ALL { + for &compressor in scheme.compressors() { + let codec = scheme.codec(compressor); + let id = BenchmarkId::new( + format!("{}/{}", codec.name(), compressor.label()), + shape.label(), + ); + let _ = write_group.bench_with_input(id, shape, |b, _| { + b.iter_batched( + || otap.clone(), + |input| black_box(codec.write(input).expect("write")), + BatchSize::SmallInput, + ); + }); + } + } + } + write_group.finish(); + + let mut read_group = c.benchmark_group("parquet_study/read"); + let _ = read_group.sample_size(20); + let _ = read_group.warm_up_time(Duration::from_millis(500)); + let _ = read_group.measurement_time(Duration::from_secs(2)); + for shape in &shapes { + let (otap, _) = gen_logs_otap(shape); + for scheme in Scheme::ALL { + for &compressor in scheme.compressors() { + let codec = scheme.codec(compressor); + let bytes = codec.write(otap.clone()).expect("write"); + let id = BenchmarkId::new( + format!("{}/{}", codec.name(), compressor.label()), + shape.label(), + ); + let _ = read_group.bench_with_input(id, shape, |b, _| { + b.iter(|| black_box(codec.read(&bytes).expect("read"))); + }); + } + } + } + read_group.finish(); +} + +fn bench_server_cost(c: &mut Criterion) { + let shapes = input_shapes(); + + let mut group = c.benchmark_group("parquet_study/server_cost"); + let _ = group.sample_size(20); + let _ = group.warm_up_time(Duration::from_millis(500)); + let _ = group.measurement_time(Duration::from_secs(2)); + + for shape in &shapes { + let (otap, _) = gen_logs_otap(shape); + // Client -> server wire is OTAP/IPC; decode auto-detects compression. + let ipc_bytes = Scheme::Ipc + .codec(Compressor::Zstd) + .write(otap.clone()) + .expect("ipc write"); + + for scheme in Scheme::PARQUET { + for compressor in Compressor::ALL { + let pcodec = scheme.codec(compressor); + + // Option A: server converts received OTAP/IPC to Parquet. + let id_a = BenchmarkId::new( + format!("convert-A/{}/{}", scheme.name(), compressor.label()), + shape.label(), + ); + let _ = group.bench_with_input(id_a, shape, |b, _| { + b.iter(|| { + black_box( + server::convert_ipc_to_parquet(&ipc_bytes, &*pcodec).expect("convert"), + ) + }); + }); + + // Option B: server reparses client-precomputed Parquet. + let parquet_bytes = pcodec.write(otap.clone()).expect("parquet write"); + let id_b = BenchmarkId::new( + format!("accept-B/{}/{}", scheme.name(), compressor.label()), + shape.label(), + ); + let _ = group.bench_with_input(id_b, shape, |b, _| { + b.iter(|| black_box(server::reparse_parquet(&parquet_bytes).expect("reparse"))); + }); + } + } + } + group.finish(); +} + +#[allow(missing_docs)] +mod bench_entry { + use super::*; + criterion_group!(benches, bench_round_trip, bench_server_cost); +} + +criterion_main!(bench_entry::benches); diff --git a/rust/otap-dataflow/benchmarks/src/lib.rs b/rust/otap-dataflow/benchmarks/src/lib.rs new file mode 100644 index 0000000000..572d4153f6 --- /dev/null +++ b/rust/otap-dataflow/benchmarks/src/lib.rs @@ -0,0 +1,13 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +//! Support library for the `otap-dataflow` benchmarks. +//! +//! The heavier benchmark logic lives here (rather than directly in the Criterion +//! `benches/` targets) so it can be unit-tested for correctness. Currently this +//! hosts the [`parquet_study`] module, which compares the cost of moving OTAP +//! logs as compressed Arrow IPC versus several flattened-Parquet encodings. + +#![allow(missing_docs)] + +pub mod parquet_study; diff --git a/rust/otap-dataflow/benchmarks/src/parquet_study/attrs.rs b/rust/otap-dataflow/benchmarks/src/parquet_study/attrs.rs new file mode 100644 index 0000000000..e2f04b8d2b --- /dev/null +++ b/rust/otap-dataflow/benchmarks/src/parquet_study/attrs.rs @@ -0,0 +1,390 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +//! Shared machinery for the flattened-Parquet contenders. +//! +//! OTAP logs are stored as four interleaved record batches: the root `Logs` +//! batch plus three attribute batches (`ResourceAttrs`, `ScopeAttrs`, +//! `LogAttrs`) linked to it by parent id. Each attribute batch has the columns +//! `parent_id, key, type, str, int, double, bool, bytes, ser` where `type` +//! selects which value column is populated. +//! +//! "Flattening" denormalizes those attribute batches onto the log rows. This +//! module provides the pieces every flattened layout needs: +//! +//! - [`extract_attr_value_arrays`] reads the eight value columns from a source +//! attribute batch (synthesizing null columns for any the encoder omitted and +//! normalizing dictionary-encoded keys to plain UTF-8). +//! - [`gather_by_parent`] computes, for each log row, the source attribute rows +//! that belong to it (joining on `resource.id` / `scope.id` / log `id`). +//! - [`build_attr_list_column`] / [`rebuild_attr_batch`] move attributes into a +//! `List` column and back into an OTAP attribute batch. +//! - [`logs_resource_id`], [`logs_scope_id`], [`logs_id`] expose the join keys. +//! - [`assert_logs_equivalent`] checks a round-tripped batch structurally. + +use std::collections::{HashMap, HashSet}; +use std::sync::Arc; + +use arrow::array::{ + Array, ArrayRef, BooleanArray, ListArray, RecordBatch, StructArray, UInt16Array, UInt32Array, + new_empty_array, new_null_array, +}; +use arrow::buffer::{OffsetBuffer, ScalarBuffer}; +use arrow::compute::{cast, take}; +use arrow::datatypes::{DataType, Field, Fields, Schema}; + +use otap_df_pdata::otap::OtapArrowRecords; +use otap_df_pdata::proto::opentelemetry::arrow::v1::ArrowPayloadType; +use otap_df_pdata::schema::consts; + +use super::StudyResult; + +/// Flat-table column holding each log row's denormalized resource attributes. +pub const RESOURCE_ATTRS_COL: &str = "resource_attributes"; +/// Flat-table column holding each log row's denormalized scope attributes. +pub const SCOPE_ATTRS_COL: &str = "scope_attributes"; +/// Flat-table column holding each log row's own attributes. +pub const LOG_ATTRS_COL: &str = "log_attributes"; + +/// The eight value columns of an attribute batch, in canonical order, with the +/// fixed Arrow types used by the flattened representation. `key` and `type` are +/// required; the typed value columns are nullable. +#[must_use] +pub fn attr_struct_fields() -> Fields { + Fields::from(vec![ + Field::new(consts::ATTRIBUTE_KEY, DataType::Utf8, false), + Field::new(consts::ATTRIBUTE_TYPE, DataType::UInt8, false), + Field::new(consts::ATTRIBUTE_STR, DataType::Utf8, true), + Field::new(consts::ATTRIBUTE_INT, DataType::Int64, true), + Field::new(consts::ATTRIBUTE_DOUBLE, DataType::Float64, true), + Field::new(consts::ATTRIBUTE_BOOL, DataType::Boolean, true), + Field::new(consts::ATTRIBUTE_BYTES, DataType::Binary, true), + Field::new(consts::ATTRIBUTE_SER, DataType::Binary, true), + ]) +} + +/// The list element field used for the `List` attribute columns. +#[must_use] +fn attr_list_element_field() -> Arc { + Arc::new(Field::new( + "item", + DataType::Struct(attr_struct_fields()), + false, + )) +} + +/// The seven value fields of an attribute (everything except `key`): used as the +/// `value` struct of the Map attribute layout. +#[must_use] +pub fn attr_value_struct_fields() -> Fields { + Fields::from( + attr_struct_fields() + .iter() + .skip(1) + .map(|f| f.as_ref().clone()) + .collect::>(), + ) +} + +/// The flat-table field for one attribute-container column. +#[must_use] +pub fn attr_list_column_field(name: &str) -> Field { + Field::new(name, DataType::List(attr_list_element_field()), false) +} + +fn normalize(array: &ArrayRef, want: &DataType) -> StudyResult { + if array.data_type() == want { + Ok(array.clone()) + } else { + Ok(cast(array, want)?) + } +} + +/// Read the eight value columns (`key, type, str, int, double, bool, bytes, +/// ser`) of an attribute batch in canonical order. Columns the encoder omitted +/// (because no attribute used that value type) are materialized as all-null +/// arrays so the flattened schema is stable, and dictionary-encoded keys are +/// normalized to plain UTF-8. +pub fn extract_attr_value_arrays(attr_batch: &RecordBatch) -> StudyResult> { + let len = attr_batch.num_rows(); + let fields = attr_struct_fields(); + let mut out = Vec::with_capacity(fields.len()); + for field in fields.iter() { + let array = match attr_batch.column_by_name(field.name()) { + Some(col) => normalize(col, field.data_type())?, + None => new_null_array(field.data_type(), len), + }; + out.push(array); + } + Ok(out) +} + +fn downcast_u16(array: &ArrayRef) -> StudyResult { + if array.data_type() == &DataType::UInt16 { + Ok(array + .as_any() + .downcast_ref::() + .expect("checked UInt16") + .clone()) + } else { + let cast = cast(array, &DataType::UInt16)?; + Ok(cast + .as_any() + .downcast_ref::() + .expect("cast to UInt16") + .clone()) + } +} + +/// The `id` column of the root `Logs` batch (parent of `LogAttrs`). +pub fn logs_id(logs: &RecordBatch) -> StudyResult { + let col = logs + .column_by_name(consts::ID) + .ok_or("Logs batch missing `id` column")?; + downcast_u16(col) +} + +fn struct_child_u16(logs: &RecordBatch, struct_col: &str) -> StudyResult { + let col = logs + .column_by_name(struct_col) + .ok_or_else(|| format!("Logs batch missing `{struct_col}` column"))?; + let st = col + .as_any() + .downcast_ref::() + .ok_or_else(|| format!("`{struct_col}` is not a struct"))?; + let id = st + .column_by_name(consts::ID) + .ok_or_else(|| format!("`{struct_col}` struct missing `id`"))?; + downcast_u16(id) +} + +/// The `resource.id` column of the root `Logs` batch (parent of `ResourceAttrs`). +pub fn logs_resource_id(logs: &RecordBatch) -> StudyResult { + struct_child_u16(logs, consts::RESOURCE) +} + +/// The `scope.id` column of the root `Logs` batch (parent of `ScopeAttrs`). +pub fn logs_scope_id(logs: &RecordBatch) -> StudyResult { + struct_child_u16(logs, consts::SCOPE) +} + +/// For each value of `parents` (one per log row), the source attribute rows that +/// share that parent id, as `take` indices plus per-log-row list offsets. +pub struct Gathered { + /// `take` indices into the source attribute batch, concatenated by log row. + pub indices: UInt32Array, + /// List offsets, length `parents.len() + 1`. + pub offsets: Vec, +} + +/// Join a source attribute batch onto the log rows by parent id. +pub fn gather_by_parent(attr_batch: &RecordBatch, parents: &UInt16Array) -> StudyResult { + let parent_col = attr_batch + .column_by_name(consts::PARENT_ID) + .ok_or("attribute batch missing `parent_id`")?; + let parent_id = downcast_u16(parent_col)?; + + let mut by_parent: HashMap> = HashMap::new(); + for row in 0..parent_id.len() { + if parent_id.is_valid(row) { + by_parent + .entry(parent_id.value(row)) + .or_default() + .push(row as u32); + } + } + + let mut indices: Vec = Vec::new(); + let mut offsets: Vec = Vec::with_capacity(parents.len() + 1); + offsets.push(0); + for i in 0..parents.len() { + if parents.is_valid(i) { + if let Some(rows) = by_parent.get(&parents.value(i)) { + indices.extend_from_slice(rows); + } + } + offsets.push(i32::try_from(indices.len()).expect("offset fits i32")); + } + + Ok(Gathered { + indices: UInt32Array::from(indices), + offsets, + }) +} + +/// Take the gathered attribute rows out of `attr_batch` as an eight-field +/// `Struct{key,type,str,int,double,bool,bytes,ser}` array (one struct row per +/// gathered attribute, concatenated across log rows). +pub fn taken_attr_struct( + attr_batch: &RecordBatch, + gathered: &Gathered, +) -> StudyResult { + let value_arrays = extract_attr_value_arrays(attr_batch)?; + let taken: Vec = value_arrays + .iter() + .map(|a| take(a, &gathered.indices, None)) + .collect::>()?; + Ok(StructArray::new(attr_struct_fields(), taken, None)) +} + +/// Build a `List` column denormalizing `attr_batch` +/// onto the log rows described by `gathered`. +pub fn build_attr_list_column( + attr_batch: &RecordBatch, + gathered: &Gathered, +) -> StudyResult { + let struct_array = taken_attr_struct(attr_batch, gathered)?; + let offsets = OffsetBuffer::new(ScalarBuffer::from(gathered.offsets.clone())); + let list = ListArray::new( + attr_list_element_field(), + offsets, + Arc::new(struct_array), + None, + ); + Ok(Arc::new(list)) +} + +/// Rebuild an OTAP attribute batch (`parent_id` + the eight value columns) from +/// a flattened `List` column. `entries` selects which log rows to emit +/// and the parent id to stamp on each emitted attribute row. +pub fn rebuild_attr_batch(list: &ListArray, entries: &[(usize, u16)]) -> StudyResult { + let values = list + .values() + .as_any() + .downcast_ref::() + .ok_or("attribute list values are not a struct")?; + rebuild_attr_batch_from_parts(values, list.value_offsets(), entries) +} + +/// Rebuild an OTAP attribute batch from an eight-field `Struct` of attribute +/// values plus per-log-row `offsets`. Used by both the nested (`List`) +/// and map (`Map`) layouts; the map layout reassembles its `keys`/`values` +/// children into the eight-field struct first. +pub fn rebuild_attr_batch_from_parts( + values: &StructArray, + offsets: &[i32], + entries: &[(usize, u16)], +) -> StudyResult { + let mut child_indices: Vec = Vec::new(); + let mut parent_ids: Vec = Vec::new(); + for (row, pid) in entries { + let start = offsets[*row] as usize; + let end = offsets[*row + 1] as usize; + for child in start..end { + child_indices.push(u32::try_from(child).expect("index fits u32")); + parent_ids.push(*pid); + } + } + let idx = UInt32Array::from(child_indices); + + let mut fields: Vec = Vec::with_capacity(9); + let mut columns: Vec = Vec::with_capacity(9); + fields.push(Field::new(consts::PARENT_ID, DataType::UInt16, false)); + columns.push(Arc::new(UInt16Array::from(parent_ids)) as ArrayRef); + for (i, field) in attr_struct_fields().iter().enumerate() { + fields.push(field.as_ref().clone()); + columns.push(take(values.column(i), &idx, None)?); + } + + Ok(RecordBatch::try_new( + Arc::new(Schema::new(fields)), + columns, + )?) +} + +/// Entries for a child attribute group (`LogAttrs`): every log row with a valid +/// id, stamped with its own id. +#[must_use] +pub fn entries_per_row(id_arr: &UInt16Array) -> Vec<(usize, u16)> { + (0..id_arr.len()) + .filter(|&i| id_arr.is_valid(i)) + .map(|i| (i, id_arr.value(i))) + .collect() +} + +/// Entries for a parent attribute group (`ResourceAttrs` / `ScopeAttrs`): the +/// first log row observed for each distinct id, stamped with that id. This +/// re-normalizes the denormalized resource/scope attributes back to one set per +/// resource/scope, using the join id preserved in the flat table. +#[must_use] +pub fn entries_dedup(id_arr: &UInt16Array) -> Vec<(usize, u16)> { + let mut seen: HashSet = HashSet::new(); + let mut entries = Vec::new(); + for i in 0..id_arr.len() { + if id_arr.is_valid(i) && seen.insert(id_arr.value(i)) { + entries.push((i, id_arr.value(i))); + } + } + entries +} + +/// An attribute-container column where every log row has an empty attribute +/// list (used when an attribute payload is absent from the source batch). +#[must_use] +pub fn empty_attr_list_column(num_rows: usize) -> ArrayRef { + let children: Vec = attr_struct_fields() + .iter() + .map(|f| new_empty_array(f.data_type())) + .collect(); + let struct_array = StructArray::new(attr_struct_fields(), children, None); + let offsets = OffsetBuffer::new(ScalarBuffer::from(vec![0i32; num_rows + 1])); + Arc::new(ListArray::new( + attr_list_element_field(), + offsets, + Arc::new(struct_array), + None, + )) +} + +/// Downcast a flat-table column to `ListArray`. +pub fn as_list<'a>(batch: &'a RecordBatch, name: &str) -> StudyResult<&'a ListArray> { + batch + .column_by_name(name) + .ok_or_else(|| format!("flat batch missing `{name}` column"))? + .as_any() + .downcast_ref::() + .ok_or_else(|| format!("`{name}` is not a list").into()) +} + +/// The OTAP `Logs` payload batch out of an [`OtapArrowRecords`]. +pub fn logs_batch(otap: &OtapArrowRecords) -> StudyResult<&RecordBatch> { + otap.get(ArrowPayloadType::Logs) + .ok_or_else(|| "missing Logs payload".into()) +} + +/// Convenience helper used by the boolean attribute path in tests. +#[must_use] +pub fn bool_value(array: &ArrayRef, row: usize) -> Option { + let b = array.as_any().downcast_ref::()?; + b.is_valid(row).then(|| b.value(row)) +} + +/// Structurally compare a round-tripped logs batch against the original: equal +/// log-record count and equal per-payload attribute-row counts. +pub fn assert_logs_equivalent( + original: &OtapArrowRecords, + decoded: &OtapArrowRecords, + codec: &str, + compressor: &str, +) { + let orig_logs = logs_batch(original).expect("original logs"); + let dec_logs = logs_batch(decoded).expect("decoded logs"); + assert_eq!( + orig_logs.num_rows(), + dec_logs.num_rows(), + "{codec}/{compressor}: log record count differs" + ); + + for pt in [ + ArrowPayloadType::ResourceAttrs, + ArrowPayloadType::ScopeAttrs, + ArrowPayloadType::LogAttrs, + ] { + let orig = original.get(pt).map_or(0, RecordBatch::num_rows); + let dec = decoded.get(pt).map_or(0, RecordBatch::num_rows); + assert_eq!( + orig, dec, + "{codec}/{compressor}: {pt:?} attribute count differs" + ); + } +} diff --git a/rust/otap-dataflow/benchmarks/src/parquet_study/datagen.rs b/rust/otap-dataflow/benchmarks/src/parquet_study/datagen.rs new file mode 100644 index 0000000000..5dac7cb7d6 --- /dev/null +++ b/rust/otap-dataflow/benchmarks/src/parquet_study/datagen.rs @@ -0,0 +1,189 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +//! Generates OTAP logs batches for the parquet study. +//! +//! The generated data mirrors the `otap_encoder` benchmark: a configurable +//! resource x scope x log fan-out where every log record carries a fixed set of +//! mixed-type attributes (string, bool, int, double, bytes, empty, array, +//! kvlist). This exercises every attribute value column (`str/int/double/bool/ +//! bytes/ser`) so the flatten/unflatten round-trips are meaningfully tested. + +use otap_df_pdata::encode::encode_logs_otap_batch; +use otap_df_pdata::otap::OtapArrowRecords; +use otap_df_pdata::proto::opentelemetry::common::v1::{AnyValue, InstrumentationScope, KeyValue}; +use otap_df_pdata::proto::opentelemetry::logs::v1::{ + LogRecord, LogRecordFlags, LogsData, ResourceLogs, ScopeLogs, SeverityNumber, +}; +use otap_df_pdata::proto::opentelemetry::resource::v1::Resource; +use otap_df_pdata::views::otlp::bytes::logs::RawLogsData; +use prost::Message; + +/// Shape of the generated logs data. +#[derive(Clone, Copy, Debug)] +pub struct LogsGenParams { + /// Number of distinct resources. + pub num_resources: usize, + /// Number of scopes within each resource. + pub num_scopes: usize, + /// Number of log records within each scope. + pub num_logs: usize, +} + +impl LogsGenParams { + /// A short id usable as a Criterion benchmark parameter label. + #[must_use] + pub fn label(&self) -> String { + format!( + "r{}_s{}_l{}_n{}", + self.num_resources, + self.num_scopes, + self.num_logs, + self.total_logs(), + ) + } + + /// Total number of log records produced. + #[must_use] + pub fn total_logs(&self) -> usize { + self.num_resources * self.num_scopes * self.num_logs + } +} + +fn sample_log_attributes() -> Vec { + let attr_values = vec![ + AnyValue::new_string("terry"), + AnyValue::new_bool(true), + AnyValue::new_int(5), + AnyValue::new_double(2.0), + AnyValue::new_bytes(b"hi"), + AnyValue { value: None }, + AnyValue::new_array(vec![AnyValue::new_bool(true)]), + AnyValue::new_kvlist(vec![KeyValue::new("key1", AnyValue::new_bool(true))]), + ]; + let mut log_attributes = attr_values + .into_iter() + .enumerate() + .map(|(i, val)| KeyValue { + key: format!("attr{i}"), + value: Some(val), + }) + .collect::>(); + + // an attribute whose value is absent + log_attributes.push(KeyValue { + key: "noneval".to_string(), + value: None, + }); + log_attributes +} + +/// Build an OTLP [`LogsData`] proto message with the requested shape. +#[must_use] +pub fn create_logs_data(params: &LogsGenParams) -> LogsData { + let log_attributes = sample_log_attributes(); + + LogsData::new( + (0..params.num_resources) + .map(|_| { + ResourceLogs::new( + Resource::build() + .attributes(vec![KeyValue::new( + "resource_attr1", + AnyValue::new_string("resource_value"), + )]) + .dropped_attributes_count(1u32), + (0..params.num_scopes) + .map(|_| { + ScopeLogs::new( + InstrumentationScope::build() + .name("library") + .version("scopev1") + .attributes(vec![ + KeyValue::new( + "scope_attr1", + AnyValue::new_string("scope_val1"), + ), + KeyValue::new( + "scope_attr2", + AnyValue::new_string("scope_val2"), + ), + ]) + .dropped_attributes_count(2u32) + .finish(), + (0..params.num_logs) + .map(|_| { + LogRecord::build() + .time_unix_nano(2_000_000_000u64) + .severity_number(SeverityNumber::Info) + .event_name("event1") + .observed_time_unix_nano(3_000_000_000u64) + .trace_id(vec![ + 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, + ]) + .span_id(vec![0, 0, 0, 0, 1, 1, 1, 1]) + .severity_text("Info") + .attributes(log_attributes.clone()) + .dropped_attributes_count(3u32) + .flags(LogRecordFlags::TraceFlagsMask) + .body(AnyValue::new_string("log_body")) + .finish() + }) + .collect::>(), + ) + .set_schema_url("https://schema.opentelemetry.io/scope_schema") + }) + .collect::>(), + ) + .set_schema_url("https://schema.opentelemetry.io/resource_schema") + }) + .collect::>(), + ) +} + +/// Generate an OTAP logs batch ([`OtapArrowRecords::Logs`]) for the given shape, +/// alongside the size in bytes of the equivalent OTLP protobuf encoding (a +/// useful reference point for the "on the wire" comparison). +#[must_use] +pub fn gen_logs_otap(params: &LogsGenParams) -> (OtapArrowRecords, usize) { + let logs_data = create_logs_data(params); + let mut proto_bytes = Vec::new(); + logs_data + .encode(&mut proto_bytes) + .expect("can encode OTLP proto bytes"); + + let view = RawLogsData::new(proto_bytes.as_ref()); + let otap = encode_logs_otap_batch(&view).expect("can encode OTAP logs batch"); + (otap, proto_bytes.len()) +} + +#[cfg(test)] +mod tests { + use super::*; + use otap_df_pdata::proto::opentelemetry::arrow::v1::ArrowPayloadType; + + #[test] + fn generates_expected_record_counts() { + let params = LogsGenParams { + num_resources: 2, + num_scopes: 3, + num_logs: 4, + }; + let (otap, proto_len) = gen_logs_otap(¶ms); + assert!(proto_len > 0); + + let logs = otap + .get(ArrowPayloadType::Logs) + .expect("logs payload present"); + assert_eq!(logs.num_rows(), params.total_logs()); + + // Every payload type we expect to flatten should be present. + for pt in [ + ArrowPayloadType::ResourceAttrs, + ArrowPayloadType::ScopeAttrs, + ArrowPayloadType::LogAttrs, + ] { + assert!(otap.get(pt).is_some(), "missing payload {pt:?}"); + } + } +} diff --git a/rust/otap-dataflow/benchmarks/src/parquet_study/ipc.rs b/rust/otap-dataflow/benchmarks/src/parquet_study/ipc.rs new file mode 100644 index 0000000000..7070e92f62 --- /dev/null +++ b/rust/otap-dataflow/benchmarks/src/parquet_study/ipc.rs @@ -0,0 +1,84 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +//! IPC baseline contender: the OTAP representation "as we have it today" -- the +//! interleaved Arrow IPC streams produced by [`Producer`] / consumed by +//! [`Consumer`], with the per-payload IPC streams optionally compressed. +//! +//! - write: [`Producer::produce_bar`] (applies transport-optimized encoding and +//! serializes each payload's record batch to an Arrow IPC stream) then +//! prost-encodes the resulting [`BatchArrowRecords`] to bytes. +//! - read: prost-decodes the [`BatchArrowRecords`], [`Consumer::consume_bar`] +//! deserializes the IPC streams, [`from_record_messages`] reassembles the +//! [`OtapArrowRecords`], and `decode_transport_optimized_ids` restores the +//! logical (non-transport-optimized) batch so it matches the input exactly. + +use otap_df_pdata::Consumer; +use otap_df_pdata::encode::producer::{Producer, ProducerOptions}; +use otap_df_pdata::otap::{Logs, OtapArrowRecords, from_record_messages}; +use otap_df_pdata::proto::opentelemetry::arrow::v1::BatchArrowRecords; +use prost::Message; + +use super::{Codec, Compressor, StudyResult}; + +/// Contender that encodes OTAP logs as compressed interleaved Arrow IPC streams. +pub struct IpcCodec { + /// Compression applied to each per-payload IPC stream. + pub compressor: Compressor, +} + +impl Codec for IpcCodec { + fn name(&self) -> &'static str { + "ipc" + } + + fn write(&self, mut logs: OtapArrowRecords) -> StudyResult> { + let mut producer = Producer::new_with_options(ProducerOptions { + ipc_compression: self.compressor.ipc(), + }); + let bar = producer.produce_bar(&mut logs)?; + let mut buf = Vec::with_capacity(1024); + bar.encode(&mut buf)?; + Ok(buf) + } + + fn read(&self, bytes: &[u8]) -> StudyResult { + let mut bar = BatchArrowRecords::decode(bytes)?; + let mut consumer = Consumer::default(); + let messages = consumer.consume_bar(&mut bar)?; + let mut logs = OtapArrowRecords::Logs(from_record_messages::(messages)?); + logs.decode_transport_optimized_ids()?; + Ok(logs) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::parquet_study::Compressor; + use crate::parquet_study::datagen::{LogsGenParams, gen_logs_otap}; + + #[test] + fn ipc_round_trip_preserves_structure() { + let params = LogsGenParams { + num_resources: 2, + num_scopes: 2, + num_logs: 3, + }; + let (otap, _) = gen_logs_otap(¶ms); + + for compressor in Compressor::IPC { + let codec = IpcCodec { compressor }; + let bytes = codec.write(otap.clone()).expect("write"); + let decoded = codec.read(&bytes).expect("read"); + // The IPC round-trip is lossless, though the decoded batch may + // dictionary-encode some value columns, so compare structurally. + crate::parquet_study::attrs::assert_logs_equivalent( + &otap, + &decoded, + codec.name(), + compressor.label(), + ); + } + } +} diff --git a/rust/otap-dataflow/benchmarks/src/parquet_study/map.rs b/rust/otap-dataflow/benchmarks/src/parquet_study/map.rs new file mode 100644 index 0000000000..c6dda0a708 --- /dev/null +++ b/rust/otap-dataflow/benchmarks/src/parquet_study/map.rs @@ -0,0 +1,234 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +//! Map flattened-Parquet contender. +//! +//! Identical to the [`nested`](super::nested) layout except the three attribute +//! containers are encoded as Arrow `Map` instead of `List`. The key is pulled out of the +//! attribute struct into the map key; the remaining seven value columns form the +//! map value struct. This exercises Parquet's map encoding versus the +//! list-of-struct encoding for the exact same logical attributes. + +use std::sync::Arc; + +use arrow::array::{ArrayRef, MapArray, RecordBatch, StructArray}; +use arrow::buffer::{OffsetBuffer, ScalarBuffer}; +use arrow::datatypes::{DataType, Field, Fields, Schema}; + +use otap_df_pdata::otap::OtapArrowRecords; +use otap_df_pdata::proto::opentelemetry::arrow::v1::ArrowPayloadType; + +use super::attrs::{ + Gathered, LOG_ATTRS_COL, RESOURCE_ATTRS_COL, SCOPE_ATTRS_COL, attr_struct_fields, + attr_value_struct_fields, entries_dedup, entries_per_row, gather_by_parent, logs_batch, + logs_id, logs_resource_id, logs_scope_id, rebuild_attr_batch_from_parts, taken_attr_struct, +}; +use super::{Codec, Compressor, StudyResult, parquet_io}; + +/// Contender that flattens OTAP logs into a single Parquet file using `Map` +/// attribute columns. +pub struct MapParquetCodec { + /// Parquet compression codec. + pub compressor: Compressor, +} + +fn map_entries_fields() -> Fields { + Fields::from(vec![ + Field::new("keys", DataType::Utf8, false), + Field::new( + "values", + DataType::Struct(attr_value_struct_fields()), + false, + ), + ]) +} + +fn map_entries_field() -> Arc { + Arc::new(Field::new( + "entries", + DataType::Struct(map_entries_fields()), + false, + )) +} + +fn map_column_field(name: &str) -> Field { + Field::new(name, DataType::Map(map_entries_field(), false), false) +} + +fn build_attr_map_column(attr_batch: &RecordBatch, gathered: &Gathered) -> StudyResult { + let full = taken_attr_struct(attr_batch, gathered)?; + let keys = full.column(0).clone(); + let value_struct = StructArray::new( + attr_value_struct_fields(), + full.columns()[1..].to_vec(), + None, + ); + let entries = StructArray::new( + map_entries_fields(), + vec![keys, Arc::new(value_struct)], + None, + ); + let offsets = OffsetBuffer::new(ScalarBuffer::from(gathered.offsets.clone())); + let map = MapArray::try_new(map_entries_field(), offsets, entries, None, false)?; + Ok(Arc::new(map)) +} + +fn empty_map_column(num_rows: usize) -> StudyResult { + let keys = arrow::array::new_empty_array(&DataType::Utf8); + let value_children: Vec = attr_value_struct_fields() + .iter() + .map(|f| arrow::array::new_empty_array(f.data_type())) + .collect(); + let value_struct = StructArray::new(attr_value_struct_fields(), value_children, None); + let entries = StructArray::new( + map_entries_fields(), + vec![keys, Arc::new(value_struct)], + None, + ); + let offsets = OffsetBuffer::new(ScalarBuffer::from(vec![0i32; num_rows + 1])); + let map = MapArray::try_new(map_entries_field(), offsets, entries, None, false)?; + Ok(Arc::new(map)) +} + +/// Flatten an OTAP logs batch into the map flat record batch. +pub fn flatten(otap: &OtapArrowRecords) -> StudyResult { + let logs = logs_batch(otap)?; + let num_rows = logs.num_rows(); + let resource_id = logs_resource_id(logs)?; + let scope_id = logs_scope_id(logs)?; + let log_id = logs_id(logs)?; + + let resource_col = match otap.get(ArrowPayloadType::ResourceAttrs) { + Some(b) => build_attr_map_column(b, &gather_by_parent(b, &resource_id)?)?, + None => empty_map_column(num_rows)?, + }; + let scope_col = match otap.get(ArrowPayloadType::ScopeAttrs) { + Some(b) => build_attr_map_column(b, &gather_by_parent(b, &scope_id)?)?, + None => empty_map_column(num_rows)?, + }; + let log_col = match otap.get(ArrowPayloadType::LogAttrs) { + Some(b) => build_attr_map_column(b, &gather_by_parent(b, &log_id)?)?, + None => empty_map_column(num_rows)?, + }; + + let mut fields: Vec = logs + .schema() + .fields() + .iter() + .map(|f| f.as_ref().clone()) + .collect(); + let mut columns: Vec = logs.columns().to_vec(); + for (name, col) in [ + (RESOURCE_ATTRS_COL, resource_col), + (SCOPE_ATTRS_COL, scope_col), + (LOG_ATTRS_COL, log_col), + ] { + fields.push(map_column_field(name)); + columns.push(col); + } + + Ok(RecordBatch::try_new( + Arc::new(Schema::new(fields)), + columns, + )?) +} + +fn map_column<'a>(flat: &'a RecordBatch, name: &str) -> StudyResult<&'a MapArray> { + flat.column_by_name(name) + .ok_or_else(|| format!("flat batch missing `{name}` column"))? + .as_any() + .downcast_ref::() + .ok_or_else(|| format!("`{name}` is not a map").into()) +} + +fn rebuild_from_map(map: &MapArray, entries: &[(usize, u16)]) -> StudyResult { + let keys = map.keys().clone(); + let value_struct = map + .values() + .as_any() + .downcast_ref::() + .ok_or("map values are not a struct")?; + let mut cols: Vec = Vec::with_capacity(8); + cols.push(keys); + cols.extend(value_struct.columns().iter().cloned()); + let full = StructArray::new(attr_struct_fields(), cols, None); + rebuild_attr_batch_from_parts(&full, map.value_offsets(), entries) +} + +/// Reconstruct an OTAP logs batch from the map flat record batch. +pub fn unflatten(flat: &RecordBatch) -> StudyResult { + let container = [RESOURCE_ATTRS_COL, SCOPE_ATTRS_COL, LOG_ATTRS_COL]; + + let mut fields: Vec = Vec::new(); + let mut columns: Vec = Vec::new(); + for (field, column) in flat.schema().fields().iter().zip(flat.columns()) { + if !container.contains(&field.name().as_str()) { + fields.push(field.as_ref().clone()); + columns.push(column.clone()); + } + } + let logs = RecordBatch::try_new(Arc::new(Schema::new(fields)), columns)?; + + let resource_id = logs_resource_id(&logs)?; + let scope_id = logs_scope_id(&logs)?; + let log_id = logs_id(&logs)?; + + let resource_attrs = rebuild_from_map( + map_column(flat, RESOURCE_ATTRS_COL)?, + &entries_dedup(&resource_id), + )?; + let scope_attrs = rebuild_from_map( + map_column(flat, SCOPE_ATTRS_COL)?, + &entries_dedup(&scope_id), + )?; + let log_attrs = rebuild_from_map(map_column(flat, LOG_ATTRS_COL)?, &entries_per_row(&log_id))?; + + let mut otap = OtapArrowRecords::Logs(Default::default()); + otap.set(ArrowPayloadType::Logs, logs)?; + otap.set(ArrowPayloadType::ResourceAttrs, resource_attrs)?; + otap.set(ArrowPayloadType::ScopeAttrs, scope_attrs)?; + otap.set(ArrowPayloadType::LogAttrs, log_attrs)?; + Ok(otap) +} + +impl Codec for MapParquetCodec { + fn name(&self) -> &'static str { + "parquet-map" + } + + fn write(&self, logs: OtapArrowRecords) -> StudyResult> { + let flat = flatten(&logs)?; + parquet_io::write_parquet(&flat, self.compressor.parquet()) + } + + fn read(&self, bytes: &[u8]) -> StudyResult { + let flat = parquet_io::read_parquet(bytes)?; + unflatten(&flat) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::parquet_study::Compressor; + use crate::parquet_study::attrs::assert_logs_equivalent; + use crate::parquet_study::datagen::{LogsGenParams, gen_logs_otap}; + + #[test] + fn map_round_trip_preserves_structure() { + let params = LogsGenParams { + num_resources: 3, + num_scopes: 2, + num_logs: 5, + }; + let (otap, _) = gen_logs_otap(¶ms); + + for compressor in Compressor::ALL { + let codec = MapParquetCodec { compressor }; + let bytes = codec.write(otap.clone()).expect("write"); + let decoded = codec.read(&bytes).expect("read"); + assert_logs_equivalent(&otap, &decoded, codec.name(), compressor.label()); + } + } +} diff --git a/rust/otap-dataflow/benchmarks/src/parquet_study/mod.rs b/rust/otap-dataflow/benchmarks/src/parquet_study/mod.rs new file mode 100644 index 0000000000..43c309ec29 --- /dev/null +++ b/rust/otap-dataflow/benchmarks/src/parquet_study/mod.rs @@ -0,0 +1,211 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +//! Parquet study: compare the read/write cost and serialized size of OTAP logs +//! encoded as compressed Arrow IPC versus several flattened-Parquet layouts. +//! +//! Every contender implements [`Codec`], which turns an [`OtapArrowRecords`] +//! logs batch into "wire" bytes ([`Codec::write`]) and back ([`Codec::read`]). +//! Contenders are enumerated by [`Scheme`] and parameterized by a [`Compressor`]. +//! Compressors are explicit codecs (`zstd`, `lz4`, `snappy`, `none`) so that +//! zstd can be compared head-to-head with lz4 -- important when a consumer's +//! Arrow/Parquet stack (for example some .NET implementations) may not support +//! zstd. Arrow IPC only supports zstd and lz4 (frame), so snappy is offered for +//! the Parquet schemes only. + +use otap_df_pdata::otap::OtapArrowRecords; + +pub mod attrs; +pub mod datagen; +pub mod ipc; +pub mod map; +pub mod nested; +pub mod parquet_io; +pub mod server; +pub mod wide; + +/// Error type used throughout the study (benchmark/test code, so a boxed error +/// is sufficient). +pub type StudyResult = Result>; + +/// Compression codec applied by a [`Codec`]. +/// +/// | variant | Arrow IPC | Parquet | +/// |----------|---------------|----------------| +/// | `Zstd` | `ZSTD` | `ZSTD` | +/// | `Lz4` | `LZ4_FRAME` | `LZ4_RAW` | +/// | `Snappy` | *unsupported* | `SNAPPY` | +/// | `None` | uncompressed | uncompressed | +/// +/// `LZ4_RAW` (not the deprecated Hadoop-framed `LZ4`) is used for Parquet +/// because it is the cross-language interoperable variant. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Compressor { + /// zstd. + Zstd, + /// lz4 (frame for IPC, raw for Parquet). + Lz4, + /// snappy (Parquet only). + Snappy, + /// No compression. + None, +} + +impl Compressor { + /// All compressors, in reporting order (valid for the Parquet schemes). + pub const ALL: [Compressor; 4] = [ + Compressor::Zstd, + Compressor::Lz4, + Compressor::Snappy, + Compressor::None, + ]; + + /// Compressors Arrow IPC supports (snappy is not an Arrow IPC codec). + pub const IPC: [Compressor; 3] = [Compressor::Zstd, Compressor::Lz4, Compressor::None]; + + /// Short label used in benchmark ids and size tables. + #[must_use] + pub fn label(self) -> &'static str { + match self { + Compressor::Zstd => "zstd", + Compressor::Lz4 => "lz4", + Compressor::Snappy => "snappy", + Compressor::None => "none", + } + } + + /// IPC compression for this setting. Panics for [`Compressor::Snappy`], which + /// Arrow IPC does not support; callers use [`Scheme::compressors`] to avoid + /// that combination. + #[must_use] + pub fn ipc(self) -> Option { + match self { + Compressor::Zstd => Some(arrow_ipc::CompressionType::ZSTD), + Compressor::Lz4 => Some(arrow_ipc::CompressionType::LZ4_FRAME), + Compressor::None => None, + Compressor::Snappy => unreachable!("Arrow IPC does not support snappy"), + } + } + + /// Parquet compression for this setting. + #[must_use] + pub fn parquet(self) -> parquet::basic::Compression { + use parquet::basic::{Compression, ZstdLevel}; + match self { + Compressor::Zstd => Compression::ZSTD(ZstdLevel::try_new(3).expect("valid zstd level")), + Compressor::Lz4 => Compression::LZ4_RAW, + Compressor::Snappy => Compression::SNAPPY, + Compressor::None => Compression::UNCOMPRESSED, + } + } +} + +/// A round-trippable encoding of an OTAP logs batch. +pub trait Codec { + /// Stable name of this contender (e.g. `"ipc"`, `"parquet-nested"`). + fn name(&self) -> &'static str; + + /// Encode an OTAP logs batch into serialized "wire" bytes. + /// + /// Takes the batch by value: the IPC producer mutates it in place + /// (transport-optimized encoding), and the Criterion harness clones the + /// input in its (untimed) setup closure. + fn write(&self, logs: OtapArrowRecords) -> StudyResult>; + + /// Decode serialized bytes back into an OTAP logs batch. + fn read(&self, bytes: &[u8]) -> StudyResult; +} + +/// The contenders compared by the study. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Scheme { + /// OTAP interleaved Arrow IPC streams (the representation we have today). + Ipc, + /// Flattened Parquet, attributes as `List`. + Nested, + /// Flattened Parquet, attributes as `Map`. + Map, + /// Flattened Parquet, attributes exploded to one typed column per key. + Wide, +} + +impl Scheme { + /// All schemes, in reporting order. + pub const ALL: [Scheme; 4] = [Scheme::Ipc, Scheme::Nested, Scheme::Map, Scheme::Wide]; + + /// Only the flattened-Parquet schemes (used by the server-cost model, where + /// IPC is the input rather than an output format). + pub const PARQUET: [Scheme; 3] = [Scheme::Nested, Scheme::Map, Scheme::Wide]; + + /// Stable contender name. + #[must_use] + pub fn name(self) -> &'static str { + match self { + Scheme::Ipc => "ipc", + Scheme::Nested => "parquet-nested", + Scheme::Map => "parquet-map", + Scheme::Wide => "parquet-wide", + } + } + + /// Whether this scheme produces Parquet (as opposed to Arrow IPC). + #[must_use] + pub fn is_parquet(self) -> bool { + !matches!(self, Scheme::Ipc) + } + + /// The compressors valid for this scheme (IPC excludes snappy). + #[must_use] + pub fn compressors(self) -> &'static [Compressor] { + if self.is_parquet() { + &Compressor::ALL + } else { + &Compressor::IPC + } + } + + /// Construct the [`Codec`] for this scheme with the given compressor. + #[must_use] + pub fn codec(self, compressor: Compressor) -> Box { + match self { + Scheme::Ipc => Box::new(ipc::IpcCodec { compressor }), + Scheme::Nested => Box::new(nested::NestedParquetCodec { compressor }), + Scheme::Map => Box::new(map::MapParquetCodec { compressor }), + Scheme::Wide => Box::new(wide::WideParquetCodec { compressor }), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::parquet_study::datagen::{LogsGenParams, gen_logs_otap}; + + /// Round-trip every scheme x its valid compressors and assert the result is + /// logically equivalent to the input. + #[test] + fn all_contenders_round_trip() { + let params = LogsGenParams { + num_resources: 2, + num_scopes: 3, + num_logs: 4, + }; + let (otap, _) = gen_logs_otap(¶ms); + + for scheme in Scheme::ALL { + for &compressor in scheme.compressors() { + let codec = scheme.codec(compressor); + let bytes = codec + .write(otap.clone()) + .unwrap_or_else(|e| panic!("{} write failed: {e}", codec.name())); + assert!(!bytes.is_empty(), "{} produced no bytes", codec.name()); + + let decoded = codec + .read(&bytes) + .unwrap_or_else(|e| panic!("{} read failed: {e}", codec.name())); + + attrs::assert_logs_equivalent(&otap, &decoded, codec.name(), compressor.label()); + } + } + } +} diff --git a/rust/otap-dataflow/benchmarks/src/parquet_study/nested.rs b/rust/otap-dataflow/benchmarks/src/parquet_study/nested.rs new file mode 100644 index 0000000000..9843cd08ae --- /dev/null +++ b/rust/otap-dataflow/benchmarks/src/parquet_study/nested.rs @@ -0,0 +1,161 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +//! Nested flattened-Parquet contender. +//! +//! The flat table keeps the entire root `Logs` record batch unchanged (so the +//! decode side can carry the scalar/struct columns straight back without +//! re-walking them, matching what the IPC path gets "for free") and appends +//! three `List` columns holding +//! each log row's denormalized resource, scope, and log attributes. +//! +//! On decode the `Logs` batch is the flat table minus those three columns; the +//! attribute batches are rebuilt from the list columns, re-normalizing the +//! resource/scope sets via the `resource.id` / `scope.id` join keys that the +//! `Logs` batch still carries. + +use std::sync::Arc; + +use arrow::array::{ArrayRef, RecordBatch}; +use arrow::datatypes::{Field, Schema}; + +use otap_df_pdata::otap::OtapArrowRecords; +use otap_df_pdata::proto::opentelemetry::arrow::v1::ArrowPayloadType; + +use super::attrs::{ + self, LOG_ATTRS_COL, RESOURCE_ATTRS_COL, SCOPE_ATTRS_COL, attr_list_column_field, + build_attr_list_column, empty_attr_list_column, entries_dedup, entries_per_row, + gather_by_parent, logs_batch, logs_id, logs_resource_id, logs_scope_id, rebuild_attr_batch, +}; +use super::{Codec, Compressor, StudyResult, parquet_io}; + +/// Contender that flattens OTAP logs into a single Parquet file using +/// `List` attribute columns. +pub struct NestedParquetCodec { + /// Parquet compression codec. + pub compressor: Compressor, +} + +/// Flatten an OTAP logs batch into the nested flat record batch. +pub fn flatten(otap: &OtapArrowRecords) -> StudyResult { + let logs = logs_batch(otap)?; + let num_rows = logs.num_rows(); + let resource_id = logs_resource_id(logs)?; + let scope_id = logs_scope_id(logs)?; + let log_id = logs_id(logs)?; + + let resource_col = match otap.get(ArrowPayloadType::ResourceAttrs) { + Some(b) => build_attr_list_column(b, &gather_by_parent(b, &resource_id)?)?, + None => empty_attr_list_column(num_rows), + }; + let scope_col = match otap.get(ArrowPayloadType::ScopeAttrs) { + Some(b) => build_attr_list_column(b, &gather_by_parent(b, &scope_id)?)?, + None => empty_attr_list_column(num_rows), + }; + let log_col = match otap.get(ArrowPayloadType::LogAttrs) { + Some(b) => build_attr_list_column(b, &gather_by_parent(b, &log_id)?)?, + None => empty_attr_list_column(num_rows), + }; + + let mut fields: Vec = logs + .schema() + .fields() + .iter() + .map(|f| f.as_ref().clone()) + .collect(); + let mut columns: Vec = logs.columns().to_vec(); + for (name, col) in [ + (RESOURCE_ATTRS_COL, resource_col), + (SCOPE_ATTRS_COL, scope_col), + (LOG_ATTRS_COL, log_col), + ] { + fields.push(attr_list_column_field(name)); + columns.push(col); + } + + Ok(RecordBatch::try_new( + Arc::new(Schema::new(fields)), + columns, + )?) +} + +/// Reconstruct an OTAP logs batch from the nested flat record batch. +pub fn unflatten(flat: &RecordBatch) -> StudyResult { + let container = [RESOURCE_ATTRS_COL, SCOPE_ATTRS_COL, LOG_ATTRS_COL]; + + // The Logs batch is the flat table minus the three attribute columns. + let mut fields: Vec = Vec::new(); + let mut columns: Vec = Vec::new(); + for (field, column) in flat.schema().fields().iter().zip(flat.columns()) { + if !container.contains(&field.name().as_str()) { + fields.push(field.as_ref().clone()); + columns.push(column.clone()); + } + } + let logs = RecordBatch::try_new(Arc::new(Schema::new(fields)), columns)?; + + let resource_id = logs_resource_id(&logs)?; + let scope_id = logs_scope_id(&logs)?; + let log_id = logs_id(&logs)?; + + let resource_attrs = rebuild_attr_batch( + attrs::as_list(flat, RESOURCE_ATTRS_COL)?, + &entries_dedup(&resource_id), + )?; + let scope_attrs = rebuild_attr_batch( + attrs::as_list(flat, SCOPE_ATTRS_COL)?, + &entries_dedup(&scope_id), + )?; + let log_attrs = rebuild_attr_batch( + attrs::as_list(flat, LOG_ATTRS_COL)?, + &entries_per_row(&log_id), + )?; + + let mut otap = OtapArrowRecords::Logs(Default::default()); + otap.set(ArrowPayloadType::Logs, logs)?; + otap.set(ArrowPayloadType::ResourceAttrs, resource_attrs)?; + otap.set(ArrowPayloadType::ScopeAttrs, scope_attrs)?; + otap.set(ArrowPayloadType::LogAttrs, log_attrs)?; + Ok(otap) +} + +impl Codec for NestedParquetCodec { + fn name(&self) -> &'static str { + "parquet-nested" + } + + fn write(&self, logs: OtapArrowRecords) -> StudyResult> { + let flat = flatten(&logs)?; + parquet_io::write_parquet(&flat, self.compressor.parquet()) + } + + fn read(&self, bytes: &[u8]) -> StudyResult { + let flat = parquet_io::read_parquet(bytes)?; + unflatten(&flat) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::parquet_study::Compressor; + use crate::parquet_study::attrs::assert_logs_equivalent; + use crate::parquet_study::datagen::{LogsGenParams, gen_logs_otap}; + + #[test] + fn nested_round_trip_preserves_structure() { + let params = LogsGenParams { + num_resources: 3, + num_scopes: 2, + num_logs: 5, + }; + let (otap, _) = gen_logs_otap(¶ms); + + for compressor in Compressor::ALL { + let codec = NestedParquetCodec { compressor }; + let bytes = codec.write(otap.clone()).expect("write"); + let decoded = codec.read(&bytes).expect("read"); + assert_logs_equivalent(&otap, &decoded, codec.name(), compressor.label()); + } + } +} diff --git a/rust/otap-dataflow/benchmarks/src/parquet_study/parquet_io.rs b/rust/otap-dataflow/benchmarks/src/parquet_study/parquet_io.rs new file mode 100644 index 0000000000..89da442ebb --- /dev/null +++ b/rust/otap-dataflow/benchmarks/src/parquet_study/parquet_io.rs @@ -0,0 +1,47 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +//! In-memory Parquet read/write used by the flattened-Parquet contenders. +//! +//! The production `parquet_exporter` writes through `object_store`; for a +//! read/write cost microbenchmark we instead encode to and decode from an +//! in-memory `Vec` using the synchronous Arrow Parquet reader/writer. + +use arrow::array::RecordBatch; +use arrow::compute::concat_batches; +use bytes::Bytes; +use parquet::arrow::ArrowWriter; +use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder; +use parquet::basic::Compression; +use parquet::file::properties::WriterProperties; + +use super::StudyResult; + +/// Encode a single record batch as a Parquet file in memory. +pub fn write_parquet(batch: &RecordBatch, compression: Compression) -> StudyResult> { + let props = WriterProperties::builder() + .set_compression(compression) + .build(); + let mut buf: Vec = Vec::with_capacity(4096); + let mut writer = ArrowWriter::try_new(&mut buf, batch.schema(), Some(props))?; + writer.write(batch)?; + let _metadata = writer.close()?; + Ok(buf) +} + +/// Decode an in-memory Parquet file into a single record batch (concatenating +/// the reader's row-group batches). +pub fn read_parquet(bytes: &[u8]) -> StudyResult { + let builder = ParquetRecordBatchReaderBuilder::try_new(Bytes::copy_from_slice(bytes))?; + let schema = builder.schema().clone(); + let reader = builder.build()?; + let mut batches: Vec = Vec::new(); + for batch in reader { + batches.push(batch?); + } + match batches.len() { + 0 => Ok(RecordBatch::new_empty(schema)), + 1 => Ok(batches.into_iter().next().expect("len checked")), + _ => Ok(concat_batches(&schema, &batches)?), + } +} diff --git a/rust/otap-dataflow/benchmarks/src/parquet_study/server.rs b/rust/otap-dataflow/benchmarks/src/parquet_study/server.rs new file mode 100644 index 0000000000..498dd23f1e --- /dev/null +++ b/rust/otap-dataflow/benchmarks/src/parquet_study/server.rs @@ -0,0 +1,92 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +//! Server-side CPU model for the OTAP-to-Parquet debate. +//! +//! In the target system the data ends up as flattened Parquet on the server +//! *either way*; the only question is where the OTAP -> Parquet conversion CPU +//! is spent. +//! +//! - **Option A -- client sends OTAP/IPC, server converts.** The server pays to +//! decode the IPC, flatten to the single Parquet table, and encode Parquet. +//! This is [`convert_ipc_to_parquet`]. +//! - **Option B -- client sends precomputed Parquet.** The client already paid +//! the conversion; the server only persists the bytes (essentially an I/O +//! copy, ~0 CPU) or, if it must partition / index / validate, reparses the +//! Parquet back into Arrow without going all the way to OTAP. That reparse is +//! [`reparse_parquet`]; persisting is [`persist_bytes`]. +//! +//! The server-side CPU *saved* by accepting client Parquet is therefore roughly +//! the whole of Option A minus whatever minimal handling Option B needs. + +use arrow::array::RecordBatch; + +use super::ipc::IpcCodec; +use super::{Codec, Compressor, StudyResult, parquet_io}; + +/// Option A: the server receives OTAP/IPC and produces flattened Parquet. +/// +/// `parquet` selects the flattening layout (nested / map / wide) and the Parquet +/// compressor. The IPC decode auto-detects each stream's compression, so the +/// compressor used to construct the internal [`IpcCodec`] does not affect the +/// read. +pub fn convert_ipc_to_parquet(ipc_bytes: &[u8], parquet: &dyn Codec) -> StudyResult> { + let otap = IpcCodec { + compressor: Compressor::Zstd, + } + .read(ipc_bytes)?; + parquet.write(otap) +} + +/// Option B (server needs the data): reparse client-precomputed Parquet into an +/// Arrow record batch. This decodes Parquet but does not rebuild OTAP -- a +/// server that partitions or indexes the flat table works in Arrow directly. +pub fn reparse_parquet(parquet_bytes: &[u8]) -> StudyResult { + parquet_io::read_parquet(parquet_bytes) +} + +/// Option B (server only stores): persist the received bytes. Modeled as the +/// single copy the server makes handing the buffer to storage; real storage +/// cost is I/O, so this is a generous upper bound on the server's CPU. +#[must_use] +pub fn persist_bytes(parquet_bytes: &[u8]) -> usize { + let sink: Vec = parquet_bytes.to_vec(); + sink.len() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::parquet_study::attrs::assert_logs_equivalent; + use crate::parquet_study::datagen::{LogsGenParams, gen_logs_otap}; + use crate::parquet_study::nested::NestedParquetCodec; + use crate::parquet_study::{Codec, Compressor}; + + #[test] + fn server_convert_matches_direct_flatten() { + let params = LogsGenParams { + num_resources: 2, + num_scopes: 2, + num_logs: 4, + }; + let (otap, _) = gen_logs_otap(¶ms); + + let ipc = IpcCodec { + compressor: Compressor::Zstd, + }; + let ipc_bytes = ipc.write(otap.clone()).expect("ipc write"); + + let nested = NestedParquetCodec { + compressor: Compressor::Zstd, + }; + // Option A server output must round-trip back to an equivalent batch. + let parquet_bytes = convert_ipc_to_parquet(&ipc_bytes, &nested).expect("convert"); + let decoded = nested.read(&parquet_bytes).expect("read back"); + assert_logs_equivalent(&otap, &decoded, "server-convert", "zstd"); + + // Option B reparse yields the flat table without error. + let flat = reparse_parquet(&parquet_bytes).expect("reparse"); + assert_eq!(flat.num_rows(), params.total_logs()); + assert!(persist_bytes(&parquet_bytes) > 0); + } +} diff --git a/rust/otap-dataflow/benchmarks/src/parquet_study/wide.rs b/rust/otap-dataflow/benchmarks/src/parquet_study/wide.rs new file mode 100644 index 0000000000..33948cbdd9 --- /dev/null +++ b/rust/otap-dataflow/benchmarks/src/parquet_study/wide.rs @@ -0,0 +1,558 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +//! Wide / exploded flattened-Parquet contender. +//! +//! Each distinct attribute key becomes its own top-level, typed Parquet column, +//! prefixed by its group (`resource.`, `scope.`, `log.`). The +//! column's Arrow type is chosen from the key's first-seen OTAP value type +//! (string -> Utf8, int -> Int64, double -> Float64, bool -> Boolean, +//! bytes/map/slice -> Binary, empty -> a Boolean presence marker). A row is null +//! in a column when it lacks that attribute. This is the "analytics-flat" layout +//! that compresses and queries well column-by-column. +//! +//! To stay lossless and keep round-trip counts exact even when a key appears +//! with more than one value type (which the single typed column cannot hold), +//! any attribute that does not fit its typed column is spilled into a per-group +//! `List` overflow column. With type-consistent keys (the +//! common case, and what this study generates) the overflow columns are empty. +//! +//! Every added column carries field metadata describing its group, key, and +//! OTAP type, so decode is fully self-describing after a Parquet round-trip. + +use std::collections::{BTreeMap, HashMap}; +use std::sync::Arc; + +use arrow::array::{ + Array, ArrayRef, BinaryArray, BinaryBuilder, BooleanArray, BooleanBuilder, Float64Array, + Float64Builder, Int64Array, Int64Builder, ListArray, RecordBatch, StringArray, StringBuilder, + StructArray, UInt8Array, +}; +use arrow::datatypes::{Field, Schema}; + +use otap_df_pdata::encode::record::attributes::AttributesRecordBatchBuilder; +use otap_df_pdata::otap::OtapArrowRecords; +use otap_df_pdata::proto::opentelemetry::arrow::v1::ArrowPayloadType; + +use super::attrs::{ + Gathered, build_attr_list_column, extract_attr_value_arrays, gather_by_parent, logs_batch, + logs_id, logs_resource_id, logs_scope_id, +}; +use super::{Codec, Compressor, StudyResult, parquet_io}; + +// OTAP attribute value type discriminators (see AttributeValueType). +const T_STR: u8 = 1; +const T_INT: u8 = 2; +const T_DOUBLE: u8 = 3; +const T_BOOL: u8 = 4; +const T_MAP: u8 = 5; +const T_SLICE: u8 = 6; +const T_BYTES: u8 = 7; + +const MD_GROUP: &str = "wide_group"; +const MD_KEY: &str = "wide_key"; +const MD_TYPE: &str = "wide_otap_type"; +const MD_OVERFLOW: &str = "wide_overflow"; + +/// Contender that flattens OTAP logs into a single Parquet file using one typed +/// column per distinct attribute key. +pub struct WideParquetCodec { + /// Parquet compression codec. + pub compressor: Compressor, +} + +/// Group identifier used in column prefixes and metadata. +#[derive(Clone, Copy)] +struct Group { + name: &'static str, + prefix: &'static str, + payload: ArrowPayloadType, + overflow_col: &'static str, +} + +const GROUPS: [Group; 3] = [ + Group { + name: "resource", + prefix: "resource.", + payload: ArrowPayloadType::ResourceAttrs, + overflow_col: "resource_overflow", + }, + Group { + name: "scope", + prefix: "scope.", + payload: ArrowPayloadType::ScopeAttrs, + overflow_col: "scope_overflow", + }, + Group { + name: "log", + prefix: "log.", + payload: ArrowPayloadType::LogAttrs, + overflow_col: "log_overflow", + }, +]; + +/// Borrowed, typed view over the eight value columns of a source attribute +/// batch (`key, type, str, int, double, bool, bytes, ser`). +struct SourceAttrs { + key: ArrayRef, + atype: ArrayRef, + values: Vec, // str,int,double,bool,bytes,ser in canonical order +} + +impl SourceAttrs { + fn new(attr_batch: &RecordBatch) -> StudyResult { + let arrays = extract_attr_value_arrays(attr_batch)?; + Ok(Self { + key: arrays[0].clone(), + atype: arrays[1].clone(), + values: arrays[2..].to_vec(), + }) + } + + fn key_at(&self, row: usize) -> &str { + self.key + .as_any() + .downcast_ref::() + .expect("key is Utf8") + .value(row) + } + + fn type_at(&self, row: usize) -> u8 { + self.atype + .as_any() + .downcast_ref::() + .expect("type is UInt8") + .value(row) + } + + fn str(&self) -> &StringArray { + self.values[0].as_any().downcast_ref().expect("str Utf8") + } + fn int(&self) -> &Int64Array { + self.values[1].as_any().downcast_ref().expect("int Int64") + } + fn double(&self) -> &Float64Array { + self.values[2] + .as_any() + .downcast_ref() + .expect("double Float64") + } + fn boolean(&self) -> &BooleanArray { + self.values[3] + .as_any() + .downcast_ref() + .expect("bool Boolean") + } + fn bytes(&self) -> &BinaryArray { + self.values[4] + .as_any() + .downcast_ref() + .expect("bytes Binary") + } + fn ser(&self) -> &BinaryArray { + self.values[5].as_any().downcast_ref().expect("ser Binary") + } +} + +/// Build the typed Arrow column for one exploded key. `row_src[i]` is the source +/// attribute row that supplies row `i`'s value, or `None` if absent. +fn build_typed_column(otap_type: u8, src: &SourceAttrs, row_src: &[Option]) -> ArrayRef { + match otap_type { + T_STR => { + let a = src.str(); + let mut b = StringBuilder::new(); + for r in row_src { + match r { + Some(i) => b.append_value(a.value(*i)), + None => b.append_null(), + } + } + Arc::new(b.finish()) + } + T_INT => { + let a = src.int(); + let mut b = Int64Builder::new(); + for r in row_src { + match r { + Some(i) => b.append_value(a.value(*i)), + None => b.append_null(), + } + } + Arc::new(b.finish()) + } + T_DOUBLE => { + let a = src.double(); + let mut b = Float64Builder::new(); + for r in row_src { + match r { + Some(i) => b.append_value(a.value(*i)), + None => b.append_null(), + } + } + Arc::new(b.finish()) + } + T_BOOL => { + let a = src.boolean(); + let mut b = BooleanBuilder::new(); + for r in row_src { + match r { + Some(i) => b.append_value(a.value(*i)), + None => b.append_null(), + } + } + Arc::new(b.finish()) + } + T_BYTES => binary_column(src.bytes(), row_src), + T_MAP | T_SLICE => binary_column(src.ser(), row_src), + // Empty: a Boolean presence marker (value is meaningless). + _ => { + let mut b = BooleanBuilder::new(); + for r in row_src { + match r { + Some(_) => b.append_value(true), + None => b.append_null(), + } + } + Arc::new(b.finish()) + } + } +} + +fn binary_column(src: &BinaryArray, row_src: &[Option]) -> ArrayRef { + let mut b = BinaryBuilder::new(); + for r in row_src { + match r { + Some(i) => b.append_value(src.value(*i)), + None => b.append_null(), + } + } + Arc::new(b.finish()) +} + +fn field_with_md(name: String, array: &ArrayRef, md: HashMap) -> Field { + Field::new(name, array.data_type().clone(), true).with_metadata(md) +} + +/// Explode one attribute group into typed columns plus an overflow column. +fn explode_group( + otap: &OtapArrowRecords, + group: Group, + parents: &arrow::array::UInt16Array, + num_rows: usize, +) -> StudyResult> { + let Some(attr_batch) = otap.get(group.payload) else { + // No attributes for this group: emit only an empty overflow column. + let overflow = super::attrs::empty_attr_list_column(num_rows); + let mut md = HashMap::new(); + let _ = md.insert(MD_OVERFLOW.to_string(), group.name.to_string()); + return Ok(vec![( + field_with_md(group.overflow_col.to_string(), &overflow, md), + overflow, + )]); + }; + + let src = SourceAttrs::new(attr_batch)?; + let gathered = gather_by_parent(attr_batch, parents)?; + + // Discover columns: first-seen OTAP type per key, ordered deterministically. + let mut col_type: BTreeMap = BTreeMap::new(); + for row in 0..attr_batch.num_rows() { + let key = src.key_at(row).to_string(); + let _ = col_type.entry(key).or_insert_with(|| src.type_at(row)); + } + let keys: Vec = col_type.keys().cloned().collect(); + let key_ordinal: HashMap<&str, usize> = keys + .iter() + .enumerate() + .map(|(i, k)| (k.as_str(), i)) + .collect(); + + // For each column, the source row supplying each log row's value. + let mut per_col_row_src: Vec>> = vec![vec![None; num_rows]; keys.len()]; + let mut overflow_indices: Vec = Vec::new(); + let mut overflow_offsets: Vec = Vec::with_capacity(num_rows + 1); + overflow_offsets.push(0); + + // `row` indexes per_col_row_src, gathered.offsets, and overflow tracking. + #[allow(clippy::needless_range_loop)] + for row in 0..num_rows { + let start = gathered.offsets[row] as usize; + let end = gathered.offsets[row + 1] as usize; + for slot in start..end { + let s = gathered.indices.value(slot) as usize; + let key = src.key_at(s); + let t = src.type_at(s); + let ord = key_ordinal[key]; + if col_type[key] == t && per_col_row_src[ord][row].is_none() { + per_col_row_src[ord][row] = Some(s); + } else { + overflow_indices.push(u32::try_from(s).expect("index fits u32")); + } + } + overflow_offsets.push(i32::try_from(overflow_indices.len()).expect("offset fits i32")); + } + + let mut out: Vec<(Field, ArrayRef)> = Vec::with_capacity(keys.len() + 1); + for (ord, key) in keys.iter().enumerate() { + let t = col_type[key]; + let array = build_typed_column(t, &src, &per_col_row_src[ord]); + let mut md = HashMap::new(); + let _ = md.insert(MD_GROUP.to_string(), group.name.to_string()); + let _ = md.insert(MD_KEY.to_string(), key.clone()); + let _ = md.insert(MD_TYPE.to_string(), t.to_string()); + let name = format!("{}{}", group.prefix, key); + out.push((field_with_md(name, &array, md), array)); + } + + // Overflow column (empty for type-consistent keys). + let overflow_gathered = Gathered { + indices: arrow::array::UInt32Array::from(overflow_indices), + offsets: overflow_offsets, + }; + let overflow = build_attr_list_column(attr_batch, &overflow_gathered)?; + let mut md = HashMap::new(); + let _ = md.insert(MD_OVERFLOW.to_string(), group.name.to_string()); + out.push(( + field_with_md(group.overflow_col.to_string(), &overflow, md), + overflow, + )); + + Ok(out) +} + +/// Flatten an OTAP logs batch into the wide flat record batch. +pub fn flatten(otap: &OtapArrowRecords) -> StudyResult { + let logs = logs_batch(otap)?; + let num_rows = logs.num_rows(); + let resource_id = logs_resource_id(logs)?; + let scope_id = logs_scope_id(logs)?; + let log_id = logs_id(logs)?; + let parents = [resource_id, scope_id, log_id]; + + let mut fields: Vec = logs + .schema() + .fields() + .iter() + .map(|f| f.as_ref().clone()) + .collect(); + let mut columns: Vec = logs.columns().to_vec(); + + for (group, parent) in GROUPS.iter().zip(parents.iter()) { + for (field, array) in explode_group(otap, *group, parent, num_rows)? { + fields.push(field); + columns.push(array); + } + } + + Ok(RecordBatch::try_new( + Arc::new(Schema::new(fields)), + columns, + )?) +} + +fn append_typed( + builder: &mut AttributesRecordBatchBuilder, + otap_type: u8, + array: &ArrayRef, + row: usize, +) { + match otap_type { + T_STR => builder + .any_values_builder + .append_str(downcast::(array).value(row).as_bytes()), + T_INT => builder + .any_values_builder + .append_int(downcast::(array).value(row)), + T_DOUBLE => builder + .any_values_builder + .append_double(downcast::(array).value(row)), + T_BOOL => builder + .any_values_builder + .append_bool(downcast::(array).value(row)), + T_BYTES => builder + .any_values_builder + .append_bytes(downcast::(array).value(row)), + T_MAP => builder + .any_values_builder + .append_map(downcast::(array).value(row)), + T_SLICE => builder + .any_values_builder + .append_slice(downcast::(array).value(row)), + _ => builder.any_values_builder.append_empty(), + } +} + +fn downcast(array: &ArrayRef) -> &T { + array + .as_any() + .downcast_ref::() + .expect("wide column has expected type") +} + +fn append_overflow_row( + builder: &mut AttributesRecordBatchBuilder, + parent_id: u16, + values: &StructArray, + elem: usize, +) { + let key = downcast::(values.column(0)).value(elem); + let t = downcast::(values.column(1)).value(elem); + builder.append_parent_id(&parent_id); + builder.append_key(key.as_bytes()); + // struct columns: 2=str,3=int,4=double,5=bool,6=bytes,7=ser + match t { + T_STR => builder.any_values_builder.append_str( + downcast::(values.column(2)) + .value(elem) + .as_bytes(), + ), + T_INT => builder + .any_values_builder + .append_int(downcast::(values.column(3)).value(elem)), + T_DOUBLE => builder + .any_values_builder + .append_double(downcast::(values.column(4)).value(elem)), + T_BOOL => builder + .any_values_builder + .append_bool(downcast::(values.column(5)).value(elem)), + T_BYTES => builder + .any_values_builder + .append_bytes(downcast::(values.column(6)).value(elem)), + T_MAP => builder + .any_values_builder + .append_map(downcast::(values.column(7)).value(elem)), + T_SLICE => builder + .any_values_builder + .append_slice(downcast::(values.column(7)).value(elem)), + _ => builder.any_values_builder.append_empty(), + } +} + +struct WideColumn { + key: String, + otap_type: u8, + array: ArrayRef, +} + +/// Reconstruct an OTAP logs batch from the wide flat record batch. +pub fn unflatten(flat: &RecordBatch) -> StudyResult { + // Partition columns: plain Logs columns vs. wide attribute / overflow columns. + let mut logs_fields: Vec = Vec::new(); + let mut logs_columns: Vec = Vec::new(); + let mut group_columns: HashMap> = HashMap::new(); + let mut overflow_columns: HashMap = HashMap::new(); + + for (field, column) in flat.schema().fields().iter().zip(flat.columns()) { + let md = field.metadata(); + if let Some(group) = md.get(MD_OVERFLOW) { + let list = column + .as_any() + .downcast_ref::() + .ok_or("overflow column is not a list")? + .clone(); + let _ = overflow_columns.insert(group.clone(), list); + } else if let (Some(group), Some(key), Some(t)) = + (md.get(MD_GROUP), md.get(MD_KEY), md.get(MD_TYPE)) + { + group_columns + .entry(group.clone()) + .or_default() + .push(WideColumn { + key: key.clone(), + otap_type: t.parse().map_err(|_| "bad wide_otap_type")?, + array: column.clone(), + }); + } else { + logs_fields.push(field.as_ref().clone()); + logs_columns.push(column.clone()); + } + } + + let logs = RecordBatch::try_new(Arc::new(Schema::new(logs_fields)), logs_columns)?; + let resource_id = logs_resource_id(&logs)?; + let scope_id = logs_scope_id(&logs)?; + let log_id = logs_id(&logs)?; + let ids = [resource_id, scope_id, log_id]; + + let mut otap = OtapArrowRecords::Logs(Default::default()); + otap.set(ArrowPayloadType::Logs, logs)?; + + for (group, id_arr) in GROUPS.iter().zip(ids.iter()) { + let entries = if group.name == "log" { + super::attrs::entries_per_row(id_arr) + } else { + super::attrs::entries_dedup(id_arr) + }; + let cols = group_columns.get(group.name); + let overflow = overflow_columns.get(group.name); + + let mut builder = AttributesRecordBatchBuilder::::new(); + for (row, pid) in &entries { + if let Some(cols) = cols { + for col in cols { + if col.array.is_valid(*row) { + builder.append_parent_id(pid); + builder.append_key(col.key.as_bytes()); + append_typed(&mut builder, col.otap_type, &col.array, *row); + } + } + } + if let Some(list) = overflow { + let values = list + .values() + .as_any() + .downcast_ref::() + .ok_or("overflow values are not a struct")?; + let offs = list.value_offsets(); + let (start, end) = (offs[*row] as usize, offs[*row + 1] as usize); + for elem in start..end { + append_overflow_row(&mut builder, *pid, values, elem); + } + } + } + otap.set(group.payload, builder.finish()?)?; + } + + Ok(otap) +} + +impl Codec for WideParquetCodec { + fn name(&self) -> &'static str { + "parquet-wide" + } + + fn write(&self, logs: OtapArrowRecords) -> StudyResult> { + let flat = flatten(&logs)?; + parquet_io::write_parquet(&flat, self.compressor.parquet()) + } + + fn read(&self, bytes: &[u8]) -> StudyResult { + let flat = parquet_io::read_parquet(bytes)?; + unflatten(&flat) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::parquet_study::Compressor; + use crate::parquet_study::attrs::assert_logs_equivalent; + use crate::parquet_study::datagen::{LogsGenParams, gen_logs_otap}; + + #[test] + fn wide_round_trip_preserves_structure() { + let params = LogsGenParams { + num_resources: 3, + num_scopes: 2, + num_logs: 5, + }; + let (otap, _) = gen_logs_otap(¶ms); + + for compressor in Compressor::ALL { + let codec = WideParquetCodec { compressor }; + let bytes = codec.write(otap.clone()).expect("write"); + let decoded = codec.read(&bytes).expect("read"); + assert_logs_equivalent(&otap, &decoded, codec.name(), compressor.label()); + } + } +} From af29f711d0e3be8edc42ca93306142ab7fad6d88 Mon Sep 17 00:00:00 2001 From: Joshua MacDonald Date: Tue, 30 Jun 2026 19:29:23 -0700 Subject: [PATCH 02/18] chore(benchmarks): add Vortex contender to the otap_parquet study Adds a Vortex file-format contender behind the optional `vortex` cargo feature. It reuses the nested flattening, stores the flat Arrow batch in an in-memory Vortex file, then reads it back and unflattens to OTAP. Vortex 0.75 pins the same arrow-rs 58.3 as this workspace, so Arrow arrays interoperate directly and the round-trip runs fully in memory. Vortex 0.75 has no FixedSizeBinary encoding, so trace_id and span_id are cast to Binary before writing and restored on read. Decode targets an explicit plain Arrow schema so the OTAP schema check accepts the result. The Vortex path is gated so the default benchmarks build does not pull the heavy dependency. On this OTAP-logs full round-trip, Vortex at default settings is larger on the wire and much slower to write than zstd-Parquet. The benchmark README records the numbers and the caveats, including that Vortex targets selective reads that this workload does not exercise. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- rust/otap-dataflow/benchmarks/Cargo.toml | 9 + .../benchmarks/benches/otap_parquet/README.md | 249 ++++++++++++------ .../benchmarks/benches/otap_parquet/main.rs | 32 ++- .../benchmarks/src/parquet_study/mod.rs | 54 ++-- .../benchmarks/src/parquet_study/vortex.rs | 249 ++++++++++++++++++ 5 files changed, 479 insertions(+), 114 deletions(-) create mode 100644 rust/otap-dataflow/benchmarks/src/parquet_study/vortex.rs diff --git a/rust/otap-dataflow/benchmarks/Cargo.toml b/rust/otap-dataflow/benchmarks/Cargo.toml index 2382022ca5..66f2992718 100644 --- a/rust/otap-dataflow/benchmarks/Cargo.toml +++ b/rust/otap-dataflow/benchmarks/Cargo.toml @@ -23,6 +23,15 @@ otap-df-otap = { workspace = true } otap-df-pdata = { workspace = true, features = ["bench", "testing"] } otap-df-pdata-views = { workspace = true } +# Optional: the Vortex file format contender for the parquet study. Heavy +# dependency, so gated behind the `vortex` feature. Vortex 0.75 pins arrow 58.3 +# (same as this workspace), so Arrow types interoperate directly. +vortex = { version = "0.75", optional = true, features = ["tokio"] } + +[features] +# Enables the Vortex contender in the otap_parquet benchmark. +vortex = ["dep:vortex"] + [dev-dependencies] criterion = { workspace = true, features = ["html_reports", "async_tokio"] } tonic = { workspace = true } diff --git a/rust/otap-dataflow/benchmarks/benches/otap_parquet/README.md b/rust/otap-dataflow/benchmarks/benches/otap_parquet/README.md index e223568a55..5c2d281711 100644 --- a/rust/otap-dataflow/benchmarks/benches/otap_parquet/README.md +++ b/rust/otap-dataflow/benchmarks/benches/otap_parquet/README.md @@ -2,52 +2,56 @@ # `otap_parquet` benchmark: OTAP/IPC vs flattened Parquet -**Status:** *experiment* (branch `jmacd/parquet_study`) +**Status:** experiment on branch `jmacd/parquet_study`. -This benchmark quantifies moving OTAP **logs** from a client to a server that -ultimately stores **flattened, single-file Parquet**. Because the data ends up -as Parquet on the server either way, the central question is where the -`OTAP -> Parquet` conversion CPU is spent: +This benchmark quantifies moving OTAP logs from a client to a server that +ultimately stores flattened, single-file Parquet. Because the data ends up as +Parquet on the server either way, the central question is where the OTAP to +Parquet conversion CPU is spent. Option A has the client send OTAP as Arrow IPC +and the server convert it to Parquet, paying for IPC decode plus flatten plus +Parquet encode. Option B has the client precompute the flattened Parquet and +send that, so the server only persists the bytes, or reparses them into Arrow if +it must partition, index, or validate. -- **Option A** — the client sends OTAP as Arrow IPC and the **server** converts - it to Parquet (IPC-decode + flatten + Parquet-encode). -- **Option B** — the **client** precomputes the flattened Parquet and sends - that; the server just persists the bytes (or reparses them into Arrow if it - needs to partition/index/validate). - -It measures **write time**, **read time**, **serialized size**, and the -**server-side conversion cost** for the same logs batch across four -representations and four compressors. +The benchmark measures write time, read time, serialized size, and the +server-side conversion cost for the same logs batch across four representations +and four compressors. ## Contenders -- `ipc` — the OTAP representation we have today: interleaved Arrow IPC streams - (`Producer` / `Consumer`), each per-payload stream compressed. -- `parquet-nested` — a single flattened Parquet file where each log row carries - its denormalized resource / scope / log attributes as +- `ipc` is the OTAP representation we have today: interleaved Arrow IPC streams + produced by `Producer` and consumed by `Consumer`, with each per-payload + stream compressed. +- `parquet-nested` is a single flattened Parquet file where each log row carries + its denormalized resource, scope, and log attributes as `List` columns. -- `parquet-map` — the same, but attributes are `Map`. -- `parquet-wide` — the "analytics-flat" extreme: every distinct attribute key - becomes its own typed top-level column (`resource.`, `scope.`, - `log.`). +- `parquet-map` is the same, with attributes stored as + `Map`. +- `parquet-wide` is the analytics-flat extreme, where every distinct attribute + key becomes its own typed top-level column named `resource.`, + `scope.`, or `log.`. +- `vortex` applies the same nested flattening but stores the result in a + [Vortex](https://vortex.dev) file instead of Parquet. It requires the `vortex` + cargo feature, which is a heavy dependency. Vortex applies its own cascading + compression, so it exposes a single setting reported as `none`. ## Compressors Compressors are explicit codecs so `zstd` can be compared head-to-head with -`lz4`. This matters for cross-language consumers: some Arrow/Parquet stacks (for -example certain .NET builds) may not support `zstd`, so `lz4` (and `snappy` for -Parquet) need first-class numbers. +`lz4`. This matters for cross-language consumers, because some Arrow and Parquet +stacks, for example certain .NET builds, may not support `zstd`. In that case +`lz4`, and `snappy` for Parquet, need first-class numbers. -| compressor | Arrow IPC | Parquet | -|------------|---------------|-------------| -| `zstd` | `ZSTD` | `ZSTD` | -| `lz4` | `LZ4_FRAME` | `LZ4_RAW` | -| `snappy` | *unsupported* | `SNAPPY` | -| `none` | uncompressed | uncompressed| +| compressor | Arrow IPC | Parquet | +|------------|---------------|--------------| +| `zstd` | `ZSTD` | `ZSTD` | +| `lz4` | `LZ4_FRAME` | `LZ4_RAW` | +| `snappy` | *unsupported* | `SNAPPY` | +| `none` | uncompressed | uncompressed | Arrow IPC only supports `zstd` and `lz4`, so `snappy` is offered for the Parquet -schemes only. Parquet uses `LZ4_RAW` (the cross-language interoperable variant, -not the deprecated Hadoop-framed `LZ4`). +schemes only. Parquet uses `LZ4_RAW`, the cross-language interoperable variant, +rather than the deprecated Hadoop-framed `LZ4`. ## Running @@ -55,85 +59,156 @@ not the deprecated Hadoop-framed `LZ4`). cargo bench -p benchmarks --bench otap_parquet ``` -Two tables (serialized size, and the server-side CPU model) are printed to -stdout before the timed benchmarks run. For a quick pass: +Two tables, one for serialized size and one for the server-side CPU model, are +printed to stdout before the timed benchmarks run. For a quick pass: ```bash cargo bench -p benchmarks --bench otap_parquet -- \ --warm-up-time 0.3 --measurement-time 0.6 --sample-size 10 ``` +To include the Vortex contender: + +```bash +cargo bench -p benchmarks --bench otap_parquet --features vortex +``` + ## What each measurement covers -- **IPC write:** `Producer::produce_bar` + prost-encode `BatchArrowRecords`. -- **IPC read:** prost-decode, `Consumer::consume_bar`, `from_record_messages`, - and `decode_transport_optimized_ids` back to `OtapArrowRecords`. -- **Parquet write:** flatten `OtapArrowRecords` → one Arrow `RecordBatch` → - `ArrowWriter` → `Vec`. -- **Parquet read:** `ParquetRecordBatchReader` → Arrow `RecordBatch` → unflatten - back to `OtapArrowRecords`. -- **server_cost `convert-A`:** IPC bytes → decode → flatten → Parquet bytes - (Option A: the server converts). -- **server_cost `accept-B`:** Parquet bytes → Arrow `RecordBatch`, i.e. reparse - without rebuilding OTAP (Option B when the server must touch the data). If the - server only persists the received bytes, its conversion CPU is ~0. - -The flattened Parquet layouts keep the entire root `Logs` record batch intact -(so decode carries its scalar/struct columns straight back, just like the IPC -path does — not penalized by re-walking the body) and only the attribute tables -are denormalized and rebuilt. Resource/scope attribute sets are re-normalized on -decode using the `resource.id` / `scope.id` join keys the `Logs` batch still -carries. `parquet-wide` is lossless for type-consistent keys; any attribute that -does not fit its single typed column spills into a per-group `List` -overflow column, so the round-trip stays exact. +- IPC write runs `Producer::produce_bar` and prost-encodes the + `BatchArrowRecords`. +- IPC read runs prost-decode, then `Consumer::consume_bar`, + `from_record_messages`, and `decode_transport_optimized_ids` back to + `OtapArrowRecords`. +- Parquet write flattens `OtapArrowRecords` to one Arrow `RecordBatch`, then runs + `ArrowWriter` to a `Vec`. +- Parquet read runs `ParquetRecordBatchReader` to an Arrow `RecordBatch`, then + unflattens back to `OtapArrowRecords`. +- The server_cost `convert-A` measurement takes IPC bytes through decode, + flatten, and Parquet encode. This is the cost the server pays in Option A. +- The server_cost `accept-B` measurement takes Parquet bytes back to an Arrow + `RecordBatch` without rebuilding OTAP. This is what the server pays in Option B + when it must touch the data. If the server only persists the received bytes, + its conversion CPU is roughly zero. + +The flattened Parquet layouts keep the entire root `Logs` record batch intact, so +decode carries its scalar and struct columns straight back just as the IPC path +does, and the comparison is not penalized by re-walking the body. Only the +attribute tables are denormalized and rebuilt. Resource and scope attribute sets +are re-normalized on decode using the `resource.id` and `scope.id` join keys the +`Logs` batch still carries. `parquet-wide` is lossless for type-consistent keys. +Any attribute that does not fit its single typed column spills into a per-group +`List` overflow column, so the round-trip stays exact. ## Illustrative results -From one development machine (WSL, jemalloc); absolute values vary by host, but -the relationships are stable. Shape `r1_s1_l5000` = 5000 log records under a -single resource/scope (OTLP proto = 1,135,223 bytes). +These numbers come from one development machine running WSL with jemalloc. +Absolute values vary by host, but the relationships are stable. Shape +`r1_s1_l5000` is 5000 log records under a single resource and scope, and its OTLP +protobuf encoding is 1,135,223 bytes. -Serialized size (bytes): +Serialized size in bytes: | contender | zstd | lz4 | snappy | none | |----------------|--------|--------|--------|-----------| -| ipc | 53,614 | 63,598 | — | 1,236,336 | +| ipc | 53,614 | 63,598 | n/a | 1,236,336 | | parquet-nested | 35,634 | 44,904 | 52,471 | 223,441 | | parquet-map | 35,946 | 45,216 | 52,783 | 223,753 | | parquet-wide | 40,524 | 48,907 | 48,939 | 50,532 | -Server-side conversion cost (indicative ms), shape `r1_s1_l5000`: +Write and read time for shape `r1_s1_l5000` with `zstd`: + +| contender | write | read | +|----------------|----------|---------| +| ipc | 1.31 ms | 0.69 ms | +| parquet-nested | 13.88 ms | 8.00 ms | +| parquet-map | 16.26 ms | 8.73 ms | +| parquet-wide | 9.14 ms | 6.37 ms | -| flatten / comp | A: server converts | B: reparse | saved if server just stores | -|-----------------------|--------------------|------------|-----------------------------| -| parquet-nested / zstd | 14.46 ms | 2.88 ms | 14.46 ms | -| parquet-nested / lz4 | 13.49 ms | 2.80 ms | 13.49 ms | -| parquet-wide / lz4 | 9.95 ms | 1.19 ms | 9.95 ms | +Server-side conversion cost for shape `r1_s1_l5000`, in indicative +milliseconds. Column A is the server converting received OTAP/IPC, column B is +the server reparsing client Parquet, and the last column is the server CPU saved +when the server simply stores client Parquet. + +| flatten / comp | A convert | B reparse | saved on store | +|-----------------------|-----------|-----------|----------------| +| parquet-nested / zstd | 14.46 ms | 2.88 ms | 14.46 ms | +| parquet-nested / lz4 | 13.49 ms | 2.80 ms | 13.49 ms | +| parquet-wide / lz4 | 9.95 ms | 1.19 ms | 9.95 ms | ## Takeaways -- **The conversion is the expensive part.** Flatten + Parquet-encode dominates; - IPC decode is a small fraction. Accepting client-precomputed Parquet (Option - B, persist) removes essentially all of that server CPU — here ~10–14 ms per - 5000-log batch, an ~80–100% reduction versus converting server-side. Even if - the server must reparse the Parquet into Arrow, it still avoids the - flatten+encode and saves ~80%. -- **zstd vs lz4.** For IPC, lz4 is ~19% larger than zstd; for Parquet-nested, - lz4 is ~26% larger and snappy ~47% larger than zstd. But lz4 encode/decode is - a touch *cheaper* on CPU, and `parquet-nested/lz4` (44.9 KB) is still smaller - than `ipc/zstd` (53.6 KB) — so a zstd-less client (e.g. some .NET stacks) can - use lz4 (or snappy) and remain competitive on the wire. -- **Layout.** `parquet-nested`/`parquet-map` compress smallest with zstd; - `parquet-wide` is far smaller uncompressed and is the cheapest to write and to - reparse (typed columns), at a modest size premium under zstd. -- **For the debate:** if server CPU is the constraint (it is here), have clients +- The conversion is the expensive part. Flatten plus Parquet encode dominates, + and IPC decode is a small fraction. Accepting client-precomputed Parquet and + persisting it removes essentially all of that server CPU, which is roughly 10 + to 14 ms per 5000-log batch, an 80 to 100 percent reduction versus converting + server-side. Even a server that must reparse the Parquet into Arrow still + avoids the flatten and encode and saves about 80 percent. +- On zstd versus lz4, IPC with lz4 is about 19 percent larger than zstd, and + Parquet-nested with lz4 is about 26 percent larger while snappy is about 47 + percent larger than zstd. lz4 encode and decode is a touch cheaper on CPU, and + `parquet-nested/lz4` at 44.9 KB is still smaller than `ipc/zstd` at 53.6 KB, so + a client that cannot use zstd, such as some .NET stacks, can use lz4 or snappy + and remain competitive on the wire. +- On layout, `parquet-nested` and `parquet-map` compress smallest with zstd, + while `parquet-wide` is far smaller uncompressed and is the cheapest to write + and to reparse thanks to typed columns, at a modest size premium under zstd. +- For the debate, if server CPU is the constraint, and here it is, have clients send precomputed flattened Parquet and let the server persist it. lz4 is a safe cross-language codec choice with a small size cost relative to zstd. +## Vortex + +Vortex is a next-generation, Arrow-native columnar file format. Vortex 0.75 pins +the same arrow-rs 58.3 as this workspace, so Arrow arrays flow in and out with no +version bridging, and the whole round-trip runs in memory over a `Vec` write +sink and a `ByteBuffer` reader. The contender reuses the nested flattening and +feeds the flat `RecordBatch` to Vortex. + +Integration notes: + +- Vortex 0.75 has no `FixedSizeBinary` encoding, so `trace_id` and `span_id` are + cast to `Binary` before writing and restored on read. +- Vortex prefers view and plain Arrow types on read, so decode targets an + explicit plain schema, with dictionaries decoded and `FixedSizeBinary` + restored, that the OTAP schema check accepts. +- Vortex applies its own cascading BtrBlocks compression, so there is no external + compressor knob. + +Illustrative results for shape `r1_s1_l5000`, which is 5000 log records: + +| metric | vortex | parquet-nested/zstd | ipc/zstd | +|---------------------------|----------|---------------------|----------| +| serialized size | 107.0 KB | 35.6 KB | 53.6 KB | +| size vs OTLP proto | 10.6x | 31.9x | 21.2x | +| server convert IPC to file| ~510 ms | ~14 ms | n/a | +| reparse file to Arrow | ~11.9 ms | ~2.9 ms | n/a | + +Vortex write time at smaller shapes is about 43 ms for 250 logs and about 101 ms +for 1000 logs. + +The finding is that for this OTAP-logs full round-trip, Vortex at default +settings is both larger on the wire, roughly twice `ipc/zstd` and three times +`parquet-nested/zstd`, and much slower to write, by an order of magnitude or +more, than zstd-Parquet. It therefore does not help the goal of having the client +precompute so the server offloads CPU. + +Several caveats make this a starting point rather than a verdict on Vortex. The +benchmark does full batch round-trips, whereas Vortex is designed for selective +reads with column and row-range projection, pushdown, and random access, none of +which this workload exercises. The default BtrBlocks compressor samples encodings +to favor decode speed and random access over write speed and maximum ratio, and +neither a lighter nor a heavier write strategy was explored. Each write and read +creates a fresh `VortexSession`, whereas a real deployment would reuse it, +although the dominant cost is the compression itself. Finally, the nested +`List` attribute layout may not play to Vortex's strengths, and the wide +layout was not tried with Vortex. Tuning the Vortex write strategy and exercising +selective reads are the obvious next steps. + ## Extending -Contenders are the `Scheme` enum and its `Codec` impls in -`benchmarks/src/parquet_study`; add a variant to include it everywhere. Input -shapes are in `input_shapes()` in `benches/otap_parquet/main.rs`. The -flatten/unflatten round-trips and the server-cost helpers have unit tests -(`cargo test -p benchmarks --lib parquet_study`). +Contenders are the `Scheme` enum and its `Codec` implementations in +`benchmarks/src/parquet_study`. Add a variant to include a contender everywhere. +Input shapes are defined in `input_shapes()` in `benches/otap_parquet/main.rs`. +The flatten and unflatten round-trips and the server-cost helpers have unit tests +runnable with `cargo test -p benchmarks --lib parquet_study`. diff --git a/rust/otap-dataflow/benchmarks/benches/otap_parquet/main.rs b/rust/otap-dataflow/benchmarks/benches/otap_parquet/main.rs index 84d37eb300..4448d6718f 100644 --- a/rust/otap-dataflow/benchmarks/benches/otap_parquet/main.rs +++ b/rust/otap-dataflow/benchmarks/benches/otap_parquet/main.rs @@ -35,7 +35,19 @@ use std::hint::black_box; use std::time::{Duration, Instant}; use benchmarks::parquet_study::datagen::{LogsGenParams, gen_logs_otap}; -use benchmarks::parquet_study::{Compressor, Scheme, server}; +use benchmarks::parquet_study::{Compressor, Scheme, StudyResult, server}; + +/// Reparse a stored flattened file back into an Arrow record batch (no +/// unflatten), dispatching on the scheme's format. Used by the server-cost +/// "accept-B" measurement. +#[allow(unused_variables)] +fn reparse(scheme: Scheme, bytes: &[u8]) -> StudyResult { + #[cfg(feature = "vortex")] + if matches!(scheme, Scheme::Vortex) { + return benchmarks::parquet_study::vortex::reparse_to_arrow(bytes); + } + server::reparse_parquet(bytes) +} #[cfg(not(windows))] use tikv_jemallocator::Jemalloc; @@ -98,7 +110,7 @@ fn print_size_table(shapes: &[LogsGenParams]) { .expect("ipc zstd write") .len(); - for scheme in Scheme::ALL { + for scheme in Scheme::all() { for &compressor in scheme.compressors() { let codec = scheme.codec(compressor); let bytes = codec @@ -167,8 +179,8 @@ fn print_server_cost_table(shapes: &[LogsGenParams]) { .write(otap.clone()) .expect("ipc write"); - for scheme in Scheme::PARQUET { - for compressor in Compressor::ALL { + for scheme in Scheme::flattened() { + for &compressor in scheme.compressors() { let pcodec = scheme.codec(compressor); let parquet_bytes = pcodec.write(otap.clone()).expect("parquet write"); @@ -176,7 +188,7 @@ fn print_server_cost_table(shapes: &[LogsGenParams]) { let _ = server::convert_ipc_to_parquet(&ipc_bytes, &*pcodec).expect("convert"); }); let reparse_b = median_ms(|| { - let _ = server::reparse_parquet(&parquet_bytes).expect("reparse"); + let _ = reparse(scheme, &parquet_bytes).expect("reparse"); }); println!( @@ -205,7 +217,7 @@ fn bench_round_trip(c: &mut Criterion) { let _ = write_group.measurement_time(Duration::from_secs(2)); for shape in &shapes { let (otap, _) = gen_logs_otap(shape); - for scheme in Scheme::ALL { + for scheme in Scheme::all() { for &compressor in scheme.compressors() { let codec = scheme.codec(compressor); let id = BenchmarkId::new( @@ -230,7 +242,7 @@ fn bench_round_trip(c: &mut Criterion) { let _ = read_group.measurement_time(Duration::from_secs(2)); for shape in &shapes { let (otap, _) = gen_logs_otap(shape); - for scheme in Scheme::ALL { + for scheme in Scheme::all() { for &compressor in scheme.compressors() { let codec = scheme.codec(compressor); let bytes = codec.write(otap.clone()).expect("write"); @@ -263,8 +275,8 @@ fn bench_server_cost(c: &mut Criterion) { .write(otap.clone()) .expect("ipc write"); - for scheme in Scheme::PARQUET { - for compressor in Compressor::ALL { + for scheme in Scheme::flattened() { + for &compressor in scheme.compressors() { let pcodec = scheme.codec(compressor); // Option A: server converts received OTAP/IPC to Parquet. @@ -287,7 +299,7 @@ fn bench_server_cost(c: &mut Criterion) { shape.label(), ); let _ = group.bench_with_input(id_b, shape, |b, _| { - b.iter(|| black_box(server::reparse_parquet(&parquet_bytes).expect("reparse"))); + b.iter(|| black_box(reparse(scheme, &parquet_bytes).expect("reparse"))); }); } } diff --git a/rust/otap-dataflow/benchmarks/src/parquet_study/mod.rs b/rust/otap-dataflow/benchmarks/src/parquet_study/mod.rs index 43c309ec29..872eb526e9 100644 --- a/rust/otap-dataflow/benchmarks/src/parquet_study/mod.rs +++ b/rust/otap-dataflow/benchmarks/src/parquet_study/mod.rs @@ -22,6 +22,8 @@ pub mod map; pub mod nested; pub mod parquet_io; pub mod server; +#[cfg(feature = "vortex")] +pub mod vortex; pub mod wide; /// Error type used throughout the study (benchmark/test code, so a boxed error @@ -127,15 +129,33 @@ pub enum Scheme { Map, /// Flattened Parquet, attributes exploded to one typed column per key. Wide, + /// Flattened Vortex file (nested layout). Requires the `vortex` feature. + #[cfg(feature = "vortex")] + Vortex, } impl Scheme { - /// All schemes, in reporting order. - pub const ALL: [Scheme; 4] = [Scheme::Ipc, Scheme::Nested, Scheme::Map, Scheme::Wide]; + /// All schemes, in reporting order (includes Vortex when the `vortex` + /// feature is enabled). + #[must_use] + pub fn all() -> Vec { + #[allow(unused_mut)] + let mut v = vec![Scheme::Ipc, Scheme::Nested, Scheme::Map, Scheme::Wide]; + #[cfg(feature = "vortex")] + v.push(Scheme::Vortex); + v + } - /// Only the flattened-Parquet schemes (used by the server-cost model, where - /// IPC is the input rather than an output format). - pub const PARQUET: [Scheme; 3] = [Scheme::Nested, Scheme::Map, Scheme::Wide]; + /// Only the flattened-file schemes (used by the server-cost model, where IPC + /// is the input rather than an output format). Includes Vortex when enabled. + #[must_use] + pub fn flattened() -> Vec { + #[allow(unused_mut)] + let mut v = vec![Scheme::Nested, Scheme::Map, Scheme::Wide]; + #[cfg(feature = "vortex")] + v.push(Scheme::Vortex); + v + } /// Stable contender name. #[must_use] @@ -145,22 +165,20 @@ impl Scheme { Scheme::Nested => "parquet-nested", Scheme::Map => "parquet-map", Scheme::Wide => "parquet-wide", + #[cfg(feature = "vortex")] + Scheme::Vortex => "vortex", } } - /// Whether this scheme produces Parquet (as opposed to Arrow IPC). - #[must_use] - pub fn is_parquet(self) -> bool { - !matches!(self, Scheme::Ipc) - } - - /// The compressors valid for this scheme (IPC excludes snappy). + /// The compressors valid for this scheme. IPC excludes snappy; Vortex applies + /// its own cascading compression, so it exposes only a single `none` setting. #[must_use] pub fn compressors(self) -> &'static [Compressor] { - if self.is_parquet() { - &Compressor::ALL - } else { - &Compressor::IPC + match self { + Scheme::Ipc => &Compressor::IPC, + #[cfg(feature = "vortex")] + Scheme::Vortex => &[Compressor::None], + _ => &Compressor::ALL, } } @@ -172,6 +190,8 @@ impl Scheme { Scheme::Nested => Box::new(nested::NestedParquetCodec { compressor }), Scheme::Map => Box::new(map::MapParquetCodec { compressor }), Scheme::Wide => Box::new(wide::WideParquetCodec { compressor }), + #[cfg(feature = "vortex")] + Scheme::Vortex => Box::new(vortex::VortexCodec), } } } @@ -192,7 +212,7 @@ mod tests { }; let (otap, _) = gen_logs_otap(¶ms); - for scheme in Scheme::ALL { + for scheme in Scheme::all() { for &compressor in scheme.compressors() { let codec = scheme.codec(compressor); let bytes = codec diff --git a/rust/otap-dataflow/benchmarks/src/parquet_study/vortex.rs b/rust/otap-dataflow/benchmarks/src/parquet_study/vortex.rs new file mode 100644 index 0000000000..28b7669c3a --- /dev/null +++ b/rust/otap-dataflow/benchmarks/src/parquet_study/vortex.rs @@ -0,0 +1,249 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +//! Vortex file-format contender (enabled by the `vortex` cargo feature). +//! +//! Vortex is a next-generation columnar file format that interoperates with +//! Arrow. This contender reuses the same flattening as [`super::nested`] (one +//! flat Arrow `RecordBatch` per logs batch) but serializes it to an in-memory +//! Vortex file instead of Parquet, then reads it back and unflattens to OTAP. +//! +//! Vortex applies its own cascading compression (BtrBlocks), so the study's +//! [`Compressor`](super::Compressor) axis does not apply; the single setting is +//! reported as `none` (meaning "Vortex default encodings"). + +use std::sync::{Arc, OnceLock}; + +use arrow::array::{Array, ArrayRef as ArrowArrayRef, RecordBatch, StructArray}; +use arrow::compute::cast; +use arrow::datatypes::{DataType, Field, Fields, Schema}; +use otap_df_pdata::otap::OtapArrowRecords; +use otap_df_pdata::schema::consts; + +use vortex::VortexSessionDefault; +use vortex::array::ArrayRef; +use vortex::array::VortexSessionExecute; +use vortex::array::arrow::ArrowSessionExt; +use vortex::array::arrow::FromArrowArray; +use vortex::array::stream::ArrayStreamExt; +use vortex::buffer::ByteBuffer; +use vortex::file::{OpenOptionsSessionExt, WriteOptionsSessionExt}; +use vortex::io::session::RuntimeSessionExt; +use vortex::session::VortexSession; + +use super::{Codec, StudyResult, nested}; + +/// Contender that flattens OTAP logs (nested layout) and stores them in an +/// in-memory Vortex file. +pub struct VortexCodec; + +fn runtime() -> &'static tokio::runtime::Runtime { + static RT: OnceLock = OnceLock::new(); + RT.get_or_init(|| { + tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .expect("build tokio runtime") + }) +} + +/// True for `FixedSizeBinary` or a dictionary whose values are `FixedSizeBinary`. +/// Vortex 0.75 has no `FixedSizeBinary` encoding, so such columns (the logs +/// `trace_id`/`span_id`, which are dictionary-encoded in the OTAP form) must be +/// rewritten before writing. +fn is_fixed_size_binary(dt: &DataType) -> bool { + match dt { + DataType::FixedSizeBinary(_) => true, + DataType::Dictionary(_, value) => matches!(value.as_ref(), DataType::FixedSizeBinary(_)), + _ => false, + } +} + +/// Cast any `FixedSizeBinary`-valued column to plain `Binary` before writing. +fn to_vortex_compatible(flat: &RecordBatch) -> StudyResult { + let mut fields: Vec = Vec::with_capacity(flat.num_columns()); + let mut columns: Vec = Vec::with_capacity(flat.num_columns()); + for (field, column) in flat.schema().fields().iter().zip(flat.columns()) { + if is_fixed_size_binary(field.data_type()) { + fields.push(Field::new( + field.name(), + DataType::Binary, + field.is_nullable(), + )); + columns.push(cast(column, &DataType::Binary)?); + } else { + fields.push(field.as_ref().clone()); + columns.push(column.clone()); + } + } + Ok(RecordBatch::try_new( + Arc::new(Schema::new(fields)), + columns, + )?) +} + +/// Reduce a data type to a "plain" Arrow type: decode dictionaries to their +/// value type and turn `FixedSizeBinary` into `Binary`, recursing through struct +/// and list children. Used as the target for Vortex's `execute_arrow` so the +/// decoded batch has deterministic, non-view, non-dictionary types that the OTAP +/// schema check accepts. (`trace_id`/`span_id` are later restored to +/// `FixedSizeBinary`.) +fn plainify(dt: &DataType) -> DataType { + match dt { + DataType::Dictionary(_, value) => plainify(value), + DataType::FixedSizeBinary(_) => DataType::Binary, + DataType::Struct(fields) => DataType::Struct(plainify_fields(fields)), + DataType::List(field) => DataType::List(Arc::new(plainify_field(field))), + DataType::LargeList(field) => DataType::List(Arc::new(plainify_field(field))), + other => other.clone(), + } +} + +fn plainify_field(field: &Field) -> Field { + Field::new( + field.name(), + plainify(field.data_type()), + field.is_nullable(), + ) +} + +fn plainify_fields(fields: &Fields) -> Fields { + Fields::from(fields.iter().map(|f| plainify_field(f)).collect::>()) +} + +/// The Arrow struct field describing the plain type Vortex should decode into. +/// Derived once from a sample flattened batch (the flat schema is deterministic). +fn vortex_target_field() -> &'static Field { + static TARGET: OnceLock = OnceLock::new(); + TARGET.get_or_init(|| { + use crate::parquet_study::datagen::{LogsGenParams, gen_logs_otap}; + let (otap, _) = gen_logs_otap(&LogsGenParams { + num_resources: 1, + num_scopes: 1, + num_logs: 1, + }); + let flat = nested::flatten(&otap).expect("sample flatten"); + let compat = to_vortex_compatible(&flat).expect("sample compat"); + Field::new( + "", + DataType::Struct(plainify_fields(compat.schema().fields())), + false, + ) + }) +} + +/// Restore the `FixedSizeBinary` columns that [`to_vortex_compatible`] cast to +/// `Binary`, so the reconstructed Logs batch matches the OTAP schema. +fn restore_fixed_size_binary(batch: RecordBatch) -> StudyResult { + let restores = [(consts::TRACE_ID, 16i32), (consts::SPAN_ID, 8i32)]; + let mut fields: Vec = batch + .schema() + .fields() + .iter() + .map(|f| f.as_ref().clone()) + .collect(); + let mut columns: Vec = batch.columns().to_vec(); + for (name, size) in restores { + if let Ok(idx) = batch.schema().index_of(name) { + if matches!( + columns[idx].data_type(), + DataType::Binary | DataType::LargeBinary + ) { + columns[idx] = cast(&columns[idx], &DataType::FixedSizeBinary(size))?; + fields[idx] = Field::new( + name, + DataType::FixedSizeBinary(size), + fields[idx].is_nullable(), + ); + } + } + } + Ok(RecordBatch::try_new( + Arc::new(Schema::new(fields)), + columns, + )?) +} + +/// Encode a flat Arrow record batch as an in-memory Vortex file. +fn encode(flat: RecordBatch) -> StudyResult> { + let flat = to_vortex_compatible(&flat)?; + runtime() + .block_on(async move { + let session = VortexSession::default().with_tokio(); + let array = ArrayRef::from_arrow(flat, false)?; + let mut sink: Vec = Vec::with_capacity(4096); + let _summary = session + .write_options() + .write(&mut sink, array.to_array_stream()) + .await?; + Ok::, vortex::error::VortexError>(sink) + }) + .map_err(Into::into) +} + +/// Decode an in-memory Vortex file back into a flat Arrow record batch. +fn decode(bytes: &[u8]) -> StudyResult { + let buffer = ByteBuffer::from(bytes.to_vec()); + let batch = runtime() + .block_on(async move { + let session = VortexSession::default().with_tokio(); + let file = session.open_options().open_buffer(buffer)?; + let array = file.scan()?.into_array_stream()?.read_all().await?; + let mut ctx = session.create_execution_ctx(); + let arrow_session = session.arrow(); + let arrow = + arrow_session.execute_arrow(array, Some(vortex_target_field()), &mut ctx)?; + let struct_array = arrow + .as_any() + .downcast_ref::() + .expect("vortex top-level array is a struct"); + Ok::(RecordBatch::from(struct_array.clone())) + }) + .map_err(Box::::from)?; + restore_fixed_size_binary(batch) +} + +impl Codec for VortexCodec { + fn name(&self) -> &'static str { + "vortex" + } + + fn write(&self, logs: OtapArrowRecords) -> StudyResult> { + let flat = nested::flatten(&logs)?; + encode(flat) + } + + fn read(&self, bytes: &[u8]) -> StudyResult { + let flat = decode(bytes)?; + nested::unflatten(&flat) + } +} + +/// Decode a Vortex file back into the flat Arrow record batch, without +/// unflattening to OTAP. Used by the server-cost model's "reparse" measurement. +pub fn reparse_to_arrow(bytes: &[u8]) -> StudyResult { + decode(bytes) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::parquet_study::attrs::assert_logs_equivalent; + use crate::parquet_study::datagen::{LogsGenParams, gen_logs_otap}; + + #[test] + fn vortex_round_trip_preserves_structure() { + let params = LogsGenParams { + num_resources: 3, + num_scopes: 2, + num_logs: 5, + }; + let (otap, _) = gen_logs_otap(¶ms); + + let codec = VortexCodec; + let bytes = codec.write(otap.clone()).expect("write"); + assert!(!bytes.is_empty()); + let decoded = codec.read(&bytes).expect("read"); + assert_logs_equivalent(&otap, &decoded, codec.name(), "none"); + } +} From 33a853adf3ce4bcb49882830aa67c71348e23c36 Mon Sep 17 00:00:00 2001 From: Joshua MacDonald Date: Tue, 30 Jun 2026 21:11:06 -0700 Subject: [PATCH 03/18] chore(benchmarks): study Vortex write throughput at 10k/100k block sizes Adds a vortex-fast contender that writes the Vortex file with BtrBlocks disabled, to test whether skipping compression lets Vortex write faster than Parquet, and switches the input shapes to realistic 10k and 100k log-record blocks. The result is that Vortex does not write faster than Parquet here. Even with compression off, Vortex write is slower than Parquet write at both sizes, and disabling BtrBlocks also disables dictionary encoding, so the denormalized data fully expands to 13 MB at 10k and 186 MB at 100k. The write bottleneck is the Vortex write pipeline itself, which canonicalizes, repartitions, computes zone statistics, and builds a layout and footer, not the compressor. The README records the numbers and notes that Vortex targets selective reads this workload does not exercise. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../benchmarks/benches/otap_parquet/README.md | 221 ++++++++---------- .../benchmarks/benches/otap_parquet/main.rs | 29 +-- .../benchmarks/src/parquet_study/mod.rs | 16 +- .../benchmarks/src/parquet_study/vortex.rs | 50 ++-- 4 files changed, 156 insertions(+), 160 deletions(-) diff --git a/rust/otap-dataflow/benchmarks/benches/otap_parquet/README.md b/rust/otap-dataflow/benchmarks/benches/otap_parquet/README.md index 5c2d281711..f82d0c7bcd 100644 --- a/rust/otap-dataflow/benchmarks/benches/otap_parquet/README.md +++ b/rust/otap-dataflow/benchmarks/benches/otap_parquet/README.md @@ -14,8 +14,8 @@ send that, so the server only persists the bytes, or reparses them into Arrow if it must partition, index, or validate. The benchmark measures write time, read time, serialized size, and the -server-side conversion cost for the same logs batch across four representations -and four compressors. +server-side conversion cost for the same logs batch across several +representations and compressors. ## Contenders @@ -31,16 +31,19 @@ and four compressors. key becomes its own typed top-level column named `resource.`, `scope.`, or `log.`. - `vortex` applies the same nested flattening but stores the result in a - [Vortex](https://vortex.dev) file instead of Parquet. It requires the `vortex` - cargo feature, which is a heavy dependency. Vortex applies its own cascading - compression, so it exposes a single setting reported as `none`. + [Vortex](https://vortex.dev) file with Vortex's default BtrBlocks compression. +- `vortex-fast` is the same Vortex layout written with BtrBlocks disabled, to + test whether skipping compression lets Vortex write faster than Parquet. The + Vortex contenders require the `vortex` cargo feature, which is a heavy + dependency. ## Compressors Compressors are explicit codecs so `zstd` can be compared head-to-head with `lz4`. This matters for cross-language consumers, because some Arrow and Parquet stacks, for example certain .NET builds, may not support `zstd`. In that case -`lz4`, and `snappy` for Parquet, need first-class numbers. +`lz4`, and `snappy` for Parquet, need first-class numbers. The Vortex contenders +manage their own encoding, so they expose a single setting reported as `none`. | compressor | Arrow IPC | Parquet | |------------|---------------|--------------| @@ -60,19 +63,18 @@ cargo bench -p benchmarks --bench otap_parquet ``` Two tables, one for serialized size and one for the server-side CPU model, are -printed to stdout before the timed benchmarks run. For a quick pass: - -```bash -cargo bench -p benchmarks --bench otap_parquet -- \ - --warm-up-time 0.3 --measurement-time 0.6 --sample-size 10 -``` - -To include the Vortex contender: +printed to stdout before the timed benchmarks run. To include the Vortex +contenders: ```bash cargo bench -p benchmarks --bench otap_parquet --features vortex ``` +The default input shapes are 10k and 100k log records, so the full Criterion +sweep is slow. The printed tables are the main output. For a quick pass, add +`-- --measurement-time 0.5 --sample-size 10`, or read the printed tables and +stop the run. + ## What each measurement covers - IPC write runs `Producer::produce_bar` and prost-encodes the @@ -80,22 +82,24 @@ cargo bench -p benchmarks --bench otap_parquet --features vortex - IPC read runs prost-decode, then `Consumer::consume_bar`, `from_record_messages`, and `decode_transport_optimized_ids` back to `OtapArrowRecords`. -- Parquet write flattens `OtapArrowRecords` to one Arrow `RecordBatch`, then runs - `ArrowWriter` to a `Vec`. -- Parquet read runs `ParquetRecordBatchReader` to an Arrow `RecordBatch`, then - unflattens back to `OtapArrowRecords`. +- File write flattens `OtapArrowRecords` to one Arrow `RecordBatch`, then encodes + it to Parquet or Vortex bytes. +- File read decodes the file to an Arrow `RecordBatch`, then unflattens back to + `OtapArrowRecords`. - The server_cost `convert-A` measurement takes IPC bytes through decode, - flatten, and Parquet encode. This is the cost the server pays in Option A. -- The server_cost `accept-B` measurement takes Parquet bytes back to an Arrow + flatten, and file encode. This is the cost the server pays in Option A. The + flatten and IPC decode are shared across schemes, so differences between + schemes isolate the file-encode cost. +- The server_cost `accept-B` measurement takes file bytes back to an Arrow `RecordBatch` without rebuilding OTAP. This is what the server pays in Option B when it must touch the data. If the server only persists the received bytes, its conversion CPU is roughly zero. -The flattened Parquet layouts keep the entire root `Logs` record batch intact, so -decode carries its scalar and struct columns straight back just as the IPC path -does, and the comparison is not penalized by re-walking the body. Only the -attribute tables are denormalized and rebuilt. Resource and scope attribute sets -are re-normalized on decode using the `resource.id` and `scope.id` join keys the +The flattened layouts keep the entire root `Logs` record batch intact, so decode +carries its scalar and struct columns straight back just as the IPC path does, +and the comparison is not penalized by re-walking the body. Only the attribute +tables are denormalized and rebuilt. Resource and scope attribute sets are +re-normalized on decode using the `resource.id` and `scope.id` join keys the `Logs` batch still carries. `parquet-wide` is lossless for type-consistent keys. Any attribute that does not fit its single typed column spills into a per-group `List` overflow column, so the round-trip stays exact. @@ -103,107 +107,86 @@ Any attribute that does not fit its single typed column spills into a per-group ## Illustrative results These numbers come from one development machine running WSL with jemalloc. -Absolute values vary by host, but the relationships are stable. Shape -`r1_s1_l5000` is 5000 log records under a single resource and scope, and its OTLP -protobuf encoding is 1,135,223 bytes. +Absolute values vary by host, but the relationships are stable. The shape is +10,000 log records under a single resource and scope, whose OTLP protobuf +encoding is 2,270,225 bytes. Serialized size in bytes: -| contender | zstd | lz4 | snappy | none | -|----------------|--------|--------|--------|-----------| -| ipc | 53,614 | 63,598 | n/a | 1,236,336 | -| parquet-nested | 35,634 | 44,904 | 52,471 | 223,441 | -| parquet-map | 35,946 | 45,216 | 52,783 | 223,753 | -| parquet-wide | 40,524 | 48,907 | 48,939 | 50,532 | - -Write and read time for shape `r1_s1_l5000` with `zstd`: - -| contender | write | read | -|----------------|----------|---------| -| ipc | 1.31 ms | 0.69 ms | -| parquet-nested | 13.88 ms | 8.00 ms | -| parquet-map | 16.26 ms | 8.73 ms | -| parquet-wide | 9.14 ms | 6.37 ms | - -Server-side conversion cost for shape `r1_s1_l5000`, in indicative -milliseconds. Column A is the server converting received OTAP/IPC, column B is -the server reparsing client Parquet, and the last column is the server CPU saved -when the server simply stores client Parquet. - -| flatten / comp | A convert | B reparse | saved on store | -|-----------------------|-----------|-----------|----------------| -| parquet-nested / zstd | 14.46 ms | 2.88 ms | 14.46 ms | -| parquet-nested / lz4 | 13.49 ms | 2.80 ms | 13.49 ms | -| parquet-wide / lz4 | 9.95 ms | 1.19 ms | 9.95 ms | +| contender | zstd | lz4 | snappy | none | +|----------------|--------|--------|--------|------------| +| ipc | 92,270 | 108,846| n/a | 2,457,008 | +| parquet-nested | 57,023 | 75,336 | 90,529 | 432,935 | +| parquet-wide | 62,088 | 78,840 | 78,851 | 82,219 | +| vortex | n/a | n/a | n/a | 199,088 | +| vortex-fast | n/a | n/a | n/a | 13,138,288 | + +Server convert cost, which is IPC decode plus flatten plus file encode, in +indicative milliseconds. The flatten and decode are shared, so differences +isolate the encode. The 100k column is the same shape scaled to 100,000 records. + +| contender / comp | convert 10k | convert 100k | +|------------------------|-------------|--------------| +| parquet-nested / zstd | 93 ms | 889 ms | +| parquet-wide / zstd | 44 ms | 577 ms | +| vortex | 692 ms | 1727 ms | +| vortex-fast | 935 ms | 1858 ms | ## Takeaways -- The conversion is the expensive part. Flatten plus Parquet encode dominates, - and IPC decode is a small fraction. Accepting client-precomputed Parquet and - persisting it removes essentially all of that server CPU, which is roughly 10 - to 14 ms per 5000-log batch, an 80 to 100 percent reduction versus converting - server-side. Even a server that must reparse the Parquet into Arrow still - avoids the flatten and encode and saves about 80 percent. -- On zstd versus lz4, IPC with lz4 is about 19 percent larger than zstd, and - Parquet-nested with lz4 is about 26 percent larger while snappy is about 47 - percent larger than zstd. lz4 encode and decode is a touch cheaper on CPU, and - `parquet-nested/lz4` at 44.9 KB is still smaller than `ipc/zstd` at 53.6 KB, so - a client that cannot use zstd, such as some .NET stacks, can use lz4 or snappy - and remain competitive on the wire. +- The conversion is the expensive part. Flatten plus file encode dominates, and + IPC decode is a small fraction. Accepting client-precomputed Parquet and + persisting it removes essentially all of that server CPU. Even a server that + must reparse the Parquet into Arrow still avoids the flatten and encode. +- On zstd versus lz4, IPC with lz4 is about 18 percent larger than zstd, and + Parquet-nested with lz4 is about 32 percent larger while snappy is about 59 + percent larger than zstd. lz4 stays competitive on the wire, so a client that + cannot use zstd, such as some .NET stacks, can use lz4 or snappy. - On layout, `parquet-nested` and `parquet-map` compress smallest with zstd, - while `parquet-wide` is far smaller uncompressed and is the cheapest to write - and to reparse thanks to typed columns, at a modest size premium under zstd. -- For the debate, if server CPU is the constraint, and here it is, have clients - send precomputed flattened Parquet and let the server persist it. lz4 is a safe - cross-language codec choice with a small size cost relative to zstd. - -## Vortex - -Vortex is a next-generation, Arrow-native columnar file format. Vortex 0.75 pins -the same arrow-rs 58.3 as this workspace, so Arrow arrays flow in and out with no -version bridging, and the whole round-trip runs in memory over a `Vec` write -sink and a `ByteBuffer` reader. The contender reuses the nested flattening and -feeds the flat `RecordBatch` to Vortex. - -Integration notes: - -- Vortex 0.75 has no `FixedSizeBinary` encoding, so `trace_id` and `span_id` are - cast to `Binary` before writing and restored on read. -- Vortex prefers view and plain Arrow types on read, so decode targets an - explicit plain schema, with dictionaries decoded and `FixedSizeBinary` - restored, that the OTAP schema check accepts. -- Vortex applies its own cascading BtrBlocks compression, so there is no external - compressor knob. - -Illustrative results for shape `r1_s1_l5000`, which is 5000 log records: - -| metric | vortex | parquet-nested/zstd | ipc/zstd | -|---------------------------|----------|---------------------|----------| -| serialized size | 107.0 KB | 35.6 KB | 53.6 KB | -| size vs OTLP proto | 10.6x | 31.9x | 21.2x | -| server convert IPC to file| ~510 ms | ~14 ms | n/a | -| reparse file to Arrow | ~11.9 ms | ~2.9 ms | n/a | - -Vortex write time at smaller shapes is about 43 ms for 250 logs and about 101 ms -for 1000 logs. - -The finding is that for this OTAP-logs full round-trip, Vortex at default -settings is both larger on the wire, roughly twice `ipc/zstd` and three times -`parquet-nested/zstd`, and much slower to write, by an order of magnitude or -more, than zstd-Parquet. It therefore does not help the goal of having the client -precompute so the server offloads CPU. - -Several caveats make this a starting point rather than a verdict on Vortex. The -benchmark does full batch round-trips, whereas Vortex is designed for selective -reads with column and row-range projection, pushdown, and random access, none of -which this workload exercises. The default BtrBlocks compressor samples encodings -to favor decode speed and random access over write speed and maximum ratio, and -neither a lighter nor a heavier write strategy was explored. Each write and read -creates a fresh `VortexSession`, whereas a real deployment would reuse it, -although the dominant cost is the compression itself. Finally, the nested -`List` attribute layout may not play to Vortex's strengths, and the wide -layout was not tried with Vortex. Tuning the Vortex write strategy and exercising -selective reads are the obvious next steps. + while `parquet-wide` is smaller uncompressed and cheaper to encode and reparse + thanks to typed columns, at a small size premium under zstd. +- For the debate, if server CPU is the constraint, have clients send precomputed + flattened Parquet and let the server persist it. + +## Vortex write experiment + +The goal here was to see whether Vortex, which is Arrow-native, could be written +faster than Parquet at realistic block sizes of 10k and 100k records, ideally by +skipping compression. The answer in this experiment is no. + +Vortex 0.75 pins the same arrow-rs 58.3 as this workspace, so Arrow arrays flow +in and out with no version bridging and the whole round-trip runs in memory. +Vortex 0.75 has no `FixedSizeBinary` encoding, so `trace_id` and `span_id` are +cast to `Binary` and restored on read, and decode targets an explicit plain +Arrow schema that the OTAP schema check accepts. The `vortex-fast` contender +disables BtrBlocks by building the write strategy with an empty compressor. + +Two results stand out. First, Vortex write is slower than Parquet write at both +sizes. At 10k the convert cost is 692 ms for `vortex` and 935 ms for +`vortex-fast`, versus 93 ms for `parquet-nested/zstd`. At 100k it is 1727 ms and +1858 ms versus 889 ms. Second, disabling compression made Vortex both larger and +slower, not faster. `vortex-fast` produces 13 MB at 10k and 186 MB at 100k, +because turning off BtrBlocks also turns off dictionary encoding, so the +denormalized resource and scope attributes fully expand. Serializing that volume +through Vortex's write pipeline outweighs any saving from skipping compression, +which is why `vortex-fast` is slower than the compressed `vortex`. + +The write bottleneck is therefore the Vortex write pipeline itself, not the +compressor. Even with compression off, Vortex still canonicalizes arrays, +repartitions into row blocks, computes zone statistics, builds a layout, and +writes a flatbuffer footer through an async pipeline, and for this wide, deeply +nested schema with many small columns that fixed work is much heavier than +Parquet's encoder. Vortex's default compression also did not improve with scale +on this data, going from about 20 bytes per record at 10k to about 62 at 100k, +while Parquet-zstd improved from about 5.7 to about 3.8. + +This is a fair result for using Vortex as a Parquet drop-in for this write path, +but it is not a verdict on Vortex overall. Vortex is designed for selective reads +with column and row-range projection, pushdown, and random access over large +files, none of which this full-batch write-and-read workload exercises. Reducing +the write pipeline overhead, for example a dictionary-only encoding that stays +small while skipping the compression search, and exercising selective reads are +the obvious next steps. ## Extending diff --git a/rust/otap-dataflow/benchmarks/benches/otap_parquet/main.rs b/rust/otap-dataflow/benchmarks/benches/otap_parquet/main.rs index 4448d6718f..7a00d2db89 100644 --- a/rust/otap-dataflow/benchmarks/benches/otap_parquet/main.rs +++ b/rust/otap-dataflow/benchmarks/benches/otap_parquet/main.rs @@ -43,7 +43,7 @@ use benchmarks::parquet_study::{Compressor, Scheme, StudyResult, server}; #[allow(unused_variables)] fn reparse(scheme: Scheme, bytes: &[u8]) -> StudyResult { #[cfg(feature = "vortex")] - if matches!(scheme, Scheme::Vortex) { + if matches!(scheme, Scheme::Vortex | Scheme::VortexFast) { return benchmarks::parquet_study::vortex::reparse_to_arrow(bytes); } server::reparse_parquet(bytes) @@ -56,31 +56,20 @@ use tikv_jemallocator::Jemalloc; #[global_allocator] static GLOBAL: Jemalloc = Jemalloc; -/// Input shapes covering a few resource/scope/log fan-outs. The first is the -/// "many small groups" shape from the `otap_encoder` bench; the rest scale the -/// number of log records under a single resource/scope (so resource/scope -/// attribute denormalization is amortized over more rows). +/// Input shapes. Parquet and Vortex both carry per-file overhead, so the study +/// uses realistic block sizes of several thousand to 100k log records under a +/// single resource/scope. A small multi-group shape is kept for reference. fn input_shapes() -> Vec { vec![ - LogsGenParams { - num_resources: 5, - num_scopes: 10, - num_logs: 5, - }, - LogsGenParams { - num_resources: 50, - num_scopes: 2, - num_logs: 10, - }, LogsGenParams { num_resources: 1, num_scopes: 1, - num_logs: 1000, + num_logs: 10_000, }, LogsGenParams { num_resources: 1, num_scopes: 1, - num_logs: 5000, + num_logs: 100_000, }, ] } @@ -138,10 +127,8 @@ fn print_size_table(shapes: &[LogsGenParams]) { /// Median wall-clock milliseconds of `f` over a few iterations (with warm-up). /// Indicative only; the `server_cost` Criterion group provides rigorous numbers. fn median_ms(mut f: impl FnMut()) -> f64 { - for _ in 0..2 { - f(); - } - let iters = 11; + f(); + let iters = 3; let mut samples = Vec::with_capacity(iters); for _ in 0..iters { let start = Instant::now(); diff --git a/rust/otap-dataflow/benchmarks/src/parquet_study/mod.rs b/rust/otap-dataflow/benchmarks/src/parquet_study/mod.rs index 872eb526e9..468bfe31d2 100644 --- a/rust/otap-dataflow/benchmarks/src/parquet_study/mod.rs +++ b/rust/otap-dataflow/benchmarks/src/parquet_study/mod.rs @@ -132,6 +132,10 @@ pub enum Scheme { /// Flattened Vortex file (nested layout). Requires the `vortex` feature. #[cfg(feature = "vortex")] Vortex, + /// Flattened Vortex file written with no compression, prioritizing write + /// throughput. Requires the `vortex` feature. + #[cfg(feature = "vortex")] + VortexFast, } impl Scheme { @@ -142,7 +146,7 @@ impl Scheme { #[allow(unused_mut)] let mut v = vec![Scheme::Ipc, Scheme::Nested, Scheme::Map, Scheme::Wide]; #[cfg(feature = "vortex")] - v.push(Scheme::Vortex); + v.extend([Scheme::Vortex, Scheme::VortexFast]); v } @@ -153,7 +157,7 @@ impl Scheme { #[allow(unused_mut)] let mut v = vec![Scheme::Nested, Scheme::Map, Scheme::Wide]; #[cfg(feature = "vortex")] - v.push(Scheme::Vortex); + v.extend([Scheme::Vortex, Scheme::VortexFast]); v } @@ -167,6 +171,8 @@ impl Scheme { Scheme::Wide => "parquet-wide", #[cfg(feature = "vortex")] Scheme::Vortex => "vortex", + #[cfg(feature = "vortex")] + Scheme::VortexFast => "vortex-fast", } } @@ -177,7 +183,7 @@ impl Scheme { match self { Scheme::Ipc => &Compressor::IPC, #[cfg(feature = "vortex")] - Scheme::Vortex => &[Compressor::None], + Scheme::Vortex | Scheme::VortexFast => &[Compressor::None], _ => &Compressor::ALL, } } @@ -191,7 +197,9 @@ impl Scheme { Scheme::Map => Box::new(map::MapParquetCodec { compressor }), Scheme::Wide => Box::new(wide::WideParquetCodec { compressor }), #[cfg(feature = "vortex")] - Scheme::Vortex => Box::new(vortex::VortexCodec), + Scheme::Vortex => Box::new(vortex::VortexCodec { fast: false }), + #[cfg(feature = "vortex")] + Scheme::VortexFast => Box::new(vortex::VortexCodec { fast: true }), } } } diff --git a/rust/otap-dataflow/benchmarks/src/parquet_study/vortex.rs b/rust/otap-dataflow/benchmarks/src/parquet_study/vortex.rs index 28b7669c3a..988292cd4f 100644 --- a/rust/otap-dataflow/benchmarks/src/parquet_study/vortex.rs +++ b/rust/otap-dataflow/benchmarks/src/parquet_study/vortex.rs @@ -27,15 +27,20 @@ use vortex::array::arrow::ArrowSessionExt; use vortex::array::arrow::FromArrowArray; use vortex::array::stream::ArrayStreamExt; use vortex::buffer::ByteBuffer; -use vortex::file::{OpenOptionsSessionExt, WriteOptionsSessionExt}; +use vortex::compressor::BtrBlocksCompressorBuilder; +use vortex::file::{OpenOptionsSessionExt, WriteOptionsSessionExt, WriteStrategyBuilder}; use vortex::io::session::RuntimeSessionExt; use vortex::session::VortexSession; use super::{Codec, StudyResult, nested}; /// Contender that flattens OTAP logs (nested layout) and stores them in an -/// in-memory Vortex file. -pub struct VortexCodec; +/// in-memory Vortex file. `fast` selects an uncompressed, near-zero-copy write +/// strategy instead of the default BtrBlocks cascading compressor. +pub struct VortexCodec { + /// When true, write with no compression to prioritize write throughput. + pub fast: bool, +} fn runtime() -> &'static tokio::runtime::Runtime { static RT: OnceLock = OnceLock::new(); @@ -164,18 +169,29 @@ fn restore_fixed_size_binary(batch: RecordBatch) -> StudyResult { )?) } -/// Encode a flat Arrow record batch as an in-memory Vortex file. -fn encode(flat: RecordBatch) -> StudyResult> { +/// Encode a flat Arrow record batch as an in-memory Vortex file. When `fast` is +/// set, a no-op compressor is used so the write is essentially a canonical +/// (uncompressed) layout, skipping the BtrBlocks encoding search. +fn encode(flat: RecordBatch, fast: bool) -> StudyResult> { let flat = to_vortex_compatible(&flat)?; runtime() .block_on(async move { let session = VortexSession::default().with_tokio(); let array = ArrayRef::from_arrow(flat, false)?; let mut sink: Vec = Vec::with_capacity(4096); - let _summary = session - .write_options() - .write(&mut sink, array.to_array_stream()) - .await?; + let write_options = session.write_options(); + let stream = array.to_array_stream(); + let _summary = if fast { + let strategy = WriteStrategyBuilder::default() + .with_btrblocks_builder(BtrBlocksCompressorBuilder::empty()) + .build(); + write_options + .with_strategy(strategy) + .write(&mut sink, stream) + .await? + } else { + write_options.write(&mut sink, stream).await? + }; Ok::, vortex::error::VortexError>(sink) }) .map_err(Into::into) @@ -205,12 +221,12 @@ fn decode(bytes: &[u8]) -> StudyResult { impl Codec for VortexCodec { fn name(&self) -> &'static str { - "vortex" + if self.fast { "vortex-fast" } else { "vortex" } } fn write(&self, logs: OtapArrowRecords) -> StudyResult> { let flat = nested::flatten(&logs)?; - encode(flat) + encode(flat, self.fast) } fn read(&self, bytes: &[u8]) -> StudyResult { @@ -240,10 +256,12 @@ mod tests { }; let (otap, _) = gen_logs_otap(¶ms); - let codec = VortexCodec; - let bytes = codec.write(otap.clone()).expect("write"); - assert!(!bytes.is_empty()); - let decoded = codec.read(&bytes).expect("read"); - assert_logs_equivalent(&otap, &decoded, codec.name(), "none"); + for fast in [false, true] { + let codec = VortexCodec { fast }; + let bytes = codec.write(otap.clone()).expect("write"); + assert!(!bytes.is_empty()); + let decoded = codec.read(&bytes).expect("read"); + assert_logs_equivalent(&otap, &decoded, codec.name(), "none"); + } } } From a89d06221adc8f6a8543473f32239bb87ec2ebb0 Mon Sep 17 00:00:00 2001 From: Joshua MacDonald Date: Tue, 30 Jun 2026 21:36:37 -0700 Subject: [PATCH 04/18] chore(benchmarks): drop Vortex and add per-step pipeline breakdown Removes the Vortex contender and its optional dependency, and refines the otap_parquet study to break each pipeline into sub-steps at 10k, 50k, and 100k block sizes. The OTAP/IPC path is split into transport-optimize and Arrow IPC serialize on the encode side, and IPC deserialize and transport-decode on the decode side. The Parquet path is split into flatten and Parquet write on encode, and Parquet read and unflatten on decode. The benchmark prints an OTAP/IPC breakdown table and a Parquet breakdown table alongside the size table. The breakdown shows that IPC encode is dominated by the transport-optimized encoding, Parquet encode is dominated by the Parquet writer, and IPC is about an order of magnitude cheaper than Parquet on both the encode and decode side. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- rust/otap-dataflow/benchmarks/Cargo.toml | 9 - .../benchmarks/benches/otap_parquet/README.md | 230 ++++++-------- .../benchmarks/benches/otap_parquet/main.rs | 286 ++++++++---------- .../benchmarks/src/parquet_study/ipc.rs | 59 +++- .../benchmarks/src/parquet_study/mod.rs | 63 ++-- .../benchmarks/src/parquet_study/vortex.rs | 267 ---------------- 6 files changed, 291 insertions(+), 623 deletions(-) delete mode 100644 rust/otap-dataflow/benchmarks/src/parquet_study/vortex.rs diff --git a/rust/otap-dataflow/benchmarks/Cargo.toml b/rust/otap-dataflow/benchmarks/Cargo.toml index 66f2992718..2382022ca5 100644 --- a/rust/otap-dataflow/benchmarks/Cargo.toml +++ b/rust/otap-dataflow/benchmarks/Cargo.toml @@ -23,15 +23,6 @@ otap-df-otap = { workspace = true } otap-df-pdata = { workspace = true, features = ["bench", "testing"] } otap-df-pdata-views = { workspace = true } -# Optional: the Vortex file format contender for the parquet study. Heavy -# dependency, so gated behind the `vortex` feature. Vortex 0.75 pins arrow 58.3 -# (same as this workspace), so Arrow types interoperate directly. -vortex = { version = "0.75", optional = true, features = ["tokio"] } - -[features] -# Enables the Vortex contender in the otap_parquet benchmark. -vortex = ["dep:vortex"] - [dev-dependencies] criterion = { workspace = true, features = ["html_reports", "async_tokio"] } tonic = { workspace = true } diff --git a/rust/otap-dataflow/benchmarks/benches/otap_parquet/README.md b/rust/otap-dataflow/benchmarks/benches/otap_parquet/README.md index f82d0c7bcd..842db8f6e6 100644 --- a/rust/otap-dataflow/benchmarks/benches/otap_parquet/README.md +++ b/rust/otap-dataflow/benchmarks/benches/otap_parquet/README.md @@ -4,24 +4,23 @@ **Status:** experiment on branch `jmacd/parquet_study`. -This benchmark quantifies moving OTAP logs from a client to a server that -ultimately stores flattened, single-file Parquet. Because the data ends up as -Parquet on the server either way, the central question is where the OTAP to -Parquet conversion CPU is spent. Option A has the client send OTAP as Arrow IPC -and the server convert it to Parquet, paying for IPC decode plus flatten plus -Parquet encode. Option B has the client precompute the flattened Parquet and -send that, so the server only persists the bytes, or reparses them into Arrow if -it must partition, index, or validate. - -The benchmark measures write time, read time, serialized size, and the -server-side conversion cost for the same logs batch across several -representations and compressors. +This benchmark studies the cost of moving OTAP logs between a client and a server +two ways: as compressed Arrow IPC, which is the representation we have today, and +as a flattened single-file Parquet, which a server would store. It starts from an +OTAP logs batch, which is four Arrow record batches (Logs, ResourceAttrs, +ScopeAttrs, LogAttrs), and breaks each pipeline into its sub-steps so the cost of +every stage is visible on both the encode and decode side. + +- OTAP/IPC encode is transport-optimize, then Arrow IPC serialize with + compression. Decode is IPC deserialize, then transport-decode. +- Parquet encode is flatten to one Arrow record batch, then write Parquet. Decode + is read Parquet, then unflatten. ## Contenders - `ipc` is the OTAP representation we have today: interleaved Arrow IPC streams - produced by `Producer` and consumed by `Consumer`, with each per-payload - stream compressed. + produced by `Producer` and consumed by `Consumer`, with each per-payload stream + compressed. - `parquet-nested` is a single flattened Parquet file where each log row carries its denormalized resource, scope, and log attributes as `List` columns. @@ -30,20 +29,13 @@ representations and compressors. - `parquet-wide` is the analytics-flat extreme, where every distinct attribute key becomes its own typed top-level column named `resource.`, `scope.`, or `log.`. -- `vortex` applies the same nested flattening but stores the result in a - [Vortex](https://vortex.dev) file with Vortex's default BtrBlocks compression. -- `vortex-fast` is the same Vortex layout written with BtrBlocks disabled, to - test whether skipping compression lets Vortex write faster than Parquet. The - Vortex contenders require the `vortex` cargo feature, which is a heavy - dependency. ## Compressors Compressors are explicit codecs so `zstd` can be compared head-to-head with `lz4`. This matters for cross-language consumers, because some Arrow and Parquet stacks, for example certain .NET builds, may not support `zstd`. In that case -`lz4`, and `snappy` for Parquet, need first-class numbers. The Vortex contenders -manage their own encoding, so they expose a single setting reported as `none`. +`lz4`, and `snappy` for Parquet, need first-class numbers. | compressor | Arrow IPC | Parquet | |------------|---------------|--------------| @@ -62,136 +54,96 @@ rather than the deprecated Hadoop-framed `LZ4`. cargo bench -p benchmarks --bench otap_parquet ``` -Two tables, one for serialized size and one for the server-side CPU model, are -printed to stdout before the timed benchmarks run. To include the Vortex -contenders: - -```bash -cargo bench -p benchmarks --bench otap_parquet --features vortex -``` - -The default input shapes are 10k and 100k log records, so the full Criterion -sweep is slow. The printed tables are the main output. For a quick pass, add -`-- --measurement-time 0.5 --sample-size 10`, or read the printed tables and -stop the run. - -## What each measurement covers - -- IPC write runs `Producer::produce_bar` and prost-encodes the - `BatchArrowRecords`. -- IPC read runs prost-decode, then `Consumer::consume_bar`, - `from_record_messages`, and `decode_transport_optimized_ids` back to - `OtapArrowRecords`. -- File write flattens `OtapArrowRecords` to one Arrow `RecordBatch`, then encodes - it to Parquet or Vortex bytes. -- File read decodes the file to an Arrow `RecordBatch`, then unflattens back to - `OtapArrowRecords`. -- The server_cost `convert-A` measurement takes IPC bytes through decode, - flatten, and file encode. This is the cost the server pays in Option A. The - flatten and IPC decode are shared across schemes, so differences between - schemes isolate the file-encode cost. -- The server_cost `accept-B` measurement takes file bytes back to an Arrow - `RecordBatch` without rebuilding OTAP. This is what the server pays in Option B - when it must touch the data. If the server only persists the received bytes, - its conversion CPU is roughly zero. +Three tables are printed to stdout before the timed round-trip benchmarks run: +serialized size, the OTAP/IPC pipeline breakdown, and the Parquet pipeline +breakdown. The default shapes are 10k, 50k, and 100k log records, so the full +Criterion sweep is slow. The printed tables are the main output; for a quick pass +add `-- --measurement-time 0.5 --sample-size 10`, or read the tables and stop the +run. + +## Pipeline steps + +- IPC `t-enc` is the OTAP transport-optimized encoding, which applies + delta and dictionary encodings to id and value columns and remaps parent ids. + `ipc-ser` is the Arrow IPC serialization with compression plus the prost + encoding of the `BatchArrowRecords`. Because `Producer::produce_bar` bundles + the two, `ipc-ser` is reported as the encode total minus `t-enc`. +- IPC `ipc-des` is the prost decode plus `Consumer::consume_bar` plus + `from_record_messages`, which yields a batch still in the transport-optimized + encoding. `t-dec` is `decode_transport_optimized_ids`, which restores the + logical batch. +- Parquet `flatten` builds the single flat Arrow record batch. `pq-write` is the + Arrow Parquet writer. `pq-read` is the Arrow Parquet reader, and `unflat` + reconstructs the four OTAP record batches. The flattened layouts keep the entire root `Logs` record batch intact, so decode -carries its scalar and struct columns straight back just as the IPC path does, -and the comparison is not penalized by re-walking the body. Only the attribute -tables are denormalized and rebuilt. Resource and scope attribute sets are -re-normalized on decode using the `resource.id` and `scope.id` join keys the -`Logs` batch still carries. `parquet-wide` is lossless for type-consistent keys. -Any attribute that does not fit its single typed column spills into a per-group -`List` overflow column, so the round-trip stays exact. +carries its scalar and struct columns straight back just as the IPC path does. +Only the attribute tables are denormalized and rebuilt, re-normalizing resource +and scope sets with the `resource.id` and `scope.id` join keys the `Logs` batch +carries. ## Illustrative results -These numbers come from one development machine running WSL with jemalloc. -Absolute values vary by host, but the relationships are stable. The shape is -10,000 log records under a single resource and scope, whose OTLP protobuf -encoding is 2,270,225 bytes. +These numbers come from one development machine running WSL with jemalloc, at the +100,000 log-record shape whose OTLP protobuf encoding is 22,700,225 bytes. +Absolute values vary by host, but the relationships are stable. Serialized size in bytes: -| contender | zstd | lz4 | snappy | none | -|----------------|--------|--------|--------|------------| -| ipc | 92,270 | 108,846| n/a | 2,457,008 | -| parquet-nested | 57,023 | 75,336 | 90,529 | 432,935 | -| parquet-wide | 62,088 | 78,840 | 78,851 | 82,219 | -| vortex | n/a | n/a | n/a | 199,088 | -| vortex-fast | n/a | n/a | n/a | 13,138,288 | - -Server convert cost, which is IPC decode plus flatten plus file encode, in -indicative milliseconds. The flatten and decode are shared, so differences -isolate the encode. The 100k column is the same shape scaled to 100,000 records. - -| contender / comp | convert 10k | convert 100k | -|------------------------|-------------|--------------| -| parquet-nested / zstd | 93 ms | 889 ms | -| parquet-wide / zstd | 44 ms | 577 ms | -| vortex | 692 ms | 1727 ms | -| vortex-fast | 935 ms | 1858 ms | +| contender | zstd | lz4 | snappy | none | +|----------------|---------|---------|---------|------------| +| ipc | 702,702 | 915,632 | n/a | 24,428,084 | +| parquet-nested | 384,740 | 518,503 | 800,452 | 7,079,037 | +| parquet-wide | 392,782 | 509,659 | 608,717 | 2,838,767 | + +OTAP/IPC pipeline breakdown, milliseconds: + +| comp | t-enc | ipc-ser | enc-tot | ipc-des | t-dec | dec-tot | +|------|-------|---------|---------|---------|-------|---------| +| zstd | 35.2 | 9.2 | 44.4 | 6.6 | 3.2 | 9.8 | +| lz4 | 34.5 | 7.1 | 41.6 | 42.2 | 2.4 | 44.7 | +| none | 39.5 | 64.9 | 104.4 | 54.6 | 2.5 | 57.1 | + +Parquet pipeline breakdown, milliseconds: + +| scheme / comp | flatten | pq-write | enc-tot | pq-read | unflat | dec-tot | +|-----------------------|---------|----------|---------|---------|--------|---------| +| parquet-nested / zstd | 148 | 404 | 552 | 152 | 65 | 217 | +| parquet-map / zstd | 139 | 363 | 503 | 160 | 63 | 223 | +| parquet-wide / zstd | 183 | 147 | 330 | 56 | 91 | 147 | ## Takeaways -- The conversion is the expensive part. Flatten plus file encode dominates, and - IPC decode is a small fraction. Accepting client-precomputed Parquet and - persisting it removes essentially all of that server CPU. Even a server that - must reparse the Parquet into Arrow still avoids the flatten and encode. -- On zstd versus lz4, IPC with lz4 is about 18 percent larger than zstd, and - Parquet-nested with lz4 is about 32 percent larger while snappy is about 59 - percent larger than zstd. lz4 stays competitive on the wire, so a client that - cannot use zstd, such as some .NET stacks, can use lz4 or snappy. -- On layout, `parquet-nested` and `parquet-map` compress smallest with zstd, - while `parquet-wide` is smaller uncompressed and cheaper to encode and reparse - thanks to typed columns, at a small size premium under zstd. -- For the debate, if server CPU is the constraint, have clients send precomputed - flattened Parquet and let the server persist it. - -## Vortex write experiment - -The goal here was to see whether Vortex, which is Arrow-native, could be written -faster than Parquet at realistic block sizes of 10k and 100k records, ideally by -skipping compression. The answer in this experiment is no. - -Vortex 0.75 pins the same arrow-rs 58.3 as this workspace, so Arrow arrays flow -in and out with no version bridging and the whole round-trip runs in memory. -Vortex 0.75 has no `FixedSizeBinary` encoding, so `trace_id` and `span_id` are -cast to `Binary` and restored on read, and decode targets an explicit plain -Arrow schema that the OTAP schema check accepts. The `vortex-fast` contender -disables BtrBlocks by building the write strategy with an empty compressor. - -Two results stand out. First, Vortex write is slower than Parquet write at both -sizes. At 10k the convert cost is 692 ms for `vortex` and 935 ms for -`vortex-fast`, versus 93 ms for `parquet-nested/zstd`. At 100k it is 1727 ms and -1858 ms versus 889 ms. Second, disabling compression made Vortex both larger and -slower, not faster. `vortex-fast` produces 13 MB at 10k and 186 MB at 100k, -because turning off BtrBlocks also turns off dictionary encoding, so the -denormalized resource and scope attributes fully expand. Serializing that volume -through Vortex's write pipeline outweighs any saving from skipping compression, -which is why `vortex-fast` is slower than the compressed `vortex`. - -The write bottleneck is therefore the Vortex write pipeline itself, not the -compressor. Even with compression off, Vortex still canonicalizes arrays, -repartitions into row blocks, computes zone statistics, builds a layout, and -writes a flatbuffer footer through an async pipeline, and for this wide, deeply -nested schema with many small columns that fixed work is much heavier than -Parquet's encoder. Vortex's default compression also did not improve with scale -on this data, going from about 20 bytes per record at 10k to about 62 at 100k, -while Parquet-zstd improved from about 5.7 to about 3.8. - -This is a fair result for using Vortex as a Parquet drop-in for this write path, -but it is not a verdict on Vortex overall. Vortex is designed for selective reads -with column and row-range projection, pushdown, and random access over large -files, none of which this full-batch write-and-read workload exercises. Reducing -the write pipeline overhead, for example a dictionary-only encoding that stays -small while skipping the compression search, and exercising selective reads are -the obvious next steps. +- IPC is far cheaper than Parquet on both sides. At 100k with zstd, IPC encodes + in about 44 ms and decodes in about 10 ms, while `parquet-nested` encodes in + about 552 ms and decodes in about 217 ms. That is roughly 12 times cheaper to + encode and 22 times cheaper to decode. +- Inside IPC encode, the transport-optimized encoding dominates, about 35 ms of + the 44 ms, and it is essentially compressor-independent because it runs before + compression. Inside IPC decode, the deserialization dominates and the transport + decode is small. +- Inside Parquet encode, the Parquet writer dominates and the flatten is roughly + a third to a half of the total. `parquet-wide` writes fastest because it has + typed scalar columns rather than nested `List` or `Map`, but it pays + more in flatten, and it ends up the cheapest Parquet encoder overall at 330 ms. +- Compression choice matters in surprising ways. For IPC, compression makes the + serialize step faster because there is far less data to move, so `none` is the + slowest to serialize, and `lz4` is much slower to deserialize than `zstd` in + this Arrow IPC implementation. For Parquet, the writer time is largely + insensitive to the compressor, while the size is not. +- For the debate, if server CPU is the constraint, the client should keep sending + OTAP/IPC, which is an order of magnitude cheaper to produce and consume than + Parquet. Producing Parquet is worth it only where the columnar file and its + smaller `zstd` size are needed at rest, and that cost lands wherever the flatten + and Parquet write run. ## Extending Contenders are the `Scheme` enum and its `Codec` implementations in `benchmarks/src/parquet_study`. Add a variant to include a contender everywhere. -Input shapes are defined in `input_shapes()` in `benches/otap_parquet/main.rs`. -The flatten and unflatten round-trips and the server-cost helpers have unit tests -runnable with `cargo test -p benchmarks --lib parquet_study`. +The IPC sub-steps are `ipc::transport_encode`, `ipc::encode_to_bytes`, +`ipc::deserialize`, and `ipc::transport_decode`; the Parquet steps are +`Scheme::flatten`, `parquet_io::write_parquet`, `parquet_io::read_parquet`, and +`Scheme::unflatten`. Input shapes are defined in `input_shapes()` in +`benches/otap_parquet/main.rs`. The round-trips have unit tests runnable with +`cargo test -p benchmarks --lib parquet_study`. diff --git a/rust/otap-dataflow/benchmarks/benches/otap_parquet/main.rs b/rust/otap-dataflow/benchmarks/benches/otap_parquet/main.rs index 7a00d2db89..4dc2976c74 100644 --- a/rust/otap-dataflow/benchmarks/benches/otap_parquet/main.rs +++ b/rust/otap-dataflow/benchmarks/benches/otap_parquet/main.rs @@ -5,17 +5,16 @@ //! //! Compares the read/write cost and serialized size of OTAP logs encoded as //! compressed Arrow IPC (the representation we have today) versus several -//! flattened single-file Parquet layouts, across the `zstd`, `lz4`, `snappy`, -//! and `none` compressors. +//! flattened single-file Parquet layouts, and breaks each pipeline into its +//! sub-steps so the cost of every stage is visible. //! -//! Two questions are answered: +//! Starting from an OTAP logs batch (four record batches: Logs, ResourceAttrs, +//! ScopeAttrs, LogAttrs): //! -//! 1. Round-trip cost and size of each representation (the `write` / `read` and -//! size-table output). -//! 2. Where the OTAP -> Parquet conversion CPU lands, given the data ends up as -//! Parquet on the server either way (the `server_cost` output): the server -//! converting received OTAP/IPC (Option A) versus accepting Parquet the -//! client already produced (Option B). +//! - OTAP/IPC encode = transport-optimize, then Arrow IPC serialize (+compress). +//! Decode = IPC deserialize, then transport-decode. +//! - Parquet encode = flatten to one Arrow record batch, then write Parquet. +//! Decode = read Parquet, then unflatten. //! //! Run with: //! @@ -23,7 +22,8 @@ //! cargo bench -p benchmarks --bench otap_parquet //! ``` //! -//! Two tables are printed to stdout before the timed benchmarks run. +//! Three tables (size, OTAP/IPC breakdown, Parquet breakdown) are printed to +//! stdout before the timed round-trip benchmarks run. #![allow(missing_docs)] // This benchmark intentionally prints comparison tables to stdout before @@ -35,19 +35,8 @@ use std::hint::black_box; use std::time::{Duration, Instant}; use benchmarks::parquet_study::datagen::{LogsGenParams, gen_logs_otap}; -use benchmarks::parquet_study::{Compressor, Scheme, StudyResult, server}; - -/// Reparse a stored flattened file back into an Arrow record batch (no -/// unflatten), dispatching on the scheme's format. Used by the server-cost -/// "accept-B" measurement. -#[allow(unused_variables)] -fn reparse(scheme: Scheme, bytes: &[u8]) -> StudyResult { - #[cfg(feature = "vortex")] - if matches!(scheme, Scheme::Vortex | Scheme::VortexFast) { - return benchmarks::parquet_study::vortex::reparse_to_arrow(bytes); - } - server::reparse_parquet(bytes) -} +use benchmarks::parquet_study::parquet_io::{read_parquet, write_parquet}; +use benchmarks::parquet_study::{Compressor, Scheme, ipc}; #[cfg(not(windows))] use tikv_jemallocator::Jemalloc; @@ -56,67 +45,66 @@ use tikv_jemallocator::Jemalloc; #[global_allocator] static GLOBAL: Jemalloc = Jemalloc; -/// Input shapes. Parquet and Vortex both carry per-file overhead, so the study -/// uses realistic block sizes of several thousand to 100k log records under a -/// single resource/scope. A small multi-group shape is kept for reference. +/// Input shapes: several block sizes larger than a few thousand log records, +/// under a single resource/scope, which is the realistic case for a Parquet +/// file. fn input_shapes() -> Vec { - vec![ - LogsGenParams { - num_resources: 1, - num_scopes: 1, - num_logs: 10_000, - }, - LogsGenParams { + [10_000usize, 50_000, 100_000] + .into_iter() + .map(|num_logs| LogsGenParams { num_resources: 1, num_scopes: 1, - num_logs: 100_000, - }, - ] + num_logs, + }) + .collect() } -/// Print a serialized-size comparison table to stdout (not part of the timed -/// measurements, but the most important output for the "on the wire" debate). +/// Median wall-clock milliseconds of `f` over a few iterations (with one warm-up +/// pass). Indicative only; the Criterion round-trip group gives rigorous totals. +fn median_ms(mut f: impl FnMut()) -> f64 { + f(); + let iters = 3; + let mut samples = Vec::with_capacity(iters); + for _ in 0..iters { + let start = Instant::now(); + f(); + samples.push(start.elapsed().as_secs_f64() * 1e3); + } + samples.sort_by(|a, b| a.partial_cmp(b).expect("no NaN")); + samples[samples.len() / 2] +} + +/// Serialized-size comparison for every contender x compressor x shape. fn print_size_table(shapes: &[LogsGenParams]) { println!("\n=== OTAP logs serialized size (bytes) ==="); println!( "{:<16} {:<8} {:>12} {:>10} {:>10} {:>12}", "contender", "comp", "bytes", "vs-otlp", "b/log", "vs-ipc-zstd" ); - for shape in shapes { let (otap, proto_len) = gen_logs_otap(shape); let total_logs = shape.total_logs(); println!( - "-- shape {} : {} log records, OTLP proto = {} bytes --", - shape.label(), - total_logs, - proto_len + "-- shape {} log records, OTLP proto = {} bytes --", + total_logs, proto_len ); - let ipc_zstd = Scheme::Ipc .codec(Compressor::Zstd) .write(otap.clone()) .expect("ipc zstd write") .len(); - for scheme in Scheme::all() { for &compressor in scheme.compressors() { let codec = scheme.codec(compressor); - let bytes = codec - .write(otap.clone()) - .unwrap_or_else(|e| panic!("{} write failed: {e}", codec.name())) - .len(); - let vs_otlp = proto_len as f64 / bytes as f64; - let per_log = bytes as f64 / total_logs as f64; - let vs_ipc = bytes as f64 / ipc_zstd as f64; + let bytes = codec.write(otap.clone()).expect("write").len(); println!( "{:<16} {:<8} {:>12} {:>9.2}x {:>10.1} {:>11.2}x", codec.name(), compressor.label(), bytes, - vs_otlp, - per_log, - vs_ipc + proto_len as f64 / bytes as f64, + bytes as f64 / total_logs as f64, + bytes as f64 / ipc_zstd as f64, ); } } @@ -124,68 +112,93 @@ fn print_size_table(shapes: &[LogsGenParams]) { } } -/// Median wall-clock milliseconds of `f` over a few iterations (with warm-up). -/// Indicative only; the `server_cost` Criterion group provides rigorous numbers. -fn median_ms(mut f: impl FnMut()) -> f64 { - f(); - let iters = 3; - let mut samples = Vec::with_capacity(iters); - for _ in 0..iters { - let start = Instant::now(); - f(); - samples.push(start.elapsed().as_secs_f64() * 1e3); +/// Per-step breakdown of the OTAP/IPC encode and decode pipelines. +fn print_ipc_breakdown(shapes: &[LogsGenParams]) { + println!("\n=== OTAP/IPC pipeline breakdown (indicative ms) ==="); + println!("encode = transport-optimize + Arrow-IPC-serialize(+compress)"); + println!("decode = IPC-deserialize + transport-decode"); + println!( + "{:<6} {:>9} {:>9} {:>9} {:>9} {:>9} {:>9} {:>10}", + "comp", "t-enc", "ipc-ser", "enc-tot", "ipc-des", "t-dec", "dec-tot", "bytes" + ); + for shape in shapes { + let (otap, _) = gen_logs_otap(shape); + println!("-- shape {} log records --", shape.total_logs()); + for &comp in Scheme::Ipc.compressors() { + let t_enc = median_ms(|| { + let mut o = otap.clone(); + ipc::transport_encode(&mut o).expect("transport encode"); + }); + let enc_tot = median_ms(|| { + let _ = ipc::encode_to_bytes(otap.clone(), comp).expect("encode"); + }); + let ipc_ser = (enc_tot - t_enc).max(0.0); + + let bytes = ipc::encode_to_bytes(otap.clone(), comp).expect("encode"); + let optimized = ipc::deserialize(&bytes).expect("deserialize"); + let ipc_des = median_ms(|| { + let _ = ipc::deserialize(&bytes).expect("deserialize"); + }); + let t_dec = median_ms(|| { + let mut o = optimized.clone(); + ipc::transport_decode(&mut o).expect("transport decode"); + }); + + println!( + "{:<6} {:>8.2} {:>8.2} {:>8.2} {:>8.2} {:>8.2} {:>8.2} {:>10}", + comp.label(), + t_enc, + ipc_ser, + enc_tot, + ipc_des, + t_dec, + ipc_des + t_dec, + bytes.len(), + ); + } + println!(); } - samples.sort_by(|a, b| a.partial_cmp(b).expect("no NaN")); - samples[samples.len() / 2] } -/// Print the server-side CPU model: where the OTAP -> Parquet conversion lands. -fn print_server_cost_table(shapes: &[LogsGenParams]) { - println!( - "\n=== Server-side CPU: where does OTAP->Parquet conversion land? (indicative ms) ===" - ); - println!("Option A (client sends OTAP/IPC): server = IPC-decode + flatten + Parquet-encode"); - println!( - "Option B (client sends Parquet): server = persist (~0 CPU) or reparse Parquet->Arrow" - ); - println!("IPC input fixed at zstd; 'comp' is the Parquet output compressor."); +/// Per-step breakdown of the Parquet encode and decode pipelines. +fn print_parquet_breakdown(shapes: &[LogsGenParams]) { + println!("\n=== Parquet pipeline breakdown (indicative ms) ==="); + println!("encode = flatten + parquet-write decode = parquet-read + unflatten"); println!( - "{:<16} {:<8} {:>12} {:>12} {:>13} {:>15}", - "flatten", "comp", "A:convert", "B:reparse", "save(store)", "save(reparse)" + "{:<16} {:<8} {:>9} {:>9} {:>9} {:>9} {:>9} {:>9} {:>10}", + "scheme", "comp", "flatten", "pq-write", "enc-tot", "pq-read", "unflat", "dec-tot", "bytes" ); - for shape in shapes { let (otap, _) = gen_logs_otap(shape); - println!( - "-- shape {} : {} log records --", - shape.label(), - shape.total_logs() - ); - let ipc_bytes = Scheme::Ipc - .codec(Compressor::Zstd) - .write(otap.clone()) - .expect("ipc write"); - + println!("-- shape {} log records --", shape.total_logs()); for scheme in Scheme::flattened() { - for &compressor in scheme.compressors() { - let pcodec = scheme.codec(compressor); - let parquet_bytes = pcodec.write(otap.clone()).expect("parquet write"); - - let convert_a = median_ms(|| { - let _ = server::convert_ipc_to_parquet(&ipc_bytes, &*pcodec).expect("convert"); + let flat = scheme.flatten(&otap).expect("flatten"); + let flatten_t = median_ms(|| { + let _ = scheme.flatten(&otap).expect("flatten"); + }); + for compressor in Compressor::ALL { + let write_t = median_ms(|| { + let _ = write_parquet(&flat, compressor.parquet()).expect("write"); }); - let reparse_b = median_ms(|| { - let _ = reparse(scheme, &parquet_bytes).expect("reparse"); + let bytes = write_parquet(&flat, compressor.parquet()).expect("write"); + let read_flat = read_parquet(&bytes).expect("read"); + let read_t = median_ms(|| { + let _ = read_parquet(&bytes).expect("read"); + }); + let unflatten_t = median_ms(|| { + let _ = scheme.unflatten(&read_flat).expect("unflatten"); }); - println!( - "{:<16} {:<8} {:>10.3}ms {:>10.3}ms {:>11.3}ms {:>13.3}ms", + "{:<16} {:<8} {:>8.2} {:>8.2} {:>8.2} {:>8.2} {:>8.2} {:>8.2} {:>10}", scheme.name(), compressor.label(), - convert_a, - reparse_b, - convert_a, - convert_a - reparse_b + flatten_t, + write_t, + flatten_t + write_t, + read_t, + unflatten_t, + read_t + unflatten_t, + bytes.len(), ); } } @@ -196,12 +209,13 @@ fn print_server_cost_table(shapes: &[LogsGenParams]) { fn bench_round_trip(c: &mut Criterion) { let shapes = input_shapes(); print_size_table(&shapes); - print_server_cost_table(&shapes); + print_ipc_breakdown(&shapes); + print_parquet_breakdown(&shapes); let mut write_group = c.benchmark_group("parquet_study/write"); - let _ = write_group.sample_size(20); + let _ = write_group.sample_size(10); let _ = write_group.warm_up_time(Duration::from_millis(500)); - let _ = write_group.measurement_time(Duration::from_secs(2)); + let _ = write_group.measurement_time(Duration::from_secs(3)); for shape in &shapes { let (otap, _) = gen_logs_otap(shape); for scheme in Scheme::all() { @@ -209,7 +223,7 @@ fn bench_round_trip(c: &mut Criterion) { let codec = scheme.codec(compressor); let id = BenchmarkId::new( format!("{}/{}", codec.name(), compressor.label()), - shape.label(), + shape.total_logs(), ); let _ = write_group.bench_with_input(id, shape, |b, _| { b.iter_batched( @@ -224,9 +238,9 @@ fn bench_round_trip(c: &mut Criterion) { write_group.finish(); let mut read_group = c.benchmark_group("parquet_study/read"); - let _ = read_group.sample_size(20); + let _ = read_group.sample_size(10); let _ = read_group.warm_up_time(Duration::from_millis(500)); - let _ = read_group.measurement_time(Duration::from_secs(2)); + let _ = read_group.measurement_time(Duration::from_secs(3)); for shape in &shapes { let (otap, _) = gen_logs_otap(shape); for scheme in Scheme::all() { @@ -235,7 +249,7 @@ fn bench_round_trip(c: &mut Criterion) { let bytes = codec.write(otap.clone()).expect("write"); let id = BenchmarkId::new( format!("{}/{}", codec.name(), compressor.label()), - shape.label(), + shape.total_logs(), ); let _ = read_group.bench_with_input(id, shape, |b, _| { b.iter(|| black_box(codec.read(&bytes).expect("read"))); @@ -246,58 +260,10 @@ fn bench_round_trip(c: &mut Criterion) { read_group.finish(); } -fn bench_server_cost(c: &mut Criterion) { - let shapes = input_shapes(); - - let mut group = c.benchmark_group("parquet_study/server_cost"); - let _ = group.sample_size(20); - let _ = group.warm_up_time(Duration::from_millis(500)); - let _ = group.measurement_time(Duration::from_secs(2)); - - for shape in &shapes { - let (otap, _) = gen_logs_otap(shape); - // Client -> server wire is OTAP/IPC; decode auto-detects compression. - let ipc_bytes = Scheme::Ipc - .codec(Compressor::Zstd) - .write(otap.clone()) - .expect("ipc write"); - - for scheme in Scheme::flattened() { - for &compressor in scheme.compressors() { - let pcodec = scheme.codec(compressor); - - // Option A: server converts received OTAP/IPC to Parquet. - let id_a = BenchmarkId::new( - format!("convert-A/{}/{}", scheme.name(), compressor.label()), - shape.label(), - ); - let _ = group.bench_with_input(id_a, shape, |b, _| { - b.iter(|| { - black_box( - server::convert_ipc_to_parquet(&ipc_bytes, &*pcodec).expect("convert"), - ) - }); - }); - - // Option B: server reparses client-precomputed Parquet. - let parquet_bytes = pcodec.write(otap.clone()).expect("parquet write"); - let id_b = BenchmarkId::new( - format!("accept-B/{}/{}", scheme.name(), compressor.label()), - shape.label(), - ); - let _ = group.bench_with_input(id_b, shape, |b, _| { - b.iter(|| black_box(reparse(scheme, &parquet_bytes).expect("reparse"))); - }); - } - } - } - group.finish(); -} - #[allow(missing_docs)] mod bench_entry { use super::*; - criterion_group!(benches, bench_round_trip, bench_server_cost); + criterion_group!(benches, bench_round_trip); } criterion_main!(bench_entry::benches); diff --git a/rust/otap-dataflow/benchmarks/src/parquet_study/ipc.rs b/rust/otap-dataflow/benchmarks/src/parquet_study/ipc.rs index 7070e92f62..c83768cab5 100644 --- a/rust/otap-dataflow/benchmarks/src/parquet_study/ipc.rs +++ b/rust/otap-dataflow/benchmarks/src/parquet_study/ipc.rs @@ -27,27 +27,60 @@ pub struct IpcCodec { pub compressor: Compressor, } +/// Pipeline sub-step: apply the OTAP transport-optimized encoding in place +/// (delta/dictionary encodings on id and value columns, parent-id remapping). +/// This is the first thing `Producer::produce_bar` does; measuring it alone lets +/// the benchmark separate it from the Arrow IPC serialization. +pub fn transport_encode(logs: &mut OtapArrowRecords) -> StudyResult<()> { + logs.encode_transport_optimized()?; + Ok(()) +} + +/// Pipeline sub-step: serialize a logs batch to wire bytes. This runs the whole +/// encode side (transport-optimized encoding, then Arrow IPC serialization with +/// compression, then prost-encoding the `BatchArrowRecords`). The IPC +/// serialization time alone is this minus [`transport_encode`]. +pub fn encode_to_bytes(mut logs: OtapArrowRecords, compressor: Compressor) -> StudyResult> { + let mut producer = Producer::new_with_options(ProducerOptions { + ipc_compression: compressor.ipc(), + }); + let bar = producer.produce_bar(&mut logs)?; + let mut buf = Vec::with_capacity(1024); + bar.encode(&mut buf)?; + Ok(buf) +} + +/// Pipeline sub-step: deserialize wire bytes into a logs batch that is still in +/// the transport-optimized encoding (prost-decode, then `Consumer::consume_bar`, +/// then `from_record_messages`). Does not run the transport decode. +pub fn deserialize(bytes: &[u8]) -> StudyResult { + let mut bar = BatchArrowRecords::decode(bytes)?; + let mut consumer = Consumer::default(); + let messages = consumer.consume_bar(&mut bar)?; + Ok(OtapArrowRecords::Logs(from_record_messages::( + messages, + )?)) +} + +/// Pipeline sub-step: reverse the transport-optimized encoding in place, leaving +/// the logical OTAP logs batch. +pub fn transport_decode(logs: &mut OtapArrowRecords) -> StudyResult<()> { + logs.decode_transport_optimized_ids()?; + Ok(()) +} + impl Codec for IpcCodec { fn name(&self) -> &'static str { "ipc" } - fn write(&self, mut logs: OtapArrowRecords) -> StudyResult> { - let mut producer = Producer::new_with_options(ProducerOptions { - ipc_compression: self.compressor.ipc(), - }); - let bar = producer.produce_bar(&mut logs)?; - let mut buf = Vec::with_capacity(1024); - bar.encode(&mut buf)?; - Ok(buf) + fn write(&self, logs: OtapArrowRecords) -> StudyResult> { + encode_to_bytes(logs, self.compressor) } fn read(&self, bytes: &[u8]) -> StudyResult { - let mut bar = BatchArrowRecords::decode(bytes)?; - let mut consumer = Consumer::default(); - let messages = consumer.consume_bar(&mut bar)?; - let mut logs = OtapArrowRecords::Logs(from_record_messages::(messages)?); - logs.decode_transport_optimized_ids()?; + let mut logs = deserialize(bytes)?; + transport_decode(&mut logs)?; Ok(logs) } } diff --git a/rust/otap-dataflow/benchmarks/src/parquet_study/mod.rs b/rust/otap-dataflow/benchmarks/src/parquet_study/mod.rs index 468bfe31d2..ede1ac8f47 100644 --- a/rust/otap-dataflow/benchmarks/src/parquet_study/mod.rs +++ b/rust/otap-dataflow/benchmarks/src/parquet_study/mod.rs @@ -13,6 +13,7 @@ //! zstd. Arrow IPC only supports zstd and lz4 (frame), so snappy is offered for //! the Parquet schemes only. +use arrow::array::RecordBatch; use otap_df_pdata::otap::OtapArrowRecords; pub mod attrs; @@ -22,8 +23,6 @@ pub mod map; pub mod nested; pub mod parquet_io; pub mod server; -#[cfg(feature = "vortex")] -pub mod vortex; pub mod wide; /// Error type used throughout the study (benchmark/test code, so a boxed error @@ -129,36 +128,20 @@ pub enum Scheme { Map, /// Flattened Parquet, attributes exploded to one typed column per key. Wide, - /// Flattened Vortex file (nested layout). Requires the `vortex` feature. - #[cfg(feature = "vortex")] - Vortex, - /// Flattened Vortex file written with no compression, prioritizing write - /// throughput. Requires the `vortex` feature. - #[cfg(feature = "vortex")] - VortexFast, } impl Scheme { - /// All schemes, in reporting order (includes Vortex when the `vortex` - /// feature is enabled). + /// All schemes, in reporting order. #[must_use] pub fn all() -> Vec { - #[allow(unused_mut)] - let mut v = vec![Scheme::Ipc, Scheme::Nested, Scheme::Map, Scheme::Wide]; - #[cfg(feature = "vortex")] - v.extend([Scheme::Vortex, Scheme::VortexFast]); - v + vec![Scheme::Ipc, Scheme::Nested, Scheme::Map, Scheme::Wide] } - /// Only the flattened-file schemes (used by the server-cost model, where IPC - /// is the input rather than an output format). Includes Vortex when enabled. + /// Only the flattened-file schemes (used by the server-cost model and the + /// pipeline-step breakdown, where IPC is handled separately). #[must_use] pub fn flattened() -> Vec { - #[allow(unused_mut)] - let mut v = vec![Scheme::Nested, Scheme::Map, Scheme::Wide]; - #[cfg(feature = "vortex")] - v.extend([Scheme::Vortex, Scheme::VortexFast]); - v + vec![Scheme::Nested, Scheme::Map, Scheme::Wide] } /// Stable contender name. @@ -169,21 +152,14 @@ impl Scheme { Scheme::Nested => "parquet-nested", Scheme::Map => "parquet-map", Scheme::Wide => "parquet-wide", - #[cfg(feature = "vortex")] - Scheme::Vortex => "vortex", - #[cfg(feature = "vortex")] - Scheme::VortexFast => "vortex-fast", } } - /// The compressors valid for this scheme. IPC excludes snappy; Vortex applies - /// its own cascading compression, so it exposes only a single `none` setting. + /// The compressors valid for this scheme. IPC excludes snappy. #[must_use] pub fn compressors(self) -> &'static [Compressor] { match self { Scheme::Ipc => &Compressor::IPC, - #[cfg(feature = "vortex")] - Scheme::Vortex | Scheme::VortexFast => &[Compressor::None], _ => &Compressor::ALL, } } @@ -196,10 +172,27 @@ impl Scheme { Scheme::Nested => Box::new(nested::NestedParquetCodec { compressor }), Scheme::Map => Box::new(map::MapParquetCodec { compressor }), Scheme::Wide => Box::new(wide::WideParquetCodec { compressor }), - #[cfg(feature = "vortex")] - Scheme::Vortex => Box::new(vortex::VortexCodec { fast: false }), - #[cfg(feature = "vortex")] - Scheme::VortexFast => Box::new(vortex::VortexCodec { fast: true }), + } + } + + /// Flatten an OTAP logs batch into this Parquet scheme's flat record batch. + /// Errors for [`Scheme::Ipc`], which is not a flattened-file scheme. + pub fn flatten(self, otap: &OtapArrowRecords) -> StudyResult { + match self { + Scheme::Nested => nested::flatten(otap), + Scheme::Map => map::flatten(otap), + Scheme::Wide => wide::flatten(otap), + Scheme::Ipc => Err("ipc has no flatten step".into()), + } + } + + /// Reconstruct an OTAP logs batch from this scheme's flat record batch. + pub fn unflatten(self, flat: &RecordBatch) -> StudyResult { + match self { + Scheme::Nested => nested::unflatten(flat), + Scheme::Map => map::unflatten(flat), + Scheme::Wide => wide::unflatten(flat), + Scheme::Ipc => Err("ipc has no unflatten step".into()), } } } diff --git a/rust/otap-dataflow/benchmarks/src/parquet_study/vortex.rs b/rust/otap-dataflow/benchmarks/src/parquet_study/vortex.rs deleted file mode 100644 index 988292cd4f..0000000000 --- a/rust/otap-dataflow/benchmarks/src/parquet_study/vortex.rs +++ /dev/null @@ -1,267 +0,0 @@ -// Copyright The OpenTelemetry Authors -// SPDX-License-Identifier: Apache-2.0 - -//! Vortex file-format contender (enabled by the `vortex` cargo feature). -//! -//! Vortex is a next-generation columnar file format that interoperates with -//! Arrow. This contender reuses the same flattening as [`super::nested`] (one -//! flat Arrow `RecordBatch` per logs batch) but serializes it to an in-memory -//! Vortex file instead of Parquet, then reads it back and unflattens to OTAP. -//! -//! Vortex applies its own cascading compression (BtrBlocks), so the study's -//! [`Compressor`](super::Compressor) axis does not apply; the single setting is -//! reported as `none` (meaning "Vortex default encodings"). - -use std::sync::{Arc, OnceLock}; - -use arrow::array::{Array, ArrayRef as ArrowArrayRef, RecordBatch, StructArray}; -use arrow::compute::cast; -use arrow::datatypes::{DataType, Field, Fields, Schema}; -use otap_df_pdata::otap::OtapArrowRecords; -use otap_df_pdata::schema::consts; - -use vortex::VortexSessionDefault; -use vortex::array::ArrayRef; -use vortex::array::VortexSessionExecute; -use vortex::array::arrow::ArrowSessionExt; -use vortex::array::arrow::FromArrowArray; -use vortex::array::stream::ArrayStreamExt; -use vortex::buffer::ByteBuffer; -use vortex::compressor::BtrBlocksCompressorBuilder; -use vortex::file::{OpenOptionsSessionExt, WriteOptionsSessionExt, WriteStrategyBuilder}; -use vortex::io::session::RuntimeSessionExt; -use vortex::session::VortexSession; - -use super::{Codec, StudyResult, nested}; - -/// Contender that flattens OTAP logs (nested layout) and stores them in an -/// in-memory Vortex file. `fast` selects an uncompressed, near-zero-copy write -/// strategy instead of the default BtrBlocks cascading compressor. -pub struct VortexCodec { - /// When true, write with no compression to prioritize write throughput. - pub fast: bool, -} - -fn runtime() -> &'static tokio::runtime::Runtime { - static RT: OnceLock = OnceLock::new(); - RT.get_or_init(|| { - tokio::runtime::Builder::new_multi_thread() - .enable_all() - .build() - .expect("build tokio runtime") - }) -} - -/// True for `FixedSizeBinary` or a dictionary whose values are `FixedSizeBinary`. -/// Vortex 0.75 has no `FixedSizeBinary` encoding, so such columns (the logs -/// `trace_id`/`span_id`, which are dictionary-encoded in the OTAP form) must be -/// rewritten before writing. -fn is_fixed_size_binary(dt: &DataType) -> bool { - match dt { - DataType::FixedSizeBinary(_) => true, - DataType::Dictionary(_, value) => matches!(value.as_ref(), DataType::FixedSizeBinary(_)), - _ => false, - } -} - -/// Cast any `FixedSizeBinary`-valued column to plain `Binary` before writing. -fn to_vortex_compatible(flat: &RecordBatch) -> StudyResult { - let mut fields: Vec = Vec::with_capacity(flat.num_columns()); - let mut columns: Vec = Vec::with_capacity(flat.num_columns()); - for (field, column) in flat.schema().fields().iter().zip(flat.columns()) { - if is_fixed_size_binary(field.data_type()) { - fields.push(Field::new( - field.name(), - DataType::Binary, - field.is_nullable(), - )); - columns.push(cast(column, &DataType::Binary)?); - } else { - fields.push(field.as_ref().clone()); - columns.push(column.clone()); - } - } - Ok(RecordBatch::try_new( - Arc::new(Schema::new(fields)), - columns, - )?) -} - -/// Reduce a data type to a "plain" Arrow type: decode dictionaries to their -/// value type and turn `FixedSizeBinary` into `Binary`, recursing through struct -/// and list children. Used as the target for Vortex's `execute_arrow` so the -/// decoded batch has deterministic, non-view, non-dictionary types that the OTAP -/// schema check accepts. (`trace_id`/`span_id` are later restored to -/// `FixedSizeBinary`.) -fn plainify(dt: &DataType) -> DataType { - match dt { - DataType::Dictionary(_, value) => plainify(value), - DataType::FixedSizeBinary(_) => DataType::Binary, - DataType::Struct(fields) => DataType::Struct(plainify_fields(fields)), - DataType::List(field) => DataType::List(Arc::new(plainify_field(field))), - DataType::LargeList(field) => DataType::List(Arc::new(plainify_field(field))), - other => other.clone(), - } -} - -fn plainify_field(field: &Field) -> Field { - Field::new( - field.name(), - plainify(field.data_type()), - field.is_nullable(), - ) -} - -fn plainify_fields(fields: &Fields) -> Fields { - Fields::from(fields.iter().map(|f| plainify_field(f)).collect::>()) -} - -/// The Arrow struct field describing the plain type Vortex should decode into. -/// Derived once from a sample flattened batch (the flat schema is deterministic). -fn vortex_target_field() -> &'static Field { - static TARGET: OnceLock = OnceLock::new(); - TARGET.get_or_init(|| { - use crate::parquet_study::datagen::{LogsGenParams, gen_logs_otap}; - let (otap, _) = gen_logs_otap(&LogsGenParams { - num_resources: 1, - num_scopes: 1, - num_logs: 1, - }); - let flat = nested::flatten(&otap).expect("sample flatten"); - let compat = to_vortex_compatible(&flat).expect("sample compat"); - Field::new( - "", - DataType::Struct(plainify_fields(compat.schema().fields())), - false, - ) - }) -} - -/// Restore the `FixedSizeBinary` columns that [`to_vortex_compatible`] cast to -/// `Binary`, so the reconstructed Logs batch matches the OTAP schema. -fn restore_fixed_size_binary(batch: RecordBatch) -> StudyResult { - let restores = [(consts::TRACE_ID, 16i32), (consts::SPAN_ID, 8i32)]; - let mut fields: Vec = batch - .schema() - .fields() - .iter() - .map(|f| f.as_ref().clone()) - .collect(); - let mut columns: Vec = batch.columns().to_vec(); - for (name, size) in restores { - if let Ok(idx) = batch.schema().index_of(name) { - if matches!( - columns[idx].data_type(), - DataType::Binary | DataType::LargeBinary - ) { - columns[idx] = cast(&columns[idx], &DataType::FixedSizeBinary(size))?; - fields[idx] = Field::new( - name, - DataType::FixedSizeBinary(size), - fields[idx].is_nullable(), - ); - } - } - } - Ok(RecordBatch::try_new( - Arc::new(Schema::new(fields)), - columns, - )?) -} - -/// Encode a flat Arrow record batch as an in-memory Vortex file. When `fast` is -/// set, a no-op compressor is used so the write is essentially a canonical -/// (uncompressed) layout, skipping the BtrBlocks encoding search. -fn encode(flat: RecordBatch, fast: bool) -> StudyResult> { - let flat = to_vortex_compatible(&flat)?; - runtime() - .block_on(async move { - let session = VortexSession::default().with_tokio(); - let array = ArrayRef::from_arrow(flat, false)?; - let mut sink: Vec = Vec::with_capacity(4096); - let write_options = session.write_options(); - let stream = array.to_array_stream(); - let _summary = if fast { - let strategy = WriteStrategyBuilder::default() - .with_btrblocks_builder(BtrBlocksCompressorBuilder::empty()) - .build(); - write_options - .with_strategy(strategy) - .write(&mut sink, stream) - .await? - } else { - write_options.write(&mut sink, stream).await? - }; - Ok::, vortex::error::VortexError>(sink) - }) - .map_err(Into::into) -} - -/// Decode an in-memory Vortex file back into a flat Arrow record batch. -fn decode(bytes: &[u8]) -> StudyResult { - let buffer = ByteBuffer::from(bytes.to_vec()); - let batch = runtime() - .block_on(async move { - let session = VortexSession::default().with_tokio(); - let file = session.open_options().open_buffer(buffer)?; - let array = file.scan()?.into_array_stream()?.read_all().await?; - let mut ctx = session.create_execution_ctx(); - let arrow_session = session.arrow(); - let arrow = - arrow_session.execute_arrow(array, Some(vortex_target_field()), &mut ctx)?; - let struct_array = arrow - .as_any() - .downcast_ref::() - .expect("vortex top-level array is a struct"); - Ok::(RecordBatch::from(struct_array.clone())) - }) - .map_err(Box::::from)?; - restore_fixed_size_binary(batch) -} - -impl Codec for VortexCodec { - fn name(&self) -> &'static str { - if self.fast { "vortex-fast" } else { "vortex" } - } - - fn write(&self, logs: OtapArrowRecords) -> StudyResult> { - let flat = nested::flatten(&logs)?; - encode(flat, self.fast) - } - - fn read(&self, bytes: &[u8]) -> StudyResult { - let flat = decode(bytes)?; - nested::unflatten(&flat) - } -} - -/// Decode a Vortex file back into the flat Arrow record batch, without -/// unflattening to OTAP. Used by the server-cost model's "reparse" measurement. -pub fn reparse_to_arrow(bytes: &[u8]) -> StudyResult { - decode(bytes) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::parquet_study::attrs::assert_logs_equivalent; - use crate::parquet_study::datagen::{LogsGenParams, gen_logs_otap}; - - #[test] - fn vortex_round_trip_preserves_structure() { - let params = LogsGenParams { - num_resources: 3, - num_scopes: 2, - num_logs: 5, - }; - let (otap, _) = gen_logs_otap(¶ms); - - for fast in [false, true] { - let codec = VortexCodec { fast }; - let bytes = codec.write(otap.clone()).expect("write"); - assert!(!bytes.is_empty()); - let decoded = codec.read(&bytes).expect("read"); - assert_logs_equivalent(&otap, &decoded, codec.name(), "none"); - } - } -} From a8adc68138088c25880e011589530375614833a5 Mon Sep 17 00:00:00 2001 From: Joshua MacDonald Date: Wed, 1 Jul 2026 08:02:17 -0700 Subject: [PATCH 05/18] docs(benchmarks): add ANALYSIS.md explaining the measured cost ratios Adds a short analysis of the otap_parquet ratios, with lz4 included in the Parquet tables since that is the compressor used in the comparison elsewhere. The analysis explains that IPC encode is dominated by the transport-optimized encoding rather than compression, that IPC serialize is faster when compressed because there is less data to move, that Parquet encode is dominated by the writer with a large flatten tax, and that lz4 IPC decode is about six times slower than zstd in this Arrow IPC build. That last point shrinks the IPC-over-Parquet decode advantage from about 22x with zstd to under 5x with lz4, so measurements must hold the compressor fixed. The lz4 rows are also added to the README Parquet breakdown table. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../benches/otap_parquet/ANALYSIS.md | 111 ++++++++++++++++++ .../benchmarks/benches/otap_parquet/README.md | 6 + 2 files changed, 117 insertions(+) create mode 100644 rust/otap-dataflow/benchmarks/benches/otap_parquet/ANALYSIS.md diff --git a/rust/otap-dataflow/benchmarks/benches/otap_parquet/ANALYSIS.md b/rust/otap-dataflow/benchmarks/benches/otap_parquet/ANALYSIS.md new file mode 100644 index 0000000000..8c6baa9bb0 --- /dev/null +++ b/rust/otap-dataflow/benchmarks/benches/otap_parquet/ANALYSIS.md @@ -0,0 +1,111 @@ + + +# Analysis: why OTAP/IPC is cheaper than flattened Parquet + +This explains the ratios the `otap_parquet` benchmark measures. Numbers are from +one development machine running WSL with jemalloc, at the 100,000 log-record +shape under a single resource and scope. The OTLP protobuf encoding of that batch +is 22,700,225 bytes. Times are indicative medians in milliseconds. Both `zstd` +and `lz4` are shown, because `lz4` is how the comparison was measured elsewhere. + +## Headline ratios (100k records) + +| contender | comp | encode ms | decode ms | size bytes | size vs ipc | +|----------------|------|-----------|-----------|------------|-------------| +| ipc | zstd | 44.4 | 9.8 | 702,702 | 1.00x | +| ipc | lz4 | 41.6 | 44.7 | 915,632 | 1.30x | +| parquet-nested | zstd | 552 | 217 | 384,740 | 0.55x | +| parquet-nested | lz4 | 477 | 213 | 518,503 | 0.57x | +| parquet-wide | zstd | 330 | 147 | 392,782 | 0.56x | +| parquet-wide | lz4 | 330 | 142 | 509,659 | 0.56x | + +Reading the ratios with `lz4`, which is the relevant compressor here: + +- Parquet-nested is 0.57x the IPC size, so it is smaller on the wire. +- Parquet-nested costs 11.5x more to encode than IPC, 477 ms versus 41.6 ms. +- Parquet-nested costs 4.8x more to decode than IPC, 213 ms versus 44.7 ms. + +With `zstd` the encode ratio is about the same, 12.4x, but the decode ratio jumps +to 22.2x, because IPC decode with `zstd` is only 9.8 ms. The reason is explained +below and it is the single most important thing to know when comparing `lz4` +measurements to `zstd` measurements. + +## Where the time goes + +The benchmark breaks each side into two steps. + +OTAP/IPC, 100k, per step: + +| comp | transport-encode | ipc-serialize | ipc-deserialize | transport-decode | +|------|------------------|---------------|-----------------|------------------| +| zstd | 35.2 | 9.2 | 6.6 | 3.2 | +| lz4 | 34.5 | 7.1 | 42.2 | 2.4 | + +Parquet, 100k, per step: + +| scheme / comp | flatten | pq-write | pq-read | unflatten | +|-----------------------|---------|----------|---------|-----------| +| parquet-nested / zstd | 148 | 404 | 152 | 65 | +| parquet-nested / lz4 | 148 | 329 | 143 | 70 | +| parquet-wide / zstd | 183 | 147 | 56 | 91 | +| parquet-wide / lz4 | 183 | 147 | 55 | 87 | + +## Explaining each ratio + +### IPC encode is dominated by transport-optimize, not compression + +The transport-optimized encoding is about 35 ms of the 42 to 44 ms IPC encode, +regardless of compressor. It makes a full pass over the four record batches, +applying delta and dictionary encodings to the id and value columns and remapping +parent ids so the child batches still reference the right rows. That pass touches +all of the data once, which is why it is the largest single IPC cost and why it +does not change with the compressor. The Arrow IPC serialization that follows is +small, 7 to 9 ms, because it writes already-compact columns and the compressor +runs on far less data. + +### IPC serialize is faster when compressed + +Uncompressed IPC serialize is 65 ms, far more than the 7 to 9 ms with `zstd` or +`lz4`. Writing the record batch means copying bytes, and without compression that +is 24 MB instead of about 700 KB. The compressor pays for itself on the write +side by shrinking what has to be moved. + +### IPC decode is where lz4 and zstd diverge sharply + +Deserialize dominates IPC decode, and the transport decode is cheap, 2 to 3 ms. +The striking result is that `lz4` deserialize is 42 ms while `zstd` deserialize +is 6.6 ms, a 6x difference. In this Arrow IPC implementation the LZ4 frame +decompression path is much slower than zstd for this data. That single step is +why IPC decode is 9.8 ms with `zstd` but 44.7 ms with `lz4`, and therefore why +the IPC-over-Parquet decode advantage shrinks from 22x with `zstd` to under 5x +with `lz4`. Anyone measuring with `lz4` will see IPC decode look far worse than a +`zstd` measurement would suggest, even though the data is identical. + +### Parquet encode is dominated by the writer, with a large flatten tax + +Flatten is 140 to 185 ms and is compressor-independent, because it is a pure +Arrow transformation that joins the attribute batches onto the log rows and +builds the nested columns. For `parquet-nested` and `parquet-map` the Parquet +writer then costs 330 to 400 ms, because encoding `List` and `Map` +columns requires Parquet definition and repetition levels over many leaf fields. +`parquet-wide` writes in about 147 ms instead, because its attributes are flat +typed scalar columns that Parquet encodes cheaply, but it pays more in flatten, +183 ms, to explode the keys into columns. The compressor barely moves the writer +time, which is why `pq-write` is nearly constant across `zstd`, `lz4`, and +`snappy`, while the output size is not. + +### Parquet decode is dominated by the reader + +Parquet read is 55 to 152 ms and unflatten is 65 to 91 ms. `parquet-wide` reads +much faster, 55 ms, because flat columns decode directly, but its unflatten is a +little more expensive because it must reassemble attributes from many columns. + +## Bottom line + +Flattened Parquet is smaller on the wire, about 0.55 to 0.57x the IPC size, but +it costs roughly an order of magnitude more CPU to produce and, with `zstd`, to +consume. If the constraint is server CPU, keep the client on OTAP/IPC. Produce +Parquet only where the columnar file and its smaller size at rest are needed, and +expect that cost to land wherever the flatten and Parquet write run. When +comparing measurements, hold the compressor fixed, because the `lz4` IPC decode +penalty alone changes the decode ratio by more than 4x. diff --git a/rust/otap-dataflow/benchmarks/benches/otap_parquet/README.md b/rust/otap-dataflow/benchmarks/benches/otap_parquet/README.md index 842db8f6e6..028b89c159 100644 --- a/rust/otap-dataflow/benchmarks/benches/otap_parquet/README.md +++ b/rust/otap-dataflow/benchmarks/benches/otap_parquet/README.md @@ -109,8 +109,14 @@ Parquet pipeline breakdown, milliseconds: | scheme / comp | flatten | pq-write | enc-tot | pq-read | unflat | dec-tot | |-----------------------|---------|----------|---------|---------|--------|---------| | parquet-nested / zstd | 148 | 404 | 552 | 152 | 65 | 217 | +| parquet-nested / lz4 | 148 | 329 | 477 | 143 | 70 | 213 | | parquet-map / zstd | 139 | 363 | 503 | 160 | 63 | 223 | +| parquet-map / lz4 | 139 | 334 | 474 | 167 | 67 | 234 | | parquet-wide / zstd | 183 | 147 | 330 | 56 | 91 | 147 | +| parquet-wide / lz4 | 183 | 147 | 330 | 55 | 87 | 142 | + +A companion write-up of what these ratios mean is in +[`ANALYSIS.md`](./ANALYSIS.md). ## Takeaways From ff1e89e84d38768fe03afb12cdda72c236d0a3d7 Mon Sep 17 00:00:00 2001 From: Joshua MacDonald Date: Wed, 1 Jul 2026 08:41:51 -0700 Subject: [PATCH 06/18] feat(benchmarks): account for OTAP/IPC streaming amortization The single-batch size comparison left the Arrow IPC streaming benefit off the table, so this adds a streaming measurement and documents the effect. A long-lived Producer writes the Arrow schema once and delta-encodes dictionaries, so every batch after the first omits the schema and re-sends only new dictionary entries. The measured amortization is a fixed cost of about 11 KB per batch. For small frequent batches this flips the size verdict, because at 1000 records the steady-state IPC batch is 0.69x the Parquet file, and Parquet has no equivalent per-batch amortization. ipc::stream_batch_sizes measures this and the bench prints a streaming table. This also corrects the block sizes. A single OTAP logs batch holds at most 65535 records because the log id is a u16, which overflowed silently in release at 100k and panicked in debug. The breakdown shapes are now 10k, 30k, and 60k, and the illustrative numbers use the valid 50k batch. ANALYSIS.md and the README are updated with the streaming table and the u16 limit. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../benches/otap_parquet/ANALYSIS.md | 168 +++++++++++------- .../benchmarks/benches/otap_parquet/README.md | 85 +++++---- .../benchmarks/benches/otap_parquet/main.rs | 62 ++++++- .../benchmarks/src/parquet_study/ipc.rs | 51 ++++++ 4 files changed, 268 insertions(+), 98 deletions(-) diff --git a/rust/otap-dataflow/benchmarks/benches/otap_parquet/ANALYSIS.md b/rust/otap-dataflow/benchmarks/benches/otap_parquet/ANALYSIS.md index 8c6baa9bb0..3ebe0bdfdf 100644 --- a/rust/otap-dataflow/benchmarks/benches/otap_parquet/ANALYSIS.md +++ b/rust/otap-dataflow/benchmarks/benches/otap_parquet/ANALYSIS.md @@ -3,109 +3,155 @@ # Analysis: why OTAP/IPC is cheaper than flattened Parquet This explains the ratios the `otap_parquet` benchmark measures. Numbers are from -one development machine running WSL with jemalloc, at the 100,000 log-record -shape under a single resource and scope. The OTLP protobuf encoding of that batch -is 22,700,225 bytes. Times are indicative medians in milliseconds. Both `zstd` -and `lz4` are shown, because `lz4` is how the comparison was measured elsewhere. +one development machine running WSL with jemalloc. Times are indicative medians +in milliseconds. Both `zstd` and `lz4` are shown, because `lz4` is how the +comparison was measured elsewhere. -## Headline ratios (100k records) +A single OTAP logs batch holds at most 65,535 log records, because the log id +that links a log row to its attributes is a `u16`. The breakdown below therefore +uses a 50,000-record batch as its headline, which is a large but valid single +batch. Volumes larger than 65,535 records must be streamed as several batches, +which is analyzed at the end and turns out to change the size comparison. + +## Headline ratios (50k records, one batch) | contender | comp | encode ms | decode ms | size bytes | size vs ipc | |----------------|------|-----------|-----------|------------|-------------| -| ipc | zstd | 44.4 | 9.8 | 702,702 | 1.00x | -| ipc | lz4 | 41.6 | 44.7 | 915,632 | 1.30x | -| parquet-nested | zstd | 552 | 217 | 384,740 | 0.55x | -| parquet-nested | lz4 | 477 | 213 | 518,503 | 0.57x | -| parquet-wide | zstd | 330 | 147 | 392,782 | 0.56x | -| parquet-wide | lz4 | 330 | 142 | 509,659 | 0.56x | +| ipc | zstd | 18.9 | 4.2 | 401,966 | 1.00x | +| ipc | lz4 | 16.1 | 20.0 | 466,416 | 1.16x | +| parquet-nested | zstd | 171 | 54.5 | 240,211 | 0.60x | +| parquet-nested | lz4 | 160 | 51.4 | 329,298 | 0.71x | +| parquet-wide | zstd | 94.4 | 34.2 | 245,757 | 0.61x | +| parquet-wide | lz4 | 91.7 | 32.2 | 327,547 | 0.70x | -Reading the ratios with `lz4`, which is the relevant compressor here: +Reading the ratios with `lz4`, which is the relevant compressor here, for a +single 50k batch: -- Parquet-nested is 0.57x the IPC size, so it is smaller on the wire. -- Parquet-nested costs 11.5x more to encode than IPC, 477 ms versus 41.6 ms. -- Parquet-nested costs 4.8x more to decode than IPC, 213 ms versus 44.7 ms. +- Parquet-nested is 0.71x the IPC size, so it is smaller at rest. +- Parquet-nested costs about 10x more to encode than IPC, 160 ms versus 16 ms. +- Parquet-nested costs about 2.6x more to decode than IPC, 51 ms versus 20 ms. -With `zstd` the encode ratio is about the same, 12.4x, but the decode ratio jumps -to 22.2x, because IPC decode with `zstd` is only 9.8 ms. The reason is explained -below and it is the single most important thing to know when comparing `lz4` -measurements to `zstd` measurements. +With `zstd` the encode ratio is about the same, 9x, but the decode ratio is 13x, +because IPC decode with `zstd` is only 4.2 ms. The reason is the `lz4` decode +penalty explained below. ## Where the time goes The benchmark breaks each side into two steps. -OTAP/IPC, 100k, per step: +OTAP/IPC, 50k, per step: | comp | transport-encode | ipc-serialize | ipc-deserialize | transport-decode | |------|------------------|---------------|-----------------|------------------| -| zstd | 35.2 | 9.2 | 6.6 | 3.2 | -| lz4 | 34.5 | 7.1 | 42.2 | 2.4 | +| zstd | 13.0 | 5.9 | 3.0 | 1.2 | +| lz4 | 13.5 | 2.6 | 18.7 | 1.3 | -Parquet, 100k, per step: +Parquet, 50k, per step: | scheme / comp | flatten | pq-write | pq-read | unflatten | |-----------------------|---------|----------|---------|-----------| -| parquet-nested / zstd | 148 | 404 | 152 | 65 | -| parquet-nested / lz4 | 148 | 329 | 143 | 70 | -| parquet-wide / zstd | 183 | 147 | 56 | 91 | -| parquet-wide / lz4 | 183 | 147 | 55 | 87 | +| parquet-nested / zstd | 44 | 127 | 39 | 16 | +| parquet-nested / lz4 | 44 | 116 | 35 | 17 | +| parquet-wide / zstd | 69 | 25 | 14 | 20 | +| parquet-wide / lz4 | 69 | 22 | 12 | 20 | ## Explaining each ratio ### IPC encode is dominated by transport-optimize, not compression -The transport-optimized encoding is about 35 ms of the 42 to 44 ms IPC encode, +The transport-optimized encoding is about 13 ms of the 16 to 19 ms IPC encode, regardless of compressor. It makes a full pass over the four record batches, applying delta and dictionary encodings to the id and value columns and remapping parent ids so the child batches still reference the right rows. That pass touches all of the data once, which is why it is the largest single IPC cost and why it does not change with the compressor. The Arrow IPC serialization that follows is -small, 7 to 9 ms, because it writes already-compact columns and the compressor +small, 3 to 6 ms, because it writes already-compact columns and the compressor runs on far less data. ### IPC serialize is faster when compressed -Uncompressed IPC serialize is 65 ms, far more than the 7 to 9 ms with `zstd` or -`lz4`. Writing the record batch means copying bytes, and without compression that -is 24 MB instead of about 700 KB. The compressor pays for itself on the write -side by shrinking what has to be moved. +Uncompressed IPC serialize is much slower than compressed, because writing the +record batch means copying bytes and there are far more of them without +compression. The compressor pays for itself on the write side by shrinking what +has to be moved. ### IPC decode is where lz4 and zstd diverge sharply -Deserialize dominates IPC decode, and the transport decode is cheap, 2 to 3 ms. -The striking result is that `lz4` deserialize is 42 ms while `zstd` deserialize -is 6.6 ms, a 6x difference. In this Arrow IPC implementation the LZ4 frame -decompression path is much slower than zstd for this data. That single step is -why IPC decode is 9.8 ms with `zstd` but 44.7 ms with `lz4`, and therefore why -the IPC-over-Parquet decode advantage shrinks from 22x with `zstd` to under 5x -with `lz4`. Anyone measuring with `lz4` will see IPC decode look far worse than a -`zstd` measurement would suggest, even though the data is identical. +Deserialize dominates IPC decode, and the transport decode is cheap. The striking +result is that `lz4` deserialize is about six times slower than `zstd` for this +data. In this Arrow IPC implementation the LZ4 frame decompression path is much +slower than zstd. That single step is why IPC decode is 4.2 ms with `zstd` but +20 ms with `lz4`, and therefore why the IPC-over-Parquet decode advantage shrinks +from about 13x with `zstd` to under 3x with `lz4`. Anyone measuring with `lz4` +will see IPC decode look far worse than a `zstd` measurement would, even though +the data is identical. ### Parquet encode is dominated by the writer, with a large flatten tax -Flatten is 140 to 185 ms and is compressor-independent, because it is a pure -Arrow transformation that joins the attribute batches onto the log rows and -builds the nested columns. For `parquet-nested` and `parquet-map` the Parquet -writer then costs 330 to 400 ms, because encoding `List` and `Map` -columns requires Parquet definition and repetition levels over many leaf fields. -`parquet-wide` writes in about 147 ms instead, because its attributes are flat -typed scalar columns that Parquet encodes cheaply, but it pays more in flatten, -183 ms, to explode the keys into columns. The compressor barely moves the writer -time, which is why `pq-write` is nearly constant across `zstd`, `lz4`, and -`snappy`, while the output size is not. +Flatten is compressor-independent, because it is a pure Arrow transformation that +joins the attribute batches onto the log rows and builds the nested columns. For +`parquet-nested` and `parquet-map` the Parquet writer then dominates, because +encoding `List` and `Map` columns requires Parquet definition and +repetition levels over many leaf fields. `parquet-wide` writes much faster +because its attributes are flat typed scalar columns that Parquet encodes +cheaply, but it pays more in flatten to explode the keys into columns. The +compressor barely moves the writer time, which is why `pq-write` is nearly +constant across `zstd`, `lz4`, and `snappy`, while the output size is not. ### Parquet decode is dominated by the reader -Parquet read is 55 to 152 ms and unflatten is 65 to 91 ms. `parquet-wide` reads -much faster, 55 ms, because flat columns decode directly, but its unflatten is a -little more expensive because it must reassemble attributes from many columns. +Parquet read is larger than unflatten. `parquet-wide` reads fastest because flat +columns decode directly, though its unflatten is a little more expensive because +it reassembles attributes from many columns. + +## Streaming changes the size story + +The size numbers above encode each batch as a self-contained Arrow IPC stream, +which pays the full schema and dictionary cost every time. That is the cold, or +worst, case. In real OTAP streaming the `Producer` is long-lived: it writes the +Arrow schema into the stream once, and it delta-encodes dictionaries, so every +batch after the first omits the schema and re-sends only new dictionary entries. +Parquet has no equivalent per-batch amortization, because each Parquet file is +self-contained with its own schema, footer, and per-row-group dictionary pages. + +Measured cold versus steady-state IPC size per batch, with the equivalent single +Parquet file for reference: + +| logs | comp | cold | warm | saved | pq-nested | warm/pq | +|--------|------|---------|---------|--------|-----------|---------| +| 1,000 | zstd | 24,748 | 13,422 | 11,326 | 19,512 | 0.69x | +| 1,000 | lz4 | 27,692 | 16,366 | 11,326 | 21,136 | 0.77x | +| 10,000 | zstd | 92,270 | 80,944 | 11,326 | 57,023 | 1.42x | +| 10,000 | lz4 | 108,846 | 97,520 | 11,326 | 75,336 | 1.29x | +| 50,000 | zstd | 401,966 | 390,640 | 11,326 | 240,211 | 1.63x | +| 50,000 | lz4 | 466,416 | 455,088 | 11,326 | 329,298 | 1.38x | + +The amortization is a fixed cost of about 11,326 bytes per batch, which is the +schema message plus the initial dictionaries, and it is the same at every batch +size. What changes is how large that fixed cost is relative to the batch: + +- At 1,000 records per batch it is about 46 percent of the cold size, and it + flips the verdict: the steady-state IPC batch is smaller than the Parquet file, + 0.69x with `zstd` and 0.77x with `lz4`. +- At 10,000 records it is about 12 percent, and Parquet is still smaller on the + wire, though the gap narrows. +- At 50,000 records it is about 3 percent, and Parquet keeps its size advantage. + +So the single-batch size comparison understates OTAP/IPC, and it understates it +most for the small, frequent batches that low-latency telemetry actually sends. +The `u16` log-id limit reinforces this: because one batch cannot exceed 65,535 +records, high volume is delivered as a stream of batches, which is exactly where +the schema and dictionary amortization applies. ## Bottom line -Flattened Parquet is smaller on the wire, about 0.55 to 0.57x the IPC size, but -it costs roughly an order of magnitude more CPU to produce and, with `zstd`, to -consume. If the constraint is server CPU, keep the client on OTAP/IPC. Produce -Parquet only where the columnar file and its smaller size at rest are needed, and -expect that cost to land wherever the flatten and Parquet write run. When -comparing measurements, hold the compressor fixed, because the `lz4` IPC decode -penalty alone changes the decode ratio by more than 4x. +For a single large batch, flattened Parquet is smaller on the wire, about 0.60 to +0.71x the IPC size, but it costs roughly an order of magnitude more CPU to +produce and, with `zstd`, to consume. When the traffic is streamed, which is the +normal case and is required above 65,535 records per batch, OTAP/IPC amortizes a +fixed schema and dictionary cost of about 11 KB per batch, and for small frequent +batches that makes IPC smaller on the wire than Parquet as well as far cheaper to +produce and consume. Produce Parquet where the columnar file and its smaller size +for large data at rest are needed, and keep the streaming client on OTAP/IPC. +When comparing measurements, hold the compressor fixed and state whether the size +is cold or steady-state, because both choices move the ratio by a large factor. diff --git a/rust/otap-dataflow/benchmarks/benches/otap_parquet/README.md b/rust/otap-dataflow/benchmarks/benches/otap_parquet/README.md index 028b89c159..6830b5464c 100644 --- a/rust/otap-dataflow/benchmarks/benches/otap_parquet/README.md +++ b/rust/otap-dataflow/benchmarks/benches/otap_parquet/README.md @@ -54,12 +54,14 @@ rather than the deprecated Hadoop-framed `LZ4`. cargo bench -p benchmarks --bench otap_parquet ``` -Three tables are printed to stdout before the timed round-trip benchmarks run: -serialized size, the OTAP/IPC pipeline breakdown, and the Parquet pipeline -breakdown. The default shapes are 10k, 50k, and 100k log records, so the full -Criterion sweep is slow. The printed tables are the main output; for a quick pass -add `-- --measurement-time 0.5 --sample-size 10`, or read the tables and stop the -run. +Four tables are printed to stdout before the timed round-trip benchmarks run: +serialized size, the OTAP/IPC pipeline breakdown, the Parquet pipeline breakdown, +and the OTAP/IPC streaming amortization. The breakdown shapes are 10k, 30k, and +60k log records. A single OTAP logs batch holds at most 65,535 records because +the log id is a `u16`, so the shapes stay below that and larger volumes are +streamed. The full Criterion sweep is slow; the printed tables are the main +output. For a quick pass add `-- --measurement-time 0.5 --sample-size 10`, or +read the tables and stop the run. ## Pipeline steps @@ -85,53 +87,69 @@ carries. ## Illustrative results These numbers come from one development machine running WSL with jemalloc, at the -100,000 log-record shape whose OTLP protobuf encoding is 22,700,225 bytes. -Absolute values vary by host, but the relationships are stable. +50,000 log-record shape, which is a large but valid single OTAP batch. Absolute +values vary by host, but the relationships are stable. Serialized size in bytes: | contender | zstd | lz4 | snappy | none | |----------------|---------|---------|---------|------------| -| ipc | 702,702 | 915,632 | n/a | 24,428,084 | -| parquet-nested | 384,740 | 518,503 | 800,452 | 7,079,037 | -| parquet-wide | 392,782 | 509,659 | 608,717 | 2,838,767 | +| ipc | 401,966 | 466,416 | n/a | 12,221,748 | +| parquet-nested | 240,211 | 329,298 | 405,397 | 2,118,185 | +| parquet-wide | 245,757 | 327,547 | 327,294 | 344,708 | OTAP/IPC pipeline breakdown, milliseconds: | comp | t-enc | ipc-ser | enc-tot | ipc-des | t-dec | dec-tot | |------|-------|---------|---------|---------|-------|---------| -| zstd | 35.2 | 9.2 | 44.4 | 6.6 | 3.2 | 9.8 | -| lz4 | 34.5 | 7.1 | 41.6 | 42.2 | 2.4 | 44.7 | -| none | 39.5 | 64.9 | 104.4 | 54.6 | 2.5 | 57.1 | +| zstd | 13.0 | 5.9 | 18.9 | 3.0 | 1.2 | 4.2 | +| lz4 | 13.5 | 2.6 | 16.1 | 18.7 | 1.3 | 20.0 | +| none | 12.6 | 29.6 | 42.2 | 27.1 | 1.2 | 28.3 | Parquet pipeline breakdown, milliseconds: | scheme / comp | flatten | pq-write | enc-tot | pq-read | unflat | dec-tot | |-----------------------|---------|----------|---------|---------|--------|---------| -| parquet-nested / zstd | 148 | 404 | 552 | 152 | 65 | 217 | -| parquet-nested / lz4 | 148 | 329 | 477 | 143 | 70 | 213 | -| parquet-map / zstd | 139 | 363 | 503 | 160 | 63 | 223 | -| parquet-map / lz4 | 139 | 334 | 474 | 167 | 67 | 234 | -| parquet-wide / zstd | 183 | 147 | 330 | 56 | 91 | 147 | -| parquet-wide / lz4 | 183 | 147 | 330 | 55 | 87 | 142 | - -A companion write-up of what these ratios mean is in -[`ANALYSIS.md`](./ANALYSIS.md). +| parquet-nested / zstd | 44 | 127 | 171 | 39 | 16 | 55 | +| parquet-nested / lz4 | 44 | 116 | 160 | 35 | 17 | 51 | +| parquet-map / zstd | 52 | 133 | 185 | 53 | 14 | 67 | +| parquet-map / lz4 | 52 | 110 | 162 | 53 | 14 | 66 | +| parquet-wide / zstd | 69 | 25 | 94 | 14 | 20 | 34 | +| parquet-wide / lz4 | 69 | 22 | 92 | 12 | 20 | 32 | + +Streaming amortization, IPC bytes per batch when a long-lived Producer streams +many batches, with the equivalent single Parquet file for reference. `cold` is +the first batch, `warm` is steady-state, and `saved` is the fixed schema and +dictionary cost that streaming amortizes: + +| logs | comp | cold | warm | saved | pq-nested | warm/pq | +|--------|------|---------|---------|--------|-----------|---------| +| 1,000 | zstd | 24,748 | 13,422 | 11,326 | 19,512 | 0.69x | +| 10,000 | zstd | 92,270 | 80,944 | 11,326 | 57,023 | 1.42x | +| 50,000 | zstd | 401,966 | 390,640 | 11,326 | 240,211 | 1.63x | + +A companion write-up of what these ratios mean, including the streaming effect, +is in [`ANALYSIS.md`](./ANALYSIS.md). ## Takeaways -- IPC is far cheaper than Parquet on both sides. At 100k with zstd, IPC encodes - in about 44 ms and decodes in about 10 ms, while `parquet-nested` encodes in - about 552 ms and decodes in about 217 ms. That is roughly 12 times cheaper to - encode and 22 times cheaper to decode. -- Inside IPC encode, the transport-optimized encoding dominates, about 35 ms of - the 44 ms, and it is essentially compressor-independent because it runs before +- IPC is far cheaper than Parquet on both sides. At 50k with zstd, IPC encodes in + about 19 ms and decodes in about 4 ms, while `parquet-nested` encodes in about + 171 ms and decodes in about 55 ms. That is roughly 9 times cheaper to encode and + 13 times cheaper to decode. +- Inside IPC encode, the transport-optimized encoding dominates, about 13 ms of + the 19 ms, and it is essentially compressor-independent because it runs before compression. Inside IPC decode, the deserialization dominates and the transport decode is small. +- Streaming amortizes a fixed cost of about 11 KB per batch, which is the schema + plus initial dictionaries. For small frequent batches this flips the size + verdict: at 1,000 records the steady-state IPC batch is 0.69x the Parquet file. + Parquet has no equivalent per-batch amortization. Because a batch cannot exceed + 65,535 records, high volume is streamed, which is where this applies. - Inside Parquet encode, the Parquet writer dominates and the flatten is roughly a third to a half of the total. `parquet-wide` writes fastest because it has typed scalar columns rather than nested `List` or `Map`, but it pays - more in flatten, and it ends up the cheapest Parquet encoder overall at 330 ms. + more in flatten, and it ends up the cheapest Parquet encoder overall at 94 ms. - Compression choice matters in surprising ways. For IPC, compression makes the serialize step faster because there is far less data to move, so `none` is the slowest to serialize, and `lz4` is much slower to deserialize than `zstd` in @@ -148,8 +166,9 @@ A companion write-up of what these ratios mean is in Contenders are the `Scheme` enum and its `Codec` implementations in `benchmarks/src/parquet_study`. Add a variant to include a contender everywhere. The IPC sub-steps are `ipc::transport_encode`, `ipc::encode_to_bytes`, -`ipc::deserialize`, and `ipc::transport_decode`; the Parquet steps are -`Scheme::flatten`, `parquet_io::write_parquet`, `parquet_io::read_parquet`, and -`Scheme::unflatten`. Input shapes are defined in `input_shapes()` in +`ipc::deserialize`, and `ipc::transport_decode`, and `ipc::stream_batch_sizes` +measures streaming amortization; the Parquet steps are `Scheme::flatten`, +`parquet_io::write_parquet`, `parquet_io::read_parquet`, and `Scheme::unflatten`. +Input shapes are defined in `input_shapes()` and `streaming_shapes()` in `benches/otap_parquet/main.rs`. The round-trips have unit tests runnable with `cargo test -p benchmarks --lib parquet_study`. diff --git a/rust/otap-dataflow/benchmarks/benches/otap_parquet/main.rs b/rust/otap-dataflow/benchmarks/benches/otap_parquet/main.rs index 4dc2976c74..94a462c380 100644 --- a/rust/otap-dataflow/benchmarks/benches/otap_parquet/main.rs +++ b/rust/otap-dataflow/benchmarks/benches/otap_parquet/main.rs @@ -45,11 +45,25 @@ use tikv_jemallocator::Jemalloc; #[global_allocator] static GLOBAL: Jemalloc = Jemalloc; -/// Input shapes: several block sizes larger than a few thousand log records, -/// under a single resource/scope, which is the realistic case for a Parquet -/// file. +/// Input shapes for the breakdown: block sizes larger than a few thousand log +/// records, under a single resource/scope. A single OTAP logs batch caps at +/// 65,535 records because log ids are u16, so these stay below that limit; larger +/// volumes must be streamed as multiple batches (see the streaming table). fn input_shapes() -> Vec { - [10_000usize, 50_000, 100_000] + [10_000usize, 30_000, 60_000] + .into_iter() + .map(|num_logs| LogsGenParams { + num_resources: 1, + num_scopes: 1, + num_logs, + }) + .collect() +} + +/// Batch sizes for the streaming table, spanning small to large so the fixed +/// per-batch schema/dictionary overhead is visible as a fraction of the batch. +fn streaming_shapes() -> Vec { + [1_000usize, 10_000, 50_000] .into_iter() .map(|num_logs| LogsGenParams { num_resources: 1, @@ -206,11 +220,51 @@ fn print_parquet_breakdown(shapes: &[LogsGenParams]) { } } +/// OTAP/IPC streaming amortization: cold (first) versus warm (steady-state) +/// per-batch size when a single long-lived Producer streams many batches, with +/// the equivalent single Parquet file for reference. +fn print_streaming_table(shapes: &[LogsGenParams]) { + println!("\n=== OTAP/IPC streaming amortization (bytes per batch) ==="); + println!("One long-lived Producer streams batches: schema once, delta dictionaries."); + println!("cold = first batch (schema + full dictionaries + data); warm = steady-state batch."); + println!( + "pq-nested is the same batch as one Parquet file, which has no per-batch amortization." + ); + println!( + "{:<8} {:<6} {:>12} {:>12} {:>10} {:>12} {:>9}", + "logs", "comp", "cold", "warm", "saved", "pq-nested", "warm/pq" + ); + for shape in shapes { + let (otap, _) = gen_logs_otap(shape); + let flat = Scheme::Nested.flatten(&otap).expect("flatten"); + for &comp in Scheme::Ipc.compressors() { + let sizes = ipc::stream_batch_sizes(&otap, comp, 6).expect("stream sizes"); + let cold = sizes[0]; + let warm = *sizes.last().expect("non-empty"); + let pq = write_parquet(&flat, comp.parquet()) + .expect("parquet write") + .len(); + println!( + "{:<8} {:<6} {:>12} {:>12} {:>10} {:>12} {:>8.2}x", + shape.total_logs(), + comp.label(), + cold, + warm, + cold - warm, + pq, + warm as f64 / pq as f64, + ); + } + } + println!(); +} + fn bench_round_trip(c: &mut Criterion) { let shapes = input_shapes(); print_size_table(&shapes); print_ipc_breakdown(&shapes); print_parquet_breakdown(&shapes); + print_streaming_table(&streaming_shapes()); let mut write_group = c.benchmark_group("parquet_study/write"); let _ = write_group.sample_size(10); diff --git a/rust/otap-dataflow/benchmarks/src/parquet_study/ipc.rs b/rust/otap-dataflow/benchmarks/src/parquet_study/ipc.rs index c83768cab5..6860c70d77 100644 --- a/rust/otap-dataflow/benchmarks/src/parquet_study/ipc.rs +++ b/rust/otap-dataflow/benchmarks/src/parquet_study/ipc.rs @@ -69,6 +69,34 @@ pub fn transport_decode(logs: &mut OtapArrowRecords) -> StudyResult<()> { Ok(()) } +/// Serialize the same logs batch `count` times through a single long-lived +/// [`Producer`], returning the wire size of each batch. +/// +/// This models OTAP streaming: the Arrow schema is written once into the stream +/// and dictionaries are delta-encoded, so `sizes[0]` is the cold size (schema +/// plus full dictionaries plus data) while `sizes[1..]` are the steady-state +/// sizes (data plus only new dictionary entries). Sending the identical batch +/// repeatedly is a best case for dictionary amortization; real telemetry with +/// varying values falls between the cold and steady-state sizes. +pub fn stream_batch_sizes( + logs: &OtapArrowRecords, + compressor: Compressor, + count: usize, +) -> StudyResult> { + let mut producer = Producer::new_with_options(ProducerOptions { + ipc_compression: compressor.ipc(), + }); + let mut sizes = Vec::with_capacity(count); + for _ in 0..count { + let mut batch = logs.clone(); + let bar = producer.produce_bar(&mut batch)?; + let mut buf = Vec::with_capacity(1024); + bar.encode(&mut buf)?; + sizes.push(buf.len()); + } + Ok(sizes) +} + impl Codec for IpcCodec { fn name(&self) -> &'static str { "ipc" @@ -114,4 +142,27 @@ mod tests { ); } } + + #[test] + fn streaming_amortizes_schema_and_dictionaries() { + let params = LogsGenParams { + num_resources: 1, + num_scopes: 1, + num_logs: 2000, + }; + let (otap, _) = gen_logs_otap(¶ms); + + for compressor in Compressor::IPC { + let sizes = stream_batch_sizes(&otap, compressor, 4).expect("stream sizes"); + // The steady-state batch (2nd onward) omits the schema header and + // re-sends only new dictionary entries, so it is smaller than the + // cold first batch. + assert!( + sizes[1] < sizes[0], + "{compressor:?}: steady {} not smaller than cold {}", + sizes[1], + sizes[0] + ); + } + } } From 9313dbc0b45b1a0c1c0f83fdb0c1c35ea4f5c6cf Mon Sep 17 00:00:00 2001 From: Joshua MacDonald Date: Wed, 1 Jul 2026 09:22:42 -0700 Subject: [PATCH 07/18] docs(benchmarks): split the streaming savings into schema and dictionaries The steady-state IPC saving looked schema-shaped because it is constant with batch size, but splitting the fixed per-batch amortization by Arrow IPC message type shows it is mostly dictionaries. Of the roughly 11.3 KB saved per steady-state batch at 1000 records with zstd, 7.7 KB are dictionary messages and 3.6 KB are schema, about two thirds dictionaries and one third schema. The cost is fixed with batch size because both parts are per-stream rather than per-row. The schema describes columns, and the set of distinct dictionary values does not grow with the row count in this synthetic data, so the dictionary messages do not grow either. The analysis also notes that this is a best case for dictionary amortization, because real high-cardinality columns such as a trace id or a log body carry new values every batch and their delta dictionaries do not amortize. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../benches/otap_parquet/ANALYSIS.md | 50 +++++++++++++++---- .../benchmarks/benches/otap_parquet/README.md | 11 ++-- 2 files changed, 47 insertions(+), 14 deletions(-) diff --git a/rust/otap-dataflow/benchmarks/benches/otap_parquet/ANALYSIS.md b/rust/otap-dataflow/benchmarks/benches/otap_parquet/ANALYSIS.md index 3ebe0bdfdf..c09fa21f60 100644 --- a/rust/otap-dataflow/benchmarks/benches/otap_parquet/ANALYSIS.md +++ b/rust/otap-dataflow/benchmarks/benches/otap_parquet/ANALYSIS.md @@ -126,9 +126,30 @@ Parquet file for reference: | 50,000 | zstd | 401,966 | 390,640 | 11,326 | 240,211 | 1.63x | | 50,000 | lz4 | 466,416 | 455,088 | 11,326 | 329,298 | 1.38x | -The amortization is a fixed cost of about 11,326 bytes per batch, which is the -schema message plus the initial dictionaries, and it is the same at every batch -size. What changes is how large that fixed cost is relative to the batch: +The amortization is a fixed cost of about 11,326 bytes per batch, and it is the +same at every batch size. Splitting that fixed cost by Arrow IPC message type, at +1,000 records with `zstd`, shows it is mostly dictionaries rather than schema: + +| payload | schema | dictionaries | data (warm) | +|---------------|--------|--------------|-------------| +| Logs | 1,856 | 4,032 | 3,968 | +| LogAttrs | 832 | 2,048 | 7,744 | +| ResourceAttrs | 448 | 832 | 832 | +| ScopeAttrs | 448 | 832 | 832 | +| total | 3,584 | 7,744 | 13,376 | + +So of the 11,328 bytes saved per steady-state batch, 7,744 are dictionary +messages and only 3,584 are schema, about 68 percent dictionaries and 32 percent +schema. The dictionary messages are large not because the values are large, since +this data has few distinct values, but because the four batches carry many +dictionary columns, and each carries per-message framing that the stream sends +once and then reuses. + +The cost is fixed with batch size because both parts are per-stream, not +per-row. The schema describes columns, not rows. The dictionaries are the set of +distinct values, and in this synthetic data that set is the same whether the +batch has 1,000 or 50,000 rows, so the dictionary messages do not grow. What +changes is how large that fixed cost is relative to the batch: - At 1,000 records per batch it is about 46 percent of the cold size, and it flips the verdict: the steady-state IPC batch is smaller than the Parquet file, @@ -137,6 +158,16 @@ size. What changes is how large that fixed cost is relative to the batch: wire, though the gap narrows. - At 50,000 records it is about 3 percent, and Parquet keeps its size advantage. +This synthetic data is a best case for dictionary amortization, because every +batch carries the identical low-cardinality values, so batches after the first +send no new dictionary entries at all. Real telemetry amortizes only the stable +low-cardinality columns, such as attribute keys, severity, and scope names. A +high-cardinality column such as a trace id or a log body carries new values in +every batch, so its delta dictionary keeps sending new entries and does not +amortize, and such a column is often better left non-dictionary. The measured +amortization here should be read as the ceiling for dictionaries plus the schema, +which is always recovered. + So the single-batch size comparison understates OTAP/IPC, and it understates it most for the small, frequent batches that low-latency telemetry actually sends. The `u16` log-id limit reinforces this: because one batch cannot exceed 65,535 @@ -149,9 +180,10 @@ For a single large batch, flattened Parquet is smaller on the wire, about 0.60 t 0.71x the IPC size, but it costs roughly an order of magnitude more CPU to produce and, with `zstd`, to consume. When the traffic is streamed, which is the normal case and is required above 65,535 records per batch, OTAP/IPC amortizes a -fixed schema and dictionary cost of about 11 KB per batch, and for small frequent -batches that makes IPC smaller on the wire than Parquet as well as far cheaper to -produce and consume. Produce Parquet where the columnar file and its smaller size -for large data at rest are needed, and keep the streaming client on OTAP/IPC. -When comparing measurements, hold the compressor fixed and state whether the size -is cold or steady-state, because both choices move the ratio by a large factor. +fixed cost of about 11 KB per batch that is roughly two thirds dictionaries and +one third schema, and for small frequent batches that makes IPC smaller on the +wire than Parquet as well as far cheaper to produce and consume. Produce Parquet +where the columnar file and its smaller size for large data at rest are needed, +and keep the streaming client on OTAP/IPC. When comparing measurements, hold the +compressor fixed and state whether the size is cold or steady-state, because both +choices move the ratio by a large factor. diff --git a/rust/otap-dataflow/benchmarks/benches/otap_parquet/README.md b/rust/otap-dataflow/benchmarks/benches/otap_parquet/README.md index 6830b5464c..4d682570fe 100644 --- a/rust/otap-dataflow/benchmarks/benches/otap_parquet/README.md +++ b/rust/otap-dataflow/benchmarks/benches/otap_parquet/README.md @@ -141,11 +141,12 @@ is in [`ANALYSIS.md`](./ANALYSIS.md). the 19 ms, and it is essentially compressor-independent because it runs before compression. Inside IPC decode, the deserialization dominates and the transport decode is small. -- Streaming amortizes a fixed cost of about 11 KB per batch, which is the schema - plus initial dictionaries. For small frequent batches this flips the size - verdict: at 1,000 records the steady-state IPC batch is 0.69x the Parquet file. - Parquet has no equivalent per-batch amortization. Because a batch cannot exceed - 65,535 records, high volume is streamed, which is where this applies. +- Streaming amortizes a fixed cost of about 11 KB per batch, which is roughly two + thirds dictionary messages and one third schema, and is independent of the row + count. For small frequent batches this flips the size verdict: at 1,000 records + the steady-state IPC batch is 0.69x the Parquet file. Parquet has no equivalent + per-batch amortization. Because a batch cannot exceed 65,535 records, high + volume is streamed, which is where this applies. - Inside Parquet encode, the Parquet writer dominates and the flatten is roughly a third to a half of the total. `parquet-wide` writes fastest because it has typed scalar columns rather than nested `List` or `Map`, but it pays From 63137087092ff19fd493131638724bc6793ae32a Mon Sep 17 00:00:00 2001 From: Joshua MacDonald Date: Wed, 1 Jul 2026 09:39:34 -0700 Subject: [PATCH 08/18] docs(benchmarks): explain flat IPC steady-state (no cross-frame compression) Assert that streaming batches after the second are identical in size, and document that Arrow IPC amortizes the schema and dictionary value tables once but does not compress batches against each other, so the per-row payload is re-sent every batch. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../benchmarks/benches/otap_parquet/ANALYSIS.md | 14 ++++++++++++++ .../benchmarks/src/parquet_study/ipc.rs | 11 ++++++++++- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/rust/otap-dataflow/benchmarks/benches/otap_parquet/ANALYSIS.md b/rust/otap-dataflow/benchmarks/benches/otap_parquet/ANALYSIS.md index c09fa21f60..a52a16c577 100644 --- a/rust/otap-dataflow/benchmarks/benches/otap_parquet/ANALYSIS.md +++ b/rust/otap-dataflow/benchmarks/benches/otap_parquet/ANALYSIS.md @@ -168,6 +168,20 @@ amortize, and such a column is often better left non-dictionary. The measured amortization here should be read as the ceiling for dictionaries plus the schema, which is always recovered. +This also explains why the steady-state batches do not keep shrinking. The drop +is one-time, at the first batch, and every batch after that is the same size. The +reason is that Arrow IPC amortizes the schema and the dictionary value tables +once, but it does not compress one batch against another: each batch's column +buffers are compressed independently so a reader can decode any batch on its own. +So even though the batches here are identical, every steady-state batch re-sends +and re-compresses the full per-row payload, which is the dictionary indices plus +the non-dictionary columns. Dictionary reuse saves the value tables, not the +per-row references, and the per-row payload is the actual information in the +batch. A streaming compressor with cross-batch context could shrink near-identical +batches much further, but that is outside what Arrow IPC does, and real telemetry, +where every batch carries different records, has less cross-batch redundancy to +exploit anyway. + So the single-batch size comparison understates OTAP/IPC, and it understates it most for the small, frequent batches that low-latency telemetry actually sends. The `u16` log-id limit reinforces this: because one batch cannot exceed 65,535 diff --git a/rust/otap-dataflow/benchmarks/src/parquet_study/ipc.rs b/rust/otap-dataflow/benchmarks/src/parquet_study/ipc.rs index 6860c70d77..631fd6c1ad 100644 --- a/rust/otap-dataflow/benchmarks/src/parquet_study/ipc.rs +++ b/rust/otap-dataflow/benchmarks/src/parquet_study/ipc.rs @@ -153,7 +153,7 @@ mod tests { let (otap, _) = gen_logs_otap(¶ms); for compressor in Compressor::IPC { - let sizes = stream_batch_sizes(&otap, compressor, 4).expect("stream sizes"); + let sizes = stream_batch_sizes(&otap, compressor, 5).expect("stream sizes"); // The steady-state batch (2nd onward) omits the schema header and // re-sends only new dictionary entries, so it is smaller than the // cold first batch. @@ -163,6 +163,15 @@ mod tests { sizes[1], sizes[0] ); + // The drop is one-time: with identical batches, every steady-state + // batch is the same size. Arrow IPC does not compress frames against + // each other, so frame N is not smaller than frame 2 despite carrying + // identical data. + assert!( + sizes[2..].iter().all(|&s| s == sizes[1]), + "{compressor:?}: steady-state not flat: {:?}", + sizes + ); } } } From b9cd57c57151fd716cf2f30ba39859b7767b7da5 Mon Sep 17 00:00:00 2001 From: Joshua MacDonald Date: Wed, 1 Jul 2026 10:03:11 -0700 Subject: [PATCH 09/18] docs(benchmarks): quantify the cross-batch redundancy IPC leaves unexploited Add a measurement showing that Arrow IPC's per-batch independent compression stores near-duplicate streaming batches at full size (about 81 KB each at 10k logs), while a whole-stream zstd recovers the cross-batch redundancy only with a large window and long-distance matching (level 19 collapses eight near-duplicate batches to 69 KB, a marginal 269 bytes each), and a default-effort whole-stream zstd does not, because each uncompressed 2.4 MB batch exceeds its match window. - ipc.rs: new cross_batch_redundancy_needs_large_window test and helper. - ANALYSIS.md: new section with the measured table and three caveats (best-case duplicates, high CPU, loss of per-batch independent decode). - Cargo.toml: zstd dev-dependency for the whole-stream measurement. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- rust/otap-dataflow/benchmarks/Cargo.toml | 1 + .../benches/otap_parquet/ANALYSIS.md | 40 +++++++++-- .../benchmarks/src/parquet_study/ipc.rs | 68 +++++++++++++++++++ 3 files changed, 105 insertions(+), 4 deletions(-) diff --git a/rust/otap-dataflow/benchmarks/Cargo.toml b/rust/otap-dataflow/benchmarks/Cargo.toml index 2382022ca5..ae2504af16 100644 --- a/rust/otap-dataflow/benchmarks/Cargo.toml +++ b/rust/otap-dataflow/benchmarks/Cargo.toml @@ -48,6 +48,7 @@ unsync.workspace = true portpicker.workspace = true tokio-stream.workspace = true weaver_common.workspace = true +zstd.workspace = true [target.'cfg(not(windows))'.dev-dependencies] tikv-jemallocator.workspace = true diff --git a/rust/otap-dataflow/benchmarks/benches/otap_parquet/ANALYSIS.md b/rust/otap-dataflow/benchmarks/benches/otap_parquet/ANALYSIS.md index a52a16c577..6b869b3c14 100644 --- a/rust/otap-dataflow/benchmarks/benches/otap_parquet/ANALYSIS.md +++ b/rust/otap-dataflow/benchmarks/benches/otap_parquet/ANALYSIS.md @@ -177,10 +177,42 @@ So even though the batches here are identical, every steady-state batch re-sends and re-compresses the full per-row payload, which is the dictionary indices plus the non-dictionary columns. Dictionary reuse saves the value tables, not the per-row references, and the per-row payload is the actual information in the -batch. A streaming compressor with cross-batch context could shrink near-identical -batches much further, but that is outside what Arrow IPC does, and real telemetry, -where every batch carries different records, has less cross-batch redundancy to -exploit anyway. +batch. How much redundancy this leaves unexploited, and why a naive whole-stream +compressor does not recover it, is measured in the next section. + +### The redundancy Arrow IPC leaves unexploited + +Because each batch is compressed on its own, the steady-state batches are stored +at full size even when they are nearly identical. To measure how much that leaves +unexploited, the study also compresses a whole stream of eight batches as a single +unit and compares the per-batch cost. At 10,000 logs per batch with `zstd`: + +| approach | 8 batches | per extra batch | +|------------------------|-----------|-----------------| +| ipc, per-batch zstd | 659 KB | 81 KB | +| whole stream, zstd L3 | 595 KB | 74 KB | +| whole stream, zstd L19 | 69 KB | 269 B | + +The uncompressed batches here differ by a single byte, a batch counter, so they +are almost pure duplicates, yet Arrow IPC still spends about 81 KB on each one. A +default-effort whole-stream `zstd` barely does better, about 74 KB per extra +batch, because each uncompressed batch is roughly 2.4 MB, larger than the match +window at that level, so the compressor cannot see that the previous batch was a +duplicate. Only a large-window, long-distance configuration at level 19 finds the +match and collapses each extra batch to about 269 bytes, storing all eight in +69 KB, close to the size of one. That factor of roughly nine is the cross-batch +redundancy Arrow IPC leaves on the table in this best case. + +Three caveats keep this from being free savings. First, it is a best case, +because these batches are byte-for-byte duplicates, whereas real telemetry batches +carry different records and share far less. Second, the large-window configuration +costs much more CPU than the light per-buffer codec Arrow IPC uses, so this is a +size ceiling, not a drop-in win. Third, whole-stream compression gives up the +per-batch independent decode that Arrow IPC provides, where any batch can be read +without the others. Arrow IPC trades cross-batch compression for low CPU and +independently decodable batches, which is the right trade for streaming transport, +so capturing the remaining redundancy is a job for a storage-side recompression +pass rather than the wire format. So the single-batch size comparison understates OTAP/IPC, and it understates it most for the small, frequent batches that low-latency telemetry actually sends. diff --git a/rust/otap-dataflow/benchmarks/src/parquet_study/ipc.rs b/rust/otap-dataflow/benchmarks/src/parquet_study/ipc.rs index 631fd6c1ad..30ffe049ca 100644 --- a/rust/otap-dataflow/benchmarks/src/parquet_study/ipc.rs +++ b/rust/otap-dataflow/benchmarks/src/parquet_study/ipc.rs @@ -174,4 +174,72 @@ mod tests { ); } } + + /// Produce `count` batches through one long-lived producer with IPC + /// compression disabled, concatenate the wire bytes, and compress the whole + /// stream once with zstd at `level`. This models the size a stream-level + /// (cross-batch) compressor could reach, whereas Arrow IPC compresses each + /// batch independently and cannot exploit redundancy across batches. + fn stream_whole_zstd(logs: &OtapArrowRecords, count: usize, level: i32) -> usize { + let mut producer = Producer::new_with_options(ProducerOptions { + ipc_compression: None, + }); + let mut stream = Vec::new(); + for _ in 0..count { + let mut batch = logs.clone(); + let bar = producer.produce_bar(&mut batch).expect("produce"); + bar.encode(&mut stream).expect("encode"); + } + zstd::stream::encode_all(&stream[..], level) + .expect("zstd") + .len() + } + + /// Arrow IPC compresses each batch independently, so it never exploits the + /// large redundancy across the near-identical steady-state batches. This test + /// shows two things about that unexploited redundancy: + /// + /// 1. A whole-stream compressor at default effort recovers almost none of it, + /// because each uncompressed batch (~2.4 MB at 10k logs) is larger than the + /// match window, so an extra near-duplicate batch still costs about as much + /// as one Arrow IPC batch. + /// 2. A whole-stream compressor with a large window and long-distance matching + /// does find it, collapsing each extra near-duplicate batch to a tiny + /// fraction of an Arrow IPC batch. + /// + /// The identical-batch case here is a best case; real telemetry batches differ + /// substantially, so the recoverable cross-batch redundancy is far smaller. + #[test] + fn cross_batch_redundancy_needs_large_window() { + let params = LogsGenParams { + num_resources: 1, + num_scopes: 1, + num_logs: 10_000, + }; + let (otap, _) = gen_logs_otap(¶ms); + + let count = 8; + // Arrow IPC steady-state batch size (per-batch independent compression). + let warm = stream_batch_sizes(&otap, Compressor::Zstd, count).expect("sizes")[1]; + + // Default effort: window smaller than one uncompressed batch, so an extra + // near-duplicate batch costs about the same as an Arrow IPC batch. + let low_1 = stream_whole_zstd(&otap, 1, 3); + let low_n = stream_whole_zstd(&otap, count, 3); + let low_marginal = (low_n - low_1) / (count - 1); + assert!( + low_marginal * 2 > warm, + "default-effort cross-batch unexpectedly cheap: {low_marginal} vs warm {warm}" + ); + + // High effort: large window plus long-distance matching finds the + // cross-batch redundancy and collapses each extra batch to near zero. + let high_1 = stream_whole_zstd(&otap, 1, 19); + let high_n = stream_whole_zstd(&otap, count, 19); + let high_marginal = (high_n - high_1) / (count - 1); + assert!( + high_marginal * 10 < warm, + "high-effort cross-batch did not collapse: {high_marginal} vs warm {warm}" + ); + } } From 9be2e4912561d05de090f63b9126ce656caf073e Mon Sep 17 00:00:00 2001 From: Joshua MacDonald Date: Wed, 1 Jul 2026 10:20:22 -0700 Subject: [PATCH 10/18] docs(benchmarks): add gateway-precompute perspective to the analysis Apply the read/write cost model to the deployment where one organization owns both the client exporter and the storage schema and is optimizing server-side ingestion CPU. Covers: the coupling objection dissolving into a layout-version surface, the store-versus-read hinge and Parquet footer/statistics enabling metadata-only routing without a full decode, moving transforms to the exporter while cross-source metric re-aggregation stays server-side, dual OTAP/IPC plus Parquet intake, and the zstd-versus-lz4 interaction on a .NET sender. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../benches/otap_parquet/ANALYSIS.md | 58 ++++++++++++++++++- 1 file changed, 57 insertions(+), 1 deletion(-) diff --git a/rust/otap-dataflow/benchmarks/benches/otap_parquet/ANALYSIS.md b/rust/otap-dataflow/benchmarks/benches/otap_parquet/ANALYSIS.md index 6b869b3c14..e64a0be0e7 100644 --- a/rust/otap-dataflow/benchmarks/benches/otap_parquet/ANALYSIS.md +++ b/rust/otap-dataflow/benchmarks/benches/otap_parquet/ANALYSIS.md @@ -220,6 +220,59 @@ The `u16` log-id limit reinforces this: because one batch cannot exceed 65,535 records, high volume is delivered as a stream of batches, which is exactly where the schema and dictionary amortization applies. +## Applying the model: precompute Parquet at the gateway + +The read and write costs above assume the converter runs on the server. The +deployment that motivates this study is different. A large sender, a customer +collector gateway, ships to a .NET SaaS ingestion service whose CPU is the +resource being optimized, and the same organization owns both the client exporter +and the storage schema. That ownership changes the trade in several ways. + +First, the coupling objection to client-side Parquet mostly goes away. When a +third party owns the storage format, making it the wire contract is brittle. When +the ingestion service owns the exporter, it can emit exactly the flattened layout +its store wants, the columns, partitioning, sort order, row-group sizing, and +compression, and it can version the exporter and the store together. The gateway +also aggregates many hosts, so it forms the large batches where flattened Parquet +is 0.60 to 0.71x the IPC size, which cuts ingress bandwidth into the service. + +Second, the decisive question becomes how much the ingestion service must read. +Precomputing Parquet removes server CPU only if ingestion is close to a validated +append. Two facts from the cost model bound this. If the service fully decodes, +Parquet is the most expensive input, about 13x the IPC decode with zstd, and the +service would re-encode anyway, so precomputing helps only when it avoids that +decode. But Parquet also carries a footer and per-row-group statistics, so tenant +routing, quota and cardinality checks, and min or max pruning can run on that +metadata without decoding column data. A service that inspects metadata and +appends pays far less than the 13x figure, and that is the regime where accepting +client Parquet wins. + +Third, any work that needs the row values still forces a full decode, so the goal +is to move that work into the exporter. Per-record transforms and redaction move +cleanly to the gateway, since the gateway is the last writer. Cross-source metric +re-aggregation does not, because one gateway sees only its own slice, so temporal +rollups across gateways remain a server-side read. Logs and traces are therefore +the strong case for precomputed Parquet, while aggregated metrics are the residual +case that still wants OTAP/IPC and a server-side convert. + +Two operational costs remain. Because exporters run on customer-managed +collectors, a storage-layout change cannot deploy atomically, so the ingestion +service must accept several layout versions at once and conform older ones itself, +which returns some CPU to the server. And not every sender is a large gateway, so +small and legacy senders still emit small batches where IPC is smaller and cheaper +than Parquet. The robust intake accepts both, OTAP/IPC for small senders and for +traffic that needs a server-side transform, and precomputed Parquet for large +gateways running the custom exporter. + +Finally, the compressor interacts with the .NET stack. The measured IPC decode +advantage depends on zstd. With lz4 the IPC decode is about 20 ms at 50k against +4 ms for zstd, so if the service does decode, the Parquet decode penalty over IPC +falls from about 13x to about 2.6x. And if the .NET Parquet writer can emit zstd +even where the .NET Arrow IPC writer cannot, the exporter can ship zstd Parquet, +which is smaller than lz4 and cheaper for any server read than lz4 IPC. That is +worth confirming on the .NET stack, because it removes the main compressor +argument against the Parquet path for that sender. + ## Bottom line For a single large batch, flattened Parquet is smaller on the wire, about 0.60 to @@ -232,4 +285,7 @@ wire than Parquet as well as far cheaper to produce and consume. Produce Parquet where the columnar file and its smaller size for large data at rest are needed, and keep the streaming client on OTAP/IPC. When comparing measurements, hold the compressor fixed and state whether the size is cold or steady-state, because both -choices move the ratio by a large factor. +choices move the ratio by a large factor. And when one organization owns both the +exporter and the store, the applied section above shows the convert step can move +to the sending gateway, as long as ingestion stays close to a metadata-validated +append rather than a full decode. From 51388dd152e20115d3ebc72c90426fe7853e13ce Mon Sep 17 00:00:00 2001 From: Joshua MacDonald Date: Wed, 1 Jul 2026 10:31:07 -0700 Subject: [PATCH 11/18] docs(benchmarks): correct the .NET IPC compression facts in the analysis Parquet is written by arrow-rs in the Rust sender, so it always has zstd. The .NET receiver's Arrow reader supports both lz4 and zstd IPC decompression as of arrow-dotnet v23, but only via the separate Apache.Arrow.Compression package and its codec factory; the core package decompresses neither and there is no build where lz4 works but zstd does not. Split the consequence into the package-present case, where zstd IPC stays available, and the package-absent case, where IPC must travel uncompressed and most favors precomputed Parquet, and note that the table decode times are arrow-rs while the .NET receiver decode is a separate cost. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../benches/otap_parquet/ANALYSIS.md | 28 +++++++++++++------ 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/rust/otap-dataflow/benchmarks/benches/otap_parquet/ANALYSIS.md b/rust/otap-dataflow/benchmarks/benches/otap_parquet/ANALYSIS.md index e64a0be0e7..9ce233141f 100644 --- a/rust/otap-dataflow/benchmarks/benches/otap_parquet/ANALYSIS.md +++ b/rust/otap-dataflow/benchmarks/benches/otap_parquet/ANALYSIS.md @@ -264,14 +264,26 @@ than Parquet. The robust intake accepts both, OTAP/IPC for small senders and for traffic that needs a server-side transform, and precomputed Parquet for large gateways running the custom exporter. -Finally, the compressor interacts with the .NET stack. The measured IPC decode -advantage depends on zstd. With lz4 the IPC decode is about 20 ms at 50k against -4 ms for zstd, so if the service does decode, the Parquet decode penalty over IPC -falls from about 13x to about 2.6x. And if the .NET Parquet writer can emit zstd -even where the .NET Arrow IPC writer cannot, the exporter can ship zstd Parquet, -which is smaller than lz4 and cheaper for any server read than lz4 IPC. That is -worth confirming on the .NET stack, because it removes the main compressor -argument against the Parquet path for that sender. +The compressor question is really about the .NET receiver, because the Parquet +here is written by arrow-rs in the Rust sender and can always use zstd. For the +IPC path, the current Apache Arrow .NET library, as of version 23, can decompress +both lz4 and zstd record-batch buffers, but only when the service adds the +Apache.Arrow.Compression package and passes its codec factory to the reader. The +core Apache.Arrow package decompresses neither and throws on any compressed body. +The two codecs ship together, on the pure-managed ZstdSharp.Port and K4os LZ4 +implementations, so there is no configuration where lz4 IPC works but zstd does +not. + +This splits into two cases. If the service takes that package, zstd IPC is +available, the IPC path keeps its smaller zstd wire size, and the compressor is +not by itself a reason to prefer Parquet. If the service cannot take it, +compressed IPC is unavailable at all, so IPC must travel uncompressed and far +larger than Parquet-zstd, which is the case that most favors precomputed Parquet. +Either way, the decode times in the tables above are arrow-rs in Rust, while the +.NET receiver decodes with the managed codecs, a separate cost this study does not +measure and one that precomputing Parquet lets the service avoid entirely. The +primary server-CPU argument does not depend on the compressor at all, because +precompute moves the Parquet encode itself off the server. ## Bottom line From e33d4e77251b0e43da268534dee502a4eb9b1258 Mon Sep 17 00:00:00 2001 From: Joshua MacDonald Date: Wed, 1 Jul 2026 10:34:43 -0700 Subject: [PATCH 12/18] docs(benchmarks): trim the .NET IPC compression tangent from the analysis The .NET receiver's IPC compression support changes arrow-ipc performance but not the conclusion, so replace the two-case discussion with a single note that the Parquet is written by arrow-rs in the sender and the server-CPU argument does not depend on the compressor. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../benches/otap_parquet/ANALYSIS.md | 24 ++++--------------- 1 file changed, 4 insertions(+), 20 deletions(-) diff --git a/rust/otap-dataflow/benchmarks/benches/otap_parquet/ANALYSIS.md b/rust/otap-dataflow/benchmarks/benches/otap_parquet/ANALYSIS.md index 9ce233141f..1118143974 100644 --- a/rust/otap-dataflow/benchmarks/benches/otap_parquet/ANALYSIS.md +++ b/rust/otap-dataflow/benchmarks/benches/otap_parquet/ANALYSIS.md @@ -264,26 +264,10 @@ than Parquet. The robust intake accepts both, OTAP/IPC for small senders and for traffic that needs a server-side transform, and precomputed Parquet for large gateways running the custom exporter. -The compressor question is really about the .NET receiver, because the Parquet -here is written by arrow-rs in the Rust sender and can always use zstd. For the -IPC path, the current Apache Arrow .NET library, as of version 23, can decompress -both lz4 and zstd record-batch buffers, but only when the service adds the -Apache.Arrow.Compression package and passes its codec factory to the reader. The -core Apache.Arrow package decompresses neither and throws on any compressed body. -The two codecs ship together, on the pure-managed ZstdSharp.Port and K4os LZ4 -implementations, so there is no configuration where lz4 IPC works but zstd does -not. - -This splits into two cases. If the service takes that package, zstd IPC is -available, the IPC path keeps its smaller zstd wire size, and the compressor is -not by itself a reason to prefer Parquet. If the service cannot take it, -compressed IPC is unavailable at all, so IPC must travel uncompressed and far -larger than Parquet-zstd, which is the case that most favors precomputed Parquet. -Either way, the decode times in the tables above are arrow-rs in Rust, while the -.NET receiver decodes with the managed codecs, a separate cost this study does not -measure and one that precomputing Parquet lets the service avoid entirely. The -primary server-CPU argument does not depend on the compressor at all, because -precompute moves the Parquet encode itself off the server. +The Parquet here is written by arrow-rs in the Rust sender, so its compression is +independent of the .NET receiver, and the primary server-CPU argument does not +depend on the compressor at all, because precompute moves the Parquet encode +itself off the server. ## Bottom line From 9ee7b269d610a7177998f1605bfe0597e53cfcf6 Mon Sep 17 00:00:00 2001 From: Joshua MacDonald Date: Wed, 1 Jul 2026 10:40:55 -0700 Subject: [PATCH 13/18] docs(benchmarks): keep the analysis logs-specific The study is logs-only, so drop the cross-signal asides: replace the metric re-aggregation paragraph with a logs-specific version about per-record work moving to the gateway, and drop the trace-id example from the dictionary discussion. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../benchmarks/benches/otap_parquet/ANALYSIS.md | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/rust/otap-dataflow/benchmarks/benches/otap_parquet/ANALYSIS.md b/rust/otap-dataflow/benchmarks/benches/otap_parquet/ANALYSIS.md index 1118143974..a7e3b6071c 100644 --- a/rust/otap-dataflow/benchmarks/benches/otap_parquet/ANALYSIS.md +++ b/rust/otap-dataflow/benchmarks/benches/otap_parquet/ANALYSIS.md @@ -162,7 +162,7 @@ This synthetic data is a best case for dictionary amortization, because every batch carries the identical low-cardinality values, so batches after the first send no new dictionary entries at all. Real telemetry amortizes only the stable low-cardinality columns, such as attribute keys, severity, and scope names. A -high-cardinality column such as a trace id or a log body carries new values in +high-cardinality column such as a log body carries new values in every batch, so its delta dictionary keeps sending new entries and does not amortize, and such a column is often better left non-dictionary. The measured amortization here should be read as the ceiling for dictionaries plus the schema, @@ -248,12 +248,11 @@ appends pays far less than the 13x figure, and that is the regime where acceptin client Parquet wins. Third, any work that needs the row values still forces a full decode, so the goal -is to move that work into the exporter. Per-record transforms and redaction move -cleanly to the gateway, since the gateway is the last writer. Cross-source metric -re-aggregation does not, because one gateway sees only its own slice, so temporal -rollups across gateways remain a server-side read. Logs and traces are therefore -the strong case for precomputed Parquet, while aggregated metrics are the residual -case that still wants OTAP/IPC and a server-side convert. +is to move that work into the exporter. For logs this is a clean fit, because the +per-record work, parsing, filtering, redaction, and attribute shaping, is all done +at the gateway, which is the last writer. Once it has run, the gateway holds the +final records and can write them straight to Parquet, leaving the ingestion +service to validate and append. Two operational costs remain. Because exporters run on customer-managed collectors, a storage-layout change cannot deploy atomically, so the ingestion From 452580ba9f27acb5a90e7d53e6d48c797e047598 Mon Sep 17 00:00:00 2001 From: Joshua MacDonald Date: Wed, 1 Jul 2026 14:35:12 -0700 Subject: [PATCH 14/18] feat(benchmarks): study OTAP-flat single columnar view and pipeline conversions Extend the otap_parquet study with an "OTAP-flat" intermediate: a single columnar view of OTAP logs that exploits the encoder emitting attributes grouped by parent_id, so the per-record log attributes are a zero-copy List and only the shared resource/scope attributes vary by layout. - parquet_study::otap_flat: flatten/unflatten across three shared-attribute layouts (Materialized, RunEndEncoded, Dictionary), plus in_memory_bytes. - parquet_study::ipc_flat: plain Arrow-IPC serialize/read of the flat batch, which unlike Parquet carries run-end and dictionary columns on the wire. - parquet_study::parquet_io::to_parquet_ready: transform only the columns arrow-rs cannot round-trip (expand REE->List, materialize dict-FixedSizeBinary trace_id/span_id) and copy the rest. - datagen::RichGenParams: realistic high-cardinality data (unique trace ids, mixed-type attributes) so wire-size comparisons are not flattered by identical rows. - New printed tables in main.rs: OTAP-flat view cost/size, service-to-service transfer (ipc-standard vs ipc-flat vs parquet), and a per-edge conversion-cost matrix. - OTAP_FLAT_ANALYSIS.md: companion write-up positioning OTAP-flat as a natural center between OTAP-standard and Parquet, with a column-level copy-vs-transform analysis, sort-order notes, and the arrow-rs REE/dict-FixedSizeBinary limits. Benchmark/analysis only; not user-facing. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../benches/otap_parquet/ANALYSIS.md | 6 + .../otap_parquet/OTAP_FLAT_ANALYSIS.md | 405 +++++++++++++ .../benchmarks/benches/otap_parquet/README.md | 82 ++- .../benchmarks/benches/otap_parquet/main.rs | 375 +++++++++++- .../benchmarks/src/parquet_study/attrs.rs | 2 +- .../benchmarks/src/parquet_study/datagen.rs | 181 ++++++ .../benchmarks/src/parquet_study/ipc_flat.rs | 84 +++ .../benchmarks/src/parquet_study/mod.rs | 2 + .../benchmarks/src/parquet_study/otap_flat.rs | 544 ++++++++++++++++++ .../src/parquet_study/parquet_io.rs | 102 +++- 10 files changed, 1770 insertions(+), 13 deletions(-) create mode 100644 rust/otap-dataflow/benchmarks/benches/otap_parquet/OTAP_FLAT_ANALYSIS.md create mode 100644 rust/otap-dataflow/benchmarks/src/parquet_study/ipc_flat.rs create mode 100644 rust/otap-dataflow/benchmarks/src/parquet_study/otap_flat.rs diff --git a/rust/otap-dataflow/benchmarks/benches/otap_parquet/ANALYSIS.md b/rust/otap-dataflow/benchmarks/benches/otap_parquet/ANALYSIS.md index a7e3b6071c..ca8c35a6b1 100644 --- a/rust/otap-dataflow/benchmarks/benches/otap_parquet/ANALYSIS.md +++ b/rust/otap-dataflow/benchmarks/benches/otap_parquet/ANALYSIS.md @@ -284,3 +284,9 @@ choices move the ratio by a large factor. And when one organization owns both th exporter and the store, the applied section above shows the convert step can move to the sending gateway, as long as ingestion stays close to a metadata-validated append rather than a full decode. + +The flatten step that this analysis treats as a fixed part of the Parquet cost is +itself examined in [`OTAP_FLAT_ANALYSIS.md`](./OTAP_FLAT_ANALYSIS.md), which shows +that most of it is avoidable because OTAP attribute batches are already grouped by +parent, and which measures how cheaply OTAP can be presented as a single columnar +view. diff --git a/rust/otap-dataflow/benchmarks/benches/otap_parquet/OTAP_FLAT_ANALYSIS.md b/rust/otap-dataflow/benchmarks/benches/otap_parquet/OTAP_FLAT_ANALYSIS.md new file mode 100644 index 0000000000..5dd3d00a09 --- /dev/null +++ b/rust/otap-dataflow/benchmarks/benches/otap_parquet/OTAP_FLAT_ANALYSIS.md @@ -0,0 +1,405 @@ + + +# Analysis: an efficient OTAP-flat single columnar view + +This is a companion to [`ANALYSIS.md`](./ANALYSIS.md). That study measured the +cost of moving OTAP logs as compressed Arrow IPC versus flattened Parquet, and +found that producing Parquet costs roughly an order of magnitude more CPU than +IPC. A large and compressor-independent part of that Parquet cost is the +`flatten` step, which turns OTAP's four normalized record batches into one +denormalized table before the Parquet writer ever runs. This document focuses on +that intermediate. It asks how cheaply OTAP can be presented as a single +columnar view, and it measures three ways to do it. It then asks whether that +flat view is also a good format to move between two services, and measures it +against OTAP-standard and Parquet on the wire. Finally it measures every +conversion in the pipeline edge by edge, to show that the flat view sits at a +natural center where each move to a neighbor rewrites only a handful of columns +and copies the rest. + +Numbers below are indicative medians in milliseconds and bytes from one +development machine running WSL with jemalloc. Both scenarios hold the record +count fixed at 60,000 logs and vary only the attribute mix, because the layouts +studied here differ only in how they treat attributes. + +## The flatten tax and where it comes from + +An OTAP logs batch is four Arrow record batches. The root `Logs` batch holds one +row per record with the scalar and struct columns. Three attribute batches, +`ResourceAttrs`, `ScopeAttrs`, and `LogAttrs`, hold the attributes, each linked +to its parent by a `parent_id` column. This is a normalized, relational shape +that is compact on the wire because a resource's attributes are stored once and +referenced by every log record under that resource. + +A single columnar view is the opposite shape. It is one table with one row per +log record, where each row can reach its resource, scope, and log attributes +without a join. Parquet needs this shape, and so does a query engine that wants +to scan the data as columns. The `flatten` step performs the join. The existing +flattened contenders build it with `gather_by_parent`, which constructs a +`HashMap>` for each attribute batch and then materializes every +value column with a random-access `take`. That hash join and full materialization +is the flatten tax. + +## The structural opportunity + +The OTAP encoder does not emit attributes in arbitrary order. It walks resources, +scopes, and records in order, and for each parent it appends that parent's +attributes contiguously before moving on. The result is that every attribute +batch arrives already grouped by `parent_id` in ascending, contiguous runs, and +the `Logs.id` column that keys the log attributes is a sequential, nullable +`u16`. A probe over generated data confirms it directly. `ResourceAttrs.parent_id` +is `0, 1, 2`, `ScopeAttrs.parent_id` is `0, 0, 1, 1, 2, 2`, and +`LogAttrs.parent_id` is a run of zeros, then a run of ones, and so on. + +This ordering makes the hash join unnecessary. Two facts follow from it. + +First, the per-record log attributes can become a `List` almost for free. +The struct children are the existing `LogAttrs` value columns used as they are, +with no `take`, and the list offsets come from a single linear scan that walks +the sorted `parent_id` runs alongside the sequential `Logs.id` column. Every log +attribute belongs to exactly one record, so no value is copied and no value is +shared. + +Second, the shared resource and scope attributes line up with runs of log rows, +because the records are already grouped by resource and then by scope. A resource +therefore spans a contiguous block of rows, which is exactly the structure that +run-end and dictionary encodings capture without physically repeating anything. + +## Three layouts of the shared attributes + +All three layouts studied here build the log attributes the same zero-copy way. +They differ only in how they present the shared resource and scope sets in the +single view. + +- Materialized repeats each set physically across its rows as a plain + `List`. This is the same shape the `parquet-nested` contender produces, + and it is the only one arrow-rs can write to Parquet. +- Run-end encoded stores each set once as `RunEndEncoded>`. The + view is logically per-row, and the physical storage holds one list per resource + or scope plus the run boundaries. +- Dictionary stores each set once as `Dictionary>`. The view + carries a per-row `u16` index into a small table that holds one list per + resource or scope. + +A spike settled how far each form travels. The arrow-rs Parquet writer at version +58.3 cannot serialize a run-end-encoded column, and it cannot serialize a +dictionary whose values are `List`. It reports the run-end case as not +supported and the nested-dictionary case as not yet implemented. A dictionary of +a primitive such as `Utf8` does write and round-trips as a dictionary. The +conclusion is that run-end and nested-dictionary views are in-memory and query +representations rather than on-disk ones. That fits a design where the live store +is Arrow-native and Parquet is an export and interchange format rather than the +serving format. + +## Measurements + +The data models realistic telemetry rather than identical rows. Every record +carries a unique pseudo-random `trace_id` and `span_id`, a varying timestamp, a +templated body, and mixed-type log attributes that blend low-cardinality enums +with high-cardinality per-record values. Resource and scope attributes are +distinct per resource or scope but shared across that resource's records. This +matters because a degenerate feed of identical records collapses under +dictionary and delta encoding and would flatter whichever format encodes best, +which distorts the comparison. + +The table reports the conversion cost from OTAP to a single record batch, the +in-memory footprint of that view, and the Parquet encode of the view where +arrow-rs can write it. The baseline is the existing nested flatten with its hash +join and full `take`. + +Log-heavy, 60,000 logs under one resource with one resource attribute, two scope +attributes, and nine attributes per record: + +| contender | convert ms | view mem | pq-write ms | pq bytes | writable | +| ---------------------- | ---------: | --------: | ----------: | --------: | -------- | +| nested, baseline | 74.1 | 41.8 MB | 236.0 | 3,863,351 | yes | +| otap-flat-materialized | 18.0 | 46.7 MB | 277.4 | 3,863,351 | yes | +| otap-flat-ree | 8.4 | 36.0 MB | n/a | n/a | no | +| otap-flat-dict | 8.9 | 36.2 MB | n/a | n/a | no | + +Resource-heavy, 60,000 logs across 600 resources with twenty attributes each, +five scope attributes, and two attributes per record: + +| contender | convert ms | view mem | pq-write ms | pq bytes | writable | +| ---------------------- | ---------: | --------: | ----------: | --------: | -------- | +| nested, baseline | 94.3 | 101.9 MB | 355.6 | 2,750,262 | yes | +| otap-flat-materialized | 86.8 | 101.9 MB | 380.0 | 2,750,262 | yes | +| otap-flat-ree | 4.4 | 12.7 MB | n/a | n/a | no | +| otap-flat-dict | 4.3 | 12.9 MB | n/a | n/a | no | + +## Reading the results + +There are two independent levers, and the two scenarios separate them. + +The first lever is the zero-copy log attributes, and the materialized layout +already captures it because it shares the same log-attribute path. In the +log-heavy scenario the record attributes dominate, and building them without a +hash join and without a full `take` cuts the conversion from 74 milliseconds to +18, about four times faster, while producing a byte-identical Parquet file. The +run-end and dictionary layouts go a little further in this scenario, to about 8 +and 9 milliseconds, because they also stop repeating the small resource and scope +sets, and that trims the in-memory view from 47 to 36 megabytes. + +The second lever is the shared attributes, and only the run-end and dictionary +layouts capture it. The resource-heavy scenario is where this shows. The +materialized layout has to repeat twenty resource attributes across every one of +the 60,000 rows, which is 1.2 million struct rows of pure duplication, so it is +no cheaper than the baseline at about 87 milliseconds and it holds a 102 megabyte +view. The run-end layout stores each resource's attributes once, 600 lists in +total, so it builds the same logical view in 4.4 milliseconds and holds it in +12.7 megabytes. That is about twenty times less conversion work and eight times +less memory for a view that answers the same per-row questions. The dictionary +layout is close behind at 4.3 milliseconds and 12.9 megabytes. + +The Parquet column is the counterpoint. The on-disk size is the same for the +materialized layout as for the baseline, about 3.9 and 2.8 megabytes, because the +Parquet writer applies its own run-length and dictionary encodings and recovers +the repetition that the materialized view spelled out in memory. The physical +duplication in the materialized layout is therefore a cost paid in build time and +peak memory, not in file size. Writing Parquet still requires that materialized +form, so the conversion cannot be avoided when Parquet is the target. It can only +be avoided when the consumer reads the columnar view directly. + +## Transferring between two services + +The measurements so far are about building the single view and holding it in +memory. A different question is whether the flat view is a good format to move +between two services, which is what OTAP-standard does today. To answer it the +study serializes each form the way it would travel. OTAP-standard is the +transport-optimized `Producer` over the four normalized batches. Each flat layout +is one record batch written as a plain Arrow IPC stream, which unlike Parquet can +carry run-end and dictionary columns. Parquet is the flat batch written as a +file. The table reports the compressed wire size under zstd and lz4, the encode +cost from an OTAP batch to wire bytes, and the decode cost from wire bytes to the +receiver's working form. + +Log-heavy: + +| contender | zstd wire | lz4 wire | encode ms | decode ms | receiver form | +| --------------------- | ---------: | ---------: | --------: | --------: | --------------- | +| ipc-standard | 4,479,026 | 6,442,420 | 84.3 | 12.6 | normalized OTAP | +| ipc-flat-materialized | 7,013,768 | 11,015,624 | 88.6 | 27.8 | flat table | +| ipc-flat-ree | 5,663,048 | 9,061,640 | 63.7 | 23.0 | flat table | +| ipc-flat-dict | 5,663,240 | 9,062,728 | 71.8 | 20.4 | flat table | +| parquet-flat | 3,863,351 | 5,777,270 | 238.1 | 57.9 | flat table | + +Resource-heavy: + +| contender | zstd wire | lz4 wire | encode ms | decode ms | receiver form | +| --------------------- | ---------: | ---------: | --------: | --------: | --------------- | +| ipc-standard | 3,013,172 | 3,841,908 | 28.0 | 5.6 | normalized OTAP | +| ipc-flat-materialized | 12,137,288 | 17,397,896 | 153.3 | 65.4 | flat table | +| ipc-flat-ree | 3,345,992 | 4,736,200 | 15.1 | 6.5 | flat table | +| ipc-flat-dict | 3,345,544 | 4,737,672 | 15.5 | 6.3 | flat table | +| parquet-flat | 2,750,262 | 3,783,505 | 416.4 | 172.6 | flat table | + +On realistic data the wire sizes land close together, and the ordering is +Parquet first, then OTAP-standard, then the run-end flat form, with the +materialized flat form well behind. Parquet is the smallest on the wire, about +3.9 megabytes log-heavy and 2.8 resource-heavy, because it applies run-length and +dictionary encoding to every column. OTAP-standard is next and close, about 4.5 +and 3.0 megabytes, because it never denormalizes the shared attributes and +transport-optimizes ids and values. The run-end flat form is close behind again, +about 5.7 and 3.3 megabytes. Only the materialized flat form is far off, 7.0 and +12.1 megabytes, because it repeats the shared resource attributes on the wire and +Arrow IPC does not fold that repetition away the way Parquet does. + +This is the reconciliation with the companion analysis, which found Parquet +smaller on the wire than OTAP-standard for a single large batch. The same holds +here, by a smaller margin because the high-cardinality per-record data sets a +floor that both formats carry. An earlier pass of this study used identical +records and reported OTAP-standard many times smaller than any flat form, which +was an artifact of that degenerate data, not a real result. With realistic +records the four forms are within about a factor of two on the wire, apart from +the materialized flat form. + +The CPU picture is where the forms separate. OTAP-standard and the run-end flat +form are the cheap pair. OTAP-standard decodes fastest, about 6 to 13 +milliseconds, because it is a light Arrow IPC deserialize plus a transport +decode. The run-end flat form is often the cheapest to encode, and in the +resource-heavy scenario it encodes in 15 milliseconds against OTAP-standard's 28, +because it only flattens with a run-end layout and writes plain Arrow IPC, +whereas OTAP-standard pays for its transport optimization. Parquet is the +expensive outlier on both ends, 238 to 416 milliseconds to encode and 58 to 173 +to decode, five to fifteen times the others, which is the same result the +companion analysis reported. + +Two caveats remain. The per-record trace ids here are unique, which is the +high-cardinality end for logs; correlated logs that share a trace id across a +span would compress somewhat better and lower every wire number together. And to +write the flat batch to Parquet the study first materializes the encoder's +dictionary columns, because arrow-rs 58.3 cannot read a dictionary-encoded +`FixedSizeBinary` such as `trace_id` back from Parquet. Arrow IPC has no such +limit and carries those dictionaries directly. + +The reason this matters for the original hypothesis is that a flat wire format is +not obviously worse. The run-end flat form is within about a quarter of +OTAP-standard on the wire, is sometimes cheaper to encode, and hands the receiver +a query-ready table with no projection step. Against that, OTAP-standard is +slightly smaller and decodes fastest and yields the normalized form, and Parquet +is the smallest at rest but by far the most expensive to produce and consume. So +the choice is a real trade rather than a rout. If the receiver wants columns and +values CPU, shipping the run-end flat form is defensible. If it wants the smallest +wire or the normalized model, or the cheapest decode, OTAP-standard is the better +default, and the flat view is then a cheap projection at the consumer. + +## OTAP-flat as the natural center + +The pipeline this study considers has five representations. OTAP-standard is the +four normalized batches, written `S`. OTAP-flat with run-end shared attributes is +the single batch, written `F`. Their serialized forms are OTAP/IPC-standard `Ws` +and OTAP/IPC-flat `Wf`, and the storage form is Parquet `P`. The question is what +it costs to move between them, and how much of each move is a genuine transform +rather than a copy of columns that do not change. + +Each directed edge, timed in isolation: + +| edge | log-heavy | resource-heavy | +| ---------------------------------- | --------: | -------------: | +| S -> F flatten to REE | 6.1 | 1.7 | +| F -> S unflatten | 12.9 | 2.9 | +| S -> Ws standard serialize | 62.9 | 25.8 | +| Ws -> S standard deserialize | 9.9 | 4.5 | +| F -> Wf flat serialize | 37.0 | 12.2 | +| Wf -> F flat deserialize | 16.1 | 6.4 | +| F -> P parquet-ready (REE+FSB) | 13.8 | 58.7 | +| F -> P parquet write | 144.9 | 242.8 | +| P -> F parquet read | 52.8 | 140.1 | + +Absolute milliseconds on this shared machine vary by up to about a factor of two +between runs, so the matrix should be read for the relative cost of the edges +rather than exact values. The relationships are stable. The two OTAP forms are +the cheapest pair to move between, a few milliseconds each way. Serializing +standard to its wire form is the most expensive of the Arrow IPC edges because it +runs the transport optimization, while serializing flat is lighter because it +only compresses. Parquet is the heavy end on both write and read. + +### What actually changes, column by column + +The flat table has thirteen columns: ten that come straight from the standard +`Logs` batch, which are `id`, `resource`, `scope`, the two timestamps, +`trace_id`, `span_id`, the two severity columns, and `body`, plus the three +attribute columns `resource_attributes`, `scope_attributes`, and +`log_attributes`. Classifying each conversion by how many of the thirteen it +copies unchanged versus how many it must rewrite: + +| conversion | copied | transformed | what changes | +| ----------- | ------: | -----------: | ---------------------------------------------------------- | +| S <-> F | 10 | 3 | attribute containers: keyed batches to REE/List columns | +| F <-> P | 9 | 4 | trace_id/span_id dict to plain; resource/scope REE to List | +| S <-> Ws | 0 | 13 | transport-optimize sort, delta, dict, then compress | +| F <-> Wf | 13 | 0 | buffer compression and framing only | + +`S` to `F` copies the ten `Logs` columns verbatim and only builds the three +attribute containers, wrapping the resource and scope batches as run-end columns +and the log attributes as a list. `F` to `P` copies nine columns, including the +dictionary `Utf8` and `Int32` columns and the log-attribute list, all of which +Parquet round-trips, and rewrites only four: it expands the two run-end columns +to plain lists and materializes `trace_id` and `span_id` from dictionary to +plain. That last pair is forced by the reader, not the writer, since arrow-rs can +write a dictionary of `FixedSizeBinary` but cannot read one back. The two +serialize edges are the opposite extremes. Standard to its wire form rewrites +every column, because the transport optimization sorts each batch and delta and +dictionary encodes it before compression. Flat to its wire form rewrites none of +them structurally, because plain Arrow IPC carries the run-end and dictionary +encodings as they are and only compresses the buffers. + +### Why flat is the center + +Put together, `F` sits between `S` and `P` and is a short hop from each. It shares +its ten `Logs` columns with `S`, so reaching it from `S` is a copy plus three +small container builds. It shares its whole schema with `P`, so reaching `P` from +`F` is a copy plus four column rewrites plus the writer. And its own wire form is +a light compress rather than a re-encode. No single move rewrites more than a +handful of columns, and the bulk of every move is memory sharing. That is what it +means for a representation to be a natural center. It is close to standard on one +side and close to Parquet on the other, and the conversions in every direction +touch only the columns that genuinely differ between the encodings. + +### Where sort order and future support change the picture + +One transform in the matrix is not cheap. Expanding the run-end resource and +scope columns for Parquet costs about 59 milliseconds in the resource-heavy +scenario, because it re-materializes the repetition that run-end encoding had +folded away. This is the memory and wire saving being paid back at export time. +Two changes would remove it. If arrow-rs learns to write run-end columns to +Parquet, the expansion becomes a passthrough from run-end runs to Parquet +run-length pages, and if its reader learns to read a dictionary of +`FixedSizeBinary`, the `trace_id` and `span_id` materialization disappears too. +At that point `F` to `P` is nearly all copy plus the writer. An asserted sort +order compounds this, because a flat batch that declares its order lets the +Parquet writer skip its own sort and produces meaningful row-group statistics, +and it keeps the run-end runs maximal. The sort is optional, and its value grows +after a merge or shuffle where the natural resource clustering has been broken. + +## What this means for the pipeline + +The applied section of the companion analysis argued that when the gateway owns +both the exporter and the store, the OTAP to Parquet conversion can move to the +sending gateway. This study refines where the remaining cost lives and when it is +avoidable. + +If the target is a Parquet file, the materialized single view is the right +intermediate, and the win available is the zero-copy log-attribute build, which +removed about three quarters of the conversion time in the log-heavy case without +changing the output. The resource and scope repetition still has to be written +out for the Parquet writer to consume, though the file it produces is no larger +for it. + +If the target is an Arrow-native store answering queries, which is the serving +path for this system, the run-end or dictionary single view is dramatically +cheaper to build and to hold, and the advantage grows with the weight of the +shared attributes. Real resource attributes in production telemetry are numerous +and highly shared, so the resource-heavy scenario is the representative one for a +host or a gateway that aggregates many records under a small number of resources. +In that regime the run-end view is the most efficient single columnar +presentation of OTAP measured here, because it never materializes a value that +OTAP already stored once. + +## Caveats and limits + +The zero-copy log-attribute build relies on the encoder emitting attributes +grouped by `parent_id`, which the current OTAP producer does and which the probe +confirmed. A path that reordered or interleaved attributes would need a stable +sort first, which would add a pass but would still avoid the hash join. The key +column is normalized from its dictionary encoding to plain `Utf8`, so that one +column is cast rather than shared, while the other value columns are shared +unchanged. + +The generated data models realistic telemetry with unique per-record ids and +mixed-type attributes, which is what keeps the wire comparison honest, since +identical rows collapse under dictionary and delta encoding and mislead. The wire +magnitudes still depend on cardinality: correlated logs that share a trace id +would compress better and lower every number together, so the comparison should +be read as one point on a spectrum rather than a fixed ratio. + +Two arrow-rs 58.3 limits shape the results. The Parquet writer cannot serialize +run-end or nested-dictionary columns, so those layouts are in-memory forms only. +The Parquet reader cannot read a dictionary-encoded `FixedSizeBinary` such as +`trace_id` back, so the study materializes the encoder's dictionary columns +before writing Parquet. Both may change as arrow-rs adds support, at which point +the run-end view could also become a Parquet write target. + +## Bottom line + +OTAP attribute batches are already grouped by parent, so presenting them as a +single columnar view does not need a hash join. Building the log attributes zero +copy makes the materialized view, which is the one Parquet can write, about four +times cheaper to produce for log-heavy data while leaving the file identical. For +the shared resource and scope attributes, a run-end or dictionary view stores +each set once and is the most efficient single presentation, about twenty times +cheaper to build and eight times smaller in memory for resource-heavy data, at +the cost of not being directly writable to Parquet today. The choice follows the +consumer. Materialize for a Parquet file, and keep the run-end view for an +Arrow-native store that serves queries. + +For moving data between two services the answer is a genuine trade rather than a +rout. On realistic data the four forms sit within about a factor of two on the +wire, apart from the materialized flat form, which repeats shared attributes and +falls behind. Parquet is the smallest at rest but by far the most expensive to +produce and consume. OTAP-standard is close on size, decodes fastest, and yields +the normalized model. The run-end flat form is close again on size, is sometimes +cheaper to encode than OTAP-standard, and hands the receiver a query-ready table. +So a flat wire format is defensible when the receiver wants columns and values +CPU, while OTAP-standard remains the better default for the smallest wire, the +cheapest decode, or the normalized model, with the flat view computed as a cheap +projection at the consumer. diff --git a/rust/otap-dataflow/benchmarks/benches/otap_parquet/README.md b/rust/otap-dataflow/benchmarks/benches/otap_parquet/README.md index 4d682570fe..81ae87340d 100644 --- a/rust/otap-dataflow/benchmarks/benches/otap_parquet/README.md +++ b/rust/otap-dataflow/benchmarks/benches/otap_parquet/README.md @@ -54,14 +54,15 @@ rather than the deprecated Hadoop-framed `LZ4`. cargo bench -p benchmarks --bench otap_parquet ``` -Four tables are printed to stdout before the timed round-trip benchmarks run: +Seven tables are printed to stdout before the timed round-trip benchmarks run: serialized size, the OTAP/IPC pipeline breakdown, the Parquet pipeline breakdown, -and the OTAP/IPC streaming amortization. The breakdown shapes are 10k, 30k, and -60k log records. A single OTAP logs batch holds at most 65,535 records because -the log id is a `u16`, so the shapes stay below that and larger volumes are -streamed. The full Criterion sweep is slow; the printed tables are the main -output. For a quick pass add `-- --measurement-time 0.5 --sample-size 10`, or -read the tables and stop the run. +the OTAP-flat single columnar view study, the service-to-service transfer +comparison, the conversion-cost matrix, and the OTAP/IPC streaming amortization. +The breakdown shapes are 10k, 30k, and 60k log records. A single OTAP logs batch +holds at most 65,535 records because the log id is a `u16`, so the shapes stay +below that and larger volumes are streamed. The full Criterion sweep is slow; the +printed tables are the main output. For a quick pass add `-- --measurement-time +0.5 --sample-size 10`, or read the tables and stop the run. ## Pipeline steps @@ -128,8 +129,61 @@ dictionary cost that streaming amortizes: | 10,000 | zstd | 92,270 | 80,944 | 11,326 | 57,023 | 1.42x | | 50,000 | zstd | 401,966 | 390,640 | 11,326 | 240,211 | 1.63x | +OTAP-flat single columnar view. `convert` is the cost of turning the four OTAP +record batches into one, `view-mem` is the in-memory footprint of that view, and +`pq-write`/`pq-bytes` are the Parquet encode where arrow-rs can write it. The +`otap-flat-*` rows exploit the fact that OTAP attribute batches are already +grouped by `parent_id`, so the per-record log attributes are a zero-copy +`List` and only the layout of the shared resource/scope sets differs. The +resource-heavy shape is 60k logs across 600 resources with twenty attributes +each: + +| contender | convert ms | view mem | pq-write ms | pq bytes | writable | +| ---------------------- | ---------: | --------: | ----------: | --------: | -------- | +| nested, baseline | 94.3 | 101.9 MB | 355.6 | 2,750,262 | yes | +| otap-flat-materialized | 86.8 | 101.9 MB | 380.0 | 2,750,262 | yes | +| otap-flat-ree | 4.4 | 12.7 MB | n/a | n/a | no | +| otap-flat-dict | 4.3 | 12.9 MB | n/a | n/a | no | + +Service-to-service transfer. Each form is serialized the way it would travel: +OTAP-standard is the transport-optimized `Producer` over the normalized batches, +each flat layout is one Arrow IPC stream, and Parquet is the flat file. `zstd +wire` and `lz4 wire` are the compressed bytes, `encode`/`decode` are the CPU to +and from the wire, and `receiver form` is what the decoder yields. On realistic +data the forms sit within about a factor of two on the wire, apart from the +materialized flat form, and Parquet is smallest at rest but most expensive on +CPU: + +| contender | zstd wire | lz4 wire | encode ms | decode ms | receiver form | +| --------------------- | ---------: | ---------: | --------: | --------: | --------------- | +| ipc-standard | 3,013,172 | 3,841,908 | 28.0 | 5.6 | normalized OTAP | +| ipc-flat-materialized | 12,137,288 | 17,397,896 | 153.3 | 65.4 | flat table | +| ipc-flat-ree | 3,345,992 | 4,736,200 | 15.1 | 6.5 | flat table | +| ipc-flat-dict | 3,345,544 | 4,737,672 | 15.5 | 6.3 | flat table | +| parquet-flat | 2,750,262 | 3,783,505 | 416.4 | 172.6 | flat table | + +Conversion-cost matrix. Every directed edge of the pipeline, timed in isolation, +where `S` is OTAP-standard, `F` is OTAP-flat/REE, `Ws`/`Wf` are their Arrow IPC +wire forms, and `P` is Parquet. The two OTAP forms are the cheapest pair to move +between, because they share ten of the flat table's thirteen columns and only the +attribute containers change: + +| edge | log-heavy | resource-heavy | +| ---------------------------------- | --------: | -------------: | +| S -> F flatten to REE | 6.1 | 1.7 | +| F -> S unflatten | 12.9 | 2.9 | +| S -> Ws standard serialize | 62.9 | 25.8 | +| Ws -> S standard deserialize | 9.9 | 4.5 | +| F -> Wf flat serialize | 37.0 | 12.2 | +| Wf -> F flat deserialize | 16.1 | 6.4 | +| F -> P parquet-ready (REE+FSB) | 13.8 | 58.7 | +| F -> P parquet write | 144.9 | 242.8 | +| P -> F parquet read | 52.8 | 140.1 | + A companion write-up of what these ratios mean, including the streaming effect, -is in [`ANALYSIS.md`](./ANALYSIS.md). +is in [`ANALYSIS.md`](./ANALYSIS.md). The OTAP-flat single columnar view, the +transfer comparison, and the natural-center conversion analysis are in +[`OTAP_FLAT_ANALYSIS.md`](./OTAP_FLAT_ANALYSIS.md). ## Takeaways @@ -173,3 +227,15 @@ measures streaming amortization; the Parquet steps are `Scheme::flatten`, Input shapes are defined in `input_shapes()` and `streaming_shapes()` in `benches/otap_parquet/main.rs`. The round-trips have unit tests runnable with `cargo test -p benchmarks --lib parquet_study`. + +The OTAP-flat single columnar view lives in `parquet_study::otap_flat`. Its +`flatten` and `unflatten` take a `Layout` of `Materialized`, `RunEndEncoded`, or +`Dictionary`, and `in_memory_bytes` reports the view footprint. The two attribute +mixes it sweeps are defined by `RichGenParams` in `parquet_study::datagen`. The +service-to-service transfer table serializes each flat layout as plain Arrow IPC +with `parquet_study::ipc_flat`, which unlike Parquet can carry run-end and +dictionary columns on the wire. `parquet_study::parquet_io::to_parquet_ready` +prepares a flat batch for Parquet by expanding run-end columns and materializing +the dictionary `FixedSizeBinary` `trace_id`/`span_id`, the only two encodings +arrow-rs cannot round-trip, and the conversion-cost matrix in `main.rs` times +every pipeline edge in isolation. diff --git a/rust/otap-dataflow/benchmarks/benches/otap_parquet/main.rs b/rust/otap-dataflow/benchmarks/benches/otap_parquet/main.rs index 94a462c380..3d1491c404 100644 --- a/rust/otap-dataflow/benchmarks/benches/otap_parquet/main.rs +++ b/rust/otap-dataflow/benchmarks/benches/otap_parquet/main.rs @@ -35,8 +35,9 @@ use std::hint::black_box; use std::time::{Duration, Instant}; use benchmarks::parquet_study::datagen::{LogsGenParams, gen_logs_otap}; -use benchmarks::parquet_study::parquet_io::{read_parquet, write_parquet}; -use benchmarks::parquet_study::{Compressor, Scheme, ipc}; +use benchmarks::parquet_study::otap_flat::{self, Layout}; +use benchmarks::parquet_study::parquet_io::{read_parquet, to_parquet_ready, write_parquet}; +use benchmarks::parquet_study::{Compressor, Scheme, ipc, ipc_flat}; #[cfg(not(windows))] use tikv_jemallocator::Jemalloc; @@ -220,6 +221,373 @@ fn print_parquet_breakdown(shapes: &[LogsGenParams]) { } } +/// OTAP-flat study: the cost of presenting the four OTAP record batches as a +/// single columnar view, and the size of that view, for three layouts of the +/// shared resource/scope attributes. `nested` is the baseline flatten (hash join +/// plus full `take`); the `otap-flat-*` rows exploit the fact that OTAP +/// attribute batches are already grouped by `parent_id`, so the per-row log +/// attributes are a zero-copy `List` and only the layout of the shared +/// resource/scope sets differs. `convert` is OTAP -> single view; `view-mem` is +/// the in-memory footprint of the view; `pq-write`/`pq-bytes` are the Parquet +/// encode of the view when arrow-rs can write it (materialized only -- REE and +/// dictionary-of-`List` are in-memory/query forms). +/// +/// Two scenarios hold the record count fixed and vary the attribute mix, because +/// REE and dictionary only save on the *shared* resource/scope attributes: +/// +/// - `log-heavy`: many per-record log attributes, few resource/scope attributes. +/// - `resource-heavy`: many resources each with many attributes, few log +/// attributes, which is where storing the shared sets once pays off. +fn print_otap_flat_table() { + use benchmarks::parquet_study::datagen::{RichGenParams, gen_logs_otap_rich}; + + let scenarios = [ + RichGenParams { + label: "log-heavy", + num_resources: 1, + num_scopes: 1, + num_logs: 60_000, + num_resource_attrs: 1, + num_scope_attrs: 2, + num_log_attrs: 9, + }, + RichGenParams { + label: "resource-heavy", + num_resources: 600, + num_scopes: 1, + num_logs: 100, + num_resource_attrs: 20, + num_scope_attrs: 5, + num_log_attrs: 2, + }, + ]; + + println!("\n=== OTAP-flat single columnar view (indicative ms, bytes) ==="); + println!("convert = OTAP -> one RecordBatch; view-mem = in-memory footprint of the view."); + println!("pq-write/pq-bytes use zstd; REE and dict cannot be written to Parquet by arrow-rs."); + println!( + "{:<24} {:>10} {:>12} {:>10} {:>12} {:>6}", + "contender", "convert-ms", "view-mem", "pq-write", "pq-bytes", "pq-ok" + ); + for scenario in &scenarios { + let otap = gen_logs_otap_rich(scenario); + println!( + "-- {} : {} logs, {} resources x {} resource-attrs, {} log-attrs --", + scenario.label, + scenario.total_logs(), + scenario.num_resources, + scenario.num_resource_attrs, + scenario.num_log_attrs, + ); + + // Baseline: the existing nested flatten (hash join + full take). + let base_convert = median_ms(|| { + let _ = Scheme::Nested.flatten(&otap).expect("nested flatten"); + }); + let base_flat = Scheme::Nested.flatten(&otap).expect("nested flatten"); + let base_mem = otap_flat::in_memory_bytes(&base_flat); + let base_pq_input = to_parquet_ready(&base_flat).expect("parquet-ready"); + let base_pq_ms = median_ms(|| { + let _ = write_parquet(&base_pq_input, Compressor::Zstd.parquet()).expect("write"); + }); + let base_pq = write_parquet(&base_pq_input, Compressor::Zstd.parquet()) + .expect("write") + .len(); + println!( + "{:<24} {:>10.2} {:>12} {:>10.2} {:>12} {:>6}", + "nested (baseline)", base_convert, base_mem, base_pq_ms, base_pq, "yes" + ); + + for layout in [ + Layout::Materialized, + Layout::RunEndEncoded, + Layout::Dictionary, + ] { + let convert = median_ms(|| { + let _ = otap_flat::flatten(&otap, layout).expect("flatten"); + }); + let flat = otap_flat::flatten(&otap, layout).expect("flatten"); + let mem = otap_flat::in_memory_bytes(&flat); + if layout.parquet_writable() { + let pq_input = to_parquet_ready(&flat).expect("parquet-ready"); + let pq_ms = median_ms(|| { + let _ = write_parquet(&pq_input, Compressor::Zstd.parquet()).expect("write"); + }); + let pq_bytes = write_parquet(&pq_input, Compressor::Zstd.parquet()) + .expect("write") + .len(); + println!( + "{:<24} {:>10.2} {:>12} {:>10.2} {:>12} {:>6}", + layout.name(), + convert, + mem, + pq_ms, + pq_bytes, + "yes" + ); + } else { + println!( + "{:<24} {:>10.2} {:>12} {:>10} {:>12} {:>6}", + layout.name(), + convert, + mem, + "n/a", + "n/a", + "no" + ); + } + } + println!(); + } +} + +/// Transfer study: how OTAP-flat compares to OTAP-standard and to Parquet when +/// the question is moving data between two large services. Each row reports the +/// serialized wire size under zstd and lz4, the encode cost from an OTAP batch to +/// wire bytes, the decode cost from wire bytes to the receiver's working form, +/// and what that working form is. +/// +/// - `ipc-standard` is the OTAP representation today: the transport-optimized +/// `Producer` over the four normalized batches, decoded back to normalized +/// OTAP. +/// - `ipc-flat-*` is one flat record batch serialized as plain Arrow IPC. Arrow +/// IPC can carry `RunEndEncoded` and dictionary columns, so the compact +/// resource/scope layouts survive on the wire. The receiver gets a single +/// query-ready table; projecting it back to normalized OTAP is an extra +/// `otap_flat::unflatten` that this table does not include. +/// - `parquet-flat` is the same flat batch written as a Parquet file. +/// +/// encode and decode are timed with zstd. +fn print_transfer_table() { + use benchmarks::parquet_study::datagen::{RichGenParams, gen_logs_otap_rich}; + + let scenarios = [ + RichGenParams { + label: "log-heavy", + num_resources: 1, + num_scopes: 1, + num_logs: 60_000, + num_resource_attrs: 1, + num_scope_attrs: 2, + num_log_attrs: 9, + }, + RichGenParams { + label: "resource-heavy", + num_resources: 600, + num_scopes: 1, + num_logs: 100, + num_resource_attrs: 20, + num_scope_attrs: 5, + num_log_attrs: 2, + }, + ]; + + println!("\n=== Transfer between two services: wire size and CPU (indicative) ==="); + println!("encode = OTAP batch -> wire bytes; decode = wire bytes -> receiver working form."); + println!("ipc-flat carries REE/dict on the wire (Parquet cannot); decode yields one table."); + println!( + "{:<22} {:>12} {:>12} {:>10} {:>10} {:<18}", + "contender", "zstd-bytes", "lz4-bytes", "encode-ms", "decode-ms", "receiver-form" + ); + for scenario in &scenarios { + let otap = gen_logs_otap_rich(scenario); + println!( + "-- {} : {} logs, {} resources x {} resource-attrs, {} log-attrs --", + scenario.label, + scenario.total_logs(), + scenario.num_resources, + scenario.num_resource_attrs, + scenario.num_log_attrs, + ); + + // OTAP standard: transport-optimized Producer over the normalized batches. + { + let zstd_bytes = ipc::encode_to_bytes(otap.clone(), Compressor::Zstd) + .expect("encode") + .len(); + let lz4_bytes = ipc::encode_to_bytes(otap.clone(), Compressor::Lz4) + .expect("encode") + .len(); + let encode_ms = median_ms(|| { + let _ = ipc::encode_to_bytes(otap.clone(), Compressor::Zstd).expect("encode"); + }); + let bytes = ipc::encode_to_bytes(otap.clone(), Compressor::Zstd).expect("encode"); + let decode_ms = median_ms(|| { + let mut o = ipc::deserialize(&bytes).expect("deserialize"); + ipc::transport_decode(&mut o).expect("transport decode"); + }); + println!( + "{:<22} {:>12} {:>12} {:>10.2} {:>10.2} {:<18}", + "ipc-standard", zstd_bytes, lz4_bytes, encode_ms, decode_ms, "normalized OTAP" + ); + } + + // OTAP flat: one Arrow IPC batch, three shared-attribute layouts. + for layout in [ + Layout::Materialized, + Layout::RunEndEncoded, + Layout::Dictionary, + ] { + let flat = otap_flat::flatten(&otap, layout).expect("flatten"); + let zstd_bytes = ipc_flat::write_ipc(&flat, Compressor::Zstd) + .expect("write ipc") + .len(); + let lz4_bytes = ipc_flat::write_ipc(&flat, Compressor::Lz4) + .expect("write ipc") + .len(); + let encode_ms = median_ms(|| { + let f = otap_flat::flatten(&otap, layout).expect("flatten"); + let _ = ipc_flat::write_ipc(&f, Compressor::Zstd).expect("write ipc"); + }); + let bytes = ipc_flat::write_ipc(&flat, Compressor::Zstd).expect("write ipc"); + let decode_ms = median_ms(|| { + let _ = ipc_flat::read_ipc(&bytes).expect("read ipc"); + }); + let name = format!("ipc-flat-{}", layout_suffix(layout)); + println!( + "{:<22} {:>12} {:>12} {:>10.2} {:>10.2} {:<18}", + name, zstd_bytes, lz4_bytes, encode_ms, decode_ms, "flat table" + ); + } + + // Parquet: the same flat batch as a Parquet file. Arrow dictionaries are + // materialized first because arrow-rs cannot read a dictionary-encoded + // FixedSizeBinary (trace_id/span_id) back from Parquet. + { + let flat = Scheme::Nested.flatten(&otap).expect("nested flatten"); + let pq_input = to_parquet_ready(&flat).expect("parquet-ready"); + let zstd_bytes = write_parquet(&pq_input, Compressor::Zstd.parquet()) + .expect("write") + .len(); + let lz4_bytes = write_parquet(&pq_input, Compressor::Lz4.parquet()) + .expect("write") + .len(); + let encode_ms = median_ms(|| { + let f = Scheme::Nested.flatten(&otap).expect("nested flatten"); + let d = to_parquet_ready(&f).expect("parquet-ready"); + let _ = write_parquet(&d, Compressor::Zstd.parquet()).expect("write"); + }); + let bytes = write_parquet(&pq_input, Compressor::Zstd.parquet()).expect("write"); + let decode_ms = median_ms(|| { + let _ = read_parquet(&bytes).expect("read"); + }); + println!( + "{:<22} {:>12} {:>12} {:>10.2} {:>10.2} {:<18}", + "parquet-flat", zstd_bytes, lz4_bytes, encode_ms, decode_ms, "flat table" + ); + } + println!(); + } +} + +/// Short suffix for a shared-attribute layout, used in transfer row names. +fn layout_suffix(layout: Layout) -> &'static str { + match layout { + Layout::Materialized => "materialized", + Layout::RunEndEncoded => "ree", + Layout::Dictionary => "dict", + } +} + +/// Conversion-cost matrix: every directed edge of the pipeline, timed in +/// isolation, so the cost of moving between representations is visible edge by +/// edge. The nodes are OTAP-standard (`S`, four normalized batches), +/// OTAP-flat/REE (`F`, one batch with run-end resource/scope columns), +/// OTAP/IPC-standard (`Ws`), OTAP/IPC-flat (`Wf`), and Parquet (`P`). The flat to +/// Parquet edge is split into the parquet-ready transform, which expands the +/// run-end columns and materializes `trace_id`/`span_id`, and the Parquet write. +fn print_conversion_matrix() { + use benchmarks::parquet_study::datagen::{RichGenParams, gen_logs_otap_rich}; + use otap_df_pdata::otap::OtapArrowRecords; + + let scenarios = [ + RichGenParams { + label: "log-heavy", + num_resources: 1, + num_scopes: 1, + num_logs: 60_000, + num_resource_attrs: 1, + num_scope_attrs: 2, + num_log_attrs: 9, + }, + RichGenParams { + label: "resource-heavy", + num_resources: 600, + num_scopes: 1, + num_logs: 100, + num_resource_attrs: 20, + num_scope_attrs: 5, + num_log_attrs: 2, + }, + ]; + + let edges = [ + "S -> F flatten to REE", + "F -> S unflatten", + "S -> Ws standard serialize", + "Ws -> S standard deserialize", + "F -> Wf flat serialize", + "Wf -> F flat deserialize", + "F -> P parquet-ready (REE+FSB)", + "F -> P parquet write", + "P -> F parquet read", + ]; + + fn measure(otap: &OtapArrowRecords) -> Vec { + let flat = otap_flat::flatten(otap, Layout::RunEndEncoded).expect("flatten"); + let ws = ipc::encode_to_bytes(otap.clone(), Compressor::Zstd).expect("ws"); + let wf = ipc_flat::write_ipc(&flat, Compressor::Zstd).expect("wf"); + let ready = to_parquet_ready(&flat).expect("ready"); + let pq = write_parquet(&ready, Compressor::Zstd.parquet()).expect("pq"); + + vec![ + median_ms(|| { + let _ = otap_flat::flatten(otap, Layout::RunEndEncoded).expect("flatten"); + }), + median_ms(|| { + let _ = otap_flat::unflatten(&flat).expect("unflatten"); + }), + median_ms(|| { + let _ = ipc::encode_to_bytes(otap.clone(), Compressor::Zstd).expect("ws"); + }), + median_ms(|| { + let mut o = ipc::deserialize(&ws).expect("des"); + ipc::transport_decode(&mut o).expect("tdec"); + }), + median_ms(|| { + let _ = ipc_flat::write_ipc(&flat, Compressor::Zstd).expect("wf"); + }), + median_ms(|| { + let _ = ipc_flat::read_ipc(&wf).expect("rf"); + }), + median_ms(|| { + let _ = to_parquet_ready(&flat).expect("ready"); + }), + median_ms(|| { + let _ = write_parquet(&ready, Compressor::Zstd.parquet()).expect("pq"); + }), + median_ms(|| { + let _ = read_parquet(&pq).expect("read"); + }), + ] + } + + println!("\n=== Conversion cost matrix (indicative ms) ==="); + println!( + "S=OTAP-standard, F=OTAP-flat(REE), Ws=OTAP/IPC-standard, Wf=OTAP/IPC-flat, P=Parquet." + ); + let log = measure(&gen_logs_otap_rich(&scenarios[0])); + let res = measure(&gen_logs_otap_rich(&scenarios[1])); + println!( + "{:<34} {:>12} {:>14}", + "edge", "log-heavy", "resource-heavy" + ); + for (i, name) in edges.iter().enumerate() { + println!("{:<34} {:>12.2} {:>14.2}", name, log[i], res[i]); + } + println!(); +} + /// OTAP/IPC streaming amortization: cold (first) versus warm (steady-state) /// per-batch size when a single long-lived Producer streams many batches, with /// the equivalent single Parquet file for reference. @@ -264,6 +632,9 @@ fn bench_round_trip(c: &mut Criterion) { print_size_table(&shapes); print_ipc_breakdown(&shapes); print_parquet_breakdown(&shapes); + print_otap_flat_table(); + print_transfer_table(); + print_conversion_matrix(); print_streaming_table(&streaming_shapes()); let mut write_group = c.benchmark_group("parquet_study/write"); diff --git a/rust/otap-dataflow/benchmarks/src/parquet_study/attrs.rs b/rust/otap-dataflow/benchmarks/src/parquet_study/attrs.rs index e2f04b8d2b..3bc21d5f5c 100644 --- a/rust/otap-dataflow/benchmarks/src/parquet_study/attrs.rs +++ b/rust/otap-dataflow/benchmarks/src/parquet_study/attrs.rs @@ -65,7 +65,7 @@ pub fn attr_struct_fields() -> Fields { /// The list element field used for the `List` attribute columns. #[must_use] -fn attr_list_element_field() -> Arc { +pub fn attr_list_element_field() -> Arc { Arc::new(Field::new( "item", DataType::Struct(attr_struct_fields()), diff --git a/rust/otap-dataflow/benchmarks/src/parquet_study/datagen.rs b/rust/otap-dataflow/benchmarks/src/parquet_study/datagen.rs index 5dac7cb7d6..dea29d41e3 100644 --- a/rust/otap-dataflow/benchmarks/src/parquet_study/datagen.rs +++ b/rust/otap-dataflow/benchmarks/src/parquet_study/datagen.rs @@ -157,6 +157,187 @@ pub fn gen_logs_otap(params: &LogsGenParams) -> (OtapArrowRecords, usize) { (otap, proto_bytes.len()) } +/// Shape of generated logs data with configurable attribute richness, used by +/// the OTAP-flat study to vary the ratio of *shared* resource/scope attributes +/// (which the REE and dictionary layouts store once) to *per-record* log +/// attributes (which every layout stores per row). +/// +/// The generated records model realistic telemetry rather than identical rows: +/// each record has a unique pseudo-random `trace_id`/`span_id`, a varying +/// timestamp, a templated body, and mixed-type log attributes that blend +/// low-cardinality enums with high-cardinality per-record values. Resource and +/// scope attributes are distinct per resource/scope but shared across that +/// resource/scope's records, which is the low-cardinality-shared case the +/// run-end and dictionary layouts target. +#[derive(Clone, Copy, Debug)] +pub struct RichGenParams { + /// A short label for tables. + pub label: &'static str, + /// Number of distinct resources. + pub num_resources: usize, + /// Number of scopes within each resource. + pub num_scopes: usize, + /// Number of log records within each scope. + pub num_logs: usize, + /// Attributes attached to each resource (distinct per resource, shared by + /// that resource's records). + pub num_resource_attrs: usize, + /// Attributes attached to each scope. + pub num_scope_attrs: usize, + /// Mixed-type attributes attached to each log record. + pub num_log_attrs: usize, +} + +impl RichGenParams { + /// Total number of log records produced. + #[must_use] + pub fn total_logs(&self) -> usize { + self.num_resources * self.num_scopes * self.num_logs + } +} + +/// splitmix64: a cheap deterministic mixer used to synthesize high-entropy, +/// unique-per-record trace ids and attribute values, so the generated data does +/// not compress like identical rows. +fn mix64(mut x: u64) -> u64 { + x = x.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = x; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) +} + +/// A unique, high-entropy 16-byte trace id for a global record index. +fn trace_id_for(idx: u64) -> Vec { + let mut b = Vec::with_capacity(16); + b.extend_from_slice(&mix64(idx).to_be_bytes()); + b.extend_from_slice(&mix64(idx ^ 0xDEAD_BEEF).to_be_bytes()); + b +} + +/// A unique 8-byte span id for a global record index. +fn span_id_for(idx: u64) -> Vec { + mix64(idx.wrapping_mul(0x100_0001)).to_be_bytes().to_vec() +} + +/// Distinct-per-resource, low-cardinality resource attributes. +fn resource_attrs_for(res_idx: usize, n: usize) -> Vec { + (0..n) + .map(|i| { + let value = if i == 0 { + format!("service-{res_idx}") + } else { + format!("res{res_idx}-attr{i}") + }; + KeyValue::new(format!("resource_attr{i}"), AnyValue::new_string(value)) + }) + .collect() +} + +/// Distinct-per-scope, low-cardinality scope attributes. +fn scope_attrs_for(scope_idx: usize, n: usize) -> Vec { + (0..n) + .map(|i| { + KeyValue::new( + format!("scope_attr{i}"), + AnyValue::new_string(format!("scope{scope_idx}-attr{i}")), + ) + }) + .collect() +} + +const HTTP_METHODS: [&str; 4] = ["GET", "POST", "PUT", "DELETE"]; + +/// Mixed-type log attributes for a global record index, blending +/// low-cardinality enums with high-cardinality per-record values. +fn log_attrs_for(idx: u64, n: usize) -> Vec { + (0..n) + .map(|i| { + let key = format!("log_attr{i}"); + let value = match i % 5 { + // low-cardinality enum string + 0 => AnyValue::new_string(HTTP_METHODS[(idx as usize + i) % 4].to_string()), + // high-cardinality int + 1 => AnyValue::new_int((mix64(idx + i as u64) % 1_000_000) as i64), + // high-cardinality double + 2 => AnyValue::new_double((idx as f64) * 1.5 + i as f64), + // bool + 3 => AnyValue::new_bool((idx as usize + i).is_multiple_of(2)), + // high-cardinality id-like string + _ => AnyValue::new_string(format!("id-{:016x}", mix64(idx.wrapping_add(i as u64)))), + }; + KeyValue::new(key, value) + }) + .collect() +} + +/// Build an OTLP [`LogsData`] with configurable per-scope/resource/log attribute +/// counts. Unlike [`create_logs_data`], attribute richness is a parameter and +/// each record carries realistic high-cardinality content so the study can +/// measure resource-heavy versus log-heavy telemetry without identical rows. +#[must_use] +pub fn create_logs_data_rich(params: &RichGenParams) -> LogsData { + let base_time = 1_700_000_000_000_000_000u64; + LogsData::new( + (0..params.num_resources) + .map(|res_idx| { + ResourceLogs::new( + Resource::build() + .attributes(resource_attrs_for(res_idx, params.num_resource_attrs)) + .dropped_attributes_count(0u32), + (0..params.num_scopes) + .map(|scope_idx| { + ScopeLogs::new( + InstrumentationScope::build() + .name("library") + .version("scopev1") + .attributes(scope_attrs_for(scope_idx, params.num_scope_attrs)) + .dropped_attributes_count(0u32) + .finish(), + (0..params.num_logs) + .map(|log_idx| { + let idx = ((res_idx * params.num_scopes + scope_idx) + * params.num_logs + + log_idx) + as u64; + LogRecord::build() + .time_unix_nano(base_time + idx * 1_000_000) + .observed_time_unix_nano( + base_time + idx * 1_000_000 + 500, + ) + .severity_number(SeverityNumber::Info) + .severity_text("Info") + .trace_id(trace_id_for(idx)) + .span_id(span_id_for(idx)) + .attributes(log_attrs_for(idx, params.num_log_attrs)) + .body(AnyValue::new_string(format!( + "request {idx} handled in {}ms", + idx % 500 + ))) + .finish() + }) + .collect::>(), + ) + }) + .collect::>(), + ) + }) + .collect::>(), + ) +} + +/// Generate an OTAP logs batch for a [`RichGenParams`] shape. +#[must_use] +pub fn gen_logs_otap_rich(params: &RichGenParams) -> OtapArrowRecords { + let logs_data = create_logs_data_rich(params); + let mut proto_bytes = Vec::new(); + logs_data + .encode(&mut proto_bytes) + .expect("can encode OTLP proto bytes"); + let view = RawLogsData::new(proto_bytes.as_ref()); + encode_logs_otap_batch(&view).expect("can encode OTAP logs batch") +} + #[cfg(test)] mod tests { use super::*; diff --git a/rust/otap-dataflow/benchmarks/src/parquet_study/ipc_flat.rs b/rust/otap-dataflow/benchmarks/src/parquet_study/ipc_flat.rs new file mode 100644 index 0000000000..815e4db0f2 --- /dev/null +++ b/rust/otap-dataflow/benchmarks/src/parquet_study/ipc_flat.rs @@ -0,0 +1,84 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +//! In-memory Arrow IPC read/write for a single flat record batch. +//! +//! This is the transport counterpart to [`parquet_io`](super::parquet_io). It +//! serializes an already-flattened OTAP-flat record batch as a compressed Arrow +//! IPC stream, which is how an "OTAP-flat" wire format would move between two +//! services. Unlike Parquet, Arrow IPC can serialize `RunEndEncoded` and +//! dictionary columns, so the compact resource/scope layouts survive on the +//! wire. +//! +//! Note this is plain Arrow IPC of one batch, not the OTAP [`Producer`] path, +//! which applies transport-optimized delta and dictionary encoding to the four +//! normalized batches. The [`ipc`](super::ipc) contender measures that OTAP +//! standard path; this measures the flat alternative. +//! +//! [`Producer`]: otap_df_pdata::encode::producer::Producer + +use arrow::array::RecordBatch; +use arrow::compute::concat_batches; +use arrow_ipc::reader::StreamReader; +use arrow_ipc::writer::{IpcWriteOptions, StreamWriter}; + +use super::{Compressor, StudyResult}; + +/// Serialize a single record batch as a compressed Arrow IPC stream in memory. +pub fn write_ipc(batch: &RecordBatch, compressor: Compressor) -> StudyResult> { + let options = IpcWriteOptions::default().try_with_compression(compressor.ipc())?; + let mut buf: Vec = Vec::with_capacity(4096); + { + let mut writer = StreamWriter::try_new_with_options(&mut buf, &batch.schema(), options)?; + writer.write(batch)?; + writer.finish()?; + } + Ok(buf) +} + +/// Decode an in-memory Arrow IPC stream into a single record batch. +pub fn read_ipc(bytes: &[u8]) -> StudyResult { + let reader = StreamReader::try_new(bytes, None)?; + let schema = reader.schema(); + let mut batches: Vec = Vec::new(); + for batch in reader { + batches.push(batch?); + } + match batches.len() { + 0 => Ok(RecordBatch::new_empty(schema)), + 1 => Ok(batches.into_iter().next().expect("len checked")), + _ => Ok(concat_batches(&schema, &batches)?), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::parquet_study::datagen::{LogsGenParams, gen_logs_otap}; + use crate::parquet_study::otap_flat::{Layout, flatten}; + + #[test] + fn ipc_flat_round_trips_every_layout() { + let params = LogsGenParams { + num_resources: 3, + num_scopes: 2, + num_logs: 6, + }; + let (otap, _) = gen_logs_otap(¶ms); + + for layout in [ + Layout::Materialized, + Layout::RunEndEncoded, + Layout::Dictionary, + ] { + let flat = flatten(&otap, layout).expect("flatten"); + for compressor in [Compressor::Zstd, Compressor::Lz4, Compressor::None] { + let bytes = write_ipc(&flat, compressor).expect("write ipc"); + assert!(!bytes.is_empty()); + let back = read_ipc(&bytes).expect("read ipc"); + assert_eq!(back.num_rows(), flat.num_rows(), "{}", layout.name()); + assert_eq!(back.schema(), flat.schema(), "{}", layout.name()); + } + } + } +} diff --git a/rust/otap-dataflow/benchmarks/src/parquet_study/mod.rs b/rust/otap-dataflow/benchmarks/src/parquet_study/mod.rs index ede1ac8f47..c1543a5f96 100644 --- a/rust/otap-dataflow/benchmarks/src/parquet_study/mod.rs +++ b/rust/otap-dataflow/benchmarks/src/parquet_study/mod.rs @@ -19,8 +19,10 @@ use otap_df_pdata::otap::OtapArrowRecords; pub mod attrs; pub mod datagen; pub mod ipc; +pub mod ipc_flat; pub mod map; pub mod nested; +pub mod otap_flat; pub mod parquet_io; pub mod server; pub mod wide; diff --git a/rust/otap-dataflow/benchmarks/src/parquet_study/otap_flat.rs b/rust/otap-dataflow/benchmarks/src/parquet_study/otap_flat.rs new file mode 100644 index 0000000000..4ea9976f6c --- /dev/null +++ b/rust/otap-dataflow/benchmarks/src/parquet_study/otap_flat.rs @@ -0,0 +1,544 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +//! Efficient "OTAP-flat": present the four OTAP logs record batches as a single +//! columnar view at minimal conversion cost. +//! +//! The flattened-Parquet contenders in [`nested`](super::nested) pay a large +//! "flatten tax" because [`gather_by_parent`](super::attrs::gather_by_parent) +//! builds a `HashMap>` per attribute batch and materializes every +//! value column with a random-access `take`. That work is avoidable: the OTAP +//! encoder emits each attribute batch already **grouped by `parent_id` in +//! ascending, contiguous runs** (`LogAttrs.parent_id = [0..,1..,2..]`, +//! `ResourceAttrs.parent_id = [0,1,2]`, `Logs.id` sequential). This module +//! exploits that ordering. +//! +//! All three variants share one move: the per-row **log attributes** become a +//! `List` whose struct children are the *existing* `LogAttrs` value +//! columns (zero-copy) and whose offsets come from a single linear scan of the +//! sorted `parent_id`. No hash join, no `take`. +//! +//! They differ only in how the *shared* resource and scope attributes (one set +//! per resource/scope, spanning a contiguous run of log rows) are presented: +//! +//! - [`Layout::Materialized`] repeats each set physically per row (a plain +//! `List`, the same shape [`nested`](super::nested) produces, so it is +//! directly Parquet-writable). +//! - [`Layout::RunEndEncoded`] stores each set once as `RunEndEncoded>` (logical per-row, physical one-per-resource). In-memory / +//! query form only: arrow-rs cannot write REE to Parquet. +//! - [`Layout::Dictionary`] stores each set once as `Dictionary>` (per-row u16 index + one list per resource). Also in-memory +//! only: arrow-rs cannot write a dictionary of `List` to Parquet. + +use std::collections::HashMap; +use std::sync::Arc; + +use arrow::array::{ + Array, ArrayRef, DictionaryArray, Int32Array, ListArray, RecordBatch, RunArray, StructArray, + UInt16Array, UInt32Array, +}; +use arrow::buffer::{OffsetBuffer, ScalarBuffer}; +use arrow::compute::take; +use arrow::datatypes::{DataType, Field, Int32Type, Schema, UInt16Type}; + +use otap_df_pdata::otap::OtapArrowRecords; +use otap_df_pdata::proto::opentelemetry::arrow::v1::ArrowPayloadType; + +use super::StudyResult; +use super::attrs::{ + LOG_ATTRS_COL, RESOURCE_ATTRS_COL, SCOPE_ATTRS_COL, as_list, attr_list_element_field, + attr_struct_fields, empty_attr_list_column, entries_dedup, entries_per_row, + extract_attr_value_arrays, logs_batch, logs_id, logs_resource_id, logs_scope_id, + rebuild_attr_batch, +}; + +/// How the shared resource/scope attribute sets are presented in the flat view. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Layout { + /// Physically repeat each set per log row (`List`). Parquet-writable. + Materialized, + /// `RunEndEncoded>`: one physical set per resource/scope. + RunEndEncoded, + /// `Dictionary>`: per-row index + one set per resource. + Dictionary, +} + +impl Layout { + /// Stable contender name. + #[must_use] + pub fn name(self) -> &'static str { + match self { + Layout::Materialized => "otap-flat-materialized", + Layout::RunEndEncoded => "otap-flat-ree", + Layout::Dictionary => "otap-flat-dict", + } + } + + /// Whether the arrow-rs Parquet writer can serialize this layout directly. + /// Only [`Layout::Materialized`] can; REE and dictionary-of-`List` + /// are in-memory / query representations (see module docs). + #[must_use] + pub fn parquet_writable(self) -> bool { + matches!(self, Layout::Materialized) + } +} + +/// Downcast an attribute batch's `parent_id` column to `UInt16Array`. +fn parent_id_u16(attr_batch: &RecordBatch) -> StudyResult { + let col = attr_batch + .column_by_name(otap_df_pdata::schema::consts::PARENT_ID) + .ok_or("attribute batch missing `parent_id`")?; + Ok(col + .as_any() + .downcast_ref::() + .ok_or("`parent_id` is not UInt16")? + .clone()) +} + +/// The eight-field attribute `Struct` over *all* rows of an attribute batch, +/// built zero-copy from its value columns (no `take`). +fn attr_struct_all(attr_batch: &RecordBatch) -> StudyResult { + let values = extract_attr_value_arrays(attr_batch)?; + Ok(StructArray::new(attr_struct_fields(), values, None)) +} + +/// Given an attribute batch sorted by `parent_id`, return a `List` with +/// one row per distinct parent (its contiguous run, zero-copy) plus the parent +/// id of each row. This is the compact "one set per resource/scope" form. +fn per_parent_lists(attr_batch: &RecordBatch) -> StudyResult<(ListArray, Vec)> { + let struct_all = attr_struct_all(attr_batch)?; + let parent = parent_id_u16(attr_batch)?; + let n = parent.len(); + + let mut offsets: Vec = vec![0]; + let mut parent_ids: Vec = Vec::new(); + if n > 0 { + let mut start = 0usize; + for i in 1..n { + if parent.value(i) != parent.value(i - 1) { + offsets.push(i32::try_from(i).expect("offset fits i32")); + parent_ids.push(parent.value(start)); + start = i; + } + } + offsets.push(i32::try_from(n).expect("offset fits i32")); + parent_ids.push(parent.value(start)); + } + + let list = ListArray::new( + attr_list_element_field(), + OffsetBuffer::new(ScalarBuffer::from(offsets)), + Arc::new(struct_all), + None, + ); + Ok((list, parent_ids)) +} + +/// The zero-copy per-row log-attribute `List` column. Struct children +/// are the `LogAttrs` value columns unchanged; offsets come from one linear scan +/// aligning the sorted `LogAttrs.parent_id` runs with the sequential, nullable +/// `Logs.id` column. No hash join, no `take`. +fn log_attr_list(otap: &OtapArrowRecords, logs: &RecordBatch) -> StudyResult { + let num_rows = logs.num_rows(); + let Some(attr_batch) = otap.get(ArrowPayloadType::LogAttrs) else { + return Ok(empty_attr_list_column(num_rows)); + }; + let struct_all = attr_struct_all(attr_batch)?; + let parent = parent_id_u16(attr_batch)?; + let log_id = logs_id(logs)?; + + // Run length of each distinct parent, in ascending parent order. + let n = parent.len(); + let mut run_lengths: Vec = Vec::new(); + if n > 0 { + let mut start = 0usize; + for i in 1..n { + if parent.value(i) != parent.value(i - 1) { + run_lengths.push(i32::try_from(i - start).expect("len fits i32")); + start = i; + } + } + run_lengths.push(i32::try_from(n - start).expect("len fits i32")); + } + + // Each log row with a valid id consumes the next run, in order (the encoder + // assigns ids sequentially in the same order it emits the runs). + let mut offsets: Vec = Vec::with_capacity(num_rows + 1); + offsets.push(0); + let mut acc = 0i32; + let mut next_run = 0usize; + for i in 0..num_rows { + if log_id.is_valid(i) { + acc += run_lengths.get(next_run).copied().unwrap_or(0); + next_run += 1; + } + offsets.push(acc); + } + debug_assert_eq!( + next_run, + run_lengths.len(), + "every LogAttrs run should map to exactly one log row with an id" + ); + + Ok(Arc::new(ListArray::new( + attr_list_element_field(), + OffsetBuffer::new(ScalarBuffer::from(offsets)), + Arc::new(struct_all), + None, + ))) +} + +/// The contiguous runs of a dense (non-null) run-structured id column: the id of +/// each run and the exclusive end row index of each run. +fn runs(id_arr: &UInt16Array) -> (Vec, Vec) { + let n = id_arr.len(); + let mut ids = Vec::new(); + let mut ends = Vec::new(); + if n > 0 { + for i in 1..n { + if id_arr.value(i) != id_arr.value(i - 1) { + ids.push(id_arr.value(i - 1)); + ends.push(i32::try_from(i).expect("end fits i32")); + } + } + ids.push(id_arr.value(n - 1)); + ends.push(i32::try_from(n).expect("end fits i32")); + } + (ids, ends) +} + +/// Byte span (start,end) of each parent's run in the flattened struct, keyed by +/// parent id. The map is sized by the number of distinct resources/scopes (tiny) +/// -- not by the attribute-row count -- so it is not the join `HashMap` the +/// baseline flatten pays. +fn spans_by_id(list: &ListArray, parent_ids: &[u16]) -> HashMap { + let offs = list.value_offsets(); + parent_ids + .iter() + .enumerate() + .map(|(j, &pid)| (pid, (offs[j], offs[j + 1]))) + .collect() +} + +/// Build the shared resource/scope attribute column for a given [`Layout`] from +/// the per-parent lists and the log rows' run structure. +fn shared_column( + layout: Layout, + per_parent: &ListArray, + parent_ids: &[u16], + id_arr: &UInt16Array, +) -> StudyResult { + let (run_ids, run_ends) = runs(id_arr); + let spans = spans_by_id(per_parent, parent_ids); + let base_struct = per_parent + .values() + .as_any() + .downcast_ref::() + .ok_or("per-parent values are not a struct")?; + + match layout { + Layout::Materialized => { + // Repeat each run's set across its rows: gather indices row by row. + let num_rows = id_arr.len(); + let mut idx: Vec = Vec::new(); + let mut offsets: Vec = Vec::with_capacity(num_rows + 1); + offsets.push(0); + let mut run = 0usize; + for i in 0..num_rows { + if run + 1 < run_ends.len() && i32::try_from(i).expect("fits") >= run_ends[run] { + run += 1; + } + let (s, e) = spans.get(&run_ids[run]).copied().unwrap_or((0, 0)); + for k in s..e { + idx.push(u32::try_from(k).expect("index fits u32")); + } + offsets.push(i32::try_from(idx.len()).expect("offset fits i32")); + } + let taken = take_struct(base_struct, &idx)?; + Ok(Arc::new(ListArray::new( + attr_list_element_field(), + OffsetBuffer::new(ScalarBuffer::from(offsets)), + Arc::new(taken), + None, + ))) + } + Layout::RunEndEncoded => { + // One list per run; run_ends give the logical row spans. + let values = align_runs(base_struct, &spans, &run_ids)?; + let run_ends = Int32Array::from(run_ends); + let ree = RunArray::::try_new(&run_ends, &values)?; + Ok(Arc::new(ree)) + } + Layout::Dictionary => { + // keys index into the per-parent lists by position; values are those + // lists unchanged (zero-copy). + let pos_of_id: HashMap = parent_ids + .iter() + .enumerate() + .map(|(j, &pid)| (pid, i32::try_from(j).expect("fits"))) + .collect(); + let keys: UInt16Array = (0..id_arr.len()) + .map(|i| { + let id = id_arr.value(i); + u16::try_from(*pos_of_id.get(&id).unwrap_or(&0)).expect("key fits u16") + }) + .collect(); + let dict = DictionaryArray::::try_new( + keys, + Arc::new(per_parent.clone()) as ArrayRef, + )?; + Ok(Arc::new(dict)) + } + } +} + +/// One `List` per run (in run order), each holding that run's resource +/// set (or empty if the resource carried no attributes). +fn align_runs( + base_struct: &StructArray, + spans: &HashMap, + run_ids: &[u16], +) -> StudyResult { + let mut idx: Vec = Vec::new(); + let mut offsets: Vec = vec![0]; + for rid in run_ids { + let (s, e) = spans.get(rid).copied().unwrap_or((0, 0)); + for k in s..e { + idx.push(u32::try_from(k).expect("index fits u32")); + } + offsets.push(i32::try_from(idx.len()).expect("offset fits i32")); + } + let taken = take_struct(base_struct, &idx)?; + Ok(Arc::new(ListArray::new( + attr_list_element_field(), + OffsetBuffer::new(ScalarBuffer::from(offsets)), + Arc::new(taken), + None, + ))) +} + +/// `take` each field of a struct by the given indices (used only for the tiny +/// resource/scope sets, never for the per-row log attributes). +fn take_struct(base: &StructArray, idx: &[u32]) -> StudyResult { + let indices = UInt32Array::from(idx.to_vec()); + let cols: Vec = base + .columns() + .iter() + .map(|c| take(c, &indices, None)) + .collect::>()?; + Ok(StructArray::new(attr_struct_fields(), cols, None)) +} + +/// The flat-table field for a shared attribute column of the given layout. +fn shared_field(name: &str, array: &ArrayRef) -> Field { + Field::new(name, array.data_type().clone(), array.is_nullable()) +} + +/// Flatten an OTAP logs batch into a single columnar view with the given layout. +pub fn flatten(otap: &OtapArrowRecords, layout: Layout) -> StudyResult { + let logs = logs_batch(otap)?; + let resource_id = logs_resource_id(logs)?; + let scope_id = logs_scope_id(logs)?; + + let mut fields: Vec = logs + .schema() + .fields() + .iter() + .map(|f| f.as_ref().clone()) + .collect(); + let mut columns: Vec = logs.columns().to_vec(); + + // Shared resource/scope columns. + for (name, payload, id_arr) in [ + ( + RESOURCE_ATTRS_COL, + ArrowPayloadType::ResourceAttrs, + &resource_id, + ), + (SCOPE_ATTRS_COL, ArrowPayloadType::ScopeAttrs, &scope_id), + ] { + let col: ArrayRef = match otap.get(payload) { + Some(attr_batch) => { + let (per_parent, parent_ids) = per_parent_lists(attr_batch)?; + shared_column(layout, &per_parent, &parent_ids, id_arr)? + } + None => empty_attr_list_column(logs.num_rows()), + }; + fields.push(shared_field(name, &col)); + columns.push(col); + } + + // Per-row log attributes (zero-copy for every layout). + let log_col = log_attr_list(otap, logs)?; + fields.push(shared_field(LOG_ATTRS_COL, &log_col)); + columns.push(log_col); + + Ok(RecordBatch::try_new( + Arc::new(Schema::new(fields)), + columns, + )?) +} + +/// Read a shared attribute column back to a `List` of one row per +/// distinct resource/scope plus the id to stamp on each, regardless of layout. +fn shared_lists(flat: &RecordBatch, name: &str, id_arr: &UInt16Array) -> StudyResult { + let column = flat + .column_by_name(name) + .ok_or_else(|| format!("flat batch missing `{name}`"))?; + + match column.data_type() { + DataType::List(_) => { + // Materialized: per-row lists; dedup to first row per id. + rebuild_attr_batch(as_list(flat, name)?, &entries_dedup(id_arr)) + } + DataType::RunEndEncoded(_, _) => { + let ree = column + .as_any() + .downcast_ref::>() + .ok_or("expected RunEndEncoded")?; + let list = ree + .values() + .as_any() + .downcast_ref::() + .ok_or("REE values are not a list")?; + let (run_ids, _) = runs(id_arr); + let entries: Vec<(usize, u16)> = + run_ids.iter().enumerate().map(|(j, &id)| (j, id)).collect(); + rebuild_attr_batch(list, &entries) + } + DataType::Dictionary(_, _) => { + let dict = column + .as_any() + .downcast_ref::>() + .ok_or("expected Dictionary")?; + let list = dict + .values() + .as_any() + .downcast_ref::() + .ok_or("dictionary values are not a list")?; + // The per-parent lists are in ascending id order; distinct ids in + // ascending order recover the stamp for each list row. + let (mut ids, _) = runs(id_arr); + ids.sort_unstable(); + ids.dedup(); + let entries: Vec<(usize, u16)> = + ids.iter().enumerate().map(|(j, &id)| (j, id)).collect(); + rebuild_attr_batch(list, &entries) + } + other => Err(format!("unexpected shared column type {other:?}").into()), + } +} + +/// Reconstruct an OTAP logs batch from a flat view (any layout). +pub fn unflatten(flat: &RecordBatch) -> StudyResult { + let container = [RESOURCE_ATTRS_COL, SCOPE_ATTRS_COL, LOG_ATTRS_COL]; + let mut fields: Vec = Vec::new(); + let mut columns: Vec = Vec::new(); + for (field, column) in flat.schema().fields().iter().zip(flat.columns()) { + if !container.contains(&field.name().as_str()) { + fields.push(field.as_ref().clone()); + columns.push(column.clone()); + } + } + let logs = RecordBatch::try_new(Arc::new(Schema::new(fields)), columns)?; + + let resource_id = logs_resource_id(&logs)?; + let scope_id = logs_scope_id(&logs)?; + let log_id = logs_id(&logs)?; + + let resource_attrs = shared_lists(flat, RESOURCE_ATTRS_COL, &resource_id)?; + let scope_attrs = shared_lists(flat, SCOPE_ATTRS_COL, &scope_id)?; + let log_attrs = rebuild_attr_batch(as_list(flat, LOG_ATTRS_COL)?, &entries_per_row(&log_id))?; + + let mut otap = OtapArrowRecords::Logs(Default::default()); + otap.set(ArrowPayloadType::Logs, logs)?; + otap.set(ArrowPayloadType::ResourceAttrs, resource_attrs)?; + otap.set(ArrowPayloadType::ScopeAttrs, scope_attrs)?; + otap.set(ArrowPayloadType::LogAttrs, log_attrs)?; + Ok(otap) +} + +/// Sum of the in-memory footprint of every column in the flat view. +#[must_use] +pub fn in_memory_bytes(flat: &RecordBatch) -> usize { + flat.columns() + .iter() + .map(|c| c.get_array_memory_size()) + .sum() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::parquet_study::attrs::assert_logs_equivalent; + use crate::parquet_study::datagen::{LogsGenParams, gen_logs_otap}; + + #[test] + fn all_layouts_round_trip() { + let params = LogsGenParams { + num_resources: 3, + num_scopes: 2, + num_logs: 7, + }; + let (otap, _) = gen_logs_otap(¶ms); + + for layout in [ + Layout::Materialized, + Layout::RunEndEncoded, + Layout::Dictionary, + ] { + let flat = flatten(&otap, layout).expect("flatten"); + assert_eq!(flat.num_rows(), params.total_logs(), "{}", layout.name()); + let decoded = unflatten(&flat).expect("unflatten"); + assert_logs_equivalent(&otap, &decoded, layout.name(), "n/a"); + } + } + + #[test] + fn resource_heavy_round_trips() { + use crate::parquet_study::datagen::{RichGenParams, gen_logs_otap_rich}; + + let params = RichGenParams { + label: "test", + num_resources: 40, + num_scopes: 3, + num_logs: 5, + num_resource_attrs: 12, + num_scope_attrs: 4, + num_log_attrs: 3, + }; + let otap = gen_logs_otap_rich(¶ms); + + for layout in [ + Layout::Materialized, + Layout::RunEndEncoded, + Layout::Dictionary, + ] { + let flat = flatten(&otap, layout).expect("flatten"); + assert_eq!(flat.num_rows(), params.total_logs(), "{}", layout.name()); + let decoded = unflatten(&flat).expect("unflatten"); + assert_logs_equivalent(&otap, &decoded, layout.name(), "n/a"); + } + } + + #[test] + fn ree_and_dict_are_smaller_in_memory_than_materialized() { + let params = LogsGenParams { + num_resources: 1, + num_scopes: 1, + num_logs: 20_000, + }; + let (otap, _) = gen_logs_otap(¶ms); + + let mat = in_memory_bytes(&flatten(&otap, Layout::Materialized).expect("mat")); + let ree = in_memory_bytes(&flatten(&otap, Layout::RunEndEncoded).expect("ree")); + let dict = in_memory_bytes(&flatten(&otap, Layout::Dictionary).expect("dict")); + + assert!(ree < mat, "REE {ree} not smaller than materialized {mat}"); + assert!( + dict < mat, + "dict {dict} not smaller than materialized {mat}" + ); + } +} diff --git a/rust/otap-dataflow/benchmarks/src/parquet_study/parquet_io.rs b/rust/otap-dataflow/benchmarks/src/parquet_study/parquet_io.rs index 89da442ebb..6232f60292 100644 --- a/rust/otap-dataflow/benchmarks/src/parquet_study/parquet_io.rs +++ b/rust/otap-dataflow/benchmarks/src/parquet_study/parquet_io.rs @@ -7,16 +7,73 @@ //! read/write cost microbenchmark we instead encode to and decode from an //! in-memory `Vec` using the synchronous Arrow Parquet reader/writer. -use arrow::array::RecordBatch; -use arrow::compute::concat_batches; +use arrow::array::{Array, ArrayRef, RecordBatch}; +use arrow::compute::{cast, concat_batches}; +use arrow::datatypes::{DataType, Field, Schema}; use bytes::Bytes; use parquet::arrow::ArrowWriter; use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder; use parquet::basic::Compression; use parquet::file::properties::WriterProperties; +use std::sync::Arc; + use super::StudyResult; +/// Convert a flat OTAP batch into the form the Parquet writer can serialize, +/// transforming only the columns arrow-rs cannot handle and copying the rest. +/// +/// Two column encodings do not survive Parquet in arrow-rs 58.3, so exactly +/// those are materialized here: +/// +/// - `RunEndEncoded` columns (the run-end resource/scope attributes) cannot be +/// written; they are cast to their plain `List` value type. +/// - `Dictionary(_, FixedSizeBinary)` columns (`trace_id`, `span_id`) cannot be +/// read back, because the reader rebuilds the values through an offset buffer +/// and then asserts the `FixedSizeBinaryArray` has one buffer; they are cast to +/// plain `FixedSizeBinary`. +/// +/// Every other column, including dictionary-encoded `Utf8`/`Int32` such as +/// `severity_text` and the nested `scope.name` and `body.str`, round-trips +/// through Parquet unchanged and is copied by reference. Parquet then applies its +/// own run-length and dictionary encoding on write. +pub fn to_parquet_ready(batch: &RecordBatch) -> StudyResult { + let mut fields: Vec = Vec::with_capacity(batch.num_columns()); + let mut columns: Vec = Vec::with_capacity(batch.num_columns()); + for (field, column) in batch.schema().fields().iter().zip(batch.columns()) { + match column.data_type() { + DataType::RunEndEncoded(_, values) => { + let decoded = cast(column, values.data_type())?; + fields.push(Field::new( + field.name(), + decoded.data_type().clone(), + field.is_nullable(), + )); + columns.push(decoded); + } + DataType::Dictionary(_, value_type) + if matches!(value_type.as_ref(), DataType::FixedSizeBinary(_)) => + { + let decoded = cast(column, value_type)?; + fields.push(Field::new( + field.name(), + decoded.data_type().clone(), + field.is_nullable(), + )); + columns.push(decoded); + } + _ => { + fields.push(field.as_ref().clone()); + columns.push(column.clone()); + } + } + } + Ok(RecordBatch::try_new( + Arc::new(Schema::new(fields)), + columns, + )?) +} + /// Encode a single record batch as a Parquet file in memory. pub fn write_parquet(batch: &RecordBatch, compression: Compression) -> StudyResult> { let props = WriterProperties::builder() @@ -45,3 +102,44 @@ pub fn read_parquet(bytes: &[u8]) -> StudyResult { _ => Ok(concat_batches(&schema, &batches)?), } } + +#[cfg(test)] +mod tests { + use super::*; + use arrow::array::{DictionaryArray, FixedSizeBinaryArray}; + use arrow::datatypes::UInt16Type; + + #[test] + fn parquet_ready_materializes_fixed_size_binary() { + // Dictionary(UInt16, FixedSizeBinary(4)) is the shape the OTAP encoder + // produces for trace_id/span_id and that the Parquet reader mishandles. + let values = FixedSizeBinaryArray::try_from_iter( + vec![b"aaaa".to_vec(), b"bbbb".to_vec(), b"cccc".to_vec()].into_iter(), + ) + .expect("values"); + let keys = arrow::array::UInt16Array::from(vec![0u16, 1, 2, 1, 0]); + let dict = DictionaryArray::::try_new(keys, Arc::new(values)).expect("dict"); + let batch = RecordBatch::try_new( + Arc::new(Schema::new(vec![Field::new( + "trace_id", + dict.data_type().clone(), + false, + )])), + vec![Arc::new(dict)], + ) + .expect("batch"); + + let decoded = to_parquet_ready(&batch).expect("parquet-ready"); + assert_eq!( + decoded.schema().field(0).data_type(), + &DataType::FixedSizeBinary(4), + "dictionary should be materialized to plain fixed-size binary" + ); + + // The plain batch round-trips through Parquet, where the dictionary form + // would trip the arrow-rs reader. + let bytes = write_parquet(&decoded, Compression::UNCOMPRESSED).expect("write"); + let back = read_parquet(&bytes).expect("read"); + assert_eq!(back.num_rows(), 5); + } +} From a686efe89d722f336e6284e92ba446e367f3ca58 Mon Sep 17 00:00:00 2001 From: Joshua MacDonald Date: Wed, 1 Jul 2026 15:19:07 -0700 Subject: [PATCH 15/18] docs(benchmarks): explain the direct standard-to-Parquet path and its ordering precondition Recognizing the shared columns yields a direct OTAP-standard to Parquet path that skips the naive hash join: copy the Logs columns, attach log attributes zero-copy as List, materialize resource/scope as lists, then write. The zero-copy log-attribute attach reads contiguous parent_id runs, so it is a correctness precondition, not just a perf tweak, that the attribute batches be grouped by parent_id. A freshly encoded batch is grouped; a transport-optimized one is sorted by (type, key, ...), which scatters parent_id. Verified with a probe: the fresh batch round-trips zero-copy while the wire-decoded batch fails the precondition and must be regrouped first. This splits the pipeline: a gateway that encodes OTLP to OTAP and writes Parquet in place gets the direct path free (precompute case), while a receiver of transport-optimized OTAP pays a regroup. Noted tension: transport optimization sorts Logs by (resource, scope, trace_id), which helps Parquet clustering, but sorts attributes by key, which the flatten must undo. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../otap_parquet/OTAP_FLAT_ANALYSIS.md | 46 +++++++++++++++++-- 1 file changed, 41 insertions(+), 5 deletions(-) diff --git a/rust/otap-dataflow/benchmarks/benches/otap_parquet/OTAP_FLAT_ANALYSIS.md b/rust/otap-dataflow/benchmarks/benches/otap_parquet/OTAP_FLAT_ANALYSIS.md index 5dd3d00a09..2d174b2130 100644 --- a/rust/otap-dataflow/benchmarks/benches/otap_parquet/OTAP_FLAT_ANALYSIS.md +++ b/rust/otap-dataflow/benchmarks/benches/otap_parquet/OTAP_FLAT_ANALYSIS.md @@ -331,6 +331,40 @@ Parquet writer skip its own sort and produces meaningful row-group statistics, and it keeps the run-end runs maximal. The sort is optional, and its value grows after a merge or shuffle where the natural resource clustering has been broken. +### A direct standard-to-Parquet path, and its ordering precondition + +Recognizing which columns are shared also yields a direct OTAP-standard to +Parquet path that skips the hash join the naive flatten performs. It copies the +ten `Logs` columns, attaches the log attributes as a `List` whose struct +children are the existing `LogAttrs` value columns, and materializes the resource +and scope attributes as lists, then writes. The log-attribute attach is the +zero-copy step, and it works by reading contiguous `parent_id` runs, so it +depends on the attribute batches being grouped by `parent_id`. + +That grouping is present in a freshly encoded batch but not in a +transport-optimized one, and the difference is not a matter of speed but of +correctness. The OTAP encoder emits each parent's attributes contiguously, so a +batch straight from OTLP has `LogAttrs.parent_id` as `[0, 0, ..., 1, 1, ...]`. +The transport optimization then sorts each attribute batch by `(type, key, +value, parent_id)` to compress the value columns, which scatters `parent_id`. +A probe confirms it: on the fresh batch the zero-copy flatten round-trips +exactly, while on the same batch after a wire round-trip the `parent_id` column +is no longer grouped and the zero-copy path fails its own precondition. A +converter that receives a transport-optimized batch must therefore regroup the +attributes by `parent_id` first, which is a stable sort, or fall back to the hash +join. + +This splits the pipeline cleanly. A gateway that encodes OTLP to OTAP and writes +Parquet in the same place holds the fresh, `parent_id`-grouped batch and gets the +direct path for free, which is the precompute-at-the-gateway case. A service that +receives OTAP-standard off the wire holds a transport-optimized batch and pays a +regroup before the direct path applies. There is a tension worth naming: the +transport optimization sorts the `Logs` batch by `(resource, scope, trace_id)`, +which helps Parquet by clustering the low-cardinality columns, but it sorts the +attribute batches by key, which the flatten must undo. The attribute sort that +shrinks the standard wire is wasted, and then some, for a receiver that flattens +to Parquet. + ## What this means for the pipeline The applied section of the companion analysis argued that when the gateway owns @@ -359,11 +393,13 @@ OTAP already stored once. The zero-copy log-attribute build relies on the encoder emitting attributes grouped by `parent_id`, which the current OTAP producer does and which the probe -confirmed. A path that reordered or interleaved attributes would need a stable -sort first, which would add a pass but would still avoid the hash join. The key -column is normalized from its dictionary encoding to plain `Utf8`, so that one -column is cast rather than shared, while the other value columns are shared -unchanged. +confirmed. This is a correctness precondition, not just a performance one: the +transport optimization re-sorts each attribute batch by key, so a +transport-optimized batch is not grouped by `parent_id` and the zero-copy path +must regroup it first or fall back to the hash join, as the standard-to-Parquet +subsection above details. The key column is normalized from its dictionary +encoding to plain `Utf8`, so that one column is cast rather than shared, while +the other value columns are shared unchanged. The generated data models realistic telemetry with unique per-record ids and mixed-type attributes, which is what keeps the wire comparison honest, since From 1fdde7d6d086bca97163415d21d6ea9665909932 Mon Sep 17 00:00:00 2001 From: Joshua MacDonald Date: Wed, 1 Jul 2026 15:32:22 -0700 Subject: [PATCH 16/18] feat(benchmarks): measure naive vs optimized OTAP-to-Parquet flatten Add a table for the common OTLP -> batch -> Parquet case that times the naive hash-join flatten from ANALYSIS.md against the optimized shared-column zero-copy build, each followed by the same parquet-ready transform and Parquet write. Both produce byte-identical files, so only the flatten differs. The optimized flatten is several times cheaper on its own (log-heavy 47.7 -> 10.7 ms), but the Parquet writer is the shared floor, so the end-to-end saving is about 17% log-heavy and 6% resource-heavy. The gap reflects that the zero-copy build only helps the per-record log attributes; resource attributes must be materialized per row for Parquet on either path. Documented in OTAP_FLAT_ANALYSIS.md and README. Benchmark/analysis only; not user-facing. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../otap_parquet/OTAP_FLAT_ANALYSIS.md | 31 ++++++++ .../benchmarks/benches/otap_parquet/README.md | 26 +++++-- .../benchmarks/benches/otap_parquet/main.rs | 78 +++++++++++++++++++ 3 files changed, 128 insertions(+), 7 deletions(-) diff --git a/rust/otap-dataflow/benchmarks/benches/otap_parquet/OTAP_FLAT_ANALYSIS.md b/rust/otap-dataflow/benchmarks/benches/otap_parquet/OTAP_FLAT_ANALYSIS.md index 2d174b2130..bde6bbd198 100644 --- a/rust/otap-dataflow/benchmarks/benches/otap_parquet/OTAP_FLAT_ANALYSIS.md +++ b/rust/otap-dataflow/benchmarks/benches/otap_parquet/OTAP_FLAT_ANALYSIS.md @@ -365,6 +365,37 @@ attribute batches by key, which the flatten must undo. The attribute sort that shrinks the standard wire is wasted, and then some, for a receiver that flattens to Parquet. +### Measuring the optimized path against the naive one + +For the common OTLP to batch to Parquet case the batch is fresh and grouped, so +the direct path applies. Timing it against the naive hash-join flatten, both +followed by the same parquet-ready transform and Parquet write, since both +produce byte-identical files: + +| scenario | flatten naive | flatten opt | prep+write | total naive | total opt | +| -------------- | ------------: | ----------: | ---------: | ----------: | --------: | +| log-heavy | 47.7 | 10.7 | 168.5 | 216.3 | 179.3 | +| resource-heavy | 79.6 | 60.4 | 265.1 | 344.7 | 325.5 | + +The optimized flatten is much cheaper on its own, from 47.7 to 10.7 milliseconds +in the log-heavy case, a bit over four times faster, and from 79.6 to 60.4 in the +resource-heavy case. But the Parquet prepare-and-write is the floor and is shared +by both paths, so the end-to-end saving is smaller than the flatten saving alone, +about seventeen percent log-heavy and six percent resource-heavy. + +The gap between the two scenarios is the point. The zero-copy build only helps the +columns it can share, which are the per-record log attributes. Log-heavy has nine +of them, so avoiding the join and the full `take` removes most of the flatten +cost. Resource-heavy has only two log attributes but twenty resource attributes, +and those must be materialized per row for Parquet whichever path builds them, so +the optimized path saves the join overhead and the small log-attribute `take` but +still pays the resource materialization. In both cases the flatten is roughly a +fifth of the OTAP-to-Parquet total, so this refines the companion analysis: the +flatten tax is real and the shared-column build cuts it several fold, but the +Parquet writer sets a floor that leaves the end-to-end win in the single to low +double digits until the writer itself is made cheaper, for instance by the +run-end passthrough discussed above. + ## What this means for the pipeline The applied section of the companion analysis argued that when the gateway owns diff --git a/rust/otap-dataflow/benchmarks/benches/otap_parquet/README.md b/rust/otap-dataflow/benchmarks/benches/otap_parquet/README.md index 81ae87340d..b447522b04 100644 --- a/rust/otap-dataflow/benchmarks/benches/otap_parquet/README.md +++ b/rust/otap-dataflow/benchmarks/benches/otap_parquet/README.md @@ -54,15 +54,16 @@ rather than the deprecated Hadoop-framed `LZ4`. cargo bench -p benchmarks --bench otap_parquet ``` -Seven tables are printed to stdout before the timed round-trip benchmarks run: +Eight tables are printed to stdout before the timed round-trip benchmarks run: serialized size, the OTAP/IPC pipeline breakdown, the Parquet pipeline breakdown, the OTAP-flat single columnar view study, the service-to-service transfer -comparison, the conversion-cost matrix, and the OTAP/IPC streaming amortization. -The breakdown shapes are 10k, 30k, and 60k log records. A single OTAP logs batch -holds at most 65,535 records because the log id is a `u16`, so the shapes stay -below that and larger volumes are streamed. The full Criterion sweep is slow; the -printed tables are the main output. For a quick pass add `-- --measurement-time -0.5 --sample-size 10`, or read the tables and stop the run. +comparison, the conversion-cost matrix, the naive-versus-optimized OTAP-to-Parquet +comparison, and the OTAP/IPC streaming amortization. The breakdown shapes are 10k, +30k, and 60k log records. A single OTAP logs batch holds at most 65,535 records +because the log id is a `u16`, so the shapes stay below that and larger volumes +are streamed. The full Criterion sweep is slow; the printed tables are the main +output. For a quick pass add `-- --measurement-time 0.5 --sample-size 10`, or read +the tables and stop the run. ## Pipeline steps @@ -180,6 +181,17 @@ attribute containers change: | F -> P parquet write | 144.9 | 242.8 | | P -> F parquet read | 52.8 | 140.1 | +Naive versus optimized OTAP-to-Parquet for the common OTLP -> batch -> Parquet +case, where the batch is fresh and `parent_id`-grouped. `naive` is the hash-join +flatten from `ANALYSIS.md`; `optimized` is the shared-column zero-copy build. +Both write byte-identical Parquet, so only the flatten differs and the +prepare-and-write is shared: + +| scenario | flatten naive | flatten opt | prep+write | total naive | total opt | +| -------------- | ------------: | ----------: | ---------: | ----------: | --------: | +| log-heavy | 47.7 | 10.7 | 168.5 | 216.3 | 179.3 | +| resource-heavy | 79.6 | 60.4 | 265.1 | 344.7 | 325.5 | + A companion write-up of what these ratios mean, including the streaming effect, is in [`ANALYSIS.md`](./ANALYSIS.md). The OTAP-flat single columnar view, the transfer comparison, and the natural-center conversion analysis are in diff --git a/rust/otap-dataflow/benchmarks/benches/otap_parquet/main.rs b/rust/otap-dataflow/benchmarks/benches/otap_parquet/main.rs index 3d1491c404..e001fe152b 100644 --- a/rust/otap-dataflow/benchmarks/benches/otap_parquet/main.rs +++ b/rust/otap-dataflow/benchmarks/benches/otap_parquet/main.rs @@ -588,6 +588,83 @@ fn print_conversion_matrix() { println!(); } +/// The common OTLP -> batch -> Parquet path, comparing the naive hash-join +/// flatten used in `ANALYSIS.md` against the optimized shared-column build. Both +/// produce byte-identical Parquet, so only the flatten differs; the Parquet +/// prepare-and-write cost is shared. This assumes a fresh, `parent_id`-grouped +/// batch straight from the encoder, which is exactly the OTLP -> batch -> Parquet +/// case and is where the zero-copy log-attribute build applies. +fn print_parquet_fastpath() { + use benchmarks::parquet_study::datagen::{RichGenParams, gen_logs_otap_rich}; + + let scenarios = [ + RichGenParams { + label: "log-heavy", + num_resources: 1, + num_scopes: 1, + num_logs: 60_000, + num_resource_attrs: 1, + num_scope_attrs: 2, + num_log_attrs: 9, + }, + RichGenParams { + label: "resource-heavy", + num_resources: 600, + num_scopes: 1, + num_logs: 100, + num_resource_attrs: 20, + num_scope_attrs: 5, + num_log_attrs: 2, + }, + ]; + + println!("\n=== OTAP -> Parquet: naive vs optimized flatten (indicative ms) ==="); + println!("naive = hash-join flatten (ANALYSIS.md); optimized = shared-column zero-copy build."); + println!("Both write byte-identical Parquet; only the flatten differs. Assumes a fresh"); + println!("parent_id-grouped batch, i.e. the OTLP -> batch -> Parquet case."); + println!( + "{:<16} {:>11} {:>10} {:>11} {:>12} {:>11} {:>12}", + "scenario", "flat-naive", "flat-opt", "prep+write", "total-naive", "total-opt", "pq-bytes" + ); + for s in &scenarios { + let otap = gen_logs_otap_rich(s); + let flat_opt = otap_flat::flatten(&otap, Layout::Materialized).expect("opt flatten"); + let ready = to_parquet_ready(&flat_opt).expect("ready"); + let pq_opt = write_parquet(&ready, Compressor::Zstd.parquet()).expect("write"); + // Confirm the naive path yields the same Parquet size. + let naive_ready = + to_parquet_ready(&Scheme::Nested.flatten(&otap).expect("naive")).expect("ready"); + let pq_naive = write_parquet(&naive_ready, Compressor::Zstd.parquet()).expect("write"); + debug_assert_eq!( + pq_opt.len(), + pq_naive.len(), + "naive/opt Parquet size differs" + ); + + let flat_naive_ms = median_ms(|| { + let _ = Scheme::Nested.flatten(&otap).expect("naive flatten"); + }); + let flat_opt_ms = median_ms(|| { + let _ = otap_flat::flatten(&otap, Layout::Materialized).expect("opt flatten"); + }); + let prep_write_ms = median_ms(|| { + let r = to_parquet_ready(&flat_opt).expect("ready"); + let _ = write_parquet(&r, Compressor::Zstd.parquet()).expect("write"); + }); + println!( + "{:<16} {:>11.1} {:>10.1} {:>11.1} {:>12.1} {:>11.1} {:>12}", + s.label, + flat_naive_ms, + flat_opt_ms, + prep_write_ms, + flat_naive_ms + prep_write_ms, + flat_opt_ms + prep_write_ms, + pq_opt.len(), + ); + } + println!(); +} + /// OTAP/IPC streaming amortization: cold (first) versus warm (steady-state) /// per-batch size when a single long-lived Producer streams many batches, with /// the equivalent single Parquet file for reference. @@ -635,6 +712,7 @@ fn bench_round_trip(c: &mut Criterion) { print_otap_flat_table(); print_transfer_table(); print_conversion_matrix(); + print_parquet_fastpath(); print_streaming_table(&streaming_shapes()); let mut write_group = c.benchmark_group("parquet_study/write"); From fdb5512bb6b83d94b78024ccbc8271bc68bece6f Mon Sep 17 00:00:00 2001 From: Joshua MacDonald Date: Wed, 1 Jul 2026 15:52:24 -0700 Subject: [PATCH 17/18] docs(benchmarks): make the parquet study vendor-neutral Remove the .NET and SaaS references so the study is not tied to a specific vendor. The applied section now frames the motivating deployment generically: a single vendor controls both ends of the path, supplying the gateway software the customer runs on premises and operating the ingestion service, and because it owns both and wants to protect the ingestion service's CPU, it shifts the encoding cost onto the customer-run gateway. The compressor notes drop the ".NET builds" example and just say some Arrow/Parquet stacks may not support zstd. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../benches/otap_parquet/ANALYSIS.md | 25 +++++++++++-------- .../benchmarks/benches/otap_parquet/README.md | 4 +-- .../benchmarks/src/parquet_study/mod.rs | 5 ++-- 3 files changed, 19 insertions(+), 15 deletions(-) diff --git a/rust/otap-dataflow/benchmarks/benches/otap_parquet/ANALYSIS.md b/rust/otap-dataflow/benchmarks/benches/otap_parquet/ANALYSIS.md index ca8c35a6b1..c7d88bd219 100644 --- a/rust/otap-dataflow/benchmarks/benches/otap_parquet/ANALYSIS.md +++ b/rust/otap-dataflow/benchmarks/benches/otap_parquet/ANALYSIS.md @@ -223,18 +223,23 @@ the schema and dictionary amortization applies. ## Applying the model: precompute Parquet at the gateway The read and write costs above assume the converter runs on the server. The -deployment that motivates this study is different. A large sender, a customer -collector gateway, ships to a .NET SaaS ingestion service whose CPU is the -resource being optimized, and the same organization owns both the client exporter -and the storage schema. That ownership changes the trade in several ways. +deployment that motivates this study is different. A single vendor controls both +ends of the path: it supplies the gateway software that the customer runs on +premises, and it operates the ingestion service that receives the data. Because +it owns both, and because the ingestion service's CPU is the resource it most +wants to protect, it prefers to shift the encoding cost onto the customer-run +gateway. The same ownership means it controls the storage schema as well, so it +can version the on-premises exporter and the store together. That ownership +changes the trade in several ways. First, the coupling objection to client-side Parquet mostly goes away. When a third party owns the storage format, making it the wire contract is brittle. When -the ingestion service owns the exporter, it can emit exactly the flattened layout -its store wants, the columns, partitioning, sort order, row-group sizing, and -compression, and it can version the exporter and the store together. The gateway -also aggregates many hosts, so it forms the large batches where flattened Parquet -is 0.60 to 0.71x the IPC size, which cuts ingress bandwidth into the service. +the vendor owns both the exporter and the store, the gateway can emit exactly the +flattened layout the store wants, the columns, partitioning, sort order, +row-group sizing, and compression, and the two are versioned together. The +gateway also aggregates many hosts, so it forms the large batches where flattened +Parquet is 0.60 to 0.71x the IPC size, which cuts ingress bandwidth into the +service. Second, the decisive question becomes how much the ingestion service must read. Precomputing Parquet removes server CPU only if ingestion is close to a validated @@ -264,7 +269,7 @@ traffic that needs a server-side transform, and precomputed Parquet for large gateways running the custom exporter. The Parquet here is written by arrow-rs in the Rust sender, so its compression is -independent of the .NET receiver, and the primary server-CPU argument does not +independent of the receiver's stack, and the primary server-CPU argument does not depend on the compressor at all, because precompute moves the Parquet encode itself off the server. diff --git a/rust/otap-dataflow/benchmarks/benches/otap_parquet/README.md b/rust/otap-dataflow/benchmarks/benches/otap_parquet/README.md index b447522b04..5dafd5d691 100644 --- a/rust/otap-dataflow/benchmarks/benches/otap_parquet/README.md +++ b/rust/otap-dataflow/benchmarks/benches/otap_parquet/README.md @@ -34,8 +34,8 @@ every stage is visible on both the encode and decode side. Compressors are explicit codecs so `zstd` can be compared head-to-head with `lz4`. This matters for cross-language consumers, because some Arrow and Parquet -stacks, for example certain .NET builds, may not support `zstd`. In that case -`lz4`, and `snappy` for Parquet, need first-class numbers. +stacks may not support `zstd`. In that case `lz4`, and `snappy` for Parquet, need +first-class numbers. | compressor | Arrow IPC | Parquet | |------------|---------------|--------------| diff --git a/rust/otap-dataflow/benchmarks/src/parquet_study/mod.rs b/rust/otap-dataflow/benchmarks/src/parquet_study/mod.rs index c1543a5f96..064c8e55ed 100644 --- a/rust/otap-dataflow/benchmarks/src/parquet_study/mod.rs +++ b/rust/otap-dataflow/benchmarks/src/parquet_study/mod.rs @@ -9,9 +9,8 @@ //! Contenders are enumerated by [`Scheme`] and parameterized by a [`Compressor`]. //! Compressors are explicit codecs (`zstd`, `lz4`, `snappy`, `none`) so that //! zstd can be compared head-to-head with lz4 -- important when a consumer's -//! Arrow/Parquet stack (for example some .NET implementations) may not support -//! zstd. Arrow IPC only supports zstd and lz4 (frame), so snappy is offered for -//! the Parquet schemes only. +//! Arrow/Parquet stack may not support zstd. Arrow IPC only supports zstd and +//! lz4 (frame), so snappy is offered for the Parquet schemes only. use arrow::array::RecordBatch; use otap_df_pdata::otap::OtapArrowRecords; From f74d7114d1209758b3b1c429c54cff06d8713401 Mon Sep 17 00:00:00 2001 From: Joshua MacDonald Date: Wed, 1 Jul 2026 16:10:49 -0700 Subject: [PATCH 18/18] docs(otap_parquet): pair and tighten the two analysis docs Retitle the analysis docs as "part 1" and "part 2" with explicit reading order, share an identical units sentence, and make the cross-references consistent. Tighten wordy sections and consolidate the three overlapping closing sections of the flat analysis. No measurement tables changed. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../benches/otap_parquet/ANALYSIS.md | 179 +++++------ .../otap_parquet/OTAP_FLAT_ANALYSIS.md | 285 ++++++++---------- 2 files changed, 216 insertions(+), 248 deletions(-) diff --git a/rust/otap-dataflow/benchmarks/benches/otap_parquet/ANALYSIS.md b/rust/otap-dataflow/benchmarks/benches/otap_parquet/ANALYSIS.md index c7d88bd219..ac24162e55 100644 --- a/rust/otap-dataflow/benchmarks/benches/otap_parquet/ANALYSIS.md +++ b/rust/otap-dataflow/benchmarks/benches/otap_parquet/ANALYSIS.md @@ -1,17 +1,22 @@ -# Analysis: why OTAP/IPC is cheaper than flattened Parquet +# Analysis part 1: OTAP/IPC versus flattened Parquet -This explains the ratios the `otap_parquet` benchmark measures. Numbers are from -one development machine running WSL with jemalloc. Times are indicative medians -in milliseconds. Both `zstd` and `lz4` are shown, because `lz4` is how the -comparison was measured elsewhere. +Read this first. It measures the cost of moving OTAP logs as compressed Arrow +IPC versus flattened Parquet, on both size and CPU, and treats the Parquet +`flatten` step as a fixed cost. Part 2, +[`OTAP_FLAT_ANALYSIS.md`](./OTAP_FLAT_ANALYSIS.md), opens up that flatten step +and the single columnar view behind it. + +Numbers are indicative medians in milliseconds and bytes from one development +machine running WSL with jemalloc. Both `zstd` and `lz4` are shown, because +`lz4` is how the comparison was measured elsewhere. A single OTAP logs batch holds at most 65,535 log records, because the log id -that links a log row to its attributes is a `u16`. The breakdown below therefore -uses a 50,000-record batch as its headline, which is a large but valid single -batch. Volumes larger than 65,535 records must be streamed as several batches, -which is analyzed at the end and turns out to change the size comparison. +that links a log row to its attributes is a `u16`. The breakdown below uses a +50,000-record batch as its headline, a large but valid single batch. Volumes +above 65,535 records must be streamed as several batches, analyzed at the end, +which changes the size comparison. ## Headline ratios (50k records, one batch) @@ -169,16 +174,15 @@ amortization here should be read as the ceiling for dictionaries plus the schema which is always recovered. This also explains why the steady-state batches do not keep shrinking. The drop -is one-time, at the first batch, and every batch after that is the same size. The -reason is that Arrow IPC amortizes the schema and the dictionary value tables -once, but it does not compress one batch against another: each batch's column -buffers are compressed independently so a reader can decode any batch on its own. -So even though the batches here are identical, every steady-state batch re-sends -and re-compresses the full per-row payload, which is the dictionary indices plus -the non-dictionary columns. Dictionary reuse saves the value tables, not the -per-row references, and the per-row payload is the actual information in the -batch. How much redundancy this leaves unexploited, and why a naive whole-stream -compressor does not recover it, is measured in the next section. +is one-time, at the first batch, and every batch after that is the same size. +Arrow IPC amortizes the schema and the dictionary value tables once, but it does +not compress one batch against another: each batch's column buffers are +compressed independently so a reader can decode any batch on its own. So every +steady-state batch re-sends and re-compresses its full per-row payload, the +dictionary indices plus the non-dictionary columns, which is the actual +information in the batch. Dictionary reuse saves the value tables, not the +per-row references. How much redundancy that leaves unexploited, and why a +whole-stream compressor does not easily recover it, is the next section. ### The redundancy Arrow IPC leaves unexploited @@ -203,16 +207,15 @@ match and collapses each extra batch to about 269 bytes, storing all eight in 69 KB, close to the size of one. That factor of roughly nine is the cross-batch redundancy Arrow IPC leaves on the table in this best case. -Three caveats keep this from being free savings. First, it is a best case, -because these batches are byte-for-byte duplicates, whereas real telemetry batches -carry different records and share far less. Second, the large-window configuration -costs much more CPU than the light per-buffer codec Arrow IPC uses, so this is a -size ceiling, not a drop-in win. Third, whole-stream compression gives up the -per-batch independent decode that Arrow IPC provides, where any batch can be read -without the others. Arrow IPC trades cross-batch compression for low CPU and -independently decodable batches, which is the right trade for streaming transport, -so capturing the remaining redundancy is a job for a storage-side recompression -pass rather than the wire format. +Three caveats keep this from being free. It is a best case, since these batches +are byte-for-byte duplicates while real telemetry shares far less. The +large-window configuration costs far more CPU than the light per-buffer codec +Arrow IPC uses, so it is a size ceiling, not a drop-in win. And whole-stream +compression gives up the per-batch independent decode Arrow IPC provides, where +any batch can be read without the others. Arrow IPC trades cross-batch +compression for low CPU and independently decodable batches, the right trade for +streaming transport, so capturing the rest is a job for a storage-side +recompression pass rather than the wire format. So the single-batch size comparison understates OTAP/IPC, and it understates it most for the small, frequent batches that low-latency telemetry actually sends. @@ -222,76 +225,60 @@ the schema and dictionary amortization applies. ## Applying the model: precompute Parquet at the gateway -The read and write costs above assume the converter runs on the server. The -deployment that motivates this study is different. A single vendor controls both -ends of the path: it supplies the gateway software that the customer runs on -premises, and it operates the ingestion service that receives the data. Because -it owns both, and because the ingestion service's CPU is the resource it most -wants to protect, it prefers to shift the encoding cost onto the customer-run -gateway. The same ownership means it controls the storage schema as well, so it -can version the on-premises exporter and the store together. That ownership -changes the trade in several ways. - -First, the coupling objection to client-side Parquet mostly goes away. When a -third party owns the storage format, making it the wire contract is brittle. When -the vendor owns both the exporter and the store, the gateway can emit exactly the -flattened layout the store wants, the columns, partitioning, sort order, -row-group sizing, and compression, and the two are versioned together. The -gateway also aggregates many hosts, so it forms the large batches where flattened -Parquet is 0.60 to 0.71x the IPC size, which cuts ingress bandwidth into the -service. - -Second, the decisive question becomes how much the ingestion service must read. -Precomputing Parquet removes server CPU only if ingestion is close to a validated -append. Two facts from the cost model bound this. If the service fully decodes, -Parquet is the most expensive input, about 13x the IPC decode with zstd, and the -service would re-encode anyway, so precomputing helps only when it avoids that -decode. But Parquet also carries a footer and per-row-group statistics, so tenant -routing, quota and cardinality checks, and min or max pruning can run on that -metadata without decoding column data. A service that inspects metadata and -appends pays far less than the 13x figure, and that is the regime where accepting -client Parquet wins. - -Third, any work that needs the row values still forces a full decode, so the goal -is to move that work into the exporter. For logs this is a clean fit, because the -per-record work, parsing, filtering, redaction, and attribute shaping, is all done -at the gateway, which is the last writer. Once it has run, the gateway holds the -final records and can write them straight to Parquet, leaving the ingestion -service to validate and append. - -Two operational costs remain. Because exporters run on customer-managed -collectors, a storage-layout change cannot deploy atomically, so the ingestion -service must accept several layout versions at once and conform older ones itself, -which returns some CPU to the server. And not every sender is a large gateway, so -small and legacy senders still emit small batches where IPC is smaller and cheaper -than Parquet. The robust intake accepts both, OTAP/IPC for small senders and for -traffic that needs a server-side transform, and precomputed Parquet for large -gateways running the custom exporter. - -The Parquet here is written by arrow-rs in the Rust sender, so its compression is -independent of the receiver's stack, and the primary server-CPU argument does not -depend on the compressor at all, because precompute moves the Parquet encode -itself off the server. +The costs above assume the converter runs on the server. The motivating +deployment is different: one vendor owns both ends, supplying the gateway the +customer runs on premises and operating the ingestion service that receives the +data. Because it owns both, it can shift the encoding cost onto the customer-run +gateway and version the on-premises exporter and the store together. + +That ownership dissolves the usual coupling objection to client-side Parquet. +When a third party owns the storage format, making it the wire contract is +brittle; when the vendor owns both, the gateway can emit exactly the layout the +store wants, the columns, partitioning, sort order, row-group sizing, and +compression, and the two are versioned together. The gateway also aggregates +many hosts, so it forms the large batches where flattened Parquet is 0.60 to +0.71x the IPC size, which cuts ingress bandwidth. + +The decisive question is how much the ingestion service must read. If it fully +decodes, Parquet is the most expensive input, about 13x the IPC decode with +zstd, so precompute helps only when it avoids that decode. But Parquet carries a +footer and per-row-group statistics, so tenant routing, quota and cardinality +checks, and min or max pruning can run on that metadata without touching column +data. A service that inspects metadata and appends pays far less than 13x, and +that is the regime where accepting client Parquet wins. Any work that needs the +row values still forces a full decode, so the goal is to move that work into the +exporter. Logs fit cleanly, because parsing, filtering, redaction, and attribute +shaping already happen at the gateway, the last writer, which can then write the +final records straight to Parquet. + +Two operational costs remain. Exporters run on customer-managed collectors, so a +layout change cannot deploy atomically; the service must accept several layout +versions and conform older ones itself, returning some CPU to the server. And +not every sender is a large gateway, so small and legacy senders still emit the +small batches where IPC is smaller and cheaper. The robust intake accepts both, +OTAP/IPC for small senders and for traffic that needs a server-side transform, +and precomputed Parquet for large gateways running the custom exporter. Because +that Parquet is written by arrow-rs in the sender, the server-CPU argument does +not depend on the compressor at all; precompute moves the Parquet encode itself +off the server. ## Bottom line For a single large batch, flattened Parquet is smaller on the wire, about 0.60 to -0.71x the IPC size, but it costs roughly an order of magnitude more CPU to -produce and, with `zstd`, to consume. When the traffic is streamed, which is the -normal case and is required above 65,535 records per batch, OTAP/IPC amortizes a -fixed cost of about 11 KB per batch that is roughly two thirds dictionaries and -one third schema, and for small frequent batches that makes IPC smaller on the -wire than Parquet as well as far cheaper to produce and consume. Produce Parquet -where the columnar file and its smaller size for large data at rest are needed, -and keep the streaming client on OTAP/IPC. When comparing measurements, hold the -compressor fixed and state whether the size is cold or steady-state, because both -choices move the ratio by a large factor. And when one organization owns both the -exporter and the store, the applied section above shows the convert step can move -to the sending gateway, as long as ingestion stays close to a metadata-validated +0.71x the IPC size, but costs roughly an order of magnitude more CPU to produce +and, with `zstd`, to consume. Streaming, the normal case and required above +65,535 records per batch, changes this: OTAP/IPC amortizes a fixed cost of about +11 KB per batch, roughly two thirds dictionaries and one third schema, so for +small frequent batches IPC is smaller on the wire than Parquet as well as far +cheaper to produce and consume. Produce Parquet where the columnar file and its +smaller size at rest are needed, and keep the streaming client on OTAP/IPC. When +comparing measurements, hold the compressor fixed and state whether the size is +cold or steady-state, since both move the ratio by a large factor. And when one +organization owns both the exporter and the store, the convert step can move to +the sending gateway, as long as ingestion stays close to a metadata-validated append rather than a full decode. -The flatten step that this analysis treats as a fixed part of the Parquet cost is -itself examined in [`OTAP_FLAT_ANALYSIS.md`](./OTAP_FLAT_ANALYSIS.md), which shows -that most of it is avoidable because OTAP attribute batches are already grouped by -parent, and which measures how cheaply OTAP can be presented as a single columnar -view. +Part 2, [`OTAP_FLAT_ANALYSIS.md`](./OTAP_FLAT_ANALYSIS.md), opens up the flatten +step this analysis treats as fixed, and shows that most of it is avoidable +because OTAP attribute batches are already grouped by parent, and measures how +cheaply OTAP can be presented as a single columnar view. diff --git a/rust/otap-dataflow/benchmarks/benches/otap_parquet/OTAP_FLAT_ANALYSIS.md b/rust/otap-dataflow/benchmarks/benches/otap_parquet/OTAP_FLAT_ANALYSIS.md index bde6bbd198..e07b371c14 100644 --- a/rust/otap-dataflow/benchmarks/benches/otap_parquet/OTAP_FLAT_ANALYSIS.md +++ b/rust/otap-dataflow/benchmarks/benches/otap_parquet/OTAP_FLAT_ANALYSIS.md @@ -1,25 +1,23 @@ -# Analysis: an efficient OTAP-flat single columnar view - -This is a companion to [`ANALYSIS.md`](./ANALYSIS.md). That study measured the -cost of moving OTAP logs as compressed Arrow IPC versus flattened Parquet, and -found that producing Parquet costs roughly an order of magnitude more CPU than -IPC. A large and compressor-independent part of that Parquet cost is the -`flatten` step, which turns OTAP's four normalized record batches into one -denormalized table before the Parquet writer ever runs. This document focuses on -that intermediate. It asks how cheaply OTAP can be presented as a single -columnar view, and it measures three ways to do it. It then asks whether that -flat view is also a good format to move between two services, and measures it -against OTAP-standard and Parquet on the wire. Finally it measures every -conversion in the pipeline edge by edge, to show that the flat view sits at a -natural center where each move to a neighbor rewrites only a handful of columns -and copies the rest. - -Numbers below are indicative medians in milliseconds and bytes from one -development machine running WSL with jemalloc. Both scenarios hold the record -count fixed at 60,000 logs and vary only the attribute mix, because the layouts -studied here differ only in how they treat attributes. +# Analysis part 2: the OTAP-flat single columnar view + +Read [`ANALYSIS.md`](./ANALYSIS.md) first. Part 1 measured OTAP logs as +compressed Arrow IPC versus flattened Parquet and found that producing Parquet +costs roughly an order of magnitude more CPU than IPC. A large, +compressor-independent part of that cost is the `flatten` step, which turns +OTAP's four normalized record batches into one denormalized table before the +Parquet writer runs. This part opens up that step. It asks how cheaply OTAP can +be presented as a single columnar view and measures three ways to do it; then +whether that flat view is a good format to move between two services, against +OTAP-standard and Parquet on the wire; and finally every conversion in the +pipeline edge by edge, showing the flat view sits at a natural center where each +move to a neighbor rewrites only a handful of columns and copies the rest. + +Numbers are indicative medians in milliseconds and bytes from one development +machine running WSL with jemalloc. Both scenarios hold the record count fixed at +60,000 logs and vary only the attribute mix, because the layouts studied here +differ only in how they treat attributes. ## The flatten tax and where it comes from @@ -130,34 +128,32 @@ five scope attributes, and two attributes per record: There are two independent levers, and the two scenarios separate them. -The first lever is the zero-copy log attributes, and the materialized layout -already captures it because it shares the same log-attribute path. In the -log-heavy scenario the record attributes dominate, and building them without a -hash join and without a full `take` cuts the conversion from 74 milliseconds to -18, about four times faster, while producing a byte-identical Parquet file. The -run-end and dictionary layouts go a little further in this scenario, to about 8 -and 9 milliseconds, because they also stop repeating the small resource and scope -sets, and that trims the in-memory view from 47 to 36 megabytes. - -The second lever is the shared attributes, and only the run-end and dictionary -layouts capture it. The resource-heavy scenario is where this shows. The -materialized layout has to repeat twenty resource attributes across every one of -the 60,000 rows, which is 1.2 million struct rows of pure duplication, so it is -no cheaper than the baseline at about 87 milliseconds and it holds a 102 megabyte -view. The run-end layout stores each resource's attributes once, 600 lists in -total, so it builds the same logical view in 4.4 milliseconds and holds it in -12.7 megabytes. That is about twenty times less conversion work and eight times -less memory for a view that answers the same per-row questions. The dictionary -layout is close behind at 4.3 milliseconds and 12.9 megabytes. - -The Parquet column is the counterpoint. The on-disk size is the same for the -materialized layout as for the baseline, about 3.9 and 2.8 megabytes, because the -Parquet writer applies its own run-length and dictionary encodings and recovers -the repetition that the materialized view spelled out in memory. The physical -duplication in the materialized layout is therefore a cost paid in build time and -peak memory, not in file size. Writing Parquet still requires that materialized -form, so the conversion cannot be avoided when Parquet is the target. It can only -be avoided when the consumer reads the columnar view directly. +The first is the zero-copy log attributes, which the materialized layout already +captures because it shares the same log-attribute path. In the log-heavy scenario +the record attributes dominate, so building them without a hash join and without +a full `take` cuts the conversion from 74 to 18 milliseconds, about four times +faster, while producing a byte-identical Parquet file. The run-end and dictionary +layouts reach about 8 and 9 milliseconds by also not repeating the small resource +and scope sets, which trims the in-memory view from 47 to 36 megabytes. + +The second lever is the shared attributes, which only the run-end and dictionary +layouts capture, and the resource-heavy scenario is where it shows. The +materialized layout repeats twenty resource attributes across every one of the +60,000 rows, 1.2 million struct rows of pure duplication, so at about 87 +milliseconds and a 102 megabyte view it is no cheaper than the baseline. The +run-end layout stores each resource's attributes once, 600 lists in total, and +builds the same logical view in 4.4 milliseconds and 12.7 megabytes, about twenty +times less work and eight times less memory. The dictionary layout is close +behind at 4.3 milliseconds and 12.9 megabytes. + +The Parquet column is the counterpoint. On-disk size is identical for the +materialized layout and the baseline, about 3.9 and 2.8 megabytes, because the +writer applies its own run-length and dictionary encodings and recovers the +repetition the materialized view spelled out in memory. That physical +duplication is a cost paid in build time and peak memory, not in file size. +Writing Parquet still requires the materialized form, so the conversion is +unavoidable when Parquet is the target and can be skipped only when the consumer +reads the columnar view directly. ## Transferring between two services @@ -192,55 +188,48 @@ Resource-heavy: | ipc-flat-dict | 3,345,544 | 4,737,672 | 15.5 | 6.3 | flat table | | parquet-flat | 2,750,262 | 3,783,505 | 416.4 | 172.6 | flat table | -On realistic data the wire sizes land close together, and the ordering is -Parquet first, then OTAP-standard, then the run-end flat form, with the -materialized flat form well behind. Parquet is the smallest on the wire, about +On realistic data the wire sizes land close together. Parquet is smallest, about 3.9 megabytes log-heavy and 2.8 resource-heavy, because it applies run-length and dictionary encoding to every column. OTAP-standard is next and close, about 4.5 -and 3.0 megabytes, because it never denormalizes the shared attributes and -transport-optimizes ids and values. The run-end flat form is close behind again, -about 5.7 and 3.3 megabytes. Only the materialized flat form is far off, 7.0 and -12.1 megabytes, because it repeats the shared resource attributes on the wire and +and 3.0 megabytes, since it never denormalizes the shared attributes and +transport-optimizes ids and values. The run-end flat form follows again, about +5.7 and 3.3 megabytes. Only the materialized flat form is far off, 7.0 and 12.1 +megabytes, because it repeats the shared resource attributes on the wire and Arrow IPC does not fold that repetition away the way Parquet does. -This is the reconciliation with the companion analysis, which found Parquet -smaller on the wire than OTAP-standard for a single large batch. The same holds -here, by a smaller margin because the high-cardinality per-record data sets a -floor that both formats carry. An earlier pass of this study used identical -records and reported OTAP-standard many times smaller than any flat form, which -was an artifact of that degenerate data, not a real result. With realistic -records the four forms are within about a factor of two on the wire, apart from -the materialized flat form. - -The CPU picture is where the forms separate. OTAP-standard and the run-end flat -form are the cheap pair. OTAP-standard decodes fastest, about 6 to 13 -milliseconds, because it is a light Arrow IPC deserialize plus a transport -decode. The run-end flat form is often the cheapest to encode, and in the -resource-heavy scenario it encodes in 15 milliseconds against OTAP-standard's 28, -because it only flattens with a run-end layout and writes plain Arrow IPC, -whereas OTAP-standard pays for its transport optimization. Parquet is the -expensive outlier on both ends, 238 to 416 milliseconds to encode and 58 to 173 -to decode, five to fifteen times the others, which is the same result the -companion analysis reported. - -Two caveats remain. The per-record trace ids here are unique, which is the -high-cardinality end for logs; correlated logs that share a trace id across a -span would compress somewhat better and lower every wire number together. And to -write the flat batch to Parquet the study first materializes the encoder's -dictionary columns, because arrow-rs 58.3 cannot read a dictionary-encoded -`FixedSizeBinary` such as `trace_id` back from Parquet. Arrow IPC has no such -limit and carries those dictionaries directly. - -The reason this matters for the original hypothesis is that a flat wire format is -not obviously worse. The run-end flat form is within about a quarter of -OTAP-standard on the wire, is sometimes cheaper to encode, and hands the receiver -a query-ready table with no projection step. Against that, OTAP-standard is -slightly smaller and decodes fastest and yields the normalized form, and Parquet -is the smallest at rest but by far the most expensive to produce and consume. So -the choice is a real trade rather than a rout. If the receiver wants columns and -values CPU, shipping the run-end flat form is defensible. If it wants the smallest -wire or the normalized model, or the cheapest decode, OTAP-standard is the better -default, and the flat view is then a cheap projection at the consumer. +This reconciles with part 1, which found Parquet smaller on the wire than +OTAP-standard for a single large batch. The same holds here, by a smaller margin +because the high-cardinality per-record data sets a floor both formats carry. An +earlier pass of this study used identical records and reported OTAP-standard many +times smaller than any flat form, an artifact of that degenerate data. With +realistic records the four forms are within about a factor of two on the wire, +apart from the materialized flat form. + +CPU is where the forms separate. OTAP-standard and the run-end flat form are the +cheap pair. OTAP-standard decodes fastest, about 6 to 13 milliseconds, being a +light Arrow IPC deserialize plus a transport decode. The run-end flat form is +often the cheapest to encode, 15 milliseconds against OTAP-standard's 28 in the +resource-heavy scenario, because it only flattens with a run-end layout and +writes plain Arrow IPC while OTAP-standard pays for its transport optimization. +Parquet is the expensive outlier on both ends, 238 to 416 milliseconds to encode +and 58 to 173 to decode, five to fifteen times the others. + +Two caveats remain. The per-record trace ids here are unique, the +high-cardinality end for logs; correlated logs that share a trace id would +compress somewhat better and lower every wire number together. And to write the +flat batch to Parquet the study first materializes the encoder's dictionary +columns, because arrow-rs 58.3 cannot read a dictionary-encoded `FixedSizeBinary` +such as `trace_id` back, while Arrow IPC has no such limit and carries those +dictionaries directly. + +So a flat wire format is not obviously worse. The run-end flat form is within +about a quarter of OTAP-standard on the wire, is sometimes cheaper to encode, and +hands the receiver a query-ready table with no projection step. Against that, +OTAP-standard is slightly smaller, decodes fastest, and yields the normalized +form, while Parquet is smallest at rest but by far the most expensive to produce +and consume. The choice is a real trade rather than a rout: ship the run-end flat +form when the receiver wants columns and values CPU; otherwise OTAP-standard is +the better default and the flat view is a cheap projection at the consumer. ## OTAP-flat as the natural center @@ -383,88 +372,80 @@ resource-heavy case. But the Parquet prepare-and-write is the floor and is share by both paths, so the end-to-end saving is smaller than the flatten saving alone, about seventeen percent log-heavy and six percent resource-heavy. -The gap between the two scenarios is the point. The zero-copy build only helps the -columns it can share, which are the per-record log attributes. Log-heavy has nine -of them, so avoiding the join and the full `take` removes most of the flatten -cost. Resource-heavy has only two log attributes but twenty resource attributes, -and those must be materialized per row for Parquet whichever path builds them, so -the optimized path saves the join overhead and the small log-attribute `take` but +The gap between the two scenarios is the point. The zero-copy build only helps +the columns it can share, the per-record log attributes. Log-heavy has nine of +them, so avoiding the join and the full `take` removes most of the flatten cost. +Resource-heavy has only two log attributes but twenty resource attributes, and +those must be materialized per row for Parquet whichever path builds them, so the +optimized path saves the join overhead and the small log-attribute `take` but still pays the resource materialization. In both cases the flatten is roughly a -fifth of the OTAP-to-Parquet total, so this refines the companion analysis: the -flatten tax is real and the shared-column build cuts it several fold, but the -Parquet writer sets a floor that leaves the end-to-end win in the single to low -double digits until the writer itself is made cheaper, for instance by the -run-end passthrough discussed above. +fifth of the OTAP-to-Parquet total, so this refines part 1: the flatten tax is +real and the shared-column build cuts it several fold, but the Parquet writer +sets a floor that leaves the end-to-end win in the single to low double digits +until the writer itself is made cheaper, for instance by the run-end passthrough +above. ## What this means for the pipeline -The applied section of the companion analysis argued that when the gateway owns -both the exporter and the store, the OTAP to Parquet conversion can move to the -sending gateway. This study refines where the remaining cost lives and when it is -avoidable. - -If the target is a Parquet file, the materialized single view is the right -intermediate, and the win available is the zero-copy log-attribute build, which -removed about three quarters of the conversion time in the log-heavy case without -changing the output. The resource and scope repetition still has to be written -out for the Parquet writer to consume, though the file it produces is no larger -for it. - -If the target is an Arrow-native store answering queries, which is the serving -path for this system, the run-end or dictionary single view is dramatically -cheaper to build and to hold, and the advantage grows with the weight of the -shared attributes. Real resource attributes in production telemetry are numerous -and highly shared, so the resource-heavy scenario is the representative one for a -host or a gateway that aggregates many records under a small number of resources. -In that regime the run-end view is the most efficient single columnar -presentation of OTAP measured here, because it never materializes a value that +Part 1 argued that when the gateway owns both the exporter and the store, the +OTAP-to-Parquet conversion can move to the sending gateway. This study refines +where the remaining cost lives. If the target is a Parquet file, the materialized +view is the right intermediate and the zero-copy log-attribute build is the win, +removing about three quarters of the conversion time log-heavy without changing +the output; the shared resource and scope repetition still has to be written for +the writer, though the file is no larger for it. If the target is an Arrow-native +store answering queries, the serving path for this system, the run-end or +dictionary view is dramatically cheaper to build and hold, and the advantage +grows with the weight of the shared attributes. Real resource attributes in +production telemetry are numerous and highly shared, so the resource-heavy +scenario is the representative one for a host or gateway that aggregates many +records under a few resources, and there the run-end view is the most efficient +single columnar presentation measured here, because it never materializes a value OTAP already stored once. ## Caveats and limits The zero-copy log-attribute build relies on the encoder emitting attributes -grouped by `parent_id`, which the current OTAP producer does and which the probe -confirmed. This is a correctness precondition, not just a performance one: the -transport optimization re-sorts each attribute batch by key, so a -transport-optimized batch is not grouped by `parent_id` and the zero-copy path -must regroup it first or fall back to the hash join, as the standard-to-Parquet -subsection above details. The key column is normalized from its dictionary -encoding to plain `Utf8`, so that one column is cast rather than shared, while -the other value columns are shared unchanged. +grouped by `parent_id`, which the current producer does and the probe confirmed. +This is a correctness precondition, not just performance: a transport-optimized +batch is re-sorted by key, so it is not grouped by `parent_id` and must be +regrouped first or fall back to the hash join, as the direct-path subsection +details. The key column is normalized from its dictionary encoding to plain +`Utf8`, so that one column is cast rather than shared. The generated data models realistic telemetry with unique per-record ids and -mixed-type attributes, which is what keeps the wire comparison honest, since -identical rows collapse under dictionary and delta encoding and mislead. The wire -magnitudes still depend on cardinality: correlated logs that share a trace id -would compress better and lower every number together, so the comparison should -be read as one point on a spectrum rather than a fixed ratio. - -Two arrow-rs 58.3 limits shape the results. The Parquet writer cannot serialize -run-end or nested-dictionary columns, so those layouts are in-memory forms only. -The Parquet reader cannot read a dictionary-encoded `FixedSizeBinary` such as -`trace_id` back, so the study materializes the encoder's dictionary columns -before writing Parquet. Both may change as arrow-rs adds support, at which point -the run-end view could also become a Parquet write target. +mixed-type attributes, which keeps the wire comparison honest, since identical +rows collapse under dictionary and delta encoding and mislead. The magnitudes +still depend on cardinality: correlated logs that share a trace id would compress +better and lower every number together, so read the comparison as one point on a +spectrum rather than a fixed ratio. + +Two arrow-rs 58.3 limits shape the results. The writer cannot serialize run-end +or nested-dictionary columns, so those layouts are in-memory forms only, and the +reader cannot read a dictionary-encoded `FixedSizeBinary` such as `trace_id` +back, so the study materializes those columns before writing Parquet. Both may +lift as arrow-rs adds support, at which point the run-end view could also become +a Parquet write target. ## Bottom line OTAP attribute batches are already grouped by parent, so presenting them as a single columnar view does not need a hash join. Building the log attributes zero -copy makes the materialized view, which is the one Parquet can write, about four -times cheaper to produce for log-heavy data while leaving the file identical. For -the shared resource and scope attributes, a run-end or dictionary view stores -each set once and is the most efficient single presentation, about twenty times +copy makes the materialized view, the one Parquet can write, about four times +cheaper to produce for log-heavy data while leaving the file identical. For the +shared resource and scope attributes, a run-end or dictionary view stores each +set once and is the most efficient single presentation, about twenty times cheaper to build and eight times smaller in memory for resource-heavy data, at the cost of not being directly writable to Parquet today. The choice follows the -consumer. Materialize for a Parquet file, and keep the run-end view for an +consumer: materialize for a Parquet file, and keep the run-end view for an Arrow-native store that serves queries. For moving data between two services the answer is a genuine trade rather than a rout. On realistic data the four forms sit within about a factor of two on the wire, apart from the materialized flat form, which repeats shared attributes and -falls behind. Parquet is the smallest at rest but by far the most expensive to -produce and consume. OTAP-standard is close on size, decodes fastest, and yields -the normalized model. The run-end flat form is close again on size, is sometimes +falls behind. Parquet is smallest at rest but by far the most expensive to +produce and consume; OTAP-standard is close on size, decodes fastest, and yields +the normalized model; the run-end flat form is close again on size, is sometimes cheaper to encode than OTAP-standard, and hands the receiver a query-ready table. So a flat wire format is defensible when the receiver wants columns and values CPU, while OTAP-standard remains the better default for the smallest wire, the