diff --git a/rust/otap-dataflow/benchmarks/Cargo.toml b/rust/otap-dataflow/benchmarks/Cargo.toml index b56527df16..ae2504af16 100644 --- a/rust/otap-dataflow/benchmarks/Cargo.toml +++ b/rust/otap-dataflow/benchmarks/Cargo.toml @@ -10,23 +10,28 @@ rust-version.workspace = true [dependencies] arrow.workspace = true +arrow-ipc.workspace = true +# Parquet study (benches/otap_parquet) needs the compression codecs enabled so +# the Arrow writer can emit zstd/snappy/lz4-compressed files. +parquet = { workspace = true, features = ["zstd", "snap", "lz4"] } +bytes.workspace = true +prost = { workspace = true } tokio.workspace = true serde_json.workspace = true otap-df-core-nodes = { workspace = true, features = ["dev-tools", "bench"] } otap-df-otap = { workspace = true } +otap-df-pdata = { workspace = true, features = ["bench", "testing"] } +otap-df-pdata-views = { workspace = true } [dev-dependencies] criterion = { workspace = true, features = ["html_reports", "async_tokio"] } tonic = { workspace = true } -prost = { workspace = true } otap-df-control-channel = { workspace = true } otap-df-config = { workspace = true } otap-df-channel = { workspace = true } otap-df-engine = { workspace = true } otap-df-telemetry = { workspace = true } -otap-df-pdata = { workspace = true, features = ["bench", "testing"] } -otap-df-pdata-views = { workspace = true } tracing.workspace = true tracing-subscriber = { workspace = true, features = ["registry"] } @@ -43,7 +48,7 @@ unsync.workspace = true portpicker.workspace = true tokio-stream.workspace = true weaver_common.workspace = true -bytes.workspace = true +zstd.workspace = true [target.'cfg(not(windows))'.dev-dependencies] tikv-jemallocator.workspace = true @@ -51,6 +56,10 @@ tikv-jemallocator.workspace = true [lints] workspace = true +[[bench]] +name = "otap_parquet" +harness = false + [[bench]] name = "channel" harness = false diff --git a/rust/otap-dataflow/benchmarks/benches/otap_parquet/ANALYSIS.md b/rust/otap-dataflow/benchmarks/benches/otap_parquet/ANALYSIS.md new file mode 100644 index 0000000000..ac24162e55 --- /dev/null +++ b/rust/otap-dataflow/benchmarks/benches/otap_parquet/ANALYSIS.md @@ -0,0 +1,284 @@ + + +# Analysis part 1: OTAP/IPC versus flattened Parquet + +Read this first. It measures the cost of moving OTAP logs as compressed Arrow +IPC versus flattened Parquet, on both size and CPU, and treats the Parquet +`flatten` step as a fixed cost. Part 2, +[`OTAP_FLAT_ANALYSIS.md`](./OTAP_FLAT_ANALYSIS.md), opens up that flatten step +and the single columnar view behind it. + +Numbers are indicative medians in milliseconds and bytes from one development +machine running WSL with jemalloc. Both `zstd` and `lz4` are shown, because +`lz4` is how the comparison was measured elsewhere. + +A single OTAP logs batch holds at most 65,535 log records, because the log id +that links a log row to its attributes is a `u16`. The breakdown below uses a +50,000-record batch as its headline, a large but valid single batch. Volumes +above 65,535 records must be streamed as several batches, analyzed at the end, +which changes the size comparison. + +## Headline ratios (50k records, one batch) + +| contender | comp | encode ms | decode ms | size bytes | size vs ipc | +|----------------|------|-----------|-----------|------------|-------------| +| ipc | zstd | 18.9 | 4.2 | 401,966 | 1.00x | +| ipc | lz4 | 16.1 | 20.0 | 466,416 | 1.16x | +| parquet-nested | zstd | 171 | 54.5 | 240,211 | 0.60x | +| parquet-nested | lz4 | 160 | 51.4 | 329,298 | 0.71x | +| parquet-wide | zstd | 94.4 | 34.2 | 245,757 | 0.61x | +| parquet-wide | lz4 | 91.7 | 32.2 | 327,547 | 0.70x | + +Reading the ratios with `lz4`, which is the relevant compressor here, for a +single 50k batch: + +- Parquet-nested is 0.71x the IPC size, so it is smaller at rest. +- Parquet-nested costs about 10x more to encode than IPC, 160 ms versus 16 ms. +- Parquet-nested costs about 2.6x more to decode than IPC, 51 ms versus 20 ms. + +With `zstd` the encode ratio is about the same, 9x, but the decode ratio is 13x, +because IPC decode with `zstd` is only 4.2 ms. The reason is the `lz4` decode +penalty explained below. + +## Where the time goes + +The benchmark breaks each side into two steps. + +OTAP/IPC, 50k, per step: + +| comp | transport-encode | ipc-serialize | ipc-deserialize | transport-decode | +|------|------------------|---------------|-----------------|------------------| +| zstd | 13.0 | 5.9 | 3.0 | 1.2 | +| lz4 | 13.5 | 2.6 | 18.7 | 1.3 | + +Parquet, 50k, per step: + +| scheme / comp | flatten | pq-write | pq-read | unflatten | +|-----------------------|---------|----------|---------|-----------| +| parquet-nested / zstd | 44 | 127 | 39 | 16 | +| parquet-nested / lz4 | 44 | 116 | 35 | 17 | +| parquet-wide / zstd | 69 | 25 | 14 | 20 | +| parquet-wide / lz4 | 69 | 22 | 12 | 20 | + +## Explaining each ratio + +### IPC encode is dominated by transport-optimize, not compression + +The transport-optimized encoding is about 13 ms of the 16 to 19 ms IPC encode, +regardless of compressor. It makes a full pass over the four record batches, +applying delta and dictionary encodings to the id and value columns and remapping +parent ids so the child batches still reference the right rows. That pass touches +all of the data once, which is why it is the largest single IPC cost and why it +does not change with the compressor. The Arrow IPC serialization that follows is +small, 3 to 6 ms, because it writes already-compact columns and the compressor +runs on far less data. + +### IPC serialize is faster when compressed + +Uncompressed IPC serialize is much slower than compressed, because writing the +record batch means copying bytes and there are far more of them without +compression. The compressor pays for itself on the write side by shrinking what +has to be moved. + +### IPC decode is where lz4 and zstd diverge sharply + +Deserialize dominates IPC decode, and the transport decode is cheap. The striking +result is that `lz4` deserialize is about six times slower than `zstd` for this +data. In this Arrow IPC implementation the LZ4 frame decompression path is much +slower than zstd. That single step is why IPC decode is 4.2 ms with `zstd` but +20 ms with `lz4`, and therefore why the IPC-over-Parquet decode advantage shrinks +from about 13x with `zstd` to under 3x with `lz4`. Anyone measuring with `lz4` +will see IPC decode look far worse than a `zstd` measurement would, even though +the data is identical. + +### Parquet encode is dominated by the writer, with a large flatten tax + +Flatten is compressor-independent, because it is a pure Arrow transformation that +joins the attribute batches onto the log rows and builds the nested columns. For +`parquet-nested` and `parquet-map` the Parquet writer then dominates, because +encoding `List` and `Map` columns requires Parquet definition and +repetition levels over many leaf fields. `parquet-wide` writes much faster +because its attributes are flat typed scalar columns that Parquet encodes +cheaply, but it pays more in flatten to explode the keys into columns. The +compressor barely moves the writer time, which is why `pq-write` is nearly +constant across `zstd`, `lz4`, and `snappy`, while the output size is not. + +### Parquet decode is dominated by the reader + +Parquet read is larger than unflatten. `parquet-wide` reads fastest because flat +columns decode directly, though its unflatten is a little more expensive because +it reassembles attributes from many columns. + +## Streaming changes the size story + +The size numbers above encode each batch as a self-contained Arrow IPC stream, +which pays the full schema and dictionary cost every time. That is the cold, or +worst, case. In real OTAP streaming the `Producer` is long-lived: it writes the +Arrow schema into the stream once, and it delta-encodes dictionaries, so every +batch after the first omits the schema and re-sends only new dictionary entries. +Parquet has no equivalent per-batch amortization, because each Parquet file is +self-contained with its own schema, footer, and per-row-group dictionary pages. + +Measured cold versus steady-state IPC size per batch, with the equivalent single +Parquet file for reference: + +| logs | comp | cold | warm | saved | pq-nested | warm/pq | +|--------|------|---------|---------|--------|-----------|---------| +| 1,000 | zstd | 24,748 | 13,422 | 11,326 | 19,512 | 0.69x | +| 1,000 | lz4 | 27,692 | 16,366 | 11,326 | 21,136 | 0.77x | +| 10,000 | zstd | 92,270 | 80,944 | 11,326 | 57,023 | 1.42x | +| 10,000 | lz4 | 108,846 | 97,520 | 11,326 | 75,336 | 1.29x | +| 50,000 | zstd | 401,966 | 390,640 | 11,326 | 240,211 | 1.63x | +| 50,000 | lz4 | 466,416 | 455,088 | 11,326 | 329,298 | 1.38x | + +The amortization is a fixed cost of about 11,326 bytes per batch, and it is the +same at every batch size. Splitting that fixed cost by Arrow IPC message type, at +1,000 records with `zstd`, shows it is mostly dictionaries rather than schema: + +| payload | schema | dictionaries | data (warm) | +|---------------|--------|--------------|-------------| +| Logs | 1,856 | 4,032 | 3,968 | +| LogAttrs | 832 | 2,048 | 7,744 | +| ResourceAttrs | 448 | 832 | 832 | +| ScopeAttrs | 448 | 832 | 832 | +| total | 3,584 | 7,744 | 13,376 | + +So of the 11,328 bytes saved per steady-state batch, 7,744 are dictionary +messages and only 3,584 are schema, about 68 percent dictionaries and 32 percent +schema. The dictionary messages are large not because the values are large, since +this data has few distinct values, but because the four batches carry many +dictionary columns, and each carries per-message framing that the stream sends +once and then reuses. + +The cost is fixed with batch size because both parts are per-stream, not +per-row. The schema describes columns, not rows. The dictionaries are the set of +distinct values, and in this synthetic data that set is the same whether the +batch has 1,000 or 50,000 rows, so the dictionary messages do not grow. What +changes is how large that fixed cost is relative to the batch: + +- At 1,000 records per batch it is about 46 percent of the cold size, and it + flips the verdict: the steady-state IPC batch is smaller than the Parquet file, + 0.69x with `zstd` and 0.77x with `lz4`. +- At 10,000 records it is about 12 percent, and Parquet is still smaller on the + wire, though the gap narrows. +- At 50,000 records it is about 3 percent, and Parquet keeps its size advantage. + +This synthetic data is a best case for dictionary amortization, because every +batch carries the identical low-cardinality values, so batches after the first +send no new dictionary entries at all. Real telemetry amortizes only the stable +low-cardinality columns, such as attribute keys, severity, and scope names. A +high-cardinality column such as a log body carries new values in +every batch, so its delta dictionary keeps sending new entries and does not +amortize, and such a column is often better left non-dictionary. The measured +amortization here should be read as the ceiling for dictionaries plus the schema, +which is always recovered. + +This also explains why the steady-state batches do not keep shrinking. The drop +is one-time, at the first batch, and every batch after that is the same size. +Arrow IPC amortizes the schema and the dictionary value tables once, but it does +not compress one batch against another: each batch's column buffers are +compressed independently so a reader can decode any batch on its own. So every +steady-state batch re-sends and re-compresses its full per-row payload, the +dictionary indices plus the non-dictionary columns, which is the actual +information in the batch. Dictionary reuse saves the value tables, not the +per-row references. How much redundancy that leaves unexploited, and why a +whole-stream compressor does not easily recover it, is the next section. + +### The redundancy Arrow IPC leaves unexploited + +Because each batch is compressed on its own, the steady-state batches are stored +at full size even when they are nearly identical. To measure how much that leaves +unexploited, the study also compresses a whole stream of eight batches as a single +unit and compares the per-batch cost. At 10,000 logs per batch with `zstd`: + +| approach | 8 batches | per extra batch | +|------------------------|-----------|-----------------| +| ipc, per-batch zstd | 659 KB | 81 KB | +| whole stream, zstd L3 | 595 KB | 74 KB | +| whole stream, zstd L19 | 69 KB | 269 B | + +The uncompressed batches here differ by a single byte, a batch counter, so they +are almost pure duplicates, yet Arrow IPC still spends about 81 KB on each one. A +default-effort whole-stream `zstd` barely does better, about 74 KB per extra +batch, because each uncompressed batch is roughly 2.4 MB, larger than the match +window at that level, so the compressor cannot see that the previous batch was a +duplicate. Only a large-window, long-distance configuration at level 19 finds the +match and collapses each extra batch to about 269 bytes, storing all eight in +69 KB, close to the size of one. That factor of roughly nine is the cross-batch +redundancy Arrow IPC leaves on the table in this best case. + +Three caveats keep this from being free. It is a best case, since these batches +are byte-for-byte duplicates while real telemetry shares far less. The +large-window configuration costs far more CPU than the light per-buffer codec +Arrow IPC uses, so it is a size ceiling, not a drop-in win. And whole-stream +compression gives up the per-batch independent decode Arrow IPC provides, where +any batch can be read without the others. Arrow IPC trades cross-batch +compression for low CPU and independently decodable batches, the right trade for +streaming transport, so capturing the rest is a job for a storage-side +recompression pass rather than the wire format. + +So the single-batch size comparison understates OTAP/IPC, and it understates it +most for the small, frequent batches that low-latency telemetry actually sends. +The `u16` log-id limit reinforces this: because one batch cannot exceed 65,535 +records, high volume is delivered as a stream of batches, which is exactly where +the schema and dictionary amortization applies. + +## Applying the model: precompute Parquet at the gateway + +The costs above assume the converter runs on the server. The motivating +deployment is different: one vendor owns both ends, supplying the gateway the +customer runs on premises and operating the ingestion service that receives the +data. Because it owns both, it can shift the encoding cost onto the customer-run +gateway and version the on-premises exporter and the store together. + +That ownership dissolves the usual coupling objection to client-side Parquet. +When a third party owns the storage format, making it the wire contract is +brittle; when the vendor owns both, the gateway can emit exactly the layout the +store wants, the columns, partitioning, sort order, row-group sizing, and +compression, and the two are versioned together. The gateway also aggregates +many hosts, so it forms the large batches where flattened Parquet is 0.60 to +0.71x the IPC size, which cuts ingress bandwidth. + +The decisive question is how much the ingestion service must read. If it fully +decodes, Parquet is the most expensive input, about 13x the IPC decode with +zstd, so precompute helps only when it avoids that decode. But Parquet carries a +footer and per-row-group statistics, so tenant routing, quota and cardinality +checks, and min or max pruning can run on that metadata without touching column +data. A service that inspects metadata and appends pays far less than 13x, and +that is the regime where accepting client Parquet wins. Any work that needs the +row values still forces a full decode, so the goal is to move that work into the +exporter. Logs fit cleanly, because parsing, filtering, redaction, and attribute +shaping already happen at the gateway, the last writer, which can then write the +final records straight to Parquet. + +Two operational costs remain. Exporters run on customer-managed collectors, so a +layout change cannot deploy atomically; the service must accept several layout +versions and conform older ones itself, returning some CPU to the server. And +not every sender is a large gateway, so small and legacy senders still emit the +small batches where IPC is smaller and cheaper. The robust intake accepts both, +OTAP/IPC for small senders and for traffic that needs a server-side transform, +and precomputed Parquet for large gateways running the custom exporter. Because +that Parquet is written by arrow-rs in the sender, the server-CPU argument does +not depend on the compressor at all; precompute moves the Parquet encode itself +off the server. + +## Bottom line + +For a single large batch, flattened Parquet is smaller on the wire, about 0.60 to +0.71x the IPC size, but costs roughly an order of magnitude more CPU to produce +and, with `zstd`, to consume. Streaming, the normal case and required above +65,535 records per batch, changes this: OTAP/IPC amortizes a fixed cost of about +11 KB per batch, roughly two thirds dictionaries and one third schema, so for +small frequent batches IPC is smaller on the wire than Parquet as well as far +cheaper to produce and consume. Produce Parquet where the columnar file and its +smaller size at rest are needed, and keep the streaming client on OTAP/IPC. When +comparing measurements, hold the compressor fixed and state whether the size is +cold or steady-state, since both move the ratio by a large factor. And when one +organization owns both the exporter and the store, the convert step can move to +the sending gateway, as long as ingestion stays close to a metadata-validated +append rather than a full decode. + +Part 2, [`OTAP_FLAT_ANALYSIS.md`](./OTAP_FLAT_ANALYSIS.md), opens up the flatten +step this analysis treats as fixed, and shows that most of it is avoidable +because OTAP attribute batches are already grouped by parent, and measures how +cheaply OTAP can be presented as a single columnar view. diff --git a/rust/otap-dataflow/benchmarks/benches/otap_parquet/OTAP_FLAT_ANALYSIS.md b/rust/otap-dataflow/benchmarks/benches/otap_parquet/OTAP_FLAT_ANALYSIS.md new file mode 100644 index 0000000000..e07b371c14 --- /dev/null +++ b/rust/otap-dataflow/benchmarks/benches/otap_parquet/OTAP_FLAT_ANALYSIS.md @@ -0,0 +1,453 @@ + + +# Analysis part 2: the OTAP-flat single columnar view + +Read [`ANALYSIS.md`](./ANALYSIS.md) first. Part 1 measured OTAP logs as +compressed Arrow IPC versus flattened Parquet and found that producing Parquet +costs roughly an order of magnitude more CPU than IPC. A large, +compressor-independent part of that cost is the `flatten` step, which turns +OTAP's four normalized record batches into one denormalized table before the +Parquet writer runs. This part opens up that step. It asks how cheaply OTAP can +be presented as a single columnar view and measures three ways to do it; then +whether that flat view is a good format to move between two services, against +OTAP-standard and Parquet on the wire; and finally every conversion in the +pipeline edge by edge, showing the flat view sits at a natural center where each +move to a neighbor rewrites only a handful of columns and copies the rest. + +Numbers are indicative medians in milliseconds and bytes from one development +machine running WSL with jemalloc. Both scenarios hold the record count fixed at +60,000 logs and vary only the attribute mix, because the layouts studied here +differ only in how they treat attributes. + +## The flatten tax and where it comes from + +An OTAP logs batch is four Arrow record batches. The root `Logs` batch holds one +row per record with the scalar and struct columns. Three attribute batches, +`ResourceAttrs`, `ScopeAttrs`, and `LogAttrs`, hold the attributes, each linked +to its parent by a `parent_id` column. This is a normalized, relational shape +that is compact on the wire because a resource's attributes are stored once and +referenced by every log record under that resource. + +A single columnar view is the opposite shape. It is one table with one row per +log record, where each row can reach its resource, scope, and log attributes +without a join. Parquet needs this shape, and so does a query engine that wants +to scan the data as columns. The `flatten` step performs the join. The existing +flattened contenders build it with `gather_by_parent`, which constructs a +`HashMap>` for each attribute batch and then materializes every +value column with a random-access `take`. That hash join and full materialization +is the flatten tax. + +## The structural opportunity + +The OTAP encoder does not emit attributes in arbitrary order. It walks resources, +scopes, and records in order, and for each parent it appends that parent's +attributes contiguously before moving on. The result is that every attribute +batch arrives already grouped by `parent_id` in ascending, contiguous runs, and +the `Logs.id` column that keys the log attributes is a sequential, nullable +`u16`. A probe over generated data confirms it directly. `ResourceAttrs.parent_id` +is `0, 1, 2`, `ScopeAttrs.parent_id` is `0, 0, 1, 1, 2, 2`, and +`LogAttrs.parent_id` is a run of zeros, then a run of ones, and so on. + +This ordering makes the hash join unnecessary. Two facts follow from it. + +First, the per-record log attributes can become a `List` almost for free. +The struct children are the existing `LogAttrs` value columns used as they are, +with no `take`, and the list offsets come from a single linear scan that walks +the sorted `parent_id` runs alongside the sequential `Logs.id` column. Every log +attribute belongs to exactly one record, so no value is copied and no value is +shared. + +Second, the shared resource and scope attributes line up with runs of log rows, +because the records are already grouped by resource and then by scope. A resource +therefore spans a contiguous block of rows, which is exactly the structure that +run-end and dictionary encodings capture without physically repeating anything. + +## Three layouts of the shared attributes + +All three layouts studied here build the log attributes the same zero-copy way. +They differ only in how they present the shared resource and scope sets in the +single view. + +- Materialized repeats each set physically across its rows as a plain + `List`. This is the same shape the `parquet-nested` contender produces, + and it is the only one arrow-rs can write to Parquet. +- Run-end encoded stores each set once as `RunEndEncoded>`. The + view is logically per-row, and the physical storage holds one list per resource + or scope plus the run boundaries. +- Dictionary stores each set once as `Dictionary>`. The view + carries a per-row `u16` index into a small table that holds one list per + resource or scope. + +A spike settled how far each form travels. The arrow-rs Parquet writer at version +58.3 cannot serialize a run-end-encoded column, and it cannot serialize a +dictionary whose values are `List`. It reports the run-end case as not +supported and the nested-dictionary case as not yet implemented. A dictionary of +a primitive such as `Utf8` does write and round-trips as a dictionary. The +conclusion is that run-end and nested-dictionary views are in-memory and query +representations rather than on-disk ones. That fits a design where the live store +is Arrow-native and Parquet is an export and interchange format rather than the +serving format. + +## Measurements + +The data models realistic telemetry rather than identical rows. Every record +carries a unique pseudo-random `trace_id` and `span_id`, a varying timestamp, a +templated body, and mixed-type log attributes that blend low-cardinality enums +with high-cardinality per-record values. Resource and scope attributes are +distinct per resource or scope but shared across that resource's records. This +matters because a degenerate feed of identical records collapses under +dictionary and delta encoding and would flatter whichever format encodes best, +which distorts the comparison. + +The table reports the conversion cost from OTAP to a single record batch, the +in-memory footprint of that view, and the Parquet encode of the view where +arrow-rs can write it. The baseline is the existing nested flatten with its hash +join and full `take`. + +Log-heavy, 60,000 logs under one resource with one resource attribute, two scope +attributes, and nine attributes per record: + +| contender | convert ms | view mem | pq-write ms | pq bytes | writable | +| ---------------------- | ---------: | --------: | ----------: | --------: | -------- | +| nested, baseline | 74.1 | 41.8 MB | 236.0 | 3,863,351 | yes | +| otap-flat-materialized | 18.0 | 46.7 MB | 277.4 | 3,863,351 | yes | +| otap-flat-ree | 8.4 | 36.0 MB | n/a | n/a | no | +| otap-flat-dict | 8.9 | 36.2 MB | n/a | n/a | no | + +Resource-heavy, 60,000 logs across 600 resources with twenty attributes each, +five scope attributes, and two attributes per record: + +| contender | convert ms | view mem | pq-write ms | pq bytes | writable | +| ---------------------- | ---------: | --------: | ----------: | --------: | -------- | +| nested, baseline | 94.3 | 101.9 MB | 355.6 | 2,750,262 | yes | +| otap-flat-materialized | 86.8 | 101.9 MB | 380.0 | 2,750,262 | yes | +| otap-flat-ree | 4.4 | 12.7 MB | n/a | n/a | no | +| otap-flat-dict | 4.3 | 12.9 MB | n/a | n/a | no | + +## Reading the results + +There are two independent levers, and the two scenarios separate them. + +The first is the zero-copy log attributes, which the materialized layout already +captures because it shares the same log-attribute path. In the log-heavy scenario +the record attributes dominate, so building them without a hash join and without +a full `take` cuts the conversion from 74 to 18 milliseconds, about four times +faster, while producing a byte-identical Parquet file. The run-end and dictionary +layouts reach about 8 and 9 milliseconds by also not repeating the small resource +and scope sets, which trims the in-memory view from 47 to 36 megabytes. + +The second lever is the shared attributes, which only the run-end and dictionary +layouts capture, and the resource-heavy scenario is where it shows. The +materialized layout repeats twenty resource attributes across every one of the +60,000 rows, 1.2 million struct rows of pure duplication, so at about 87 +milliseconds and a 102 megabyte view it is no cheaper than the baseline. The +run-end layout stores each resource's attributes once, 600 lists in total, and +builds the same logical view in 4.4 milliseconds and 12.7 megabytes, about twenty +times less work and eight times less memory. The dictionary layout is close +behind at 4.3 milliseconds and 12.9 megabytes. + +The Parquet column is the counterpoint. On-disk size is identical for the +materialized layout and the baseline, about 3.9 and 2.8 megabytes, because the +writer applies its own run-length and dictionary encodings and recovers the +repetition the materialized view spelled out in memory. That physical +duplication is a cost paid in build time and peak memory, not in file size. +Writing Parquet still requires the materialized form, so the conversion is +unavoidable when Parquet is the target and can be skipped only when the consumer +reads the columnar view directly. + +## Transferring between two services + +The measurements so far are about building the single view and holding it in +memory. A different question is whether the flat view is a good format to move +between two services, which is what OTAP-standard does today. To answer it the +study serializes each form the way it would travel. OTAP-standard is the +transport-optimized `Producer` over the four normalized batches. Each flat layout +is one record batch written as a plain Arrow IPC stream, which unlike Parquet can +carry run-end and dictionary columns. Parquet is the flat batch written as a +file. The table reports the compressed wire size under zstd and lz4, the encode +cost from an OTAP batch to wire bytes, and the decode cost from wire bytes to the +receiver's working form. + +Log-heavy: + +| contender | zstd wire | lz4 wire | encode ms | decode ms | receiver form | +| --------------------- | ---------: | ---------: | --------: | --------: | --------------- | +| ipc-standard | 4,479,026 | 6,442,420 | 84.3 | 12.6 | normalized OTAP | +| ipc-flat-materialized | 7,013,768 | 11,015,624 | 88.6 | 27.8 | flat table | +| ipc-flat-ree | 5,663,048 | 9,061,640 | 63.7 | 23.0 | flat table | +| ipc-flat-dict | 5,663,240 | 9,062,728 | 71.8 | 20.4 | flat table | +| parquet-flat | 3,863,351 | 5,777,270 | 238.1 | 57.9 | flat table | + +Resource-heavy: + +| contender | zstd wire | lz4 wire | encode ms | decode ms | receiver form | +| --------------------- | ---------: | ---------: | --------: | --------: | --------------- | +| ipc-standard | 3,013,172 | 3,841,908 | 28.0 | 5.6 | normalized OTAP | +| ipc-flat-materialized | 12,137,288 | 17,397,896 | 153.3 | 65.4 | flat table | +| ipc-flat-ree | 3,345,992 | 4,736,200 | 15.1 | 6.5 | flat table | +| ipc-flat-dict | 3,345,544 | 4,737,672 | 15.5 | 6.3 | flat table | +| parquet-flat | 2,750,262 | 3,783,505 | 416.4 | 172.6 | flat table | + +On realistic data the wire sizes land close together. Parquet is smallest, about +3.9 megabytes log-heavy and 2.8 resource-heavy, because it applies run-length and +dictionary encoding to every column. OTAP-standard is next and close, about 4.5 +and 3.0 megabytes, since it never denormalizes the shared attributes and +transport-optimizes ids and values. The run-end flat form follows again, about +5.7 and 3.3 megabytes. Only the materialized flat form is far off, 7.0 and 12.1 +megabytes, because it repeats the shared resource attributes on the wire and +Arrow IPC does not fold that repetition away the way Parquet does. + +This reconciles with part 1, which found Parquet smaller on the wire than +OTAP-standard for a single large batch. The same holds here, by a smaller margin +because the high-cardinality per-record data sets a floor both formats carry. An +earlier pass of this study used identical records and reported OTAP-standard many +times smaller than any flat form, an artifact of that degenerate data. With +realistic records the four forms are within about a factor of two on the wire, +apart from the materialized flat form. + +CPU is where the forms separate. OTAP-standard and the run-end flat form are the +cheap pair. OTAP-standard decodes fastest, about 6 to 13 milliseconds, being a +light Arrow IPC deserialize plus a transport decode. The run-end flat form is +often the cheapest to encode, 15 milliseconds against OTAP-standard's 28 in the +resource-heavy scenario, because it only flattens with a run-end layout and +writes plain Arrow IPC while OTAP-standard pays for its transport optimization. +Parquet is the expensive outlier on both ends, 238 to 416 milliseconds to encode +and 58 to 173 to decode, five to fifteen times the others. + +Two caveats remain. The per-record trace ids here are unique, the +high-cardinality end for logs; correlated logs that share a trace id would +compress somewhat better and lower every wire number together. And to write the +flat batch to Parquet the study first materializes the encoder's dictionary +columns, because arrow-rs 58.3 cannot read a dictionary-encoded `FixedSizeBinary` +such as `trace_id` back, while Arrow IPC has no such limit and carries those +dictionaries directly. + +So a flat wire format is not obviously worse. The run-end flat form is within +about a quarter of OTAP-standard on the wire, is sometimes cheaper to encode, and +hands the receiver a query-ready table with no projection step. Against that, +OTAP-standard is slightly smaller, decodes fastest, and yields the normalized +form, while Parquet is smallest at rest but by far the most expensive to produce +and consume. The choice is a real trade rather than a rout: ship the run-end flat +form when the receiver wants columns and values CPU; otherwise OTAP-standard is +the better default and the flat view is a cheap projection at the consumer. + +## OTAP-flat as the natural center + +The pipeline this study considers has five representations. OTAP-standard is the +four normalized batches, written `S`. OTAP-flat with run-end shared attributes is +the single batch, written `F`. Their serialized forms are OTAP/IPC-standard `Ws` +and OTAP/IPC-flat `Wf`, and the storage form is Parquet `P`. The question is what +it costs to move between them, and how much of each move is a genuine transform +rather than a copy of columns that do not change. + +Each directed edge, timed in isolation: + +| edge | log-heavy | resource-heavy | +| ---------------------------------- | --------: | -------------: | +| S -> F flatten to REE | 6.1 | 1.7 | +| F -> S unflatten | 12.9 | 2.9 | +| S -> Ws standard serialize | 62.9 | 25.8 | +| Ws -> S standard deserialize | 9.9 | 4.5 | +| F -> Wf flat serialize | 37.0 | 12.2 | +| Wf -> F flat deserialize | 16.1 | 6.4 | +| F -> P parquet-ready (REE+FSB) | 13.8 | 58.7 | +| F -> P parquet write | 144.9 | 242.8 | +| P -> F parquet read | 52.8 | 140.1 | + +Absolute milliseconds on this shared machine vary by up to about a factor of two +between runs, so the matrix should be read for the relative cost of the edges +rather than exact values. The relationships are stable. The two OTAP forms are +the cheapest pair to move between, a few milliseconds each way. Serializing +standard to its wire form is the most expensive of the Arrow IPC edges because it +runs the transport optimization, while serializing flat is lighter because it +only compresses. Parquet is the heavy end on both write and read. + +### What actually changes, column by column + +The flat table has thirteen columns: ten that come straight from the standard +`Logs` batch, which are `id`, `resource`, `scope`, the two timestamps, +`trace_id`, `span_id`, the two severity columns, and `body`, plus the three +attribute columns `resource_attributes`, `scope_attributes`, and +`log_attributes`. Classifying each conversion by how many of the thirteen it +copies unchanged versus how many it must rewrite: + +| conversion | copied | transformed | what changes | +| ----------- | ------: | -----------: | ---------------------------------------------------------- | +| S <-> F | 10 | 3 | attribute containers: keyed batches to REE/List columns | +| F <-> P | 9 | 4 | trace_id/span_id dict to plain; resource/scope REE to List | +| S <-> Ws | 0 | 13 | transport-optimize sort, delta, dict, then compress | +| F <-> Wf | 13 | 0 | buffer compression and framing only | + +`S` to `F` copies the ten `Logs` columns verbatim and only builds the three +attribute containers, wrapping the resource and scope batches as run-end columns +and the log attributes as a list. `F` to `P` copies nine columns, including the +dictionary `Utf8` and `Int32` columns and the log-attribute list, all of which +Parquet round-trips, and rewrites only four: it expands the two run-end columns +to plain lists and materializes `trace_id` and `span_id` from dictionary to +plain. That last pair is forced by the reader, not the writer, since arrow-rs can +write a dictionary of `FixedSizeBinary` but cannot read one back. The two +serialize edges are the opposite extremes. Standard to its wire form rewrites +every column, because the transport optimization sorts each batch and delta and +dictionary encodes it before compression. Flat to its wire form rewrites none of +them structurally, because plain Arrow IPC carries the run-end and dictionary +encodings as they are and only compresses the buffers. + +### Why flat is the center + +Put together, `F` sits between `S` and `P` and is a short hop from each. It shares +its ten `Logs` columns with `S`, so reaching it from `S` is a copy plus three +small container builds. It shares its whole schema with `P`, so reaching `P` from +`F` is a copy plus four column rewrites plus the writer. And its own wire form is +a light compress rather than a re-encode. No single move rewrites more than a +handful of columns, and the bulk of every move is memory sharing. That is what it +means for a representation to be a natural center. It is close to standard on one +side and close to Parquet on the other, and the conversions in every direction +touch only the columns that genuinely differ between the encodings. + +### Where sort order and future support change the picture + +One transform in the matrix is not cheap. Expanding the run-end resource and +scope columns for Parquet costs about 59 milliseconds in the resource-heavy +scenario, because it re-materializes the repetition that run-end encoding had +folded away. This is the memory and wire saving being paid back at export time. +Two changes would remove it. If arrow-rs learns to write run-end columns to +Parquet, the expansion becomes a passthrough from run-end runs to Parquet +run-length pages, and if its reader learns to read a dictionary of +`FixedSizeBinary`, the `trace_id` and `span_id` materialization disappears too. +At that point `F` to `P` is nearly all copy plus the writer. An asserted sort +order compounds this, because a flat batch that declares its order lets the +Parquet writer skip its own sort and produces meaningful row-group statistics, +and it keeps the run-end runs maximal. The sort is optional, and its value grows +after a merge or shuffle where the natural resource clustering has been broken. + +### A direct standard-to-Parquet path, and its ordering precondition + +Recognizing which columns are shared also yields a direct OTAP-standard to +Parquet path that skips the hash join the naive flatten performs. It copies the +ten `Logs` columns, attaches the log attributes as a `List` whose struct +children are the existing `LogAttrs` value columns, and materializes the resource +and scope attributes as lists, then writes. The log-attribute attach is the +zero-copy step, and it works by reading contiguous `parent_id` runs, so it +depends on the attribute batches being grouped by `parent_id`. + +That grouping is present in a freshly encoded batch but not in a +transport-optimized one, and the difference is not a matter of speed but of +correctness. The OTAP encoder emits each parent's attributes contiguously, so a +batch straight from OTLP has `LogAttrs.parent_id` as `[0, 0, ..., 1, 1, ...]`. +The transport optimization then sorts each attribute batch by `(type, key, +value, parent_id)` to compress the value columns, which scatters `parent_id`. +A probe confirms it: on the fresh batch the zero-copy flatten round-trips +exactly, while on the same batch after a wire round-trip the `parent_id` column +is no longer grouped and the zero-copy path fails its own precondition. A +converter that receives a transport-optimized batch must therefore regroup the +attributes by `parent_id` first, which is a stable sort, or fall back to the hash +join. + +This splits the pipeline cleanly. A gateway that encodes OTLP to OTAP and writes +Parquet in the same place holds the fresh, `parent_id`-grouped batch and gets the +direct path for free, which is the precompute-at-the-gateway case. A service that +receives OTAP-standard off the wire holds a transport-optimized batch and pays a +regroup before the direct path applies. There is a tension worth naming: the +transport optimization sorts the `Logs` batch by `(resource, scope, trace_id)`, +which helps Parquet by clustering the low-cardinality columns, but it sorts the +attribute batches by key, which the flatten must undo. The attribute sort that +shrinks the standard wire is wasted, and then some, for a receiver that flattens +to Parquet. + +### Measuring the optimized path against the naive one + +For the common OTLP to batch to Parquet case the batch is fresh and grouped, so +the direct path applies. Timing it against the naive hash-join flatten, both +followed by the same parquet-ready transform and Parquet write, since both +produce byte-identical files: + +| scenario | flatten naive | flatten opt | prep+write | total naive | total opt | +| -------------- | ------------: | ----------: | ---------: | ----------: | --------: | +| log-heavy | 47.7 | 10.7 | 168.5 | 216.3 | 179.3 | +| resource-heavy | 79.6 | 60.4 | 265.1 | 344.7 | 325.5 | + +The optimized flatten is much cheaper on its own, from 47.7 to 10.7 milliseconds +in the log-heavy case, a bit over four times faster, and from 79.6 to 60.4 in the +resource-heavy case. But the Parquet prepare-and-write is the floor and is shared +by both paths, so the end-to-end saving is smaller than the flatten saving alone, +about seventeen percent log-heavy and six percent resource-heavy. + +The gap between the two scenarios is the point. The zero-copy build only helps +the columns it can share, the per-record log attributes. Log-heavy has nine of +them, so avoiding the join and the full `take` removes most of the flatten cost. +Resource-heavy has only two log attributes but twenty resource attributes, and +those must be materialized per row for Parquet whichever path builds them, so the +optimized path saves the join overhead and the small log-attribute `take` but +still pays the resource materialization. In both cases the flatten is roughly a +fifth of the OTAP-to-Parquet total, so this refines part 1: the flatten tax is +real and the shared-column build cuts it several fold, but the Parquet writer +sets a floor that leaves the end-to-end win in the single to low double digits +until the writer itself is made cheaper, for instance by the run-end passthrough +above. + +## What this means for the pipeline + +Part 1 argued that when the gateway owns both the exporter and the store, the +OTAP-to-Parquet conversion can move to the sending gateway. This study refines +where the remaining cost lives. If the target is a Parquet file, the materialized +view is the right intermediate and the zero-copy log-attribute build is the win, +removing about three quarters of the conversion time log-heavy without changing +the output; the shared resource and scope repetition still has to be written for +the writer, though the file is no larger for it. If the target is an Arrow-native +store answering queries, the serving path for this system, the run-end or +dictionary view is dramatically cheaper to build and hold, and the advantage +grows with the weight of the shared attributes. Real resource attributes in +production telemetry are numerous and highly shared, so the resource-heavy +scenario is the representative one for a host or gateway that aggregates many +records under a few resources, and there the run-end view is the most efficient +single columnar presentation measured here, because it never materializes a value +OTAP already stored once. + +## Caveats and limits + +The zero-copy log-attribute build relies on the encoder emitting attributes +grouped by `parent_id`, which the current producer does and the probe confirmed. +This is a correctness precondition, not just performance: a transport-optimized +batch is re-sorted by key, so it is not grouped by `parent_id` and must be +regrouped first or fall back to the hash join, as the direct-path subsection +details. The key column is normalized from its dictionary encoding to plain +`Utf8`, so that one column is cast rather than shared. + +The generated data models realistic telemetry with unique per-record ids and +mixed-type attributes, which keeps the wire comparison honest, since identical +rows collapse under dictionary and delta encoding and mislead. The magnitudes +still depend on cardinality: correlated logs that share a trace id would compress +better and lower every number together, so read the comparison as one point on a +spectrum rather than a fixed ratio. + +Two arrow-rs 58.3 limits shape the results. The writer cannot serialize run-end +or nested-dictionary columns, so those layouts are in-memory forms only, and the +reader cannot read a dictionary-encoded `FixedSizeBinary` such as `trace_id` +back, so the study materializes those columns before writing Parquet. Both may +lift as arrow-rs adds support, at which point the run-end view could also become +a Parquet write target. + +## Bottom line + +OTAP attribute batches are already grouped by parent, so presenting them as a +single columnar view does not need a hash join. Building the log attributes zero +copy makes the materialized view, the one Parquet can write, about four times +cheaper to produce for log-heavy data while leaving the file identical. For the +shared resource and scope attributes, a run-end or dictionary view stores each +set once and is the most efficient single presentation, about twenty times +cheaper to build and eight times smaller in memory for resource-heavy data, at +the cost of not being directly writable to Parquet today. The choice follows the +consumer: materialize for a Parquet file, and keep the run-end view for an +Arrow-native store that serves queries. + +For moving data between two services the answer is a genuine trade rather than a +rout. On realistic data the four forms sit within about a factor of two on the +wire, apart from the materialized flat form, which repeats shared attributes and +falls behind. Parquet is smallest at rest but by far the most expensive to +produce and consume; OTAP-standard is close on size, decodes fastest, and yields +the normalized model; the run-end flat form is close again on size, is sometimes +cheaper to encode than OTAP-standard, and hands the receiver a query-ready table. +So a flat wire format is defensible when the receiver wants columns and values +CPU, while OTAP-standard remains the better default for the smallest wire, the +cheapest decode, or the normalized model, with the flat view computed as a cheap +projection at the consumer. diff --git a/rust/otap-dataflow/benchmarks/benches/otap_parquet/README.md b/rust/otap-dataflow/benchmarks/benches/otap_parquet/README.md new file mode 100644 index 0000000000..5dafd5d691 --- /dev/null +++ b/rust/otap-dataflow/benchmarks/benches/otap_parquet/README.md @@ -0,0 +1,253 @@ + + +# `otap_parquet` benchmark: OTAP/IPC vs flattened Parquet + +**Status:** experiment on branch `jmacd/parquet_study`. + +This benchmark studies the cost of moving OTAP logs between a client and a server +two ways: as compressed Arrow IPC, which is the representation we have today, and +as a flattened single-file Parquet, which a server would store. It starts from an +OTAP logs batch, which is four Arrow record batches (Logs, ResourceAttrs, +ScopeAttrs, LogAttrs), and breaks each pipeline into its sub-steps so the cost of +every stage is visible on both the encode and decode side. + +- OTAP/IPC encode is transport-optimize, then Arrow IPC serialize with + compression. Decode is IPC deserialize, then transport-decode. +- Parquet encode is flatten to one Arrow record batch, then write Parquet. Decode + is read Parquet, then unflatten. + +## Contenders + +- `ipc` is the OTAP representation we have today: interleaved Arrow IPC streams + produced by `Producer` and consumed by `Consumer`, with each per-payload stream + compressed. +- `parquet-nested` is a single flattened Parquet file where each log row carries + its denormalized resource, scope, and log attributes as + `List` columns. +- `parquet-map` is the same, with attributes stored as + `Map`. +- `parquet-wide` is the analytics-flat extreme, where every distinct attribute + key becomes its own typed top-level column named `resource.`, + `scope.`, or `log.`. + +## Compressors + +Compressors are explicit codecs so `zstd` can be compared head-to-head with +`lz4`. This matters for cross-language consumers, because some Arrow and Parquet +stacks may not support `zstd`. In that case `lz4`, and `snappy` for Parquet, need +first-class numbers. + +| compressor | Arrow IPC | Parquet | +|------------|---------------|--------------| +| `zstd` | `ZSTD` | `ZSTD` | +| `lz4` | `LZ4_FRAME` | `LZ4_RAW` | +| `snappy` | *unsupported* | `SNAPPY` | +| `none` | uncompressed | uncompressed | + +Arrow IPC only supports `zstd` and `lz4`, so `snappy` is offered for the Parquet +schemes only. Parquet uses `LZ4_RAW`, the cross-language interoperable variant, +rather than the deprecated Hadoop-framed `LZ4`. + +## Running + +```bash +cargo bench -p benchmarks --bench otap_parquet +``` + +Eight tables are printed to stdout before the timed round-trip benchmarks run: +serialized size, the OTAP/IPC pipeline breakdown, the Parquet pipeline breakdown, +the OTAP-flat single columnar view study, the service-to-service transfer +comparison, the conversion-cost matrix, the naive-versus-optimized OTAP-to-Parquet +comparison, and the OTAP/IPC streaming amortization. The breakdown shapes are 10k, +30k, and 60k log records. A single OTAP logs batch holds at most 65,535 records +because the log id is a `u16`, so the shapes stay below that and larger volumes +are streamed. The full Criterion sweep is slow; the printed tables are the main +output. For a quick pass add `-- --measurement-time 0.5 --sample-size 10`, or read +the tables and stop the run. + +## Pipeline steps + +- IPC `t-enc` is the OTAP transport-optimized encoding, which applies + delta and dictionary encodings to id and value columns and remaps parent ids. + `ipc-ser` is the Arrow IPC serialization with compression plus the prost + encoding of the `BatchArrowRecords`. Because `Producer::produce_bar` bundles + the two, `ipc-ser` is reported as the encode total minus `t-enc`. +- IPC `ipc-des` is the prost decode plus `Consumer::consume_bar` plus + `from_record_messages`, which yields a batch still in the transport-optimized + encoding. `t-dec` is `decode_transport_optimized_ids`, which restores the + logical batch. +- Parquet `flatten` builds the single flat Arrow record batch. `pq-write` is the + Arrow Parquet writer. `pq-read` is the Arrow Parquet reader, and `unflat` + reconstructs the four OTAP record batches. + +The flattened layouts keep the entire root `Logs` record batch intact, so decode +carries its scalar and struct columns straight back just as the IPC path does. +Only the attribute tables are denormalized and rebuilt, re-normalizing resource +and scope sets with the `resource.id` and `scope.id` join keys the `Logs` batch +carries. + +## Illustrative results + +These numbers come from one development machine running WSL with jemalloc, at the +50,000 log-record shape, which is a large but valid single OTAP batch. Absolute +values vary by host, but the relationships are stable. + +Serialized size in bytes: + +| contender | zstd | lz4 | snappy | none | +|----------------|---------|---------|---------|------------| +| ipc | 401,966 | 466,416 | n/a | 12,221,748 | +| parquet-nested | 240,211 | 329,298 | 405,397 | 2,118,185 | +| parquet-wide | 245,757 | 327,547 | 327,294 | 344,708 | + +OTAP/IPC pipeline breakdown, milliseconds: + +| comp | t-enc | ipc-ser | enc-tot | ipc-des | t-dec | dec-tot | +|------|-------|---------|---------|---------|-------|---------| +| zstd | 13.0 | 5.9 | 18.9 | 3.0 | 1.2 | 4.2 | +| lz4 | 13.5 | 2.6 | 16.1 | 18.7 | 1.3 | 20.0 | +| none | 12.6 | 29.6 | 42.2 | 27.1 | 1.2 | 28.3 | + +Parquet pipeline breakdown, milliseconds: + +| scheme / comp | flatten | pq-write | enc-tot | pq-read | unflat | dec-tot | +|-----------------------|---------|----------|---------|---------|--------|---------| +| parquet-nested / zstd | 44 | 127 | 171 | 39 | 16 | 55 | +| parquet-nested / lz4 | 44 | 116 | 160 | 35 | 17 | 51 | +| parquet-map / zstd | 52 | 133 | 185 | 53 | 14 | 67 | +| parquet-map / lz4 | 52 | 110 | 162 | 53 | 14 | 66 | +| parquet-wide / zstd | 69 | 25 | 94 | 14 | 20 | 34 | +| parquet-wide / lz4 | 69 | 22 | 92 | 12 | 20 | 32 | + +Streaming amortization, IPC bytes per batch when a long-lived Producer streams +many batches, with the equivalent single Parquet file for reference. `cold` is +the first batch, `warm` is steady-state, and `saved` is the fixed schema and +dictionary cost that streaming amortizes: + +| logs | comp | cold | warm | saved | pq-nested | warm/pq | +|--------|------|---------|---------|--------|-----------|---------| +| 1,000 | zstd | 24,748 | 13,422 | 11,326 | 19,512 | 0.69x | +| 10,000 | zstd | 92,270 | 80,944 | 11,326 | 57,023 | 1.42x | +| 50,000 | zstd | 401,966 | 390,640 | 11,326 | 240,211 | 1.63x | + +OTAP-flat single columnar view. `convert` is the cost of turning the four OTAP +record batches into one, `view-mem` is the in-memory footprint of that view, and +`pq-write`/`pq-bytes` are the Parquet encode where arrow-rs can write it. The +`otap-flat-*` rows exploit the fact that OTAP attribute batches are already +grouped by `parent_id`, so the per-record log attributes are a zero-copy +`List` and only the layout of the shared resource/scope sets differs. The +resource-heavy shape is 60k logs across 600 resources with twenty attributes +each: + +| contender | convert ms | view mem | pq-write ms | pq bytes | writable | +| ---------------------- | ---------: | --------: | ----------: | --------: | -------- | +| nested, baseline | 94.3 | 101.9 MB | 355.6 | 2,750,262 | yes | +| otap-flat-materialized | 86.8 | 101.9 MB | 380.0 | 2,750,262 | yes | +| otap-flat-ree | 4.4 | 12.7 MB | n/a | n/a | no | +| otap-flat-dict | 4.3 | 12.9 MB | n/a | n/a | no | + +Service-to-service transfer. Each form is serialized the way it would travel: +OTAP-standard is the transport-optimized `Producer` over the normalized batches, +each flat layout is one Arrow IPC stream, and Parquet is the flat file. `zstd +wire` and `lz4 wire` are the compressed bytes, `encode`/`decode` are the CPU to +and from the wire, and `receiver form` is what the decoder yields. On realistic +data the forms sit within about a factor of two on the wire, apart from the +materialized flat form, and Parquet is smallest at rest but most expensive on +CPU: + +| contender | zstd wire | lz4 wire | encode ms | decode ms | receiver form | +| --------------------- | ---------: | ---------: | --------: | --------: | --------------- | +| ipc-standard | 3,013,172 | 3,841,908 | 28.0 | 5.6 | normalized OTAP | +| ipc-flat-materialized | 12,137,288 | 17,397,896 | 153.3 | 65.4 | flat table | +| ipc-flat-ree | 3,345,992 | 4,736,200 | 15.1 | 6.5 | flat table | +| ipc-flat-dict | 3,345,544 | 4,737,672 | 15.5 | 6.3 | flat table | +| parquet-flat | 2,750,262 | 3,783,505 | 416.4 | 172.6 | flat table | + +Conversion-cost matrix. Every directed edge of the pipeline, timed in isolation, +where `S` is OTAP-standard, `F` is OTAP-flat/REE, `Ws`/`Wf` are their Arrow IPC +wire forms, and `P` is Parquet. The two OTAP forms are the cheapest pair to move +between, because they share ten of the flat table's thirteen columns and only the +attribute containers change: + +| edge | log-heavy | resource-heavy | +| ---------------------------------- | --------: | -------------: | +| S -> F flatten to REE | 6.1 | 1.7 | +| F -> S unflatten | 12.9 | 2.9 | +| S -> Ws standard serialize | 62.9 | 25.8 | +| Ws -> S standard deserialize | 9.9 | 4.5 | +| F -> Wf flat serialize | 37.0 | 12.2 | +| Wf -> F flat deserialize | 16.1 | 6.4 | +| F -> P parquet-ready (REE+FSB) | 13.8 | 58.7 | +| F -> P parquet write | 144.9 | 242.8 | +| P -> F parquet read | 52.8 | 140.1 | + +Naive versus optimized OTAP-to-Parquet for the common OTLP -> batch -> Parquet +case, where the batch is fresh and `parent_id`-grouped. `naive` is the hash-join +flatten from `ANALYSIS.md`; `optimized` is the shared-column zero-copy build. +Both write byte-identical Parquet, so only the flatten differs and the +prepare-and-write is shared: + +| scenario | flatten naive | flatten opt | prep+write | total naive | total opt | +| -------------- | ------------: | ----------: | ---------: | ----------: | --------: | +| log-heavy | 47.7 | 10.7 | 168.5 | 216.3 | 179.3 | +| resource-heavy | 79.6 | 60.4 | 265.1 | 344.7 | 325.5 | + +A companion write-up of what these ratios mean, including the streaming effect, +is in [`ANALYSIS.md`](./ANALYSIS.md). The OTAP-flat single columnar view, the +transfer comparison, and the natural-center conversion analysis are in +[`OTAP_FLAT_ANALYSIS.md`](./OTAP_FLAT_ANALYSIS.md). + +## Takeaways + +- IPC is far cheaper than Parquet on both sides. At 50k with zstd, IPC encodes in + about 19 ms and decodes in about 4 ms, while `parquet-nested` encodes in about + 171 ms and decodes in about 55 ms. That is roughly 9 times cheaper to encode and + 13 times cheaper to decode. +- Inside IPC encode, the transport-optimized encoding dominates, about 13 ms of + the 19 ms, and it is essentially compressor-independent because it runs before + compression. Inside IPC decode, the deserialization dominates and the transport + decode is small. +- Streaming amortizes a fixed cost of about 11 KB per batch, which is roughly two + thirds dictionary messages and one third schema, and is independent of the row + count. For small frequent batches this flips the size verdict: at 1,000 records + the steady-state IPC batch is 0.69x the Parquet file. Parquet has no equivalent + per-batch amortization. Because a batch cannot exceed 65,535 records, high + volume is streamed, which is where this applies. +- Inside Parquet encode, the Parquet writer dominates and the flatten is roughly + a third to a half of the total. `parquet-wide` writes fastest because it has + typed scalar columns rather than nested `List` or `Map`, but it pays + more in flatten, and it ends up the cheapest Parquet encoder overall at 94 ms. +- Compression choice matters in surprising ways. For IPC, compression makes the + serialize step faster because there is far less data to move, so `none` is the + slowest to serialize, and `lz4` is much slower to deserialize than `zstd` in + this Arrow IPC implementation. For Parquet, the writer time is largely + insensitive to the compressor, while the size is not. +- For the debate, if server CPU is the constraint, the client should keep sending + OTAP/IPC, which is an order of magnitude cheaper to produce and consume than + Parquet. Producing Parquet is worth it only where the columnar file and its + smaller `zstd` size are needed at rest, and that cost lands wherever the flatten + and Parquet write run. + +## Extending + +Contenders are the `Scheme` enum and its `Codec` implementations in +`benchmarks/src/parquet_study`. Add a variant to include a contender everywhere. +The IPC sub-steps are `ipc::transport_encode`, `ipc::encode_to_bytes`, +`ipc::deserialize`, and `ipc::transport_decode`, and `ipc::stream_batch_sizes` +measures streaming amortization; the Parquet steps are `Scheme::flatten`, +`parquet_io::write_parquet`, `parquet_io::read_parquet`, and `Scheme::unflatten`. +Input shapes are defined in `input_shapes()` and `streaming_shapes()` in +`benches/otap_parquet/main.rs`. The round-trips have unit tests runnable with +`cargo test -p benchmarks --lib parquet_study`. + +The OTAP-flat single columnar view lives in `parquet_study::otap_flat`. Its +`flatten` and `unflatten` take a `Layout` of `Materialized`, `RunEndEncoded`, or +`Dictionary`, and `in_memory_bytes` reports the view footprint. The two attribute +mixes it sweeps are defined by `RichGenParams` in `parquet_study::datagen`. The +service-to-service transfer table serializes each flat layout as plain Arrow IPC +with `parquet_study::ipc_flat`, which unlike Parquet can carry run-end and +dictionary columns on the wire. `parquet_study::parquet_io::to_parquet_ready` +prepares a flat batch for Parquet by expanding run-end columns and materializing +the dictionary `FixedSizeBinary` `trace_id`/`span_id`, the only two encodings +arrow-rs cannot round-trip, and the conversion-cost matrix in `main.rs` times +every pipeline edge in isolation. diff --git a/rust/otap-dataflow/benchmarks/benches/otap_parquet/main.rs b/rust/otap-dataflow/benchmarks/benches/otap_parquet/main.rs new file mode 100644 index 0000000000..e001fe152b --- /dev/null +++ b/rust/otap-dataflow/benchmarks/benches/otap_parquet/main.rs @@ -0,0 +1,772 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +//! Parquet study benchmark. +//! +//! Compares the read/write cost and serialized size of OTAP logs encoded as +//! compressed Arrow IPC (the representation we have today) versus several +//! flattened single-file Parquet layouts, and breaks each pipeline into its +//! sub-steps so the cost of every stage is visible. +//! +//! Starting from an OTAP logs batch (four record batches: Logs, ResourceAttrs, +//! ScopeAttrs, LogAttrs): +//! +//! - OTAP/IPC encode = transport-optimize, then Arrow IPC serialize (+compress). +//! Decode = IPC deserialize, then transport-decode. +//! - Parquet encode = flatten to one Arrow record batch, then write Parquet. +//! Decode = read Parquet, then unflatten. +//! +//! Run with: +//! +//! ```bash +//! cargo bench -p benchmarks --bench otap_parquet +//! ``` +//! +//! Three tables (size, OTAP/IPC breakdown, Parquet breakdown) are printed to +//! stdout before the timed round-trip benchmarks run. + +#![allow(missing_docs)] +// This benchmark intentionally prints comparison tables to stdout before +// running the timed measurements. +#![allow(clippy::print_stdout)] + +use criterion::{BatchSize, BenchmarkId, Criterion, criterion_group, criterion_main}; +use std::hint::black_box; +use std::time::{Duration, Instant}; + +use benchmarks::parquet_study::datagen::{LogsGenParams, gen_logs_otap}; +use benchmarks::parquet_study::otap_flat::{self, Layout}; +use benchmarks::parquet_study::parquet_io::{read_parquet, to_parquet_ready, write_parquet}; +use benchmarks::parquet_study::{Compressor, Scheme, ipc, ipc_flat}; + +#[cfg(not(windows))] +use tikv_jemallocator::Jemalloc; + +#[cfg(not(windows))] +#[global_allocator] +static GLOBAL: Jemalloc = Jemalloc; + +/// Input shapes for the breakdown: block sizes larger than a few thousand log +/// records, under a single resource/scope. A single OTAP logs batch caps at +/// 65,535 records because log ids are u16, so these stay below that limit; larger +/// volumes must be streamed as multiple batches (see the streaming table). +fn input_shapes() -> Vec { + [10_000usize, 30_000, 60_000] + .into_iter() + .map(|num_logs| LogsGenParams { + num_resources: 1, + num_scopes: 1, + num_logs, + }) + .collect() +} + +/// Batch sizes for the streaming table, spanning small to large so the fixed +/// per-batch schema/dictionary overhead is visible as a fraction of the batch. +fn streaming_shapes() -> Vec { + [1_000usize, 10_000, 50_000] + .into_iter() + .map(|num_logs| LogsGenParams { + num_resources: 1, + num_scopes: 1, + num_logs, + }) + .collect() +} + +/// Median wall-clock milliseconds of `f` over a few iterations (with one warm-up +/// pass). Indicative only; the Criterion round-trip group gives rigorous totals. +fn median_ms(mut f: impl FnMut()) -> f64 { + f(); + let iters = 3; + let mut samples = Vec::with_capacity(iters); + for _ in 0..iters { + let start = Instant::now(); + f(); + samples.push(start.elapsed().as_secs_f64() * 1e3); + } + samples.sort_by(|a, b| a.partial_cmp(b).expect("no NaN")); + samples[samples.len() / 2] +} + +/// Serialized-size comparison for every contender x compressor x shape. +fn print_size_table(shapes: &[LogsGenParams]) { + println!("\n=== OTAP logs serialized size (bytes) ==="); + println!( + "{:<16} {:<8} {:>12} {:>10} {:>10} {:>12}", + "contender", "comp", "bytes", "vs-otlp", "b/log", "vs-ipc-zstd" + ); + for shape in shapes { + let (otap, proto_len) = gen_logs_otap(shape); + let total_logs = shape.total_logs(); + println!( + "-- shape {} log records, OTLP proto = {} bytes --", + total_logs, proto_len + ); + let ipc_zstd = Scheme::Ipc + .codec(Compressor::Zstd) + .write(otap.clone()) + .expect("ipc zstd write") + .len(); + for scheme in Scheme::all() { + for &compressor in scheme.compressors() { + let codec = scheme.codec(compressor); + let bytes = codec.write(otap.clone()).expect("write").len(); + println!( + "{:<16} {:<8} {:>12} {:>9.2}x {:>10.1} {:>11.2}x", + codec.name(), + compressor.label(), + bytes, + proto_len as f64 / bytes as f64, + bytes as f64 / total_logs as f64, + bytes as f64 / ipc_zstd as f64, + ); + } + } + println!(); + } +} + +/// Per-step breakdown of the OTAP/IPC encode and decode pipelines. +fn print_ipc_breakdown(shapes: &[LogsGenParams]) { + println!("\n=== OTAP/IPC pipeline breakdown (indicative ms) ==="); + println!("encode = transport-optimize + Arrow-IPC-serialize(+compress)"); + println!("decode = IPC-deserialize + transport-decode"); + println!( + "{:<6} {:>9} {:>9} {:>9} {:>9} {:>9} {:>9} {:>10}", + "comp", "t-enc", "ipc-ser", "enc-tot", "ipc-des", "t-dec", "dec-tot", "bytes" + ); + for shape in shapes { + let (otap, _) = gen_logs_otap(shape); + println!("-- shape {} log records --", shape.total_logs()); + for &comp in Scheme::Ipc.compressors() { + let t_enc = median_ms(|| { + let mut o = otap.clone(); + ipc::transport_encode(&mut o).expect("transport encode"); + }); + let enc_tot = median_ms(|| { + let _ = ipc::encode_to_bytes(otap.clone(), comp).expect("encode"); + }); + let ipc_ser = (enc_tot - t_enc).max(0.0); + + let bytes = ipc::encode_to_bytes(otap.clone(), comp).expect("encode"); + let optimized = ipc::deserialize(&bytes).expect("deserialize"); + let ipc_des = median_ms(|| { + let _ = ipc::deserialize(&bytes).expect("deserialize"); + }); + let t_dec = median_ms(|| { + let mut o = optimized.clone(); + ipc::transport_decode(&mut o).expect("transport decode"); + }); + + println!( + "{:<6} {:>8.2} {:>8.2} {:>8.2} {:>8.2} {:>8.2} {:>8.2} {:>10}", + comp.label(), + t_enc, + ipc_ser, + enc_tot, + ipc_des, + t_dec, + ipc_des + t_dec, + bytes.len(), + ); + } + println!(); + } +} + +/// Per-step breakdown of the Parquet encode and decode pipelines. +fn print_parquet_breakdown(shapes: &[LogsGenParams]) { + println!("\n=== Parquet pipeline breakdown (indicative ms) ==="); + println!("encode = flatten + parquet-write decode = parquet-read + unflatten"); + println!( + "{:<16} {:<8} {:>9} {:>9} {:>9} {:>9} {:>9} {:>9} {:>10}", + "scheme", "comp", "flatten", "pq-write", "enc-tot", "pq-read", "unflat", "dec-tot", "bytes" + ); + for shape in shapes { + let (otap, _) = gen_logs_otap(shape); + println!("-- shape {} log records --", shape.total_logs()); + for scheme in Scheme::flattened() { + let flat = scheme.flatten(&otap).expect("flatten"); + let flatten_t = median_ms(|| { + let _ = scheme.flatten(&otap).expect("flatten"); + }); + for compressor in Compressor::ALL { + let write_t = median_ms(|| { + let _ = write_parquet(&flat, compressor.parquet()).expect("write"); + }); + let bytes = write_parquet(&flat, compressor.parquet()).expect("write"); + let read_flat = read_parquet(&bytes).expect("read"); + let read_t = median_ms(|| { + let _ = read_parquet(&bytes).expect("read"); + }); + let unflatten_t = median_ms(|| { + let _ = scheme.unflatten(&read_flat).expect("unflatten"); + }); + println!( + "{:<16} {:<8} {:>8.2} {:>8.2} {:>8.2} {:>8.2} {:>8.2} {:>8.2} {:>10}", + scheme.name(), + compressor.label(), + flatten_t, + write_t, + flatten_t + write_t, + read_t, + unflatten_t, + read_t + unflatten_t, + bytes.len(), + ); + } + } + println!(); + } +} + +/// OTAP-flat study: the cost of presenting the four OTAP record batches as a +/// single columnar view, and the size of that view, for three layouts of the +/// shared resource/scope attributes. `nested` is the baseline flatten (hash join +/// plus full `take`); the `otap-flat-*` rows exploit the fact that OTAP +/// attribute batches are already grouped by `parent_id`, so the per-row log +/// attributes are a zero-copy `List` and only the layout of the shared +/// resource/scope sets differs. `convert` is OTAP -> single view; `view-mem` is +/// the in-memory footprint of the view; `pq-write`/`pq-bytes` are the Parquet +/// encode of the view when arrow-rs can write it (materialized only -- REE and +/// dictionary-of-`List` are in-memory/query forms). +/// +/// Two scenarios hold the record count fixed and vary the attribute mix, because +/// REE and dictionary only save on the *shared* resource/scope attributes: +/// +/// - `log-heavy`: many per-record log attributes, few resource/scope attributes. +/// - `resource-heavy`: many resources each with many attributes, few log +/// attributes, which is where storing the shared sets once pays off. +fn print_otap_flat_table() { + use benchmarks::parquet_study::datagen::{RichGenParams, gen_logs_otap_rich}; + + let scenarios = [ + RichGenParams { + label: "log-heavy", + num_resources: 1, + num_scopes: 1, + num_logs: 60_000, + num_resource_attrs: 1, + num_scope_attrs: 2, + num_log_attrs: 9, + }, + RichGenParams { + label: "resource-heavy", + num_resources: 600, + num_scopes: 1, + num_logs: 100, + num_resource_attrs: 20, + num_scope_attrs: 5, + num_log_attrs: 2, + }, + ]; + + println!("\n=== OTAP-flat single columnar view (indicative ms, bytes) ==="); + println!("convert = OTAP -> one RecordBatch; view-mem = in-memory footprint of the view."); + println!("pq-write/pq-bytes use zstd; REE and dict cannot be written to Parquet by arrow-rs."); + println!( + "{:<24} {:>10} {:>12} {:>10} {:>12} {:>6}", + "contender", "convert-ms", "view-mem", "pq-write", "pq-bytes", "pq-ok" + ); + for scenario in &scenarios { + let otap = gen_logs_otap_rich(scenario); + println!( + "-- {} : {} logs, {} resources x {} resource-attrs, {} log-attrs --", + scenario.label, + scenario.total_logs(), + scenario.num_resources, + scenario.num_resource_attrs, + scenario.num_log_attrs, + ); + + // Baseline: the existing nested flatten (hash join + full take). + let base_convert = median_ms(|| { + let _ = Scheme::Nested.flatten(&otap).expect("nested flatten"); + }); + let base_flat = Scheme::Nested.flatten(&otap).expect("nested flatten"); + let base_mem = otap_flat::in_memory_bytes(&base_flat); + let base_pq_input = to_parquet_ready(&base_flat).expect("parquet-ready"); + let base_pq_ms = median_ms(|| { + let _ = write_parquet(&base_pq_input, Compressor::Zstd.parquet()).expect("write"); + }); + let base_pq = write_parquet(&base_pq_input, Compressor::Zstd.parquet()) + .expect("write") + .len(); + println!( + "{:<24} {:>10.2} {:>12} {:>10.2} {:>12} {:>6}", + "nested (baseline)", base_convert, base_mem, base_pq_ms, base_pq, "yes" + ); + + for layout in [ + Layout::Materialized, + Layout::RunEndEncoded, + Layout::Dictionary, + ] { + let convert = median_ms(|| { + let _ = otap_flat::flatten(&otap, layout).expect("flatten"); + }); + let flat = otap_flat::flatten(&otap, layout).expect("flatten"); + let mem = otap_flat::in_memory_bytes(&flat); + if layout.parquet_writable() { + let pq_input = to_parquet_ready(&flat).expect("parquet-ready"); + let pq_ms = median_ms(|| { + let _ = write_parquet(&pq_input, Compressor::Zstd.parquet()).expect("write"); + }); + let pq_bytes = write_parquet(&pq_input, Compressor::Zstd.parquet()) + .expect("write") + .len(); + println!( + "{:<24} {:>10.2} {:>12} {:>10.2} {:>12} {:>6}", + layout.name(), + convert, + mem, + pq_ms, + pq_bytes, + "yes" + ); + } else { + println!( + "{:<24} {:>10.2} {:>12} {:>10} {:>12} {:>6}", + layout.name(), + convert, + mem, + "n/a", + "n/a", + "no" + ); + } + } + println!(); + } +} + +/// Transfer study: how OTAP-flat compares to OTAP-standard and to Parquet when +/// the question is moving data between two large services. Each row reports the +/// serialized wire size under zstd and lz4, the encode cost from an OTAP batch to +/// wire bytes, the decode cost from wire bytes to the receiver's working form, +/// and what that working form is. +/// +/// - `ipc-standard` is the OTAP representation today: the transport-optimized +/// `Producer` over the four normalized batches, decoded back to normalized +/// OTAP. +/// - `ipc-flat-*` is one flat record batch serialized as plain Arrow IPC. Arrow +/// IPC can carry `RunEndEncoded` and dictionary columns, so the compact +/// resource/scope layouts survive on the wire. The receiver gets a single +/// query-ready table; projecting it back to normalized OTAP is an extra +/// `otap_flat::unflatten` that this table does not include. +/// - `parquet-flat` is the same flat batch written as a Parquet file. +/// +/// encode and decode are timed with zstd. +fn print_transfer_table() { + use benchmarks::parquet_study::datagen::{RichGenParams, gen_logs_otap_rich}; + + let scenarios = [ + RichGenParams { + label: "log-heavy", + num_resources: 1, + num_scopes: 1, + num_logs: 60_000, + num_resource_attrs: 1, + num_scope_attrs: 2, + num_log_attrs: 9, + }, + RichGenParams { + label: "resource-heavy", + num_resources: 600, + num_scopes: 1, + num_logs: 100, + num_resource_attrs: 20, + num_scope_attrs: 5, + num_log_attrs: 2, + }, + ]; + + println!("\n=== Transfer between two services: wire size and CPU (indicative) ==="); + println!("encode = OTAP batch -> wire bytes; decode = wire bytes -> receiver working form."); + println!("ipc-flat carries REE/dict on the wire (Parquet cannot); decode yields one table."); + println!( + "{:<22} {:>12} {:>12} {:>10} {:>10} {:<18}", + "contender", "zstd-bytes", "lz4-bytes", "encode-ms", "decode-ms", "receiver-form" + ); + for scenario in &scenarios { + let otap = gen_logs_otap_rich(scenario); + println!( + "-- {} : {} logs, {} resources x {} resource-attrs, {} log-attrs --", + scenario.label, + scenario.total_logs(), + scenario.num_resources, + scenario.num_resource_attrs, + scenario.num_log_attrs, + ); + + // OTAP standard: transport-optimized Producer over the normalized batches. + { + let zstd_bytes = ipc::encode_to_bytes(otap.clone(), Compressor::Zstd) + .expect("encode") + .len(); + let lz4_bytes = ipc::encode_to_bytes(otap.clone(), Compressor::Lz4) + .expect("encode") + .len(); + let encode_ms = median_ms(|| { + let _ = ipc::encode_to_bytes(otap.clone(), Compressor::Zstd).expect("encode"); + }); + let bytes = ipc::encode_to_bytes(otap.clone(), Compressor::Zstd).expect("encode"); + let decode_ms = median_ms(|| { + let mut o = ipc::deserialize(&bytes).expect("deserialize"); + ipc::transport_decode(&mut o).expect("transport decode"); + }); + println!( + "{:<22} {:>12} {:>12} {:>10.2} {:>10.2} {:<18}", + "ipc-standard", zstd_bytes, lz4_bytes, encode_ms, decode_ms, "normalized OTAP" + ); + } + + // OTAP flat: one Arrow IPC batch, three shared-attribute layouts. + for layout in [ + Layout::Materialized, + Layout::RunEndEncoded, + Layout::Dictionary, + ] { + let flat = otap_flat::flatten(&otap, layout).expect("flatten"); + let zstd_bytes = ipc_flat::write_ipc(&flat, Compressor::Zstd) + .expect("write ipc") + .len(); + let lz4_bytes = ipc_flat::write_ipc(&flat, Compressor::Lz4) + .expect("write ipc") + .len(); + let encode_ms = median_ms(|| { + let f = otap_flat::flatten(&otap, layout).expect("flatten"); + let _ = ipc_flat::write_ipc(&f, Compressor::Zstd).expect("write ipc"); + }); + let bytes = ipc_flat::write_ipc(&flat, Compressor::Zstd).expect("write ipc"); + let decode_ms = median_ms(|| { + let _ = ipc_flat::read_ipc(&bytes).expect("read ipc"); + }); + let name = format!("ipc-flat-{}", layout_suffix(layout)); + println!( + "{:<22} {:>12} {:>12} {:>10.2} {:>10.2} {:<18}", + name, zstd_bytes, lz4_bytes, encode_ms, decode_ms, "flat table" + ); + } + + // Parquet: the same flat batch as a Parquet file. Arrow dictionaries are + // materialized first because arrow-rs cannot read a dictionary-encoded + // FixedSizeBinary (trace_id/span_id) back from Parquet. + { + let flat = Scheme::Nested.flatten(&otap).expect("nested flatten"); + let pq_input = to_parquet_ready(&flat).expect("parquet-ready"); + let zstd_bytes = write_parquet(&pq_input, Compressor::Zstd.parquet()) + .expect("write") + .len(); + let lz4_bytes = write_parquet(&pq_input, Compressor::Lz4.parquet()) + .expect("write") + .len(); + let encode_ms = median_ms(|| { + let f = Scheme::Nested.flatten(&otap).expect("nested flatten"); + let d = to_parquet_ready(&f).expect("parquet-ready"); + let _ = write_parquet(&d, Compressor::Zstd.parquet()).expect("write"); + }); + let bytes = write_parquet(&pq_input, Compressor::Zstd.parquet()).expect("write"); + let decode_ms = median_ms(|| { + let _ = read_parquet(&bytes).expect("read"); + }); + println!( + "{:<22} {:>12} {:>12} {:>10.2} {:>10.2} {:<18}", + "parquet-flat", zstd_bytes, lz4_bytes, encode_ms, decode_ms, "flat table" + ); + } + println!(); + } +} + +/// Short suffix for a shared-attribute layout, used in transfer row names. +fn layout_suffix(layout: Layout) -> &'static str { + match layout { + Layout::Materialized => "materialized", + Layout::RunEndEncoded => "ree", + Layout::Dictionary => "dict", + } +} + +/// Conversion-cost matrix: every directed edge of the pipeline, timed in +/// isolation, so the cost of moving between representations is visible edge by +/// edge. The nodes are OTAP-standard (`S`, four normalized batches), +/// OTAP-flat/REE (`F`, one batch with run-end resource/scope columns), +/// OTAP/IPC-standard (`Ws`), OTAP/IPC-flat (`Wf`), and Parquet (`P`). The flat to +/// Parquet edge is split into the parquet-ready transform, which expands the +/// run-end columns and materializes `trace_id`/`span_id`, and the Parquet write. +fn print_conversion_matrix() { + use benchmarks::parquet_study::datagen::{RichGenParams, gen_logs_otap_rich}; + use otap_df_pdata::otap::OtapArrowRecords; + + let scenarios = [ + RichGenParams { + label: "log-heavy", + num_resources: 1, + num_scopes: 1, + num_logs: 60_000, + num_resource_attrs: 1, + num_scope_attrs: 2, + num_log_attrs: 9, + }, + RichGenParams { + label: "resource-heavy", + num_resources: 600, + num_scopes: 1, + num_logs: 100, + num_resource_attrs: 20, + num_scope_attrs: 5, + num_log_attrs: 2, + }, + ]; + + let edges = [ + "S -> F flatten to REE", + "F -> S unflatten", + "S -> Ws standard serialize", + "Ws -> S standard deserialize", + "F -> Wf flat serialize", + "Wf -> F flat deserialize", + "F -> P parquet-ready (REE+FSB)", + "F -> P parquet write", + "P -> F parquet read", + ]; + + fn measure(otap: &OtapArrowRecords) -> Vec { + let flat = otap_flat::flatten(otap, Layout::RunEndEncoded).expect("flatten"); + let ws = ipc::encode_to_bytes(otap.clone(), Compressor::Zstd).expect("ws"); + let wf = ipc_flat::write_ipc(&flat, Compressor::Zstd).expect("wf"); + let ready = to_parquet_ready(&flat).expect("ready"); + let pq = write_parquet(&ready, Compressor::Zstd.parquet()).expect("pq"); + + vec![ + median_ms(|| { + let _ = otap_flat::flatten(otap, Layout::RunEndEncoded).expect("flatten"); + }), + median_ms(|| { + let _ = otap_flat::unflatten(&flat).expect("unflatten"); + }), + median_ms(|| { + let _ = ipc::encode_to_bytes(otap.clone(), Compressor::Zstd).expect("ws"); + }), + median_ms(|| { + let mut o = ipc::deserialize(&ws).expect("des"); + ipc::transport_decode(&mut o).expect("tdec"); + }), + median_ms(|| { + let _ = ipc_flat::write_ipc(&flat, Compressor::Zstd).expect("wf"); + }), + median_ms(|| { + let _ = ipc_flat::read_ipc(&wf).expect("rf"); + }), + median_ms(|| { + let _ = to_parquet_ready(&flat).expect("ready"); + }), + median_ms(|| { + let _ = write_parquet(&ready, Compressor::Zstd.parquet()).expect("pq"); + }), + median_ms(|| { + let _ = read_parquet(&pq).expect("read"); + }), + ] + } + + println!("\n=== Conversion cost matrix (indicative ms) ==="); + println!( + "S=OTAP-standard, F=OTAP-flat(REE), Ws=OTAP/IPC-standard, Wf=OTAP/IPC-flat, P=Parquet." + ); + let log = measure(&gen_logs_otap_rich(&scenarios[0])); + let res = measure(&gen_logs_otap_rich(&scenarios[1])); + println!( + "{:<34} {:>12} {:>14}", + "edge", "log-heavy", "resource-heavy" + ); + for (i, name) in edges.iter().enumerate() { + println!("{:<34} {:>12.2} {:>14.2}", name, log[i], res[i]); + } + println!(); +} + +/// The common OTLP -> batch -> Parquet path, comparing the naive hash-join +/// flatten used in `ANALYSIS.md` against the optimized shared-column build. Both +/// produce byte-identical Parquet, so only the flatten differs; the Parquet +/// prepare-and-write cost is shared. This assumes a fresh, `parent_id`-grouped +/// batch straight from the encoder, which is exactly the OTLP -> batch -> Parquet +/// case and is where the zero-copy log-attribute build applies. +fn print_parquet_fastpath() { + use benchmarks::parquet_study::datagen::{RichGenParams, gen_logs_otap_rich}; + + let scenarios = [ + RichGenParams { + label: "log-heavy", + num_resources: 1, + num_scopes: 1, + num_logs: 60_000, + num_resource_attrs: 1, + num_scope_attrs: 2, + num_log_attrs: 9, + }, + RichGenParams { + label: "resource-heavy", + num_resources: 600, + num_scopes: 1, + num_logs: 100, + num_resource_attrs: 20, + num_scope_attrs: 5, + num_log_attrs: 2, + }, + ]; + + println!("\n=== OTAP -> Parquet: naive vs optimized flatten (indicative ms) ==="); + println!("naive = hash-join flatten (ANALYSIS.md); optimized = shared-column zero-copy build."); + println!("Both write byte-identical Parquet; only the flatten differs. Assumes a fresh"); + println!("parent_id-grouped batch, i.e. the OTLP -> batch -> Parquet case."); + println!( + "{:<16} {:>11} {:>10} {:>11} {:>12} {:>11} {:>12}", + "scenario", "flat-naive", "flat-opt", "prep+write", "total-naive", "total-opt", "pq-bytes" + ); + for s in &scenarios { + let otap = gen_logs_otap_rich(s); + let flat_opt = otap_flat::flatten(&otap, Layout::Materialized).expect("opt flatten"); + let ready = to_parquet_ready(&flat_opt).expect("ready"); + let pq_opt = write_parquet(&ready, Compressor::Zstd.parquet()).expect("write"); + // Confirm the naive path yields the same Parquet size. + let naive_ready = + to_parquet_ready(&Scheme::Nested.flatten(&otap).expect("naive")).expect("ready"); + let pq_naive = write_parquet(&naive_ready, Compressor::Zstd.parquet()).expect("write"); + debug_assert_eq!( + pq_opt.len(), + pq_naive.len(), + "naive/opt Parquet size differs" + ); + + let flat_naive_ms = median_ms(|| { + let _ = Scheme::Nested.flatten(&otap).expect("naive flatten"); + }); + let flat_opt_ms = median_ms(|| { + let _ = otap_flat::flatten(&otap, Layout::Materialized).expect("opt flatten"); + }); + let prep_write_ms = median_ms(|| { + let r = to_parquet_ready(&flat_opt).expect("ready"); + let _ = write_parquet(&r, Compressor::Zstd.parquet()).expect("write"); + }); + println!( + "{:<16} {:>11.1} {:>10.1} {:>11.1} {:>12.1} {:>11.1} {:>12}", + s.label, + flat_naive_ms, + flat_opt_ms, + prep_write_ms, + flat_naive_ms + prep_write_ms, + flat_opt_ms + prep_write_ms, + pq_opt.len(), + ); + } + println!(); +} + +/// OTAP/IPC streaming amortization: cold (first) versus warm (steady-state) +/// per-batch size when a single long-lived Producer streams many batches, with +/// the equivalent single Parquet file for reference. +fn print_streaming_table(shapes: &[LogsGenParams]) { + println!("\n=== OTAP/IPC streaming amortization (bytes per batch) ==="); + println!("One long-lived Producer streams batches: schema once, delta dictionaries."); + println!("cold = first batch (schema + full dictionaries + data); warm = steady-state batch."); + println!( + "pq-nested is the same batch as one Parquet file, which has no per-batch amortization." + ); + println!( + "{:<8} {:<6} {:>12} {:>12} {:>10} {:>12} {:>9}", + "logs", "comp", "cold", "warm", "saved", "pq-nested", "warm/pq" + ); + for shape in shapes { + let (otap, _) = gen_logs_otap(shape); + let flat = Scheme::Nested.flatten(&otap).expect("flatten"); + for &comp in Scheme::Ipc.compressors() { + let sizes = ipc::stream_batch_sizes(&otap, comp, 6).expect("stream sizes"); + let cold = sizes[0]; + let warm = *sizes.last().expect("non-empty"); + let pq = write_parquet(&flat, comp.parquet()) + .expect("parquet write") + .len(); + println!( + "{:<8} {:<6} {:>12} {:>12} {:>10} {:>12} {:>8.2}x", + shape.total_logs(), + comp.label(), + cold, + warm, + cold - warm, + pq, + warm as f64 / pq as f64, + ); + } + } + println!(); +} + +fn bench_round_trip(c: &mut Criterion) { + let shapes = input_shapes(); + print_size_table(&shapes); + print_ipc_breakdown(&shapes); + print_parquet_breakdown(&shapes); + print_otap_flat_table(); + print_transfer_table(); + print_conversion_matrix(); + print_parquet_fastpath(); + print_streaming_table(&streaming_shapes()); + + let mut write_group = c.benchmark_group("parquet_study/write"); + let _ = write_group.sample_size(10); + let _ = write_group.warm_up_time(Duration::from_millis(500)); + let _ = write_group.measurement_time(Duration::from_secs(3)); + for shape in &shapes { + let (otap, _) = gen_logs_otap(shape); + for scheme in Scheme::all() { + for &compressor in scheme.compressors() { + let codec = scheme.codec(compressor); + let id = BenchmarkId::new( + format!("{}/{}", codec.name(), compressor.label()), + shape.total_logs(), + ); + let _ = write_group.bench_with_input(id, shape, |b, _| { + b.iter_batched( + || otap.clone(), + |input| black_box(codec.write(input).expect("write")), + BatchSize::SmallInput, + ); + }); + } + } + } + write_group.finish(); + + let mut read_group = c.benchmark_group("parquet_study/read"); + let _ = read_group.sample_size(10); + let _ = read_group.warm_up_time(Duration::from_millis(500)); + let _ = read_group.measurement_time(Duration::from_secs(3)); + for shape in &shapes { + let (otap, _) = gen_logs_otap(shape); + for scheme in Scheme::all() { + for &compressor in scheme.compressors() { + let codec = scheme.codec(compressor); + let bytes = codec.write(otap.clone()).expect("write"); + let id = BenchmarkId::new( + format!("{}/{}", codec.name(), compressor.label()), + shape.total_logs(), + ); + let _ = read_group.bench_with_input(id, shape, |b, _| { + b.iter(|| black_box(codec.read(&bytes).expect("read"))); + }); + } + } + } + read_group.finish(); +} + +#[allow(missing_docs)] +mod bench_entry { + use super::*; + criterion_group!(benches, bench_round_trip); +} + +criterion_main!(bench_entry::benches); diff --git a/rust/otap-dataflow/benchmarks/src/lib.rs b/rust/otap-dataflow/benchmarks/src/lib.rs new file mode 100644 index 0000000000..572d4153f6 --- /dev/null +++ b/rust/otap-dataflow/benchmarks/src/lib.rs @@ -0,0 +1,13 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +//! Support library for the `otap-dataflow` benchmarks. +//! +//! The heavier benchmark logic lives here (rather than directly in the Criterion +//! `benches/` targets) so it can be unit-tested for correctness. Currently this +//! hosts the [`parquet_study`] module, which compares the cost of moving OTAP +//! logs as compressed Arrow IPC versus several flattened-Parquet encodings. + +#![allow(missing_docs)] + +pub mod parquet_study; diff --git a/rust/otap-dataflow/benchmarks/src/parquet_study/attrs.rs b/rust/otap-dataflow/benchmarks/src/parquet_study/attrs.rs new file mode 100644 index 0000000000..3bc21d5f5c --- /dev/null +++ b/rust/otap-dataflow/benchmarks/src/parquet_study/attrs.rs @@ -0,0 +1,390 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +//! Shared machinery for the flattened-Parquet contenders. +//! +//! OTAP logs are stored as four interleaved record batches: the root `Logs` +//! batch plus three attribute batches (`ResourceAttrs`, `ScopeAttrs`, +//! `LogAttrs`) linked to it by parent id. Each attribute batch has the columns +//! `parent_id, key, type, str, int, double, bool, bytes, ser` where `type` +//! selects which value column is populated. +//! +//! "Flattening" denormalizes those attribute batches onto the log rows. This +//! module provides the pieces every flattened layout needs: +//! +//! - [`extract_attr_value_arrays`] reads the eight value columns from a source +//! attribute batch (synthesizing null columns for any the encoder omitted and +//! normalizing dictionary-encoded keys to plain UTF-8). +//! - [`gather_by_parent`] computes, for each log row, the source attribute rows +//! that belong to it (joining on `resource.id` / `scope.id` / log `id`). +//! - [`build_attr_list_column`] / [`rebuild_attr_batch`] move attributes into a +//! `List` column and back into an OTAP attribute batch. +//! - [`logs_resource_id`], [`logs_scope_id`], [`logs_id`] expose the join keys. +//! - [`assert_logs_equivalent`] checks a round-tripped batch structurally. + +use std::collections::{HashMap, HashSet}; +use std::sync::Arc; + +use arrow::array::{ + Array, ArrayRef, BooleanArray, ListArray, RecordBatch, StructArray, UInt16Array, UInt32Array, + new_empty_array, new_null_array, +}; +use arrow::buffer::{OffsetBuffer, ScalarBuffer}; +use arrow::compute::{cast, take}; +use arrow::datatypes::{DataType, Field, Fields, Schema}; + +use otap_df_pdata::otap::OtapArrowRecords; +use otap_df_pdata::proto::opentelemetry::arrow::v1::ArrowPayloadType; +use otap_df_pdata::schema::consts; + +use super::StudyResult; + +/// Flat-table column holding each log row's denormalized resource attributes. +pub const RESOURCE_ATTRS_COL: &str = "resource_attributes"; +/// Flat-table column holding each log row's denormalized scope attributes. +pub const SCOPE_ATTRS_COL: &str = "scope_attributes"; +/// Flat-table column holding each log row's own attributes. +pub const LOG_ATTRS_COL: &str = "log_attributes"; + +/// The eight value columns of an attribute batch, in canonical order, with the +/// fixed Arrow types used by the flattened representation. `key` and `type` are +/// required; the typed value columns are nullable. +#[must_use] +pub fn attr_struct_fields() -> Fields { + Fields::from(vec![ + Field::new(consts::ATTRIBUTE_KEY, DataType::Utf8, false), + Field::new(consts::ATTRIBUTE_TYPE, DataType::UInt8, false), + Field::new(consts::ATTRIBUTE_STR, DataType::Utf8, true), + Field::new(consts::ATTRIBUTE_INT, DataType::Int64, true), + Field::new(consts::ATTRIBUTE_DOUBLE, DataType::Float64, true), + Field::new(consts::ATTRIBUTE_BOOL, DataType::Boolean, true), + Field::new(consts::ATTRIBUTE_BYTES, DataType::Binary, true), + Field::new(consts::ATTRIBUTE_SER, DataType::Binary, true), + ]) +} + +/// The list element field used for the `List` attribute columns. +#[must_use] +pub fn attr_list_element_field() -> Arc { + Arc::new(Field::new( + "item", + DataType::Struct(attr_struct_fields()), + false, + )) +} + +/// The seven value fields of an attribute (everything except `key`): used as the +/// `value` struct of the Map attribute layout. +#[must_use] +pub fn attr_value_struct_fields() -> Fields { + Fields::from( + attr_struct_fields() + .iter() + .skip(1) + .map(|f| f.as_ref().clone()) + .collect::>(), + ) +} + +/// The flat-table field for one attribute-container column. +#[must_use] +pub fn attr_list_column_field(name: &str) -> Field { + Field::new(name, DataType::List(attr_list_element_field()), false) +} + +fn normalize(array: &ArrayRef, want: &DataType) -> StudyResult { + if array.data_type() == want { + Ok(array.clone()) + } else { + Ok(cast(array, want)?) + } +} + +/// Read the eight value columns (`key, type, str, int, double, bool, bytes, +/// ser`) of an attribute batch in canonical order. Columns the encoder omitted +/// (because no attribute used that value type) are materialized as all-null +/// arrays so the flattened schema is stable, and dictionary-encoded keys are +/// normalized to plain UTF-8. +pub fn extract_attr_value_arrays(attr_batch: &RecordBatch) -> StudyResult> { + let len = attr_batch.num_rows(); + let fields = attr_struct_fields(); + let mut out = Vec::with_capacity(fields.len()); + for field in fields.iter() { + let array = match attr_batch.column_by_name(field.name()) { + Some(col) => normalize(col, field.data_type())?, + None => new_null_array(field.data_type(), len), + }; + out.push(array); + } + Ok(out) +} + +fn downcast_u16(array: &ArrayRef) -> StudyResult { + if array.data_type() == &DataType::UInt16 { + Ok(array + .as_any() + .downcast_ref::() + .expect("checked UInt16") + .clone()) + } else { + let cast = cast(array, &DataType::UInt16)?; + Ok(cast + .as_any() + .downcast_ref::() + .expect("cast to UInt16") + .clone()) + } +} + +/// The `id` column of the root `Logs` batch (parent of `LogAttrs`). +pub fn logs_id(logs: &RecordBatch) -> StudyResult { + let col = logs + .column_by_name(consts::ID) + .ok_or("Logs batch missing `id` column")?; + downcast_u16(col) +} + +fn struct_child_u16(logs: &RecordBatch, struct_col: &str) -> StudyResult { + let col = logs + .column_by_name(struct_col) + .ok_or_else(|| format!("Logs batch missing `{struct_col}` column"))?; + let st = col + .as_any() + .downcast_ref::() + .ok_or_else(|| format!("`{struct_col}` is not a struct"))?; + let id = st + .column_by_name(consts::ID) + .ok_or_else(|| format!("`{struct_col}` struct missing `id`"))?; + downcast_u16(id) +} + +/// The `resource.id` column of the root `Logs` batch (parent of `ResourceAttrs`). +pub fn logs_resource_id(logs: &RecordBatch) -> StudyResult { + struct_child_u16(logs, consts::RESOURCE) +} + +/// The `scope.id` column of the root `Logs` batch (parent of `ScopeAttrs`). +pub fn logs_scope_id(logs: &RecordBatch) -> StudyResult { + struct_child_u16(logs, consts::SCOPE) +} + +/// For each value of `parents` (one per log row), the source attribute rows that +/// share that parent id, as `take` indices plus per-log-row list offsets. +pub struct Gathered { + /// `take` indices into the source attribute batch, concatenated by log row. + pub indices: UInt32Array, + /// List offsets, length `parents.len() + 1`. + pub offsets: Vec, +} + +/// Join a source attribute batch onto the log rows by parent id. +pub fn gather_by_parent(attr_batch: &RecordBatch, parents: &UInt16Array) -> StudyResult { + let parent_col = attr_batch + .column_by_name(consts::PARENT_ID) + .ok_or("attribute batch missing `parent_id`")?; + let parent_id = downcast_u16(parent_col)?; + + let mut by_parent: HashMap> = HashMap::new(); + for row in 0..parent_id.len() { + if parent_id.is_valid(row) { + by_parent + .entry(parent_id.value(row)) + .or_default() + .push(row as u32); + } + } + + let mut indices: Vec = Vec::new(); + let mut offsets: Vec = Vec::with_capacity(parents.len() + 1); + offsets.push(0); + for i in 0..parents.len() { + if parents.is_valid(i) { + if let Some(rows) = by_parent.get(&parents.value(i)) { + indices.extend_from_slice(rows); + } + } + offsets.push(i32::try_from(indices.len()).expect("offset fits i32")); + } + + Ok(Gathered { + indices: UInt32Array::from(indices), + offsets, + }) +} + +/// Take the gathered attribute rows out of `attr_batch` as an eight-field +/// `Struct{key,type,str,int,double,bool,bytes,ser}` array (one struct row per +/// gathered attribute, concatenated across log rows). +pub fn taken_attr_struct( + attr_batch: &RecordBatch, + gathered: &Gathered, +) -> StudyResult { + let value_arrays = extract_attr_value_arrays(attr_batch)?; + let taken: Vec = value_arrays + .iter() + .map(|a| take(a, &gathered.indices, None)) + .collect::>()?; + Ok(StructArray::new(attr_struct_fields(), taken, None)) +} + +/// Build a `List` column denormalizing `attr_batch` +/// onto the log rows described by `gathered`. +pub fn build_attr_list_column( + attr_batch: &RecordBatch, + gathered: &Gathered, +) -> StudyResult { + let struct_array = taken_attr_struct(attr_batch, gathered)?; + let offsets = OffsetBuffer::new(ScalarBuffer::from(gathered.offsets.clone())); + let list = ListArray::new( + attr_list_element_field(), + offsets, + Arc::new(struct_array), + None, + ); + Ok(Arc::new(list)) +} + +/// Rebuild an OTAP attribute batch (`parent_id` + the eight value columns) from +/// a flattened `List` column. `entries` selects which log rows to emit +/// and the parent id to stamp on each emitted attribute row. +pub fn rebuild_attr_batch(list: &ListArray, entries: &[(usize, u16)]) -> StudyResult { + let values = list + .values() + .as_any() + .downcast_ref::() + .ok_or("attribute list values are not a struct")?; + rebuild_attr_batch_from_parts(values, list.value_offsets(), entries) +} + +/// Rebuild an OTAP attribute batch from an eight-field `Struct` of attribute +/// values plus per-log-row `offsets`. Used by both the nested (`List`) +/// and map (`Map`) layouts; the map layout reassembles its `keys`/`values` +/// children into the eight-field struct first. +pub fn rebuild_attr_batch_from_parts( + values: &StructArray, + offsets: &[i32], + entries: &[(usize, u16)], +) -> StudyResult { + let mut child_indices: Vec = Vec::new(); + let mut parent_ids: Vec = Vec::new(); + for (row, pid) in entries { + let start = offsets[*row] as usize; + let end = offsets[*row + 1] as usize; + for child in start..end { + child_indices.push(u32::try_from(child).expect("index fits u32")); + parent_ids.push(*pid); + } + } + let idx = UInt32Array::from(child_indices); + + let mut fields: Vec = Vec::with_capacity(9); + let mut columns: Vec = Vec::with_capacity(9); + fields.push(Field::new(consts::PARENT_ID, DataType::UInt16, false)); + columns.push(Arc::new(UInt16Array::from(parent_ids)) as ArrayRef); + for (i, field) in attr_struct_fields().iter().enumerate() { + fields.push(field.as_ref().clone()); + columns.push(take(values.column(i), &idx, None)?); + } + + Ok(RecordBatch::try_new( + Arc::new(Schema::new(fields)), + columns, + )?) +} + +/// Entries for a child attribute group (`LogAttrs`): every log row with a valid +/// id, stamped with its own id. +#[must_use] +pub fn entries_per_row(id_arr: &UInt16Array) -> Vec<(usize, u16)> { + (0..id_arr.len()) + .filter(|&i| id_arr.is_valid(i)) + .map(|i| (i, id_arr.value(i))) + .collect() +} + +/// Entries for a parent attribute group (`ResourceAttrs` / `ScopeAttrs`): the +/// first log row observed for each distinct id, stamped with that id. This +/// re-normalizes the denormalized resource/scope attributes back to one set per +/// resource/scope, using the join id preserved in the flat table. +#[must_use] +pub fn entries_dedup(id_arr: &UInt16Array) -> Vec<(usize, u16)> { + let mut seen: HashSet = HashSet::new(); + let mut entries = Vec::new(); + for i in 0..id_arr.len() { + if id_arr.is_valid(i) && seen.insert(id_arr.value(i)) { + entries.push((i, id_arr.value(i))); + } + } + entries +} + +/// An attribute-container column where every log row has an empty attribute +/// list (used when an attribute payload is absent from the source batch). +#[must_use] +pub fn empty_attr_list_column(num_rows: usize) -> ArrayRef { + let children: Vec = attr_struct_fields() + .iter() + .map(|f| new_empty_array(f.data_type())) + .collect(); + let struct_array = StructArray::new(attr_struct_fields(), children, None); + let offsets = OffsetBuffer::new(ScalarBuffer::from(vec![0i32; num_rows + 1])); + Arc::new(ListArray::new( + attr_list_element_field(), + offsets, + Arc::new(struct_array), + None, + )) +} + +/// Downcast a flat-table column to `ListArray`. +pub fn as_list<'a>(batch: &'a RecordBatch, name: &str) -> StudyResult<&'a ListArray> { + batch + .column_by_name(name) + .ok_or_else(|| format!("flat batch missing `{name}` column"))? + .as_any() + .downcast_ref::() + .ok_or_else(|| format!("`{name}` is not a list").into()) +} + +/// The OTAP `Logs` payload batch out of an [`OtapArrowRecords`]. +pub fn logs_batch(otap: &OtapArrowRecords) -> StudyResult<&RecordBatch> { + otap.get(ArrowPayloadType::Logs) + .ok_or_else(|| "missing Logs payload".into()) +} + +/// Convenience helper used by the boolean attribute path in tests. +#[must_use] +pub fn bool_value(array: &ArrayRef, row: usize) -> Option { + let b = array.as_any().downcast_ref::()?; + b.is_valid(row).then(|| b.value(row)) +} + +/// Structurally compare a round-tripped logs batch against the original: equal +/// log-record count and equal per-payload attribute-row counts. +pub fn assert_logs_equivalent( + original: &OtapArrowRecords, + decoded: &OtapArrowRecords, + codec: &str, + compressor: &str, +) { + let orig_logs = logs_batch(original).expect("original logs"); + let dec_logs = logs_batch(decoded).expect("decoded logs"); + assert_eq!( + orig_logs.num_rows(), + dec_logs.num_rows(), + "{codec}/{compressor}: log record count differs" + ); + + for pt in [ + ArrowPayloadType::ResourceAttrs, + ArrowPayloadType::ScopeAttrs, + ArrowPayloadType::LogAttrs, + ] { + let orig = original.get(pt).map_or(0, RecordBatch::num_rows); + let dec = decoded.get(pt).map_or(0, RecordBatch::num_rows); + assert_eq!( + orig, dec, + "{codec}/{compressor}: {pt:?} attribute count differs" + ); + } +} diff --git a/rust/otap-dataflow/benchmarks/src/parquet_study/datagen.rs b/rust/otap-dataflow/benchmarks/src/parquet_study/datagen.rs new file mode 100644 index 0000000000..dea29d41e3 --- /dev/null +++ b/rust/otap-dataflow/benchmarks/src/parquet_study/datagen.rs @@ -0,0 +1,370 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +//! Generates OTAP logs batches for the parquet study. +//! +//! The generated data mirrors the `otap_encoder` benchmark: a configurable +//! resource x scope x log fan-out where every log record carries a fixed set of +//! mixed-type attributes (string, bool, int, double, bytes, empty, array, +//! kvlist). This exercises every attribute value column (`str/int/double/bool/ +//! bytes/ser`) so the flatten/unflatten round-trips are meaningfully tested. + +use otap_df_pdata::encode::encode_logs_otap_batch; +use otap_df_pdata::otap::OtapArrowRecords; +use otap_df_pdata::proto::opentelemetry::common::v1::{AnyValue, InstrumentationScope, KeyValue}; +use otap_df_pdata::proto::opentelemetry::logs::v1::{ + LogRecord, LogRecordFlags, LogsData, ResourceLogs, ScopeLogs, SeverityNumber, +}; +use otap_df_pdata::proto::opentelemetry::resource::v1::Resource; +use otap_df_pdata::views::otlp::bytes::logs::RawLogsData; +use prost::Message; + +/// Shape of the generated logs data. +#[derive(Clone, Copy, Debug)] +pub struct LogsGenParams { + /// Number of distinct resources. + pub num_resources: usize, + /// Number of scopes within each resource. + pub num_scopes: usize, + /// Number of log records within each scope. + pub num_logs: usize, +} + +impl LogsGenParams { + /// A short id usable as a Criterion benchmark parameter label. + #[must_use] + pub fn label(&self) -> String { + format!( + "r{}_s{}_l{}_n{}", + self.num_resources, + self.num_scopes, + self.num_logs, + self.total_logs(), + ) + } + + /// Total number of log records produced. + #[must_use] + pub fn total_logs(&self) -> usize { + self.num_resources * self.num_scopes * self.num_logs + } +} + +fn sample_log_attributes() -> Vec { + let attr_values = vec![ + AnyValue::new_string("terry"), + AnyValue::new_bool(true), + AnyValue::new_int(5), + AnyValue::new_double(2.0), + AnyValue::new_bytes(b"hi"), + AnyValue { value: None }, + AnyValue::new_array(vec![AnyValue::new_bool(true)]), + AnyValue::new_kvlist(vec![KeyValue::new("key1", AnyValue::new_bool(true))]), + ]; + let mut log_attributes = attr_values + .into_iter() + .enumerate() + .map(|(i, val)| KeyValue { + key: format!("attr{i}"), + value: Some(val), + }) + .collect::>(); + + // an attribute whose value is absent + log_attributes.push(KeyValue { + key: "noneval".to_string(), + value: None, + }); + log_attributes +} + +/// Build an OTLP [`LogsData`] proto message with the requested shape. +#[must_use] +pub fn create_logs_data(params: &LogsGenParams) -> LogsData { + let log_attributes = sample_log_attributes(); + + LogsData::new( + (0..params.num_resources) + .map(|_| { + ResourceLogs::new( + Resource::build() + .attributes(vec![KeyValue::new( + "resource_attr1", + AnyValue::new_string("resource_value"), + )]) + .dropped_attributes_count(1u32), + (0..params.num_scopes) + .map(|_| { + ScopeLogs::new( + InstrumentationScope::build() + .name("library") + .version("scopev1") + .attributes(vec![ + KeyValue::new( + "scope_attr1", + AnyValue::new_string("scope_val1"), + ), + KeyValue::new( + "scope_attr2", + AnyValue::new_string("scope_val2"), + ), + ]) + .dropped_attributes_count(2u32) + .finish(), + (0..params.num_logs) + .map(|_| { + LogRecord::build() + .time_unix_nano(2_000_000_000u64) + .severity_number(SeverityNumber::Info) + .event_name("event1") + .observed_time_unix_nano(3_000_000_000u64) + .trace_id(vec![ + 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, + ]) + .span_id(vec![0, 0, 0, 0, 1, 1, 1, 1]) + .severity_text("Info") + .attributes(log_attributes.clone()) + .dropped_attributes_count(3u32) + .flags(LogRecordFlags::TraceFlagsMask) + .body(AnyValue::new_string("log_body")) + .finish() + }) + .collect::>(), + ) + .set_schema_url("https://schema.opentelemetry.io/scope_schema") + }) + .collect::>(), + ) + .set_schema_url("https://schema.opentelemetry.io/resource_schema") + }) + .collect::>(), + ) +} + +/// Generate an OTAP logs batch ([`OtapArrowRecords::Logs`]) for the given shape, +/// alongside the size in bytes of the equivalent OTLP protobuf encoding (a +/// useful reference point for the "on the wire" comparison). +#[must_use] +pub fn gen_logs_otap(params: &LogsGenParams) -> (OtapArrowRecords, usize) { + let logs_data = create_logs_data(params); + let mut proto_bytes = Vec::new(); + logs_data + .encode(&mut proto_bytes) + .expect("can encode OTLP proto bytes"); + + let view = RawLogsData::new(proto_bytes.as_ref()); + let otap = encode_logs_otap_batch(&view).expect("can encode OTAP logs batch"); + (otap, proto_bytes.len()) +} + +/// Shape of generated logs data with configurable attribute richness, used by +/// the OTAP-flat study to vary the ratio of *shared* resource/scope attributes +/// (which the REE and dictionary layouts store once) to *per-record* log +/// attributes (which every layout stores per row). +/// +/// The generated records model realistic telemetry rather than identical rows: +/// each record has a unique pseudo-random `trace_id`/`span_id`, a varying +/// timestamp, a templated body, and mixed-type log attributes that blend +/// low-cardinality enums with high-cardinality per-record values. Resource and +/// scope attributes are distinct per resource/scope but shared across that +/// resource/scope's records, which is the low-cardinality-shared case the +/// run-end and dictionary layouts target. +#[derive(Clone, Copy, Debug)] +pub struct RichGenParams { + /// A short label for tables. + pub label: &'static str, + /// Number of distinct resources. + pub num_resources: usize, + /// Number of scopes within each resource. + pub num_scopes: usize, + /// Number of log records within each scope. + pub num_logs: usize, + /// Attributes attached to each resource (distinct per resource, shared by + /// that resource's records). + pub num_resource_attrs: usize, + /// Attributes attached to each scope. + pub num_scope_attrs: usize, + /// Mixed-type attributes attached to each log record. + pub num_log_attrs: usize, +} + +impl RichGenParams { + /// Total number of log records produced. + #[must_use] + pub fn total_logs(&self) -> usize { + self.num_resources * self.num_scopes * self.num_logs + } +} + +/// splitmix64: a cheap deterministic mixer used to synthesize high-entropy, +/// unique-per-record trace ids and attribute values, so the generated data does +/// not compress like identical rows. +fn mix64(mut x: u64) -> u64 { + x = x.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = x; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) +} + +/// A unique, high-entropy 16-byte trace id for a global record index. +fn trace_id_for(idx: u64) -> Vec { + let mut b = Vec::with_capacity(16); + b.extend_from_slice(&mix64(idx).to_be_bytes()); + b.extend_from_slice(&mix64(idx ^ 0xDEAD_BEEF).to_be_bytes()); + b +} + +/// A unique 8-byte span id for a global record index. +fn span_id_for(idx: u64) -> Vec { + mix64(idx.wrapping_mul(0x100_0001)).to_be_bytes().to_vec() +} + +/// Distinct-per-resource, low-cardinality resource attributes. +fn resource_attrs_for(res_idx: usize, n: usize) -> Vec { + (0..n) + .map(|i| { + let value = if i == 0 { + format!("service-{res_idx}") + } else { + format!("res{res_idx}-attr{i}") + }; + KeyValue::new(format!("resource_attr{i}"), AnyValue::new_string(value)) + }) + .collect() +} + +/// Distinct-per-scope, low-cardinality scope attributes. +fn scope_attrs_for(scope_idx: usize, n: usize) -> Vec { + (0..n) + .map(|i| { + KeyValue::new( + format!("scope_attr{i}"), + AnyValue::new_string(format!("scope{scope_idx}-attr{i}")), + ) + }) + .collect() +} + +const HTTP_METHODS: [&str; 4] = ["GET", "POST", "PUT", "DELETE"]; + +/// Mixed-type log attributes for a global record index, blending +/// low-cardinality enums with high-cardinality per-record values. +fn log_attrs_for(idx: u64, n: usize) -> Vec { + (0..n) + .map(|i| { + let key = format!("log_attr{i}"); + let value = match i % 5 { + // low-cardinality enum string + 0 => AnyValue::new_string(HTTP_METHODS[(idx as usize + i) % 4].to_string()), + // high-cardinality int + 1 => AnyValue::new_int((mix64(idx + i as u64) % 1_000_000) as i64), + // high-cardinality double + 2 => AnyValue::new_double((idx as f64) * 1.5 + i as f64), + // bool + 3 => AnyValue::new_bool((idx as usize + i).is_multiple_of(2)), + // high-cardinality id-like string + _ => AnyValue::new_string(format!("id-{:016x}", mix64(idx.wrapping_add(i as u64)))), + }; + KeyValue::new(key, value) + }) + .collect() +} + +/// Build an OTLP [`LogsData`] with configurable per-scope/resource/log attribute +/// counts. Unlike [`create_logs_data`], attribute richness is a parameter and +/// each record carries realistic high-cardinality content so the study can +/// measure resource-heavy versus log-heavy telemetry without identical rows. +#[must_use] +pub fn create_logs_data_rich(params: &RichGenParams) -> LogsData { + let base_time = 1_700_000_000_000_000_000u64; + LogsData::new( + (0..params.num_resources) + .map(|res_idx| { + ResourceLogs::new( + Resource::build() + .attributes(resource_attrs_for(res_idx, params.num_resource_attrs)) + .dropped_attributes_count(0u32), + (0..params.num_scopes) + .map(|scope_idx| { + ScopeLogs::new( + InstrumentationScope::build() + .name("library") + .version("scopev1") + .attributes(scope_attrs_for(scope_idx, params.num_scope_attrs)) + .dropped_attributes_count(0u32) + .finish(), + (0..params.num_logs) + .map(|log_idx| { + let idx = ((res_idx * params.num_scopes + scope_idx) + * params.num_logs + + log_idx) + as u64; + LogRecord::build() + .time_unix_nano(base_time + idx * 1_000_000) + .observed_time_unix_nano( + base_time + idx * 1_000_000 + 500, + ) + .severity_number(SeverityNumber::Info) + .severity_text("Info") + .trace_id(trace_id_for(idx)) + .span_id(span_id_for(idx)) + .attributes(log_attrs_for(idx, params.num_log_attrs)) + .body(AnyValue::new_string(format!( + "request {idx} handled in {}ms", + idx % 500 + ))) + .finish() + }) + .collect::>(), + ) + }) + .collect::>(), + ) + }) + .collect::>(), + ) +} + +/// Generate an OTAP logs batch for a [`RichGenParams`] shape. +#[must_use] +pub fn gen_logs_otap_rich(params: &RichGenParams) -> OtapArrowRecords { + let logs_data = create_logs_data_rich(params); + let mut proto_bytes = Vec::new(); + logs_data + .encode(&mut proto_bytes) + .expect("can encode OTLP proto bytes"); + let view = RawLogsData::new(proto_bytes.as_ref()); + encode_logs_otap_batch(&view).expect("can encode OTAP logs batch") +} + +#[cfg(test)] +mod tests { + use super::*; + use otap_df_pdata::proto::opentelemetry::arrow::v1::ArrowPayloadType; + + #[test] + fn generates_expected_record_counts() { + let params = LogsGenParams { + num_resources: 2, + num_scopes: 3, + num_logs: 4, + }; + let (otap, proto_len) = gen_logs_otap(¶ms); + assert!(proto_len > 0); + + let logs = otap + .get(ArrowPayloadType::Logs) + .expect("logs payload present"); + assert_eq!(logs.num_rows(), params.total_logs()); + + // Every payload type we expect to flatten should be present. + for pt in [ + ArrowPayloadType::ResourceAttrs, + ArrowPayloadType::ScopeAttrs, + ArrowPayloadType::LogAttrs, + ] { + assert!(otap.get(pt).is_some(), "missing payload {pt:?}"); + } + } +} diff --git a/rust/otap-dataflow/benchmarks/src/parquet_study/ipc.rs b/rust/otap-dataflow/benchmarks/src/parquet_study/ipc.rs new file mode 100644 index 0000000000..30ffe049ca --- /dev/null +++ b/rust/otap-dataflow/benchmarks/src/parquet_study/ipc.rs @@ -0,0 +1,245 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +//! IPC baseline contender: the OTAP representation "as we have it today" -- the +//! interleaved Arrow IPC streams produced by [`Producer`] / consumed by +//! [`Consumer`], with the per-payload IPC streams optionally compressed. +//! +//! - write: [`Producer::produce_bar`] (applies transport-optimized encoding and +//! serializes each payload's record batch to an Arrow IPC stream) then +//! prost-encodes the resulting [`BatchArrowRecords`] to bytes. +//! - read: prost-decodes the [`BatchArrowRecords`], [`Consumer::consume_bar`] +//! deserializes the IPC streams, [`from_record_messages`] reassembles the +//! [`OtapArrowRecords`], and `decode_transport_optimized_ids` restores the +//! logical (non-transport-optimized) batch so it matches the input exactly. + +use otap_df_pdata::Consumer; +use otap_df_pdata::encode::producer::{Producer, ProducerOptions}; +use otap_df_pdata::otap::{Logs, OtapArrowRecords, from_record_messages}; +use otap_df_pdata::proto::opentelemetry::arrow::v1::BatchArrowRecords; +use prost::Message; + +use super::{Codec, Compressor, StudyResult}; + +/// Contender that encodes OTAP logs as compressed interleaved Arrow IPC streams. +pub struct IpcCodec { + /// Compression applied to each per-payload IPC stream. + pub compressor: Compressor, +} + +/// Pipeline sub-step: apply the OTAP transport-optimized encoding in place +/// (delta/dictionary encodings on id and value columns, parent-id remapping). +/// This is the first thing `Producer::produce_bar` does; measuring it alone lets +/// the benchmark separate it from the Arrow IPC serialization. +pub fn transport_encode(logs: &mut OtapArrowRecords) -> StudyResult<()> { + logs.encode_transport_optimized()?; + Ok(()) +} + +/// Pipeline sub-step: serialize a logs batch to wire bytes. This runs the whole +/// encode side (transport-optimized encoding, then Arrow IPC serialization with +/// compression, then prost-encoding the `BatchArrowRecords`). The IPC +/// serialization time alone is this minus [`transport_encode`]. +pub fn encode_to_bytes(mut logs: OtapArrowRecords, compressor: Compressor) -> StudyResult> { + let mut producer = Producer::new_with_options(ProducerOptions { + ipc_compression: compressor.ipc(), + }); + let bar = producer.produce_bar(&mut logs)?; + let mut buf = Vec::with_capacity(1024); + bar.encode(&mut buf)?; + Ok(buf) +} + +/// Pipeline sub-step: deserialize wire bytes into a logs batch that is still in +/// the transport-optimized encoding (prost-decode, then `Consumer::consume_bar`, +/// then `from_record_messages`). Does not run the transport decode. +pub fn deserialize(bytes: &[u8]) -> StudyResult { + let mut bar = BatchArrowRecords::decode(bytes)?; + let mut consumer = Consumer::default(); + let messages = consumer.consume_bar(&mut bar)?; + Ok(OtapArrowRecords::Logs(from_record_messages::( + messages, + )?)) +} + +/// Pipeline sub-step: reverse the transport-optimized encoding in place, leaving +/// the logical OTAP logs batch. +pub fn transport_decode(logs: &mut OtapArrowRecords) -> StudyResult<()> { + logs.decode_transport_optimized_ids()?; + Ok(()) +} + +/// Serialize the same logs batch `count` times through a single long-lived +/// [`Producer`], returning the wire size of each batch. +/// +/// This models OTAP streaming: the Arrow schema is written once into the stream +/// and dictionaries are delta-encoded, so `sizes[0]` is the cold size (schema +/// plus full dictionaries plus data) while `sizes[1..]` are the steady-state +/// sizes (data plus only new dictionary entries). Sending the identical batch +/// repeatedly is a best case for dictionary amortization; real telemetry with +/// varying values falls between the cold and steady-state sizes. +pub fn stream_batch_sizes( + logs: &OtapArrowRecords, + compressor: Compressor, + count: usize, +) -> StudyResult> { + let mut producer = Producer::new_with_options(ProducerOptions { + ipc_compression: compressor.ipc(), + }); + let mut sizes = Vec::with_capacity(count); + for _ in 0..count { + let mut batch = logs.clone(); + let bar = producer.produce_bar(&mut batch)?; + let mut buf = Vec::with_capacity(1024); + bar.encode(&mut buf)?; + sizes.push(buf.len()); + } + Ok(sizes) +} + +impl Codec for IpcCodec { + fn name(&self) -> &'static str { + "ipc" + } + + fn write(&self, logs: OtapArrowRecords) -> StudyResult> { + encode_to_bytes(logs, self.compressor) + } + + fn read(&self, bytes: &[u8]) -> StudyResult { + let mut logs = deserialize(bytes)?; + transport_decode(&mut logs)?; + Ok(logs) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::parquet_study::Compressor; + use crate::parquet_study::datagen::{LogsGenParams, gen_logs_otap}; + + #[test] + fn ipc_round_trip_preserves_structure() { + let params = LogsGenParams { + num_resources: 2, + num_scopes: 2, + num_logs: 3, + }; + let (otap, _) = gen_logs_otap(¶ms); + + for compressor in Compressor::IPC { + let codec = IpcCodec { compressor }; + let bytes = codec.write(otap.clone()).expect("write"); + let decoded = codec.read(&bytes).expect("read"); + // The IPC round-trip is lossless, though the decoded batch may + // dictionary-encode some value columns, so compare structurally. + crate::parquet_study::attrs::assert_logs_equivalent( + &otap, + &decoded, + codec.name(), + compressor.label(), + ); + } + } + + #[test] + fn streaming_amortizes_schema_and_dictionaries() { + let params = LogsGenParams { + num_resources: 1, + num_scopes: 1, + num_logs: 2000, + }; + let (otap, _) = gen_logs_otap(¶ms); + + for compressor in Compressor::IPC { + let sizes = stream_batch_sizes(&otap, compressor, 5).expect("stream sizes"); + // The steady-state batch (2nd onward) omits the schema header and + // re-sends only new dictionary entries, so it is smaller than the + // cold first batch. + assert!( + sizes[1] < sizes[0], + "{compressor:?}: steady {} not smaller than cold {}", + sizes[1], + sizes[0] + ); + // The drop is one-time: with identical batches, every steady-state + // batch is the same size. Arrow IPC does not compress frames against + // each other, so frame N is not smaller than frame 2 despite carrying + // identical data. + assert!( + sizes[2..].iter().all(|&s| s == sizes[1]), + "{compressor:?}: steady-state not flat: {:?}", + sizes + ); + } + } + + /// Produce `count` batches through one long-lived producer with IPC + /// compression disabled, concatenate the wire bytes, and compress the whole + /// stream once with zstd at `level`. This models the size a stream-level + /// (cross-batch) compressor could reach, whereas Arrow IPC compresses each + /// batch independently and cannot exploit redundancy across batches. + fn stream_whole_zstd(logs: &OtapArrowRecords, count: usize, level: i32) -> usize { + let mut producer = Producer::new_with_options(ProducerOptions { + ipc_compression: None, + }); + let mut stream = Vec::new(); + for _ in 0..count { + let mut batch = logs.clone(); + let bar = producer.produce_bar(&mut batch).expect("produce"); + bar.encode(&mut stream).expect("encode"); + } + zstd::stream::encode_all(&stream[..], level) + .expect("zstd") + .len() + } + + /// Arrow IPC compresses each batch independently, so it never exploits the + /// large redundancy across the near-identical steady-state batches. This test + /// shows two things about that unexploited redundancy: + /// + /// 1. A whole-stream compressor at default effort recovers almost none of it, + /// because each uncompressed batch (~2.4 MB at 10k logs) is larger than the + /// match window, so an extra near-duplicate batch still costs about as much + /// as one Arrow IPC batch. + /// 2. A whole-stream compressor with a large window and long-distance matching + /// does find it, collapsing each extra near-duplicate batch to a tiny + /// fraction of an Arrow IPC batch. + /// + /// The identical-batch case here is a best case; real telemetry batches differ + /// substantially, so the recoverable cross-batch redundancy is far smaller. + #[test] + fn cross_batch_redundancy_needs_large_window() { + let params = LogsGenParams { + num_resources: 1, + num_scopes: 1, + num_logs: 10_000, + }; + let (otap, _) = gen_logs_otap(¶ms); + + let count = 8; + // Arrow IPC steady-state batch size (per-batch independent compression). + let warm = stream_batch_sizes(&otap, Compressor::Zstd, count).expect("sizes")[1]; + + // Default effort: window smaller than one uncompressed batch, so an extra + // near-duplicate batch costs about the same as an Arrow IPC batch. + let low_1 = stream_whole_zstd(&otap, 1, 3); + let low_n = stream_whole_zstd(&otap, count, 3); + let low_marginal = (low_n - low_1) / (count - 1); + assert!( + low_marginal * 2 > warm, + "default-effort cross-batch unexpectedly cheap: {low_marginal} vs warm {warm}" + ); + + // High effort: large window plus long-distance matching finds the + // cross-batch redundancy and collapses each extra batch to near zero. + let high_1 = stream_whole_zstd(&otap, 1, 19); + let high_n = stream_whole_zstd(&otap, count, 19); + let high_marginal = (high_n - high_1) / (count - 1); + assert!( + high_marginal * 10 < warm, + "high-effort cross-batch did not collapse: {high_marginal} vs warm {warm}" + ); + } +} diff --git a/rust/otap-dataflow/benchmarks/src/parquet_study/ipc_flat.rs b/rust/otap-dataflow/benchmarks/src/parquet_study/ipc_flat.rs new file mode 100644 index 0000000000..815e4db0f2 --- /dev/null +++ b/rust/otap-dataflow/benchmarks/src/parquet_study/ipc_flat.rs @@ -0,0 +1,84 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +//! In-memory Arrow IPC read/write for a single flat record batch. +//! +//! This is the transport counterpart to [`parquet_io`](super::parquet_io). It +//! serializes an already-flattened OTAP-flat record batch as a compressed Arrow +//! IPC stream, which is how an "OTAP-flat" wire format would move between two +//! services. Unlike Parquet, Arrow IPC can serialize `RunEndEncoded` and +//! dictionary columns, so the compact resource/scope layouts survive on the +//! wire. +//! +//! Note this is plain Arrow IPC of one batch, not the OTAP [`Producer`] path, +//! which applies transport-optimized delta and dictionary encoding to the four +//! normalized batches. The [`ipc`](super::ipc) contender measures that OTAP +//! standard path; this measures the flat alternative. +//! +//! [`Producer`]: otap_df_pdata::encode::producer::Producer + +use arrow::array::RecordBatch; +use arrow::compute::concat_batches; +use arrow_ipc::reader::StreamReader; +use arrow_ipc::writer::{IpcWriteOptions, StreamWriter}; + +use super::{Compressor, StudyResult}; + +/// Serialize a single record batch as a compressed Arrow IPC stream in memory. +pub fn write_ipc(batch: &RecordBatch, compressor: Compressor) -> StudyResult> { + let options = IpcWriteOptions::default().try_with_compression(compressor.ipc())?; + let mut buf: Vec = Vec::with_capacity(4096); + { + let mut writer = StreamWriter::try_new_with_options(&mut buf, &batch.schema(), options)?; + writer.write(batch)?; + writer.finish()?; + } + Ok(buf) +} + +/// Decode an in-memory Arrow IPC stream into a single record batch. +pub fn read_ipc(bytes: &[u8]) -> StudyResult { + let reader = StreamReader::try_new(bytes, None)?; + let schema = reader.schema(); + let mut batches: Vec = Vec::new(); + for batch in reader { + batches.push(batch?); + } + match batches.len() { + 0 => Ok(RecordBatch::new_empty(schema)), + 1 => Ok(batches.into_iter().next().expect("len checked")), + _ => Ok(concat_batches(&schema, &batches)?), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::parquet_study::datagen::{LogsGenParams, gen_logs_otap}; + use crate::parquet_study::otap_flat::{Layout, flatten}; + + #[test] + fn ipc_flat_round_trips_every_layout() { + let params = LogsGenParams { + num_resources: 3, + num_scopes: 2, + num_logs: 6, + }; + let (otap, _) = gen_logs_otap(¶ms); + + for layout in [ + Layout::Materialized, + Layout::RunEndEncoded, + Layout::Dictionary, + ] { + let flat = flatten(&otap, layout).expect("flatten"); + for compressor in [Compressor::Zstd, Compressor::Lz4, Compressor::None] { + let bytes = write_ipc(&flat, compressor).expect("write ipc"); + assert!(!bytes.is_empty()); + let back = read_ipc(&bytes).expect("read ipc"); + assert_eq!(back.num_rows(), flat.num_rows(), "{}", layout.name()); + assert_eq!(back.schema(), flat.schema(), "{}", layout.name()); + } + } + } +} diff --git a/rust/otap-dataflow/benchmarks/src/parquet_study/map.rs b/rust/otap-dataflow/benchmarks/src/parquet_study/map.rs new file mode 100644 index 0000000000..c6dda0a708 --- /dev/null +++ b/rust/otap-dataflow/benchmarks/src/parquet_study/map.rs @@ -0,0 +1,234 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +//! Map flattened-Parquet contender. +//! +//! Identical to the [`nested`](super::nested) layout except the three attribute +//! containers are encoded as Arrow `Map` instead of `List`. The key is pulled out of the +//! attribute struct into the map key; the remaining seven value columns form the +//! map value struct. This exercises Parquet's map encoding versus the +//! list-of-struct encoding for the exact same logical attributes. + +use std::sync::Arc; + +use arrow::array::{ArrayRef, MapArray, RecordBatch, StructArray}; +use arrow::buffer::{OffsetBuffer, ScalarBuffer}; +use arrow::datatypes::{DataType, Field, Fields, Schema}; + +use otap_df_pdata::otap::OtapArrowRecords; +use otap_df_pdata::proto::opentelemetry::arrow::v1::ArrowPayloadType; + +use super::attrs::{ + Gathered, LOG_ATTRS_COL, RESOURCE_ATTRS_COL, SCOPE_ATTRS_COL, attr_struct_fields, + attr_value_struct_fields, entries_dedup, entries_per_row, gather_by_parent, logs_batch, + logs_id, logs_resource_id, logs_scope_id, rebuild_attr_batch_from_parts, taken_attr_struct, +}; +use super::{Codec, Compressor, StudyResult, parquet_io}; + +/// Contender that flattens OTAP logs into a single Parquet file using `Map` +/// attribute columns. +pub struct MapParquetCodec { + /// Parquet compression codec. + pub compressor: Compressor, +} + +fn map_entries_fields() -> Fields { + Fields::from(vec![ + Field::new("keys", DataType::Utf8, false), + Field::new( + "values", + DataType::Struct(attr_value_struct_fields()), + false, + ), + ]) +} + +fn map_entries_field() -> Arc { + Arc::new(Field::new( + "entries", + DataType::Struct(map_entries_fields()), + false, + )) +} + +fn map_column_field(name: &str) -> Field { + Field::new(name, DataType::Map(map_entries_field(), false), false) +} + +fn build_attr_map_column(attr_batch: &RecordBatch, gathered: &Gathered) -> StudyResult { + let full = taken_attr_struct(attr_batch, gathered)?; + let keys = full.column(0).clone(); + let value_struct = StructArray::new( + attr_value_struct_fields(), + full.columns()[1..].to_vec(), + None, + ); + let entries = StructArray::new( + map_entries_fields(), + vec![keys, Arc::new(value_struct)], + None, + ); + let offsets = OffsetBuffer::new(ScalarBuffer::from(gathered.offsets.clone())); + let map = MapArray::try_new(map_entries_field(), offsets, entries, None, false)?; + Ok(Arc::new(map)) +} + +fn empty_map_column(num_rows: usize) -> StudyResult { + let keys = arrow::array::new_empty_array(&DataType::Utf8); + let value_children: Vec = attr_value_struct_fields() + .iter() + .map(|f| arrow::array::new_empty_array(f.data_type())) + .collect(); + let value_struct = StructArray::new(attr_value_struct_fields(), value_children, None); + let entries = StructArray::new( + map_entries_fields(), + vec![keys, Arc::new(value_struct)], + None, + ); + let offsets = OffsetBuffer::new(ScalarBuffer::from(vec![0i32; num_rows + 1])); + let map = MapArray::try_new(map_entries_field(), offsets, entries, None, false)?; + Ok(Arc::new(map)) +} + +/// Flatten an OTAP logs batch into the map flat record batch. +pub fn flatten(otap: &OtapArrowRecords) -> StudyResult { + let logs = logs_batch(otap)?; + let num_rows = logs.num_rows(); + let resource_id = logs_resource_id(logs)?; + let scope_id = logs_scope_id(logs)?; + let log_id = logs_id(logs)?; + + let resource_col = match otap.get(ArrowPayloadType::ResourceAttrs) { + Some(b) => build_attr_map_column(b, &gather_by_parent(b, &resource_id)?)?, + None => empty_map_column(num_rows)?, + }; + let scope_col = match otap.get(ArrowPayloadType::ScopeAttrs) { + Some(b) => build_attr_map_column(b, &gather_by_parent(b, &scope_id)?)?, + None => empty_map_column(num_rows)?, + }; + let log_col = match otap.get(ArrowPayloadType::LogAttrs) { + Some(b) => build_attr_map_column(b, &gather_by_parent(b, &log_id)?)?, + None => empty_map_column(num_rows)?, + }; + + let mut fields: Vec = logs + .schema() + .fields() + .iter() + .map(|f| f.as_ref().clone()) + .collect(); + let mut columns: Vec = logs.columns().to_vec(); + for (name, col) in [ + (RESOURCE_ATTRS_COL, resource_col), + (SCOPE_ATTRS_COL, scope_col), + (LOG_ATTRS_COL, log_col), + ] { + fields.push(map_column_field(name)); + columns.push(col); + } + + Ok(RecordBatch::try_new( + Arc::new(Schema::new(fields)), + columns, + )?) +} + +fn map_column<'a>(flat: &'a RecordBatch, name: &str) -> StudyResult<&'a MapArray> { + flat.column_by_name(name) + .ok_or_else(|| format!("flat batch missing `{name}` column"))? + .as_any() + .downcast_ref::() + .ok_or_else(|| format!("`{name}` is not a map").into()) +} + +fn rebuild_from_map(map: &MapArray, entries: &[(usize, u16)]) -> StudyResult { + let keys = map.keys().clone(); + let value_struct = map + .values() + .as_any() + .downcast_ref::() + .ok_or("map values are not a struct")?; + let mut cols: Vec = Vec::with_capacity(8); + cols.push(keys); + cols.extend(value_struct.columns().iter().cloned()); + let full = StructArray::new(attr_struct_fields(), cols, None); + rebuild_attr_batch_from_parts(&full, map.value_offsets(), entries) +} + +/// Reconstruct an OTAP logs batch from the map flat record batch. +pub fn unflatten(flat: &RecordBatch) -> StudyResult { + let container = [RESOURCE_ATTRS_COL, SCOPE_ATTRS_COL, LOG_ATTRS_COL]; + + let mut fields: Vec = Vec::new(); + let mut columns: Vec = Vec::new(); + for (field, column) in flat.schema().fields().iter().zip(flat.columns()) { + if !container.contains(&field.name().as_str()) { + fields.push(field.as_ref().clone()); + columns.push(column.clone()); + } + } + let logs = RecordBatch::try_new(Arc::new(Schema::new(fields)), columns)?; + + let resource_id = logs_resource_id(&logs)?; + let scope_id = logs_scope_id(&logs)?; + let log_id = logs_id(&logs)?; + + let resource_attrs = rebuild_from_map( + map_column(flat, RESOURCE_ATTRS_COL)?, + &entries_dedup(&resource_id), + )?; + let scope_attrs = rebuild_from_map( + map_column(flat, SCOPE_ATTRS_COL)?, + &entries_dedup(&scope_id), + )?; + let log_attrs = rebuild_from_map(map_column(flat, LOG_ATTRS_COL)?, &entries_per_row(&log_id))?; + + let mut otap = OtapArrowRecords::Logs(Default::default()); + otap.set(ArrowPayloadType::Logs, logs)?; + otap.set(ArrowPayloadType::ResourceAttrs, resource_attrs)?; + otap.set(ArrowPayloadType::ScopeAttrs, scope_attrs)?; + otap.set(ArrowPayloadType::LogAttrs, log_attrs)?; + Ok(otap) +} + +impl Codec for MapParquetCodec { + fn name(&self) -> &'static str { + "parquet-map" + } + + fn write(&self, logs: OtapArrowRecords) -> StudyResult> { + let flat = flatten(&logs)?; + parquet_io::write_parquet(&flat, self.compressor.parquet()) + } + + fn read(&self, bytes: &[u8]) -> StudyResult { + let flat = parquet_io::read_parquet(bytes)?; + unflatten(&flat) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::parquet_study::Compressor; + use crate::parquet_study::attrs::assert_logs_equivalent; + use crate::parquet_study::datagen::{LogsGenParams, gen_logs_otap}; + + #[test] + fn map_round_trip_preserves_structure() { + let params = LogsGenParams { + num_resources: 3, + num_scopes: 2, + num_logs: 5, + }; + let (otap, _) = gen_logs_otap(¶ms); + + for compressor in Compressor::ALL { + let codec = MapParquetCodec { compressor }; + let bytes = codec.write(otap.clone()).expect("write"); + let decoded = codec.read(&bytes).expect("read"); + assert_logs_equivalent(&otap, &decoded, codec.name(), compressor.label()); + } + } +} diff --git a/rust/otap-dataflow/benchmarks/src/parquet_study/mod.rs b/rust/otap-dataflow/benchmarks/src/parquet_study/mod.rs new file mode 100644 index 0000000000..064c8e55ed --- /dev/null +++ b/rust/otap-dataflow/benchmarks/src/parquet_study/mod.rs @@ -0,0 +1,233 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +//! Parquet study: compare the read/write cost and serialized size of OTAP logs +//! encoded as compressed Arrow IPC versus several flattened-Parquet layouts. +//! +//! Every contender implements [`Codec`], which turns an [`OtapArrowRecords`] +//! logs batch into "wire" bytes ([`Codec::write`]) and back ([`Codec::read`]). +//! Contenders are enumerated by [`Scheme`] and parameterized by a [`Compressor`]. +//! Compressors are explicit codecs (`zstd`, `lz4`, `snappy`, `none`) so that +//! zstd can be compared head-to-head with lz4 -- important when a consumer's +//! Arrow/Parquet stack may not support zstd. Arrow IPC only supports zstd and +//! lz4 (frame), so snappy is offered for the Parquet schemes only. + +use arrow::array::RecordBatch; +use otap_df_pdata::otap::OtapArrowRecords; + +pub mod attrs; +pub mod datagen; +pub mod ipc; +pub mod ipc_flat; +pub mod map; +pub mod nested; +pub mod otap_flat; +pub mod parquet_io; +pub mod server; +pub mod wide; + +/// Error type used throughout the study (benchmark/test code, so a boxed error +/// is sufficient). +pub type StudyResult = Result>; + +/// Compression codec applied by a [`Codec`]. +/// +/// | variant | Arrow IPC | Parquet | +/// |----------|---------------|----------------| +/// | `Zstd` | `ZSTD` | `ZSTD` | +/// | `Lz4` | `LZ4_FRAME` | `LZ4_RAW` | +/// | `Snappy` | *unsupported* | `SNAPPY` | +/// | `None` | uncompressed | uncompressed | +/// +/// `LZ4_RAW` (not the deprecated Hadoop-framed `LZ4`) is used for Parquet +/// because it is the cross-language interoperable variant. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Compressor { + /// zstd. + Zstd, + /// lz4 (frame for IPC, raw for Parquet). + Lz4, + /// snappy (Parquet only). + Snappy, + /// No compression. + None, +} + +impl Compressor { + /// All compressors, in reporting order (valid for the Parquet schemes). + pub const ALL: [Compressor; 4] = [ + Compressor::Zstd, + Compressor::Lz4, + Compressor::Snappy, + Compressor::None, + ]; + + /// Compressors Arrow IPC supports (snappy is not an Arrow IPC codec). + pub const IPC: [Compressor; 3] = [Compressor::Zstd, Compressor::Lz4, Compressor::None]; + + /// Short label used in benchmark ids and size tables. + #[must_use] + pub fn label(self) -> &'static str { + match self { + Compressor::Zstd => "zstd", + Compressor::Lz4 => "lz4", + Compressor::Snappy => "snappy", + Compressor::None => "none", + } + } + + /// IPC compression for this setting. Panics for [`Compressor::Snappy`], which + /// Arrow IPC does not support; callers use [`Scheme::compressors`] to avoid + /// that combination. + #[must_use] + pub fn ipc(self) -> Option { + match self { + Compressor::Zstd => Some(arrow_ipc::CompressionType::ZSTD), + Compressor::Lz4 => Some(arrow_ipc::CompressionType::LZ4_FRAME), + Compressor::None => None, + Compressor::Snappy => unreachable!("Arrow IPC does not support snappy"), + } + } + + /// Parquet compression for this setting. + #[must_use] + pub fn parquet(self) -> parquet::basic::Compression { + use parquet::basic::{Compression, ZstdLevel}; + match self { + Compressor::Zstd => Compression::ZSTD(ZstdLevel::try_new(3).expect("valid zstd level")), + Compressor::Lz4 => Compression::LZ4_RAW, + Compressor::Snappy => Compression::SNAPPY, + Compressor::None => Compression::UNCOMPRESSED, + } + } +} + +/// A round-trippable encoding of an OTAP logs batch. +pub trait Codec { + /// Stable name of this contender (e.g. `"ipc"`, `"parquet-nested"`). + fn name(&self) -> &'static str; + + /// Encode an OTAP logs batch into serialized "wire" bytes. + /// + /// Takes the batch by value: the IPC producer mutates it in place + /// (transport-optimized encoding), and the Criterion harness clones the + /// input in its (untimed) setup closure. + fn write(&self, logs: OtapArrowRecords) -> StudyResult>; + + /// Decode serialized bytes back into an OTAP logs batch. + fn read(&self, bytes: &[u8]) -> StudyResult; +} + +/// The contenders compared by the study. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Scheme { + /// OTAP interleaved Arrow IPC streams (the representation we have today). + Ipc, + /// Flattened Parquet, attributes as `List`. + Nested, + /// Flattened Parquet, attributes as `Map`. + Map, + /// Flattened Parquet, attributes exploded to one typed column per key. + Wide, +} + +impl Scheme { + /// All schemes, in reporting order. + #[must_use] + pub fn all() -> Vec { + vec![Scheme::Ipc, Scheme::Nested, Scheme::Map, Scheme::Wide] + } + + /// Only the flattened-file schemes (used by the server-cost model and the + /// pipeline-step breakdown, where IPC is handled separately). + #[must_use] + pub fn flattened() -> Vec { + vec![Scheme::Nested, Scheme::Map, Scheme::Wide] + } + + /// Stable contender name. + #[must_use] + pub fn name(self) -> &'static str { + match self { + Scheme::Ipc => "ipc", + Scheme::Nested => "parquet-nested", + Scheme::Map => "parquet-map", + Scheme::Wide => "parquet-wide", + } + } + + /// The compressors valid for this scheme. IPC excludes snappy. + #[must_use] + pub fn compressors(self) -> &'static [Compressor] { + match self { + Scheme::Ipc => &Compressor::IPC, + _ => &Compressor::ALL, + } + } + + /// Construct the [`Codec`] for this scheme with the given compressor. + #[must_use] + pub fn codec(self, compressor: Compressor) -> Box { + match self { + Scheme::Ipc => Box::new(ipc::IpcCodec { compressor }), + Scheme::Nested => Box::new(nested::NestedParquetCodec { compressor }), + Scheme::Map => Box::new(map::MapParquetCodec { compressor }), + Scheme::Wide => Box::new(wide::WideParquetCodec { compressor }), + } + } + + /// Flatten an OTAP logs batch into this Parquet scheme's flat record batch. + /// Errors for [`Scheme::Ipc`], which is not a flattened-file scheme. + pub fn flatten(self, otap: &OtapArrowRecords) -> StudyResult { + match self { + Scheme::Nested => nested::flatten(otap), + Scheme::Map => map::flatten(otap), + Scheme::Wide => wide::flatten(otap), + Scheme::Ipc => Err("ipc has no flatten step".into()), + } + } + + /// Reconstruct an OTAP logs batch from this scheme's flat record batch. + pub fn unflatten(self, flat: &RecordBatch) -> StudyResult { + match self { + Scheme::Nested => nested::unflatten(flat), + Scheme::Map => map::unflatten(flat), + Scheme::Wide => wide::unflatten(flat), + Scheme::Ipc => Err("ipc has no unflatten step".into()), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::parquet_study::datagen::{LogsGenParams, gen_logs_otap}; + + /// Round-trip every scheme x its valid compressors and assert the result is + /// logically equivalent to the input. + #[test] + fn all_contenders_round_trip() { + let params = LogsGenParams { + num_resources: 2, + num_scopes: 3, + num_logs: 4, + }; + let (otap, _) = gen_logs_otap(¶ms); + + for scheme in Scheme::all() { + for &compressor in scheme.compressors() { + let codec = scheme.codec(compressor); + let bytes = codec + .write(otap.clone()) + .unwrap_or_else(|e| panic!("{} write failed: {e}", codec.name())); + assert!(!bytes.is_empty(), "{} produced no bytes", codec.name()); + + let decoded = codec + .read(&bytes) + .unwrap_or_else(|e| panic!("{} read failed: {e}", codec.name())); + + attrs::assert_logs_equivalent(&otap, &decoded, codec.name(), compressor.label()); + } + } + } +} diff --git a/rust/otap-dataflow/benchmarks/src/parquet_study/nested.rs b/rust/otap-dataflow/benchmarks/src/parquet_study/nested.rs new file mode 100644 index 0000000000..9843cd08ae --- /dev/null +++ b/rust/otap-dataflow/benchmarks/src/parquet_study/nested.rs @@ -0,0 +1,161 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +//! Nested flattened-Parquet contender. +//! +//! The flat table keeps the entire root `Logs` record batch unchanged (so the +//! decode side can carry the scalar/struct columns straight back without +//! re-walking them, matching what the IPC path gets "for free") and appends +//! three `List` columns holding +//! each log row's denormalized resource, scope, and log attributes. +//! +//! On decode the `Logs` batch is the flat table minus those three columns; the +//! attribute batches are rebuilt from the list columns, re-normalizing the +//! resource/scope sets via the `resource.id` / `scope.id` join keys that the +//! `Logs` batch still carries. + +use std::sync::Arc; + +use arrow::array::{ArrayRef, RecordBatch}; +use arrow::datatypes::{Field, Schema}; + +use otap_df_pdata::otap::OtapArrowRecords; +use otap_df_pdata::proto::opentelemetry::arrow::v1::ArrowPayloadType; + +use super::attrs::{ + self, LOG_ATTRS_COL, RESOURCE_ATTRS_COL, SCOPE_ATTRS_COL, attr_list_column_field, + build_attr_list_column, empty_attr_list_column, entries_dedup, entries_per_row, + gather_by_parent, logs_batch, logs_id, logs_resource_id, logs_scope_id, rebuild_attr_batch, +}; +use super::{Codec, Compressor, StudyResult, parquet_io}; + +/// Contender that flattens OTAP logs into a single Parquet file using +/// `List` attribute columns. +pub struct NestedParquetCodec { + /// Parquet compression codec. + pub compressor: Compressor, +} + +/// Flatten an OTAP logs batch into the nested flat record batch. +pub fn flatten(otap: &OtapArrowRecords) -> StudyResult { + let logs = logs_batch(otap)?; + let num_rows = logs.num_rows(); + let resource_id = logs_resource_id(logs)?; + let scope_id = logs_scope_id(logs)?; + let log_id = logs_id(logs)?; + + let resource_col = match otap.get(ArrowPayloadType::ResourceAttrs) { + Some(b) => build_attr_list_column(b, &gather_by_parent(b, &resource_id)?)?, + None => empty_attr_list_column(num_rows), + }; + let scope_col = match otap.get(ArrowPayloadType::ScopeAttrs) { + Some(b) => build_attr_list_column(b, &gather_by_parent(b, &scope_id)?)?, + None => empty_attr_list_column(num_rows), + }; + let log_col = match otap.get(ArrowPayloadType::LogAttrs) { + Some(b) => build_attr_list_column(b, &gather_by_parent(b, &log_id)?)?, + None => empty_attr_list_column(num_rows), + }; + + let mut fields: Vec = logs + .schema() + .fields() + .iter() + .map(|f| f.as_ref().clone()) + .collect(); + let mut columns: Vec = logs.columns().to_vec(); + for (name, col) in [ + (RESOURCE_ATTRS_COL, resource_col), + (SCOPE_ATTRS_COL, scope_col), + (LOG_ATTRS_COL, log_col), + ] { + fields.push(attr_list_column_field(name)); + columns.push(col); + } + + Ok(RecordBatch::try_new( + Arc::new(Schema::new(fields)), + columns, + )?) +} + +/// Reconstruct an OTAP logs batch from the nested flat record batch. +pub fn unflatten(flat: &RecordBatch) -> StudyResult { + let container = [RESOURCE_ATTRS_COL, SCOPE_ATTRS_COL, LOG_ATTRS_COL]; + + // The Logs batch is the flat table minus the three attribute columns. + let mut fields: Vec = Vec::new(); + let mut columns: Vec = Vec::new(); + for (field, column) in flat.schema().fields().iter().zip(flat.columns()) { + if !container.contains(&field.name().as_str()) { + fields.push(field.as_ref().clone()); + columns.push(column.clone()); + } + } + let logs = RecordBatch::try_new(Arc::new(Schema::new(fields)), columns)?; + + let resource_id = logs_resource_id(&logs)?; + let scope_id = logs_scope_id(&logs)?; + let log_id = logs_id(&logs)?; + + let resource_attrs = rebuild_attr_batch( + attrs::as_list(flat, RESOURCE_ATTRS_COL)?, + &entries_dedup(&resource_id), + )?; + let scope_attrs = rebuild_attr_batch( + attrs::as_list(flat, SCOPE_ATTRS_COL)?, + &entries_dedup(&scope_id), + )?; + let log_attrs = rebuild_attr_batch( + attrs::as_list(flat, LOG_ATTRS_COL)?, + &entries_per_row(&log_id), + )?; + + let mut otap = OtapArrowRecords::Logs(Default::default()); + otap.set(ArrowPayloadType::Logs, logs)?; + otap.set(ArrowPayloadType::ResourceAttrs, resource_attrs)?; + otap.set(ArrowPayloadType::ScopeAttrs, scope_attrs)?; + otap.set(ArrowPayloadType::LogAttrs, log_attrs)?; + Ok(otap) +} + +impl Codec for NestedParquetCodec { + fn name(&self) -> &'static str { + "parquet-nested" + } + + fn write(&self, logs: OtapArrowRecords) -> StudyResult> { + let flat = flatten(&logs)?; + parquet_io::write_parquet(&flat, self.compressor.parquet()) + } + + fn read(&self, bytes: &[u8]) -> StudyResult { + let flat = parquet_io::read_parquet(bytes)?; + unflatten(&flat) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::parquet_study::Compressor; + use crate::parquet_study::attrs::assert_logs_equivalent; + use crate::parquet_study::datagen::{LogsGenParams, gen_logs_otap}; + + #[test] + fn nested_round_trip_preserves_structure() { + let params = LogsGenParams { + num_resources: 3, + num_scopes: 2, + num_logs: 5, + }; + let (otap, _) = gen_logs_otap(¶ms); + + for compressor in Compressor::ALL { + let codec = NestedParquetCodec { compressor }; + let bytes = codec.write(otap.clone()).expect("write"); + let decoded = codec.read(&bytes).expect("read"); + assert_logs_equivalent(&otap, &decoded, codec.name(), compressor.label()); + } + } +} diff --git a/rust/otap-dataflow/benchmarks/src/parquet_study/otap_flat.rs b/rust/otap-dataflow/benchmarks/src/parquet_study/otap_flat.rs new file mode 100644 index 0000000000..4ea9976f6c --- /dev/null +++ b/rust/otap-dataflow/benchmarks/src/parquet_study/otap_flat.rs @@ -0,0 +1,544 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +//! Efficient "OTAP-flat": present the four OTAP logs record batches as a single +//! columnar view at minimal conversion cost. +//! +//! The flattened-Parquet contenders in [`nested`](super::nested) pay a large +//! "flatten tax" because [`gather_by_parent`](super::attrs::gather_by_parent) +//! builds a `HashMap>` per attribute batch and materializes every +//! value column with a random-access `take`. That work is avoidable: the OTAP +//! encoder emits each attribute batch already **grouped by `parent_id` in +//! ascending, contiguous runs** (`LogAttrs.parent_id = [0..,1..,2..]`, +//! `ResourceAttrs.parent_id = [0,1,2]`, `Logs.id` sequential). This module +//! exploits that ordering. +//! +//! All three variants share one move: the per-row **log attributes** become a +//! `List` whose struct children are the *existing* `LogAttrs` value +//! columns (zero-copy) and whose offsets come from a single linear scan of the +//! sorted `parent_id`. No hash join, no `take`. +//! +//! They differ only in how the *shared* resource and scope attributes (one set +//! per resource/scope, spanning a contiguous run of log rows) are presented: +//! +//! - [`Layout::Materialized`] repeats each set physically per row (a plain +//! `List`, the same shape [`nested`](super::nested) produces, so it is +//! directly Parquet-writable). +//! - [`Layout::RunEndEncoded`] stores each set once as `RunEndEncoded>` (logical per-row, physical one-per-resource). In-memory / +//! query form only: arrow-rs cannot write REE to Parquet. +//! - [`Layout::Dictionary`] stores each set once as `Dictionary>` (per-row u16 index + one list per resource). Also in-memory +//! only: arrow-rs cannot write a dictionary of `List` to Parquet. + +use std::collections::HashMap; +use std::sync::Arc; + +use arrow::array::{ + Array, ArrayRef, DictionaryArray, Int32Array, ListArray, RecordBatch, RunArray, StructArray, + UInt16Array, UInt32Array, +}; +use arrow::buffer::{OffsetBuffer, ScalarBuffer}; +use arrow::compute::take; +use arrow::datatypes::{DataType, Field, Int32Type, Schema, UInt16Type}; + +use otap_df_pdata::otap::OtapArrowRecords; +use otap_df_pdata::proto::opentelemetry::arrow::v1::ArrowPayloadType; + +use super::StudyResult; +use super::attrs::{ + LOG_ATTRS_COL, RESOURCE_ATTRS_COL, SCOPE_ATTRS_COL, as_list, attr_list_element_field, + attr_struct_fields, empty_attr_list_column, entries_dedup, entries_per_row, + extract_attr_value_arrays, logs_batch, logs_id, logs_resource_id, logs_scope_id, + rebuild_attr_batch, +}; + +/// How the shared resource/scope attribute sets are presented in the flat view. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Layout { + /// Physically repeat each set per log row (`List`). Parquet-writable. + Materialized, + /// `RunEndEncoded>`: one physical set per resource/scope. + RunEndEncoded, + /// `Dictionary>`: per-row index + one set per resource. + Dictionary, +} + +impl Layout { + /// Stable contender name. + #[must_use] + pub fn name(self) -> &'static str { + match self { + Layout::Materialized => "otap-flat-materialized", + Layout::RunEndEncoded => "otap-flat-ree", + Layout::Dictionary => "otap-flat-dict", + } + } + + /// Whether the arrow-rs Parquet writer can serialize this layout directly. + /// Only [`Layout::Materialized`] can; REE and dictionary-of-`List` + /// are in-memory / query representations (see module docs). + #[must_use] + pub fn parquet_writable(self) -> bool { + matches!(self, Layout::Materialized) + } +} + +/// Downcast an attribute batch's `parent_id` column to `UInt16Array`. +fn parent_id_u16(attr_batch: &RecordBatch) -> StudyResult { + let col = attr_batch + .column_by_name(otap_df_pdata::schema::consts::PARENT_ID) + .ok_or("attribute batch missing `parent_id`")?; + Ok(col + .as_any() + .downcast_ref::() + .ok_or("`parent_id` is not UInt16")? + .clone()) +} + +/// The eight-field attribute `Struct` over *all* rows of an attribute batch, +/// built zero-copy from its value columns (no `take`). +fn attr_struct_all(attr_batch: &RecordBatch) -> StudyResult { + let values = extract_attr_value_arrays(attr_batch)?; + Ok(StructArray::new(attr_struct_fields(), values, None)) +} + +/// Given an attribute batch sorted by `parent_id`, return a `List` with +/// one row per distinct parent (its contiguous run, zero-copy) plus the parent +/// id of each row. This is the compact "one set per resource/scope" form. +fn per_parent_lists(attr_batch: &RecordBatch) -> StudyResult<(ListArray, Vec)> { + let struct_all = attr_struct_all(attr_batch)?; + let parent = parent_id_u16(attr_batch)?; + let n = parent.len(); + + let mut offsets: Vec = vec![0]; + let mut parent_ids: Vec = Vec::new(); + if n > 0 { + let mut start = 0usize; + for i in 1..n { + if parent.value(i) != parent.value(i - 1) { + offsets.push(i32::try_from(i).expect("offset fits i32")); + parent_ids.push(parent.value(start)); + start = i; + } + } + offsets.push(i32::try_from(n).expect("offset fits i32")); + parent_ids.push(parent.value(start)); + } + + let list = ListArray::new( + attr_list_element_field(), + OffsetBuffer::new(ScalarBuffer::from(offsets)), + Arc::new(struct_all), + None, + ); + Ok((list, parent_ids)) +} + +/// The zero-copy per-row log-attribute `List` column. Struct children +/// are the `LogAttrs` value columns unchanged; offsets come from one linear scan +/// aligning the sorted `LogAttrs.parent_id` runs with the sequential, nullable +/// `Logs.id` column. No hash join, no `take`. +fn log_attr_list(otap: &OtapArrowRecords, logs: &RecordBatch) -> StudyResult { + let num_rows = logs.num_rows(); + let Some(attr_batch) = otap.get(ArrowPayloadType::LogAttrs) else { + return Ok(empty_attr_list_column(num_rows)); + }; + let struct_all = attr_struct_all(attr_batch)?; + let parent = parent_id_u16(attr_batch)?; + let log_id = logs_id(logs)?; + + // Run length of each distinct parent, in ascending parent order. + let n = parent.len(); + let mut run_lengths: Vec = Vec::new(); + if n > 0 { + let mut start = 0usize; + for i in 1..n { + if parent.value(i) != parent.value(i - 1) { + run_lengths.push(i32::try_from(i - start).expect("len fits i32")); + start = i; + } + } + run_lengths.push(i32::try_from(n - start).expect("len fits i32")); + } + + // Each log row with a valid id consumes the next run, in order (the encoder + // assigns ids sequentially in the same order it emits the runs). + let mut offsets: Vec = Vec::with_capacity(num_rows + 1); + offsets.push(0); + let mut acc = 0i32; + let mut next_run = 0usize; + for i in 0..num_rows { + if log_id.is_valid(i) { + acc += run_lengths.get(next_run).copied().unwrap_or(0); + next_run += 1; + } + offsets.push(acc); + } + debug_assert_eq!( + next_run, + run_lengths.len(), + "every LogAttrs run should map to exactly one log row with an id" + ); + + Ok(Arc::new(ListArray::new( + attr_list_element_field(), + OffsetBuffer::new(ScalarBuffer::from(offsets)), + Arc::new(struct_all), + None, + ))) +} + +/// The contiguous runs of a dense (non-null) run-structured id column: the id of +/// each run and the exclusive end row index of each run. +fn runs(id_arr: &UInt16Array) -> (Vec, Vec) { + let n = id_arr.len(); + let mut ids = Vec::new(); + let mut ends = Vec::new(); + if n > 0 { + for i in 1..n { + if id_arr.value(i) != id_arr.value(i - 1) { + ids.push(id_arr.value(i - 1)); + ends.push(i32::try_from(i).expect("end fits i32")); + } + } + ids.push(id_arr.value(n - 1)); + ends.push(i32::try_from(n).expect("end fits i32")); + } + (ids, ends) +} + +/// Byte span (start,end) of each parent's run in the flattened struct, keyed by +/// parent id. The map is sized by the number of distinct resources/scopes (tiny) +/// -- not by the attribute-row count -- so it is not the join `HashMap` the +/// baseline flatten pays. +fn spans_by_id(list: &ListArray, parent_ids: &[u16]) -> HashMap { + let offs = list.value_offsets(); + parent_ids + .iter() + .enumerate() + .map(|(j, &pid)| (pid, (offs[j], offs[j + 1]))) + .collect() +} + +/// Build the shared resource/scope attribute column for a given [`Layout`] from +/// the per-parent lists and the log rows' run structure. +fn shared_column( + layout: Layout, + per_parent: &ListArray, + parent_ids: &[u16], + id_arr: &UInt16Array, +) -> StudyResult { + let (run_ids, run_ends) = runs(id_arr); + let spans = spans_by_id(per_parent, parent_ids); + let base_struct = per_parent + .values() + .as_any() + .downcast_ref::() + .ok_or("per-parent values are not a struct")?; + + match layout { + Layout::Materialized => { + // Repeat each run's set across its rows: gather indices row by row. + let num_rows = id_arr.len(); + let mut idx: Vec = Vec::new(); + let mut offsets: Vec = Vec::with_capacity(num_rows + 1); + offsets.push(0); + let mut run = 0usize; + for i in 0..num_rows { + if run + 1 < run_ends.len() && i32::try_from(i).expect("fits") >= run_ends[run] { + run += 1; + } + let (s, e) = spans.get(&run_ids[run]).copied().unwrap_or((0, 0)); + for k in s..e { + idx.push(u32::try_from(k).expect("index fits u32")); + } + offsets.push(i32::try_from(idx.len()).expect("offset fits i32")); + } + let taken = take_struct(base_struct, &idx)?; + Ok(Arc::new(ListArray::new( + attr_list_element_field(), + OffsetBuffer::new(ScalarBuffer::from(offsets)), + Arc::new(taken), + None, + ))) + } + Layout::RunEndEncoded => { + // One list per run; run_ends give the logical row spans. + let values = align_runs(base_struct, &spans, &run_ids)?; + let run_ends = Int32Array::from(run_ends); + let ree = RunArray::::try_new(&run_ends, &values)?; + Ok(Arc::new(ree)) + } + Layout::Dictionary => { + // keys index into the per-parent lists by position; values are those + // lists unchanged (zero-copy). + let pos_of_id: HashMap = parent_ids + .iter() + .enumerate() + .map(|(j, &pid)| (pid, i32::try_from(j).expect("fits"))) + .collect(); + let keys: UInt16Array = (0..id_arr.len()) + .map(|i| { + let id = id_arr.value(i); + u16::try_from(*pos_of_id.get(&id).unwrap_or(&0)).expect("key fits u16") + }) + .collect(); + let dict = DictionaryArray::::try_new( + keys, + Arc::new(per_parent.clone()) as ArrayRef, + )?; + Ok(Arc::new(dict)) + } + } +} + +/// One `List` per run (in run order), each holding that run's resource +/// set (or empty if the resource carried no attributes). +fn align_runs( + base_struct: &StructArray, + spans: &HashMap, + run_ids: &[u16], +) -> StudyResult { + let mut idx: Vec = Vec::new(); + let mut offsets: Vec = vec![0]; + for rid in run_ids { + let (s, e) = spans.get(rid).copied().unwrap_or((0, 0)); + for k in s..e { + idx.push(u32::try_from(k).expect("index fits u32")); + } + offsets.push(i32::try_from(idx.len()).expect("offset fits i32")); + } + let taken = take_struct(base_struct, &idx)?; + Ok(Arc::new(ListArray::new( + attr_list_element_field(), + OffsetBuffer::new(ScalarBuffer::from(offsets)), + Arc::new(taken), + None, + ))) +} + +/// `take` each field of a struct by the given indices (used only for the tiny +/// resource/scope sets, never for the per-row log attributes). +fn take_struct(base: &StructArray, idx: &[u32]) -> StudyResult { + let indices = UInt32Array::from(idx.to_vec()); + let cols: Vec = base + .columns() + .iter() + .map(|c| take(c, &indices, None)) + .collect::>()?; + Ok(StructArray::new(attr_struct_fields(), cols, None)) +} + +/// The flat-table field for a shared attribute column of the given layout. +fn shared_field(name: &str, array: &ArrayRef) -> Field { + Field::new(name, array.data_type().clone(), array.is_nullable()) +} + +/// Flatten an OTAP logs batch into a single columnar view with the given layout. +pub fn flatten(otap: &OtapArrowRecords, layout: Layout) -> StudyResult { + let logs = logs_batch(otap)?; + let resource_id = logs_resource_id(logs)?; + let scope_id = logs_scope_id(logs)?; + + let mut fields: Vec = logs + .schema() + .fields() + .iter() + .map(|f| f.as_ref().clone()) + .collect(); + let mut columns: Vec = logs.columns().to_vec(); + + // Shared resource/scope columns. + for (name, payload, id_arr) in [ + ( + RESOURCE_ATTRS_COL, + ArrowPayloadType::ResourceAttrs, + &resource_id, + ), + (SCOPE_ATTRS_COL, ArrowPayloadType::ScopeAttrs, &scope_id), + ] { + let col: ArrayRef = match otap.get(payload) { + Some(attr_batch) => { + let (per_parent, parent_ids) = per_parent_lists(attr_batch)?; + shared_column(layout, &per_parent, &parent_ids, id_arr)? + } + None => empty_attr_list_column(logs.num_rows()), + }; + fields.push(shared_field(name, &col)); + columns.push(col); + } + + // Per-row log attributes (zero-copy for every layout). + let log_col = log_attr_list(otap, logs)?; + fields.push(shared_field(LOG_ATTRS_COL, &log_col)); + columns.push(log_col); + + Ok(RecordBatch::try_new( + Arc::new(Schema::new(fields)), + columns, + )?) +} + +/// Read a shared attribute column back to a `List` of one row per +/// distinct resource/scope plus the id to stamp on each, regardless of layout. +fn shared_lists(flat: &RecordBatch, name: &str, id_arr: &UInt16Array) -> StudyResult { + let column = flat + .column_by_name(name) + .ok_or_else(|| format!("flat batch missing `{name}`"))?; + + match column.data_type() { + DataType::List(_) => { + // Materialized: per-row lists; dedup to first row per id. + rebuild_attr_batch(as_list(flat, name)?, &entries_dedup(id_arr)) + } + DataType::RunEndEncoded(_, _) => { + let ree = column + .as_any() + .downcast_ref::>() + .ok_or("expected RunEndEncoded")?; + let list = ree + .values() + .as_any() + .downcast_ref::() + .ok_or("REE values are not a list")?; + let (run_ids, _) = runs(id_arr); + let entries: Vec<(usize, u16)> = + run_ids.iter().enumerate().map(|(j, &id)| (j, id)).collect(); + rebuild_attr_batch(list, &entries) + } + DataType::Dictionary(_, _) => { + let dict = column + .as_any() + .downcast_ref::>() + .ok_or("expected Dictionary")?; + let list = dict + .values() + .as_any() + .downcast_ref::() + .ok_or("dictionary values are not a list")?; + // The per-parent lists are in ascending id order; distinct ids in + // ascending order recover the stamp for each list row. + let (mut ids, _) = runs(id_arr); + ids.sort_unstable(); + ids.dedup(); + let entries: Vec<(usize, u16)> = + ids.iter().enumerate().map(|(j, &id)| (j, id)).collect(); + rebuild_attr_batch(list, &entries) + } + other => Err(format!("unexpected shared column type {other:?}").into()), + } +} + +/// Reconstruct an OTAP logs batch from a flat view (any layout). +pub fn unflatten(flat: &RecordBatch) -> StudyResult { + let container = [RESOURCE_ATTRS_COL, SCOPE_ATTRS_COL, LOG_ATTRS_COL]; + let mut fields: Vec = Vec::new(); + let mut columns: Vec = Vec::new(); + for (field, column) in flat.schema().fields().iter().zip(flat.columns()) { + if !container.contains(&field.name().as_str()) { + fields.push(field.as_ref().clone()); + columns.push(column.clone()); + } + } + let logs = RecordBatch::try_new(Arc::new(Schema::new(fields)), columns)?; + + let resource_id = logs_resource_id(&logs)?; + let scope_id = logs_scope_id(&logs)?; + let log_id = logs_id(&logs)?; + + let resource_attrs = shared_lists(flat, RESOURCE_ATTRS_COL, &resource_id)?; + let scope_attrs = shared_lists(flat, SCOPE_ATTRS_COL, &scope_id)?; + let log_attrs = rebuild_attr_batch(as_list(flat, LOG_ATTRS_COL)?, &entries_per_row(&log_id))?; + + let mut otap = OtapArrowRecords::Logs(Default::default()); + otap.set(ArrowPayloadType::Logs, logs)?; + otap.set(ArrowPayloadType::ResourceAttrs, resource_attrs)?; + otap.set(ArrowPayloadType::ScopeAttrs, scope_attrs)?; + otap.set(ArrowPayloadType::LogAttrs, log_attrs)?; + Ok(otap) +} + +/// Sum of the in-memory footprint of every column in the flat view. +#[must_use] +pub fn in_memory_bytes(flat: &RecordBatch) -> usize { + flat.columns() + .iter() + .map(|c| c.get_array_memory_size()) + .sum() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::parquet_study::attrs::assert_logs_equivalent; + use crate::parquet_study::datagen::{LogsGenParams, gen_logs_otap}; + + #[test] + fn all_layouts_round_trip() { + let params = LogsGenParams { + num_resources: 3, + num_scopes: 2, + num_logs: 7, + }; + let (otap, _) = gen_logs_otap(¶ms); + + for layout in [ + Layout::Materialized, + Layout::RunEndEncoded, + Layout::Dictionary, + ] { + let flat = flatten(&otap, layout).expect("flatten"); + assert_eq!(flat.num_rows(), params.total_logs(), "{}", layout.name()); + let decoded = unflatten(&flat).expect("unflatten"); + assert_logs_equivalent(&otap, &decoded, layout.name(), "n/a"); + } + } + + #[test] + fn resource_heavy_round_trips() { + use crate::parquet_study::datagen::{RichGenParams, gen_logs_otap_rich}; + + let params = RichGenParams { + label: "test", + num_resources: 40, + num_scopes: 3, + num_logs: 5, + num_resource_attrs: 12, + num_scope_attrs: 4, + num_log_attrs: 3, + }; + let otap = gen_logs_otap_rich(¶ms); + + for layout in [ + Layout::Materialized, + Layout::RunEndEncoded, + Layout::Dictionary, + ] { + let flat = flatten(&otap, layout).expect("flatten"); + assert_eq!(flat.num_rows(), params.total_logs(), "{}", layout.name()); + let decoded = unflatten(&flat).expect("unflatten"); + assert_logs_equivalent(&otap, &decoded, layout.name(), "n/a"); + } + } + + #[test] + fn ree_and_dict_are_smaller_in_memory_than_materialized() { + let params = LogsGenParams { + num_resources: 1, + num_scopes: 1, + num_logs: 20_000, + }; + let (otap, _) = gen_logs_otap(¶ms); + + let mat = in_memory_bytes(&flatten(&otap, Layout::Materialized).expect("mat")); + let ree = in_memory_bytes(&flatten(&otap, Layout::RunEndEncoded).expect("ree")); + let dict = in_memory_bytes(&flatten(&otap, Layout::Dictionary).expect("dict")); + + assert!(ree < mat, "REE {ree} not smaller than materialized {mat}"); + assert!( + dict < mat, + "dict {dict} not smaller than materialized {mat}" + ); + } +} diff --git a/rust/otap-dataflow/benchmarks/src/parquet_study/parquet_io.rs b/rust/otap-dataflow/benchmarks/src/parquet_study/parquet_io.rs new file mode 100644 index 0000000000..6232f60292 --- /dev/null +++ b/rust/otap-dataflow/benchmarks/src/parquet_study/parquet_io.rs @@ -0,0 +1,145 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +//! In-memory Parquet read/write used by the flattened-Parquet contenders. +//! +//! The production `parquet_exporter` writes through `object_store`; for a +//! read/write cost microbenchmark we instead encode to and decode from an +//! in-memory `Vec` using the synchronous Arrow Parquet reader/writer. + +use arrow::array::{Array, ArrayRef, RecordBatch}; +use arrow::compute::{cast, concat_batches}; +use arrow::datatypes::{DataType, Field, Schema}; +use bytes::Bytes; +use parquet::arrow::ArrowWriter; +use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder; +use parquet::basic::Compression; +use parquet::file::properties::WriterProperties; + +use std::sync::Arc; + +use super::StudyResult; + +/// Convert a flat OTAP batch into the form the Parquet writer can serialize, +/// transforming only the columns arrow-rs cannot handle and copying the rest. +/// +/// Two column encodings do not survive Parquet in arrow-rs 58.3, so exactly +/// those are materialized here: +/// +/// - `RunEndEncoded` columns (the run-end resource/scope attributes) cannot be +/// written; they are cast to their plain `List` value type. +/// - `Dictionary(_, FixedSizeBinary)` columns (`trace_id`, `span_id`) cannot be +/// read back, because the reader rebuilds the values through an offset buffer +/// and then asserts the `FixedSizeBinaryArray` has one buffer; they are cast to +/// plain `FixedSizeBinary`. +/// +/// Every other column, including dictionary-encoded `Utf8`/`Int32` such as +/// `severity_text` and the nested `scope.name` and `body.str`, round-trips +/// through Parquet unchanged and is copied by reference. Parquet then applies its +/// own run-length and dictionary encoding on write. +pub fn to_parquet_ready(batch: &RecordBatch) -> StudyResult { + let mut fields: Vec = Vec::with_capacity(batch.num_columns()); + let mut columns: Vec = Vec::with_capacity(batch.num_columns()); + for (field, column) in batch.schema().fields().iter().zip(batch.columns()) { + match column.data_type() { + DataType::RunEndEncoded(_, values) => { + let decoded = cast(column, values.data_type())?; + fields.push(Field::new( + field.name(), + decoded.data_type().clone(), + field.is_nullable(), + )); + columns.push(decoded); + } + DataType::Dictionary(_, value_type) + if matches!(value_type.as_ref(), DataType::FixedSizeBinary(_)) => + { + let decoded = cast(column, value_type)?; + fields.push(Field::new( + field.name(), + decoded.data_type().clone(), + field.is_nullable(), + )); + columns.push(decoded); + } + _ => { + fields.push(field.as_ref().clone()); + columns.push(column.clone()); + } + } + } + Ok(RecordBatch::try_new( + Arc::new(Schema::new(fields)), + columns, + )?) +} + +/// Encode a single record batch as a Parquet file in memory. +pub fn write_parquet(batch: &RecordBatch, compression: Compression) -> StudyResult> { + let props = WriterProperties::builder() + .set_compression(compression) + .build(); + let mut buf: Vec = Vec::with_capacity(4096); + let mut writer = ArrowWriter::try_new(&mut buf, batch.schema(), Some(props))?; + writer.write(batch)?; + let _metadata = writer.close()?; + Ok(buf) +} + +/// Decode an in-memory Parquet file into a single record batch (concatenating +/// the reader's row-group batches). +pub fn read_parquet(bytes: &[u8]) -> StudyResult { + let builder = ParquetRecordBatchReaderBuilder::try_new(Bytes::copy_from_slice(bytes))?; + let schema = builder.schema().clone(); + let reader = builder.build()?; + let mut batches: Vec = Vec::new(); + for batch in reader { + batches.push(batch?); + } + match batches.len() { + 0 => Ok(RecordBatch::new_empty(schema)), + 1 => Ok(batches.into_iter().next().expect("len checked")), + _ => Ok(concat_batches(&schema, &batches)?), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use arrow::array::{DictionaryArray, FixedSizeBinaryArray}; + use arrow::datatypes::UInt16Type; + + #[test] + fn parquet_ready_materializes_fixed_size_binary() { + // Dictionary(UInt16, FixedSizeBinary(4)) is the shape the OTAP encoder + // produces for trace_id/span_id and that the Parquet reader mishandles. + let values = FixedSizeBinaryArray::try_from_iter( + vec![b"aaaa".to_vec(), b"bbbb".to_vec(), b"cccc".to_vec()].into_iter(), + ) + .expect("values"); + let keys = arrow::array::UInt16Array::from(vec![0u16, 1, 2, 1, 0]); + let dict = DictionaryArray::::try_new(keys, Arc::new(values)).expect("dict"); + let batch = RecordBatch::try_new( + Arc::new(Schema::new(vec![Field::new( + "trace_id", + dict.data_type().clone(), + false, + )])), + vec![Arc::new(dict)], + ) + .expect("batch"); + + let decoded = to_parquet_ready(&batch).expect("parquet-ready"); + assert_eq!( + decoded.schema().field(0).data_type(), + &DataType::FixedSizeBinary(4), + "dictionary should be materialized to plain fixed-size binary" + ); + + // The plain batch round-trips through Parquet, where the dictionary form + // would trip the arrow-rs reader. + let bytes = write_parquet(&decoded, Compression::UNCOMPRESSED).expect("write"); + let back = read_parquet(&bytes).expect("read"); + assert_eq!(back.num_rows(), 5); + } +} diff --git a/rust/otap-dataflow/benchmarks/src/parquet_study/server.rs b/rust/otap-dataflow/benchmarks/src/parquet_study/server.rs new file mode 100644 index 0000000000..498dd23f1e --- /dev/null +++ b/rust/otap-dataflow/benchmarks/src/parquet_study/server.rs @@ -0,0 +1,92 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +//! Server-side CPU model for the OTAP-to-Parquet debate. +//! +//! In the target system the data ends up as flattened Parquet on the server +//! *either way*; the only question is where the OTAP -> Parquet conversion CPU +//! is spent. +//! +//! - **Option A -- client sends OTAP/IPC, server converts.** The server pays to +//! decode the IPC, flatten to the single Parquet table, and encode Parquet. +//! This is [`convert_ipc_to_parquet`]. +//! - **Option B -- client sends precomputed Parquet.** The client already paid +//! the conversion; the server only persists the bytes (essentially an I/O +//! copy, ~0 CPU) or, if it must partition / index / validate, reparses the +//! Parquet back into Arrow without going all the way to OTAP. That reparse is +//! [`reparse_parquet`]; persisting is [`persist_bytes`]. +//! +//! The server-side CPU *saved* by accepting client Parquet is therefore roughly +//! the whole of Option A minus whatever minimal handling Option B needs. + +use arrow::array::RecordBatch; + +use super::ipc::IpcCodec; +use super::{Codec, Compressor, StudyResult, parquet_io}; + +/// Option A: the server receives OTAP/IPC and produces flattened Parquet. +/// +/// `parquet` selects the flattening layout (nested / map / wide) and the Parquet +/// compressor. The IPC decode auto-detects each stream's compression, so the +/// compressor used to construct the internal [`IpcCodec`] does not affect the +/// read. +pub fn convert_ipc_to_parquet(ipc_bytes: &[u8], parquet: &dyn Codec) -> StudyResult> { + let otap = IpcCodec { + compressor: Compressor::Zstd, + } + .read(ipc_bytes)?; + parquet.write(otap) +} + +/// Option B (server needs the data): reparse client-precomputed Parquet into an +/// Arrow record batch. This decodes Parquet but does not rebuild OTAP -- a +/// server that partitions or indexes the flat table works in Arrow directly. +pub fn reparse_parquet(parquet_bytes: &[u8]) -> StudyResult { + parquet_io::read_parquet(parquet_bytes) +} + +/// Option B (server only stores): persist the received bytes. Modeled as the +/// single copy the server makes handing the buffer to storage; real storage +/// cost is I/O, so this is a generous upper bound on the server's CPU. +#[must_use] +pub fn persist_bytes(parquet_bytes: &[u8]) -> usize { + let sink: Vec = parquet_bytes.to_vec(); + sink.len() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::parquet_study::attrs::assert_logs_equivalent; + use crate::parquet_study::datagen::{LogsGenParams, gen_logs_otap}; + use crate::parquet_study::nested::NestedParquetCodec; + use crate::parquet_study::{Codec, Compressor}; + + #[test] + fn server_convert_matches_direct_flatten() { + let params = LogsGenParams { + num_resources: 2, + num_scopes: 2, + num_logs: 4, + }; + let (otap, _) = gen_logs_otap(¶ms); + + let ipc = IpcCodec { + compressor: Compressor::Zstd, + }; + let ipc_bytes = ipc.write(otap.clone()).expect("ipc write"); + + let nested = NestedParquetCodec { + compressor: Compressor::Zstd, + }; + // Option A server output must round-trip back to an equivalent batch. + let parquet_bytes = convert_ipc_to_parquet(&ipc_bytes, &nested).expect("convert"); + let decoded = nested.read(&parquet_bytes).expect("read back"); + assert_logs_equivalent(&otap, &decoded, "server-convert", "zstd"); + + // Option B reparse yields the flat table without error. + let flat = reparse_parquet(&parquet_bytes).expect("reparse"); + assert_eq!(flat.num_rows(), params.total_logs()); + assert!(persist_bytes(&parquet_bytes) > 0); + } +} diff --git a/rust/otap-dataflow/benchmarks/src/parquet_study/wide.rs b/rust/otap-dataflow/benchmarks/src/parquet_study/wide.rs new file mode 100644 index 0000000000..33948cbdd9 --- /dev/null +++ b/rust/otap-dataflow/benchmarks/src/parquet_study/wide.rs @@ -0,0 +1,558 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +//! Wide / exploded flattened-Parquet contender. +//! +//! Each distinct attribute key becomes its own top-level, typed Parquet column, +//! prefixed by its group (`resource.`, `scope.`, `log.`). The +//! column's Arrow type is chosen from the key's first-seen OTAP value type +//! (string -> Utf8, int -> Int64, double -> Float64, bool -> Boolean, +//! bytes/map/slice -> Binary, empty -> a Boolean presence marker). A row is null +//! in a column when it lacks that attribute. This is the "analytics-flat" layout +//! that compresses and queries well column-by-column. +//! +//! To stay lossless and keep round-trip counts exact even when a key appears +//! with more than one value type (which the single typed column cannot hold), +//! any attribute that does not fit its typed column is spilled into a per-group +//! `List` overflow column. With type-consistent keys (the +//! common case, and what this study generates) the overflow columns are empty. +//! +//! Every added column carries field metadata describing its group, key, and +//! OTAP type, so decode is fully self-describing after a Parquet round-trip. + +use std::collections::{BTreeMap, HashMap}; +use std::sync::Arc; + +use arrow::array::{ + Array, ArrayRef, BinaryArray, BinaryBuilder, BooleanArray, BooleanBuilder, Float64Array, + Float64Builder, Int64Array, Int64Builder, ListArray, RecordBatch, StringArray, StringBuilder, + StructArray, UInt8Array, +}; +use arrow::datatypes::{Field, Schema}; + +use otap_df_pdata::encode::record::attributes::AttributesRecordBatchBuilder; +use otap_df_pdata::otap::OtapArrowRecords; +use otap_df_pdata::proto::opentelemetry::arrow::v1::ArrowPayloadType; + +use super::attrs::{ + Gathered, build_attr_list_column, extract_attr_value_arrays, gather_by_parent, logs_batch, + logs_id, logs_resource_id, logs_scope_id, +}; +use super::{Codec, Compressor, StudyResult, parquet_io}; + +// OTAP attribute value type discriminators (see AttributeValueType). +const T_STR: u8 = 1; +const T_INT: u8 = 2; +const T_DOUBLE: u8 = 3; +const T_BOOL: u8 = 4; +const T_MAP: u8 = 5; +const T_SLICE: u8 = 6; +const T_BYTES: u8 = 7; + +const MD_GROUP: &str = "wide_group"; +const MD_KEY: &str = "wide_key"; +const MD_TYPE: &str = "wide_otap_type"; +const MD_OVERFLOW: &str = "wide_overflow"; + +/// Contender that flattens OTAP logs into a single Parquet file using one typed +/// column per distinct attribute key. +pub struct WideParquetCodec { + /// Parquet compression codec. + pub compressor: Compressor, +} + +/// Group identifier used in column prefixes and metadata. +#[derive(Clone, Copy)] +struct Group { + name: &'static str, + prefix: &'static str, + payload: ArrowPayloadType, + overflow_col: &'static str, +} + +const GROUPS: [Group; 3] = [ + Group { + name: "resource", + prefix: "resource.", + payload: ArrowPayloadType::ResourceAttrs, + overflow_col: "resource_overflow", + }, + Group { + name: "scope", + prefix: "scope.", + payload: ArrowPayloadType::ScopeAttrs, + overflow_col: "scope_overflow", + }, + Group { + name: "log", + prefix: "log.", + payload: ArrowPayloadType::LogAttrs, + overflow_col: "log_overflow", + }, +]; + +/// Borrowed, typed view over the eight value columns of a source attribute +/// batch (`key, type, str, int, double, bool, bytes, ser`). +struct SourceAttrs { + key: ArrayRef, + atype: ArrayRef, + values: Vec, // str,int,double,bool,bytes,ser in canonical order +} + +impl SourceAttrs { + fn new(attr_batch: &RecordBatch) -> StudyResult { + let arrays = extract_attr_value_arrays(attr_batch)?; + Ok(Self { + key: arrays[0].clone(), + atype: arrays[1].clone(), + values: arrays[2..].to_vec(), + }) + } + + fn key_at(&self, row: usize) -> &str { + self.key + .as_any() + .downcast_ref::() + .expect("key is Utf8") + .value(row) + } + + fn type_at(&self, row: usize) -> u8 { + self.atype + .as_any() + .downcast_ref::() + .expect("type is UInt8") + .value(row) + } + + fn str(&self) -> &StringArray { + self.values[0].as_any().downcast_ref().expect("str Utf8") + } + fn int(&self) -> &Int64Array { + self.values[1].as_any().downcast_ref().expect("int Int64") + } + fn double(&self) -> &Float64Array { + self.values[2] + .as_any() + .downcast_ref() + .expect("double Float64") + } + fn boolean(&self) -> &BooleanArray { + self.values[3] + .as_any() + .downcast_ref() + .expect("bool Boolean") + } + fn bytes(&self) -> &BinaryArray { + self.values[4] + .as_any() + .downcast_ref() + .expect("bytes Binary") + } + fn ser(&self) -> &BinaryArray { + self.values[5].as_any().downcast_ref().expect("ser Binary") + } +} + +/// Build the typed Arrow column for one exploded key. `row_src[i]` is the source +/// attribute row that supplies row `i`'s value, or `None` if absent. +fn build_typed_column(otap_type: u8, src: &SourceAttrs, row_src: &[Option]) -> ArrayRef { + match otap_type { + T_STR => { + let a = src.str(); + let mut b = StringBuilder::new(); + for r in row_src { + match r { + Some(i) => b.append_value(a.value(*i)), + None => b.append_null(), + } + } + Arc::new(b.finish()) + } + T_INT => { + let a = src.int(); + let mut b = Int64Builder::new(); + for r in row_src { + match r { + Some(i) => b.append_value(a.value(*i)), + None => b.append_null(), + } + } + Arc::new(b.finish()) + } + T_DOUBLE => { + let a = src.double(); + let mut b = Float64Builder::new(); + for r in row_src { + match r { + Some(i) => b.append_value(a.value(*i)), + None => b.append_null(), + } + } + Arc::new(b.finish()) + } + T_BOOL => { + let a = src.boolean(); + let mut b = BooleanBuilder::new(); + for r in row_src { + match r { + Some(i) => b.append_value(a.value(*i)), + None => b.append_null(), + } + } + Arc::new(b.finish()) + } + T_BYTES => binary_column(src.bytes(), row_src), + T_MAP | T_SLICE => binary_column(src.ser(), row_src), + // Empty: a Boolean presence marker (value is meaningless). + _ => { + let mut b = BooleanBuilder::new(); + for r in row_src { + match r { + Some(_) => b.append_value(true), + None => b.append_null(), + } + } + Arc::new(b.finish()) + } + } +} + +fn binary_column(src: &BinaryArray, row_src: &[Option]) -> ArrayRef { + let mut b = BinaryBuilder::new(); + for r in row_src { + match r { + Some(i) => b.append_value(src.value(*i)), + None => b.append_null(), + } + } + Arc::new(b.finish()) +} + +fn field_with_md(name: String, array: &ArrayRef, md: HashMap) -> Field { + Field::new(name, array.data_type().clone(), true).with_metadata(md) +} + +/// Explode one attribute group into typed columns plus an overflow column. +fn explode_group( + otap: &OtapArrowRecords, + group: Group, + parents: &arrow::array::UInt16Array, + num_rows: usize, +) -> StudyResult> { + let Some(attr_batch) = otap.get(group.payload) else { + // No attributes for this group: emit only an empty overflow column. + let overflow = super::attrs::empty_attr_list_column(num_rows); + let mut md = HashMap::new(); + let _ = md.insert(MD_OVERFLOW.to_string(), group.name.to_string()); + return Ok(vec![( + field_with_md(group.overflow_col.to_string(), &overflow, md), + overflow, + )]); + }; + + let src = SourceAttrs::new(attr_batch)?; + let gathered = gather_by_parent(attr_batch, parents)?; + + // Discover columns: first-seen OTAP type per key, ordered deterministically. + let mut col_type: BTreeMap = BTreeMap::new(); + for row in 0..attr_batch.num_rows() { + let key = src.key_at(row).to_string(); + let _ = col_type.entry(key).or_insert_with(|| src.type_at(row)); + } + let keys: Vec = col_type.keys().cloned().collect(); + let key_ordinal: HashMap<&str, usize> = keys + .iter() + .enumerate() + .map(|(i, k)| (k.as_str(), i)) + .collect(); + + // For each column, the source row supplying each log row's value. + let mut per_col_row_src: Vec>> = vec![vec![None; num_rows]; keys.len()]; + let mut overflow_indices: Vec = Vec::new(); + let mut overflow_offsets: Vec = Vec::with_capacity(num_rows + 1); + overflow_offsets.push(0); + + // `row` indexes per_col_row_src, gathered.offsets, and overflow tracking. + #[allow(clippy::needless_range_loop)] + for row in 0..num_rows { + let start = gathered.offsets[row] as usize; + let end = gathered.offsets[row + 1] as usize; + for slot in start..end { + let s = gathered.indices.value(slot) as usize; + let key = src.key_at(s); + let t = src.type_at(s); + let ord = key_ordinal[key]; + if col_type[key] == t && per_col_row_src[ord][row].is_none() { + per_col_row_src[ord][row] = Some(s); + } else { + overflow_indices.push(u32::try_from(s).expect("index fits u32")); + } + } + overflow_offsets.push(i32::try_from(overflow_indices.len()).expect("offset fits i32")); + } + + let mut out: Vec<(Field, ArrayRef)> = Vec::with_capacity(keys.len() + 1); + for (ord, key) in keys.iter().enumerate() { + let t = col_type[key]; + let array = build_typed_column(t, &src, &per_col_row_src[ord]); + let mut md = HashMap::new(); + let _ = md.insert(MD_GROUP.to_string(), group.name.to_string()); + let _ = md.insert(MD_KEY.to_string(), key.clone()); + let _ = md.insert(MD_TYPE.to_string(), t.to_string()); + let name = format!("{}{}", group.prefix, key); + out.push((field_with_md(name, &array, md), array)); + } + + // Overflow column (empty for type-consistent keys). + let overflow_gathered = Gathered { + indices: arrow::array::UInt32Array::from(overflow_indices), + offsets: overflow_offsets, + }; + let overflow = build_attr_list_column(attr_batch, &overflow_gathered)?; + let mut md = HashMap::new(); + let _ = md.insert(MD_OVERFLOW.to_string(), group.name.to_string()); + out.push(( + field_with_md(group.overflow_col.to_string(), &overflow, md), + overflow, + )); + + Ok(out) +} + +/// Flatten an OTAP logs batch into the wide flat record batch. +pub fn flatten(otap: &OtapArrowRecords) -> StudyResult { + let logs = logs_batch(otap)?; + let num_rows = logs.num_rows(); + let resource_id = logs_resource_id(logs)?; + let scope_id = logs_scope_id(logs)?; + let log_id = logs_id(logs)?; + let parents = [resource_id, scope_id, log_id]; + + let mut fields: Vec = logs + .schema() + .fields() + .iter() + .map(|f| f.as_ref().clone()) + .collect(); + let mut columns: Vec = logs.columns().to_vec(); + + for (group, parent) in GROUPS.iter().zip(parents.iter()) { + for (field, array) in explode_group(otap, *group, parent, num_rows)? { + fields.push(field); + columns.push(array); + } + } + + Ok(RecordBatch::try_new( + Arc::new(Schema::new(fields)), + columns, + )?) +} + +fn append_typed( + builder: &mut AttributesRecordBatchBuilder, + otap_type: u8, + array: &ArrayRef, + row: usize, +) { + match otap_type { + T_STR => builder + .any_values_builder + .append_str(downcast::(array).value(row).as_bytes()), + T_INT => builder + .any_values_builder + .append_int(downcast::(array).value(row)), + T_DOUBLE => builder + .any_values_builder + .append_double(downcast::(array).value(row)), + T_BOOL => builder + .any_values_builder + .append_bool(downcast::(array).value(row)), + T_BYTES => builder + .any_values_builder + .append_bytes(downcast::(array).value(row)), + T_MAP => builder + .any_values_builder + .append_map(downcast::(array).value(row)), + T_SLICE => builder + .any_values_builder + .append_slice(downcast::(array).value(row)), + _ => builder.any_values_builder.append_empty(), + } +} + +fn downcast(array: &ArrayRef) -> &T { + array + .as_any() + .downcast_ref::() + .expect("wide column has expected type") +} + +fn append_overflow_row( + builder: &mut AttributesRecordBatchBuilder, + parent_id: u16, + values: &StructArray, + elem: usize, +) { + let key = downcast::(values.column(0)).value(elem); + let t = downcast::(values.column(1)).value(elem); + builder.append_parent_id(&parent_id); + builder.append_key(key.as_bytes()); + // struct columns: 2=str,3=int,4=double,5=bool,6=bytes,7=ser + match t { + T_STR => builder.any_values_builder.append_str( + downcast::(values.column(2)) + .value(elem) + .as_bytes(), + ), + T_INT => builder + .any_values_builder + .append_int(downcast::(values.column(3)).value(elem)), + T_DOUBLE => builder + .any_values_builder + .append_double(downcast::(values.column(4)).value(elem)), + T_BOOL => builder + .any_values_builder + .append_bool(downcast::(values.column(5)).value(elem)), + T_BYTES => builder + .any_values_builder + .append_bytes(downcast::(values.column(6)).value(elem)), + T_MAP => builder + .any_values_builder + .append_map(downcast::(values.column(7)).value(elem)), + T_SLICE => builder + .any_values_builder + .append_slice(downcast::(values.column(7)).value(elem)), + _ => builder.any_values_builder.append_empty(), + } +} + +struct WideColumn { + key: String, + otap_type: u8, + array: ArrayRef, +} + +/// Reconstruct an OTAP logs batch from the wide flat record batch. +pub fn unflatten(flat: &RecordBatch) -> StudyResult { + // Partition columns: plain Logs columns vs. wide attribute / overflow columns. + let mut logs_fields: Vec = Vec::new(); + let mut logs_columns: Vec = Vec::new(); + let mut group_columns: HashMap> = HashMap::new(); + let mut overflow_columns: HashMap = HashMap::new(); + + for (field, column) in flat.schema().fields().iter().zip(flat.columns()) { + let md = field.metadata(); + if let Some(group) = md.get(MD_OVERFLOW) { + let list = column + .as_any() + .downcast_ref::() + .ok_or("overflow column is not a list")? + .clone(); + let _ = overflow_columns.insert(group.clone(), list); + } else if let (Some(group), Some(key), Some(t)) = + (md.get(MD_GROUP), md.get(MD_KEY), md.get(MD_TYPE)) + { + group_columns + .entry(group.clone()) + .or_default() + .push(WideColumn { + key: key.clone(), + otap_type: t.parse().map_err(|_| "bad wide_otap_type")?, + array: column.clone(), + }); + } else { + logs_fields.push(field.as_ref().clone()); + logs_columns.push(column.clone()); + } + } + + let logs = RecordBatch::try_new(Arc::new(Schema::new(logs_fields)), logs_columns)?; + let resource_id = logs_resource_id(&logs)?; + let scope_id = logs_scope_id(&logs)?; + let log_id = logs_id(&logs)?; + let ids = [resource_id, scope_id, log_id]; + + let mut otap = OtapArrowRecords::Logs(Default::default()); + otap.set(ArrowPayloadType::Logs, logs)?; + + for (group, id_arr) in GROUPS.iter().zip(ids.iter()) { + let entries = if group.name == "log" { + super::attrs::entries_per_row(id_arr) + } else { + super::attrs::entries_dedup(id_arr) + }; + let cols = group_columns.get(group.name); + let overflow = overflow_columns.get(group.name); + + let mut builder = AttributesRecordBatchBuilder::::new(); + for (row, pid) in &entries { + if let Some(cols) = cols { + for col in cols { + if col.array.is_valid(*row) { + builder.append_parent_id(pid); + builder.append_key(col.key.as_bytes()); + append_typed(&mut builder, col.otap_type, &col.array, *row); + } + } + } + if let Some(list) = overflow { + let values = list + .values() + .as_any() + .downcast_ref::() + .ok_or("overflow values are not a struct")?; + let offs = list.value_offsets(); + let (start, end) = (offs[*row] as usize, offs[*row + 1] as usize); + for elem in start..end { + append_overflow_row(&mut builder, *pid, values, elem); + } + } + } + otap.set(group.payload, builder.finish()?)?; + } + + Ok(otap) +} + +impl Codec for WideParquetCodec { + fn name(&self) -> &'static str { + "parquet-wide" + } + + fn write(&self, logs: OtapArrowRecords) -> StudyResult> { + let flat = flatten(&logs)?; + parquet_io::write_parquet(&flat, self.compressor.parquet()) + } + + fn read(&self, bytes: &[u8]) -> StudyResult { + let flat = parquet_io::read_parquet(bytes)?; + unflatten(&flat) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::parquet_study::Compressor; + use crate::parquet_study::attrs::assert_logs_equivalent; + use crate::parquet_study::datagen::{LogsGenParams, gen_logs_otap}; + + #[test] + fn wide_round_trip_preserves_structure() { + let params = LogsGenParams { + num_resources: 3, + num_scopes: 2, + num_logs: 5, + }; + let (otap, _) = gen_logs_otap(¶ms); + + for compressor in Compressor::ALL { + let codec = WideParquetCodec { compressor }; + let bytes = codec.write(otap.clone()).expect("write"); + let decoded = codec.read(&bytes).expect("read"); + assert_logs_equivalent(&otap, &decoded, codec.name(), compressor.label()); + } + } +}