diff --git a/Cargo.lock b/Cargo.lock index dbf5aac..942f849 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + [[package]] name = "ahash" version = "0.8.12" @@ -98,9 +104,9 @@ checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" [[package]] name = "arrow-array" -version = "58.1.0" +version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "772bd34cacdda8baec9418d80d23d0fb4d50ef0735685bd45158b83dfeb6e62d" +checksum = "cfd33d3e92f207444098c75b42de99d329562be0cf686b307b097cc52b4e999e" dependencies = [ "ahash", "arrow-buffer", @@ -108,7 +114,7 @@ dependencies = [ "arrow-schema", "chrono", "half", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "num-complex", "num-integer", "num-traits", @@ -116,9 +122,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "58.1.0" +version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "898f4cf1e9598fdb77f356fdf2134feedfd0ee8d5a4e0a5f573e7d0aec16baa4" +checksum = "0c6cd424c2693bcdbc150d843dc9d4d137dd2de4782ce6df491ad11a3a0416c0" dependencies = [ "bytes", "half", @@ -128,9 +134,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "58.1.0" +version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42d10beeab2b1c3bb0b53a00f7c944a178b622173a5c7bcabc3cb45d90238df4" +checksum = "3c88210023a2bfee1896af366309a3028fc3bcbd6515fa29a7990ee1baa08ee0" dependencies = [ "arrow-buffer", "arrow-schema", @@ -141,9 +147,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "58.1.0" +version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "609a441080e338147a84e8e6904b6da482cefb957c5cdc0f3398872f69a315d0" +checksum = "238438f0834483703d88896db6fe5a7138b2230debc31b34c0336c2996e3c64f" dependencies = [ "arrow-array", "arrow-buffer", @@ -155,15 +161,15 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "58.1.0" +version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c30a1365d7a7dc50cc847e54154e6af49e4c4b0fddc9f607b687f29212082743" +checksum = "f633dbfdf39c039ada1bf9e34c694816eb71fbb7dc78f613993b7245e078a1ed" [[package]] name = "arrow-select" -version = "58.1.0" +version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78694888660a9e8ac949853db393af2a8b8fc82c19ce333132dfa2e72cc1a7fe" +checksum = "8cd065c54172ac787cf3f2f8d4107e0d3fdc26edba76fdf4f4cc170258942222" dependencies = [ "ahash", "arrow-array", @@ -331,9 +337,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.60" +version = "1.2.62" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43c5703da9466b66a946814e1adf53ea2c90f10063b86290cc9eb67ce3478a20" +checksum = "a1dce859f0832a7d088c4f1119888ab94ef4b5d6795d1ce05afb7fe159d79f98" dependencies = [ "find-msvc-tools", "jobserver", @@ -442,9 +448,9 @@ dependencies = [ [[package]] name = "const-hex" -version = "1.18.1" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "531185e432bb31db1ecda541e9e7ab21468d4d844ad7505e0546a49b4945d49b" +checksum = "20d9a563d167a9cce0f94153382b33cb6eded6dfabff03c69ad65a28ea1514e0" dependencies = [ "cfg-if", "cpufeatures", @@ -505,6 +511,15 @@ dependencies = [ "rustc_version", ] +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + [[package]] name = "crossterm" version = "0.28.1" @@ -756,6 +771,16 @@ dependencies = [ "rustc_version", ] +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "fnv" version = "1.0.7" @@ -885,9 +910,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.13" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f44da3a8150a6703ed5d34e164b875fd14c2cdab9af1252a9a1020bde2bdc54" +checksum = "171fefbc92fe4a4de27e0698d6a5b392d6a0e333506bc49133760b3bcf948733" dependencies = [ "atomic-waker", "bytes", @@ -936,9 +961,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.17.0" +version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" [[package]] name = "hashlink" @@ -1104,7 +1129,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" dependencies = [ "equivalent", - "hashbrown 0.17.0", + "hashbrown 0.17.1", "serde", "serde_core", ] @@ -1170,10 +1195,12 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.95" +version = "0.3.98" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2964e92d1d9dc3364cae4d718d93f227e3abb088e747d92e0395bfdedf1c12ca" +checksum = "67df7112613f8bfd9150013a0314e196f4800d3201ae742489d999db2f979f08" dependencies = [ + "cfg-if", + "futures-util", "once_cell", "wasm-bindgen", ] @@ -1209,9 +1236,9 @@ checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" [[package]] name = "libc" -version = "0.2.185" +version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52ff2c0fe9bc6cb6b14a0592c2ff4fa9ceb83eea9db979b0487cd054946a2b8f" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" [[package]] name = "libloading" @@ -1305,6 +1332,7 @@ version = "0.8.0" dependencies = [ "clap", "colored", + "flate2", "libc", "libloading", "liblogjet", @@ -1450,6 +1478,16 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + [[package]] name = "mio" version = "1.2.0" @@ -1666,9 +1704,9 @@ dependencies = [ [[package]] name = "parquet" -version = "58.1.0" +version = "58.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d3f9f2205199603564127932b89695f52b62322f541d0fc7179d57c2e1c9877" +checksum = "5dafa7d01085b62a47dd0c1829550a0a36710ea9c4fe358a05a85477cec8a908" dependencies = [ "ahash", "arrow-array", @@ -1681,7 +1719,7 @@ dependencies = [ "bytes", "chrono", "half", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "num-bigint", "num-integer", "num-traits", @@ -1811,18 +1849,18 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.1.11" +version = "1.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1749c7ed4bcaf4c3d0a3efc28538844fb29bcdd7d2b67b2be7e20ba861ff517" +checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.11" +version = "1.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9b20ed30f105399776b9c883e68e536ef602a16ae6f596d2c473591d6ad64c6" +checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" dependencies = [ "proc-macro2", "quote", @@ -2193,9 +2231,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.38" +version = "0.23.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69f9466fb2c14ea04357e91413efb882e2a6d4a406e625449bc0a5d360d53a21" +checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" dependencies = [ "log", "once_cell", @@ -2217,9 +2255,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.14.0" +version = "1.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be040f8b0a225e40375822a563fa9524378b9d63112f53e19ffff34df5d33fdd" +checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" dependencies = [ "zeroize", ] @@ -2369,11 +2407,17 @@ dependencies = [ "libc", ] +[[package]] +name = "simd-adler32" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" + [[package]] name = "siphasher" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2aa850e253778c88a04c3d7323b043aeda9d3e30d5971937c1855769763678e" +checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" [[package]] name = "slab" @@ -2632,9 +2676,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.52.1" +version = "1.52.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b67dee974fe86fd92cc45b7a95fdd2f99a36a6d7b0d431a231178d3d670bbcc6" +checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" dependencies = [ "bytes", "libc", @@ -2692,9 +2736,9 @@ dependencies = [ [[package]] name = "tonic" -version = "0.14.5" +version = "0.14.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fec7c61a0695dc1887c1b53952990f3ad2e3a31453e1f49f10e75424943a93ec" +checksum = "ac2a5518c70fa84342385732db33fb3f44bc4cc748936eb5833d2df34d6445ef" dependencies = [ "async-trait", "axum", @@ -2722,9 +2766,9 @@ dependencies = [ [[package]] name = "tonic-prost" -version = "0.14.5" +version = "0.14.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a55376a0bbaa4975a3f10d009ad763d8f4108f067c7c2e74f3001fb49778d309" +checksum = "50849f68853be452acf590cde0b146665b8d507b3b8af17261df47e02c209ea0" dependencies = [ "bytes", "prost", @@ -2944,9 +2988,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.118" +version = "0.2.121" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bf938a0bacb0469e83c1e148908bd7d5a6010354cf4fb73279b7447422e3a89" +checksum = "49ace1d07c165b0864824eee619580c4689389afa9dc9ed3a4c75040d82e6790" dependencies = [ "cfg-if", "once_cell", @@ -2957,9 +3001,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.118" +version = "0.2.121" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eeff24f84126c0ec2db7a449f0c2ec963c6a49efe0698c4242929da037ca28ed" +checksum = "8e68e6f4afd367a562002c05637acb8578ff2dea1943df76afb9e83d177c8578" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -2967,9 +3011,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.118" +version = "0.2.121" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d08065faf983b2b80a79fd87d8254c409281cf7de75fc4b773019824196c904" +checksum = "d95a9ec35c64b2a7cb35d3fead40c4238d0940c86d107136999567a4703259f2" dependencies = [ "bumpalo", "proc-macro2", @@ -2980,9 +3024,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.118" +version = "0.2.121" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fd04d9e306f1907bd13c6361b5c6bfc7b3b3c095ed3f8a9246390f8dbdee129" +checksum = "c4e0100b01e9f0d03189a92b96772a1fb998639d981193d7dbab487302513441" dependencies = [ "unicode-ident", ] diff --git a/Makefile b/Makefile index 67166cb..8e269c5 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: build dev devel check fix test test-unit test-integration test-abi-matrix test-exporter-release-smoke setup clean stats arm-devel arm x86-devel x86 setup-arm setup-x86 demo man +.PHONY: build dev devel check fix test test-unit test-integration test-abi-matrix test-exporter-release-smoke setup clean stats arm-devel arm x86-devel x86 setup-arm setup-x86 demo man advisory DEFAULT_TARGET := build ARM_TARGET ?= aarch64-unknown-linux-musl @@ -60,10 +60,13 @@ x86: setup setup-x86 demo: devel cargo build -p otlp-demo + cargo build -p otlp-demo --bin traces-emitter + cargo build -p otlp-demo --bin traces-grpc-emitter + cargo build -p otlp-demo --bin multi-signal-emitter + cargo build -p otlp-demo --bin metrics-grpc-emitter cargo build -p lj-syslog-ingest cargo build -p lj-logcat-ingest cargo build -p lj-stress-ingest - man: $(MANPAGE_OUT) $(MANPAGE_OUT): doc/manpage/%.1: doc/manpage/%.1.md @@ -71,6 +74,10 @@ $(MANPAGE_OUT): doc/manpage/%.1: doc/manpage/%.1.md @mkdir -p doc/manpage pandoc --standalone --to man $< -o $@ +advisory: + @cargo audit --version >/dev/null 2>&1 || { echo "Installing cargo-audit..."; cargo install cargo-audit --locked; } + @scripts/audit-table.sh + clean: cargo clean diff --git a/demo/Cargo.toml b/demo/Cargo.toml index 6e37842..ede1802 100644 --- a/demo/Cargo.toml +++ b/demo/Cargo.toml @@ -8,7 +8,7 @@ license.workspace = true colored = "3" logjet = { path = ".." } lz4_flex = { version = "0.11", default-features = false, features = ["std"] } -opentelemetry-proto = { version = "0.31", features = ["gen-tonic", "logs"] } +opentelemetry-proto = { version = "0.31", features = ["gen-tonic", "logs", "metrics", "trace"] } prost = "0.14" rustls = { version = "0.23", default-features = false, features = [ "ring", diff --git a/demo/README.md b/demo/README.md index 9b3da27..9368695 100644 --- a/demo/README.md +++ b/demo/README.md @@ -45,6 +45,16 @@ It also contains scenario demos under subdirectories: - generate about 5K BOFH log entries, then export that `.logjet` file to Parquet through the external exporter plugin - [`tui-view`](./tui-view) - generate 1000 randomized log entries and open `ljx view` on the result +- [`metrics-view`](./metrics-view) + - generate OTLP metrics batches, ingest them into `ljd`, and open `ljx view` on the result +- [`metrics-grpc-view`](./metrics-grpc-view) + - generate OTLP/gRPC metrics batches, ingest them into `ljd`, and open `ljx view` on the result +- [`traces-view`](./traces-view) + - generate OTLP traces batches, ingest them into `ljd`, and open `ljx view` on the result +- [`traces-grpc-view`](./traces-grpc-view) + - generate OTLP/gRPC traces batches, ingest them into `ljd`, and open `ljx view` on the result +- [`multi-signal-view`](./multi-signal-view) + - interleave logs, metrics, and traces into a single `ljd` file, then open `ljx view` - [`multiscan-view`](./multiscan-view) - generate a tree of `.logjet` files and open `ljx view` across the whole dataset - [`multiscan-discover`](./multiscan-discover) diff --git a/demo/metrics-grpc-view/README.md b/demo/metrics-grpc-view/README.md new file mode 100644 index 0000000..225fcf5 --- /dev/null +++ b/demo/metrics-grpc-view/README.md @@ -0,0 +1,18 @@ +# metrics-grpc-view + +Ingest OTLP/gRPC metrics into `ljd`, then open `ljx view` on the result. + +## Run + +```bash +make demo +cd demo/metrics-grpc-view +./run-demo.sh +``` + +The demo: +1. Starts `ljd` with OTLP/gRPC ingest on `127.0.0.1:4317` +2. Emits 15 metrics batches via `metrics-grpc-emitter` (`MetricsService/Export`) +3. Stops `ljd` after flush +4. Opens `ljx view` on the resulting `.logjet` file +5. Cleans up after the viewer exits diff --git a/demo/metrics-grpc-view/logjetd.conf b/demo/metrics-grpc-view/logjetd.conf new file mode 100644 index 0000000..6c96953 --- /dev/null +++ b/demo/metrics-grpc-view/logjetd.conf @@ -0,0 +1,7 @@ +output: file +file.path: ./logs +file.size: 100000 +file.name: metrics.logjet +ingest.protocol: otlp-grpc +ingest.listen: 127.0.0.1:4317 +replay.listen: 127.0.0.1:7002 diff --git a/demo/metrics-grpc-view/run-demo.sh b/demo/metrics-grpc-view/run-demo.sh new file mode 100755 index 0000000..60c7aa5 --- /dev/null +++ b/demo/metrics-grpc-view/run-demo.sh @@ -0,0 +1,72 @@ +#!/bin/sh +set -eu + +SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +TARGET_DIR="$SCRIPT_DIR/../../target/debug" +LJD="$TARGET_DIR/ljd" +EMITTER="$TARGET_DIR/metrics-grpc-emitter" +LJX="$TARGET_DIR/ljx" +CONFIG="$SCRIPT_DIR/logjetd.conf" +OUTPUT_DIR="$SCRIPT_DIR/logs" +OUTPUT_FILE="$OUTPUT_DIR/metrics.logjet" + +if [ ! -x "$LJD" ]; then + echo "missing $LJD" + echo "build it first with: make demo" + exit 1 +fi + +if [ ! -x "$EMITTER" ]; then + echo "missing $EMITTER" + echo "build it first with: make demo" + exit 1 +fi + +if [ ! -x "$LJX" ]; then + echo "missing $LJX" + echo "build it first with: make demo" + exit 1 +fi + +cd "$SCRIPT_DIR" + +mkdir -p "$OUTPUT_DIR" +rm -f "$OUTPUT_FILE" "$OUTPUT_DIR/metrics-"*.logjet "$OUTPUT_DIR/metrics.stream-id" + +echo "starting ljd with config $CONFIG" +"$LJD" --config "$CONFIG" & +LJD_PID=$! + +cleanup() { + if [ -n "${EMITTER_PID:-}" ]; then + kill "$EMITTER_PID" 2>/dev/null || true + wait "$EMITTER_PID" 2>/dev/null || true + fi + if [ -n "${LJD_PID:-}" ]; then + kill "$LJD_PID" 2>/dev/null || true + wait "$LJD_PID" 2>/dev/null || true + fi +} +trap cleanup EXIT INT TERM + +sleep 1 + +METRIC_COUNT=15 +echo "starting metrics-emitter toward 127.0.0.1:4317 ($METRIC_COUNT batches)" +"$EMITTER" 127.0.0.1:4317 "$METRIC_COUNT" + +echo "emitter finished; giving ljd time to flush" +sleep 2 + +echo "stopping ljd" +kill "$LJD_PID" 2>/dev/null || true +wait "$LJD_PID" 2>/dev/null || true +LJD_PID="" + +echo "opening ljx view on $OUTPUT_FILE" +"$LJX" view "$OUTPUT_FILE" + +echo "cleaning up demo artefacts" +rm -rf "$OUTPUT_DIR" + +echo "done" diff --git a/demo/metrics-view/README.md b/demo/metrics-view/README.md new file mode 100644 index 0000000..af02510 --- /dev/null +++ b/demo/metrics-view/README.md @@ -0,0 +1,35 @@ +# metrics-view + +Generate OTLP metrics, ingest them into `ljd`, and browse the resulting `.logjet` file with `ljx view`. + +## What it does + +1. Starts `ljd` in file mode listening for OTLP/HTTP on `127.0.0.1:4318`. +2. Runs `metrics-emitter` which POSTs 15 `ExportMetricsServiceRequest` batches to `/v1/metrics`. + Each batch contains: + - a Gauge `cpu.usage` with a value that drifts between 10 % and 90 % + - a cumulative Sum `requests.total` that grows monotonically +3. Stops `ljd` after the emitter finishes. +4. Opens `ljx view` on the stored `metrics.logjet` file. + +## Prerequisites + +Build the demo binaries: + +```bash +make demo +``` + +## Run + +```bash +./run-demo.sh +``` + +## Keybindings in `ljx view` + +- `↑` / `↓` – move through records +- `Enter` – open detail modal for the selected record +- `Esc` – close modal +- `i` – toggle hex/inspection panel +- `q` – quit diff --git a/demo/metrics-view/logjetd.conf b/demo/metrics-view/logjetd.conf new file mode 100644 index 0000000..cb44657 --- /dev/null +++ b/demo/metrics-view/logjetd.conf @@ -0,0 +1,7 @@ +output: file +file.path: ./logs +file.size: 100000 +file.name: metrics.logjet +ingest.protocol: otlp-http +ingest.listen: 127.0.0.1:4318 +replay.listen: 127.0.0.1:7002 diff --git a/demo/metrics-view/run-demo.sh b/demo/metrics-view/run-demo.sh new file mode 100755 index 0000000..84f03ad --- /dev/null +++ b/demo/metrics-view/run-demo.sh @@ -0,0 +1,72 @@ +#!/bin/sh +set -eu + +SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +TARGET_DIR="$SCRIPT_DIR/../../target/debug" +LJD="$TARGET_DIR/ljd" +EMITTER="$TARGET_DIR/metrics-emitter" +LJX="$TARGET_DIR/ljx" +CONFIG="$SCRIPT_DIR/logjetd.conf" +OUTPUT_DIR="$SCRIPT_DIR/logs" +OUTPUT_FILE="$OUTPUT_DIR/metrics.logjet" + +if [ ! -x "$LJD" ]; then + echo "missing $LJD" + echo "build it first with: make demo" + exit 1 +fi + +if [ ! -x "$EMITTER" ]; then + echo "missing $EMITTER" + echo "build it first with: make demo" + exit 1 +fi + +if [ ! -x "$LJX" ]; then + echo "missing $LJX" + echo "build it first with: make demo" + exit 1 +fi + +cd "$SCRIPT_DIR" + +mkdir -p "$OUTPUT_DIR" +rm -f "$OUTPUT_FILE" "$OUTPUT_DIR/metrics-"*.logjet "$OUTPUT_DIR/metrics.stream-id" + +echo "starting ljd with config $CONFIG" +"$LJD" --config "$CONFIG" & +LJD_PID=$! + +cleanup() { + if [ -n "${EMITTER_PID:-}" ]; then + kill "$EMITTER_PID" 2>/dev/null || true + wait "$EMITTER_PID" 2>/dev/null || true + fi + if [ -n "${LJD_PID:-}" ]; then + kill "$LJD_PID" 2>/dev/null || true + wait "$LJD_PID" 2>/dev/null || true + fi +} +trap cleanup EXIT INT TERM + +sleep 1 + +METRIC_COUNT=15 +echo "starting metrics-emitter toward 127.0.0.1:4318 ($METRIC_COUNT batches)" +"$EMITTER" 127.0.0.1:4318 "$METRIC_COUNT" + +echo "emitter finished; giving ljd time to flush" +sleep 2 + +echo "stopping ljd" +kill "$LJD_PID" 2>/dev/null || true +wait "$LJD_PID" 2>/dev/null || true +LJD_PID="" + +echo "opening ljx view on $OUTPUT_FILE" +"$LJX" view "$OUTPUT_FILE" + +echo "cleaning up demo artefacts" +rm -rf "$OUTPUT_DIR" + +echo "done" diff --git a/demo/multi-signal-view/README.md b/demo/multi-signal-view/README.md new file mode 100644 index 0000000..3e03ae1 --- /dev/null +++ b/demo/multi-signal-view/README.md @@ -0,0 +1,27 @@ +# multi-signal-view + +Ingest logs, metrics, and traces (all over OTLP/HTTP) into a single `ljd` instance, then open `ljx view` to verify all three signals decode correctly side-by-side. + +## Run + +```bash +make demo +cd demo/multi-signal-view +./run-demo.sh +``` + +The demo: +1. Starts `ljd` with OTLP/HTTP ingest on `127.0.0.1:4318` +2. Emits 6 batches per signal (logs → metrics → traces), interleaved, via `multi-signal-emitter` +3. Stops `ljd` after flush +4. Opens `ljx view` on the resulting `.logjet` file +5. Cleans up after the viewer exits + +## What to look for in `ljx view` + +- **Logs rows**: show BOFH excuse text (body preview) +- **Metrics rows**: show `cpu.usage=N%` and `requests.total=N` summaries +- **Traces rows**: show `GET /api/items/N?page=M` span names with kind +- All three record types coexist in one file in arrival order +- Press `Enter` on any row → full decoded payload +- Press `i` → info panel with signal-specific metadata diff --git a/demo/multi-signal-view/logjetd.conf b/demo/multi-signal-view/logjetd.conf new file mode 100644 index 0000000..b209b0f --- /dev/null +++ b/demo/multi-signal-view/logjetd.conf @@ -0,0 +1,7 @@ +output: file +file.path: ./logs +file.size: 100000 +file.name: mixed.logjet +ingest.protocol: otlp-http +ingest.listen: 127.0.0.1:4318 +replay.listen: 127.0.0.1:7002 diff --git a/demo/multi-signal-view/run-demo.sh b/demo/multi-signal-view/run-demo.sh new file mode 100755 index 0000000..08476d3 --- /dev/null +++ b/demo/multi-signal-view/run-demo.sh @@ -0,0 +1,81 @@ +#!/bin/sh +set -eu + +SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +TARGET_DIR="$SCRIPT_DIR/../../target/debug" +LJD="$TARGET_DIR/ljd" +EMITTER="$TARGET_DIR/multi-signal-emitter" +LJX="$TARGET_DIR/ljx" +CONFIG="$SCRIPT_DIR/logjetd.conf" +OUTPUT_DIR="$SCRIPT_DIR/logs" +OUTPUT_FILE="$OUTPUT_DIR/mixed.logjet" + +if [ ! -x "$LJD" ]; then + echo "missing $LJD" + echo "build it first with: make demo" + exit 1 +fi + +if [ ! -x "$EMITTER" ]; then + echo "missing $EMITTER" + echo "build it first with: make demo" + exit 1 +fi + +if [ ! -x "$LJX" ]; then + echo "missing $LJX" + echo "build it first with: make demo" + exit 1 +fi + +cd "$SCRIPT_DIR" + +mkdir -p "$OUTPUT_DIR" +rm -f "$OUTPUT_FILE" "$OUTPUT_DIR/mixed-"*.logjet "$OUTPUT_DIR/mixed.stream-id" + +echo "starting ljd with config $CONFIG" +"$LJD" --config "$CONFIG" & +LJD_PID=$! + +cleanup() { + if [ -n "${EMITTER_PID:-}" ]; then + kill "$EMITTER_PID" 2>/dev/null || true + wait "$EMITTER_PID" 2>/dev/null || true + fi + if [ -n "${LJD_PID:-}" ]; then + kill "$LJD_PID" 2>/dev/null || true + wait "$LJD_PID" 2>/dev/null || true + fi +} +trap cleanup EXIT INT TERM + +sleep 1 + +BATCH_COUNT=6 +echo "starting multi-signal-emitter toward 127.0.0.1:4318 ($BATCH_COUNT batches per signal)" +"$EMITTER" 127.0.0.1:4318 "$BATCH_COUNT" + +echo "emitter finished; giving ljd time to flush" +sleep 2 + +echo "stopping ljd" +kill "$LJD_PID" 2>/dev/null || true +wait "$LJD_PID" 2>/dev/null || true +LJD_PID="" + +echo "opening ljx view on $OUTPUT_FILE" +echo "" +echo "TIP: Navigate through the list. You will see:" +echo " - BOFH log entries (body text preview)" +echo " - Metrics entries (cpu.usage=N%, requests.total=N)" +echo " - Traces entries (GET /api/items/N?page=M)" +echo "" +echo "Press Enter on any row to see the full decoded payload." +echo "Press 'i' for the info panel with signal-specific metadata." +echo "" +"$LJX" view "$OUTPUT_FILE" + +echo "cleaning up demo artefacts" +rm -rf "$OUTPUT_DIR" + +echo "done" diff --git a/demo/src/bin/metrics-emitter.rs b/demo/src/bin/metrics-emitter.rs new file mode 100644 index 0000000..493dc6e --- /dev/null +++ b/demo/src/bin/metrics-emitter.rs @@ -0,0 +1,42 @@ +use std::env; +use std::process; +use std::thread; +use std::time::Duration; + +fn main() { + let args: Vec = env::args().collect(); + if args.len() < 2 { + eprintln!("usage: metrics-emitter [count]"); + eprintln!(" addr - host:port of the OTLP/HTTP metrics endpoint (e.g. 127.0.0.1:4318)"); + eprintln!(" count - number of metric batches to emit (default: 20)"); + process::exit(1); + } + + let addr = &args[1]; + let count: u64 = args.get(2).and_then(|s| s.parse().ok()).unwrap_or(20); + + let endpoint = if addr.starts_with("http://") || addr.starts_with("https://") { + format!("{addr}/v1/metrics") + } else { + format!("http://{addr}/v1/metrics") + }; + + println!("metrics-emitter sending {count} batches to {endpoint}"); + + for sequence in 1..=count { + let request = otlp_demo::build_metrics_request(sequence); + match otlp_demo::post_otlp_http_metrics(&endpoint, &request) { + Ok(()) => { + println!("seq={sequence} -> sent metrics batch"); + } + Err(err) => { + eprintln!("seq={sequence} -> error: {err}"); + } + } + if sequence < count { + thread::sleep(Duration::from_millis(200)); + } + } + + println!("metrics-emitter finished"); +} diff --git a/demo/src/bin/metrics-grpc-emitter.rs b/demo/src/bin/metrics-grpc-emitter.rs new file mode 100644 index 0000000..abfbeda --- /dev/null +++ b/demo/src/bin/metrics-grpc-emitter.rs @@ -0,0 +1,37 @@ +use std::env; +use std::time::Duration; + +use opentelemetry_proto::tonic::collector::metrics::v1::{ExportMetricsServiceRequest, metrics_service_client::MetricsServiceClient}; +use tonic::Request; + +#[tokio::main] +async fn main() -> Result<(), Box> { + let addr = env::args().nth(1).unwrap_or_else(|| "127.0.0.1:4317".to_string()); + let count: u64 = env::args().nth(2).and_then(|s| s.parse().ok()).unwrap_or(15); + let endpoint = if addr.starts_with("http://") || addr.starts_with("https://") { addr } else { format!("http://{addr}") }; + + eprintln!("metrics-grpc-emitter sending {count} OTLP metrics batches to {endpoint}"); + + let client = MetricsServiceClient::connect(endpoint.clone()).await?; + let mut client = client; + + for sequence in 1..=count { + let request = otlp_demo::build_metrics_request(sequence); + match send_batch(&mut client, request).await { + Ok(()) => eprintln!("sent OTLP gRPC metrics batch #{sequence} to {endpoint}"), + Err(err) => eprintln!("send failed for batch #{sequence}: {err}"), + } + + if sequence < count { + tokio::time::sleep(Duration::from_millis(200)).await; + } + } + + eprintln!("metrics-grpc-emitter finished"); + Ok(()) +} + +async fn send_batch(client: &mut MetricsServiceClient, request: ExportMetricsServiceRequest) -> Result<(), Box> { + client.export(Request::new(request)).await?; + Ok(()) +} diff --git a/demo/src/bin/multi-signal-emitter.rs b/demo/src/bin/multi-signal-emitter.rs new file mode 100644 index 0000000..f5fa313 --- /dev/null +++ b/demo/src/bin/multi-signal-emitter.rs @@ -0,0 +1,65 @@ +use std::env; +use std::process; +use std::thread; +use std::time::Duration; + +fn main() { + let args: Vec = env::args().collect(); + if args.len() < 2 { + eprintln!("usage: multi-signal-emitter [count]"); + eprintln!(" addr - host:port of the OTLP/HTTP endpoint (e.g. 127.0.0.1:4318)"); + eprintln!(" count - number of batches per signal to emit (default: 8)"); + process::exit(1); + } + + let addr = &args[1]; + let count: u64 = args.get(2).and_then(|s| s.parse().ok()).unwrap_or(8); + + let logs_endpoint = if addr.starts_with("http://") || addr.starts_with("https://") { + format!("{addr}/v1/logs") + } else { + format!("http://{addr}/v1/logs") + }; + let metrics_endpoint = if addr.starts_with("http://") || addr.starts_with("https://") { + format!("{addr}/v1/metrics") + } else { + format!("http://{addr}/v1/metrics") + }; + let traces_endpoint = if addr.starts_with("http://") || addr.starts_with("https://") { + format!("{addr}/v1/traces") + } else { + format!("http://{addr}/v1/traces") + }; + + println!("multi-signal-emitter sending {count} batches per signal (logs, metrics, traces) to {addr}"); + + for sequence in 1..=count { + let log_request = otlp_demo::build_excuse_request(sequence); + match otlp_demo::post_otlp_http(&logs_endpoint, &log_request) { + Ok(()) => println!("seq={sequence} -> sent logs batch"), + Err(err) => eprintln!("seq={sequence} -> logs error: {err}"), + } + + thread::sleep(Duration::from_millis(200)); + + let metric_request = otlp_demo::build_metrics_request(sequence); + match otlp_demo::post_otlp_http_metrics(&metrics_endpoint, &metric_request) { + Ok(()) => println!("seq={sequence} -> sent metrics batch"), + Err(err) => eprintln!("seq={sequence} -> metrics error: {err}"), + } + + thread::sleep(Duration::from_millis(200)); + + let trace_request = otlp_demo::build_trace_request(sequence); + match otlp_demo::post_otlp_http_traces(&traces_endpoint, &trace_request) { + Ok(()) => println!("seq={sequence} -> sent traces batch"), + Err(err) => eprintln!("seq={sequence} -> traces error: {err}"), + } + + if sequence < count { + thread::sleep(Duration::from_millis(500)); + } + } + + println!("multi-signal-emitter finished"); +} diff --git a/demo/src/bin/traces-emitter.rs b/demo/src/bin/traces-emitter.rs new file mode 100644 index 0000000..dadc277 --- /dev/null +++ b/demo/src/bin/traces-emitter.rs @@ -0,0 +1,42 @@ +use std::env; +use std::process; +use std::thread; +use std::time::Duration; + +fn main() { + let args: Vec = env::args().collect(); + if args.len() < 2 { + eprintln!("usage: traces-emitter [count]"); + eprintln!(" addr - host:port of the OTLP/HTTP traces endpoint (e.g. 127.0.0.1:4318)"); + eprintln!(" count - number of trace batches to emit (default: 15)"); + process::exit(1); + } + + let addr = &args[1]; + let count: u64 = args.get(2).and_then(|s| s.parse().ok()).unwrap_or(15); + + let endpoint = if addr.starts_with("http://") || addr.starts_with("https://") { + format!("{addr}/v1/traces") + } else { + format!("http://{addr}/v1/traces") + }; + + println!("traces-emitter sending {count} batches to {endpoint}"); + + for sequence in 1..=count { + let request = otlp_demo::build_trace_request(sequence); + match otlp_demo::post_otlp_http_traces(&endpoint, &request) { + Ok(()) => { + println!("seq={sequence} -> sent traces batch"); + } + Err(err) => { + eprintln!("seq={sequence} -> error: {err}"); + } + } + if sequence < count { + thread::sleep(Duration::from_millis(300)); + } + } + + println!("traces-emitter finished"); +} diff --git a/demo/src/bin/traces-grpc-emitter.rs b/demo/src/bin/traces-grpc-emitter.rs new file mode 100644 index 0000000..6a73a31 --- /dev/null +++ b/demo/src/bin/traces-grpc-emitter.rs @@ -0,0 +1,37 @@ +use std::env; +use std::time::Duration; + +use opentelemetry_proto::tonic::collector::trace::v1::{ExportTraceServiceRequest, trace_service_client::TraceServiceClient}; +use tonic::Request; + +#[tokio::main] +async fn main() -> Result<(), Box> { + let addr = env::args().nth(1).unwrap_or_else(|| "127.0.0.1:4317".to_string()); + let count: u64 = env::args().nth(2).and_then(|s| s.parse().ok()).unwrap_or(15); + let endpoint = if addr.starts_with("http://") || addr.starts_with("https://") { addr } else { format!("http://{addr}") }; + + eprintln!("traces-grpc-emitter sending {count} OTLP trace batches to {endpoint}"); + + let client = TraceServiceClient::connect(endpoint.clone()).await?; + let mut client = client; + + for sequence in 1..=count { + let request = otlp_demo::build_trace_request(sequence); + match send_batch(&mut client, request).await { + Ok(()) => eprintln!("sent OTLP gRPC trace batch #{sequence} to {endpoint}"), + Err(err) => eprintln!("send failed for batch #{sequence}: {err}"), + } + + if sequence < count { + tokio::time::sleep(Duration::from_millis(300)).await; + } + } + + eprintln!("traces-grpc-emitter finished"); + Ok(()) +} + +async fn send_batch(client: &mut TraceServiceClient, request: ExportTraceServiceRequest) -> Result<(), Box> { + client.export(Request::new(request)).await?; + Ok(()) +} diff --git a/demo/src/lib.rs b/demo/src/lib.rs index 79fd7ec..0805dc4 100644 --- a/demo/src/lib.rs +++ b/demo/src/lib.rs @@ -8,9 +8,16 @@ use std::time::{Duration, SystemTime, UNIX_EPOCH}; use colored::Colorize; use opentelemetry_proto::tonic::collector::logs::v1::ExportLogsServiceRequest; +use opentelemetry_proto::tonic::collector::metrics::v1::ExportMetricsServiceRequest; use opentelemetry_proto::tonic::common::v1::{AnyValue, InstrumentationScope, KeyValue}; use opentelemetry_proto::tonic::logs::v1::{LogRecord, ResourceLogs, ScopeLogs, SeverityNumber}; +use opentelemetry_proto::tonic::metrics::v1::number_data_point::Value as DataPointValue; +use opentelemetry_proto::tonic::metrics::v1::{ + AggregationTemporality, Gauge, Metric, NumberDataPoint, ResourceMetrics, ScopeMetrics, Sum, +}; use opentelemetry_proto::tonic::resource::v1::Resource; +use opentelemetry_proto::tonic::collector::trace::v1::ExportTraceServiceRequest; +use opentelemetry_proto::tonic::trace::v1::{ResourceSpans, ScopeSpans, Span}; use prost::Message; use rustls::pki_types::{CertificateDer, PrivateKeyDer, ServerName}; use rustls::{ClientConfig, ClientConnection, RootCertStore, ServerConfig, StreamOwned}; @@ -114,8 +121,78 @@ pub fn post_otlp_http(addr: &str, request: &ExportLogsServiceRequest) -> io::Res DemoConnection::open(addr, None, None)?.post(&request.encode_to_vec()) } -pub fn post_raw_otlp_http(addr: &str, body: &[u8], ca_file: Option<&Path>, server_name: Option<&str>) -> io::Result<()> { - DemoConnection::open(addr, ca_file, server_name)?.post(body) +pub fn post_otlp_http_metrics(addr: &str, request: &ExportMetricsServiceRequest) -> io::Result<()> { + DemoConnection::open(addr, None, None)?.post(&request.encode_to_vec()) +} + +pub fn build_metrics_request(sequence: u64) -> ExportMetricsServiceRequest { + let nanos = unix_time_nanos(); + let cpu_value = 10.0 + ((sequence % 80) as f64) + (sequence % 7) as f64 * 0.5; + let request_count = sequence * 100 + 42; + + let resource = Resource { + attributes: vec![ + string_attr("service.name", "metrics-demo"), + string_attr("host.name", "garage-rig"), + ], + dropped_attributes_count: 0, + entity_refs: Vec::new(), + }; + + let scope = InstrumentationScope { + name: "demo-metrics-emitter".to_string(), + version: "0.1.0".to_string(), + attributes: Vec::new(), + dropped_attributes_count: 0, + }; + + let cpu_metric = Metric { + name: "cpu.usage".to_string(), + description: "Current CPU usage percentage".to_string(), + unit: "%".to_string(), + data: Some(opentelemetry_proto::tonic::metrics::v1::metric::Data::Gauge(Gauge { + data_points: vec![NumberDataPoint { + attributes: vec![string_attr("cpu", "all")], + start_time_unix_nano: 0, + time_unix_nano: nanos, + value: Some(DataPointValue::AsDouble(cpu_value)), + flags: 0, + exemplars: Vec::new(), + }], + })), + metadata: Vec::new(), + }; + + let requests_metric = Metric { + name: "requests.total".to_string(), + description: "Total number of requests served".to_string(), + unit: "1".to_string(), + data: Some(opentelemetry_proto::tonic::metrics::v1::metric::Data::Sum(Sum { + data_points: vec![NumberDataPoint { + attributes: vec![string_attr("method", "GET")], + start_time_unix_nano: nanos - 1_000_000_000, + time_unix_nano: nanos, + value: Some(DataPointValue::AsInt(request_count as i64)), + flags: 0, + exemplars: Vec::new(), + }], + aggregation_temporality: AggregationTemporality::Cumulative as i32, + is_monotonic: true, + })), + metadata: Vec::new(), + }; + + ExportMetricsServiceRequest { + resource_metrics: vec![ResourceMetrics { + resource: Some(resource), + scope_metrics: vec![ScopeMetrics { + scope: Some(scope), + metrics: vec![cpu_metric, requests_metric], + schema_url: String::new(), + }], + schema_url: String::new(), + }], + } } enum DemoStream { @@ -453,3 +530,79 @@ fn any_value_to_string(value: &AnyValue) -> Option<&str> { fn unix_time_nanos() -> u64 { SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_nanos() as u64 } + +pub fn build_trace_request(sequence: u64) -> ExportTraceServiceRequest { + let base_nanos = 1_700_000_000_000_000_000u64; + + ExportTraceServiceRequest { + resource_spans: vec![ResourceSpans { + resource: Some(Resource { + attributes: vec![ + string_attr("service.name", "traces-demo"), + string_attr("host.name", "garage-rig"), + ], + dropped_attributes_count: 0, + entity_refs: Vec::new(), + }), + scope_spans: vec![ScopeSpans { + scope: Some(InstrumentationScope { + name: "demo-traces-emitter".to_string(), + version: "0.1.0".to_string(), + attributes: Vec::new(), + dropped_attributes_count: 0, + }), + spans: vec![ + Span { + trace_id: vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, (sequence % 256) as u8], + span_id: vec![16, 17, 18, 19, 20, 21, 22, (sequence % 256) as u8], + parent_span_id: vec![], + name: format!("GET /api/items/{}?page={}", sequence, sequence % 5), + kind: 2, + start_time_unix_nano: base_nanos + sequence * 1_000_000, + end_time_unix_nano: base_nanos + sequence * 1_000_000 + ((sequence % 50 + 1) * 1_000_000), + attributes: vec![ + string_attr("http.method", "GET"), + string_attr("http.route", "/api/items/:id"), + int_attr("http.status_code", 200), + ], + dropped_attributes_count: 0, + events: vec![], + dropped_events_count: 0, + links: vec![], + dropped_links_count: 0, + status: Some(opentelemetry_proto::tonic::trace::v1::Status { code: 1, message: String::new() }), + flags: 0, + trace_state: String::new(), + }, + Span { + trace_id: vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, (sequence % 256) as u8], + span_id: vec![23, 24, 25, 26, 27, 28, 29, (sequence % 256) as u8], + parent_span_id: vec![16, 17, 18, 19, 20, 21, 22, (sequence % 256) as u8], + name: "SELECT items".to_string(), + kind: 3, + start_time_unix_nano: base_nanos + sequence * 1_000_000 + 2_000_000, + end_time_unix_nano: base_nanos + sequence * 1_000_000 + ((sequence % 50 + 1) * 1_000_000) - 1_000_000, + attributes: vec![ + string_attr("db.system", "postgres"), + string_attr("db.statement", "SELECT * FROM items WHERE id = $1"), + ], + dropped_attributes_count: 0, + events: vec![], + dropped_events_count: 0, + links: vec![], + dropped_links_count: 0, + status: Some(opentelemetry_proto::tonic::trace::v1::Status { code: 0, message: String::new() }), + flags: 0, + trace_state: String::new(), + }, + ], + schema_url: String::new(), + }], + schema_url: String::new(), + }], + } +} + +pub fn post_otlp_http_traces(addr: &str, request: &ExportTraceServiceRequest) -> io::Result<()> { + DemoConnection::open(addr, None, None)?.post(&request.encode_to_vec()) +} diff --git a/demo/traces-grpc-view/README.md b/demo/traces-grpc-view/README.md new file mode 100644 index 0000000..90ebce0 --- /dev/null +++ b/demo/traces-grpc-view/README.md @@ -0,0 +1,18 @@ +# traces-grpc-view + +Ingest OTLP/gRPC traces into `ljd`, then open `ljx view` on the result. + +## Run + +```bash +make demo +cd demo/traces-grpc-view +./run-demo.sh +``` + +The demo: +1. Starts `ljd` with OTLP/gRPC ingest on `127.0.0.1:4317` +2. Emits 12 trace batches via `traces-grpc-emitter` (`TraceService/Export`) +3. Stops `ljd` after flush +4. Opens `ljx view` on the resulting `.logjet` file +5. Cleans up after the viewer exits diff --git a/demo/traces-grpc-view/logjetd.conf b/demo/traces-grpc-view/logjetd.conf new file mode 100644 index 0000000..2ef8d77 --- /dev/null +++ b/demo/traces-grpc-view/logjetd.conf @@ -0,0 +1,7 @@ +output: file +file.path: ./logs +file.size: 100000 +file.name: traces.logjet +ingest.protocol: otlp-grpc +ingest.listen: 127.0.0.1:4317 +replay.listen: 127.0.0.1:7002 diff --git a/demo/traces-grpc-view/run-demo.sh b/demo/traces-grpc-view/run-demo.sh new file mode 100755 index 0000000..c0205cc --- /dev/null +++ b/demo/traces-grpc-view/run-demo.sh @@ -0,0 +1,72 @@ +#!/bin/sh +set -eu + +SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +TARGET_DIR="$SCRIPT_DIR/../../target/debug" +LJD="$TARGET_DIR/ljd" +EMITTER="$TARGET_DIR/traces-grpc-emitter" +LJX="$TARGET_DIR/ljx" +CONFIG="$SCRIPT_DIR/logjetd.conf" +OUTPUT_DIR="$SCRIPT_DIR/logs" +OUTPUT_FILE="$OUTPUT_DIR/traces.logjet" + +if [ ! -x "$LJD" ]; then + echo "missing $LJD" + echo "build it first with: make demo" + exit 1 +fi + +if [ ! -x "$EMITTER" ]; then + echo "missing $EMITTER" + echo "build it first with: make demo" + exit 1 +fi + +if [ ! -x "$LJX" ]; then + echo "missing $LJX" + echo "build it first with: make demo" + exit 1 +fi + +cd "$SCRIPT_DIR" + +mkdir -p "$OUTPUT_DIR" +rm -f "$OUTPUT_FILE" "$OUTPUT_DIR/traces-"*.logjet "$OUTPUT_DIR/traces.stream-id" + +echo "starting ljd with config $CONFIG" +"$LJD" --config "$CONFIG" & +LJD_PID=$! + +cleanup() { + if [ -n "${EMITTER_PID:-}" ]; then + kill "$EMITTER_PID" 2>/dev/null || true + wait "$EMITTER_PID" 2>/dev/null || true + fi + if [ -n "${LJD_PID:-}" ]; then + kill "$LJD_PID" 2>/dev/null || true + wait "$LJD_PID" 2>/dev/null || true + fi +} +trap cleanup EXIT INT TERM + +sleep 1 + +TRACE_COUNT=12 +echo "starting traces-grpc-emitter toward 127.0.0.1:4317 ($TRACE_COUNT batches)" +"$EMITTER" 127.0.0.1:4317 "$TRACE_COUNT" + +echo "emitter finished; giving ljd time to flush" +sleep 2 + +echo "stopping ljd" +kill "$LJD_PID" 2>/dev/null || true +wait "$LJD_PID" 2>/dev/null || true +LJD_PID="" + +echo "opening ljx view on $OUTPUT_FILE" +"$LJX" view "$OUTPUT_FILE" + +echo "cleaning up demo artefacts" +rm -rf "$OUTPUT_DIR" + +echo "done" diff --git a/demo/traces-view/README.md b/demo/traces-view/README.md new file mode 100644 index 0000000..5463569 --- /dev/null +++ b/demo/traces-view/README.md @@ -0,0 +1,24 @@ +# traces-view + +Ingest OTLP/HTTP traces into `ljd`, then open `ljx view` on the result to verify traces decode. + +## Run + +```bash +make demo +cd demo/traces-view +./run-demo.sh +``` + +The demo: +1. Starts `ljd` with OTLP/HTTP ingest on `127.0.0.1:4318` +2. Emits 12 trace batches via `traces-emitter` (HTTP `POST /v1/traces`) +3. Stops `ljd` after flush +4. Opens `ljx view` on the resulting `.logjet` file +5. Cleans up after the viewer exits + +## What to look for in `ljx view` + +- List rows should show span names like `GET /api/items/N?page=M` and span kind +- Press `Enter` to open the modal and see trace IDs, span IDs, parent-child relationships, attributes +- Press `i` to see per-kind and per-status span counts in the info panel diff --git a/demo/traces-view/logjetd.conf b/demo/traces-view/logjetd.conf new file mode 100644 index 0000000..5580b78 --- /dev/null +++ b/demo/traces-view/logjetd.conf @@ -0,0 +1,7 @@ +output: file +file.path: ./logs +file.size: 100000 +file.name: traces.logjet +ingest.protocol: otlp-http +ingest.listen: 127.0.0.1:4318 +replay.listen: 127.0.0.1:7002 diff --git a/demo/traces-view/run-demo.sh b/demo/traces-view/run-demo.sh new file mode 100755 index 0000000..3b8f1d3 --- /dev/null +++ b/demo/traces-view/run-demo.sh @@ -0,0 +1,72 @@ +#!/bin/sh +set -eu + +SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +TARGET_DIR="$SCRIPT_DIR/../../target/debug" +LJD="$TARGET_DIR/ljd" +EMITTER="$TARGET_DIR/traces-emitter" +LJX="$TARGET_DIR/ljx" +CONFIG="$SCRIPT_DIR/logjetd.conf" +OUTPUT_DIR="$SCRIPT_DIR/logs" +OUTPUT_FILE="$OUTPUT_DIR/traces.logjet" + +if [ ! -x "$LJD" ]; then + echo "missing $LJD" + echo "build it first with: make demo" + exit 1 +fi + +if [ ! -x "$EMITTER" ]; then + echo "missing $EMITTER" + echo "build it first with: make demo" + exit 1 +fi + +if [ ! -x "$LJX" ]; then + echo "missing $LJX" + echo "build it first with: make demo" + exit 1 +fi + +cd "$SCRIPT_DIR" + +mkdir -p "$OUTPUT_DIR" +rm -f "$OUTPUT_FILE" "$OUTPUT_DIR/traces-"*.logjet "$OUTPUT_DIR/traces.stream-id" + +echo "starting ljd with config $CONFIG" +"$LJD" --config "$CONFIG" & +LJD_PID=$! + +cleanup() { + if [ -n "${EMITTER_PID:-}" ]; then + kill "$EMITTER_PID" 2>/dev/null || true + wait "$EMITTER_PID" 2>/dev/null || true + fi + if [ -n "${LJD_PID:-}" ]; then + kill "$LJD_PID" 2>/dev/null || true + wait "$LJD_PID" 2>/dev/null || true + fi +} +trap cleanup EXIT INT TERM + +sleep 1 + +TRACE_COUNT=12 +echo "starting traces-emitter toward 127.0.0.1:4318 ($TRACE_COUNT batches)" +"$EMITTER" 127.0.0.1:4318 "$TRACE_COUNT" + +echo "emitter finished; giving ljd time to flush" +sleep 2 + +echo "stopping ljd" +kill "$LJD_PID" 2>/dev/null || true +wait "$LJD_PID" 2>/dev/null || true +LJD_PID="" + +echo "opening ljx view on $OUTPUT_FILE" +"$LJX" view "$OUTPUT_FILE" + +echo "cleaning up demo artefacts" +rm -rf "$OUTPUT_DIR" + +echo "done" diff --git a/doc/configuration.md b/doc/configuration.md index 979de06..6e04f2c 100644 --- a/doc/configuration.md +++ b/doc/configuration.md @@ -575,10 +575,10 @@ Values: - this is not OTLP - `otlp-http` - OTLP over HTTP protobuf - - accepts `POST /v1/logs` + - accepts `POST /v1/logs`, `POST /v1/metrics`, and `POST /v1/traces` - `otlp-grpc` - OTLP over gRPC - - accepts the standard logs `Export` RPC + - accepts the standard `LogsService/Export`, `MetricsService/Export`, and `TraceService/Export` RPCs - `plugin` - loads an ingest plugin shared library - passive plugins receive bytes from the `ingest.listen` TCP listener @@ -641,7 +641,7 @@ Enable TLS for OTLP ingest listeners. Behaviour: -- with `ingest.protocol: otlp-http`, `ljd` accepts HTTPS on `/v1/logs` +- with `ingest.protocol: otlp-http`, `ljd` accepts HTTPS on `/v1/logs`, `/v1/metrics`, and `/v1/traces` - with `ingest.protocol: otlp-grpc`, `ljd` accepts gRPC over TLS ### `ingest.ca-file` diff --git a/doc/daemon.md b/doc/daemon.md index 58b6303..c85d346 100644 --- a/doc/daemon.md +++ b/doc/daemon.md @@ -86,8 +86,7 @@ ljd --config /path/to/logjet.conf bridge --source 10.0.0.15:7002 - supports `ingest.protocol: plugin` with `ingest.plugin-path` - supports TLS on OTLP/HTTP and OTLP/gRPC ingest with `ingest.*` - can rate-limit accepted ingest batches through `ingest.max-batches-per-second` -- can keep higher-severity OTLP log batches during overload through `ingest.priority-severity-at-least` -- can emit overload counters on stderr through `ingest.overload-report-ms` +- can keep higher-severity OTLP batches during overload through `ingest.priority-severity-at-least` - stores raw OTLP protobuf bytes in configured storage For plugin ingest, explicit `.so` values in `ingest.plugin-path` are loaded @@ -131,7 +130,7 @@ upstream. - `ljd bridge` connects to another `ljd` replay listener - requests either `keep` or `drain` mode from the upstream side - continues forwarding newly replayed records -- forwards OTLP log payloads to every destination configured in `collector.url` +- forwards OTLP payloads to every destination configured in `collector.url` - reconnects after disconnect using the last in-process forwarded sequence - can also load and save the last forwarded sequence through `upstream.state-file` - resets saved bridge sequence state automatically when upstream stream identity changes @@ -150,7 +149,7 @@ upstream. - `ljd replay --path ... --name ...` - reads ordered rotated `.logjet` files -- sends stored OTLP log batches to the configured destination set +- sends stored OTLP batches to the configured destination set - uses `collector.url` by default - `--dest` can override the collector destination - if `collector.url` is a list, replay fans out to every configured destination diff --git a/doc/features.md b/doc/features.md index a815739..c39d516 100644 --- a/doc/features.md +++ b/doc/features.md @@ -5,28 +5,31 @@ It is meant to evolve as the daemon grows. ## Current Features -### 1. OTLP log ingest +### 1. OTLP ingest -`ljd` can accept real OTLP/HTTP protobuf log export requests on: +`ljd` can accept real OTLP/HTTP protobuf export requests on: ```text POST /v1/logs +POST /v1/metrics +POST /v1/traces ``` -It can also accept OTLP/gRPC log export requests on the standard -`LogsService/Export` endpoint when `ingest.protocol: otlp-grpc` is configured. +It can also accept OTLP/gRPC export requests on the standard +`LogsService/Export`, `MetricsService/Export`, and `TraceService/Export` +endpoints when `ingest.protocol: otlp-grpc` is configured. Current behaviour: -- accepts OTLP log batches over HTTP and gRPC +- accepts OTLP log, metrics, and trace batches over HTTP and gRPC - OTLP/HTTP ingest can also run over HTTPS - OTLP/gRPC ingest can also run over TLS - rejects oversized batches through `ingest.max-batch-bytes` - can cap concurrent ingest handling through `ingest.max-clients` - can rate-limit accepted ingest batches through `ingest.max-batches-per-second` -- can keep higher-severity OTLP log batches during overload through `ingest.priority-severity-at-least` +- can keep higher-severity OTLP batches during overload through `ingest.priority-severity-at-least` - emits overload counters on stderr through `ingest.overload-report-ms` -- validates that the request decodes as `ExportLogsServiceRequest` +- validates that the request decodes as `ExportLogsServiceRequest`, `ExportMetricsServiceRequest`, or `ExportTraceServiceRequest` - stores the raw OTLP protobuf bytes - assigns a local sequence number for internal replay ordering @@ -101,7 +104,7 @@ Current behaviour: - connects to another `ljd` replay listener - requests replay starting after the last sequence already forwarded - can keep upstream records or drain them, depending on `upstream.mode` -- stays attached and forwards new log records live +- stays attached and forwards new records live - posts raw stored OTLP protobuf payloads to every destination configured in `collector.url` - supports OTLP/HTTP export, HTTPS export, plain OTLP/gRPC export, and gRPC export over TLS or mutual TLS - `grpcs://...` uses server certificate validation through `collector.ca-file` @@ -155,7 +158,7 @@ Current behaviour: - scans for `name.logjet`, `name-1.logjet`, `name-2.logjet`, and so on - replays them in that order -- reads stored `logs` records +- reads stored logs, metrics, and traces records - posts the raw OTLP protobuf payloads to every configured replay destination - supports `http://`, `https://`, and `grpc://` collector URLs - sends as fast as the destination socket allows, with no artificial delay @@ -258,7 +261,7 @@ Useful when: Use case: - use the OTLP demo emitter -- send logs into `ljd` +- send logs, metrics, and traces into `ljd` - store them or inspect them locally Useful when: @@ -277,7 +280,7 @@ Use case: Useful when: - you want a fast demo -- you need bulk backfill of recorded OTLP logs +- you need bulk backfill of recorded OTLP logs, metrics, and traces - you want to validate stored files against a collector pipeline ### 6. File archive housekeeping outside the daemon @@ -300,7 +303,7 @@ Use case: - one `ljd` instance runs next to `OA` - a second `ljd` instance connects to the first over the network -- the second instance forwards retained backlog and live OTLP logs into an OTel Collector +- the second instance forwards retained backlog and live OTLP records into an OTel Collector Useful when: diff --git a/doc/ljx.md b/doc/ljx.md index e6e0d68..0cde140 100644 --- a/doc/ljx.md +++ b/doc/ljx.md @@ -107,9 +107,10 @@ streams matches across the full selection. Use `--nfs` to favour sequential reads on network file systems. When it is set, index lookups are skipped so scans avoid random access. -For OTLP log records, NDJSON output includes the core record fields when -present, including: +For OTLP records, NDJSON output includes structured fields depending on the +signal type: +**Logs:** - `body` - `timestamp` - `observed_timestamp` @@ -123,6 +124,35 @@ present, including: - `scope_version` - flattened resource, scope, and record attributes such as `service_name` +**Metrics:** +- `metric_name` +- `metric_description` +- `metric_unit` +- `metric_type` (`Gauge`, `Sum`, `Histogram`, `ExponentialHistogram`, `Summary`) +- `timestamp` +- `start_time` +- `value` (for Gauge/Sum) +- `is_monotonic` (for Sum) +- `aggregation_temporality` (for Sum) +- `count` (for Histogram/ExponentialHistogram/Summary) +- `sum` (for Histogram/ExponentialHistogram/Summary) +- flattened resource, scope, and datapoint attributes + +**Traces:** +- `trace_id` +- `span_id` +- `parent_span_id` +- `name` +- `kind` (`Internal`, `Server`, `Client`, `Producer`, `Consumer`) +- `start_time` +- `end_time` +- `duration_ns` +- `status_code` +- `status_message` +- `scope_name` +- `scope_version` +- flattened resource, scope, and span attributes + ## `ljx count` Count records in one `.logjet` file, optionally subject to a record-aware diff --git a/doc/manpage/ljd.1.md b/doc/manpage/ljd.1.md index 389c91b..84e88aa 100644 --- a/doc/manpage/ljd.1.md +++ b/doc/manpage/ljd.1.md @@ -28,8 +28,8 @@ ljd - OTLP ingest, `.logjet` storage, replay, and file blasting daemon It can: -- accept OTLP/HTTP log batches on `POST /v1/logs` -- accept OTLP/gRPC log batches on the standard `LogsService/Export` endpoint +- accept OTLP/HTTP batches on `POST /v1/logs`, `POST /v1/metrics`, and `POST /v1/traces` +- accept OTLP/gRPC batches on the standard `LogsService/Export`, `MetricsService/Export`, and `TraceService/Export` endpoints - optionally run OTLP/HTTP ingest over HTTPS and OTLP/gRPC ingest over TLS - store raw OTLP protobuf payloads either in memory or in append-only `.logjet` files - expose a replay listener for downstream consumers over the current internal wire protocol @@ -67,7 +67,7 @@ Current serve behaviour: ## bridge Connect to another `ljd` replay listener, drain retained backlog, stay -attached for live records, and forward OTLP log payloads to the configured +attached for live records, and forward OTLP payloads to the configured collector. Example: @@ -111,7 +111,7 @@ ljd segments --path /var/lib/logjet --name app.logjet ## replay -Read ordered `.logjet` files from a directory and blast the stored OTLP log +Read ordered `.logjet` files from a directory and blast the stored OTLP payloads into OTLP collectors. Example: @@ -324,8 +324,8 @@ Append-only file behaviour: # CURRENT FEATURES -- OTLP/HTTP log ingest on `POST /v1/logs` -- OTLP/gRPC log ingest on the standard logs export service +- OTLP/HTTP ingest on `POST /v1/logs`, `POST /v1/metrics`, and `POST /v1/traces` +- OTLP/gRPC ingest on the standard logs, metrics, and traces export services - optional TLS on OTLP/HTTP and OTLP/gRPC ingest - basic ingest guardrails through `ingest.max-batch-bytes` and `ingest.max-clients` - ingest rate limiting with severity-aware overload shedding diff --git a/doc/overview.md b/doc/overview.md index 4a8e90f..d9791ee 100644 --- a/doc/overview.md +++ b/doc/overview.md @@ -3,7 +3,7 @@ `logjet` is split into two parts: - `logjet`: a Rust library and `.logjet` block format for storing raw OTLP protobuf batches -- `ljd`: a daemon that accepts OTLP logs, keeps a backlog, and replays or blasts stored data later +- `ljd`: a daemon that accepts OTLP logs, metrics, and traces, keeps a backlog, and replays or blasts stored data later ## Components @@ -20,8 +20,8 @@ The library provides: The daemon provides: -- OTLP/HTTP ingest listener -- OTLP/gRPC ingest listener +- OTLP/HTTP ingest listener for logs, metrics, and traces +- OTLP/gRPC ingest listener for logs, metrics, and traces - optional TLS for OTLP ingest - internal wire-protocol ingest listener - replay listener diff --git a/ljx/Cargo.toml b/ljx/Cargo.toml index 13c5477..facef30 100644 --- a/ljx/Cargo.toml +++ b/ljx/Cargo.toml @@ -12,7 +12,7 @@ crossterm = "0.28" libloading = "0.8" liblogjet = { path = "../liblogjet" } logjet = { path = ".." } -opentelemetry-proto = { version = "0.31", features = ["gen-tonic", "logs"] } +opentelemetry-proto = { version = "0.31", features = ["gen-tonic", "logs", "metrics", "trace"] } prost = "0.14" ratatui = "0.30" regex = "1.12" diff --git a/ljx/src/commands/export.rs b/ljx/src/commands/export.rs index 03d20a1..de61bdb 100644 --- a/ljx/src/commands/export.rs +++ b/ljx/src/commands/export.rs @@ -9,9 +9,12 @@ use std::path::{Path, PathBuf}; use chrono::{TimeZone, Utc}; use logjet::{LogjetReader, OwnedRecord, RecordType}; use opentelemetry_proto::tonic::collector::logs::v1::ExportLogsServiceRequest; +use opentelemetry_proto::tonic::collector::metrics::v1::ExportMetricsServiceRequest; +use opentelemetry_proto::tonic::collector::trace::v1::ExportTraceServiceRequest; use opentelemetry_proto::tonic::common::v1::AnyValue; use opentelemetry_proto::tonic::common::v1::any_value::Value; use opentelemetry_proto::tonic::logs::v1::LogRecord; +use opentelemetry_proto::tonic::metrics::v1::metric::Data as MetricData; use prost::Message; use serde_json::{Map as JsonMap, Value as JsonValue}; @@ -88,45 +91,18 @@ pub(crate) fn export_ndjson_objects(record: &OwnedRecord, fields: &[String]) -> } pub(crate) fn export_ndjson_objects_with_preview(record: &OwnedRecord, fields: &[String], preview_bytes: Option) -> Vec { - if record.record_type != RecordType::Logs { - let mut obj = JsonMap::new(); - obj.insert("record_type".to_string(), JsonValue::String(record_kind_label(record.record_type).to_string())); - obj.insert("timestamp".to_string(), JsonValue::String(format_timestamp(record.ts_unix_ns))); - obj.insert("payload".to_string(), JsonValue::String(truncate_preview(&String::from_utf8_lossy(&record.payload), preview_bytes))); - return vec![select_json_fields(obj, fields)]; - } - - let Ok(batch) = ExportLogsServiceRequest::decode(record.payload.as_slice()) else { - let mut obj = JsonMap::new(); - obj.insert("timestamp".to_string(), JsonValue::String(format_timestamp(record.ts_unix_ns))); - obj.insert("payload".to_string(), JsonValue::String(truncate_preview(&String::from_utf8_lossy(&record.payload), preview_bytes))); - return vec![select_json_fields(obj, fields)]; - }; - - let mut out = Vec::new(); - for resource_logs in &batch.resource_logs { - let resource_attrs = resource_logs.resource.as_ref().map(|r| &r.attributes).map(Vec::as_slice).unwrap_or(&[]); - for scope_logs in &resource_logs.scope_logs { - let scope_attrs = scope_logs.scope.as_ref().map(|s| s.attributes.as_slice()).unwrap_or(&[]); - for log_record in &scope_logs.log_records { - let mut obj = JsonMap::new(); - if let Some(scope) = &scope_logs.scope { - if !scope.name.is_empty() { - obj.insert("scope_name".to_string(), JsonValue::String(scope.name.clone())); - } - if !scope.version.is_empty() { - obj.insert("scope_version".to_string(), JsonValue::String(scope.version.clone())); - } - } - insert_otlp_log_fields_with_preview(&mut obj, log_record, record.ts_unix_ns, preview_bytes); - flatten_otlp_attrs_into_json(&mut obj, resource_attrs, preview_bytes); - flatten_otlp_attrs_into_json(&mut obj, scope_attrs, preview_bytes); - flatten_otlp_attrs_into_json(&mut obj, &log_record.attributes, preview_bytes); - out.push(select_json_fields(obj, fields)); - } + match record.record_type { + RecordType::Logs => export_logs_ndjson(record, fields, preview_bytes), + RecordType::Metrics => export_metrics_ndjson(record, fields, preview_bytes), + RecordType::Traces => export_traces_ndjson(record, fields, preview_bytes), + _ => { + let mut obj = JsonMap::new(); + obj.insert("record_type".to_string(), JsonValue::String(record_kind_label(record.record_type).to_string())); + obj.insert("timestamp".to_string(), JsonValue::String(format_timestamp(record.ts_unix_ns))); + obj.insert("payload".to_string(), JsonValue::String(truncate_preview(&String::from_utf8_lossy(&record.payload), preview_bytes))); + vec![select_json_fields(obj, fields)] } } - out } fn insert_otlp_log_fields_with_preview( @@ -208,8 +184,296 @@ fn select_json_fields(mut obj: JsonMap, fields: &[String]) -> JsonValue::Object(selected) } -fn flatten_otlp_attrs_into_json( - target: &mut JsonMap, attrs: &[opentelemetry_proto::tonic::common::v1::KeyValue], preview_bytes: Option, +fn export_logs_ndjson(record: &OwnedRecord, fields: &[String], preview_bytes: Option) -> Vec { + let Ok(batch) = ExportLogsServiceRequest::decode(record.payload.as_slice()) else { + let mut obj = JsonMap::new(); + obj.insert("timestamp".to_string(), JsonValue::String(format_timestamp(record.ts_unix_ns))); + obj.insert("payload".to_string(), JsonValue::String(truncate_preview(&String::from_utf8_lossy(&record.payload), preview_bytes))); + return vec![select_json_fields(obj, fields)]; + }; + + let mut out = Vec::new(); + for resource_logs in &batch.resource_logs { + let resource_attrs = resource_logs.resource.as_ref().map(|r| &r.attributes).map(Vec::as_slice).unwrap_or(&[]); + for scope_logs in &resource_logs.scope_logs { + let scope_attrs = scope_logs.scope.as_ref().map(|s| s.attributes.as_slice()).unwrap_or(&[]); + for log_record in &scope_logs.log_records { + let mut obj = JsonMap::new(); + if let Some(scope) = &scope_logs.scope { + if !scope.name.is_empty() { + obj.insert("scope_name".to_string(), JsonValue::String(scope.name.clone())); + } + if !scope.version.is_empty() { + obj.insert("scope_version".to_string(), JsonValue::String(scope.version.clone())); + } + } + insert_otlp_log_fields_with_preview(&mut obj, log_record, record.ts_unix_ns, preview_bytes); + flatten_otlp_attrs_into_json(&mut obj, resource_attrs, preview_bytes); + flatten_otlp_attrs_into_json(&mut obj, scope_attrs, preview_bytes); + flatten_otlp_attrs_into_json(&mut obj, &log_record.attributes, preview_bytes); + out.push(select_json_fields(obj, fields)); + } + } + } + out +} + +fn export_metrics_ndjson(record: &OwnedRecord, fields: &[String], preview_bytes: Option) -> Vec { + let Ok(batch) = ExportMetricsServiceRequest::decode(record.payload.as_slice()) else { + let mut obj = JsonMap::new(); + obj.insert("timestamp".to_string(), JsonValue::String(format_timestamp(record.ts_unix_ns))); + obj.insert("payload".to_string(), JsonValue::String(truncate_preview(&String::from_utf8_lossy(&record.payload), preview_bytes))); + return vec![select_json_fields(obj, fields)]; + }; + + let mut out = Vec::new(); + for resource_metrics in &batch.resource_metrics { + let resource_attrs = resource_metrics.resource.as_ref().map(|r| &r.attributes).map(Vec::as_slice).unwrap_or(&[]); + for scope_metrics in &resource_metrics.scope_metrics { + let scope_attrs = scope_metrics.scope.as_ref().map(|s| s.attributes.as_slice()).unwrap_or(&[]); + let scope_name = scope_metrics.scope.as_ref().map(|s| s.name.clone()).unwrap_or_default(); + let scope_version = scope_metrics.scope.as_ref().map(|s| s.version.clone()).unwrap_or_default(); + for metric in &scope_metrics.metrics { + let metric_name = metric.name.clone(); + let metric_description = metric.description.clone(); + let metric_unit = metric.unit.clone(); + + match &metric.data { + Some(MetricData::Gauge(gauge)) => { + for dp in &gauge.data_points { + let mut obj = JsonMap::new(); + obj.insert("metric_name".to_string(), JsonValue::String(metric_name.clone())); + obj.insert("metric_description".to_string(), JsonValue::String(metric_description.clone())); + obj.insert("metric_unit".to_string(), JsonValue::String(metric_unit.clone())); + obj.insert("metric_type".to_string(), JsonValue::String("Gauge".to_string())); + if dp.time_unix_nano > 0 { + obj.insert("timestamp".to_string(), JsonValue::String(format_timestamp(dp.time_unix_nano))); + } + if dp.start_time_unix_nano > 0 { + obj.insert("start_time".to_string(), JsonValue::String(format_timestamp(dp.start_time_unix_nano))); + } + if let Some(ref value) = dp.value { + obj.insert("value".to_string(), data_point_value_to_json(value)); + } + flatten_otlp_attrs_into_json(&mut obj, resource_attrs, preview_bytes); + flatten_otlp_attrs_into_json(&mut obj, scope_attrs, preview_bytes); + flatten_otlp_attrs_into_json(&mut obj, &dp.attributes, preview_bytes); + if !scope_name.is_empty() { + obj.insert("scope_name".to_string(), JsonValue::String(scope_name.clone())); + } + if !scope_version.is_empty() { + obj.insert("scope_version".to_string(), JsonValue::String(scope_version.clone())); + } + out.push(select_json_fields(obj, fields)); + } + } + Some(MetricData::Sum(sum)) => { + for dp in &sum.data_points { + let mut obj = JsonMap::new(); + obj.insert("metric_name".to_string(), JsonValue::String(metric_name.clone())); + obj.insert("metric_description".to_string(), JsonValue::String(metric_description.clone())); + obj.insert("metric_unit".to_string(), JsonValue::String(metric_unit.clone())); + obj.insert("metric_type".to_string(), JsonValue::String("Sum".to_string())); + if dp.time_unix_nano > 0 { + obj.insert("timestamp".to_string(), JsonValue::String(format_timestamp(dp.time_unix_nano))); + } + if dp.start_time_unix_nano > 0 { + obj.insert("start_time".to_string(), JsonValue::String(format_timestamp(dp.start_time_unix_nano))); + } + if let Some(ref value) = dp.value { + obj.insert("value".to_string(), data_point_value_to_json(value)); + } + obj.insert("is_monotonic".to_string(), JsonValue::Bool(sum.is_monotonic)); + obj.insert("aggregation_temporality".to_string(), JsonValue::Number(sum.aggregation_temporality.into())); + flatten_otlp_attrs_into_json(&mut obj, resource_attrs, preview_bytes); + flatten_otlp_attrs_into_json(&mut obj, scope_attrs, preview_bytes); + flatten_otlp_attrs_into_json(&mut obj, &dp.attributes, preview_bytes); + if !scope_name.is_empty() { + obj.insert("scope_name".to_string(), JsonValue::String(scope_name.clone())); + } + if !scope_version.is_empty() { + obj.insert("scope_version".to_string(), JsonValue::String(scope_version.clone())); + } + out.push(select_json_fields(obj, fields)); + } + } + Some(MetricData::Histogram(hist)) => { + for dp in &hist.data_points { + let mut obj = JsonMap::new(); + obj.insert("metric_name".to_string(), JsonValue::String(metric_name.clone())); + obj.insert("metric_description".to_string(), JsonValue::String(metric_description.clone())); + obj.insert("metric_unit".to_string(), JsonValue::String(metric_unit.clone())); + obj.insert("metric_type".to_string(), JsonValue::String("Histogram".to_string())); + if dp.time_unix_nano > 0 { + obj.insert("timestamp".to_string(), JsonValue::String(format_timestamp(dp.time_unix_nano))); + } + if dp.start_time_unix_nano > 0 { + obj.insert("start_time".to_string(), JsonValue::String(format_timestamp(dp.start_time_unix_nano))); + } + obj.insert("count".to_string(), JsonValue::Number(dp.count.into())); + if let Some(sum) = dp.sum + && let Some(num) = serde_json::Number::from_f64(sum) + { + obj.insert("sum".to_string(), JsonValue::Number(num)); + } + obj.insert("aggregation_temporality".to_string(), JsonValue::Number(hist.aggregation_temporality.into())); + flatten_otlp_attrs_into_json(&mut obj, resource_attrs, preview_bytes); + flatten_otlp_attrs_into_json(&mut obj, scope_attrs, preview_bytes); + flatten_otlp_attrs_into_json(&mut obj, &dp.attributes, preview_bytes); + if !scope_name.is_empty() { + obj.insert("scope_name".to_string(), JsonValue::String(scope_name.clone())); + } + if !scope_version.is_empty() { + obj.insert("scope_version".to_string(), JsonValue::String(scope_version.clone())); + } + out.push(select_json_fields(obj, fields)); + } + } + Some(MetricData::ExponentialHistogram(ehist)) => { + for dp in &ehist.data_points { + let mut obj = JsonMap::new(); + obj.insert("metric_name".to_string(), JsonValue::String(metric_name.clone())); + obj.insert("metric_description".to_string(), JsonValue::String(metric_description.clone())); + obj.insert("metric_unit".to_string(), JsonValue::String(metric_unit.clone())); + obj.insert("metric_type".to_string(), JsonValue::String("ExponentialHistogram".to_string())); + if dp.time_unix_nano > 0 { + obj.insert("timestamp".to_string(), JsonValue::String(format_timestamp(dp.time_unix_nano))); + } + if dp.start_time_unix_nano > 0 { + obj.insert("start_time".to_string(), JsonValue::String(format_timestamp(dp.start_time_unix_nano))); + } + obj.insert("count".to_string(), JsonValue::Number(dp.count.into())); + if let Some(sum) = dp.sum + && let Some(num) = serde_json::Number::from_f64(sum) + { + obj.insert("sum".to_string(), JsonValue::Number(num)); + } + obj.insert("aggregation_temporality".to_string(), JsonValue::Number(ehist.aggregation_temporality.into())); + flatten_otlp_attrs_into_json(&mut obj, resource_attrs, preview_bytes); + flatten_otlp_attrs_into_json(&mut obj, scope_attrs, preview_bytes); + flatten_otlp_attrs_into_json(&mut obj, &dp.attributes, preview_bytes); + if !scope_name.is_empty() { + obj.insert("scope_name".to_string(), JsonValue::String(scope_name.clone())); + } + if !scope_version.is_empty() { + obj.insert("scope_version".to_string(), JsonValue::String(scope_version.clone())); + } + out.push(select_json_fields(obj, fields)); + } + } + Some(MetricData::Summary(summary)) => { + for dp in &summary.data_points { + let mut obj = JsonMap::new(); + obj.insert("metric_name".to_string(), JsonValue::String(metric_name.clone())); + obj.insert("metric_description".to_string(), JsonValue::String(metric_description.clone())); + obj.insert("metric_unit".to_string(), JsonValue::String(metric_unit.clone())); + obj.insert("metric_type".to_string(), JsonValue::String("Summary".to_string())); + if dp.time_unix_nano > 0 { + obj.insert("timestamp".to_string(), JsonValue::String(format_timestamp(dp.time_unix_nano))); + } + if dp.start_time_unix_nano > 0 { + obj.insert("start_time".to_string(), JsonValue::String(format_timestamp(dp.start_time_unix_nano))); + } + obj.insert("count".to_string(), JsonValue::Number(dp.count.into())); + if let Some(num) = serde_json::Number::from_f64(dp.sum) { + obj.insert("sum".to_string(), JsonValue::Number(num)); + } + flatten_otlp_attrs_into_json(&mut obj, resource_attrs, preview_bytes); + flatten_otlp_attrs_into_json(&mut obj, scope_attrs, preview_bytes); + flatten_otlp_attrs_into_json(&mut obj, &dp.attributes, preview_bytes); + if !scope_name.is_empty() { + obj.insert("scope_name".to_string(), JsonValue::String(scope_name.clone())); + } + if !scope_version.is_empty() { + obj.insert("scope_version".to_string(), JsonValue::String(scope_version.clone())); + } + out.push(select_json_fields(obj, fields)); + } + } + _ => {} + } + } + } + } + out +} + +fn export_traces_ndjson(record: &OwnedRecord, fields: &[String], preview_bytes: Option) -> Vec { + let Ok(batch) = ExportTraceServiceRequest::decode(record.payload.as_slice()) else { + let mut obj = JsonMap::new(); + obj.insert("timestamp".to_string(), JsonValue::String(format_timestamp(record.ts_unix_ns))); + obj.insert("payload".to_string(), JsonValue::String(truncate_preview(&String::from_utf8_lossy(&record.payload), preview_bytes))); + return vec![select_json_fields(obj, fields)]; + }; + + let mut out = Vec::new(); + for resource_spans in &batch.resource_spans { + let resource_attrs = resource_spans.resource.as_ref().map(|r| &r.attributes).map(Vec::as_slice).unwrap_or(&[]); + for scope_spans in &resource_spans.scope_spans { + let scope_attrs = scope_spans.scope.as_ref().map(|s| s.attributes.as_slice()).unwrap_or(&[]); + let scope_name = scope_spans.scope.as_ref().map(|s| s.name.clone()).unwrap_or_default(); + let scope_version = scope_spans.scope.as_ref().map(|s| s.version.clone()).unwrap_or_default(); + for span in &scope_spans.spans { + let mut obj = JsonMap::new(); + obj.insert("trace_id".to_string(), JsonValue::String(hex_encode(&span.trace_id))); + obj.insert("span_id".to_string(), JsonValue::String(hex_encode(&span.span_id))); + if !span.parent_span_id.is_empty() { + obj.insert("parent_span_id".to_string(), JsonValue::String(hex_encode(&span.parent_span_id))); + } + obj.insert("name".to_string(), JsonValue::String(span.name.clone())); + obj.insert("kind".to_string(), JsonValue::String(format_span_kind(span.kind))); + if span.start_time_unix_nano > 0 { + obj.insert("start_time".to_string(), JsonValue::String(format_timestamp(span.start_time_unix_nano))); + } + if span.end_time_unix_nano > 0 { + obj.insert("end_time".to_string(), JsonValue::String(format_timestamp(span.end_time_unix_nano))); + } + if span.end_time_unix_nano > span.start_time_unix_nano { + obj.insert("duration_ns".to_string(), JsonValue::Number((span.end_time_unix_nano - span.start_time_unix_nano).into())); + } + if let Some(status) = &span.status { + obj.insert("status_code".to_string(), JsonValue::Number(status.code.into())); + if !status.message.is_empty() { + obj.insert("status_message".to_string(), JsonValue::String(status.message.clone())); + } + } + if !scope_name.is_empty() { + obj.insert("scope_name".to_string(), JsonValue::String(scope_name.clone())); + } + if !scope_version.is_empty() { + obj.insert("scope_version".to_string(), JsonValue::String(scope_version.clone())); + } + flatten_otlp_attrs_into_json(&mut obj, resource_attrs, preview_bytes); + flatten_otlp_attrs_into_json(&mut obj, scope_attrs, preview_bytes); + flatten_otlp_attrs_into_json(&mut obj, &span.attributes, preview_bytes); + out.push(select_json_fields(obj, fields)); + } + } + } + out +} + +fn format_span_kind(kind: i32) -> String { + match kind { + 1 => "Internal".to_string(), + 2 => "Server".to_string(), + 3 => "Client".to_string(), + 4 => "Producer".to_string(), + 5 => "Consumer".to_string(), + _ => format!("Unknown({kind})"), + } +} + +fn data_point_value_to_json(value: &opentelemetry_proto::tonic::metrics::v1::number_data_point::Value) -> JsonValue { + match value { + opentelemetry_proto::tonic::metrics::v1::number_data_point::Value::AsDouble(v) => { + serde_json::Number::from_f64(*v).map(JsonValue::Number).unwrap_or(JsonValue::Null) + } + opentelemetry_proto::tonic::metrics::v1::number_data_point::Value::AsInt(v) => JsonValue::Number((*v).into()), + } +} + +fn flatten_otlp_attrs_into_json( target: &mut JsonMap, attrs: &[opentelemetry_proto::tonic::common::v1::KeyValue], preview_bytes: Option, ) { for attr in attrs { let key = attr.key.replace('.', "_"); @@ -284,6 +548,10 @@ fn format_timestamp(ts_unix_ns: u64) -> String { } } +#[cfg(test)] +#[path = "../../tests/unit/commands/export_ut.rs"] +mod export_ut; + #[cfg(test)] #[path = "../../tests/unit/commands/top_level_query_ut.rs"] mod top_level_query_ut; diff --git a/ljx/src/commands/view/detail.rs b/ljx/src/commands/view/detail.rs index b1fee2d..588628a 100644 --- a/ljx/src/commands/view/detail.rs +++ b/ljx/src/commands/view/detail.rs @@ -1,7 +1,10 @@ use chrono::{TimeZone, Utc}; use opentelemetry_proto::tonic::collector::logs::v1::ExportLogsServiceRequest; +use opentelemetry_proto::tonic::collector::metrics::v1::ExportMetricsServiceRequest; use opentelemetry_proto::tonic::common::v1::any_value::Value; use opentelemetry_proto::tonic::common::v1::{AnyValue, KeyValue}; +use opentelemetry_proto::tonic::collector::trace::v1::ExportTraceServiceRequest; +use opentelemetry_proto::tonic::metrics::v1::metric::Data as MetricData; use prost::Message; use ratatui::style::{Color, Modifier, Style}; use ratatui::text::{Line, Span}; @@ -14,6 +17,10 @@ use logjet::RecordType; pub(crate) fn format_summary(detail: &DetailRecord, hex_payload: bool) -> String { if hex_payload { hex_preview(&detail.payload, 32) + } else if detail.meta.record_type == RecordType::Metrics { + extract_otlp_metrics_summary(&detail.payload).unwrap_or_else(|| text_preview(&detail.payload, 160)) + } else if detail.meta.record_type == RecordType::Traces { + extract_otlp_traces_summary(&detail.payload).unwrap_or_else(|| text_preview(&detail.payload, 160)) } else if let Some(message) = extract_otlp_log_message(&detail.payload) { trim_single_line(&message, 160) } else { @@ -44,10 +51,113 @@ pub(super) fn render_detail_lines(detail: &DetailRecord, hex_payload: bool) -> V } fn render_otlp_lines(detail: &DetailRecord) -> Vec> { - if detail.meta.record_type != RecordType::Logs { - return Vec::new(); + match detail.meta.record_type { + RecordType::Logs => render_otlp_log_lines(detail), + RecordType::Metrics => render_otlp_metrics_lines(detail), + RecordType::Traces => render_otlp_traces_lines(detail), + _ => Vec::new(), + } +} + +fn render_otlp_traces_lines(detail: &DetailRecord) -> Vec> { + let Ok(batch) = ExportTraceServiceRequest::decode(detail.payload.as_slice()) else { + return vec![Line::from(vec![ + Span::styled("OTLP traces: ", Style::default().add_modifier(Modifier::BOLD)), + Span::raw("payload decode failed; showing raw preview"), + ])]; + }; + + let mut span_count = 0usize; + let mut services = Vec::new(); + + for resource_spans in &batch.resource_spans { + if let Some(resource) = &resource_spans.resource { + for attr in &resource.attributes { + if attr.key == "service.name" + && let Some(value) = &attr.value + && let Some(Value::StringValue(service)) = &value.value + && !services.iter().any(|existing| existing == service) + { + services.push(service.clone()); + } + } + } + for scope_spans in &resource_spans.scope_spans { + span_count += scope_spans.spans.len(); + } + } + + let mut lines = vec![ + key_value_line("OTLP kind:", "traces".to_string(), Style::default().fg(Color::White)), + key_value_line("Resources:", batch.resource_spans.len().to_string(), Style::default().fg(Color::White)), + key_value_line("Spans:", span_count.to_string(), Style::default().fg(Color::White)), + ]; + + if !services.is_empty() { + lines.push(key_value_line("Services:", services.join(", "), Style::default().fg(Color::White))); + } + + lines +} + +fn render_otlp_metrics_lines(detail: &DetailRecord) -> Vec> { + use opentelemetry_proto::tonic::collector::metrics::v1::ExportMetricsServiceRequest; + use opentelemetry_proto::tonic::metrics::v1::metric::Data; + + let Ok(batch) = ExportMetricsServiceRequest::decode(detail.payload.as_slice()) else { + return vec![Line::from(vec![ + Span::styled("OTLP metrics: ", Style::default().add_modifier(Modifier::BOLD)), + Span::raw("payload decode failed; showing raw preview"), + ])]; + }; + + let mut metric_count = 0usize; + let mut datapoint_count = 0usize; + let mut services = Vec::new(); + + for resource_metrics in &batch.resource_metrics { + if let Some(resource) = &resource_metrics.resource { + for attr in &resource.attributes { + if attr.key == "service.name" + && let Some(value) = &attr.value + && let Some(Value::StringValue(service)) = &value.value + && !services.iter().any(|existing| existing == service) + { + services.push(service.clone()); + } + } + } + for scope_metrics in &resource_metrics.scope_metrics { + for metric in &scope_metrics.metrics { + metric_count += 1; + let dp_len = match metric.data.as_ref() { + Some(Data::Gauge(g)) => g.data_points.len(), + Some(Data::Sum(s)) => s.data_points.len(), + Some(Data::Histogram(h)) => h.data_points.len(), + Some(Data::ExponentialHistogram(eh)) => eh.data_points.len(), + Some(Data::Summary(s)) => s.data_points.len(), + None => 0, + }; + datapoint_count += dp_len; + } + } + } + + let mut lines = vec![ + key_value_line("OTLP kind:", "metrics".to_string(), Style::default().fg(Color::White)), + key_value_line("Resources:", batch.resource_metrics.len().to_string(), Style::default().fg(Color::White)), + key_value_line("Metrics:", metric_count.to_string(), Style::default().fg(Color::White)), + key_value_line("Datapoints:", datapoint_count.to_string(), Style::default().fg(Color::White)), + ]; + + if !services.is_empty() { + lines.push(key_value_line("Services:", services.join(", "), Style::default().fg(Color::White))); } + lines +} + +fn render_otlp_log_lines(detail: &DetailRecord) -> Vec> { let Ok(batch) = ExportLogsServiceRequest::decode(detail.payload.as_slice()) else { return vec![Line::from(vec![ Span::styled("OTLP logs: ", Style::default().add_modifier(Modifier::BOLD)), @@ -138,10 +248,199 @@ pub(crate) fn render_modal_message(detail: &DetailRecord, hex_payload: bool) -> if let Some(message) = extract_otlp_log_message(&detail.payload) { return message; } + if detail.meta.record_type == RecordType::Metrics + && let Some(message) = extract_otlp_metrics_message(&detail.payload) + { + return message; + } + if detail.meta.record_type == RecordType::Traces + && let Some(message) = extract_otlp_traces_message(&detail.payload) + { + return message; + } if hex_payload { hex_dump(&detail.payload) } else { String::from_utf8_lossy(&detail.payload).into_owned() } } +pub(crate) fn extract_otlp_metrics_summary(payload: &[u8]) -> Option { + use opentelemetry_proto::tonic::collector::metrics::v1::ExportMetricsServiceRequest; + use opentelemetry_proto::tonic::metrics::v1::metric::Data; + + let batch = ExportMetricsServiceRequest::decode(payload).ok()?; + let mut parts = Vec::new(); + + for resource_metrics in &batch.resource_metrics { + for scope_metrics in &resource_metrics.scope_metrics { + for metric in &scope_metrics.metrics { + let value = match metric.data.as_ref() { + Some(Data::Gauge(g)) => g.data_points.first().and_then(|dp| dp.value.as_ref()).map(format_data_point_value), + Some(Data::Sum(s)) => s.data_points.first().and_then(|dp| dp.value.as_ref()).map(format_data_point_value), + Some(Data::Histogram(h)) => h.data_points.first().map(|dp| format!("count={}", dp.count)), + Some(Data::ExponentialHistogram(eh)) => eh.data_points.first().map(|dp| format!("count={}", dp.count)), + Some(Data::Summary(s)) => s.data_points.first().map(|dp| format!("count={}", dp.count)), + None => None, + }; + if let Some(v) = value { + parts.push(format!("{}={}{}", metric.name, v, metric.unit)); + } else { + parts.push(metric.name.clone()); + } + } + } + } + + if parts.is_empty() { + None + } else { + Some(parts.join(", ")) + } +} + +pub(crate) fn extract_otlp_traces_summary(payload: &[u8]) -> Option { + let batch = ExportTraceServiceRequest::decode(payload).ok()?; + let mut parts = Vec::new(); + + for resource_spans in &batch.resource_spans { + for scope_spans in &resource_spans.scope_spans { + for span in &scope_spans.spans { + let status = span.status.as_ref().map(|s| s.code.to_string()).unwrap_or_default(); + let name = &span.name; + let kind = format_span_kind(span.kind); + if status.is_empty() { + parts.push(format!("{name} ({kind})")); + } else { + parts.push(format!("{name} ({kind}, status={status})")); + } + } + } + } + + if parts.is_empty() { + None + } else { + Some(parts.join(", ")) + } +} + +pub(crate) fn extract_otlp_traces_message(payload: &[u8]) -> Option { + let batch = ExportTraceServiceRequest::decode(payload).ok()?; + let mut lines = Vec::new(); + + for resource_spans in &batch.resource_spans { + for scope_spans in &resource_spans.scope_spans { + for span in &scope_spans.spans { + lines.push(format!("Span: {}", span.name)); + if !span.trace_id.is_empty() { + lines.push(format!(" Trace ID: {}", hex_encode(&span.trace_id))); + } + if !span.span_id.is_empty() { + lines.push(format!(" Span ID: {}", hex_encode(&span.span_id))); + } + if !span.parent_span_id.is_empty() { + lines.push(format!(" Parent Span ID: {}", hex_encode(&span.parent_span_id))); + } + lines.push(format!(" Kind: {}", format_span_kind(span.kind))); + lines.push(format!(" Start: {}", format_timestamp(span.start_time_unix_nano))); + lines.push(format!(" End: {}", format_timestamp(span.end_time_unix_nano))); + if let Some(status) = &span.status { + lines.push(format!(" Status: code={} message={}", status.code, status.message)); + } + for attr in &span.attributes { + lines.push(format!(" Attr: {}={}", attr.key, attr.value.as_ref().map(|v| format_any_value(Some(v))).unwrap_or_default())); + } + lines.push(String::new()); + } + } + } + + if lines.is_empty() { + None + } else { + Some(lines.join("\n")) + } +} + +fn format_span_kind(kind: i32) -> String { + match kind { + 1 => "Internal".to_string(), + 2 => "Server".to_string(), + 3 => "Client".to_string(), + 4 => "Producer".to_string(), + 5 => "Consumer".to_string(), + _ => format!("Unknown({kind})"), + } +} + +pub(crate) fn extract_otlp_metrics_message(payload: &[u8]) -> Option { + use opentelemetry_proto::tonic::collector::metrics::v1::ExportMetricsServiceRequest; + use opentelemetry_proto::tonic::metrics::v1::metric::Data; + + let batch = ExportMetricsServiceRequest::decode(payload).ok()?; + let mut lines = Vec::new(); + + for resource_metrics in &batch.resource_metrics { + for scope_metrics in &resource_metrics.scope_metrics { + for metric in &scope_metrics.metrics { + lines.push(format!("Metric: {}", metric.name)); + if !metric.description.is_empty() { + lines.push(format!(" Description: {}", metric.description)); + } + if !metric.unit.is_empty() { + lines.push(format!(" Unit: {}", metric.unit)); + } + + match metric.data.as_ref() { + Some(Data::Gauge(g)) => { + lines.push(" Type: Gauge".to_string()); + for dp in &g.data_points { + lines.push(format!(" - time={}, value={}", format_timestamp(dp.time_unix_nano), dp.value.as_ref().map(format_data_point_value).unwrap_or_default())); + } + } + Some(Data::Sum(s)) => { + lines.push(format!(" Type: Sum (monotonic={}, temporality={})", s.is_monotonic, s.aggregation_temporality)); + for dp in &s.data_points { + lines.push(format!(" - time={}, start_time={}, value={}", format_timestamp(dp.time_unix_nano), format_timestamp(dp.start_time_unix_nano), dp.value.as_ref().map(format_data_point_value).unwrap_or_default())); + } + } + Some(Data::Histogram(h)) => { + lines.push(" Type: Histogram".to_string()); + for dp in &h.data_points { + lines.push(format!(" - time={}, count={}, sum={:?}", format_timestamp(dp.time_unix_nano), dp.count, dp.sum)); + } + } + Some(Data::ExponentialHistogram(eh)) => { + lines.push(" Type: ExponentialHistogram".to_string()); + for dp in &eh.data_points { + lines.push(format!(" - time={}, count={}, scale={}", format_timestamp(dp.time_unix_nano), dp.count, dp.scale)); + } + } + Some(Data::Summary(s)) => { + lines.push(" Type: Summary".to_string()); + for dp in &s.data_points { + lines.push(format!(" - time={}, count={}, sum={}", format_timestamp(dp.time_unix_nano), dp.count, dp.sum)); + } + } + None => {} + } + lines.push(String::new()); + } + } + } + + if lines.is_empty() { + None + } else { + Some(lines.join("\n")) + } +} + +fn format_data_point_value(value: &opentelemetry_proto::tonic::metrics::v1::number_data_point::Value) -> String { + match value { + opentelemetry_proto::tonic::metrics::v1::number_data_point::Value::AsDouble(v) => format!("{v}"), + opentelemetry_proto::tonic::metrics::v1::number_data_point::Value::AsInt(v) => v.to_string(), + } +} + pub(super) fn render_modal_footer(detail: &DetailRecord) -> Line<'static> { let (size_num, size_unit) = format_size_parts(detail.meta.payload_len); Line::from(vec![ @@ -178,10 +477,211 @@ pub(crate) fn render_modal_info_entries(detail: &DetailRecord) -> Vec<(String, S ("payload_bytes".to_string(), detail.meta.payload_len.to_string()), ]; - if detail.meta.record_type != RecordType::Logs { - return lines; + match detail.meta.record_type { + RecordType::Logs => lines.extend(render_modal_log_info_entries(detail)), + RecordType::Metrics => lines.extend(render_modal_metrics_info_entries(detail)), + RecordType::Traces => lines.extend(render_modal_traces_info_entries(detail)), + _ => {} + } + + lines +} + +fn render_modal_traces_info_entries(detail: &DetailRecord) -> Vec<(String, String)> { + let Ok(batch) = ExportTraceServiceRequest::decode(detail.payload.as_slice()) else { + return vec![("otlp".to_string(), "decode failed".to_string())]; + }; + + let mut entries = vec![("otlp.kind".to_string(), "traces".to_string())]; + entries.push(("resources".to_string(), batch.resource_spans.len().to_string())); + + let mut service_names = Vec::new(); + let mut scope_names = Vec::new(); + let mut span_names = Vec::new(); + let mut span_count = 0usize; + let mut kind_counts = std::collections::BTreeMap::new(); + let mut status_counts = std::collections::BTreeMap::new(); + + for resource_spans in &batch.resource_spans { + if let Some(resource) = &resource_spans.resource { + for attr in &resource.attributes { + if attr.key == "service.name" + && let Some(value) = &attr.value + && let Some(Value::StringValue(service)) = &value.value + && !service_names.iter().any(|existing| existing == service) + { + service_names.push(service.clone()); + } + } + } + for scope_spans in &resource_spans.scope_spans { + if let Some(scope) = &scope_spans.scope + && !scope.name.is_empty() + && !scope_names.iter().any(|existing| existing == &scope.name) + { + scope_names.push(scope.name.clone()); + } + for span in &scope_spans.spans { + span_count += 1; + if !span.name.is_empty() && !span_names.iter().any(|existing| existing == &span.name) { + span_names.push(span.name.clone()); + } + *kind_counts.entry(format_span_kind(span.kind)).or_insert(0usize) += 1; + if let Some(status) = &span.status { + *status_counts.entry(status.code.to_string()).or_insert(0usize) += 1; + } + } + } + } + + if !service_names.is_empty() { + entries.push(("service.name".to_string(), service_names.join(", "))); + } + if !scope_names.is_empty() { + entries.push(("scope".to_string(), scope_names.join(", "))); + } + entries.push(("spans".to_string(), span_count.to_string())); + if !span_names.is_empty() { + entries.push(("span.names".to_string(), span_names.join(", "))); + } + for (kind, count) in kind_counts { + entries.push((format!("span.kind.{kind}"), count.to_string())); + } + for (code, count) in status_counts { + entries.push((format!("span.status.{code}"), count.to_string())); + } + + entries +} + +fn render_modal_metrics_info_entries(detail: &DetailRecord) -> Vec<(String, String)> { + use opentelemetry_proto::tonic::collector::metrics::v1::ExportMetricsServiceRequest; + use opentelemetry_proto::tonic::metrics::v1::metric::Data; + + let Ok(batch) = ExportMetricsServiceRequest::decode(detail.payload.as_slice()) else { + return vec![("otlp".to_string(), "decode failed".to_string())]; + }; + + let mut entries = vec![("otlp.kind".to_string(), "metrics".to_string())]; + entries.push(("resources".to_string(), batch.resource_metrics.len().to_string())); + + let mut metric_names = Vec::new(); + let mut metric_count = 0usize; + let mut datapoint_count = 0usize; + let mut service_names = Vec::new(); + let mut scope_names = Vec::new(); + + for resource_metrics in &batch.resource_metrics { + if let Some(resource) = &resource_metrics.resource { + for attr in &resource.attributes { + if attr.key == "service.name" + && let Some(value) = &attr.value + && let Some(Value::StringValue(service)) = &value.value + && !service_names.iter().any(|existing| existing == service) + { + service_names.push(service.clone()); + } + } + } + for scope_metrics in &resource_metrics.scope_metrics { + if let Some(scope) = &scope_metrics.scope + && !scope.name.is_empty() + && !scope_names.iter().any(|existing| existing == &scope.name) + { + scope_names.push(scope.name.clone()); + } + for metric in &scope_metrics.metrics { + metric_count += 1; + if !metric.name.is_empty() && !metric_names.iter().any(|existing| existing == &metric.name) { + metric_names.push(metric.name.clone()); + } + let dp_len = match metric.data.as_ref() { + Some(Data::Gauge(g)) => g.data_points.len(), + Some(Data::Sum(s)) => s.data_points.len(), + Some(Data::Histogram(h)) => h.data_points.len(), + Some(Data::ExponentialHistogram(eh)) => eh.data_points.len(), + Some(Data::Summary(s)) => s.data_points.len(), + None => 0, + }; + datapoint_count += dp_len; + } + } + } + + if !service_names.is_empty() { + entries.push(("service.name".to_string(), service_names.join(", "))); + } + if !scope_names.is_empty() { + entries.push(("scope".to_string(), scope_names.join(", "))); + } + entries.push(("metrics".to_string(), metric_count.to_string())); + entries.push(("datapoints".to_string(), datapoint_count.to_string())); + if !metric_names.is_empty() { + entries.push(("metric.names".to_string(), metric_names.join(", "))); + } + + // Add per-metric detail entries + for resource_metrics in &batch.resource_metrics { + for scope_metrics in &resource_metrics.scope_metrics { + for metric in &scope_metrics.metrics { + let prefix = format!("metric.{}" , metric.name); + entries.push((format!("{prefix}.unit"), metric.unit.clone())); + if !metric.description.is_empty() { + entries.push((format!("{prefix}.description"), metric.description.clone())); + } + match metric.data.as_ref() { + Some(Data::Gauge(g)) => { + entries.push((format!("{prefix}.kind"), "Gauge".to_string())); + for (i, dp) in g.data_points.iter().enumerate() { + let val = dp.value.as_ref().map(format_data_point_value).unwrap_or_default(); + entries.push((format!("{prefix}.dp{i}.value"), val)); + entries.push((format!("{prefix}.dp{i}.time"), format_timestamp(dp.time_unix_nano))); + } + } + Some(Data::Sum(s)) => { + entries.push((format!("{prefix}.kind"), "Sum".to_string())); + entries.push((format!("{prefix}.monotonic"), s.is_monotonic.to_string())); + entries.push((format!("{prefix}.temporality"), s.aggregation_temporality.to_string())); + for (i, dp) in s.data_points.iter().enumerate() { + let val = dp.value.as_ref().map(format_data_point_value).unwrap_or_default(); + entries.push((format!("{prefix}.dp{i}.value"), val)); + entries.push((format!("{prefix}.dp{i}.time"), format_timestamp(dp.time_unix_nano))); + entries.push((format!("{prefix}.dp{i}.start_time"), format_timestamp(dp.start_time_unix_nano))); + } + } + Some(Data::Histogram(h)) => { + entries.push((format!("{prefix}.kind"), "Histogram".to_string())); + for (i, dp) in h.data_points.iter().enumerate() { + entries.push((format!("{prefix}.dp{i}.count"), dp.count.to_string())); + entries.push((format!("{prefix}.dp{i}.time"), format_timestamp(dp.time_unix_nano))); + } + } + Some(Data::ExponentialHistogram(eh)) => { + entries.push((format!("{prefix}.kind"), "ExpHistogram".to_string())); + for (i, dp) in eh.data_points.iter().enumerate() { + entries.push((format!("{prefix}.dp{i}.count"), dp.count.to_string())); + entries.push((format!("{prefix}.dp{i}.time"), format_timestamp(dp.time_unix_nano))); + } + } + Some(Data::Summary(s)) => { + entries.push((format!("{prefix}.kind"), "Summary".to_string())); + for (i, dp) in s.data_points.iter().enumerate() { + entries.push((format!("{prefix}.dp{i}.count"), dp.count.to_string())); + entries.push((format!("{prefix}.dp{i}.sum"), dp.sum.to_string())); + entries.push((format!("{prefix}.dp{i}.time"), format_timestamp(dp.time_unix_nano))); + } + } + None => {} + } + } + } } + entries +} + +fn render_modal_log_info_entries(detail: &DetailRecord) -> Vec<(String, String)> { + let mut lines = vec![]; let Ok(batch) = ExportLogsServiceRequest::decode(detail.payload.as_slice()) else { lines.push(("otlp".to_string(), "decode failed".to_string())); return lines; @@ -336,14 +836,21 @@ pub(crate) fn parse_export_selection(input: &str, total: usize, selected: usize) } pub(super) fn export_ndjson_objects(detail: &DetailRecord) -> Vec { - if detail.meta.record_type != RecordType::Logs { - let mut obj = JsonMap::new(); - obj.insert("record_type".to_string(), JsonValue::String(record_kind_label(detail.meta.record_type).to_string())); - obj.insert("timestamp".to_string(), JsonValue::String(format_timestamp(detail.meta.ts_unix_ns))); - obj.insert("payload".to_string(), JsonValue::String(String::from_utf8_lossy(&detail.payload).to_string())); - return vec![JsonValue::Object(obj)]; + match detail.meta.record_type { + RecordType::Logs => export_logs_ndjson(detail), + RecordType::Metrics => export_metrics_ndjson(detail), + RecordType::Traces => export_traces_ndjson(detail), + _ => { + let mut obj = JsonMap::new(); + obj.insert("record_type".to_string(), JsonValue::String(record_kind_label(detail.meta.record_type).to_string())); + obj.insert("timestamp".to_string(), JsonValue::String(format_timestamp(detail.meta.ts_unix_ns))); + obj.insert("payload".to_string(), JsonValue::String(String::from_utf8_lossy(&detail.payload).to_string())); + vec![JsonValue::Object(obj)] + } } +} +fn export_logs_ndjson(detail: &DetailRecord) -> Vec { let Ok(batch) = ExportLogsServiceRequest::decode(detail.payload.as_slice()) else { let mut obj = JsonMap::new(); obj.insert("timestamp".to_string(), JsonValue::String(format_timestamp(detail.meta.ts_unix_ns))); @@ -377,6 +884,203 @@ pub(super) fn export_ndjson_objects(detail: &DetailRecord) -> Vec { out } +fn export_metrics_ndjson(detail: &DetailRecord) -> Vec { + let Ok(batch) = ExportMetricsServiceRequest::decode(detail.payload.as_slice()) else { + let mut obj = JsonMap::new(); + obj.insert("timestamp".to_string(), JsonValue::String(format_timestamp(detail.meta.ts_unix_ns))); + obj.insert("payload".to_string(), JsonValue::String(String::from_utf8_lossy(&detail.payload).to_string())); + return vec![JsonValue::Object(obj)]; + }; + + let mut out = Vec::new(); + for resource_metrics in &batch.resource_metrics { + let resource_attrs = resource_metrics.resource.as_ref().map(|r| &r.attributes).map(Vec::as_slice).unwrap_or(&[]); + for scope_metrics in &resource_metrics.scope_metrics { + let scope_attrs = scope_metrics.scope.as_ref().map(|s| s.attributes.as_slice()).unwrap_or(&[]); + for metric in &scope_metrics.metrics { + match metric.data.as_ref() { + Some(MetricData::Gauge(g)) => { + for dp in &g.data_points { + let mut obj = JsonMap::new(); + obj.insert("metric_name".to_string(), JsonValue::String(metric.name.clone())); + if !metric.description.is_empty() { + obj.insert("metric_description".to_string(), JsonValue::String(metric.description.clone())); + } + if !metric.unit.is_empty() { + obj.insert("metric_unit".to_string(), JsonValue::String(metric.unit.clone())); + } + obj.insert("metric_type".to_string(), JsonValue::String("Gauge".to_string())); + obj.insert("timestamp".to_string(), JsonValue::String(format_timestamp(dp.time_unix_nano))); + if let Some(value) = &dp.value { + obj.insert("value".to_string(), data_point_value_to_json(value)); + } + flatten_otlp_attrs_into_json(&mut obj, resource_attrs); + flatten_otlp_attrs_into_json(&mut obj, scope_attrs); + flatten_otlp_attrs_into_json(&mut obj, &dp.attributes); + out.push(JsonValue::Object(obj)); + } + } + Some(MetricData::Sum(s)) => { + for dp in &s.data_points { + let mut obj = JsonMap::new(); + obj.insert("metric_name".to_string(), JsonValue::String(metric.name.clone())); + if !metric.description.is_empty() { + obj.insert("metric_description".to_string(), JsonValue::String(metric.description.clone())); + } + if !metric.unit.is_empty() { + obj.insert("metric_unit".to_string(), JsonValue::String(metric.unit.clone())); + } + obj.insert("metric_type".to_string(), JsonValue::String("Sum".to_string())); + obj.insert("is_monotonic".to_string(), JsonValue::Bool(s.is_monotonic)); + obj.insert("aggregation_temporality".to_string(), JsonValue::String(s.aggregation_temporality.to_string())); + obj.insert("timestamp".to_string(), JsonValue::String(format_timestamp(dp.time_unix_nano))); + if dp.start_time_unix_nano > 0 { + obj.insert("start_time".to_string(), JsonValue::String(format_timestamp(dp.start_time_unix_nano))); + } + if let Some(value) = &dp.value { + obj.insert("value".to_string(), data_point_value_to_json(value)); + } + flatten_otlp_attrs_into_json(&mut obj, resource_attrs); + flatten_otlp_attrs_into_json(&mut obj, scope_attrs); + flatten_otlp_attrs_into_json(&mut obj, &dp.attributes); + out.push(JsonValue::Object(obj)); + } + } + Some(MetricData::Histogram(h)) => { + for dp in &h.data_points { + let mut obj = JsonMap::new(); + obj.insert("metric_name".to_string(), JsonValue::String(metric.name.clone())); + if !metric.description.is_empty() { + obj.insert("metric_description".to_string(), JsonValue::String(metric.description.clone())); + } + if !metric.unit.is_empty() { + obj.insert("metric_unit".to_string(), JsonValue::String(metric.unit.clone())); + } + obj.insert("metric_type".to_string(), JsonValue::String("Histogram".to_string())); + obj.insert("timestamp".to_string(), JsonValue::String(format_timestamp(dp.time_unix_nano))); + obj.insert("count".to_string(), JsonValue::Number(dp.count.into())); + if let Some(sum) = dp.sum { + obj.insert("sum".to_string(), JsonValue::Number(serde_json::Number::from_f64(sum).unwrap_or(0.into()))); + } + flatten_otlp_attrs_into_json(&mut obj, resource_attrs); + flatten_otlp_attrs_into_json(&mut obj, scope_attrs); + flatten_otlp_attrs_into_json(&mut obj, &dp.attributes); + out.push(JsonValue::Object(obj)); + } + } + Some(MetricData::ExponentialHistogram(eh)) => { + for dp in &eh.data_points { + let mut obj = JsonMap::new(); + obj.insert("metric_name".to_string(), JsonValue::String(metric.name.clone())); + if !metric.description.is_empty() { + obj.insert("metric_description".to_string(), JsonValue::String(metric.description.clone())); + } + if !metric.unit.is_empty() { + obj.insert("metric_unit".to_string(), JsonValue::String(metric.unit.clone())); + } + obj.insert("metric_type".to_string(), JsonValue::String("ExponentialHistogram".to_string())); + obj.insert("timestamp".to_string(), JsonValue::String(format_timestamp(dp.time_unix_nano))); + obj.insert("count".to_string(), JsonValue::Number(dp.count.into())); + obj.insert("scale".to_string(), JsonValue::Number(dp.scale.into())); + flatten_otlp_attrs_into_json(&mut obj, resource_attrs); + flatten_otlp_attrs_into_json(&mut obj, scope_attrs); + flatten_otlp_attrs_into_json(&mut obj, &dp.attributes); + out.push(JsonValue::Object(obj)); + } + } + Some(MetricData::Summary(s)) => { + for dp in &s.data_points { + let mut obj = JsonMap::new(); + obj.insert("metric_name".to_string(), JsonValue::String(metric.name.clone())); + if !metric.description.is_empty() { + obj.insert("metric_description".to_string(), JsonValue::String(metric.description.clone())); + } + if !metric.unit.is_empty() { + obj.insert("metric_unit".to_string(), JsonValue::String(metric.unit.clone())); + } + obj.insert("metric_type".to_string(), JsonValue::String("Summary".to_string())); + obj.insert("timestamp".to_string(), JsonValue::String(format_timestamp(dp.time_unix_nano))); + obj.insert("count".to_string(), JsonValue::Number(dp.count.into())); + obj.insert("sum".to_string(), JsonValue::Number(serde_json::Number::from_f64(dp.sum).unwrap_or(0.into()))); + flatten_otlp_attrs_into_json(&mut obj, resource_attrs); + flatten_otlp_attrs_into_json(&mut obj, scope_attrs); + flatten_otlp_attrs_into_json(&mut obj, &dp.attributes); + out.push(JsonValue::Object(obj)); + } + } + None => {} + } + } + } + } + out +} + +fn data_point_value_to_json(value: &opentelemetry_proto::tonic::metrics::v1::number_data_point::Value) -> JsonValue { + match value { + opentelemetry_proto::tonic::metrics::v1::number_data_point::Value::AsDouble(v) => serde_json::Number::from_f64(*v).map(JsonValue::Number).unwrap_or(JsonValue::Null), + opentelemetry_proto::tonic::metrics::v1::number_data_point::Value::AsInt(v) => JsonValue::Number((*v).into()), + } +} + +fn export_traces_ndjson(detail: &DetailRecord) -> Vec { + let Ok(batch) = ExportTraceServiceRequest::decode(detail.payload.as_slice()) else { + let mut obj = JsonMap::new(); + obj.insert("timestamp".to_string(), JsonValue::String(format_timestamp(detail.meta.ts_unix_ns))); + obj.insert("payload".to_string(), JsonValue::String(String::from_utf8_lossy(&detail.payload).to_string())); + return vec![JsonValue::Object(obj)]; + }; + + let mut out = Vec::new(); + for resource_spans in &batch.resource_spans { + let resource_attrs = resource_spans.resource.as_ref().map(|r| &r.attributes).map(Vec::as_slice).unwrap_or(&[]); + for scope_spans in &resource_spans.scope_spans { + let scope_attrs = scope_spans.scope.as_ref().map(|s| s.attributes.as_slice()).unwrap_or(&[]); + for span in &scope_spans.spans { + let mut obj = JsonMap::new(); + if !span.trace_id.is_empty() { + obj.insert("trace_id".to_string(), JsonValue::String(hex_encode(&span.trace_id))); + } + if !span.span_id.is_empty() { + obj.insert("span_id".to_string(), JsonValue::String(hex_encode(&span.span_id))); + } + if !span.parent_span_id.is_empty() { + obj.insert("parent_span_id".to_string(), JsonValue::String(hex_encode(&span.parent_span_id))); + } + obj.insert("name".to_string(), JsonValue::String(span.name.clone())); + obj.insert("kind".to_string(), JsonValue::String(format_span_kind(span.kind))); + obj.insert("start_time".to_string(), JsonValue::String(format_timestamp(span.start_time_unix_nano))); + obj.insert("end_time".to_string(), JsonValue::String(format_timestamp(span.end_time_unix_nano))); + if span.end_time_unix_nano > span.start_time_unix_nano { + obj.insert("duration_ns".to_string(), JsonValue::Number((span.end_time_unix_nano - span.start_time_unix_nano).into())); + } + if let Some(status) = &span.status { + obj.insert("status_code".to_string(), JsonValue::Number(status.code.into())); + if !status.message.is_empty() { + obj.insert("status_message".to_string(), JsonValue::String(status.message.clone())); + } + } + if !span.trace_state.is_empty() { + obj.insert("trace_state".to_string(), JsonValue::String(span.trace_state.clone())); + } + if let Some(scope) = &scope_spans.scope { + if !scope.name.is_empty() { + obj.insert("scope_name".to_string(), JsonValue::String(scope.name.clone())); + } + if !scope.version.is_empty() { + obj.insert("scope_version".to_string(), JsonValue::String(scope.version.clone())); + } + } + flatten_otlp_attrs_into_json(&mut obj, resource_attrs); + flatten_otlp_attrs_into_json(&mut obj, scope_attrs); + flatten_otlp_attrs_into_json(&mut obj, &span.attributes); + out.push(JsonValue::Object(obj)); + } + } + } + out +} + fn insert_otlp_log_fields( target: &mut JsonMap, record: &opentelemetry_proto::tonic::logs::v1::LogRecord, fallback_ts_unix_ns: u64, ) { @@ -548,6 +1252,7 @@ fn is_otlp_attribute_entry(key: &str) -> bool { (key.starts_with("resource.") && key != "resource.attrs") || (key.starts_with("scope.") && key != "scope.attrs") || (key.starts_with("record.") && key != "record.attrs") + || (key.starts_with("span.") && key != "span.attrs") } fn is_standard_otlp_attribute_entry(key: &str) -> bool { diff --git a/ljx/src/commands/view/render.rs b/ljx/src/commands/view/render.rs index 5f08823..d144534 100644 --- a/ljx/src/commands/view/render.rs +++ b/ljx/src/commands/view/render.rs @@ -562,7 +562,7 @@ impl ViewApp { frame.render_widget(Clear, area); let block = Block::default() - .title(Span::styled(" Log record ", Style::default().fg(Color::Black).bg(Color::Indexed(30)).add_modifier(Modifier::BOLD))) + .title(Span::styled(" Record ", Style::default().fg(Color::Black).bg(Color::Indexed(30)).add_modifier(Modifier::BOLD))) .borders(Borders::ALL) .border_type(BorderType::Double) .border_style(Style::default().fg(Color::Black).bg(Color::Gray)) @@ -604,7 +604,7 @@ impl ViewApp { frame.render_widget(Clear, area); let block = Block::default() - .title(Span::styled(" Log record ", Style::default().fg(Color::Black).bg(Color::Indexed(30)).add_modifier(Modifier::BOLD))) + .title(Span::styled(" Record ", Style::default().fg(Color::Black).bg(Color::Indexed(30)).add_modifier(Modifier::BOLD))) .borders(Borders::ALL) .border_type(BorderType::Double) .border_style(Style::default().fg(Color::Gray)) diff --git a/ljx/tests/unit/commands/export_ut.rs b/ljx/tests/unit/commands/export_ut.rs index c308c90..0694de1 100644 --- a/ljx/tests/unit/commands/export_ut.rs +++ b/ljx/tests/unit/commands/export_ut.rs @@ -1,9 +1,14 @@ use logjet::{OwnedRecord, RecordType}; use opentelemetry_proto::tonic::collector::logs::v1::ExportLogsServiceRequest; +use opentelemetry_proto::tonic::collector::metrics::v1::ExportMetricsServiceRequest; +use opentelemetry_proto::tonic::collector::trace::v1::ExportTraceServiceRequest; use opentelemetry_proto::tonic::common::v1::any_value::Value; use opentelemetry_proto::tonic::common::v1::{AnyValue, InstrumentationScope, KeyValue}; use opentelemetry_proto::tonic::logs::v1::{LogRecord, ResourceLogs, ScopeLogs}; +use opentelemetry_proto::tonic::metrics::v1::number_data_point::Value as DataPointValue; +use opentelemetry_proto::tonic::metrics::v1::{Gauge, Metric, NumberDataPoint, ResourceMetrics, ScopeMetrics}; use opentelemetry_proto::tonic::resource::v1::Resource; +use opentelemetry_proto::tonic::trace::v1::{ResourceSpans, ScopeSpans, Span}; use prost::Message; use serde_json::{Map as JsonMap, Value as JsonValue}; @@ -135,3 +140,103 @@ fn export_ndjson_objects_with_preview_truncates_long_fields() { assert_eq!(obj.get("body"), Some(&JsonValue::String("hello-...".to_string()))); assert_eq!(obj.get("http_target"), Some(&JsonValue::String("/api/v...".to_string()))); } + +#[test] +fn export_ndjson_objects_includes_otlp_metrics_fields() { + let metric = Metric { + name: "cpu.usage".to_string(), + description: "CPU usage".to_string(), + unit: "%".to_string(), + data: Some(opentelemetry_proto::tonic::metrics::v1::metric::Data::Gauge(Gauge { + data_points: vec![NumberDataPoint { + attributes: vec![KeyValue { key: "cpu".to_string(), value: Some(AnyValue { value: Some(Value::StringValue("all".to_string())) }) }], + start_time_unix_nano: 0, + time_unix_nano: 1_700_000_000_000_000_000, + value: Some(DataPointValue::AsDouble(45.5)), + flags: 0, + exemplars: vec![], + }], + })), + metadata: vec![], + }; + let batch = ExportMetricsServiceRequest { + resource_metrics: vec![ResourceMetrics { + resource: Some(Resource { + attributes: vec![KeyValue { key: "service.name".to_string(), value: Some(AnyValue { value: Some(Value::StringValue("metrics-svc".to_string())) }) }], + dropped_attributes_count: 0, + entity_refs: vec![], + }), + scope_metrics: vec![ScopeMetrics { + scope: Some(InstrumentationScope { name: "demo.metrics.scope".to_string(), version: "1.0.0".to_string(), attributes: vec![], dropped_attributes_count: 0 }), + metrics: vec![metric], + schema_url: String::new(), + }], + schema_url: String::new(), + }], + }; + let record = OwnedRecord { record_type: RecordType::Metrics, seq: 1, ts_unix_ns: 1_700_000_000_000_000_000, payload: batch.encode_to_vec() }; + + let docs = export_ndjson_objects(&record, &[]); + assert_eq!(docs.len(), 1); + let Some(obj) = docs.first().and_then(JsonValue::as_object) else { panic!("expected object") }; + assert_eq!(obj.get("metric_name"), Some(&JsonValue::String("cpu.usage".to_string()))); + assert_eq!(obj.get("metric_type"), Some(&JsonValue::String("Gauge".to_string()))); + assert_eq!(obj.get("metric_unit"), Some(&JsonValue::String("%".to_string()))); + assert_eq!(obj.get("metric_description"), Some(&JsonValue::String("CPU usage".to_string()))); + assert_eq!(obj.get("value"), Some(&JsonValue::Number(serde_json::Number::from_f64(45.5).unwrap()))); + assert_eq!(obj.get("service_name"), Some(&JsonValue::String("metrics-svc".to_string()))); + assert_eq!(obj.get("cpu"), Some(&JsonValue::String("all".to_string()))); + assert!(obj.get("timestamp").is_some()); +} + +#[test] +fn export_ndjson_objects_includes_otlp_traces_fields() { + let batch = ExportTraceServiceRequest { + resource_spans: vec![ResourceSpans { + resource: Some(Resource { + attributes: vec![KeyValue { key: "service.name".to_string(), value: Some(AnyValue { value: Some(Value::StringValue("traces-svc".to_string())) }) }], + dropped_attributes_count: 0, + entity_refs: vec![], + }), + scope_spans: vec![ScopeSpans { + scope: Some(InstrumentationScope { name: "demo.traces.scope".to_string(), version: "2.0.0".to_string(), attributes: vec![], dropped_attributes_count: 0 }), + spans: vec![Span { + trace_id: vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16], + span_id: vec![16, 17, 18, 19, 20, 21, 22, 23], + parent_span_id: vec![], + name: "GET /api".to_string(), + kind: 2, + start_time_unix_nano: 1_700_000_000_000_000_000, + end_time_unix_nano: 1_700_000_000_000_000_100, + attributes: vec![KeyValue { key: "http.method".to_string(), value: Some(AnyValue { value: Some(Value::StringValue("GET".to_string())) }) }], + dropped_attributes_count: 0, + events: vec![], + dropped_events_count: 0, + links: vec![], + dropped_links_count: 0, + status: Some(opentelemetry_proto::tonic::trace::v1::Status { code: 1, message: String::new() }), + flags: 0, + trace_state: String::new(), + }], + schema_url: String::new(), + }], + schema_url: String::new(), + }], + }; + let record = OwnedRecord { record_type: RecordType::Traces, seq: 1, ts_unix_ns: 1_700_000_000_000_000_000, payload: batch.encode_to_vec() }; + + let docs = export_ndjson_objects(&record, &[]); + assert_eq!(docs.len(), 1); + let Some(obj) = docs.first().and_then(JsonValue::as_object) else { panic!("expected object") }; + assert_eq!(obj.get("name"), Some(&JsonValue::String("GET /api".to_string()))); + assert_eq!(obj.get("kind"), Some(&JsonValue::String("Server".to_string()))); + assert_eq!(obj.get("trace_id"), Some(&JsonValue::String("0102030405060708090a0b0c0d0e0f10".to_string()))); + assert_eq!(obj.get("span_id"), Some(&JsonValue::String("1011121314151617".to_string()))); + assert_eq!(obj.get("status_code"), Some(&JsonValue::Number(1.into()))); + assert_eq!(obj.get("duration_ns"), Some(&JsonValue::Number(100.into()))); + assert_eq!(obj.get("service_name"), Some(&JsonValue::String("traces-svc".to_string()))); + assert_eq!(obj.get("scope_name"), Some(&JsonValue::String("demo.traces.scope".to_string()))); + assert_eq!(obj.get("http_method"), Some(&JsonValue::String("GET".to_string()))); + assert!(obj.get("start_time").is_some()); + assert!(obj.get("end_time").is_some()); +} diff --git a/ljx/tests/unit/commands/view_ut.rs b/ljx/tests/unit/commands/view_ut.rs index 28194e1..37daf65 100644 --- a/ljx/tests/unit/commands/view_ut.rs +++ b/ljx/tests/unit/commands/view_ut.rs @@ -8,10 +8,12 @@ use crossterm::event::{KeyCode, KeyEvent, KeyModifiers}; use logjet::OwnedRecord; use logjet::{LogjetWriter, RecordType}; use opentelemetry_proto::tonic::collector::logs::v1::ExportLogsServiceRequest; +use opentelemetry_proto::tonic::collector::trace::v1::ExportTraceServiceRequest; use opentelemetry_proto::tonic::common::v1::any_value::Value; use opentelemetry_proto::tonic::common::v1::{AnyValue, InstrumentationScope, KeyValue}; use opentelemetry_proto::tonic::logs::v1::{LogRecord, ResourceLogs, ScopeLogs}; use opentelemetry_proto::tonic::resource::v1::Resource; +use opentelemetry_proto::tonic::trace::v1::{ResourceSpans, ScopeSpans, Span}; use prost::Message; use serde_json::Value as JsonValue; use std::fs::File; @@ -862,3 +864,309 @@ fn export_current_results_can_export_selected_row_only() { let _ = std::fs::remove_file(input); let _ = std::fs::remove_file(output); } + +#[test] +fn summary_decodes_otlp_metrics_payload() { + use opentelemetry_proto::tonic::collector::metrics::v1::ExportMetricsServiceRequest; + use opentelemetry_proto::tonic::metrics::v1::number_data_point::Value as DataPointValue; + use opentelemetry_proto::tonic::metrics::v1::{Gauge, Metric, NumberDataPoint, ResourceMetrics, ScopeMetrics}; + + let metric = Metric { + name: "cpu.usage".to_string(), + description: String::new(), + unit: "%".to_string(), + data: Some(opentelemetry_proto::tonic::metrics::v1::metric::Data::Gauge(Gauge { + data_points: vec![NumberDataPoint { + attributes: vec![], + start_time_unix_nano: 0, + time_unix_nano: 1_700_000_000_000_000_000, + value: Some(DataPointValue::AsDouble(45.5)), + flags: 0, + exemplars: vec![], + }], + })), + metadata: vec![], + }; + let batch = ExportMetricsServiceRequest { + resource_metrics: vec![ResourceMetrics { + resource: Some(Resource { attributes: vec![], dropped_attributes_count: 0, entity_refs: vec![] }), + scope_metrics: vec![ScopeMetrics { + scope: Some(InstrumentationScope { name: "test".to_string(), version: String::new(), attributes: vec![], dropped_attributes_count: 0 }), + metrics: vec![metric], + schema_url: String::new(), + }], + schema_url: String::new(), + }], + }; + let payload = batch.encode_to_vec(); + let detail = DetailRecord { + meta: EntryMeta { offset: 0, record_type: RecordType::Metrics, seq: 1, ts_unix_ns: 2, payload_len: payload.len() as u64, source_path: "a.logjet".into() }, + payload, + }; + assert_eq!(format_summary(&detail, false), "cpu.usage=45.5%"); +} + +#[test] +fn modal_message_decodes_otlp_metrics_payload() { + use opentelemetry_proto::tonic::collector::metrics::v1::ExportMetricsServiceRequest; + use opentelemetry_proto::tonic::metrics::v1::number_data_point::Value as DataPointValue; + use opentelemetry_proto::tonic::metrics::v1::{Gauge, Metric, NumberDataPoint, ResourceMetrics, ScopeMetrics}; + + let metric = Metric { + name: "cpu.usage".to_string(), + description: String::new(), + unit: String::new(), + data: Some(opentelemetry_proto::tonic::metrics::v1::metric::Data::Gauge(Gauge { + data_points: vec![NumberDataPoint { + attributes: vec![], + start_time_unix_nano: 0, + time_unix_nano: 1_700_000_000_000_000_000, + value: Some(DataPointValue::AsDouble(45.5)), + flags: 0, + exemplars: vec![], + }], + })), + metadata: vec![], + }; + let batch = ExportMetricsServiceRequest { + resource_metrics: vec![ResourceMetrics { + resource: Some(Resource { attributes: vec![], dropped_attributes_count: 0, entity_refs: vec![] }), + scope_metrics: vec![ScopeMetrics { + scope: Some(InstrumentationScope { name: "test".to_string(), version: String::new(), attributes: vec![], dropped_attributes_count: 0 }), + metrics: vec![metric], + schema_url: String::new(), + }], + schema_url: String::new(), + }], + }; + let payload = batch.encode_to_vec(); + let detail = DetailRecord { + meta: EntryMeta { offset: 0, record_type: RecordType::Metrics, seq: 1, ts_unix_ns: 2, payload_len: payload.len() as u64, source_path: "a.logjet".into() }, + payload, + }; + let message = render_modal_message(&detail, false); + assert!(message.contains("Metric: cpu.usage"), "modal body should contain metric name: {message}"); + assert!(message.contains("45.5"), "modal body should contain metric value: {message}"); +} + +#[test] +fn modal_info_entries_decodes_otlp_metrics_payload() { + use opentelemetry_proto::tonic::collector::metrics::v1::ExportMetricsServiceRequest; + use opentelemetry_proto::tonic::metrics::v1::number_data_point::Value as DataPointValue; + use opentelemetry_proto::tonic::metrics::v1::{Gauge, Metric, NumberDataPoint, ResourceMetrics, ScopeMetrics}; + + let metric = Metric { + name: "cpu.usage".to_string(), + description: "Current CPU usage".to_string(), + unit: "%".to_string(), + data: Some(opentelemetry_proto::tonic::metrics::v1::metric::Data::Gauge(Gauge { + data_points: vec![NumberDataPoint { + attributes: vec![], + start_time_unix_nano: 0, + time_unix_nano: 1_700_000_000_000_000_000, + value: Some(DataPointValue::AsDouble(45.5)), + flags: 0, + exemplars: vec![], + }], + })), + metadata: vec![], + }; + let batch = ExportMetricsServiceRequest { + resource_metrics: vec![ResourceMetrics { + resource: Some(Resource { attributes: vec![], dropped_attributes_count: 0, entity_refs: vec![] }), + scope_metrics: vec![ScopeMetrics { + scope: Some(InstrumentationScope { name: "test".to_string(), version: String::new(), attributes: vec![], dropped_attributes_count: 0 }), + metrics: vec![metric], + schema_url: String::new(), + }], + schema_url: String::new(), + }], + }; + let payload = batch.encode_to_vec(); + let detail = DetailRecord { + meta: EntryMeta { offset: 0, record_type: RecordType::Metrics, seq: 1, ts_unix_ns: 2, payload_len: payload.len() as u64, source_path: "a.logjet".into() }, + payload, + }; + let entries = render_modal_info_entries(&detail); + assert!(entries.iter().any(|(k, _)| k == "otlp.kind"), "should have otlp.kind entry"); + assert!(entries.iter().any(|(k, v)| k == "metrics" && v == "1"), "should have metrics count"); + assert!(entries.iter().any(|(k, v)| k == "metric.cpu.usage.unit" && v == "%"), "should have metric unit"); +} + +#[test] +fn summary_decodes_otlp_traces_payload() { + let batch = ExportTraceServiceRequest { + resource_spans: vec![ResourceSpans { + resource: Some(Resource { + attributes: vec![KeyValue { + key: "service.name".to_string(), + value: Some(AnyValue { value: Some(Value::StringValue("trace-demo".to_string())) }), + }], + dropped_attributes_count: 0, + entity_refs: vec![], + }), + scope_spans: vec![ScopeSpans { + scope: Some(InstrumentationScope { name: "test".to_string(), version: String::new(), attributes: vec![], dropped_attributes_count: 0 }), + spans: vec![ + Span { + trace_id: vec![1, 2, 3, 4], + span_id: vec![5, 6, 7, 8], + parent_span_id: vec![], + name: "GET /api".to_string(), + kind: 2, + start_time_unix_nano: 1_700_000_000_000_000_000, + end_time_unix_nano: 1_700_000_000_000_000_100, + attributes: vec![], + dropped_attributes_count: 0, + events: vec![], + dropped_events_count: 0, + links: vec![], + dropped_links_count: 0, + status: None, + flags: 0, + trace_state: String::new(), + }, + Span { + trace_id: vec![1, 2, 3, 4], + span_id: vec![9, 10, 11, 12], + parent_span_id: vec![5, 6, 7, 8], + name: "SELECT".to_string(), + kind: 3, + start_time_unix_nano: 1_700_000_000_000_000_050, + end_time_unix_nano: 1_700_000_000_000_000_080, + attributes: vec![], + dropped_attributes_count: 0, + events: vec![], + dropped_events_count: 0, + links: vec![], + dropped_links_count: 0, + status: None, + flags: 0, + trace_state: String::new(), + }, + ], + schema_url: String::new(), + }], + schema_url: String::new(), + }], + }; + let payload = batch.encode_to_vec(); + let detail = DetailRecord { + meta: EntryMeta { offset: 0, record_type: RecordType::Traces, seq: 1, ts_unix_ns: 2, payload_len: payload.len() as u64, source_path: "a.logjet".into() }, + payload, + }; + let summary = format_summary(&detail, false); + assert!(summary.contains("GET /api"), "summary should contain span name: {summary}"); + assert!(summary.contains("Server"), "summary should contain span kind: {summary}"); +} + +#[test] +fn modal_message_decodes_otlp_traces_payload() { + let batch = ExportTraceServiceRequest { + resource_spans: vec![ResourceSpans { + resource: Some(Resource { attributes: vec![], dropped_attributes_count: 0, entity_refs: vec![] }), + scope_spans: vec![ScopeSpans { + scope: Some(InstrumentationScope { name: "test".to_string(), version: String::new(), attributes: vec![], dropped_attributes_count: 0 }), + spans: vec![Span { + trace_id: vec![1, 2, 3, 4], + span_id: vec![5, 6, 7, 8], + parent_span_id: vec![], + name: "POST /events".to_string(), + kind: 2, + start_time_unix_nano: 1_700_000_000_000_000_000, + end_time_unix_nano: 1_700_000_000_000_000_200, + attributes: vec![], + dropped_attributes_count: 0, + events: vec![], + dropped_events_count: 0, + links: vec![], + dropped_links_count: 0, + status: None, + flags: 0, + trace_state: String::new(), + }], + schema_url: String::new(), + }], + schema_url: String::new(), + }], + }; + let payload = batch.encode_to_vec(); + let detail = DetailRecord { + meta: EntryMeta { offset: 0, record_type: RecordType::Traces, seq: 1, ts_unix_ns: 2, payload_len: payload.len() as u64, source_path: "a.logjet".into() }, + payload, + }; + let message = render_modal_message(&detail, false); + assert!(message.contains("Span: POST /events"), "modal body should contain span name: {message}"); + assert!(message.contains("Trace ID:"), "modal body should contain trace ID: {message}"); + assert!(message.contains("Kind: Server"), "modal body should contain kind: {message}"); +} + +#[test] +fn modal_info_entries_decodes_otlp_traces_payload() { + let batch = ExportTraceServiceRequest { + resource_spans: vec![ResourceSpans { + resource: Some(Resource { + attributes: vec![KeyValue { + key: "service.name".to_string(), + value: Some(AnyValue { value: Some(Value::StringValue("trace-demo".to_string())) }), + }], + dropped_attributes_count: 0, + entity_refs: vec![], + }), + scope_spans: vec![ScopeSpans { + scope: Some(InstrumentationScope { name: "test".to_string(), version: String::new(), attributes: vec![], dropped_attributes_count: 0 }), + spans: vec![ + Span { + trace_id: vec![1, 2, 3, 4], + span_id: vec![5, 6, 7, 8], + parent_span_id: vec![], + name: "GET /api".to_string(), + kind: 2, + start_time_unix_nano: 1_700_000_000_000_000_000, + end_time_unix_nano: 1_700_000_000_000_000_100, + attributes: vec![], + dropped_attributes_count: 0, + events: vec![], + dropped_events_count: 0, + links: vec![], + dropped_links_count: 0, + status: Some(opentelemetry_proto::tonic::trace::v1::Status { code: 1, message: String::new() }), + flags: 0, + trace_state: String::new(), + }, + Span { + trace_id: vec![1, 2, 3, 4], + span_id: vec![9, 10, 11, 12], + parent_span_id: vec![5, 6, 7, 8], + name: "SELECT".to_string(), + kind: 3, + start_time_unix_nano: 1_700_000_000_000_000_050, + end_time_unix_nano: 1_700_000_000_000_000_080, + attributes: vec![], + dropped_attributes_count: 0, + events: vec![], + dropped_events_count: 0, + links: vec![], + dropped_links_count: 0, + status: Some(opentelemetry_proto::tonic::trace::v1::Status { code: 0, message: String::new() }), + flags: 0, + trace_state: String::new(), + }, + ], + schema_url: String::new(), + }], + schema_url: String::new(), + }], + }; + let payload = batch.encode_to_vec(); + let detail = DetailRecord { + meta: EntryMeta { offset: 0, record_type: RecordType::Traces, seq: 1, ts_unix_ns: 2, payload_len: payload.len() as u64, source_path: "a.logjet".into() }, + payload, + }; + let entries = render_modal_info_entries(&detail); + assert!(entries.iter().any(|(k, v)| k == "otlp.kind" && v == "traces"), "should have otlp.kind entry"); + assert!(entries.iter().any(|(k, v)| k == "service.name" && v == "trace-demo"), "should have service name"); + assert!(entries.iter().any(|(k, v)| k == "spans" && v == "2"), "should have span count"); + assert!(entries.iter().any(|(k, v)| k == "span.kind.Server" && v == "1"), "should have Server kind count"); + assert!(entries.iter().any(|(k, v)| k == "span.kind.Client" && v == "1"), "should have Client kind count"); +} diff --git a/logjetd/Cargo.toml b/logjetd/Cargo.toml index ad55cf8..5bfbb42 100644 --- a/logjetd/Cargo.toml +++ b/logjetd/Cargo.toml @@ -7,6 +7,7 @@ license.workspace = true [dependencies] clap = { version = "4.5", features = ["derive", "wrap_help"] } colored = "3" +flate2 = { version = "1.1", default-features = false, features = ["rust_backend"] } libc = "0.2" libloading = "0.8" liblogjet = { path = "../liblogjet" } diff --git a/logjetd/src/daemon.rs b/logjetd/src/daemon.rs index b0af3d5..72b6893 100644 --- a/logjetd/src/daemon.rs +++ b/logjetd/src/daemon.rs @@ -12,6 +12,14 @@ use opentelemetry_proto::tonic::collector::logs::v1::{ ExportLogsServiceRequest, ExportLogsServiceResponse, logs_service_server::{LogsService, LogsServiceServer}, }; +use opentelemetry_proto::tonic::collector::metrics::v1::{ + ExportMetricsServiceRequest, ExportMetricsServiceResponse, + metrics_service_server::{MetricsService, MetricsServiceServer}, +}; +use opentelemetry_proto::tonic::collector::trace::v1::{ + ExportTraceServiceRequest, ExportTraceServiceResponse, + trace_service_server::{TraceService, TraceServiceServer}, +}; use prost::Message; use rustls::{ServerConfig, ServerConnection, StreamOwned}; use tiny_http::{Method, Response, Server, StatusCode}; @@ -260,14 +268,18 @@ fn ingest_loop( return otlp_http_tls_loop(bind_addr, ingest_tls, ingest_limits, ingest_policy, spool, next_seq, limiter); } let server = Server::http(&bind_addr).map_err(|err| io::Error::other(err.to_string()))?; - eprintln!("ljd ingest listening on http://{bind_addr}/v1/logs using otlp-http max-batch-bytes={}", ingest_limits.max_batch_bytes); + eprintln!("ljd ingest listening on http://{bind_addr}/v1/logs /v1/metrics /v1/traces using otlp-http max-batch-bytes={}", ingest_limits.max_batch_bytes); for mut request in server.incoming_requests() { - if request.method() != &Method::Post || request.url() != "/v1/logs" { + if request.method() != &Method::Post || !matches!(request.url(), "/v1/logs" | "/v1/metrics" | "/v1/traces") { let response = Response::from_string("not found").with_status_code(StatusCode(404)); let _ = request.respond(response); continue; } + let is_metrics = request.url() == "/v1/metrics"; + let is_traces = request.url() == "/v1/traces"; + + let content_encoding = request.headers().iter().find(|h| h.field.equiv("content-encoding")).map(|h| h.value.to_string()); let mut body = Vec::with_capacity(ingest_limits.max_batch_bytes.min(8192)); request.as_reader().take((ingest_limits.max_batch_bytes + 1) as u64).read_to_end(&mut body)?; @@ -277,28 +289,88 @@ fn ingest_loop( request.respond(response).map_err(|err| io::Error::other(err.to_string()))?; continue; } - match ExportLogsServiceRequest::decode(body.as_slice()) { - Ok(batch) => { - let decision = ingest_policy.decide(classify_otlp_batch_priority(&batch))?; - if matches!(decision, IngestDecision::RejectRateLimited) { - let response = Response::from_string("rate limit exceeded").with_status_code(StatusCode(429)); + let body = match maybe_decompress_body(body, content_encoding.as_deref()) { + Ok(b) => b, + Err(err) => { + let response = Response::from_string(format!("decompression error: {err}")).with_status_code(StatusCode(400)); + request.respond(response).map_err(|resp_err| io::Error::other(resp_err.to_string()))?; + continue; + } + }; + if is_metrics { + match ExportMetricsServiceRequest::decode(body.as_slice()) { + Ok(batch) => { + let decision = ingest_policy.decide(BatchPriority::Unknown)?; + if matches!(decision, IngestDecision::RejectRateLimited) { + let response = Response::from_string("rate limit exceeded").with_status_code(StatusCode(429)); + request.respond(response).map_err(|err| io::Error::other(err.to_string()))?; + continue; + } + let record = WireRecord { + record_type: logjet::RecordType::Metrics, + seq: next_seq.fetch_add(1, Ordering::Relaxed), + ts_unix_ns: extract_batch_timestamp_metrics(&batch).unwrap_or_else(unix_time_nanos), + payload: body, + }; + append_batch_record(&spool, record)?; + + let response = Response::empty(200); request.respond(response).map_err(|err| io::Error::other(err.to_string()))?; - continue; } - let record = WireRecord { - record_type: logjet::RecordType::Logs, - seq: next_seq.fetch_add(1, Ordering::Relaxed), - ts_unix_ns: extract_batch_timestamp(&batch).unwrap_or_else(unix_time_nanos), - payload: body, - }; - append_batch_record(&spool, record)?; - - let response = Response::empty(200); - request.respond(response).map_err(|err| io::Error::other(err.to_string()))?; + Err(err) => { + let response = Response::from_string(format!("decode error: {err}")).with_status_code(StatusCode(400)); + request.respond(response).map_err(|resp_err| io::Error::other(resp_err.to_string()))?; + } } - Err(err) => { - let response = Response::from_string(format!("decode error: {err}")).with_status_code(StatusCode(400)); - request.respond(response).map_err(|resp_err| io::Error::other(resp_err.to_string()))?; + } else if is_traces { + match ExportTraceServiceRequest::decode(body.as_slice()) { + Ok(batch) => { + let decision = ingest_policy.decide(BatchPriority::Unknown)?; + if matches!(decision, IngestDecision::RejectRateLimited) { + let response = Response::from_string("rate limit exceeded").with_status_code(StatusCode(429)); + request.respond(response).map_err(|err| io::Error::other(err.to_string()))?; + continue; + } + let record = WireRecord { + record_type: logjet::RecordType::Traces, + seq: next_seq.fetch_add(1, Ordering::Relaxed), + ts_unix_ns: extract_batch_timestamp_traces(&batch).unwrap_or_else(unix_time_nanos), + payload: body, + }; + append_batch_record(&spool, record)?; + + let response = Response::empty(200); + request.respond(response).map_err(|err| io::Error::other(err.to_string()))?; + } + Err(err) => { + let response = Response::from_string(format!("decode error: {err}")).with_status_code(StatusCode(400)); + request.respond(response).map_err(|resp_err| io::Error::other(resp_err.to_string()))?; + } + } + } else { + match ExportLogsServiceRequest::decode(body.as_slice()) { + Ok(batch) => { + let decision = ingest_policy.decide(classify_otlp_batch_priority(&batch))?; + if matches!(decision, IngestDecision::RejectRateLimited) { + let response = Response::from_string("rate limit exceeded").with_status_code(StatusCode(429)); + request.respond(response).map_err(|err| io::Error::other(err.to_string()))?; + continue; + } + let record = WireRecord { + record_type: logjet::RecordType::Logs, + seq: next_seq.fetch_add(1, Ordering::Relaxed), + ts_unix_ns: extract_batch_timestamp(&batch).unwrap_or_else(unix_time_nanos), + payload: body, + }; + append_batch_record(&spool, record)?; + + let response = Response::empty(200); + request.respond(response).map_err(|err| io::Error::other(err.to_string()))?; + } + Err(err) => { + let response = Response::from_string(format!("decode error: {err}")).with_status_code(StatusCode(400)); + request.respond(response).map_err(|resp_err| io::Error::other(resp_err.to_string()))?; + } } } } @@ -314,7 +386,9 @@ fn ingest_loop( ); let runtime = tokio::runtime::Builder::new_multi_thread().enable_all().build().map_err(|err| io::Error::other(err.to_string()))?; - let service = OtlpGrpcLogsService { spool, next_seq, ingest_policy }; + let logs_service = OtlpGrpcLogsService { spool: Arc::clone(&spool), next_seq: Arc::clone(&next_seq), ingest_policy: Arc::clone(&ingest_policy) }; + let metrics_service = OtlpGrpcMetricsService { spool: Arc::clone(&spool), next_seq: Arc::clone(&next_seq), ingest_policy: Arc::clone(&ingest_policy) }; + let traces_service = OtlpGrpcTracesService { spool, next_seq, ingest_policy }; let grpc_tls = if ingest_tls.enable { Some(build_grpc_server_tls_config(&ingest_tls)?) } else { None }; runtime.block_on(async move { @@ -324,7 +398,9 @@ fn ingest_loop( builder .concurrency_limit_per_connection(ingest_limits.max_clients) - .add_service(LogsServiceServer::new(service).max_decoding_message_size(ingest_limits.max_batch_bytes)) + .add_service(LogsServiceServer::new(logs_service).max_decoding_message_size(ingest_limits.max_batch_bytes)) + .add_service(MetricsServiceServer::new(metrics_service).max_decoding_message_size(ingest_limits.max_batch_bytes)) + .add_service(TraceServiceServer::new(traces_service).max_decoding_message_size(ingest_limits.max_batch_bytes)) .serve(addr) .await .map_err(|err| io::Error::other(err.to_string())) @@ -342,7 +418,7 @@ fn otlp_http_tls_loop( let listener = TcpListener::bind(&bind_addr)?; let tls_server = load_ingest_server_config(&ingest_tls)?; eprintln!( - "ljd ingest listening on https://{bind_addr}/v1/logs using otlp-http max-batch-bytes={} max-clients={}", + "ljd ingest listening on https://{bind_addr}/v1/logs /v1/metrics /v1/traces using otlp-http max-batch-bytes={} max-clients={}", ingest_limits.max_batch_bytes, ingest_limits.max_clients ); @@ -393,29 +469,78 @@ fn handle_otlp_http_transport( } Err(err) => return Err(err), }; - if request.method != "POST" || request.path != "/v1/logs" { + let is_metrics = request.path == "/v1/metrics"; + let is_traces = request.path == "/v1/traces"; + if request.method != "POST" || !matches!(request.path.as_str(), "/v1/logs" | "/v1/metrics" | "/v1/traces") { write_http_response(transport, 404, "not found")?; return Ok(()); } - match ExportLogsServiceRequest::decode(request.body.as_slice()) { - Ok(batch) => { - let decision = ingest_policy.decide(classify_otlp_batch_priority(&batch))?; - if matches!(decision, IngestDecision::RejectRateLimited) { - write_http_response(transport, 429, "rate limit exceeded")?; - return Ok(()); + if is_metrics { + let body = maybe_decompress_body(request.body, request.content_encoding.as_deref())?; + match ExportMetricsServiceRequest::decode(body.as_slice()) { + Ok(batch) => { + let decision = ingest_policy.decide(BatchPriority::Unknown)?; + if matches!(decision, IngestDecision::RejectRateLimited) { + write_http_response(transport, 429, "rate limit exceeded")?; + return Ok(()); + } + let record = WireRecord { + record_type: logjet::RecordType::Metrics, + seq: next_seq.fetch_add(1, Ordering::Relaxed), + ts_unix_ns: extract_batch_timestamp_metrics(&batch).unwrap_or_else(unix_time_nanos), + payload: body, + }; + append_batch_record(&spool, record)?; + write_http_response(transport, 200, "")?; + } + Err(err) => { + write_http_response(transport, 400, &format!("decode error: {err}"))?; + } + } + } else if is_traces { + let body = maybe_decompress_body(request.body, request.content_encoding.as_deref())?; + match ExportTraceServiceRequest::decode(body.as_slice()) { + Ok(batch) => { + let decision = ingest_policy.decide(BatchPriority::Unknown)?; + if matches!(decision, IngestDecision::RejectRateLimited) { + write_http_response(transport, 429, "rate limit exceeded")?; + return Ok(()); + } + let record = WireRecord { + record_type: logjet::RecordType::Traces, + seq: next_seq.fetch_add(1, Ordering::Relaxed), + ts_unix_ns: extract_batch_timestamp_traces(&batch).unwrap_or_else(unix_time_nanos), + payload: body, + }; + append_batch_record(&spool, record)?; + write_http_response(transport, 200, "")?; + } + Err(err) => { + write_http_response(transport, 400, &format!("decode error: {err}"))?; } - let record = WireRecord { - record_type: logjet::RecordType::Logs, - seq: next_seq.fetch_add(1, Ordering::Relaxed), - ts_unix_ns: extract_batch_timestamp(&batch).unwrap_or_else(unix_time_nanos), - payload: request.body, - }; - append_batch_record(&spool, record)?; - write_http_response(transport, 200, "")?; } - Err(err) => { - write_http_response(transport, 400, &format!("decode error: {err}"))?; + } else { + let body = maybe_decompress_body(request.body, request.content_encoding.as_deref())?; + match ExportLogsServiceRequest::decode(body.as_slice()) { + Ok(batch) => { + let decision = ingest_policy.decide(classify_otlp_batch_priority(&batch))?; + if matches!(decision, IngestDecision::RejectRateLimited) { + write_http_response(transport, 429, "rate limit exceeded")?; + return Ok(()); + } + let record = WireRecord { + record_type: logjet::RecordType::Logs, + seq: next_seq.fetch_add(1, Ordering::Relaxed), + ts_unix_ns: extract_batch_timestamp(&batch).unwrap_or_else(unix_time_nanos), + payload: body, + }; + append_batch_record(&spool, record)?; + write_http_response(transport, 200, "")?; + } + Err(err) => { + write_http_response(transport, 400, &format!("decode error: {err}"))?; + } } } @@ -439,6 +564,20 @@ struct ParsedHttpRequest { method: String, path: String, body: Vec, + content_encoding: Option, +} + +fn maybe_decompress_body(body: Vec, encoding: Option<&str>) -> io::Result> { + match encoding { + Some("gzip") | Some("x-gzip") => { + use flate2::read::GzDecoder; + let mut decoder = GzDecoder::new(body.as_slice()); + let mut out = Vec::with_capacity(body.len().saturating_mul(3)); + decoder.read_to_end(&mut out).map_err(|err| io::Error::new(io::ErrorKind::InvalidData, format!("gzip decompression failed: {err}")))?; + Ok(out) + } + _ => Ok(body), + } } fn read_http_request(transport: &mut T, max_batch_bytes: usize) -> io::Result { @@ -467,11 +606,16 @@ fn read_http_request(transport: &mut T, max_batch_bytes: usize) -> io:: let path = parts.next().ok_or_else(|| io::Error::new(io::ErrorKind::InvalidData, "missing http path"))?.to_string(); let mut content_length = None; + let mut content_encoding = None; for line in lines { - if let Some((name, value)) = line.split_once(':') - && name.eq_ignore_ascii_case("content-length") - { - content_length = Some(value.trim().parse::().map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "invalid content-length"))?); + if let Some((name, value)) = line.split_once(':') { + let name_trimmed = name.trim(); + let value_trimmed = value.trim(); + if name_trimmed.eq_ignore_ascii_case("content-length") { + content_length = Some(value_trimmed.parse::().map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "invalid content-length"))?); + } else if name_trimmed.eq_ignore_ascii_case("content-encoding") { + content_encoding = Some(value_trimmed.to_string()); + } } } @@ -485,7 +629,7 @@ fn read_http_request(transport: &mut T, max_batch_bytes: usize) -> io:: return Err(io::Error::new(io::ErrorKind::UnexpectedEof, "short http body")); } - Ok(ParsedHttpRequest { method, path, body }) + Ok(ParsedHttpRequest { method, path, body, content_encoding }) } fn write_http_response(transport: &mut T, status: u16, body: &str) -> io::Result<()> { @@ -764,6 +908,104 @@ fn unix_time_nanos() -> u64 { SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_nanos() as u64 } +fn extract_batch_timestamp_metrics(batch: &ExportMetricsServiceRequest) -> Option { + use opentelemetry_proto::tonic::metrics::v1::metric::Data; + for resource_metrics in &batch.resource_metrics { + for scope_metrics in &resource_metrics.scope_metrics { + for metric in &scope_metrics.metrics { + let ts = match metric.data.as_ref()? { + Data::Gauge(g) => g.data_points.iter().find_map(|dp| if dp.time_unix_nano != 0 { Some(dp.time_unix_nano) } else { None }), + Data::Sum(s) => s.data_points.iter().find_map(|dp| if dp.time_unix_nano != 0 { Some(dp.time_unix_nano) } else { None }), + Data::Histogram(h) => h.data_points.iter().find_map(|dp| if dp.time_unix_nano != 0 { Some(dp.time_unix_nano) } else { None }), + Data::ExponentialHistogram(eh) => { + eh.data_points.iter().find_map(|dp| if dp.time_unix_nano != 0 { Some(dp.time_unix_nano) } else { None }) + } + Data::Summary(s) => s.data_points.iter().find_map(|dp| if dp.time_unix_nano != 0 { Some(dp.time_unix_nano) } else { None }), + }; + if ts.is_some() { + return ts; + } + } + } + } + None +} + +fn extract_batch_timestamp_traces(batch: &ExportTraceServiceRequest) -> Option { + for resource_spans in &batch.resource_spans { + for scope_spans in &resource_spans.scope_spans { + for span in &scope_spans.spans { + if span.start_time_unix_nano != 0 { + return Some(span.start_time_unix_nano); + } + } + } + } + None +} + +#[derive(Clone)] +struct OtlpGrpcTracesService { + spool: Arc, + next_seq: Arc, + ingest_policy: Arc, +} + +#[tonic::async_trait] +impl TraceService for OtlpGrpcTracesService { + async fn export(&self, request: Request) -> Result, Status> { + let batch = request.into_inner(); + match self.ingest_policy.decide(BatchPriority::Unknown).map_err(|err| Status::internal(err.to_string()))? { + IngestDecision::Accept | IngestDecision::AcceptPriorityBypass => {} + IngestDecision::RejectRateLimited => { + return Err(Status::resource_exhausted("ingest rate limit exceeded")); + } + } + let payload = batch.encode_to_vec(); + let record = WireRecord { + record_type: logjet::RecordType::Traces, + seq: self.next_seq.fetch_add(1, Ordering::Relaxed), + ts_unix_ns: extract_batch_timestamp_traces(&batch).unwrap_or_else(unix_time_nanos), + payload, + }; + + append_batch_record(&self.spool, record).map_err(|err| Status::internal(err.to_string()))?; + + Ok(GrpcResponse::new(ExportTraceServiceResponse { partial_success: None })) + } +} + +#[derive(Clone)] +struct OtlpGrpcMetricsService { + spool: Arc, + next_seq: Arc, + ingest_policy: Arc, +} + +#[tonic::async_trait] +impl MetricsService for OtlpGrpcMetricsService { + async fn export(&self, request: Request) -> Result, Status> { + let batch = request.into_inner(); + match self.ingest_policy.decide(BatchPriority::Unknown).map_err(|err| Status::internal(err.to_string()))? { + IngestDecision::Accept | IngestDecision::AcceptPriorityBypass => {} + IngestDecision::RejectRateLimited => { + return Err(Status::resource_exhausted("ingest rate limit exceeded")); + } + } + let payload = batch.encode_to_vec(); + let record = WireRecord { + record_type: logjet::RecordType::Metrics, + seq: self.next_seq.fetch_add(1, Ordering::Relaxed), + ts_unix_ns: extract_batch_timestamp_metrics(&batch).unwrap_or_else(unix_time_nanos), + payload, + }; + + append_batch_record(&self.spool, record).map_err(|err| Status::internal(err.to_string()))?; + + Ok(GrpcResponse::new(ExportMetricsServiceResponse { partial_success: None })) + } +} + #[derive(Clone)] struct OtlpGrpcLogsService { spool: Arc, diff --git a/logjetd/src/replay.rs b/logjetd/src/replay.rs index 1e0e1e9..6f79ad9 100644 --- a/logjetd/src/replay.rs +++ b/logjetd/src/replay.rs @@ -8,6 +8,8 @@ use std::time::{Duration, Instant}; use logjet::{LogjetReader, ReaderConfig, RecordType}; use opentelemetry_proto::tonic::collector::logs::v1::{ExportLogsServiceRequest, logs_service_client::LogsServiceClient}; +use opentelemetry_proto::tonic::collector::metrics::v1::{ExportMetricsServiceRequest, metrics_service_client::MetricsServiceClient}; +use opentelemetry_proto::tonic::collector::trace::v1::{ExportTraceServiceRequest, trace_service_client::TraceServiceClient}; use prost::Message; use rustls::{ClientConfig, ClientConnection, StreamOwned}; use tokio::runtime::{Builder, Runtime}; @@ -23,7 +25,7 @@ pub fn replay_path_to_otlp_http(path: &Path, name: &str, collector: &CollectorCo let mut sent = 0u64; let endpoints = parse_collector_endpoints(collector)?; let mut conn = MultiCollectorConnection::connect(&endpoints, Duration::from_millis(collector.timeout_ms), collector)?; - let mut batcher = OtlpBatcher::new(collector.batch_size, collector.batch_timeout_ms); + let mut logs_batcher = OtlpBatcher::new(collector.batch_size, collector.batch_timeout_ms); for segment in list_named_segments(path, name)? { let file = File::open(&segment.path)?; @@ -31,13 +33,15 @@ pub fn replay_path_to_otlp_http(path: &Path, name: &str, collector: &CollectorCo while let Some(record) = reader.next_record().map_err(to_io_error)? { if record.record_type == RecordType::Logs { - batcher.add(&record.payload, &mut conn)?; - sent = sent.saturating_add(1); + logs_batcher.add(&record.payload, &mut conn)?; + } else { + conn.post_for(record.record_type, &record.payload)?; } + sent = sent.saturating_add(1); } } - batcher.flush(&mut conn)?; + logs_batcher.flush(&mut conn)?; Ok(sent) } @@ -125,9 +129,7 @@ fn bridge_transport( if !collector_transport.backpressure_enabled { let mut conn = collector_transport.open_connection()?; while let Some(record) = read_record(transport)? { - if record.record_type == RecordType::Logs { - conn.post(&record.payload)?; - } + conn.post_for(record.record_type, &record.payload)?; commit_record(transport, state, state_file, consume, record.seq)?; } return Ok(()); @@ -142,13 +144,8 @@ fn bridge_transport( while let Some(record) = read_record(transport)? { flush_ready_results(transport, state, state_file, consume, &mut pending, &result_rx, false)?; - if record.record_type != RecordType::Logs { - commit_record(transport, state, state_file, consume, record.seq)?; - continue; - } - let seq = record.seq; - match enqueue_export_task(&task_tx, collector_transport, ExportTask { seq, payload: record.payload }) { + match enqueue_export_task(&task_tx, collector_transport, ExportTask { seq, record_type: record.record_type, payload: record.payload }) { Ok(EnqueueOutcome::Queued) => pending.push_back(PendingExport::Queued(seq)), Ok(EnqueueOutcome::DroppedNewest) => pending.push_back(PendingExport::Dropped(seq)), Err(err) => { @@ -196,19 +193,24 @@ fn export_worker( collector_transport: CollectorTransport, task_rx: mpsc::Receiver, result_tx: mpsc::Sender, ) -> io::Result<()> { let mut conn = collector_transport.open_connection()?; - let mut batcher = OtlpBatcher::new(collector_transport.collector.batch_size, collector_transport.collector.batch_timeout_ms); + let mut logs_batcher = OtlpBatcher::new(collector_transport.collector.batch_size, collector_transport.collector.batch_timeout_ms); let recv_timeout = Duration::from_millis(collector_transport.collector.batch_timeout_ms.max(50)); loop { let task = match task_rx.recv_timeout(recv_timeout) { Ok(task) => task, Err(mpsc::RecvTimeoutError::Timeout) => { - batcher.flush_if_expired(&mut conn)?; + logs_batcher.flush_if_expired(&mut conn)?; continue; } Err(mpsc::RecvTimeoutError::Disconnected) => break, }; - let outcome = batcher.add(&task.payload, &mut conn).map(|()| ExportOutcome::Delivered); - batcher.flush_if_expired(&mut conn)?; + let outcome = if task.record_type == RecordType::Logs { + let result = logs_batcher.add(&task.payload, &mut conn).map(|()| ExportOutcome::Delivered); + logs_batcher.flush_if_expired(&mut conn)?; + result + } else { + conn.post_for(task.record_type, &task.payload).map(|()| ExportOutcome::Delivered) + }; let failed = outcome.is_err(); if result_tx.send(ExportResult { seq: task.seq, outcome }).is_err() { break; @@ -217,7 +219,7 @@ fn export_worker( break; } } - let _ = batcher.flush(&mut conn); + let _ = logs_batcher.flush(&mut conn); Ok(()) } @@ -477,12 +479,61 @@ struct HttpCollectorConnection { struct GrpcCollectorConnection { client: LogsServiceClient, + metrics_client: MetricsServiceClient, + traces_client: TraceServiceClient, endpoint: CollectorEndpoint, collector: CollectorConfig, timeout: Duration, runtime: Runtime, } +impl GrpcCollectorConnection { + fn connect(endpoint: &CollectorEndpoint, timeout: Duration, collector: &CollectorConfig) -> io::Result { + let runtime = + Builder::new_current_thread().enable_all().build().map_err(|err| io::Error::other(format!("failed to build gRPC runtime: {err}")))?; + let (client, metrics_client, traces_client) = runtime.block_on(connect_grpc_with_collector(endpoint, timeout, Some(collector)))?; + Ok(Self { client, metrics_client, traces_client, endpoint: endpoint.clone(), collector: collector.clone(), timeout, runtime }) + } + + fn reconnect(&mut self) -> io::Result<()> { + let (client, metrics_client, traces_client) = self.runtime.block_on(connect_grpc_with_collector(&self.endpoint, self.timeout, Some(&self.collector)))?; + self.client = client; + self.metrics_client = metrics_client; + self.traces_client = traces_client; + Ok(()) + } + + fn post_for_inner(&mut self, record_type: RecordType, payload: &[u8]) -> io::Result<()> { + match record_type { + RecordType::Logs => { + let req = ExportLogsServiceRequest::decode(payload) + .map_err(|err| io::Error::new(io::ErrorKind::InvalidData, format!("invalid OTLP log batch: {err}")))?; + self.runtime + .block_on(self.client.export(Request::new(req))) + .map(|_| ()) + .map_err(|err| io::Error::other(format!("collector gRPC export failed: {err}"))) + } + RecordType::Metrics => { + let req = ExportMetricsServiceRequest::decode(payload) + .map_err(|err| io::Error::new(io::ErrorKind::InvalidData, format!("invalid OTLP metrics batch: {err}")))?; + self.runtime + .block_on(self.metrics_client.export(Request::new(req))) + .map(|_| ()) + .map_err(|err| io::Error::other(format!("collector gRPC metrics export failed: {err}"))) + } + RecordType::Traces => { + let req = ExportTraceServiceRequest::decode(payload) + .map_err(|err| io::Error::new(io::ErrorKind::InvalidData, format!("invalid OTLP trace batch: {err}")))?; + self.runtime + .block_on(self.traces_client.export(Request::new(req))) + .map(|_| ()) + .map_err(|err| io::Error::other(format!("collector gRPC traces export failed: {err}"))) + } + _ => Err(io::Error::other(format!("gRPC export not yet implemented for record type: {record_type:?}"))), + } + } +} + impl CollectorConnection { fn connect( endpoint: &CollectorEndpoint, timeout: Duration, tls_client: Option<&Arc>, collector: &CollectorConfig, @@ -501,20 +552,20 @@ impl CollectorConnection { } /// POST one OTLP payload, reconnecting once on transport failure. - fn post(&mut self, payload: &[u8]) -> io::Result<()> { - match self.post_inner(payload) { + fn post_for(&mut self, record_type: RecordType, payload: &[u8]) -> io::Result<()> { + match self.post_for_inner(record_type, payload) { Ok(()) => Ok(()), Err(_first) => { self.reconnect()?; - self.post_inner(payload) + self.post_for_inner(record_type, payload) } } } - fn post_inner(&mut self, payload: &[u8]) -> io::Result<()> { + fn post_for_inner(&mut self, record_type: RecordType, payload: &[u8]) -> io::Result<()> { match self { - Self::Http(conn) => conn.post_inner(payload), - Self::Grpc(conn) => conn.post_inner(payload), + Self::Http(conn) => conn.post_for_inner(record_type, payload), + Self::Grpc(conn) => conn.post_for_inner(record_type, payload), } } } @@ -529,9 +580,9 @@ impl MultiCollectorConnection { Ok(Self { collectors }) } - fn post(&mut self, payload: &[u8]) -> io::Result<()> { + fn post_for(&mut self, record_type: RecordType, payload: &[u8]) -> io::Result<()> { for collector in &mut self.collectors { - collector.post(payload)?; + collector.post_for(record_type, payload)?; } Ok(()) } @@ -559,11 +610,12 @@ impl HttpCollectorConnection { Ok(()) } - fn post_inner(&mut self, payload: &[u8]) -> io::Result<()> { + fn post_for_inner(&mut self, record_type: RecordType, payload: &[u8]) -> io::Result<()> { + let path = signal_path_for_endpoint(&self.endpoint.path, record_type); write!( self.stream, "POST {} HTTP/1.1\r\nHost: {}\r\nContent-Type: application/x-protobuf\r\nContent-Length: {}\r\nConnection: keep-alive\r\n\r\n", - self.endpoint.path, + path, self.endpoint.authority, payload.len() )?; @@ -573,26 +625,16 @@ impl HttpCollectorConnection { } } -impl GrpcCollectorConnection { - fn connect(endpoint: &CollectorEndpoint, timeout: Duration, collector: &CollectorConfig) -> io::Result { - let runtime = - Builder::new_current_thread().enable_all().build().map_err(|err| io::Error::other(format!("failed to build gRPC runtime: {err}")))?; - let client = runtime.block_on(connect_grpc_with_collector(endpoint, timeout, Some(collector)))?; - Ok(Self { client, endpoint: endpoint.clone(), collector: collector.clone(), timeout, runtime }) - } - - fn reconnect(&mut self) -> io::Result<()> { - self.client = self.runtime.block_on(connect_grpc_with_collector(&self.endpoint, self.timeout, Some(&self.collector)))?; - Ok(()) - } - - fn post_inner(&mut self, payload: &[u8]) -> io::Result<()> { - let req = ExportLogsServiceRequest::decode(payload) - .map_err(|err| io::Error::new(io::ErrorKind::InvalidData, format!("invalid OTLP log batch: {err}")))?; - self.runtime - .block_on(self.client.export(Request::new(req))) - .map(|_| ()) - .map_err(|err| io::Error::other(format!("collector gRPC export failed: {err}"))) +fn signal_path_for_endpoint(base_path: &str, record_type: RecordType) -> String { + if base_path == "/v1/logs" || base_path.is_empty() { + match record_type { + RecordType::Logs => "/v1/logs".to_string(), + RecordType::Metrics => "/v1/metrics".to_string(), + RecordType::Traces => "/v1/traces".to_string(), + RecordType::Events => "/v1/logs".to_string(), + } + } else { + base_path.to_string() } } @@ -671,11 +713,11 @@ impl OtlpBatcher { /// Add a raw OTLP payload to the batch. Flushes to conn if batch is full. fn add(&mut self, payload: &[u8], conn: &mut MultiCollectorConnection) -> io::Result<()> { if self.batch_size <= 1 { - return conn.post(payload); + return conn.post_for(RecordType::Logs, payload); } match ExportLogsServiceRequest::decode(payload) { Ok(req) => self.merge(req), - Err(_) => return conn.post(payload), + Err(_) => return conn.post_for(RecordType::Logs, payload), } if self.pending_count >= self.batch_size { self.flush(conn)?; @@ -699,7 +741,7 @@ impl OtlpBatcher { let merged = std::mem::replace(&mut self.pending, ExportLogsServiceRequest { resource_logs: Vec::new() }); self.pending_count = 0; self.first_added = None; - conn.post(&merged.encode_to_vec()) + conn.post_for(RecordType::Logs, &merged.encode_to_vec()) } fn merge(&mut self, req: ExportLogsServiceRequest) { @@ -738,6 +780,7 @@ impl OtlpBatcher { #[derive(Debug)] struct ExportTask { seq: u64, + record_type: RecordType, payload: Vec, } @@ -802,7 +845,7 @@ impl CollectorEndpoint { async fn connect_grpc_with_collector( endpoint: &CollectorEndpoint, timeout: Duration, collector: Option<&CollectorConfig>, -) -> io::Result> { +) -> io::Result<(LogsServiceClient, MetricsServiceClient, TraceServiceClient)> { let mut client = Endpoint::from_shared(format!("{}://{}", if endpoint.tls { "https" } else { "http" }, endpoint.authority)) .map_err(|err| io::Error::new(io::ErrorKind::InvalidInput, err.to_string()))? .timeout(timeout) @@ -829,7 +872,8 @@ async fn connect_grpc_with_collector( }; client = client.tls_config(tls).map_err(|err| io::Error::new(io::ErrorKind::InvalidInput, err.to_string()))?; } - client.connect().await.map(LogsServiceClient::new).map_err(|err| io::Error::other(format!("failed to connect gRPC collector: {err}"))) + let channel = client.connect().await.map_err(|err| io::Error::other(format!("failed to connect gRPC collector: {err}")))?; + Ok((LogsServiceClient::new(channel.clone()), MetricsServiceClient::new(channel.clone()), TraceServiceClient::new(channel))) } fn split_authority_and_path(input: &str) -> (&str, &str) { diff --git a/logjetd/tests/unit/daemon_utst.rs b/logjetd/tests/unit/daemon_utst.rs index 9a41d8f..cfb18e8 100644 --- a/logjetd/tests/unit/daemon_utst.rs +++ b/logjetd/tests/unit/daemon_utst.rs @@ -1,12 +1,18 @@ use super::{ - BatchPriority, ConnectionLimiter, IngestDecision, SharedIngestPolicy, classify_otlp_batch_priority, read_http_request, write_http_response, + BatchPriority, ConnectionLimiter, IngestDecision, SharedIngestPolicy, classify_otlp_batch_priority, extract_batch_timestamp_metrics, + extract_batch_timestamp_traces, maybe_decompress_body, read_http_request, write_http_response, }; use crate::config::{IngestOverloadConfig, SeverityFloor}; use opentelemetry_proto::tonic::collector::logs::v1::ExportLogsServiceRequest; +use opentelemetry_proto::tonic::collector::metrics::v1::ExportMetricsServiceRequest; +use opentelemetry_proto::tonic::collector::trace::v1::ExportTraceServiceRequest; use opentelemetry_proto::tonic::common::v1::{AnyValue, InstrumentationScope}; use opentelemetry_proto::tonic::logs::v1::{LogRecord, ResourceLogs, ScopeLogs}; +use opentelemetry_proto::tonic::metrics::v1::number_data_point::Value as DataPointValue; +use opentelemetry_proto::tonic::metrics::v1::{Gauge, Metric, NumberDataPoint, ResourceMetrics, ScopeMetrics}; use opentelemetry_proto::tonic::resource::v1::Resource; -use std::io::Cursor; +use opentelemetry_proto::tonic::trace::v1::{ResourceSpans, ScopeSpans, Span}; +use std::io::{Cursor, Write}; use std::sync::Arc; #[test] @@ -19,6 +25,47 @@ fn read_http_request_parses_valid_request() { assert_eq!(request.body, b"abc"); } +#[test] +fn read_http_request_parses_content_encoding() { + let bytes = b"POST /v1/logs HTTP/1.1\r\nHost: example\r\nContent-Length: 3\r\nContent-Encoding: gzip\r\n\r\nabc"; + let mut cursor = Cursor::new(bytes.as_slice()); + let request = read_http_request(&mut cursor, 1024).unwrap(); + assert_eq!(request.content_encoding, Some("gzip".to_string())); +} + +#[test] +fn maybe_decompress_body_passes_through_uncompressed() { + let data = b"hello"; + let out = maybe_decompress_body(data.to_vec(), None).unwrap(); + assert_eq!(out, data); +} + +#[test] +fn maybe_decompress_body_decompresses_gzip() { + let data = b"hello world"; + let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); + encoder.write_all(data).unwrap(); + let compressed = encoder.finish().unwrap(); + let out = maybe_decompress_body(compressed, Some("gzip")).unwrap(); + assert_eq!(out, data); +} + +#[test] +fn maybe_decompress_body_decompresses_x_gzip() { + let data = b"hello world"; + let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); + encoder.write_all(data).unwrap(); + let compressed = encoder.finish().unwrap(); + let out = maybe_decompress_body(compressed, Some("x-gzip")).unwrap(); + assert_eq!(out, data); +} + +#[test] +fn maybe_decompress_body_rejects_invalid_gzip() { + let err = maybe_decompress_body(b"not-gzip".to_vec(), Some("gzip")).unwrap_err(); + assert_eq!(err.kind(), std::io::ErrorKind::InvalidData); +} + #[test] fn read_http_request_rejects_missing_content_length() { let bytes = b"POST /v1/logs HTTP/1.1\r\nHost: example\r\n\r\nabc"; @@ -169,3 +216,80 @@ fn classify_otlp_batch_priority_uses_highest_log_severity() { assert_eq!(classify_otlp_batch_priority(&batch), BatchPriority::Error); } + +#[test] +fn extract_batch_timestamp_metrics_finds_first_datapoint_time() { + let metric = Metric { + name: "cpu".to_string(), + description: String::new(), + unit: "%".to_string(), + data: Some(opentelemetry_proto::tonic::metrics::v1::metric::Data::Gauge(Gauge { + data_points: vec![NumberDataPoint { + attributes: vec![], + start_time_unix_nano: 0, + time_unix_nano: 1_700_000_000_000_000_000, + value: Some(DataPointValue::AsDouble(42.0)), + flags: 0, + exemplars: vec![], + }], + })), + metadata: vec![], + }; + let batch = ExportMetricsServiceRequest { + resource_metrics: vec![ResourceMetrics { + resource: Some(Resource { attributes: vec![], dropped_attributes_count: 0, entity_refs: vec![] }), + scope_metrics: vec![ScopeMetrics { + scope: Some(InstrumentationScope { name: "test".to_string(), version: String::new(), attributes: vec![], dropped_attributes_count: 0 }), + metrics: vec![metric], + schema_url: String::new(), + }], + schema_url: String::new(), + }], + }; + assert_eq!(extract_batch_timestamp_metrics(&batch), Some(1_700_000_000_000_000_000)); +} + +#[test] +fn extract_batch_timestamp_metrics_returns_none_when_empty() { + let batch = ExportMetricsServiceRequest { resource_metrics: vec![] }; + assert_eq!(extract_batch_timestamp_metrics(&batch), None); +} + +#[test] +fn extract_batch_timestamp_traces_finds_first_span_start_time() { + let batch = ExportTraceServiceRequest { + resource_spans: vec![ResourceSpans { + resource: Some(Resource { attributes: vec![], dropped_attributes_count: 0, entity_refs: vec![] }), + scope_spans: vec![ScopeSpans { + scope: Some(InstrumentationScope { name: "test".to_string(), version: String::new(), attributes: vec![], dropped_attributes_count: 0 }), + spans: vec![Span { + trace_id: vec![1, 2, 3, 4], + span_id: vec![5, 6, 7, 8], + parent_span_id: vec![], + name: "test-span".to_string(), + kind: 1, + start_time_unix_nano: 1_700_000_000_000_000_000, + end_time_unix_nano: 1_700_000_000_000_000_001, + attributes: vec![], + dropped_attributes_count: 0, + events: vec![], + dropped_events_count: 0, + links: vec![], + dropped_links_count: 0, + status: None, + flags: 0, + trace_state: String::new(), + }], + schema_url: String::new(), + }], + schema_url: String::new(), + }], + }; + assert_eq!(extract_batch_timestamp_traces(&batch), Some(1_700_000_000_000_000_000)); +} + +#[test] +fn extract_batch_timestamp_traces_returns_none_when_empty() { + let batch = ExportTraceServiceRequest { resource_spans: vec![] }; + assert_eq!(extract_batch_timestamp_traces(&batch), None); +} diff --git a/logjetd/tests/unit/replay_utst.rs b/logjetd/tests/unit/replay_utst.rs index cdbb86a..de52e4b 100644 --- a/logjetd/tests/unit/replay_utst.rs +++ b/logjetd/tests/unit/replay_utst.rs @@ -1,9 +1,10 @@ use super::{ BridgeState, CollectorEndpoint, CollectorTransport, EnqueueOutcome, ExportTask, enqueue_export_task, parse_bridge_state, parse_content_length, - read_bridge_state, read_http_response, reconcile_bridge_state, write_bridge_state, + read_bridge_state, read_http_response, reconcile_bridge_state, signal_path_for_endpoint, write_bridge_state, }; use crate::config::{BackpressureMode, CollectorConfig, UpstreamMode}; use crate::protocol::ReplayHello; +use logjet::RecordType; use std::fs; use std::path::PathBuf; use std::sync::mpsc; @@ -102,9 +103,9 @@ fn bridge_state_resets_when_legacy_saved_seq_is_above_upstream_last_seq() { fn disconnect_mode_errors_when_export_queue_is_full() { let transport = test_collector_transport(BackpressureMode::Disconnect, 1); let (task_tx, _task_rx) = mpsc::sync_channel(1); - task_tx.send(ExportTask { seq: 1, payload: vec![1] }).unwrap(); + task_tx.send(ExportTask { seq: 1, record_type: RecordType::Logs, payload: vec![1] }).unwrap(); - let err = enqueue_export_task(&task_tx, &transport, ExportTask { seq: 2, payload: vec![2] }).unwrap_err(); + let err = enqueue_export_task(&task_tx, &transport, ExportTask { seq: 2, record_type: RecordType::Logs, payload: vec![2] }).unwrap_err(); assert_eq!(err.kind(), std::io::ErrorKind::TimedOut); } @@ -112,9 +113,9 @@ fn disconnect_mode_errors_when_export_queue_is_full() { fn drop_newest_mode_reports_drop_when_export_queue_is_full() { let transport = test_collector_transport(BackpressureMode::DropNewest, 1); let (task_tx, _task_rx) = mpsc::sync_channel(1); - task_tx.send(ExportTask { seq: 1, payload: vec![1] }).unwrap(); + task_tx.send(ExportTask { seq: 1, record_type: RecordType::Logs, payload: vec![1] }).unwrap(); - let outcome = enqueue_export_task(&task_tx, &transport, ExportTask { seq: 2, payload: vec![2] }).unwrap(); + let outcome = enqueue_export_task(&task_tx, &transport, ExportTask { seq: 2, record_type: RecordType::Logs, payload: vec![2] }).unwrap(); assert_eq!(outcome, EnqueueOutcome::DroppedNewest); } @@ -192,3 +193,24 @@ fn parse_content_length_is_case_insensitive() { fn parse_content_length_returns_zero_when_absent() { assert_eq!(parse_content_length("X-Custom: foo\r\n"), 0); } + +#[test] +fn signal_path_for_endpoint_defaults_to_signal_specific_paths() { + assert_eq!(signal_path_for_endpoint("/v1/logs", logjet::RecordType::Logs), "/v1/logs"); + assert_eq!(signal_path_for_endpoint("/v1/logs", logjet::RecordType::Metrics), "/v1/metrics"); + assert_eq!(signal_path_for_endpoint("/v1/logs", logjet::RecordType::Traces), "/v1/traces"); + assert_eq!(signal_path_for_endpoint("/v1/logs", logjet::RecordType::Events), "/v1/logs"); +} + +#[test] +fn signal_path_for_endpoint_preserves_custom_path() { + assert_eq!(signal_path_for_endpoint("/custom/ingest", logjet::RecordType::Metrics), "/custom/ingest"); + assert_eq!(signal_path_for_endpoint("/custom/ingest", logjet::RecordType::Traces), "/custom/ingest"); +} + +#[test] +fn signal_path_for_endpoint_defaults_from_empty_path() { + assert_eq!(signal_path_for_endpoint("", logjet::RecordType::Logs), "/v1/logs"); + assert_eq!(signal_path_for_endpoint("", logjet::RecordType::Metrics), "/v1/metrics"); + assert_eq!(signal_path_for_endpoint("", logjet::RecordType::Traces), "/v1/traces"); +} diff --git a/scripts/audit-table.sh b/scripts/audit-table.sh new file mode 100755 index 0000000..48b0801 --- /dev/null +++ b/scripts/audit-table.sh @@ -0,0 +1,143 @@ +#!/usr/bin/env bash +# Dependency audit table formatter. Requires: cargo-audit. +set -euo pipefail + +RED=$(tput setaf 1 2>/dev/null || echo '') +YELLOW=$(tput setaf 3 2>/dev/null || echo '') +GREEN=$(tput setaf 2 2>/dev/null || echo '') +BOLD=$(tput bold 2>/dev/null || echo '') +RST=$(tput sgr0 2>/dev/null || echo '') +DIM="\033[2m" +DKYLW="\033[33m" + +TMPFILE=$(mktemp /tmp/audit-table.XXXXXX) +trap 'rm -f "$TMPFILE" "${TMPFILE}.parsed" "${TMPFILE}.full"' EXIT + +HAS_EXCEPTIONS=false +[ -f .cargo/audit.toml ] && HAS_EXCEPTIONS=true + +# ── Reason map for suppressed advisories ───── +# Format: RUSTSEC-ID=reason +declare -A REASONS=( +) + +# ── Run with current exceptions ────────────── +cargo audit 2>&1 | tee "$TMPFILE" >/dev/null || true + +# ── Parse helper (extracts id field too) ───── +parse_audit() { + local inf="$1" out="$2" + awk ' + BEGIN { RS=""; FS="\n" } + { + crate=""; version=""; title=""; severity=""; fix="-"; warn_type=""; id="" + for (i=1; i<=NF; i++) { + line = $i + if (line ~ /^Crate:/) { split(line, a, /[[:space:]]+/); crate=a[2] } + if (line ~ /^Version:/) { split(line, a, /[[:space:]]+/); version=a[2] } + if (line ~ /^Title:/) { sub(/^Title:[[:space:]]+/, "", line); title=line } + if (line ~ /^ID:/) { split(line, a, /[[:space:]]+/); id=a[2] } + if (line ~ /^Severity:/) { split(line, a, /[[:space:]]+/); score=a[2]; + sub(/^Severity:[[:space:]]+[^[:space:]]+[[:space:]]+/, "", line); sev=line; + gsub(/[()]/, "", sev); severity=sprintf("%s (%s)", score, sev) } + if (line ~ /^Warning:/) { split(line, a, /[[:space:]]+/); warn_type=a[2] } + if (line ~ /^Solution:/) { sub(/^Solution:[[:space:]]+/, "", line); fix=line; + if (fix ~ /No fixed/) fix="-"; gsub(/^Upgrade to /, "", fix) } + } + if (length(crate) == 0) next + if (length(warn_type) > 0) + printf "warn\t%s\t%s\t%s\t%s\t%s\n", warn_type, crate, version, title, fix, id + else + printf "vuln\t%s\t%s\t%s\t%s\t%s\n", (length(severity)>0 ? severity : "?"), crate, version, title, fix, id + } + ' "$inf" > "$out" +} + +parse_audit "$TMPFILE" "${TMPFILE}.parsed" + +# ── If exceptions exist, also get the full picture ── +if $HAS_EXCEPTIONS; then + FULL_TMP=$(mktemp /tmp/audit-full.XXXXXX) + mv .cargo/audit.toml .cargo/_audit.toml.bak + cargo audit 2>&1 | tee "$FULL_TMP" >/dev/null || true + mv .cargo/_audit.toml.bak .cargo/audit.toml + parse_audit "$FULL_TMP" "${TMPFILE}.full" + rm -f "$FULL_TMP" + + comm -23 <(sort "${TMPFILE}.full") <(sort "${TMPFILE}.parsed") > "${TMPFILE}.ignored" +fi + +# ── Lookup reason by advisory id ───────────── +reason_for() { + local id="$1" r + r="${REASONS[$id]:-}" + [ -n "$r" ] && echo "$r" || echo "see .cargo/audit.toml" +} + +# ── Header ─────────────────────────────────── +echo -e "${BOLD} Dependency Audit Report${RST}" +echo + +# ── Vulnerabilities (still open) ───────────── +VULN_COUNT=$(awk '/^vuln/{n++} END{print n+0}' "${TMPFILE}.parsed") +if [ "$VULN_COUNT" -gt 0 ]; then + echo -e " ${BOLD}${RED}VULNERABILITIES${RST} ${RED}(${VULN_COUNT} found)${RST}" + echo + printf " %-20s %-10s %-14s %-55s %-30s\n" "Crate" "Version" "Severity" "Issue" "Fix" + printf " %-20s %-10s %-14s %-55s %-30s\n" "--------------------" "----------" "--------------" "-------------------------------------------------------" "------------------------------" + grep '^vuln' "${TMPFILE}.parsed" | while IFS=$'\t' read -r _ sev crate ver title fix id; do + printf " ${RED}%-20s${RST} %-10s %-14s %-55s %-30s\n" \ + "${crate:0:20}" "${ver:0:10}" "${sev:0:14}" "${title:0:55}" "${fix:0:30}" + done + echo +fi + +# ── Warnings (still open) ──────────────────── +WARN_COUNT=$(awk '/^warn/{n++} END{print n+0}' "${TMPFILE}.parsed") +if [ "$WARN_COUNT" -gt 0 ]; then + echo -e " ${BOLD}${YELLOW}WARNINGS${RST} ${YELLOW}(${WARN_COUNT} total)${RST}" + echo + printf " %-14s %-20s %-10s %-60s\n" "Kind" "Crate" "Version" "Issue" + printf " %-14s %-20s %-10s %-60s\n" "--------------" "--------------------" "----------" "------------------------------------------------------------" + grep '^warn' "${TMPFILE}.parsed" | while IFS=$'\t' read -r _ kind crate ver title fix id; do + printf " ${YELLOW}%-14s${RST} %-20s %-10s %-60s\n" \ + "${kind:0:14}" "${crate:0:20}" "${ver:0:10}" "${title:0:60}" + done + echo +fi + +# ── Clean summary ──────────────────────────── +if [ "$VULN_COUNT" -eq 0 ] && [ "$WARN_COUNT" -eq 0 ]; then + echo -e " ${BOLD}${GREEN}All clear — no open findings${RST}" + echo +fi + +# ── Ignored advisories summary ─────────────── +if $HAS_EXCEPTIONS; then + IGN_VULN=$(awk '/^vuln/{n++} END{print n+0}' "${TMPFILE}.ignored" 2>/dev/null) + IGN_WARN=$(awk '/^warn/{n++} END{print n+0}' "${TMPFILE}.ignored" 2>/dev/null) + IGN_TOTAL=$((IGN_VULN + IGN_WARN)) + + if [ "$IGN_TOTAL" -gt 0 ]; then + echo -e " ${BOLD}${DIM}Suppressed by .cargo/audit.toml${RST} ${DIM}(${IGN_TOTAL} advisories)${RST}" + echo + printf " ${DIM}%-20s %-8s %-7s %-55s %-28s${RST}\n" "Crate" "Version" "Risk" "Issue" "Why suppressed" + printf " ${DIM}%-20s %-8s %-7s %-55s %-28s${RST}\n" "--------------------" "--------" "-------" "-------------------------------------------------------" "----------------------------" + + # vulns first + grep '^vuln' "${TMPFILE}.ignored" 2>/dev/null | while IFS=$'\t' read -r _ sev crate ver title fix id; do + reason=$(reason_for "$id") + if [[ "$reason" == TODO:* ]]; then color="${DKYLW}"; else color="${DIM}"; fi + printf " ${color}%-20s %-8s %-7s %-55s %-28s${RST}\n" \ + "${crate:0:20}" "${ver:0:8}" "${sev:0:7}" "${title:0:55}" "$reason" + done + # then warnings + grep '^warn' "${TMPFILE}.ignored" 2>/dev/null | while IFS=$'\t' read -r _ kind crate ver title fix id; do + reason=$(reason_for "$id") + if [[ "$reason" == TODO:* ]]; then color="${DKYLW}"; else color="${DIM}"; fi + printf " ${color}%-20s %-8s %-7s %-55s %-28s${RST}\n" \ + "${crate:0:20}" "${ver:0:8}" "${kind:0:7}" "${title:0:55}" "$reason" + done + echo + fi +fi