Add metrics and traces support

Makefile

@@ -1,4 +1,4 @@
-.PHONY: build dev devel check fix test test-unit test-integration test-abi-matrix test-exporter-release-smoke setup clean stats arm-devel arm x86-devel x86 setup-arm setup-x86 demo man
+.PHONY: build dev devel check fix test test-unit test-integration test-abi-matrix test-exporter-release-smoke setup clean stats arm-devel arm x86-devel x86 setup-arm setup-x86 demo man advisory
 
 DEFAULT_TARGET := build
 ARM_TARGET ?= aarch64-unknown-linux-musl
@@ -60,17 +60,24 @@ x86: setup setup-x86
 
 demo: devel
 	cargo build -p otlp-demo
+	cargo build -p otlp-demo --bin traces-emitter
+	cargo build -p otlp-demo --bin traces-grpc-emitter
+	cargo build -p otlp-demo --bin multi-signal-emitter
+	cargo build -p otlp-demo --bin metrics-grpc-emitter
 	cargo build -p lj-syslog-ingest
 	cargo build -p lj-logcat-ingest
 	cargo build -p lj-stress-ingest
-
 man: $(MANPAGE_OUT)
 
 $(MANPAGE_OUT): doc/manpage/%.1: doc/manpage/%.1.md
 	@command -v pandoc >/dev/null 2>&1 || { echo "pandoc not found. Install pandoc to build manpages."; exit 1; }
 	@mkdir -p doc/manpage
 	pandoc --standalone --to man $< -o $@
 
+advisory:
+	@cargo audit --version >/dev/null 2>&1 || { echo "Installing cargo-audit..."; cargo install cargo-audit --locked; }
+	@scripts/audit-table.sh
+
 clean:
 	cargo clean
 

demo/Cargo.toml

@@ -8,7 +8,7 @@ license.workspace = true
 colored = "3"
 logjet = { path = ".." }
 lz4_flex = { version = "0.11", default-features = false, features = ["std"] }
-opentelemetry-proto = { version = "0.31", features = ["gen-tonic", "logs"] }
+opentelemetry-proto = { version = "0.31", features = ["gen-tonic", "logs", "metrics", "trace"] }
 prost = "0.14"
 rustls = { version = "0.23", default-features = false, features = [
     "ring",

demo/README.md

@@ -45,6 +45,16 @@ It also contains scenario demos under subdirectories:
   - generate about 5K BOFH log entries, then export that `.logjet` file to Parquet through the external exporter plugin
 - [`tui-view`](./tui-view)
   - generate 1000 randomized log entries and open `ljx view` on the result
+- [`metrics-view`](./metrics-view)
+  - generate OTLP metrics batches, ingest them into `ljd`, and open `ljx view` on the result
+- [`metrics-grpc-view`](./metrics-grpc-view)
+  - generate OTLP/gRPC metrics batches, ingest them into `ljd`, and open `ljx view` on the result
+- [`traces-view`](./traces-view)
+  - generate OTLP traces batches, ingest them into `ljd`, and open `ljx view` on the result
+- [`traces-grpc-view`](./traces-grpc-view)
+  - generate OTLP/gRPC traces batches, ingest them into `ljd`, and open `ljx view` on the result
+- [`multi-signal-view`](./multi-signal-view)
+  - interleave logs, metrics, and traces into a single `ljd` file, then open `ljx view`
 - [`multiscan-view`](./multiscan-view)
   - generate a tree of `.logjet` files and open `ljx view` across the whole dataset
 - [`multiscan-discover`](./multiscan-discover)

demo/metrics-grpc-view/README.md

@@ -0,0 +1,18 @@
+# metrics-grpc-view
+
+Ingest OTLP/gRPC metrics into `ljd`, then open `ljx view` on the result.
+
+## Run
+
+```bash
+make demo
+cd demo/metrics-grpc-view
+./run-demo.sh
+```
+
+The demo:
+1. Starts `ljd` with OTLP/gRPC ingest on `127.0.0.1:4317`
+2. Emits 15 metrics batches via `metrics-grpc-emitter` (`MetricsService/Export`)
+3. Stops `ljd` after flush
+4. Opens `ljx view` on the resulting `.logjet` file
+5. Cleans up after the viewer exits

demo/metrics-grpc-view/logjetd.conf

@@ -0,0 +1,7 @@
+output: file
+file.path: ./logs
+file.size: 100000
+file.name: metrics.logjet
+ingest.protocol: otlp-grpc
+ingest.listen: 127.0.0.1:4317
+replay.listen: 127.0.0.1:7002

demo/metrics-grpc-view/run-demo.sh

@@ -0,0 +1,72 @@
+#!/bin/sh
+set -eu
+
+SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
+TARGET_DIR="$SCRIPT_DIR/../../target/debug"
+LJD="$TARGET_DIR/ljd"
+EMITTER="$TARGET_DIR/metrics-grpc-emitter"
+LJX="$TARGET_DIR/ljx"
+CONFIG="$SCRIPT_DIR/logjetd.conf"
+OUTPUT_DIR="$SCRIPT_DIR/logs"
+OUTPUT_FILE="$OUTPUT_DIR/metrics.logjet"
+
+if [ ! -x "$LJD" ]; then
+    echo "missing $LJD"
+    echo "build it first with: make demo"
+    exit 1
+fi
+
+if [ ! -x "$EMITTER" ]; then
+    echo "missing $EMITTER"
+    echo "build it first with: make demo"
+    exit 1
+fi
+
+if [ ! -x "$LJX" ]; then
+    echo "missing $LJX"
+    echo "build it first with: make demo"
+    exit 1
+fi
+
+cd "$SCRIPT_DIR"
+
+mkdir -p "$OUTPUT_DIR"
+rm -f "$OUTPUT_FILE" "$OUTPUT_DIR/metrics-"*.logjet "$OUTPUT_DIR/metrics.stream-id"
+
+echo "starting ljd with config $CONFIG"
+"$LJD" --config "$CONFIG" &
+LJD_PID=$!
+
+cleanup() {
+    if [ -n "${EMITTER_PID:-}" ]; then
+        kill "$EMITTER_PID" 2>/dev/null || true
+        wait "$EMITTER_PID" 2>/dev/null || true
+    fi
+    if [ -n "${LJD_PID:-}" ]; then
+        kill "$LJD_PID" 2>/dev/null || true
+        wait "$LJD_PID" 2>/dev/null || true
+    fi
+}
+trap cleanup EXIT INT TERM
+
+sleep 1
+
+METRIC_COUNT=15
+echo "starting metrics-emitter toward 127.0.0.1:4317 ($METRIC_COUNT batches)"
+"$EMITTER" 127.0.0.1:4317 "$METRIC_COUNT"
+
+echo "emitter finished; giving ljd time to flush"
+sleep 2
+
+echo "stopping ljd"
+kill "$LJD_PID" 2>/dev/null || true
+wait "$LJD_PID" 2>/dev/null || true
+LJD_PID=""
+
+echo "opening ljx view on $OUTPUT_FILE"
+"$LJX" view "$OUTPUT_FILE"
+
+echo "cleaning up demo artefacts"
+rm -rf "$OUTPUT_DIR"
+
+echo "done"

demo/metrics-view/README.md

@@ -0,0 +1,35 @@
+# metrics-view
+
+Generate OTLP metrics, ingest them into `ljd`, and browse the resulting `.logjet` file with `ljx view`.
+
+## What it does
+
+1. Starts `ljd` in file mode listening for OTLP/HTTP on `127.0.0.1:4318`.
+2. Runs `metrics-emitter` which POSTs 15 `ExportMetricsServiceRequest` batches to `/v1/metrics`.
+   Each batch contains:
+   - a Gauge `cpu.usage` with a value that drifts between 10 % and 90 %
+   - a cumulative Sum `requests.total` that grows monotonically
+3. Stops `ljd` after the emitter finishes.
+4. Opens `ljx view` on the stored `metrics.logjet` file.
+
+## Prerequisites
+
+Build the demo binaries:
+
+```bash
+make demo
+```
+
+## Run
+
+```bash
+./run-demo.sh
+```
+
+## Keybindings in `ljx view`
+
+- `↑` / `↓` – move through records
+- `Enter` – open detail modal for the selected record
+- `Esc` – close modal
+- `i` – toggle hex/inspection panel
+- `q` – quit

demo/metrics-view/logjetd.conf

@@ -0,0 +1,7 @@
+output: file
+file.path: ./logs
+file.size: 100000
+file.name: metrics.logjet
+ingest.protocol: otlp-http
+ingest.listen: 127.0.0.1:4318
+replay.listen: 127.0.0.1:7002

demo/metrics-view/run-demo.sh

@@ -0,0 +1,72 @@
+#!/bin/sh
+set -eu
+
+SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
+TARGET_DIR="$SCRIPT_DIR/../../target/debug"
+LJD="$TARGET_DIR/ljd"
+EMITTER="$TARGET_DIR/metrics-emitter"
+LJX="$TARGET_DIR/ljx"
+CONFIG="$SCRIPT_DIR/logjetd.conf"
+OUTPUT_DIR="$SCRIPT_DIR/logs"
+OUTPUT_FILE="$OUTPUT_DIR/metrics.logjet"
+
+if [ ! -x "$LJD" ]; then
+    echo "missing $LJD"
+    echo "build it first with: make demo"
+    exit 1
+fi
+
+if [ ! -x "$EMITTER" ]; then
+    echo "missing $EMITTER"
+    echo "build it first with: make demo"
+    exit 1
+fi
+
+if [ ! -x "$LJX" ]; then
+    echo "missing $LJX"
+    echo "build it first with: make demo"
+    exit 1
+fi
+
+cd "$SCRIPT_DIR"
+
+mkdir -p "$OUTPUT_DIR"
+rm -f "$OUTPUT_FILE" "$OUTPUT_DIR/metrics-"*.logjet "$OUTPUT_DIR/metrics.stream-id"
+
+echo "starting ljd with config $CONFIG"
+"$LJD" --config "$CONFIG" &
+LJD_PID=$!
+
+cleanup() {
+    if [ -n "${EMITTER_PID:-}" ]; then
+        kill "$EMITTER_PID" 2>/dev/null || true
+        wait "$EMITTER_PID" 2>/dev/null || true
+    fi
+    if [ -n "${LJD_PID:-}" ]; then
+        kill "$LJD_PID" 2>/dev/null || true
+        wait "$LJD_PID" 2>/dev/null || true
+    fi
+}
+trap cleanup EXIT INT TERM
+
+sleep 1
+
+METRIC_COUNT=15
+echo "starting metrics-emitter toward 127.0.0.1:4318 ($METRIC_COUNT batches)"
+"$EMITTER" 127.0.0.1:4318 "$METRIC_COUNT"
+
+echo "emitter finished; giving ljd time to flush"
+sleep 2
+
+echo "stopping ljd"
+kill "$LJD_PID" 2>/dev/null || true
+wait "$LJD_PID" 2>/dev/null || true
+LJD_PID=""
+
+echo "opening ljx view on $OUTPUT_FILE"
+"$LJX" view "$OUTPUT_FILE"
+
+echo "cleaning up demo artefacts"
+rm -rf "$OUTPUT_DIR"
+
+echo "done"

demo/multi-signal-view/README.md

@@ -0,0 +1,27 @@
+# multi-signal-view
+
+Ingest logs, metrics, and traces (all over OTLP/HTTP) into a single `ljd` instance, then open `ljx view` to verify all three signals decode correctly side-by-side.
+
+## Run
+
+```bash
+make demo
+cd demo/multi-signal-view
+./run-demo.sh
+```
+
+The demo:
+1. Starts `ljd` with OTLP/HTTP ingest on `127.0.0.1:4318`
+2. Emits 6 batches per signal (logs → metrics → traces), interleaved, via `multi-signal-emitter`
+3. Stops `ljd` after flush
+4. Opens `ljx view` on the resulting `.logjet` file
+5. Cleans up after the viewer exits
+
+## What to look for in `ljx view`
+
+- **Logs rows**: show BOFH excuse text (body preview)
+- **Metrics rows**: show `cpu.usage=N%` and `requests.total=N` summaries
+- **Traces rows**: show `GET /api/items/N?page=M` span names with kind
+- All three record types coexist in one file in arrival order
+- Press `Enter` on any row → full decoded payload
+- Press `i` → info panel with signal-specific metadata

demo/multi-signal-view/logjetd.conf

@@ -0,0 +1,7 @@
+output: file
+file.path: ./logs
+file.size: 100000
+file.name: mixed.logjet
+ingest.protocol: otlp-http
+ingest.listen: 127.0.0.1:4318
+replay.listen: 127.0.0.1:7002

demo/multi-signal-view/run-demo.sh

@@ -0,0 +1,81 @@
+#!/bin/sh
+set -eu
+
+SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
+TARGET_DIR="$SCRIPT_DIR/../../target/debug"
+LJD="$TARGET_DIR/ljd"
+EMITTER="$TARGET_DIR/multi-signal-emitter"
+LJX="$TARGET_DIR/ljx"
+CONFIG="$SCRIPT_DIR/logjetd.conf"
+OUTPUT_DIR="$SCRIPT_DIR/logs"
+OUTPUT_FILE="$OUTPUT_DIR/mixed.logjet"
+
+if [ ! -x "$LJD" ]; then
+    echo "missing $LJD"
+    echo "build it first with: make demo"
+    exit 1
+fi
+
+if [ ! -x "$EMITTER" ]; then
+    echo "missing $EMITTER"
+    echo "build it first with: make demo"
+    exit 1
+fi
+
+if [ ! -x "$LJX" ]; then
+    echo "missing $LJX"
+    echo "build it first with: make demo"
+    exit 1
+fi
+
+cd "$SCRIPT_DIR"
+
+mkdir -p "$OUTPUT_DIR"
+rm -f "$OUTPUT_FILE" "$OUTPUT_DIR/mixed-"*.logjet "$OUTPUT_DIR/mixed.stream-id"
+
+echo "starting ljd with config $CONFIG"
+"$LJD" --config "$CONFIG" &
+LJD_PID=$!
+
+cleanup() {
+    if [ -n "${EMITTER_PID:-}" ]; then
+        kill "$EMITTER_PID" 2>/dev/null || true
+        wait "$EMITTER_PID" 2>/dev/null || true
+    fi
+    if [ -n "${LJD_PID:-}" ]; then
+        kill "$LJD_PID" 2>/dev/null || true
+        wait "$LJD_PID" 2>/dev/null || true
+    fi
+}
+trap cleanup EXIT INT TERM
+
+sleep 1
+
+BATCH_COUNT=6
+echo "starting multi-signal-emitter toward 127.0.0.1:4318 ($BATCH_COUNT batches per signal)"
+"$EMITTER" 127.0.0.1:4318 "$BATCH_COUNT"
+
+echo "emitter finished; giving ljd time to flush"
+sleep 2
+
+echo "stopping ljd"
+kill "$LJD_PID" 2>/dev/null || true
+wait "$LJD_PID" 2>/dev/null || true
+LJD_PID=""
+
+echo "opening ljx view on $OUTPUT_FILE"
+echo ""
+echo "TIP: Navigate through the list. You will see:"
+echo "  - BOFH log entries (body text preview)"
+echo "  - Metrics entries (cpu.usage=N%, requests.total=N)"
+echo "  - Traces entries (GET /api/items/N?page=M)"
+echo ""
+echo "Press Enter on any row to see the full decoded payload."
+echo "Press 'i' for the info panel with signal-specific metadata."
+echo ""
+"$LJX" view "$OUTPUT_FILE"
+
+echo "cleaning up demo artefacts"
+rm -rf "$OUTPUT_DIR"
+
+echo "done"