Ingestor implementation for Google Perfetto

Cargo.toml

@@ -14,6 +14,7 @@ members = [
 	"plugins/syslog-ingest",
 	"plugins/logcat-ingest",
 	"plugins/stress-ingest",
+	"plugins/perfetto-ingest",
 	"plugins/parquet-exporter",
 ]
 
@@ -30,6 +31,7 @@ default-members = [
 	"plugins/syslog-ingest",
 	"plugins/logcat-ingest",
 	"plugins/stress-ingest",
+	"plugins/perfetto-ingest",
 	"plugins/parquet-exporter",
 ]
 

Makefile

@@ -27,8 +27,8 @@ test: setup
 	cargo nextest run $(CORE_WORKSPACE)
 
 test-unit: setup
-	cargo build -p ljx-parquet-exporter
-	cargo nextest run -p logjet --lib -p ljd --bins -p ljx --bin ljx
+	cargo build -p ljx-parquet-exporter -p lj-perfetto-ingest
+	cargo nextest run -p logjet --lib -p ljd --bins -p ljx --bin ljx -p lj-perfetto-ingest
 
 test-integration: setup
 	cargo build -p ljd -p ljx -p ljx-parquet-exporter

demo/perfetto/README.md

@@ -0,0 +1,25 @@
+# Perfetto demos
+
+## Building Perfetto
+
+```bash
+# From workspace root — downloads deps and builds all needed tools.
+./scripts/build-perfetto.sh
+```
+
+Or pass a custom source path:
+```bash
+./scripts/build-perfetto.sh /path/to/perfetto
+```
+
+The script installs missing system packages (git, python3, curl, tar), downloads
+hermetic GN/Ninja/clang toolchain, and builds `trace_processor_shell`, `traced`,
+`traced_probes`, and `tracebox` into `perfetto/out/linux_release/`.
+
+Add to PATH or set `LJD_PERFETTO_TRACE_PROCESSOR` to point at
+`trace_processor_shell`.
+
+## Demos
+
+- [linux-data-record](linux-data-record/) — capture and inspect a system trace
+- [perfetto-to-logjet](perfetto-to-logjet/) — full end-to-end: record ftrace, import via plugin, view in ljx

demo/perfetto/linux-data-record/README.md

@@ -0,0 +1,53 @@
+# linux-data-record
+
+Record a short ftrace-based Perfetto trace on a Linux desktop, then open it in
+the trace processor for interactive inspection.
+
+## Prerequisites
+
+Build the Perfetto tools (from workspace root):
+
+```bash
+./scripts/build-perfetto.sh
+```
+
+The script finds tools via `PERFETTO_OUT` (default: `perfetto/out/linux_release`).
+Set `PERFETTO_TRACE_OUT` to override the output trace path.
+
+## Permissions
+
+ftrace requires root. The script runs `tracebox` with `sudo`. If passwordless
+sudo is not configured, run the script with `sudo`:
+
+```bash
+sudo ./run-demo.sh
+```
+
+Or set up passwordless ftrace: `sudo chown -R $USER /sys/kernel/tracing`.
+
+## Run
+
+```bash
+./run-demo.sh
+```
+
+## What happens
+
+1. `traced` and `traced_probes` are started in the background.
+2. `tracebox` captures 5 seconds of ftrace (scheduler, syscalls) via sudo into a `.pftrace` file.
+3. Tools are stopped.
+4. `trace_processor_shell` opens the trace in interactive SQL mode.
+5. Type `.q` to quit.
+
+## Example queries (in trace processor)
+
+```sql
+-- List process names
+SELECT DISTINCT name FROM process WHERE name IS NOT NULL;
+
+-- Show thread scheduling slices
+SELECT ts, dur, utid, cpu, end_state FROM sched ORDER BY ts LIMIT 20;
+
+-- Count kernel functions seen in ftrace
+SELECT name, count(*) FROM ftrace_event GROUP BY name ORDER BY count(*) DESC LIMIT 10;
+```

demo/perfetto/linux-data-record/run-demo.sh

@@ -0,0 +1,84 @@
+#!/bin/sh
+set -eu
+
+SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
+ROOT_DIR="$SCRIPT_DIR/../../.."
+PERFETTO_OUT="${PERFETTO_OUT:-$ROOT_DIR/perfetto/out/linux_release}"
+PERFETTO_TRACE_OUT="${PERFETTO_TRACE_OUT:-$SCRIPT_DIR/trace.pftrace}"
+
+TRACED="$PERFETTO_OUT/traced"
+TRACED_PROBES="$PERFETTO_OUT/traced_probes"
+TRACEBOX="$PERFETTO_OUT/tracebox"
+TP="$PERFETTO_OUT/trace_processor_shell"
+
+for bin in "$TRACED" "$TRACED_PROBES" "$TRACEBOX" "$TP"; do
+    if [ ! -x "$bin" ]; then
+        echo "missing $bin"
+        echo "build first from workspace root:"
+        echo "  cd perfetto && gn gen out/linux --args='is_debug=false'"
+        echo "  ninja -C out/linux trace_processor_shell traced traced_probes tracebox"
+        exit 1
+    fi
+done
+
+echo "Starting traced..."
+"$TRACED" &>/dev/null &
+TRACED_PID=$!
+
+echo "Starting traced_probes..."
+"$TRACED_PROBES" &>/dev/null &
+PROBES_PID=$!
+
+cleanup() {
+    kill "$TRACED_PID" 2>/dev/null || true
+    kill "$PROBES_PID" 2>/dev/null || true
+    wait "$TRACED_PID" 2>/dev/null || true
+    wait "$PROBES_PID" 2>/dev/null || true
+    echo "Services stopped."
+}
+
+trap cleanup EXIT INT TERM
+
+sleep 1
+
+echo "Recording 5s of ftrace to $PERFETTO_TRACE_OUT..."
+CONFIG_FILE="$SCRIPT_DIR/trace-config.txt"
+cat > "$CONFIG_FILE" <<'ENDCONFIG'
+buffers: {
+    size_kb: 4096
+    fill_policy: RING_BUFFER
+}
+data_sources: {
+    config {
+        name: "linux.ftrace"
+        ftrace_config {
+            ftrace_events: "sched/sched_switch"
+            ftrace_events: "sched/sched_waking"
+        }
+    }
+}
+duration_ms: 5000
+ENDCONFIG
+
+if [ "$(id -u)" -eq 0 ]; then
+    "$TRACEBOX" --txt -c "$CONFIG_FILE" -o "$PERFETTO_TRACE_OUT"
+else
+    sudo "$TRACEBOX" --txt -c "$CONFIG_FILE" -o "$PERFETTO_TRACE_OUT"
+    sudo chown "$(id -u):$(id -g)" "$PERFETTO_TRACE_OUT"
+fi
+
+rm -f "$CONFIG_FILE"
+
+cleanup
+
+if [ ! -f "$PERFETTO_TRACE_OUT" ]; then
+    echo "Trace file not created."
+    exit 1
+fi
+
+SIZE=$(du -h "$PERFETTO_TRACE_OUT" | cut -f1)
+echo "Trace recorded: $PERFETTO_TRACE_OUT ($SIZE)"
+echo ""
+echo "Opening in trace processor (type .q to quit)..."
+echo ""
+"$TP" "$PERFETTO_TRACE_OUT"

demo/perfetto/perfetto-to-logjet/README.md

@@ -0,0 +1,47 @@
+# Perfetto-to-logjet Demo
+
+End-to-end pipeline: record a Linux ftrace, import it via the perfetto-ingest
+plugin into a `.logjet` spool, and view the result in `ljx view`.
+
+## Build First
+
+```bash
+# From workspace root
+make dev
+./scripts/build-perfetto.sh
+```
+
+## Run
+
+```bash
+cd demo/perfetto/perfetto-to-logjet
+./run-demo.sh
+```
+
+Requires sudo for ftrace access.
+
+## What Happens
+
+1. `traced` + `traced_probes` start in the background.
+2. `tracebox` records 5s of scheduler events (CPU switches) via ftrace.
+3. `ljd` loads the perfetto-ingest plugin, which spawns `trace_processor`,
+   exports the trace as SQLite, maps `sched_slice` rows to OTel log records
+   with CPU/state/duration, and streams them into a `.logjet` spool.
+4. `ljx view` opens the spool — each CPU scheduling event appears as one line.
+
+## What You Should See
+
+- Thousands of log lines, each showing a CPU scheduling event:
+  ```
+  May  7 10:43:15 I  cpu=7 dur=7.2us state=R utid=19 ts=...
+  May  7 10:43:15 I  cpu=7 dur=2.0us state=R utid=21 ts=...
+  ```
+- Press `Enter` to see full OTel attributes (perfetto.sched.id, cpu, end_state).
+- Press `F` for field filter, `/` to search, `q` to quit.
+
+## Troubleshooting
+
+- **0 records**: The trace needs ftrace events — they require root. The script
+  uses `sudo tracebox`. If passwordless sudo isn't configured, run `sudo ./run-demo.sh`.
+- **Fewer records than expected in ljx view**: Delete stale index cache:
+  `rm -rf ~/.cache/ljx && ./run-demo.sh`