diff --git a/CURRENT_STATE.md b/CURRENT_STATE.md index 1f70cfe..2faa972 100644 --- a/CURRENT_STATE.md +++ b/CURRENT_STATE.md @@ -82,11 +82,17 @@ toolkit's: (`ui.virtualList`) in a model-owned `split`: one row per event (severity dot, time, direction, message, type), row selection, and a first-cut detail pane. The window stays viewport-sized in widget nodes no matter the trace length. - -Still ahead in S3: the trusted-ingestion capacity path for 500k-event / 100 MB -traces (#29), the full message inspector + session panel (#30), the failure panel -(#31), and search / filter (#32). Interactive open (native dialog + drag-drop) is -deferred to #33 — it needs an ejected runner (see ADR-0006). +- **Trusted ingestion + capacity (#29)** — trace files the user opens now parse + under raised `trusted_limits` (256 MB / 2M events) vs. the browser-scale + `untrusted_limits` kept for live/pasted data (ADR-0007). A 500k-event trace + parses, correlates, and stays viewport-bounded in the timeline. Failure + detection is capped at 50k events (several rules are O(n²)); past it the trace + is fully inspectable but detection is skipped and the UI says so — the O(n) + detection rewrite is tracked in #36. + +Still ahead in S3: the full message inspector + session panel (#30), the failure +panel (#31), and search / filter (#32). Interactive open (native dialog + +drag-drop) is deferred to #33 — it needs an ejected runner (see ADR-0006). ## What's next @@ -103,7 +109,7 @@ wall-clock replay, and a headless CLI mode. | --- | --- | | `repo` (tooling, CI) | ✅ done for S0 | | `docs` (docs, ADRs) | ✅ done for S0 | -| `ocpp` (engine) | ✅ done for S2 | +| `ocpp` (engine) | ✅ S2 + trusted ingestion (#29); O(n) detection pending (#36) | | `ui` (native views) | 🚧 shell + timeline (S3, #27–#28); panes + search next | | `capture` (live proxy) | ⬜ not started (S5) | | `cli` (headless) | ⬜ not started (S4) | diff --git a/docs/adr/0007-trusted-ingestion.md b/docs/adr/0007-trusted-ingestion.md new file mode 100644 index 0000000..b86f8aa --- /dev/null +++ b/docs/adr/0007-trusted-ingestion.md @@ -0,0 +1,64 @@ +# ADR-0007 — Trusted vs untrusted trace ingestion, and the detection cap + +- **Status:** Accepted +- **Date:** 2026-07-11 + +## Context + +Studio's headline capability over the browser toolkit is capacity: open and +inspect traces far larger than a browser tab can hold (target: 500 000 events / +100 MB). But the S1 parser hard-codes the toolkit's browser-scale caps — +`MAX_INPUT_SIZE_BYTES` (10 MB) and `MAX_EVENT_COUNT` (10 000). Those caps are +correct as an **untrusted-input** defense (a malicious or malformed payload from +a live socket or a paste must not exhaust memory), but wrong for a file the user +deliberately opened from their own disk. + +Two forces: + +1. **Trust is a property of the source, not the parser.** A command-line path (and + later a file dialog / drag-drop) is a file the user chose — trusted. Live + capture bytes (S5) are untrusted. The same parser should apply different caps + depending on which. +2. **Detection has O(n²) rules.** Several of the 16 detection rules match events + to each other by scanning (`FAILED_AUTHORIZATION` matches each result against + every event, `CONNECTOR_FAULT` / `UNEXPECTED_START` scan around each + transaction, etc.). They mirror the toolkit's algorithms and are fine at + 10 000 events, but at 500 000 a realistic call/result trace would stall the UI + for minutes on load. + +## Decision + +**Make the ingestion caps a `Limits` value chosen by the source's trust level, +and bound failure detection to a safe event count.** + +- `parser.Limits { max_input_bytes, max_events }`, with two presets: + `untrusted_limits` (10 MB / 10k — the `parseTrace` default, unchanged) and + `trusted_limits` (256 MB / 2 000 000 — `parseTraceTrusted`). Parsing and + validation are otherwise identical; only the caps differ. +- The workspace opens user files with `parseTraceTrusted`. Future untrusted + sources (S5 live capture) keep `parseTrace`. +- JSONL remains the streaming format: it parses one line at a time, so no + whole-file JSON tree is ever held — peak memory past the input buffer is the + arena of retained events, which `max_events` bounds. Bare-array / object formats + do build one tree; JSONL is the format for the largest traces. +- `detection.max_events_for_detection` (50 000) caps failure analysis. Past it the + workspace **skips** detection and sets `detection_skipped`; the trace still + parses, correlates, and is fully inspectable in the timeline, and the UI says + detection was skipped. `detectFailures` itself does not enforce the cap, so the + conformance harness and other small-trace callers are unaffected. + +## Consequences + +- Studio delivers the capacity headline honestly: a 500 000-event / 100 MB trace + opens, correlates, and is inspectable, with the timeline viewport-bounded in + widget nodes — all O(n) / O(n·sessions), no O(n²) on the load path. +- The trust boundary is explicit and testable: raising an untrusted-input bound is + a deliberate, reviewed decision keyed to source, not an accident. +- The detection cap is a **known, surfaced** limitation, not silent: large traces + show "detection skipped (large trace)" rather than a wrong empty result. Making + the O(n²) rules O(n) (indexing message-id → event, transaction pairing) so + detection scales to the full trusted capacity is tracked as a follow-up + (issue #36), with the `contract-v1` conformance goldens as the safety net that + the optimization keeps output bit-identical. +- The cap (50 000) is a deliberately conservative worst-case bound; it can rise as + rules are optimized. It lives as one named constant in `detection.zig`. diff --git a/src/ocpp/detection.zig b/src/ocpp/detection.zig index 51f326d..144bf74 100644 --- a/src/ocpp/detection.zig +++ b/src/ocpp/detection.zig @@ -227,6 +227,16 @@ fn ids2(arena: std.mem.Allocator, a: []const u8, b: []const u8) ![]const []const /// Detect failures across a trace's events and sessions. Failures are returned /// in contract order (rule by rule); allocations come from `arena`. +/// Above this event count, callers should skip `detectFailures`: several rules +/// are O(n²) in the event count (they mirror the toolkit's algorithms, written +/// for browser-scale traces). A trusted trace far past this still parses, +/// correlates, and is fully inspectable in the timeline — only the failure +/// analysis is bounded. Making the rules O(n) so detection scales to the full +/// trusted capacity is tracked as a follow-up (see ADR-0007). `detectFailures` +/// itself does not enforce this — the workspace does — so small-trace callers +/// (the conformance harness) are unaffected. +pub const max_events_for_detection: usize = 50_000; + pub fn detectFailures(arena: std.mem.Allocator, events: []const Event, sessions: []const Session) ![]Failure { var list: FailureList = .empty; diff --git a/src/ocpp/parser.zig b/src/ocpp/parser.zig index 18d9d0b..6ae80c0 100644 --- a/src/ocpp/parser.zig +++ b/src/ocpp/parser.zig @@ -1,10 +1,21 @@ -//! Trace parser — accepts **untrusted** input in JSON object, JSONL, or bare -//! array format and produces normalized `Event`s, with per-entry warnings for -//! malformed data. Mirrors the toolkit's `core/parser.ts` and `schemas.ts`. +//! Trace parser — accepts input in JSON object, JSONL, or bare array format and +//! produces normalized `Event`s, with per-entry warnings for malformed data. +//! Mirrors the toolkit's `core/parser.ts` and `schemas.ts`. //! -//! Security posture (untrusted input): -//! - Input size is capped before parsing (`MAX_INPUT_SIZE_BYTES`). -//! - Event count is capped after parsing (`MAX_EVENT_COUNT`). +//! Ingestion trust (ADR-0007): the size and event-count caps are a `Limits` +//! value chosen by the SOURCE. `parseTrace` applies the browser-scale +//! `untrusted_limits` (10 MB / 10k) — the default for live sockets and pasted +//! data. `parseTraceTrusted` applies `trusted_limits` (256 MB / 2M) for files the +//! user explicitly opened, so Studio can inspect traces far past the browser's +//! ceiling. Everything else is identical between the two. +//! +//! Security posture: +//! - Input size is capped before parsing (`Limits.max_input_bytes`). +//! - Event count is capped after parsing (`Limits.max_events`). +//! - JSONL parses one line at a time, so no whole-file JSON tree is held: peak +//! memory past the input buffer is the arena of retained events, which the +//! limits bound. (Bare-array / object formats do build one tree; JSONL is the +//! format for the largest traces.) //! - JSON is parsed with `std.json` (no prototype/`__proto__` semantics exist in //! Zig, so object-key pollution is a non-issue) into a caller-owned arena, //! copying every string (`alloc_always`) so the result borrows only the arena. @@ -24,12 +35,37 @@ const Direction = types.Direction; // Limits // --------------------------------------------------------------------------- -/// Maximum input size in bytes (10 MB). +/// Untrusted input size cap in bytes (10 MB) — matches the toolkit. pub const MAX_INPUT_SIZE_BYTES: usize = 10 * 1024 * 1024; -/// Maximum number of events after parsing. +/// Untrusted event-count cap (10k) — matches the toolkit. pub const MAX_EVENT_COUNT: usize = 10_000; +/// Trusted input size cap in bytes (256 MB) — for files the user opened. +pub const TRUSTED_MAX_INPUT_SIZE_BYTES: usize = 256 * 1024 * 1024; + +/// Trusted event-count cap (2,000,000) — comfortably past the 500k target, +/// bounded so a pathological file cannot exhaust memory. +pub const TRUSTED_MAX_EVENT_COUNT: usize = 2_000_000; + +/// Size and event-count caps for one parse, chosen by the source's trust level. +pub const Limits = struct { + max_input_bytes: usize, + max_events: usize, +}; + +/// Browser-scale caps for untrusted sources (live capture, pasted data). +pub const untrusted_limits = Limits{ + .max_input_bytes = MAX_INPUT_SIZE_BYTES, + .max_events = MAX_EVENT_COUNT, +}; + +/// Raised caps for trusted sources (files the user explicitly opened). +pub const trusted_limits = Limits{ + .max_input_bytes = TRUSTED_MAX_INPUT_SIZE_BYTES, + .max_events = TRUSTED_MAX_EVENT_COUNT, +}; + /// Excerpt of a malformed entry retained in a warning, in bytes. const RAW_EXCERPT_LEN: usize = 200; @@ -202,12 +238,27 @@ fn parseJsonObject(arena: std.mem.Allocator, input: []const u8, events: *Events, // parseTrace // --------------------------------------------------------------------------- -/// Parse a trace from a raw string into normalized events plus warnings. +/// Parse an **untrusted** trace (live capture, pasted data) under the +/// browser-scale `untrusted_limits`. The default entry point. /// /// Allocations come from `arena`; the returned `ParseResult` borrows from it and /// stays valid until it is freed. The caller owns the arena's lifetime. pub fn parseTrace(arena: std.mem.Allocator, input: []const u8) Error!ParseResult { - if (input.len > MAX_INPUT_SIZE_BYTES) return error.InputTooLarge; + return parseTraceWithLimits(arena, input, untrusted_limits); +} + +/// Parse a **trusted** trace (a file the user explicitly opened) under the +/// raised `trusted_limits`, so Studio can inspect traces far past the browser's +/// 10 MB / 10k ceiling. Same parsing and validation as `parseTrace` — only the +/// caps differ (ADR-0007). +pub fn parseTraceTrusted(arena: std.mem.Allocator, input: []const u8) Error!ParseResult { + return parseTraceWithLimits(arena, input, trusted_limits); +} + +/// Parse a trace under explicit ingestion `limits`. `parseTrace` / +/// `parseTraceTrusted` are the two presets. +pub fn parseTraceWithLimits(arena: std.mem.Allocator, input: []const u8, limits: Limits) Error!ParseResult { + if (input.len > limits.max_input_bytes) return error.InputTooLarge; const trimmed = std.mem.trim(u8, input, " \t\r\n"); if (trimmed.len == 0) return error.EmptyInput; @@ -231,7 +282,7 @@ pub fn parseTrace(arena: std.mem.Allocator, input: []const u8) Error!ParseResult }, } - if (events.items.len > MAX_EVENT_COUNT) return error.TooManyEvents; + if (events.items.len > limits.max_events) return error.TooManyEvents; if (events.items.len == 0) return error.NoValidEvents; return .{ @@ -414,6 +465,48 @@ test "untrusted input: size and event-count limits" { try testing.expectError(error.TooManyEvents, parseTrace(a, buf.items)); } +test "trusted ingestion parses a dataset-scale trace past the untrusted cap" { + var arena = std.heap.ArenaAllocator.init(testing.allocator); + defer arena.deinit(); + const a = arena.allocator(); + + // A 500,000-event JSONL trace — 50x the untrusted cap. JSONL parses one line + // at a time (no whole-file tree); the arena bulk-allocates from its backing, + // so this is time- and memory-bounded, not a million tracked allocations. + const count: usize = 500_000; + var buf: std.ArrayList(u8) = .empty; + defer buf.deinit(testing.allocator); + var i: usize = 0; + while (i < count) : (i += 1) { + try buf.appendSlice(testing.allocator, "{\"message\":[2,\"m\",\"Heartbeat\",{}]}\n"); + } + + // The trusted path accepts and normalizes every event; the untrusted default + // would reject the same input with TooManyEvents. + const r = try parseTraceTrusted(a, buf.items); + try testing.expectEqual(count, r.events.len); + try testing.expectEqualStrings("evt-0001", r.events[0].id); + try testing.expectEqual(types.MessageType.call, r.events[count - 1].message_type); +} + +test "parseTraceWithLimits honors explicit caps (both trust presets share it)" { + var arena = std.heap.ArenaAllocator.init(testing.allocator); + defer arena.deinit(); + const a = arena.allocator(); + + // Byte cap: an input past max_input_bytes is rejected before parsing. + try testing.expectError(error.InputTooLarge, parseTraceWithLimits(a, "{\"events\":[]}", .{ .max_input_bytes = 4, .max_events = 10 })); + + // Event cap: two events exceed a max_events of 1. + const two = + \\{"events":[{"message":[2,"a","Heartbeat",{}]},{"message":[2,"b","Heartbeat",{}]}]} + ; + try testing.expectError(error.TooManyEvents, parseTraceWithLimits(a, two, .{ .max_input_bytes = 1 << 20, .max_events = 1 })); + // The same input is fine when the cap admits it. + const ok = try parseTraceWithLimits(a, two, .{ .max_input_bytes = 1 << 20, .max_events = 2 }); + try testing.expectEqual(@as(usize, 2), ok.events.len); +} + test "untrusted input: object-key pollution is inert" { var arena = std.heap.ArenaAllocator.init(testing.allocator); defer arena.deinit(); diff --git a/src/ui/inspector.zig b/src/ui/inspector.zig index 251bcfe..803a1ed 100644 --- a/src/ui/inspector.zig +++ b/src/ui/inspector.zig @@ -295,6 +295,10 @@ fn statusBar(ui: *Ui, model: *const Model) Node { const t = model.activeTrace().?; const text = if (t.isError()) std.fmt.allocPrint(ui.arena, "Failed to load {s}: {s}", .{ t.name, t.load_error orelse "unknown error" }) catch "load failed" + else if (t.detection_skipped) + std.fmt.allocPrint(ui.arena, "{d} events \u{00B7} {d} sessions \u{00B7} detection skipped (large trace) \u{00B7} {d} warnings", .{ + t.eventCount(), t.sessionCount(), t.warningCount(), + }) catch "" else std.fmt.allocPrint(ui.arena, "{d} events \u{00B7} {d} sessions \u{00B7} {d} failures \u{00B7} {d} warnings", .{ t.eventCount(), t.sessionCount(), t.failureCount(), t.warningCount(), diff --git a/src/ui/workspace.zig b/src/ui/workspace.zig index cfde6c8..d03acc2 100644 --- a/src/ui/workspace.zig +++ b/src/ui/workspace.zig @@ -36,6 +36,10 @@ pub const LoadedTrace = struct { warnings: []types.ParseWarning = &.{}, /// Non-null when the trace could not be parsed; a human-readable reason. load_error: ?[]const u8 = null, + /// True when failure detection was skipped because the trace exceeds + /// `detection.max_events_for_detection` (ADR-0007). The trace is still fully + /// parsed, correlated, and inspectable; `failures` is empty. + detection_skipped: bool = false, /// The timeline row the user selected, if any (wired in the timeline PR). selected_event: ?usize = null, @@ -209,12 +213,28 @@ fn loadTrace(backing: std.mem.Allocator, name: []const u8, bytes: []const u8) Lo return .{ .name = "", .load_error = "out of memory" }; }; - const parsed = parser.parseTrace(a, bytes) catch |err| { + // Files reach the workspace because the user opened them (command line, and + // later a dialog / drag-drop), so they parse under the trusted limits + // (ADR-0007) — 256 MB / 2M events, far past the browser's ceiling. + const parsed = parser.parseTraceTrusted(a, bytes) catch |err| { return .{ .arena = arena_ptr, .name = name_copy, .load_error = @errorName(err) }; }; const sessions = timeline.buildSessionTimeline(a, parsed.events) catch |err| { return .{ .arena = arena_ptr, .name = name_copy, .events = parsed.events, .warnings = parsed.warnings, .load_error = @errorName(err) }; }; + + // Failure detection has O(n²) rules (ADR-0007); skip it past the cap so a + // huge trace still opens instantly and stays fully inspectable. + if (parsed.events.len > detection.max_events_for_detection) { + return .{ + .arena = arena_ptr, + .name = name_copy, + .events = parsed.events, + .sessions = sessions, + .warnings = parsed.warnings, + .detection_skipped = true, + }; + } const failures = detection.detectFailures(a, parsed.events, sessions) catch |err| { return .{ .arena = arena_ptr, .name = name_copy, .events = parsed.events, .sessions = sessions, .warnings = parsed.warnings, .load_error = @errorName(err) }; }; @@ -344,6 +364,37 @@ test "the splitter fraction is model-owned and echoed by update" { try testing.expectApproxEqAbs(@as(f32, 0.4), model.timeline_split, 0.0001); } +test "a trace past the detection cap loads fully but skips detection" { + var model = Model{ .backing = testing.allocator }; + defer model.deinitAll(); + + // One event past the cap: it must still parse and correlate, but detection + // is skipped (its O(n^2) rules would stall) — the trace stays inspectable. + const count = detection.max_events_for_detection + 1; + var buf: std.ArrayList(u8) = .empty; + defer buf.deinit(testing.allocator); + var i: usize = 0; + while (i < count) : (i += 1) { + try buf.appendSlice(testing.allocator, "{\"message\":[2,\"m\",\"Heartbeat\",{}]}\n"); + } + + model.openBytes("big.jsonl", buf.items); + const t = model.activeTrace().?; + try testing.expect(!t.isError()); + try testing.expectEqual(count, t.eventCount()); + try testing.expect(t.detection_skipped); + try testing.expectEqual(@as(usize, 0), t.failureCount()); +} + +test "the sample trace stays under the cap and runs detection" { + var model = Model{ .backing = testing.allocator }; + defer model.deinitAll(); + update(&model, .open_sample); + // 22 events, well under the cap: detection ran (and found nothing, as it's a + // clean session). + try testing.expect(!model.activeTrace().?.detection_skipped); +} + test "the workspace is bounded at max_open_traces" { var model = Model{ .backing = testing.allocator }; defer model.deinitAll();