diff --git a/.gitignore b/.gitignore
index 0dfc197..a8cb890 100644
--- a/.gitignore
+++ b/.gitignore
@@ -29,6 +29,7 @@ profile.cov
/*.log
/data/
+data/demo-state/
*.md
# Local runtime/build artifacts
diff --git a/Makefile b/Makefile
index e47aeb7..46f0d90 100644
--- a/Makefile
+++ b/Makefile
@@ -1,6 +1,6 @@
SHELL := /bin/sh
-.PHONY: help build build-examples ingest ingest-mcp waylog waylog-live checkout test test-race test-sdk lint ci fmt vet vet-sdk clean kafka-up kafka-down demo demo-stop demo-acceptance demo-up demo-down micro-demo micro-demo-stop docker-build docker-up docker-down docker-reset docker-dev docker-prod ts-install ts-build ts-test bench-gate
+.PHONY: help build build-examples ingest ingest-mcp waylog waylog-live checkout test test-race test-sdk lint ci fmt vet vet-sdk clean kafka-up kafka-down demo demo-stop demo-acceptance proof-loop rca-scorecard rollup-comparison otlp-conformance demo-up demo-down micro-demo micro-demo-stop docker-build docker-up docker-down docker-reset docker-dev docker-prod ts-install ts-build ts-test bench-gate
help:
@echo "Targets:"
@@ -19,6 +19,10 @@ help:
@echo " demo - start dashboard demo locally (detached, no Docker)"
@echo " demo-stop - stop demo processes"
@echo " demo-acceptance - verify a running local demo end-to-end"
+ @echo " proof-loop - run alert -> incident -> triage -> report -> rollup proof"
+ @echo " rca-scorecard - run deterministic RCA scorecard over the demo scenario"
+ @echo " rollup-comparison - run demo proof for root-cause vs naive rollup counts"
+ @echo " otlp-conformance - run deterministic OTLP HTTP/gRPC fixture checks"
@echo " demo-up - start v2 demo stack in Docker (detached)"
@echo " demo-down - stop Docker demo stack"
@echo " micro-demo - start 4-service micro-demo in foreground for debugging"
@@ -82,7 +86,7 @@ vet-sdk: ## Vet SDK modules
cd pkg && go vet ./...
cd pkg/transport/kafka && go vet ./...
-ci: fmt vet vet-sdk test-race test-sdk ts-test check-doc-links check-rollup-contract
+ci: fmt vet vet-sdk test-race test-sdk ts-test check-doc-links check-rollup-contract otlp-conformance
@echo "CI checks passed"
ts-install: ## Install TS SDK deps (skipped if node_modules is already present)
@@ -123,6 +127,18 @@ demo-stop:
demo-acceptance:
./scripts/demo-acceptance.sh
+proof-loop:
+ bash ./scripts/proof-loop.sh
+
+rca-scorecard:
+ bash ./scripts/rca-scorecard.sh
+
+rollup-comparison:
+ ./scripts/rollup-comparison.sh
+
+otlp-conformance:
+ ./scripts/otlp-conformance.sh
+
demo-up: docker-dev
demo-down: docker-down
diff --git a/README.md b/README.md
index 439170b..7f6d92f 100644
--- a/README.md
+++ b/README.md
@@ -1,24 +1,47 @@
-
-
+
+

-
- Structured logging that explains failure propagation.
- Drop-in SDKs (Go, TypeScript) or OTLP/HTTP. Agent-native by design.
-
+
+ Production incident triage you can hash, cite, and hand to an agent.
+ Drop-in SDKs (Go, TypeScript) or OTLP HTTP/gRPC. Deterministic. Single Go binary.
+
-
- polyglot SDKs · agent-native API · failure tree · rollup-correct root cause
-
+
+ schema 2.0 WideEvents · signal-correlated incidents · cited operator reports · rollup-correct root cause · agent-native
+
-
- Public alpha — an impact-analysis engine for backend systems built on WideEvents.
-
+
+ alpha · go 1.24+ · single-node · OTLP-compatible · MCP-native · LLM-optional
+
+
+
+> **Public alpha for single-node production-style incident triage.** APIs may break before 1.0.
+
+---
+
+## Try it in 60 seconds
+
+```bash
+git clone https://github.com/sssmaran/WaylogCLI
+cd WaylogCLI
+make demo
+```
+
+Then open and click **Run proof loop**.
+
+You'll see the full v2.1 flow run end-to-end: an external alert is accepted, a payment-failure burst spikes an error family, the spike detector opens an incident, the cause is classified as `dependency`, a triage report is built across four surfaces (CLI, read endpoint, direct tool, plan template) and verified byte-stable, and Markdown / Slack / PagerDuty operator reports are rendered with citations to every alert, signal, and trace.
+
+Stop with `make demo-stop`. No Docker. No Kafka. No bridge process. SQLite + a single Go binary.
---
-## What Waylog does
+## What Waylog is
-A request hits your API gateway, fans out to three services, and one of them fails. The gateway returns 502. Your logs say "upstream error." Waylog tells you exactly what happened:
+Waylog turns failed requests and external alerts into **incidents with deterministic, cited triage reports** that humans and agents can both consume. Three things make it different:
+
+- **Deterministic.** Every triage report has a `report_hash` that is byte-stable across CLI, REST read, direct tool call, and plan template — within a single engine tick. No LLM required to get an answer; LLMs are an optional UX layer over the same bytes.
+- **Agent-native.** The same tool registry powers the CLI, MCP stdio, REST `/v1/tools/*`, and `/v1/plans/execute`. Agents read the exact bytes the human read. Built-in triage plan template, idempotency keys, structured envelopes.
+- **Drop-in.** Go SDK (`net/http`, chi, gin, echo), TypeScript SDK (Express, Hono, Next.js, NestJS), or OTLP HTTP / gRPC. Use what you already have. Single binary + embedded SQLite. No Docker required.
```text
trace 7f3a2b9c… flow=purchase user=standard region=us-east-1
@@ -28,52 +51,44 @@ A request hits your API gateway, fans out to three services, and one of them fai
└─ db 200 — 3 ms
└─ payment 502 PMT_502 5 ms ← first failure
- blast radius: 12 requests · 8 users · 4 services
+ blast radius: 12 requests · 8 users · 4 services
+ incident: inc_a43c189fc63eff31 (cause=dependency · confidence=high)
+ report_hash: sha256:1ed7c21b… (stable across cli / read / tool / plan)
```
-This is not log search. Waylog builds a live in-memory graph from every request flowing through your services. When you ask a question — "why did this trace fail?", "who is affected by `PMT_502`?", "what changed in the last 10 minutes?" — it walks the graph and returns a precomputed, structured answer. Root-cause rollups count the originating failure once, not once per propagated hop.
-
-Run `make demo` and see it yourself.
-
-## Quick start
+Root-cause rollups count the originating failure **once**, not once per propagated hop. The same hash answers "what failed" identically whether the question came from a terminal, a webhook, or an LLM tool call.
-```bash
-make demo
-```
-
-This starts the ingest server plus four real Go demo services wired through the schema-2.0 Go SDK (`api-gateway → checkout → db/payment`), enables `WAYLOG_V2_READS=true`, and does not require Docker, Kafka, or the bridge process.
+---
-Once the stack is up:
+## The incident loop
-1. Open demo controls at , or open the dashboard at . The local demo disables dashboard login.
-2. Click **Run traffic burst** to fire a production-like mix through the checkout chain. For a focused single-trace look, click **Run payment outage** instead, or run:
- ```bash
- curl -s -X POST http://localhost:9081/purchase \
- -H 'Content-Type: application/json' \
- --data '{"sku":"X1","scenario":"payment_502"}'
- ```
-3. Investigate with the v2 CLI:
- ```bash
- ./waylog errors --window 15m
- ./waylog explain
- ./waylog blast --service checkout --step payment.charge --code PMT_502 --window 15m
- ./waylog blast --code PMT_502 --window 15m
- ```
+The v2.1 product in one paragraph: services emit WideEvents (or push OTel spans). External systems post production-context signals (deploys, dependency health, runtime events) and alerts (Alertmanager, Grafana, PagerDuty webhooks, or Waylog-native JSON). When an error family spikes, Waylog opens an **incident**, correlates the signals and alerts that overlap its window, classifies the cause deterministically (`deploy | app | dependency | runtime | unknown`), and exposes a cited **TriageReport** to humans (CLI, dashboard) and agents (REST, plan template, MCP) — same bytes, same hash.
-The demo also supports `happy` and `suppressed_payment_502` scenarios through the UI or `POST /purchase`.
+```bash
+# 1. Drive a real failure through the demo stack
+curl -X POST http://localhost:9081/purchase \
+ -H 'Content-Type: application/json' \
+ -d '{"sku":"X1","scenario":"payment_502"}'
-Stop with `make demo-stop`.
+# 2. List active incidents
+waylog incidents
-Prefer Docker? Use `make docker-dev` / `make docker-down`. Prefer foreground service logs while hacking on Go code? Use `make micro-demo` and stop with `make micro-demo-stop`.
+# 3. Get the cited triage report — and verify hash agreement across surfaces
+waylog triage inc_a43c189fc63eff31 --snapshot
+curl -H "Authorization: Bearer $WAYLOG_AGENT_KEY" \
+ -d '{"incident_id":"inc_a43c189fc63eff31","snapshot":true}' \
+ http://localhost:8080/v1/tools/triage_incident
+# 4. Render an operator report (Markdown / Slack Block Kit / PagerDuty note)
+curl -H "Authorization: Bearer $WAYLOG_READ_KEY" \
+ "http://localhost:8080/v1/triage/inc_a43c189fc63eff31/report?format=slack"
+```
-## How it works
+> **Alerts correlate; they do not create incidents.** Incidents are opened by the spike detector. Alerts and signals attach as evidence and shape the deterministic cause classification.
-1. **Capture** — services emit [WideEvents](docs/waylog-sdk-contract.md) via the Go or TypeScript SDK, or push OpenTelemetry spans to `/v1/otlp/v1/traces`. Every event is durably logged (WAL + fsync) before it enters the derived read models.
-2. **Analyze** — the ingest server projects completed execution segments into request, service, error, user, and trace views. Deterministic tools answer specific questions: propagation chain, blast radius, what-changed, deploy correlation.
-3. **Operator** — CLI, REST, MCP, TUI, and the embedded dashboard query the same derived views through the same tool registry. Every answer is also callable by agents as a structured tool with idempotency keys.
+---
-## Get traces in
+## Capture: send Waylog your traces
All three paths feed the same schema-2.0 ingest and read APIs. Pick whichever matches your stack.
@@ -101,7 +116,7 @@ app.post("/buy", (req, res) => {
});
```
-`@waylog/sdk` is ESM-only, Node 18+, and ships standalone core APIs plus Express, Hono, Next.js, and NestJS entrypoints (`@waylog/sdk/express`, `@waylog/sdk/hono`, `@waylog/sdk/next`, `@waylog/sdk/nest`).
+ESM-only, Node 18+. Standalone core plus framework entrypoints: `@waylog/sdk/express`, `@waylog/sdk/hono`, `@waylog/sdk/next`, `@waylog/sdk/nest`. Full examples in [`docs/sdk-examples.md`](docs/sdk-examples.md).
### Go SDK
@@ -127,192 +142,313 @@ func main() {
}
```
-The recommended SDK path is framework middleware plus `waylog.From(ctx)` / `useLogger(...)` inside handlers. Low-level request APIs such as `Begin`, `Finalize`, and `setField` are for adapter authors, tests, and unusual custom integrations. Full copy-paste examples for `net/http`, chi, gin, echo, standalone TypeScript, Express, Hono, Next.js, and NestJS are in [`docs/sdk-examples.md`](docs/sdk-examples.md).
+Middleware adapters for `net/http`, chi, gin, and echo are in [`docs/sdk-examples.md`](docs/sdk-examples.md). The recommended path is framework middleware plus `waylog.From(ctx)` / `useLogger(...)` inside handlers — low-level `Begin` / `Finalize` / `setField` APIs are for adapter authors.
+
+### OTLP / OpenTelemetry
+
+Point your existing OTel collector at Waylog. Both protocols, same conversion path, same downstream views.
-### OTLP/HTTP traces
+```yaml
+exporters:
+ otlphttp/waylog:
+ endpoint: http://localhost:8080/v1/otlp/v1/traces
+ headers:
+ authorization: "Bearer ${WAYLOG_WRITE_KEY}"
+ otlp/waylog:
+ endpoint: localhost:4317
+ headers:
+ authorization: "Bearer ${WAYLOG_WRITE_KEY}"
+```
+
+Sample collector config: [`examples/otel-collector/`](examples/otel-collector/). Only traces are accepted; OTLP logs and metrics are not shipping. Bind `OTLP_GRPC_ADDR=127.0.0.1:4317` for single-host installs that don't need cross-host collectors.
-Point your existing OpenTelemetry collector at `http://localhost:8080/v1/otlp/v1/traces`. Protobuf bodies are accepted (gzip optional) and HTTP spans convert to schema-2.0 WideEvents on the way in, then show up in the same errors, explain, blast, and recent-trace APIs as SDK events when `WAYLOG_V2_READS=true`. **Phase A covers traces over HTTP.** gRPC, logs, and metrics are not yet shipping.
+**Auth.** Both endpoints require `WAYLOG_WRITE_KEY` when `WAYLOG_PROFILE=prod`; the server refuses to boot with unauthenticated OTLP in prod. `make demo` runs unauthenticated by design.
-### Alternative: local ingest server (no Docker)
+### Local ingest only (no demo services)
```bash
make ingest
```
-Runs only the ingest server. Point your own services at it via an SDK or OTLP. Full env-var reference: [`docs/env.md`](docs/env.md).
+Runs only the ingest server. Point your own services at it via SDK or OTLP. Full env reference: [`docs/env.md`](docs/env.md).
-## What you can ask
+---
+
+## Operate: CLI, dashboard, agents
### CLI
```bash
-WAYLOG_V2_READS=true ./ingest
-
-waylog capabilities
-waylog recent --limit 5
-waylog errors --window 15m
-waylog blast checkout:payment.charge:PMT_502 --window 15m
-waylog explain trace_01HX...
-waylog trace trace_01HX...
-waylog event event_01HX...
-waylog search PMT_502 --window 1h
+./ingest # v2 reads are on by default
+
+waylog capabilities # diagnose server config / profile / feature flags
+waylog incidents # active incidents, deterministic order
+waylog incident --snapshot # full detail with frozen sample traces
+waylog triage # cited triage report
+waylog errors --window 15m # top error families
+waylog explain # first observable failing step
+waylog blast checkout:payment.charge:PMT_502 --window 15m
+waylog recent --limit 5
+waylog event
+waylog trace
+waylog search PMT_502 --window 1h
```
-The `waylog` binary is now the v2 operator CLI over the running ingest server's read APIs. Most verbs require the server to advertise `v2_reads.enabled=true` from `/v1/capabilities`; `waylog capabilities` is intentionally ungated so it can diagnose server setup. The CLI uses `INGEST_ADDR`, `WAYLOG_READ_KEY`, and `WAYLOG_CLI_TIMEOUT` by default. Add `--json` to any verb for machine-readable output.
+`waylog capabilities` is intentionally ungated so it can diagnose server setup; other verbs require `v2_reads.enabled=true` (the default). Defaults: `INGEST_ADDR`, `WAYLOG_READ_KEY`, `WAYLOG_CLI_TIMEOUT`. Add `--json` to any verb for machine-readable output.
+
+### Dashboard
-### REST (direct tool call)
+Embedded Geist UI at . Uses the dashboard session cookie for read-scope auth and runs against the default `WAYLOG_V2_READS=true` reader.
+
+- `#/errors` — top error families over `/v1/errors`
+- `#/incident/` — incident evidence and next checks over `/v1/incidents/{id}`
+- `#/explain/` — first observable failing step over `/v1/traces/story`
+- `#/blast/` — impact panel over `/v1/blast_radius`
+- recent-request stream from `/v1/traces/recent`, polled every 5 s
+
+No Chart.js, Cytoscape, topology-first UI, Ask panel, deploy diff, or large dashboard charts. Just the triage loop.
+
+### Agent surface
+
+Twelve deterministic tools, exposed identically through CLI, REST `/v1/tools/{name}`, MCP stdio, and plan execution. Same idempotency keys, same structured envelopes, same bytes.
+
+| Tool | Answers |
+| ---------------------- | -------------------------------------------------------------------------------------------- |
+| `triage_incident` | Structured TriageReport for an open incident (blast + first failure + signals + next checks) |
+| `render_triage_report` | Markdown, Slack Block Kit JSON, or PagerDuty note from a TriageReport |
+| `explain_request` | Why did this specific trace fail? |
+| `trace_summary` | Span tree and timing for a trace |
+| `graph_failures` | Which requests are currently failing? |
+| `failure_patterns` | What error codes dominate this window? |
+| `blast_radius` | How many requests, users, and services does this error touch? |
+| `failure_chain` | How did this failure propagate through services? |
+| `graph_query` | DSL query over the graph (`expr` + `window`) |
+| `compare_windows` | Diff error rates between two windows |
+| `graph_insights` | Windowed rollup of top errors and patterns |
+| `graph_stats` | Overall shape of the graph right now |
```bash
+# Direct tool call
curl -X POST http://localhost:8080/v1/tools/blast_radius \
- -H 'Content-Type: application/json' \
-H "Authorization: Bearer $WAYLOG_AGENT_KEY" \
-d '{"error_code":"PMT_502","window":"10m","include_services":true}'
-```
-
-### REST (multi-step plan)
-```bash
+# Built-in triage plan template — same hash as the CLI/read/tool surfaces
curl -X POST http://localhost:8080/v1/plans/execute \
- -H 'Content-Type: application/json' \
-H "Authorization: Bearer $WAYLOG_AGENT_KEY" \
- -d '{
- "steps": [
- {"id":"patterns", "tool":"failure_patterns", "params":{"window":"10m"}},
- {"id":"blast", "tool":"blast_radius", "params":{"error_code":"PMT_502","window":"10m"}}
- ]
- }'
+ -d '{"template":"triage","params":{"incident_id":"inc_...","snapshot":true}}'
```
-Plans execute deterministically server-side with SSE progress on `/v1/stream/plans/{id}`.
+Plans execute deterministically server-side with SSE progress on `/v1/stream/plans/{id}`. Full schemas: `GET /v1/tools` or [`docs/openapi.yaml`](docs/openapi.yaml).
-### Trace story
+### MCP
```bash
-curl "http://localhost:8080/v1/traces/story?trace_id=$TRACE" \
- -H "Authorization: Bearer $WAYLOG_READ_KEY"
+make ingest-mcp # MCP_STDIO=1
```
-Returns the first failing step, contributing path, logs, downstream calls, and linkage mode used by the dashboard and `waylog explain`.
+Same registry, same idempotency keys. Plugs into Claude, Cursor, and other MCP clients.
-### MCP (agent surface)
+### External alerts
```bash
-make ingest-mcp # MCP_STDIO=1
+curl -X POST http://localhost:8080/v1/alerts \
+ -H "Authorization: Bearer $WAYLOG_WRITE_KEY" \
+ -d @grafana-webhook.json
```
-Exposes the same tool registry over MCP stdio for Claude, Cursor, and other MCP clients. Same semantics as the REST API.
+`POST /v1/alerts` accepts Waylog-normalized JSON plus Alertmanager, Grafana, and PagerDuty webhook payloads. Accepted alerts are stored as `type=alert` signals and **correlated** with active incidents when possible — alerts do not create incidents. The matching alert evidence then appears in cited Markdown / Slack / PagerDuty triage reports.
-### Analysis tools
+---
-All ten tools are deterministic, idempotent, and available via CLI, REST `/v1/tools/{name}`, MCP, and plan execution.
+## Architecture
-| Tool | Answers |
-| ------------------ | ------------------------------------------------------------- |
-| `graph_stats` | Overall shape of the graph right now |
-| `explain_request` | Why did this specific trace fail? |
-| `trace_summary` | Span tree and timing for a trace |
-| `graph_failures` | Which requests are currently failing? |
-| `failure_patterns` | What error codes dominate this window? |
-| `blast_radius` | How many requests, users, and services does this error touch? |
-| `failure_chain` | How did this failure propagate through services? |
-| `graph_query` | DSL query over the graph (`expr` + `window`) |
-| `compare_windows` | Diff error rates between two windows |
-| `graph_insights` | Windowed rollup of top errors and patterns |
+```text
+ Go / TS services (SDK) OTel collectors
+ │ │
+ schema-2.0 WideEvents OTLP HTTP / gRPC
+ ╰──────────────┬─────────────────╯
+ ▼
+ ingest server
+ ┌──────────────┴──────────────┐
+ │ │
+ event log (append-only WAL, SQLite cold store
+ source of truth) (events · deployments ·
+ │ signals · incidents ·
+ ▼ causal claims)
+ derived read models
+ (errors · explain · blast ·
+ recent · incidents · triage)
+ │
+ ├──▶ /ui dashboard (Geist, no vendored chart/topology)
+ ├──▶ /v1/tools/* (deterministic agent surface)
+ ├──▶ /v1/plans/execute (server-side plan execution + SSE)
+ └──▶ waylog CLI · TUI · MCP
+```
-Full schemas: `GET /v1/tools` or [`docs/openapi.yaml`](docs/openapi.yaml).
+- **Single binary** plus embedded SQLite. No Docker, no Kafka, no bridge.
+- **WAL is source of truth.** Crash → replay on next boot rebuilds the derived read models.
+- **Hot graph + dedicated trace store.** Pruned per snapshot tick to bound memory.
+- **`report_hash` excludes `generated_at`, `plan_run_id`, and itself.** Same upstream state → same bytes across every surface.
+- **OTLP path reuses the same WAL and projector** as the SDK path. No separate ingestion plane.
-## Dashboard
+Durability model, retention, merge semantics, readiness policy, and counter buffer details: [`docs/internals.md`](docs/internals.md). Full HTTP contract: [`docs/openapi.yaml`](docs/openapi.yaml).
-The embedded dashboard at `/ui` is a v2 triage surface over the same read APIs as the CLI. It requires `WAYLOG_V2_READS=true` and uses the dashboard session cookie for read-scope auth.
+---
-- dark, minimal Geist UI with aligned KPI modules and inline SVG mini-graphs
-- `#/errors` — top error families over `/v1/errors`
-- `#/explain/` — first observable failing step over `/v1/traces/story`
-- `#/blast/` — impact panel over `/v1/blast_radius`
-- recent-request stream from `/v1/traces/recent`, polled every 5s
-- no Chart.js, Cytoscape, topology-first UI, Ask panel, deploy diff, or large dashboard charts
+## Auth & profiles
+Three independent scoped keys. The dashboard never holds the agent key.
-## Architecture
+| Key | Protects |
+| ------------------ | ----------------------------------------------------------- |
+| `WAYLOG_WRITE_KEY` | `/v1/events`, OTLP HTTP + gRPC, `/v1/signals`, `/v1/alerts` |
+| `WAYLOG_READ_KEY` | Read APIs, dashboard session |
+| `WAYLOG_AGENT_KEY` | `/v1/tools/*`, `/v1/ask`, `/v1/plans/*` |
-```text
-Go / TS services (SDK) · OTLP/HTTP collectors
- │ schema-2.0 WideEvents · OTLP/HTTP traces
- ▼
- ingest server
- ├─ event log (append-only WAL, source of truth)
- ├─ derived read models (errors · explain · blast · recent traces)
- ├─ SQLite cold store (events · deployments · causal claims)
- ├─ tool registry · Ask · plan execution
- └─ v2 dashboard · health · metrics · OpenAPI
- │
- ├──▶ /ui dashboard (Geist, no vendored chart/topology libs)
- ├──▶ /v1/tools/* · /v1/plans/execute (agent-native)
- └──▶ CLI · TUI · MCP · agents
+`WAYLOG_API_KEY` is a legacy alias for the write scope. `ParseConfig` validates the auth matrix at startup and refuses to boot with an unsafe combination.
+
+`WAYLOG_PROFILE` controls auth strictness. The current profile is reported on `/v1/capabilities`.
+
+| Profile | Use case | Defaults |
+| ------- | --------------------------- | ----------------------------------------------------------------------------------------------------- |
+| `demo` | `make demo` showcase | All endpoints open. **Not safe to expose to a network.** |
+| `dev` | Local development (default) | Open OTLP, optional read auth |
+| `prod` | Real deployments | Refuses to boot without all three scoped keys **and** without `WAYLOG_WRITE_KEY` when OTLP is enabled |
+
+Set `WAYLOG_PROFILE=prod` for any deployment that crosses a trust boundary.
+
+---
+
+## Try every claim locally
+
+```bash
+make demo # one-shot local stack (no Docker)
+make demo-acceptance # 15-check gate over CLI + browser proof
+make proof-loop # full alert → incident → triage → cited reports loop
+make rca-scorecard # cold-start latency + measured report_hash_stable
+make rollup-comparison # rollup-correct counts vs naive propagated counts
+make demo-stop
```
-Events are durably logged before projection — if the process crashes, replay rebuilds the read models from the WAL on next boot.
+`make proof-loop` writes shareable artifacts to `./data/demo-state/proof/`:
+
+- `triage.json` — the TriageReport from the read endpoint
+- `report.md`, `slack.json`, `pagerduty.txt` — the same triage rendered for three operator surfaces
+- `rollup-comparison.txt` — root-cause counts next to naive propagated counts
+- `scorecard.json` — measured `report_hash_stable`, `triage_latency_ms`, `scenario` (`cold-demo` for `rca-scorecard`, `warm-demo` chained from `proof-loop`), inflation-avoided count, and the `report_hash` itself
+
+The browser has the same flow at → **Run proof loop**.
-Durability model, retention, merge semantics, readiness policy, and counter buffer: [`docs/internals.md`](docs/internals.md). Full v2 HTTP contract: [`docs/openapi.yaml`](docs/openapi.yaml).
+> Artifacts are local-only. Alert payloads include the demo write key (`Bearer demo`). `data/demo-state/` is gitignored.
+
+---
## Development
```bash
-make build # core binaries
-make build-examples # demo services
-make fmt vet test # checks
-make test-race # race detector
-make ts-test # TypeScript SDK vitest suite
-make ci # fmt + vet + test-race + test-sdk + ts-test + doc-link + rollup-contract
-make demo-acceptance # with make demo running, verify demo + CLI triage loop
+make build # core binaries
+make build-examples # demo services
+make fmt vet test # checks
+make test-race # race detector
+make ts-test # TypeScript SDK vitest suite
+make ci # fmt + vet + test-race + test-sdk + ts-test + doc-link + rollup-contract
```
-## Auth
+Full env-var reference: [`docs/env.md`](docs/env.md). Reproducible demo gate: `make demo-acceptance`.
+
+---
-Waylog uses three scoped keys. They are independent — the dashboard never holds the agent key.
+## What's new in v2.1
-| Key | Protects |
-| ------------------ | ----------------------------------------------------- |
-| `WAYLOG_WRITE_KEY` | `/v1/events`, `/v1/otlp/v1/traces` (SDKs, collectors) |
-| `WAYLOG_READ_KEY` | Read APIs, dashboard session |
-| `WAYLOG_AGENT_KEY` | `/v1/tools/*`, `/v1/ask`, `/v1/plans/*` |
+- **Signal-correlated incident engine** at `/v1/incidents/*` with deterministic cause classification (`deploy | app | dependency | runtime | unknown`).
+- **Deterministic TriageReport** (`triage.v1`) with stable per-tick `report_hash` across CLI, read endpoint, direct tool, and plan template.
+- **Alert intake** at `POST /v1/alerts` for Waylog-native, Alertmanager, Grafana, and PagerDuty webhooks. Alerts correlate with active incidents; they do not create incidents.
+- **Cited operator reports** rendered as Markdown, Slack Block Kit, or PagerDuty notes via `GET /v1/triage/{id}/report`.
+- **OTLP/gRPC trace receiver** on `OTLP_GRPC_ADDR` (default `:4317`).
+- **Provider-neutral Ask** configuration: `gemini`, `anthropic`, `openai`, or `none`. All deterministic surfaces work with no LLM configured.
+- **`WAYLOG_PROFILE=demo|dev|prod`** gates auth defaults; `prod` hard-fails on unsafe configs.
+- **`WAYLOG_V2_READS` defaults to `true`.** Set `false` only for legacy v1-only stacks.
+- **`/v1/insight`** retained as a compat shim returning the top active incident. New clients should use `/v1/incidents/*`.
-`WAYLOG_API_KEY` is a legacy alias for the write scope. `ParseConfig` validates the auth matrix at startup and refuses to boot with an unsafe combination.
+---
## Status
-Public alpha. APIs may break before 1.0.
+Public alpha for single-node production-style incident triage. APIs may break before 1.0.
-**Shipped:**
+**Shipped**
- Go SDK v2 (`net/http`, chi, gin, echo) and TypeScript SDK v2 (`@waylog/sdk`, ESM, Node 18+, standalone core, Express, Hono, Next.js, NestJS)
-- OTLP/HTTP traces at `/v1/otlp/v1/traces` (Phase A — traces only)
-- durable ingest with WAL + replay
-- hot graph with flattened 3-node model + dedicated trace store
-- schema-2.0 recent-index read APIs behind `WAYLOG_V2_READS=true`
-- SQLite cold store (events, deployments, causal claims)
-- 10 deterministic analysis tools, rollup-correct root-cause attribution
-- agent-native REST (`/v1/tools/*`, `/v1/ask`, `/v1/plans/execute`) with idempotency and structured envelopes
-- `/v1/traces/story` and indented failure-path rendering in the dashboard
-- dashboard: minimal v2 triage loop (errors, explain, blast, recent requests)
-- v2 operator CLI (`capabilities`, `recent`, `errors`, `event`, `trace`, `explain`, `blast`, `search`) over read APIs
-- live TUI (`waylog-live --dev` streams via SSE), MCP stdio
-- scoped auth (write/read/agent) with startup validation
-
-**Planned:**
-
-- OTLP gRPC, logs, and metrics (Phase B)
+- OTLP HTTP at `/v1/otlp/v1/traces` and OTLP/gRPC at `OTLP_GRPC_ADDR` (traces only)
+- Durable ingest with WAL + replay
+- Hot graph with flattened 3-node model + dedicated trace store
+- Schema-2.0 recent-index read APIs (default)
+- SQLite cold store (events, deployments, signals, incidents, causal claims)
+- Signal-correlated incident engine with stable IDs, deterministic classification, and startup hot-window rebuild from the schema-2.0 WAL
+- Alert intake from four webhook formats, stored as signals and correlated with active incidents
+- Deterministic triage report with stable hash across CLI / read endpoint / direct tool / plan template within a single engine tick
+- Provider-neutral Ask configuration; deterministic CLI, tools, plans, triage, and MCP work with no LLM configured
+- Twelve deterministic analysis tools, rollup-correct root-cause attribution
+- Agent-native REST with idempotency and structured envelopes
+- MCP stdio, live TUI (`waylog-live --dev` streams via SSE), embedded Geist dashboard
+- Scoped auth (write / read / agent) with startup validation and `WAYLOG_PROFILE=prod` hard-fail
+
+**Planned**
+
+- OTLP logs and metrics
- Python SDK
+- Resolved-incident retention janitor
- Mintlify docs site
+---
+
## Known limitations
-- Single-node only. No HA, no clustering.
-- Alpha quality. APIs may break before 1.0.
-- OTLP is HTTP/traces only. gRPC, logs, and metrics are not shipping yet.
-- Only Go and TypeScript SDKs today. Python / Java / Ruby are not available.
-- SQLite cold store fits demos and small deployments; not sized for production-scale retention.
-- No built-in alerting or paging. Waylog answers questions, it doesn't wake you up.
-- No multi-tenancy. One instance = one trust boundary.
+- **Public alpha.** APIs may break before 1.0. Not production-ready. Not HA.
+- **Triage report hash is stable per tick, not forever.** Hash changes when the underlying recent-index window changes (≈30 s default). Use as a short-window dedup key, not a long-term incident fingerprint.
+- **Alerts correlate; they do not create incidents.** Incidents are opened by the spike detector. The alert path is for routing context, not paging primitives.
+- **Resolved incidents are not pruned automatically.** Per the v2.1 plan, the retention janitor is deferred. Manual cleanup:
+ ```sql
+ DELETE FROM incidents WHERE status = 'resolved' AND resolved_at < datetime('now', '-7 days');
+ ```
+- **Stale `active` rows after long downtime.** If the WAL has rolled past an incident's `started_at` and `WAYLOG_REBUILD_INCIDENTS_ON_START=true`, the engine transitions only the stale rows to `recovering` on next start; they resolve after `WAYLOG_INCIDENT_RESOLVE_AFTER` without new evidence.
+- **Single-node only.** No HA, no clustering, no multi-tenant.
+- **SQLite cold store** fits demos and small deployments. Postgres is not shipping.
+- **OTLP supports traces only.** Logs and metrics are not shipping yet.
+- **Only Go and TypeScript SDKs today.** Python / Java / Ruby are not available.
+- **No outbound paging.** Waylog accepts external alerts and renders operator reports; it does not page.
+- **No multi-tenancy.** One instance = one trust boundary.
+- **No full log search, Slack/PagerDuty automation, RBAC/SSO, or automatic remediation.**
+- **Incident cause classification is heuristic and deterministic.** Intentionally explainable, not ML-based.
+
+---
+
+## Documentation
+
+| File | What's in it |
+| ------------------------------------------------------------ | ---------------------------------------------------------------------------------------- |
+| [`docs/env.md`](docs/env.md) | Full env-var reference (auth, profiles, retention, OTLP, incident engine, LLM providers) |
+| [`docs/sdk-examples.md`](docs/sdk-examples.md) | Copy-paste SDK examples for every supported framework |
+| [`docs/waylog-sdk-contract.md`](docs/waylog-sdk-contract.md) | WideEvent schema and validation rules |
+| [`docs/openapi.yaml`](docs/openapi.yaml) | Full HTTP contract |
+| [`docs/internals.md`](docs/internals.md) | Durability model, retention, merge semantics, readiness policy, counter buffer |
+
+---
+
+## Project layout
+
+```
+cmd/ executable binaries (ingest, waylog, waylog-live, ...)
+pkg/ public SDK importable by external services
+internal/ private implementation (auth, incidents, triage, ingest, ...)
+examples/ demo services + collector config + microdemo
+scripts/ demo + proof + ci helpers
+docs/ reference and contracts
+```
+
+---
+
+## License
-**Fastest walkthrough:** `make demo`, open , click **Run traffic burst**, then use the dashboard or `waylog recent`, `waylog errors`, `waylog explain`, and `waylog blast` to answer what failed, which downstream was involved, and how broad the impact is.
+Not yet declared. The project is in public alpha; a license will be added before tagging 1.0. Until then, contact the maintainers if you'd like to use Waylog in a context where licensing matters to you.
diff --git a/cmd/ingest/main.go b/cmd/ingest/main.go
index 52c700e..e9c9ea1 100644
--- a/cmd/ingest/main.go
+++ b/cmd/ingest/main.go
@@ -6,6 +6,7 @@ import (
"errors"
"fmt"
"log/slog"
+ "net"
"net/http"
"os"
"os/signal"
@@ -15,6 +16,7 @@ import (
"time"
"github.com/prometheus/client_golang/prometheus"
+ "github.com/sssmaran/WaylogCLI/internal/alerts"
"github.com/sssmaran/WaylogCLI/internal/auth"
"github.com/sssmaran/WaylogCLI/internal/cli"
"github.com/sssmaran/WaylogCLI/internal/coldstore"
@@ -26,14 +28,23 @@ import (
"github.com/sssmaran/WaylogCLI/internal/graph/causal"
"github.com/sssmaran/WaylogCLI/internal/graph/core"
graphstore "github.com/sssmaran/WaylogCLI/internal/graph/store"
+ "github.com/sssmaran/WaylogCLI/internal/incidents"
"github.com/sssmaran/WaylogCLI/internal/ingest"
ingestv2 "github.com/sssmaran/WaylogCLI/internal/ingest/v2"
+ "github.com/sssmaran/WaylogCLI/internal/llm"
"github.com/sssmaran/WaylogCLI/internal/mcp/stdio"
"github.com/sssmaran/WaylogCLI/internal/metrics"
otelhttp "github.com/sssmaran/WaylogCLI/internal/otel"
"github.com/sssmaran/WaylogCLI/internal/persist"
+ "github.com/sssmaran/WaylogCLI/internal/signals"
"github.com/sssmaran/WaylogCLI/internal/tools"
"github.com/sssmaran/WaylogCLI/internal/tracestore"
+ "github.com/sssmaran/WaylogCLI/internal/triage"
+ "github.com/sssmaran/WaylogCLI/internal/triagehttp"
+ apiv2 "github.com/sssmaran/WaylogCLI/pkg/api/v2"
+ eventv2 "github.com/sssmaran/WaylogCLI/pkg/event/v2"
+ coltracepb "go.opentelemetry.io/proto/otlp/collector/trace/v1"
+ "google.golang.org/grpc"
)
var graphStore *graphstore.Store
@@ -104,7 +115,7 @@ func main() {
var sm *auth.SessionManager
if authCfg.DashboardMode != "off" {
sm = auth.NewSessionManager(authCfg.SessionSecret, auth.DefaultSessionMaxAge)
- sm.Secure = os.Getenv("WAYLOG_PROFILE") == "prod"
+ sm.Secure = authCfg.Profile == auth.ProfileProd
}
sessionCheck := auth.SessionCheckFunc(sm)
@@ -123,12 +134,43 @@ func main() {
grafanaURL := config.Getenv("GRAFANA_URL", "")
graphUI := config.GetenvBool("GRAPH_UI", false)
otlpEnabled := config.GetenvBool("OTLP_ENABLED", true)
- v2ReadsEnabled := config.GetenvBool("WAYLOG_V2_READS", false)
+ otlpGRPCAddr := config.Getenv("OTLP_GRPC_ADDR", ":4317")
+ if authCfg.Profile == auth.ProfileProd && otlpEnabled && len(authCfg.WriteKeys) == 0 {
+ slog.Error("WAYLOG_PROFILE=prod with OTLP enabled requires WAYLOG_WRITE_KEY — refusing to boot with unauthenticated OTLP")
+ os.Exit(1)
+ }
+ v2ReadsEnabled := config.GetenvBool("WAYLOG_V2_READS", true)
+ signalRetention := config.GetenvDuration("WAYLOG_SIGNAL_RETENTION", 72*time.Hour)
+ alertMatchWindow := config.GetenvDuration("ALERT_MATCH_WINDOW", 15*time.Minute)
+ if alertMatchWindow <= 0 {
+ alertMatchWindow = 15 * time.Minute
+ }
+ if alertMatchWindow > 24*time.Hour {
+ alertMatchWindow = 24 * time.Hour
+ }
+ incidentsEnabled := config.GetenvBool("WAYLOG_INCIDENTS_ENABLED", true)
+ incidentCfg := incidents.Config{
+ TickInterval: config.GetenvDuration("WAYLOG_INCIDENT_TICK_INTERVAL", 30*time.Second),
+ Window: config.GetenvDuration("WAYLOG_INCIDENT_WINDOW", 10*time.Minute),
+ MinCount: config.GetenvInt("WAYLOG_INCIDENT_MIN_COUNT", 5),
+ MinLift: config.GetenvFloat("WAYLOG_INCIDENT_MIN_LIFT", 3.0),
+ ResolveAfter: config.GetenvDuration("WAYLOG_INCIDENT_RESOLVE_AFTER", 2*time.Minute),
+ DeployCorrelationWindow: config.GetenvDuration("WAYLOG_DEPLOY_CORRELATION_WINDOW", 15*time.Minute),
+ SampleLimit: config.GetenvInt("WAYLOG_INCIDENT_SAMPLE_LIMIT", 5),
+ }
+ if signalRetention <= 0 {
+ slog.Error("WAYLOG_SIGNAL_RETENTION must be positive", "value", signalRetention)
+ os.Exit(1)
+ }
causalEnabled := config.GetenvBool("CAUSAL_ENABLED", false)
causalInterval := config.GetenvDuration("CAUSAL_INTERVAL", 30*time.Second)
trustProxy := config.GetenvBool("WAYLOG_TRUST_PROXY", false)
+ if _, err := llm.SelectFromEnv(); err != nil {
+ slog.Error("LLM provider config error", "err", err)
+ os.Exit(1)
+ }
dedupCache := ingest.NewDedupCache()
planStore := ingest.NewPlanStore()
@@ -188,6 +230,7 @@ func main() {
// Optional SQLite cold store
var coldDB coldstore.ManagedStore
var coldWriter *coldstore.BatchWriter
+ var signalStore signals.Store = signals.UnavailableStore{}
if sqlitePath != "" {
if eventLogDir == "" {
slog.Warn("SQLITE_PATH set without EVENT_LOG_DIR — cold store is async-only, " +
@@ -207,34 +250,41 @@ func main() {
FlushInterval: config.GetenvDuration("SQLITE_FLUSH_INTERVAL", 500*time.Millisecond),
}, m)
coldWriter.Start()
+ signalStore = coldstore.NewSignalStore(coldDB.(*coldstore.SQLiteStore))
slog.Info("coldstore enabled", "path", sqlitePath)
}
// Create ingest server with the store
ingestServer := ingest.NewServer(ingest.ServerConfig{
- Store: graphStore,
- TraceStore: traceStore,
- MaxBodyBytes: maxBody,
- EventLogDir: eventLogDir,
- Metrics: m,
- StartTime: time.Now(),
- AskRegistry: reg,
- AskMaxStepsDefault: askMaxStepsDefault,
- AskMaxStepsMax: askMaxStepsMax,
- DashboardRefreshSec: dashboardRefreshSec,
- PrometheusURL: prometheusURL,
- GrafanaURL: grafanaURL,
- GraphUI: graphUI,
- DedupCache: dedupCache,
- AgentKey: agentKey,
- TrustProxy: trustProxy,
- ColdWriter: coldWriter,
- ColdStore: coldDB,
- PlanStore: planStore,
- GraphHotWindow: graphHotWindow,
- OTLPEnabled: otlpEnabled,
- V2ReadsEnabled: v2ReadsEnabled,
+ Store: graphStore,
+ TraceStore: traceStore,
+ MaxBodyBytes: maxBody,
+ EventLogDir: eventLogDir,
+ Metrics: m,
+ StartTime: time.Now(),
+ AskRegistry: reg,
+ AskMaxStepsDefault: askMaxStepsDefault,
+ AskMaxStepsMax: askMaxStepsMax,
+ DashboardRefreshSec: dashboardRefreshSec,
+ PrometheusURL: prometheusURL,
+ GrafanaURL: grafanaURL,
+ GraphUI: graphUI,
+ DedupCache: dedupCache,
+ AgentKey: agentKey,
+ TrustProxy: trustProxy,
+ ColdWriter: coldWriter,
+ ColdStore: coldDB,
+ PlanStore: planStore,
+ GraphHotWindow: graphHotWindow,
+ OTLPEnabled: otlpEnabled,
+ OTLPGRPCEnabled: otlpEnabled && otlpGRPCAddr != "",
+ OTLPGRPCAddr: otlpGRPCAddr,
+ V2ReadsEnabled: v2ReadsEnabled,
+ IncidentsEnabled: v2ReadsEnabled && incidentsEnabled && sqlitePath != "",
+ IncidentsPersistent: v2ReadsEnabled && incidentsEnabled && sqlitePath != "",
+ IncidentRebuildSupported: v2ReadsEnabled && incidentsEnabled && sqlitePath != "",
+ Profile: authCfg.Profile,
})
// SSE hub for real-time dashboard updates
@@ -355,12 +405,23 @@ func main() {
}
mux.Handle("/v1/events", writeAuth(http.HandlerFunc(eventsV2.Events)))
mux.Handle("/v1/events/validate", writeAuth(http.HandlerFunc(eventsV2.Validate)))
+ signalHandler := signals.NewHandler(signalStore, m)
+ mux.Handle("/v1/signals", writeAuth(http.HandlerFunc(signalHandler.Signals)))
// OTLP/HTTP traces reuse the same schema-2.0 WAL and projector as the SDK path.
+ var otlpGRPCServer *grpc.Server
if otlpEnabled {
otlpHandler := otelhttp.NewHandler(eventsV2, m, maxBody)
mux.Handle("/v1/otlp/v1/traces", writeAuth(http.HandlerFunc(otlpHandler.ServeHTTP)))
slog.Info("otlp enabled", "endpoint", "/v1/otlp/v1/traces")
+ if otlpGRPCAddr != "" {
+ otlpGRPCServer = grpc.NewServer(
+ grpc.UnaryInterceptor(otelhttp.AuthUnaryInterceptor(authCfg.WriteKeys)),
+ grpc.MaxRecvMsgSize(int(maxBody)),
+ )
+ coltracepb.RegisterTraceServiceServer(otlpGRPCServer, otelhttp.NewTraceServiceServer(eventsV2, m, maxBody))
+ ingestServer.SetOTLPGRPC(true, otlpGRPCAddr)
+ }
}
// Read endpoints — CORS outermost so OPTIONS preflight passes without auth.
@@ -370,8 +431,11 @@ func main() {
func(w http.ResponseWriter, r *http.Request) { inner.ServeHTTP(w, r) }))
}
mux.Handle("/v1/overview", readCORS(ingestServer.Overview))
+ var v2Reader *ingestv2.Reader
+ var incidentEngine *incidents.Engine
+ incidentRunning := false
if v2ReadsEnabled {
- v2Reader := ingestv2.NewReader(v2Index)
+ v2Reader = ingestv2.NewReader(v2Index)
v2ReadHandler := ingestv2.NewReadHandler(v2Reader, m, graphHotWindow)
mux.Handle("/v1/events/search", readCORS(v2ReadHandler.EventSearch))
mux.Handle("/v1/errors", readCORS(v2ReadHandler.Errors))
@@ -383,6 +447,164 @@ func main() {
mux.Handle("/v1/events/", readCORS(v2ReadHandler.EventByID))
mux.Handle("/v1/traces/", readCORS(v2ReadHandler.TraceByID))
slog.Info("v2 read endpoints enabled")
+ if incidentsEnabled {
+ if sqlite, ok := coldDB.(*coldstore.SQLiteStore); ok {
+ incidentStore := coldstore.NewIncidentStore(sqlite)
+ incReader := incidentReaderAdapter{reader: v2Reader}
+ incidentEngine = incidents.NewEngine(
+ incReader,
+ signalStore,
+ coldDeployAdapter{store: sqlite},
+ incidentStore,
+ incidentCfg,
+ m,
+ slog.Default(),
+ )
+ if err := incidentEngine.Bootstrap(context.Background()); err != nil {
+ slog.Error("incident engine bootstrap failed", "err", err)
+ os.Exit(1)
+ }
+ if config.GetenvBool("WAYLOG_REBUILD_INCIDENTS_ON_START", false) {
+ rebuildMaxEvents := config.GetenvInt("WAYLOG_INCIDENT_REBUILD_MAX_EVENTS", 250000)
+ if rebuildMaxEvents <= 0 {
+ rebuildMaxEvents = 250000
+ }
+ replayWindow := graphHotWindow
+ if minWindow := 2 * incidentCfg.Window; minWindow > replayWindow {
+ replayWindow = minWindow
+ }
+ replaySince := time.Now().UTC().Add(-replayWindow)
+ seed := incidentEngine.SnapshotActive()
+ for _, inc := range seed {
+ if inc.StartedAt.Before(replaySince) {
+ slog.Info("incident continuity broken: started_at older than WAL retention",
+ "incident_id", inc.IncidentID,
+ "started_at", inc.StartedAt,
+ "replay_since", replaySince,
+ )
+ break
+ }
+ }
+ tempIndex := ingestv2.NewRecentIndex(nil)
+ tempDedup := ingestv2.NewDedup(dedupCapacity, nil)
+ tempProjector := ingestv2.NewProjector(tempIndex)
+ replay, err := ingestv2.ReplayWAL(eventLogV2Dir, tempDedup, tempProjector, replaySince, m)
+ if err != nil {
+ m.IncidentRebuildFailures.Inc()
+ slog.Error("incident rebuild WAL replay failed", "err", err)
+ os.Exit(1)
+ }
+ m.IncidentRebuildReplayed.Add(float64(replay.Projected))
+ if replay.Projected > rebuildMaxEvents {
+ m.IncidentRebuildFailures.Inc()
+ slog.Error("incident rebuild replay exceeded max events", "projected", replay.Projected, "max_events", rebuildMaxEvents)
+ os.Exit(1)
+ }
+ if replay.Projected == 0 {
+ // Empty WAL replay while rebuild was explicitly requested.
+ // Transition only the seed rows whose StartedAt precedes
+ // replaySince — those are stale beyond the hot window and
+ // their continuing "active" status is no longer evidence-
+ // backed. Non-stale active rows in the same seed are left
+ // untouched and will be re-evaluated by the next live tick.
+ staleTransitioned := 0
+ if len(seed) > 0 {
+ incidentStoreRef := incidentStore
+ now := time.Now().UTC()
+ for _, inc := range seed {
+ if inc.Status != incidents.StatusActive {
+ continue
+ }
+ if !inc.StartedAt.Before(replaySince) {
+ continue
+ }
+ row := inc
+ row.Status = incidents.StatusRecovering
+ t := now
+ row.RecoveringAt = &t
+ row.UpdatedAt = now
+ if err := incidentStoreRef.Upsert(context.Background(), row); err != nil {
+ slog.Warn("stale-active rebuild transition failed",
+ "incident_id", row.IncidentID, "err", err)
+ continue
+ }
+ staleTransitioned++
+ }
+ if staleTransitioned > 0 {
+ if err := incidentEngine.Bootstrap(context.Background()); err != nil {
+ slog.Error("incident engine re-bootstrap after stale transition failed", "err", err)
+ os.Exit(1)
+ }
+ slog.Info("incidents rebuild: stale active rows transitioned to recovering",
+ "transitioned", staleTransitioned,
+ "replay_since", replaySince)
+ } else {
+ slog.Warn("incidents rebuild skipped: WAL replay returned no events; preserving SQLite as-is")
+ }
+ }
+ } else {
+ result, err := incidents.Rebuild(context.Background(), incidents.RebuildDeps{
+ Engine: incidentEngine,
+ Reader: incidentReaderAdapter{reader: ingestv2.NewReader(tempIndex)},
+ Now: time.Now,
+ })
+ if err != nil {
+ m.IncidentRebuildFailures.Inc()
+ slog.Error("incident rebuild failed", "err", err)
+ os.Exit(1)
+ }
+ m.IncidentRebuildDuration.Observe(result.Duration.Seconds())
+ m.IncidentRebuildRows.Add(float64(result.RowsReplaced))
+ slog.Info("incident rebuild complete",
+ "replayed_events", replay.Projected,
+ "rows_replaced", result.RowsReplaced,
+ "duration", result.Duration,
+ )
+ }
+ }
+ incidentHandler := incidents.NewHandler(incidentEngine)
+ mux.Handle("/v1/incidents/active", readCORS(incidentHandler.Active))
+ mux.Handle("/v1/incidents/", readCORS(incidentHandler.Incident))
+ ingestServer.SetDetector(incidentInsightAdapter{engine: incidentEngine})
+
+ // Triage engine: deterministic TriageReport build for a given
+ // incident. Reuses the same v2Reader-backed adapter for blast
+ // queries, the live graph + trace store for first-failure
+ // stories, and the configured signal store. Read-scope auth.
+ triageEng, err := triage.NewEngine(triage.Deps{
+ Incidents: triage.NewIncidentLookupAdapter(incidentEngine),
+ Blast: triage.NewBlastQueryAdapter(incReader),
+ Story: triage.NewStoryBuilderAdapter(
+ incidentEngine,
+ func(traceID string) (apiv2.StoryResponse, bool) {
+ return v2Reader.TraceStoryByTraceID(traceID)
+ },
+ ),
+ Signals: triage.NewSignalQueryAdapter(signalStore),
+ Alerts: triage.NewAlertQueryAdapter(signalStore, alertMatchWindow),
+ NextChecks: triage.NewNextChecksAdapter(),
+ })
+ if err != nil {
+ slog.Error("triage engine init failed", "err", err)
+ os.Exit(1)
+ }
+ if err := tools.RegisterTriageTool(reg, triageEng); err != nil {
+ slog.Error("triage tool register failed", "err", err)
+ os.Exit(1)
+ }
+ if err := tools.RegisterTriageReportTool(reg, triageEng); err != nil {
+ slog.Error("triage report tool register failed", "err", err)
+ os.Exit(1)
+ }
+ triageHandler := triagehttp.NewHandler(triageEng)
+ mux.Handle("/v1/triage/", readCORS(triageHandler.Triage))
+
+ incidentRunning = true
+ slog.Info("incident engine enabled", "interval", incidentCfg.TickInterval, "window", incidentCfg.Window)
+ } else {
+ slog.Warn("incidents requested but SQLite not configured; running without incidents")
+ }
+ }
} else {
mux.Handle("/v1/traces/story", readCORS(ingestServer.TraceStory))
mux.Handle("/v1/blast_radius", readCORS(ingestServer.BlastRadius))
@@ -395,6 +617,8 @@ func main() {
mux.Handle("/v1/topology", readCORS(ingestServer.Topology))
mux.Handle("/v1/stream/dashboard", readCORS(ingestServer.SSEStream))
mux.Handle("/v1/insight", readCORS(ingestServer.Insight))
+ alertHandler := alerts.NewHandler(signalStore, incidentEngine, v2Reader, alertMatchWindow)
+ mux.Handle("/v1/alerts", writeAuth(http.HandlerFunc(alertHandler.Alerts)))
// Deployments — dual method: GET=read, POST=write.
mux.Handle("/v1/deployments", http.HandlerFunc(
@@ -453,6 +677,13 @@ func main() {
)
defer stop()
+ if _, ok := signalStore.(*coldstore.SignalStore); ok {
+ go signals.RunRetention(ctx, signalStore, signalRetention, 5*time.Minute, m, slog.Default())
+ }
+ if incidentRunning {
+ go incidentEngine.Run(ctx)
+ }
+
go func() {
slog.Info("ingest listening", "addr", addr, "graph_hot_window", graphHotWindow)
if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed {
@@ -461,6 +692,21 @@ func main() {
}
}()
+ if otlpGRPCServer != nil {
+ lis, err := net.Listen("tcp", otlpGRPCAddr)
+ if err != nil {
+ slog.Error("otlp grpc listen failed", "addr", otlpGRPCAddr, "err", err)
+ os.Exit(1)
+ }
+ go func() {
+ slog.Info("otlp grpc enabled", "addr", otlpGRPCAddr)
+ if err := otlpGRPCServer.Serve(lis); err != nil && !errors.Is(err, grpc.ErrServerStopped) {
+ slog.Error("otlp grpc server error", "err", err)
+ os.Exit(1)
+ }
+ }()
+ }
+
// ---------------- Embedded CLI ----------------
if mcpStdio {
@@ -595,7 +841,7 @@ func main() {
// ---------------- Anomaly detection ticker ----------------
detectCfg := detect.ParseConfig()
- if detectCfg.Enabled {
+ if detectCfg.Enabled && !incidentRunning {
var deploySrc detect.DeploySource
if coldDB != nil {
deploySrc = coldDB
@@ -603,6 +849,8 @@ func main() {
detector := detect.NewDetector(detectCfg, graphStore, traceStore, deploySrc)
ingestServer.SetDetector(detector)
go detector.Run(ctx)
+ } else if incidentRunning {
+ slog.Info("legacy anomaly detector disabled because v2.1 incident engine is running")
}
// ---------------- Causal inference ticker ----------------
@@ -707,6 +955,20 @@ func main() {
} else {
slog.Info("ingest shutdown complete")
}
+ if otlpGRPCServer != nil {
+ done := make(chan struct{})
+ go func() {
+ otlpGRPCServer.GracefulStop()
+ close(done)
+ }()
+ select {
+ case <-done:
+ slog.Info("otlp grpc shutdown complete")
+ case <-time.After(5 * time.Second):
+ otlpGRPCServer.Stop()
+ slog.Warn("otlp grpc graceful shutdown timed out; forced stop")
+ }
+ }
planStore.Close()
@@ -780,6 +1042,116 @@ func printHelp() {
os.Stdout.WriteString("\n\033[2mnotes: MCP stdio: run with MCP_STDIO=1\033[0m\n")
}
+type coldDeployAdapter struct {
+ store *coldstore.SQLiteStore
+}
+
+func (a coldDeployAdapter) DeploymentsInWindow(ctx context.Context, start, end time.Time, serviceFilter string) ([]incidents.Deployment, error) {
+ rows, err := a.store.DeploymentsInWindow(ctx, start, end, serviceFilter)
+ if err != nil {
+ return nil, err
+ }
+ out := make([]incidents.Deployment, 0, len(rows))
+ for _, row := range rows {
+ out = append(out, incidents.Deployment{
+ ID: row.ID,
+ Service: row.Service,
+ Version: row.Version,
+ Env: row.Env,
+ FirstSeen: row.FirstSeen,
+ LastSeen: row.LastSeen,
+ Metadata: row.Metadata,
+ })
+ }
+ return out, nil
+}
+
+type incidentReaderAdapter struct {
+ reader *ingestv2.Reader
+}
+
+func (a incidentReaderAdapter) Errors(f incidents.SearchFilter, limit int) incidents.ErrorsResult {
+ res := a.reader.Errors(toV2SearchFilter(f), nil, limit)
+ return incidents.ErrorsResult{Rows: res.Rows}
+}
+
+func (a incidentReaderAdapter) BlastRadius(f incidents.SearchFilter, key apiv2.BlastKey) apiv2.BlastRadiusResponse {
+ return a.reader.BlastRadius(toV2SearchFilter(f), ingestv2.BlastKeyMode{Key: key})
+}
+
+func (a incidentReaderAdapter) SearchEvents(f incidents.SearchFilter, limit int) []*eventv2.Event {
+ res := a.reader.SearchEvents(toV2SearchFilter(f), nil, limit)
+ return res.Events
+}
+
+func toV2SearchFilter(f incidents.SearchFilter) ingestv2.SearchFilter {
+ return ingestv2.SearchFilter{
+ Service: f.Service,
+ Statuses: f.Statuses,
+ ErrorCode: f.ErrorCode,
+ Since: f.Since,
+ Until: f.Until,
+ }
+}
+
+type incidentInsightAdapter struct {
+ engine *incidents.Engine
+}
+
+func (a incidentInsightAdapter) Current() *detect.Insight {
+ if a.engine == nil {
+ return nil
+ }
+ inc, err := a.engine.TopActive(context.Background())
+ if err != nil || inc == nil {
+ return nil
+ }
+ return projectIncidentInsight(*inc, time.Now().UTC())
+}
+
+func projectIncidentInsight(inc incidents.Incident, detectedAt time.Time) *detect.Insight {
+ affectedUsers := 0
+ if inc.AffectedUsers != nil {
+ affectedUsers = *inc.AffectedUsers
+ }
+ out := &detect.Insight{
+ DetectedAt: detectedAt,
+ TopErrorCode: inc.ErrorFamily.ErrorCode,
+ Lift: inc.Lift,
+ CurrentCount: inc.CurrentCount,
+ BaselineCount: inc.BaselineCount,
+ AffectedRequests: inc.AffectedRequests,
+ AffectedUsers: affectedUsers,
+ Services: append([]string(nil), inc.TopServices...),
+ SeverityScore: float64(inc.Severity),
+ }
+ if len(out.Services) == 0 {
+ out.Services = []string{inc.Service}
+ }
+ for _, ev := range inc.Evidence {
+ if ev.Kind == incidents.EvidenceDeployment && ev.DeployID != "" {
+ out.DeployCorrelation = &detect.DeployCorrelation{
+ DeploymentID: ev.DeployID,
+ Service: ev.Service,
+ Confidence: incidentConfidenceFloat(inc.Confidence),
+ }
+ break
+ }
+ }
+ return out
+}
+
+func incidentConfidenceFloat(c incidents.Confidence) float64 {
+ switch c {
+ case incidents.ConfidenceHigh:
+ return 0.9
+ case incidents.ConfidenceMedium:
+ return 0.65
+ default:
+ return 0.35
+ }
+}
+
func parseSlogLevel(s string) slog.Level {
switch strings.ToLower(s) {
case "debug":
diff --git a/docs/env.md b/docs/env.md
index f0c77ea..0ef8836 100644
--- a/docs/env.md
+++ b/docs/env.md
@@ -6,7 +6,25 @@ Reference for configuring the Waylog ingest server and SDK. All variables are re
| Variable | Purpose |
|---|---|
-| `GEMINI_API_KEY` / `GOOGLE_API_KEY` | Required when server-side Ask/tool flows use Gemini |
+| Provider credentials | Required only when natural-language Ask should call a configured LLM provider. Set the matching key for the selected provider: `GEMINI_API_KEY` or `GOOGLE_API_KEY`, `ANTHROPIC_API_KEY`, or `OPENAI_API_KEY` |
+
+## LLM provider
+
+Deterministic tools, plans, triage, MCP, and read APIs do not require an LLM provider. The provider is only used by natural-language Ask flows. If Ask cannot construct the selected provider, it returns a provider-agnostic "LLM provider not configured" error.
+
+| Variable | Default | Purpose |
+|---|---|---|
+| `WAYLOG_LLM_PROVIDER` | `none` unless a supported provider key is present | LLM provider for Ask. Supported values: `none`, `gemini`, `anthropic`, `openai` |
+| `WAYLOG_LLM_MODEL` | provider default | Provider-neutral model override. Takes precedence over provider-specific model variables |
+| `GEMINI_MODEL` | `gemini-2.5-flash` | Gemini-specific model override when `WAYLOG_LLM_MODEL` is unset |
+| `GEMINI_API_BASE` | Gemini API default | Gemini-specific API base URL override |
+| `GEMINI_TOOL_MODE` | `text` | Gemini-specific tool-calling mode |
+| `ANTHROPIC_API_KEY` | — | Anthropic API key for `WAYLOG_LLM_PROVIDER=anthropic` |
+| `ANTHROPIC_MODEL` | `claude-sonnet-4-6` | Anthropic-specific model override when `WAYLOG_LLM_MODEL` is unset |
+| `ANTHROPIC_API_BASE` | Anthropic API default | Anthropic-specific API base URL override |
+| `OPENAI_API_KEY` | — | OpenAI API key for `WAYLOG_LLM_PROVIDER=openai` |
+| `OPENAI_MODEL` | `gpt-5.4-mini` | OpenAI-specific model override when `WAYLOG_LLM_MODEL` is unset |
+| `OPENAI_API_BASE` | OpenAI API default | OpenAI-specific API base URL override |
## Auth
@@ -14,30 +32,34 @@ Scoped keys. See the Auth section of the [README](../README.md).
| Variable | Scope |
|---|---|
-| `WAYLOG_WRITE_KEY` | Write auth for `/v1/events` (preferred) |
+| `WAYLOG_PROFILE` | Auth strictness profile: `demo` (open), `dev` (default — open OTLP, optional auth), `prod` (refuses to boot without write/read/agent keys, and without write auth on OTLP HTTP+gRPC) |
+| `WAYLOG_WRITE_KEY` | Write auth for `/v1/events`, `/v1/otlp/v1/traces`, OTLP/gRPC, `/v1/signals`, `/v1/alerts` (preferred) |
| `WAYLOG_API_KEY` | Legacy alias for write scope. Supports `Authorization: Bearer` and `X-API-Key` headers |
| `WAYLOG_READ_KEY` | Read auth for read endpoints + dashboard session validation |
| `WAYLOG_AGENT_KEY` | Agent auth for `/v1/tools/*`, `/v1/ask`, `/v1/plans/*`. No session fallback |
| `DASHBOARD_AUTH` | Dashboard auth mode: `off` \| `basic::` \| `key:` |
-| `DASHBOARD_SESSION_SECRET` | Session signing key (derived from `DASHBOARD_AUTH` if unset) |
+| `DASHBOARD_SESSION_SECRET` | Session signing key (derived from `DASHBOARD_AUTH` if unset; required when `WAYLOG_PROFILE=prod`) |
-`ParseConfig` validates the auth matrix at startup and refuses to boot with an unsafe combination.
+`ParseConfig` validates the auth matrix at startup and refuses to boot with an unsafe combination. When `WAYLOG_PROFILE=prod`, all three scoped keys (`WAYLOG_WRITE_KEY`, `WAYLOG_READ_KEY`, `WAYLOG_AGENT_KEY`) are required, and OTLP cannot run unauthenticated.
## Ingest server
| Variable | Default | Purpose |
|---|---|---|
| `INGEST_ADDR` | `:8080` | Listen address |
-| `MAX_BODY_BYTES` | `1048576` (1 MB) | Max body size for `/v1/events` |
+| `OTLP_ENABLED` | `true` | Enable OTLP trace ingest over HTTP and gRPC |
+| `OTLP_GRPC_ADDR` | `:4317` | OTLP/gRPC trace receiver listen address. Set empty to disable the gRPC receiver. For single-host installs, bind `127.0.0.1:4317`. When `WAYLOG_PROFILE=prod`, the server refuses to boot if OTLP is enabled without `WAYLOG_WRITE_KEY` |
+| `MAX_BODY_BYTES` | `1048576` (1 MB) | Max body size for `/v1/events`, `/v1/otlp/v1/traces`, and OTLP/gRPC receive messages |
| `READ_HEADER_TIMEOUT` | `5s` | HTTP read header timeout |
| `READ_TIMEOUT` | `10s` | HTTP read timeout |
| `WRITE_TIMEOUT` | `10s` | HTTP write timeout |
| `IDLE_TIMEOUT` | `120s` | HTTP idle timeout |
| `CORS_ORIGIN` | `*` | Allowed CORS origin for read APIs |
+| `ALERT_MATCH_WINDOW` | `15m` | Window for matching `/v1/alerts` to active incidents by `env + service + error_code`; capped at `24h` |
## CLI
-The `waylog` CLI calls the running ingest server's v2 read APIs. The server must run with `WAYLOG_V2_READS=true`.
+The `waylog` CLI calls the running ingest server's v2 read APIs. The server runs with `WAYLOG_V2_READS=true` by default; only set it to `false` for legacy v1-only stacks.
| Variable | Default | Purpose |
|---|---|---|
@@ -56,6 +78,17 @@ The `waylog` CLI calls the running ingest server's v2 read APIs. The server must
| `EVENT_LOG_SYNC` | `true` | Per-write fsync. Set `false` for dev/load testing |
| `EVENT_LOG_MAX_FILE_MB` | `50` | Rotation size. `0` disables rotation |
| `EVENT_LOG_RETENTION` | `72h` | Event log retention. Must be positive |
+| `WAYLOG_SIGNAL_RETENTION` | `72h` | Production-context signal retention. Must be positive. `/v1/signals` requires `SQLITE_PATH` |
+| `WAYLOG_INCIDENTS_ENABLED` | `true` | Enable the v2.1 incident engine when `SQLITE_PATH` is set and `WAYLOG_V2_READS=true` |
+| `WAYLOG_INCIDENT_TICK_INTERVAL` | `30s` | Incident engine evaluation interval |
+| `WAYLOG_INCIDENT_WINDOW` | `10m` | Current error-family spike window |
+| `WAYLOG_INCIDENT_MIN_COUNT` | `5` | Minimum current-window failures needed to open an incident |
+| `WAYLOG_INCIDENT_MIN_LIFT` | `3.0` | Minimum current-vs-baseline lift when the family already exists in the baseline window |
+| `WAYLOG_INCIDENT_RESOLVE_AFTER` | `2m` | Time without renewed matching failures before a recovering incident resolves |
+| `WAYLOG_DEPLOY_CORRELATION_WINDOW` | `15m` | Window used to attach deploy signals and deployment records as incident evidence |
+| `WAYLOG_INCIDENT_SAMPLE_LIMIT` | `5` | Maximum persisted sample traces per incident |
+| `WAYLOG_REBUILD_INCIDENTS_ON_START` | `false` | Rebuild non-resolved incident rows at startup from the schema-2.0 WAL hot window plus signals |
+| `WAYLOG_INCIDENT_REBUILD_MAX_EVENTS` | `250000` | Safety cap for startup incident rebuild replay |
| `WAYLOG_V2_DEDUP_CAPACITY` | `65536` | Recent schema-2.0 `event_id` dedupe cache capacity |
| `GRAPH_HOT_WINDOW` | `GRAPH_RETENTION` or `24h` | Recent in-memory graph/index retention window and max v2 read window |
| `GRAPH_RETENTION` | `24h` | Hot graph retention. Nodes older than this are pruned every snapshot tick |
@@ -74,7 +107,7 @@ See [Internals](internals.md) for the full durability model.
| Variable | Default | Purpose |
|---|---|---|
| `GRAPH_UI` | `false` | Enable optional graph topology endpoint `/v1/graph/topology` |
-| `WAYLOG_V2_READS` | `false` | Route v2 read endpoints to the schema-2.0 recent index |
+| `WAYLOG_V2_READS` | `true` | Route v2 read endpoints to the schema-2.0 recent index. Set `false` only for legacy v1-only stacks |
| `CAUSAL_ENABLED` | `false` | Enable shadow-mode causal inference |
| `CAUSAL_INTERVAL` | `30s` | Causal inference ticker interval |
| `HAPPY_SAMPLE_RATE_PCT` | `2` | Success-event sampling rate. Set `100` in dev profiles |
diff --git a/docs/openapi.yaml b/docs/openapi.yaml
index 9d874ee..6cc08dd 100644
--- a/docs/openapi.yaml
+++ b/docs/openapi.yaml
@@ -7,7 +7,9 @@ info:
The primary product path is schema-2.0 ingest plus the v2 read APIs used by
the operator CLI and embedded dashboard: recent traces, error families,
- trace story, blast radius, event search, and direct trace/event lookup.
+ trace story, blast radius, incidents, event search, and direct trace/event
+ lookup. `/v1/insight` is a compatibility endpoint; new clients should use
+ `/v1/incidents/*`.
Write endpoints require write-scope auth when auth is configured. Read
endpoints require read-scope auth when read keys are configured.
@@ -18,7 +20,9 @@ tags:
- name: Ingest
description: Schema-2.0 event ingest and validation.
- name: OTLP
- description: OTLP/HTTP trace ingest converted into schema-2.0 events.
+ description: OTLP trace ingest converted into schema-2.0 events. HTTP is documented here; gRPC uses OTLP TraceService on the configured OTLP_GRPC_ADDR.
+ - name: Signals
+ description: Production-context facts used by incident triage.
- name: Events
description: Direct event lookup and search.
- name: Traces
@@ -136,6 +140,113 @@ paths:
schema:
$ref: '#/components/schemas/IngestEnvelope'
+ /v1/signals:
+ post:
+ tags: [Signals]
+ operationId: ingestSignal
+ summary: Ingest one production-context signal
+ description: |
+ Accepts one low-volume production-context signal, such as a deploy,
+ dependency, runtime, healthcheck, config, or alert fact. Signals require
+ SQLite persistence and are used by the v2.1 incident engine.
+ security:
+ - ApiKeyHeader: []
+ - BearerAuth: []
+ requestBody:
+ required: true
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/Signal'
+ responses:
+ '201':
+ description: Signal accepted and persisted
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/SignalAccepted'
+ '400':
+ description: Invalid signal JSON or fields
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/ReadError'
+ '401':
+ description: Unauthorized
+ '405':
+ description: Method Not Allowed
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/ReadError'
+ '413':
+ description: Request body too large
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/ReadError'
+ '503':
+ description: Signal storage unavailable; set SQLITE_PATH
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/ReadError'
+ '500':
+ description: Internal signal storage error
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/ReadError'
+
+ /v1/alerts:
+ post:
+ tags: [Signals]
+ operationId: ingestAlert
+ summary: Ingest an external alert and correlate it with an active incident
+ description: |
+ Accepts Waylog-normalized alerts plus Alertmanager, Grafana, and
+ PagerDuty webhook payloads. Accepted alerts are stored as `type=alert`
+ signals and correlated with active incidents when possible. Alerts do
+ not create incidents — the spike detector owns incident lifecycle.
+ security:
+ - ApiKeyHeader: []
+ - BearerAuth: []
+ requestBody:
+ required: true
+ content:
+ application/json:
+ schema:
+ type: object
+ additionalProperties: true
+ responses:
+ '201':
+ description: Alert accepted and stored as a signal
+ content:
+ application/json:
+ schema:
+ type: object
+ required: [signal, match]
+ properties:
+ signal:
+ $ref: '#/components/schemas/Signal'
+ match:
+ type: object
+ required: [matched, strategy]
+ properties:
+ matched: {type: boolean}
+ incident_id: {type: string}
+ strategy:
+ type: string
+ enum: [incident_id, trace_id, family, none]
+ '400':
+ description: Invalid JSON, unsupported alert shape, or missing required fields
+ '401':
+ description: Unauthorized
+ '405':
+ description: Method Not Allowed
+ '503':
+ description: Signal store unavailable
+
/v1/otlp/v1/traces:
post:
tags: [OTLP]
@@ -145,7 +256,9 @@ paths:
Accepts OTLP ExportTraceServiceRequest protobuf bodies. HTTP spans are
converted into schema-2.0 WideEvents and ingested through the same WAL
and projector as SDK events. Unsupported spans and validation rejects
- are reported via OTLP partial_success.
+ are reported via OTLP partial_success. The same trace conversion path
+ is available over OTLP/gRPC TraceService when the gRPC receiver is
+ enabled.
security:
- ApiKeyHeader: []
- BearerAuth: []
@@ -375,6 +488,209 @@ paths:
'400':
$ref: '#/components/responses/ReadBadRequest'
+ /v1/incidents/active:
+ get:
+ tags: [Triage]
+ operationId: listActiveIncidents
+ summary: Active v2.1 incidents
+ description: Returns active and recovering incidents derived from v2 error-family spikes, signals, and deployment context.
+ security:
+ - ApiKeyHeader: []
+ - BearerAuth: []
+ responses:
+ '200':
+ description: Active incidents
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/IncidentListResponse'
+ '401':
+ description: Unauthorized
+ '405':
+ description: Method Not Allowed
+
+ /v1/incidents/{id}:
+ get:
+ tags: [Triage]
+ operationId: getIncident
+ summary: Get one incident
+ security:
+ - ApiKeyHeader: []
+ - BearerAuth: []
+ parameters:
+ - $ref: '#/components/parameters/IncidentID'
+ responses:
+ '200':
+ description: Incident detail
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/IncidentDetailResponse'
+ '401':
+ description: Unauthorized
+ '404':
+ $ref: '#/components/responses/ReadNotFound'
+ '405':
+ description: Method Not Allowed
+
+ /v1/incidents/{id}/snapshot:
+ get:
+ tags: [Triage]
+ operationId: getIncidentSnapshot
+ summary: Render an incident snapshot
+ description: |
+ Defaults to text/plain. Send Accept: application/json to receive the
+ snapshot text plus the incident object.
+ security:
+ - ApiKeyHeader: []
+ - BearerAuth: []
+ parameters:
+ - $ref: '#/components/parameters/IncidentID'
+ responses:
+ '200':
+ description: Incident snapshot
+ content:
+ text/plain:
+ schema:
+ type: string
+ application/json:
+ schema:
+ $ref: '#/components/schemas/IncidentSnapshotResponse'
+ '401':
+ description: Unauthorized
+ '404':
+ $ref: '#/components/responses/ReadNotFound'
+ '405':
+ description: Method Not Allowed
+
+ /v1/triage/{incident_id}:
+ get:
+ tags: [Triage]
+ operationId: getTriageReport
+ summary: Build a deterministic TriageReport for an open incident
+ description: |
+ Returns a structured TriageReport v1 (incident_ref, blast_snapshot,
+ first_failure, sample_traces, signals, next_checks, confidence,
+ report_hash). Same builder backs `POST /v1/tools/triage_incident`;
+ both surfaces produce identical `report_hash` for the same input.
+ Read-scope auth.
+ security:
+ - ApiKeyHeader: []
+ - BearerAuth: []
+ parameters:
+ - name: incident_id
+ in: path
+ required: true
+ schema: {type: string}
+ - name: window
+ in: query
+ required: false
+ schema: {type: string, default: "15m"}
+ description: Go duration string (e.g. 15m, 1h). Default 15m.
+ - name: snapshot
+ in: query
+ required: false
+ schema: {type: boolean, default: false}
+ description: |
+ When true, freeze evaluation bounds to the incident's started_at
+ and updated_at instead of using wall-clock now.
+ responses:
+ '200':
+ description: TriageReport v1
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/TriageReport'
+ '400':
+ description: Missing or invalid parameters
+ '401':
+ description: Unauthorized
+ '500':
+ description: Triage build failed
+
+ /v1/triage/{incident_id}/report:
+ get:
+ tags: [Triage]
+ operationId: renderTriageReport
+ summary: Render a cited operator report from a TriageReport
+ description: |
+ Builds the same deterministic TriageReport as `/v1/triage/{incident_id}`
+ and renders it as Markdown, Slack Block Kit JSON, or PagerDuty note text.
+ This endpoint does not deliver messages to Slack or PagerDuty.
+ security:
+ - ApiKeyHeader: []
+ - BearerAuth: []
+ parameters:
+ - name: incident_id
+ in: path
+ required: true
+ schema: {type: string}
+ - name: format
+ in: query
+ required: false
+ schema:
+ type: string
+ enum: [markdown, slack, pagerduty]
+ default: markdown
+ - name: window
+ in: query
+ required: false
+ schema: {type: string, default: "15m"}
+ - name: snapshot
+ in: query
+ required: false
+ schema: {type: boolean, default: false}
+ responses:
+ '200':
+ description: Rendered operator report
+ content:
+ text/markdown:
+ schema: {type: string}
+ text/plain:
+ schema: {type: string}
+ application/json:
+ schema:
+ type: object
+ additionalProperties: true
+ '400':
+ description: Missing or invalid parameters
+ '401':
+ description: Unauthorized
+ '404':
+ $ref: '#/components/responses/ReadNotFound'
+ '500':
+ description: Triage build or render failed
+
+ /v1/insight:
+ get:
+ tags: [Operational]
+ operationId: getCompatibilityInsight
+ summary: Compatibility anomaly insight
+ deprecated: true
+ description: |
+ Compatibility endpoint for older dashboard/tool consumers. When the
+ v2.1 incident engine is running, this projects the top active incident
+ into the legacy insight shape. Otherwise it falls back to the legacy
+ detector. New clients should use `/v1/incidents/active` and
+ `/v1/incidents/{id}`.
+ security:
+ - ApiKeyHeader: []
+ - BearerAuth: []
+ responses:
+ '200':
+ description: Compatibility insight object
+ content:
+ application/json:
+ schema:
+ type: object
+ additionalProperties: true
+ '204':
+ description: No active insight
+ '401':
+ description: Unauthorized
+ '405':
+ description: Method Not Allowed
+
/v1/capabilities:
get:
tags: [Capabilities]
@@ -529,12 +845,78 @@ paths:
'503':
description: LLM provider unavailable
+ /v1/plans/execute:
+ post:
+ tags: [Operational]
+ operationId: executePlan
+ summary: Execute a deterministic tool plan
+ description: |
+ Executes explicit plan steps or a built-in template shorthand. The
+ `triage` template expands to one `triage_incident` tool step; the
+ resulting TriageReport is returned at `steps[0].result`.
+ security:
+ - ApiKeyHeader: []
+ - BearerAuth: []
+ requestBody:
+ required: true
+ content:
+ application/json:
+ schema:
+ oneOf:
+ - type: object
+ required: [steps]
+ properties:
+ steps:
+ type: array
+ minItems: 1
+ maxItems: 10
+ items:
+ type: object
+ required: [id, tool]
+ properties:
+ id: {type: string}
+ tool: {type: string}
+ params:
+ type: object
+ additionalProperties: true
+ - type: object
+ required: [template, params]
+ properties:
+ template:
+ type: string
+ enum: [triage]
+ params:
+ type: object
+ required: [incident_id]
+ properties:
+ incident_id: {type: string}
+ window: {type: string, default: 15m}
+ snapshot: {type: boolean, default: false}
+ responses:
+ '200':
+ description: Plan execution result
+ headers:
+ X-Plan-ID:
+ schema: {type: string}
+ description: Plan ID for `/v1/stream/plans/{id}`.
+ content:
+ application/json:
+ schema:
+ $ref: '#/components/schemas/PlanResult'
+ '400':
+ description: Invalid plan or template parameters
+ '503':
+ description: Tool registry unavailable
+
/v1/tools/{name}:
post:
tags: [Operational]
operationId: executeTool
summary: Direct tool call
- description: Executes a registered structured tool by name.
+ description: |
+ Executes a registered structured tool by name. `triage_incident`
+ returns a TriageReport; `render_triage_report` renders that report as
+ Markdown, Slack Block Kit JSON, or PagerDuty note text without delivery.
security:
- ApiKeyHeader: []
- BearerAuth: []
@@ -650,6 +1032,12 @@ components:
required: true
schema:
type: string
+ IncidentID:
+ name: id
+ in: path
+ required: true
+ schema:
+ type: string
TraceIDQuery:
name: trace_id
in: query
@@ -911,6 +1299,64 @@ components:
items:
$ref: '#/components/schemas/ErrorRef'
+ Signal:
+ type: object
+ additionalProperties: true
+ required: [type, source, service, env, severity, reason, timestamp]
+ example:
+ type: deploy
+ source: github-actions
+ service: checkout
+ env: prod
+ severity: info
+ reason: RolloutComplete
+ message: checkout 1.18.2 rolled out
+ metadata:
+ deployment_id: deploy_123
+ version: 1.18.2
+ timestamp: "2026-05-02T18:09:40Z"
+ properties:
+ signal_id:
+ type: string
+ readOnly: true
+ description: Server-generated signal id with sig_ prefix.
+ type:
+ type: string
+ enum: [deploy, runtime, healthcheck, dependency, config, alert]
+ source:
+ type: string
+ service:
+ type: string
+ env:
+ type: string
+ severity:
+ type: string
+ enum: [info, warning, critical]
+ reason:
+ type: string
+ message:
+ type: string
+ resource:
+ type: object
+ additionalProperties: true
+ metadata:
+ type: object
+ additionalProperties: true
+ timestamp:
+ type: string
+ format: date-time
+ received_at:
+ type: string
+ format: date-time
+ readOnly: true
+
+ SignalAccepted:
+ type: object
+ required: [signal]
+ properties:
+ signal:
+ $ref: '#/components/schemas/Signal'
+
Anchor:
type: object
required: [step, error_code]
@@ -1312,11 +1758,321 @@ components:
items:
type: string
+ IncidentEvidence:
+ type: object
+ required: [kind, title, occurred_at]
+ properties:
+ kind:
+ type: string
+ enum: [signal, deployment, trace, metric]
+ title:
+ type: string
+ detail:
+ type: string
+ service:
+ type: string
+ signal_id:
+ type: string
+ deployment_id:
+ type: string
+ trace_id:
+ type: string
+ occurred_at:
+ type: string
+ format: date-time
+ fields:
+ type: object
+ additionalProperties: true
+
+ Incident:
+ type: object
+ required:
+ - incident_id
+ - env
+ - service
+ - error_family
+ - status
+ - cause
+ - confidence
+ - severity
+ - started_at
+ - updated_at
+ - last_seen_at
+ - affected_requests
+ - affected_services
+ - top_services
+ - sample_traces
+ - evidence
+ - next_checks
+ - lift
+ - baseline_count
+ - current_count
+ example:
+ incident_id: inc_7d0b0b3d5a52d891
+ env: prod
+ service: checkout
+ error_family:
+ service: checkout
+ step: payment.charge
+ error_code: PMT_502
+ status: active
+ cause: dependency
+ confidence: medium
+ severity: 8
+ started_at: '2026-05-04T16:00:00Z'
+ updated_at: '2026-05-04T16:02:00Z'
+ last_seen_at: '2026-05-04T16:02:00Z'
+ affected_requests: 12
+ affected_users: 8
+ affected_services: 3
+ top_services: [api-gateway, checkout, payment]
+ sample_traces: [7f3a2b9c000000000000000000000001]
+ evidence:
+ - kind: trace
+ title: First failing trace sample
+ detail: payment.charge/PMT_502
+ service: checkout
+ trace_id: 7f3a2b9c000000000000000000000001
+ occurred_at: '2026-05-04T16:00:00Z'
+ next_checks:
+ - Check the downstream service health and recent deploys.
+ lift: 6
+ baseline_count: 2
+ current_count: 12
+ properties:
+ incident_id:
+ type: string
+ env:
+ type: string
+ service:
+ type: string
+ error_family:
+ $ref: '#/components/schemas/ErrorFamily'
+ status:
+ type: string
+ enum: [active, recovering, resolved]
+ cause:
+ type: string
+ enum: [deploy, app, dependency, unknown]
+ confidence:
+ type: string
+ enum: [high, medium, low]
+ severity:
+ type: integer
+ minimum: 1
+ maximum: 10
+ started_at:
+ type: string
+ format: date-time
+ updated_at:
+ type: string
+ format: date-time
+ last_seen_at:
+ type: string
+ format: date-time
+ recovering_at:
+ type: string
+ format: date-time
+ nullable: true
+ resolved_at:
+ type: string
+ format: date-time
+ nullable: true
+ affected_requests:
+ type: integer
+ affected_users:
+ type: integer
+ nullable: true
+ affected_services:
+ type: integer
+ top_services:
+ type: array
+ items:
+ type: string
+ sample_traces:
+ type: array
+ items:
+ type: string
+ evidence:
+ type: array
+ items:
+ $ref: '#/components/schemas/IncidentEvidence'
+ next_checks:
+ type: array
+ items:
+ type: string
+ instrumentation_warnings:
+ type: array
+ items:
+ type: string
+ lift:
+ type: number
+ format: double
+ baseline_count:
+ type: integer
+ current_count:
+ type: integer
+
+ IncidentListResponse:
+ type: object
+ required: [incidents]
+ properties:
+ incidents:
+ type: array
+ items:
+ $ref: '#/components/schemas/Incident'
+
+ IncidentDetailResponse:
+ type: object
+ required: [incident]
+ properties:
+ incident:
+ $ref: '#/components/schemas/Incident'
+
+ IncidentSnapshotResponse:
+ type: object
+ required: [snapshot, incident]
+ properties:
+ snapshot:
+ type: string
+ incident:
+ $ref: '#/components/schemas/Incident'
+
+ TriageReport:
+ type: object
+ required: [schema_version, incident_ref, blast_snapshot, confidence, generated_at, report_hash]
+ description: |
+ Deterministic triage artifact for an open incident. Versioned via
+ schema_version. report_hash is sha256 over the canonical JSON
+ excluding generated_at, plan_run_id, and report_hash itself.
+ properties:
+ schema_version:
+ type: string
+ enum: ["triage.v1"]
+ incident_ref:
+ type: object
+ required: [id]
+ properties:
+ id: {type: string}
+ window: {type: string, description: "Go duration string e.g. 15m0s"}
+ blast_snapshot:
+ type: object
+ properties:
+ requests: {type: integer}
+ users: {type: integer}
+ services: {type: integer}
+ top_error_families:
+ type: array
+ items:
+ type: object
+ properties:
+ service: {type: string}
+ step: {type: string}
+ error_code: {type: string}
+ count: {type: integer}
+ first_failure:
+ type: object
+ additionalProperties: true
+ description: Full /v1/traces/story payload for the first observed failing step.
+ sample_traces:
+ type: array
+ items:
+ type: object
+ properties:
+ trace_id: {type: string}
+ summary: {type: string}
+ signals:
+ type: array
+ items:
+ type: object
+ properties:
+ id: {type: string}
+ type: {type: string}
+ evidence_ids: {type: array, items: {type: string}}
+ alerts:
+ type: array
+ description: Alert signals linked to this incident and cited by operator reports.
+ items:
+ type: object
+ properties:
+ signal_id: {type: string}
+ alert_id: {type: string}
+ source: {type: string, enum: [waylog, alertmanager, grafana, pagerduty]}
+ severity: {type: string}
+ reason: {type: string}
+ provider_url: {type: string}
+ evidence_ids:
+ type: array
+ items: {type: string}
+ next_checks:
+ type: array
+ items:
+ type: object
+ properties:
+ id: {type: string}
+ prompt: {type: string}
+ confidence:
+ type: string
+ enum: [low, medium, high]
+ generated_at: {type: string}
+ plan_run_id:
+ type: string
+ description: Reserved for future plan-produced reports; omitted by the M1.5 triage template.
+ report_hash:
+ type: string
+ description: "sha256:"
+
+ PlanResult:
+ type: object
+ required: [plan_id, steps, completed, total, status]
+ properties:
+ plan_id: {type: string}
+ steps:
+ type: array
+ items:
+ type: object
+ required: [id, index, tool, duration_ms]
+ properties:
+ id: {type: string}
+ index: {type: integer}
+ tool: {type: string}
+ result:
+ description: Tool output. For the triage template this is a TriageReport.
+ oneOf:
+ - $ref: '#/components/schemas/TriageReport'
+ - type: object
+ additionalProperties: true
+ error:
+ type: object
+ nullable: true
+ properties:
+ code: {type: string}
+ message: {type: string}
+ retryable: {type: boolean}
+ duration_ms: {type: integer}
+ completed: {type: integer}
+ total: {type: integer}
+ status:
+ type: string
+ enum: [complete, partial, failed]
+ halted_at:
+ type: integer
+ nullable: true
+ error:
+ type: object
+ nullable: true
+ properties:
+ code: {type: string}
+ message: {type: string}
+ retryable: {type: boolean}
+
CapabilitiesResponse:
type: object
example:
otlp:
http_traces: true
+ grpc_traces: true
+ grpc_addr: ":4317"
v2_reads:
enabled: true
graph: false
@@ -1324,6 +2080,26 @@ components:
ask:
type: object
additionalProperties: true
+ llm:
+ type: object
+ description: Provider-neutral Ask configuration and runtime state.
+ properties:
+ provider:
+ type: string
+ description: Resolved Ask provider. `custom` is used for injected providers.
+ enum: [none, gemini, anthropic, openai, custom]
+ model:
+ type: string
+ description: Resolved model for the selected provider, or empty when Ask is disabled.
+ tool_mode:
+ type: string
+ description: Resolved tool-calling mode for the selected provider, or empty when Ask is disabled.
+ configured:
+ type: boolean
+ description: True when a provider was explicitly selected or inferred from credentials.
+ ask_enabled:
+ type: boolean
+ description: True when Ask has a provider implementation available.
dashboard:
type: object
additionalProperties: true
@@ -1337,11 +2113,30 @@ components:
properties:
http_traces:
type: boolean
+ grpc_traces:
+ type: boolean
+ grpc_addr:
+ type: string
v2_reads:
type: object
properties:
enabled:
type: boolean
+ incidents:
+ type: object
+ properties:
+ enabled:
+ type: boolean
+ persistent:
+ type: boolean
+ rebuild:
+ type: object
+ properties:
+ supported:
+ type: boolean
+ scope:
+ type: string
+ enum: ["", hot-window]
architecture:
type: object
additionalProperties: true
diff --git a/examples/cmd/api-gateway/main.go b/examples/cmd/api-gateway/main.go
index 12ca98e..9662c36 100644
--- a/examples/cmd/api-gateway/main.go
+++ b/examples/cmd/api-gateway/main.go
@@ -16,12 +16,22 @@ func main() {
}
checkoutURL := config.Getenv("CHECKOUT_URL", "http://localhost:9082")
+ ingestURL := config.Getenv("INGEST_URL", "http://localhost:8080")
+ writeKey := config.Getenv("WAYLOG_WRITE_KEY", "")
+ readKey := config.Getenv("WAYLOG_READ_KEY", writeKey)
+ agentKey := config.Getenv("WAYLOG_AGENT_KEY", readKey)
gateway := microdemo.NewGatewayHandler(checkoutURL)
+ gateway.SetSignalPoster(microdemo.NewDemoSignalPoster(
+ ingestURL,
+ writeKey,
+ ))
+ gateway.SetWaylogAPI(ingestURL, readKey, writeKey, agentKey)
mux := http.NewServeMux()
mux.Handle("/purchase", gateway.PurchaseHandler())
mux.HandleFunc("/demo", gateway.ServeDemo)
mux.HandleFunc("/demo/burst", gateway.ServeBurst)
+ mux.HandleFunc("/demo/proof", gateway.ServeProof)
microdemo.RunService("api-gateway", ":9081", mux)
}
diff --git a/examples/microdemo/burst.go b/examples/microdemo/burst.go
index 3c76258..cf7ccc4 100644
--- a/examples/microdemo/burst.go
+++ b/examples/microdemo/burst.go
@@ -14,6 +14,7 @@ import (
const (
defaultBurstRequests = 50
defaultBurstConcurrency = 10
+ incidentSeedPayments = 6
maxBurstRequests = 250
maxBurstConcurrency = 50
maxBurstSamples = 5
@@ -27,6 +28,7 @@ type BurstRequest struct {
type BurstSummary struct {
Requested BurstRequest `json:"requested"`
Accepted BurstRequest `json:"accepted"`
+ Signals []SignalResult `json:"signals,omitempty"`
DurationMs int64 `json:"duration_ms"`
ByScenario map[string]int `json:"by_scenario"`
OK int `json:"ok"`
@@ -87,6 +89,20 @@ func normalizeBurstRequest(raw BurstRequest) (requested, accepted BurstRequest)
return requested, accepted
}
+func pickBurstScenarioForIndex(i, requests int) string {
+ if i < incidentSeedPaymentCount(requests) {
+ return ScenarioPayment502
+ }
+ return pickBurstScenario()
+}
+
+func incidentSeedPaymentCount(requests int) int {
+ if requests < incidentSeedPayments {
+ return requests
+ }
+ return incidentSeedPayments
+}
+
func runBurst(ctx context.Context, dispatch http.Handler, raw BurstRequest) BurstSummary {
requested, accepted := normalizeBurstRequest(raw)
summary := BurstSummary{
@@ -112,11 +128,11 @@ func runBurst(ctx context.Context, dispatch http.Handler, raw BurstRequest) Burs
// concurrency instead of stacking up `requests` blocked goroutines.
sem <- struct{}{}
wg.Add(1)
- go func() {
+ scenario := pickBurstScenarioForIndex(i, accepted.Requests)
+ go func(scenario string) {
defer wg.Done()
defer func() { <-sem }()
- scenario := pickBurstScenario()
payload, _ := json.Marshal(PurchaseRequest{
SKU: "X1",
Scenario: scenario,
@@ -155,7 +171,7 @@ func runBurst(ctx context.Context, dispatch http.Handler, raw BurstRequest) Burs
summary.SampleTraceIDs = append(summary.SampleTraceIDs, resp.TraceID)
}
}
- }()
+ }(scenario)
}
wg.Wait()
summary.DurationMs = time.Since(start).Milliseconds()
diff --git a/examples/microdemo/burst_test.go b/examples/microdemo/burst_test.go
index eaa914b..8c49b87 100644
--- a/examples/microdemo/burst_test.go
+++ b/examples/microdemo/burst_test.go
@@ -1,6 +1,7 @@
package microdemo
import (
+ "context"
"encoding/json"
"net/http"
"net/http/httptest"
@@ -61,8 +62,16 @@ func TestRunBurstDispatchesEveryRequestThroughHandler(t *testing.T) {
if got := r.Header.Get("Content-Type"); got != "application/json" {
t.Fatalf("content-type = %q, want application/json", got)
}
+ var purchase PurchaseRequest
+ if err := json.NewDecoder(r.Body).Decode(&purchase); err != nil {
+ t.Fatalf("decode purchase: %v", err)
+ }
w.Header().Set("Content-Type", "application/json")
- _, _ = w.Write([]byte(`{"success":true,"trace_id":"t","scenario":"happy"}`))
+ _ = json.NewEncoder(w).Encode(map[string]any{
+ "success": purchase.Scenario == ScenarioHappy,
+ "trace_id": "trace-" + purchase.Scenario,
+ "scenario": purchase.Scenario,
+ })
})
summary := runBurst(t.Context(), dispatch, BurstRequest{Requests: 20, Concurrency: 4})
@@ -72,11 +81,25 @@ func TestRunBurstDispatchesEveryRequestThroughHandler(t *testing.T) {
if summary.Accepted.Requests != 20 || summary.Accepted.Concurrency != 4 {
t.Fatalf("accepted = %#v, want 20/4", summary.Accepted)
}
- if summary.OK != 20 || summary.Errors != 0 || summary.Suppressed != 0 {
- t.Fatalf("summary counts = ok:%d errors:%d suppressed:%d", summary.OK, summary.Errors, summary.Suppressed)
+ if summary.Errors < incidentSeedPayments {
+ t.Fatalf("errors = %d, want at least seeded payment failures %d", summary.Errors, incidentSeedPayments)
+ }
+ if summary.ByScenario[ScenarioPayment502] < incidentSeedPayments {
+ t.Fatalf("payment_502 count = %d, want at least %d", summary.ByScenario[ScenarioPayment502], incidentSeedPayments)
+ }
+ if summary.OK+summary.Errors+summary.Suppressed != 20 {
+ t.Fatalf("summary total = %d, want 20", summary.OK+summary.Errors+summary.Suppressed)
+ }
+}
+
+func TestPickBurstScenarioForIndexSeedsPaymentFailures(t *testing.T) {
+ for i := 0; i < incidentSeedPayments; i++ {
+ if got := pickBurstScenarioForIndex(i, 20); got != ScenarioPayment502 {
+ t.Fatalf("seed scenario[%d] = %q, want payment_502", i, got)
+ }
}
- if summary.ByScenario[ScenarioHappy] != 20 {
- t.Fatalf("happy count = %d, want 20", summary.ByScenario[ScenarioHappy])
+ if got := incidentSeedPaymentCount(3); got != 3 {
+ t.Fatalf("seed count = %d, want capped to request count 3", got)
}
}
@@ -139,6 +162,28 @@ func TestServeBurstAppliesDefaultsWhenZero(t *testing.T) {
}
}
+func TestServeBurstPostsDemoSignals(t *testing.T) {
+ gateway := NewGatewayHandler("http://checkout.example")
+ gateway.SetPurchaseHandler(okBurstDispatch())
+ gateway.SetSignalPoster(staticSignalPoster{results: []SignalResult{{
+ Type: "dependency", Service: "payment", Reason: "payment_gateway_5xx", Accepted: true, Status: http.StatusCreated, SignalID: "sig_demo",
+ }}})
+ rec := httptest.NewRecorder()
+ req := httptest.NewRequest(http.MethodPost, "/demo/burst", strings.NewReader(`{"requests":1,"concurrency":1}`))
+ req.Header.Set("Content-Type", "application/json")
+ gateway.ServeBurst(rec, req)
+ if rec.Code != http.StatusOK {
+ t.Fatalf("status = %d, want 200: %s", rec.Code, rec.Body.String())
+ }
+ var summary BurstSummary
+ if err := json.Unmarshal(rec.Body.Bytes(), &summary); err != nil {
+ t.Fatalf("unmarshal summary: %v", err)
+ }
+ if len(summary.Signals) != 1 || !summary.Signals[0].Accepted || summary.Signals[0].SignalID != "sig_demo" {
+ t.Fatalf("signals = %+v", summary.Signals)
+ }
+}
+
func serveBurstForTest(t *testing.T, body string) *httptest.ResponseRecorder {
t.Helper()
gateway := NewGatewayHandler("http://checkout.example")
@@ -150,6 +195,14 @@ func serveBurstForTest(t *testing.T, body string) *httptest.ResponseRecorder {
return rec
}
+type staticSignalPoster struct {
+ results []SignalResult
+}
+
+func (p staticSignalPoster) PostDemoSignals(context.Context) []SignalResult {
+ return p.results
+}
+
func okBurstDispatch() http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
var req PurchaseRequest
diff --git a/examples/microdemo/gateway.go b/examples/microdemo/gateway.go
index a6a7fac..e4a0a52 100644
--- a/examples/microdemo/gateway.go
+++ b/examples/microdemo/gateway.go
@@ -28,8 +28,14 @@ var uiHTML []byte
type GatewayHandler struct {
checkoutURL string
+ ingestURL string
+ readKey string
+ writeKey string
+ agentKey string
client *http.Client
+ proofClient *http.Client
purchase http.Handler
+ signals SignalPoster
}
type PurchaseRequest struct {
@@ -43,6 +49,7 @@ func NewGatewayHandler(checkoutURL string) *GatewayHandler {
client: &http.Client{
Transport: wayloghttp.NewTransport(demoHTTPTransport(), "checkout"),
},
+ proofClient: &http.Client{Timeout: 10 * demoSignalTimeout},
}
// Pre-wrap so the live /purchase route and /demo/burst dispatch share a
// single instance — and so callers can't forget to wire it up.
@@ -61,6 +68,19 @@ func (h *GatewayHandler) SetPurchaseHandler(handler http.Handler) {
h.purchase = handler
}
+// SetSignalPoster overrides the signal poster used by /demo/burst. Test seam.
+func (h *GatewayHandler) SetSignalPoster(poster SignalPoster) {
+ h.signals = poster
+}
+
+// SetWaylogAPI configures the demo-only proof loop proxy.
+func (h *GatewayHandler) SetWaylogAPI(ingestURL, readKey, writeKey, agentKey string) {
+ h.ingestURL = strings.TrimRight(strings.TrimSpace(ingestURL), "/")
+ h.readKey = strings.TrimSpace(readKey)
+ h.writeKey = strings.TrimSpace(writeKey)
+ h.agentKey = strings.TrimSpace(agentKey)
+}
+
func (h *GatewayHandler) ServeDemo(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/html")
_, _ = w.Write(uiHTML)
@@ -136,7 +156,12 @@ func (h *GatewayHandler) ServeBurst(w http.ResponseWriter, r *http.Request) {
}
}
+ var signalResults []SignalResult
+ if h.signals != nil {
+ signalResults = h.signals.PostDemoSignals(r.Context())
+ }
summary := runBurst(r.Context(), h.purchase, req)
+ summary.Signals = signalResults
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(summary)
}
diff --git a/examples/microdemo/proof.go b/examples/microdemo/proof.go
new file mode 100644
index 0000000..41236ac
--- /dev/null
+++ b/examples/microdemo/proof.go
@@ -0,0 +1,431 @@
+package microdemo
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "fmt"
+ "io"
+ "net/http"
+ "net/url"
+ "time"
+
+ apiv2 "github.com/sssmaran/WaylogCLI/pkg/api/v2"
+ pkgtriage "github.com/sssmaran/WaylogCLI/pkg/triage"
+)
+
+const (
+ proofWindow = "15m"
+ proofPollDelay = 750 * time.Millisecond
+ proofPollAttempts = 24
+)
+
+type ProofSummary struct {
+ AlertID string `json:"alert_id"`
+ IncidentID string `json:"incident_id"`
+ ReportHash string `json:"report_hash"`
+ Hashes map[string]string `json:"hashes"`
+ Burst BurstSummary `json:"burst"`
+ Evidence ProofEvidence `json:"evidence"`
+ Reports ProofReports `json:"reports"`
+ Scorecard ProofScorecard `json:"scorecard"`
+}
+
+type ProofEvidence struct {
+ TraceID string `json:"trace_id"`
+ AlertLinked bool `json:"alert_linked"`
+ DependencySignal bool `json:"dependency_signal"`
+ NextChecks bool `json:"next_checks"`
+}
+
+type ProofReports struct {
+ Markdown string `json:"markdown"`
+ Slack json.RawMessage `json:"slack"`
+ PagerDuty string `json:"pagerduty"`
+}
+
+type ProofScorecard struct {
+ RootCauseAccuracy bool `json:"root_cause_accuracy"`
+ CauseClassificationDependency bool `json:"cause_classification_dependency"`
+ ReportHashStable bool `json:"report_hash_stable"`
+ PropagatedErrorInflationAvoided int `json:"propagated_error_inflation_avoided"`
+ TriageLatencyMS int64 `json:"triage_latency_ms"`
+ Scenario string `json:"scenario"`
+ RootCauseCount int `json:"root_cause_count"`
+ NaivePropagatedCount int `json:"naive_propagated_count"`
+}
+
+type planResult struct {
+ Steps []struct {
+ Result json.RawMessage `json:"result"`
+ } `json:"steps"`
+}
+
+func (h *GatewayHandler) ServeProof(w http.ResponseWriter, r *http.Request) {
+ if r.Method != http.MethodPost {
+ http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+ return
+ }
+ if h.ingestURL == "" {
+ http.Error(w, "INGEST_URL is not configured for the demo proof", http.StatusServiceUnavailable)
+ return
+ }
+
+ var req BurstRequest
+ if r.Body != nil {
+ defer r.Body.Close()
+ dec := json.NewDecoder(r.Body)
+ dec.DisallowUnknownFields()
+ if err := dec.Decode(&req); err != nil && err != io.EOF {
+ http.Error(w, "invalid json: "+err.Error(), http.StatusBadRequest)
+ return
+ }
+ }
+
+ result, err := h.runProof(r.Context(), req)
+ if err != nil {
+ http.Error(w, err.Error(), http.StatusBadGateway)
+ return
+ }
+ w.Header().Set("Content-Type", "application/json")
+ _ = json.NewEncoder(w).Encode(result)
+}
+
+func (h *GatewayHandler) runProof(ctx context.Context, req BurstRequest) (ProofSummary, error) {
+ alertID := fmt.Sprintf("alert_demo_proof_pmt_502_%d", time.Now().Unix())
+ if err := h.postProofAlert(ctx, alertID); err != nil {
+ return ProofSummary{}, err
+ }
+
+ signals := []SignalResult(nil)
+ if h.signals != nil {
+ signals = h.signals.PostDemoSignals(ctx)
+ }
+ burst := runBurst(ctx, h.purchase, req)
+ burst.Signals = signals
+ answerStart := time.Now()
+
+ errorsResp, err := h.pollErrors(ctx)
+ if err != nil {
+ return ProofSummary{}, err
+ }
+ inc, incidentsResp, err := h.pollIncident(ctx)
+ if err != nil {
+ return ProofSummary{}, err
+ }
+
+ triageA, err := h.getTriage(ctx, inc.IncidentID)
+ if err != nil {
+ return ProofSummary{}, err
+ }
+ triageB, err := h.getTriage(ctx, inc.IncidentID)
+ if err != nil {
+ return ProofSummary{}, err
+ }
+ answerEnd := time.Now()
+
+ toolReport, err := h.postToolTriage(ctx, inc.IncidentID)
+ if err != nil {
+ return ProofSummary{}, err
+ }
+ planReport, err := h.postPlanTriage(ctx, inc.IncidentID)
+ if err != nil {
+ return ProofSummary{}, err
+ }
+ hashes := map[string]string{
+ "read": triageA.ReportHash,
+ "repeat": triageB.ReportHash,
+ "tool": toolReport.ReportHash,
+ "plan": planReport.ReportHash,
+ }
+ hashStable := triageA.ReportHash != "" &&
+ triageA.ReportHash == triageB.ReportHash &&
+ triageA.ReportHash == toolReport.ReportHash &&
+ triageA.ReportHash == planReport.ReportHash
+
+ blast, err := h.getBlast(ctx)
+ if err != nil {
+ return ProofSummary{}, err
+ }
+ reports, err := h.getReports(ctx, inc.IncidentID)
+ if err != nil {
+ return ProofSummary{}, err
+ }
+
+ rootCount := paymentErrorCount(errorsResp)
+ naive := rootCount * blast.AffectedServices
+ return ProofSummary{
+ AlertID: alertID,
+ IncidentID: inc.IncidentID,
+ ReportHash: triageA.ReportHash,
+ Hashes: hashes,
+ Burst: burst,
+ Evidence: ProofEvidence{
+ TraceID: firstTraceID(triageA),
+ AlertLinked: hasAlertID(triageA, alertID),
+ DependencySignal: hasSignalType(triageA, "dependency"),
+ NextChecks: len(triageA.NextChecks) > 0,
+ },
+ Reports: reports,
+ Scorecard: ProofScorecard{
+ RootCauseAccuracy: triageRootCauseAccurate(triageA),
+ CauseClassificationDependency: incidentCauseIsDependency(incidentsResp, inc.IncidentID),
+ ReportHashStable: hashStable,
+ PropagatedErrorInflationAvoided: naive - rootCount,
+ TriageLatencyMS: answerEnd.Sub(answerStart).Milliseconds(),
+ Scenario: "warm-demo",
+ RootCauseCount: rootCount,
+ NaivePropagatedCount: naive,
+ },
+ }, nil
+}
+
+func (h *GatewayHandler) postProofAlert(ctx context.Context, alertID string) error {
+ body := map[string]any{
+ "source": "waylog",
+ "alert_id": alertID,
+ "service": "checkout",
+ "env": "demo",
+ "severity": "critical",
+ "reason": "PMT_502 spike",
+ "message": "browser demo alert for checkout payment failures",
+ "error_code": "PMT_502",
+ "timestamp": time.Now().UTC().Format(time.RFC3339),
+ }
+ status, _, err := h.doJSON(ctx, http.MethodPost, "/v1/alerts", h.writeKey, body, nil)
+ if err != nil {
+ return err
+ }
+ if status != http.StatusCreated {
+ return fmt.Errorf("alert webhook failed: HTTP %d", status)
+ }
+ return nil
+}
+
+func (h *GatewayHandler) pollErrors(ctx context.Context) (apiv2.ErrorsResponse, error) {
+ var last apiv2.ErrorsResponse
+ for i := 0; i < proofPollAttempts; i++ {
+ q := url.Values{"window": {proofWindow}, "limit": {"10"}}
+ status, _, err := h.doJSON(ctx, http.MethodGet, "/v1/errors?"+q.Encode(), h.readKey, nil, &last)
+ if err == nil && status == http.StatusOK && paymentErrorCount(last) > 0 {
+ return last, nil
+ }
+ if err != nil {
+ return apiv2.ErrorsResponse{}, err
+ }
+ sleepOrDone(ctx, proofPollDelay)
+ }
+ return apiv2.ErrorsResponse{}, fmt.Errorf("payment_502 error family did not appear")
+}
+
+func (h *GatewayHandler) pollIncident(ctx context.Context) (apiv2.Incident, apiv2.IncidentListResponse, error) {
+ var last apiv2.IncidentListResponse
+ for i := 0; i < proofPollAttempts; i++ {
+ status, _, err := h.doJSON(ctx, http.MethodGet, "/v1/incidents/active", h.readKey, nil, &last)
+ if err != nil {
+ return apiv2.Incident{}, apiv2.IncidentListResponse{}, err
+ }
+ if status == http.StatusOK {
+ for _, inc := range last.Incidents {
+ if isPaymentFamily(inc.ErrorFamily) && inc.Cause == "dependency" && inc.Status == "active" {
+ return inc, last, nil
+ }
+ }
+ }
+ sleepOrDone(ctx, proofPollDelay)
+ }
+ return apiv2.Incident{}, apiv2.IncidentListResponse{}, fmt.Errorf("dependency incident did not appear")
+}
+
+func (h *GatewayHandler) getTriage(ctx context.Context, incidentID string) (*pkgtriage.Report, error) {
+ var rep pkgtriage.Report
+ status, _, err := h.doJSON(ctx, http.MethodGet, "/v1/triage/"+url.PathEscape(incidentID)+"?snapshot=true", h.readKey, nil, &rep)
+ if err != nil {
+ return nil, err
+ }
+ if status != http.StatusOK {
+ return nil, fmt.Errorf("triage read failed: HTTP %d", status)
+ }
+ return &rep, nil
+}
+
+func (h *GatewayHandler) postToolTriage(ctx context.Context, incidentID string) (*pkgtriage.Report, error) {
+ var rep pkgtriage.Report
+ status, _, err := h.doJSON(ctx, http.MethodPost, "/v1/tools/triage_incident", h.agentKey, map[string]any{"incident_id": incidentID, "snapshot": true}, &rep)
+ if err != nil {
+ return nil, err
+ }
+ if status != http.StatusOK {
+ return nil, fmt.Errorf("triage tool failed: HTTP %d", status)
+ }
+ return &rep, nil
+}
+
+func (h *GatewayHandler) postPlanTriage(ctx context.Context, incidentID string) (*pkgtriage.Report, error) {
+ var plan planResult
+ status, _, err := h.doJSON(ctx, http.MethodPost, "/v1/plans/execute", h.agentKey, map[string]any{
+ "template": "triage",
+ "params": map[string]any{"incident_id": incidentID, "snapshot": true},
+ }, &plan)
+ if err != nil {
+ return nil, err
+ }
+ if status != http.StatusOK || len(plan.Steps) == 0 {
+ return nil, fmt.Errorf("triage plan failed: HTTP %d", status)
+ }
+ var rep pkgtriage.Report
+ if err := json.Unmarshal(plan.Steps[0].Result, &rep); err != nil {
+ return nil, fmt.Errorf("triage plan result decode: %w", err)
+ }
+ return &rep, nil
+}
+
+func (h *GatewayHandler) getBlast(ctx context.Context) (apiv2.BlastRadiusResponse, error) {
+ q := url.Values{"window": {proofWindow}, "error_family": {"checkout:payment.charge:PMT_502"}}
+ var blast apiv2.BlastRadiusResponse
+ status, _, err := h.doJSON(ctx, http.MethodGet, "/v1/blast_radius?"+q.Encode(), h.readKey, nil, &blast)
+ if err != nil {
+ return apiv2.BlastRadiusResponse{}, err
+ }
+ if status != http.StatusOK {
+ return apiv2.BlastRadiusResponse{}, fmt.Errorf("blast failed: HTTP %d", status)
+ }
+ return blast, nil
+}
+
+func (h *GatewayHandler) getReports(ctx context.Context, incidentID string) (ProofReports, error) {
+ var out ProofReports
+ for _, format := range []string{"markdown", "slack", "pagerduty"} {
+ path := "/v1/triage/" + url.PathEscape(incidentID) + "/report?format=" + format + "&snapshot=true"
+ status, raw, err := h.doJSON(ctx, http.MethodGet, path, h.readKey, nil, nil)
+ if err != nil {
+ return ProofReports{}, err
+ }
+ if status != http.StatusOK {
+ return ProofReports{}, fmt.Errorf("%s report failed: HTTP %d", format, status)
+ }
+ switch format {
+ case "markdown":
+ out.Markdown = string(raw)
+ case "slack":
+ out.Slack = append(json.RawMessage(nil), raw...)
+ case "pagerduty":
+ out.PagerDuty = string(raw)
+ }
+ }
+ return out, nil
+}
+
+func (h *GatewayHandler) doJSON(ctx context.Context, method, path, key string, body any, out any) (int, []byte, error) {
+ var reader io.Reader
+ if body != nil {
+ raw, err := json.Marshal(body)
+ if err != nil {
+ return 0, nil, err
+ }
+ reader = bytes.NewReader(raw)
+ }
+ req, err := http.NewRequestWithContext(ctx, method, h.ingestURL+path, reader)
+ if err != nil {
+ return 0, nil, err
+ }
+ if body != nil {
+ req.Header.Set("Content-Type", "application/json")
+ }
+ if key != "" {
+ req.Header.Set("Authorization", "Bearer "+key)
+ }
+ client := h.proofClient
+ if client == nil {
+ client = http.DefaultClient
+ }
+ resp, err := client.Do(req)
+ if err != nil {
+ return 0, nil, err
+ }
+ defer resp.Body.Close()
+ raw, err := io.ReadAll(io.LimitReader(resp.Body, 4<<20))
+ if err != nil {
+ return resp.StatusCode, nil, err
+ }
+ if out != nil && resp.StatusCode >= 200 && resp.StatusCode < 300 {
+ if err := json.Unmarshal(raw, out); err != nil {
+ return resp.StatusCode, raw, err
+ }
+ }
+ return resp.StatusCode, raw, nil
+}
+
+func sleepOrDone(ctx context.Context, d time.Duration) {
+ timer := time.NewTimer(d)
+ defer timer.Stop()
+ select {
+ case <-ctx.Done():
+ case <-timer.C:
+ }
+}
+
+func paymentErrorCount(resp apiv2.ErrorsResponse) int {
+ for _, row := range resp.Rows {
+ if isPaymentFamily(row.ErrorFamily) {
+ return row.Count
+ }
+ }
+ return 0
+}
+
+func isPaymentFamily(f apiv2.ErrorFamily) bool {
+ return f.Service == "checkout" && f.Step == "payment.charge" && f.ErrorCode == "PMT_502"
+}
+
+func firstTraceID(rep *pkgtriage.Report) string {
+ if rep == nil || len(rep.SampleTraces) == 0 {
+ return ""
+ }
+ return rep.SampleTraces[0].TraceID
+}
+
+func hasAlertID(rep *pkgtriage.Report, alertID string) bool {
+ if rep == nil {
+ return false
+ }
+ for _, alert := range rep.Alerts {
+ if alert.AlertID == alertID && alert.SignalID != "" {
+ return true
+ }
+ }
+ return false
+}
+
+func hasSignalType(rep *pkgtriage.Report, typ string) bool {
+ if rep == nil {
+ return false
+ }
+ for _, sig := range rep.Signals {
+ if sig.ID != "" && sig.Type == typ {
+ return true
+ }
+ }
+ return false
+}
+
+func triageRootCauseAccurate(rep *pkgtriage.Report) bool {
+ if rep == nil {
+ return false
+ }
+ for _, family := range rep.BlastSnapshot.TopErrorFamilies {
+ if family.Service == "checkout" && family.Step == "payment.charge" && family.ErrorCode == "PMT_502" {
+ return true
+ }
+ }
+ return false
+}
+
+func incidentCauseIsDependency(resp apiv2.IncidentListResponse, incidentID string) bool {
+ for _, inc := range resp.Incidents {
+ if inc.IncidentID == incidentID {
+ return inc.Cause == "dependency"
+ }
+ }
+ return false
+}
diff --git a/examples/microdemo/proof_test.go b/examples/microdemo/proof_test.go
new file mode 100644
index 0000000..12904ab
--- /dev/null
+++ b/examples/microdemo/proof_test.go
@@ -0,0 +1,88 @@
+package microdemo
+
+import (
+ "encoding/json"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "testing"
+)
+
+func TestServeProofRejectsNonPOST(t *testing.T) {
+ gateway := NewGatewayHandler("http://checkout.example")
+ rec := httptest.NewRecorder()
+ req := httptest.NewRequest(http.MethodGet, "/demo/proof", nil)
+ gateway.ServeProof(rec, req)
+ if rec.Code != http.StatusMethodNotAllowed {
+ t.Fatalf("status = %d, want 405", rec.Code)
+ }
+}
+
+func TestServeProofRequiresIngestURL(t *testing.T) {
+ gateway := NewGatewayHandler("http://checkout.example")
+ gateway.SetPurchaseHandler(okBurstDispatch())
+ rec := httptest.NewRecorder()
+ req := httptest.NewRequest(http.MethodPost, "/demo/proof", strings.NewReader(`{}`))
+ gateway.ServeProof(rec, req)
+ if rec.Code != http.StatusServiceUnavailable {
+ t.Fatalf("status = %d, want 503", rec.Code)
+ }
+}
+
+func TestServeProofRejectsUnknownFields(t *testing.T) {
+ gateway := NewGatewayHandler("http://checkout.example")
+ gateway.SetWaylogAPI("http://ingest.example", "read", "write", "agent")
+ rec := httptest.NewRecorder()
+ req := httptest.NewRequest(http.MethodPost, "/demo/proof", strings.NewReader(`{"foo":1}`))
+ gateway.ServeProof(rec, req)
+ if rec.Code != http.StatusBadRequest {
+ t.Fatalf("status = %d, want 400", rec.Code)
+ }
+}
+
+func TestProofSummaryJSONShape(t *testing.T) {
+ out := ProofSummary{
+ AlertID: "alert_1",
+ IncidentID: "inc_1",
+ ReportHash: "sha256:x",
+ Hashes: map[string]string{"read": "sha256:x"},
+ Evidence: ProofEvidence{TraceID: "trace_1", AlertLinked: true, DependencySignal: true, NextChecks: true},
+ Scorecard: ProofScorecard{
+ RootCauseAccuracy: true,
+ CauseClassificationDependency: true,
+ ReportHashStable: true,
+ TriageLatencyMS: 42,
+ Scenario: "warm-demo",
+ },
+ }
+ raw, err := json.Marshal(out)
+ if err != nil {
+ t.Fatalf("marshal: %v", err)
+ }
+ for _, want := range []string{"alert_1", "inc_1", "trace_1", "cause_classification_dependency", `"triage_latency_ms":42`, `"scenario":"warm-demo"`} {
+ if !strings.Contains(string(raw), want) {
+ t.Fatalf("json missing %q: %s", want, raw)
+ }
+ }
+}
+
+func TestProofSummaryReportHashStableFalse(t *testing.T) {
+ out := ProofSummary{
+ AlertID: "alert_1",
+ IncidentID: "inc_1",
+ ReportHash: "sha256:x",
+ Hashes: map[string]string{"read": "sha256:x", "repeat": "sha256:y"},
+ Scorecard: ProofScorecard{
+ ReportHashStable: false,
+ TriageLatencyMS: 7,
+ Scenario: "warm-demo",
+ },
+ }
+ raw, err := json.Marshal(out)
+ if err != nil {
+ t.Fatalf("marshal: %v", err)
+ }
+ if !strings.Contains(string(raw), `"report_hash_stable":false`) {
+ t.Fatalf("json missing report_hash_stable:false: %s", raw)
+ }
+}
diff --git a/examples/microdemo/signals.go b/examples/microdemo/signals.go
new file mode 100644
index 0000000..b7b6611
--- /dev/null
+++ b/examples/microdemo/signals.go
@@ -0,0 +1,153 @@
+package microdemo
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "fmt"
+ "io"
+ "net/http"
+ "strings"
+ "time"
+)
+
+const demoSignalTimeout = 2 * time.Second
+
+type SignalResult struct {
+ Type string `json:"type"`
+ Service string `json:"service"`
+ Reason string `json:"reason"`
+ Accepted bool `json:"accepted"`
+ Status int `json:"status,omitempty"`
+ SignalID string `json:"signal_id,omitempty"`
+ Error string `json:"error,omitempty"`
+}
+
+type SignalPoster interface {
+ PostDemoSignals(ctx context.Context) []SignalResult
+}
+
+type DemoSignalPoster struct {
+ ingestURL string
+ apiKey string
+ client *http.Client
+ now func() time.Time
+}
+
+func NewDemoSignalPoster(ingestURL, apiKey string) *DemoSignalPoster {
+ return &DemoSignalPoster{
+ ingestURL: strings.TrimRight(strings.TrimSpace(ingestURL), "/"),
+ apiKey: strings.TrimSpace(apiKey),
+ client: &http.Client{Timeout: demoSignalTimeout},
+ now: func() time.Time { return time.Now().UTC() },
+ }
+}
+
+func (p *DemoSignalPoster) PostDemoSignals(ctx context.Context) []SignalResult {
+ specs := []demoSignalSpec{
+ {
+ Type: "deploy",
+ Service: "checkout",
+ Severity: "info",
+ Reason: "demo_checkout_rollout",
+ Message: "Demo checkout rollout before the payment dependency incident.",
+ Resource: map[string]any{"service": "checkout"},
+ Metadata: map[string]any{"version": "demo-v2.1", "demo": "traffic_burst"},
+ },
+ {
+ Type: "dependency",
+ Service: "payment",
+ Severity: "critical",
+ Reason: "payment_gateway_5xx",
+ Message: "Demo payment provider is returning intermittent 5xx responses.",
+ Resource: map[string]any{"service": "payment", "endpoint": "POST /charge"},
+ Metadata: map[string]any{"error_code": "PMT_502", "downstream": "payment", "demo": "traffic_burst"},
+ },
+ }
+
+ results := make([]SignalResult, 0, len(specs))
+ for _, spec := range specs {
+ results = append(results, p.postSignal(ctx, spec))
+ }
+ return results
+}
+
+func (p *DemoSignalPoster) postSignal(ctx context.Context, spec demoSignalSpec) SignalResult {
+ result := SignalResult{Type: spec.Type, Service: spec.Service, Reason: spec.Reason}
+ if p == nil || p.ingestURL == "" {
+ result.Error = "INGEST_URL is not configured"
+ return result
+ }
+
+ body, err := json.Marshal(spec.body(p.now()))
+ if err != nil {
+ result.Error = err.Error()
+ return result
+ }
+ reqCtx, cancel := context.WithTimeout(ctx, demoSignalTimeout)
+ defer cancel()
+ req, err := http.NewRequestWithContext(reqCtx, http.MethodPost, p.ingestURL+"/v1/signals", bytes.NewReader(body))
+ if err != nil {
+ result.Error = err.Error()
+ return result
+ }
+ req.Header.Set("Content-Type", "application/json")
+ if p.apiKey != "" {
+ req.Header.Set("X-API-Key", p.apiKey)
+ }
+
+ client := p.client
+ if client == nil {
+ client = &http.Client{Timeout: demoSignalTimeout}
+ }
+ resp, err := client.Do(req)
+ if err != nil {
+ result.Error = err.Error()
+ return result
+ }
+ defer resp.Body.Close()
+ result.Status = resp.StatusCode
+
+ raw, _ := io.ReadAll(io.LimitReader(resp.Body, 1024))
+ if resp.StatusCode != http.StatusCreated {
+ result.Error = fmt.Sprintf("signal POST returned HTTP %d: %s", resp.StatusCode, strings.TrimSpace(string(raw)))
+ return result
+ }
+ var accepted struct {
+ Signal struct {
+ SignalID string `json:"signal_id"`
+ } `json:"signal"`
+ }
+ if err := json.Unmarshal(raw, &accepted); err != nil {
+ result.Error = "accepted signal response was not valid JSON: " + err.Error()
+ return result
+ }
+ result.Accepted = true
+ result.SignalID = accepted.Signal.SignalID
+ return result
+}
+
+type demoSignalSpec struct {
+ Type string
+ Service string
+ Severity string
+ Reason string
+ Message string
+ Resource map[string]any
+ Metadata map[string]any
+}
+
+func (s demoSignalSpec) body(ts time.Time) map[string]any {
+ return map[string]any{
+ "type": s.Type,
+ "source": "waylog-demo",
+ "service": s.Service,
+ "env": "demo",
+ "severity": s.Severity,
+ "reason": s.Reason,
+ "message": s.Message,
+ "resource": s.Resource,
+ "metadata": s.Metadata,
+ "timestamp": ts.UTC(),
+ }
+}
diff --git a/examples/microdemo/signals_test.go b/examples/microdemo/signals_test.go
new file mode 100644
index 0000000..d1d530b
--- /dev/null
+++ b/examples/microdemo/signals_test.go
@@ -0,0 +1,84 @@
+package microdemo
+
+import (
+ "bytes"
+ "encoding/json"
+ "io"
+ "net/http"
+ "testing"
+ "time"
+)
+
+func TestDemoSignalPosterPostsDeployAndDependencySignals(t *testing.T) {
+ var posted []map[string]any
+ poster := NewDemoSignalPoster("http://ingest.example", "demo-write")
+ poster.client = &http.Client{Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) {
+ if r.URL.Path != "/v1/signals" {
+ t.Fatalf("path = %s, want /v1/signals", r.URL.Path)
+ }
+ if got := r.Header.Get("X-API-Key"); got != "demo-write" {
+ t.Fatalf("api key = %q, want demo-write", got)
+ }
+ var body map[string]any
+ if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
+ t.Fatalf("decode signal: %v", err)
+ }
+ posted = append(posted, body)
+ raw, _ := json.Marshal(map[string]any{
+ "signal": map[string]any{"signal_id": "sig_" + body["type"].(string)},
+ })
+ return &http.Response{
+ StatusCode: http.StatusCreated,
+ Header: http.Header{"Content-Type": []string{"application/json"}},
+ Body: io.NopCloser(bytes.NewReader(raw)),
+ }, nil
+ })}
+ poster.now = func() time.Time { return time.Date(2026, 5, 5, 12, 0, 0, 0, time.UTC) }
+ results := poster.PostDemoSignals(t.Context())
+ if len(results) != 2 {
+ t.Fatalf("results len = %d, want 2", len(results))
+ }
+ for _, result := range results {
+ if !result.Accepted || result.SignalID == "" || result.Status != http.StatusCreated {
+ t.Fatalf("result = %+v", result)
+ }
+ }
+ if len(posted) != 2 {
+ t.Fatalf("posted len = %d, want 2", len(posted))
+ }
+ if posted[0]["type"] != "deploy" || posted[0]["service"] != "checkout" || posted[0]["env"] != "demo" {
+ t.Fatalf("deploy signal = %+v", posted[0])
+ }
+ if posted[1]["type"] != "dependency" || posted[1]["service"] != "payment" || posted[1]["reason"] != "payment_gateway_5xx" {
+ t.Fatalf("dependency signal = %+v", posted[1])
+ }
+ metadata, ok := posted[1]["metadata"].(map[string]any)
+ if !ok || metadata["error_code"] != "PMT_502" {
+ t.Fatalf("dependency metadata = %+v", posted[1]["metadata"])
+ }
+}
+
+func TestDemoSignalPosterReportsNonCreatedResponse(t *testing.T) {
+ poster := NewDemoSignalPoster("http://ingest.example", "")
+ poster.client = &http.Client{Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) {
+ return &http.Response{
+ StatusCode: http.StatusServiceUnavailable,
+ Body: io.NopCloser(bytes.NewBufferString("set SQLITE_PATH to enable signals")),
+ }, nil
+ })}
+ results := poster.PostDemoSignals(t.Context())
+ if len(results) != 2 {
+ t.Fatalf("results len = %d, want 2", len(results))
+ }
+ for _, result := range results {
+ if result.Accepted || result.Status != http.StatusServiceUnavailable || result.Error == "" {
+ t.Fatalf("result = %+v", result)
+ }
+ }
+}
+
+type roundTripFunc func(*http.Request) (*http.Response, error)
+
+func (f roundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) {
+ return f(r)
+}
diff --git a/examples/microdemo/ui.html b/examples/microdemo/ui.html
index 30ede80..b0d3cf0 100644
--- a/examples/microdemo/ui.html
+++ b/examples/microdemo/ui.html
@@ -346,6 +346,150 @@
display: grid;
gap: 16px;
}
+ .proof-grid {
+ display: grid;
+ grid-template-columns: minmax(0, 1fr);
+ gap: 20px;
+ align-items: start;
+ }
+ .proof-hero {
+ display: grid;
+ gap: 16px;
+ padding-bottom: 4px;
+ }
+ .proof-headline {
+ margin: 8px 0 0;
+ font-size: clamp(1.4rem, 3vw, 2rem);
+ line-height: 1.1;
+ letter-spacing: -0.035em;
+ max-width: 22ch;
+ }
+ .proof-summary {
+ display: grid;
+ grid-template-columns: repeat(3, minmax(0, 1fr));
+ gap: 10px;
+ }
+ .proof-panel {
+ display: grid;
+ gap: 12px;
+ }
+ .proof-columns {
+ display: grid;
+ grid-template-columns: minmax(0, 0.95fr) minmax(360px, 1.05fr);
+ gap: 18px;
+ align-items: start;
+ }
+ .proof-checklist {
+ display: grid;
+ gap: 8px;
+ margin: 0;
+ padding: 0;
+ list-style: none;
+ }
+ .proof-checklist li {
+ display: grid;
+ grid-template-columns: 18px minmax(0, 1fr);
+ gap: 9px;
+ align-items: start;
+ padding: 9px 10px;
+ background: var(--surface-2);
+ border: 1px solid var(--line);
+ border-radius: var(--radius);
+ color: var(--muted);
+ font-size: 0.86rem;
+ }
+ .proof-checklist .ok { color: var(--accent); font-family: var(--font-mono); font-weight: 600; }
+ .proof-checklist .bad { color: var(--danger); font-family: var(--font-mono); font-weight: 600; }
+ .proof-table {
+ width: 100%;
+ border-collapse: collapse;
+ background: var(--surface-2);
+ border: 1px solid var(--line);
+ border-radius: var(--radius);
+ overflow: hidden;
+ font-family: var(--font-mono);
+ font-size: 0.74rem;
+ }
+ .proof-table th,
+ .proof-table td {
+ padding: 8px 9px;
+ border-bottom: 1px solid var(--line);
+ text-align: left;
+ vertical-align: top;
+ }
+ .proof-table tr:last-child td { border-bottom: 0; }
+ .proof-table td { overflow-wrap: anywhere; }
+ .proof-table .hash-short { color: var(--ink); font-weight: 600; }
+ .proof-table .hash-full { color: var(--faint); display: block; margin-top: 3px; }
+ .proof-status {
+ color: var(--accent);
+ font-weight: 600;
+ white-space: nowrap;
+ }
+ .proof-status.fail { color: var(--danger); }
+ .proof-metric {
+ display: grid;
+ gap: 4px;
+ padding: 11px 12px;
+ background: var(--surface-2);
+ border: 1px solid var(--line);
+ border-radius: var(--radius);
+ min-width: 0;
+ }
+ .proof-metric span {
+ color: var(--muted);
+ font-family: var(--font-mono);
+ font-size: 0.68rem;
+ letter-spacing: 0.06em;
+ text-transform: uppercase;
+ }
+ .proof-metric strong {
+ font-family: var(--font-mono);
+ font-size: 0.86rem;
+ font-weight: 500;
+ overflow-wrap: anywhere;
+ }
+ .proof-report {
+ max-height: 360px;
+ overflow: auto;
+ white-space: pre-wrap;
+ overflow-wrap: anywhere;
+ background: var(--surface-2);
+ border: 1px solid var(--line);
+ border-radius: var(--radius);
+ padding: 14px;
+ margin: 0;
+ font-family: var(--font-mono);
+ font-size: 0.76rem;
+ line-height: 1.5;
+ }
+ .proof-report-head {
+ display: flex;
+ flex-wrap: wrap;
+ justify-content: space-between;
+ gap: 10px;
+ align-items: end;
+ }
+ .proof-tabs {
+ display: inline-flex;
+ flex-wrap: wrap;
+ gap: 6px;
+ }
+ .proof-tab {
+ border: 1px solid var(--line);
+ background: var(--surface-2);
+ border-radius: var(--radius-sm);
+ color: var(--muted);
+ font-family: var(--font-mono);
+ font-size: 0.68rem;
+ padding: 5px 7px;
+ text-transform: uppercase;
+ letter-spacing: 0.04em;
+ }
+ .proof-tab.active {
+ color: var(--ink);
+ border-color: var(--line-strong);
+ }
.burst-counts {
display: flex;
flex-wrap: wrap;
@@ -479,6 +623,7 @@
@media (max-width: 760px) {
.expectations { grid-template-columns: 1fr; }
.result-grid { grid-template-columns: 1fr; }
+ .proof-summary, .proof-columns { grid-template-columns: 1fr; }
.burst-row { grid-template-columns: 1fr; }
.burst-form { justify-content: flex-start; }
main { padding: 36px 0 64px; }
@@ -542,7 +687,7 @@ Scenarios
+
+
Result
Choose a scenario to send a real request through the demo services.
@@ -613,12 +766,15 @@ Result
const scenarioButtons = Array.from(document.querySelectorAll("[data-scenario]"));
const burstForm = document.getElementById("burst-form");
const burstButton = document.getElementById("burst-run");
+ const proofButton = document.getElementById("proof-run");
const burstControls = Array.from(burstForm.elements);
let burstInFlight = false;
+ let proofInFlight = false;
scenarioButtons.forEach(button => {
button.addEventListener("click", () => purchase(button.dataset.scenario, button));
});
burstForm.addEventListener("submit", runBurst);
+ proofButton.addEventListener("click", runProofLoop);
function esc(value) {
return String(value ?? "").replace(/[&<>"']/g, ch => ({
@@ -627,7 +783,7 @@ Result
}
function setLoading(activeButton, loading) {
- [...scenarioButtons, ...burstControls].forEach(control => {
+ [...scenarioButtons, ...burstControls, proofButton].forEach(control => {
control.disabled = loading;
control.setAttribute("aria-busy", loading ? "true" : "false");
control.querySelector?.(".spinner")?.remove();
@@ -637,6 +793,14 @@ Result
}
}
+ function burstPayloadFromForm() {
+ const formData = new FormData(burstForm);
+ return {
+ requests: Number(formData.get("requests")),
+ concurrency: Number(formData.get("concurrency"))
+ };
+ }
+
async function purchase(scenario, activeButton) {
const result = document.getElementById("result");
setLoading(activeButton, true);
@@ -664,14 +828,10 @@ Result
if (burstInFlight) return;
burstInFlight = true;
setLoading(burstButton, true);
- const formData = new FormData(burstForm);
- const payload = {
- requests: Number(formData.get("requests")),
- concurrency: Number(formData.get("concurrency"))
- };
+ const payload = burstPayloadFromForm();
const result = document.getElementById("result");
result.className = "result-empty";
- result.textContent = "Running production-like traffic mix through the demo chain…";
+ result.textContent = "Posting demo signals and running production-like traffic through the checkout chain…";
try {
const resp = await fetch("/demo/burst", {
method: "POST",
@@ -689,6 +849,30 @@ Result
}
}
+ async function runProofLoop() {
+ if (proofInFlight) return;
+ proofInFlight = true;
+ setLoading(proofButton, true);
+ const result = document.getElementById("result");
+ result.className = "result-empty";
+ result.textContent = "Running alert → incident → triage → reports → scorecard. This takes a few seconds…";
+ try {
+ const resp = await fetch("/demo/proof", {
+ method: "POST",
+ headers: { "Content-Type": "application/json" },
+ body: JSON.stringify(burstPayloadFromForm())
+ });
+ if (!resp.ok) throw new Error(await resp.text() || "proof loop failed");
+ renderProofLoop(await resp.json());
+ } catch (err) {
+ result.className = "result-empty";
+ result.textContent = "Error: " + err.message;
+ } finally {
+ proofInFlight = false;
+ setLoading(proofButton, false);
+ }
+ }
+
async function pollStory(scenario, data) {
if (!data.trace_id) return;
const attempts = 16;
@@ -770,6 +954,11 @@ Result
const samples = (summary.sample_trace_ids || []).slice(0, 3)
.map(id => `Explain ${esc(id.slice(0, 12))}…`)
.join("");
+ const signals = (summary.signals || []).map(signal => {
+ const cls = signal.accepted ? "" : "warn";
+ const label = signal.accepted ? "accepted" : (signal.error ? "unavailable" : "skipped");
+ return `${esc(signal.type)}:${esc(signal.service)} ${esc(label)}`;
+ }).join("");
const result = document.getElementById("result");
result.className = "bracketed";
@@ -777,15 +966,99 @@ Result
Burst captured
${reqLabel} · ${summary.accepted.concurrency}× · ${summary.duration_ms}ms
+
Dashboard should now show an active incident with dependency evidence after the next incident tick.
${counts}
+ ${signals ? `${signals}
` : ""}
`;
}
+
+ function renderProofLoop(proof) {
+ const score = proof.scorecard || {};
+ const evidence = proof.evidence || {};
+ const markdown = proof.reports?.markdown || "Report not available";
+ const hashes = proof.hashes || {};
+ const hashOK = Object.values(hashes).length > 0 && Object.values(hashes).every(value => value === proof.report_hash);
+ const rootCause = "checkout:payment.charge:PMT_502";
+ const fullHash = proof.report_hash || "";
+ const shortHash = shortHashValue(fullHash);
+ const checklist = [
+ ["Alert accepted", Boolean(proof.alert_id), proof.alert_id || "not available"],
+ ["Incident opened", Boolean(proof.incident_id), proof.incident_id || "not available"],
+ ["Triage built", Boolean(proof.report_hash), shortHash || "not available"],
+ ["Read/tool/plan hashes agree", hashOK, "direct read, direct tool, plan template, repeat snapshot"],
+ ["Reports rendered", Boolean(markdown && proof.reports?.slack && proof.reports?.pagerduty), "Markdown, Slack Block Kit, PagerDuty note"],
+ ["Scorecard passed", Boolean(score.root_cause_accuracy && score.cause_classification_dependency && score.report_hash_stable), "root cause, dependency classification, hash stability"]
+ ];
+ const hashRows = [
+ ["read endpoint", hashes.read],
+ ["repeat snapshot", hashes.repeat],
+ ["direct tool", hashes.tool],
+ ["plan template", hashes.plan]
+ ].map(([label, value]) => `| ${esc(label)} | ${esc(shortHashValue(value) || "not available")}${esc(value || "")} | ${value === proof.report_hash ? "pass" : "fail"} |
`).join("");
+ const result = document.getElementById("result");
+ result.className = "bracketed";
+ result.innerHTML = `
+
+
+
Proof loop complete
+
Alert correlated. Root cause identified. Report verified.
+
Waylog correlated an external alert with an active incident, found ${esc(rootCause)}, and produced a cited operator report whose hash was stable across CLI, read, direct tool, and plan-template surfaces in this run.
+
+
+
Root cause${esc(rootCause)}
+
Stable report hash${esc(shortHash || "not available")}
+
Inflation avoided${esc(score.propagated_error_inflation_avoided ?? "not available")} propagated errors
+
+
+
+
+
+ ${checklist.map(([label, ok, detail]) => `- ${ok ? "✓" : "!"}${esc(label)}
${esc(detail)} `).join("")}
+
+
Evidence IDsalert=${esc(proof.alert_id || "not available")} · trace=${esc(evidence.trace_id || "not available")} · incident=${esc(proof.incident_id || "not available")}
+
Evidence completenessalert linked · dependency signal present · next checks ready
+
+ | Surface | Report hash | Status |
+ ${hashRows}
+
+
RCA scorecard${humanBool(score.root_cause_accuracy, "Root cause identified")} · ${humanBool(score.cause_classification_dependency, "Dependency cause confirmed")} · ${humanBool(score.report_hash_stable, "Hash stable")} · time to answer ${esc(score.time_to_answer_ms || 0)}ms
+
+
+
+
+
Operator reportCited, deterministic, LLM-free
+
+ Markdown
+ Slack JSON
+ PagerDuty note
+
+
+
${esc(markdown)}
+
+
+
`;
+ }
+
+ function shortHashValue(value) {
+ value = String(value || "");
+ if (value.length <= 22) return value;
+ if (value.startsWith("sha256:")) return "sha256:" + value.slice(7, 15) + "…" + value.slice(-6);
+ return value.slice(0, 12) + "…" + value.slice(-6);
+ }
+
+ function humanBool(ok, label) {
+ return ok ? label : "Missing " + label.toLowerCase();
+ }