diff --git a/CHANGELOG.md b/CHANGELOG.md index bd4b4bd..0db5c57 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,13 @@ surface is governed by [`COMPATIBILITY.md`](COMPATIBILITY.md). ## [Unreleased] ### Added +- **Prometheus `/metrics` endpoint.** Scrape the daemon's own state: governed runs + by status, halted runs by halt reason, total spend in dollars and tokens, priced + model calls, and the pending-approval queue depth. Plain Prometheus text + exposition (version 0.0.4), authenticated like the rest of the API, served when a + durable store is configured. It's local metrics you scrape — no phone-home, no + prompt content, no PII — and it's hand-rolled, so it adds no dependency. See + [`docs/METRICS.md`](docs/METRICS.md) for the metric list and an example scrape config. - **Shell completions.** `riskkernel completion ` prints a completion script to stdout — tab-complete the top-level commands and their sub-subcommands (`runs list|resume`, `audit export|tools|compliance`, `policy validate|dry-run`, diff --git a/docs/METRICS.md b/docs/METRICS.md new file mode 100644 index 0000000..7d71f44 --- /dev/null +++ b/docs/METRICS.md @@ -0,0 +1,79 @@ +# Metrics + +RiskKernel exposes a Prometheus `/metrics` endpoint describing the daemon's +governed-run state — runs by status, halt reasons, total spend and tokens, and the +human-in-the-loop approval-queue depth. Platform teams scrape it to watch a +reliability tool's own health alongside everything else they run. + +It honors the no-telemetry posture: this is **local** metrics the user scrapes. +Nothing is emitted anywhere; the numbers are derived on the fly from the SQLite +state the user already owns, and no prompt content or PII is exposed. + +## The endpoint + +`GET /metrics` returns the Prometheus text exposition format (version 0.0.4). It's +authenticated like the rest of the API — when `RISKKERNEL_API_TOKEN` is set, send +the bearer token. It's served only when a durable store is configured (the default +SQLite state). + +```bash +curl -s -H "Authorization: Bearer $RISKKERNEL_API_TOKEN" http://localhost:7070/metrics +``` + +## What's exposed + +| Metric | Type | Labels | Meaning | +|---|---|---|---| +| `riskkernel_runs_total` | gauge | `status` | Governed runs by lifecycle status (`running` / `halted` / `cancelled`). | +| `riskkernel_runs_halted_total` | gauge | `reason` | Halted runs by halt reason (`token_budget_exceeded`, `dollar_budget_exceeded`, `loop_budget_exceeded`, `time_budget_exceeded`, `cancelled`). | +| `riskkernel_spend_dollars_total` | counter | — | Total spend in dollars across all runs, summed from the cost ledger. | +| `riskkernel_tokens_total` | counter | — | Total tokens (prompt + completion) across all runs. | +| `riskkernel_model_calls_total` | counter | — | Priced model calls recorded in the cost ledger. | +| `riskkernel_approvals_pending` | gauge | — | Pending human-in-the-loop approvals (the queue depth). | + +Example scrape: + +``` +# HELP riskkernel_runs_total Number of governed runs by lifecycle status. +# TYPE riskkernel_runs_total gauge +riskkernel_runs_total{status="cancelled"} 1 +riskkernel_runs_total{status="halted"} 3 +riskkernel_runs_total{status="running"} 5 +# HELP riskkernel_runs_halted_total Number of halted runs by halt reason. +# TYPE riskkernel_runs_halted_total gauge +riskkernel_runs_halted_total{reason="dollar_budget_exceeded"} 2 +riskkernel_runs_halted_total{reason="loop_budget_exceeded"} 1 +# HELP riskkernel_spend_dollars_total Total spend in dollars across all runs, summed from the cost ledger. +# TYPE riskkernel_spend_dollars_total counter +riskkernel_spend_dollars_total 4.21 +# HELP riskkernel_tokens_total Total tokens (prompt + completion) across all runs. +# TYPE riskkernel_tokens_total counter +riskkernel_tokens_total 182340 +# HELP riskkernel_model_calls_total Total priced model calls recorded in the cost ledger. +# TYPE riskkernel_model_calls_total counter +riskkernel_model_calls_total 96 +# HELP riskkernel_approvals_pending Number of pending human-in-the-loop approvals. +# TYPE riskkernel_approvals_pending gauge +riskkernel_approvals_pending 0 +``` + +## Example scrape config + +Add a job to your `prometheus.yml`: + +```yaml +scrape_configs: + - job_name: riskkernel + scrape_interval: 30s + static_configs: + - targets: ["localhost:7070"] + # Only needed when RISKKERNEL_API_TOKEN is set. + authorization: + type: Bearer + credentials: "" +``` + +For the enforcement overhead a platform team cares about most — the latency +RiskKernel adds in front of each call — see [`PERFORMANCE.md`](PERFORMANCE.md): +the deterministic decision is ~150 ns with zero allocations, so it never shows up +as a meaningful contributor next to the model call itself. diff --git a/internal/httpapi/metrics.go b/internal/httpapi/metrics.go new file mode 100644 index 0000000..58adb61 --- /dev/null +++ b/internal/httpapi/metrics.go @@ -0,0 +1,205 @@ +package httpapi + +import ( + "context" + "net/http" + "sort" + "strconv" + "strings" + + "github.com/prashar32/riskkernel/internal/storage" +) + +// Prometheus text exposition format, version 0.0.4. We hand-roll it rather than +// pull in prometheus/client_golang: the surface is tiny (a handful of run/spend +// gauges scraped on demand), and a minimal dependency graph is a project rule. +// +// This is local metrics the user scrapes — it honors the no-telemetry posture: +// nothing is emitted anywhere, the numbers are derived on the fly from the +// SQLite state the user already owns, and no prompt content or PII is exposed. +const metricsContentType = "text/plain; version=0.0.4; charset=utf-8" + +// handleMetrics implements GET /metrics — a Prometheus scrape of the daemon's +// governed-run state, sourced from the durable Store. Registered only when a +// store is available (see Handler). +func (s *Server) handleMetrics(w http.ResponseWriter, r *http.Request) { + store := s.runs.Store() + if store == nil { + http.Error(w, "no durable store configured", http.StatusServiceUnavailable) + return + } + + var mw metricsWriter + if err := s.collectMetrics(r.Context(), store, &mw); err != nil { + http.Error(w, "collect metrics: "+err.Error(), http.StatusInternalServerError) + return + } + + w.Header().Set("Content-Type", metricsContentType) + w.WriteHeader(http.StatusOK) + _, _ = w.Write(mw.bytes()) +} + +// collectMetrics derives the exposition body from the store. Kept separate from +// the HTTP plumbing so it's directly unit-testable. +func (s *Server) collectMetrics(ctx context.Context, store storage.Store, mw *metricsWriter) error { + runRecs, err := store.ListRuns(ctx) + if err != nil { + return err + } + + // riskkernel_runs_total{status="..."} — the live count of runs by lifecycle + // status (running / halted / cancelled). A gauge, not a counter: runs change + // status over their life, so the value can go down as well as up. + byStatus := map[string]int{} + for _, run := range runRecs { + byStatus[run.Status]++ + } + mw.help("riskkernel_runs_total", "Number of governed runs by lifecycle status.") + mw.typ("riskkernel_runs_total", "gauge") + if len(byStatus) == 0 { + // Emit a zero so the series exists on a fresh daemon (a scrape never sees + // an empty metric and reads it as "scrape failed"). + mw.sample("riskkernel_runs_total", map[string]string{"status": "running"}, 0) + } + for _, status := range sortedKeys(byStatus) { + mw.sample("riskkernel_runs_total", map[string]string{"status": status}, float64(byStatus[status])) + } + + // riskkernel_runs_halted_total{reason="..."} — halted runs broken out by halt + // reason (token/dollar/loop/time budget, cancelled). The "why did it stop" + // view platform teams want next to the status counts. + byReason := map[string]int{} + for _, run := range runRecs { + if run.HaltReason != "" { + byReason[run.HaltReason]++ + } + } + mw.help("riskkernel_runs_halted_total", "Number of halted runs by halt reason.") + mw.typ("riskkernel_runs_halted_total", "gauge") + for _, reason := range sortedKeys(byReason) { + mw.sample("riskkernel_runs_halted_total", map[string]string{"reason": reason}, float64(byReason[reason])) + } + + // Aggregate spend across all runs, summed from the per-run ledger totals (the + // auditable source of truth for cost). riskkernel_spend_dollars_total and + // riskkernel_tokens_total are monotonic over a run's life, so counters. + var totalDollars float64 + var totalTokens, totalCalls int64 + for _, run := range runRecs { + t, err := store.Totals(ctx, run.ID) + if err != nil { + return err + } + totalDollars += t.Dollars + totalTokens += t.PromptTokens + t.CompletionTokens + totalCalls += t.Calls + } + mw.help("riskkernel_spend_dollars_total", "Total spend in dollars across all runs, summed from the cost ledger.") + mw.typ("riskkernel_spend_dollars_total", "counter") + mw.sample("riskkernel_spend_dollars_total", nil, totalDollars) + + mw.help("riskkernel_tokens_total", "Total tokens (prompt + completion) across all runs.") + mw.typ("riskkernel_tokens_total", "counter") + mw.sample("riskkernel_tokens_total", nil, float64(totalTokens)) + + mw.help("riskkernel_model_calls_total", "Total priced model calls recorded in the cost ledger.") + mw.typ("riskkernel_model_calls_total", "counter") + mw.sample("riskkernel_model_calls_total", nil, float64(totalCalls)) + + // riskkernel_approvals_pending — the human-in-the-loop queue depth. A gauge: + // it rises and falls as approvals are requested and resolved. + pending, err := store.ListApprovals(ctx, storage.ApprovalPending) + if err != nil { + return err + } + mw.help("riskkernel_approvals_pending", "Number of pending human-in-the-loop approvals.") + mw.typ("riskkernel_approvals_pending", "gauge") + mw.sample("riskkernel_approvals_pending", nil, float64(len(pending))) + + return nil +} + +// sortedKeys returns a map's keys sorted, for deterministic exposition output +// (stable scrapes and stable tests). +func sortedKeys(m map[string]int) []string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + return keys +} + +// metricsWriter builds a Prometheus exposition body. Callers write a metric's +// HELP/TYPE lines once, then its samples. +type metricsWriter struct { + b strings.Builder +} + +func (mw *metricsWriter) help(name, text string) { + mw.b.WriteString("# HELP ") + mw.b.WriteString(name) + mw.b.WriteByte(' ') + mw.b.WriteString(escapeHelp(text)) + mw.b.WriteByte('\n') +} + +func (mw *metricsWriter) typ(name, t string) { + mw.b.WriteString("# TYPE ") + mw.b.WriteString(name) + mw.b.WriteByte(' ') + mw.b.WriteString(t) + mw.b.WriteByte('\n') +} + +// sample writes one `name{labels} value` line. Labels are sorted by key so the +// output is deterministic regardless of map iteration order. +func (mw *metricsWriter) sample(name string, labels map[string]string, value float64) { + mw.b.WriteString(name) + if len(labels) > 0 { + keys := make([]string, 0, len(labels)) + for k := range labels { + keys = append(keys, k) + } + sort.Strings(keys) + mw.b.WriteByte('{') + for i, k := range keys { + if i > 0 { + mw.b.WriteByte(',') + } + mw.b.WriteString(k) + mw.b.WriteString(`="`) + mw.b.WriteString(escapeLabelValue(labels[k])) + mw.b.WriteByte('"') + } + mw.b.WriteByte('}') + } + mw.b.WriteByte(' ') + mw.b.WriteString(formatValue(value)) + mw.b.WriteByte('\n') +} + +func (mw *metricsWriter) bytes() []byte { return []byte(mw.b.String()) } + +// formatValue renders a float in the Go default ('g') format; whole numbers come +// out without a trailing ".0", which is valid Prometheus and matches what +// counters of whole units (runs, tokens, calls) should look like. +func formatValue(v float64) string { + return strconv.FormatFloat(v, 'g', -1, 64) +} + +// escapeHelp escapes a HELP string per the exposition format: backslash and +// newline only. +func escapeHelp(s string) string { + s = strings.ReplaceAll(s, `\`, `\\`) + return strings.ReplaceAll(s, "\n", `\n`) +} + +// escapeLabelValue escapes a label value per the exposition format: backslash, +// double-quote, and newline. +func escapeLabelValue(s string) string { + s = strings.ReplaceAll(s, `\`, `\\`) + s = strings.ReplaceAll(s, `"`, `\"`) + return strings.ReplaceAll(s, "\n", `\n`) +} diff --git a/internal/httpapi/metrics_test.go b/internal/httpapi/metrics_test.go new file mode 100644 index 0000000..a028453 --- /dev/null +++ b/internal/httpapi/metrics_test.go @@ -0,0 +1,167 @@ +package httpapi + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/prashar32/riskkernel/internal/governor" + "github.com/prashar32/riskkernel/internal/runs" + "github.com/prashar32/riskkernel/internal/storage" +) + +// seedMetricsRuns populates the store, via the manager, with a representative +// mix: a running run with spend, a run halted on the dollar budget, and a +// cancelled run. Returns the manager's store for any direct seeding. +func seedMetricsRuns(t *testing.T, mgr *runs.Manager) { + t.Helper() + + // A running run with one recorded call → spend + tokens on the ledger. + r := mgr.Create(runs.CreateOptions{ID: "run-running"}) + step, err := r.BeginStep() + if err != nil { + t.Fatal(err) + } + if err := r.RecordCall(runs.Call{ + StepIndex: step, Provider: "anthropic", Model: "claude-sonnet-4-5", + PromptTokens: 100, CompletionTokens: 50, Dollars: 0.25, Priced: true, ResponseID: "a", + }); err != nil { + t.Fatal(err) + } + + // A run that halts on the dollar budget: a tiny ceiling, then a call over it. + budget := governor.Budget{Dollars: 0.01} + h := mgr.Create(runs.CreateOptions{ID: "run-halted", Budget: &budget}) + hStep, _ := h.BeginStep() + if err := h.RecordCall(runs.Call{ + StepIndex: hStep, Provider: "anthropic", Model: "claude-sonnet-4-5", + PromptTokens: 10, CompletionTokens: 10, Dollars: 1.00, Priced: true, ResponseID: "b", + }); err == nil { + t.Fatal("expected the over-budget call to halt the run") + } + + // A cancelled run (kill switch). + c := mgr.Create(runs.CreateOptions{ID: "run-cancelled"}) + c.Cancel() +} + +func TestMetricsEndpoint(t *testing.T) { + srv, mgr, _ := newTestServer(t, "") + seedMetricsRuns(t, mgr) + + // A pending human-in-the-loop approval, seeded directly into the store. + store := mgr.Store() + if err := store.CreateApproval(context.Background(), storage.ApprovalRecord{ + ID: "appr-1", RunID: "run-running", StepIndex: 1, Tool: "mcp://shell", + SideEffect: "exec", Status: storage.ApprovalPending, CreatedAt: time.Now(), + }); err != nil { + t.Fatal(err) + } + + h := srv.Handler() + w := httptest.NewRecorder() + h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/metrics", nil)) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, body=%s", w.Code, w.Body.String()) + } + if ct := w.Header().Get("Content-Type"); ct != metricsContentType { + t.Fatalf("Content-Type = %q, want %q", ct, metricsContentType) + } + + body := w.Body.String() + + // Every metric carries its HELP/TYPE preamble. + for _, want := range []string{ + "# HELP riskkernel_runs_total ", + "# TYPE riskkernel_runs_total gauge", + "# HELP riskkernel_runs_halted_total ", + "# TYPE riskkernel_runs_halted_total gauge", + "# TYPE riskkernel_spend_dollars_total counter", + "# TYPE riskkernel_tokens_total counter", + "# TYPE riskkernel_model_calls_total counter", + "# TYPE riskkernel_approvals_pending gauge", + } { + if !strings.Contains(body, want) { + t.Errorf("metrics body missing preamble %q\n---\n%s", want, body) + } + } + + // Run counts by status: one each of running / halted / cancelled. + for _, want := range []string{ + `riskkernel_runs_total{status="running"} 1`, + `riskkernel_runs_total{status="halted"} 1`, + `riskkernel_runs_total{status="cancelled"} 1`, + `riskkernel_runs_halted_total{reason="dollar_budget_exceeded"} 1`, + } { + if !strings.Contains(body, want) { + t.Errorf("metrics body missing line %q\n---\n%s", want, body) + } + } + + // Spend / tokens / calls aggregate the ledger. The running run priced one call + // at $0.25 / 150 tokens; the halted run's over-budget call is still recorded + // ($1.00 / 20 tokens), so totals are $1.25, 170 tokens, 2 calls. + for _, want := range []string{ + "riskkernel_spend_dollars_total 1.25", + "riskkernel_tokens_total 170", + "riskkernel_model_calls_total 2", + "riskkernel_approvals_pending 1", + } { + if !strings.Contains(body, want) { + t.Errorf("metrics body missing line %q\n---\n%s", want, body) + } + } +} + +// On a fresh daemon the runs_total series still exists (as a zero), so a scrape +// never reads an empty metric as a failed scrape. +func TestMetricsEndpoint_Empty(t *testing.T) { + srv, _, _ := newTestServer(t, "") + h := srv.Handler() + + w := httptest.NewRecorder() + h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/metrics", nil)) + if w.Code != http.StatusOK { + t.Fatalf("status = %d", w.Code) + } + body := w.Body.String() + for _, want := range []string{ + `riskkernel_runs_total{status="running"} 0`, + "riskkernel_spend_dollars_total 0", + "riskkernel_tokens_total 0", + "riskkernel_approvals_pending 0", + } { + if !strings.Contains(body, want) { + t.Errorf("empty metrics body missing line %q\n---\n%s", want, body) + } + } +} + +func TestMetricsEndpoint_AuthRequired(t *testing.T) { + srv, mgr, _ := newTestServer(t, "sekret") + mgr.Create(runs.CreateOptions{ID: "run-1"}) + h := srv.Handler() + + // No token → 401. + w := httptest.NewRecorder() + h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/metrics", nil)) + if w.Code != http.StatusUnauthorized { + t.Fatalf("no-token status = %d, want 401", w.Code) + } + + // Correct token → 200 with the exposition Content-Type. + req := httptest.NewRequest(http.MethodGet, "/metrics", nil) + req.Header.Set("Authorization", "Bearer sekret") + w = httptest.NewRecorder() + h.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("authed status = %d, body=%s", w.Code, w.Body.String()) + } + if ct := w.Header().Get("Content-Type"); ct != metricsContentType { + t.Fatalf("Content-Type = %q, want %q", ct, metricsContentType) + } +} diff --git a/internal/httpapi/server.go b/internal/httpapi/server.go index 3b4edc5..e6ddbe6 100644 --- a/internal/httpapi/server.go +++ b/internal/httpapi/server.go @@ -100,6 +100,13 @@ func (s *Server) Handler() http.Handler { mux.HandleFunc("PUT /v1/memory/facts", s.requireAuth(s.handlePutFact)) } + // Prometheus scrape of the daemon's governed-run state (runs by status, halt + // reasons, spend, tokens, approval-queue depth). Authenticated like the rest + // of the API and only registered when a durable store is the source of truth. + if s.runs != nil && s.runs.Store() != nil { + mux.HandleFunc("GET /metrics", s.requireAuth(s.handleMetrics)) + } + return s.recoverer(mux) }