diff --git a/CHANGELOG.md b/CHANGELOG.md index dd3c863..1459dca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,17 @@ surface is governed by [`COMPATIBILITY.md`](COMPATIBILITY.md). ## [Unreleased] +### Added +- **Point a provider at a custom upstream.** Set `RISKKERNEL_OPENAI_BASE_URL` or + `RISKKERNEL_ANTHROPIC_BASE_URL` to route that provider through an OpenAI-compatible + gateway, a corporate proxy, or a local mock (e.g. for benchmarking) instead of its + default API endpoint. RiskKernel-namespaced so it never collides with the + caller-facing `OPENAI_BASE_URL` used to point an app *at* RiskKernel. +- **A reproducible cost benchmark** ([`benchmark/`](benchmark)) — runs the same + looping agent with and without a RiskKernel dollar budget against a deterministic + mock provider, and reports the spend saved straight from the cost ledger. Key-free + and tunable: `python3 benchmark/benchmark.py`. + ## [0.4.0] - 2026-06-07 Observability, rounded out: tool-call governance now shows up in your traces, a diff --git a/benchmark/.gitignore b/benchmark/.gitignore new file mode 100644 index 0000000..bc15a97 --- /dev/null +++ b/benchmark/.gitignore @@ -0,0 +1,3 @@ +# Generated by benchmark.py +results.json +.bench-data/ diff --git a/benchmark/README.md b/benchmark/README.md new file mode 100644 index 0000000..38a662a --- /dev/null +++ b/benchmark/README.md @@ -0,0 +1,77 @@ +# Cost benchmark — dollars saved on a runaway loop + +A reproducible measurement of the headline claim: **RiskKernel caps the cost of a +runaway agent.** The same looping agent runs twice against a deterministic mock +provider — once with no governance, once through RiskKernel with a hard dollar +budget — and we compare the spend. + +``` + RiskKernel cost benchmark — runaway loop + ------------------------------------------------------ + loop length (N) 50 + dollar budget $0.25 + per-call cost $0.0125 (gpt-4o, from RiskKernel's ledger) + ------------------------------------------------------ + calls spend + baseline (no governance) 50 $0.6250 + governed (RiskKernel) 20 $0.2500 + ------------------------------------------------------ + dollars saved $0.3750 (60%) + stopped by dollar_budget_exceeded +``` + +## Run it + +```bash +go install github.com/prashar32/riskkernel/cmd/riskkernel@latest # or: make build +python3 benchmark/benchmark.py +``` + +No API key, no real spend. Tunable via env: `N` (loop length), `BUDGET` (dollar +ceiling), `RK_BIN` (path to the binary, default `riskkernel` on `PATH`). + +## Methodology (why the number is honest) + +The benchmark is built so the **only** difference between the two runs is whether +RiskKernel's budget stopped the loop: + +- **Deterministic provider.** [`mock_provider.py`](mock_provider.py) is a stand-in + for the OpenAI Chat Completions API that returns a *fixed* token usage on every + call (1000 in / 1000 out). No network variance, no real money, exactly + reproducible. RiskKernel reaches it via the namespaced + `RISKKERNEL_OPENAI_BASE_URL` override — the agent never touches a real provider. +- **Real prices, pinned.** [`pricing.json`](pricing.json) pins `gpt-4o` at its list + price ($2.50 / $10.00 per 1M input / output tokens). Per call = + `1000·2.50/1e6 + 1000·10.00/1e6 = $0.0125`. +- **The governed spend is measured, not modelled.** It's read straight from + RiskKernel's own cost ledger (`GET /v1/runs/{id}` → `usage.dollars`). RiskKernel + meters each call and halts the run *pre-call* once the next call would exceed the + budget — here at exactly `20 × $0.0125 = $0.25`. +- **The baseline uses that same per-call price** across the full loop, so the two + numbers are directly comparable. + +## What the number actually says + +The governed run's spend is **capped at the budget regardless of how long the +runaway would have continued.** The baseline grows without bound: + +| If the runaway loops… | Baseline spend | Governed spend | Saved | +|---|---|---|---| +| 50× | $0.63 | **$0.25** | $0.38 | +| 1,000× | $12.50 | **$0.25** | $12.25 | +| 10,000× | $125.00 | **$0.25** | $124.75 | + +So "dollars saved" is a function of how far the loop would have run before a human +noticed — and RiskKernel's guarantee is the **flat ceiling**, not a fixed percentage. + +## Scope & honesty + +- This measures the **cost-ceiling** dimension. The **crash-recovery** dimension — + `kill -9` mid-run, resume without re-spending — is demonstrated end-to-end in + [`examples/kill-9-resume`](../examples/kill-9-resume); a *timed* recovery + benchmark is a planned addition here. +- It deliberately removes provider latency/variance to isolate the governance + effect. The enforcement overhead RiskKernel itself adds is small and measured + separately; this harness is about dollars, not milliseconds. +- The mock and pricing are in this directory — inspect and change them. Nothing is + hidden behind a wrapper. diff --git a/benchmark/benchmark.py b/benchmark/benchmark.py new file mode 100644 index 0000000..1f0b029 --- /dev/null +++ b/benchmark/benchmark.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python3 +"""RiskKernel cost benchmark — reproducible "dollars saved" on a runaway loop. + +The SAME looping agent runs twice against a deterministic mock provider: + + 1. Baseline — calls the provider directly, no governance. A stuck loop makes + all N calls and spends the full amount. + 2. Governed — calls through RiskKernel with a hard dollar budget. RiskKernel + meters cost per call and halts the run at the ceiling. + +The mock returns fixed token usage, so cost is exact and reproducible. The +governed run's spend is read from RiskKernel's own ledger (GET /v1/runs/{id}); +the baseline spend is that same per-call price across the full loop. No API key, +no real money. + +Run: python3 benchmark/benchmark.py +Env: RK_BIN (default "riskkernel"), N, BUDGET, PORT, MOCK_PORT +""" +import json +import os +import shutil +import subprocess +import sys +import time +import urllib.error +import urllib.request + +HERE = os.path.dirname(os.path.abspath(__file__)) +RK_BIN = os.environ.get("RK_BIN", "riskkernel") +N = int(os.environ.get("N", "50")) # runaway loop length +BUDGET = float(os.environ.get("BUDGET", "0.25")) # dollar ceiling for the governed run +PORT = int(os.environ.get("PORT", "7070")) +MOCK_PORT = int(os.environ.get("MOCK_PORT", "9099")) +MODEL = "gpt-4o" +RK_URL = f"http://127.0.0.1:{PORT}" +MOCK_URL = f"http://127.0.0.1:{MOCK_PORT}" +CHAT_BODY = json.dumps({"model": MODEL, "messages": [{"role": "user", "content": "continue"}]}).encode() + + +def post(url, headers=None, timeout=15): + req = urllib.request.Request(url, data=CHAT_BODY, method="POST", + headers={"Content-Type": "application/json", **(headers or {})}) + try: + with urllib.request.urlopen(req, timeout=timeout) as r: + return r.status, b"" + except urllib.error.HTTPError as e: + return e.code, e.read() + except Exception: + return 0, b"" # connection refused while a server is still starting + + +def wait_get(url, timeout=25): + deadline = time.time() + timeout + while time.time() < deadline: + try: + urllib.request.urlopen(url, timeout=1) + return True + except Exception: + time.sleep(0.2) + return False + + +def wait_post(url, timeout=10): + deadline = time.time() + timeout + while time.time() < deadline: + if post(url, timeout=1)[0] == 200: + return True + time.sleep(0.2) + return False + + +def run_loop(url, headers): + """Make up to N calls; stop early on the first non-200 (a budget halt).""" + start = time.time() + calls, halt = 0, "" + for _ in range(N): + code, body = post(url, headers=headers) + if code != 200: + try: + halt = json.loads(body).get("code") or f"http {code}" + except Exception: + halt = f"http {code}" + break + calls += 1 + return calls, halt, time.time() - start + + +def main(): + data_dir = os.path.join(HERE, ".bench-data") + shutil.rmtree(data_dir, ignore_errors=True) + mock = subprocess.Popen([sys.executable, os.path.join(HERE, "mock_provider.py"), str(MOCK_PORT)]) + env = dict(os.environ, + RISKKERNEL_PORT=str(PORT), + RISKKERNEL_DATA_DIR=data_dir, + RISKKERNEL_DEFAULT_PROVIDER="openai", + OPENAI_API_KEY="bench-dummy-key", + RISKKERNEL_OPENAI_BASE_URL=MOCK_URL, + RISKKERNEL_DEFAULT_DOLLARS=str(BUDGET), + RISKKERNEL_DEFAULT_LOOPS="0", # only the dollar budget should stop the loop + RISKKERNEL_DEFAULT_SECONDS="0", + RISKKERNEL_PRICING_FILE=os.path.join(HERE, "pricing.json")) + rk = subprocess.Popen([RK_BIN, "serve"], env=env) + try: + if not wait_post(f"{MOCK_URL}/v1/chat/completions"): + sys.exit("mock provider did not come up") + if not wait_get(f"{RK_URL}/healthz"): + sys.exit("riskkernel did not come up") + + gov_calls, halt, gov_time = run_loop(f"{RK_URL}/v1/chat/completions", + {"X-RiskKernel-Run-Id": "bench-governed"}) + run = json.loads(urllib.request.urlopen(f"{RK_URL}/v1/runs/bench-governed", timeout=5).read()) + gov_dollars = float(run.get("usage", {}).get("dollars", 0.0)) + + base_calls, _, base_time = run_loop(f"{MOCK_URL}/v1/chat/completions", None) + + per_call = gov_dollars / gov_calls if gov_calls else 0.0 + base_dollars = base_calls * per_call + saved = base_dollars - gov_dollars + pct = (saved / base_dollars * 100) if base_dollars else 0.0 + + print("\n RiskKernel cost benchmark — runaway loop") + print(" " + "-" * 54) + print(f" loop length (N) {N}") + print(f" dollar budget ${BUDGET:.2f}") + print(f" per-call cost ${per_call:.4f} ({MODEL}, from RiskKernel's ledger)") + print(" " + "-" * 54) + print(f" {'':24}{'calls':>7}{'spend':>13}") + print(f" {'baseline (no governance)':24}{base_calls:>7}{'$'+format(base_dollars, '.4f'):>13}") + print(f" {'governed (RiskKernel)':24}{gov_calls:>7}{'$'+format(gov_dollars, '.4f'):>13}") + print(" " + "-" * 54) + print(f" dollars saved ${saved:.4f} ({pct:.0f}%)") + print(f" stopped by {halt}") + print(f" wall time base / governed {base_time:.2f}s / {gov_time:.2f}s") + print() + + with open(os.path.join(HERE, "results.json"), "w") as f: + json.dump({ + "loop_length": N, "budget_dollars": BUDGET, "per_call_dollars": round(per_call, 6), + "baseline_calls": base_calls, "baseline_dollars": round(base_dollars, 6), + "governed_calls": gov_calls, "governed_dollars": round(gov_dollars, 6), + "dollars_saved": round(saved, 6), "percent_saved": round(pct, 1), "halt_reason": halt, + }, f, indent=2) + f.write("\n") + finally: + for p in (rk, mock): + p.terminate() + try: + p.wait(5) + except Exception: + p.kill() + shutil.rmtree(data_dir, ignore_errors=True) + + +if __name__ == "__main__": + main() diff --git a/benchmark/mock_provider.py b/benchmark/mock_provider.py new file mode 100644 index 0000000..df8f3dc --- /dev/null +++ b/benchmark/mock_provider.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +"""A deterministic mock of the OpenAI Chat Completions API for the benchmark. + +Every call returns a FIXED token usage, so the cost RiskKernel meters is exactly +reproducible — no API key, no real spend, no variance. This is what makes the +"dollars saved" number defensible: the only thing that differs between the two +runs is whether RiskKernel's budget stopped the loop. + +Usage: python3 mock_provider.py [port] [prompt_tokens] [completion_tokens] +""" +import json +import sys +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +PORT = int(sys.argv[1]) if len(sys.argv) > 1 else 9099 +PROMPT_TOKENS = int(sys.argv[2]) if len(sys.argv) > 2 else 1000 +COMPLETION_TOKENS = int(sys.argv[3]) if len(sys.argv) > 3 else 1000 + + +class Handler(BaseHTTPRequestHandler): + def do_POST(self): + length = int(self.headers.get("Content-Length", 0)) + try: + req = json.loads(self.rfile.read(length) or b"{}") + except Exception: + req = {} + model = req.get("model", "gpt-4o") + out = json.dumps({ + "id": "chatcmpl-bench", + "object": "chat.completion", + "model": model, + "choices": [{ + "index": 0, + "message": {"role": "assistant", "content": "...still looping..."}, + "finish_reason": "stop", + }], + "usage": { + "prompt_tokens": PROMPT_TOKENS, + "completion_tokens": COMPLETION_TOKENS, + "total_tokens": PROMPT_TOKENS + COMPLETION_TOKENS, + }, + }).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(out))) + self.end_headers() + self.wfile.write(out) + + def log_message(self, *args): # keep the benchmark output clean + pass + + +if __name__ == "__main__": + ThreadingHTTPServer(("127.0.0.1", PORT), Handler).serve_forever() diff --git a/benchmark/pricing.json b/benchmark/pricing.json new file mode 100644 index 0000000..5898bb6 --- /dev/null +++ b/benchmark/pricing.json @@ -0,0 +1,3 @@ +{ + "gpt-4o": { "inputPerM": 2.50, "outputPerM": 10.00 } +} diff --git a/internal/app/bootstrap.go b/internal/app/bootstrap.go index 3a5aea0..2af6c7f 100644 --- a/internal/app/bootstrap.go +++ b/internal/app/bootstrap.go @@ -169,12 +169,12 @@ func BuildRegistry(cfg *config.Config) (*provider.Registry, error) { // key is present; Bedrock/Ollama are stubs config can name before they're built // out. ps := []provider.Provider{ - provider.NewAnthropic(cfg.AnthropicAPIKey), + provider.NewAnthropic(cfg.AnthropicAPIKey).WithBaseURL(cfg.AnthropicBaseURL), provider.NewBedrock(), provider.NewOllama("http://localhost:11434"), } if cfg.OpenAIAPIKey != "" { - ps = append(ps, provider.NewOpenAI(cfg.OpenAIAPIKey)) + ps = append(ps, provider.NewOpenAI(cfg.OpenAIAPIKey).WithBaseURL(cfg.OpenAIBaseURL)) } return provider.NewRegistry(cfg.DefaultProvider, ps...) diff --git a/internal/config/config.go b/internal/config/config.go index 1aceb84..0ecca3c 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -47,6 +47,14 @@ type Config struct { AnthropicAPIKey string // ANTHROPIC_API_KEY OpenAIAPIKey string // OPENAI_API_KEY + // Provider upstream-base overrides — point RiskKernel's provider at a gateway, + // a corporate proxy, or a local mock. RiskKernel-namespaced on purpose: the + // bare OPENAI_BASE_URL / ANTHROPIC_BASE_URL are what a caller sets to point + // *at* RiskKernel, so reusing them here would collide (RiskKernel forwarding to + // itself in a shared shell). Empty uses the provider's default endpoint. + AnthropicBaseURL string // RISKKERNEL_ANTHROPIC_BASE_URL + OpenAIBaseURL string // RISKKERNEL_OPENAI_BASE_URL + // DefaultBudget is applied to runs created without an explicit budget — e.g. // proxy calls that supply only a run-id. Any zero field is unlimited. When no // RISKKERNEL_DEFAULT_* variable is set at all, conservative safe defaults are @@ -178,15 +186,17 @@ func Load() (*Config, error) { } cfg := &Config{ - Port: port, - DataDir: getenvDefault("RISKKERNEL_DATA_DIR", "./data"), - APIToken: os.Getenv("RISKKERNEL_API_TOKEN"), - DefaultProvider: getenvDefault("RISKKERNEL_DEFAULT_PROVIDER", "anthropic"), - AnthropicAPIKey: os.Getenv("ANTHROPIC_API_KEY"), - OpenAIAPIKey: os.Getenv("OPENAI_API_KEY"), - DefaultBudget: budget, - PricingFile: os.Getenv("RISKKERNEL_PRICING_FILE"), - OTel: loadOTel(), + Port: port, + DataDir: getenvDefault("RISKKERNEL_DATA_DIR", "./data"), + APIToken: os.Getenv("RISKKERNEL_API_TOKEN"), + DefaultProvider: getenvDefault("RISKKERNEL_DEFAULT_PROVIDER", "anthropic"), + AnthropicAPIKey: os.Getenv("ANTHROPIC_API_KEY"), + OpenAIAPIKey: os.Getenv("OPENAI_API_KEY"), + AnthropicBaseURL: os.Getenv("RISKKERNEL_ANTHROPIC_BASE_URL"), + OpenAIBaseURL: os.Getenv("RISKKERNEL_OPENAI_BASE_URL"), + DefaultBudget: budget, + PricingFile: os.Getenv("RISKKERNEL_PRICING_FILE"), + OTel: loadOTel(), Approval: ApprovalConfig{ DefaultSafe: envBoolDefault("RISKKERNEL_APPROVAL_DEFAULT_SAFE", true), WebhookURL: os.Getenv("RISKKERNEL_APPROVAL_WEBHOOK"), diff --git a/internal/config/config_test.go b/internal/config/config_test.go index e54d64a..76bde25 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -206,6 +206,21 @@ func TestLoad_OTLPHeaders(t *testing.T) { } } +func TestLoad_ProviderBaseURLs(t *testing.T) { + withCleanEnv(t) + chdirTemp(t) + t.Setenv("RISKKERNEL_ANTHROPIC_BASE_URL", "http://localhost:9001") + t.Setenv("RISKKERNEL_OPENAI_BASE_URL", "http://localhost:9002") + + cfg, err := Load() + if err != nil { + t.Fatalf("Load: %v", err) + } + if cfg.AnthropicBaseURL != "http://localhost:9001" || cfg.OpenAIBaseURL != "http://localhost:9002" { + t.Errorf("base URLs = %q / %q", cfg.AnthropicBaseURL, cfg.OpenAIBaseURL) + } +} + // --- helpers --- // withCleanEnv clears the env vars Load reads so tests are hermetic. t.Setenv @@ -218,6 +233,7 @@ func withCleanEnv(t *testing.T) { "RISKKERNEL_DEFAULT_LOOPS", "RISKKERNEL_DEFAULT_SECONDS", "OTEL_EXPORTER_OTLP_ENDPOINT", "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", "OTEL_EXPORTER_OTLP_HEADERS", "OTEL_EXPORTER_OTLP_TRACES_HEADERS", + "RISKKERNEL_ANTHROPIC_BASE_URL", "RISKKERNEL_OPENAI_BASE_URL", } { t.Setenv(k, "") os.Unsetenv(k) diff --git a/internal/provider/anthropic.go b/internal/provider/anthropic.go index 8ff016a..3606d9d 100644 --- a/internal/provider/anthropic.go +++ b/internal/provider/anthropic.go @@ -39,6 +39,16 @@ func NewAnthropic(apiKey string) *Anthropic { } } +// WithBaseURL overrides the API base — point it at a proxy that fronts Anthropic +// or a local mock (e.g. for benchmarking). Empty keeps the default. Returns the +// provider for chaining. +func (a *Anthropic) WithBaseURL(url string) *Anthropic { + if url != "" { + a.baseURL = strings.TrimRight(url, "/") + } + return a +} + // Name returns the stable provider identifier. func (a *Anthropic) Name() string { return "anthropic" } diff --git a/internal/provider/openai.go b/internal/provider/openai.go index 8370400..da79465 100644 --- a/internal/provider/openai.go +++ b/internal/provider/openai.go @@ -31,6 +31,16 @@ func NewOpenAI(apiKey string) *OpenAI { } } +// WithBaseURL overrides the API base — point it at an OpenAI-compatible gateway, +// a corporate proxy, or a local mock (e.g. for benchmarking). Empty keeps the +// default. Returns the provider for chaining. +func (o *OpenAI) WithBaseURL(url string) *OpenAI { + if url != "" { + o.baseURL = strings.TrimRight(url, "/") + } + return o +} + // Name returns the stable provider identifier. func (o *OpenAI) Name() string { return "openai" } diff --git a/internal/provider/openai_test.go b/internal/provider/openai_test.go index 78f732e..020106f 100644 --- a/internal/provider/openai_test.go +++ b/internal/provider/openai_test.go @@ -56,6 +56,41 @@ func TestOpenAIChat_Success(t *testing.T) { } } +func TestProviderWithBaseURL(t *testing.T) { + // Field logic: trailing slash trimmed; empty keeps the default. + if got := NewOpenAI("k").WithBaseURL("http://x/").baseURL; got != "http://x" { + t.Errorf("OpenAI WithBaseURL trim = %q", got) + } + if got := NewOpenAI("k").WithBaseURL("").baseURL; got != defaultOpenAIBaseURL { + t.Errorf("OpenAI empty override = %q, want default", got) + } + if got := NewAnthropic("k").WithBaseURL("http://y/").baseURL; got != "http://y" { + t.Errorf("Anthropic WithBaseURL trim = %q", got) + } + + // End-to-end: a Chat call routes to the overridden base (a mock here), not the + // default endpoint — this is what lets the benchmark point at a local provider. + var hit bool + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + hit = true + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"id":"x","model":"gpt-4o","choices":[{"message":{"role":"assistant","content":"ok"},"finish_reason":"stop"}],"usage":{"prompt_tokens":5,"completion_tokens":1,"total_tokens":6}}`)) + })) + defer srv.Close() + + resp, err := NewOpenAI("k").WithBaseURL(srv.URL).Chat(context.Background(), + Request{Model: "gpt-4o", Messages: []Message{{Role: RoleUser, Content: "hi"}}}) + if err != nil { + t.Fatalf("Chat via override: %v", err) + } + if !hit { + t.Error("call did not route to the overridden base URL") + } + if resp.Content != "ok" { + t.Errorf("resp = %+v", resp) + } +} + func TestOpenAIChat_APIError(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusTooManyRequests)