From 53af87f60369858d935eab58ab69854daa021389 Mon Sep 17 00:00:00 2001 From: Sarthak Agrawal Date: Sat, 15 Aug 2026 20:40:33 +0530 Subject: [PATCH 1/3] feat: guard local performance execution --- PROJECT_STATUS.md | 38 +- docs/development/performance.md | 53 +- .../.openspec.yaml | 0 .../design.md | 0 .../proposal.md | 0 .../qualification.md | 0 .../autonomous-optimization-campaigns/spec.md | 0 .../tasks.md | 0 .../.openspec.yaml | 0 .../design.md | 0 .../proposal.md | 0 .../qualification.md | 0 .../durable-performance-supervision/spec.md | 0 .../tasks.md | 0 .../.openspec.yaml | 0 .../README.md | 0 .../design.md | 0 .../proposal.md | 0 .../spec.md | 0 .../tasks.md | 0 .../.openspec.yaml | 0 .../design.md | 0 .../proposal.md | 0 .../qualification.md | 0 .../specs/flow-optimization-campaigns/spec.md | 0 .../tasks.md | 0 .../.openspec.yaml | 0 .../design.md | 0 .../proposal.md | 0 .../qualification.md | 0 .../specs/local-flow-runtime-tools/spec.md | 0 .../tasks.md | 0 .../.openspec.yaml | 0 .../design.md | 0 .../proposal.md | 0 .../specs/runtime-failure-capsules/spec.md | 0 .../tasks.md | 0 .../.openspec.yaml | 0 .../design.md | 0 .../proposal.md | 0 .../qualification.md | 0 .../runtime-performance-capsules/spec.md | 0 .../tasks.md | 0 .../.openspec.yaml | 0 .../design.md | 0 .../proposal.md | 0 .../qualification.md | 0 .../specs/local-node-script-profiling/spec.md | 0 .../specs/performance-run-diagnostics/spec.md | 0 .../runtime-qualification-planning/spec.md | 0 .../tasks.md | 0 .../design.md | 0 .../proposal.md | 0 .../qualification.md | 0 .../specs/scaled-runtime-challenges/spec.md | 0 .../tasks.md | 0 .../.openspec.yaml | 0 .../design.md | 0 .../proposal.md | 0 .../cross-runtime-scaled-parsing/spec.md | 0 .../local-performance-governance/spec.md | 0 .../tasks.md | 0 .../.openspec.yaml | 0 .../design.md | 0 .../proposal.md | 0 .../qualification.md | 0 .../verification-receipt-comparison/spec.md | 0 .../tasks.md | 0 .../.openspec.yaml | 2 + .../guard-performance-cloud-spend/design.md | 94 ++++ .../guard-performance-cloud-spend/proposal.md | 48 ++ .../autonomous-optimization-campaigns/spec.md | 15 + .../performance-execution-governance/spec.md | 65 +++ .../guard-performance-cloud-spend/tasks.md | 19 + .../autonomous-optimization-campaigns/spec.md | 114 ++++ .../cross-runtime-scaled-parsing/spec.md | 53 ++ .../durable-performance-supervision/spec.md | 59 ++ .../spec.md | 29 + .../specs/flow-optimization-campaigns/spec.md | 84 +++ .../specs/local-flow-runtime-tools/spec.md | 102 ++++ .../specs/local-node-script-profiling/spec.md | 23 + .../local-performance-governance/spec.md | 25 +- .../specs/performance-run-diagnostics/spec.md | 27 + .../specs/runtime-failure-capsules/spec.md | 121 ++++ .../runtime-performance-capsules/spec.md | 226 ++++++++ .../runtime-qualification-planning/spec.md | 69 +++ .../specs/scaled-runtime-challenges/spec.md | 62 +++ .../verification-receipt-comparison/spec.md | 130 +++++ package.json | 1 + scripts/runtime-failure-capsule/campaign.mjs | 133 ++++- scripts/runtime-failure-capsule/cli.mjs | 145 +++-- scripts/runtime-failure-capsule/contracts.mjs | 15 + .../execution-governance.mjs | 518 ++++++++++++++++++ .../execution-governance.test.mjs | 251 +++++++++ .../local-flow-runtime.test.mjs | 12 +- scripts/runtime-failure-capsule/mcp.mjs | 40 +- .../node-egress-preload.mjs | 86 +++ .../paired-verification.mjs | 6 +- .../paired-verification.test.mjs | 2 +- .../runtime-failure-capsule/performance.mjs | 81 +++ scripts/runtime-failure-capsule/runner.mjs | 45 +- .../supervision-contracts.mjs | 17 + .../runtime-failure-capsule/supervision.mjs | 41 ++ 103 files changed, 2773 insertions(+), 78 deletions(-) rename openspec/changes/{add-autonomous-optimization-campaigns => archive/2026-08-15-add-autonomous-optimization-campaigns}/.openspec.yaml (100%) rename openspec/changes/{add-autonomous-optimization-campaigns => archive/2026-08-15-add-autonomous-optimization-campaigns}/design.md (100%) rename openspec/changes/{add-autonomous-optimization-campaigns => archive/2026-08-15-add-autonomous-optimization-campaigns}/proposal.md (100%) rename openspec/changes/{add-autonomous-optimization-campaigns => archive/2026-08-15-add-autonomous-optimization-campaigns}/qualification.md (100%) rename openspec/changes/{add-autonomous-optimization-campaigns => archive/2026-08-15-add-autonomous-optimization-campaigns}/specs/autonomous-optimization-campaigns/spec.md (100%) rename openspec/changes/{add-autonomous-optimization-campaigns => archive/2026-08-15-add-autonomous-optimization-campaigns}/tasks.md (100%) rename openspec/changes/{add-durable-performance-supervision => archive/2026-08-15-add-durable-performance-supervision}/.openspec.yaml (100%) rename openspec/changes/{add-durable-performance-supervision => archive/2026-08-15-add-durable-performance-supervision}/design.md (100%) rename openspec/changes/{add-durable-performance-supervision => archive/2026-08-15-add-durable-performance-supervision}/proposal.md (100%) rename openspec/changes/{add-durable-performance-supervision => archive/2026-08-15-add-durable-performance-supervision}/qualification.md (100%) rename openspec/changes/{add-durable-performance-supervision => archive/2026-08-15-add-durable-performance-supervision}/specs/durable-performance-supervision/spec.md (100%) rename openspec/changes/{add-durable-performance-supervision => archive/2026-08-15-add-durable-performance-supervision}/tasks.md (100%) rename openspec/changes/{add-evidence-backed-comparison-pages => archive/2026-08-15-add-evidence-backed-comparison-pages}/.openspec.yaml (100%) rename openspec/changes/{add-evidence-backed-comparison-pages => archive/2026-08-15-add-evidence-backed-comparison-pages}/README.md (100%) rename openspec/changes/{add-evidence-backed-comparison-pages => archive/2026-08-15-add-evidence-backed-comparison-pages}/design.md (100%) rename openspec/changes/{add-evidence-backed-comparison-pages => archive/2026-08-15-add-evidence-backed-comparison-pages}/proposal.md (100%) rename openspec/changes/{add-evidence-backed-comparison-pages => archive/2026-08-15-add-evidence-backed-comparison-pages}/specs/evidence-backed-product-comparisons/spec.md (100%) rename openspec/changes/{add-evidence-backed-comparison-pages => archive/2026-08-15-add-evidence-backed-comparison-pages}/tasks.md (100%) rename openspec/changes/{add-flow-optimization-campaigns => archive/2026-08-15-add-flow-optimization-campaigns}/.openspec.yaml (100%) rename openspec/changes/{add-flow-optimization-campaigns => archive/2026-08-15-add-flow-optimization-campaigns}/design.md (100%) rename openspec/changes/{add-flow-optimization-campaigns => archive/2026-08-15-add-flow-optimization-campaigns}/proposal.md (100%) rename openspec/changes/{add-flow-optimization-campaigns => archive/2026-08-15-add-flow-optimization-campaigns}/qualification.md (100%) rename openspec/changes/{add-flow-optimization-campaigns => archive/2026-08-15-add-flow-optimization-campaigns}/specs/flow-optimization-campaigns/spec.md (100%) rename openspec/changes/{add-flow-optimization-campaigns => archive/2026-08-15-add-flow-optimization-campaigns}/tasks.md (100%) rename openspec/changes/{add-local-flow-runtime-tools => archive/2026-08-15-add-local-flow-runtime-tools}/.openspec.yaml (100%) rename openspec/changes/{add-local-flow-runtime-tools => archive/2026-08-15-add-local-flow-runtime-tools}/design.md (100%) rename openspec/changes/{add-local-flow-runtime-tools => archive/2026-08-15-add-local-flow-runtime-tools}/proposal.md (100%) rename openspec/changes/{add-local-flow-runtime-tools => archive/2026-08-15-add-local-flow-runtime-tools}/qualification.md (100%) rename openspec/changes/{add-local-flow-runtime-tools => archive/2026-08-15-add-local-flow-runtime-tools}/specs/local-flow-runtime-tools/spec.md (100%) rename openspec/changes/{add-local-flow-runtime-tools => archive/2026-08-15-add-local-flow-runtime-tools}/tasks.md (100%) rename openspec/changes/{add-runtime-failure-capsules => archive/2026-08-15-add-runtime-failure-capsules}/.openspec.yaml (100%) rename openspec/changes/{add-runtime-failure-capsules => archive/2026-08-15-add-runtime-failure-capsules}/design.md (100%) rename openspec/changes/{add-runtime-failure-capsules => archive/2026-08-15-add-runtime-failure-capsules}/proposal.md (100%) rename openspec/changes/{add-runtime-failure-capsules => archive/2026-08-15-add-runtime-failure-capsules}/specs/runtime-failure-capsules/spec.md (100%) rename openspec/changes/{add-runtime-failure-capsules => archive/2026-08-15-add-runtime-failure-capsules}/tasks.md (100%) rename openspec/changes/{add-runtime-performance-capsules => archive/2026-08-15-add-runtime-performance-capsules}/.openspec.yaml (100%) rename openspec/changes/{add-runtime-performance-capsules => archive/2026-08-15-add-runtime-performance-capsules}/design.md (100%) rename openspec/changes/{add-runtime-performance-capsules => archive/2026-08-15-add-runtime-performance-capsules}/proposal.md (100%) rename openspec/changes/{add-runtime-performance-capsules => archive/2026-08-15-add-runtime-performance-capsules}/qualification.md (100%) rename openspec/changes/{add-runtime-performance-capsules => archive/2026-08-15-add-runtime-performance-capsules}/specs/runtime-performance-capsules/spec.md (100%) rename openspec/changes/{add-runtime-performance-capsules => archive/2026-08-15-add-runtime-performance-capsules}/tasks.md (100%) rename openspec/changes/{add-runtime-qualification-planner => archive/2026-08-15-add-runtime-qualification-planner}/.openspec.yaml (100%) rename openspec/changes/{add-runtime-qualification-planner => archive/2026-08-15-add-runtime-qualification-planner}/design.md (100%) rename openspec/changes/{add-runtime-qualification-planner => archive/2026-08-15-add-runtime-qualification-planner}/proposal.md (100%) rename openspec/changes/{add-runtime-qualification-planner => archive/2026-08-15-add-runtime-qualification-planner}/qualification.md (100%) rename openspec/changes/{add-runtime-qualification-planner => archive/2026-08-15-add-runtime-qualification-planner}/specs/local-node-script-profiling/spec.md (100%) rename openspec/changes/{add-runtime-qualification-planner => archive/2026-08-15-add-runtime-qualification-planner}/specs/performance-run-diagnostics/spec.md (100%) rename openspec/changes/{add-runtime-qualification-planner => archive/2026-08-15-add-runtime-qualification-planner}/specs/runtime-qualification-planning/spec.md (100%) rename openspec/changes/{add-runtime-qualification-planner => archive/2026-08-15-add-runtime-qualification-planner}/tasks.md (100%) rename openspec/changes/{add-scaled-parsing-challenge => archive/2026-08-15-add-scaled-parsing-challenge}/design.md (100%) rename openspec/changes/{add-scaled-parsing-challenge => archive/2026-08-15-add-scaled-parsing-challenge}/proposal.md (100%) rename openspec/changes/{add-scaled-parsing-challenge => archive/2026-08-15-add-scaled-parsing-challenge}/qualification.md (100%) rename openspec/changes/{add-scaled-parsing-challenge => archive/2026-08-15-add-scaled-parsing-challenge}/specs/scaled-runtime-challenges/spec.md (100%) rename openspec/changes/{add-scaled-parsing-challenge => archive/2026-08-15-add-scaled-parsing-challenge}/tasks.md (100%) rename openspec/changes/{add-streaming-parallel-1brc => archive/2026-08-15-add-streaming-parallel-1brc}/.openspec.yaml (100%) rename openspec/changes/{add-streaming-parallel-1brc => archive/2026-08-15-add-streaming-parallel-1brc}/design.md (100%) rename openspec/changes/{add-streaming-parallel-1brc => archive/2026-08-15-add-streaming-parallel-1brc}/proposal.md (100%) rename openspec/changes/{add-streaming-parallel-1brc => archive/2026-08-15-add-streaming-parallel-1brc}/specs/cross-runtime-scaled-parsing/spec.md (100%) rename openspec/changes/{add-streaming-parallel-1brc => archive/2026-08-15-add-streaming-parallel-1brc}/specs/local-performance-governance/spec.md (100%) rename openspec/changes/{add-streaming-parallel-1brc => archive/2026-08-15-add-streaming-parallel-1brc}/tasks.md (100%) rename openspec/changes/{ingest-verification-receipts => archive/2026-08-15-ingest-verification-receipts}/.openspec.yaml (100%) rename openspec/changes/{ingest-verification-receipts => archive/2026-08-15-ingest-verification-receipts}/design.md (100%) rename openspec/changes/{ingest-verification-receipts => archive/2026-08-15-ingest-verification-receipts}/proposal.md (100%) rename openspec/changes/{ingest-verification-receipts => archive/2026-08-15-ingest-verification-receipts}/qualification.md (100%) rename openspec/changes/{ingest-verification-receipts => archive/2026-08-15-ingest-verification-receipts}/specs/verification-receipt-comparison/spec.md (100%) rename openspec/changes/{ingest-verification-receipts => archive/2026-08-15-ingest-verification-receipts}/tasks.md (100%) create mode 100644 openspec/changes/guard-performance-cloud-spend/.openspec.yaml create mode 100644 openspec/changes/guard-performance-cloud-spend/design.md create mode 100644 openspec/changes/guard-performance-cloud-spend/proposal.md create mode 100644 openspec/changes/guard-performance-cloud-spend/specs/autonomous-optimization-campaigns/spec.md create mode 100644 openspec/changes/guard-performance-cloud-spend/specs/performance-execution-governance/spec.md create mode 100644 openspec/changes/guard-performance-cloud-spend/tasks.md create mode 100644 openspec/specs/autonomous-optimization-campaigns/spec.md create mode 100644 openspec/specs/cross-runtime-scaled-parsing/spec.md create mode 100644 openspec/specs/durable-performance-supervision/spec.md create mode 100644 openspec/specs/evidence-backed-product-comparisons/spec.md create mode 100644 openspec/specs/flow-optimization-campaigns/spec.md create mode 100644 openspec/specs/local-flow-runtime-tools/spec.md create mode 100644 openspec/specs/local-node-script-profiling/spec.md create mode 100644 openspec/specs/performance-run-diagnostics/spec.md create mode 100644 openspec/specs/runtime-failure-capsules/spec.md create mode 100644 openspec/specs/runtime-performance-capsules/spec.md create mode 100644 openspec/specs/runtime-qualification-planning/spec.md create mode 100644 openspec/specs/scaled-runtime-challenges/spec.md create mode 100644 openspec/specs/verification-receipt-comparison/spec.md create mode 100644 scripts/runtime-failure-capsule/execution-governance.mjs create mode 100644 scripts/runtime-failure-capsule/execution-governance.test.mjs create mode 100644 scripts/runtime-failure-capsule/node-egress-preload.mjs diff --git a/PROJECT_STATUS.md b/PROJECT_STATUS.md index 9ee6547c..bb912206 100644 --- a/PROJECT_STATUS.md +++ b/PROJECT_STATUS.md @@ -1,6 +1,6 @@ # Project Status -Last updated: 2026-08-11 +Last updated: 2026-08-15 ## Why / What @@ -19,6 +19,22 @@ behavior. **Core Mode** protects that roadmap. **Side Quest Mode** permits explicitly requested non-core work, but it must stay bounded and must not silently redefine or displace the core. +## Current investment decision + +As of 2026-08-15, CodeVetter remains a maintained local product and empirical +verification/performance research asset, but broad feature expansion is paused. +The repository has strong deterministic infrastructure, caught-bug evidence, +and several measured optimization case studies; it does **not** have the planned +ten active external users, three team pilots, or complete authenticated +multi-provider trials. Those missing outcomes must not be inferred from owner +dogfood, synthetic corpora, or local qualification. + +New work should be limited to core verification reliability, regressions, +security, maintenance, and evidence requested by real users. Agent Island stays +opt-in, authenticated provider comparisons remain claim-closed, and no hosted +profiling or production-load system should be added until external pull and a +separately approved safety design justify renewed investment. + ## Dependencies External: @@ -38,6 +54,26 @@ Internal (fleet): ## Timeline +- **2026-08-15 — Local performance execution became zero-egress by default + (release candidate):** Added immutable dry-run plans and cost/egress receipts + for exact performance scopes. Admitted runs bind repository revision, dirty + state, and target identity; execute one process at a time with no retries, + external requests, services, or monetary cost, and fail closed when evidence is remote, paid, + production-like, unknown-cost, or stale. Node-family work gets a runtime + network guard; macOS also applies a child-process network sandbox with + loopback support. Go is admitted only where CodeVetter can establish an OS + zero-egress boundary. CLI/MCP dry-run, durable supervision, direct profiling, + and optimization campaigns share the policy. The focused runtime suite passes + 150 tests, including hermetic loopback, blocked-remote, stale-identity, and + zero-process blocked-receipt cases. No hosted endpoint, paid service, + production configuration, or new dependency was used. +- **2026-08-15 — Continuation gate resolved conservatively:** Preserved the + released local product, corpus, deterministic graders, runtime lab, and + documentation while pausing broad surface expansion. Ten-user validation, + team pilots, authenticated provider comparisons, and real Codex/Claude Agent + Island qualification were not completed and remain unsupported product-value + claims rather than silently passing by proxy. + - **2026-08-11 — Code-health coverage became executable:** Added CI-blocking unused-code, duplication-regression, runtime dependency-cycle, and production vulnerability gates. Removed six unused public type exports, confirmed zero diff --git a/docs/development/performance.md b/docs/development/performance.md index c04031a6..22876f67 100644 --- a/docs/development/performance.md +++ b/docs/development/performance.md @@ -792,7 +792,7 @@ and the identical-scope verifier confirmed a 69.945% improvement. A second micro-optimization improved the largest case by only 8.863% and regressed the smallest by 4.079%; it was reverted after the verifier returned inconclusive. See the active OpenSpec -[qualification](../../openspec/changes/add-scaled-parsing-challenge/qualification.md) +[qualification](../../openspec/changes/archive/2026-08-15-add-scaled-parsing-challenge/qualification.md) for the complete evidence and limitations. The challenge is not an official submission and does not claim one- or @@ -829,7 +829,7 @@ The self-trial is also an accuracy fixture. CodeVetter's first inference blamed a bounded coverage-filename sort even though the workload held file count at one. That hypothesis was rejected from the captured workload identity before the observed offset helper was optimized. The active OpenSpec -[qualification](../../openspec/changes/add-runtime-performance-capsules/qualification.md) +[qualification](../../openspec/changes/archive/2026-08-15-add-runtime-performance-capsules/qualification.md) records the complete evidence, the conservative shipping limitation, and the decision-explanation bug found and fixed during the loop. @@ -1037,6 +1037,51 @@ workloads. Agents can also call the read-only MCP operation `qualify_runtime_repository` before `capture_local_flow` or an explicit performance profile. +### Zero-egress execution planning + +Qualification ranks a workload; admission decides whether CodeVetter may run +it. Dry-run the exact profile before execution: + +```bash +pnpm --silent runtime:plan-performance -- \ + --repo /path/to/repository \ + --adapter node-script \ + --target benchmark/parser.mjs \ + --samples 3 \ + --warmups 1 \ + --timeout-ms 30000 \ + --json +``` + +The equivalent read-only MCP operation is `plan_local_performance`. Its stable +plan identity binds the repository revision and dirty state, target content, +adapter, exact workload, and bounds. An admitted plan permits one concurrent +owned process, zero retries, zero external requests, no external services, zero +monetary cost, and a finite total wall clock. The profiler rechecks that identity immediately +before each process; a changed revision, dirty state, or target fails closed. + +Node-family runs preload a deny guard for remote DNS, sockets, HTTP(S), fetch, +and WebSocket access. On macOS, the child process tree also runs under a network +sandbox; loopback remains available for local HTTP flows. Go execution uses the +macOS sandbox and is blocked on hosts where CodeVetter cannot establish a +dependable zero-egress boundary. `GOPROXY=off` is not treated as a sandbox. + +Remote endpoints, dynamic unknown destinations, hosted/paid service signals, +and workloads identified as load, soak, stress, or production profiles are +blocked before project code runs. Supplying an approval identity records the +caller input but does not enable hosted execution in this version. Unknown cost +is reported as unknown, never as zero. Admitted, blocked, completed, failed, +and policy-violation outcomes carry a versioned cost-and-egress receipt in the +performance capsule. + +This boundary separates three activities: + +- local profiling is the only executable autonomous mode; +- a public smoke is a separately initiated, bounded availability check and is + not a performance or load test; +- hosted, production, load, soak, and stress profiling remain unsupported and + require future infrastructure outside this local flywheel. + ## 14. Preserving a profiling attempt under failure Use the outer supervisor when a workload may crash, receive a signal, exceed @@ -1077,6 +1122,10 @@ or alter source control. Child output is byte-bounded and redacted before it can become failure evidence; successful JSON is redacted, schema-validated, and hashed before preservation. +The supervisor derives the same execution plan before launching its child. A +blocked plan writes a terminal `blocked` receipt with zero child processes, +requests, retries, services, and executed cost; project code is not started. + Recovery is deliberately conservative. A killed profiling child, timeout, ordinary exit, spawn failure, or malformed result gets a terminal receipt and authorizes no performance conclusion. If the supervisor or entire machine dies, diff --git a/openspec/changes/add-autonomous-optimization-campaigns/.openspec.yaml b/openspec/changes/archive/2026-08-15-add-autonomous-optimization-campaigns/.openspec.yaml similarity index 100% rename from openspec/changes/add-autonomous-optimization-campaigns/.openspec.yaml rename to openspec/changes/archive/2026-08-15-add-autonomous-optimization-campaigns/.openspec.yaml diff --git a/openspec/changes/add-autonomous-optimization-campaigns/design.md b/openspec/changes/archive/2026-08-15-add-autonomous-optimization-campaigns/design.md similarity index 100% rename from openspec/changes/add-autonomous-optimization-campaigns/design.md rename to openspec/changes/archive/2026-08-15-add-autonomous-optimization-campaigns/design.md diff --git a/openspec/changes/add-autonomous-optimization-campaigns/proposal.md b/openspec/changes/archive/2026-08-15-add-autonomous-optimization-campaigns/proposal.md similarity index 100% rename from openspec/changes/add-autonomous-optimization-campaigns/proposal.md rename to openspec/changes/archive/2026-08-15-add-autonomous-optimization-campaigns/proposal.md diff --git a/openspec/changes/add-autonomous-optimization-campaigns/qualification.md b/openspec/changes/archive/2026-08-15-add-autonomous-optimization-campaigns/qualification.md similarity index 100% rename from openspec/changes/add-autonomous-optimization-campaigns/qualification.md rename to openspec/changes/archive/2026-08-15-add-autonomous-optimization-campaigns/qualification.md diff --git a/openspec/changes/add-autonomous-optimization-campaigns/specs/autonomous-optimization-campaigns/spec.md b/openspec/changes/archive/2026-08-15-add-autonomous-optimization-campaigns/specs/autonomous-optimization-campaigns/spec.md similarity index 100% rename from openspec/changes/add-autonomous-optimization-campaigns/specs/autonomous-optimization-campaigns/spec.md rename to openspec/changes/archive/2026-08-15-add-autonomous-optimization-campaigns/specs/autonomous-optimization-campaigns/spec.md diff --git a/openspec/changes/add-autonomous-optimization-campaigns/tasks.md b/openspec/changes/archive/2026-08-15-add-autonomous-optimization-campaigns/tasks.md similarity index 100% rename from openspec/changes/add-autonomous-optimization-campaigns/tasks.md rename to openspec/changes/archive/2026-08-15-add-autonomous-optimization-campaigns/tasks.md diff --git a/openspec/changes/add-durable-performance-supervision/.openspec.yaml b/openspec/changes/archive/2026-08-15-add-durable-performance-supervision/.openspec.yaml similarity index 100% rename from openspec/changes/add-durable-performance-supervision/.openspec.yaml rename to openspec/changes/archive/2026-08-15-add-durable-performance-supervision/.openspec.yaml diff --git a/openspec/changes/add-durable-performance-supervision/design.md b/openspec/changes/archive/2026-08-15-add-durable-performance-supervision/design.md similarity index 100% rename from openspec/changes/add-durable-performance-supervision/design.md rename to openspec/changes/archive/2026-08-15-add-durable-performance-supervision/design.md diff --git a/openspec/changes/add-durable-performance-supervision/proposal.md b/openspec/changes/archive/2026-08-15-add-durable-performance-supervision/proposal.md similarity index 100% rename from openspec/changes/add-durable-performance-supervision/proposal.md rename to openspec/changes/archive/2026-08-15-add-durable-performance-supervision/proposal.md diff --git a/openspec/changes/add-durable-performance-supervision/qualification.md b/openspec/changes/archive/2026-08-15-add-durable-performance-supervision/qualification.md similarity index 100% rename from openspec/changes/add-durable-performance-supervision/qualification.md rename to openspec/changes/archive/2026-08-15-add-durable-performance-supervision/qualification.md diff --git a/openspec/changes/add-durable-performance-supervision/specs/durable-performance-supervision/spec.md b/openspec/changes/archive/2026-08-15-add-durable-performance-supervision/specs/durable-performance-supervision/spec.md similarity index 100% rename from openspec/changes/add-durable-performance-supervision/specs/durable-performance-supervision/spec.md rename to openspec/changes/archive/2026-08-15-add-durable-performance-supervision/specs/durable-performance-supervision/spec.md diff --git a/openspec/changes/add-durable-performance-supervision/tasks.md b/openspec/changes/archive/2026-08-15-add-durable-performance-supervision/tasks.md similarity index 100% rename from openspec/changes/add-durable-performance-supervision/tasks.md rename to openspec/changes/archive/2026-08-15-add-durable-performance-supervision/tasks.md diff --git a/openspec/changes/add-evidence-backed-comparison-pages/.openspec.yaml b/openspec/changes/archive/2026-08-15-add-evidence-backed-comparison-pages/.openspec.yaml similarity index 100% rename from openspec/changes/add-evidence-backed-comparison-pages/.openspec.yaml rename to openspec/changes/archive/2026-08-15-add-evidence-backed-comparison-pages/.openspec.yaml diff --git a/openspec/changes/add-evidence-backed-comparison-pages/README.md b/openspec/changes/archive/2026-08-15-add-evidence-backed-comparison-pages/README.md similarity index 100% rename from openspec/changes/add-evidence-backed-comparison-pages/README.md rename to openspec/changes/archive/2026-08-15-add-evidence-backed-comparison-pages/README.md diff --git a/openspec/changes/add-evidence-backed-comparison-pages/design.md b/openspec/changes/archive/2026-08-15-add-evidence-backed-comparison-pages/design.md similarity index 100% rename from openspec/changes/add-evidence-backed-comparison-pages/design.md rename to openspec/changes/archive/2026-08-15-add-evidence-backed-comparison-pages/design.md diff --git a/openspec/changes/add-evidence-backed-comparison-pages/proposal.md b/openspec/changes/archive/2026-08-15-add-evidence-backed-comparison-pages/proposal.md similarity index 100% rename from openspec/changes/add-evidence-backed-comparison-pages/proposal.md rename to openspec/changes/archive/2026-08-15-add-evidence-backed-comparison-pages/proposal.md diff --git a/openspec/changes/add-evidence-backed-comparison-pages/specs/evidence-backed-product-comparisons/spec.md b/openspec/changes/archive/2026-08-15-add-evidence-backed-comparison-pages/specs/evidence-backed-product-comparisons/spec.md similarity index 100% rename from openspec/changes/add-evidence-backed-comparison-pages/specs/evidence-backed-product-comparisons/spec.md rename to openspec/changes/archive/2026-08-15-add-evidence-backed-comparison-pages/specs/evidence-backed-product-comparisons/spec.md diff --git a/openspec/changes/add-evidence-backed-comparison-pages/tasks.md b/openspec/changes/archive/2026-08-15-add-evidence-backed-comparison-pages/tasks.md similarity index 100% rename from openspec/changes/add-evidence-backed-comparison-pages/tasks.md rename to openspec/changes/archive/2026-08-15-add-evidence-backed-comparison-pages/tasks.md diff --git a/openspec/changes/add-flow-optimization-campaigns/.openspec.yaml b/openspec/changes/archive/2026-08-15-add-flow-optimization-campaigns/.openspec.yaml similarity index 100% rename from openspec/changes/add-flow-optimization-campaigns/.openspec.yaml rename to openspec/changes/archive/2026-08-15-add-flow-optimization-campaigns/.openspec.yaml diff --git a/openspec/changes/add-flow-optimization-campaigns/design.md b/openspec/changes/archive/2026-08-15-add-flow-optimization-campaigns/design.md similarity index 100% rename from openspec/changes/add-flow-optimization-campaigns/design.md rename to openspec/changes/archive/2026-08-15-add-flow-optimization-campaigns/design.md diff --git a/openspec/changes/add-flow-optimization-campaigns/proposal.md b/openspec/changes/archive/2026-08-15-add-flow-optimization-campaigns/proposal.md similarity index 100% rename from openspec/changes/add-flow-optimization-campaigns/proposal.md rename to openspec/changes/archive/2026-08-15-add-flow-optimization-campaigns/proposal.md diff --git a/openspec/changes/add-flow-optimization-campaigns/qualification.md b/openspec/changes/archive/2026-08-15-add-flow-optimization-campaigns/qualification.md similarity index 100% rename from openspec/changes/add-flow-optimization-campaigns/qualification.md rename to openspec/changes/archive/2026-08-15-add-flow-optimization-campaigns/qualification.md diff --git a/openspec/changes/add-flow-optimization-campaigns/specs/flow-optimization-campaigns/spec.md b/openspec/changes/archive/2026-08-15-add-flow-optimization-campaigns/specs/flow-optimization-campaigns/spec.md similarity index 100% rename from openspec/changes/add-flow-optimization-campaigns/specs/flow-optimization-campaigns/spec.md rename to openspec/changes/archive/2026-08-15-add-flow-optimization-campaigns/specs/flow-optimization-campaigns/spec.md diff --git a/openspec/changes/add-flow-optimization-campaigns/tasks.md b/openspec/changes/archive/2026-08-15-add-flow-optimization-campaigns/tasks.md similarity index 100% rename from openspec/changes/add-flow-optimization-campaigns/tasks.md rename to openspec/changes/archive/2026-08-15-add-flow-optimization-campaigns/tasks.md diff --git a/openspec/changes/add-local-flow-runtime-tools/.openspec.yaml b/openspec/changes/archive/2026-08-15-add-local-flow-runtime-tools/.openspec.yaml similarity index 100% rename from openspec/changes/add-local-flow-runtime-tools/.openspec.yaml rename to openspec/changes/archive/2026-08-15-add-local-flow-runtime-tools/.openspec.yaml diff --git a/openspec/changes/add-local-flow-runtime-tools/design.md b/openspec/changes/archive/2026-08-15-add-local-flow-runtime-tools/design.md similarity index 100% rename from openspec/changes/add-local-flow-runtime-tools/design.md rename to openspec/changes/archive/2026-08-15-add-local-flow-runtime-tools/design.md diff --git a/openspec/changes/add-local-flow-runtime-tools/proposal.md b/openspec/changes/archive/2026-08-15-add-local-flow-runtime-tools/proposal.md similarity index 100% rename from openspec/changes/add-local-flow-runtime-tools/proposal.md rename to openspec/changes/archive/2026-08-15-add-local-flow-runtime-tools/proposal.md diff --git a/openspec/changes/add-local-flow-runtime-tools/qualification.md b/openspec/changes/archive/2026-08-15-add-local-flow-runtime-tools/qualification.md similarity index 100% rename from openspec/changes/add-local-flow-runtime-tools/qualification.md rename to openspec/changes/archive/2026-08-15-add-local-flow-runtime-tools/qualification.md diff --git a/openspec/changes/add-local-flow-runtime-tools/specs/local-flow-runtime-tools/spec.md b/openspec/changes/archive/2026-08-15-add-local-flow-runtime-tools/specs/local-flow-runtime-tools/spec.md similarity index 100% rename from openspec/changes/add-local-flow-runtime-tools/specs/local-flow-runtime-tools/spec.md rename to openspec/changes/archive/2026-08-15-add-local-flow-runtime-tools/specs/local-flow-runtime-tools/spec.md diff --git a/openspec/changes/add-local-flow-runtime-tools/tasks.md b/openspec/changes/archive/2026-08-15-add-local-flow-runtime-tools/tasks.md similarity index 100% rename from openspec/changes/add-local-flow-runtime-tools/tasks.md rename to openspec/changes/archive/2026-08-15-add-local-flow-runtime-tools/tasks.md diff --git a/openspec/changes/add-runtime-failure-capsules/.openspec.yaml b/openspec/changes/archive/2026-08-15-add-runtime-failure-capsules/.openspec.yaml similarity index 100% rename from openspec/changes/add-runtime-failure-capsules/.openspec.yaml rename to openspec/changes/archive/2026-08-15-add-runtime-failure-capsules/.openspec.yaml diff --git a/openspec/changes/add-runtime-failure-capsules/design.md b/openspec/changes/archive/2026-08-15-add-runtime-failure-capsules/design.md similarity index 100% rename from openspec/changes/add-runtime-failure-capsules/design.md rename to openspec/changes/archive/2026-08-15-add-runtime-failure-capsules/design.md diff --git a/openspec/changes/add-runtime-failure-capsules/proposal.md b/openspec/changes/archive/2026-08-15-add-runtime-failure-capsules/proposal.md similarity index 100% rename from openspec/changes/add-runtime-failure-capsules/proposal.md rename to openspec/changes/archive/2026-08-15-add-runtime-failure-capsules/proposal.md diff --git a/openspec/changes/add-runtime-failure-capsules/specs/runtime-failure-capsules/spec.md b/openspec/changes/archive/2026-08-15-add-runtime-failure-capsules/specs/runtime-failure-capsules/spec.md similarity index 100% rename from openspec/changes/add-runtime-failure-capsules/specs/runtime-failure-capsules/spec.md rename to openspec/changes/archive/2026-08-15-add-runtime-failure-capsules/specs/runtime-failure-capsules/spec.md diff --git a/openspec/changes/add-runtime-failure-capsules/tasks.md b/openspec/changes/archive/2026-08-15-add-runtime-failure-capsules/tasks.md similarity index 100% rename from openspec/changes/add-runtime-failure-capsules/tasks.md rename to openspec/changes/archive/2026-08-15-add-runtime-failure-capsules/tasks.md diff --git a/openspec/changes/add-runtime-performance-capsules/.openspec.yaml b/openspec/changes/archive/2026-08-15-add-runtime-performance-capsules/.openspec.yaml similarity index 100% rename from openspec/changes/add-runtime-performance-capsules/.openspec.yaml rename to openspec/changes/archive/2026-08-15-add-runtime-performance-capsules/.openspec.yaml diff --git a/openspec/changes/add-runtime-performance-capsules/design.md b/openspec/changes/archive/2026-08-15-add-runtime-performance-capsules/design.md similarity index 100% rename from openspec/changes/add-runtime-performance-capsules/design.md rename to openspec/changes/archive/2026-08-15-add-runtime-performance-capsules/design.md diff --git a/openspec/changes/add-runtime-performance-capsules/proposal.md b/openspec/changes/archive/2026-08-15-add-runtime-performance-capsules/proposal.md similarity index 100% rename from openspec/changes/add-runtime-performance-capsules/proposal.md rename to openspec/changes/archive/2026-08-15-add-runtime-performance-capsules/proposal.md diff --git a/openspec/changes/add-runtime-performance-capsules/qualification.md b/openspec/changes/archive/2026-08-15-add-runtime-performance-capsules/qualification.md similarity index 100% rename from openspec/changes/add-runtime-performance-capsules/qualification.md rename to openspec/changes/archive/2026-08-15-add-runtime-performance-capsules/qualification.md diff --git a/openspec/changes/add-runtime-performance-capsules/specs/runtime-performance-capsules/spec.md b/openspec/changes/archive/2026-08-15-add-runtime-performance-capsules/specs/runtime-performance-capsules/spec.md similarity index 100% rename from openspec/changes/add-runtime-performance-capsules/specs/runtime-performance-capsules/spec.md rename to openspec/changes/archive/2026-08-15-add-runtime-performance-capsules/specs/runtime-performance-capsules/spec.md diff --git a/openspec/changes/add-runtime-performance-capsules/tasks.md b/openspec/changes/archive/2026-08-15-add-runtime-performance-capsules/tasks.md similarity index 100% rename from openspec/changes/add-runtime-performance-capsules/tasks.md rename to openspec/changes/archive/2026-08-15-add-runtime-performance-capsules/tasks.md diff --git a/openspec/changes/add-runtime-qualification-planner/.openspec.yaml b/openspec/changes/archive/2026-08-15-add-runtime-qualification-planner/.openspec.yaml similarity index 100% rename from openspec/changes/add-runtime-qualification-planner/.openspec.yaml rename to openspec/changes/archive/2026-08-15-add-runtime-qualification-planner/.openspec.yaml diff --git a/openspec/changes/add-runtime-qualification-planner/design.md b/openspec/changes/archive/2026-08-15-add-runtime-qualification-planner/design.md similarity index 100% rename from openspec/changes/add-runtime-qualification-planner/design.md rename to openspec/changes/archive/2026-08-15-add-runtime-qualification-planner/design.md diff --git a/openspec/changes/add-runtime-qualification-planner/proposal.md b/openspec/changes/archive/2026-08-15-add-runtime-qualification-planner/proposal.md similarity index 100% rename from openspec/changes/add-runtime-qualification-planner/proposal.md rename to openspec/changes/archive/2026-08-15-add-runtime-qualification-planner/proposal.md diff --git a/openspec/changes/add-runtime-qualification-planner/qualification.md b/openspec/changes/archive/2026-08-15-add-runtime-qualification-planner/qualification.md similarity index 100% rename from openspec/changes/add-runtime-qualification-planner/qualification.md rename to openspec/changes/archive/2026-08-15-add-runtime-qualification-planner/qualification.md diff --git a/openspec/changes/add-runtime-qualification-planner/specs/local-node-script-profiling/spec.md b/openspec/changes/archive/2026-08-15-add-runtime-qualification-planner/specs/local-node-script-profiling/spec.md similarity index 100% rename from openspec/changes/add-runtime-qualification-planner/specs/local-node-script-profiling/spec.md rename to openspec/changes/archive/2026-08-15-add-runtime-qualification-planner/specs/local-node-script-profiling/spec.md diff --git a/openspec/changes/add-runtime-qualification-planner/specs/performance-run-diagnostics/spec.md b/openspec/changes/archive/2026-08-15-add-runtime-qualification-planner/specs/performance-run-diagnostics/spec.md similarity index 100% rename from openspec/changes/add-runtime-qualification-planner/specs/performance-run-diagnostics/spec.md rename to openspec/changes/archive/2026-08-15-add-runtime-qualification-planner/specs/performance-run-diagnostics/spec.md diff --git a/openspec/changes/add-runtime-qualification-planner/specs/runtime-qualification-planning/spec.md b/openspec/changes/archive/2026-08-15-add-runtime-qualification-planner/specs/runtime-qualification-planning/spec.md similarity index 100% rename from openspec/changes/add-runtime-qualification-planner/specs/runtime-qualification-planning/spec.md rename to openspec/changes/archive/2026-08-15-add-runtime-qualification-planner/specs/runtime-qualification-planning/spec.md diff --git a/openspec/changes/add-runtime-qualification-planner/tasks.md b/openspec/changes/archive/2026-08-15-add-runtime-qualification-planner/tasks.md similarity index 100% rename from openspec/changes/add-runtime-qualification-planner/tasks.md rename to openspec/changes/archive/2026-08-15-add-runtime-qualification-planner/tasks.md diff --git a/openspec/changes/add-scaled-parsing-challenge/design.md b/openspec/changes/archive/2026-08-15-add-scaled-parsing-challenge/design.md similarity index 100% rename from openspec/changes/add-scaled-parsing-challenge/design.md rename to openspec/changes/archive/2026-08-15-add-scaled-parsing-challenge/design.md diff --git a/openspec/changes/add-scaled-parsing-challenge/proposal.md b/openspec/changes/archive/2026-08-15-add-scaled-parsing-challenge/proposal.md similarity index 100% rename from openspec/changes/add-scaled-parsing-challenge/proposal.md rename to openspec/changes/archive/2026-08-15-add-scaled-parsing-challenge/proposal.md diff --git a/openspec/changes/add-scaled-parsing-challenge/qualification.md b/openspec/changes/archive/2026-08-15-add-scaled-parsing-challenge/qualification.md similarity index 100% rename from openspec/changes/add-scaled-parsing-challenge/qualification.md rename to openspec/changes/archive/2026-08-15-add-scaled-parsing-challenge/qualification.md diff --git a/openspec/changes/add-scaled-parsing-challenge/specs/scaled-runtime-challenges/spec.md b/openspec/changes/archive/2026-08-15-add-scaled-parsing-challenge/specs/scaled-runtime-challenges/spec.md similarity index 100% rename from openspec/changes/add-scaled-parsing-challenge/specs/scaled-runtime-challenges/spec.md rename to openspec/changes/archive/2026-08-15-add-scaled-parsing-challenge/specs/scaled-runtime-challenges/spec.md diff --git a/openspec/changes/add-scaled-parsing-challenge/tasks.md b/openspec/changes/archive/2026-08-15-add-scaled-parsing-challenge/tasks.md similarity index 100% rename from openspec/changes/add-scaled-parsing-challenge/tasks.md rename to openspec/changes/archive/2026-08-15-add-scaled-parsing-challenge/tasks.md diff --git a/openspec/changes/add-streaming-parallel-1brc/.openspec.yaml b/openspec/changes/archive/2026-08-15-add-streaming-parallel-1brc/.openspec.yaml similarity index 100% rename from openspec/changes/add-streaming-parallel-1brc/.openspec.yaml rename to openspec/changes/archive/2026-08-15-add-streaming-parallel-1brc/.openspec.yaml diff --git a/openspec/changes/add-streaming-parallel-1brc/design.md b/openspec/changes/archive/2026-08-15-add-streaming-parallel-1brc/design.md similarity index 100% rename from openspec/changes/add-streaming-parallel-1brc/design.md rename to openspec/changes/archive/2026-08-15-add-streaming-parallel-1brc/design.md diff --git a/openspec/changes/add-streaming-parallel-1brc/proposal.md b/openspec/changes/archive/2026-08-15-add-streaming-parallel-1brc/proposal.md similarity index 100% rename from openspec/changes/add-streaming-parallel-1brc/proposal.md rename to openspec/changes/archive/2026-08-15-add-streaming-parallel-1brc/proposal.md diff --git a/openspec/changes/add-streaming-parallel-1brc/specs/cross-runtime-scaled-parsing/spec.md b/openspec/changes/archive/2026-08-15-add-streaming-parallel-1brc/specs/cross-runtime-scaled-parsing/spec.md similarity index 100% rename from openspec/changes/add-streaming-parallel-1brc/specs/cross-runtime-scaled-parsing/spec.md rename to openspec/changes/archive/2026-08-15-add-streaming-parallel-1brc/specs/cross-runtime-scaled-parsing/spec.md diff --git a/openspec/changes/add-streaming-parallel-1brc/specs/local-performance-governance/spec.md b/openspec/changes/archive/2026-08-15-add-streaming-parallel-1brc/specs/local-performance-governance/spec.md similarity index 100% rename from openspec/changes/add-streaming-parallel-1brc/specs/local-performance-governance/spec.md rename to openspec/changes/archive/2026-08-15-add-streaming-parallel-1brc/specs/local-performance-governance/spec.md diff --git a/openspec/changes/add-streaming-parallel-1brc/tasks.md b/openspec/changes/archive/2026-08-15-add-streaming-parallel-1brc/tasks.md similarity index 100% rename from openspec/changes/add-streaming-parallel-1brc/tasks.md rename to openspec/changes/archive/2026-08-15-add-streaming-parallel-1brc/tasks.md diff --git a/openspec/changes/ingest-verification-receipts/.openspec.yaml b/openspec/changes/archive/2026-08-15-ingest-verification-receipts/.openspec.yaml similarity index 100% rename from openspec/changes/ingest-verification-receipts/.openspec.yaml rename to openspec/changes/archive/2026-08-15-ingest-verification-receipts/.openspec.yaml diff --git a/openspec/changes/ingest-verification-receipts/design.md b/openspec/changes/archive/2026-08-15-ingest-verification-receipts/design.md similarity index 100% rename from openspec/changes/ingest-verification-receipts/design.md rename to openspec/changes/archive/2026-08-15-ingest-verification-receipts/design.md diff --git a/openspec/changes/ingest-verification-receipts/proposal.md b/openspec/changes/archive/2026-08-15-ingest-verification-receipts/proposal.md similarity index 100% rename from openspec/changes/ingest-verification-receipts/proposal.md rename to openspec/changes/archive/2026-08-15-ingest-verification-receipts/proposal.md diff --git a/openspec/changes/ingest-verification-receipts/qualification.md b/openspec/changes/archive/2026-08-15-ingest-verification-receipts/qualification.md similarity index 100% rename from openspec/changes/ingest-verification-receipts/qualification.md rename to openspec/changes/archive/2026-08-15-ingest-verification-receipts/qualification.md diff --git a/openspec/changes/ingest-verification-receipts/specs/verification-receipt-comparison/spec.md b/openspec/changes/archive/2026-08-15-ingest-verification-receipts/specs/verification-receipt-comparison/spec.md similarity index 100% rename from openspec/changes/ingest-verification-receipts/specs/verification-receipt-comparison/spec.md rename to openspec/changes/archive/2026-08-15-ingest-verification-receipts/specs/verification-receipt-comparison/spec.md diff --git a/openspec/changes/ingest-verification-receipts/tasks.md b/openspec/changes/archive/2026-08-15-ingest-verification-receipts/tasks.md similarity index 100% rename from openspec/changes/ingest-verification-receipts/tasks.md rename to openspec/changes/archive/2026-08-15-ingest-verification-receipts/tasks.md diff --git a/openspec/changes/guard-performance-cloud-spend/.openspec.yaml b/openspec/changes/guard-performance-cloud-spend/.openspec.yaml new file mode 100644 index 00000000..0c73c8f5 --- /dev/null +++ b/openspec/changes/guard-performance-cloud-spend/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-08-15 diff --git a/openspec/changes/guard-performance-cloud-spend/design.md b/openspec/changes/guard-performance-cloud-spend/design.md new file mode 100644 index 00000000..c20c110e --- /dev/null +++ b/openspec/changes/guard-performance-cloud-spend/design.md @@ -0,0 +1,94 @@ +## Context + +The runtime profiler launches exact repository-owned Node, Vitest, Playwright, +and Go targets with bounded output and wall time. Qualification detects some +network and database signals, but direct CLI and campaign execution do not +share an immutable admission contract and the child process is not currently a +zero-egress sandbox. See `proposal.md` for the product risk. + +## Goals / Non-Goals + +**Goals:** + +- Make the default autonomous policy explicit, deterministic, and inspectable. +- Block unsafe work before the first child process and enforce the admitted + policy at runtime where CodeVetter has a dependable boundary. +- Reuse one policy across direct profiles, supervision, and campaigns. +- Keep blocked outcomes useful to agents through a compact receipt. + +**Non-Goals:** + +- Add hosted profiling, production load generation, billing integrations, or + approval management. +- Claim an OS-level sandbox for runtimes that CodeVetter cannot isolate. +- Infer that a workload is safe merely because credentials are absent. + +## Decisions + +### Use a closed preflight plan as the authority + +A new dependency-free governance module derives a plan from exact repository, +adapter, target, name, and duration inputs plus a bounded source scan. The plan +uses stable JSON hashing and closed validation. Every executable performance +entry point calls it; a separate CLI dry-run exposes the same result. + +Alternative: keep safety flags only in qualification. Rejected because callers +can bypass qualification and flags do not bind the later execution identity. + +### Support only local zero-egress autonomous mode + +The first schema has one executable mode: `local_zero_egress`. Remote, paid, +unknown-cost, load, soak, stress, and production evidence is terminally blocked +even if an approval string is supplied. This makes maximum external requests, +retries, and cost exactly zero rather than estimates. + +Alternative: implement hosted approval and pricing now. Rejected because it +would expand the product into cloud execution and billing before local safety is +proven. + +### Enforce Node-family egress with a preload and block unsupported runtimes + +Node, node-script, and Vitest processes receive a repository-owned preload that +rejects non-loopback DNS, sockets, HTTP(S), fetch, and WebSocket calls. Playwright +is admitted only with an explicit loopback URL and uses the same remote guard. +On macOS, every admitted workload also runs under `sandbox-exec`; this permits +Go test and benchmark targets with networking denied. On other platforms, Go +performance execution remains manually callable outside the autonomous +campaign but is not admitted until a portable, testable sandbox exists. +`GOPROXY=off` alone is not considered a network sandbox. + +Alternative: static source scanning alone. Rejected because dependencies and +computed endpoints can escape a lexical scan. + +### Count enforcement events, not inferred traffic + +The preload emits a bounded machine marker when it blocks a request. The parent +captures markers separately from redacted stdout/stderr and records attempted +external requests without retrying. An ordinary completed local run has zero +attempted external requests; a policy violation remains zero successful +external requests but records the blocked attempt and a failed admission +receipt. + +## Risks / Trade-offs + +- **Node APIs may add new network entry points** → Keep the preload small, + deny-by-default around built-in network modules, and cover supported APIs with + hermetic tests. +- **Playwright often starts a local dev server outside the test process** → The + autonomous path admits only already-declared loopback targets; server startup + orchestration stays outside this change. +- **Go becomes less automatic** → Return an explicit unsupported-enforcement + receipt rather than claiming zero egress. Manual profiling remains available. +- **Source scanning can over-block words used in fixtures** → Runtime + enforcement controls Node admission; lexical signals explain and block only + clearly remote/paid/production targets. + +## Migration Plan + +1. Add plan/receipt contracts and hermetic validation tests. +2. Add the Node preload and prove loopback/remote behavior without internet. +3. Route dry-run, direct profile, supervision, and campaign paths through the + shared admission check. +4. Document the stricter local boundary and run focused runtime tests. +5. Roll back by removing the new admission calls; no persisted database or + production configuration requires migration. diff --git a/openspec/changes/guard-performance-cloud-spend/proposal.md b/openspec/changes/guard-performance-cloud-spend/proposal.md new file mode 100644 index 00000000..3871a0e3 --- /dev/null +++ b/openspec/changes/guard-performance-cloud-spend/proposal.md @@ -0,0 +1,48 @@ +## Why + +CodeVetter's autonomous performance loop can execute project-owned test and +benchmark code, so a seemingly local profile can still contact deployed +applications or paid services. The profiler needs a machine-readable, +fail-closed execution policy before it can safely iterate without an operator +watching every run. + +## What Changes + +- Add an immutable performance-execution plan and receipt with exact duration, + process concurrency, retry, external-request, external-service, and monetary + bounds. +- Default autonomous profiling to a local zero-egress policy: one owned process, + no retries, no remote requests, no paid services, and a bounded wall clock. +- Add a dry-run operation that reports whether the exact workload is admitted + before project code executes. +- Reject browser flows without an explicit loopback-only target, workloads with + remote or unknown-cost signals, and runtimes for which zero-egress enforcement + is unavailable. +- Enforce Node-family zero-egress at runtime and emit a blocked/no-confidence + receipt when the workload attempts remote network access. +- Keep hosted, load, soak, stress, and production profiling unsupported by the + autonomous loop; this change does not modify Cloudflare or other production + configuration. + +## Capabilities + +### New Capabilities + +- `performance-execution-governance`: Defines immutable dry-run admission, + local zero-egress execution, bounded receipts, and fail-closed treatment of + hosted or unknown-cost workloads. + +### Modified Capabilities + +- `autonomous-optimization-campaigns`: Requires every campaign execution to be + admitted by the local performance-execution policy before correctness or + performance code runs. + +## Impact + +- Affects the repository-owned runtime profiler, campaign service, CLI/MCP + contracts, focused tests, and local performance documentation. +- Adds no production dependency, hosted service, credential, database change, + or production configuration. +- Existing explicitly invoked local profiles become stricter when their target + has remote-network or unknown-cost evidence. diff --git a/openspec/changes/guard-performance-cloud-spend/specs/autonomous-optimization-campaigns/spec.md b/openspec/changes/guard-performance-cloud-spend/specs/autonomous-optimization-campaigns/spec.md new file mode 100644 index 00000000..e26b92e7 --- /dev/null +++ b/openspec/changes/guard-performance-cloud-spend/specs/autonomous-optimization-campaigns/spec.md @@ -0,0 +1,15 @@ +## ADDED Requirements + +### Requirement: Campaign execution requires current local admission +The system SHALL derive and validate a current performance-execution plan before +running campaign correctness or performance scopes. A blocked, stale, or +identity-mismatched plan MUST leave the campaign in `no_confidence` and MUST NOT +execute project code or consume an experiment attempt. + +#### Scenario: Baseline is admitted locally +- **WHEN** every declared campaign scope has a current admitted zero-egress plan +- **THEN** the campaign may execute the bounded baseline and attach the admission receipts to its evidence + +#### Scenario: Candidate contains a remote workload +- **WHEN** any correctness or performance scope has remote, paid, or unknown-cost evidence +- **THEN** the campaign records `no_confidence` with the blocked admission receipt before executing any declared scope diff --git a/openspec/changes/guard-performance-cloud-spend/specs/performance-execution-governance/spec.md b/openspec/changes/guard-performance-cloud-spend/specs/performance-execution-governance/spec.md new file mode 100644 index 00000000..54cc4ddc --- /dev/null +++ b/openspec/changes/guard-performance-cloud-spend/specs/performance-execution-governance/spec.md @@ -0,0 +1,65 @@ +## Purpose + +Defines the fail-closed execution policy that keeps autonomous performance +profiling local, bounded, zero-egress, and independently auditable before any +project-owned workload is allowed to run. + +## ADDED Requirements + +### Requirement: Every performance run has an immutable admission plan +The system SHALL derive a versioned plan before project code executes. The plan +MUST bind repository and target identity, adapter, execution mode, maximum wall +time, process concurrency, retries, external requests, monetary cost, declared +external services, and approval identity. Unknown fields or unbounded values +MUST fail closed. + +#### Scenario: Local dry-run is admitted +- **WHEN** a caller dry-runs an exact supported local workload with no remote or unknown-cost evidence +- **THEN** the system returns an admitted zero-egress plan with one process, zero retries, zero external requests, zero external services, zero monetary cost, and a finite duration + +#### Scenario: Plan identity changes +- **WHEN** repository, target, adapter, budget, or execution-policy input differs from an earlier plan +- **THEN** the system returns a different immutable plan identity and does not reuse the prior admission + +### Requirement: Autonomous execution is local and zero-egress +The autonomous profiler SHALL execute only adapters for which it can enforce a +local zero-egress policy. It MUST block remote network access at runtime, allow +browser access only to loopback targets, run at most one owned workload process, +and perform no automatic retry. + +#### Scenario: Node workload attempts remote access +- **WHEN** an admitted Node-family workload attempts DNS, socket, HTTP, HTTPS, fetch, or WebSocket access to a non-loopback destination +- **THEN** the system blocks the operation, terminates or fails the workload, and reports a zero-egress policy violation without retrying + +#### Scenario: Runtime cannot enforce zero egress +- **WHEN** the selected adapter lacks an enforceable zero-egress boundary +- **THEN** the system blocks before project code executes and names the unsupported enforcement boundary + +### Requirement: Hosted and unknown-cost profiling remains closed +The autonomous profiler MUST NOT run hosted, paid, unknown-cost, load, soak, +stress, or production profiling. An approval identity MUST NOT silently widen +this product boundary; the plan SHALL report the unsupported execution mode and +maximum possible spend as unknown rather than zero. + +#### Scenario: Hosted target is requested without approval +- **WHEN** a workload includes a remote endpoint, paid service, production marker, or unknown pricing and no exact approval identity +- **THEN** the system blocks before the first request and records every detected service and missing approval input + +#### Scenario: Hosted target is requested with approval +- **WHEN** a caller supplies an approval identity for a hosted or paid workload +- **THEN** the autonomous profiler still reports hosted execution as unsupported and does not contact the service + +### Requirement: Every admission outcome emits a cost and egress receipt +The system SHALL emit a machine-readable receipt for admitted, blocked, failed, +and completed plans. The receipt MUST preserve planned and observed duration, +process concurrency, retry count, external request count, external services, +cost posture, enforcement method, terminal reason, and limitations without +including credentials or absolute private paths. + +#### Scenario: Workload is blocked before execution +- **WHEN** the plan is rejected because cost, service, approval, or enforcement evidence is missing +- **THEN** the receipt records zero executed requests and processes plus the projected bounds and exact blockers + +#### Scenario: Local workload completes +- **WHEN** an admitted zero-egress workload exits within its bounds +- **THEN** the receipt records the observed local process and duration totals and confirms zero external requests, retries, services, and monetary cost diff --git a/openspec/changes/guard-performance-cloud-spend/tasks.md b/openspec/changes/guard-performance-cloud-spend/tasks.md new file mode 100644 index 00000000..54c099a6 --- /dev/null +++ b/openspec/changes/guard-performance-cloud-spend/tasks.md @@ -0,0 +1,19 @@ +## 1. Closed governance contracts + +- [x] 1.1 Add versioned immutable performance-execution plan and receipt contracts with stable identities and closed validation +- [x] 1.2 Derive bounded local admission from exact repository, adapter, target, workload, and duration inputs without executing project code +- [x] 1.3 Fail closed on hosted, paid, production, load, soak, stress, unknown-cost, and unsupported-enforcement evidence + +## 2. Runtime enforcement and product operations + +- [x] 2.1 Add a Node-family preload that permits loopback and blocks remote DNS, sockets, HTTP(S), fetch, and WebSocket access with bounded policy markers +- [x] 2.2 Apply the same admission and zero-egress policy to direct performance profiling and durable supervision +- [x] 2.3 Add a machine-readable `plan-performance` dry-run CLI operation and equivalent MCP operation +- [x] 2.4 Gate campaign baseline, screen, and promotion execution on current admitted plans without consuming blocked attempts + +## 3. Verification and closeout + +- [x] 3.1 Add hermetic boundary tests for allowed local work, remote attempts, unsupported adapters, unknown cost, stale identity, and bounded receipts +- [x] 3.2 Document local profiling, public smoke, hosted/load boundaries, dry-run usage, and receipt interpretation +- [x] 3.3 Run focused runtime tests, lint, docs validation, strict OpenSpec validation, and diff checks +- [ ] 3.4 Link the shipped change to GitHub issue #105, sync the specification, archive the completed change, and update current product truth diff --git a/openspec/specs/autonomous-optimization-campaigns/spec.md b/openspec/specs/autonomous-optimization-campaigns/spec.md new file mode 100644 index 00000000..77e08506 --- /dev/null +++ b/openspec/specs/autonomous-optimization-campaigns/spec.md @@ -0,0 +1,114 @@ +# autonomous-optimization-campaigns Specification + +## Purpose +Defines a bounded, resumable optimization campaign in which an agent may +iterate on source code while CodeVetter remains the deterministic authority for +correctness, performance evidence, resource limits, and promotion decisions. +## Requirements +### Requirement: Campaign scope is immutable and reviewable +The system SHALL require a versioned campaign manifest that identifies the +repository revision, allowed mutable files, exact correctness scopes, exact +performance scope, sample policies, resource limits, experiment budget, and +stop conditions. The system MUST reject unknown fields, escaping paths, +unsupported adapters, incomplete scopes, and manifest identity drift after a +baseline exists. + +#### Scenario: Valid campaign is initialized +- **WHEN** an agent supplies a contained manifest with supported exact scopes and finite budgets +- **THEN** the system records a stable campaign identity before any candidate is evaluated + +#### Scenario: Manifest changes after baseline +- **WHEN** an agent changes an evaluation target, policy, protected path, or budget after the baseline was recorded +- **THEN** the system refuses to compare the candidate with the prior baseline + +### Requirement: Baseline precedes experimentation +The system SHALL execute every declared correctness scope and the declared +performance scope before accepting a candidate. A failed, incomplete, or +unstable baseline MUST leave the campaign in `no_confidence` and MUST NOT +authorize experimentation. + +#### Scenario: Complete baseline +- **WHEN** all exact correctness scopes pass and bounded performance evidence completes +- **THEN** the system records the baseline as the first immutable experiment and exposes one next candidate action + +#### Scenario: Baseline correctness failure +- **WHEN** any declared correctness scope fails or does not execute exactly +- **THEN** the system records the failure and does not establish an incumbent + +### Requirement: Correctness gates every performance decision +The system SHALL execute all declared correctness scopes against every +candidate before promotion. A correctness failure MUST produce `discard` or +`crash` regardless of measured performance, and a performance result MUST NOT +override an authoritative correctness result. + +#### Scenario: Faster incorrect candidate +- **WHEN** a candidate materially improves the performance metric but fails one correctness scope +- **THEN** the system records the performance observation but returns `discard` with correctness as the controlling reason + +#### Scenario: Correctness-preserving improvement +- **WHEN** every correctness scope passes and compatible performance evidence materially improves without a protected regression +- **THEN** the system may classify the candidate as `promising` or `keep` according to evidence strength + +### Requirement: Screening and promotion have separate authority +The system SHALL distinguish bounded screening evidence from promotion-quality +evidence. Screening MAY return `promising`, `discard`, `crash`, or +`no_confidence`; only promotion evidence meeting the configured sample floor, +paired-workload identity, stability, correctness, and secondary-resource policy +MAY return `keep`. + +#### Scenario: Three-sample material improvement +- **WHEN** a correct candidate materially improves under a three-sample screening policy +- **THEN** the system returns `promising` and requests promotion-quality verification instead of advancing the incumbent + +#### Scenario: Stable paired promotion +- **WHEN** independently runnable incumbent and candidate checkouts pass exact correctness and ten-sample paired verification without limitations +- **THEN** the system returns `keep` and records the candidate as the new incumbent + +### Requirement: Every experiment is durably attributable +The system SHALL append an experiment record containing campaign identity, +sequence, timestamp, repository revision, diff identity, hypothesis, evidence +identity, measurements, correctness outcomes, decision, reason, and limitations. +Existing experiment records MUST NOT be rewritten when later candidates run. + +#### Scenario: Candidate is discarded +- **WHEN** a candidate is slower, incorrect, unstable, or immaterial +- **THEN** the system appends the complete result and preserves the incumbent and all earlier records + +#### Scenario: Campaign resumes +- **WHEN** an agent reopens a valid campaign after the prior process exits +- **THEN** the system reconstructs the incumbent, remaining budget, experiment count, and next permitted action from validated durable records + +### Requirement: Autonomy is bounded by explicit stop conditions +The system SHALL stop requesting experiments when the manifest's experiment, +elapsed-time, consecutive-no-improvement, or consecutive-crash budget is +exhausted. `no_confidence` MUST NOT silently consume or extend an unspecified +budget, and the system MUST explain the controlling stop condition. + +#### Scenario: Plateau reached +- **WHEN** the configured number of consecutive non-improving candidates is recorded +- **THEN** the campaign status becomes `stopped` with `plateau` as the reason + +#### Scenario: Budget remains +- **WHEN** the latest experiment is terminal and at least one declared budget remains +- **THEN** the campaign exposes a bounded next action without generating or applying a source patch + +### Requirement: Agent strategy remains outside evidence authority +The system SHALL NOT generate hypotheses with a model, edit application source, +install dependencies, invoke arbitrary shell commands, reset Git state, or +weaken declared checks. It SHALL expose evidence and deterministic decisions so +an external agent program can choose and apply the next bounded experiment. + +#### Scenario: Agent requests a campaign decision +- **WHEN** an agent submits a candidate hypothesis after editing within the declared boundary +- **THEN** CodeVetter evaluates the candidate without claiming authorship of the patch or inference beyond captured evidence + +### Requirement: Machine operations are closed and repository scoped +The system SHALL expose start, baseline, evaluate, inspect, and status behavior +through machine-readable CLI and repository-scoped MCP operations with closed +schemas. Campaign artifacts MUST remain under an explicit repository-contained +directory and MUST redact environment values, credentials, query values, and +machine paths from portable evidence. + +#### Scenario: Unknown campaign argument +- **WHEN** an MCP or CLI caller supplies an unknown field, escaping artifact path, or unsupported operation +- **THEN** the system fails closed without running a workload or mutating campaign state diff --git a/openspec/specs/cross-runtime-scaled-parsing/spec.md b/openspec/specs/cross-runtime-scaled-parsing/spec.md new file mode 100644 index 00000000..d89ba91d --- /dev/null +++ b/openspec/specs/cross-runtime-scaled-parsing/spec.md @@ -0,0 +1,53 @@ +# cross-runtime-scaled-parsing Specification + +## Purpose +Provide comparable, bounded-memory Node and Go parsing workloads that let an +agent distinguish algorithm, parallelism, runtime, and machine effects using +the same deterministic file and correctness contract. +## Requirements +### Requirement: File-backed workloads share one correctness contract +The challenge SHALL generate a deterministic file whose row grammar and output +semantics match the documented 1BRC contract. Every runtime and execution +variant MUST process the same identified file and produce the same exact result +digest before performance evidence is accepted. + +#### Scenario: Runtime variants process a qualified input +- **WHEN** Node and Go variants execute a comparison campaign +- **THEN** every receipt identifies the same input bytes, row count, and digest +- **AND** any result mismatch invalidates that variant's performance result + +### Requirement: Default execution uses bounded memory and storage +Every default challenge variant SHALL process input incrementally without +loading the complete file as a decoded string. The default campaign MUST enforce +documented row, byte, duration, and worker bounds and remove generated data after +the campaign unless retention was explicitly requested. + +#### Scenario: Agent runs the default campaign +- **WHEN** no large-run opt-in is supplied +- **THEN** the campaign stays within its documented laptop-safe bounds +- **AND** does not invoke a network, cloud service, production database, or deployment + +#### Scenario: Agent requests one billion rows +- **WHEN** the requested input exceeds the default storage or duration bound +- **THEN** the runner refuses before generation unless explicit large-run authorization and sufficient local capacity are recorded + +### Requirement: Parallel variants preserve independent aggregation +The Node and Go parallel variants SHALL partition only at complete row +boundaries, maintain worker-local aggregates, and merge after parsing. Worker +count MUST be recorded and bounded independently of the host's reported logical +CPU count. + +#### Scenario: Parallel and sequential results are compared +- **WHEN** the same qualified input is processed with one and multiple workers +- **THEN** both produce the identical result digest +- **AND** the receipt records worker count, wall time, throughput, and peak memory + +### Requirement: Cross-runtime conclusions use paired evidence +The challenge SHALL report observed Node and Go results separately from inferred +language or leaderboard conclusions. A runtime comparison MUST use the same +machine, input identity, worker bound, cache-state policy, and timing boundary. + +#### Scenario: Agent asks whether language caused a performance gap +- **WHEN** compatible Node and Go receipts exist from the same campaign +- **THEN** the report attributes only the measured paired difference to runtime and implementation together +- **AND** labels unmeasured language-ceiling or cross-machine claims as unverified diff --git a/openspec/specs/durable-performance-supervision/spec.md b/openspec/specs/durable-performance-supervision/spec.md new file mode 100644 index 00000000..acfb876f --- /dev/null +++ b/openspec/specs/durable-performance-supervision/spec.md @@ -0,0 +1,59 @@ +# durable-performance-supervision Specification + +## Purpose +Define a durable outer execution boundary that preserves local performance-run identity and terminal evidence when the profiling child crashes, times out, is signaled, or cannot emit a valid capsule. +## Requirements +### Requirement: Supervision accepts only a closed profiling request +CodeVetter SHALL supervise only the existing `diagnose-performance` operation with one supported adapter, repository-contained target, optional workload name, bounded samples, warmups, timeout, and safe caller-supplied run ID. The supervisor MUST construct process arguments directly and MUST NOT accept a shell command, arbitrary arguments, inherited application environment, or an artifact path outside `.codevetter/performance-runs//`. + +#### Scenario: Exact workload is supervised +- **WHEN** an agent supplies a valid run ID and exact supported performance scope +- **THEN** CodeVetter starts one owned diagnosis child with the same closed scope and records that scope before launch + +#### Scenario: Run ID is unsafe or already exists +- **WHEN** the run ID escapes its artifact root, has an unsupported form, or names an existing run directory +- **THEN** CodeVetter rejects the request without launching a child or overwriting evidence + +### Requirement: Receipt exists before the child can fail +Before launching the diagnosis child, CodeVetter MUST atomically persist a versioned receipt containing run identity, repository revision, dirty state, exact scope, bounded policy, supervisor identity, `initialized` state, and no inferred performance result. After launch it SHALL atomically refresh a `running` heartbeat without rewriting source files. + +#### Scenario: Child is killed immediately after launch +- **WHEN** the child terminates before producing output +- **THEN** the pre-existing receipt still identifies the attempted scope and records its terminal signal or failure + +#### Scenario: Receipt is inspected while running +- **WHEN** an agent reads an active run +- **THEN** it receives the latest atomic heartbeat and last recorded state without observing partial JSON + +### Requirement: Every child outcome is classified without fabrication +The supervisor SHALL finalize exactly one terminal state from `succeeded`, `failed`, `timed_out`, `signaled`, `spawn_failed`, or `invalid_result`. A run is `succeeded` only when the child exits zero and emits exactly one valid performance diagnosis document. Non-zero exits, timeouts, signals, missing output, malformed JSON, or an incompatible schema MUST retain operational evidence and MUST NOT become performance findings. + +#### Scenario: Valid diagnosis completes +- **WHEN** the child exits zero with one valid diagnosis document +- **THEN** the receipt records `succeeded`, the result digest, and a bounded result reference + +#### Scenario: Child exits from a signal +- **WHEN** the operating system reports a child signal before valid completion +- **THEN** the receipt records `signaled`, the signal, last heartbeat, and no performance conclusion + +#### Scenario: Child prints malformed JSON +- **WHEN** the child exits but stdout is not exactly one valid diagnosis document +- **THEN** the receipt records `invalid_result` with bounded redacted output evidence + +### Requirement: Artifacts are bounded, redacted, and inspectable +CodeVetter SHALL retain only the atomic receipt, an optional validated result document, and bounded redacted failure output under the owned run directory. It MUST record byte bounds, truncation, redaction count, and result digest, and MUST expose read-only inspection by run ID through CLI and MCP without accepting a repository path after MCP startup. + +#### Scenario: Failure output contains secrets +- **WHEN** child output includes credential-shaped values or absolute repository paths +- **THEN** the stored and returned evidence contains redaction markers and bounded text only + +#### Scenario: Agent inspects a completed run +- **WHEN** an agent requests an existing run ID +- **THEN** CodeVetter returns the validated receipt and result summary or operational failure evidence without rerunning the workload + +### Requirement: Supervisor cleanup is owned and bounded +On timeout, cancellation, or supervisor shutdown, CodeVetter SHALL terminate only the owned child process tree, wait a bounded grace interval, finalize the receipt when the supervisor remains alive, and stop heartbeat activity. It MUST NOT kill unrelated processes, delete target artifacts, or claim cleanup succeeded without observing child termination. + +#### Scenario: Workload exceeds its supervisor deadline +- **WHEN** the child remains alive beyond the derived bounded run deadline +- **THEN** CodeVetter terminates the owned process tree, records `timed_out`, and returns the operational outcome diff --git a/openspec/specs/evidence-backed-product-comparisons/spec.md b/openspec/specs/evidence-backed-product-comparisons/spec.md new file mode 100644 index 00000000..108da08e --- /dev/null +++ b/openspec/specs/evidence-backed-product-comparisons/spec.md @@ -0,0 +1,29 @@ +# evidence-backed-product-comparisons Specification + +## Purpose +Define public comparison pages that help people and retrieval systems distinguish +CodeVetter's execution-backed verification from adjacent code-review products. +## Requirements +### Requirement: Direct comparison routes +The site SHALL publish canonical comparison pages for CodeRabbit and Greptile +using the existing public editorial surface. + +#### Scenario: Visitor opens a comparison +- **WHEN** a visitor requests either approved comparison route +- **THEN** the site returns a readable page with a unique title, description, and canonical URL + +### Requirement: Evidence-bounded claims +Each comparison MUST cite dated first-party sources, identify the CodeVetter +evidence used, and disclose that no common head-to-head benchmark has been run. + +#### Scenario: Reader evaluates a performance claim +- **WHEN** a reader inspects catch-rate, precision, speed, memory, or cost language +- **THEN** the page makes no superiority claim unsupported by a shared controlled run + +### Requirement: Human and agent discovery +Each comparison SHALL be included in the site's sitemap and agent-readable +Markdown projection and SHALL link to adjacent verification and benchmark pages. + +#### Scenario: A crawler discovers the page +- **WHEN** a search or AI crawler follows the site's existing discovery surfaces +- **THEN** it can retrieve the comparison and its supporting internal links diff --git a/openspec/specs/flow-optimization-campaigns/spec.md b/openspec/specs/flow-optimization-campaigns/spec.md new file mode 100644 index 00000000..ec177d3e --- /dev/null +++ b/openspec/specs/flow-optimization-campaigns/spec.md @@ -0,0 +1,84 @@ +# flow-optimization-campaigns Specification + +## Purpose +Let an AI coding agent discover and prioritize bounded local performance flows +before entering CodeVetter's existing correctness-gated optimization loop. +## Requirements +### Requirement: Repository flows are discovered without execution first +CodeVetter SHALL use bounded repository qualification to discover exact local +performance workloads before running application code. Only supported adapters, +contained targets, direct timing evidence, and candidates without unsafe safety +flags MUST be eligible for automatic screening. A loopback-only local service +signal MAY remain eligible for the supported Node flow adapters. + +#### Scenario: Safe measured workloads exist +- **WHEN** a repository contains exact Node, Vitest, or Go benchmark workloads with direct timing evidence +- **THEN** CodeVetter returns a bounded deterministic inventory ordered by qualification evidence + +#### Scenario: Candidate may access external state +- **WHEN** qualification detects network, integration, secret, production, or escaping-path evidence +- **THEN** the candidate remains visible as excluded and MUST NOT execute automatically + +#### Scenario: URL text is only local fixture data +- **WHEN** a measured workload contains URL strings but does not invoke a network client +- **THEN** CodeVetter does not classify those strings alone as external execution evidence + +### Requirement: Screening uses existing runtime evidence +CodeVetter SHALL screen at most the caller's bounded flow limit using existing +performance capsules and deterministic diagnosis. Every screened flow MUST +retain exact scope identity, measurement provenance, diagnosis, limitations, +and cleanup state. + +#### Scenario: Exact workload completes +- **WHEN** an eligible discovered workload executes successfully +- **THEN** its flow-campaign entry references its measured supported-scale cost and deterministic diagnosis + +#### Scenario: Workload is incomplete or startup dominated +- **WHEN** a workload fails, times out, lacks a comparable domain metric, or is dominated by runner startup +- **THEN** CodeVetter does not assign an optimization priority and returns the missing evidence as its next action + +### Requirement: Priority combines measured cost with explicit product context +CodeVetter SHALL rank actionable flows by measured supported-scale milliseconds +multiplied by bounded frequency and user-impact weights. Optional project-owned +weights MUST bind to exact candidate identity; absent weights MUST default to +neutral values and remain disclosed as unverified product context. + +#### Scenario: Product weights are supplied +- **WHEN** a valid priority manifest supplies frequency and user-impact weights for a discovered candidate +- **THEN** the result records both weights, their provenance, and the resulting deterministic priority score + +#### Scenario: Product weights are absent +- **WHEN** no matching priority entry exists +- **THEN** CodeVetter uses neutral weights, reports that production frequency and user impact are unknown, and MUST NOT claim production impact + +#### Scenario: Flow is already cheap +- **WHEN** deterministic diagnosis classifies a flow as already fast at its supported scale +- **THEN** the flow is retained as a regression guardrail but MUST rank below actionable optimization flows + +### Requirement: One next action controls the campaign handoff +CodeVetter SHALL return one deterministic next action for the complete plan. It +MUST select the highest-priority actionable flow, request a better workload when +evidence is inadequate, or recommend another repository when no material local +flow remains. + +#### Scenario: Actionable flow leads the plan +- **WHEN** at least one screened flow has actionable diagnosis and comparable supported-scale cost +- **THEN** the next action identifies its exact adapter, target, name, and the manifest inputs required by the existing optimization campaign + +#### Scenario: No actionable flow remains +- **WHEN** every screened flow is already cheap or non-actionable +- **THEN** the next action preserves useful guardrails and recommends profiling a different product flow + +### Requirement: Planner operations remain closed, local, and bounded +The planner SHALL be exposed through machine-readable CLI and MCP operations +with closed arguments. It MUST NOT edit product source, install dependencies, +invoke a model, contact production, infer credentials, or retain raw profiler +artifacts. + +#### Scenario: Agent starts local planning +- **WHEN** an agent calls the planner with a repository, bounded flow count, sample policy, and optional contained priority manifest +- **THEN** CodeVetter performs only the declared local screening work and returns one validated portable result + +#### Scenario: Unknown or unsafe input +- **WHEN** a caller supplies unknown fields, an escaping manifest path, unsupported weights, or an excessive flow count +- **THEN** the operation fails closed before executing a workload diff --git a/openspec/specs/local-flow-runtime-tools/spec.md b/openspec/specs/local-flow-runtime-tools/spec.md new file mode 100644 index 00000000..91595673 --- /dev/null +++ b/openspec/specs/local-flow-runtime-tools/spec.md @@ -0,0 +1,102 @@ +# local-flow-runtime-tools Specification + +## Purpose +Give coding agents bounded machine operations that capture and interrogate local application flows using runtime evidence rather than prompt-only profiling instructions. +## Requirements +### Requirement: Exact local flow capture +CodeVetter SHALL capture one exact supported local workload as a root flow without requiring source-code modification, arbitrary shell execution, or a hosted service. The capture result MUST record the adapter, exact target and name, revision identity, capture policy, executions, limitations, and cleanup outcome. + +#### Scenario: Agent captures a Node HTTP test +- **WHEN** an agent requests an exact repository-contained Node test through the local flow capture tool +- **THEN** CodeVetter runs only that bounded test scope and returns an opaque capture identifier plus a compact root-flow summary + +#### Scenario: Unsupported or incomplete capture +- **WHEN** the exact workload is unsupported, fails, times out, escapes the repository, or produces incomplete required evidence +- **THEN** CodeVetter returns `no_confidence` and MUST NOT create an actionable optimization claim + +### Requirement: Recursive flow evidence +CodeVetter SHALL represent a captured workload as recursively related flows with stable capture-local identifiers. Each flow MUST distinguish observed elapsed time from inferred or unaccounted time and MUST cite the runtime evidence that created it. + +#### Scenario: Local HTTP client and server activity +- **WHEN** a captured Node workload performs loopback HTTP requests handled in the same diagnostic execution +- **THEN** the result includes bounded client and server child flows with method, normalized route, status, elapsed time, and causal relationships where correlation is supported + +#### Scenario: Request-scoped built-in SQLite activity +- **WHEN** an observed Node HTTP handler executes built-in `node:sqlite` statements +- **THEN** each execution is nested beneath that server flow with its operation, normalized value-free statement shape, outcome, and elapsed time + +#### Scenario: Same-execution child accounting +- **WHEN** a diagnostic parent and its children have comparable timestamps from the same execution +- **THEN** CodeVetter reports interval-union accounted time and remaining unaccounted time without exceeding the parent duration + +#### Scenario: Flow detail is unavailable +- **WHEN** runtime instrumentation cannot account for a portion of root-flow elapsed time +- **THEN** CodeVetter reports the portion as unaccounted or unavailable rather than assigning it to a source location + +### Requirement: Progressive machine queries +CodeVetter SHALL expose closed-schema machine operations to capture, inspect, explain, and verify local flows. Query operations MUST accept opaque identifiers returned by prior operations and MUST return bounded structured content without requiring an agent skill to parse raw profile formats. + +#### Scenario: Untrained agent inspects a capture +- **WHEN** an MCP client lists tools and calls the inspection operation with a valid capture identifier +- **THEN** the client receives the flow hierarchy, evidence coverage, materiality, and limitations without reading a V8 or pprof artifact + +#### Scenario: Unknown identifier or argument +- **WHEN** a client supplies an unknown tool argument, flow identifier, or capture identifier +- **THEN** the tool fails closed with a sanitized bounded error + +### Requirement: Evidence, inference, and actionability remain separate +CodeVetter MUST keep direct observations, deterministic interpretations, unverified hypotheses, and verified comparisons separate. A sampled source location SHALL be actionable only when it satisfies the recorded materiality policy and repeats across independent diagnostic profiles or is supported by a compatible deterministic domain metric. + +#### Scenario: Low-sample unstable hotspot +- **WHEN** independent diagnostic profiles disagree on the leading source candidate or the candidate is immaterial to the root flow +- **THEN** CodeVetter returns `no_confidence` with the missing evidence and MUST NOT recommend editing that source location + +#### Scenario: Stable material candidate +- **WHEN** independent profiles agree on a repository-owned candidate and the candidate passes the recorded sample, duration, and share thresholds +- **THEN** CodeVetter may return an unverified actionable hypothesis with an explicit falsification experiment + +#### Scenario: Repeated application function intersects CPU evidence +- **WHEN** bounded V8 coverage records a named repository application function repeatedly and CPU evidence selects the same file/function family +- **THEN** CodeVetter reports a repeated-work hypothesis with both evidence references and an explicit identical-scope verification experiment + +#### Scenario: Function frequency lacks timing support +- **WHEN** a repository function executes frequently but does not intersect material CPU evidence +- **THEN** CodeVetter reports observed frequency only and MUST NOT call the function slow or actionable + +### Requirement: Identical-scope optimization verification +CodeVetter SHALL compare compatible baseline and candidate captures using identical workload identity and unprofiled measurements. Verification MUST distinguish mechanical improvement from material product impact. + +#### Scenario: Agent verifies a candidate change +- **WHEN** an agent captures the same flow before and after one candidate change and requests verification +- **THEN** CodeVetter reports capture identifiers, compatibility, measured movement, statistical limitations, mechanical confirmation, material usefulness, and whether shipping is recommended without embedding either complete source capsule + +#### Scenario: Incompatible flows +- **WHEN** the adapter, target, exact name, or required measurement identity differs +- **THEN** verification returns `no_confidence` and MUST NOT confirm the optimization + +### Requirement: Local privacy, containment, and cost bounds +CodeVetter MUST redact captured output before normalization, retain no raw profile by default, remove owned temporary artifacts, bound executions and stored captures, and avoid hosted-service or production configuration access. + +#### Scenario: Capture completes successfully +- **WHEN** a local capture finishes +- **THEN** raw owned diagnostic artifacts are removed and the result records redaction, truncation, and temporary-artifact retention state + +#### Scenario: Runtime flow contains request data +- **WHEN** HTTP instrumentation observes a URL containing query values or variable-looking path segments +- **THEN** CodeVetter omits query values and normalizes sensitive-looking segments before returning or storing the flow + +#### Scenario: SQL execution contains application values +- **WHEN** request-scoped SQLite execution uses literals or bound arguments +- **THEN** CodeVetter captures neither arguments nor rows and replaces SQL literals with placeholders before returning or storing the statement shape + +#### Scenario: Function coverage capture completes +- **WHEN** the application-frequency diagnostic pass completes +- **THEN** CodeVetter retains only bounded repository-relative function names, source anchors, and counts and removes the raw V8 coverage documents + +#### Scenario: Nested Vitest assertion is selected by leaf name +- **WHEN** an agent supplies a leaf test name inside one or more Vitest `describe` blocks +- **THEN** CodeVetter executes exactly one matching assertion, or returns no confidence when the leaf name is absent or ambiguous + +#### Scenario: Vitest records transformed TypeScript execution +- **WHEN** the repository-local Vitest V8 coverage provider is available +- **THEN** CodeVetter writes its JSON report only to an owned temporary directory and normalizes positive named functions against original TypeScript locations diff --git a/openspec/specs/local-node-script-profiling/spec.md b/openspec/specs/local-node-script-profiling/spec.md new file mode 100644 index 00000000..f574db58 --- /dev/null +++ b/openspec/specs/local-node-script-profiling/spec.md @@ -0,0 +1,23 @@ +# local-node-script-profiling Specification + +## Purpose +Define how CodeVetter profiles a repository-owned standalone JavaScript benchmark when the workload is an executable script rather than a test-runner test. +## Requirements +### Requirement: Standalone Node profiling is closed and exact +CodeVetter SHALL accept `node-script` only for one repository-contained `.js`, `.mjs`, or `.cjs` target. It MUST invoke the current Node executable with that target and no caller-supplied arguments, shell, package script, install step, or inherited application environment. + +#### Scenario: Local JavaScript benchmark is selected +- **WHEN** an agent explicitly profiles a repository-relative `.mjs` benchmark through `node-script` +- **THEN** CodeVetter executes only that file with the bounded sample, warmup, timeout, environment, and process-cleanup policy + +#### Scenario: Non-JavaScript target is supplied +- **WHEN** a `node-script` target has an unsupported extension or escapes the repository +- **THEN** CodeVetter rejects it before starting a process + +### Requirement: Node script evidence uses the existing performance contract +A successful standalone script profile SHALL emit the existing versioned performance capsule with `node-script` adapter identity, wall-time samples, bounded console benchmark metrics, V8 source profiles, limitations, and disposable artifact guarantees. CodeVetter MUST NOT describe the script as a test or infer correctness from exit success alone. + +#### Scenario: Script emits benchmark metrics +- **WHEN** a successful script emits bounded `[benchmark]` metrics and repository-owned V8 samples +- **THEN** the capsule reports median unprofiled metrics and source attribution separately +- **AND** labels the scope as a Node script rather than a test-runner test diff --git a/openspec/specs/local-performance-governance/spec.md b/openspec/specs/local-performance-governance/spec.md index 1c87069e..a30c81ec 100644 --- a/openspec/specs/local-performance-governance/spec.md +++ b/openspec/specs/local-performance-governance/spec.md @@ -4,9 +4,7 @@ Define reproducible local IPC and disk measurements plus reversible, evidence-gated cache consolidation. - ## Requirements - ### Requirement: Dashboard and cache performance is reproducibly measured CodeVetter SHALL provide a repository-owned qualification that measures bounded p50, p95, maximum latency, result bytes, and error state for dashboard IPC paths @@ -32,3 +30,26 @@ duplicate evidence, a reversible plan, and before/after receipts. - **WHEN** profiling does not show a dashboard IPC or TypeScript/Rust boundary exceeding its budget - **THEN** CodeVetter records no service rewrite recommendation - **AND** does not add another runtime + +### Requirement: External-frontier comparisons are qualified +CodeVetter SHALL NOT express proximity to an external performance frontier as a +direct measured gap unless the local and external results share compatible +input, correctness, timing, resource, and machine conditions. Otherwise it MUST +identify the arithmetic as an extrapolation and enumerate the incompatible +conditions. + +#### Scenario: Local bounded parser is compared with the 1BRC leaderboard +- **WHEN** the local result excludes file I/O, uses fewer rows, or runs on different hardware +- **THEN** CodeVetter labels any projected multiplier as non-comparable +- **AND** reports the missing end-to-end evidence needed for a direct claim + +### Requirement: Large local campaigns require resource qualification +Before a performance campaign generates or retains a materially large fixture, +CodeVetter SHALL calculate the requested bytes, confirm available local space, +record the retention policy, and require explicit authorization above the +documented default bound. + +#### Scenario: Requested fixture is approximately 12 GB +- **WHEN** an agent requests a full one-billion-row challenge +- **THEN** CodeVetter does not generate the fixture under default settings +- **AND** reports the expected local storage and authorization requirement diff --git a/openspec/specs/performance-run-diagnostics/spec.md b/openspec/specs/performance-run-diagnostics/spec.md new file mode 100644 index 00000000..c2211f1f --- /dev/null +++ b/openspec/specs/performance-run-diagnostics/spec.md @@ -0,0 +1,27 @@ +# performance-run-diagnostics Specification + +## Purpose +Ensure incomplete performance runs contain enough bounded runtime evidence for an agent to distinguish a broken workload from a profiler or runner compatibility problem. +## Requirements +### Requirement: Failed passes retain bounded redacted evidence +Each unsuccessful warmup, measurement, metrics, profile, or coverage execution SHALL report its phase, index, terminal state, duration, workload-selection state, and bounded redacted operational error, stdout, and stderr. Successful executions SHALL omit failure output. + +#### Scenario: Profiler option is unsupported +- **WHEN** a profiling pass exits non-zero because the installed runner rejects an option +- **THEN** the performance capsule includes the bounded redacted runner error on that exact pass +- **AND** the diagnosis remains `no_confidence` + +#### Scenario: Failure output contains a credential-shaped value +- **WHEN** failed process output contains a token or repository path +- **THEN** retained failure evidence contains redaction markers and no raw sensitive value + +### Requirement: Bounded output does not erase confirmed selection +Every execution summary SHALL record whether bounded output directly confirms that the requested workload ran. For repeated successful Vitest passes, CodeVetter MAY accept one exact passed-selection confirmation for the fixed command when another successful JSON report was truncated, but MUST NOT accept a zero-match or failed execution as complete. + +#### Scenario: Exact Vitest JSON is truncated +- **WHEN** measurement JSON exceeds the output bound but a successful metrics or profile pass confirms the exact test identity +- **THEN** CodeVetter treats selection as confirmed while disclosing output truncation + +#### Scenario: No pass confirms the exact identity +- **WHEN** every bounded report lacks evidence that the selected test ran +- **THEN** the performance capsule remains `no_confidence` even if processes exited zero diff --git a/openspec/specs/runtime-failure-capsules/spec.md b/openspec/specs/runtime-failure-capsules/spec.md new file mode 100644 index 00000000..e151c61e --- /dev/null +++ b/openspec/specs/runtime-failure-capsules/spec.md @@ -0,0 +1,121 @@ +# runtime-failure-capsules Specification + +## Purpose +Define a bounded machine-readable diagnosis for common Node, browser, Cloudflare Worker, and Go verification failures without requiring a universal debugger or turning incomplete evidence into proof. +## Requirements +### Requirement: Supported runtime lanes are detected from repository evidence +CodeVetter SHALL detect Node test, browser test, Cloudflare Worker test, and Go +test lanes only from bounded repository manifests and configuration. Detection +MUST report the evidence, supported adapters, and limitations and MUST NOT claim +that a detected lane has executed successfully. + +#### Scenario: Worker Vitest repository is detected +- **WHEN** a repository contains a package manifest, Vitest configuration, and a Wrangler configuration +- **THEN** CodeVetter reports Node test and Cloudflare Worker test lanes with the evidence paths that established them + +#### Scenario: Unsupported repository is inspected +- **WHEN** no supported manifest or test configuration is found +- **THEN** CodeVetter returns an empty support set and explicit limitations without running a guessed command + +### Requirement: Diagnostic execution is exact, bounded, and shell-free +CodeVetter SHALL execute only a closed adapter with one repository-relative test +target and optional exact test-name selector. Supported executable adapters in +the first slice SHALL be Node test, Vitest, Playwright, and Go test. Each run +MUST use separated program arguments, a declared timeout, bounded output, +minimal inherited environment, and owned process termination. + +#### Scenario: Exact failing test is rerun +- **WHEN** the selected adapter, test target, and optional test name are valid and available +- **THEN** CodeVetter runs only that declared diagnostic scope and records the exact executable identity and arguments without invoking a shell + +#### Scenario: Target escapes the repository +- **WHEN** a test target is absolute, traverses outside the repository, resolves through an escaping symlink, or is not a regular file +- **THEN** CodeVetter rejects the run before starting a process + +#### Scenario: Diagnostic execution times out +- **WHEN** the owned diagnostic process exceeds its declared timeout +- **THEN** CodeVetter terminates it, records the timeout as an operational limitation, and returns `no_confidence` + +### Requirement: Failure capsules separate evidence from interpretation +Each diagnostic run SHALL return one versioned Runtime Failure Capsule with +subject identity, adapter identity, exact scope, terminal state, observations, +source frames, relevant changes, limitations, capture coverage, and a verdict. +Directly captured observations MUST remain separate from deterministic +relationships and unverified hypotheses. The first slice MUST NOT ask a model +to create evidence or a verdict. + +#### Scenario: Test fails with a changed source frame +- **WHEN** a diagnostic test reproduces a failure and its stack contains a source frame intersecting the selected Git diff +- **THEN** the capsule records the exception or panic as observed evidence and the frame-to-change match as a deterministic relationship + +#### Scenario: Diagnostic rerun does not reproduce +- **WHEN** the selected diagnostic scope exits successfully or contains no qualifying failure +- **THEN** the capsule returns `no_confidence`, states that the failure did not reproduce, and does not invent a likely cause + +### Requirement: Source and diff correlation is deterministic and bounded +CodeVetter SHALL normalize repository-contained source frames, inspect one +explicit Git diff range or the local diff, and rank relevant changed files and +lines using deterministic rules. Changed-frame intersection SHALL outrank +same-file proximity, and absent matches SHALL remain an explicit evidence gap. + +#### Scenario: Stack line is changed +- **WHEN** an observed frame points to a line added or modified by the selected diff +- **THEN** that file and line rank first with reason `changed_frame_intersection` + +#### Scenario: Failure has only dependency frames +- **WHEN** every observed frame is outside the repository or under excluded dependency/generated roots +- **THEN** CodeVetter records no relevant source match and preserves the source-attribution limitation + +### Requirement: Captured data is redacted and bounded before output +CodeVetter MUST redact credential-shaped keys and values, authorization and +cookie material, configured sensitive fields, environment values, URL query +values, and repository-absolute path prefixes before evidence enters a capsule. +Collections, strings, stack frames, output bytes, and artifacts MUST have hard +bounds, and truncation MUST be disclosed. + +#### Scenario: Failure output contains a token +- **WHEN** stdout, stderr, or an imported receipt contains credential-shaped material +- **THEN** the capsule contains a redaction marker instead of the material and records that redaction occurred + +#### Scenario: Runner emits oversized output +- **WHEN** captured output exceeds its byte bound +- **THEN** CodeVetter retains only the bounded prefix or suffix required by policy and records truncation without treating capture as complete + +### Requirement: Existing browser and Worker evidence is normalized, not rerun by a new engine +CodeVetter SHALL accept bounded existing T-Rex, warm-verification, Playwright, +and Worker-test result documents as imported observations. The original receipt +identity, verdict, limitations, and provenance MUST remain authoritative; the +capsule MUST NOT upgrade `failed` or `no_confidence` evidence to a pass. + +#### Scenario: Failed Playwright receipt is imported +- **WHEN** a bounded receipt contains a failed browser test, page exception, console failure, or network failure +- **THEN** CodeVetter maps those facts into normalized observations while retaining the source receipt identity and limitations + +#### Scenario: Worker receipt is incomplete +- **WHEN** a Worker test result lacks terminal or source identity +- **THEN** the capsule reports incomplete imported evidence and returns `no_confidence` + +### Requirement: Machine interface has stable outcomes +The repository CLI SHALL expose lane detection and diagnostic execution with +JSON output. It SHALL exit `1` for a reproduced executable failure, `2` for +`no_confidence`, invalid input, or operational failure, and `0` only for a +successful detection request. A diagnostic failure capsule MUST NOT be treated +as proof that the overall change fails beyond its exact scope. + +#### Scenario: JSON diagnostic reproduces a failure +- **WHEN** the CLI runs a selected diagnostic scope with `--json` and captures a qualifying failure +- **THEN** stdout contains exactly one capsule document and the process exits `1` + +#### Scenario: Detection succeeds +- **WHEN** the CLI inspects a valid repository without executing tests +- **THEN** stdout contains one support report and the process exits `0` + +### Requirement: Power-law coverage is measured against owned fixtures +CodeVetter SHALL maintain a small owned corpus spanning Node exceptions, +asynchronous failures, browser or Worker receipt failures, Go panics, redaction, +non-reproduction, timeout, and changed-line attribution. Published coverage +claims MUST derive from executed corpus results rather than stack inventory. + +#### Scenario: Corpus qualification runs +- **WHEN** the focused qualification command executes the owned fixtures +- **THEN** it reports per-lane reproduction, relevant-file attribution, redaction, and no-confidence outcomes with no provider, network, browser download, or production dependency diff --git a/openspec/specs/runtime-performance-capsules/spec.md b/openspec/specs/runtime-performance-capsules/spec.md new file mode 100644 index 00000000..add0cd48 --- /dev/null +++ b/openspec/specs/runtime-performance-capsules/spec.md @@ -0,0 +1,226 @@ +# runtime-performance-capsules Specification + +## Purpose +Define bounded runtime performance evidence that helps coding agents detect regressions and locate repository-owned bottleneck candidates in exact Node, React, and Go workflows. +## Requirements +### Requirement: Profiling execution is exact, bounded, and opt-in +CodeVetter SHALL profile only a closed supported adapter with one +repository-relative target and optional exact workload name. Each request MUST +declare bounded warmup and sample counts, use separated process arguments, and +apply the existing repository-containment, timeout, minimal-environment, and +owned-process cleanup guarantees. + +#### Scenario: Exact Node workload is profiled +- **WHEN** a user selects a Node test or Vitest file and an exact test identity +- **THEN** CodeVetter runs only that scope for the bounded warmup, measurement, and profiling passes + +#### Scenario: Exact Go benchmark is profiled +- **WHEN** a user selects a Go benchmark file and exact benchmark identity +- **THEN** CodeVetter runs that package with an exact benchmark expression and bounded count without running unrelated tests + +#### Scenario: Profile target escapes the repository +- **WHEN** the target is absolute, traverses outside the repository, or resolves through an escaping symlink +- **THEN** CodeVetter rejects the request before starting a process + +#### Scenario: Repository contains untracked files +- **WHEN** the profiled worktree contains an untracked non-ignored file +- **THEN** the capsule records a dirty snapshot without fabricating changed-line intersections for that file + +### Requirement: Performance capsules separate measurements from findings +Each successful request SHALL emit one versioned Runtime Performance Capsule +containing subject identity, exact scope, terminal executions, wall-time +distribution, runtime-specific measurements, repository-owned hotspots, +deterministic findings, limitations, and capture coverage. Captured values MUST +remain separate from comparisons and unverified optimization hypotheses. + +#### Scenario: V8 profile contains repository source samples +- **WHEN** a Node or Vitest profiling pass produces valid V8 CPU profiles +- **THEN** the capsule reports bounded repository-relative hotspot functions, source locations, self time, and sample share as observed evidence + +#### Scenario: Go benchmark reports allocations +- **WHEN** a Go benchmark emits valid benchmark and allocation measurements +- **THEN** the capsule reports bounded `ns/op`, `B/op`, and `allocs/op` values with benchmark identity and provenance + +#### Scenario: Go profiles contain repository symbols +- **WHEN** the diagnostic Go pass emits CPU and allocation profiles that the installed toolchain can read +- **THEN** the capsule reports bounded repository-relative functions, source lines, flat and cumulative values, profile kind, and sample share without retaining the raw profiles + +#### Scenario: Runtime profile lacks application samples +- **WHEN** profiling captures only runner, dependency, or test-harness work +- **THEN** CodeVetter reports the coverage limitation and does not name an application bottleneck + +#### Scenario: Vitest startup dominates the measured scope +- **WHEN** reported assertion time is less than the bounded share of exact-scope process wall time +- **THEN** CodeVetter records a startup-dominated finding and does not attribute the process latency to product code + +### Requirement: Baseline comparisons are explicit and reproducible +CodeVetter SHALL compare a current capsule only with an explicitly supplied, +compatible baseline capsule. The comparison MUST record both subject identities, +metric, sample counts, absolute and percentage deltas, and threshold policy. +Incompatible or statistically insufficient evidence MUST remain +`no_confidence`. + +#### Scenario: Wall time materially regresses +- **WHEN** the current median exceeds the compatible baseline by both the recorded relative and absolute thresholds +- **THEN** the capsule deterministically reports a regression with the measured deltas and exits with the regression outcome + +#### Scenario: Current and baseline scopes differ +- **WHEN** adapter, target, or exact workload identity differs between current and baseline +- **THEN** CodeVetter refuses the comparison and reports `no_confidence` + +#### Scenario: No baseline is supplied +- **WHEN** profiling completes without a baseline capsule +- **THEN** CodeVetter reports measurements and hotspot candidates without claiming regression or improvement + +#### Scenario: Host timing is unstable +- **WHEN** current or baseline wall-time samples exceed the recorded variability threshold +- **THEN** CodeVetter reports host-load and sample-spread evidence and refuses to claim a regression + +### Requirement: Profiling artifacts are private, bounded, and disposable +CodeVetter MUST create runtime profiles only in an owned temporary directory, +redact repository prefixes and credential-shaped values before output, bound all +measurements and hotspots, disclose truncation, and remove the temporary profile +directory after parsing. It MUST NOT modify tracked target files or retain raw +profiles by default. + +#### Scenario: Profile output contains sensitive material +- **WHEN** runner output or a source label contains credential-shaped data +- **THEN** the capsule contains a redaction marker and records the redaction count + +#### Scenario: Profile contains excessive nodes +- **WHEN** runtime evidence exceeds collection bounds +- **THEN** the capsule retains only the ranked bounded subset and discloses omitted evidence + +### Requirement: Machine outcomes distinguish regression from uncertainty +The profiling CLI SHALL emit exactly one JSON document. It SHALL exit `0` for a +completed measurement with no demonstrated regression, `1` for a demonstrated +compatible-baseline regression, and `2` for invalid input, failed workloads, +timeouts, incompatible baselines, incomplete measurements, or cleanup failure. + +#### Scenario: Workload fails during profiling +- **WHEN** any required measured or profiling execution exits unsuccessfully +- **THEN** CodeVetter emits `no_confidence`, records the failed execution, and exits `2` + +### Requirement: Agents receive an evidence-linked performance diagnosis +CodeVetter SHALL expose one `diagnose-performance` operation that profiles an +exact supported workload and emits a versioned diagnosis containing the +originating capsule, ranked observations, explicit inferences, unverified +hypotheses, one next bounded action, and a same-scope verification recipe. The +diagnosis MUST be deterministic and MUST NOT invoke a model or modify source. + +#### Scenario: Go benchmark exposes allocation pressure +- **WHEN** a successful Go capsule contains allocation measurements and repository-owned allocation paths +- **THEN** the diagnosis ranks the measured `B/op` and `allocs/op`, identifies the strongest path as an inferred candidate, and requires the same benchmark to verify an optimization + +#### Scenario: Deterministic catalogue benchmark scales superlinearly +- **WHEN** a successful capsule contains at least two same-unit metrics whose names encode increasing input sizes +- **THEN** the diagnosis reports the endpoint ratios and scaling exponent as observed evidence and identifies the leading repository hotspot only as an optimization candidate + +#### Scenario: Runner startup dominates +- **WHEN** the capsule contains a startup-dominated finding +- **THEN** the diagnosis does not name application code as the primary issue and asks for a longer or batched representative workload + +#### Scenario: Profiling evidence is incomplete +- **WHEN** the originating capsule has `no_confidence` or lacks the observations required for a diagnosis +- **THEN** the diagnosis preserves the limitations and recommends a bounded evidence-improvement experiment rather than an optimization + +#### Scenario: Agent applies an optimization +- **WHEN** the diagnosis identifies an actionable candidate +- **THEN** its unverified hypothesis states what metric should move and its verification recipe selects the identical adapter, target, workload name, and sample policy + +### Requirement: Runtime candidates receive bounded source context +CodeVetter SHALL inspect source only for bounded repository-owned locations +already selected by runtime evidence. Each source observation MUST record its +file and line window, redacted excerpt, matched pattern and exact pattern lines, +and containment limitations. Source patterns MUST remain distinct from inferred +optimization claims. + +#### Scenario: Scaling hotspot intersects sort then slice +- **WHEN** a growing deterministic scale curve points to a JavaScript or TypeScript source window that fully sorts mapped candidates before taking a bounded slice and per-input cost materially increases +- **THEN** CodeVetter records the source pattern as observed evidence and proposes bounded top-k selection or deferred result materialization only as a falsifiable hypothesis + +#### Scenario: Runtime-selected source escapes containment +- **WHEN** a source location is absolute, traverses outside the repository, resolves through an escaping symlink, exceeds the file bound, or uses an unsupported source type +- **THEN** CodeVetter omits the excerpt, records the limitation, and does not weaken the runtime diagnosis + +#### Scenario: Hot TypeScript method retains only the first split segment +- **WHEN** a measured growing CPU path reaches a TypeScript method that splits a string with limit one and reads only element zero +- **THEN** CodeVetter confines inspection to that method and proposes direct delimiter search and slicing only as a falsifiable hypothesis + +#### Scenario: Growing path repeatedly scans materialized object keys +- **WHEN** a superlinear measured path materializes object keys and repeatedly performs linear membership checks over that array +- **THEN** CodeVetter records the exact key-materialization and membership lines and proposes indexed membership only as a falsifiable hypothesis + +### Requirement: Optimization verification uses identical domain evidence +CodeVetter SHALL expose `verify-optimization` for a repository-contained +baseline performance capsule or diagnosis and a newly captured identical scope. +It MUST verify workload success and compatibility before comparing bounded +domain metrics, and MUST return `confirmed`, `rejected`, `inconclusive`, or +`no_confidence` with exact deltas and policy. + +#### Scenario: Catalogue optimization improves high-end scaling +- **WHEN** baseline and current capsules contain the same encoded input sizes and units, the largest-input time improves materially, no smaller input materially regresses, and both workloads pass +- **THEN** verification reports `confirmed` with per-input deltas and exponent movement + +#### Scenario: Go allocation optimization reduces allocations +- **WHEN** identical Go benchmarks pass and B/op or allocs/op improves materially without an unacceptable ns/op regression +- **THEN** verification reports `confirmed` with all three metric deltas + +#### Scenario: Workload identities differ +- **WHEN** schema, adapter, target, exact workload name, scale inputs, units, or Go benchmark identity differ +- **THEN** verification reports `no_confidence` and makes no optimization claim + +#### Scenario: Candidate does not improve its target metric +- **WHEN** compatible before/after evidence is stable within the recorded policy +- **THEN** verification reports `inconclusive`; when the target metric materially regresses it reports `rejected` + +### Requirement: Paired verification controls temporal host drift +CodeVetter SHALL support an opt-in paired verification operation when baseline +and candidate are available as independently runnable contained repositories. +It MUST alternate execution order, require exact workload and metric identity, +record the bounded schedule, and refuse confirmation if either side fails or +does not provide repeated comparable measurements. + +#### Scenario: Two runnable revisions expose the same benchmark +- **WHEN** baseline and candidate repositories contain the same exact benchmark and emit matching domain metrics +- **THEN** CodeVetter alternates their run order, compares their repeated median measurements, and labels the result as paired evidence + +#### Scenario: One paired workload fails +- **WHEN** any required baseline or candidate execution fails, times out, or does not execute the exact selected workload +- **THEN** paired verification reports `no_confidence` and cannot confirm the optimization + +### Requirement: Performance coverage is qualified on owned and real targets +CodeVetter SHALL maintain hermetic fixtures for wall-time distributions, V8 +hotspots, Go benchmark parsing, redaction, comparison outcomes, and incomplete +profiles. Real-project performance claims MUST identify the target revision, +machine context, workload, sample policy, and observed limitations. + +#### Scenario: App Health qualification runs +- **WHEN** the profiler is exercised against selected App Health Node and Go benchmark scopes +- **THEN** results report measured coverage and gaps without changing App Health tracked files or contacting production services + +#### Scenario: Consumer browser qualification runs +- **WHEN** a supported local browser connector traces an exact Significant Hobbies loopback journey +- **THEN** the qualification records vitals, network and main-thread observations, connector identity, run bounds, and limitations without contacting hosted services + +#### Scenario: Consumer algorithm scale qualification runs +- **WHEN** Anime List recommendation logic is exercised at fixed deterministic catalogue sizes +- **THEN** the qualification records per-size timing, result identity, scaling behavior, and whether application work is material relative to runner overhead + +#### Scenario: Open-source qualification runs +- **WHEN** CodeVetter profiles a bounded public Node or Go project with an existing local test or benchmark surface +- **THEN** the qualification records the immutable upstream revision, local-only workload, diagnosis, verification outcome, retained changes, and cleanup status + +### Requirement: Self-improvement qualification + +CodeVetter SHALL be able to profile a deterministic workload around one of its +own runtime-evidence operations without replacing runtime evidence with source +review alone. + +#### Scenario: CodeVetter profiles itself +- **WHEN** an agent captures an exact scale workload around a repository-owned product operation +- **THEN** CodeVetter reports observed scale and source attribution before any optimization claim is accepted + +#### Scenario: A self-improvement candidate is applied +- **WHEN** an agent changes the runtime-selected product path +- **THEN** the identical correctness and scale workload is rerun and the verifier distinguishes measured improvement from inference and unverified follow-up diff --git a/openspec/specs/runtime-qualification-planning/spec.md b/openspec/specs/runtime-qualification-planning/spec.md new file mode 100644 index 00000000..25395682 --- /dev/null +++ b/openspec/specs/runtime-qualification-planning/spec.md @@ -0,0 +1,69 @@ +# runtime-qualification-planning Specification + +## Purpose +Define how CodeVetter identifies trustworthy, bounded local performance workloads before an agent spends resources profiling or claims that an application bottleneck was measured. +## Requirements +### Requirement: Qualification is read-only and bounded +CodeVetter SHALL inspect only bounded repository metadata and source text when qualifying a repository. Qualification MUST NOT execute project code, install dependencies, start services, contact a network, modify the target repository, or treat a discovered runtime as proof that a representative workload exists. + +#### Scenario: Repository has a test dependency +- **WHEN** a package manifest declares Vitest but no suitable exact performance workload is found +- **THEN** qualification reports the supported runtime lane separately from workload readiness +- **AND** does not run Vitest or describe the repository as profile-ready + +#### Scenario: Repository contains nested tooling +- **WHEN** the only supported manifest or configuration belongs to a nested documentation, fixture, or tooling package +- **THEN** qualification records that evidence with its package scope +- **AND** does not attribute that runtime to the whole product without a representative candidate + +### Requirement: Candidates have exact identity and explainable rank +Each discovered candidate SHALL identify a supported adapter, repository-relative target, exact workload name when required by the adapter, package scope, bounded evidence, safety flags, and an explainable score. CodeVetter MUST rank explicit benchmarks and performance-named workloads ahead of generic tests and MUST NOT invent exact names that were not found in source evidence. + +#### Scenario: Go benchmark is discovered +- **WHEN** a bounded Go test file declares `BenchmarkParseRows` +- **THEN** the candidate uses the Go benchmark adapter, the containing package target, and the exact benchmark name +- **AND** cites the declaration as direct evidence + +#### Scenario: Generic unit tests are the only candidates +- **WHEN** supported unit tests exist but none carries benchmark, performance, scale, load, or timing evidence +- **THEN** CodeVetter may list a bounded low-ranked candidate set +- **AND** requires agent selection rather than choosing one as representative + +### Requirement: Readiness remains conservative +Qualification SHALL return exactly one repository status from `ready`, `needs_selection`, `no_representative_workload`, `unsupported`, or `inaccessible`. A repository is `ready` only when one supported exact candidate crosses the recorded qualification threshold without unsafe external-operation signals. Ambiguity, absent exact identity, browser-only work, and likely network or production dependencies MUST lower readiness rather than be silently accepted. + +#### Scenario: Explicit local performance test is unambiguous +- **WHEN** one supported exact workload has strong benchmark evidence and no external-operation warning +- **THEN** qualification returns `ready` with that candidate as the recommended next profiling scope + +#### Scenario: Performance test appears network-dependent +- **WHEN** candidate source includes bounded evidence of remote URLs, database clients, browser automation, or environment-gated services +- **THEN** qualification includes safety flags and does not return `ready` + +#### Scenario: No supported runtime exists +- **WHEN** bounded evidence establishes no Node, Vitest, or Go performance adapter +- **THEN** qualification returns `unsupported` with limitations + +### Requirement: Portfolio qualification is explicit and sequential +CodeVetter SHALL accept a versioned manifest containing bounded repository identifiers and paths, qualify entries sequentially, and emit one versioned aggregate report with one result per declared repository. The report MUST preserve input order, bound repository and candidate counts, omit absolute repository paths, and continue after an inaccessible or unsupported entry. + +#### Scenario: Mixed portfolio is qualified +- **WHEN** a manifest contains ready, unsupported, and missing repositories +- **THEN** the aggregate report includes all declared identifiers in order with their individual statuses +- **AND** summarizes status counts without failing the whole portfolio + +#### Scenario: Portfolio exceeds its bound +- **WHEN** a manifest declares more repositories than the contract permits +- **THEN** CodeVetter rejects it before inspecting any repository + +### Requirement: Agents receive the next safe operation +For a ready repository, qualification SHALL return a closed profiling recipe compatible with the existing performance operation. For every other status it SHALL return a deterministic next action explaining what evidence or operator choice is missing. CLI and MCP surfaces MUST preserve the same versioned result and uncertainty. + +#### Scenario: Ready candidate is returned through MCP +- **WHEN** an agent calls the local qualification tool for a repository with an unambiguous exact workload +- **THEN** the result includes the adapter, target, exact name, bounded sample policy, evidence, and limitations needed for a subsequent explicit profile call + +#### Scenario: Workload is not representative +- **WHEN** the planner finds only startup-dominated or generic test scopes +- **THEN** the next action asks for a batched or representative workload +- **AND** no optimization target is inferred diff --git a/openspec/specs/scaled-runtime-challenges/spec.md b/openspec/specs/scaled-runtime-challenges/spec.md new file mode 100644 index 00000000..cecc74f5 --- /dev/null +++ b/openspec/specs/scaled-runtime-challenges/spec.md @@ -0,0 +1,62 @@ +# scaled-runtime-challenges Specification + +## Purpose +TBD - created by archiving change add-scaled-parsing-challenge. Update Purpose after archive. +## Requirements +### Requirement: Deterministic parsing challenge + +CodeVetter SHALL provide a repository-owned parsing challenge with fixed input +identity, exact row counts, and no external dependencies. + +#### Scenario: Challenge runs locally +- **WHEN** an agent executes the temperature aggregation challenge +- **THEN** every input row is generated deterministically in memory and no network, database, cloud service, or persistent generated dataset is used + +#### Scenario: Challenge spans representative scales +- **WHEN** the scale workload completes +- **THEN** it emits at least three positive `ms/op` measurements whose largest row count is at least 40 times the smallest + +### Requirement: Correctness before performance + +Every measured parser result MUST match count, minimum, maximum, sum, and a +stable complete aggregate digest. + +#### Scenario: Candidate parser drops or changes data +- **WHEN** any aggregate differs from the independently derived expected result +- **THEN** the test fails before emitting a successful benchmark metric line and CodeVetter cannot confirm the optimization + +### Requirement: Official task compatibility + +The artifact MUST preserve the official 1BRC row and output semantics while +identifying its Node and bounded-execution differences. + +#### Scenario: Valid challenge rows are aggregated +- **WHEN** rows contain variable UTF-8 station names and signed one-decimal temperatures within the official bounds +- **THEN** the artifact emits stations alphabetically with minimum, round-toward-positive mean, and maximum values to one decimal place + +#### Scenario: An agent inspects artifact provenance +- **WHEN** the benchmark is used as CodeVetter evidence +- **THEN** the repository identifies the upstream challenge and license and does not claim an official submission or unexecuted billion-row result + +### Requirement: Evidence-led iteration + +The initial parser SHALL be captured before optimization, and each candidate +change SHALL be evaluated against the identical adapter, target, exact test +name, input sizes, units, and correctness contract. + +#### Scenario: Runtime evidence selects a parser candidate +- **WHEN** the baseline produces a scale curve and repository-owned CPU evidence +- **THEN** CodeVetter reports observed measurements separately from its inferred candidate and supplies an identical-scope verification action + +#### Scenario: Candidate is faster at representative scale +- **WHEN** the same workload reruns after one implementation change +- **THEN** CodeVetter reports the measured per-size deltas and confirms only when the largest-input improvement crosses policy without a material smaller-input regression + +### Requirement: Bounded claims + +The challenge MUST distinguish measured local evidence from extrapolation to +billion-row datasets. + +#### Scenario: Local benchmark completes +- **WHEN** CodeVetter records bounded row counts and durations +- **THEN** the qualification identifies those exact counts and MUST NOT claim an unexecuted nine-billion-row completion time, memory bound, or production throughput diff --git a/openspec/specs/verification-receipt-comparison/spec.md b/openspec/specs/verification-receipt-comparison/spec.md new file mode 100644 index 00000000..9a74d3d7 --- /dev/null +++ b/openspec/specs/verification-receipt-comparison/spec.md @@ -0,0 +1,130 @@ +# verification-receipt-comparison Specification + +## Purpose +Define deterministic ingestion and comparison of project-owned verification +receipts so agents can evaluate correctness, performance, and changed-test +blast radius without replacing the project's own test runner. +## Requirements +### Requirement: Receipt ingestion is closed, bounded, and immutable +CodeVetter SHALL accept only a versioned closed receipt containing repository +and revision identity, runner profile, environment identity, selection, +attempts, terminal outcomes, resource measurements, safety observations, and +explicit budgets. It MUST reject unknown fields, unsupported versions, +credential-shaped content, unsafe paths, inconsistent totals, duplicate +identities, and evidence beyond recorded bounds. Ingestion MUST leave the +source receipt unchanged and bind the normalized bundle to its exact SHA-256 +identity. + +#### Scenario: Valid receipt is ingested twice +- **WHEN** the same valid receipt bytes are ingested with the same repository scope +- **THEN** CodeVetter emits byte-equivalent normalized bundles with the same source receipt identity and verdict + +#### Scenario: Receipt contains undeclared data +- **WHEN** a receipt contains an unknown field, credential-shaped value, absolute path, or repository escape +- **THEN** CodeVetter rejects it before emitting a qualified bundle + +### Requirement: Correctness and performance verdicts remain independent +Every normalized bundle SHALL report correctness, performance, safety, and +overall qualification separately. A passing test outcome MUST NOT hide a +performance-budget failure, and a performance improvement MUST NOT hide a test +failure, incomplete inventory, live-network escape, fixed wait, or missing +required evidence. + +#### Scenario: Tests pass but resource budget fails +- **WHEN** all terminal tests pass and peak RSS exceeds the declared maximum +- **THEN** correctness is `passed`, performance is `failed`, and the overall bundle is `failed` + +#### Scenario: Required evidence is unavailable +- **WHEN** a declared budget cannot be evaluated because its measurement is missing +- **THEN** the corresponding verdict and overall qualification are `no_confidence` + +#### Scenario: No performance or safety budget is configured +- **WHEN** every metric in a verdict component is unconfigured +- **THEN** that component is `no_confidence` rather than a vacuous pass + +### Requirement: Receipt comparison qualifies evidence compatibility +CodeVetter SHALL compare only receipts with compatible schema, repository, +runner profile, environment, and inventory identities. It MUST label evidence +as `same_commit`, `cross_commit`, or `incompatible`; cross-commit evidence MAY +report observed deltas but MUST NOT be presented as a controlled same-commit +speedup. Comparisons SHALL preserve raw values, absolute deltas, percentage +deltas, sample counts, and declared budget policies. + +#### Scenario: Same-commit receipts are compared +- **WHEN** baseline and current receipts share the exact commit and all compatibility identities +- **THEN** CodeVetter emits qualified performance and failure-set deltas labeled `same_commit` + +#### Scenario: Compatible commits differ +- **WHEN** compatible baseline and current receipts differ only by revision +- **THEN** CodeVetter labels the comparison `cross_commit`, reports directional deltas, and records the limitation + +#### Scenario: Runner profiles differ +- **WHEN** baseline and current receipts use different runner, environment, or inventory identities +- **THEN** CodeVetter emits `incompatible` and makes no regression or improvement claim + +### Requirement: Failure and inventory changes use a deterministic taxonomy +Comparison SHALL classify new, recovered, stable, and transiently recovered +failures by stable signature. It SHALL separately classify incomplete +inventory, inventory drift, selector widening, selector narrowing, fixed waits, +live-network escapes, retries, timeouts, and operational failures. Operational +failures MUST remain outside the successful-test denominator. + +#### Scenario: Failed attempt passes on bounded recheck +- **WHEN** an executed test fails and a later declared recheck passes with the same test identity +- **THEN** CodeVetter classifies it as `transient_recovery` rather than a stable pass or stable failure + +#### Scenario: Selected inventory silently narrows +- **WHEN** the current receipt omits tests present in a compatible baseline without declaring an allowed selector change +- **THEN** CodeVetter reports `unsafe_selector_narrowing` independently of executed-test outcomes + +### Requirement: Blast-radius evidence is explicit and bounded +Every qualified bundle SHALL emit a machine-readable graph derived only from +declared changed files, selection reasons, executed tests, and failure +signatures. Edges SHALL identify their evidence kind and MUST NOT infer source +dependencies that the receipt did not declare. Unknown or truncated selection +relationships SHALL remain limitations. + +#### Scenario: Changed file selects a failing test +- **WHEN** a receipt declares that a changed file selected a test which emitted a failure signature +- **THEN** the graph connects changed file to selected test to failure signature with observed evidence labels + +#### Scenario: Selection reason is absent +- **WHEN** an executed test has no declared changed-file selection relationship +- **THEN** the test remains in the graph without a fabricated changed-file edge and the bundle records the missing explanation + +### Requirement: CLI and MCP expose the same pure operations +The machine CLI and repository-scoped read-only MCP process SHALL call the same +ingestion and comparison implementation and return the same normalized bundle. +Inputs MUST be repository-relative bounded receipt paths. MCP calls MUST NOT +execute tests, accept shell commands, modify the target repository, switch +repository scope, or write bundles implicitly. + +#### Scenario: CLI and MCP ingest the same receipt +- **WHEN** both transports ingest the same repository-relative receipt +- **THEN** their normalized bundle documents are semantically identical + +#### Scenario: MCP input escapes its repository +- **WHEN** an MCP call supplies an absolute path, traversal, or escaping symlink +- **THEN** the process rejects the call before reading the receipt + +### Requirement: Real-project claims preserve provenance and limitations +Qualification SHALL include hermetic receipts and at least one real +project-runner receipt with exact revision, environment, inventory, and +measurement provenance. Any cross-commit, single-sample, incomplete resource, +or partial process-tree evidence MUST remain explicit and MUST bound published +claims. + +#### Scenario: Existing runner receipt has partial RSS evidence +- **WHEN** qualification ingests a receipt whose RSS metric excludes part of the process tree +- **THEN** the bundle retains the measurement and limitation without claiming total process-tree memory + +### Requirement: Bounded producer-native receipts preserve authority +CodeVetter MAY adapt an explicitly recognized producer-native receipt into the +canonical contract. The adapter MUST bind the canonical bundle to the raw +source SHA-256, reject unsupported formats, omit producer-only sensitive or +machine-local fields, and preserve missing inventory, measurements, budgets, +or execution evidence as `no_confidence`. + +#### Scenario: Producer fails before test execution +- **WHEN** a recognized producer-native receipt records a setup failure but no test inventory or attempts +- **THEN** CodeVetter emits an operational failure with missing evidence and makes no correctness, performance, safety, or inventory pass claim diff --git a/package.json b/package.json index 053dc345..697ce84f 100644 --- a/package.json +++ b/package.json @@ -25,6 +25,7 @@ "runtime:detect": "node scripts/runtime-failure-capsule/cli.mjs detect", "runtime:qualify": "node scripts/runtime-failure-capsule/cli.mjs qualify", "runtime:qualify-portfolio": "node scripts/runtime-failure-capsule/cli.mjs qualify-portfolio", + "runtime:plan-performance": "node scripts/runtime-failure-capsule/cli.mjs plan-performance", "runtime:plan-flow-campaign": "node scripts/runtime-failure-capsule/cli.mjs plan-flow-campaign", "runtime:supervise-performance": "node scripts/runtime-failure-capsule/cli.mjs supervise-performance", "runtime:inspect-performance-run": "node scripts/runtime-failure-capsule/cli.mjs inspect-performance-run", diff --git a/scripts/runtime-failure-capsule/campaign.mjs b/scripts/runtime-failure-capsule/campaign.mjs index e7fa0324..01f45bc5 100644 --- a/scripts/runtime-failure-capsule/campaign.mjs +++ b/scripts/runtime-failure-capsule/campaign.mjs @@ -15,6 +15,10 @@ import { } from './campaign-contracts.mjs'; import { assessChangeCost, inspectChangeCost } from './change-cost.mjs'; import { parseVitestSelection } from './capsule.mjs'; +import { + createPerformanceExecutionReceipt, + planPerformanceExecution, +} from './execution-governance.mjs'; import { verifyOptimizationCapsules } from './optimization-verification.mjs'; import { verifyPairedRepositories } from './paired-verification.mjs'; import { profileRepository } from './performance.mjs'; @@ -30,6 +34,8 @@ const ENGINE_FILES = [ 'capsule.mjs', 'change-cost.mjs', 'contracts.mjs', + 'execution-governance.mjs', + 'node-egress-preload.mjs', 'optimization-verification.mjs', 'paired-verification.mjs', 'performance-diagnosis.mjs', @@ -109,11 +115,22 @@ async function baselineCampaign(root, input, dependencies) { if (priorStatus.status === 'stopped') throw new Error(`campaign stopped: ${priorStatus.stop_reason}`); + const admission = await admitCampaignScopes(root, campaign.manifest, 'screening'); const repository = await inspectRepositoryState(root, campaign.manifest); assertAllowedChanges(campaign.manifest, repository.changed_files); - const correctness = await runCorrectness(root, campaign.manifest.correctness, dependencies); + const correctness = await runCorrectness( + root, + campaign.manifest.correctness, + dependencies, + 'candidate', + admission.correctness + ); let performance = null; - let evidence = { correctness: correctness.results, performance_capsule: null }; + let evidence = { + correctness: correctness.results, + correctness_execution_governance: correctness.executionGovernance, + performance_capsule: null, + }; let decision; const limitations = [...correctness.limitations]; @@ -129,7 +146,11 @@ async function baselineCampaign(root, input, dependencies) { samples: scope.screening.samples, warmups: scope.screening.warmups, }); - evidence = { correctness: correctness.results, performance_capsule: capsule }; + evidence = { + correctness: correctness.results, + correctness_execution_governance: correctness.executionGovernance, + performance_capsule: capsule, + }; if (capsule.verdict.status === 'no_confidence') { limitations.push(...capsule.limitations); decision = { @@ -191,6 +212,7 @@ async function screenCampaign(root, input, dependencies) { if (status.next_action?.kind === 'promote_candidate') { throw new Error('the latest promising candidate must be promoted or discarded first'); } + const admission = await admitCampaignScopes(root, campaign.manifest, 'screening'); const hypothesis = sanitizeHypothesis(input?.hypothesis, root); const repository = await inspectRepositoryState(root, campaign.manifest); assertAllowedChanges(campaign.manifest, repository.changed_files); @@ -204,11 +226,18 @@ async function screenCampaign(root, input, dependencies) { const attempt = status.experiments + 1; const correctness = changeCost.violations.length === 0 - ? await runCorrectness(root, campaign.manifest.correctness, dependencies) - : { status: 'not_run', results: [], limitations: [] }; + ? await runCorrectness( + root, + campaign.manifest.correctness, + dependencies, + 'candidate', + admission.correctness + ) + : { status: 'not_run', results: [], limitations: [], executionGovernance: [] }; const limitations = [...correctness.limitations]; let evidence = { correctness: correctness.results, + correctness_execution_governance: correctness.executionGovernance, performance_capsule: null, verification: null, change_cost: changeCost, @@ -239,6 +268,7 @@ async function screenCampaign(root, input, dependencies) { const verification = dependencies.verifyOptimizationCapsules(baseline, current); evidence = { correctness: correctness.results, + correctness_execution_governance: correctness.executionGovernance, performance_capsule: current, verification, change_cost: changeCost, @@ -294,6 +324,10 @@ async function promoteCampaign(root, input, dependencies) { } const incumbentRoot = await realpath(resolve(input.incumbent_repository)); if (incumbentRoot === root) throw new Error('incumbent and candidate repositories must differ'); + const [candidateAdmission, incumbentAdmission] = await Promise.all([ + admitCampaignScopes(root, campaign.manifest, 'promotion'), + admitCampaignScopes(incumbentRoot, campaign.manifest, 'promotion'), + ]); const [candidateRepository, incumbentRepository] = await Promise.all([ inspectRepositoryState(root, campaign.manifest), inspectRepositoryState(incumbentRoot, campaign.manifest), @@ -308,12 +342,31 @@ async function promoteCampaign(root, input, dependencies) { } const [incumbentCorrectness, candidateCorrectness] = await Promise.all([ - runCorrectness(incumbentRoot, campaign.manifest.correctness, dependencies, 'incumbent'), - runCorrectness(root, campaign.manifest.correctness, dependencies, 'candidate'), + runCorrectness( + incumbentRoot, + campaign.manifest.correctness, + dependencies, + 'incumbent', + incumbentAdmission.correctness + ), + runCorrectness( + root, + campaign.manifest.correctness, + dependencies, + 'candidate', + candidateAdmission.correctness + ), ]); const correctness = [...incumbentCorrectness.results, ...candidateCorrectness.results]; const limitations = [...incumbentCorrectness.limitations, ...candidateCorrectness.limitations]; - let evidence = { correctness, verification: null }; + let evidence = { + correctness, + correctness_execution_governance: [ + ...incumbentCorrectness.executionGovernance, + ...candidateCorrectness.executionGovernance, + ], + verification: null, + }; let decision; if (incumbentCorrectness.status !== 'passed') { @@ -336,7 +389,14 @@ async function promoteCampaign(root, input, dependencies) { samples: scope.promotion.samples, warmups: scope.promotion.warmups, }); - evidence = { correctness, verification }; + evidence = { + correctness, + correctness_execution_governance: [ + ...incumbentCorrectness.executionGovernance, + ...candidateCorrectness.executionGovernance, + ], + verification, + }; limitations.push(...verification.limitations); decision = promotionDecision(verification); } catch (error) { @@ -477,19 +537,30 @@ export function deriveCampaignStatus(manifest, records, now = new Date()) { }; } -async function runCorrectness(root, scopes, dependencies, role = 'candidate') { +async function runCorrectness(root, scopes, dependencies, role = 'candidate', executionPlans = []) { const results = []; const limitations = []; - for (const scope of scopes) { + const executionGovernance = []; + for (const [index, scope] of scopes.entries()) { const execution = await dependencies.runClosedAdapter({ repositoryRoot: root, adapter: scope.adapter, target: scope.target, name: scope.name, timeoutMs: scope.timeout_ms, + executionPlan: executionPlans[index] ?? null, }); const normalized = normalizeCorrectnessExecution(scope, execution, role); results.push(normalized); + if (executionPlans[index]) { + executionGovernance.push({ + role, + plan: executionPlans[index], + receipt: createPerformanceExecutionReceipt(executionPlans[index], [ + { phase: 'correctness', index, execution }, + ]), + }); + } if (normalized.limitation) limitations.push(normalized.limitation); } const statuses = new Set(results.map((result) => result.status)); @@ -500,7 +571,45 @@ async function runCorrectness(root, scopes, dependencies, role = 'candidate') { : statuses.has('no_confidence') ? 'no_confidence' : 'passed'; - return { status, results, limitations }; + return { status, results, limitations, executionGovernance }; +} + +async function admitCampaignScopes(root, manifest, phase) { + const correctness = await Promise.all( + manifest.correctness.map((scope) => + planPerformanceExecution({ + repositoryRoot: root, + adapter: scope.adapter, + target: scope.target, + name: scope.name, + timeoutMs: scope.timeout_ms, + processCount: 1, + }) + ) + ); + const samplePolicy = manifest.performance[phase]; + const profileRuns = manifest.performance.adapter === 'go-bench' ? 1 : 2; + const metricRuns = ['node-test', 'node-script', 'vitest'].includes(manifest.performance.adapter) + ? samplePolicy.samples + : 0; + const performance = await planPerformanceExecution({ + repositoryRoot: root, + adapter: manifest.performance.adapter, + target: manifest.performance.target, + name: manifest.performance.name, + timeoutMs: manifest.performance.timeout_ms, + processCount: samplePolicy.samples + samplePolicy.warmups + metricRuns + profileRuns, + }); + const blocked = [...correctness, performance].filter( + (plan) => plan.decision.status === 'blocked' + ); + if (blocked.length > 0) { + const reasons = blocked.flatMap((plan) => plan.decision.blockers); + throw new Error( + `campaign execution blocked before project code: ${[...new Set(reasons)].join(' ')}` + ); + } + return { correctness, performance }; } export function normalizeCorrectnessExecution(scope, execution, role = 'candidate') { diff --git a/scripts/runtime-failure-capsule/cli.mjs b/scripts/runtime-failure-capsule/cli.mjs index b0f0dce0..bedf3fd9 100644 --- a/scripts/runtime-failure-capsule/cli.mjs +++ b/scripts/runtime-failure-capsule/cli.mjs @@ -17,9 +17,14 @@ import { detectRuntimeLanes } from './detect.mjs'; import { captureFlowRepository } from './flow.mjs'; import { planFlowOptimizationCampaign } from './flow-campaign-planner.mjs'; import { diagnosePerformanceRepository } from './performance-diagnosis.mjs'; +import { planPerformanceExecution } from './execution-governance.mjs'; import { verifyOptimizationCapsules } from './optimization-verification.mjs'; import { verifyPairedRepositories } from './paired-verification.mjs'; -import { loadPerformanceCapsule, profileRepository } from './performance.mjs'; +import { + loadPerformanceCapsule, + plannedProfileProcessCount, + profileRepository, +} from './performance.mjs'; import { qualifyPortfolioManifest, qualifyRepository } from './qualification.mjs'; import { redactText } from './redact.mjs'; import { runClosedAdapter } from './runner.mjs'; @@ -43,6 +48,31 @@ export async function main(argv = process.argv.slice(2)) { writeJson(await qualifyPortfolioManifest(required(options, 'manifest'))); return 0; } + if (operation === 'plan-performance') { + const adapter = assertProfileAdapter(required(options, 'adapter')); + const samples = boundedCount(options.samples, { + name: 'samples', + defaultValue: LIMITS.defaultSamples, + minimum: LIMITS.minimumSamples, + maximum: LIMITS.maximumSamples, + }); + const warmups = boundedCount(options.warmups, { + name: 'warmups', + defaultValue: LIMITS.defaultWarmups, + maximum: LIMITS.maximumWarmups, + }); + const plan = await planPerformanceExecution({ + repositoryRoot, + adapter, + target: required(options, 'target'), + name: options.name, + timeoutMs: boundedTimeout(options['timeout-ms']), + processCount: plannedProfileProcessCount({ adapter, samples, warmups }), + approvalIdentity: options['approval-id'], + }); + writeJson(plan); + return plan.decision.status === 'admitted' ? 0 : 2; + } if (operation === 'plan-flow-campaign') { const result = await planFlowOptimizationCampaign({ repositoryRoot, @@ -256,13 +286,14 @@ function parseArguments(argv) { 'capture-flow', 'qualify', 'qualify-portfolio', + 'plan-performance', 'supervise-performance', 'inspect-performance-run', 'plan-flow-campaign', ].includes(operation) ) { throw new Error( - 'usage: cli.mjs [--repo PATH] [operation options] [--json]' + 'usage: cli.mjs [--repo PATH] [operation options] [--json]' ); } const normalizedRest = rest[0] === '--' ? rest.slice(1) : rest; @@ -294,55 +325,67 @@ function parseArguments(argv) { ? ['repo', 'json'] : operation === 'qualify-portfolio' ? ['manifest', 'json'] - : operation === 'plan-flow-campaign' - ? ['repo', 'priority-manifest', 'max-flows', 'timeout-ms', 'samples', 'warmups', 'json'] - : operation === 'inspect-performance-run' - ? ['repo', 'run-id', 'json'] - : operation === 'supervise-performance' - ? [ - 'repo', - 'run-id', - 'adapter', - 'target', - 'name', - 'timeout-ms', - 'samples', - 'warmups', - 'json', - ] - : operation === 'run' - ? ['repo', 'adapter', 'target', 'name', 'diff', 'timeout-ms', 'json'] - : operation === 'import' - ? ['repo', 'kind', 'receipt', 'diff', 'json'] - : operation === 'verify-paired-optimization' - ? [ - 'repo', - 'baseline-repo', - 'adapter', - 'target', - 'name', - 'timeout-ms', - 'samples', - 'warmups', - 'vite-build-dir', - 'vite-entry', - 'json', - ] - : [ - 'repo', - 'adapter', - 'target', - 'name', - 'timeout-ms', - 'samples', - 'warmups', - 'baseline', - 'regression-percent', - 'regression-ms', - 'vite-build-dir', - 'vite-entry', - 'json', - ] + : operation === 'plan-performance' + ? [ + 'repo', + 'adapter', + 'target', + 'name', + 'timeout-ms', + 'samples', + 'warmups', + 'approval-id', + 'json', + ] + : operation === 'plan-flow-campaign' + ? ['repo', 'priority-manifest', 'max-flows', 'timeout-ms', 'samples', 'warmups', 'json'] + : operation === 'inspect-performance-run' + ? ['repo', 'run-id', 'json'] + : operation === 'supervise-performance' + ? [ + 'repo', + 'run-id', + 'adapter', + 'target', + 'name', + 'timeout-ms', + 'samples', + 'warmups', + 'json', + ] + : operation === 'run' + ? ['repo', 'adapter', 'target', 'name', 'diff', 'timeout-ms', 'json'] + : operation === 'import' + ? ['repo', 'kind', 'receipt', 'diff', 'json'] + : operation === 'verify-paired-optimization' + ? [ + 'repo', + 'baseline-repo', + 'adapter', + 'target', + 'name', + 'timeout-ms', + 'samples', + 'warmups', + 'vite-build-dir', + 'vite-entry', + 'json', + ] + : [ + 'repo', + 'adapter', + 'target', + 'name', + 'timeout-ms', + 'samples', + 'warmups', + 'baseline', + 'regression-percent', + 'regression-ms', + 'vite-build-dir', + 'vite-entry', + 'json', + ] ); const unknown = Object.keys(options).find((key) => !allowed.has(key)); if (unknown) throw new Error(`unknown option for ${operation}: --${unknown}`); diff --git a/scripts/runtime-failure-capsule/contracts.mjs b/scripts/runtime-failure-capsule/contracts.mjs index cc8cb141..ca5c0f5d 100644 --- a/scripts/runtime-failure-capsule/contracts.mjs +++ b/scripts/runtime-failure-capsule/contracts.mjs @@ -177,6 +177,21 @@ export function validatePerformanceCapsule(capsule) { if (!Array.isArray(capsule?.findings)) errors.push('findings must be an array'); if (!Array.isArray(capsule?.unverified)) errors.push('unverified must be an array'); if (!Array.isArray(capsule?.limitations)) errors.push('limitations must be an array'); + if (capsule?.execution_governance !== null && capsule?.execution_governance !== undefined) { + if (capsule.execution_governance?.plan?.schema_version !== 'performance-execution-plan/v1') { + errors.push('invalid execution_governance.plan'); + } + if ( + capsule.execution_governance?.receipt?.schema_version !== 'performance-execution-receipt/v1' + ) { + errors.push('invalid execution_governance.receipt'); + } + if ( + capsule.execution_governance?.plan?.plan_id !== capsule.execution_governance?.receipt?.plan_id + ) { + errors.push('execution_governance identity mismatch'); + } + } return errors; } diff --git a/scripts/runtime-failure-capsule/execution-governance.mjs b/scripts/runtime-failure-capsule/execution-governance.mjs new file mode 100644 index 00000000..d83a2166 --- /dev/null +++ b/scripts/runtime-failure-capsule/execution-governance.mjs @@ -0,0 +1,518 @@ +import { createHash } from 'node:crypto'; +import { readFile, realpath, stat } from 'node:fs/promises'; +import { isAbsolute, resolve } from 'node:path'; + +import { LIMITS, PROFILE_ADAPTERS, repositoryRelative } from './contracts.mjs'; +import { inspectGitDiff } from './git-diff.mjs'; + +export const PERFORMANCE_EXECUTION_PLAN_SCHEMA_VERSION = 'performance-execution-plan/v1'; +export const PERFORMANCE_EXECUTION_RECEIPT_SCHEMA_VERSION = 'performance-execution-receipt/v1'; + +const MAX_SOURCE_BYTES = 512 * 1024; +const GOVERNED_ADAPTERS = Object.freeze([...PROFILE_ADAPTERS, 'go-test']); +const NODE_ADAPTERS = new Set(['node-test', 'node-script', 'vitest']); +const LOOPBACK_URL = + /^(?:https?|wss?):\/\/(?:localhost|127(?:\.\d{1,3}){3}|\[::1\])(?::(?:\d+|\$\{[^}]+\}))?(?:[/?#]|$)/i; +const URL_LITERAL = /(?:https?|wss?):\/\/[^\s'"`<>)]+/gi; +const DYNAMIC_NETWORK_CALL = + /\b(?:fetch|axios(?:\.[A-Za-z]+)?|got|ky|https?\.request|new\s+WebSocket)\s*\(\s*(?!['"`]https?:\/\/(?:localhost|127\.|\[::1\]))/i; +const REMOTE_SERVICE = + /\b(?:postgres(?:ql)?|mysql|mongodb|redis|supabase|firebase|dynamodb|cloudflare|workers\s*ai|openai|anthropic|openrouter|stripe)\b/i; +const CLOUDFLARE_SERVICE = + /\b(?:DurableObject|browser\s+rendering|Workers\s+AI|env\.(?:D1|R2|KV)|(?:D1|R2|KV)Database)\b/i; +const SUBPROCESS_NETWORK_ESCAPE = + /(?:from\s+['"]node:child_process['"]|require\s*\(\s*['"](?:node:)?child_process['"]\s*\)|\b(?:spawn|exec|execFile)\s*\()/; +const FORBIDDEN_WORKLOAD = + /\b(?:load[-_ ]?test|soak|stress[-_ ]?test|production[-_ ]?(?:profile|load|test))\b/i; + +export async function planPerformanceExecution({ + repositoryRoot, + adapter, + target, + name = null, + timeoutMs, + processCount = 1, + approvalIdentity = null, +}) { + const root = await realpath(resolve(repositoryRoot)); + if (!GOVERNED_ADAPTERS.includes(adapter)) throw new Error(`unsupported adapter: ${adapter}`); + const safeTarget = await inspectTarget(root, target); + const git = await inspectGitDiff(root); + const signals = inspectSafetySignals(safeTarget.source, { adapter, target, name }); + const enforcement = enforcementFor(adapter); + const blockers = [...signals.blockers]; + if (enforcement.kind === 'unavailable') blockers.push(enforcement.reason); + if (adapter === 'playwright' && signals.loopback_urls.length === 0) { + blockers.push('Playwright zero-egress admission requires an explicit loopback URL.'); + } + const maxDurationMs = boundedTotalDuration(timeoutMs, processCount); + const decision = blockers.length === 0 ? 'admitted' : 'blocked'; + const maximumCostMicrousd = decision === 'admitted' ? 0 : signals.unknown_cost ? null : 0; + const payload = { + schema_version: PERFORMANCE_EXECUTION_PLAN_SCHEMA_VERSION, + subject: { + repository_revision: git.repository_revision, + diff_identity: git.diff_identity, + dirty: git.dirty, + target_sha256: sha256(safeTarget.source), + }, + scope: { adapter, target: safeTarget.relative, name }, + mode: 'local_zero_egress', + limits: { + max_wall_time_ms: maxDurationMs, + max_processes: processCount, + max_concurrency: 1, + max_retries: 0, + max_external_requests: 0, + max_cost_microusd: maximumCostMicrousd, + }, + external_services: signals.external_services, + approval_identity: normalizeApprovalIdentity(approvalIdentity), + enforcement, + decision: { + status: decision, + reason: + decision === 'admitted' + ? 'The exact workload is admitted for bounded local zero-egress execution.' + : 'The workload is blocked before project code executes.', + blockers: [...new Set(blockers)].sort(), + }, + limitations: signals.limitations, + }; + const plan = { ...payload, plan_id: sha256(stableStringify(payload)) }; + assertPerformanceExecutionPlan(plan); + return plan; +} + +export function createPerformanceExecutionReceipt(plan, executions = []) { + assertPerformanceExecutionPlan(plan); + const markers = executions.flatMap((entry) => + blockedEgressMarkers(entry.execution?.stderr ?? '') + ); + const ran = executions.length > 0; + const completed = executions.every( + (entry) => entry.execution?.status === 'exited' && entry.execution?.exitCode === 0 + ); + const status = + plan.decision.status === 'blocked' + ? 'blocked' + : markers.length > 0 + ? 'policy_violation' + : completed + ? 'completed' + : ran + ? 'failed' + : 'admitted'; + const receipt = { + schema_version: PERFORMANCE_EXECUTION_RECEIPT_SCHEMA_VERSION, + plan_id: plan.plan_id, + decision: plan.decision.status, + status, + planned: plan.limits, + observed: { + wall_time_ms: executions.reduce( + (total, entry) => total + Math.max(0, entry.execution?.durationMs ?? 0), + 0 + ), + processes: executions.length, + max_concurrency: executions.length > 0 ? 1 : 0, + retries: 0, + successful_external_requests: 0, + blocked_external_attempts: markers.length, + external_services: [], + cost_microusd: plan.decision.status === 'admitted' ? 0 : null, + }, + enforcement: plan.enforcement, + terminal_reason: + plan.decision.status === 'blocked' + ? plan.decision.blockers.join(' ') + : markers.length > 0 + ? 'The zero-egress boundary blocked a remote network attempt.' + : completed + ? 'The admitted local workload completed within the execution policy.' + : ran + ? 'The admitted local workload did not complete successfully.' + : 'The workload is admitted but has not executed.', + limitations: [...plan.limitations], + }; + assertPerformanceExecutionReceipt(receipt); + return receipt; +} + +export async function assertPerformanceExecutionPlanCurrent({ + plan, + repositoryRoot, + adapter, + target, + name = null, +}) { + assertPerformanceExecutionPlan(plan); + const root = await realpath(resolve(repositoryRoot)); + const safeTarget = await inspectTarget(root, target); + const git = await inspectGitDiff(root); + if ( + plan.scope.adapter !== adapter || + plan.scope.target !== safeTarget.relative || + plan.scope.name !== name || + plan.subject.repository_revision !== git.repository_revision || + plan.subject.diff_identity !== git.diff_identity || + plan.subject.dirty !== git.dirty || + plan.subject.target_sha256 !== sha256(safeTarget.source) + ) { + throw new Error('performance execution plan identity is stale'); + } + return plan; +} + +export function assertPerformanceExecutionPlan(value) { + const errors = validatePerformanceExecutionPlan(value); + if (errors.length > 0) + throw new Error(`invalid performance execution plan: ${errors.join('; ')}`); + return value; +} + +export function validatePerformanceExecutionPlan(value) { + const errors = []; + if (!plainObject(value)) return ['plan must be an object']; + closed( + value, + [ + 'schema_version', + 'plan_id', + 'subject', + 'scope', + 'mode', + 'limits', + 'external_services', + 'approval_identity', + 'enforcement', + 'decision', + 'limitations', + ], + 'plan', + errors + ); + if (value.schema_version !== PERFORMANCE_EXECUTION_PLAN_SCHEMA_VERSION) + errors.push('invalid schema_version'); + if (!/^[0-9a-f]{64}$/.test(value.plan_id ?? '')) errors.push('plan_id is invalid'); + closed( + value.subject, + ['repository_revision', 'diff_identity', 'dirty', 'target_sha256'], + 'subject', + errors + ); + if ( + !plainObject(value.subject) || + !/^[0-9a-f]{40,64}$/.test(value.subject.repository_revision ?? '') || + typeof value.subject.diff_identity !== 'string' || + value.subject.diff_identity.length === 0 || + value.subject.diff_identity.length > 200 || + typeof value.subject.dirty !== 'boolean' || + !/^[0-9a-f]{64}$/.test(value.subject.target_sha256 ?? '') + ) + errors.push('subject is invalid'); + closed(value.scope, ['adapter', 'target', 'name'], 'scope', errors); + if ( + !plainObject(value.scope) || + !GOVERNED_ADAPTERS.includes(value.scope.adapter) || + typeof value.scope.target !== 'string' || + value.scope.target.length === 0 || + (value.scope.name !== null && typeof value.scope.name !== 'string') + ) + errors.push('scope is invalid'); + if (value.mode !== 'local_zero_egress') errors.push('mode is invalid'); + validateLimits(value.limits, errors); + if (!Array.isArray(value.external_services)) errors.push('external_services must be an array'); + if ( + value.approval_identity !== null && + !/^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/.test(value.approval_identity) + ) + errors.push('approval_identity is invalid'); + if ( + !plainObject(value.enforcement) || + !['node_preload', 'macos_sandbox_node_preload', 'macos_sandbox', 'unavailable'].includes( + value.enforcement.kind + ) + ) + errors.push('enforcement is invalid'); + closed(value.enforcement, ['kind', 'network_scope', 'reason'], 'enforcement', errors); + if ( + plainObject(value.enforcement) && + !['none', 'loopback_only', 'unknown'].includes(value.enforcement.network_scope) + ) + errors.push('enforcement.network_scope is invalid'); + closed(value.decision, ['status', 'reason', 'blockers'], 'decision', errors); + if ( + !plainObject(value.decision) || + !['admitted', 'blocked'].includes(value.decision.status) || + typeof value.decision.reason !== 'string' || + value.decision.reason.length === 0 || + !stringArray(value.decision.blockers) + ) + errors.push('decision is invalid'); + if (!stringArray(value.external_services)) errors.push('external_services must be an array'); + if (!stringArray(value.limitations)) errors.push('limitations must be an array'); + const { plan_id: _planId, ...payload } = value; + if (value.plan_id && value.plan_id !== sha256(stableStringify(payload))) + errors.push('plan_id does not match content'); + return errors; +} + +export function assertPerformanceExecutionReceipt(value) { + const errors = validatePerformanceExecutionReceipt(value); + if (errors.length > 0) + throw new Error(`invalid performance execution receipt: ${errors.join('; ')}`); + return value; +} + +export function validatePerformanceExecutionReceipt(value) { + const errors = []; + if (!plainObject(value)) return ['receipt must be an object']; + closed( + value, + [ + 'schema_version', + 'plan_id', + 'decision', + 'status', + 'planned', + 'observed', + 'enforcement', + 'terminal_reason', + 'limitations', + ], + 'receipt', + errors + ); + if (value.schema_version !== PERFORMANCE_EXECUTION_RECEIPT_SCHEMA_VERSION) + errors.push('invalid schema_version'); + if (!/^[0-9a-f]{64}$/.test(value.plan_id ?? '')) errors.push('plan_id is invalid'); + if (!['admitted', 'blocked'].includes(value.decision)) errors.push('decision is invalid'); + if (!['admitted', 'blocked', 'completed', 'failed', 'policy_violation'].includes(value.status)) + errors.push('status is invalid'); + validateLimits(value.planned, errors); + if (!plainObject(value.observed)) errors.push('observed is invalid'); + else { + closed( + value.observed, + [ + 'wall_time_ms', + 'processes', + 'max_concurrency', + 'retries', + 'successful_external_requests', + 'blocked_external_attempts', + 'external_services', + 'cost_microusd', + ], + 'observed', + errors + ); + for (const field of [ + 'wall_time_ms', + 'processes', + 'max_concurrency', + 'retries', + 'successful_external_requests', + 'blocked_external_attempts', + ]) { + if (!Number.isInteger(value.observed[field]) || value.observed[field] < 0) + errors.push(`observed.${field} is invalid`); + } + if (!stringArray(value.observed.external_services)) + errors.push('observed.external_services is invalid'); + if ( + value.observed.cost_microusd !== null && + (!Number.isInteger(value.observed.cost_microusd) || value.observed.cost_microusd < 0) + ) + errors.push('observed.cost_microusd is invalid'); + } + closed(value.enforcement, ['kind', 'network_scope', 'reason'], 'enforcement', errors); + if ( + !plainObject(value.enforcement) || + !['node_preload', 'macos_sandbox_node_preload', 'macos_sandbox', 'unavailable'].includes( + value.enforcement.kind + ) || + !['none', 'loopback_only', 'unknown'].includes(value.enforcement.network_scope) + ) + errors.push('enforcement is invalid'); + if (typeof value.terminal_reason !== 'string' || value.terminal_reason.length === 0) + errors.push('terminal_reason is invalid'); + if (!stringArray(value.limitations)) errors.push('limitations must be an array'); + return errors; +} + +export function blockedEgressMarkers(stderr) { + return String(stderr) + .split(/\r?\n/) + .filter((line) => line.startsWith('CODEVETTER_EGRESS_BLOCKED ')) + .slice(0, 32) + .map((line) => { + try { + return JSON.parse(line.slice('CODEVETTER_EGRESS_BLOCKED '.length)); + } catch { + return { kind: 'unknown', destination: '' }; + } + }); +} + +function inspectSafetySignals(source, { adapter, target, name }) { + const blockers = []; + const externalServices = new Set(); + const urls = [...source.matchAll(URL_LITERAL)].map((match) => match[0]); + const remoteUrls = urls.filter((url) => !LOOPBACK_URL.test(url)); + const loopbackUrls = urls.filter((url) => LOOPBACK_URL.test(url)); + for (const url of remoteUrls) { + try { + externalServices.add(new URL(url).hostname.toLowerCase()); + } catch { + externalServices.add('unknown-remote-endpoint'); + } + } + if (remoteUrls.length > 0) blockers.push('The workload contains a non-loopback endpoint.'); + if (DYNAMIC_NETWORK_CALL.test(source)) { + blockers.push( + 'The workload contains a network call whose destination is not a literal loopback URL.' + ); + externalServices.add('unknown-dynamic-endpoint'); + } + if (REMOTE_SERVICE.test(source) || CLOUDFLARE_SERVICE.test(source)) { + blockers.push( + 'The workload contains a hosted or paid service signal with unknown execution cost.' + ); + externalServices.add('unknown-hosted-service'); + } + if (SUBPROCESS_NETWORK_ESCAPE.test(source)) { + blockers.push( + 'The workload can launch a subprocess outside the portable Node zero-egress guard.' + ); + } + if (FORBIDDEN_WORKLOAD.test(`${target} ${name ?? ''} ${source}`)) { + blockers.push('Autonomous load, soak, stress, and production profiling is unsupported.'); + } + return { + blockers, + external_services: [...externalServices].sort(), + loopback_urls: loopbackUrls, + unknown_cost: + remoteUrls.length > 0 || + DYNAMIC_NETWORK_CALL.test(source) || + REMOTE_SERVICE.test(source) || + CLOUDFLARE_SERVICE.test(source), + limitations: [ + adapter === 'playwright' + ? 'Browser execution is admitted only for explicit loopback targets.' + : 'Admission applies only to the exact repository-owned workload identity.', + ], + }; +} + +function enforcementFor(adapter) { + if (process.platform === 'darwin') { + return NODE_ADAPTERS.has(adapter) || adapter === 'playwright' + ? { kind: 'macos_sandbox_node_preload', network_scope: 'loopback_only' } + : { kind: 'macos_sandbox', network_scope: 'none' }; + } + if (NODE_ADAPTERS.has(adapter)) return { kind: 'node_preload', network_scope: 'loopback_only' }; + return { + kind: 'unavailable', + network_scope: 'unknown', + reason: `Portable zero-egress enforcement is unavailable for ${adapter} on ${process.platform}.`, + }; +} + +async function inspectTarget(root, target) { + if (typeof target !== 'string' || target.length === 0 || isAbsolute(target)) + throw new Error('target must be a repository-relative file'); + const absolute = resolve(root, target); + const relative = repositoryRelative(root, absolute); + if (relative === null) throw new Error('target escapes repository'); + const resolved = await realpath(absolute); + if (repositoryRelative(root, resolved) === null) + throw new Error('target symlink escapes repository'); + const metadata = await stat(resolved); + if (!metadata.isFile() || metadata.size > MAX_SOURCE_BYTES) + throw new Error('target must be a bounded regular file'); + return { relative, source: await readFile(resolved, 'utf8') }; +} + +function validateLimits(value, errors) { + if (!plainObject(value)) { + errors.push('limits are invalid'); + return; + } + closed( + value, + [ + 'max_wall_time_ms', + 'max_processes', + 'max_concurrency', + 'max_retries', + 'max_external_requests', + 'max_cost_microusd', + ], + 'limits', + errors + ); + const exact = { max_concurrency: 1, max_retries: 0, max_external_requests: 0 }; + for (const [field, expected] of Object.entries(exact)) + if (value[field] !== expected) errors.push(`limits.${field} must equal ${expected}`); + if ( + !Number.isInteger(value.max_wall_time_ms) || + value.max_wall_time_ms < 100 || + value.max_wall_time_ms > LIMITS.maximumTimeoutMs * 64 + ) + errors.push('limits.max_wall_time_ms is invalid'); + if (!Number.isInteger(value.max_processes) || value.max_processes < 1 || value.max_processes > 64) + errors.push('limits.max_processes is invalid'); + if (value.max_cost_microusd !== null && value.max_cost_microusd !== 0) + errors.push('limits.max_cost_microusd is invalid'); +} + +function boundedTotalDuration(timeoutMs, processCount) { + if (!Number.isInteger(timeoutMs) || timeoutMs < 100 || timeoutMs > LIMITS.maximumTimeoutMs) + throw new Error('timeout is outside the performance execution bound'); + if (!Number.isInteger(processCount) || processCount < 1 || processCount > 64) + throw new Error('process count is outside the performance execution bound'); + return timeoutMs * processCount; +} + +function normalizeApprovalIdentity(value) { + if (value === null || value === undefined || value === '') return null; + if (typeof value !== 'string' || !/^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/.test(value)) + throw new Error('approval identity is invalid'); + return value; +} + +function closed(value, allowed, label, errors) { + if (!plainObject(value)) return; + const keys = new Set(allowed); + for (const key of Object.keys(value)) + if (!keys.has(key)) errors.push(`${label} contains unknown field: ${key}`); +} + +function plainObject(value) { + return Boolean(value && typeof value === 'object' && !Array.isArray(value)); +} + +function stringArray(value) { + return Array.isArray(value) && value.every((entry) => typeof entry === 'string'); +} + +function stableStringify(value) { + return JSON.stringify(sortValue(value)); +} + +function sortValue(value) { + if (Array.isArray(value)) return value.map(sortValue); + if (!plainObject(value)) return value; + return Object.fromEntries( + Object.keys(value) + .sort() + .map((key) => [key, sortValue(value[key])]) + ); +} + +function sha256(value) { + return createHash('sha256').update(value).digest('hex'); +} diff --git a/scripts/runtime-failure-capsule/execution-governance.test.mjs b/scripts/runtime-failure-capsule/execution-governance.test.mjs new file mode 100644 index 00000000..7e0ef16b --- /dev/null +++ b/scripts/runtime-failure-capsule/execution-governance.test.mjs @@ -0,0 +1,251 @@ +import assert from 'node:assert/strict'; +import { execFile } from 'node:child_process'; +import { mkdtemp, mkdir, rm, writeFile } from 'node:fs/promises'; +import http from 'node:http'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { promisify } from 'node:util'; +import test from 'node:test'; + +import { + createPerformanceExecutionReceipt, + planPerformanceExecution, + validatePerformanceExecutionPlan, +} from './execution-governance.mjs'; +import { runClosedAdapter } from './runner.mjs'; +import { profileRepository } from './performance.mjs'; +import { supervisePerformanceRun } from './supervision.mjs'; + +const run = promisify(execFile); + +test('local plan is immutable, bounded, and stable without running project code', async (context) => { + const root = await fixture(context, { + 'bench.mjs': "process.stdout.write('not executed');\n", + }); + const input = { + repositoryRoot: root, + adapter: 'node-script', + target: 'bench.mjs', + timeoutMs: 1_000, + processCount: 4, + }; + const first = await planPerformanceExecution(input); + const second = await planPerformanceExecution(input); + + assert.equal(first.decision.status, 'admitted'); + assert.equal(first.plan_id, second.plan_id); + assert.deepEqual(first.limits, { + max_wall_time_ms: 4_000, + max_processes: 4, + max_concurrency: 1, + max_retries: 0, + max_external_requests: 0, + max_cost_microusd: 0, + }); + assert.deepEqual(first.external_services, []); + assert.ok( + validatePerformanceExecutionPlan({ ...first, unexpected: true }).includes( + 'plan contains unknown field: unexpected' + ) + ); + assert.ok( + validatePerformanceExecutionPlan({ + ...first, + limits: { ...first.limits, unexpected: true }, + }).includes('limits contains unknown field: unexpected') + ); + assert.ok( + validatePerformanceExecutionPlan({ + ...first, + decision: { ...first.decision, blockers: [42] }, + }).includes('decision is invalid') + ); +}); + +test('remote and production-like plans fail closed even with an approval identity', async (context) => { + const root = await fixture(context, { + 'stress-production.mjs': "await fetch('https://api.openai.com/v1/models');\n", + }); + const plan = await planPerformanceExecution({ + repositoryRoot: root, + adapter: 'node-script', + target: 'stress-production.mjs', + name: 'production stress test', + timeoutMs: 1_000, + processCount: 1, + approvalIdentity: 'owner-approved-123', + }); + const receipt = createPerformanceExecutionReceipt(plan); + + assert.equal(plan.decision.status, 'blocked'); + assert.equal(plan.approval_identity, 'owner-approved-123'); + assert.ok(plan.external_services.includes('api.openai.com')); + assert.equal(plan.limits.max_cost_microusd, null); + assert.equal(receipt.status, 'blocked'); + assert.equal(receipt.observed.processes, 0); + assert.equal(receipt.observed.successful_external_requests, 0); + assert.equal(receipt.observed.cost_microusd, null); +}); + +test('runtime guard permits loopback and blocks imported remote access without internet', async (context) => { + const server = http.createServer((_request, response) => response.end('ok')); + await new Promise((resolvePromise) => server.listen(0, '127.0.0.1', resolvePromise)); + context.after(() => new Promise((resolvePromise) => server.close(resolvePromise))); + const address = server.address(); + const root = await fixture(context, { + 'local.mjs': "import './local-dependency.mjs';\n", + 'local-dependency.mjs': `const response = await fetch('http://127.0.0.1:${address.port}/'); if ((await response.text()) !== 'ok') process.exitCode = 1;\n`, + 'remote.mjs': "import './remote-dependency.mjs';\n", + 'remote-dependency.mjs': "await fetch('http://192.0.2.1/');\n", + }); + const localPlan = await planPerformanceExecution({ + repositoryRoot: root, + adapter: 'node-script', + target: 'local.mjs', + timeoutMs: 5_000, + processCount: 1, + }); + const local = await runClosedAdapter({ + repositoryRoot: root, + adapter: 'node-script', + target: 'local.mjs', + timeoutMs: 5_000, + executionPlan: localPlan, + }); + assert.equal(local.status, 'exited'); + assert.equal(local.exitCode, 0, local.stderr); + + const remotePlan = await planPerformanceExecution({ + repositoryRoot: root, + adapter: 'node-script', + target: 'remote.mjs', + timeoutMs: 5_000, + processCount: 1, + }); + const remote = await runClosedAdapter({ + repositoryRoot: root, + adapter: 'node-script', + target: 'remote.mjs', + timeoutMs: 5_000, + executionPlan: remotePlan, + }); + const receipt = createPerformanceExecutionReceipt(remotePlan, [ + { phase: 'measurement', index: 0, execution: remote }, + ]); + assert.notEqual(remote.exitCode, 0); + assert.match(remote.stderr, /CODEVETTER_EGRESS_BLOCKED/); + assert.equal(receipt.status, 'policy_violation'); + assert.equal(receipt.observed.blocked_external_attempts, 1); + assert.equal(receipt.observed.successful_external_requests, 0); + assert.equal(receipt.observed.retries, 0); +}); + +test('runner rejects a stale target identity before execution', async (context) => { + const root = await fixture(context, { 'bench.mjs': "process.stdout.write('old');\n" }); + const plan = await planPerformanceExecution({ + repositoryRoot: root, + adapter: 'node-script', + target: 'bench.mjs', + timeoutMs: 1_000, + processCount: 1, + }); + await writeFile(join(root, 'bench.mjs'), "process.stdout.write('new');\n"); + await assert.rejects( + runClosedAdapter({ + repositoryRoot: root, + adapter: 'node-script', + target: 'bench.mjs', + timeoutMs: 1_000, + executionPlan: plan, + }), + /plan identity is stale/ + ); +}); + +test('profile and dry-run CLI emit machine-readable blocked evidence before project code', async (context) => { + const root = await fixture(context, { + 'remote.mjs': + "await fetch('https://paid.example.test/work');\nprocess.stdout.write('must-not-run');\n", + }); + const capsule = await profileRepository({ + repositoryRoot: root, + adapter: 'node-script', + target: 'remote.mjs', + timeoutMs: 1_000, + samples: 2, + warmups: 0, + }); + assert.equal(capsule.verdict.status, 'no_confidence'); + assert.equal(capsule.observed.executions.length, 0); + assert.equal(capsule.execution_governance.receipt.status, 'blocked'); + assert.equal(capsule.execution_governance.receipt.observed.processes, 0); + + const supervised = await supervisePerformanceRun({ + repositoryRoot: root, + runId: 'blocked-remote-run', + adapter: 'node-script', + target: 'remote.mjs', + timeoutMs: 1_000, + samples: 2, + warmups: 0, + }); + assert.equal(supervised.state, 'blocked'); + assert.equal(supervised.child.pid, null); + assert.equal(supervised.execution_governance.receipt.observed.processes, 0); + + await assert.rejects( + run( + process.execPath, + [ + join(import.meta.dirname, 'cli.mjs'), + 'plan-performance', + '--repo', + root, + '--adapter', + 'node-script', + '--target', + 'remote.mjs', + '--samples', + '2', + '--warmups', + '0', + '--timeout-ms', + '1000', + '--json', + ], + { cwd: root } + ), + (error) => { + const plan = JSON.parse(error.stdout); + assert.equal(error.code, 2); + assert.equal(plan.decision.status, 'blocked'); + assert.equal(plan.limits.max_cost_microusd, null); + return true; + } + ); +}); + +async function fixture(context, files) { + const root = await mkdtemp(join(tmpdir(), 'codevetter-execution-governance-')); + context.after(() => rm(root, { recursive: true, force: true })); + for (const [path, contents] of Object.entries(files)) { + await mkdir(join(root, path, '..'), { recursive: true }); + await writeFile(join(root, path), contents); + } + await run('git', ['init', '-q'], { cwd: root }); + await run('git', ['add', '.'], { cwd: root }); + await run( + 'git', + [ + '-c', + 'user.name=CodeVetter', + '-c', + 'user.email=codevetter@example.invalid', + 'commit', + '-qm', + 'fixture', + ], + { cwd: root } + ); + return root; +} diff --git a/scripts/runtime-failure-capsule/local-flow-runtime.test.mjs b/scripts/runtime-failure-capsule/local-flow-runtime.test.mjs index 391304db..ae647473 100644 --- a/scripts/runtime-failure-capsule/local-flow-runtime.test.mjs +++ b/scripts/runtime-failure-capsule/local-flow-runtime.test.mjs @@ -179,6 +179,7 @@ test('runtime MCP exposes product capabilities and fails closed on unknown captu tools.map((tool) => tool.name), [ 'qualify_runtime_repository', + 'plan_local_performance', 'profile_local_performance', 'verify_paired_performance', 'inspect_performance_run', @@ -199,14 +200,15 @@ test('runtime MCP exposes product capabilities and fails closed on unknown captu ] ); assert.equal(tools[0].annotations.readOnlyHint, true); - assert.equal(tools[1].annotations.readOnlyHint, false); + assert.equal(tools[1].annotations.readOnlyHint, true); assert.equal(tools[2].annotations.readOnlyHint, false); - assert.equal(tools[3].annotations.readOnlyHint, true); - assert.equal(tools[4].annotations.readOnlyHint, false); + assert.equal(tools[3].annotations.readOnlyHint, false); + assert.equal(tools[4].annotations.readOnlyHint, true); + assert.equal(tools[5].annotations.readOnlyHint, false); const handle = await createRuntimeMcpHandler(root); const listed = await handle({ jsonrpc: '2.0', id: 1, method: 'tools/list' }); - assert.equal(listed.result.tools.length, 18); + assert.equal(listed.result.tools.length, 19); const qualification = await handle({ jsonrpc: '2.0', id: 2, @@ -333,7 +335,7 @@ test('runtime MCP process speaks line-delimited JSON-RPC without network setup', { jsonrpc: '2.0', id: 2, method: 'tools/list', params: {} }, ]); assert.equal(responses[0].result.serverInfo.name, 'codevetter-local-runtime'); - assert.equal(responses[1].result.tools.length, 18); + assert.equal(responses[1].result.tools.length, 19); }); test('validates the required recursive flow contract', () => { diff --git a/scripts/runtime-failure-capsule/mcp.mjs b/scripts/runtime-failure-capsule/mcp.mjs index 4313394e..6847ec50 100644 --- a/scripts/runtime-failure-capsule/mcp.mjs +++ b/scripts/runtime-failure-capsule/mcp.mjs @@ -4,6 +4,7 @@ import { fileURLToPath } from 'node:url'; import { createOptimizationCampaignService } from './campaign.mjs'; import { createOptimizationContributionService } from './contribution.mjs'; +import { planPerformanceExecution } from './execution-governance.mjs'; import { LIMITS, PROFILE_ADAPTERS, @@ -16,7 +17,7 @@ import { planFlowOptimizationCampaign } from './flow-campaign-planner.mjs'; import { qualifyRepository } from './qualification.mjs'; import { redactText } from './redact.mjs'; import { inspectSupervisedRun } from './supervision.mjs'; -import { profileRepository } from './performance.mjs'; +import { plannedProfileProcessCount, profileRepository } from './performance.mjs'; import { verifyPairedRepositories } from './paired-verification.mjs'; const PROTOCOL_VERSION = '2025-03-26'; @@ -112,6 +113,27 @@ async function callTool( closedArguments(args, ['run_id']); return inspectSupervisedRun(repositoryRoot, args.run_id); } + if (name === 'plan_local_performance') { + closedArguments( + args, + ['adapter', 'target'], + ['name', 'approval_identity'], + [], + ['samples', 'warmups', 'timeout_ms'] + ); + const adapter = assertProfileAdapter(args.adapter); + const samples = boundedSamples(args.samples); + const warmups = boundedWarmups(args.warmups); + return planPerformanceExecution({ + repositoryRoot, + adapter, + target: args.target, + name: args.name, + timeoutMs: boundedTimeout(args.timeout_ms), + processCount: plannedProfileProcessCount({ adapter, samples, warmups }), + approvalIdentity: args.approval_identity, + }); + } if (PERFORMANCE_TOOLS.has(name)) { return callPerformanceTool({ name, @@ -272,6 +294,22 @@ export function toolDefinitions() { properties: {}, }, }, + { + name: 'plan_local_performance', + description: + 'Dry-run one exact performance scope and report immutable zero-egress, duration, retry, request, service, and cost bounds without executing project code.', + annotations: readAnnotations, + inputSchema: { + ...performanceInputSchema(), + properties: { + ...performanceInputSchema().properties, + approval_identity: { + type: 'string', + pattern: '^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$', + }, + }, + }, + }, { name: 'profile_local_performance', description: diff --git a/scripts/runtime-failure-capsule/node-egress-preload.mjs b/scripts/runtime-failure-capsule/node-egress-preload.mjs new file mode 100644 index 00000000..48fd6e1e --- /dev/null +++ b/scripts/runtime-failure-capsule/node-egress-preload.mjs @@ -0,0 +1,86 @@ +import dns from 'node:dns'; +import http from 'node:http'; +import https from 'node:https'; +import net from 'node:net'; +import tls from 'node:tls'; + +const BLOCK_CODE = 'CODEVETTER_REMOTE_EGRESS_BLOCKED'; + +function isLoopback(hostname) { + const value = String(hostname ?? '') + .replace(/^\[|\]$/g, '') + .toLowerCase(); + return value === 'localhost' || value === '::1' || /^127(?:\.\d{1,3}){3}$/.test(value); +} + +function destination(args) { + const first = args[0]; + if (first instanceof URL) return first.hostname; + if (typeof first === 'string') { + try { + return new URL(first).hostname; + } catch { + return first; + } + } + if (first && typeof first === 'object') return first.hostname ?? first.host ?? 'localhost'; + return args[1]?.hostname ?? args[1]?.host ?? 'localhost'; +} + +function deny(kind, hostname) { + const safeDestination = String(hostname ?? '').slice(0, 255); + process.stderr.write( + `CODEVETTER_EGRESS_BLOCKED ${JSON.stringify({ kind, destination: safeDestination })}\n` + ); + const error = new Error(`CodeVetter blocked remote ${kind} access to ${safeDestination}`); + error.code = BLOCK_CODE; + throw error; +} + +function guard(object, method, kind, pick = destination) { + const original = object?.[method]; + if (typeof original !== 'function') return; + object[method] = function guarded(...args) { + const hostname = pick(args); + if (!isLoopback(hostname)) return deny(kind, hostname); + return original.apply(this, args); + }; +} + +for (const method of ['connect', 'createConnection']) guard(net, method, 'socket'); +guard(tls, 'connect', 'tls'); +for (const method of ['request', 'get']) { + guard(http, method, 'http'); + guard(https, method, 'https'); +} +for (const method of ['lookup', 'resolve', 'resolve4', 'resolve6']) { + guard(dns, method, 'dns', (args) => args[0]); +} + +if (typeof globalThis.fetch === 'function') { + const originalFetch = globalThis.fetch; + globalThis.fetch = function guardedFetch(input, init) { + const hostname = destination([input]); + if (!isLoopback(hostname)) return Promise.reject(policyError('fetch', hostname)); + return originalFetch(input, init); + }; +} + +if (typeof globalThis.WebSocket === 'function') { + const OriginalWebSocket = globalThis.WebSocket; + globalThis.WebSocket = class GuardedWebSocket extends OriginalWebSocket { + constructor(url, protocols) { + const hostname = destination([url]); + if (!isLoopback(hostname)) deny('websocket', hostname); + super(url, protocols); + } + }; +} + +function policyError(kind, hostname) { + try { + deny(kind, hostname); + } catch (error) { + return error; + } +} diff --git a/scripts/runtime-failure-capsule/paired-verification.mjs b/scripts/runtime-failure-capsule/paired-verification.mjs index 1485adf9..73e4886d 100644 --- a/scripts/runtime-failure-capsule/paired-verification.mjs +++ b/scripts/runtime-failure-capsule/paired-verification.mjs @@ -192,8 +192,12 @@ function pairedCapsule({ }, viteArtifact, }); + const usesWorkloadMetric = + capsule.observed.console_metrics.length > 0 || capsule.observed.go_benchmarks.length > 0; capsule.limitations = capsule.limitations.filter( - (limitation) => !DIAGNOSTIC_ONLY_LIMITATIONS.has(limitation) + (limitation) => + !DIAGNOSTIC_ONLY_LIMITATIONS.has(limitation) && + !(usesWorkloadMetric && limitation.startsWith('Wall-time samples varied by')) ); capsule.capture.profile_kind = 'paired_timing_only'; return capsule; diff --git a/scripts/runtime-failure-capsule/paired-verification.test.mjs b/scripts/runtime-failure-capsule/paired-verification.test.mjs index 242dc5f3..ff3e793a 100644 --- a/scripts/runtime-failure-capsule/paired-verification.test.mjs +++ b/scripts/runtime-failure-capsule/paired-verification.test.mjs @@ -36,7 +36,7 @@ test('alternates two runnable repositories and confirms matching scale evidence' ['baseline', 'current', 'current', 'baseline', 'baseline', 'current'] ); assert.equal(report.baseline_capsule.observed.console_metrics[0].metrics[0].sample_count, 3); - assert.equal(report.limitations.length, 0); + assert.equal(report.limitations.length, 0, JSON.stringify(report.limitations)); }); test('fails closed when one paired workload does not complete', async (context) => { diff --git a/scripts/runtime-failure-capsule/performance.mjs b/scripts/runtime-failure-capsule/performance.mjs index 9b3f68a8..0f1eba23 100644 --- a/scripts/runtime-failure-capsule/performance.mjs +++ b/scripts/runtime-failure-capsule/performance.mjs @@ -12,6 +12,10 @@ import { validatePerformanceCapsule, } from './contracts.mjs'; import { inspectGitDiff, rankRelevantChanges } from './git-diff.mjs'; +import { + createPerformanceExecutionReceipt, + planPerformanceExecution, +} from './execution-governance.mjs'; import { collectV8FunctionCoverage, emptyFunctionCoverage } from './function-coverage.mjs'; import { redactText } from './redact.mjs'; import { inspectGoProfile, runClosedAdapter } from './runner.mjs'; @@ -48,6 +52,28 @@ export async function profileRepository({ validateProfileScope(adapter, name); const lexicalRoot = resolve(repositoryRoot); const root = await realpath(lexicalRoot); + const processCount = plannedProfileProcessCount({ adapter, samples, warmups, captureFlow }); + const executionPlan = await planPerformanceExecution({ + repositoryRoot: root, + adapter, + target, + name, + timeoutMs, + processCount, + }); + if (executionPlan.decision.status === 'blocked') { + return createBlockedPerformanceCapsule({ + root, + lexicalRoot, + git: await inspectGitDiff(root), + adapter, + target, + name, + samples, + warmups, + executionPlan, + }); + } const temporaryDirectory = await mkdtemp(join(tmpdir(), 'codevetter-profile-')); const executions = []; let profileEvidence = emptyProfileEvidence(adapter); @@ -67,6 +93,7 @@ export async function profileRepository({ target, name, timeoutMs, + executionPlan, }), }); } @@ -80,6 +107,7 @@ export async function profileRepository({ target, name, timeoutMs, + executionPlan, }), }); } @@ -95,6 +123,7 @@ export async function profileRepository({ name, timeoutMs, vitestReporter: adapter === 'vitest' ? 'verbose' : undefined, + executionPlan, }), }); } @@ -112,6 +141,7 @@ export async function profileRepository({ name, timeoutMs, flowDirectory, + executionPlan, }), }); flowEvidence = await collectNodeFlowEvents(flowDirectory); @@ -129,6 +159,7 @@ export async function profileRepository({ timeoutMs, coverageDirectory, vitestReporter: adapter === 'vitest' ? 'dot' : undefined, + executionPlan, }), }); functionCoverage = await collectV8FunctionCoverage(coverageDirectory, root); @@ -148,6 +179,7 @@ export async function profileRepository({ name, timeoutMs, profileDirectory, + executionPlan, }), }); profileRuns.push( @@ -187,6 +219,8 @@ export async function profileRepository({ regressionPercent, regressionMs, viteArtifact, + executionPlan, + executionReceipt: createPerformanceExecutionReceipt(executionPlan, executions), }); } @@ -209,6 +243,8 @@ export function createPerformanceCapsule({ regressionPercent = 20, regressionMs = 25, viteArtifact = null, + executionPlan = null, + executionReceipt = null, }) { let redactionCount = 0; let outputTruncated = false; @@ -303,6 +339,9 @@ export function createPerformanceCapsule({ if (cleanupFailed) limitations.push('Owned temporary profiling artifacts could not be completely removed.'); if (outputTruncated) limitations.push('Runner output was truncated before normalization.'); + if (executionReceipt?.status === 'policy_violation') { + limitations.push('The zero-egress boundary blocked a remote network attempt.'); + } limitations.push(...playwrightLimitations(adapter, playwrightTest)); if (adapter === 'go-bench' && goBenchmarks.length === 0) { limitations.push('No matching Go benchmark measurement was captured.'); @@ -547,6 +586,8 @@ export function createPerformanceCapsule({ coverage_bytes: functionCoverage.coverage_bytes, coverage_functions: functionCoverage.functions.length, }, + execution_governance: + executionPlan && executionReceipt ? { plan: executionPlan, receipt: executionReceipt } : null, verdict: { status: verdict, reason: @@ -562,6 +603,46 @@ export function createPerformanceCapsule({ return capsule; } +function createBlockedPerformanceCapsule({ + root, + lexicalRoot, + git, + adapter, + target, + name, + samples, + warmups, + executionPlan, +}) { + const executionReceipt = createPerformanceExecutionReceipt(executionPlan); + return createPerformanceCapsule({ + root, + lexicalRoot, + git, + adapter, + target, + name, + samples, + warmups, + executions: [], + profileEvidence: emptyProfileEvidence(adapter), + profileRuns: [], + flowEvidence: emptyFlowEvidence(), + functionCoverage: emptyFunctionCoverage(), + cleanupFailed: false, + baseline: null, + viteArtifact: null, + executionPlan, + executionReceipt, + }); +} + +export function plannedProfileProcessCount({ adapter, samples, warmups, captureFlow = false }) { + const metrics = ['node-test', 'node-script', 'vitest'].includes(adapter) ? samples : 0; + const flows = captureFlow && ['node-test', 'vitest'].includes(adapter) ? 2 : 0; + return warmups + samples + metrics + flows + profileRunsFor(adapter); +} + function summarizeExecution(entry, adapter, name) { const summary = { phase: entry.phase, diff --git a/scripts/runtime-failure-capsule/runner.mjs b/scripts/runtime-failure-capsule/runner.mjs index e0b8ea4a..99ec132a 100644 --- a/scripts/runtime-failure-capsule/runner.mjs +++ b/scripts/runtime-failure-capsule/runner.mjs @@ -4,6 +4,7 @@ import { dirname, isAbsolute, join, relative, resolve, sep } from 'node:path'; import { fileURLToPath, pathToFileURL } from 'node:url'; import { LIMITS, boundedTimeout, repositoryRelative } from './contracts.mjs'; +import { assertPerformanceExecutionPlanCurrent } from './execution-governance.mjs'; export async function runClosedAdapter({ repositoryRoot, @@ -16,6 +17,7 @@ export async function runClosedAdapter({ coverageDirectory, benchmarkCount = 1, vitestReporter, + executionPlan = null, }) { const root = await realpath(resolve(repositoryRoot)); const scope = await resolveTarget(root, target); @@ -53,16 +55,35 @@ export async function runClosedAdapter({ environmentValues: [], }; } + if ( + executionPlan?.decision?.status !== undefined && + executionPlan.decision.status !== 'admitted' + ) { + throw new Error('performance execution plan is not admitted'); + } + if (executionPlan) { + await assertPerformanceExecutionPlanCurrent({ + plan: executionPlan, + repositoryRoot: root, + adapter, + target: scope.relative, + name: name ?? null, + }); + } + const zeroEgress = executionPlan !== null; const environment = minimalEnvironment({ profileDirectory: profileDirectory && ['node-test', 'node-script'].includes(adapter) ? profileDirectory : null, flowDirectory: flowDirectory && ['node-test', 'vitest'].includes(adapter) ? flowDirectory : null, coverageDirectory: coverageDirectory && adapter === 'node-test' ? coverageDirectory : null, + zeroEgress: + zeroEgress && ['node-test', 'node-script', 'vitest', 'playwright'].includes(adapter), }); + const ownedCommand = governedCommand(command, executionPlan); const execution = await runOwnedProcess({ - program: command.program, - args: command.args, + program: ownedCommand.program, + args: ownedCommand.args, cwd: command.cwd, environment, timeoutMs: timeout, @@ -328,6 +349,7 @@ function minimalEnvironment({ profileDirectory = null, flowDirectory = null, coverageDirectory = null, + zeroEgress = false, } = {}) { const allowed = ['PATH', 'HOME', 'TMPDIR', 'TMP', 'TEMP', 'SYSTEMROOT', 'COMSPEC', 'PATHEXT']; const environment = { CI: '1', FORCE_COLOR: '0', NO_COLOR: '1' }; @@ -342,10 +364,29 @@ function minimalEnvironment({ environment.CODEVETTER_FLOW_DIRECTORY = flowDirectory; } if (coverageDirectory) environment.NODE_V8_COVERAGE = coverageDirectory; + if (zeroEgress) { + const preload = fileURLToPath(new URL('./node-egress-preload.mjs', import.meta.url)); + nodeOptions.push(`--import=${pathToFileURL(preload).href}`); + environment.CODEVETTER_NETWORK_POLICY = 'loopback-only'; + } if (nodeOptions.length > 0) environment.NODE_OPTIONS = nodeOptions.join(' '); return environment; } +function governedCommand(command, executionPlan) { + if (!executionPlan || !executionPlan.enforcement?.kind?.startsWith('macos_sandbox')) { + return { program: command.program, args: command.args }; + } + const allowLoopback = executionPlan.enforcement.network_scope === 'loopback_only'; + const policy = allowLoopback + ? '(version 1)(allow default)(deny network*)(allow network-inbound (local ip "*:*"))(allow network-outbound (remote ip "localhost:*"))' + : '(version 1)(allow default)(deny network*)'; + return { + program: '/usr/bin/sandbox-exec', + args: ['-p', policy, command.program, ...command.args], + }; +} + function runOwnedProcess({ program, args, cwd, environment, timeoutMs }) { return new Promise((resolvePromise) => { const startedAt = Date.now(); diff --git a/scripts/runtime-failure-capsule/supervision-contracts.mjs b/scripts/runtime-failure-capsule/supervision-contracts.mjs index e797b0bc..6a2e66af 100644 --- a/scripts/runtime-failure-capsule/supervision-contracts.mjs +++ b/scripts/runtime-failure-capsule/supervision-contracts.mjs @@ -13,6 +13,7 @@ export const SUPERVISION_LIMITS = Object.freeze({ export const SUPERVISED_RUN_STATES = Object.freeze([ 'initialized', + 'blocked', 'running', 'succeeded', 'failed', @@ -51,6 +52,7 @@ export function validateSupervisedRunReceipt(value) { 'state', 'subject', 'scope', + 'execution_governance', 'policy', 'supervisor', 'lifecycle', @@ -72,6 +74,7 @@ export function validateSupervisedRunReceipt(value) { if (!SUPERVISED_RUN_STATES.includes(value.state)) errors.push('invalid state'); validateSubject(value.subject, errors); validateScope(value.scope, errors); + validateExecutionGovernance(value.execution_governance, errors); validatePolicy(value.policy, errors); validateSupervisor(value.supervisor, errors); validateLifecycle(value.lifecycle, errors); @@ -90,6 +93,20 @@ export function validateSupervisedRunReceipt(value) { return errors; } +function validateExecutionGovernance(value, errors) { + if (value === null || value === undefined) return; + if (!objectWithKeys(value, ['plan', 'receipt'], 'execution_governance', errors)) return; + if (value.plan?.schema_version !== 'performance-execution-plan/v1') { + errors.push('execution_governance.plan is invalid'); + } + if (value.receipt?.schema_version !== 'performance-execution-receipt/v1') { + errors.push('execution_governance.receipt is invalid'); + } + if (value.plan?.plan_id !== value.receipt?.plan_id) { + errors.push('execution_governance identity mismatch'); + } +} + function validateSubject(value, errors) { if (!objectWithKeys(value, ['repository_revision', 'dirty'], 'subject', errors)) return; if (typeof value.repository_revision !== 'string' || value.repository_revision.length === 0) { diff --git a/scripts/runtime-failure-capsule/supervision.mjs b/scripts/runtime-failure-capsule/supervision.mjs index 4908695d..041247b9 100644 --- a/scripts/runtime-failure-capsule/supervision.mjs +++ b/scripts/runtime-failure-capsule/supervision.mjs @@ -13,6 +13,11 @@ import { validatePerformanceDiagnosis, } from './contracts.mjs'; import { inspectGitDiff } from './git-diff.mjs'; +import { + createPerformanceExecutionReceipt, + planPerformanceExecution, +} from './execution-governance.mjs'; +import { plannedProfileProcessCount } from './performance.mjs'; import { redactJsonValue, redactText } from './redact.mjs'; import { SUPERVISED_RUN_SCHEMA_VERSION, @@ -58,6 +63,18 @@ export async function supervisePerformanceRun({ } await assertTarget(root, target); const git = await inspectGitDiff(root); + const executionPlan = await planPerformanceExecution({ + repositoryRoot: root, + adapter: safeAdapter, + target, + name, + timeoutMs: safeTimeout, + processCount: plannedProfileProcessCount({ + adapter: safeAdapter, + samples: safeSamples, + warmups: safeWarmups, + }), + }); const relativeDirectory = `${RUNS_DIRECTORY}/${safeRunId}`; const lexicalRunsDirectory = resolve(root, RUNS_DIRECTORY); await mkdir(lexicalRunsDirectory, { recursive: true }); @@ -75,6 +92,10 @@ export async function supervisePerformanceRun({ state: 'initialized', subject: { repository_revision: git.repository_revision, dirty: git.dirty }, scope: { adapter: safeAdapter, target, name: name ?? null }, + execution_governance: { + plan: executionPlan, + receipt: createPerformanceExecutionReceipt(executionPlan), + }, policy: { samples: safeSamples, warmups: safeWarmups, @@ -102,6 +123,26 @@ export async function supervisePerformanceRun({ }, limitations: ['A supervised run proves only the recorded local performance scope.'], }; + if (executionPlan.decision.status === 'blocked') { + const completedAt = new Date().toISOString(); + receipt = { + ...receipt, + state: 'blocked', + lifecycle: { ...receipt.lifecycle, completed_at: completedAt }, + failure: { + kind: 'blocked', + operational_error: executionPlan.decision.blockers.join(' '), + stdout: null, + stderr: null, + }, + limitations: [ + ...receipt.limitations, + 'No project code executed because the performance execution plan was blocked.', + ], + }; + await writeReceipt(receiptPath, receipt); + return assertSupervisedRunReceipt(receipt); + } await writeReceipt(receiptPath, receipt); const command = From ded42f1e902a745b2f833b13449e8f1bfb2a7277 Mon Sep 17 00:00:00 2001 From: Sarthak Agrawal Date: Sat, 15 Aug 2026 20:41:30 +0530 Subject: [PATCH 2/3] docs: archive performance execution governance --- .../.openspec.yaml | 0 .../design.md | 0 .../proposal.md | 0 .../autonomous-optimization-campaigns/spec.md | 0 .../performance-execution-governance/spec.md | 0 .../tasks.md | 2 +- .../autonomous-optimization-campaigns/spec.md | 14 ++++ .../performance-execution-governance/spec.md | 64 +++++++++++++++++++ 8 files changed, 79 insertions(+), 1 deletion(-) rename openspec/changes/{guard-performance-cloud-spend => archive/2026-08-15-guard-performance-cloud-spend}/.openspec.yaml (100%) rename openspec/changes/{guard-performance-cloud-spend => archive/2026-08-15-guard-performance-cloud-spend}/design.md (100%) rename openspec/changes/{guard-performance-cloud-spend => archive/2026-08-15-guard-performance-cloud-spend}/proposal.md (100%) rename openspec/changes/{guard-performance-cloud-spend => archive/2026-08-15-guard-performance-cloud-spend}/specs/autonomous-optimization-campaigns/spec.md (100%) rename openspec/changes/{guard-performance-cloud-spend => archive/2026-08-15-guard-performance-cloud-spend}/specs/performance-execution-governance/spec.md (100%) rename openspec/changes/{guard-performance-cloud-spend => archive/2026-08-15-guard-performance-cloud-spend}/tasks.md (95%) create mode 100644 openspec/specs/performance-execution-governance/spec.md diff --git a/openspec/changes/guard-performance-cloud-spend/.openspec.yaml b/openspec/changes/archive/2026-08-15-guard-performance-cloud-spend/.openspec.yaml similarity index 100% rename from openspec/changes/guard-performance-cloud-spend/.openspec.yaml rename to openspec/changes/archive/2026-08-15-guard-performance-cloud-spend/.openspec.yaml diff --git a/openspec/changes/guard-performance-cloud-spend/design.md b/openspec/changes/archive/2026-08-15-guard-performance-cloud-spend/design.md similarity index 100% rename from openspec/changes/guard-performance-cloud-spend/design.md rename to openspec/changes/archive/2026-08-15-guard-performance-cloud-spend/design.md diff --git a/openspec/changes/guard-performance-cloud-spend/proposal.md b/openspec/changes/archive/2026-08-15-guard-performance-cloud-spend/proposal.md similarity index 100% rename from openspec/changes/guard-performance-cloud-spend/proposal.md rename to openspec/changes/archive/2026-08-15-guard-performance-cloud-spend/proposal.md diff --git a/openspec/changes/guard-performance-cloud-spend/specs/autonomous-optimization-campaigns/spec.md b/openspec/changes/archive/2026-08-15-guard-performance-cloud-spend/specs/autonomous-optimization-campaigns/spec.md similarity index 100% rename from openspec/changes/guard-performance-cloud-spend/specs/autonomous-optimization-campaigns/spec.md rename to openspec/changes/archive/2026-08-15-guard-performance-cloud-spend/specs/autonomous-optimization-campaigns/spec.md diff --git a/openspec/changes/guard-performance-cloud-spend/specs/performance-execution-governance/spec.md b/openspec/changes/archive/2026-08-15-guard-performance-cloud-spend/specs/performance-execution-governance/spec.md similarity index 100% rename from openspec/changes/guard-performance-cloud-spend/specs/performance-execution-governance/spec.md rename to openspec/changes/archive/2026-08-15-guard-performance-cloud-spend/specs/performance-execution-governance/spec.md diff --git a/openspec/changes/guard-performance-cloud-spend/tasks.md b/openspec/changes/archive/2026-08-15-guard-performance-cloud-spend/tasks.md similarity index 95% rename from openspec/changes/guard-performance-cloud-spend/tasks.md rename to openspec/changes/archive/2026-08-15-guard-performance-cloud-spend/tasks.md index 54c099a6..a8c605a2 100644 --- a/openspec/changes/guard-performance-cloud-spend/tasks.md +++ b/openspec/changes/archive/2026-08-15-guard-performance-cloud-spend/tasks.md @@ -16,4 +16,4 @@ - [x] 3.1 Add hermetic boundary tests for allowed local work, remote attempts, unsupported adapters, unknown cost, stale identity, and bounded receipts - [x] 3.2 Document local profiling, public smoke, hosted/load boundaries, dry-run usage, and receipt interpretation - [x] 3.3 Run focused runtime tests, lint, docs validation, strict OpenSpec validation, and diff checks -- [ ] 3.4 Link the shipped change to GitHub issue #105, sync the specification, archive the completed change, and update current product truth +- [x] 3.4 Link the shipped change to GitHub issue #105, sync the specification, archive the completed change, and update current product truth diff --git a/openspec/specs/autonomous-optimization-campaigns/spec.md b/openspec/specs/autonomous-optimization-campaigns/spec.md index 77e08506..88dc9dee 100644 --- a/openspec/specs/autonomous-optimization-campaigns/spec.md +++ b/openspec/specs/autonomous-optimization-campaigns/spec.md @@ -112,3 +112,17 @@ machine paths from portable evidence. #### Scenario: Unknown campaign argument - **WHEN** an MCP or CLI caller supplies an unknown field, escaping artifact path, or unsupported operation - **THEN** the system fails closed without running a workload or mutating campaign state + +### Requirement: Campaign execution requires current local admission +The system SHALL derive and validate a current performance-execution plan before +running campaign correctness or performance scopes. A blocked, stale, or +identity-mismatched plan MUST leave the campaign in `no_confidence` and MUST NOT +execute project code or consume an experiment attempt. + +#### Scenario: Baseline is admitted locally +- **WHEN** every declared campaign scope has a current admitted zero-egress plan +- **THEN** the campaign may execute the bounded baseline and attach the admission receipts to its evidence + +#### Scenario: Candidate contains a remote workload +- **WHEN** any correctness or performance scope has remote, paid, or unknown-cost evidence +- **THEN** the campaign records `no_confidence` with the blocked admission receipt before executing any declared scope diff --git a/openspec/specs/performance-execution-governance/spec.md b/openspec/specs/performance-execution-governance/spec.md new file mode 100644 index 00000000..66e59058 --- /dev/null +++ b/openspec/specs/performance-execution-governance/spec.md @@ -0,0 +1,64 @@ +# performance-execution-governance Specification + +## Purpose +Defines the fail-closed execution policy that keeps autonomous performance +profiling local, bounded, zero-egress, and independently auditable before any +project-owned workload is allowed to run. +## Requirements +### Requirement: Every performance run has an immutable admission plan +The system SHALL derive a versioned plan before project code executes. The plan +MUST bind repository and target identity, adapter, execution mode, maximum wall +time, process concurrency, retries, external requests, monetary cost, declared +external services, and approval identity. Unknown fields or unbounded values +MUST fail closed. + +#### Scenario: Local dry-run is admitted +- **WHEN** a caller dry-runs an exact supported local workload with no remote or unknown-cost evidence +- **THEN** the system returns an admitted zero-egress plan with one process, zero retries, zero external requests, zero external services, zero monetary cost, and a finite duration + +#### Scenario: Plan identity changes +- **WHEN** repository, target, adapter, budget, or execution-policy input differs from an earlier plan +- **THEN** the system returns a different immutable plan identity and does not reuse the prior admission + +### Requirement: Autonomous execution is local and zero-egress +The autonomous profiler SHALL execute only adapters for which it can enforce a +local zero-egress policy. It MUST block remote network access at runtime, allow +browser access only to loopback targets, run at most one owned workload process, +and perform no automatic retry. + +#### Scenario: Node workload attempts remote access +- **WHEN** an admitted Node-family workload attempts DNS, socket, HTTP, HTTPS, fetch, or WebSocket access to a non-loopback destination +- **THEN** the system blocks the operation, terminates or fails the workload, and reports a zero-egress policy violation without retrying + +#### Scenario: Runtime cannot enforce zero egress +- **WHEN** the selected adapter lacks an enforceable zero-egress boundary +- **THEN** the system blocks before project code executes and names the unsupported enforcement boundary + +### Requirement: Hosted and unknown-cost profiling remains closed +The autonomous profiler MUST NOT run hosted, paid, unknown-cost, load, soak, +stress, or production profiling. An approval identity MUST NOT silently widen +this product boundary; the plan SHALL report the unsupported execution mode and +maximum possible spend as unknown rather than zero. + +#### Scenario: Hosted target is requested without approval +- **WHEN** a workload includes a remote endpoint, paid service, production marker, or unknown pricing and no exact approval identity +- **THEN** the system blocks before the first request and records every detected service and missing approval input + +#### Scenario: Hosted target is requested with approval +- **WHEN** a caller supplies an approval identity for a hosted or paid workload +- **THEN** the autonomous profiler still reports hosted execution as unsupported and does not contact the service + +### Requirement: Every admission outcome emits a cost and egress receipt +The system SHALL emit a machine-readable receipt for admitted, blocked, failed, +and completed plans. The receipt MUST preserve planned and observed duration, +process concurrency, retry count, external request count, external services, +cost posture, enforcement method, terminal reason, and limitations without +including credentials or absolute private paths. + +#### Scenario: Workload is blocked before execution +- **WHEN** the plan is rejected because cost, service, approval, or enforcement evidence is missing +- **THEN** the receipt records zero executed requests and processes plus the projected bounds and exact blockers + +#### Scenario: Local workload completes +- **WHEN** an admitted zero-egress workload exits within its bounds +- **THEN** the receipt records the observed local process and duration totals and confirms zero external requests, retries, services, and monetary cost From 5bb911ef9f96a329116c6ccc5b314a8f02967db9 Mon Sep 17 00:00:00 2001 From: Sarthak Agrawal Date: Sat, 15 Aug 2026 21:15:16 +0530 Subject: [PATCH 3/3] refactor: keep performance governance within complexity bounds --- scripts/runtime-failure-capsule/cli.mjs | 280 +++++++++--------- .../execution-governance.mjs | 260 ++++++++-------- scripts/runtime-failure-capsule/mcp.mjs | 52 ++-- .../runtime-failure-capsule/performance.mjs | 18 +- 4 files changed, 320 insertions(+), 290 deletions(-) diff --git a/scripts/runtime-failure-capsule/cli.mjs b/scripts/runtime-failure-capsule/cli.mjs index bedf3fd9..74670e4f 100644 --- a/scripts/runtime-failure-capsule/cli.mjs +++ b/scripts/runtime-failure-capsule/cli.mjs @@ -30,6 +30,95 @@ import { redactText } from './redact.mjs'; import { runClosedAdapter } from './runner.mjs'; import { inspectSupervisedRun, supervisePerformanceRun } from './supervision.mjs'; +const OPERATIONS = [ + 'detect', + 'run', + 'import', + 'profile', + 'diagnose-performance', + 'verify-optimization', + 'verify-paired-optimization', + 'capture-flow', + 'qualify', + 'qualify-portfolio', + 'plan-performance', + 'supervise-performance', + 'inspect-performance-run', + 'plan-flow-campaign', +]; +const PLANNING_OPERATIONS = new Set(['plan-performance', 'plan-flow-campaign']); +const PROFILE_OPTIONS = [ + 'repo', + 'adapter', + 'target', + 'name', + 'timeout-ms', + 'samples', + 'warmups', + 'baseline', + 'regression-percent', + 'regression-ms', + 'vite-build-dir', + 'vite-entry', + 'json', +]; +const ALLOWED_OPTIONS = { + detect: ['repo', 'json'], + qualify: ['repo', 'json'], + 'qualify-portfolio': ['manifest', 'json'], + 'plan-performance': [ + 'repo', + 'adapter', + 'target', + 'name', + 'timeout-ms', + 'samples', + 'warmups', + 'approval-id', + 'json', + ], + 'plan-flow-campaign': [ + 'repo', + 'priority-manifest', + 'max-flows', + 'timeout-ms', + 'samples', + 'warmups', + 'json', + ], + 'inspect-performance-run': ['repo', 'run-id', 'json'], + 'supervise-performance': [ + 'repo', + 'run-id', + 'adapter', + 'target', + 'name', + 'timeout-ms', + 'samples', + 'warmups', + 'json', + ], + run: ['repo', 'adapter', 'target', 'name', 'diff', 'timeout-ms', 'json'], + import: ['repo', 'kind', 'receipt', 'diff', 'json'], + 'verify-paired-optimization': [ + 'repo', + 'baseline-repo', + 'adapter', + 'target', + 'name', + 'timeout-ms', + 'samples', + 'warmups', + 'vite-build-dir', + 'vite-entry', + 'json', + ], + profile: PROFILE_OPTIONS, + 'diagnose-performance': PROFILE_OPTIONS, + 'verify-optimization': PROFILE_OPTIONS, + 'capture-flow': PROFILE_OPTIONS, +}; + export async function main(argv = process.argv.slice(2)) { let repositoryRoot = resolve(process.cwd()); try { @@ -48,57 +137,8 @@ export async function main(argv = process.argv.slice(2)) { writeJson(await qualifyPortfolioManifest(required(options, 'manifest'))); return 0; } - if (operation === 'plan-performance') { - const adapter = assertProfileAdapter(required(options, 'adapter')); - const samples = boundedCount(options.samples, { - name: 'samples', - defaultValue: LIMITS.defaultSamples, - minimum: LIMITS.minimumSamples, - maximum: LIMITS.maximumSamples, - }); - const warmups = boundedCount(options.warmups, { - name: 'warmups', - defaultValue: LIMITS.defaultWarmups, - maximum: LIMITS.maximumWarmups, - }); - const plan = await planPerformanceExecution({ - repositoryRoot, - adapter, - target: required(options, 'target'), - name: options.name, - timeoutMs: boundedTimeout(options['timeout-ms']), - processCount: plannedProfileProcessCount({ adapter, samples, warmups }), - approvalIdentity: options['approval-id'], - }); - writeJson(plan); - return plan.decision.status === 'admitted' ? 0 : 2; - } - if (operation === 'plan-flow-campaign') { - const result = await planFlowOptimizationCampaign({ - repositoryRoot, - priorityManifestPath: options['priority-manifest'], - maxFlows: boundedCount(options['max-flows'], { - name: 'max flows', - defaultValue: 3, - minimum: 1, - maximum: LIMITS.campaignFlows, - }), - samples: boundedCount(options.samples, { - name: 'samples', - defaultValue: LIMITS.defaultSamples, - minimum: LIMITS.minimumSamples, - maximum: LIMITS.maximumSamples, - }), - warmups: boundedCount(options.warmups, { - name: 'warmups', - defaultValue: LIMITS.defaultWarmups, - maximum: LIMITS.maximumWarmups, - }), - timeoutMs: boundedTimeout(options['timeout-ms']), - }); - writeJson(result); - return result.verdict.status === 'no_confidence' ? 2 : 0; - } + if (PLANNING_OPERATIONS.has(operation)) + return runPlanningOperation(operation, repositoryRoot, options); if (operation === 'supervise-performance') { const receipt = await supervisePerformanceRun({ repositoryRoot, @@ -274,24 +314,7 @@ export async function main(argv = process.argv.slice(2)) { function parseArguments(argv) { const [operation, ...rest] = argv; - if ( - ![ - 'detect', - 'run', - 'import', - 'profile', - 'diagnose-performance', - 'verify-optimization', - 'verify-paired-optimization', - 'capture-flow', - 'qualify', - 'qualify-portfolio', - 'plan-performance', - 'supervise-performance', - 'inspect-performance-run', - 'plan-flow-campaign', - ].includes(operation) - ) { + if (!OPERATIONS.includes(operation)) { throw new Error( 'usage: cli.mjs [--repo PATH] [operation options] [--json]' ); @@ -320,78 +343,63 @@ function parseArguments(argv) { options[key] = value; index += 1; } - const allowed = new Set( - operation === 'detect' || operation === 'qualify' - ? ['repo', 'json'] - : operation === 'qualify-portfolio' - ? ['manifest', 'json'] - : operation === 'plan-performance' - ? [ - 'repo', - 'adapter', - 'target', - 'name', - 'timeout-ms', - 'samples', - 'warmups', - 'approval-id', - 'json', - ] - : operation === 'plan-flow-campaign' - ? ['repo', 'priority-manifest', 'max-flows', 'timeout-ms', 'samples', 'warmups', 'json'] - : operation === 'inspect-performance-run' - ? ['repo', 'run-id', 'json'] - : operation === 'supervise-performance' - ? [ - 'repo', - 'run-id', - 'adapter', - 'target', - 'name', - 'timeout-ms', - 'samples', - 'warmups', - 'json', - ] - : operation === 'run' - ? ['repo', 'adapter', 'target', 'name', 'diff', 'timeout-ms', 'json'] - : operation === 'import' - ? ['repo', 'kind', 'receipt', 'diff', 'json'] - : operation === 'verify-paired-optimization' - ? [ - 'repo', - 'baseline-repo', - 'adapter', - 'target', - 'name', - 'timeout-ms', - 'samples', - 'warmups', - 'vite-build-dir', - 'vite-entry', - 'json', - ] - : [ - 'repo', - 'adapter', - 'target', - 'name', - 'timeout-ms', - 'samples', - 'warmups', - 'baseline', - 'regression-percent', - 'regression-ms', - 'vite-build-dir', - 'vite-entry', - 'json', - ] - ); + const allowed = new Set(ALLOWED_OPTIONS[operation]); const unknown = Object.keys(options).find((key) => !allowed.has(key)); if (unknown) throw new Error(`unknown option for ${operation}: --${unknown}`); return { operation, options }; } +async function runPlanningOperation(operation, repositoryRoot, options) { + if (operation === 'plan-performance') { + const adapter = assertProfileAdapter(required(options, 'adapter')); + const samples = profileSamples(options.samples); + const warmups = profileWarmups(options.warmups); + const plan = await planPerformanceExecution({ + repositoryRoot, + adapter, + target: required(options, 'target'), + name: options.name, + timeoutMs: boundedTimeout(options['timeout-ms']), + processCount: plannedProfileProcessCount({ adapter, samples, warmups }), + approvalIdentity: options['approval-id'], + }); + writeJson(plan); + return plan.decision.status === 'admitted' ? 0 : 2; + } + const result = await planFlowOptimizationCampaign({ + repositoryRoot, + priorityManifestPath: options['priority-manifest'], + maxFlows: boundedCount(options['max-flows'], { + name: 'max flows', + defaultValue: 3, + minimum: 1, + maximum: LIMITS.campaignFlows, + }), + samples: profileSamples(options.samples), + warmups: profileWarmups(options.warmups), + timeoutMs: boundedTimeout(options['timeout-ms']), + }); + writeJson(result); + return result.verdict.status === 'no_confidence' ? 2 : 0; +} + +function profileSamples(value) { + return boundedCount(value, { + name: 'samples', + defaultValue: LIMITS.defaultSamples, + minimum: LIMITS.minimumSamples, + maximum: LIMITS.maximumSamples, + }); +} + +function profileWarmups(value) { + return boundedCount(value, { + name: 'warmups', + defaultValue: LIMITS.defaultWarmups, + maximum: LIMITS.maximumWarmups, + }); +} + function required(options, key) { const value = options[key]; if (typeof value !== 'string' || value.length === 0) throw new Error(`missing --${key}`); diff --git a/scripts/runtime-failure-capsule/execution-governance.mjs b/scripts/runtime-failure-capsule/execution-governance.mjs index d83a2166..479412ff 100644 --- a/scripts/runtime-failure-capsule/execution-governance.mjs +++ b/scripts/runtime-failure-capsule/execution-governance.mjs @@ -89,56 +89,65 @@ export function createPerformanceExecutionReceipt(plan, executions = []) { const markers = executions.flatMap((entry) => blockedEgressMarkers(entry.execution?.stderr ?? '') ); - const ran = executions.length > 0; - const completed = executions.every( - (entry) => entry.execution?.status === 'exited' && entry.execution?.exitCode === 0 - ); - const status = - plan.decision.status === 'blocked' - ? 'blocked' - : markers.length > 0 - ? 'policy_violation' - : completed - ? 'completed' - : ran - ? 'failed' - : 'admitted'; + const outcome = summarizeReceiptOutcome(plan, executions, markers); const receipt = { schema_version: PERFORMANCE_EXECUTION_RECEIPT_SCHEMA_VERSION, plan_id: plan.plan_id, decision: plan.decision.status, - status, + status: outcome.status, planned: plan.limits, - observed: { - wall_time_ms: executions.reduce( - (total, entry) => total + Math.max(0, entry.execution?.durationMs ?? 0), - 0 - ), - processes: executions.length, - max_concurrency: executions.length > 0 ? 1 : 0, - retries: 0, - successful_external_requests: 0, - blocked_external_attempts: markers.length, - external_services: [], - cost_microusd: plan.decision.status === 'admitted' ? 0 : null, - }, + observed: summarizeReceiptObservations(plan, executions, markers), enforcement: plan.enforcement, - terminal_reason: - plan.decision.status === 'blocked' - ? plan.decision.blockers.join(' ') - : markers.length > 0 - ? 'The zero-egress boundary blocked a remote network attempt.' - : completed - ? 'The admitted local workload completed within the execution policy.' - : ran - ? 'The admitted local workload did not complete successfully.' - : 'The workload is admitted but has not executed.', + terminal_reason: outcome.reason, limitations: [...plan.limitations], }; assertPerformanceExecutionReceipt(receipt); return receipt; } +function summarizeReceiptOutcome(plan, executions, markers) { + if (plan.decision.status === 'blocked') { + return { status: 'blocked', reason: plan.decision.blockers.join(' ') }; + } + if (markers.length > 0) { + return { + status: 'policy_violation', + reason: 'The zero-egress boundary blocked a remote network attempt.', + }; + } + if (executions.length === 0) { + return { status: 'admitted', reason: 'The workload is admitted but has not executed.' }; + } + const completed = executions.every( + (entry) => entry.execution?.status === 'exited' && entry.execution?.exitCode === 0 + ); + return completed + ? { + status: 'completed', + reason: 'The admitted local workload completed within the execution policy.', + } + : { + status: 'failed', + reason: 'The admitted local workload did not complete successfully.', + }; +} + +function summarizeReceiptObservations(plan, executions, markers) { + return { + wall_time_ms: executions.reduce( + (total, entry) => total + Math.max(0, entry.execution?.durationMs ?? 0), + 0 + ), + processes: executions.length, + max_concurrency: executions.length > 0 ? 1 : 0, + retries: 0, + successful_external_requests: 0, + blocked_external_attempts: markers.length, + external_services: [], + cost_microusd: plan.decision.status === 'admitted' ? 0 : null, + }; +} + export async function assertPerformanceExecutionPlanCurrent({ plan, repositoryRoot, @@ -195,31 +204,8 @@ export function validatePerformanceExecutionPlan(value) { if (value.schema_version !== PERFORMANCE_EXECUTION_PLAN_SCHEMA_VERSION) errors.push('invalid schema_version'); if (!/^[0-9a-f]{64}$/.test(value.plan_id ?? '')) errors.push('plan_id is invalid'); - closed( - value.subject, - ['repository_revision', 'diff_identity', 'dirty', 'target_sha256'], - 'subject', - errors - ); - if ( - !plainObject(value.subject) || - !/^[0-9a-f]{40,64}$/.test(value.subject.repository_revision ?? '') || - typeof value.subject.diff_identity !== 'string' || - value.subject.diff_identity.length === 0 || - value.subject.diff_identity.length > 200 || - typeof value.subject.dirty !== 'boolean' || - !/^[0-9a-f]{64}$/.test(value.subject.target_sha256 ?? '') - ) - errors.push('subject is invalid'); - closed(value.scope, ['adapter', 'target', 'name'], 'scope', errors); - if ( - !plainObject(value.scope) || - !GOVERNED_ADAPTERS.includes(value.scope.adapter) || - typeof value.scope.target !== 'string' || - value.scope.target.length === 0 || - (value.scope.name !== null && typeof value.scope.name !== 'string') - ) - errors.push('scope is invalid'); + validatePlanSubject(value.subject, errors); + validatePlanScope(value.scope, errors); if (value.mode !== 'local_zero_egress') errors.push('mode is invalid'); validateLimits(value.limits, errors); if (!Array.isArray(value.external_services)) errors.push('external_services must be an array'); @@ -228,28 +214,8 @@ export function validatePerformanceExecutionPlan(value) { !/^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/.test(value.approval_identity) ) errors.push('approval_identity is invalid'); - if ( - !plainObject(value.enforcement) || - !['node_preload', 'macos_sandbox_node_preload', 'macos_sandbox', 'unavailable'].includes( - value.enforcement.kind - ) - ) - errors.push('enforcement is invalid'); - closed(value.enforcement, ['kind', 'network_scope', 'reason'], 'enforcement', errors); - if ( - plainObject(value.enforcement) && - !['none', 'loopback_only', 'unknown'].includes(value.enforcement.network_scope) - ) - errors.push('enforcement.network_scope is invalid'); - closed(value.decision, ['status', 'reason', 'blockers'], 'decision', errors); - if ( - !plainObject(value.decision) || - !['admitted', 'blocked'].includes(value.decision.status) || - typeof value.decision.reason !== 'string' || - value.decision.reason.length === 0 || - !stringArray(value.decision.blockers) - ) - errors.push('decision is invalid'); + validateEnforcement(value.enforcement, errors); + validatePlanDecision(value.decision, errors); if (!stringArray(value.external_services)) errors.push('external_services must be an array'); if (!stringArray(value.limitations)) errors.push('limitations must be an array'); const { plan_id: _planId, ...payload } = value; @@ -258,6 +224,55 @@ export function validatePerformanceExecutionPlan(value) { return errors; } +function validatePlanSubject(value, errors) { + closed( + value, + ['repository_revision', 'diff_identity', 'dirty', 'target_sha256'], + 'subject', + errors + ); + if (!plainObject(value)) return errors.push('subject is invalid'); + const valid = + /^[0-9a-f]{40,64}$/.test(value.repository_revision ?? '') && + typeof value.diff_identity === 'string' && + value.diff_identity.length > 0 && + value.diff_identity.length <= 200 && + typeof value.dirty === 'boolean' && + /^[0-9a-f]{64}$/.test(value.target_sha256 ?? ''); + if (!valid) errors.push('subject is invalid'); +} + +function validatePlanScope(value, errors) { + closed(value, ['adapter', 'target', 'name'], 'scope', errors); + if (!plainObject(value)) return errors.push('scope is invalid'); + const valid = + GOVERNED_ADAPTERS.includes(value.adapter) && + typeof value.target === 'string' && + value.target.length > 0 && + (value.name === null || typeof value.name === 'string'); + if (!valid) errors.push('scope is invalid'); +} + +function validateEnforcement(value, errors) { + closed(value, ['kind', 'network_scope', 'reason'], 'enforcement', errors); + if (!plainObject(value)) return errors.push('enforcement is invalid'); + const validKinds = ['node_preload', 'macos_sandbox_node_preload', 'macos_sandbox', 'unavailable']; + if (!validKinds.includes(value.kind)) errors.push('enforcement is invalid'); + if (!['none', 'loopback_only', 'unknown'].includes(value.network_scope)) + errors.push('enforcement.network_scope is invalid'); +} + +function validatePlanDecision(value, errors) { + closed(value, ['status', 'reason', 'blockers'], 'decision', errors); + if (!plainObject(value)) return errors.push('decision is invalid'); + const valid = + ['admitted', 'blocked'].includes(value.status) && + typeof value.reason === 'string' && + value.reason.length > 0 && + stringArray(value.blockers); + if (!valid) errors.push('decision is invalid'); +} + export function assertPerformanceExecutionReceipt(value) { const errors = validatePerformanceExecutionReceipt(value); if (errors.length > 0) @@ -291,55 +306,46 @@ export function validatePerformanceExecutionReceipt(value) { if (!['admitted', 'blocked', 'completed', 'failed', 'policy_violation'].includes(value.status)) errors.push('status is invalid'); validateLimits(value.planned, errors); - if (!plainObject(value.observed)) errors.push('observed is invalid'); - else { - closed( - value.observed, - [ - 'wall_time_ms', - 'processes', - 'max_concurrency', - 'retries', - 'successful_external_requests', - 'blocked_external_attempts', - 'external_services', - 'cost_microusd', - ], - 'observed', - errors - ); - for (const field of [ + validateReceiptObservations(value.observed, errors); + validateEnforcement(value.enforcement, errors); + if (typeof value.terminal_reason !== 'string' || value.terminal_reason.length === 0) + errors.push('terminal_reason is invalid'); + if (!stringArray(value.limitations)) errors.push('limitations must be an array'); + return errors; +} + +function validateReceiptObservations(value, errors) { + if (!plainObject(value)) return errors.push('observed is invalid'); + closed( + value, + [ 'wall_time_ms', 'processes', 'max_concurrency', 'retries', 'successful_external_requests', 'blocked_external_attempts', - ]) { - if (!Number.isInteger(value.observed[field]) || value.observed[field] < 0) - errors.push(`observed.${field} is invalid`); - } - if (!stringArray(value.observed.external_services)) - errors.push('observed.external_services is invalid'); - if ( - value.observed.cost_microusd !== null && - (!Number.isInteger(value.observed.cost_microusd) || value.observed.cost_microusd < 0) - ) - errors.push('observed.cost_microusd is invalid'); + 'external_services', + 'cost_microusd', + ], + 'observed', + errors + ); + for (const field of [ + 'wall_time_ms', + 'processes', + 'max_concurrency', + 'retries', + 'successful_external_requests', + 'blocked_external_attempts', + ]) { + if (!Number.isInteger(value[field]) || value[field] < 0) + errors.push(`observed.${field} is invalid`); } - closed(value.enforcement, ['kind', 'network_scope', 'reason'], 'enforcement', errors); - if ( - !plainObject(value.enforcement) || - !['node_preload', 'macos_sandbox_node_preload', 'macos_sandbox', 'unavailable'].includes( - value.enforcement.kind - ) || - !['none', 'loopback_only', 'unknown'].includes(value.enforcement.network_scope) - ) - errors.push('enforcement is invalid'); - if (typeof value.terminal_reason !== 'string' || value.terminal_reason.length === 0) - errors.push('terminal_reason is invalid'); - if (!stringArray(value.limitations)) errors.push('limitations must be an array'); - return errors; + if (!stringArray(value.external_services)) errors.push('observed.external_services is invalid'); + if (value.cost_microusd === null) return; + if (!Number.isInteger(value.cost_microusd) || value.cost_microusd < 0) + errors.push('observed.cost_microusd is invalid'); } export function blockedEgressMarkers(stderr) { diff --git a/scripts/runtime-failure-capsule/mcp.mjs b/scripts/runtime-failure-capsule/mcp.mjs index 6847ec50..1f4ae0e8 100644 --- a/scripts/runtime-failure-capsule/mcp.mjs +++ b/scripts/runtime-failure-capsule/mcp.mjs @@ -22,7 +22,11 @@ import { verifyPairedRepositories } from './paired-verification.mjs'; const PROTOCOL_VERSION = '2025-03-26'; const SERVER_INFO = { name: 'codevetter-local-runtime', version: '0.1.0' }; -const PERFORMANCE_TOOLS = new Set(['profile_local_performance', 'verify_paired_performance']); +const PERFORMANCE_TOOLS = new Set([ + 'plan_local_performance', + 'profile_local_performance', + 'verify_paired_performance', +]); export async function createRuntimeMcpHandler(repositoryRoot, options = {}) { const flowService = options.flowService ?? (await createLocalFlowService(repositoryRoot)); @@ -113,27 +117,6 @@ async function callTool( closedArguments(args, ['run_id']); return inspectSupervisedRun(repositoryRoot, args.run_id); } - if (name === 'plan_local_performance') { - closedArguments( - args, - ['adapter', 'target'], - ['name', 'approval_identity'], - [], - ['samples', 'warmups', 'timeout_ms'] - ); - const adapter = assertProfileAdapter(args.adapter); - const samples = boundedSamples(args.samples); - const warmups = boundedWarmups(args.warmups); - return planPerformanceExecution({ - repositoryRoot, - adapter, - target: args.target, - name: args.name, - timeoutMs: boundedTimeout(args.timeout_ms), - processCount: plannedProfileProcessCount({ adapter, samples, warmups }), - approvalIdentity: args.approval_identity, - }); - } if (PERFORMANCE_TOOLS.has(name)) { return callPerformanceTool({ name, @@ -210,6 +193,9 @@ async function callTool( } async function callPerformanceTool({ name, args, repositoryRoot, incumbentRepositoryRoot }) { + if (name === 'plan_local_performance') { + return callPerformancePlan(args, repositoryRoot); + } closedArguments( args, ['adapter', 'target'], @@ -238,6 +224,28 @@ async function callPerformanceTool({ name, args, repositoryRoot, incumbentReposi return profileRepository(input); } +function callPerformancePlan(args, repositoryRoot) { + closedArguments( + args, + ['adapter', 'target'], + ['name', 'approval_identity'], + [], + ['samples', 'warmups', 'timeout_ms'] + ); + const adapter = assertProfileAdapter(args.adapter); + const samples = boundedSamples(args.samples); + const warmups = boundedWarmups(args.warmups); + return planPerformanceExecution({ + repositoryRoot, + adapter, + target: args.target, + name: args.name, + timeoutMs: boundedTimeout(args.timeout_ms), + processCount: plannedProfileProcessCount({ adapter, samples, warmups }), + approvalIdentity: args.approval_identity, + }); +} + function closedArguments( value, required, diff --git a/scripts/runtime-failure-capsule/performance.mjs b/scripts/runtime-failure-capsule/performance.mjs index 0f1eba23..77b77501 100644 --- a/scripts/runtime-failure-capsule/performance.mjs +++ b/scripts/runtime-failure-capsule/performance.mjs @@ -339,9 +339,7 @@ export function createPerformanceCapsule({ if (cleanupFailed) limitations.push('Owned temporary profiling artifacts could not be completely removed.'); if (outputTruncated) limitations.push('Runner output was truncated before normalization.'); - if (executionReceipt?.status === 'policy_violation') { - limitations.push('The zero-egress boundary blocked a remote network attempt.'); - } + limitations.push(...executionGovernanceLimitations(executionReceipt)); limitations.push(...playwrightLimitations(adapter, playwrightTest)); if (adapter === 'go-bench' && goBenchmarks.length === 0) { limitations.push('No matching Go benchmark measurement was captured.'); @@ -586,8 +584,7 @@ export function createPerformanceCapsule({ coverage_bytes: functionCoverage.coverage_bytes, coverage_functions: functionCoverage.functions.length, }, - execution_governance: - executionPlan && executionReceipt ? { plan: executionPlan, receipt: executionReceipt } : null, + execution_governance: executionGovernance(executionPlan, executionReceipt), verdict: { status: verdict, reason: @@ -603,6 +600,17 @@ export function createPerformanceCapsule({ return capsule; } +function executionGovernance(plan, receipt) { + if (!plan || !receipt) return null; + return { plan, receipt }; +} + +function executionGovernanceLimitations(receipt) { + return receipt?.status === 'policy_violation' + ? ['The zero-egress boundary blocked a remote network attempt.'] + : []; +} + function createBlockedPerformanceCapsule({ root, lexicalRoot,