From fc68a9d0c9ae6ac483458cb2af61d6a7bd87320e Mon Sep 17 00:00:00 2001 From: Nate Summers Date: Sat, 11 Jul 2026 14:03:21 -0700 Subject: [PATCH 01/12] feat(schemas): add test_plan artifact contract and plan-review gate Add the test_plan artifact type to the loop manifest schemas and the artifact_type Postgres enum, require a plan-review approval gate in the default development-loop manifest, and change the agent_plans agent_name default to planner. Seed data gains a demo test_plan artifact. Part of #47. Co-Authored-By: Claude Fable 5 --- docs/loop-manifest.md | 13 +- drizzle/0003_dusty_nico_minoru.sql | 2 + drizzle/meta/0003_snapshot.json | 2258 ++++++++++++++++++++++++++++ drizzle/meta/_journal.json | 7 + schemas/loop-manifest.schema.json | 1 + schemas/loop-manifest.ts | 1 + src/db/schema.ts | 3 +- src/lib/loops/manifest.ts | 14 + src/lib/seed/demo-data.ts | 11 +- tests/unit/loops/manifest.test.ts | 12 +- 10 files changed, 2316 insertions(+), 6 deletions(-) create mode 100644 drizzle/0003_dusty_nico_minoru.sql create mode 100644 drizzle/meta/0003_snapshot.json diff --git a/docs/loop-manifest.md b/docs/loop-manifest.md index 0e9f0f8..ce6eec4 100644 --- a/docs/loop-manifest.md +++ b/docs/loop-manifest.md @@ -76,7 +76,8 @@ planning, test-writing, development, validation, code review, commit, PR, and done. Each stage has a required visible artifact contract: 1. Planning: plan artifact. -2. Test-writing: red test evidence. +2. Test-writing: red test evidence and an automated test plan with explicit + fixtures and a bounded test-only patch. 3. Development: patch artifact. 4. Validation: validation report. 5. Code review: code review notes. @@ -90,6 +91,16 @@ skipped/no-op reason such as `loop_disabled` so operators can explain why an `agent-ready` issue did not start. Research-loop disabled evidence is tracked separately from the development-loop skeleton. +The planning-to-test-writing boundary requires an `approved` `plan-review` +record tied to the exact run, plan row, and canonical plan digest. The +test-writing stage succeeds only when every approved-plan acceptance criterion +has expected assertion-failure evidence. Its `validation_report` row carries +`loopworks.red_test_evidence.v1`; its `test_plan` row carries +`loopworks.test_plan.v1`. Both remain separate from the later green validation +report. Expected-red entries include a verified execution receipt bound to the +persisted test patch; setup, infrastructure, timeout, crash, unrelated, or +passing outcomes cannot advance the stage. + ## Validation Report Artifact The `validation_report` artifact uses schema diff --git a/drizzle/0003_dusty_nico_minoru.sql b/drizzle/0003_dusty_nico_minoru.sql new file mode 100644 index 0000000..4dbaded --- /dev/null +++ b/drizzle/0003_dusty_nico_minoru.sql @@ -0,0 +1,2 @@ +ALTER TYPE "public"."artifact_type" ADD VALUE 'test_plan' BEFORE 'patch';--> statement-breakpoint +ALTER TABLE "agent_plans" ALTER COLUMN "agent_name" SET DEFAULT 'planner'; diff --git a/drizzle/meta/0003_snapshot.json b/drizzle/meta/0003_snapshot.json new file mode 100644 index 0000000..f3f81e7 --- /dev/null +++ b/drizzle/meta/0003_snapshot.json @@ -0,0 +1,2258 @@ +{ + "id": "1ab74bee-a441-4310-bb27-05726691115c", + "prevId": "435e50c8-43e9-4d38-8bb5-563bd0a960f6", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.accounts": { + "name": "accounts", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_account_id": { + "name": "provider_account_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "token_type": { + "name": "token_type", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "session_state": { + "name": "session_state", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "accounts_user_id_users_id_fk": { + "name": "accounts_user_id_users_id_fk", + "tableFrom": "accounts", + "tableTo": "users", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "accounts_provider_provider_account_id_pk": { + "name": "accounts_provider_provider_account_id_pk", + "columns": ["provider", "provider_account_id"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.agent_plans": { + "name": "agent_plans", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "loop_id": { + "name": "loop_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "run_id": { + "name": "run_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "issue_number": { + "name": "issue_number", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "agent_name": { + "name": "agent_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'planner'" + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "input": { + "name": "input", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "plan": { + "name": "plan", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "agent_plans_loop_id_created_at_idx": { + "name": "agent_plans_loop_id_created_at_idx", + "columns": [ + { + "expression": "loop_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "agent_plans_run_id_created_at_idx": { + "name": "agent_plans_run_id_created_at_idx", + "columns": [ + { + "expression": "run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "agent_plans_loop_id_loops_id_fk": { + "name": "agent_plans_loop_id_loops_id_fk", + "tableFrom": "agent_plans", + "tableTo": "loops", + "columnsFrom": ["loop_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "agent_plans_run_id_loop_runs_id_fk": { + "name": "agent_plans_run_id_loop_runs_id_fk", + "tableFrom": "agent_plans", + "tableTo": "loop_runs", + "columnsFrom": ["run_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.approval_transition_events": { + "name": "approval_transition_events", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "approval_id": { + "name": "approval_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "run_id": { + "name": "run_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "from_status": { + "name": "from_status", + "type": "approval_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "to_status": { + "name": "to_status", + "type": "approval_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "action": { + "name": "action", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "occurred_at": { + "name": "occurred_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "note": { + "name": "note", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "approval_transition_events_approval_created_at_idx": { + "name": "approval_transition_events_approval_created_at_idx", + "columns": [ + { + "expression": "approval_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "occurred_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "approval_transition_events_run_created_at_idx": { + "name": "approval_transition_events_run_created_at_idx", + "columns": [ + { + "expression": "run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "occurred_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "approval_transition_events_approval_id_approvals_id_fk": { + "name": "approval_transition_events_approval_id_approvals_id_fk", + "tableFrom": "approval_transition_events", + "tableTo": "approvals", + "columnsFrom": ["approval_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "approval_transition_events_run_id_loop_runs_id_fk": { + "name": "approval_transition_events_run_id_loop_runs_id_fk", + "tableFrom": "approval_transition_events", + "tableTo": "loop_runs", + "columnsFrom": ["run_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.approvals": { + "name": "approvals", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "loop_id": { + "name": "loop_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "run_id": { + "name": "run_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "approval_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'requested'" + }, + "requested_by": { + "name": "requested_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "requested_at": { + "name": "requested_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "resolved_by": { + "name": "resolved_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "resolved_at": { + "name": "resolved_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "note": { + "name": "note", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "approvals_loop_id_status_idx": { + "name": "approvals_loop_id_status_idx", + "columns": [ + { + "expression": "loop_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "approvals_run_id_status_idx": { + "name": "approvals_run_id_status_idx", + "columns": [ + { + "expression": "run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "approvals_loop_id_loops_id_fk": { + "name": "approvals_loop_id_loops_id_fk", + "tableFrom": "approvals", + "tableTo": "loops", + "columnsFrom": ["loop_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "approvals_run_id_loop_runs_id_fk": { + "name": "approvals_run_id_loop_runs_id_fk", + "tableFrom": "approvals", + "tableTo": "loop_runs", + "columnsFrom": ["run_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.artifacts": { + "name": "artifacts", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "run_id": { + "name": "run_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "step_id": { + "name": "step_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "type": { + "name": "type", + "type": "artifact_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'other'" + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "uri": { + "name": "uri", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "sha256": { + "name": "sha256", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "artifacts_run_type_idx": { + "name": "artifacts_run_type_idx", + "columns": [ + { + "expression": "run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "artifacts_step_id_idx": { + "name": "artifacts_step_id_idx", + "columns": [ + { + "expression": "step_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "artifacts_run_id_loop_runs_id_fk": { + "name": "artifacts_run_id_loop_runs_id_fk", + "tableFrom": "artifacts", + "tableTo": "loop_runs", + "columnsFrom": ["run_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "artifacts_step_id_run_steps_id_fk": { + "name": "artifacts_step_id_run_steps_id_fk", + "tableFrom": "artifacts", + "tableTo": "run_steps", + "columnsFrom": ["step_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.deployments": { + "name": "deployments", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "repository_id": { + "name": "repository_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "loop_id": { + "name": "loop_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "run_id": { + "name": "run_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'vercel'" + }, + "external_id": { + "name": "external_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "project_name": { + "name": "project_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "deployment_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "environment": { + "name": "environment", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "branch": { + "name": "branch", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "commit_sha": { + "name": "commit_sha", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "inspector_url": { + "name": "inspector_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "alias_urls": { + "name": "alias_urls", + "type": "text[]", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "ready_at": { + "name": "ready_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "deployments_project_id_created_at_idx": { + "name": "deployments_project_id_created_at_idx", + "columns": [ + { + "expression": "project_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "deployments_run_id_created_at_idx": { + "name": "deployments_run_id_created_at_idx", + "columns": [ + { + "expression": "run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "deployments_repository_id_repositories_id_fk": { + "name": "deployments_repository_id_repositories_id_fk", + "tableFrom": "deployments", + "tableTo": "repositories", + "columnsFrom": ["repository_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "deployments_loop_id_loops_id_fk": { + "name": "deployments_loop_id_loops_id_fk", + "tableFrom": "deployments", + "tableTo": "loops", + "columnsFrom": ["loop_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "deployments_run_id_loop_runs_id_fk": { + "name": "deployments_run_id_loop_runs_id_fk", + "tableFrom": "deployments", + "tableTo": "loop_runs", + "columnsFrom": ["run_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "deployments_external_id_unique": { + "name": "deployments_external_id_unique", + "nullsNotDistinct": false, + "columns": ["external_id"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.idempotency_locks": { + "name": "idempotency_locks", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "idempotency_lock_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'acquired'" + }, + "acquired_at": { + "name": "acquired_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "released_at": { + "name": "released_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "idempotency_locks_scope_status_idx": { + "name": "idempotency_locks_scope_status_idx", + "columns": [ + { + "expression": "scope", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idempotency_locks_expires_at_idx": { + "name": "idempotency_locks_expires_at_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "idempotency_locks_key_unique": { + "name": "idempotency_locks_key_unique", + "nullsNotDistinct": false, + "columns": ["key"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.loop_events": { + "name": "loop_events", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "loop_id": { + "name": "loop_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "actor_type": { + "name": "actor_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "event_type": { + "name": "event_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "from_state": { + "name": "from_state", + "type": "loop_state", + "typeSchema": "public", + "primaryKey": false, + "notNull": false + }, + "to_state": { + "name": "to_state", + "type": "loop_state", + "typeSchema": "public", + "primaryKey": false, + "notNull": false + }, + "reason": { + "name": "reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "loop_events_loop_id_created_at_idx": { + "name": "loop_events_loop_id_created_at_idx", + "columns": [ + { + "expression": "loop_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "loop_events_loop_id_loops_id_fk": { + "name": "loop_events_loop_id_loops_id_fk", + "tableFrom": "loop_events", + "tableTo": "loops", + "columnsFrom": ["loop_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.loop_runs": { + "name": "loop_runs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "repository_id": { + "name": "repository_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "loop_key": { + "name": "loop_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "github_issue_number": { + "name": "github_issue_number", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "github_issue_url": { + "name": "github_issue_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "run_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'queued'" + }, + "current_stage": { + "name": "current_stage", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'planning'" + }, + "trace_id": { + "name": "trace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "cost_cents": { + "name": "cost_cents", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "queued_at": { + "name": "queued_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "started_at": { + "name": "started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "canceled_at": { + "name": "canceled_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "loop_runs_repository_status_idx": { + "name": "loop_runs_repository_status_idx", + "columns": [ + { + "expression": "repository_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "loop_runs_repository_issue_idx": { + "name": "loop_runs_repository_issue_idx", + "columns": [ + { + "expression": "repository_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "github_issue_number", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "loop_runs_repository_id_repositories_id_fk": { + "name": "loop_runs_repository_id_repositories_id_fk", + "tableFrom": "loop_runs", + "tableTo": "repositories", + "columnsFrom": ["repository_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.loops": { + "name": "loops", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "repository_id": { + "name": "repository_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "github_issue_number": { + "name": "github_issue_number", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "loop_state", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'intake'" + }, + "milestone": { + "name": "milestone", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "area_label": { + "name": "area_label", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "priority_label": { + "name": "priority_label", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "owner_github_login": { + "name": "owner_github_login", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "source_url": { + "name": "source_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_synced_at": { + "name": "last_synced_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "loops_repository_issue_number_idx": { + "name": "loops_repository_issue_number_idx", + "columns": [ + { + "expression": "repository_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "github_issue_number", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "loops_repository_id_repositories_id_fk": { + "name": "loops_repository_id_repositories_id_fk", + "tableFrom": "loops", + "tableTo": "repositories", + "columnsFrom": ["repository_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.observability_events": { + "name": "observability_events", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "repository_id": { + "name": "repository_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "run_id": { + "name": "run_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "step_id": { + "name": "step_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "event_type": { + "name": "event_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "severity": { + "name": "severity", + "type": "observability_severity", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'info'" + }, + "correlation_id": { + "name": "correlation_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "trace_id": { + "name": "trace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "metric_name": { + "name": "metric_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "metric_value": { + "name": "metric_value", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "observability_events_type_created_at_idx": { + "name": "observability_events_type_created_at_idx", + "columns": [ + { + "expression": "event_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "observability_events_run_created_at_idx": { + "name": "observability_events_run_created_at_idx", + "columns": [ + { + "expression": "run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "observability_events_trace_id_idx": { + "name": "observability_events_trace_id_idx", + "columns": [ + { + "expression": "trace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "observability_events_repository_id_repositories_id_fk": { + "name": "observability_events_repository_id_repositories_id_fk", + "tableFrom": "observability_events", + "tableTo": "repositories", + "columnsFrom": ["repository_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "observability_events_run_id_loop_runs_id_fk": { + "name": "observability_events_run_id_loop_runs_id_fk", + "tableFrom": "observability_events", + "tableTo": "loop_runs", + "columnsFrom": ["run_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "observability_events_step_id_run_steps_id_fk": { + "name": "observability_events_step_id_run_steps_id_fk", + "tableFrom": "observability_events", + "tableTo": "run_steps", + "columnsFrom": ["step_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.repositories": { + "name": "repositories", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "github_repo_id": { + "name": "github_repo_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "full_name": { + "name": "full_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "installation_id": { + "name": "installation_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "is_active": { + "name": "is_active", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "health": { + "name": "health", + "type": "repo_health", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'healthy'" + }, + "framework": { + "name": "framework", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'Unknown'" + }, + "default_branch": { + "name": "default_branch", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'main'" + }, + "ci_commands": { + "name": "ci_commands", + "type": "text[]", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "docs_href": { + "name": "docs_href", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "observability_href": { + "name": "observability_href", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "design_system_href": { + "name": "design_system_href", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "enabled_loops": { + "name": "enabled_loops", + "type": "text[]", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "validation_gates": { + "name": "validation_gates", + "type": "text[]", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "last_synced_at": { + "name": "last_synced_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "repositories_github_repo_id_unique": { + "name": "repositories_github_repo_id_unique", + "nullsNotDistinct": false, + "columns": ["github_repo_id"] + }, + "repositories_full_name_unique": { + "name": "repositories_full_name_unique", + "nullsNotDistinct": false, + "columns": ["full_name"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.run_steps": { + "name": "run_steps", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "run_id": { + "name": "run_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "stage": { + "name": "stage", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "run_step_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'queued'" + }, + "attempt": { + "name": "attempt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "actor_type": { + "name": "actor_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "summary": { + "name": "summary", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "validation_command": { + "name": "validation_command", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "validation_status": { + "name": "validation_status", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "trace_id": { + "name": "trace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "queued_at": { + "name": "queued_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "started_at": { + "name": "started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "run_steps_run_stage_idx": { + "name": "run_steps_run_stage_idx", + "columns": [ + { + "expression": "run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "stage", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "run_steps_run_status_idx": { + "name": "run_steps_run_status_idx", + "columns": [ + { + "expression": "run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "run_steps_run_id_loop_runs_id_fk": { + "name": "run_steps_run_id_loop_runs_id_fk", + "tableFrom": "run_steps", + "tableTo": "loop_runs", + "columnsFrom": ["run_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sessions": { + "name": "sessions", + "schema": "", + "columns": { + "session_token": { + "name": "session_token", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "expires": { + "name": "expires", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "sessions_user_id_users_id_fk": { + "name": "sessions_user_id_users_id_fk", + "tableFrom": "sessions", + "tableTo": "users", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.users": { + "name": "users", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "email_verified": { + "name": "email_verified", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "github_login": { + "name": "github_login", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "users_email_unique": { + "name": "users_email_unique", + "nullsNotDistinct": false, + "columns": ["email"] + }, + "users_github_login_unique": { + "name": "users_github_login_unique", + "nullsNotDistinct": false, + "columns": ["github_login"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.vercel_projects": { + "name": "vercel_projects", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "repository_id": { + "name": "repository_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "project_name": { + "name": "project_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "team_id": { + "name": "team_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "team_slug": { + "name": "team_slug", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "production_url": { + "name": "production_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "dashboard_url": { + "name": "dashboard_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "vercel_projects_repository_id_idx": { + "name": "vercel_projects_repository_id_idx", + "columns": [ + { + "expression": "repository_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "vercel_projects_repository_id_repositories_id_fk": { + "name": "vercel_projects_repository_id_repositories_id_fk", + "tableFrom": "vercel_projects", + "tableTo": "repositories", + "columnsFrom": ["repository_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "vercel_projects_project_id_unique": { + "name": "vercel_projects_project_id_unique", + "nullsNotDistinct": false, + "columns": ["project_id"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.verification_tokens": { + "name": "verification_tokens", + "schema": "", + "columns": { + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires": { + "name": "expires", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "verification_tokens_identifier_token_pk": { + "name": "verification_tokens_identifier_token_pk", + "columns": ["identifier", "token"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.webhook_deliveries": { + "name": "webhook_deliveries", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'github'" + }, + "delivery_id": { + "name": "delivery_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "event": { + "name": "event", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "action": { + "name": "action", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "repository_full_name": { + "name": "repository_full_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "webhook_delivery_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'received'" + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "received_at": { + "name": "received_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "processed_at": { + "name": "processed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "webhook_deliveries_delivery_id_unique": { + "name": "webhook_deliveries_delivery_id_unique", + "nullsNotDistinct": false, + "columns": ["delivery_id"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": { + "public.approval_status": { + "name": "approval_status", + "schema": "public", + "values": ["requested", "approved", "rejected", "cancelled", "expired", "applied", "bypassed"] + }, + "public.artifact_type": { + "name": "artifact_type", + "schema": "public", + "values": [ + "plan", + "validation_report", + "test_plan", + "patch", + "pr_intent", + "deployment_summary", + "log_summary", + "trace", + "other" + ] + }, + "public.deployment_status": { + "name": "deployment_status", + "schema": "public", + "values": ["queued", "building", "ready", "error", "canceled"] + }, + "public.idempotency_lock_status": { + "name": "idempotency_lock_status", + "schema": "public", + "values": ["acquired", "released", "expired"] + }, + "public.loop_state": { + "name": "loop_state", + "schema": "public", + "values": [ + "intake", + "triage", + "planned", + "in_progress", + "waiting_on_review", + "validating", + "blocked", + "done" + ] + }, + "public.observability_severity": { + "name": "observability_severity", + "schema": "public", + "values": ["debug", "info", "warn", "error"] + }, + "public.repo_health": { + "name": "repo_health", + "schema": "public", + "values": ["healthy", "watch", "blocked", "disconnected"] + }, + "public.run_status": { + "name": "run_status", + "schema": "public", + "values": [ + "queued", + "running", + "waiting_for_approval", + "blocked", + "failed", + "succeeded", + "canceled" + ] + }, + "public.run_step_status": { + "name": "run_step_status", + "schema": "public", + "values": ["queued", "running", "skipped", "failed", "succeeded"] + }, + "public.webhook_delivery_status": { + "name": "webhook_delivery_status", + "schema": "public", + "values": ["received", "processed", "ignored", "failed"] + } + }, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} diff --git a/drizzle/meta/_journal.json b/drizzle/meta/_journal.json index 2a40c3f..73c753b 100644 --- a/drizzle/meta/_journal.json +++ b/drizzle/meta/_journal.json @@ -22,6 +22,13 @@ "when": 1783028394100, "tag": "0002_absurd_proteus", "breakpoints": true + }, + { + "idx": 3, + "version": "7", + "when": 1783795849940, + "tag": "0003_dusty_nico_minoru", + "breakpoints": true } ] } diff --git a/schemas/loop-manifest.schema.json b/schemas/loop-manifest.schema.json index 9662300..bfef426 100644 --- a/schemas/loop-manifest.schema.json +++ b/schemas/loop-manifest.schema.json @@ -424,6 +424,7 @@ "enum": [ "plan", "validation_report", + "test_plan", "patch", "pr_intent", "summary", diff --git a/schemas/loop-manifest.ts b/schemas/loop-manifest.ts index 0e52b04..bd6501d 100644 --- a/schemas/loop-manifest.ts +++ b/schemas/loop-manifest.ts @@ -75,6 +75,7 @@ export const approvalBypassPolicyValues = ["none", "maintainer_override"] as con export const artifactContractTypeValues = [ "plan", "validation_report", + "test_plan", "patch", "pr_intent", "summary", diff --git a/src/db/schema.ts b/src/db/schema.ts index d437b8d..f2ed60d 100644 --- a/src/db/schema.ts +++ b/src/db/schema.ts @@ -56,6 +56,7 @@ export const runStepStatusEnum = pgEnum("run_step_status", [ export const artifactTypeEnum = pgEnum("artifact_type", [ "plan", "validation_report", + "test_plan", "patch", "pr_intent", "deployment_summary", @@ -460,7 +461,7 @@ export const agentPlans = pgTable( loopId: uuid("loop_id").references(() => loops.id, { onDelete: "set null" }), runId: uuid("run_id").references(() => loopRuns.id, { onDelete: "set null" }), issueNumber: integer("issue_number"), - agentName: text("agent_name").default("planning-agent").notNull(), + agentName: text("agent_name").default("planner").notNull(), status: text("status").default("pending").notNull(), input: jsonb("input").$type>().notNull(), plan: jsonb("plan").$type>(), diff --git a/src/lib/loops/manifest.ts b/src/lib/loops/manifest.ts index 025b840..1152365 100644 --- a/src/lib/loops/manifest.ts +++ b/src/lib/loops/manifest.ts @@ -64,6 +64,13 @@ export const defaultLoopManifest: LoopManifest = loopManifestSchema.parse({ requiredFor: ["external_write", "pr_creation", "manifest_rollout"], bypassPolicy: "none", gates: [ + { + key: "plan-review", + name: "Plan review", + required: true, + reviewers: ["maintainer"], + evidence: ["plan"], + }, { key: "external-write-review", name: "External write review", @@ -87,6 +94,13 @@ export const defaultLoopManifest: LoopManifest = loopManifestSchema.parse({ description: "Deterministic command evidence collected before review or rollout.", retention: "audit", }, + { + type: "test_plan", + required: true, + description: + "AC-mapped automated test plan with explicit fixtures and a bounded test-only patch.", + retention: "audit", + }, { type: "diff_summary", required: true, diff --git a/src/lib/seed/demo-data.ts b/src/lib/seed/demo-data.ts index ce29bec..9066126 100644 --- a/src/lib/seed/demo-data.ts +++ b/src/lib/seed/demo-data.ts @@ -2,8 +2,8 @@ import { inArray } from "drizzle-orm"; import type { db } from "@/db/client"; import { - approvalTransitionEvents, approvals, + approvalTransitionEvents, artifacts, deployments, loopRuns, @@ -158,6 +158,7 @@ export const demoSeedIds = { trace: seedId("artifacts", 6), other: seedId("artifacts", 7), validationReportSucceeded: seedId("artifacts", 8), + testPlan: seedId("artifacts", 9), }, approvals: { requested: seedId("approvals", 0), @@ -585,6 +586,14 @@ export function buildDemoSeedData(): DemoSeedData { uri: "https://github.com/ncolesummers/delivery-ops/actions/runs/demo-validation-report", metadata: demoValidationReportMetadata, }, + { + id: ids.artifacts.testPlan, + runId: ids.loopRuns.running, + stepId: ids.runSteps.running, + type: "test_plan", + title: "Automated test plan", + uri: "artifact://demo/test-plan", + }, { id: ids.artifacts.validationReportSucceeded, runId: ids.loopRuns.succeeded, diff --git a/tests/unit/loops/manifest.test.ts b/tests/unit/loops/manifest.test.ts index 940ac60..5965703 100644 --- a/tests/unit/loops/manifest.test.ts +++ b/tests/unit/loops/manifest.test.ts @@ -1,8 +1,7 @@ +import { defaultLoopManifest, parseLoopManifest, validateLoopManifest } from "@/lib/loops/manifest"; import { loopManifestSchema, retryableStatusValues } from "../../../schemas/loop-manifest"; import loopManifestJsonSchema from "../../../schemas/loop-manifest.schema.json"; -import { defaultLoopManifest, parseLoopManifest, validateLoopManifest } from "@/lib/loops/manifest"; - type JsonSchemaObject = { required?: string[]; properties?: Record; @@ -71,6 +70,10 @@ describe("loop manifest schema", () => { ); expect(developmentLoop.approvals.gates).toEqual( expect.arrayContaining([ + expect.objectContaining({ + key: "plan-review", + required: true, + }), expect.objectContaining({ key: "external-write-review", required: true, @@ -78,7 +81,10 @@ describe("loop manifest schema", () => { ]), ); expect(developmentLoop.artifacts).toEqual( - expect.arrayContaining([expect.objectContaining({ type: "pr_intent", required: true })]), + expect.arrayContaining([ + expect.objectContaining({ type: "test_plan", required: true }), + expect.objectContaining({ type: "pr_intent", required: true }), + ]), ); expect(developmentLoop.retryPolicy).toMatchObject({ maxAttempts: 2, From 2ed7df51e2572c2e6de9ded30a8b2ad6333248f8 Mon Sep 17 00:00:00 2001 From: Nate Summers Date: Sat, 11 Jul 2026 14:03:36 -0700 Subject: [PATCH 02/12] feat(agent): add bounded repository inspection library Shared repository-scoped discovery, text search, and line-range read helpers for stage subagents: commit-pinned provenance, secret/generated/ dependency path exclusions, symlink-escape rejection, and fail-closed query and output bounds. Part of #47. Co-Authored-By: Claude Fable 5 --- agent/lib/repository-inspection-runtime.ts | 191 +++++++++++++++++ agent/lib/repository-inspection.ts | 201 ++++++++++++++++++ .../unit/agent/repository-inspection.test.ts | 156 ++++++++++++++ 3 files changed, 548 insertions(+) create mode 100644 agent/lib/repository-inspection-runtime.ts create mode 100644 agent/lib/repository-inspection.ts create mode 100644 tests/unit/agent/repository-inspection.test.ts diff --git a/agent/lib/repository-inspection-runtime.ts b/agent/lib/repository-inspection-runtime.ts new file mode 100644 index 0000000..9b67b2a --- /dev/null +++ b/agent/lib/repository-inspection-runtime.ts @@ -0,0 +1,191 @@ +import { z } from "zod"; + +import { + assertPinnedRepositoryCommit, + assertSafeRepositoryGlob, + assertSafeRepositoryPath, + assertSafeRepositorySearch, + buildGitGrepCommand, + buildGitListCommand, + buildGitReadCommand, + buildGitTreeEntryCommand, + isSafeNonSymlinkTreeEntry, + parseSafeGitTreePaths, + parseSafeRepositorySearchLines, + redactRepositoryInspectionOutput, + truncateRepositoryInspectionOutput, +} from "./repository-inspection"; + +type RunResult = { exitCode: number; stdout: string }; +export type RepositorySandbox = { + readTextFile(input: { path: string }): PromiseLike; + run(input: { command: string; abortSignal?: AbortSignal }): PromiseLike; +}; + +const commitSha = z.string().regex(/^[a-f0-9]{40}$/); +const fileResult = z.object({ + path: z.string(), + startLine: z.number().int().positive(), + requestedEndLine: z.number().int().positive(), + returnedEndLine: z.number().int().nonnegative(), + content: z.string(), + truncated: z.boolean(), +}); + +export const repositoryListOutputSchema = z.object({ + commitSha, + fixtureMode: z.boolean(), + paths: z.array(z.string()), + truncated: z.boolean(), +}); +export const repositorySearchOutputSchema = z.object({ + commitSha, + fixtureMode: z.boolean(), + content: z.string(), + matchCount: z.number().int().nonnegative(), + truncated: z.boolean(), +}); +export const repositoryReadOutputSchema = z.object({ + commitSha, + fixtureMode: z.boolean(), + files: z.array(fileResult), + truncated: z.boolean(), +}); + +export async function readPinnedRepositoryCommit(sandbox: RepositorySandbox): Promise { + return assertPinnedRepositoryCommit( + await sandbox.readTextFile({ path: ".loopworks/repository-commit" }), + ); +} + +export async function listRepositoryFiles(sandbox: RepositorySandbox, patterns: readonly string[]) { + patterns.forEach(assertSafeRepositoryGlob); + const pinned = await readPinnedRepositoryCommit(sandbox); + const listing = await sandbox.run({ + command: buildGitListCommand(pinned, patterns), + abortSignal: AbortSignal.timeout(10_000), + }); + if (listing.exitCode !== 0) throw new Error("Repository listing failed."); + const rawLines = listing.stdout.split(/\r?\n/).filter(Boolean); + const paths = parseSafeGitTreePaths(rawLines); + return { + commitSha: pinned, + fixtureMode: false, + paths, + truncated: Buffer.byteLength(listing.stdout) > 64 * 1024 || rawLines.length > paths.length, + }; +} + +export async function searchRepository( + sandbox: RepositorySandbox, + input: { pattern: string; paths: readonly string[] }, +) { + assertSafeRepositorySearch(input.pattern); + input.paths.forEach(assertSafeRepositoryGlob); + const pinned = await readPinnedRepositoryCommit(sandbox); + const search = await sandbox.run({ + command: buildGitGrepCommand({ ...input, commitSha: pinned }), + abortSignal: AbortSignal.timeout(10_000), + }); + if (![0, 1].includes(search.exitCode)) throw new Error("Repository search failed."); + const rawLines = search.stdout.split(/\r?\n/).filter(Boolean); + const parsed = parseSafeRepositorySearchLines(rawLines, pinned); + const paths = [...new Set(parsed.map(({ path }) => path))]; + const treeEntries = await Promise.all( + paths.map(async (path) => { + const result = await sandbox.run({ + command: buildGitTreeEntryCommand(pinned, path), + abortSignal: AbortSignal.timeout(5_000), + }); + return [path, result] as const; + }), + ); + const safePaths = new Set( + treeEntries + .filter( + ([path, result]) => result.exitCode === 0 && isSafeNonSymlinkTreeEntry(result.stdout, path), + ) + .map(([path]) => path), + ); + const safeLines = parsed.filter(({ path }) => safePaths.has(path)).map(({ line }) => line); + const output = truncateRepositoryInspectionOutput( + redactRepositoryInspectionOutput(safeLines.join("\n")), + ); + return { + commitSha: pinned, + fixtureMode: false, + content: output.content, + matchCount: safeLines.length, + truncated: + output.truncated || + Buffer.byteLength(search.stdout) > 64 * 1024 || + rawLines.length > safeLines.length, + }; +} + +export async function readRepositoryFiles( + sandbox: RepositorySandbox, + files: readonly { path: string; startLine: number; endLine?: number }[], +) { + const normalized = files.map((entry) => ({ + ...entry, + path: assertSafeRepositoryPath(entry.path), + })); + for (const entry of normalized) { + const requestedEndLine = entry.endLine ?? entry.startLine + 399; + if (requestedEndLine < entry.startLine || requestedEndLine - entry.startLine + 1 > 400) { + throw new Error("Repository read range is invalid or too large."); + } + } + const pinned = await readPinnedRepositoryCommit(sandbox); + const results = []; + let remainingBytes = 64 * 1024; + let truncated = false; + for (const entry of normalized) { + const requestedEndLine = entry.endLine ?? entry.startLine + 399; + const tree = await sandbox.run({ + command: buildGitTreeEntryCommand(pinned, entry.path), + abortSignal: AbortSignal.timeout(5_000), + }); + if (tree.exitCode !== 0 || !isSafeNonSymlinkTreeEntry(tree.stdout, entry.path)) { + throw new Error(`Repository file ${entry.path} is missing or is a symlink.`); + } + const maxBytes = Math.min(32 * 1024, remainingBytes); + const read = await sandbox.run({ + command: buildGitReadCommand({ + commitSha: pinned, + path: entry.path, + startLine: entry.startLine, + endLine: requestedEndLine, + maxBytes, + }), + abortSignal: AbortSignal.timeout(10_000), + }); + if (read.exitCode !== 0) throw new Error(`Repository file ${entry.path} could not be read.`); + const bounded = truncateRepositoryInspectionOutput( + redactRepositoryInspectionOutput(read.stdout), + maxBytes, + ); + const newlineCount = (bounded.content.match(/\n/g) ?? []).length; + const returnedLineCount = + bounded.content.length === 0 ? 0 : newlineCount + (bounded.content.endsWith("\n") ? 0 : 1); + const returnedEndLine = + returnedLineCount === 0 ? entry.startLine - 1 : entry.startLine + returnedLineCount - 1; + const fileTruncated = bounded.truncated || Buffer.byteLength(read.stdout) > maxBytes; + remainingBytes -= bounded.byteCount; + truncated ||= fileTruncated; + results.push({ + path: entry.path, + startLine: entry.startLine, + requestedEndLine, + returnedEndLine, + content: bounded.content, + truncated: fileTruncated, + }); + if (remainingBytes <= 0) { + truncated = true; + break; + } + } + return { commitSha: pinned, fixtureMode: false, files: results, truncated }; +} diff --git a/agent/lib/repository-inspection.ts b/agent/lib/repository-inspection.ts new file mode 100644 index 0000000..d4ea68e --- /dev/null +++ b/agent/lib/repository-inspection.ts @@ -0,0 +1,201 @@ +const forbiddenInputPattern = /[;|&`$<>\n\r\0]/; +const forbiddenSearchPattern = /[;&`$<>\n\r\0]/; +const simpleGlobPattern = /^[A-Za-z0-9._/*-]+$/; +const excludedRootPattern = + /(?:^|\/)(?:\.git|\.next|node_modules|coverage|storybook-static|dist|build|out|vendor)(?:\/|$)/; +const secretPathPattern = + /(?:^|\/)(?:\.env[^/]*|\.npmrc|\.netrc|\.ssh|\.aws|secrets?|credentials?)(?:\/|$)|\.(?:pem|p12|key)$/i; + +export const repositoryCommitPattern = /^[a-f0-9]{40}$/; + +function shellQuote(value: string): string { + return `'${value.replaceAll("'", `'"'"'`)}'`; +} + +export function assertPinnedRepositoryCommit(value: string | null): string { + const commitSha = value?.trim() ?? ""; + if (!repositoryCommitPattern.test(commitSha)) { + throw new Error("Repository context is not pinned to a valid commit."); + } + return commitSha; +} + +export function assertSafeRepositoryPath(value: string): string { + const path = value.trim().replaceAll("\\", "/"); + if ( + !path || + path.startsWith("/") || + /^[A-Za-z]:\//.test(path) || + path.includes("//") || + path.split("/").includes(".") || + path.startsWith(":") || + path.split("/").includes("..") || + forbiddenInputPattern.test(path) || + excludedRootPattern.test(path) || + secretPathPattern.test(path) + ) { + throw new Error(`Unsafe repository path: ${value}`); + } + return path; +} + +export function assertSafeRepositoryGlob(value: string): string { + const pattern = value.trim().replaceAll("\\", "/"); + if ( + !pattern || + pattern.length > 256 || + !simpleGlobPattern.test(pattern) || + pattern.startsWith("/") || + /^[A-Za-z]:\//.test(pattern) || + pattern.includes("//") || + pattern.startsWith(":") || + pattern.split("/").includes("..") || + forbiddenInputPattern.test(pattern) || + excludedRootPattern.test(pattern.replaceAll("*", "")) || + secretPathPattern.test(pattern.replaceAll("*", "")) || + /(?:^|\/)\*+\/(?:\.env|\.npmrc|\.netrc|\.ssh|\.aws|secrets?|credentials?)(?:[^/]*)(?:\/|$)/i.test( + pattern, + ) || + /\.(?:pem|p12|key)$/i.test(pattern) + ) { + throw new Error(`Unsafe repository glob: ${value}`); + } + return pattern; +} + +export function assertSafeRepositorySearch(value: string): string { + const pattern = value.trim(); + if (!pattern || pattern.length > 256 || forbiddenSearchPattern.test(pattern)) { + throw new Error("Unsafe repository search pattern."); + } + try { + new RegExp(pattern); + } catch { + throw new Error("Unsafe repository search pattern."); + } + return pattern; +} + +function pathspecs(patterns: readonly string[]): string { + return patterns + .map((pattern) => shellQuote(`:(glob)${assertSafeRepositoryGlob(pattern)}`)) + .join(" "); +} + +function globToExtendedRegex(patternValue: string): string { + const pattern = assertSafeRepositoryGlob(patternValue); + let result = ""; + for (let index = 0; index < pattern.length; index += 1) { + const character = pattern[index]; + if (character === "*" && pattern[index + 1] === "*") { + if (pattern[index + 2] === "/") { + result += "(.*/)?"; + index += 2; + } else { + result += ".*"; + index += 1; + } + } else if (character === "*") { + result += "[^/]*"; + } else if (character === ".") { + result += "\\."; + } else { + result += character; + } + } + return result; +} + +export function buildGitListCommand(commitSha: string, patterns: readonly string[]): string { + const commit = shellQuote(assertPinnedRepositoryCommit(commitSha)); + const matcher = shellQuote( + `^[0-9]{6}[[:space:]](${patterns.map(globToExtendedRegex).join("|")})$`, + ); + return `cd repo && git ls-tree -r --format='%(objectmode)%x09%(path)' ${commit} | grep -E ${matcher} | head -n 1001 | head -c 65537`; +} + +export function buildGitGrepCommand(input: { + commitSha: string; + pattern: string; + paths: readonly string[]; +}): string { + const commit = shellQuote(assertPinnedRepositoryCommit(input.commitSha)); + const pattern = shellQuote(assertSafeRepositorySearch(input.pattern)); + return `cd repo && git grep -n -I -E -- ${pattern} ${commit} -- ${pathspecs(input.paths)} | head -n 101 | head -c 65537`; +} + +export function buildGitTreeEntryCommand(commitSha: string, pathValue: string): string { + const commit = shellQuote(assertPinnedRepositoryCommit(commitSha)); + const path = shellQuote(assertSafeRepositoryPath(pathValue)); + return `cd repo && git ls-tree ${commit} -- ${path}`; +} + +export function buildGitReadCommand(input: { + commitSha: string; + path: string; + startLine: number; + endLine: number; + maxBytes: number; +}): string { + const commit = assertPinnedRepositoryCommit(input.commitSha); + const path = assertSafeRepositoryPath(input.path); + return `cd repo && git show ${shellQuote(`${commit}:${path}`)} | sed -n '${input.startLine},${input.endLine}p' | head -c ${input.maxBytes + 1}`; +} + +export function parseSafeGitTreePaths(lines: readonly string[], limit = 200): string[] { + const safe: string[] = []; + for (const line of lines) { + const match = line.match(/^(\d{6})\t(.+)$/); + if (!match?.[2] || match[1] === "120000") continue; + try { + safe.push(assertSafeRepositoryPath(match[2])); + } catch { + // Omit generated, dependency, secret-like, malformed, and symlink entries. + } + if (safe.length >= limit) break; + } + return safe; +} + +export function parseSafeRepositorySearchLines( + lines: readonly string[], + commitSha: string, + limit = 100, +): Array<{ path: string; line: string }> { + const safe: Array<{ path: string; line: string }> = []; + const prefix = `${assertPinnedRepositoryCommit(commitSha)}:`; + for (const rawLine of lines) { + if (!rawLine.startsWith(prefix)) continue; + const line = rawLine.slice(prefix.length); + const match = line.match(/^([^:]+):(\d+):(.*)$/); + if (!match?.[1]) continue; + try { + safe.push({ path: assertSafeRepositoryPath(match[1]), line }); + } catch { + // Never return matches from excluded or secret-like paths. + } + if (safe.length >= limit) break; + } + return safe; +} + +export function isSafeNonSymlinkTreeEntry(value: string, expectedPath: string): boolean { + const match = value.trim().match(/^(\d{6})\s+\w+\s+[a-f0-9]+\t(.+)$/); + return Boolean(match && match[1] !== "120000" && match[2] === expectedPath); +} + +export function truncateRepositoryInspectionOutput(value: string, maxBytes = 64 * 1024) { + const bytes = Buffer.from(value, "utf8"); + if (bytes.byteLength <= maxBytes) { + return { byteCount: bytes.byteLength, content: value, truncated: false }; + } + const content = bytes.subarray(0, maxBytes).toString("utf8"); + return { byteCount: Buffer.byteLength(content), content, truncated: true }; +} + +export function redactRepositoryInspectionOutput(value: string): string { + return value + .replace(/(authorization:\s*bearer\s+)\S+/gi, "$1[REDACTED]") + .replace(/\b(?:gh[pousr]_|github_pat_|sk-)[A-Za-z0-9_-]+\b/g, "[REDACTED]") + .replace(/((?:password|secret|token|api[_-]?key)\s*[=:]\s*)\S+/gi, "$1[REDACTED]"); +} diff --git a/tests/unit/agent/repository-inspection.test.ts b/tests/unit/agent/repository-inspection.test.ts new file mode 100644 index 0000000..b7ccf7c --- /dev/null +++ b/tests/unit/agent/repository-inspection.test.ts @@ -0,0 +1,156 @@ +/** @vitest-environment node */ +import { execFileSync, spawnSync } from "node:child_process"; +import { mkdir, mkdtemp, readFile, rm, symlink, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { + assertSafeRepositoryGlob, + assertSafeRepositoryPath, + assertSafeRepositorySearch, + buildGitGrepCommand, + buildGitListCommand, + parseSafeRepositorySearchLines, + truncateRepositoryInspectionOutput, +} from "@agent/lib/repository-inspection"; +import { + listRepositoryFiles, + type RepositorySandbox, + readRepositoryFiles, + searchRepository, +} from "@agent/lib/repository-inspection-runtime"; + +const commitSha = "a".repeat(40); + +describe("repository inspection policy", () => { + it("rejects escape, secret, generated, glob-obfuscation, and shell surfaces", () => { + expect(assertSafeRepositoryPath("src/lib/loops/manifest.ts")).toBe("src/lib/loops/manifest.ts"); + for (const path of [ + "../outside.ts", + "C:/Windows/system.ini", + "/etc/passwd", + ".env.local", + ".git/config", + "node_modules/eve/index.ts", + "packages/app/vendor/library.ts", + "src//lib/file.ts", + ".next/server/app.js", + "secrets/private-key.pem", + ]) + expect(() => assertSafeRepositoryPath(path)).toThrow("Unsafe repository path"); + + expect(assertSafeRepositoryGlob("tests/**/*.test.ts")).toBe("tests/**/*.test.ts"); + expect(assertSafeRepositoryGlob("**/AGENTS.md")).toBe("**/AGENTS.md"); + for (const pattern of [ + "../**", + "node_modules/**", + "**/node_modules/**", + ".next/**", + "**/.env*", + "**/.npmrc", + "**/*.pem", + "src/[ab].ts", + "**/*.ts; git status", + ]) + expect(() => assertSafeRepositoryGlob(pattern)).toThrow("Unsafe repository glob"); + + expect( + parseSafeRepositorySearchLines( + [`${commitSha}:.npmrc:1:not-a-key:123:SUPERSECRET`], + commitSha, + ), + ).toEqual([]); + }); + + it("bounds regex search and pins bounded Git-object commands", () => { + expect(assertSafeRepositorySearch("test-writing|plan-review")).toBe("test-writing|plan-review"); + expect(() => assertSafeRepositorySearch("a".repeat(257))).toThrow("Unsafe repository search"); + expect(() => assertSafeRepositorySearch("[invalid")).toThrow("Unsafe repository search"); + expect(() => assertSafeRepositorySearch("token=$(cat .env)")).toThrow( + "Unsafe repository search", + ); + expect( + buildGitGrepCommand({ commitSha, pattern: "plan-review", paths: ["src/**/*.ts"] }), + ).toContain(`git grep -n -I -E -- 'plan-review' '${commitSha}'`); + expect(buildGitListCommand(commitSha, ["**/AGENTS.md"])).toContain( + `git ls-tree -r --format='%(objectmode)%x09%(path)' '${commitSha}'`, + ); + expect(buildGitListCommand(commitSha, ["**/AGENTS.md"])).toContain("head -c 65537"); + }); + + it("truncates inspection output deterministically", () => { + expect(truncateRepositoryInspectionOutput("abcdef", 4)).toEqual({ + byteCount: 4, + content: "abcd", + truncated: true, + }); + }); +}); + +describe("repository inspection runtime", () => { + it("reads immutable commit objects, omits symlinks, and reports exact provenance", async () => { + const root = await mkdtemp(join(tmpdir(), "loopworks-repository-inspection-")); + const repo = join(root, "repo"); + try { + await mkdir(join(repo, "src"), { recursive: true }); + await mkdir(join(root, ".loopworks"), { recursive: true }); + await writeFile( + join(repo, "src", "safe.ts"), + "export const value = 'committed';\nline two\n", + ); + await writeFile(join(repo, ".npmrc"), "token=SUPERSECRET\n"); + await symlink("safe.ts", join(repo, "src", "link.ts")); + execFileSync("git", ["init"], { cwd: repo }); + execFileSync("git", ["config", "user.email", "test@example.com"], { cwd: repo }); + execFileSync("git", ["config", "user.name", "Test"], { cwd: repo }); + execFileSync("git", ["add", "."], { cwd: repo }); + execFileSync("git", ["commit", "-m", "fixture"], { cwd: repo }); + const pinned = execFileSync("git", ["rev-parse", "HEAD"], { + cwd: repo, + encoding: "utf8", + }).trim(); + await writeFile(join(root, ".loopworks", "repository-commit"), pinned); + await writeFile(join(repo, "src", "safe.ts"), "export const value = 'dirty';\n"); + + const sandbox: RepositorySandbox = { + readTextFile: async ({ path }) => { + try { + return await readFile(join(root, path), "utf8"); + } catch { + return null; + } + }, + run: async ({ command }) => { + const result = spawnSync("bash", ["-lc", command], { cwd: root, encoding: "utf8" }); + return { exitCode: result.status ?? 1, stdout: result.stdout }; + }, + }; + + const listed = await listRepositoryFiles(sandbox, ["src/**/*.ts"]); + expect(listed.paths).toEqual(["src/safe.ts"]); + const searched = await searchRepository(sandbox, { + pattern: "committed|dirty", + paths: ["src/**/*.ts"], + }); + expect(searched.content).toContain("src/safe.ts:1:export const value = 'committed'"); + expect(searched.content).not.toContain("dirty"); + const read = await readRepositoryFiles(sandbox, [ + { path: "src/safe.ts", startLine: 1, endLine: 2 }, + ]); + expect(read.files[0]).toMatchObject({ + content: "export const value = 'committed';\nline two\n", + requestedEndLine: 2, + returnedEndLine: 2, + truncated: false, + }); + await expect( + readRepositoryFiles(sandbox, [{ path: "src/link.ts", startLine: 1 }]), + ).rejects.toThrow("symlink"); + await expect( + readRepositoryFiles(sandbox, [{ path: "src/safe.ts", startLine: 1, endLine: 401 }]), + ).rejects.toThrow("too large"); + } finally { + await rm(root, { recursive: true, force: true }); + } + }, 30_000); +}); From 384a5e030d74ef4fe250ce24ae6999afa2569ace Mon Sep 17 00:00:00 2001 From: Nate Summers Date: Sat, 11 Jul 2026 14:04:14 -0700 Subject: [PATCH 03/12] feat(agent): move planning into a declared planner subagent with pinned plan digests Relocate the planning tool surface to a declared planner subagent with an isolated deny-all sandbox and repository inspection tools. Plan artifacts gain a stable identity, a canonical SHA-256 digest, and a pinned repository revision enforced by pinnedPlanningAgentOutputSchema. Adopts openai/gpt-5.6-sol with xhigh reasoning per the issue #47 model routing contract, superseding the gpt-5.5 label recorded in ADR 0013. Part of #47. Co-Authored-By: Claude Fable 5 --- agent/planning-agent.ts | 88 ++++++++++++++++++- agent/subagents/planner/agent.ts | 13 +++ agent/subagents/planner/instructions.md | 16 ++++ agent/subagents/planner/sandbox.ts | 10 +++ agent/subagents/planner/tools/ask_question.ts | 2 + agent/subagents/planner/tools/bash.ts | 16 ++++ .../planner/tools/emit_plan_artifact.ts | 10 +++ agent/subagents/planner/tools/glob.ts | 2 + agent/subagents/planner/tools/grep.ts | 2 + .../planner/tools/list_repository_files.ts | 34 +++++++ agent/subagents/planner/tools/load_skill.ts | 3 + .../tools/prepare_repository_context.ts | 86 ++++++++++++++++++ agent/subagents/planner/tools/read_file.ts | 2 + .../planner}/tools/read_issue_context.ts | 9 +- .../planner/tools/read_repository_files.ts | 43 +++++++++ .../planner/tools/search_repository.ts | 28 ++++++ .../summarize_validation_requirements.ts | 17 ++++ agent/subagents/planner/tools/todo.ts | 2 + agent/subagents/planner/tools/web_fetch.ts | 2 + agent/subagents/planner/tools/web_search.ts | 2 + agent/subagents/planner/tools/write_file.ts | 2 + agent/tools/emit_plan_artifact.ts | 13 --- .../summarize_validation_requirements.ts | 25 ------ docs/adr/0013-planning-agent-contract.md | 15 +++- evals/planning/issue-13-plan.eval.ts | 6 +- tests/unit/agent/planning-agent.test.ts | 28 +++++- tests/unit/agent/planning-eval.test.ts | 8 +- tests/unit/agent/planning-tools.test.ts | 3 +- tests/unit/loops/development-run.test.ts | 2 +- 29 files changed, 429 insertions(+), 60 deletions(-) create mode 100644 agent/subagents/planner/agent.ts create mode 100644 agent/subagents/planner/instructions.md create mode 100644 agent/subagents/planner/sandbox.ts create mode 100644 agent/subagents/planner/tools/ask_question.ts create mode 100644 agent/subagents/planner/tools/bash.ts create mode 100644 agent/subagents/planner/tools/emit_plan_artifact.ts create mode 100644 agent/subagents/planner/tools/glob.ts create mode 100644 agent/subagents/planner/tools/grep.ts create mode 100644 agent/subagents/planner/tools/list_repository_files.ts create mode 100644 agent/subagents/planner/tools/load_skill.ts create mode 100644 agent/subagents/planner/tools/prepare_repository_context.ts create mode 100644 agent/subagents/planner/tools/read_file.ts rename agent/{ => subagents/planner}/tools/read_issue_context.ts (75%) create mode 100644 agent/subagents/planner/tools/read_repository_files.ts create mode 100644 agent/subagents/planner/tools/search_repository.ts create mode 100644 agent/subagents/planner/tools/summarize_validation_requirements.ts create mode 100644 agent/subagents/planner/tools/todo.ts create mode 100644 agent/subagents/planner/tools/web_fetch.ts create mode 100644 agent/subagents/planner/tools/web_search.ts create mode 100644 agent/subagents/planner/tools/write_file.ts delete mode 100644 agent/tools/emit_plan_artifact.ts delete mode 100644 agent/tools/summarize_validation_requirements.ts diff --git a/agent/planning-agent.ts b/agent/planning-agent.ts index d26843e..0bc3199 100644 --- a/agent/planning-agent.ts +++ b/agent/planning-agent.ts @@ -1,10 +1,12 @@ +import { createHash } from "node:crypto"; + import { z } from "zod"; import { defaultLoopManifest } from "@/lib/loops/manifest"; import { type LoopManifest, loopStateValues } from "../schemas/loop-manifest"; -export const planningAgentModelLabel = "openai/gpt-5.5-xhigh"; +export const planningAgentModelLabel = "openai/gpt-5.6-sol-xhigh"; export const planningAgentInputSchema = z.object({ repositoryFullName: z.string().min(1), @@ -14,6 +16,13 @@ export const planningAgentInputSchema = z.object({ body: z.string().default(""), labels: z.array(z.string().min(1)).default([]), milestone: z.string().min(1).nullable().default(null), + repositoryRevision: z + .object({ + ref: z.string().min(1), + commitSha: z.string().regex(/^[a-f0-9]{40}$/), + }) + .nullable() + .default(null), }); export const planningAgentIssueSchema = z.object({ @@ -93,9 +102,19 @@ export const planningAgentToolContractSchema = z.object({ }); export const planningAgentOutputSchema = z.object({ + identity: z.object({ + id: z.string().min(1), + sha256: z.string().regex(/^[a-f0-9]{64}$/), + }), summary: z.string().min(1), model: z.literal(planningAgentModelLabel), issue: planningAgentIssueSchema, + repositoryRevision: z + .object({ + ref: z.string().min(1), + commitSha: z.string().regex(/^[a-f0-9]{40}$/), + }) + .nullable(), initialState: z.enum(loopStateValues).default("planned"), checkpoints: z.array(z.string().min(1)).min(1), stages: z.array(planningAgentStageSchema).min(1), @@ -111,6 +130,27 @@ export const planningAgentOutputSchema = z.object({ export type PlanningAgentInput = z.infer; export type PlanningAgentOutput = z.infer; +export const pinnedPlanningAgentOutputSchema = planningAgentOutputSchema.refine( + ( + plan, + ): plan is PlanningAgentOutput & { + repositoryRevision: NonNullable; + } => plan.repositoryRevision !== null, + "Planning artifacts require an inspected, pinned repository revision.", +); + +export function computePlanningArtifactDigest( + artifact: Omit & { + identity?: { id: string; sha256?: string }; + }, +): string { + const canonical = { + ...artifact, + identity: artifact.identity ? { id: artifact.identity.id } : undefined, + }; + return createHash("sha256").update(JSON.stringify(canonical)).digest("hex"); +} + export const planningAgentDefinition = { name: "planning-agent", runtime: "eve", @@ -199,7 +239,10 @@ export function createPlanningAgentSeedPlan(input: PlanningAgentInput): Planning const acceptanceCriteria = extractAcceptanceCriteria(parsed.body); const issueUrl = getIssueUrl(parsed); - return planningAgentOutputSchema.parse({ + const planWithoutDigest = { + identity: { + id: `plan:${parsed.repositoryFullName}#${parsed.issueNumber}`, + }, summary: `Initial plan for issue #${parsed.issueNumber} in ${parsed.repositoryFullName}.`, model: planningAgentModelLabel, issue: { @@ -211,6 +254,7 @@ export function createPlanningAgentSeedPlan(input: PlanningAgentInput): Planning title: parsed.title, url: issueUrl, }, + repositoryRevision: parsed.repositoryRevision, initialState: "planned", checkpoints, stages: [ @@ -358,6 +402,30 @@ export function createPlanningAgentSeedPlan(input: PlanningAgentInput): Planning planningOnly: true, planArtifactOnlyWrite: true, allowedTools: [ + { + name: "prepare_repository_context", + capability: "Prepare a read-only isolated checkout pinned to an exact commit.", + mutates: false, + auditFields: ["agent", "repo", "issue", "run", "step", "traceId"], + }, + { + name: "list_repository_files", + capability: "Discover bounded tracked repository paths by safe glob patterns.", + mutates: false, + auditFields: ["agent", "repo", "issue", "run", "step", "traceId"], + }, + { + name: "search_repository", + capability: "Search bounded repository text with path and line provenance.", + mutates: false, + auditFields: ["agent", "repo", "issue", "run", "step", "traceId"], + }, + { + name: "read_repository_files", + capability: "Read bounded file ranges from the pinned repository context.", + mutates: false, + auditFields: ["agent", "repo", "issue", "run", "step", "traceId"], + }, { name: "read_issue_context", capability: "Read supplied GitHub issue context and summarize acceptance criteria.", @@ -392,5 +460,21 @@ export function createPlanningAgentSeedPlan(input: PlanningAgentInput): Planning "copy-agent delegation", ], }, + }; + + const normalizedPlan = planningAgentOutputSchema.parse({ + ...planWithoutDigest, + identity: { + id: planWithoutDigest.identity.id, + sha256: "0".repeat(64), + }, + }); + + return planningAgentOutputSchema.parse({ + ...normalizedPlan, + identity: { + id: normalizedPlan.identity.id, + sha256: computePlanningArtifactDigest(normalizedPlan), + }, }); } diff --git a/agent/subagents/planner/agent.ts b/agent/subagents/planner/agent.ts new file mode 100644 index 0000000..9c903f8 --- /dev/null +++ b/agent/subagents/planner/agent.ts @@ -0,0 +1,13 @@ +import { defineAgent } from "eve"; + +import { pinnedPlanningAgentOutputSchema } from "../../planning-agent"; + +export default defineAgent({ + description: "Create an approved executable plan artifact from durable GitHub issue context.", + model: "openai/gpt-5.6-sol", + modelContextWindowTokens: 400_000, + modelOptions: { + providerOptions: { openai: { reasoningEffort: "xhigh" } }, + }, + outputSchema: pinnedPlanningAgentOutputSchema, +}); diff --git a/agent/subagents/planner/instructions.md b/agent/subagents/planner/instructions.md new file mode 100644 index 0000000..9a68763 --- /dev/null +++ b/agent/subagents/planner/instructions.md @@ -0,0 +1,16 @@ +# Loopworks Planner Subagent + +Read GitHub issue context as the durable source of truth and produce only the +validated executable plan artifact. Preserve acceptance criteria, pin the +repository revision, and identify validation and approval gates. + +Use the durable run ID supplied by the orchestrator to prepare its bound, +commit-pinned repository context, then use bounded repository file +listing, content search, and line-range reads to find applicable `AGENTS.md`, +existing architecture, tests, and validation conventions. Cite repository paths +and line ranges in the plan evidence. Planner web search remains separately +guarded by issue #68. + +Do not edit repository files, change branches, mutate GitHub or SaaS state, or +delegate to another agent. Structured logs carry correlation fields only and +must not capture raw prompts, issue bodies, or tool output. diff --git a/agent/subagents/planner/sandbox.ts b/agent/subagents/planner/sandbox.ts new file mode 100644 index 0000000..763a55c --- /dev/null +++ b/agent/subagents/planner/sandbox.ts @@ -0,0 +1,10 @@ +import { defaultBackend, defineSandbox } from "eve/sandbox"; + +export default defineSandbox({ + backend: defaultBackend({ + docker: { networkPolicy: "deny-all" }, + microsandbox: { networkPolicy: "deny-all" }, + vercel: { networkPolicy: "deny-all" }, + }), + description: "Planning-only isolated checkout with deny-all network after preparation.", +}); diff --git a/agent/subagents/planner/tools/ask_question.ts b/agent/subagents/planner/tools/ask_question.ts new file mode 100644 index 0000000..a908e44 --- /dev/null +++ b/agent/subagents/planner/tools/ask_question.ts @@ -0,0 +1,2 @@ +import { disableTool } from "eve/tools"; +export default disableTool(); diff --git a/agent/subagents/planner/tools/bash.ts b/agent/subagents/planner/tools/bash.ts new file mode 100644 index 0000000..28125d1 --- /dev/null +++ b/agent/subagents/planner/tools/bash.ts @@ -0,0 +1,16 @@ +import { defineTool } from "eve/tools"; +import { z } from "zod"; + +import { executeCliInspectionCommand } from "../../../lib/cli-inspection"; + +export default defineTool({ + description: "Run a guarded read-only CLI inspection command for planning context.", + inputSchema: z.object({ command: z.string().min(1) }), + outputSchema: z.object({ + exitCode: z.number(), + stderr: z.string(), + stdout: z.string(), + truncated: z.boolean(), + }), + execute: ({ command }) => executeCliInspectionCommand(command), +}); diff --git a/agent/subagents/planner/tools/emit_plan_artifact.ts b/agent/subagents/planner/tools/emit_plan_artifact.ts new file mode 100644 index 0000000..d87caf1 --- /dev/null +++ b/agent/subagents/planner/tools/emit_plan_artifact.ts @@ -0,0 +1,10 @@ +import { defineTool } from "eve/tools"; + +import { pinnedPlanningAgentOutputSchema } from "../../../planning-agent"; + +export default defineTool({ + description: "Emit the final validated planning artifact.", + inputSchema: pinnedPlanningAgentOutputSchema, + outputSchema: pinnedPlanningAgentOutputSchema, + execute: (input) => pinnedPlanningAgentOutputSchema.parse(input), +}); diff --git a/agent/subagents/planner/tools/glob.ts b/agent/subagents/planner/tools/glob.ts new file mode 100644 index 0000000..a908e44 --- /dev/null +++ b/agent/subagents/planner/tools/glob.ts @@ -0,0 +1,2 @@ +import { disableTool } from "eve/tools"; +export default disableTool(); diff --git a/agent/subagents/planner/tools/grep.ts b/agent/subagents/planner/tools/grep.ts new file mode 100644 index 0000000..a908e44 --- /dev/null +++ b/agent/subagents/planner/tools/grep.ts @@ -0,0 +1,2 @@ +import { disableTool } from "eve/tools"; +export default disableTool(); diff --git a/agent/subagents/planner/tools/list_repository_files.ts b/agent/subagents/planner/tools/list_repository_files.ts new file mode 100644 index 0000000..f656413 --- /dev/null +++ b/agent/subagents/planner/tools/list_repository_files.ts @@ -0,0 +1,34 @@ +import { defineTool } from "eve/tools"; +import { z } from "zod"; +import { resolvePlanningAgentFixtureMode } from "../../../lib/fixture-mode"; +import { assertSafeRepositoryGlob } from "../../../lib/repository-inspection"; +import { + listRepositoryFiles, + repositoryListOutputSchema, +} from "../../../lib/repository-inspection-runtime"; + +const glob = z.string().refine((value) => { + try { + assertSafeRepositoryGlob(value); + return true; + } catch { + return false; + } +}, "Unsafe repository glob."); + +export default defineTool({ + description: "List bounded regular-file paths from the prepared pinned Git commit.", + inputSchema: z.object({ patterns: z.array(glob).min(1).max(5) }), + outputSchema: repositoryListOutputSchema, + async execute({ patterns }, ctx) { + if (resolvePlanningAgentFixtureMode().enabled) { + return { + commitSha: "a".repeat(40), + fixtureMode: true, + paths: ["AGENTS.md", "agent/AGENTS.md", "tests/AGENTS.md"], + truncated: false, + }; + } + return listRepositoryFiles(await ctx.getSandbox(), patterns); + }, +}); diff --git a/agent/subagents/planner/tools/load_skill.ts b/agent/subagents/planner/tools/load_skill.ts new file mode 100644 index 0000000..04bd054 --- /dev/null +++ b/agent/subagents/planner/tools/load_skill.ts @@ -0,0 +1,3 @@ +import { disableTool } from "eve/tools"; + +export default disableTool(); diff --git a/agent/subagents/planner/tools/prepare_repository_context.ts b/agent/subagents/planner/tools/prepare_repository_context.ts new file mode 100644 index 0000000..f0d6cf9 --- /dev/null +++ b/agent/subagents/planner/tools/prepare_repository_context.ts @@ -0,0 +1,86 @@ +import { eq } from "drizzle-orm"; +import { defineTool } from "eve/tools"; +import { z } from "zod"; +import { db } from "@/db/client"; +import { loopRuns, repositories } from "@/db/schema"; + +import { resolvePlanningAgentFixtureMode } from "../../../lib/fixture-mode"; + +const repositoryName = z.string().regex(/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/); +const revision = z + .object({ + commitSha: z + .string() + .regex(/^[a-f0-9]{40}$/) + .optional(), + ref: z + .string() + .regex(/^[A-Za-z0-9._/-]+$/) + .optional(), + }) + .optional(); + +export default defineTool({ + description: "Prepare an isolated repository checkout and return its exact pinned commit.", + inputSchema: z.object({ + runId: z.string().uuid().optional(), + repositoryFullName: repositoryName.optional(), + revision, + }), + outputSchema: z.object({ + commitSha: z.string().regex(/^[a-f0-9]{40}$/), + fixtureMode: z.boolean(), + }), + async execute(input, ctx) { + const fixtureMode = resolvePlanningAgentFixtureMode().enabled; + if (fixtureMode) { + return { commitSha: input.revision?.commitSha ?? "a".repeat(40), fixtureMode: true }; + } + if (!input.runId) + throw new Error("Production repository preparation requires a durable run ID."); + const [boundRepository] = await db + .select({ fullName: repositories.fullName }) + .from(loopRuns) + .innerJoin(repositories, eq(loopRuns.repositoryId, repositories.id)) + .where(eq(loopRuns.id, input.runId)) + .limit(1); + if (!boundRepository) throw new Error("Run repository binding was not found."); + if (input.repositoryFullName && input.repositoryFullName !== boundRepository.fullName) { + throw new Error("Requested repository does not match the durable run."); + } + const repositoryFullName = boundRepository.fullName; + const sandbox = await ctx.getSandbox(); + await sandbox.setNetworkPolicy({ allow: ["github.com", "objects.githubusercontent.com"] }); + try { + const clone = await sandbox.run({ + command: `git clone --filter=blob:none 'https://github.com/${repositoryFullName}.git' repo`, + abortSignal: AbortSignal.timeout(120_000), + }); + if (clone.exitCode !== 0) throw new Error("Repository checkout failed."); + const target = input.revision?.commitSha ?? input.revision?.ref; + if (target) { + const checkout = await sandbox.run({ + command: `cd repo && git checkout --detach '${target}'`, + abortSignal: AbortSignal.timeout(60_000), + }); + if (checkout.exitCode !== 0) throw new Error("Repository revision checkout failed."); + } + const resolved = await sandbox.run({ + command: "cd repo && git rev-parse HEAD", + abortSignal: AbortSignal.timeout(10_000), + }); + const commitSha = resolved.stdout.trim(); + if (resolved.exitCode !== 0 || !/^[a-f0-9]{40}$/.test(commitSha)) { + throw new Error("Repository revision could not be pinned."); + } + await sandbox.run({ + command: "mkdir -p .loopworks", + abortSignal: AbortSignal.timeout(5_000), + }); + await sandbox.writeTextFile({ path: ".loopworks/repository-commit", content: commitSha }); + return { commitSha, fixtureMode: false }; + } finally { + await sandbox.setNetworkPolicy("deny-all"); + } + }, +}); diff --git a/agent/subagents/planner/tools/read_file.ts b/agent/subagents/planner/tools/read_file.ts new file mode 100644 index 0000000..a908e44 --- /dev/null +++ b/agent/subagents/planner/tools/read_file.ts @@ -0,0 +1,2 @@ +import { disableTool } from "eve/tools"; +export default disableTool(); diff --git a/agent/tools/read_issue_context.ts b/agent/subagents/planner/tools/read_issue_context.ts similarity index 75% rename from agent/tools/read_issue_context.ts rename to agent/subagents/planner/tools/read_issue_context.ts index 8c800cd..61935df 100644 --- a/agent/tools/read_issue_context.ts +++ b/agent/subagents/planner/tools/read_issue_context.ts @@ -1,9 +1,9 @@ import { defineTool } from "eve/tools"; import { z } from "zod"; -import { extractAcceptanceCriteria } from "../planning-agent"; +import { extractAcceptanceCriteria } from "../../../planning-agent"; -const readIssueContextInputSchema = z.object({ +const schema = z.object({ body: z.string().default(""), labels: z.array(z.string()).default([]), milestone: z.string().nullable().default(null), @@ -14,9 +14,8 @@ const readIssueContextInputSchema = z.object({ }); export default defineTool({ - description: - "Read supplied GitHub issue context and return normalized planning metadata without mutating GitHub.", - inputSchema: readIssueContextInputSchema, + description: "Normalize supplied GitHub issue context without mutating GitHub.", + inputSchema: schema, execute(input) { return { acceptanceCriteria: extractAcceptanceCriteria(input.body), diff --git a/agent/subagents/planner/tools/read_repository_files.ts b/agent/subagents/planner/tools/read_repository_files.ts new file mode 100644 index 0000000..113fe6a --- /dev/null +++ b/agent/subagents/planner/tools/read_repository_files.ts @@ -0,0 +1,43 @@ +import { defineTool } from "eve/tools"; +import { z } from "zod"; +import { resolvePlanningAgentFixtureMode } from "../../../lib/fixture-mode"; +import { + readRepositoryFiles, + repositoryReadOutputSchema, +} from "../../../lib/repository-inspection-runtime"; + +const request = z.union([ + z.string().min(1), + z.object({ + path: z.string().min(1), + startLine: z.number().int().positive().default(1), + endLine: z.number().int().positive().optional(), + }), +]); + +export default defineTool({ + description: "Read bounded line ranges from regular files at the pinned Git commit.", + inputSchema: z.object({ files: z.array(request).min(1).max(20) }), + outputSchema: repositoryReadOutputSchema, + async execute({ files }, ctx) { + const normalized = files.map((entry) => + typeof entry === "string" ? { path: entry, startLine: 1 } : entry, + ); + if (resolvePlanningAgentFixtureMode().enabled) { + return { + commitSha: "a".repeat(40), + fixtureMode: true, + files: normalized.map((entry) => ({ + path: entry.path, + startLine: entry.startLine, + requestedEndLine: entry.endLine ?? entry.startLine + 399, + returnedEndLine: entry.startLine, + content: "// Deterministic fixture repository context.", + truncated: false, + })), + truncated: false, + }; + } + return readRepositoryFiles(await ctx.getSandbox(), normalized); + }, +}); diff --git a/agent/subagents/planner/tools/search_repository.ts b/agent/subagents/planner/tools/search_repository.ts new file mode 100644 index 0000000..3694bee --- /dev/null +++ b/agent/subagents/planner/tools/search_repository.ts @@ -0,0 +1,28 @@ +import { defineTool } from "eve/tools"; +import { z } from "zod"; +import { resolvePlanningAgentFixtureMode } from "../../../lib/fixture-mode"; +import { + repositorySearchOutputSchema, + searchRepository, +} from "../../../lib/repository-inspection-runtime"; + +export default defineTool({ + description: "Search regular files from the pinned Git commit with bounded regex output.", + inputSchema: z.object({ + pattern: z.string().min(1).max(256), + paths: z.array(z.string()).min(1).max(5), + }), + outputSchema: repositorySearchOutputSchema, + async execute(input, ctx) { + if (resolvePlanningAgentFixtureMode().enabled) { + return { + commitSha: "a".repeat(40), + fixtureMode: true, + content: "tests/unit/agent/example.test.ts:1:fixture match", + matchCount: 1, + truncated: false, + }; + } + return searchRepository(await ctx.getSandbox(), input); + }, +}); diff --git a/agent/subagents/planner/tools/summarize_validation_requirements.ts b/agent/subagents/planner/tools/summarize_validation_requirements.ts new file mode 100644 index 0000000..adeb69a --- /dev/null +++ b/agent/subagents/planner/tools/summarize_validation_requirements.ts @@ -0,0 +1,17 @@ +import { defineTool } from "eve/tools"; +import { z } from "zod"; + +export default defineTool({ + description: "Map acceptance criteria to deterministic validation gates.", + inputSchema: z.object({ acceptanceCriteria: z.array(z.string().min(1)).min(1) }), + execute(input) { + return { + gates: input.acceptanceCriteria.map((acceptanceCriterion, index) => ({ + acceptanceCriterion, + key: `ac-${index + 1}`, + validation: "focused unit/eval coverage required before review", + })), + requiredCommands: ["bun test tests/unit/agent", "bunx eve eval --list"], + }; + }, +}); diff --git a/agent/subagents/planner/tools/todo.ts b/agent/subagents/planner/tools/todo.ts new file mode 100644 index 0000000..a908e44 --- /dev/null +++ b/agent/subagents/planner/tools/todo.ts @@ -0,0 +1,2 @@ +import { disableTool } from "eve/tools"; +export default disableTool(); diff --git a/agent/subagents/planner/tools/web_fetch.ts b/agent/subagents/planner/tools/web_fetch.ts new file mode 100644 index 0000000..a908e44 --- /dev/null +++ b/agent/subagents/planner/tools/web_fetch.ts @@ -0,0 +1,2 @@ +import { disableTool } from "eve/tools"; +export default disableTool(); diff --git a/agent/subagents/planner/tools/web_search.ts b/agent/subagents/planner/tools/web_search.ts new file mode 100644 index 0000000..a908e44 --- /dev/null +++ b/agent/subagents/planner/tools/web_search.ts @@ -0,0 +1,2 @@ +import { disableTool } from "eve/tools"; +export default disableTool(); diff --git a/agent/subagents/planner/tools/write_file.ts b/agent/subagents/planner/tools/write_file.ts new file mode 100644 index 0000000..a908e44 --- /dev/null +++ b/agent/subagents/planner/tools/write_file.ts @@ -0,0 +1,2 @@ +import { disableTool } from "eve/tools"; +export default disableTool(); diff --git a/agent/tools/emit_plan_artifact.ts b/agent/tools/emit_plan_artifact.ts deleted file mode 100644 index 3e75be1..0000000 --- a/agent/tools/emit_plan_artifact.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { defineTool } from "eve/tools"; - -import { planningAgentOutputSchema } from "../planning-agent"; - -export default defineTool({ - description: - "Emit the final validated planning agent artifact. This is the only planning tool whose contract produces a write-like artifact.", - inputSchema: planningAgentOutputSchema, - outputSchema: planningAgentOutputSchema, - execute(input) { - return planningAgentOutputSchema.parse(input); - }, -}); diff --git a/agent/tools/summarize_validation_requirements.ts b/agent/tools/summarize_validation_requirements.ts deleted file mode 100644 index 7cdb385..0000000 --- a/agent/tools/summarize_validation_requirements.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { defineTool } from "eve/tools"; -import { z } from "zod"; - -const validationInputSchema = z.object({ - acceptanceCriteria: z.array(z.string().min(1)).min(1), -}); - -export default defineTool({ - description: - "Map acceptance criteria to deterministic validation gates for the planning artifact.", - inputSchema: validationInputSchema, - execute(input) { - return { - gates: input.acceptanceCriteria.map((acceptanceCriterion, index) => ({ - acceptanceCriterion, - key: `ac-${index + 1}`, - validation: "focused unit/eval coverage required before review", - })), - requiredCommands: [ - "bun test tests/unit/agent/planning-agent.test.ts tests/unit/agent/planning-tools.test.ts tests/unit/agent/planning-fixture.test.ts tests/unit/agent/planning-observability.test.ts", - "bunx eve eval --list", - ], - }; - }, -}); diff --git a/docs/adr/0013-planning-agent-contract.md b/docs/adr/0013-planning-agent-contract.md index f33c1e6..be0ecd2 100644 --- a/docs/adr/0013-planning-agent-contract.md +++ b/docs/adr/0013-planning-agent-contract.md @@ -14,11 +14,15 @@ and metric contract. ## Decision -Loopworks will define the planning agent as an Eve-backed planning-only runtime. The -agent emits a typed plan artifact containing issue metadata, stages, validation +Loopworks defines the planner as an Eve-backed planning-only declared subagent +under the neutral stage orchestrator established by ADR +[0015](0015-stage-orchestrator-and-isolated-subagent-handoffs.md). This 2026-07-11 +placement update preserves the original planning contract while removing stage +orchestration from the planner. The subagent emits a typed plan artifact containing issue metadata, stages, validation gates, approval points, risks, fixture mode, eval coverage, and tool-contract -summary. The selected planning model is `openai/gpt-5.5` with OpenAI reasoning -effort `xhigh`, reported in artifacts as `openai/gpt-5.5-xhigh`. +summary. The selected planning model is `openai/gpt-5.6-sol` with OpenAI +reasoning effort `xhigh`, reported in artifacts as +`openai/gpt-5.6-sol-xhigh`. The model-visible CLI surface is a guarded `bash` replacement for read-only inspection through tools such as `gh`, `az`, and read-only `git` commands. @@ -42,6 +46,9 @@ shell or source mutation surface. The first agent contract is testable through deterministic golden fixtures and Eve eval discovery before broader model, prompt, or tool changes. +The planner no longer owns the root Eve runtime. It is a sibling of other stage +subagents and cannot invoke them or transition durable run state. + The implementation intentionally defers telemetry exporter wiring, production masking policy, metrics backend activation, and trace collector setup to ADR 0012 implementation work. diff --git a/evals/planning/issue-13-plan.eval.ts b/evals/planning/issue-13-plan.eval.ts index a4cdf5e..58897a7 100644 --- a/evals/planning/issue-13-plan.eval.ts +++ b/evals/planning/issue-13-plan.eval.ts @@ -4,7 +4,7 @@ import { defineEval } from "eve/evals"; import { includes, matches } from "eve/evals/expect"; import type { EveEvalToolCall } from "eve/evals"; -import { planningAgentOutputSchema } from "@agent/planning-agent"; +import { pinnedPlanningAgentOutputSchema } from "@agent/planning-agent"; type IssueFixture = { body: string; @@ -88,9 +88,9 @@ export default defineEval({ t.completed(); t.noFailedActions(); t.calledTool("emit_plan_artifact", { - output: (output: unknown) => planningAgentOutputSchema.safeParse(output).success, + output: (output: unknown) => pinnedPlanningAgentOutputSchema.safeParse(output).success, }); - t.check(artifact, matches(planningAgentOutputSchema)); + t.check(artifact, matches(pinnedPlanningAgentOutputSchema)); t.notCalledTool("write_file"); t.notCalledTool("web_fetch"); t.maxToolCalls(5); diff --git a/tests/unit/agent/planning-agent.test.ts b/tests/unit/agent/planning-agent.test.ts index b80ff2a..51dd05b 100644 --- a/tests/unit/agent/planning-agent.test.ts +++ b/tests/unit/agent/planning-agent.test.ts @@ -1,8 +1,9 @@ /** @vitest-environment node */ import { createPlanningAgentSeedPlan, - planningAgentOutputSchema, + pinnedPlanningAgentOutputSchema, planningAgentModelLabel, + planningAgentOutputSchema, } from "@agent/planning-agent"; const issue13Input = { @@ -18,6 +19,10 @@ const issue13Input = { ].join("\n"), labels: ["loop:development", "area:agents", "priority:p1"], milestone: null, + repositoryRevision: { + ref: "main", + commitSha: "a".repeat(40), + }, }; describe("Planning agent artifact contract", () => { @@ -26,7 +31,7 @@ describe("Planning agent artifact contract", () => { expect(planningAgentOutputSchema.parse(plan)).toEqual(plan); expect(plan.model).toBe(planningAgentModelLabel); - expect(plan.model).toBe("openai/gpt-5.5-xhigh"); + expect(plan.model).toBe("openai/gpt-5.6-sol-xhigh"); expect(plan.issue).toMatchObject({ number: 13, repositoryFullName: "ncolesummers/loopworks", @@ -38,6 +43,14 @@ describe("Planning agent artifact contract", () => { "Agent tools are narrow and auditable.", "Future model/prompt/tool changes have a path to eval coverage.", ]); + expect(plan.identity).toMatchObject({ + id: "plan:ncolesummers/loopworks#13", + sha256: expect.stringMatching(/^[a-f0-9]{64}$/), + }); + expect(plan.repositoryRevision).toEqual({ + ref: "main", + commitSha: "a".repeat(40), + }); expect(plan.stages.map((stage) => stage.key)).toEqual([ "resolve-issue", "plan-artifact", @@ -66,6 +79,13 @@ describe("Planning agent artifact contract", () => { ); }); + it("rejects an emitted planning artifact that has no inspected repository pin", () => { + const plan = createPlanningAgentSeedPlan({ ...issue13Input, repositoryRevision: null }); + + expect(planningAgentOutputSchema.safeParse(plan).success).toBe(true); + expect(pinnedPlanningAgentOutputSchema.safeParse(plan).success).toBe(false); + }); + it("documents the planning-only tool contract and fixture/eval coverage", () => { const plan = createPlanningAgentSeedPlan(issue13Input); @@ -85,6 +105,10 @@ describe("Planning agent artifact contract", () => { mutates: true, capability: expect.stringContaining("plan artifact"), }), + expect.objectContaining({ name: "prepare_repository_context", mutates: false }), + expect.objectContaining({ name: "list_repository_files", mutates: false }), + expect.objectContaining({ name: "search_repository", mutates: false }), + expect.objectContaining({ name: "read_repository_files", mutates: false }), ]), ); expect(plan.fixtureMode).toMatchObject({ diff --git a/tests/unit/agent/planning-eval.test.ts b/tests/unit/agent/planning-eval.test.ts index 9387a59..c2ee5c6 100644 --- a/tests/unit/agent/planning-eval.test.ts +++ b/tests/unit/agent/planning-eval.test.ts @@ -1,14 +1,13 @@ import { describe, expect, it } from "vitest"; - +import { createPlanningAgentSeedPlan } from "../../../agent/planning-agent"; import sandbox from "../../../agent/sandbox"; import { getPlanningArtifactCandidate, - planningEvalTimeoutMs, parsePlanningArtifactReply, + planningEvalTimeoutMs, readIssueFixture, resolveIssueFixturePath, } from "../../../evals/planning/issue-13-plan.eval"; -import { createPlanningAgentSeedPlan } from "../../../agent/planning-agent"; function resolveSandboxBackendName(): string { const backend = typeof sandbox.backend === "function" ? sandbox.backend() : sandbox.backend; @@ -40,11 +39,12 @@ describe("planning eval fixture loading", () => { labels: ["loop:development"], milestone: null, repositoryFullName: "ncolesummers/loopworks", + repositoryRevision: { commitSha: "a".repeat(40), ref: "main" }, title: "Initial Eve planning agent", }); expect(parsePlanningArtifactReply(JSON.stringify(artifact))).toMatchObject({ - model: "openai/gpt-5.5-xhigh", + model: "openai/gpt-5.6-sol-xhigh", issue: { number: 13 }, }); expect( diff --git a/tests/unit/agent/planning-tools.test.ts b/tests/unit/agent/planning-tools.test.ts index 772c07e..de29807 100644 --- a/tests/unit/agent/planning-tools.test.ts +++ b/tests/unit/agent/planning-tools.test.ts @@ -1,5 +1,5 @@ /** @vitest-environment node */ -import { readFile, readdir } from "node:fs/promises"; +import { readdir, readFile } from "node:fs/promises"; import { basename, join } from "node:path"; import { @@ -9,6 +9,7 @@ import { } from "@agent/lib/cli-inspection"; const eveFrameworkToolNames = new Set([ + "agent", "ask_question", "bash", "glob", diff --git a/tests/unit/loops/development-run.test.ts b/tests/unit/loops/development-run.test.ts index 2022661..41119aa 100644 --- a/tests/unit/loops/development-run.test.ts +++ b/tests/unit/loops/development-run.test.ts @@ -204,7 +204,7 @@ describe("agent-ready development loop run skeleton", () => { issue: expect.objectContaining({ acceptanceCriteria: ["Development-loop planning stores the issue acceptance criteria."], }), - model: "openai/gpt-5.5-xhigh", + model: "openai/gpt-5.6-sol-xhigh", stages: expect.arrayContaining([ expect.objectContaining({ key: "plan-artifact", From 032d7f4615ca4563d9a78072c4b7c709c4679aed Mon Sep 17 00:00:00 2001 From: Nate Summers Date: Sat, 11 Jul 2026 14:04:26 -0700 Subject: [PATCH 04/12] feat(agent): add test-writer subagent with receipt-verified red evidence Declared test-writer sibling subagent (openai/gpt-5.6-terra, xhigh) with an isolated deny-all sandbox. It turns an approved plan into focused failing tests and emits loopworks.test_plan.v1 plus loopworks.red_test_evidence.v1: AC-mapped tests, explicit fixtures, a bounded SHA-256-bound test-only unified patch, and HMAC execution receipts minted only by the guarded run_test_suite tool so unexecuted tests cannot self-attest as red. Includes a golden fixture and Eve eval. Part of #47. Co-Authored-By: Claude Fable 5 --- agent/subagents/test-writer/agent.ts | 13 + agent/subagents/test-writer/instructions.md | 28 ++ .../subagents/test-writer/lib/fixture-mode.ts | 20 ++ .../subagents/test-writer/lib/tool-policy.ts | 105 ++++++ agent/subagents/test-writer/sandbox.ts | 11 + .../test-writer/tools/ask_question.ts | 2 + agent/subagents/test-writer/tools/bash.ts | 2 + .../tools/emit_test_writing_artifacts.ts | 10 + agent/subagents/test-writer/tools/glob.ts | 2 + agent/subagents/test-writer/tools/grep.ts | 2 + .../tools/list_repository_files.ts | 34 ++ .../subagents/test-writer/tools/load_skill.ts | 3 + .../test-writer/tools/read_approved_plan.ts | 117 +++++++ .../subagents/test-writer/tools/read_file.ts | 2 + .../tools/read_repository_files.ts | 43 +++ .../test-writer/tools/run_test_suite.ts | 103 ++++++ .../test-writer/tools/search_repository.ts | 29 ++ agent/subagents/test-writer/tools/todo.ts | 2 + .../subagents/test-writer/tools/web_fetch.ts | 2 + .../subagents/test-writer/tools/web_search.ts | 2 + .../subagents/test-writer/tools/write_file.ts | 2 + .../test-writer/tools/write_test_files.ts | 109 ++++++ agent/test-writing-agent.ts | 329 ++++++++++++++++++ evals/test-writing/fixtures/issue-47.json | 12 + evals/test-writing/issue-47-red.eval.ts | 56 +++ tests/unit/agent/test-writing-agent.test.ts | 217 ++++++++++++ .../unit/agent/test-writing-discovery.test.ts | 103 ++++++ tests/unit/agent/test-writing-eval.test.ts | 23 ++ tests/unit/agent/test-writing-tools.test.ts | 88 +++++ 29 files changed, 1471 insertions(+) create mode 100644 agent/subagents/test-writer/agent.ts create mode 100644 agent/subagents/test-writer/instructions.md create mode 100644 agent/subagents/test-writer/lib/fixture-mode.ts create mode 100644 agent/subagents/test-writer/lib/tool-policy.ts create mode 100644 agent/subagents/test-writer/sandbox.ts create mode 100644 agent/subagents/test-writer/tools/ask_question.ts create mode 100644 agent/subagents/test-writer/tools/bash.ts create mode 100644 agent/subagents/test-writer/tools/emit_test_writing_artifacts.ts create mode 100644 agent/subagents/test-writer/tools/glob.ts create mode 100644 agent/subagents/test-writer/tools/grep.ts create mode 100644 agent/subagents/test-writer/tools/list_repository_files.ts create mode 100644 agent/subagents/test-writer/tools/load_skill.ts create mode 100644 agent/subagents/test-writer/tools/read_approved_plan.ts create mode 100644 agent/subagents/test-writer/tools/read_file.ts create mode 100644 agent/subagents/test-writer/tools/read_repository_files.ts create mode 100644 agent/subagents/test-writer/tools/run_test_suite.ts create mode 100644 agent/subagents/test-writer/tools/search_repository.ts create mode 100644 agent/subagents/test-writer/tools/todo.ts create mode 100644 agent/subagents/test-writer/tools/web_fetch.ts create mode 100644 agent/subagents/test-writer/tools/web_search.ts create mode 100644 agent/subagents/test-writer/tools/write_file.ts create mode 100644 agent/subagents/test-writer/tools/write_test_files.ts create mode 100644 agent/test-writing-agent.ts create mode 100644 evals/test-writing/fixtures/issue-47.json create mode 100644 evals/test-writing/issue-47-red.eval.ts create mode 100644 tests/unit/agent/test-writing-agent.test.ts create mode 100644 tests/unit/agent/test-writing-discovery.test.ts create mode 100644 tests/unit/agent/test-writing-eval.test.ts create mode 100644 tests/unit/agent/test-writing-tools.test.ts diff --git a/agent/subagents/test-writer/agent.ts b/agent/subagents/test-writer/agent.ts new file mode 100644 index 0000000..76127e9 --- /dev/null +++ b/agent/subagents/test-writer/agent.ts @@ -0,0 +1,13 @@ +import { defineAgent } from "eve"; + +import { testWritingAgentOutputSchema } from "../../test-writing-agent"; + +export default defineAgent({ + description: "Write focused failing tests from an approved plan and emit reusable red evidence.", + model: "openai/gpt-5.6-terra", + modelContextWindowTokens: 400_000, + modelOptions: { + providerOptions: { openai: { reasoningEffort: "xhigh" } }, + }, + outputSchema: testWritingAgentOutputSchema, +}); diff --git a/agent/subagents/test-writer/instructions.md b/agent/subagents/test-writer/instructions.md new file mode 100644 index 0000000..460be4b --- /dev/null +++ b/agent/subagents/test-writer/instructions.md @@ -0,0 +1,28 @@ +# Loopworks Test-Writer Subagent + +Accept only an approved, digest-bound plan from the stage orchestrator. Write +the smallest focused tests needed to cover every acceptance criterion and run +only the planned focused commands until they fail for the expected assertions. + +Emit a versioned automated test plan, explicit seed/fixture data, a bounded +test-only unified patch, and red evidence with redacted output references. +Infrastructure, setup, timeout, crash, passing, or unrelated failures are not +valid red evidence. + +Use `read_approved_plan`, inspect only necessary repository files with +`read_repository_files`, write the complete test set once with +`write_test_files`, and pass the returned patch digest into every +`run_test_suite` call. The emitted patch must be the exact patch returned by +that write tool. + +Before authoring tests, use `list_repository_files`, `search_repository`, and +bounded `read_repository_files` calls to discover applicable `AGENTS.md`, +adjacent tests, fixture conventions, and deterministic commands at the approved +commit. Do not guess paths or use web access. + +Call `read_approved_plan` with only the durable run ID. Its control-plane tool +loads and validates the exact plan row and approval; never reconstruct approval +evidence from prompt text. + +Do not edit production source, change branches, mutate GitHub, transition run +state, delegate, or log raw plans, source, patches, fixtures, or command output. diff --git a/agent/subagents/test-writer/lib/fixture-mode.ts b/agent/subagents/test-writer/lib/fixture-mode.ts new file mode 100644 index 0000000..7aa6618 --- /dev/null +++ b/agent/subagents/test-writer/lib/fixture-mode.ts @@ -0,0 +1,20 @@ +import { isProductionRuntime, isTruthyEnvValue } from "@/lib/runtime"; + +export type TestWriterFixtureMode = + | { enabled: true; reason: "explicit_non_production_fixture" } + | { + enabled: false; + reason: "not_requested" | "production_runtime_blocked"; + }; + +export function resolveTestWriterFixtureMode( + env: Partial = process.env, +): TestWriterFixtureMode { + if (!isTruthyEnvValue(env.LOOPWORKS_EVE_TEST_WRITER_FIXTURE_MODE)) { + return { enabled: false, reason: "not_requested" }; + } + if (isProductionRuntime(env)) { + return { enabled: false, reason: "production_runtime_blocked" }; + } + return { enabled: true, reason: "explicit_non_production_fixture" }; +} diff --git a/agent/subagents/test-writer/lib/tool-policy.ts b/agent/subagents/test-writer/lib/tool-policy.ts new file mode 100644 index 0000000..35026e8 --- /dev/null +++ b/agent/subagents/test-writer/lib/tool-policy.ts @@ -0,0 +1,105 @@ +import { createHash, createHmac, timingSafeEqual } from "node:crypto"; + +import { + isAllowedFocusedTestCommand, + isAllowedTestArtifactPath, +} from "../../../test-writing-agent"; + +export type PlannedTestCommand = { + path: string; + type: "unit" | "integration" | "browser"; +}; + +export type TestExecutionReceiptPayload = { + command: string; + exitCode: number; + expectedAssertions: string[]; + outcome: "expected_failure" | "invalid_failure"; + outputSha256: string; + patchSha256: string; + testPaths: string[]; +}; + +export function assertAllowedTestFiles(files: readonly { path: string }[]): void { + if (files.length === 0) throw new Error("At least one test file is required."); + for (const file of files) { + if (!isAllowedTestArtifactPath(file.path)) { + throw new Error(`Unsafe test artifact path: ${file.path}`); + } + } +} + +export function assertCommandMatchesPlannedTests( + command: string, + tests: readonly PlannedTestCommand[], +): void { + assertAllowedTestFiles(tests); + if (!isAllowedFocusedTestCommand(command) || tests.length === 0) { + throw new Error("Command is not an allowed focused test command."); + } + const browser = tests.every((test) => test.type === "browser"); + const nonBrowser = tests.every((test) => test.type !== "browser"); + if (!browser && !nonBrowser) + throw new Error("Browser and non-browser tests must run separately."); + if (tests.length !== 1) throw new Error("Each focused red command must target exactly one test."); + const prefix = browser ? "bunx playwright test" : "bun run test --"; + const expected = `${prefix} ${tests.map((test) => test.path).join(" ")}`; + if (command.trim().replace(/\s+/g, " ") !== expected) { + throw new Error("Command must exactly match the approved test paths."); + } +} + +export function redactTestOutput(value: string): string { + return value + .replace(/(authorization:\s*bearer\s+)\S+/gi, "$1[REDACTED]") + .replace(/\b(?:gh[pousr]_|sk-)[A-Za-z0-9_-]+\b/g, "[REDACTED]") + .replace(/\bgithub_pat_[A-Za-z0-9_]+\b/g, "[REDACTED]") + .replace(/\bAKIA[0-9A-Z]{16}\b/g, "[REDACTED]") + .replace(/\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\b/g, "[REDACTED]") + .replace( + /((?:password|secret|token|api[_-]?key|cookie|set-cookie)\s*[=:]\s*)\S+/gi, + "$1[REDACTED]", + ) + .replace(/([a-z][a-z0-9+.-]*:\/\/[^\s:/]+:)[^@\s]+@/gi, "$1[REDACTED]@"); +} + +export function classifyTestRun(input: { + exitCode: number; + expectedAssertions: readonly string[]; + output: string; + testPaths: readonly string[]; +}): "expected_failure" | "invalid_failure" { + const infrastructureFailure = + /(cannot find module|module not found|failed to load|command not found|no tests? found|syntaxerror|timed? out|timeout|killed|segmentation fault|out of memory|unhandled error)/i.test( + input.output, + ); + const assertionsMatched = input.expectedAssertions.every((assertion) => + input.output.includes(assertion), + ); + const pathsMatched = input.testPaths.every((path) => input.output.includes(path)); + return input.exitCode > 0 && assertionsMatched && pathsMatched && !infrastructureFailure + ? "expected_failure" + : "invalid_failure"; +} + +export function createTestExecutionReceipt( + payload: TestExecutionReceiptPayload, + secret: string, +): string { + if (!secret.trim()) throw new Error("Test execution receipt secret is required."); + return createHmac("sha256", secret).update(JSON.stringify(payload)).digest("hex"); +} + +export function verifyTestExecutionReceipt( + payload: TestExecutionReceiptPayload, + receipt: string, + secret: string, +): boolean { + const expected = Buffer.from(createTestExecutionReceipt(payload, secret), "hex"); + const actual = Buffer.from(receipt, "hex"); + return actual.length === expected.length && timingSafeEqual(actual, expected); +} + +export function sha256(value: string): string { + return createHash("sha256").update(value).digest("hex"); +} diff --git a/agent/subagents/test-writer/sandbox.ts b/agent/subagents/test-writer/sandbox.ts new file mode 100644 index 0000000..71b113c --- /dev/null +++ b/agent/subagents/test-writer/sandbox.ts @@ -0,0 +1,11 @@ +import { defaultBackend, defineSandbox } from "eve/sandbox"; + +export default defineSandbox({ + backend: defaultBackend({ + docker: { networkPolicy: "deny-all" }, + microsandbox: { networkPolicy: "deny-all" }, + vercel: { networkPolicy: "deny-all" }, + }), + description: + "Isolated real-binary test-writing sandbox. Checkout is commit-pinned and egress is disabled before tests run.", +}); diff --git a/agent/subagents/test-writer/tools/ask_question.ts b/agent/subagents/test-writer/tools/ask_question.ts new file mode 100644 index 0000000..a908e44 --- /dev/null +++ b/agent/subagents/test-writer/tools/ask_question.ts @@ -0,0 +1,2 @@ +import { disableTool } from "eve/tools"; +export default disableTool(); diff --git a/agent/subagents/test-writer/tools/bash.ts b/agent/subagents/test-writer/tools/bash.ts new file mode 100644 index 0000000..a908e44 --- /dev/null +++ b/agent/subagents/test-writer/tools/bash.ts @@ -0,0 +1,2 @@ +import { disableTool } from "eve/tools"; +export default disableTool(); diff --git a/agent/subagents/test-writer/tools/emit_test_writing_artifacts.ts b/agent/subagents/test-writer/tools/emit_test_writing_artifacts.ts new file mode 100644 index 0000000..22962d5 --- /dev/null +++ b/agent/subagents/test-writer/tools/emit_test_writing_artifacts.ts @@ -0,0 +1,10 @@ +import { defineTool } from "eve/tools"; + +import { testWritingAgentOutputSchema } from "../../../test-writing-agent"; + +export default defineTool({ + description: "Emit the final validated test-plan and red-evidence artifacts.", + inputSchema: testWritingAgentOutputSchema, + outputSchema: testWritingAgentOutputSchema, + execute: (input) => testWritingAgentOutputSchema.parse(input), +}); diff --git a/agent/subagents/test-writer/tools/glob.ts b/agent/subagents/test-writer/tools/glob.ts new file mode 100644 index 0000000..a908e44 --- /dev/null +++ b/agent/subagents/test-writer/tools/glob.ts @@ -0,0 +1,2 @@ +import { disableTool } from "eve/tools"; +export default disableTool(); diff --git a/agent/subagents/test-writer/tools/grep.ts b/agent/subagents/test-writer/tools/grep.ts new file mode 100644 index 0000000..a908e44 --- /dev/null +++ b/agent/subagents/test-writer/tools/grep.ts @@ -0,0 +1,2 @@ +import { disableTool } from "eve/tools"; +export default disableTool(); diff --git a/agent/subagents/test-writer/tools/list_repository_files.ts b/agent/subagents/test-writer/tools/list_repository_files.ts new file mode 100644 index 0000000..5fef38f --- /dev/null +++ b/agent/subagents/test-writer/tools/list_repository_files.ts @@ -0,0 +1,34 @@ +import { defineTool } from "eve/tools"; +import { z } from "zod"; +import { assertSafeRepositoryGlob } from "../../../lib/repository-inspection"; +import { + listRepositoryFiles, + repositoryListOutputSchema, +} from "../../../lib/repository-inspection-runtime"; +import { resolveTestWriterFixtureMode } from "../lib/fixture-mode"; + +const glob = z.string().refine((value) => { + try { + assertSafeRepositoryGlob(value); + return true; + } catch { + return false; + } +}, "Unsafe repository glob."); + +export default defineTool({ + description: "List bounded regular-file paths from the approved pinned Git commit.", + inputSchema: z.object({ patterns: z.array(glob).min(1).max(5) }), + outputSchema: repositoryListOutputSchema, + async execute({ patterns }, ctx) { + if (resolveTestWriterFixtureMode().enabled) { + return { + commitSha: "a".repeat(40), + fixtureMode: true, + paths: ["AGENTS.md", "tests/AGENTS.md", "tests/unit/agent/example.test.ts"], + truncated: false, + }; + } + return listRepositoryFiles(await ctx.getSandbox(), patterns); + }, +}); diff --git a/agent/subagents/test-writer/tools/load_skill.ts b/agent/subagents/test-writer/tools/load_skill.ts new file mode 100644 index 0000000..04bd054 --- /dev/null +++ b/agent/subagents/test-writer/tools/load_skill.ts @@ -0,0 +1,3 @@ +import { disableTool } from "eve/tools"; + +export default disableTool(); diff --git a/agent/subagents/test-writer/tools/read_approved_plan.ts b/agent/subagents/test-writer/tools/read_approved_plan.ts new file mode 100644 index 0000000..40aaf43 --- /dev/null +++ b/agent/subagents/test-writer/tools/read_approved_plan.ts @@ -0,0 +1,117 @@ +import { and, eq } from "drizzle-orm"; +import { defineTool } from "eve/tools"; +import { z } from "zod"; +import { db } from "@/db/client"; +import { agentPlans, approvals } from "@/db/schema"; + +import { + computePlanningArtifactDigest, + createPlanningAgentSeedPlan, + planningAgentOutputSchema, +} from "../../../planning-agent"; +import { resolveTestWriterFixtureMode } from "../lib/fixture-mode"; + +const inputSchema = z.object({ + runId: z.string().uuid(), +}); + +export default defineTool({ + description: "Validate an exact approved plan and prepare its commit-pinned sandbox checkout.", + inputSchema, + async execute(input, ctx) { + const fixtureMode = resolveTestWriterFixtureMode(); + const fixturePlan = fixtureMode.enabled + ? createPlanningAgentSeedPlan({ + body: [ + "## Acceptance Criteria", + "- Red evidence is tied to the plan acceptance criteria.", + "- The automated test plan and fixture data are reusable downstream.", + "- Future model, prompt, and tool changes have eval coverage.", + "- ADR 0015 records the orchestrator, sibling subagents, isolated sandboxes, and artifact handoff.", + ].join("\n"), + issueNumber: 47, + labels: ["area:agents"], + milestone: null, + repositoryFullName: "ncolesummers/loopworks", + repositoryRevision: { commitSha: "a".repeat(40), ref: "main" }, + title: "Test-writing subagent for the development loop", + }) + : undefined; + const planRows = fixtureMode.enabled + ? [{ id: "00000000-0000-4000-8000-000000000147", plan: fixturePlan, status: "approved" }] + : await db.select().from(agentPlans).where(eq(agentPlans.runId, input.runId)); + const planApprovals = fixtureMode.enabled + ? [ + { + metadata: { + planId: "00000000-0000-4000-8000-000000000147", + planSha256: fixturePlan?.identity.sha256, + }, + status: "approved", + }, + ] + : await db + .select() + .from(approvals) + .where(and(eq(approvals.runId, input.runId), eq(approvals.scope, "plan-review"))); + if (planRows.length !== 1 || planApprovals.length !== 1) { + throw new Error("Test writing requires exactly one durable plan and plan review."); + } + const planRow = planRows[0]; + const approval = planApprovals[0]; + const plan = planningAgentOutputSchema.parse(planRow?.plan); + if (!plan.repositoryRevision) throw new Error("Approved plan must pin a repository revision."); + if ( + planRow?.status !== "approved" || + approval?.status !== "approved" || + approval.metadata?.planId !== planRow.id || + approval.metadata?.planSha256 !== plan.identity.sha256 || + computePlanningArtifactDigest(plan) !== plan.identity.sha256 + ) { + throw new Error("Approval is not bound to the exact planning artifact."); + } + if (!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(plan.issue.repositoryFullName)) { + throw new Error("Repository name is not safe for checkout."); + } + + if (!fixtureMode.enabled) { + const sandbox = await ctx.getSandbox(); + const repoUrl = `https://github.com/${plan.issue.repositoryFullName}.git`; + await sandbox.setNetworkPolicy({ + allow: ["github.com", "objects.githubusercontent.com", "registry.npmjs.org"], + }); + try { + const command = [ + `git clone --filter=blob:none ${JSON.stringify(repoUrl)} repo`, + `cd repo`, + `git checkout --detach ${plan.repositoryRevision.commitSha}`, + `command -v bun`, + `bun install --frozen-lockfile --ignore-scripts`, + `test -z "$(git status --porcelain)"`, + ].join(" && "); + const result = await sandbox.run({ command, abortSignal: AbortSignal.timeout(120_000) }); + if (result.exitCode !== 0) throw new Error("Commit-pinned repository checkout failed."); + await sandbox.run({ + command: "mkdir -p .loopworks", + abortSignal: AbortSignal.timeout(5_000), + }); + await sandbox.writeTextFile({ + path: ".loopworks/repository-commit", + content: plan.repositoryRevision.commitSha, + }); + } finally { + await sandbox.setNetworkPolicy("deny-all"); + } + } + + return { + fixtureMode: fixtureMode.enabled, + planId: plan.identity.id, + planRowId: planRow.id, + planSha256: plan.identity.sha256, + repositoryFullName: plan.issue.repositoryFullName, + repositoryRevision: plan.repositoryRevision, + runId: input.runId, + }; + }, +}); diff --git a/agent/subagents/test-writer/tools/read_file.ts b/agent/subagents/test-writer/tools/read_file.ts new file mode 100644 index 0000000..a908e44 --- /dev/null +++ b/agent/subagents/test-writer/tools/read_file.ts @@ -0,0 +1,2 @@ +import { disableTool } from "eve/tools"; +export default disableTool(); diff --git a/agent/subagents/test-writer/tools/read_repository_files.ts b/agent/subagents/test-writer/tools/read_repository_files.ts new file mode 100644 index 0000000..fb8e1a0 --- /dev/null +++ b/agent/subagents/test-writer/tools/read_repository_files.ts @@ -0,0 +1,43 @@ +import { defineTool } from "eve/tools"; +import { z } from "zod"; +import { + readRepositoryFiles, + repositoryReadOutputSchema, +} from "../../../lib/repository-inspection-runtime"; +import { resolveTestWriterFixtureMode } from "../lib/fixture-mode"; + +const request = z.union([ + z.string().min(1), + z.object({ + path: z.string().min(1), + startLine: z.number().int().positive().default(1), + endLine: z.number().int().positive().optional(), + }), +]); + +export default defineTool({ + description: "Read bounded line ranges from regular files at the approved pinned Git commit.", + inputSchema: z.object({ files: z.array(request).min(1).max(20) }), + outputSchema: repositoryReadOutputSchema, + async execute({ files }, ctx) { + const normalized = files.map((entry) => + typeof entry === "string" ? { path: entry, startLine: 1 } : entry, + ); + if (resolveTestWriterFixtureMode().enabled) { + return { + commitSha: "a".repeat(40), + fixtureMode: true, + files: normalized.map((entry) => ({ + path: entry.path, + startLine: entry.startLine, + requestedEndLine: entry.endLine ?? entry.startLine + 399, + returnedEndLine: entry.startLine, + content: "// Deterministic fixture repository context.", + truncated: false, + })), + truncated: false, + }; + } + return readRepositoryFiles(await ctx.getSandbox(), normalized); + }, +}); diff --git a/agent/subagents/test-writer/tools/run_test_suite.ts b/agent/subagents/test-writer/tools/run_test_suite.ts new file mode 100644 index 0000000..88fa1d2 --- /dev/null +++ b/agent/subagents/test-writer/tools/run_test_suite.ts @@ -0,0 +1,103 @@ +import { SpanStatusCode } from "@opentelemetry/api"; +import { defineTool } from "eve/tools"; +import { z } from "zod"; +import { startLoopworksSpan } from "@/lib/observability/trace-context"; +import { resolveTestWriterFixtureMode } from "../lib/fixture-mode"; +import { + assertCommandMatchesPlannedTests, + classifyTestRun, + createTestExecutionReceipt, + redactTestOutput, + sha256, +} from "../lib/tool-policy"; + +const inputSchema = z.object({ + command: z.string().min(1), + tests: z + .array(z.object({ path: z.string().min(1), type: z.enum(["unit", "integration", "browser"]) })) + .min(1), + expectedAssertions: z.array(z.string().min(1)).min(1), + patchSha256: z.string().regex(/^[a-f0-9]{64}$/), +}); + +export default defineTool({ + description: "Run one exact approved focused test command and persist only redacted evidence.", + inputSchema, + async execute(input, ctx) { + assertCommandMatchesPlannedTests(input.command, input.tests); + const startedAt = Date.now(); + const span = startLoopworksSpan("loopworks.test_writing.execution", { + attributes: { + "loopworks.agent": "test-writer", + "loopworks.stage": "test-writing", + "loopworks.test.count": input.tests.length, + }, + }); + try { + const sandbox = await ctx.getSandbox(); + const fixtureMode = resolveTestWriterFixtureMode(); + const result = fixtureMode.enabled + ? { + exitCode: 1, + stdout: `${input.tests[0]?.path}\n${input.expectedAssertions.join("\n")}`, + stderr: "", + } + : await sandbox.run({ + command: `cd repo && ${input.command}`, + abortSignal: AbortSignal.timeout(120_000), + }); + const redacted = redactTestOutput(`${result.stdout}\n${result.stderr}`); + const digest = sha256(redacted); + const uri = `.loopworks/red-evidence/${digest}.log`; + await sandbox.run({ command: "mkdir -p .loopworks/red-evidence" }); + await sandbox.writeTextFile({ path: uri, content: redacted }); + const testPaths = input.tests.map(({ path }) => path); + const outcome = classifyTestRun({ + exitCode: result.exitCode, + expectedAssertions: input.expectedAssertions, + output: redacted, + testPaths, + }); + const receiptSecret = process.env.LOOPWORKS_EVE_TEST_RECEIPT_SECRET; + if (!receiptSecret) throw new Error("Test execution receipt secret is not configured."); + const executionReceipt = createTestExecutionReceipt( + { + command: input.command, + exitCode: result.exitCode, + expectedAssertions: input.expectedAssertions, + outcome, + outputSha256: digest, + patchSha256: input.patchSha256, + testPaths, + }, + receiptSecret, + ); + const durationMs = Math.max(0, Date.now() - startedAt); + span.setAttributes({ + "loopworks.duration_ms": durationMs, + "loopworks.outcome": outcome, + }); + span.setStatus({ + code: outcome === "expected_failure" ? SpanStatusCode.OK : SpanStatusCode.ERROR, + }); + return { + durationMs, + exitCode: result.exitCode, + outcome, + executionReceipt, + outputReference: { + uri: `artifact://sandbox/${sandbox.id}/${uri}`, + sha256: digest, + byteCount: Buffer.byteLength(redacted), + redacted: true, + }, + }; + } catch (error) { + span.recordException(error instanceof Error ? error : String(error)); + span.setStatus({ code: SpanStatusCode.ERROR }); + throw error; + } finally { + span.end(); + } + }, +}); diff --git a/agent/subagents/test-writer/tools/search_repository.ts b/agent/subagents/test-writer/tools/search_repository.ts new file mode 100644 index 0000000..6a6058a --- /dev/null +++ b/agent/subagents/test-writer/tools/search_repository.ts @@ -0,0 +1,29 @@ +import { defineTool } from "eve/tools"; +import { z } from "zod"; +import { + repositorySearchOutputSchema, + searchRepository, +} from "../../../lib/repository-inspection-runtime"; +import { resolveTestWriterFixtureMode } from "../lib/fixture-mode"; + +export default defineTool({ + description: + "Search regular files from the approved pinned Git commit with bounded regex output.", + inputSchema: z.object({ + pattern: z.string().min(1).max(256), + paths: z.array(z.string()).min(1).max(5), + }), + outputSchema: repositorySearchOutputSchema, + async execute(input, ctx) { + if (resolveTestWriterFixtureMode().enabled) { + return { + commitSha: "a".repeat(40), + fixtureMode: true, + content: "tests/unit/agent/example.test.ts:1:fixture match", + matchCount: 1, + truncated: false, + }; + } + return searchRepository(await ctx.getSandbox(), input); + }, +}); diff --git a/agent/subagents/test-writer/tools/todo.ts b/agent/subagents/test-writer/tools/todo.ts new file mode 100644 index 0000000..a908e44 --- /dev/null +++ b/agent/subagents/test-writer/tools/todo.ts @@ -0,0 +1,2 @@ +import { disableTool } from "eve/tools"; +export default disableTool(); diff --git a/agent/subagents/test-writer/tools/web_fetch.ts b/agent/subagents/test-writer/tools/web_fetch.ts new file mode 100644 index 0000000..a908e44 --- /dev/null +++ b/agent/subagents/test-writer/tools/web_fetch.ts @@ -0,0 +1,2 @@ +import { disableTool } from "eve/tools"; +export default disableTool(); diff --git a/agent/subagents/test-writer/tools/web_search.ts b/agent/subagents/test-writer/tools/web_search.ts new file mode 100644 index 0000000..a908e44 --- /dev/null +++ b/agent/subagents/test-writer/tools/web_search.ts @@ -0,0 +1,2 @@ +import { disableTool } from "eve/tools"; +export default disableTool(); diff --git a/agent/subagents/test-writer/tools/write_file.ts b/agent/subagents/test-writer/tools/write_file.ts new file mode 100644 index 0000000..a908e44 --- /dev/null +++ b/agent/subagents/test-writer/tools/write_file.ts @@ -0,0 +1,2 @@ +import { disableTool } from "eve/tools"; +export default disableTool(); diff --git a/agent/subagents/test-writer/tools/write_test_files.ts b/agent/subagents/test-writer/tools/write_test_files.ts new file mode 100644 index 0000000..5dcb302 --- /dev/null +++ b/agent/subagents/test-writer/tools/write_test_files.ts @@ -0,0 +1,109 @@ +import { defineTool } from "eve/tools"; +import { z } from "zod"; +import { buildGitTreeEntryCommand } from "../../../lib/repository-inspection"; +import { readPinnedRepositoryCommit } from "../../../lib/repository-inspection-runtime"; +import { resolveTestWriterFixtureMode } from "../lib/fixture-mode"; +import { assertAllowedTestFiles, sha256 } from "../lib/tool-policy"; + +const inputSchema = z.object({ + files: z.array(z.object({ path: z.string().min(1), content: z.string().max(256 * 1024) })).min(1), +}); + +function symlinkGuardCommand(path: string): string { + const segments = path.split("/").slice(0, -1); + const parents = segments.map((_, index) => `repo/${segments.slice(0, index + 1).join("/")}`); + return parents.map((parent) => `test ! -L ${JSON.stringify(parent)}`).join(" && "); +} + +export default defineTool({ + description: "Write only approved test, eval, fixture, or story files in the isolated checkout.", + inputSchema, + async execute(input, ctx) { + assertAllowedTestFiles(input.files); + if (resolveTestWriterFixtureMode().enabled) { + const content = input.files + .map((file) => { + const lines = file.content.split("\n"); + return [ + `diff --git a/${file.path} b/${file.path}`, + "new file mode 100644", + "--- /dev/null", + `+++ b/${file.path}`, + `@@ -0,0 +1,${lines.length} @@`, + ...lines.map((line) => `+${line}`), + ].join("\n"); + }) + .join("\n"); + const byteCount = Buffer.byteLength(content); + if (byteCount > 256 * 1024) throw new Error("Test-only patch exceeds 256 KiB."); + return { + patch: { + byteCount, + content, + paths: input.files.map(({ path }) => path), + sha256: sha256(content), + }, + written: input.files.map((file) => ({ + byteCount: Buffer.byteLength(file.content), + path: file.path, + sha256: sha256(file.content), + })), + }; + } + const sandbox = await ctx.getSandbox(); + const pinnedCommit = await readPinnedRepositoryCommit(sandbox); + const written = []; + const tracked = new Map(); + for (const file of input.files) { + const treeEntry = await sandbox.run({ + command: buildGitTreeEntryCommand(pinnedCommit, file.path), + abortSignal: AbortSignal.timeout(5_000), + }); + tracked.set(file.path, treeEntry.exitCode === 0 && treeEntry.stdout.trim().length > 0); + const target = `repo/${file.path}`; + const directory = target.slice(0, target.lastIndexOf("/")); + const ancestorGuard = symlinkGuardCommand(file.path); + const guard = await sandbox.run({ + command: `${ancestorGuard} && mkdir -p ${JSON.stringify(directory)} && ${ancestorGuard}`, + }); + if (guard.exitCode !== 0) throw new Error(`Unsafe symlink encountered for ${file.path}.`); + await sandbox.writeTextFile({ path: target, content: file.content }); + written.push({ + path: file.path, + byteCount: Buffer.byteLength(file.content), + sha256: sha256(file.content), + }); + } + const paths = input.files.map(({ path }) => path); + const check = await sandbox.run({ + command: "cd repo && git diff --check", + abortSignal: AbortSignal.timeout(10_000), + }); + if (check.exitCode !== 0) throw new Error("Test-only patch could not be produced safely."); + const diffs = await Promise.all( + paths.map(async (path) => { + const command = tracked.get(path) + ? `cd repo && git diff --no-ext-diff --no-color ${pinnedCommit} -- ${JSON.stringify(path)}` + : `cd repo && git diff --no-index --no-color -- /dev/null ${JSON.stringify(path)}`; + const result = await sandbox.run({ + command, + abortSignal: AbortSignal.timeout(10_000), + }); + if (tracked.get(path) ? result.exitCode !== 0 : ![0, 1].includes(result.exitCode)) { + throw new Error("Test-only patch could not be produced safely."); + } + return result.stdout; + }), + ); + const content = diffs.join("\n"); + if (!content.trim()) { + throw new Error("Test-only patch could not be produced safely."); + } + const byteCount = Buffer.byteLength(content); + if (byteCount > 256 * 1024) throw new Error("Test-only patch exceeds 256 KiB."); + return { + patch: { content, sha256: sha256(content), byteCount, paths }, + written, + }; + }, +}); diff --git a/agent/test-writing-agent.ts b/agent/test-writing-agent.ts new file mode 100644 index 0000000..c06af4a --- /dev/null +++ b/agent/test-writing-agent.ts @@ -0,0 +1,329 @@ +import { createHash } from "node:crypto"; + +import { z } from "zod"; + +export const testWriterModelLabel = "openai/gpt-5.6-terra-xhigh"; +export const testPlanSchemaId = "loopworks.test_plan.v1"; +export const redTestEvidenceSchemaId = "loopworks.red_test_evidence.v1"; +export const maxTestPatchBytes = 256 * 1024; + +export function computeTestPlanDigest(value: unknown): string { + return createHash("sha256").update(JSON.stringify(value)).digest("hex"); +} + +const safeTestPathPattern = + /(^|\/)(tests?|evals?|fixtures?|stories)(\/|$)|\.(test|spec|stories)\.[^/]+$/; + +export function isAllowedTestArtifactPath(path: string): boolean { + const normalized = path.trim().replaceAll("\\", "/"); + return ( + normalized.length > 0 && + !normalized.startsWith("/") && + !/^[A-Za-z]:\//.test(normalized) && + !normalized.includes("//") && + !/[;&|`$<>\n\r\0]/.test(normalized) && + !normalized.split("/").includes(".") && + !normalized.split("/").includes("..") && + !normalized.startsWith(".git/") && + safeTestPathPattern.test(normalized) + ); +} + +export function isAllowedFocusedTestCommand(command: string): boolean { + if (/[;&|`$<>\n\r]/.test(command)) return false; + const normalized = command.trim().replace(/\s+/g, " "); + return /^(bun run test --|bunx playwright test)(\s+[^-\s][^\s]*)+$/.test(normalized); +} + +function duplicateIds(values: readonly string[]): string[] { + const seen = new Set(); + const duplicates = new Set(); + for (const value of values) { + if (seen.has(value)) duplicates.add(value); + seen.add(value); + } + return [...duplicates]; +} + +function containsSecretLikeFixtureData(value: unknown): boolean { + if (!value || typeof value !== "object") return false; + return Object.entries(value).some(([key, entry]) => { + const secretKey = /(password|passwd|secret|token|api[_-]?key|private[_-]?key)/i.test(key); + const secretValue = + typeof entry === "string" && /(bearer\s+|gh[pousr]_|sk-[a-z0-9])/i.test(entry); + return secretKey || secretValue; + }); +} + +function patchHeaderPaths(content: string): string[] { + const paths = new Set(); + for (const line of content.split(/\r?\n/)) { + const diffMatch = line.match(/^diff --git a\/(.+) b\/(.+)$/); + if (diffMatch) { + if (diffMatch[1]) paths.add(diffMatch[1]); + if (diffMatch[2]) paths.add(diffMatch[2]); + continue; + } + const headerMatch = line.match(/^(?:---|\+\+\+) (?:[ab]\/(.+)|\/dev\/null)$/); + if (headerMatch?.[1]) paths.add(headerMatch[1]); + } + return [...paths]; +} + +const sha256Schema = z.string().regex(/^[a-f0-9]{64}$/); +const identifierSchema = z.string().regex(/^[a-z0-9][a-z0-9-]*$/); + +export const testPlanArtifactSchema = z + .object({ + version: z.literal(1), + schemaId: z.literal(testPlanSchemaId), + plan: z.object({ + id: z.string().min(1), + sha256: sha256Schema, + repositoryFullName: z.string().min(1), + commitSha: z.string().regex(/^[a-f0-9]{40}$/), + }), + acceptanceCriteria: z + .array(z.object({ id: identifierSchema, text: z.string().min(1) }).strict()) + .min(1), + tests: z + .array( + z + .object({ + id: identifierSchema, + acceptanceCriterionIds: z.array(identifierSchema).min(1), + type: z.enum(["unit", "integration", "browser"]), + path: z.string().refine(isAllowedTestArtifactPath, "Unsafe test artifact path."), + command: z.string().refine(isAllowedFocusedTestCommand, "Unsafe focused test command."), + steps: z.array(z.string().min(1)).min(1), + expectedFailure: z.object({ kind: z.literal("assertion"), message: z.string().min(1) }), + fixtureIds: z.array(identifierSchema).default([]), + }) + .strict(), + ) + .min(1), + fixtures: z.array( + z + .object({ + id: identifierSchema, + kind: z.enum(["fixture", "seed"]), + description: z.string().min(1), + data: z.record(z.string(), z.union([z.string(), z.number(), z.boolean(), z.null()])), + }) + .strict(), + ), + patch: z + .object({ + format: z.literal("unified-diff"), + content: z.string().min(1), + sha256: sha256Schema, + byteCount: z.number().int().positive().max(maxTestPatchBytes), + paths: z.array(z.string().refine(isAllowedTestArtifactPath)).min(1), + }) + .strict(), + }) + .strict() + .superRefine((artifact, ctx) => { + const criterionIds = new Set(artifact.acceptanceCriteria.map((criterion) => criterion.id)); + const fixtureIds = new Set(artifact.fixtures.map((fixture) => fixture.id)); + const coveredCriteria = new Set(); + for (const duplicate of duplicateIds(artifact.acceptanceCriteria.map(({ id }) => id))) { + ctx.addIssue({ code: "custom", message: `Duplicate acceptance criterion ${duplicate}.` }); + } + for (const duplicate of duplicateIds(artifact.tests.map(({ id }) => id))) { + ctx.addIssue({ code: "custom", message: `Duplicate test ${duplicate}.` }); + } + for (const duplicate of duplicateIds(artifact.fixtures.map(({ id }) => id))) { + ctx.addIssue({ code: "custom", message: `Duplicate fixture ${duplicate}.` }); + } + for (const test of artifact.tests) { + for (const criterionId of test.acceptanceCriterionIds) { + if (!criterionIds.has(criterionId)) { + ctx.addIssue({ code: "custom", message: `Unknown acceptance criterion ${criterionId}.` }); + } + coveredCriteria.add(criterionId); + } + for (const fixtureId of test.fixtureIds) { + if (!fixtureIds.has(fixtureId)) { + ctx.addIssue({ code: "custom", message: `Unknown fixture ${fixtureId}.` }); + } + } + } + for (const criterionId of criterionIds) { + if (!coveredCriteria.has(criterionId)) { + ctx.addIssue({ + code: "custom", + message: `Acceptance criterion ${criterionId} is uncovered.`, + }); + } + } + if (artifact.fixtures.some((fixture) => containsSecretLikeFixtureData(fixture.data))) { + ctx.addIssue({ code: "custom", message: "Artifact contains secret-like fixture data." }); + } + const declaredPaths = new Set(artifact.patch.paths); + const parsedPatchPaths = patchHeaderPaths(artifact.patch.content); + if (parsedPatchPaths.length === 0) { + ctx.addIssue({ code: "custom", message: "Patch must contain a parseable diff header." }); + } + if ( + /^(?:rename|copy) (?:from|to) |^GIT binary patch$|^(?:new file mode|old mode|new mode) 120000$/m.test( + artifact.patch.content, + ) + ) { + ctx.addIssue({ + code: "custom", + message: "Patch contains a forbidden link, rename, copy, or binary operation.", + }); + } + for (const path of parsedPatchPaths) { + if (!declaredPaths.has(path) || !isAllowedTestArtifactPath(path)) { + ctx.addIssue({ + code: "custom", + message: `Patch contains undeclared or unsafe path ${path}.`, + }); + } + } + for (const path of declaredPaths) { + if (!parsedPatchPaths.includes(path)) { + ctx.addIssue({ + code: "custom", + message: `Declared patch path ${path} is missing from diff headers.`, + }); + } + } + const bytes = Buffer.byteLength(artifact.patch.content, "utf8"); + if (bytes !== artifact.patch.byteCount) { + ctx.addIssue({ code: "custom", message: "Patch byte count does not match content." }); + } + if ( + createHash("sha256").update(artifact.patch.content).digest("hex") !== artifact.patch.sha256 + ) { + ctx.addIssue({ code: "custom", message: "Patch digest does not match content." }); + } + }); + +export const redTestEvidenceSchema = z + .object({ + version: z.literal(1), + schemaId: z.literal(redTestEvidenceSchemaId), + planId: z.string().min(1), + planSha256: sha256Schema, + testPlanSha256: sha256Schema, + results: z + .array( + z + .object({ + id: identifierSchema, + testId: identifierSchema, + acceptanceCriterionIds: z.array(identifierSchema).min(1), + command: z.string().refine(isAllowedFocusedTestCommand), + outcome: z.literal("expected_failure"), + exitCode: z.number().int().positive(), + durationMs: z.number().int().nonnegative(), + expectedAssertion: z.string().min(1), + executionReceipt: sha256Schema, + outputReference: z.object({ + uri: z.string().min(1), + sha256: sha256Schema, + byteCount: z.number().int().nonnegative(), + redacted: z.literal(true), + }), + }) + .strict(), + ) + .min(1), + }) + .strict(); + +export const testWritingAgentOutputSchema = z + .object({ + model: z.literal(testWriterModelLabel), + testPlan: testPlanArtifactSchema, + redEvidence: redTestEvidenceSchema, + }) + .strict() + .superRefine((output, ctx) => { + const criterionIds = new Set( + output.testPlan.acceptanceCriteria.map((criterion) => criterion.id), + ); + const testIds = new Set(output.testPlan.tests.map((test) => test.id)); + const evidenceTestIds = new Set(output.redEvidence.results.map((result) => result.testId)); + if ( + output.redEvidence.planId !== output.testPlan.plan.id || + output.redEvidence.planSha256 !== output.testPlan.plan.sha256 + ) { + ctx.addIssue({ code: "custom", message: "Red evidence is not bound to the test plan." }); + } + if (output.redEvidence.testPlanSha256 !== computeTestPlanDigest(output.testPlan)) { + ctx.addIssue({ code: "custom", message: "Red evidence test-plan digest does not match." }); + } + for (const duplicate of duplicateIds(output.redEvidence.results.map(({ id }) => id))) { + ctx.addIssue({ code: "custom", message: `Duplicate red evidence ${duplicate}.` }); + } + for (const result of output.redEvidence.results) { + if (!testIds.has(result.testId)) { + ctx.addIssue({ + code: "custom", + message: `Red evidence references unknown test ${result.testId}.`, + }); + } + for (const criterionId of result.acceptanceCriterionIds) { + if (!criterionIds.has(criterionId)) { + ctx.addIssue({ + code: "custom", + message: `Red evidence references unknown acceptance criterion ${criterionId}.`, + }); + } + } + const plannedTest = output.testPlan.tests.find((test) => test.id === result.testId); + if ( + plannedTest && + (result.command !== plannedTest.command || + result.expectedAssertion !== plannedTest.expectedFailure.message || + JSON.stringify(result.acceptanceCriterionIds) !== + JSON.stringify(plannedTest.acceptanceCriterionIds)) + ) { + ctx.addIssue({ + code: "custom", + message: `Red evidence does not exactly match planned test ${result.testId}.`, + }); + } + } + for (const duplicate of duplicateIds(output.redEvidence.results.map(({ testId }) => testId))) { + ctx.addIssue({ code: "custom", message: `Duplicate evidence for test ${duplicate}.` }); + } + const plannedPaths = new Set(output.testPlan.tests.map((test) => test.path)); + const patchPaths = new Set(output.testPlan.patch.paths); + if ( + plannedPaths.size !== patchPaths.size || + [...plannedPaths].some((path) => !patchPaths.has(path)) + ) { + ctx.addIssue({ + code: "custom", + message: "Patch paths must exactly match planned test paths.", + }); + } + for (const testId of testIds) { + if (!evidenceTestIds.has(testId)) { + ctx.addIssue({ code: "custom", message: `Test ${testId} has no red evidence.` }); + } + } + }); + +export type TestWritingAgentOutput = z.infer; + +export function createRedTestEvidenceArtifactContractMetadata() { + return { + expectedRedTestEvidenceSchemaId: redTestEvidenceSchemaId, + redTestEvidenceMetadataKind: "red_test_evidence_contract" as const, + redTestEvidenceVersion: 1 as const, + }; +} + +export function createTestPlanArtifactContractMetadata() { + return { + expectedTestPlanSchemaId: testPlanSchemaId, + testPlanMetadataKind: "test_plan_contract" as const, + testPlanVersion: 1 as const, + }; +} diff --git a/evals/test-writing/fixtures/issue-47.json b/evals/test-writing/fixtures/issue-47.json new file mode 100644 index 0000000..90dda40 --- /dev/null +++ b/evals/test-writing/fixtures/issue-47.json @@ -0,0 +1,12 @@ +{ + "repositoryFullName": "ncolesummers/loopworks", + "issueNumber": 47, + "title": "Test-writing subagent for the development loop", + "commitSha": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "acceptanceCriteria": [ + "Red evidence is tied to the plan acceptance criteria.", + "The automated test plan and fixture data are reusable downstream.", + "Future model, prompt, and tool changes have eval coverage.", + "ADR 0015 records the orchestrator, sibling subagents, isolated sandboxes, and artifact handoff." + ] +} diff --git a/evals/test-writing/issue-47-red.eval.ts b/evals/test-writing/issue-47-red.eval.ts new file mode 100644 index 0000000..4b015db --- /dev/null +++ b/evals/test-writing/issue-47-red.eval.ts @@ -0,0 +1,56 @@ +import { readFile } from "node:fs/promises"; +import { join } from "node:path"; +import { testWritingAgentOutputSchema } from "@agent/test-writing-agent"; +import { defineEval } from "eve/evals"; +import { includes } from "eve/evals/expect"; + +type TestWritingFixture = { + acceptanceCriteria: string[]; + commitSha: string; + issueNumber: number; + repositoryFullName: string; + title: string; +}; + +export const testWritingEvalTimeoutMs = 180_000; + +export function resolveTestWritingFixturePath(appRoot = process.cwd()): string { + return join(appRoot, "evals", "test-writing", "fixtures", "issue-47.json"); +} + +export async function readTestWritingFixture(appRoot = process.cwd()): Promise { + return JSON.parse(await readFile(resolveTestWritingFixturePath(appRoot), "utf8")); +} + +export default defineEval({ + description: + "The root orchestrator delegates issue #47 to test-writer and receives loopworks.test_plan.v1 plus expected-red evidence.", + tags: ["test-writing", "issue-47"], + timeoutMs: testWritingEvalTimeoutMs, + async test(t) { + const fixture = await readTestWritingFixture(); + await t.send( + [ + "Process durable run 00000000-0000-4000-8000-000000000047.", + "Read its durable state, delegate only when the exact plan-review is approved, and persist the result through the control plane.", + `Repository: ${fixture.repositoryFullName}`, + `Commit: ${fixture.commitSha}`, + `Issue: #${fixture.issueNumber} ${fixture.title}`, + "Acceptance criteria:", + ...fixture.acceptanceCriteria.map((criterion, index) => `ac-${index + 1}: ${criterion}`), + "Use explicit fixture mode. Emit the typed test-writing output only.", + ].join("\n"), + ); + + t.completed(); + t.noFailedActions(); + t.calledTool("read_run_stage_context"); + t.calledSubagent("test-writer", { + output: (output: unknown) => testWritingAgentOutputSchema.safeParse(output).success, + }); + t.calledTool("apply_test_writing_result"); + t.check(JSON.stringify(t.events), includes("loopworks.test_plan.v1")); + t.notCalledTool("write_file"); + t.notCalledTool("bash"); + }, +}); diff --git a/tests/unit/agent/test-writing-agent.test.ts b/tests/unit/agent/test-writing-agent.test.ts new file mode 100644 index 0000000..a2451bc --- /dev/null +++ b/tests/unit/agent/test-writing-agent.test.ts @@ -0,0 +1,217 @@ +/** @vitest-environment node */ +import { createHash } from "node:crypto"; + +import { + isAllowedFocusedTestCommand, + isAllowedTestArtifactPath, + redTestEvidenceSchemaId, + testPlanSchemaId, + testWriterModelLabel, + testWritingAgentOutputSchema, +} from "@agent/test-writing-agent"; + +function sha256(value: string): string { + return createHash("sha256").update(value).digest("hex"); +} + +const patch = [ + "diff --git a/tests/unit/example.test.ts b/tests/unit/example.test.ts", + "new file mode 100644", + "--- /dev/null", + "+++ b/tests/unit/example.test.ts", + "@@ -0,0 +1 @@", + "+expect(actual).toBe(expected);", +].join("\n"); + +function validOutput() { + const testPlan = { + version: 1 as const, + schemaId: testPlanSchemaId, + plan: { + id: "plan-47", + sha256: sha256("plan-47"), + repositoryFullName: "ncolesummers/loopworks", + commitSha: "a".repeat(40), + }, + acceptanceCriteria: [ + { id: "ac-1", text: "Red evidence is tied to the plan." }, + { id: "ac-2", text: "The test plan is reusable downstream." }, + ], + tests: [ + { + id: "test-red-evidence", + acceptanceCriterionIds: ["ac-1", "ac-2"], + type: "unit" as const, + path: "tests/unit/example.test.ts", + command: "bun run test -- tests/unit/example.test.ts", + steps: ["Run the focused contract test."], + expectedFailure: { kind: "assertion" as const, message: "expected false to be true" }, + fixtureIds: ["approved-plan"], + }, + ], + fixtures: [ + { + id: "approved-plan", + kind: "fixture" as const, + description: "Approved plan fixture.", + data: { approved: true }, + }, + ], + patch: { + format: "unified-diff" as const, + content: patch, + sha256: sha256(patch), + byteCount: Buffer.byteLength(patch), + paths: ["tests/unit/example.test.ts"], + }, + }; + + return { + model: testWriterModelLabel, + testPlan, + redEvidence: { + version: 1 as const, + schemaId: redTestEvidenceSchemaId, + planId: testPlan.plan.id, + planSha256: testPlan.plan.sha256, + testPlanSha256: sha256(JSON.stringify(testPlan)), + results: [ + { + id: "red-1", + testId: "test-red-evidence", + acceptanceCriterionIds: ["ac-1", "ac-2"], + command: "bun run test -- tests/unit/example.test.ts", + outcome: "expected_failure" as const, + exitCode: 1, + durationMs: 25, + expectedAssertion: "expected false to be true", + executionReceipt: "c".repeat(64), + outputReference: { + uri: "artifact://redacted/red-1.log", + sha256: sha256("redacted output"), + byteCount: 15, + redacted: true as const, + }, + }, + ], + }, + }; +} + +describe("Test-writing artifact contract", () => { + it("accepts a complete AC-mapped test plan and expected-red evidence", () => { + expect(testWritingAgentOutputSchema.parse(validOutput())).toMatchObject({ + model: "openai/gpt-5.6-terra-xhigh", + testPlan: { schemaId: "loopworks.test_plan.v1" }, + redEvidence: { schemaId: "loopworks.red_test_evidence.v1" }, + }); + }); + + it("rejects duplicate IDs and evidence that references unknown criteria", () => { + const duplicate = validOutput(); + duplicate.testPlan.acceptanceCriteria.push({ + id: "ac-1", + text: "Duplicate criterion.", + }); + duplicate.redEvidence.results[0]?.acceptanceCriterionIds.push("ac-missing"); + + const parsed = testWritingAgentOutputSchema.safeParse(duplicate); + expect(parsed.success).toBe(false); + if (!parsed.success) { + expect(parsed.error.issues.map((issue) => issue.message)).toEqual( + expect.arrayContaining([ + expect.stringContaining("Duplicate acceptance criterion"), + expect.stringContaining("unknown acceptance criterion"), + ]), + ); + } + }); + + it("binds evidence and the persisted patch to the exact planned test", () => { + const mismatchedEvidence = validOutput(); + const evidence = mismatchedEvidence.redEvidence.results[0]; + if (!evidence) throw new Error("Expected evidence fixture."); + evidence.command = "bun run test -- tests/unit/other.test.ts"; + evidence.expectedAssertion = "different assertion"; + evidence.acceptanceCriterionIds = ["ac-1"]; + expect(testWritingAgentOutputSchema.safeParse(mismatchedEvidence).success).toBe(false); + + const mismatchedPatch = validOutput(); + const test = mismatchedPatch.testPlan.tests[0]; + const patchEvidence = mismatchedPatch.redEvidence.results[0]; + if (!test || !patchEvidence) throw new Error("Expected test and evidence fixture."); + test.path = "tests/unit/other.test.ts"; + test.command = "bun run test -- tests/unit/other.test.ts"; + patchEvidence.command = "bun run test -- tests/unit/other.test.ts"; + mismatchedPatch.redEvidence.testPlanSha256 = sha256(JSON.stringify(mismatchedPatch.testPlan)); + expect(testWritingAgentOutputSchema.safeParse(mismatchedPatch).success).toBe(false); + }); + + it("rejects secret-like fixture data and production paths hidden in patch headers", () => { + const unsafe = validOutput(); + ( + unsafe.testPlan.fixtures[0] as { data: Record } + ).data = { + API_TOKEN: "super-secret-token", + }; + unsafe.testPlan.patch.content = unsafe.testPlan.patch.content.replaceAll( + "tests/unit/example.test.ts", + "src/lib/runtime.ts", + ); + unsafe.testPlan.patch.sha256 = sha256(unsafe.testPlan.patch.content); + unsafe.testPlan.patch.byteCount = Buffer.byteLength(unsafe.testPlan.patch.content); + + const parsed = testWritingAgentOutputSchema.safeParse(unsafe); + expect(parsed.success).toBe(false); + if (!parsed.success) { + expect(parsed.error.issues.map((issue) => issue.message)).toEqual( + expect.arrayContaining([ + expect.stringContaining("secret-like fixture data"), + expect.stringContaining("undeclared or unsafe path"), + ]), + ); + } + }); + + it("rejects symlink, rename, binary, and pathless patch payloads", () => { + for (const dangerousLine of [ + "new file mode 120000", + "rename from tests/unit/example.test.ts", + "GIT binary patch", + ]) { + const unsafe = validOutput(); + unsafe.testPlan.patch.content = `${unsafe.testPlan.patch.content}\n${dangerousLine}`; + unsafe.testPlan.patch.sha256 = sha256(unsafe.testPlan.patch.content); + unsafe.testPlan.patch.byteCount = Buffer.byteLength(unsafe.testPlan.patch.content); + unsafe.redEvidence.testPlanSha256 = sha256(JSON.stringify(unsafe.testPlan)); + expect(testWritingAgentOutputSchema.safeParse(unsafe).success).toBe(false); + } + + const pathless = validOutput(); + pathless.testPlan.patch.content = "+expect(false).toBe(true);"; + pathless.testPlan.patch.sha256 = sha256(pathless.testPlan.patch.content); + pathless.testPlan.patch.byteCount = Buffer.byteLength(pathless.testPlan.patch.content); + pathless.redEvidence.testPlanSha256 = sha256(JSON.stringify(pathless.testPlan)); + expect(testWritingAgentOutputSchema.safeParse(pathless).success).toBe(false); + }); +}); + +describe("Test-writing write and command allowlists", () => { + it("allows focused test artifacts and rejects source, traversal, and shell mutation", () => { + expect(isAllowedTestArtifactPath("tests/unit/example.test.ts")).toBe(true); + expect(isAllowedTestArtifactPath("src/components/button.stories.tsx")).toBe(true); + expect(isAllowedTestArtifactPath("src/lib/runtime.ts")).toBe(false); + expect(isAllowedTestArtifactPath("../tests/escape.test.ts")).toBe(false); + expect(isAllowedTestArtifactPath("tests/$(touch PWNED)/escape.test.ts")).toBe(false); + expect(isAllowedTestArtifactPath("tests/`touch PWNED`/escape.test.ts")).toBe(false); + + expect(isAllowedFocusedTestCommand("bun run test -- tests/unit/example.test.ts")).toBe(true); + expect(isAllowedFocusedTestCommand("bunx playwright test tests/e2e/example.spec.ts")).toBe( + true, + ); + expect(isAllowedFocusedTestCommand("bun run validate")).toBe(false); + expect( + isAllowedFocusedTestCommand("bun run test -- tests/unit/example.test.ts && git status"), + ).toBe(false); + }); +}); diff --git a/tests/unit/agent/test-writing-discovery.test.ts b/tests/unit/agent/test-writing-discovery.test.ts new file mode 100644 index 0000000..4853a02 --- /dev/null +++ b/tests/unit/agent/test-writing-discovery.test.ts @@ -0,0 +1,103 @@ +/** @vitest-environment node */ +import { readdir, readFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { artifactTypeEnum } from "@/db/schema"; +import { developmentLoopStages } from "@/lib/loops/development-run"; +import * as developmentLoopTransitions from "@/lib/loops/development-run-transitions"; + +describe("Eve stage orchestrator discovery", () => { + it("uses a neutral root orchestrator with planner and test-writer siblings", async () => { + const root = process.cwd(); + const [rootInstructions, rootAgent, plannerAgent, testWriterAgent] = await Promise.all([ + readFile(join(root, "agent", "instructions.md"), "utf8"), + readFile(join(root, "agent", "agent.ts"), "utf8"), + readFile(join(root, "agent", "subagents", "planner", "agent.ts"), "utf8"), + readFile(join(root, "agent", "subagents", "test-writer", "agent.ts"), "utf8"), + ]); + + expect(rootInstructions).toContain("stage orchestrator"); + expect(rootAgent).toContain('model: "openai/gpt-5.6-sol"'); + expect(plannerAgent).toContain('description: "Create an approved executable plan artifact'); + expect(plannerAgent).toContain('model: "openai/gpt-5.6-sol"'); + expect(testWriterAgent).toContain('description: "Write focused failing tests'); + expect(testWriterAgent).toContain('model: "openai/gpt-5.6-terra"'); + }); + + it("requires a dedicated test-plan artifact beside red evidence", () => { + expect(artifactTypeEnum.enumValues).toContain("test_plan"); + + const testWriting = developmentLoopStages.find((stage) => stage.key === "test-writing"); + expect(testWriting).toMatchObject({ actorId: "test-writer" }); + expect(testWriting?.artifacts).toEqual([ + { label: "Red test evidence", required: true, type: "validation_report" }, + { label: "Automated test plan", required: true, type: "test_plan" }, + ]); + }); + + it("exposes a deterministic test-writing transition owned by the control plane", () => { + expect("applyDevelopmentLoopTestWritingResult" in developmentLoopTransitions).toBe(true); + }); + + it("gives each stage agent a narrow, explicit tool surface", async () => { + const root = process.cwd(); + const [rootTools, plannerTools, testWriterTools] = await Promise.all([ + readdir(join(root, "agent", "tools")), + readdir(join(root, "agent", "subagents", "planner", "tools")), + readdir(join(root, "agent", "subagents", "test-writer", "tools")), + ]); + + expect(rootTools).toEqual( + expect.arrayContaining([ + "apply_test_writing_result.ts", + "bash.ts", + "read_run_stage_context.ts", + "record_plan_artifact.ts", + ]), + ); + expect(plannerTools).toEqual( + expect.arrayContaining([ + "bash.ts", + "emit_plan_artifact.ts", + "list_repository_files.ts", + "prepare_repository_context.ts", + "read_repository_files.ts", + "read_issue_context.ts", + "search_repository.ts", + "summarize_validation_requirements.ts", + ]), + ); + expect(testWriterTools).toEqual( + expect.arrayContaining([ + "emit_test_writing_artifacts.ts", + "list_repository_files.ts", + "read_repository_files.ts", + "read_approved_plan.ts", + "run_test_suite.ts", + "write_test_files.ts", + ]), + ); + + expect(rootTools).not.toContain("emit_plan_artifact.ts"); + }); + + it("provides a discoverable issue #47 Eve eval path", async () => { + const evalSource = await readFile( + join(process.cwd(), "evals", "test-writing", "issue-47-red.eval.ts"), + "utf8", + ); + expect(evalSource).toContain("test-writer"); + expect(evalSource).toContain("loopworks.test_plan.v1"); + expect(evalSource).toContain('calledTool("read_run_stage_context"'); + expect(evalSource).toContain('calledSubagent("test-writer"'); + expect(evalSource).toContain('calledTool("apply_test_writing_result"'); + }); + + it("labels root telemetry as orchestration without raw agent IO", async () => { + const source = await readFile(join(process.cwd(), "agent", "instrumentation.ts"), "utf8"); + expect(source).toContain('"loopworks.agent": "stage-orchestrator"'); + expect(source).not.toContain('"loopworks.agent": "planning-agent"'); + expect(source).toContain("recordInputs: telemetryPolicy.recordInputs"); + expect(source).toContain("recordOutputs: telemetryPolicy.recordOutputs"); + }); +}); diff --git a/tests/unit/agent/test-writing-eval.test.ts b/tests/unit/agent/test-writing-eval.test.ts new file mode 100644 index 0000000..57815e2 --- /dev/null +++ b/tests/unit/agent/test-writing-eval.test.ts @@ -0,0 +1,23 @@ +/** @vitest-environment node */ +import { + readTestWritingFixture, + resolveTestWritingFixturePath, + testWritingEvalTimeoutMs, +} from "../../../evals/test-writing/issue-47-red.eval"; + +describe("test-writing Eve eval fixture", () => { + it("loads the pinned issue #47 fixture from the repository root", async () => { + expect( + resolveTestWritingFixturePath().endsWith("evals/test-writing/fixtures/issue-47.json"), + ).toBe(true); + await expect(readTestWritingFixture()).resolves.toMatchObject({ + issueNumber: 47, + repositoryFullName: "ncolesummers/loopworks", + commitSha: "a".repeat(40), + }); + }); + + it("allows enough time for root routing and an xhigh child session", () => { + expect(testWritingEvalTimeoutMs).toBeGreaterThanOrEqual(180_000); + }); +}); diff --git a/tests/unit/agent/test-writing-tools.test.ts b/tests/unit/agent/test-writing-tools.test.ts new file mode 100644 index 0000000..eb54397 --- /dev/null +++ b/tests/unit/agent/test-writing-tools.test.ts @@ -0,0 +1,88 @@ +/** @vitest-environment node */ +import { resolveTestWriterFixtureMode } from "../../../agent/subagents/test-writer/lib/fixture-mode"; +import { + assertAllowedTestFiles, + assertCommandMatchesPlannedTests, + classifyTestRun, + redactTestOutput, +} from "../../../agent/subagents/test-writer/lib/tool-policy"; + +describe("test-writer fixture mode", () => { + it("is explicit, local-only, and fail-closed in production", () => { + expect(resolveTestWriterFixtureMode({})).toEqual({ enabled: false, reason: "not_requested" }); + expect( + resolveTestWriterFixtureMode({ + LOOPWORKS_EVE_TEST_WRITER_FIXTURE_MODE: "true", + NODE_ENV: "development", + }), + ).toEqual({ enabled: true, reason: "explicit_non_production_fixture" }); + expect( + resolveTestWriterFixtureMode({ + LOOPWORKS_EVE_TEST_WRITER_FIXTURE_MODE: "true", + VERCEL_ENV: "production", + }), + ).toEqual({ enabled: false, reason: "production_runtime_blocked" }); + }); +}); + +describe("test-writer command and output policy", () => { + it("binds the exact command to approved test paths and test type", () => { + expect(() => + assertCommandMatchesPlannedTests("bun run test -- tests/unit/a.test.ts", [ + { path: "tests/unit/a.test.ts", type: "unit" }, + ]), + ).not.toThrow(); + expect(() => + assertCommandMatchesPlannedTests("bunx playwright test tests/e2e/a.spec.ts", [ + { path: "tests/e2e/a.spec.ts", type: "browser" }, + ]), + ).not.toThrow(); + expect(() => + assertCommandMatchesPlannedTests("bun run test -- ../../evil.test.ts", [ + { path: "../../evil.test.ts", type: "unit" }, + ]), + ).toThrow("Unsafe test artifact path"); + expect(() => + assertCommandMatchesPlannedTests("bun run test -- tests/unit/a.test.ts --preload ./evil.ts", [ + { path: "tests/unit/a.test.ts", type: "unit" }, + ]), + ).toThrow(); + expect(() => + assertCommandMatchesPlannedTests("bun run test -- tests/unit/a.test.ts tests/e2e/a.spec.ts", [ + { path: "tests/unit/a.test.ts", type: "unit" }, + { path: "tests/e2e/a.spec.ts", type: "browser" }, + ]), + ).toThrow("must run separately"); + }); + + it("rejects unsafe writes and redacts credential-shaped output", () => { + expect(() => assertAllowedTestFiles([{ path: "src/lib/runtime.ts" }])).toThrow(); + expect(() => assertAllowedTestFiles([{ path: "tests/unit/a.test.ts" }])).not.toThrow(); + const redacted = redactTestOutput( + "authorization: Bearer abc123\nAPI_TOKEN=secret-value\nghp_abcdefghijklmnopqrstuvwxyz", + ); + expect(redacted).not.toContain("abc123"); + expect(redacted).not.toContain("secret-value"); + expect(redacted).not.toContain("ghp_"); + expect(redacted).toContain("[REDACTED]"); + }); + + it("does not classify setup, crash, timeout, or unrelated output as expected red", () => { + expect( + classifyTestRun({ + exitCode: 1, + output: "Cannot find module x\nexpected false to be true", + expectedAssertions: ["expected false to be true"], + testPaths: ["tests/unit/a.test.ts"], + }), + ).toBe("invalid_failure"); + expect( + classifyTestRun({ + exitCode: 1, + output: "tests/unit/a.test.ts\nexpected false to be true", + expectedAssertions: ["expected false to be true"], + testPaths: ["tests/unit/a.test.ts"], + }), + ).toBe("expected_failure"); + }); +}); From d853a7c876509ad731bd3f5ae3cd7436e8d23da6 Mon Sep 17 00:00:00 2001 From: Nate Summers Date: Sat, 11 Jul 2026 14:04:38 -0700 Subject: [PATCH 05/12] feat(loops): gate test writing on exact plan approval and verified red evidence Run creation now seeds a plan-review approval bound to the plan row and canonical digest, and provisions separate red-evidence and test-plan artifacts for the test-writing stage. recordDevelopmentLoopPlanArtifact persists digest-valid pinned plans and requests review; an approved plan-review advances the run to test-writing while other resolutions block it. applyDevelopmentLoopTestWritingResult advances to development only for exact plan/approval/digest matches with HMAC-verified expected-failure receipts, claim-guarded against concurrent transitions. Part of #47. Co-Authored-By: Claude Fable 5 --- src/lib/approval-transitions.ts | 35 +- src/lib/loops/development-run-transitions.ts | 416 +++++++++++++++++- src/lib/loops/development-run.ts | 131 ++++-- src/lib/observability/trace-context.ts | 17 +- .../approval-transitions.integration.test.ts | 113 +++++ .../github/webhook-store.integration.test.ts | 6 +- tests/unit/github/webhooks.test.ts | 4 +- tests/unit/loops/development-run.test.ts | 82 ++-- .../loops/test-writing-transition.test.ts | 305 +++++++++++++ 9 files changed, 1031 insertions(+), 78 deletions(-) create mode 100644 tests/unit/approval-transitions.integration.test.ts create mode 100644 tests/unit/loops/test-writing-transition.test.ts diff --git a/src/lib/approval-transitions.ts b/src/lib/approval-transitions.ts index 6ef3b6b..b8be849 100644 --- a/src/lib/approval-transitions.ts +++ b/src/lib/approval-transitions.ts @@ -1,14 +1,14 @@ import { and, eq, sql } from "drizzle-orm"; -import { approvals, approvalTransitionEvents } from "@/db/schema"; +import { agentPlans, approvals, approvalTransitionEvents, loopRuns, runSteps } from "@/db/schema"; import { type ApprovalAction, ApprovalExpectedStatusError, ApprovalNotFoundError, - ApprovalWriteInProgressError, type ApprovalStatus, type ApprovalTransition, type ApprovalTransitionDatabase, + ApprovalWriteInProgressError, transitionApproval, } from "@/lib/approvals"; import type { LoopworksLogger } from "@/lib/observability/logger"; @@ -95,6 +95,7 @@ export async function applyApprovalTransition( ) .returning({ id: approvals.id, + metadata: approvals.metadata, requestedAt: approvals.requestedAt, runId: approvals.runId, scope: approvals.scope, @@ -145,6 +146,36 @@ export async function applyApprovalTransition( toStatus: transition.to, }); + if (approval.scope === "plan-review") { + const planId = + approval.metadata && typeof approval.metadata.planId === "string" + ? approval.metadata.planId + : undefined; + if (!planId) { + throw new Error("Plan-review approval metadata must include planId."); + } + if (!approval.runId) { + throw new Error("Plan-review approval must belong to a run."); + } + await tx + .update(agentPlans) + .set({ status: transition.to }) + .where(and(eq(agentPlans.id, planId), eq(agentPlans.runId, approval.runId))); + await tx + .update(loopRuns) + .set({ + ...(transition.to === "approved" ? { currentStage: "test-writing" } : {}), + status: transition.to === "approved" ? "running" : "blocked", + }) + .where(eq(loopRuns.id, approval.runId)); + if (transition.to === "approved") { + await tx + .update(runSteps) + .set({ status: "queued" }) + .where(and(eq(runSteps.runId, approval.runId), eq(runSteps.stage, "test-writing"))); + } + } + input.logger?.info( { action: transition.action, diff --git a/src/lib/loops/development-run-transitions.ts b/src/lib/loops/development-run-transitions.ts index 29312ba..d959b57 100644 --- a/src/lib/loops/development-run-transitions.ts +++ b/src/lib/loops/development-run-transitions.ts @@ -1,7 +1,20 @@ -import { and, desc, eq } from "drizzle-orm"; +import { randomUUID } from "node:crypto"; +import { + computePlanningArtifactDigest, + pinnedPlanningAgentOutputSchema, + planningAgentOutputSchema, +} from "@agent/planning-agent"; +import { verifyTestExecutionReceipt } from "@agent/subagents/test-writer/lib/tool-policy"; +import { + computeTestPlanDigest, + type TestWritingAgentOutput, + testWritingAgentOutputSchema, +} from "@agent/test-writing-agent"; +import { and, desc, eq, sql } from "drizzle-orm"; import type { db } from "@/db/client"; import { + agentPlans, approvals, approvalTransitionEvents, artifacts, @@ -37,9 +50,160 @@ import { recordDevelopmentLoopValidationDurationMetric, recordDevelopmentLoopValidationOutcomeMetric, } from "@/lib/observability/metrics"; +import { + markLoopworksSpanError, + markLoopworksSpanOk, + startLoopworksSpan, +} from "@/lib/observability/trace-context"; export type DevelopmentLoopTransitionDatabase = Pick; +export type TestWritingTransitionResult = { + idempotent?: boolean; + runId: string; + stage: "test-writing"; + status: "advanced"; + stepId: string; + traceId?: string; +}; + +export type ApplyDevelopmentLoopTestWritingResultInput = { + database: DevelopmentLoopTransitionDatabase; + logger?: LoopworksLogger; + occurredAt?: Date; + output: TestWritingAgentOutput; + receiptSecret?: string; + runId: string; +}; + +export type RecordDevelopmentLoopPlanArtifactInput = { + database: DevelopmentLoopTransitionDatabase; + occurredAt?: Date; + plan: unknown; + runId: string; +}; + +export async function recordDevelopmentLoopPlanArtifact( + input: RecordDevelopmentLoopPlanArtifactInput, +): Promise<{ approvalId: string; planId: string; runId: string; status: "waiting_for_approval" }> { + const plan = pinnedPlanningAgentOutputSchema.parse(input.plan); + if (!plan.repositoryRevision || computePlanningArtifactDigest(plan) !== plan.identity.sha256) { + throw new DevelopmentLoopTransitionError( + "Plan review requires a valid digest and pinned repository revision.", + ); + } + const occurredAt = input.occurredAt ?? new Date(); + + return input.database.transaction(async (tx) => { + const [run] = await tx + .select({ + currentStage: loopRuns.currentStage, + id: loopRuns.id, + queuedAt: loopRuns.queuedAt, + repositoryFullName: repositories.fullName, + }) + .from(loopRuns) + .innerJoin(repositories, eq(loopRuns.repositoryId, repositories.id)) + .where(eq(loopRuns.id, input.runId)) + .limit(1); + if (run?.currentStage !== "planning") { + throw new DevelopmentLoopTransitionError(`Run ${input.runId} is not available for planning.`); + } + if (plan.issue.repositoryFullName !== run.repositoryFullName) { + throw new DevelopmentLoopTransitionError("Plan repository does not match the run."); + } + + const [planRow] = await tx + .select() + .from(agentPlans) + .where(eq(agentPlans.runId, input.runId)) + .limit(1); + const [planningStep] = await tx + .select() + .from(runSteps) + .where(and(eq(runSteps.runId, input.runId), eq(runSteps.stage, "planning"))) + .limit(1); + if (!planRow || !planningStep) { + throw new DevelopmentLoopTransitionError("Run is missing its planning records."); + } + + await tx + .update(agentPlans) + .set({ agentName: "planner", plan, status: "requested" }) + .where(eq(agentPlans.id, planRow.id)); + + const [existingApproval] = await tx + .select() + .from(approvals) + .where(and(eq(approvals.runId, input.runId), eq(approvals.scope, "plan-review"))) + .limit(1); + if (existingApproval && existingApproval.status !== "requested") { + throw new DevelopmentLoopTransitionError( + "A resolved plan review cannot be rebound to a new plan.", + ); + } + const approvalMetadata = { + planId: planRow.id, + planSha256: plan.identity.sha256, + }; + const approvalId = existingApproval?.id ?? randomUUID(); + if (existingApproval) { + await tx + .update(approvals) + .set({ metadata: approvalMetadata, requestedBy: "planner" }) + .where(eq(approvals.id, existingApproval.id)); + } else { + await tx.insert(approvals).values({ + id: approvalId, + metadata: approvalMetadata, + requestedBy: "planner", + runId: input.runId, + scope: "plan-review", + status: "requested", + }); + } + + const [planArtifact] = await tx + .select() + .from(artifacts) + .where( + and( + eq(artifacts.runId, input.runId), + eq(artifacts.stepId, planningStep.id), + eq(artifacts.type, "plan"), + ), + ) + .limit(1); + if (!planArtifact) throw new DevelopmentLoopTransitionError("Planning artifact is missing."); + await tx + .update(artifacts) + .set({ + metadata: { + plan, + planId: plan.identity.id, + planMetadataKind: "plan_result", + planSha256: plan.identity.sha256, + }, + sha256: plan.identity.sha256, + }) + .where(eq(artifacts.id, planArtifact.id)); + await tx + .update(runSteps) + .set({ + completedAt: occurredAt, + startedAt: planningStep.startedAt ?? occurredAt, + status: "succeeded", + }) + .where(eq(runSteps.id, planningStep.id)); + await tx + .update(loopRuns) + .set({ startedAt: run.queuedAt, status: "waiting_for_approval" }) + .where(eq(loopRuns.id, input.runId)); + + return { approvalId, planId: planRow.id, runId: input.runId, status: "waiting_for_approval" }; + }); +} + export type DevelopmentLoopValidationTransitionStatus = "advanced" | "blocked"; export type DevelopmentLoopTerminalStatus = "succeeded" | "failed" | "canceled"; @@ -119,6 +283,256 @@ export class DevelopmentLoopTransitionError extends Error { } } +export async function applyDevelopmentLoopTestWritingResult( + input: ApplyDevelopmentLoopTestWritingResultInput, +): Promise { + const occurredAt = input.occurredAt ?? new Date(); + const transitionStartedAt = Date.now(); + const output = testWritingAgentOutputSchema.parse(input.output); + const span = startLoopworksSpan("loopworks.test_writing.transition", { + attributes: { + "loopworks.agent": "test-writer", + "loopworks.run_id": input.runId, + "loopworks.stage": "test-writing", + "loopworks.test_count": output.testPlan.tests.length, + "loopworks.acceptance_criterion_count": output.testPlan.acceptanceCriteria.length, + }, + }); + + try { + const result = await input.database.transaction(async (tx) => { + const [run] = await tx.select().from(loopRuns).where(eq(loopRuns.id, input.runId)).limit(1); + if (!run) throw new DevelopmentLoopTransitionError(`Run ${input.runId} was not found.`); + + const [step] = await tx + .select() + .from(runSteps) + .where(and(eq(runSteps.runId, input.runId), eq(runSteps.stage, "test-writing"))) + .limit(1); + if (!step) { + throw new DevelopmentLoopTransitionError( + `Run ${input.runId} does not have a test-writing step.`, + ); + } + if (step.status === "succeeded" && step.completedAt) { + return { + idempotent: true, + runId: input.runId, + stage: "test-writing", + status: "advanced", + stepId: step.id, + ...((step.traceId ?? run.traceId) + ? { traceId: step.traceId ?? run.traceId ?? undefined } + : {}), + } satisfies TestWritingTransitionResult; + } + if (run.currentStage !== "test-writing") { + throw new DevelopmentLoopTransitionError( + `Run ${input.runId} is at ${run.currentStage}, not test-writing.`, + ); + } + + const planRows = await tx.select().from(agentPlans).where(eq(agentPlans.runId, input.runId)); + if (planRows.length !== 1) { + throw new DevelopmentLoopTransitionError("Test writing requires exactly one current plan."); + } + const planRow = planRows[0]; + const parsedPlan = planningAgentOutputSchema.safeParse(planRow?.plan); + if (!planRow || !parsedPlan.success || !parsedPlan.data.repositoryRevision) { + throw new DevelopmentLoopTransitionError( + "Test writing requires a pinned planning artifact.", + ); + } + const plan = parsedPlan.data; + const repositoryRevision = plan.repositoryRevision; + if (!repositoryRevision) { + throw new DevelopmentLoopTransitionError( + "Test writing requires a pinned repository revision.", + ); + } + if ( + plan.identity.sha256 !== computePlanningArtifactDigest(plan) || + plan.identity.id !== output.testPlan.plan.id || + plan.identity.sha256 !== output.testPlan.plan.sha256 || + plan.issue.repositoryFullName !== output.testPlan.plan.repositoryFullName || + repositoryRevision.commitSha !== output.testPlan.plan.commitSha + ) { + throw new DevelopmentLoopTransitionError( + "Test-writing output does not match the persisted plan.", + ); + } + + const planApprovals = await tx + .select() + .from(approvals) + .where(and(eq(approvals.runId, input.runId), eq(approvals.scope, "plan-review"))); + if (planApprovals.length !== 1) { + throw new DevelopmentLoopTransitionError( + "Test writing requires exactly one current plan-review approval.", + ); + } + const approval = planApprovals[0]; + const approvalMetadata = approval?.metadata; + if ( + approval?.status !== "approved" || + approvalMetadata?.planId !== planRow.id || + approvalMetadata?.planSha256 !== plan.identity.sha256 + ) { + throw new DevelopmentLoopTransitionError( + "Test writing requires an approved plan-review bound to the exact plan.", + ); + } + + const expectedCriteria = plan.issue.acceptanceCriteria.map((text, index) => ({ + id: `ac-${index + 1}`, + text, + })); + if (JSON.stringify(output.testPlan.acceptanceCriteria) !== JSON.stringify(expectedCriteria)) { + throw new DevelopmentLoopTransitionError( + "Test plan acceptance criteria do not exactly match the approved plan.", + ); + } + + const receiptSecret = input.receiptSecret ?? process.env.LOOPWORKS_EVE_TEST_RECEIPT_SECRET; + if (!receiptSecret) { + throw new DevelopmentLoopTransitionError( + "Test execution receipt verification is not configured.", + ); + } + for (const result of output.redEvidence.results) { + const test = output.testPlan.tests.find(({ id }) => id === result.testId); + if ( + !test || + !verifyTestExecutionReceipt( + { + command: result.command, + exitCode: result.exitCode, + expectedAssertions: [result.expectedAssertion], + outcome: result.outcome, + outputSha256: result.outputReference.sha256, + patchSha256: output.testPlan.patch.sha256, + testPaths: [test.path], + }, + result.executionReceipt, + receiptSecret, + ) + ) { + throw new DevelopmentLoopTransitionError( + `Red evidence receipt is invalid for test ${result.testId}.`, + ); + } + } + + const claimId = randomUUID(); + const [claimedStep] = await tx + .update(runSteps) + .set({ metadata: { ...(step.metadata ?? {}), testWritingClaim: claimId } }) + .where( + and( + eq(runSteps.id, step.id), + sql`not coalesce(${runSteps.metadata} ? 'testWritingClaim', false)`, + ), + ) + .returning({ id: runSteps.id }); + if (!claimedStep) { + throw new DevelopmentLoopTransitionError( + `Test-writing transition is already in progress for run ${input.runId}.`, + ); + } + + const stageArtifacts = await tx + .select() + .from(artifacts) + .where(and(eq(artifacts.runId, input.runId), eq(artifacts.stepId, step.id))); + const redArtifact = stageArtifacts.find((artifact) => artifact.type === "validation_report"); + const testPlanArtifact = stageArtifacts.find((artifact) => artifact.type === "test_plan"); + if (!redArtifact || !testPlanArtifact) { + throw new DevelopmentLoopTransitionError( + "Test-writing step requires validation_report and test_plan artifacts.", + ); + } + + await tx + .update(artifacts) + .set({ + metadata: { + redTestEvidence: output.redEvidence, + redTestEvidenceMetadataKind: "red_test_evidence_result", + redTestEvidenceSchemaId: output.redEvidence.schemaId, + redTestEvidenceVersion: output.redEvidence.version, + }, + sha256: computeTestPlanDigest(output.redEvidence), + }) + .where(eq(artifacts.id, redArtifact.id)); + await tx + .update(artifacts) + .set({ + metadata: { + testPlan: output.testPlan, + testPlanMetadataKind: "test_plan_result", + testPlanSchemaId: output.testPlan.schemaId, + testPlanVersion: output.testPlan.version, + }, + sha256: computeTestPlanDigest(output.testPlan), + }) + .where(eq(artifacts.id, testPlanArtifact.id)); + + const startedAt = step.startedAt ?? occurredAt; + await tx + .update(runSteps) + .set({ + completedAt: occurredAt, + startedAt, + status: "succeeded", + validationStatus: "red", + }) + .where(eq(runSteps.id, step.id)); + await tx + .update(loopRuns) + .set({ + currentStage: "development", + startedAt: run.startedAt ?? run.queuedAt, + status: "running", + }) + .where(eq(loopRuns.id, input.runId)); + + input.logger?.info( + { + acceptanceCriterionCount: output.testPlan.acceptanceCriteria.length, + durationMs: Math.max(0, Date.now() - transitionStartedAt), + outcome: "advanced", + planSha256: plan.identity.sha256, + runId: input.runId, + stepId: step.id, + testCount: output.testPlan.tests.length, + }, + "test_writing_stage_advanced", + ); + + return { + runId: input.runId, + stage: "test-writing", + status: "advanced", + stepId: step.id, + ...((step.traceId ?? run.traceId) + ? { traceId: step.traceId ?? run.traceId ?? undefined } + : {}), + } satisfies TestWritingTransitionResult; + }); + span.setAttributes({ + "loopworks.duration_ms": Math.max(0, Date.now() - transitionStartedAt), + "loopworks.outcome": "advanced", + }); + markLoopworksSpanOk(span); + return result; + } catch (error) { + markLoopworksSpanError(span, error); + throw error; + } finally { + span.end(); + } +} + function durationSecondsBetween(startedAt: Date, completedAt: Date): number { return Math.max(0, (completedAt.getTime() - startedAt.getTime()) / 1000); } diff --git a/src/lib/loops/development-run.ts b/src/lib/loops/development-run.ts index 6b7b05b..e5a2b2e 100644 --- a/src/lib/loops/development-run.ts +++ b/src/lib/loops/development-run.ts @@ -1,21 +1,24 @@ import { randomUUID } from "node:crypto"; - +import { createPlanningAgentSeedPlan } from "@agent/planning-agent"; +import { + createRedTestEvidenceArtifactContractMetadata, + createTestPlanArtifactContractMetadata, +} from "@agent/test-writing-agent"; import { and, eq, sql } from "drizzle-orm"; - import type { db } from "@/db/client"; import { agentPlans, + approvals, artifacts, loopRuns, observabilityEvents, repositories, runSteps, } from "@/db/schema"; -import { createPlanningAgentSeedPlan } from "@agent/planning-agent"; import { createPrIntentArtifactContractMetadata } from "@/lib/loops/pr-intent"; +import { createValidationReportArtifactContractMetadata } from "@/lib/loops/validation-report"; import { recordDevelopmentLoopRunCreatedObservability } from "@/lib/observability/metrics"; import { getActiveTraceId, isValidW3cTraceId } from "@/lib/observability/trace-context"; -import { createValidationReportArtifactContractMetadata } from "@/lib/loops/validation-report"; import type { ArtifactRecord, TimelineEvent, TimelineKind } from "@/lib/types"; export const developmentLoopKey = "development-loop"; @@ -34,6 +37,7 @@ export type DevelopmentLoopStageKey = export type DevelopmentLoopArtifactType = | "plan" | "validation_report" + | "test_plan" | "patch" | "pr_intent" | "log_summary" @@ -48,7 +52,7 @@ type DevelopmentLoopArtifactContract = { type DevelopmentLoopStageContract = { actorId: string; actorType: "agent" | "ci" | "human" | "system"; - artifact: DevelopmentLoopArtifactContract; + artifacts: readonly DevelopmentLoopArtifactContract[]; key: DevelopmentLoopStageKey; summary: string; timelineKind: TimelineKind; @@ -61,7 +65,7 @@ export const developmentLoopStages = [ { actorId: "planning-agent", actorType: "agent", - artifact: { label: "Plan artifact", required: true, type: "plan" }, + artifacts: [{ label: "Plan artifact", required: true, type: "plan" }], key: "planning", summary: "Create an issue-backed execution plan with acceptance criteria and validation mapping.", @@ -69,9 +73,12 @@ export const developmentLoopStages = [ title: "Planning", }, { - actorId: "eve-builder-agent", + actorId: "test-writer", actorType: "agent", - artifact: { label: "Red test evidence", required: true, type: "validation_report" }, + artifacts: [ + { label: "Red test evidence", required: true, type: "validation_report" }, + { label: "Automated test plan", required: true, type: "test_plan" }, + ], key: "test-writing", summary: "Write focused failing tests before production code changes.", timelineKind: "test", @@ -82,7 +89,7 @@ export const developmentLoopStages = [ { actorId: "eve-builder-agent", actorType: "agent", - artifact: { label: "Patch artifact", required: true, type: "patch" }, + artifacts: [{ label: "Patch artifact", required: true, type: "patch" }], key: "development", summary: "Implement the smallest green change for the issue scope.", timelineKind: "development", @@ -91,7 +98,7 @@ export const developmentLoopStages = [ { actorId: "ci-runner", actorType: "ci", - artifact: { label: "Validation report", required: true, type: "validation_report" }, + artifacts: [{ label: "Validation report", required: true, type: "validation_report" }], key: "validation", summary: "Run deterministic checks before review, LLM judgment, commit, or PR stages.", timelineKind: "validation", @@ -102,7 +109,7 @@ export const developmentLoopStages = [ { actorId: "reviewer", actorType: "human", - artifact: { label: "Code review notes", required: true, type: "log_summary" }, + artifacts: [{ label: "Code review notes", required: true, type: "log_summary" }], key: "code-review", summary: "Review assumptions, security/a11y risks, and validation evidence.", timelineKind: "review", @@ -111,7 +118,7 @@ export const developmentLoopStages = [ { actorId: "maintainer", actorType: "human", - artifact: { label: "Commit intent", required: true, type: "other" }, + artifacts: [{ label: "Commit intent", required: true, type: "other" }], key: "commit", summary: "Prepare an atomic conventional commit only after validation and review.", timelineKind: "commit", @@ -120,7 +127,7 @@ export const developmentLoopStages = [ { actorId: "maintainer", actorType: "human", - artifact: { label: "PR intent", required: true, type: "pr_intent" }, + artifacts: [{ label: "PR intent", required: true, type: "pr_intent" }], key: "pr", summary: "Prepare PR metadata linking the source issue, run, and validation evidence.", timelineKind: "pull_request", @@ -129,7 +136,7 @@ export const developmentLoopStages = [ { actorId: "loopworks", actorType: "system", - artifact: { label: "Completion summary", required: true, type: "other" }, + artifacts: [{ label: "Completion summary", required: true, type: "other" }], key: "done", summary: "Close the run only after deterministic validation and review evidence are present.", timelineKind: "done", @@ -145,6 +152,10 @@ export type DevelopmentLoopTrigger = { labels?: readonly string[]; milestone?: string | null; repositoryFullName: string; + repositoryRevision?: { + ref: string; + commitSha: string; + }; title?: string | null; }; @@ -176,7 +187,7 @@ type DevelopmentLoopRunTransactionResult = { type DevelopmentLoopStageInstance = { actorId: string; actorType: string; - artifact: DevelopmentLoopArtifactInstance; + artifacts: DevelopmentLoopArtifactInstance[]; completedAt?: Date; key: DevelopmentLoopStageKey; queuedAt: Date; @@ -192,6 +203,7 @@ type DevelopmentLoopArtifactInstance = { detail: string; label: string; required: true; + stageKey: DevelopmentLoopStageKey; type: DevelopmentLoopArtifactType; uri: string; }; @@ -217,8 +229,16 @@ function getIssueUrl(trigger: DevelopmentLoopTrigger): string { return `https://github.com/${trigger.repositoryFullName}/issues/${trigger.issueNumber}`; } -function getArtifactUri(trigger: DevelopmentLoopTrigger, stage: DevelopmentLoopStageContract) { - return `${getIssueUrl(trigger)}#development-loop-${stage.key}`; +function getArtifactUri( + trigger: DevelopmentLoopTrigger, + stage: DevelopmentLoopStageContract, + artifact: DevelopmentLoopArtifactContract, +) { + const suffix = + artifact.type === "validation_report" + ? artifact.label.toLowerCase().replaceAll(" ", "-") + : artifact.type.replaceAll("_", "-"); + return `${getIssueUrl(trigger)}#development-loop-${stage.key}-${suffix}`; } function formatTimelineTime(date: Date): string { @@ -245,18 +265,19 @@ export function createDevelopmentLoopRunSkeleton(input: { }): DevelopmentLoopRunSkeleton { const stages = developmentLoopStages.map((stageDefinition, index) => { const stage: DevelopmentLoopStageContract = stageDefinition; - const artifact = { + const stageArtifacts = stage.artifacts.map((artifact) => ({ detail: stage.summary, - label: stage.artifact.label, - required: stage.artifact.required, - type: stage.artifact.type, - uri: getArtifactUri(input.trigger, stage), - }; + label: artifact.label, + required: artifact.required, + stageKey: stage.key, + type: artifact.type, + uri: getArtifactUri(input.trigger, stage, artifact), + })); return { actorId: stage.actorId, actorType: stage.actorType, - artifact, + artifacts: stageArtifacts, key: stage.key, queuedAt: minutesAfter(input.now, index), status: "queued" as const, @@ -269,7 +290,7 @@ export function createDevelopmentLoopRunSkeleton(input: { }); return { - artifacts: stages.map((stage) => stage.artifact), + artifacts: stages.flatMap((stage) => stage.artifacts), loopKey: developmentLoopKey, mode: input.mode, ...(input.runId ? { runId: input.runId } : {}), @@ -283,7 +304,7 @@ export function projectDevelopmentLoopTimeline( ): TimelineEvent[] { return skeleton.stages.map((stage) => ({ actor: stage.actorId, - artifact: stage.artifact.label, + artifact: stage.artifacts.map((artifact) => artifact.label).join(", "), at: formatTimelineTime(stage.queuedAt), detail: stage.summary, kind: stage.timelineKind, @@ -409,17 +430,17 @@ export async function createDevelopmentLoopRun(input: { traceId, }); - const stepIds: string[] = []; + const stepIdsByStage = new Map(); for (const stage of skeleton.stages) { const stepId = randomUUID(); - stepIds.push(stepId); + stepIdsByStage.set(stage.key, stepId); await tx.insert(runSteps).values({ id: stepId, actorId: stage.actorId, actorType: stage.actorType, metadata: { - artifactLabel: stage.artifact.label, - requiredArtifact: stage.artifact.required, + artifactLabels: stage.artifacts.map((artifact) => artifact.label), + requiredArtifacts: stage.artifacts.every((artifact) => artifact.required), }, queuedAt: stage.queuedAt, runId, @@ -433,28 +454,45 @@ export async function createDevelopmentLoopRun(input: { } await tx.insert(artifacts).values( - skeleton.artifacts.map((artifact, index) => ({ + skeleton.artifacts.map((artifact) => ({ id: randomUUID(), metadata: { required: artifact.required, - stage: skeleton.stages[index]?.key, - ...(artifact.type === "validation_report" + stage: artifact.stageKey, + ...(artifact.type === "validation_report" && artifact.stageKey === "validation" ? createValidationReportArtifactContractMetadata({ detail: artifact.detail, }) : {}), + ...(artifact.type === "validation_report" && artifact.stageKey === "test-writing" + ? createRedTestEvidenceArtifactContractMetadata() + : {}), + ...(artifact.type === "test_plan" ? createTestPlanArtifactContractMetadata() : {}), ...(artifact.type === "pr_intent" ? createPrIntentArtifactContractMetadata() : {}), }, runId, - stepId: stepIds[index], + stepId: stepIdsByStage.get(artifact.stageKey), title: artifact.label, type: artifact.type, uri: artifact.uri, })), ); + const planId = randomUUID(); + const plan = createPlanningAgentSeedPlan({ + body: input.trigger.body ?? "", + issueNumber: input.trigger.issueNumber, + issueUrl: getIssueUrl(input.trigger), + labels: [...(input.trigger.labels ?? [])], + milestone: input.trigger.milestone ?? null, + repositoryFullName: input.trigger.repositoryFullName, + repositoryRevision: input.trigger.repositoryRevision ?? null, + title: input.trigger.title ?? `Issue #${input.trigger.issueNumber}`, + }); + await tx.insert(agentPlans).values({ - agentName: "planning-agent", + id: planId, + agentName: "planner", input: { issueNumber: input.trigger.issueNumber, labels: input.trigger.labels ?? [], @@ -463,19 +501,24 @@ export async function createDevelopmentLoopRun(input: { title: input.trigger.title ?? "", }, issueNumber: input.trigger.issueNumber, - plan: createPlanningAgentSeedPlan({ - body: input.trigger.body ?? "", - issueNumber: input.trigger.issueNumber, - issueUrl: getIssueUrl(input.trigger), - labels: [...(input.trigger.labels ?? [])], - milestone: input.trigger.milestone ?? null, - repositoryFullName: input.trigger.repositoryFullName, - title: input.trigger.title ?? `Issue #${input.trigger.issueNumber}`, - }), + plan, runId, status: "pending", }); + if (plan.repositoryRevision) { + await tx.insert(approvals).values({ + metadata: { + planId, + planSha256: plan.identity.sha256, + }, + requestedBy: "planner", + runId, + scope: "plan-review", + status: "requested", + }); + } + const emitObservability = await recordDevelopmentLoopRunCreatedObservability({ artifactCount: skeleton.artifacts.length, deliveryId: input.trigger.deliveryId, diff --git a/src/lib/observability/trace-context.ts b/src/lib/observability/trace-context.ts index 17f9dad..b55a165 100644 --- a/src/lib/observability/trace-context.ts +++ b/src/lib/observability/trace-context.ts @@ -1,4 +1,10 @@ -import { trace, type Span, type SpanOptions, type Tracer } from "@opentelemetry/api"; +import { + type Span, + type SpanOptions, + SpanStatusCode, + type Tracer, + trace, +} from "@opentelemetry/api"; const w3cTraceIdPattern = /^[0-9a-f]{32}$/; const emptyTraceId = "00000000000000000000000000000000"; @@ -16,6 +22,15 @@ export function startLoopworksSpan( return tracer.startSpan(name, options); } +export function markLoopworksSpanOk(span: Span): void { + span.setStatus({ code: SpanStatusCode.OK }); +} + +export function markLoopworksSpanError(span: Span, error: unknown): void { + span.recordException(error instanceof Error ? error : String(error)); + span.setStatus({ code: SpanStatusCode.ERROR }); +} + export function isValidW3cTraceId(traceId: unknown): traceId is string { return typeof traceId === "string" && w3cTraceIdPattern.test(traceId) && traceId !== emptyTraceId; } diff --git a/tests/unit/approval-transitions.integration.test.ts b/tests/unit/approval-transitions.integration.test.ts new file mode 100644 index 0000000..3371121 --- /dev/null +++ b/tests/unit/approval-transitions.integration.test.ts @@ -0,0 +1,113 @@ +/** @vitest-environment node */ + +import { createPlanningAgentSeedPlan } from "@agent/planning-agent"; +import { eq } from "drizzle-orm"; + +import { agentPlans, approvals, loopRuns, repositories } from "@/db/schema"; +import { applyApprovalTransition } from "@/lib/approval-transitions"; +import type { ApprovalTransitionDatabase } from "@/lib/approvals"; +import { + createDevelopmentLoopRun, + type DevelopmentLoopRunDatabase, +} from "@/lib/loops/development-run"; +import { + type DevelopmentLoopTransitionDatabase, + recordDevelopmentLoopPlanArtifact, +} from "@/lib/loops/development-run-transitions"; +import { createPgliteTestDatabase, type PgliteTestDatabase } from "../helpers/pglite"; + +describe("plan-review approval synchronization", () => { + let context: PgliteTestDatabase; + + beforeEach(async () => { + context = await createPgliteTestDatabase(); + await context.db.insert(repositories).values({ + githubRepoId: 47_000_001, + owner: "ncolesummers", + name: "loopworks", + fullName: "ncolesummers/loopworks", + enabledLoops: ["Agent-ready development loop"], + validationGates: ["Focused tests"], + }); + }); + + afterEach(async () => context.close()); + + it("updates the exact agent plan when its durable review is approved", async () => { + const run = await createDevelopmentLoopRun({ + database: context.db as unknown as DevelopmentLoopRunDatabase, + trigger: { + issueNumber: 47, + repositoryFullName: "ncolesummers/loopworks", + repositoryRevision: { ref: "main", commitSha: "a".repeat(40) }, + title: "Test-writing subagent", + }, + }); + if (run.mode !== "created") throw new Error("Expected created run."); + + const [approval] = await context.db + .select() + .from(approvals) + .where(eq(approvals.runId, run.runId)); + const [plan] = await context.db + .select() + .from(agentPlans) + .where(eq(agentPlans.runId, run.runId)); + if (!approval || !plan) throw new Error("Expected plan review fixtures."); + + await applyApprovalTransition({ + action: "approve", + actorId: "maintainer", + approvalId: approval.id, + database: context.db as unknown as ApprovalTransitionDatabase, + expectedStatus: "requested", + }); + + const [updatedPlan] = await context.db + .select() + .from(agentPlans) + .where(eq(agentPlans.id, plan.id)); + expect(updatedPlan?.status).toBe("approved"); + }); + + it("records a planner-pinned revision before creating the durable review", async () => { + const run = await createDevelopmentLoopRun({ + database: context.db as unknown as DevelopmentLoopRunDatabase, + trigger: { + body: "## Acceptance Criteria\n- Planner output is pinned before review.", + issueNumber: 47, + repositoryFullName: "ncolesummers/loopworks", + title: "Test-writing subagent", + }, + }); + if (run.mode !== "created") throw new Error("Expected created run."); + + const plan = createPlanningAgentSeedPlan({ + body: "## Acceptance Criteria\n- Planner output is pinned before review.", + issueNumber: 47, + labels: [], + milestone: null, + repositoryFullName: "ncolesummers/loopworks", + repositoryRevision: { commitSha: "a".repeat(40), ref: "main" }, + title: "Test-writing subagent", + }); + const result = await recordDevelopmentLoopPlanArtifact({ + database: context.db as unknown as DevelopmentLoopTransitionDatabase, + plan, + runId: run.runId, + }); + + expect(result.status).toBe("waiting_for_approval"); + const [approval] = await context.db + .select() + .from(approvals) + .where(eq(approvals.id, result.approvalId)); + const [updatedRun] = await context.db.select().from(loopRuns).where(eq(loopRuns.id, run.runId)); + expect(approval).toMatchObject({ + scope: "plan-review", + status: "requested", + metadata: { planId: result.planId, planSha256: plan.identity.sha256 }, + }); + expect(updatedRun?.status).toBe("waiting_for_approval"); + }); +}); diff --git a/tests/unit/github/webhook-store.integration.test.ts b/tests/unit/github/webhook-store.integration.test.ts index 530d654..14eeef8 100644 --- a/tests/unit/github/webhook-store.integration.test.ts +++ b/tests/unit/github/webhook-store.integration.test.ts @@ -307,7 +307,7 @@ describe("GitHub webhook delivery store (pglite integration)", () => { duplicate: false, agentReadyTrigger: { shouldTrigger: true, workflow: "development" }, developmentRun: { - artifactCount: 8, + artifactCount: 9, mode: "created", stageCount: 8, }, @@ -351,7 +351,7 @@ describe("GitHub webhook delivery store (pglite integration)", () => { .from(observabilityEvents) .where(eq(observabilityEvents.eventType, "development_loop_run_created")); expect(event.traceId).toBe(runRows[0]?.traceId); - expect(artifactRows).toHaveLength(8); + expect(artifactRows).toHaveLength(9); expect(planRows).toHaveLength(1); // A replayed delivery is rejected at the route boundary without creating new rows. @@ -369,7 +369,7 @@ describe("GitHub webhook delivery store (pglite integration)", () => { expect(await context.db.select().from(idempotencyLocks)).toHaveLength(1); expect(await context.db.select().from(loopRuns)).toHaveLength(1); expect(await context.db.select().from(runSteps)).toHaveLength(8); - expect(await context.db.select().from(artifacts)).toHaveLength(8); + expect(await context.db.select().from(artifacts)).toHaveLength(9); expect(await context.db.select().from(agentPlans)).toHaveLength(1); }); diff --git a/tests/unit/github/webhooks.test.ts b/tests/unit/github/webhooks.test.ts index 8122b2a..17cfb73 100644 --- a/tests/unit/github/webhooks.test.ts +++ b/tests/unit/github/webhooks.test.ts @@ -484,7 +484,7 @@ describe("GitHub webhook helpers", () => { workflow: "development", }, developmentRun: { - artifactCount: 8, + artifactCount: 9, mode: "simulated", stageCount: 8, }, @@ -605,7 +605,7 @@ describe("GitHub webhook helpers", () => { deliveryId: "successful-processing-route-delivery", metadata: { developmentRun: { - artifactCount: 8, + artifactCount: 9, mode: "simulated", stageCount: 8, }, diff --git a/tests/unit/loops/development-run.test.ts b/tests/unit/loops/development-run.test.ts index 41119aa..b6219d3 100644 --- a/tests/unit/loops/development-run.test.ts +++ b/tests/unit/loops/development-run.test.ts @@ -1,9 +1,11 @@ /** @vitest-environment node */ -import { eq } from "drizzle-orm"; -import { context as otelContext, trace, TraceFlags, type Span } from "@opentelemetry/api"; +import { redTestEvidenceSchemaId, testPlanSchemaId } from "@agent/test-writing-agent"; +import { context as otelContext, type Span, TraceFlags, trace } from "@opentelemetry/api"; +import { eq } from "drizzle-orm"; import { agentPlans, + approvals, artifacts, loopRuns, observabilityEvents, @@ -13,11 +15,11 @@ import { import { createDevelopmentLoopRun, createDevelopmentLoopRunSkeleton, + type DevelopmentLoopRunDatabase, developmentLoopStages, projectDevelopmentLoopArtifacts, projectDevelopmentLoopTimeline, recordDevelopmentLoopNoop, - type DevelopmentLoopRunDatabase, } from "@/lib/loops/development-run"; import { prIntentSchemaId } from "@/lib/loops/pr-intent"; import { validationReportSchemaId } from "@/lib/loops/validation-runner"; @@ -34,6 +36,10 @@ const issueTrigger = { labels: ["agent-ready", "area:loops", "area:agents", "loop:development", "priority:p0"], milestone: "M3 Durable Loop MVP", repositoryFullName: "ncolesummers/loopworks", + repositoryRevision: { + ref: "main", + commitSha: "a".repeat(40), + }, title: "Agent-ready development loop skeleton", }; @@ -86,7 +92,11 @@ describe("agent-ready development loop run skeleton", () => { "pr", "done", ]); - expect(developmentLoopStages.every((stage) => stage.artifact.required)).toBe(true); + expect( + developmentLoopStages.every((stage) => + stage.artifacts.every((artifact) => artifact.required), + ), + ).toBe(true); expect(developmentLoopStages.findIndex((stage) => stage.key === "validation")).toBeLessThan( developmentLoopStages.findIndex((stage) => stage.key === "code-review"), ); @@ -106,7 +116,7 @@ describe("agent-ready development loop run skeleton", () => { const artifactRecords = projectDevelopmentLoopArtifacts(skeleton); expect(skeleton.stages).toHaveLength(8); - expect(skeleton.artifacts).toHaveLength(8); + expect(skeleton.artifacts).toHaveLength(9); expect(timeline.map((event) => event.title)).toEqual([ "Planning", "Test writing", @@ -121,6 +131,7 @@ describe("agent-ready development loop run skeleton", () => { expect(artifactRecords.map((artifact) => artifact.label)).toEqual([ "Plan artifact", "Red test evidence", + "Automated test plan", "Patch artifact", "Validation report", "Code review notes", @@ -130,19 +141,20 @@ describe("agent-ready development loop run skeleton", () => { ]); }); - it("creates one durable run, eight stage rows, eight artifacts, and an agent plan", async () => { + it("creates one durable run, eight stage rows, nine artifacts, and an agent plan", async () => { await insertRepository(context); const result = await withTestTrace(() => createDevelopmentLoopRun({ database: testDatabase(context), now: () => new Date("2026-07-02T16:00:00.000Z"), + traceId: "4bf92f3577b34da6a3ce929d0e0e4736", trigger: issueTrigger, }), ); expect(result).toMatchObject({ - artifactCount: 8, + artifactCount: 9, mode: "created", stageCount: 8, }); @@ -151,6 +163,7 @@ describe("agent-ready development loop run skeleton", () => { const stepRows = await context.db.select().from(runSteps); const artifactRows = await context.db.select().from(artifacts); const planRows = await context.db.select().from(agentPlans); + const approvalRows = await context.db.select().from(approvals); expect(runRows).toHaveLength(1); expect(runRows[0]).toMatchObject({ @@ -173,32 +186,51 @@ describe("agent-ready development loop run skeleton", () => { expect(stepRows.map((step) => step.stage)).toEqual( developmentLoopStages.map((stage) => stage.key), ); - expect(artifactRows).toHaveLength(8); + expect(artifactRows).toHaveLength(9); expect(artifactRows.every((artifact) => artifact.runId === runRows[0]?.id)).toBe(true); expect( - artifactRows - .filter((artifact) => artifact.type === "validation_report") - .map((artifact) => artifact.metadata), - ).toEqual([ - expect.objectContaining({ - expectedValidationReportSchemaId: validationReportSchemaId, - validationReportMetadataKind: "validation_report_contract", - validationReportVersion: 1, - }), - expect.objectContaining({ - expectedValidationReportSchemaId: validationReportSchemaId, - validationReportMetadataKind: "validation_report_contract", - validationReportVersion: 1, - }), - ]); + artifactRows.find( + (artifact) => + artifact.type === "validation_report" && artifact.metadata?.stage === "test-writing", + )?.metadata, + ).toMatchObject({ + expectedRedTestEvidenceSchemaId: redTestEvidenceSchemaId, + redTestEvidenceMetadataKind: "red_test_evidence_contract", + redTestEvidenceVersion: 1, + }); + expect( + artifactRows.find( + (artifact) => + artifact.type === "validation_report" && artifact.metadata?.stage === "validation", + )?.metadata, + ).toMatchObject({ + expectedValidationReportSchemaId: validationReportSchemaId, + validationReportMetadataKind: "validation_report_contract", + validationReportVersion: 1, + }); + expect(artifactRows.find((artifact) => artifact.type === "test_plan")?.metadata).toMatchObject({ + expectedTestPlanSchemaId: testPlanSchemaId, + testPlanMetadataKind: "test_plan_contract", + testPlanVersion: 1, + }); expect(artifactRows.find((artifact) => artifact.type === "pr_intent")?.metadata).toMatchObject({ expectedPrIntentSchemaId: prIntentSchemaId, prIntentMetadataKind: "pr_intent_contract", prIntentVersion: 1, }); expect(planRows).toHaveLength(1); + expect(approvalRows).toHaveLength(1); + expect(approvalRows[0]).toMatchObject({ + runId: runRows[0]?.id, + scope: "plan-review", + status: "requested", + metadata: { + planId: planRows[0]?.id, + planSha256: expect.stringMatching(/^[a-f0-9]{64}$/), + }, + }); expect(planRows[0]).toMatchObject({ - agentName: "planning-agent", + agentName: "planner", issueNumber: 11, plan: expect.objectContaining({ issue: expect.objectContaining({ @@ -241,7 +273,7 @@ describe("agent-ready development loop run skeleton", () => { expect(second).toEqual(first); expect(await context.db.select().from(loopRuns)).toHaveLength(1); expect(await context.db.select().from(runSteps)).toHaveLength(8); - expect(await context.db.select().from(artifacts)).toHaveLength(8); + expect(await context.db.select().from(artifacts)).toHaveLength(9); expect(await context.db.select().from(agentPlans)).toHaveLength(1); }); diff --git a/tests/unit/loops/test-writing-transition.test.ts b/tests/unit/loops/test-writing-transition.test.ts new file mode 100644 index 0000000..5c4214b --- /dev/null +++ b/tests/unit/loops/test-writing-transition.test.ts @@ -0,0 +1,305 @@ +/** @vitest-environment node */ +import { createHash } from "node:crypto"; +import { createTestExecutionReceipt } from "@agent/subagents/test-writer/lib/tool-policy"; +import { + redTestEvidenceSchemaId, + type TestWritingAgentOutput, + testPlanSchemaId, + testWriterModelLabel, +} from "@agent/test-writing-agent"; +import { and, eq } from "drizzle-orm"; +import { approvals, artifacts, loopRuns, repositories, runSteps } from "@/db/schema"; +import { applyApprovalTransition } from "@/lib/approval-transitions"; +import type { ApprovalTransitionDatabase } from "@/lib/approvals"; +import { + createDevelopmentLoopRun, + type DevelopmentLoopRunDatabase, +} from "@/lib/loops/development-run"; +import { + applyDevelopmentLoopTestWritingResult, + type DevelopmentLoopTransitionDatabase, + DevelopmentLoopTransitionError, +} from "@/lib/loops/development-run-transitions"; +import { createPgliteTestDatabase, type PgliteTestDatabase } from "../../helpers/pglite"; + +function sha256(value: string): string { + return createHash("sha256").update(value).digest("hex"); +} + +function outputForPlan(plan: { id: string; sha256: string }): TestWritingAgentOutput { + const patch = [ + "diff --git a/tests/unit/red.test.ts b/tests/unit/red.test.ts", + "new file mode 100644", + "--- /dev/null", + "+++ b/tests/unit/red.test.ts", + "@@ -0,0 +1 @@", + "+expect(false).toBe(true);", + ].join("\n"); + const testPlan = { + version: 1 as const, + schemaId: testPlanSchemaId as typeof testPlanSchemaId, + plan: { + id: plan.id, + sha256: plan.sha256, + repositoryFullName: "ncolesummers/loopworks", + commitSha: "a".repeat(40), + }, + acceptanceCriteria: [{ id: "ac-1", text: "Expected red evidence is required." }], + tests: [ + { + id: "test-ac-1", + acceptanceCriterionIds: ["ac-1"], + type: "unit" as const, + path: "tests/unit/red.test.ts", + command: "bun run test -- tests/unit/red.test.ts", + steps: ["Run focused test."], + expectedFailure: { kind: "assertion" as const, message: "expected false to be true" }, + fixtureIds: [], + }, + ], + fixtures: [], + patch: { + format: "unified-diff" as const, + content: patch, + sha256: sha256(patch), + byteCount: Buffer.byteLength(patch), + paths: ["tests/unit/red.test.ts"], + }, + }; + const outputSha256 = sha256("redacted"); + const executionReceipt = createTestExecutionReceipt( + { + command: "bun run test -- tests/unit/red.test.ts", + exitCode: 1, + expectedAssertions: ["expected false to be true"], + outcome: "expected_failure", + outputSha256, + patchSha256: testPlan.patch.sha256, + testPaths: ["tests/unit/red.test.ts"], + }, + "test-secret", + ); + return { + model: testWriterModelLabel, + testPlan, + redEvidence: { + version: 1, + schemaId: redTestEvidenceSchemaId, + planId: plan.id, + planSha256: plan.sha256, + testPlanSha256: sha256(JSON.stringify(testPlan)), + results: [ + { + id: "red-ac-1", + testId: "test-ac-1", + acceptanceCriterionIds: ["ac-1"], + command: "bun run test -- tests/unit/red.test.ts", + outcome: "expected_failure", + exitCode: 1, + durationMs: 12, + expectedAssertion: "expected false to be true", + executionReceipt, + outputReference: { + uri: "artifact://redacted/red-ac-1.log", + sha256: outputSha256, + byteCount: 8, + redacted: true, + }, + }, + ], + }, + }; +} + +describe("test-writing stage transition", () => { + let context: PgliteTestDatabase; + + beforeEach(async () => { + context = await createPgliteTestDatabase(); + await context.db.insert(repositories).values({ + githubRepoId: 47_000_002, + owner: "ncolesummers", + name: "loopworks", + fullName: "ncolesummers/loopworks", + enabledLoops: ["Agent-ready development loop"], + validationGates: ["Focused tests"], + }); + }); + + afterEach(async () => context.close()); + + async function prepareRun( + approvalAction: "approve" | "bypass" | "expire" | "reject" | null = "approve", + ) { + const created = await createDevelopmentLoopRun({ + database: context.db as unknown as DevelopmentLoopRunDatabase, + trigger: { + body: "## Acceptance Criteria\n- Expected red evidence is required.", + issueNumber: 47, + repositoryFullName: "ncolesummers/loopworks", + repositoryRevision: { ref: "main", commitSha: "a".repeat(40) }, + title: "Test-writing subagent", + }, + }); + if (created.mode !== "created") throw new Error("Expected created run."); + + const [approval] = await context.db + .select() + .from(approvals) + .where(eq(approvals.runId, created.runId)); + if (!approval) throw new Error("Expected plan approval."); + if (approvalAction) { + await applyApprovalTransition({ + action: approvalAction, + actorId: "maintainer", + approvalId: approval.id, + database: context.db as unknown as ApprovalTransitionDatabase, + expectedStatus: "requested", + }); + } + + const [planRow] = await context.db.query.agentPlans.findMany({ + where: (plan, { eq: equals }) => equals(plan.runId, created.runId), + limit: 1, + }); + const plan = planRow?.plan as { identity?: { id?: string; sha256?: string } } | null; + if (!plan?.identity?.id || !plan.identity.sha256) throw new Error("Expected plan identity."); + + await context.db + .update(loopRuns) + .set({ currentStage: "test-writing", status: "running" }) + .where(eq(loopRuns.id, created.runId)); + await context.db + .update(runSteps) + .set({ status: "running", startedAt: new Date("2026-07-11T16:00:00.000Z") }) + .where(and(eq(runSteps.runId, created.runId), eq(runSteps.stage, "test-writing"))); + + return { + approval, + plan: { id: plan.identity.id, sha256: plan.identity.sha256 }, + runId: created.runId, + }; + } + + it("persists both artifacts and advances expected red to development", async () => { + const prepared = await prepareRun(); + const result = await applyDevelopmentLoopTestWritingResult({ + database: context.db as unknown as DevelopmentLoopTransitionDatabase, + occurredAt: new Date("2026-07-11T16:01:00.000Z"), + output: outputForPlan(prepared.plan), + receiptSecret: "test-secret", + runId: prepared.runId, + }); + + expect(result).toMatchObject({ status: "advanced", stage: "test-writing" }); + const [run] = await context.db.select().from(loopRuns).where(eq(loopRuns.id, prepared.runId)); + const [step] = await context.db + .select() + .from(runSteps) + .where(and(eq(runSteps.runId, prepared.runId), eq(runSteps.stage, "test-writing"))); + if (!step) throw new Error("Expected test-writing step."); + const artifactRows = await context.db + .select() + .from(artifacts) + .where(and(eq(artifacts.runId, prepared.runId), eq(artifacts.stepId, step.id))); + + expect(run).toMatchObject({ currentStage: "development", status: "running" }); + expect(step).toMatchObject({ status: "succeeded", validationStatus: "red" }); + expect(artifactRows).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + type: "validation_report", + metadata: expect.objectContaining({ redTestEvidenceSchemaId }), + }), + expect.objectContaining({ + type: "test_plan", + metadata: expect.objectContaining({ testPlanSchemaId }), + }), + ]), + ); + expect(artifactRows.find((artifact) => artifact.type === "validation_report")?.sha256).toBe( + sha256(JSON.stringify(outputForPlan(prepared.plan).redEvidence)), + ); + expect(artifactRows.find((artifact) => artifact.type === "test_plan")?.sha256).toBe( + sha256(JSON.stringify(outputForPlan(prepared.plan).testPlan)), + ); + expect(new Set(artifactRows.map((artifact) => artifact.sha256)).size).toBe(2); + }); + + it.each([ + ["requested", null], + ["bypassed", "bypass" as const], + ["rejected", "reject" as const], + ["expired", "expire" as const], + ])("fails closed for a %s plan review", async (_status, action) => { + const prepared = await prepareRun(action); + await expect( + applyDevelopmentLoopTestWritingResult({ + database: context.db as unknown as DevelopmentLoopTransitionDatabase, + output: outputForPlan(prepared.plan), + receiptSecret: "test-secret", + runId: prepared.runId, + }), + ).rejects.toThrow(DevelopmentLoopTransitionError); + }); + + it("rejects a stale approval digest", async () => { + const prepared = await prepareRun(); + await context.db + .update(approvals) + .set({ metadata: { planId: prepared.plan.id, planSha256: "b".repeat(64) } }) + .where(eq(approvals.id, prepared.approval.id)); + + await expect( + applyDevelopmentLoopTestWritingResult({ + database: context.db as unknown as DevelopmentLoopTransitionDatabase, + output: outputForPlan(prepared.plan), + receiptSecret: "test-secret", + runId: prepared.runId, + }), + ).rejects.toThrow(DevelopmentLoopTransitionError); + }); + + it("rejects self-attested evidence without a valid execution receipt", async () => { + const prepared = await prepareRun(); + const output = outputForPlan(prepared.plan); + const result = output.redEvidence.results[0]; + if (!result) throw new Error("Expected red evidence fixture."); + result.executionReceipt = "f".repeat(64); + + await expect( + applyDevelopmentLoopTestWritingResult({ + database: context.db as unknown as DevelopmentLoopTransitionDatabase, + output, + receiptSecret: "test-secret", + runId: prepared.runId, + }), + ).rejects.toThrow("Red evidence receipt is invalid"); + }); + + it("serializes concurrent persistence so artifacts cannot be overwritten", async () => { + const prepared = await prepareRun(); + const apply = () => + applyDevelopmentLoopTestWritingResult({ + database: context.db as unknown as DevelopmentLoopTransitionDatabase, + output: outputForPlan(prepared.plan), + receiptSecret: "test-secret", + runId: prepared.runId, + }); + + const results = await Promise.allSettled([apply(), apply()]); + expect(results.filter(({ status }) => status === "fulfilled")).toHaveLength(2); + const values = results + .filter( + (result): result is PromiseFulfilledResult>> => + result.status === "fulfilled", + ) + .map(({ value }) => value); + expect(values).toEqual( + expect.arrayContaining([ + expect.objectContaining({ status: "advanced" }), + expect.objectContaining({ idempotent: true }), + ]), + ); + }); +}); From b1373e289b88387624ea5b825193bce29b7bad70 Mon Sep 17 00:00:00 2001 From: Nate Summers Date: Sat, 11 Jul 2026 14:04:52 -0700 Subject: [PATCH 06/12] feat(agent): convert the root agent into a neutral stage orchestrator The root Eve runtime now only reads durable run state, delegates one stage to the matching declared subagent, and invokes deterministic control-plane tools (read_run_stage_context, record_plan_artifact, apply_test_writing_result). Root bash and load_skill are disabled; the orchestrator never infers approval from prompts or mutates GitHub. Part of #47. Co-Authored-By: Claude Fable 5 --- agent/agent.ts | 5 +- agent/instructions.md | 32 +++++----- agent/instrumentation.ts | 2 +- agent/tools/apply_test_writing_result.ts | 50 +++++++++++++++ agent/tools/bash.ts | 26 +------- agent/tools/load_skill.ts | 3 + agent/tools/read_run_stage_context.ts | 77 ++++++++++++++++++++++++ agent/tools/record_plan_artifact.ts | 12 ++++ 8 files changed, 162 insertions(+), 45 deletions(-) create mode 100644 agent/tools/apply_test_writing_result.ts create mode 100644 agent/tools/load_skill.ts create mode 100644 agent/tools/read_run_stage_context.ts create mode 100644 agent/tools/record_plan_artifact.ts diff --git a/agent/agent.ts b/agent/agent.ts index f2bbd2a..825bd89 100644 --- a/agent/agent.ts +++ b/agent/agent.ts @@ -1,9 +1,7 @@ import { defineAgent } from "eve"; -import { planningAgentOutputSchema } from "./planning-agent"; - export default defineAgent({ - model: "openai/gpt-5.5", + model: "openai/gpt-5.6-sol", modelContextWindowTokens: 400_000, modelOptions: { providerOptions: { @@ -12,5 +10,4 @@ export default defineAgent({ }, }, }, - outputSchema: planningAgentOutputSchema, }); diff --git a/agent/instructions.md b/agent/instructions.md index 460801b..f5ce6a7 100644 --- a/agent/instructions.md +++ b/agent/instructions.md @@ -1,21 +1,21 @@ -# Eve Planning Agent +# Loopworks Stage Orchestrator -You are Eve's Loopworks planning agent. +You are Loopworks' neutral stage orchestrator. -Read GitHub issue context as the durable source of truth. Produce only the -validated executable plan artifact. Do not edit code, write repository files, -change branches, open pull requests, change labels, transition approvals, deploy -resources, or mutate SaaS state. +Read durable run state and delegate exactly one stage to the matching declared +subagent. Planning belongs to `planner`; approved test writing belongs to +`test-writer`. Stage subagents have isolated sandboxes and communicate only with +typed artifacts. -Use tools only for planning: +Always begin with `read_run_stage_context`. After planner delegation, call +`record_plan_artifact`; after test-writer delegation, call +`apply_test_writing_result`. A subagent response alone never changes durable +state. -- Read supplied issue context. -- Summarize validation requirements. -- Run guarded read-only CLI inspection through `bash` when SaaS context is - needed. -- Emit the final plan artifact. +Never infer approval from a prompt. Test writing requires a persisted +`plan-review` approval bound to the exact run, plan row, and plan digest. Durable +artifact persistence and stage transitions belong to deterministic control-plane +tools, not to subagents. -Every plan must include stages, validation gates, approval points, risks, -fixture mode, eval coverage, and tool-contract summary. Structured logs should -carry correlation fields only; production must not capture raw prompts, raw -issue bodies, or raw tool output until issue #21 defines masking and export. +Do not edit source directly, mutate GitHub, change branches, or log raw prompts, +plan bodies, patches, test source, fixture values, or command output. diff --git a/agent/instrumentation.ts b/agent/instrumentation.ts index 795780f..2b902c5 100644 --- a/agent/instrumentation.ts +++ b/agent/instrumentation.ts @@ -11,7 +11,7 @@ export default defineInstrumentation({ "step.started"(input) { return { runtimeContext: { - "loopworks.agent": "planning-agent", + "loopworks.agent": "stage-orchestrator", "loopworks.telemetry.policy": telemetryPolicy.reason, "loopworks.raw_io_capture": telemetryPolicy.captureRawIO, "loopworks.session.id": input.session.id, diff --git a/agent/tools/apply_test_writing_result.ts b/agent/tools/apply_test_writing_result.ts new file mode 100644 index 0000000..0f90b52 --- /dev/null +++ b/agent/tools/apply_test_writing_result.ts @@ -0,0 +1,50 @@ +import { defineTool } from "eve/tools"; +import { z } from "zod"; + +import { db } from "@/db/client"; +import { applyDevelopmentLoopTestWritingResult } from "@/lib/loops/development-run-transitions"; +import { resolveTestWriterFixtureMode } from "../subagents/test-writer/lib/fixture-mode"; +import { verifyTestExecutionReceipt } from "../subagents/test-writer/lib/tool-policy"; +import { testWritingAgentOutputSchema } from "../test-writing-agent"; + +export default defineTool({ + description: "Persist validated expected-red artifacts and advance the durable run.", + inputSchema: z.object({ runId: z.string().uuid(), output: testWritingAgentOutputSchema }), + execute: ({ output, runId }) => { + const parsed = testWritingAgentOutputSchema.parse(output); + if (resolveTestWriterFixtureMode().enabled) { + const receiptSecret = process.env.LOOPWORKS_EVE_TEST_RECEIPT_SECRET; + if (!receiptSecret) throw new Error("Test execution receipt secret is not configured."); + for (const result of parsed.redEvidence.results) { + const test = parsed.testPlan.tests.find(({ id }) => id === result.testId); + if ( + !test || + !verifyTestExecutionReceipt( + { + command: result.command, + exitCode: result.exitCode, + expectedAssertions: [result.expectedAssertion], + outcome: result.outcome, + outputSha256: result.outputReference.sha256, + patchSha256: parsed.testPlan.patch.sha256, + testPaths: [test.path], + }, + result.executionReceipt, + receiptSecret, + ) + ) { + throw new Error(`Invalid execution receipt for ${result.testId}.`); + } + } + return { + persistedArtifactTypes: ["validation_report", "test_plan"], + runId, + stage: "test-writing" as const, + status: "advanced" as const, + stepId: "00000000-0000-4000-8000-000000000347", + testCount: parsed.testPlan.tests.length, + }; + } + return applyDevelopmentLoopTestWritingResult({ database: db, output: parsed, runId }); + }, +}); diff --git a/agent/tools/bash.ts b/agent/tools/bash.ts index 7aa0949..04bd054 100644 --- a/agent/tools/bash.ts +++ b/agent/tools/bash.ts @@ -1,25 +1,3 @@ -import { defineTool } from "eve/tools"; -import { z } from "zod"; +import { disableTool } from "eve/tools"; -import { executeCliInspectionCommand } from "../lib/cli-inspection"; - -const bashInputSchema = z.object({ - command: z.string().min(1), -}); - -const bashOutputSchema = z.object({ - exitCode: z.number(), - stderr: z.string(), - stdout: z.string(), - truncated: z.boolean(), -}); - -export default defineTool({ - description: - "Run a guarded read-only CLI inspection command for planning context. Allows audited read-only SaaS and repository inspection commands such as gh issue view, gh pr view, az account show, and git status. Rejects shell constructs, file writes, repo mutation, and SaaS mutation verbs.", - inputSchema: bashInputSchema, - outputSchema: bashOutputSchema, - async execute(input) { - return executeCliInspectionCommand(input.command); - }, -}); +export default disableTool(); diff --git a/agent/tools/load_skill.ts b/agent/tools/load_skill.ts new file mode 100644 index 0000000..04bd054 --- /dev/null +++ b/agent/tools/load_skill.ts @@ -0,0 +1,3 @@ +import { disableTool } from "eve/tools"; + +export default disableTool(); diff --git a/agent/tools/read_run_stage_context.ts b/agent/tools/read_run_stage_context.ts new file mode 100644 index 0000000..db65cf3 --- /dev/null +++ b/agent/tools/read_run_stage_context.ts @@ -0,0 +1,77 @@ +import { and, eq } from "drizzle-orm"; +import { defineTool } from "eve/tools"; +import { z } from "zod"; + +import { db } from "@/db/client"; +import { agentPlans, approvals, artifacts, loopRuns, runSteps } from "@/db/schema"; +import { createPlanningAgentSeedPlan } from "../planning-agent"; +import { resolveTestWriterFixtureMode } from "../subagents/test-writer/lib/fixture-mode"; + +export default defineTool({ + description: "Read durable run, plan, approval, step, and artifact context for stage routing.", + inputSchema: z.object({ runId: z.string().uuid() }), + async execute({ runId }) { + if (resolveTestWriterFixtureMode().enabled) { + const planId = "00000000-0000-4000-8000-000000000147"; + const plan = createPlanningAgentSeedPlan({ + body: [ + "## Acceptance Criteria", + "- Red evidence is tied to the plan acceptance criteria.", + "- The automated test plan and fixture data are reusable downstream.", + "- Future model, prompt, and tool changes have eval coverage.", + "- ADR 0015 records the orchestrator, sibling subagents, isolated sandboxes, and artifact handoff.", + ].join("\n"), + issueNumber: 47, + labels: ["area:agents"], + milestone: null, + repositoryFullName: "ncolesummers/loopworks", + repositoryRevision: { commitSha: "a".repeat(40), ref: "main" }, + title: "Test-writing subagent for the development loop", + }); + return { + approvals: [ + { + id: "00000000-0000-4000-8000-000000000247", + metadata: { planId, planSha256: plan.identity.sha256 }, + status: "approved", + }, + ], + artifacts: [], + plans: [{ id: planId, plan, status: "approved" }], + run: { currentStage: "test-writing", id: runId, status: "running" }, + steps: [ + { + id: "00000000-0000-4000-8000-000000000347", + stage: "test-writing", + status: "queued", + }, + ], + }; + } + const [run] = await db.select().from(loopRuns).where(eq(loopRuns.id, runId)).limit(1); + if (!run) throw new Error(`Run ${runId} was not found.`); + const [steps, plans, planApprovals, runArtifacts] = await Promise.all([ + db.select().from(runSteps).where(eq(runSteps.runId, runId)), + db.select().from(agentPlans).where(eq(agentPlans.runId, runId)), + db + .select() + .from(approvals) + .where(and(eq(approvals.runId, runId), eq(approvals.scope, "plan-review"))), + db.select().from(artifacts).where(eq(artifacts.runId, runId)), + ]); + return { + run: { currentStage: run.currentStage, id: run.id, status: run.status }, + steps: steps.map(({ id, stage, status }) => ({ id, stage, status })), + plans: plans.map(({ id, plan, status }) => ({ id, plan, status })), + approvals: planApprovals.map(({ id, metadata, status }) => ({ id, metadata, status })), + artifacts: runArtifacts.map(({ id, metadata, sha256, stepId, type, uri }) => ({ + id, + metadata, + sha256, + stepId, + type, + uri, + })), + }; + }, +}); diff --git a/agent/tools/record_plan_artifact.ts b/agent/tools/record_plan_artifact.ts new file mode 100644 index 0000000..aa698fe --- /dev/null +++ b/agent/tools/record_plan_artifact.ts @@ -0,0 +1,12 @@ +import { defineTool } from "eve/tools"; +import { z } from "zod"; + +import { db } from "@/db/client"; +import { recordDevelopmentLoopPlanArtifact } from "@/lib/loops/development-run-transitions"; +import { pinnedPlanningAgentOutputSchema } from "../planning-agent"; + +export default defineTool({ + description: "Persist a digest-valid pinned plan and request durable plan review.", + inputSchema: z.object({ runId: z.string().uuid(), plan: pinnedPlanningAgentOutputSchema }), + execute: ({ plan, runId }) => recordDevelopmentLoopPlanArtifact({ database: db, plan, runId }), +}); From 3852bc6349b369bff4afe181371e832f10d0bcfe Mon Sep 17 00:00:00 2001 From: Nate Summers Date: Sat, 11 Jul 2026 14:05:09 -0700 Subject: [PATCH 07/12] docs: record stage orchestrator and subagent handoff decision (ADR 0015) ADR 0015 captures the neutral root orchestrator, sibling subagents with isolated sandboxes, typed artifact handoffs, and fail-closed plan-review gating. Architecture and persona scenarios updated to match. Part of #47. Co-Authored-By: Claude Fable 5 --- ...estrator-and-isolated-subagent-handoffs.md | 81 +++++++++++++++++++ docs/adr/README.md | 1 + docs/architecture.md | 23 ++++-- docs/personas-and-test-scenarios.md | 6 +- 4 files changed, 100 insertions(+), 11 deletions(-) create mode 100644 docs/adr/0015-stage-orchestrator-and-isolated-subagent-handoffs.md diff --git a/docs/adr/0015-stage-orchestrator-and-isolated-subagent-handoffs.md b/docs/adr/0015-stage-orchestrator-and-isolated-subagent-handoffs.md new file mode 100644 index 0000000..e6d6110 --- /dev/null +++ b/docs/adr/0015-stage-orchestrator-and-isolated-subagent-handoffs.md @@ -0,0 +1,81 @@ +# ADR 0015: Stage Orchestrator And Isolated Subagent Handoffs + +Status: Proposed +Date: 2026-07-11 + +Driving issue: [#47](https://github.com/ncolesummers/loopworks/issues/47) + +## Context + +The development loop needs planning, test-writing, implementation, review, PR, +and completion specialists. Eve declared subagents can select independent models +and narrow tool surfaces, but each owns an isolated sandbox. Nesting later stages +under the planning agent would give planning the wrong authority, while relying +on shared workspace state would prevent per-stage model selection and durable +handoffs. + +## Decision + +Loopworks uses one neutral Eve root as the stage orchestrator. Planner, +test-writer, and future stage specialists are declared sibling subagents with +independent model, instruction, tool, and sandbox contracts. The orchestrator +reads durable run state, verifies approval and artifact prerequisites, delegates +one stage, validates the typed result, and invokes deterministic control-plane +transitions. + +The root orchestrator and planner use `openai/gpt-5.6-sol`; the test-writer +uses `openai/gpt-5.6-terra`. Each retains independent `xhigh` reasoning +configuration so model routing can evolve per stage without changing the +shared topology. + +Planner and test-writer receive repository-scoped discovery, text search, and +line-range read tools against their isolated commit-pinned checkouts. The tools +exclude secret, generated, dependency, and traversal paths; bound queries and +outputs; reject symlink escape; and return commit/path/line provenance. The +framework's permissive filesystem tools remain disabled. Planner web access is +a separate guarded capability tracked by issue #68. + +Stage subagents do not mutate GitHub or durable workflow state. They communicate +through versioned artifacts. Because declared subagent sandboxes are isolated, +test-writing hands its repository changes forward as a bounded, SHA-256-bound, +test-only unified patch inside `loopworks.test_plan.v1`. Red evidence is stored +with a control-plane-verifiable execution receipt binding the exact command, +test path, expected assertion, redacted-output digest, and patch digest. This +prevents a stage response from self-attesting that unexecuted tests are red. +The evidence is stored separately as `loopworks.red_test_evidence.v1` in the stage's +`validation_report` artifact. + +Test writing requires an `approved` `plan-review` record bound to the run, plan +row, and canonical plan digest. Requested, rejected, expired, bypassed, stale, +or mismatched approvals fail closed. Expected assertion failures complete the +test-writing stage successfully; environment, setup, timeout, crash, unrelated, +or passing results do not advance the run. + +## Consequences + +Each stage can tune its model and capabilities independently without granting +the planner repository mutation rights. Durable artifact handoffs make isolated +sandboxes reproducible and reviewable, at the cost of versioned patch contracts, +additional artifact persistence, and strict transition checks. + +The root becomes orchestration-only. Stage ordering and approval checks remain +deterministic control-plane behavior rather than model judgment. + +## Validation + +1. Eve discovery reports the root plus declared `planner` and `test-writer` + subagents without diagnostics. +2. Unit tests cover tool allowlists, fixture fail-closed behavior, patch safety, + AC coverage, red-evidence classification, and sanitized telemetry. +3. PGlite tests prove exact plan approval, two-artifact persistence, idempotency, + and advancement only for complete expected-red evidence. +4. Eve eval discovery includes planner and test-writing routing scenarios. +5. `bun run validate` and `bun run build` pass before review. + +## Follow-Ups + +1. Issue #48 consumes the persisted test-only patch and produces the smallest + green implementation patch in a separate sandbox. +2. Issues #44-#46 and #49-#51 implement additional sibling subagents under this + orchestration contract. +3. Accept this ADR only after maintainer review of issue #47. diff --git a/docs/adr/README.md b/docs/adr/README.md index fbdf9f7..8be258d 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -35,6 +35,7 @@ Loopworks uses ADRs for durable technical and product architecture decisions. Gi | [0012](0012-telemetry-backend-and-metric-contract.md) | Proposed | Adopt Axiom as the OTel telemetry backend and define the metric, span, and correlation contract. | | [0013](0013-planning-agent-contract.md) | Proposed | Define the planning agent as a planning-only runtime with guarded CLI inspection, explicit fixtures, eval coverage, and ADR 0012-aligned telemetry boundaries. | | [0014](0014-guarded-github-pr-write-reconciliation.md) | Proposed | Guard GitHub PR writes with digest-bound approval, deterministic branches, and two-phase reconciliation. | +| [0015](0015-stage-orchestrator-and-isolated-subagent-handoffs.md) | Proposed | Use a neutral root orchestrator, independent sibling subagents, and typed artifact handoffs between isolated sandboxes. | ## Template diff --git a/docs/architecture.md b/docs/architecture.md index 62aaf23..9dac964 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -166,7 +166,14 @@ Non-MVP: ## Agent Architecture -The first agent is a planning agent. It should: +The root Eve runtime is a neutral stage orchestrator. It reads durable run +state, verifies approvals and artifact prerequisites, delegates to one declared +stage subagent, validates the typed result, and invokes deterministic +control-plane transitions. Stage subagents are siblings with independent model, +tool, and isolated sandbox contracts; they do not own GitHub writes or workflow +state transitions. + +The planner subagent should: 1. Read a GitHub issue and repo metadata. 2. Create or update an executable plan artifact. @@ -174,16 +181,16 @@ The first agent is a planning agent. It should: 4. Produce a clear next-action summary. 5. Avoid mutating code or GitHub state outside explicit tool contracts. -Later agents are tracked as backlog items, one per development-loop stage that still lacks an LLM agent: +Stage subagents are tracked as backlog items, one per loop stage that still lacks an LLM specialist: 1. Research loop skeleton, research planner, researcher, and research author agents — the `spike` plus `agent-ready` research loop, run in parallel to the development loop. -2. Test-writing agent — test-writing stage, red test evidence plus a reusable automated test plan and seed data. -3. Implementation agent — development stage, patch artifact. -4. Validation review agent — code review stage, code review notes and UI screenshot evidence. -5. PR preparation agent — PR stage, PR intent content including screenshots. -6. Release notes agent — done stage, completion summary. +2. Test-writer subagent — test-writing stage, red test evidence plus a reusable automated test plan, explicit seed data, and a bounded test-only patch. +3. Implementation subagent — development stage, patch artifact. +4. Validation review subagent — code review stage, code review notes and UI screenshot evidence. +5. PR preparation subagent — PR stage, PR intent content including screenshots. +6. Release notes subagent — done stage, completion summary. -The commit stage intentionally stays mechanical, owned by the PR creation path rather than a dedicated agent. Each agent needs eval coverage before promotion to default use. +The commit stage intentionally stays mechanical, owned by the PR creation path rather than a dedicated subagent. Typed artifacts bridge isolated subagent sandboxes, and each subagent needs eval coverage before promotion to default use. ## Security Architecture diff --git a/docs/personas-and-test-scenarios.md b/docs/personas-and-test-scenarios.md index 2b28dce..0e5ebee 100644 --- a/docs/personas-and-test-scenarios.md +++ b/docs/personas-and-test-scenarios.md @@ -95,9 +95,9 @@ Risks: | M01 | Maintainer | Catalog rows show owner, framework, CI commands, docs, observability, design-system, enabled loops, Vercel project links, and search/filter controls. | Playwright, Storybook | | M02 | Maintainer | Turning a loop off prevents trigger execution and records a skipped reason. | Unit, Playwright | | M03 | Maintainer | Missing Vercel credentials in dev returns explicit fixture fallback metadata; production does not silently return fixtures. | Unit, integration | -| A01 | Agent Supervisor | Run detail shows planning, test-writing, development, validation, code review, commit, PR, and done stages with artifacts. | Storybook, Playwright | -| A02 | Agent Supervisor | Approval gates show requested, approved, rejected, bypassed, and expired states with actor and evidence. | Unit, Storybook, Playwright | -| A03 | Agent Supervisor | Deterministic validation output appears before LLM review or judgment. | Integration, Playwright | +| A01 | Agent Supervisor | Run detail shows planning, test-writing, development, validation, code review, commit, PR, and done stages, including separate red-evidence and automated-test-plan artifacts. | Unit, Storybook, Playwright | +| A02 | Agent Supervisor | Approval gates show requested, approved, rejected, bypassed, and expired states with actor and evidence; test writing requires an exact approved plan review. | Unit, Storybook, Playwright | +| A03 | Agent Supervisor | AC-mapped expected-red evidence appears before implementation, and green deterministic validation appears before LLM review or judgment. | Integration, Playwright | | R01 | Reviewer | A PR intent or created PR links to the source issue, run, validation artifacts, and Vercel preview. | Integration, Playwright | | R02 | Reviewer | UI changes include Storybook stories, design-token usage, and browser workflow coverage with axe passing in light and dark before the task is complete. | CI, Storybook build, Playwright, a11y | | S01 | Security Reviewer | GitHub webhook requests with invalid signatures are rejected before payload processing. | Unit, integration | From 0da962702b9612004cb59d8c9650609a647ca538 Mon Sep 17 00:00:00 2001 From: Nate Summers Date: Sat, 11 Jul 2026 14:05:59 -0700 Subject: [PATCH 08/12] docs(env): document test-writer receipt secret and fixture mode LOOPWORKS_EVE_TEST_RECEIPT_SECRET is required for the test-writing transition (it fails closed without it); the fixture-mode flag is non-production only. Part of #47. Co-Authored-By: Claude Fable 5 --- .env.example | 2 ++ README.md | 2 ++ 2 files changed, 4 insertions(+) diff --git a/.env.example b/.env.example index 9ec96b5..770b35f 100644 --- a/.env.example +++ b/.env.example @@ -8,6 +8,8 @@ LOOPWORKS_AGENT_READY_LOOP_ENABLED="true" LOOPWORKS_DEVELOPMENT_LOOP_ENABLED="true" LOOPWORKS_RESEARCH_LOOP_ENABLED="true" LOOPWORKS_PORTAL_DATA_MODE="" +LOOPWORKS_EVE_TEST_RECEIPT_SECRET="replace-with-test-receipt-secret" +LOOPWORKS_EVE_TEST_WRITER_FIXTURE_MODE="false" LOG_LEVEL="info" DATABASE_URL="postgres://loopworks:loopworks@127.0.0.1:5432/loopworks" OTEL_EXPORTER_OTLP_PROTOCOL="http/protobuf" diff --git a/README.md b/README.md index eae8f2a..3f9da75 100644 --- a/README.md +++ b/README.md @@ -51,6 +51,8 @@ Copy `.env.example` to `.env.local` for local development. The fixture server on - `LOOPWORKS_DEVELOPMENT_LOOP_ENABLED` - `LOOPWORKS_RESEARCH_LOOP_ENABLED` - `LOOPWORKS_PORTAL_DATA_MODE` +- `LOOPWORKS_EVE_TEST_RECEIPT_SECRET` +- `LOOPWORKS_EVE_TEST_WRITER_FIXTURE_MODE` - `LOG_LEVEL` - `DATABASE_URL` - `OTEL_EXPORTER_OTLP_PROTOCOL` From e809aec9b29c823bcb224c5b50605b2ade374e9a Mon Sep 17 00:00:00 2001 From: Nate Summers Date: Sat, 11 Jul 2026 14:10:28 -0700 Subject: [PATCH 09/12] fix(agent): canonicalize plan and test-plan digests Plan digests, test-plan digests, and execution-receipt HMACs previously hashed JSON.stringify output, which depends on object key order and only stayed deterministic while every caller digested schema-parsed values (jsonb roundtrips reorder keys). Serialize through a recursive key-sorted canonical JSON encoder so digests are stable for any equivalent value. Part of #47. Co-Authored-By: Claude Fable 5 --- agent/lib/canonical-json.ts | 21 +++++++++++++++ agent/planning-agent.ts | 3 ++- .../subagents/test-writer/lib/tool-policy.ts | 3 ++- agent/test-writing-agent.ts | 4 ++- tests/unit/agent/planning-agent.test.ts | 21 +++++++++++++++ tests/unit/agent/test-writing-agent.test.ts | 27 ++++++++++++++++--- .../loops/test-writing-transition.test.ts | 7 ++--- 7 files changed, 76 insertions(+), 10 deletions(-) create mode 100644 agent/lib/canonical-json.ts diff --git a/agent/lib/canonical-json.ts b/agent/lib/canonical-json.ts new file mode 100644 index 0000000..c3fa31a --- /dev/null +++ b/agent/lib/canonical-json.ts @@ -0,0 +1,21 @@ +/** + * Deterministic JSON serialization for digest and HMAC inputs. Only supports + * JSON-safe plain data (objects, arrays, primitives); non-plain objects such + * as Date are not preserved, so callers must serialize schema-parsed values. + */ +export function canonicalJsonStringify(value: unknown): string { + return JSON.stringify(sortKeysDeep(value)); +} + +function sortKeysDeep(value: unknown): unknown { + if (Array.isArray(value)) return value.map(sortKeysDeep); + if (value !== null && typeof value === "object") { + return Object.fromEntries( + Object.entries(value) + .filter(([, entry]) => entry !== undefined) + .sort(([left], [right]) => (left < right ? -1 : left > right ? 1 : 0)) + .map(([key, entry]) => [key, sortKeysDeep(entry)]), + ); + } + return value; +} diff --git a/agent/planning-agent.ts b/agent/planning-agent.ts index 0bc3199..7e9083f 100644 --- a/agent/planning-agent.ts +++ b/agent/planning-agent.ts @@ -5,6 +5,7 @@ import { z } from "zod"; import { defaultLoopManifest } from "@/lib/loops/manifest"; import { type LoopManifest, loopStateValues } from "../schemas/loop-manifest"; +import { canonicalJsonStringify } from "./lib/canonical-json"; export const planningAgentModelLabel = "openai/gpt-5.6-sol-xhigh"; @@ -148,7 +149,7 @@ export function computePlanningArtifactDigest( ...artifact, identity: artifact.identity ? { id: artifact.identity.id } : undefined, }; - return createHash("sha256").update(JSON.stringify(canonical)).digest("hex"); + return createHash("sha256").update(canonicalJsonStringify(canonical)).digest("hex"); } export const planningAgentDefinition = { diff --git a/agent/subagents/test-writer/lib/tool-policy.ts b/agent/subagents/test-writer/lib/tool-policy.ts index 35026e8..529558f 100644 --- a/agent/subagents/test-writer/lib/tool-policy.ts +++ b/agent/subagents/test-writer/lib/tool-policy.ts @@ -1,5 +1,6 @@ import { createHash, createHmac, timingSafeEqual } from "node:crypto"; +import { canonicalJsonStringify } from "../../../lib/canonical-json"; import { isAllowedFocusedTestCommand, isAllowedTestArtifactPath, @@ -87,7 +88,7 @@ export function createTestExecutionReceipt( secret: string, ): string { if (!secret.trim()) throw new Error("Test execution receipt secret is required."); - return createHmac("sha256", secret).update(JSON.stringify(payload)).digest("hex"); + return createHmac("sha256", secret).update(canonicalJsonStringify(payload)).digest("hex"); } export function verifyTestExecutionReceipt( diff --git a/agent/test-writing-agent.ts b/agent/test-writing-agent.ts index c06af4a..9fb3262 100644 --- a/agent/test-writing-agent.ts +++ b/agent/test-writing-agent.ts @@ -2,13 +2,15 @@ import { createHash } from "node:crypto"; import { z } from "zod"; +import { canonicalJsonStringify } from "./lib/canonical-json"; + export const testWriterModelLabel = "openai/gpt-5.6-terra-xhigh"; export const testPlanSchemaId = "loopworks.test_plan.v1"; export const redTestEvidenceSchemaId = "loopworks.red_test_evidence.v1"; export const maxTestPatchBytes = 256 * 1024; export function computeTestPlanDigest(value: unknown): string { - return createHash("sha256").update(JSON.stringify(value)).digest("hex"); + return createHash("sha256").update(canonicalJsonStringify(value)).digest("hex"); } const safeTestPathPattern = diff --git a/tests/unit/agent/planning-agent.test.ts b/tests/unit/agent/planning-agent.test.ts index 51dd05b..9ef8f9a 100644 --- a/tests/unit/agent/planning-agent.test.ts +++ b/tests/unit/agent/planning-agent.test.ts @@ -1,11 +1,25 @@ /** @vitest-environment node */ import { + computePlanningArtifactDigest, createPlanningAgentSeedPlan, + type PlanningAgentOutput, pinnedPlanningAgentOutputSchema, planningAgentModelLabel, planningAgentOutputSchema, } from "@agent/planning-agent"; +function reverseKeyOrder(value: unknown): unknown { + if (Array.isArray(value)) return value.map(reverseKeyOrder); + if (value !== null && typeof value === "object") { + return Object.fromEntries( + Object.entries(value) + .reverse() + .map(([key, entry]) => [key, reverseKeyOrder(entry)]), + ); + } + return value; +} + const issue13Input = { repositoryFullName: "ncolesummers/loopworks", issueNumber: 13, @@ -26,6 +40,13 @@ const issue13Input = { }; describe("Planning agent artifact contract", () => { + it("computes the same plan digest regardless of JSON key order", () => { + const plan = createPlanningAgentSeedPlan(issue13Input); + const reordered = reverseKeyOrder(plan) as PlanningAgentOutput; + + expect(computePlanningArtifactDigest(reordered)).toBe(plan.identity.sha256); + }); + it("builds the issue #13 executable planning artifact shape", () => { const plan = createPlanningAgentSeedPlan(issue13Input); diff --git a/tests/unit/agent/test-writing-agent.test.ts b/tests/unit/agent/test-writing-agent.test.ts index a2451bc..83cb43c 100644 --- a/tests/unit/agent/test-writing-agent.test.ts +++ b/tests/unit/agent/test-writing-agent.test.ts @@ -2,6 +2,7 @@ import { createHash } from "node:crypto"; import { + computeTestPlanDigest, isAllowedFocusedTestCommand, isAllowedTestArtifactPath, redTestEvidenceSchemaId, @@ -14,6 +15,18 @@ function sha256(value: string): string { return createHash("sha256").update(value).digest("hex"); } +function reverseKeyOrder(value: unknown): unknown { + if (Array.isArray(value)) return value.map(reverseKeyOrder); + if (value !== null && typeof value === "object") { + return Object.fromEntries( + Object.entries(value) + .reverse() + .map(([key, entry]) => [key, reverseKeyOrder(entry)]), + ); + } + return value; +} + const patch = [ "diff --git a/tests/unit/example.test.ts b/tests/unit/example.test.ts", "new file mode 100644", @@ -74,7 +87,7 @@ function validOutput() { schemaId: redTestEvidenceSchemaId, planId: testPlan.plan.id, planSha256: testPlan.plan.sha256, - testPlanSha256: sha256(JSON.stringify(testPlan)), + testPlanSha256: computeTestPlanDigest(testPlan), results: [ { id: "red-1", @@ -99,6 +112,12 @@ function validOutput() { } describe("Test-writing artifact contract", () => { + it("computes the same test-plan digest regardless of JSON key order", () => { + const testPlan = validOutput().testPlan; + + expect(computeTestPlanDigest(reverseKeyOrder(testPlan))).toBe(computeTestPlanDigest(testPlan)); + }); + it("accepts a complete AC-mapped test plan and expected-red evidence", () => { expect(testWritingAgentOutputSchema.parse(validOutput())).toMatchObject({ model: "openai/gpt-5.6-terra-xhigh", @@ -143,7 +162,7 @@ describe("Test-writing artifact contract", () => { test.path = "tests/unit/other.test.ts"; test.command = "bun run test -- tests/unit/other.test.ts"; patchEvidence.command = "bun run test -- tests/unit/other.test.ts"; - mismatchedPatch.redEvidence.testPlanSha256 = sha256(JSON.stringify(mismatchedPatch.testPlan)); + mismatchedPatch.redEvidence.testPlanSha256 = computeTestPlanDigest(mismatchedPatch.testPlan); expect(testWritingAgentOutputSchema.safeParse(mismatchedPatch).success).toBe(false); }); @@ -183,7 +202,7 @@ describe("Test-writing artifact contract", () => { unsafe.testPlan.patch.content = `${unsafe.testPlan.patch.content}\n${dangerousLine}`; unsafe.testPlan.patch.sha256 = sha256(unsafe.testPlan.patch.content); unsafe.testPlan.patch.byteCount = Buffer.byteLength(unsafe.testPlan.patch.content); - unsafe.redEvidence.testPlanSha256 = sha256(JSON.stringify(unsafe.testPlan)); + unsafe.redEvidence.testPlanSha256 = computeTestPlanDigest(unsafe.testPlan); expect(testWritingAgentOutputSchema.safeParse(unsafe).success).toBe(false); } @@ -191,7 +210,7 @@ describe("Test-writing artifact contract", () => { pathless.testPlan.patch.content = "+expect(false).toBe(true);"; pathless.testPlan.patch.sha256 = sha256(pathless.testPlan.patch.content); pathless.testPlan.patch.byteCount = Buffer.byteLength(pathless.testPlan.patch.content); - pathless.redEvidence.testPlanSha256 = sha256(JSON.stringify(pathless.testPlan)); + pathless.redEvidence.testPlanSha256 = computeTestPlanDigest(pathless.testPlan); expect(testWritingAgentOutputSchema.safeParse(pathless).success).toBe(false); }); }); diff --git a/tests/unit/loops/test-writing-transition.test.ts b/tests/unit/loops/test-writing-transition.test.ts index 5c4214b..4e2ec5e 100644 --- a/tests/unit/loops/test-writing-transition.test.ts +++ b/tests/unit/loops/test-writing-transition.test.ts @@ -2,6 +2,7 @@ import { createHash } from "node:crypto"; import { createTestExecutionReceipt } from "@agent/subagents/test-writer/lib/tool-policy"; import { + computeTestPlanDigest, redTestEvidenceSchemaId, type TestWritingAgentOutput, testPlanSchemaId, @@ -87,7 +88,7 @@ function outputForPlan(plan: { id: string; sha256: string }): TestWritingAgentOu schemaId: redTestEvidenceSchemaId, planId: plan.id, planSha256: plan.sha256, - testPlanSha256: sha256(JSON.stringify(testPlan)), + testPlanSha256: computeTestPlanDigest(testPlan), results: [ { id: "red-ac-1", @@ -218,10 +219,10 @@ describe("test-writing stage transition", () => { ]), ); expect(artifactRows.find((artifact) => artifact.type === "validation_report")?.sha256).toBe( - sha256(JSON.stringify(outputForPlan(prepared.plan).redEvidence)), + computeTestPlanDigest(outputForPlan(prepared.plan).redEvidence), ); expect(artifactRows.find((artifact) => artifact.type === "test_plan")?.sha256).toBe( - sha256(JSON.stringify(outputForPlan(prepared.plan).testPlan)), + computeTestPlanDigest(outputForPlan(prepared.plan).testPlan), ); expect(new Set(artifactRows.map((artifact) => artifact.sha256)).size).toBe(2); }); From 01432a62f43e7b019539bc312338f32bb4843c4a Mon Sep 17 00:00:00 2001 From: Nate Summers Date: Sat, 11 Jul 2026 14:13:44 -0700 Subject: [PATCH 10/12] fix(agent): bind execution receipts to the approved plan digest Receipts previously bound command, paths, exit, output, and patch digests but not plan identity, so a receipt minted under one approved plan could be replayed for another run with an identical patch. The receipt payload now carries the plan digest: run_test_suite requires it at mint time and both verification paths recompute it from the persisted plan, failing closed on mismatch. Part of #47. Co-Authored-By: Claude Fable 5 --- agent/subagents/test-writer/instructions.md | 6 +++--- .../subagents/test-writer/lib/tool-policy.ts | 1 + .../test-writer/tools/run_test_suite.ts | 2 ++ agent/tools/apply_test_writing_result.ts | 1 + src/lib/loops/development-run-transitions.ts | 1 + .../loops/test-writing-transition.test.ts | 19 ++++++++++++++++++- 6 files changed, 26 insertions(+), 4 deletions(-) diff --git a/agent/subagents/test-writer/instructions.md b/agent/subagents/test-writer/instructions.md index 460be4b..86b8163 100644 --- a/agent/subagents/test-writer/instructions.md +++ b/agent/subagents/test-writer/instructions.md @@ -11,9 +11,9 @@ valid red evidence. Use `read_approved_plan`, inspect only necessary repository files with `read_repository_files`, write the complete test set once with -`write_test_files`, and pass the returned patch digest into every -`run_test_suite` call. The emitted patch must be the exact patch returned by -that write tool. +`write_test_files`, and pass the returned patch digest plus the approved plan +digest into every `run_test_suite` call. The emitted patch must be the exact +patch returned by that write tool. Before authoring tests, use `list_repository_files`, `search_repository`, and bounded `read_repository_files` calls to discover applicable `AGENTS.md`, diff --git a/agent/subagents/test-writer/lib/tool-policy.ts b/agent/subagents/test-writer/lib/tool-policy.ts index 529558f..b29d792 100644 --- a/agent/subagents/test-writer/lib/tool-policy.ts +++ b/agent/subagents/test-writer/lib/tool-policy.ts @@ -18,6 +18,7 @@ export type TestExecutionReceiptPayload = { outcome: "expected_failure" | "invalid_failure"; outputSha256: string; patchSha256: string; + planSha256: string; testPaths: string[]; }; diff --git a/agent/subagents/test-writer/tools/run_test_suite.ts b/agent/subagents/test-writer/tools/run_test_suite.ts index 88fa1d2..1eb33d9 100644 --- a/agent/subagents/test-writer/tools/run_test_suite.ts +++ b/agent/subagents/test-writer/tools/run_test_suite.ts @@ -18,6 +18,7 @@ const inputSchema = z.object({ .min(1), expectedAssertions: z.array(z.string().min(1)).min(1), patchSha256: z.string().regex(/^[a-f0-9]{64}$/), + planSha256: z.string().regex(/^[a-f0-9]{64}$/), }); export default defineTool({ @@ -68,6 +69,7 @@ export default defineTool({ outcome, outputSha256: digest, patchSha256: input.patchSha256, + planSha256: input.planSha256, testPaths, }, receiptSecret, diff --git a/agent/tools/apply_test_writing_result.ts b/agent/tools/apply_test_writing_result.ts index 0f90b52..3247c1e 100644 --- a/agent/tools/apply_test_writing_result.ts +++ b/agent/tools/apply_test_writing_result.ts @@ -27,6 +27,7 @@ export default defineTool({ outcome: result.outcome, outputSha256: result.outputReference.sha256, patchSha256: parsed.testPlan.patch.sha256, + planSha256: parsed.testPlan.plan.sha256, testPaths: [test.path], }, result.executionReceipt, diff --git a/src/lib/loops/development-run-transitions.ts b/src/lib/loops/development-run-transitions.ts index d959b57..0bef142 100644 --- a/src/lib/loops/development-run-transitions.ts +++ b/src/lib/loops/development-run-transitions.ts @@ -411,6 +411,7 @@ export async function applyDevelopmentLoopTestWritingResult( outcome: result.outcome, outputSha256: result.outputReference.sha256, patchSha256: output.testPlan.patch.sha256, + planSha256: plan.identity.sha256, testPaths: [test.path], }, result.executionReceipt, diff --git a/tests/unit/loops/test-writing-transition.test.ts b/tests/unit/loops/test-writing-transition.test.ts index 4e2ec5e..88a7546 100644 --- a/tests/unit/loops/test-writing-transition.test.ts +++ b/tests/unit/loops/test-writing-transition.test.ts @@ -27,7 +27,10 @@ function sha256(value: string): string { return createHash("sha256").update(value).digest("hex"); } -function outputForPlan(plan: { id: string; sha256: string }): TestWritingAgentOutput { +function outputForPlan( + plan: { id: string; sha256: string }, + options: { receiptPlanSha256?: string } = {}, +): TestWritingAgentOutput { const patch = [ "diff --git a/tests/unit/red.test.ts b/tests/unit/red.test.ts", "new file mode 100644", @@ -76,6 +79,7 @@ function outputForPlan(plan: { id: string; sha256: string }): TestWritingAgentOu outcome: "expected_failure", outputSha256, patchSha256: testPlan.patch.sha256, + planSha256: options.receiptPlanSha256 ?? plan.sha256, testPaths: ["tests/unit/red.test.ts"], }, "test-secret", @@ -261,6 +265,19 @@ describe("test-writing stage transition", () => { ).rejects.toThrow(DevelopmentLoopTransitionError); }); + it("rejects red evidence receipts minted for a different plan digest", async () => { + const prepared = await prepareRun(); + + await expect( + applyDevelopmentLoopTestWritingResult({ + database: context.db as unknown as DevelopmentLoopTransitionDatabase, + output: outputForPlan(prepared.plan, { receiptPlanSha256: "b".repeat(64) }), + receiptSecret: "test-secret", + runId: prepared.runId, + }), + ).rejects.toThrow("Red evidence receipt is invalid"); + }); + it("rejects self-attested evidence without a valid execution receipt", async () => { const prepared = await prepareRun(); const output = outputForPlan(prepared.plan); From 26eabf355c26d7d7dbf7670e72bdac09c72d8c53 Mon Sep 17 00:00:00 2001 From: Nate Summers Date: Sat, 11 Jul 2026 14:15:35 -0700 Subject: [PATCH 11/12] fix(loops): scope plan-review side effects to approval outcomes Applying an approved plan review previously re-blocked a run that had already advanced to test-writing, and non-approve resolutions wrote raw approval statuses (bypassed, cancelled, applied) into the plan row. Plan-review side effects now advance only on approved, fail closed to a blocked run with a rejected plan for every other resolution, and skip run/plan mutation entirely for applied. Part of #47. Co-Authored-By: Claude Fable 5 --- src/lib/approval-transitions.ts | 13 ++-- .../approval-transitions.integration.test.ts | 78 +++++++++++++++++++ 2 files changed, 86 insertions(+), 5 deletions(-) diff --git a/src/lib/approval-transitions.ts b/src/lib/approval-transitions.ts index b8be849..22ce99e 100644 --- a/src/lib/approval-transitions.ts +++ b/src/lib/approval-transitions.ts @@ -146,7 +146,9 @@ export async function applyApprovalTransition( toStatus: transition.to, }); - if (approval.scope === "plan-review") { + // "applied" records that an already-approved review was acted on; it must + // not re-block a run that advanced when the review was approved. + if (approval.scope === "plan-review" && transition.to !== "applied") { const planId = approval.metadata && typeof approval.metadata.planId === "string" ? approval.metadata.planId @@ -157,18 +159,19 @@ export async function applyApprovalTransition( if (!approval.runId) { throw new Error("Plan-review approval must belong to a run."); } + const approved = transition.to === "approved"; await tx .update(agentPlans) - .set({ status: transition.to }) + .set({ status: approved ? "approved" : "rejected" }) .where(and(eq(agentPlans.id, planId), eq(agentPlans.runId, approval.runId))); await tx .update(loopRuns) .set({ - ...(transition.to === "approved" ? { currentStage: "test-writing" } : {}), - status: transition.to === "approved" ? "running" : "blocked", + ...(approved ? { currentStage: "test-writing" } : {}), + status: approved ? "running" : "blocked", }) .where(eq(loopRuns.id, approval.runId)); - if (transition.to === "approved") { + if (approved) { await tx .update(runSteps) .set({ status: "queued" }) diff --git a/tests/unit/approval-transitions.integration.test.ts b/tests/unit/approval-transitions.integration.test.ts index 3371121..88712a1 100644 --- a/tests/unit/approval-transitions.integration.test.ts +++ b/tests/unit/approval-transitions.integration.test.ts @@ -70,6 +70,84 @@ describe("plan-review approval synchronization", () => { expect(updatedPlan?.status).toBe("approved"); }); + it("marks the plan rejected and blocks the run when its review is bypassed", async () => { + const run = await createDevelopmentLoopRun({ + database: context.db as unknown as DevelopmentLoopRunDatabase, + trigger: { + issueNumber: 47, + repositoryFullName: "ncolesummers/loopworks", + repositoryRevision: { ref: "main", commitSha: "a".repeat(40) }, + title: "Test-writing subagent", + }, + }); + if (run.mode !== "created") throw new Error("Expected created run."); + const [approval] = await context.db + .select() + .from(approvals) + .where(eq(approvals.runId, run.runId)); + if (!approval) throw new Error("Expected plan review fixture."); + + await applyApprovalTransition({ + action: "bypass", + actorId: "maintainer", + approvalId: approval.id, + database: context.db as unknown as ApprovalTransitionDatabase, + expectedStatus: "requested", + }); + + const [plan] = await context.db + .select() + .from(agentPlans) + .where(eq(agentPlans.runId, run.runId)); + const [blockedRun] = await context.db.select().from(loopRuns).where(eq(loopRuns.id, run.runId)); + expect(plan?.status).toBe("rejected"); + expect(blockedRun?.status).toBe("blocked"); + }); + + it("leaves an advanced run untouched when an approved plan review is applied", async () => { + const run = await createDevelopmentLoopRun({ + database: context.db as unknown as DevelopmentLoopRunDatabase, + trigger: { + issueNumber: 47, + repositoryFullName: "ncolesummers/loopworks", + repositoryRevision: { ref: "main", commitSha: "a".repeat(40) }, + title: "Test-writing subagent", + }, + }); + if (run.mode !== "created") throw new Error("Expected created run."); + const [approval] = await context.db + .select() + .from(approvals) + .where(eq(approvals.runId, run.runId)); + if (!approval) throw new Error("Expected plan review fixture."); + + await applyApprovalTransition({ + action: "approve", + actorId: "maintainer", + approvalId: approval.id, + database: context.db as unknown as ApprovalTransitionDatabase, + expectedStatus: "requested", + }); + await applyApprovalTransition({ + action: "apply", + actorId: "loopworks", + approvalId: approval.id, + database: context.db as unknown as ApprovalTransitionDatabase, + expectedStatus: "approved", + }); + + const [plan] = await context.db + .select() + .from(agentPlans) + .where(eq(agentPlans.runId, run.runId)); + const [advancedRun] = await context.db + .select() + .from(loopRuns) + .where(eq(loopRuns.id, run.runId)); + expect(plan?.status).toBe("approved"); + expect(advancedRun).toMatchObject({ currentStage: "test-writing", status: "running" }); + }); + it("records a planner-pinned revision before creating the durable review", async () => { const run = await createDevelopmentLoopRun({ database: context.db as unknown as DevelopmentLoopRunDatabase, From cb040f76cd7ed3886d9edf3465330ea79234b3d0 Mon Sep 17 00:00:00 2001 From: Nate Summers Date: Sat, 11 Jul 2026 14:17:11 -0700 Subject: [PATCH 12/12] fix(loops): preserve existing run start time when recording plans recordDevelopmentLoopPlanArtifact unconditionally reset loopRuns.startedAt to the queue time; keep an already-recorded start and fall back to queuedAt, matching the test-writing transition. Part of #47. Co-Authored-By: Claude Fable 5 --- src/lib/loops/development-run-transitions.ts | 3 +- .../approval-transitions.integration.test.ts | 37 +++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/src/lib/loops/development-run-transitions.ts b/src/lib/loops/development-run-transitions.ts index 0bef142..a319cad 100644 --- a/src/lib/loops/development-run-transitions.ts +++ b/src/lib/loops/development-run-transitions.ts @@ -101,6 +101,7 @@ export async function recordDevelopmentLoopPlanArtifact( id: loopRuns.id, queuedAt: loopRuns.queuedAt, repositoryFullName: repositories.fullName, + startedAt: loopRuns.startedAt, }) .from(loopRuns) .innerJoin(repositories, eq(loopRuns.repositoryId, repositories.id)) @@ -197,7 +198,7 @@ export async function recordDevelopmentLoopPlanArtifact( .where(eq(runSteps.id, planningStep.id)); await tx .update(loopRuns) - .set({ startedAt: run.queuedAt, status: "waiting_for_approval" }) + .set({ startedAt: run.startedAt ?? run.queuedAt, status: "waiting_for_approval" }) .where(eq(loopRuns.id, input.runId)); return { approvalId, planId: planRow.id, runId: input.runId, status: "waiting_for_approval" }; diff --git a/tests/unit/approval-transitions.integration.test.ts b/tests/unit/approval-transitions.integration.test.ts index 88712a1..2b0ee44 100644 --- a/tests/unit/approval-transitions.integration.test.ts +++ b/tests/unit/approval-transitions.integration.test.ts @@ -188,4 +188,41 @@ describe("plan-review approval synchronization", () => { }); expect(updatedRun?.status).toBe("waiting_for_approval"); }); + + it("preserves an existing run start time when recording the plan artifact", async () => { + const run = await createDevelopmentLoopRun({ + database: context.db as unknown as DevelopmentLoopRunDatabase, + trigger: { + body: "## Acceptance Criteria\n- Planner output is pinned before review.", + issueNumber: 47, + repositoryFullName: "ncolesummers/loopworks", + title: "Test-writing subagent", + }, + }); + if (run.mode !== "created") throw new Error("Expected created run."); + const startedAt = new Date("2026-07-11T15:30:00.000Z"); + await context.db + .update(loopRuns) + .set({ startedAt, status: "running" }) + .where(eq(loopRuns.id, run.runId)); + + const plan = createPlanningAgentSeedPlan({ + body: "## Acceptance Criteria\n- Planner output is pinned before review.", + issueNumber: 47, + labels: [], + milestone: null, + repositoryFullName: "ncolesummers/loopworks", + repositoryRevision: { commitSha: "a".repeat(40), ref: "main" }, + title: "Test-writing subagent", + }); + await recordDevelopmentLoopPlanArtifact({ + database: context.db as unknown as DevelopmentLoopTransitionDatabase, + plan, + runId: run.runId, + }); + + const [updatedRun] = await context.db.select().from(loopRuns).where(eq(loopRuns.id, run.runId)); + expect(updatedRun?.startedAt).toEqual(startedAt); + expect(updatedRun?.status).toBe("waiting_for_approval"); + }); });