diff --git a/docs/adr/0017-durable-dispatch-leases-and-retry-backoff.md b/docs/adr/0017-durable-dispatch-leases-and-retry-backoff.md new file mode 100644 index 0000000..c842a0a --- /dev/null +++ b/docs/adr/0017-durable-dispatch-leases-and-retry-backoff.md @@ -0,0 +1,90 @@ +# ADR 0017: Durable Dispatch Leases and Retry Backoff + +Status: Proposed +Date: 2026-07-24 +Issue: [#96](https://github.com/ncolesummers/loopworks/issues/96) + +## Context + +Webhook idempotency prevents replaying one delivery, but it does not enforce a +manifest concurrency cap across different deliveries or hosts. In-memory +semaphores also lose authority on restart. Retry behavior needs the same durable +boundary: delayed work cannot depend on sleeps, terminal evidence must remain +inspectable, and attempt limits must mean the same thing in every runtime path. + +## Decision + +Use Postgres admission transactions and `idempotency_locks` as the durable +dispatch authority. A persistent group-guard row is selected `FOR UPDATE` +before capacity is counted. Acquired leases in the resolved manifest group +consume capacity even after their expiry until their owner is finalized; this +prevents an expired-but-running attempt from overlapping its successor. + +Every issue-backed run is protected by a partial unique index over repository +and issue while its status is nonterminal. This applies across loop types. +Delivery IDs remain replay keys, not concurrency identities. `{repo}` resolves +to the canonical repository full name; other unresolved placeholders are +rejected. A persistent repository/issue guard serializes development and +research creation before the cross-loop uniqueness check, so contention is a +typed outcome rather than a raw constraint error. + +Over-cap work is persisted as a queued run without a lease. Queue draining is +ordered by `queuedAt`, then issue number, and acquires leases up to the manifest +cap. Lease-less runs cannot enter a development stage. The winning terminal +finalizer releases only a lease whose `run_id`, owner, and status still match; +replay does not rewrite `releasedAt`, except to repair a legacy acquired leak +using the already-persisted completion time. Terminal draining is scoped to the +released repository group. Lease expiry is derived from +`budgets.maxRunMinutes`. + +`retryPolicy.maxAttempts` counts the initial attempt. Fixed backoff is +`min(initialSeconds, maxSeconds)`; exponential backoff is +`min(initialSeconds * 2^(completedAttempt - 1), maxSeconds)`. Retryable stage +failures keep the failed step evidence, release the lease, and receive a future +eligibility time. The attempt increments only when a supervisor tick leases the +due work. Stage promotions and linked runs consume the same dispatch-attempt +budget. No hosted polling cadence is implied by this ADR. + +Reconciliation-authored `stalled` and `timed_out` outcomes preserve the source +run as terminal and create a new trace-linked run from planning. The child +inherits an immutable trigger snapshot, records `retryOfRunId`, and increments +the dispatch attempt. Cancellation, success, and untyped terminal failure do +not create linked retries. + +Admission primitives are loop-agnostic, while issue #96 integrates the complete +lease and retry lifecycle only for the development loop. Research-loop durable +transitions and enforcement require a separate M9 story. + +## Consequences + +- Restarts do not erase capacity ownership or retry eligibility. +- Database row locks and uniqueness constraints, rather than process locality, + serialize competing dispatchers. +- Terminal retry history is represented as linked immutable runs, increasing + row count while improving forensic clarity. +- Operators must reconcile an expired owner before its capacity is reusable. +- Missing lease evidence is unknown and fails open; released or expired lease + evidence is inactive. Queued acquired owners participate in reconciliation, + while queued deferred work does not. +- Migration preflight fails closed when historical active duplicates exist. + Operators must inspect and terminalize the losing rows before retrying the + migration; the migration never guesses which forensic record to mutate. +- A production scheduler may call the injected-clock supervisor, but its cadence + remains an explicit future hosting decision. + +## Validation + +1. PGlite integration tests cover deferral, contention, queue order, terminal + release, restart-equivalent reads, retry timing, exhaustion, and trace links. +2. Migration tests apply and replay the schema, including the duplicate-active + preflight and partial unique index. +3. Static observability tests keep dispatch and retry spans and metrics behind + central helpers with bounded attributes. + +## Follow-Ups + +- Draft an M9 story titled “Research-loop durable transitions, reconciliation, + lease lifecycle, and retry enforcement.” Do not create it without maintainer + authorization. +- Add a real-Postgres multi-session admission lane if production lock-wait + behavior needs evidence stronger than embedded PGlite transactions. diff --git a/docs/adr/README.md b/docs/adr/README.md index a246260..aa2ebf6 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -37,6 +37,7 @@ Loopworks uses ADRs for durable technical and product architecture decisions. Gi | [0014](0014-guarded-github-pr-write-reconciliation.md) | Proposed | Guard GitHub PR writes with digest-bound approval, deterministic branches, and two-phase reconciliation. | | [0015](0015-stage-orchestrator-and-isolated-subagent-handoffs.md) | Proposed | Use a neutral root orchestrator, independent sibling subagents, and typed artifact handoffs between isolated sandboxes. | | [0016](0016-run-reconciliation-and-terminal-reasons.md) | Proposed | Reconcile active runs with typed terminal reasons, read-only tracker refresh, and injected execution liveness. | +| [0017](0017-durable-dispatch-leases-and-retry-backoff.md) | Proposed | Serialize manifest admission with durable leases and preserve bounded retry evidence across restarts. | ## Template diff --git a/docs/architecture.md b/docs/architecture.md index a819a81..86c97be 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -66,7 +66,9 @@ The initial schema should support these tables: 7. Artifacts: run id, step id, type, title, URI, metadata, checksum where relevant. 8. Approvals: run id, gate key, status, approver identity, rationale, evidence URI, timestamps. 9. GitHub events: delivery id, event name, action, installation id, payload summary, processed status, error summary. -10. Idempotency locks: key, scope, acquired/expired timestamps, owner, status. +10. Idempotency locks: key, scope, run and trace correlation, acquired/expired + timestamps, owner, and status. Released group-guard rows serialize admission; + acquired run leases are the execution-liveness authority. 11. Observability events or projections: correlation ids, metric counters, trace links, log summary links, and alert state. ## Observability @@ -92,11 +94,23 @@ Logs are not the event store. The control plane must still persist durable run, 5. Acquire a short-lived lock for repo plus issue plus trigger type. 6. Normalize the event into Loopworks event tables. 7. Evaluate loop triggers from the active manifest snapshot. -8. Create a run or append a skipped/no-op decision with a durable reason. -9. Write a summary or durable link back to GitHub only when useful to humans. +8. Resolve the manifest concurrency group, lock its persistent guard row plus + the cross-loop repository/issue guard, and persist the run. Acquire a run + lease when capacity exists; otherwise retain the run as queued without a + lease. Stage transitions require that exact acquired lease. +9. On terminal completion, release the exact owner/run lease and drain due work + for that repository group by oldest queue time then issue number. Retryable + failures keep their evidence and use manifest-computed future eligibility + rather than process sleeps. +10. Write a summary or durable link back to GitHub only when useful to humans. Webhook processing must be repeat-safe. Re-delivery should not create duplicate runs, duplicate comments, or conflicting approval gates. +One repository issue may have only one nonterminal run across loop types. +Sequential runs are allowed after terminal completion. Expired but unreconciled +leases continue consuming capacity so work cannot overlap before ownership is +resolved. + ## Loop Manifest The loop manifest should be versioned and validated. It must include: diff --git a/docs/loop-manifest.md b/docs/loop-manifest.md index dfa083e..5d85506 100644 --- a/docs/loop-manifest.md +++ b/docs/loop-manifest.md @@ -103,6 +103,28 @@ terminal reasons are persisted separately from status: `stalled` and `timed_out` map to `failed`, while `canceled_by_reconciliation` maps to `canceled`. Execution liveness is supplied by the runtime boundary and an unknown signal fails open rather than terminating a potentially healthy run. +For persistent development runs, the exact acquired run lease is the default +liveness authority. Released or expired evidence is inactive; a missing lease +is unknown and therefore fails open. Queued runs are reconciled only when they +own an acquired lease, so deferred work is not mistaken for dead execution. + +Development-loop admission resolves `{repo}` in `concurrency.group` to the +canonical repository full name and rejects any other unresolved placeholder. +The group cap counts every acquired lease, including an expired lease whose +owner has not yet been finalized. Over-cap triggers still create durable queued +runs without leases, and stage transitions reject lease-less mutation. The +persistent creation entry point reports `dispatched`, `deferred`, or +`lease_contention` without flattening admission state. A terminal finalizer +releases the matching run/owner lease and best-effort drains due work for the +released repository group in `queuedAt`, then issue-number order. + +`retryPolicy.maxAttempts` is a total-attempt budget: attempt one is the initial +execution. Fixed and exponential backoff are calculated from the completed +attempt and capped by `maxSeconds`. Retryable stage failures remain inspectable +until their due retry is leased, at which point the step attempt increments. +`stalled` and `timed_out` create a new trace-linked run from planning when budget +remains; `canceled_by_reconciliation`, success, and untyped terminal failure do +not. The supervisor accepts an injected clock and defines no hosted cadence. The enabled `research-loop` is a parallel, fixture-backed generality probe. It requires both `spike` and `agent-ready`, uses a separate diff --git a/drizzle/0007_remarkable_legion.sql b/drizzle/0007_remarkable_legion.sql new file mode 100644 index 0000000..07d3d9d --- /dev/null +++ b/drizzle/0007_remarkable_legion.sql @@ -0,0 +1,21 @@ +ALTER TABLE "idempotency_locks" ADD COLUMN "run_id" uuid;--> statement-breakpoint +ALTER TABLE "idempotency_locks" ADD COLUMN "trace_id" text;--> statement-breakpoint +ALTER TABLE "idempotency_locks" ADD CONSTRAINT "idempotency_locks_run_id_loop_runs_id_fk" FOREIGN KEY ("run_id") REFERENCES "public"."loop_runs"("id") ON DELETE set null ON UPDATE no action;--> statement-breakpoint +CREATE INDEX "idempotency_locks_run_status_idx" ON "idempotency_locks" USING btree ("run_id","status");--> statement-breakpoint +DO $$ +BEGIN + IF EXISTS ( + SELECT 1 + FROM "loop_runs" + WHERE "github_issue_number" IS NOT NULL + AND ( + "status" IN ('queued', 'running', 'waiting_for_approval', 'blocked') + OR ("status" = 'failed' AND "completed_at" IS NULL) + ) + GROUP BY "repository_id", "github_issue_number" + HAVING count(*) > 1 + ) THEN + RAISE EXCEPTION 'Cannot enforce active issue uniqueness: duplicate nonterminal loop runs exist'; + END IF; +END $$;--> statement-breakpoint +CREATE UNIQUE INDEX "loop_runs_active_repository_issue_idx" ON "loop_runs" USING btree ("repository_id","github_issue_number") WHERE "loop_runs"."github_issue_number" IS NOT NULL AND ("loop_runs"."status" IN ('queued', 'running', 'waiting_for_approval', 'blocked') OR ("loop_runs"."status" = 'failed' AND "loop_runs"."completed_at" IS NULL)); diff --git a/drizzle/meta/0007_snapshot.json b/drizzle/meta/0007_snapshot.json new file mode 100644 index 0000000..dfcd60a --- /dev/null +++ b/drizzle/meta/0007_snapshot.json @@ -0,0 +1,2336 @@ +{ + "id": "e179e7f9-ae47-47a4-9ec5-7c4b013aeaba", + "prevId": "7c56be06-d3f2-40e4-acc2-c1663d2e43fc", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.accounts": { + "name": "accounts", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_account_id": { + "name": "provider_account_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "token_type": { + "name": "token_type", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "session_state": { + "name": "session_state", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "accounts_user_id_users_id_fk": { + "name": "accounts_user_id_users_id_fk", + "tableFrom": "accounts", + "tableTo": "users", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "accounts_provider_provider_account_id_pk": { + "name": "accounts_provider_provider_account_id_pk", + "columns": ["provider", "provider_account_id"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.agent_plans": { + "name": "agent_plans", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "loop_id": { + "name": "loop_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "run_id": { + "name": "run_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "issue_number": { + "name": "issue_number", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "agent_name": { + "name": "agent_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'planner'" + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "input": { + "name": "input", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "plan": { + "name": "plan", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "agent_plans_loop_id_created_at_idx": { + "name": "agent_plans_loop_id_created_at_idx", + "columns": [ + { + "expression": "loop_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "agent_plans_run_id_created_at_idx": { + "name": "agent_plans_run_id_created_at_idx", + "columns": [ + { + "expression": "run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "agent_plans_loop_id_loops_id_fk": { + "name": "agent_plans_loop_id_loops_id_fk", + "tableFrom": "agent_plans", + "tableTo": "loops", + "columnsFrom": ["loop_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "agent_plans_run_id_loop_runs_id_fk": { + "name": "agent_plans_run_id_loop_runs_id_fk", + "tableFrom": "agent_plans", + "tableTo": "loop_runs", + "columnsFrom": ["run_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.approval_transition_events": { + "name": "approval_transition_events", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "approval_id": { + "name": "approval_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "run_id": { + "name": "run_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "from_status": { + "name": "from_status", + "type": "approval_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "to_status": { + "name": "to_status", + "type": "approval_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "action": { + "name": "action", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "occurred_at": { + "name": "occurred_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "note": { + "name": "note", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "approval_transition_events_approval_created_at_idx": { + "name": "approval_transition_events_approval_created_at_idx", + "columns": [ + { + "expression": "approval_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "occurred_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "approval_transition_events_run_created_at_idx": { + "name": "approval_transition_events_run_created_at_idx", + "columns": [ + { + "expression": "run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "occurred_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "approval_transition_events_approval_id_approvals_id_fk": { + "name": "approval_transition_events_approval_id_approvals_id_fk", + "tableFrom": "approval_transition_events", + "tableTo": "approvals", + "columnsFrom": ["approval_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "approval_transition_events_run_id_loop_runs_id_fk": { + "name": "approval_transition_events_run_id_loop_runs_id_fk", + "tableFrom": "approval_transition_events", + "tableTo": "loop_runs", + "columnsFrom": ["run_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.approvals": { + "name": "approvals", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "loop_id": { + "name": "loop_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "run_id": { + "name": "run_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "approval_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'requested'" + }, + "requested_by": { + "name": "requested_by", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "requested_at": { + "name": "requested_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "resolved_by": { + "name": "resolved_by", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "resolved_at": { + "name": "resolved_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "note": { + "name": "note", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "approvals_loop_id_status_idx": { + "name": "approvals_loop_id_status_idx", + "columns": [ + { + "expression": "loop_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "approvals_run_id_status_idx": { + "name": "approvals_run_id_status_idx", + "columns": [ + { + "expression": "run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "approvals_loop_id_loops_id_fk": { + "name": "approvals_loop_id_loops_id_fk", + "tableFrom": "approvals", + "tableTo": "loops", + "columnsFrom": ["loop_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "approvals_run_id_loop_runs_id_fk": { + "name": "approvals_run_id_loop_runs_id_fk", + "tableFrom": "approvals", + "tableTo": "loop_runs", + "columnsFrom": ["run_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.artifacts": { + "name": "artifacts", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "run_id": { + "name": "run_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "step_id": { + "name": "step_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "type": { + "name": "type", + "type": "artifact_type", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'other'" + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "uri": { + "name": "uri", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "sha256": { + "name": "sha256", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "artifacts_run_type_idx": { + "name": "artifacts_run_type_idx", + "columns": [ + { + "expression": "run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "type", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "artifacts_step_id_idx": { + "name": "artifacts_step_id_idx", + "columns": [ + { + "expression": "step_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "artifacts_run_id_loop_runs_id_fk": { + "name": "artifacts_run_id_loop_runs_id_fk", + "tableFrom": "artifacts", + "tableTo": "loop_runs", + "columnsFrom": ["run_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "artifacts_step_id_run_steps_id_fk": { + "name": "artifacts_step_id_run_steps_id_fk", + "tableFrom": "artifacts", + "tableTo": "run_steps", + "columnsFrom": ["step_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.deployments": { + "name": "deployments", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "repository_id": { + "name": "repository_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "loop_id": { + "name": "loop_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "run_id": { + "name": "run_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'vercel'" + }, + "external_id": { + "name": "external_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "project_name": { + "name": "project_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "deployment_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "environment": { + "name": "environment", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "branch": { + "name": "branch", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "commit_sha": { + "name": "commit_sha", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "inspector_url": { + "name": "inspector_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "alias_urls": { + "name": "alias_urls", + "type": "text[]", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "ready_at": { + "name": "ready_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "deployments_project_id_created_at_idx": { + "name": "deployments_project_id_created_at_idx", + "columns": [ + { + "expression": "project_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "deployments_run_id_created_at_idx": { + "name": "deployments_run_id_created_at_idx", + "columns": [ + { + "expression": "run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "deployments_repository_id_repositories_id_fk": { + "name": "deployments_repository_id_repositories_id_fk", + "tableFrom": "deployments", + "tableTo": "repositories", + "columnsFrom": ["repository_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "deployments_loop_id_loops_id_fk": { + "name": "deployments_loop_id_loops_id_fk", + "tableFrom": "deployments", + "tableTo": "loops", + "columnsFrom": ["loop_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "deployments_run_id_loop_runs_id_fk": { + "name": "deployments_run_id_loop_runs_id_fk", + "tableFrom": "deployments", + "tableTo": "loop_runs", + "columnsFrom": ["run_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "deployments_external_id_unique": { + "name": "deployments_external_id_unique", + "nullsNotDistinct": false, + "columns": ["external_id"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.idempotency_locks": { + "name": "idempotency_locks", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "idempotency_lock_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'acquired'" + }, + "acquired_at": { + "name": "acquired_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "released_at": { + "name": "released_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "run_id": { + "name": "run_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "trace_id": { + "name": "trace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "idempotency_locks_scope_status_idx": { + "name": "idempotency_locks_scope_status_idx", + "columns": [ + { + "expression": "scope", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idempotency_locks_expires_at_idx": { + "name": "idempotency_locks_expires_at_idx", + "columns": [ + { + "expression": "expires_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "idempotency_locks_run_status_idx": { + "name": "idempotency_locks_run_status_idx", + "columns": [ + { + "expression": "run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "idempotency_locks_run_id_loop_runs_id_fk": { + "name": "idempotency_locks_run_id_loop_runs_id_fk", + "tableFrom": "idempotency_locks", + "tableTo": "loop_runs", + "columnsFrom": ["run_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "idempotency_locks_key_unique": { + "name": "idempotency_locks_key_unique", + "nullsNotDistinct": false, + "columns": ["key"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.loop_events": { + "name": "loop_events", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "loop_id": { + "name": "loop_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "actor_type": { + "name": "actor_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "event_type": { + "name": "event_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "from_state": { + "name": "from_state", + "type": "loop_state", + "typeSchema": "public", + "primaryKey": false, + "notNull": false + }, + "to_state": { + "name": "to_state", + "type": "loop_state", + "typeSchema": "public", + "primaryKey": false, + "notNull": false + }, + "reason": { + "name": "reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "loop_events_loop_id_created_at_idx": { + "name": "loop_events_loop_id_created_at_idx", + "columns": [ + { + "expression": "loop_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "loop_events_loop_id_loops_id_fk": { + "name": "loop_events_loop_id_loops_id_fk", + "tableFrom": "loop_events", + "tableTo": "loops", + "columnsFrom": ["loop_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.loop_runs": { + "name": "loop_runs", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "repository_id": { + "name": "repository_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "loop_key": { + "name": "loop_key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "github_issue_number": { + "name": "github_issue_number", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "github_issue_url": { + "name": "github_issue_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "run_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'queued'" + }, + "terminal_reason": { + "name": "terminal_reason", + "type": "run_terminal_reason", + "typeSchema": "public", + "primaryKey": false, + "notNull": false + }, + "current_stage": { + "name": "current_stage", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'planning'" + }, + "trace_id": { + "name": "trace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "cost_cents": { + "name": "cost_cents", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "queued_at": { + "name": "queued_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "started_at": { + "name": "started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "canceled_at": { + "name": "canceled_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "loop_runs_active_repository_issue_idx": { + "name": "loop_runs_active_repository_issue_idx", + "columns": [ + { + "expression": "repository_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "github_issue_number", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "where": "\"loop_runs\".\"github_issue_number\" IS NOT NULL AND (\"loop_runs\".\"status\" IN ('queued', 'running', 'waiting_for_approval', 'blocked') OR (\"loop_runs\".\"status\" = 'failed' AND \"loop_runs\".\"completed_at\" IS NULL))", + "concurrently": false, + "method": "btree", + "with": {} + }, + "loop_runs_repository_status_idx": { + "name": "loop_runs_repository_status_idx", + "columns": [ + { + "expression": "repository_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "loop_runs_repository_issue_idx": { + "name": "loop_runs_repository_issue_idx", + "columns": [ + { + "expression": "repository_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "github_issue_number", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "loop_runs_repository_id_repositories_id_fk": { + "name": "loop_runs_repository_id_repositories_id_fk", + "tableFrom": "loop_runs", + "tableTo": "repositories", + "columnsFrom": ["repository_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.loops": { + "name": "loops", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "repository_id": { + "name": "repository_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "github_issue_number": { + "name": "github_issue_number", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "state": { + "name": "state", + "type": "loop_state", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'intake'" + }, + "milestone": { + "name": "milestone", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "area_label": { + "name": "area_label", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "priority_label": { + "name": "priority_label", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "owner_github_login": { + "name": "owner_github_login", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "source_url": { + "name": "source_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "last_synced_at": { + "name": "last_synced_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "loops_repository_issue_number_idx": { + "name": "loops_repository_issue_number_idx", + "columns": [ + { + "expression": "repository_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "github_issue_number", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "loops_repository_id_repositories_id_fk": { + "name": "loops_repository_id_repositories_id_fk", + "tableFrom": "loops", + "tableTo": "repositories", + "columnsFrom": ["repository_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.observability_events": { + "name": "observability_events", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "repository_id": { + "name": "repository_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "run_id": { + "name": "run_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "step_id": { + "name": "step_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "event_type": { + "name": "event_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "severity": { + "name": "severity", + "type": "observability_severity", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'info'" + }, + "correlation_id": { + "name": "correlation_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "trace_id": { + "name": "trace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "metric_name": { + "name": "metric_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "metric_value": { + "name": "metric_value", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "observability_events_type_created_at_idx": { + "name": "observability_events_type_created_at_idx", + "columns": [ + { + "expression": "event_type", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "observability_events_run_created_at_idx": { + "name": "observability_events_run_created_at_idx", + "columns": [ + { + "expression": "run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "observability_events_trace_id_idx": { + "name": "observability_events_trace_id_idx", + "columns": [ + { + "expression": "trace_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "observability_events_repository_id_repositories_id_fk": { + "name": "observability_events_repository_id_repositories_id_fk", + "tableFrom": "observability_events", + "tableTo": "repositories", + "columnsFrom": ["repository_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "observability_events_run_id_loop_runs_id_fk": { + "name": "observability_events_run_id_loop_runs_id_fk", + "tableFrom": "observability_events", + "tableTo": "loop_runs", + "columnsFrom": ["run_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + }, + "observability_events_step_id_run_steps_id_fk": { + "name": "observability_events_step_id_run_steps_id_fk", + "tableFrom": "observability_events", + "tableTo": "run_steps", + "columnsFrom": ["step_id"], + "columnsTo": ["id"], + "onDelete": "set null", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.repositories": { + "name": "repositories", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "github_repo_id": { + "name": "github_repo_id", + "type": "bigint", + "primaryKey": false, + "notNull": true + }, + "owner": { + "name": "owner", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "full_name": { + "name": "full_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "installation_id": { + "name": "installation_id", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "is_active": { + "name": "is_active", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "health": { + "name": "health", + "type": "repo_health", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'healthy'" + }, + "framework": { + "name": "framework", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'Unknown'" + }, + "default_branch": { + "name": "default_branch", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'main'" + }, + "ci_commands": { + "name": "ci_commands", + "type": "text[]", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "docs_href": { + "name": "docs_href", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "observability_href": { + "name": "observability_href", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "design_system_href": { + "name": "design_system_href", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "enabled_loops": { + "name": "enabled_loops", + "type": "text[]", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "validation_gates": { + "name": "validation_gates", + "type": "text[]", + "primaryKey": false, + "notNull": true, + "default": "'{}'" + }, + "last_synced_at": { + "name": "last_synced_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "repositories_github_repo_id_unique": { + "name": "repositories_github_repo_id_unique", + "nullsNotDistinct": false, + "columns": ["github_repo_id"] + }, + "repositories_full_name_unique": { + "name": "repositories_full_name_unique", + "nullsNotDistinct": false, + "columns": ["full_name"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.run_steps": { + "name": "run_steps", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "run_id": { + "name": "run_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "stage": { + "name": "stage", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "run_step_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'queued'" + }, + "attempt": { + "name": "attempt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 1 + }, + "actor_type": { + "name": "actor_type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "summary": { + "name": "summary", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "validation_command": { + "name": "validation_command", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "validation_status": { + "name": "validation_status", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "trace_id": { + "name": "trace_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "queued_at": { + "name": "queued_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "started_at": { + "name": "started_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "completed_at": { + "name": "completed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "run_steps_run_stage_idx": { + "name": "run_steps_run_stage_idx", + "columns": [ + { + "expression": "run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "stage", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "run_steps_run_status_idx": { + "name": "run_steps_run_status_idx", + "columns": [ + { + "expression": "run_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "status", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "run_steps_run_id_loop_runs_id_fk": { + "name": "run_steps_run_id_loop_runs_id_fk", + "tableFrom": "run_steps", + "tableTo": "loop_runs", + "columnsFrom": ["run_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.sessions": { + "name": "sessions", + "schema": "", + "columns": { + "session_token": { + "name": "session_token", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "expires": { + "name": "expires", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": { + "sessions_user_id_users_id_fk": { + "name": "sessions_user_id_users_id_fk", + "tableFrom": "sessions", + "tableTo": "users", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.users": { + "name": "users", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "email_verified": { + "name": "email_verified", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "github_login": { + "name": "github_login", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "users_email_unique": { + "name": "users_email_unique", + "nullsNotDistinct": false, + "columns": ["email"] + }, + "users_github_login_unique": { + "name": "users_github_login_unique", + "nullsNotDistinct": false, + "columns": ["github_login"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.vercel_projects": { + "name": "vercel_projects", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "repository_id": { + "name": "repository_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "project_id": { + "name": "project_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "project_name": { + "name": "project_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "team_id": { + "name": "team_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "team_slug": { + "name": "team_slug", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "production_url": { + "name": "production_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "dashboard_url": { + "name": "dashboard_url", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "vercel_projects_repository_id_idx": { + "name": "vercel_projects_repository_id_idx", + "columns": [ + { + "expression": "repository_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "vercel_projects_repository_id_repositories_id_fk": { + "name": "vercel_projects_repository_id_repositories_id_fk", + "tableFrom": "vercel_projects", + "tableTo": "repositories", + "columnsFrom": ["repository_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "vercel_projects_project_id_unique": { + "name": "vercel_projects_project_id_unique", + "nullsNotDistinct": false, + "columns": ["project_id"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.verification_tokens": { + "name": "verification_tokens", + "schema": "", + "columns": { + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires": { + "name": "expires", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "verification_tokens_identifier_token_pk": { + "name": "verification_tokens_identifier_token_pk", + "columns": ["identifier", "token"] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.webhook_deliveries": { + "name": "webhook_deliveries", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "gen_random_uuid()" + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'github'" + }, + "delivery_id": { + "name": "delivery_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "event": { + "name": "event", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "action": { + "name": "action", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "repository_full_name": { + "name": "repository_full_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "status": { + "name": "status", + "type": "webhook_delivery_status", + "typeSchema": "public", + "primaryKey": false, + "notNull": true, + "default": "'received'" + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "received_at": { + "name": "received_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "processed_at": { + "name": "processed_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "webhook_deliveries_delivery_id_unique": { + "name": "webhook_deliveries_delivery_id_unique", + "nullsNotDistinct": false, + "columns": ["delivery_id"] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": { + "public.approval_status": { + "name": "approval_status", + "schema": "public", + "values": ["requested", "approved", "rejected", "cancelled", "expired", "applied", "bypassed"] + }, + "public.artifact_type": { + "name": "artifact_type", + "schema": "public", + "values": [ + "plan", + "validation_report", + "test_plan", + "patch", + "pr_intent", + "deployment_summary", + "log_summary", + "trace", + "screenshot", + "other" + ] + }, + "public.deployment_status": { + "name": "deployment_status", + "schema": "public", + "values": ["queued", "building", "ready", "error", "canceled"] + }, + "public.idempotency_lock_status": { + "name": "idempotency_lock_status", + "schema": "public", + "values": ["acquired", "released", "expired"] + }, + "public.loop_state": { + "name": "loop_state", + "schema": "public", + "values": [ + "intake", + "triage", + "planned", + "in_progress", + "waiting_on_review", + "validating", + "blocked", + "done" + ] + }, + "public.observability_severity": { + "name": "observability_severity", + "schema": "public", + "values": ["debug", "info", "warn", "error"] + }, + "public.repo_health": { + "name": "repo_health", + "schema": "public", + "values": ["healthy", "watch", "blocked", "disconnected"] + }, + "public.run_status": { + "name": "run_status", + "schema": "public", + "values": [ + "queued", + "running", + "waiting_for_approval", + "blocked", + "failed", + "succeeded", + "canceled" + ] + }, + "public.run_step_status": { + "name": "run_step_status", + "schema": "public", + "values": ["queued", "running", "skipped", "failed", "succeeded"] + }, + "public.run_terminal_reason": { + "name": "run_terminal_reason", + "schema": "public", + "values": ["succeeded", "failed", "timed_out", "stalled", "canceled_by_reconciliation"] + }, + "public.webhook_delivery_status": { + "name": "webhook_delivery_status", + "schema": "public", + "values": ["received", "processed", "ignored", "failed"] + } + }, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} diff --git a/drizzle/meta/_journal.json b/drizzle/meta/_journal.json index 9db3f3b..8edd869 100644 --- a/drizzle/meta/_journal.json +++ b/drizzle/meta/_journal.json @@ -50,6 +50,13 @@ "when": 1784778985115, "tag": "0006_skinny_electro", "breakpoints": true + }, + { + "idx": 7, + "version": "7", + "when": 1784956021096, + "tag": "0007_remarkable_legion", + "breakpoints": true } ] } diff --git a/src/app/api/github/webhooks/route.ts b/src/app/api/github/webhooks/route.ts index 1691ac4..9857f2e 100644 --- a/src/app/api/github/webhooks/route.ts +++ b/src/app/api/github/webhooks/route.ts @@ -128,7 +128,15 @@ const resolveAgentReadyLoopState: GithubAgentReadyLoopResolver = (trigger) => ({ ), }); -function getNextAction(agentReadyTrigger: GithubAgentReadyTrigger): string { +function getNextAction( + agentReadyTrigger: GithubAgentReadyTrigger, + developmentRun?: DevelopmentRunOutcome, + researchRun?: ResearchRunOutcome, +): string { + if (developmentRun?.mode === "deferred") return "await_dispatch_capacity"; + if (developmentRun?.mode === "lease_contention" || researchRun?.mode === "lease_contention") { + return "observe_existing_run"; + } if (agentReadyTrigger.shouldTrigger === true && agentReadyTrigger.workflow === "research") { return "queue_deep_research_loop"; } @@ -472,7 +480,6 @@ export async function handleGithubWebhookPost( event === "issues" && issuesPayload ? getAgentReadyTrigger(issuesPayload, resolveAgentReadyLoopState) : { shouldTrigger: false, reason: "unsupported_event" }; - const nextAction = getNextAction(agentReadyTrigger); const deliveryStatus = agentReadyTrigger.shouldTrigger ? "processed" : "ignored"; const processedAt = now(); const developmentRun = await resolveDevelopmentRunOutcome({ @@ -495,6 +502,7 @@ export async function handleGithubWebhookPost( selectedDeliveryStore.mode === "drizzle" || Boolean(dependencies.developmentRunDatabase), traceId, }); + const nextAction = getNextAction(agentReadyTrigger, developmentRun, researchRun); await webhookDeliveryStore.complete(claim.key, { deliveryId: claim.deliveryId, diff --git a/src/db/schema.ts b/src/db/schema.ts index 12aab6d..71c42a2 100644 --- a/src/db/schema.ts +++ b/src/db/schema.ts @@ -12,6 +12,7 @@ import { uniqueIndex, uuid, } from "drizzle-orm/pg-core"; +import { sql } from "drizzle-orm"; import { approvalStatusValues } from "@/lib/approvals"; import { loopStateValues } from "../../schemas/loop-manifest"; @@ -234,6 +235,11 @@ export const loopRuns = pgTable( canceledAt: timestamp("canceled_at", { withTimezone: true }), }, (table) => ({ + activeRepositoryIssueIndex: uniqueIndex("loop_runs_active_repository_issue_idx") + .on(table.repositoryId, table.githubIssueNumber) + .where( + sql`${table.githubIssueNumber} IS NOT NULL AND (${table.status} IN ('queued', 'running', 'waiting_for_approval', 'blocked') OR (${table.status} = 'failed' AND ${table.completedAt} IS NULL))`, + ), repositoryStatusIndex: index("loop_runs_repository_status_idx").on( table.repositoryId, table.status, @@ -328,11 +334,14 @@ export const idempotencyLocks = pgTable( acquiredAt: timestamp("acquired_at", { withTimezone: true }).defaultNow().notNull(), expiresAt: timestamp("expires_at", { withTimezone: true }).notNull(), releasedAt: timestamp("released_at", { withTimezone: true }), + runId: uuid("run_id").references(() => loopRuns.id, { onDelete: "set null" }), + traceId: text("trace_id"), metadata: jsonb("metadata").$type>(), }, (table) => ({ scopeStatusIndex: index("idempotency_locks_scope_status_idx").on(table.scope, table.status), expiresAtIndex: index("idempotency_locks_expires_at_idx").on(table.expiresAt), + runStatusIndex: index("idempotency_locks_run_status_idx").on(table.runId, table.status), }), ); diff --git a/src/lib/loops/development-run-reconciliation-store.ts b/src/lib/loops/development-run-reconciliation-store.ts index 3fca3c6..1e70b68 100644 --- a/src/lib/loops/development-run-reconciliation-store.ts +++ b/src/lib/loops/development-run-reconciliation-store.ts @@ -1,7 +1,7 @@ -import { and, eq } from "drizzle-orm"; +import { and, eq, or } from "drizzle-orm"; import type { db } from "@/db/client"; -import { loopRuns, repositories, runSteps } from "@/db/schema"; +import { idempotencyLocks, loopRuns, repositories, runSteps } from "@/db/schema"; import { finalizeDevelopmentLoopRun, DevelopmentLoopTransitionError, @@ -15,6 +15,7 @@ import type { DevelopmentLoopRunStore, } from "@/lib/loops/development-run-reconciliation"; import type { LoopworksLogger } from "@/lib/observability/logger"; +import type { LoopManifest } from "../../../schemas/loop-manifest"; export type DevelopmentLoopReconciliationDatabase = Pick; @@ -34,9 +35,11 @@ function latestActivity( } export function createDevelopmentLoopRunStore(input: { + clock?: () => Date; database: DevelopmentLoopReconciliationDatabase; - executionLiveness: (run: DevelopmentLoopActiveRun) => Promise; + executionLiveness?: (run: DevelopmentLoopActiveRun) => Promise; logger?: LoopworksLogger; + manifest?: LoopManifest; metrics?: DevelopmentLoopTransitionMetrics; }): DevelopmentLoopRunStore { return { @@ -51,14 +54,34 @@ export function createDevelopmentLoopRunStore(input: { repositoryName: repositories.name, repositoryOwner: repositories.owner, runId: loopRuns.id, + status: loopRuns.status, traceId: loopRuns.traceId, }) .from(loopRuns) .innerJoin(repositories, eq(loopRuns.repositoryId, repositories.id)) - .where(and(eq(loopRuns.status, "running"), eq(loopRuns.loopKey, "development-loop"))); + .where( + and( + or(eq(loopRuns.status, "running"), eq(loopRuns.status, "queued")), + eq(loopRuns.loopKey, "development-loop"), + ), + ); - return Promise.all( + const activeRuns = await Promise.all( rows.map(async (row) => { + if (row.status === "queued") { + const [lease] = await input.database + .select({ id: idempotencyLocks.id }) + .from(idempotencyLocks) + .where( + and( + eq(idempotencyLocks.runId, row.runId), + eq(idempotencyLocks.owner, row.runId), + eq(idempotencyLocks.status, "acquired"), + ), + ) + .limit(1); + if (!lease) return null; + } const steps = await input.database .select() .from(runSteps) @@ -80,9 +103,28 @@ export function createDevelopmentLoopRunStore(input: { } satisfies DevelopmentLoopActiveRun; }), ); + return activeRuns.filter((run): run is DevelopmentLoopActiveRun => run !== null); }, - getExecutionLiveness(run) { - return input.executionLiveness(run); + async getExecutionLiveness(run) { + if (input.executionLiveness) return input.executionLiveness(run); + try { + const [lease] = await input.database + .select({ + expiresAt: idempotencyLocks.expiresAt, + owner: idempotencyLocks.owner, + status: idempotencyLocks.status, + }) + .from(idempotencyLocks) + .where(and(eq(idempotencyLocks.runId, run.runId), eq(idempotencyLocks.owner, run.runId))) + .limit(1); + if (!lease) return "unknown"; + if (lease.status !== "acquired" || lease.owner !== run.runId) return "inactive"; + return lease.expiresAt.getTime() > (input.clock?.() ?? new Date()).getTime() + ? "active" + : "inactive"; + } catch { + return "unknown"; + } }, async finalizeRun(finalizeInput) { if (finalizeInput.expected) { @@ -120,6 +162,7 @@ export function createDevelopmentLoopRunStore(input: { ? { expectedCurrentStage: finalizeInput.expected.currentStage } : {}), logger: input.logger, + manifest: input.manifest, metrics: input.metrics, occurredAt: finalizeInput.occurredAt, reason: finalizeInput.reason, diff --git a/src/lib/loops/development-run-transitions.ts b/src/lib/loops/development-run-transitions.ts index 2f66010..907dd15 100644 --- a/src/lib/loops/development-run-transitions.ts +++ b/src/lib/loops/development-run-transitions.ts @@ -38,13 +38,14 @@ import { type ValidationReviewResult, validationReviewResultSchema, } from "@agent/validation-review-agent"; -import { and, eq, isNull, notInArray, sql } from "drizzle-orm"; +import { and, eq, isNull, sql } from "drizzle-orm"; import type { db } from "@/db/client"; import { agentPlans, approvals, approvalTransitionEvents, artifacts, + idempotencyLocks, loopRuns, repositories, runSteps, @@ -55,6 +56,13 @@ import type { GitHubPullRequestWriter, } from "@/lib/github/pull-request"; import { createGitHubPullRequest, createPullRequestChangeDigest } from "@/lib/github/pull-request"; +import { + calculateDevelopmentLoopRetryDelaySeconds, + developmentLoopKey, + drainDevelopmentLoopDispatchQueue, + insertLinkedDevelopmentLoopRetryInTransaction, + type DevelopmentLoopRunDatabase, +} from "@/lib/loops/development-run"; import { defaultLoopManifest } from "@/lib/loops/manifest"; import { createPrIntentArtifactMetadata } from "@/lib/loops/pr-intent"; import { assertCanonicalLoopworksRunUrl } from "@/lib/loops/run-url"; @@ -76,7 +84,7 @@ import { validationReportArtifactMetadataSchema, validationReportV1Schema, } from "@/lib/loops/validation-report"; -import type { LoopworksLogger } from "@/lib/observability/logger"; +import { logger as defaultLogger, type LoopworksLogger } from "@/lib/observability/logger"; import { type DevelopmentLoopRunCompletedMetricInput, type DevelopmentLoopRunDurationMetricInput, @@ -96,8 +104,10 @@ import { import { markLoopworksSpanError, markLoopworksSpanOk, + startDevelopmentLoopRetrySpan, startLoopworksSpan, } from "@/lib/observability/trace-context"; +import type { LoopManifest } from "../../../schemas/loop-manifest"; export type DevelopmentLoopTransitionDatabase = Pick; @@ -156,6 +166,7 @@ export async function recordDevelopmentLoopPlanArtifact( const occurredAt = input.occurredAt ?? new Date(); return input.database.transaction(async (tx) => { + await assertDevelopmentLoopExecutionLease(tx, input.runId, "planning"); const [run] = await tx .select({ currentStage: loopRuns.currentStage, @@ -328,6 +339,7 @@ type PrStageTransitionResult = { type PrStageTransitionBaseInput = { database: DevelopmentLoopTransitionDatabase; logger?: LoopworksLogger; + manifest?: LoopManifest; metrics?: DevelopmentLoopTransitionMetrics; now?: () => Date; occurredAt?: Date; @@ -354,6 +366,36 @@ export class DevelopmentLoopTransitionError extends Error { } } +async function assertDevelopmentLoopExecutionLease( + tx: Parameters[0]>[0], + runId: string, + stage?: string, +): Promise { + const [lease] = await tx + .select({ id: idempotencyLocks.id }) + .from(idempotencyLocks) + .where( + and( + eq(idempotencyLocks.runId, runId), + eq(idempotencyLocks.owner, runId), + eq(idempotencyLocks.status, "acquired"), + ), + ) + .limit(1); + if (lease) return; + const [completedStep] = stage + ? await tx + .select({ completedAt: runSteps.completedAt, status: runSteps.status }) + .from(runSteps) + .where(and(eq(runSteps.runId, runId), eq(runSteps.stage, stage))) + .limit(1) + : []; + if (completedStep?.status === "succeeded" && completedStep.completedAt) return; + throw new DevelopmentLoopTransitionError( + `Run ${runId} does not own an acquired execution lease.`, + ); +} + export async function applyDevelopmentLoopTestWritingResult( input: ApplyDevelopmentLoopTestWritingResultInput, ): Promise { @@ -372,6 +414,7 @@ export async function applyDevelopmentLoopTestWritingResult( try { const result = await input.database.transaction(async (tx) => { + await assertDevelopmentLoopExecutionLease(tx, input.runId, "test-writing"); const [run] = await tx.select().from(loopRuns).where(eq(loopRuns.id, input.runId)).limit(1); if (!run) throw new DevelopmentLoopTransitionError(`Run ${input.runId} was not found.`); @@ -622,6 +665,7 @@ export async function applyDevelopmentLoopImplementationResult( try { const result = await input.database.transaction(async (tx) => { + await assertDevelopmentLoopExecutionLease(tx, input.runId, "development"); const [run] = await tx.select().from(loopRuns).where(eq(loopRuns.id, input.runId)).limit(1); if (!run) throw new DevelopmentLoopTransitionError(`Run ${input.runId} was not found.`); @@ -1109,6 +1153,7 @@ export async function applyDevelopmentLoopValidationReport(input: { let metricInputs: ValidationTransitionMetricInputs | undefined; const result = await input.database.transaction(async (tx) => { + await assertDevelopmentLoopExecutionLease(tx, input.runId, "validation"); const [run] = await tx .select({ currentStage: loopRuns.currentStage, @@ -1368,6 +1413,7 @@ export async function applyDevelopmentLoopPrPreparationResult(input: { }); try { const result = await input.database.transaction(async (tx) => { + await assertDevelopmentLoopExecutionLease(tx, input.runId, "pr"); const context = await loadPrPreparationContextWithDatabase( tx as unknown as PrPreparationReadDatabase, input.runId, @@ -1598,6 +1644,7 @@ export async function applyDevelopmentLoopValidationReviewResult(input: { try { const result = await input.database.transaction( async (tx) => { + await assertDevelopmentLoopExecutionLease(tx, input.runId, "code-review"); const [run] = await tx .select({ currentStage: loopRuns.currentStage, @@ -2082,6 +2129,7 @@ export async function executeDevelopmentLoopPrStage( : undefined; const prepared = await input.database.transaction(async (tx) => { + await assertDevelopmentLoopExecutionLease(tx, input.runId, "pr"); const [run] = await tx .select({ currentStage: loopRuns.currentStage, @@ -2498,6 +2546,7 @@ export async function executeDevelopmentLoopPrStage( completedAt: failedAt, metadata: { ...(prepared.prStep.metadata ?? {}), + failure: { code: "github_pr_creation_failed", retryable: true }, failureCode: "github_pr_creation_failed", retryable: true, }, @@ -2537,6 +2586,16 @@ export async function executeDevelopmentLoopPrStage( }, "development_loop_pr_transition_failed", ); + await scheduleDevelopmentLoopStageRetry({ + database: input.database, + failure: { code: "github_pr_creation_failed", retryable: true }, + logger: input.logger, + manifest: input.manifest ?? defaultLoopManifest, + occurredAt: failedAt, + runId: input.runId, + stage: "pr", + traceId: prepared.prStep.traceId ?? prepared.repository.traceId ?? undefined, + }); throw new DevelopmentLoopTransitionError( "PR creation failed; the step is ready for inspection and retry.", ); @@ -2594,6 +2653,7 @@ export async function finalizeDevelopmentLoopRun(input: { database: DevelopmentLoopTransitionDatabase; expectedCurrentStage?: string; logger?: LoopworksLogger; + manifest?: LoopManifest; metrics?: DevelopmentLoopTransitionMetrics; occurredAt?: Date; reason: DevelopmentLoopTerminalReason; @@ -2609,6 +2669,9 @@ export async function finalizeDevelopmentLoopRun(input: { const occurredAt = input.occurredAt ?? new Date(); let runCompletedMetric: DevelopmentLoopRunCompletedMetricInput | undefined; let runDurationMetric: DevelopmentLoopRunDurationMetricInput | undefined; + let linkedRetry: { eligibleAt: Date; emitObservability: () => void; runId: string } | undefined; + let drainRepositoryFullName: string | undefined; + let repairedTerminalLease = false; const status = terminalStatusForReason(input.reason); const result = await input.database.transaction(async (tx) => { @@ -2635,11 +2698,24 @@ export async function finalizeDevelopmentLoopRun(input: { if (!run) { throw new DevelopmentLoopTransitionError(`Run ${input.runId} was not found.`); } + drainRepositoryFullName = run.repository; if ( run.completedAt && (run.status === "succeeded" || run.status === "failed" || run.status === "canceled") ) { + const repairedLeases = await tx + .update(idempotencyLocks) + .set({ releasedAt: run.completedAt, status: "released" }) + .where( + and( + eq(idempotencyLocks.runId, input.runId), + eq(idempotencyLocks.owner, input.runId), + eq(idempotencyLocks.status, "acquired"), + ), + ) + .returning({ id: idempotencyLocks.id }); + repairedTerminalLease = repairedLeases.length > 0; return { durationSeconds: durationSecondsBetween(run.startedAt ?? run.queuedAt, run.completedAt), idempotent: true, @@ -2658,7 +2734,7 @@ export async function finalizeDevelopmentLoopRun(input: { .limit(1); const updatePredicates = [ eq(loopRuns.id, input.runId), - notInArray(loopRuns.status, ["succeeded", "failed", "canceled"]), + isNull(loopRuns.completedAt), ...(input.expectedCurrentStage ? [eq(loopRuns.currentStage, input.expectedCurrentStage)] : []), @@ -2696,6 +2772,18 @@ export async function finalizeDevelopmentLoopRun(input: { `Run ${input.runId} could not be finalized because its state changed.`, ); } + const repairedLeases = await tx + .update(idempotencyLocks) + .set({ releasedAt: terminalRun.completedAt, status: "released" }) + .where( + and( + eq(idempotencyLocks.runId, input.runId), + eq(idempotencyLocks.owner, input.runId), + eq(idempotencyLocks.status, "acquired"), + ), + ) + .returning({ id: idempotencyLocks.id }); + repairedTerminalLease = repairedLeases.length > 0; return { durationSeconds: durationSecondsBetween( terminalRun.startedAt ?? terminalRun.queuedAt, @@ -2711,6 +2799,33 @@ export async function finalizeDevelopmentLoopRun(input: { }; } + await tx + .update(idempotencyLocks) + .set({ + releasedAt: occurredAt, + status: "released", + }) + .where( + and( + eq(idempotencyLocks.runId, input.runId), + eq(idempotencyLocks.owner, input.runId), + eq(idempotencyLocks.status, "acquired"), + ), + ); + + if (input.reason === "stalled" || input.reason === "timed_out") { + linkedRetry = await insertLinkedDevelopmentLoopRetryInTransaction({ + manifest: input.manifest ?? defaultLoopManifest, + occurredAt, + reason: input.reason, + repository: { fullName: run.repository, id: run.repositoryId }, + sourceMetadata: run.metadata, + sourceRunId: input.runId, + traceId: run.traceId ?? undefined, + writer: tx, + }); + } + await recordDevelopmentLoopRunCompletedObservability({ durationSeconds, loopKey: run.loopKey, @@ -2756,6 +2871,22 @@ export async function finalizeDevelopmentLoopRun(input: { runDurationMetric, ); } + try { + linkedRetry?.emitObservability(); + } catch { + // Terminal evidence and linked retry creation already committed. + } + if (linkedRetry) { + (input.logger ?? defaultLogger).info( + { + eligibleAt: linkedRetry.eligibleAt.toISOString(), + loopKey: developmentLoopKey, + reason: input.reason, + runId: linkedRetry.runId, + }, + "development_loop_retry_scheduled", + ); + } input.logger?.info( { @@ -2769,12 +2900,33 @@ export async function finalizeDevelopmentLoopRun(input: { developmentLoopRunCompletedEventType, ); + if (!("idempotent" in result && result.idempotent) || repairedTerminalLease) { + try { + await drainDevelopmentLoopDispatchQueue({ + clock: () => occurredAt, + database: input.database as DevelopmentLoopRunDatabase, + manifest: input.manifest ?? defaultLoopManifest, + repositoryFullName: drainRepositoryFullName, + }); + } catch (error) { + (input.logger ?? defaultLogger).error( + { + error: error instanceof Error ? error.message : "unknown", + loopKey: developmentLoopKey, + runId: input.runId, + }, + "development_loop_dispatch_drain_failed", + ); + } + } + return result; } export async function completeDevelopmentLoopRun(input: { database: DevelopmentLoopTransitionDatabase; logger?: LoopworksLogger; + manifest?: LoopManifest; metrics?: DevelopmentLoopTransitionMetrics; occurredAt?: Date; reason?: DevelopmentLoopTerminalReason; @@ -2795,6 +2947,7 @@ export async function completeDevelopmentLoopRun(input: { return finalizeDevelopmentLoopRun({ database: input.database, logger: input.logger, + manifest: input.manifest, metrics: input.metrics, occurredAt: input.occurredAt, reason, @@ -2802,6 +2955,173 @@ export async function completeDevelopmentLoopRun(input: { }); } +export async function scheduleDevelopmentLoopStageRetry(input: { + database: DevelopmentLoopTransitionDatabase; + failure: { code: string; retryable: boolean }; + logger?: LoopworksLogger; + manifest: LoopManifest; + occurredAt?: Date; + runId: string; + stage: string; + traceId?: string; +}): Promise<{ + attempt: number; + eligibleAt?: Date; + runId: string; + stage: string; + status: "exhausted" | "ineligible" | "scheduled"; +}> { + const occurredAt = input.occurredAt ?? new Date(); + const loopManifest = input.manifest.loops.find(({ key }) => key === developmentLoopKey); + if (!loopManifest) { + throw new DevelopmentLoopTransitionError(`Manifest does not define ${developmentLoopKey}.`); + } + const retrySpan = startDevelopmentLoopRetrySpan({ traceId: input.traceId }); + const reason = normalizeReasonCode(input.failure.code) ?? "unspecified"; + + const decision = await input.database + .transaction(async (tx) => { + const [run] = await tx + .select({ + completedAt: loopRuns.completedAt, + metadata: loopRuns.metadata, + status: loopRuns.status, + }) + .from(loopRuns) + .where(eq(loopRuns.id, input.runId)) + .for("update") + .limit(1); + const [step] = await tx + .select() + .from(runSteps) + .where(and(eq(runSteps.runId, input.runId), eq(runSteps.stage, input.stage))) + .limit(1); + if (!run || !step) { + throw new DevelopmentLoopTransitionError( + `Run ${input.runId} does not have a ${input.stage} step.`, + ); + } + const runMetadata = (run.metadata ?? {}) as Record; + const completedAttempt = + typeof runMetadata.dispatchAttempt === "number" ? runMetadata.dispatchAttempt : 1; + const existingSchedule = + runMetadata.scheduledRetry && typeof runMetadata.scheduledRetry === "object" + ? (runMetadata.scheduledRetry as Record) + : undefined; + if ( + run.status === "queued" && + existingSchedule?.stage === input.stage && + existingSchedule.completedAttempt === completedAttempt && + existingSchedule.stepAttempt === step.attempt && + typeof existingSchedule.eligibleAt === "string" + ) { + return { + attempt: completedAttempt, + eligibleAt: new Date(existingSchedule.eligibleAt), + status: "scheduled" as const, + }; + } + const storedFailure = + step.metadata && typeof step.metadata === "object" + ? (step.metadata as Record).failure + : undefined; + const hasBoundedRetryMarker = + storedFailure !== null && + typeof storedFailure === "object" && + (storedFailure as Record).retryable === true && + typeof (storedFailure as Record).code === "string"; + const authorized = + step.status === "failed" && + run.completedAt === null && + run.status === "failed" && + input.failure.retryable && + hasBoundedRetryMarker && + loopManifest.retryPolicy.retryableStatuses.includes("failed"); + if (!authorized || completedAttempt >= loopManifest.retryPolicy.maxAttempts) { + return { + attempt: completedAttempt, + status: (authorized ? "exhausted" : "ineligible") as "exhausted" | "ineligible", + }; + } + + const delaySeconds = calculateDevelopmentLoopRetryDelaySeconds({ + backoff: loopManifest.retryPolicy.backoff, + completedAttempt, + }); + const eligibleAt = new Date(occurredAt.getTime() + delaySeconds * 1_000); + await tx + .update(loopRuns) + .set({ + currentStage: input.stage, + metadata: { + ...runMetadata, + scheduledRetry: { + completedAttempt, + eligibleAt: eligibleAt.toISOString(), + reason, + stage: input.stage, + stepAttempt: step.attempt, + }, + }, + queuedAt: eligibleAt, + status: "queued", + }) + .where( + and( + eq(loopRuns.id, input.runId), + eq(loopRuns.status, "failed"), + isNull(loopRuns.completedAt), + ), + ); + await tx + .update(idempotencyLocks) + .set({ releasedAt: occurredAt, status: "released" }) + .where( + and( + eq(idempotencyLocks.runId, input.runId), + eq(idempotencyLocks.owner, input.runId), + eq(idempotencyLocks.status, "acquired"), + ), + ); + return { attempt: completedAttempt, eligibleAt, status: "scheduled" as const }; + }) + .catch((error) => { + markLoopworksSpanError(retrySpan.span, error); + retrySpan.span.end(); + throw error; + }); + + try { + if (decision.status === "exhausted") { + await finalizeDevelopmentLoopRun({ + database: input.database, + logger: input.logger, + manifest: input.manifest, + occurredAt, + reason: "failed", + runId: input.runId, + }); + (input.logger ?? defaultLogger).info( + { attempt: decision.attempt, loopKey: developmentLoopKey, reason, runId: input.runId }, + "development_loop_retry_exhausted", + ); + } else if (decision.status === "scheduled") { + (input.logger ?? defaultLogger).info( + { attempt: decision.attempt, loopKey: developmentLoopKey, reason, runId: input.runId }, + "development_loop_retry_scheduled", + ); + } + retrySpan.setOutcome(decision.status); + markLoopworksSpanOk(retrySpan.span); + retrySpan.span.end(); + return { ...decision, runId: input.runId, stage: input.stage }; + } catch (error) { + markLoopworksSpanError(retrySpan.span, error); + retrySpan.span.end(); + throw error; + } +} + export async function retryDevelopmentLoopStep(input: { database: DevelopmentLoopTransitionDatabase; logger?: LoopworksLogger; @@ -2825,6 +3145,7 @@ export async function retryDevelopmentLoopStep(input: { const result = await input.database.transaction(async (tx) => { const [run] = await tx .select({ + completedAt: loopRuns.completedAt, id: loopRuns.id, loopKey: loopRuns.loopKey, metadata: loopRuns.metadata, @@ -2837,6 +3158,9 @@ export async function retryDevelopmentLoopStep(input: { if (!run) { throw new DevelopmentLoopTransitionError(`Run ${input.runId} was not found.`); } + if (run.completedAt) { + throw new DevelopmentLoopTransitionError(`Cannot retry terminal run ${input.runId}.`); + } const [step] = await tx .select() @@ -2862,6 +3186,21 @@ export async function retryDevelopmentLoopStep(input: { }; } + const runMetadata = (run.metadata ?? {}) as Record; + if (runMetadata.scheduledRetry && typeof runMetadata.scheduledRetry === "object") { + const traceId = step.traceId ?? run.traceId ?? undefined; + return { + attempt: step.attempt, + idempotent: true, + runId: input.runId, + stage: input.stage, + stepId: step.id, + ...(traceId ? { traceId } : {}), + }; + } + + await assertDevelopmentLoopExecutionLease(tx, input.runId); + const attempt = step.attempt + 1; await tx .update(runSteps) diff --git a/src/lib/loops/development-run.ts b/src/lib/loops/development-run.ts index 08fdb41..3e17b5e 100644 --- a/src/lib/loops/development-run.ts +++ b/src/lib/loops/development-run.ts @@ -6,23 +6,35 @@ import { createTestPlanArtifactContractMetadata, } from "@agent/test-writing-agent"; import { createValidationReviewArtifactContractMetadata } from "@agent/validation-review-agent"; -import { and, eq, sql } from "drizzle-orm"; +import { and, asc, eq, isNotNull, isNull, lte, sql } from "drizzle-orm"; import type { db } from "@/db/client"; import { agentPlans, approvals, artifacts, + idempotencyLocks, loopRuns, observabilityEvents, repositories, runSteps, } from "@/db/schema"; +import { defaultLoopManifest } from "@/lib/loops/manifest"; import { createPrIntentArtifactContractMetadata } from "@/lib/loops/pr-intent"; import { createScreenshotEvidenceArtifactContractMetadata } from "@/lib/loops/screenshot-evidence"; import { createValidationReportArtifactContractMetadata } from "@/lib/loops/validation-report"; import { recordDevelopmentLoopRunCreatedObservability } from "@/lib/observability/metrics"; -import { getActiveTraceId, isValidW3cTraceId } from "@/lib/observability/trace-context"; +import { recordLockContentionMetric } from "@/lib/observability/metrics"; +import { recordDevelopmentLoopStepRetryMetric } from "@/lib/observability/metrics"; +import { logger } from "@/lib/observability/logger"; +import { + getActiveTraceId, + isValidW3cTraceId, + markLoopworksSpanError, + markLoopworksSpanOk, + startDevelopmentLoopDispatchSpan, +} from "@/lib/observability/trace-context"; import type { ArtifactRecord, TimelineEvent, TimelineKind } from "@/lib/types"; +import type { LoopManifest } from "../../../schemas/loop-manifest"; export const developmentLoopKey = "development-loop"; export const developmentLoopNoopEventType = "development_loop_noop"; @@ -167,14 +179,12 @@ export type DevelopmentLoopTrigger = { }; export type DevelopmentLoopRunDatabase = Pick; +export type DevelopmentLoopRunTransactionWriter = Parameters< + Parameters<(typeof db)["transaction"]>[0] +>[0]; export type DevelopmentLoopRunMetadata = - | { - artifactCount: number; - mode: "created"; - runId: string; - stageCount: number; - } + | DevelopmentLoopDispatchOutcome | { artifactCount: number; mode: "simulated"; @@ -188,9 +198,31 @@ export type DevelopmentLoopNoopMetadata = { type DevelopmentLoopRunTransactionResult = { emitObservability?: () => void; - metadata: DevelopmentLoopRunMetadata; + metadata: DevelopmentLoopDispatchOutcome; }; +export type DevelopmentLoopDispatchOutcome = + | { + artifactCount: number; + mode: "dispatched"; + runId: string; + stageCount: number; + } + | { + artifactCount: number; + mode: "deferred"; + reason: "max_in_flight" | "not_due"; + runId: string; + stageCount: number; + } + | { + artifactCount: number; + mode: "lease_contention"; + reason: "delivery_replay" | "issue_active"; + runId: string; + stageCount: number; + }; + type DevelopmentLoopStageInstance = { actorId: string; actorType: string; @@ -348,22 +380,78 @@ export function simulateDevelopmentLoopRun(input: { }; } -export async function createDevelopmentLoopRun(input: { +function getDevelopmentLoopManifest(manifest: LoopManifest) { + const loop = manifest.loops.find(({ key }) => key === developmentLoopKey); + if (!loop) { + throw new Error(`Manifest does not define ${developmentLoopKey}.`); + } + return loop; +} + +export function resolveDevelopmentLoopConcurrencyGroup(input: { + manifest: LoopManifest; + repositoryFullName: string; +}): string { + const template = getDevelopmentLoopManifest(input.manifest).concurrency.group; + const resolved = template.replaceAll("{repo}", input.repositoryFullName); + const unresolved = resolved.match(/\{[^}]+\}/)?.[0]; + if (unresolved) { + throw new Error(`Unsupported concurrency group placeholder: ${unresolved}`); + } + return resolved; +} + +export function calculateDevelopmentLoopRetryDelaySeconds(input: { + backoff: { + initialSeconds: number; + maxSeconds: number; + strategy: "exponential" | "fixed"; + }; + completedAttempt: number; +}): number { + const delay = + input.backoff.strategy === "fixed" + ? input.backoff.initialSeconds + : input.backoff.initialSeconds * 2 ** Math.max(0, input.completedAttempt - 1); + return Math.min(delay, input.backoff.maxSeconds); +} + +function dispatchGroupGuardKey(group: string): string { + return `loop:dispatch:group-guard:${group}`; +} + +export async function dispatchDevelopmentLoopRun(input: { + clock: () => Date; database: DevelopmentLoopRunDatabase; - now?: () => Date; + manifest: LoopManifest; + observability?: { + recordLockContentionMetric?: typeof recordLockContentionMetric; + }; + retry?: { + attempt: number; + eligibleAt: Date; + retryOfRunId: string; + rootRunId: string; + }; traceId?: string; trigger: DevelopmentLoopTrigger; -}): Promise { - const createdAt = input.now?.() ?? new Date(); +}): Promise { + const createdAt = input.clock(); + const loopManifest = getDevelopmentLoopManifest(input.manifest); + const resolvedGroup = resolveDevelopmentLoopConcurrencyGroup({ + manifest: input.manifest, + repositoryFullName: input.trigger.repositoryFullName, + }); const traceId = input.traceId === undefined ? getActiveTraceId() : isValidW3cTraceId(input.traceId) ? input.traceId : undefined; + const dispatchSpan = startDevelopmentLoopDispatchSpan({ traceId }); - const result: DevelopmentLoopRunTransactionResult = await input.database.transaction( - async (tx) => { + const result: DevelopmentLoopRunTransactionResult = await input.database + .transaction(async (tx) => { const existingRun = input.trigger.deliveryId ? await tx .select({ id: loopRuns.id }) @@ -390,7 +478,8 @@ export async function createDevelopmentLoopRun(input: { return { metadata: { artifactCount: existingArtifacts.length, - mode: "created", + mode: "lease_contention" as const, + reason: "delivery_replay" as const, runId: existingRun[0].id, stageCount: existingSteps.length, }, @@ -409,10 +498,102 @@ export async function createDevelopmentLoopRun(input: { ); } + const guardKey = dispatchGroupGuardKey(resolvedGroup); + await tx + .insert(idempotencyLocks) + .values({ + acquiredAt: createdAt, + expiresAt: createdAt, + key: guardKey, + owner: "loopworks:dispatch-admission", + releasedAt: createdAt, + scope: "loop:dispatch:group-guard", + status: "released", + }) + .onConflictDoNothing({ target: idempotencyLocks.key }); + + const [guard] = await tx + .select({ id: idempotencyLocks.id }) + .from(idempotencyLocks) + .where(eq(idempotencyLocks.key, guardKey)) + .for("update"); + if (!guard) { + throw new Error(`Dispatch group guard ${guardKey} could not be locked.`); + } + + const issueGuardKey = `loop:dispatch:issue-guard:${repository.id}:${input.trigger.issueNumber}`; + await tx + .insert(idempotencyLocks) + .values({ + acquiredAt: createdAt, + expiresAt: createdAt, + key: issueGuardKey, + owner: "loopworks:dispatch-admission", + releasedAt: createdAt, + scope: "loop:dispatch:issue-guard", + status: "released", + }) + .onConflictDoNothing({ target: idempotencyLocks.key }); + const [issueGuard] = await tx + .select({ id: idempotencyLocks.id }) + .from(idempotencyLocks) + .where(eq(idempotencyLocks.key, issueGuardKey)) + .for("update"); + if (!issueGuard) { + throw new Error(`Dispatch issue guard ${issueGuardKey} could not be locked.`); + } + + const activeRun = await tx + .select({ id: loopRuns.id }) + .from(loopRuns) + .where( + and( + eq(loopRuns.repositoryId, repository.id), + eq(loopRuns.githubIssueNumber, input.trigger.issueNumber), + sql`(${loopRuns.status} in ('queued', 'running', 'waiting_for_approval', 'blocked') or (${loopRuns.status} = 'failed' and ${loopRuns.completedAt} is null))`, + ), + ) + .limit(1); + + if (activeRun[0]) { + const existingArtifacts = await tx + .select({ id: artifacts.id }) + .from(artifacts) + .where(eq(artifacts.runId, activeRun[0].id)); + const existingSteps = await tx + .select({ id: runSteps.id }) + .from(runSteps) + .where(eq(runSteps.runId, activeRun[0].id)); + return { + metadata: { + artifactCount: existingArtifacts.length, + mode: "lease_contention" as const, + reason: "issue_active" as const, + runId: activeRun[0].id, + stageCount: existingSteps.length, + }, + }; + } + + const [capacity] = await tx + .select({ count: sql`count(*)` }) + .from(idempotencyLocks) + .where( + and( + eq(idempotencyLocks.scope, resolvedGroup), + eq(idempotencyLocks.status, "acquired"), + isNotNull(idempotencyLocks.runId), + ), + ); + const isDue = !input.retry || input.retry.eligibleAt.getTime() <= createdAt.getTime(); + const shouldAcquireLease = + isDue && Number(capacity?.count ?? 0) < loopManifest.concurrency.maxInFlight; + const runId = randomUUID(); + const queuedAt = input.retry?.eligibleAt ?? createdAt; const skeleton = createDevelopmentLoopRunSkeleton({ mode: "created", - now: createdAt, + now: queuedAt, runId, trigger: input.trigger, }); @@ -424,14 +605,27 @@ export async function createDevelopmentLoopRun(input: { githubIssueUrl: getIssueUrl(input.trigger), loopKey: skeleton.loopKey, metadata: { + dispatchAttempt: input.retry?.attempt ?? 1, deliveryId: input.trigger.deliveryId, labels: input.trigger.labels ?? [], milestone: input.trigger.milestone ?? null, issueTitle: input.trigger.title ?? `Issue #${input.trigger.issueNumber}`, + retryOfRunId: input.retry?.retryOfRunId ?? null, + rootRunId: input.retry?.rootRunId ?? runId, source: "github_issue", stageCount: skeleton.stages.length, + triggerSnapshot: { + body: input.trigger.body, + issueNumber: input.trigger.issueNumber, + issueUrl: getIssueUrl(input.trigger), + labels: input.trigger.labels ?? [], + milestone: input.trigger.milestone ?? null, + repositoryFullName: input.trigger.repositoryFullName, + repositoryRevision: input.trigger.repositoryRevision, + title: input.trigger.title ?? null, + }, }, - queuedAt: createdAt, + queuedAt, repositoryId: repository.id, status: "queued", traceId, @@ -547,21 +741,587 @@ export async function createDevelopmentLoopRun(input: { writer: tx, }); - return { - emitObservability, - metadata: { - artifactCount: skeleton.artifacts.length, - mode: "created", + if (shouldAcquireLease) { + await tx.insert(idempotencyLocks).values({ + acquiredAt: createdAt, + expiresAt: minutesAfter(createdAt, loopManifest.budgets.maxRunMinutes), + key: `loop:dispatch:lease:${runId}`, + metadata: { + loopKey: developmentLoopKey, + }, + owner: runId, runId, - stageCount: skeleton.stages.length, - }, - }; + scope: resolvedGroup, + status: "acquired", + traceId, + }); + } + + const metadata: DevelopmentLoopDispatchOutcome = shouldAcquireLease + ? { + artifactCount: skeleton.artifacts.length, + mode: "dispatched", + runId, + stageCount: skeleton.stages.length, + } + : { + artifactCount: skeleton.artifacts.length, + mode: "deferred", + reason: isDue ? "max_in_flight" : "not_due", + runId, + stageCount: skeleton.stages.length, + }; + return { emitObservability, metadata }; + }) + .catch((error) => { + markLoopworksSpanError(dispatchSpan.span, error); + dispatchSpan.span.end(); + throw error; + }); + + try { + result.emitObservability?.(); + } catch (error) { + logger.error( + { error: error instanceof Error ? error.message : "unknown", loopKey: developmentLoopKey }, + "development_loop_dispatch_observability_failed", + ); + } + + if (result.metadata.mode === "lease_contention" && result.metadata.reason === "issue_active") { + try { + (input.observability?.recordLockContentionMetric ?? recordLockContentionMetric)({ + scope: "loop:dispatch", + }); + } catch { + // Durable admission must not depend on the telemetry sink. + } + logger.info( + { + issueNumber: input.trigger.issueNumber, + loopKey: developmentLoopKey, + runId: result.metadata.runId, + }, + "development_loop_dispatch_lease_contended", + ); + } else if (result.metadata.mode === "deferred") { + logger.info( + { + issueNumber: input.trigger.issueNumber, + loopKey: developmentLoopKey, + runId: result.metadata.runId, + }, + "development_loop_dispatch_deferred", + ); + } + + dispatchSpan.setOutcome(result.metadata.mode); + markLoopworksSpanOk(dispatchSpan.span); + dispatchSpan.span.end(); + return result.metadata; +} + +export async function drainDevelopmentLoopDispatchQueue(input: { + clock: () => Date; + database: DevelopmentLoopRunDatabase; + manifest: LoopManifest; + repositoryFullName?: string; +}): Promise> { + const now = input.clock(); + const loopManifest = getDevelopmentLoopManifest(input.manifest); + + const promoted = await input.database.transaction(async (tx) => { + const candidates = await tx + .select({ + issueNumber: loopRuns.githubIssueNumber, + metadata: loopRuns.metadata, + repositoryFullName: repositories.fullName, + repositoryId: loopRuns.repositoryId, + runId: loopRuns.id, + traceId: loopRuns.traceId, + }) + .from(loopRuns) + .innerJoin(repositories, eq(loopRuns.repositoryId, repositories.id)) + .where( + and( + eq(loopRuns.loopKey, developmentLoopKey), + eq(loopRuns.status, "queued"), + isNull(loopRuns.completedAt), + lte(loopRuns.queuedAt, now), + ...(input.repositoryFullName + ? [eq(repositories.fullName, input.repositoryFullName)] + : []), + ), + ) + .orderBy(asc(loopRuns.queuedAt), asc(loopRuns.githubIssueNumber)); + + const outcomes: Array<{ + mode: "dispatched"; + retryMetric?: { reason: string; stage: string }; + runId: string; + traceId?: string; + }> = []; + for (const candidate of candidates) { + if (candidate.issueNumber === null) continue; + const resolvedGroup = resolveDevelopmentLoopConcurrencyGroup({ + manifest: input.manifest, + repositoryFullName: candidate.repositoryFullName, + }); + const guardKey = dispatchGroupGuardKey(resolvedGroup); + await tx + .insert(idempotencyLocks) + .values({ + acquiredAt: now, + expiresAt: now, + key: guardKey, + owner: "loopworks:dispatch-admission", + releasedAt: now, + scope: "loop:dispatch:group-guard", + status: "released", + }) + .onConflictDoNothing({ target: idempotencyLocks.key }); + const [guard] = await tx + .select({ id: idempotencyLocks.id }) + .from(idempotencyLocks) + .where(eq(idempotencyLocks.key, guardKey)) + .for("update"); + if (!guard) { + throw new Error(`Dispatch group guard ${guardKey} could not be locked.`); + } + + const [existingLease] = await tx + .select({ id: idempotencyLocks.id }) + .from(idempotencyLocks) + .where( + and(eq(idempotencyLocks.runId, candidate.runId), eq(idempotencyLocks.status, "acquired")), + ) + .limit(1); + if (existingLease) continue; + + const [capacity] = await tx + .select({ count: sql`count(*)` }) + .from(idempotencyLocks) + .where( + and( + eq(idempotencyLocks.scope, resolvedGroup), + eq(idempotencyLocks.status, "acquired"), + isNotNull(idempotencyLocks.runId), + ), + ); + if (Number(capacity?.count ?? 0) >= loopManifest.concurrency.maxInFlight) continue; + + const [reacquired] = await tx + .update(idempotencyLocks) + .set({ + acquiredAt: now, + expiresAt: minutesAfter(now, loopManifest.budgets.maxRunMinutes), + releasedAt: null, + scope: resolvedGroup, + status: "acquired", + traceId: candidate.traceId, + }) + .where( + and( + eq(idempotencyLocks.runId, candidate.runId), + eq(idempotencyLocks.owner, candidate.runId), + eq(idempotencyLocks.status, "released"), + ), + ) + .returning({ id: idempotencyLocks.id }); + if (!reacquired) { + await tx.insert(idempotencyLocks).values({ + acquiredAt: now, + expiresAt: minutesAfter(now, loopManifest.budgets.maxRunMinutes), + key: `loop:dispatch:lease:${candidate.runId}`, + metadata: { loopKey: developmentLoopKey }, + owner: candidate.runId, + runId: candidate.runId, + scope: resolvedGroup, + status: "acquired", + traceId: candidate.traceId, + }); + } + const runMetadata = (candidate.metadata ?? {}) as Record; + const scheduledRetry = + runMetadata.scheduledRetry && typeof runMetadata.scheduledRetry === "object" + ? (runMetadata.scheduledRetry as Record) + : undefined; + let retryMetric: { reason: string; stage: string } | undefined; + if ( + scheduledRetry && + typeof scheduledRetry.stage === "string" && + typeof scheduledRetry.completedAttempt === "number" && + typeof scheduledRetry.stepAttempt === "number" + ) { + const [failedStep] = await tx + .select() + .from(runSteps) + .where( + and( + eq(runSteps.runId, candidate.runId), + eq(runSteps.stage, scheduledRetry.stage), + eq(runSteps.status, "failed"), + eq(runSteps.attempt, scheduledRetry.stepAttempt), + ), + ) + .limit(1); + if (failedStep) { + const failedStepMetadata = (failedStep.metadata ?? {}) as Record; + const attemptHistory = Array.isArray(failedStepMetadata.attemptHistory) + ? failedStepMetadata.attemptHistory + : []; + await tx + .update(runSteps) + .set({ + attempt: failedStep.attempt + 1, + completedAt: null, + metadata: { + ...failedStepMetadata, + attemptHistory: [ + ...attemptHistory, + { + attempt: failedStep.attempt, + completedAt: failedStep.completedAt?.toISOString() ?? null, + failure: failedStepMetadata.failure ?? null, + status: "failed", + }, + ], + promotedAt: now.toISOString(), + }, + queuedAt: now, + startedAt: null, + status: "queued", + traceId: failedStep.traceId ?? candidate.traceId, + }) + .where(eq(runSteps.id, failedStep.id)); + await tx + .update(loopRuns) + .set({ + metadata: { + ...runMetadata, + dispatchAttempt: Math.max( + typeof runMetadata.dispatchAttempt === "number" ? runMetadata.dispatchAttempt : 1, + scheduledRetry.completedAttempt + 1, + ), + scheduledRetry: { + ...scheduledRetry, + promotedAt: now.toISOString(), + }, + }, + }) + .where(eq(loopRuns.id, candidate.runId)); + retryMetric = { + reason: + typeof scheduledRetry.reason === "string" ? scheduledRetry.reason : "unspecified", + stage: scheduledRetry.stage, + }; + } + } + outcomes.push({ + mode: "dispatched", + ...(retryMetric ? { retryMetric } : {}), + runId: candidate.runId, + ...(candidate.traceId ? { traceId: candidate.traceId } : {}), + }); + } + return outcomes; + }); + + for (const outcome of promoted) { + if (outcome.retryMetric) { + try { + recordDevelopmentLoopStepRetryMetric({ + loopKey: developmentLoopKey, + ...outcome.retryMetric, + }); + } catch { + // Durable retry promotion must not depend on the telemetry sink. + } + } + logger.info( + { loopKey: developmentLoopKey, runId: outcome.runId }, + "development_loop_dispatch_promoted", + ); + } + return promoted.map(({ retryMetric: _retryMetric, ...outcome }) => outcome); +} + +type StoredTriggerSnapshot = { + body?: string; + issueNumber: number; + issueUrl?: string; + labels?: readonly string[]; + milestone?: string | null; + repositoryFullName: string; + repositoryRevision?: { commitSha: string; ref: string }; + title?: string | null; +}; + +function storedTriggerSnapshot(value: unknown): StoredTriggerSnapshot | undefined { + if (!value || typeof value !== "object") return undefined; + const snapshot = value as Partial; + if (typeof snapshot.issueNumber !== "number" || typeof snapshot.repositoryFullName !== "string") { + return undefined; + } + return snapshot as StoredTriggerSnapshot; +} + +export async function insertLinkedDevelopmentLoopRetryInTransaction(input: { + manifest: LoopManifest; + occurredAt: Date; + reason: "stalled" | "timed_out"; + repository: { fullName: string; id: string }; + sourceMetadata: unknown; + sourceRunId: string; + traceId?: string; + writer: DevelopmentLoopRunTransactionWriter; +}): Promise<{ eligibleAt: Date; emitObservability: () => void; runId: string } | undefined> { + const loopManifest = getDevelopmentLoopManifest(input.manifest); + const metadata = (input.sourceMetadata ?? {}) as Record; + const completedAttempt = + typeof metadata.dispatchAttempt === "number" ? metadata.dispatchAttempt : 1; + if (completedAttempt >= loopManifest.retryPolicy.maxAttempts) return undefined; + const snapshot = storedTriggerSnapshot(metadata.triggerSnapshot); + if (!snapshot) return undefined; + + const eligibleAt = new Date( + input.occurredAt.getTime() + + calculateDevelopmentLoopRetryDelaySeconds({ + backoff: loopManifest.retryPolicy.backoff, + completedAttempt, + }) * + 1_000, + ); + const runId = randomUUID(); + const trigger: DevelopmentLoopTrigger = { + ...snapshot, + deliveryId: `loopworks-retry:${input.sourceRunId}:${completedAttempt + 1}`, + }; + const skeleton = createDevelopmentLoopRunSkeleton({ + mode: "created", + now: input.occurredAt, + runId, + trigger, + }); + await input.writer.insert(loopRuns).values({ + id: runId, + currentStage: developmentLoopStages[0].key, + githubIssueNumber: trigger.issueNumber, + githubIssueUrl: getIssueUrl(trigger), + loopKey: developmentLoopKey, + metadata: { + dispatchAttempt: completedAttempt + 1, + deliveryId: trigger.deliveryId, + issueTitle: trigger.title ?? `Issue #${trigger.issueNumber}`, + labels: trigger.labels ?? [], + milestone: trigger.milestone ?? null, + retryOfRunId: input.sourceRunId, + rootRunId: typeof metadata.rootRunId === "string" ? metadata.rootRunId : input.sourceRunId, + source: "github_issue", + stageCount: skeleton.stages.length, + triggerSnapshot: snapshot, }, + queuedAt: eligibleAt, + repositoryId: input.repository.id, + status: "queued", + traceId: input.traceId, + }); + + const stepIdsByStage = new Map(); + for (const stage of skeleton.stages) { + const stepId = randomUUID(); + stepIdsByStage.set(stage.key, stepId); + await input.writer.insert(runSteps).values({ + id: stepId, + actorId: stage.actorId, + actorType: stage.actorType, + metadata: { + artifactLabels: stage.artifacts.map((artifact) => artifact.label), + requiredArtifacts: stage.artifacts.every((artifact) => artifact.required), + }, + queuedAt: stage.queuedAt, + runId, + stage: stage.key, + status: stage.status, + summary: stage.summary, + traceId: input.traceId, + validationCommand: stage.validationCommand, + validationStatus: stage.validationStatus, + }); + } + await input.writer.insert(artifacts).values( + skeleton.artifacts.map((artifact) => ({ + id: randomUUID(), + metadata: { + required: artifact.required, + stage: artifact.stageKey, + ...(artifact.type === "validation_report" && artifact.stageKey === "validation" + ? createValidationReportArtifactContractMetadata({ detail: artifact.detail }) + : {}), + ...(artifact.type === "validation_report" && artifact.stageKey === "test-writing" + ? createRedTestEvidenceArtifactContractMetadata() + : {}), + ...(artifact.type === "test_plan" ? createTestPlanArtifactContractMetadata() : {}), + ...(artifact.type === "patch" ? createImplementationArtifactContractMetadata() : {}), + ...(artifact.type === "screenshot" + ? createScreenshotEvidenceArtifactContractMetadata() + : {}), + ...(artifact.type === "log_summary" && artifact.stageKey === "code-review" + ? createValidationReviewArtifactContractMetadata() + : {}), + ...(artifact.type === "pr_intent" ? createPrIntentArtifactContractMetadata() : {}), + }, + runId, + stepId: stepIdsByStage.get(artifact.stageKey), + title: artifact.label, + type: artifact.type, + uri: artifact.uri, + })), ); - result.emitObservability?.(); + const planId = randomUUID(); + const plan = createPlanningAgentSeedPlan({ + body: trigger.body ?? "", + issueNumber: trigger.issueNumber, + issueUrl: getIssueUrl(trigger), + labels: [...(trigger.labels ?? [])], + milestone: trigger.milestone ?? null, + repositoryFullName: trigger.repositoryFullName, + repositoryRevision: trigger.repositoryRevision ?? null, + title: trigger.title ?? `Issue #${trigger.issueNumber}`, + }); + await input.writer.insert(agentPlans).values({ + id: planId, + agentName: "planner", + input: { + issueNumber: trigger.issueNumber, + labels: trigger.labels ?? [], + milestone: trigger.milestone ?? null, + repositoryFullName: trigger.repositoryFullName, + title: trigger.title ?? "", + }, + issueNumber: trigger.issueNumber, + plan, + runId, + status: "pending", + }); + if (plan.repositoryRevision) { + await input.writer.insert(approvals).values({ + metadata: { planId, planSha256: plan.identity.sha256 }, + requestedBy: "planner", + runId, + scope: "plan-review", + status: "requested", + }); + } + const emitObservability = await recordDevelopmentLoopRunCreatedObservability({ + artifactCount: skeleton.artifacts.length, + deliveryId: trigger.deliveryId, + issueNumber: trigger.issueNumber, + loopKey: developmentLoopKey, + repositoryFullName: input.repository.fullName, + repositoryId: input.repository.id, + runId, + stageCount: skeleton.stages.length, + traceId: input.traceId, + triggerLabel: "agent-ready", + writer: input.writer, + }); + return { eligibleAt, emitObservability, runId }; +} - return result.metadata; +export async function createLinkedDevelopmentLoopRetry(input: { + clock: () => Date; + database: DevelopmentLoopRunDatabase; + manifest: LoopManifest; + reason: "stalled" | "timed_out"; + sourceRunId: string; +}): Promise { + const loopManifest = getDevelopmentLoopManifest(input.manifest); + const source = await input.database.transaction(async (tx) => { + const [run] = await tx + .select({ metadata: loopRuns.metadata, traceId: loopRuns.traceId }) + .from(loopRuns) + .where(eq(loopRuns.id, input.sourceRunId)) + .limit(1); + return run; + }); + if (!source) return undefined; + const metadata = (source.metadata ?? {}) as Record; + const completedAttempt = + typeof metadata.dispatchAttempt === "number" ? metadata.dispatchAttempt : 1; + if (completedAttempt >= loopManifest.retryPolicy.maxAttempts) { + logger.info( + { loopKey: developmentLoopKey, reason: input.reason, runId: input.sourceRunId }, + "development_loop_retry_exhausted", + ); + return undefined; + } + const triggerSnapshot = storedTriggerSnapshot(metadata.triggerSnapshot); + if (!triggerSnapshot) { + logger.error( + { loopKey: developmentLoopKey, reason: "missing_trigger_snapshot", runId: input.sourceRunId }, + "development_loop_retry_exhausted", + ); + return undefined; + } + const delaySeconds = calculateDevelopmentLoopRetryDelaySeconds({ + backoff: loopManifest.retryPolicy.backoff, + completedAttempt, + }); + const eligibleAt = new Date(input.clock().getTime() + delaySeconds * 1_000); + const rootRunId = typeof metadata.rootRunId === "string" ? metadata.rootRunId : input.sourceRunId; + const outcome = await dispatchDevelopmentLoopRun({ + clock: input.clock, + database: input.database, + manifest: input.manifest, + retry: { + attempt: completedAttempt + 1, + eligibleAt, + retryOfRunId: input.sourceRunId, + rootRunId, + }, + traceId: source.traceId ?? undefined, + trigger: { + ...triggerSnapshot, + deliveryId: `loopworks-retry:${input.sourceRunId}:${completedAttempt + 1}`, + }, + }); + logger.info( + { + attempt: completedAttempt + 1, + loopKey: developmentLoopKey, + reason: input.reason, + runId: outcome.runId, + }, + "development_loop_retry_scheduled", + ); + return outcome; +} + +export function runDevelopmentLoopRetrySupervisorTick(input: { + clock: () => Date; + database: DevelopmentLoopRunDatabase; + manifest: LoopManifest; +}) { + return drainDevelopmentLoopDispatchQueue(input); +} + +export async function createDevelopmentLoopRun(input: { + database: DevelopmentLoopRunDatabase; + now?: () => Date; + traceId?: string; + trigger: DevelopmentLoopTrigger; +}): Promise { + const result = await dispatchDevelopmentLoopRun({ + clock: input.now ?? (() => new Date()), + database: input.database, + manifest: defaultLoopManifest, + traceId: input.traceId, + trigger: input.trigger, + }); + + return result; } export async function recordDevelopmentLoopNoop(input: { diff --git a/src/lib/loops/research-run.ts b/src/lib/loops/research-run.ts index f9bd57a..2b8b9e4 100644 --- a/src/lib/loops/research-run.ts +++ b/src/lib/loops/research-run.ts @@ -141,6 +141,13 @@ export type ResearchLoopRunDatabase = Pick; export type ResearchLoopRunMetadata = | { artifactCount: number; mode: "created"; runId: string; stageCount: number } + | { + artifactCount: number; + mode: "lease_contention"; + reason: "issue_active"; + runId: string; + stageCount: number; + } | { artifactCount: number; mode: "simulated"; stageCount: number }; export type ResearchLoopNoopMetadata = { mode: "noop"; reason: "loop_disabled" }; @@ -324,6 +331,55 @@ export async function createResearchLoopRun(input: { ); } + const issueGuardKey = `loop:dispatch:issue-guard:${repository.id}:${input.trigger.issueNumber}`; + await tx + .insert(idempotencyLocks) + .values({ + acquiredAt: createdAt, + expiresAt: createdAt, + key: issueGuardKey, + owner: "loopworks:dispatch-admission", + releasedAt: createdAt, + scope: "loop:dispatch:issue-guard", + status: "released", + }) + .onConflictDoNothing({ target: idempotencyLocks.key }); + const [issueGuard] = await tx + .select({ id: idempotencyLocks.id }) + .from(idempotencyLocks) + .where(eq(idempotencyLocks.key, issueGuardKey)) + .for("update"); + if (!issueGuard) { + throw new Error(`Dispatch issue guard ${issueGuardKey} could not be locked.`); + } + + const [activeRun] = await tx + .select({ id: loopRuns.id }) + .from(loopRuns) + .where( + and( + eq(loopRuns.repositoryId, repository.id), + eq(loopRuns.githubIssueNumber, input.trigger.issueNumber), + sql`(${loopRuns.status} in ('queued', 'running', 'waiting_for_approval', 'blocked') or (${loopRuns.status} = 'failed' and ${loopRuns.completedAt} is null))`, + ), + ) + .limit(1); + if (activeRun) { + const [existingArtifacts, existingSteps] = await Promise.all([ + tx.select({ id: artifacts.id }).from(artifacts).where(eq(artifacts.runId, activeRun.id)), + tx.select({ id: runSteps.id }).from(runSteps).where(eq(runSteps.runId, activeRun.id)), + ]); + return { + metadata: { + artifactCount: existingArtifacts.length, + mode: "lease_contention" as const, + reason: "issue_active" as const, + runId: activeRun.id, + stageCount: existingSteps.length, + }, + }; + } + const runId = randomUUID(); const skeleton = createResearchLoopRunSkeleton({ mode: "created", diff --git a/src/lib/observability/trace-context.ts b/src/lib/observability/trace-context.ts index a6a02be..64acbb1 100644 --- a/src/lib/observability/trace-context.ts +++ b/src/lib/observability/trace-context.ts @@ -1,7 +1,10 @@ +import { randomBytes } from "node:crypto"; import { + context, type Span, type SpanOptions, SpanStatusCode, + TraceFlags, type Tracer, trace, } from "@opentelemetry/api"; @@ -35,6 +38,58 @@ export function startDevelopmentLoopReconciliationSpan(tracer = getLoopworksTrac }; } +type DurableTraceSpanInput = { + traceId?: string; + tracer?: Tracer; +}; + +function startDurableTraceSpan(name: string, input: DurableTraceSpanInput): Span { + const tracer = input.tracer ?? getLoopworksTracer(); + if (!input.traceId || !isValidW3cTraceId(input.traceId)) { + return startLoopworksSpan(name, undefined, tracer); + } + const activeContext = context.active(); + const activeSpanContext = trace.getSpanContext(activeContext); + const parentContext = + activeSpanContext?.traceId === input.traceId + ? activeContext + : trace.setSpanContext(activeContext, { + isRemote: true, + spanId: randomBytes(8).toString("hex"), + traceFlags: TraceFlags.SAMPLED, + traceId: input.traceId, + }); + return tracer.startSpan(name, undefined, parentContext); +} + +export function startDevelopmentLoopDispatchSpan(input: DurableTraceSpanInput | Tracer = {}): { + setOutcome(outcome: "deferred" | "dispatched" | "lease_contention"): void; + span: Span; +} { + const options = "startSpan" in input ? { tracer: input } : input; + const span = startDurableTraceSpan("loopworks.run.dispatch", options); + return { + setOutcome(outcome) { + span.setAttribute("loopworks.dispatch.outcome", outcome); + }, + span, + }; +} + +export function startDevelopmentLoopRetrySpan(input: DurableTraceSpanInput | Tracer = {}): { + setOutcome(outcome: "exhausted" | "ineligible" | "promoted" | "scheduled"): void; + span: Span; +} { + const options = "startSpan" in input ? { tracer: input } : input; + const span = startDurableTraceSpan("loopworks.run.retry", options); + return { + setOutcome(outcome) { + span.setAttribute("loopworks.retry.outcome", outcome); + }, + span, + }; +} + export function markLoopworksSpanOk(span: Span): void { span.setStatus({ code: SpanStatusCode.OK }); } diff --git a/tests/unit/approval-transitions.integration.test.ts b/tests/unit/approval-transitions.integration.test.ts index 2b0ee44..52adddb 100644 --- a/tests/unit/approval-transitions.integration.test.ts +++ b/tests/unit/approval-transitions.integration.test.ts @@ -43,7 +43,7 @@ describe("plan-review approval synchronization", () => { title: "Test-writing subagent", }, }); - if (run.mode !== "created") throw new Error("Expected created run."); + if (run.mode !== "dispatched") throw new Error("Expected dispatched run."); const [approval] = await context.db .select() @@ -80,7 +80,7 @@ describe("plan-review approval synchronization", () => { title: "Test-writing subagent", }, }); - if (run.mode !== "created") throw new Error("Expected created run."); + if (run.mode !== "dispatched") throw new Error("Expected dispatched run."); const [approval] = await context.db .select() .from(approvals) @@ -114,7 +114,7 @@ describe("plan-review approval synchronization", () => { title: "Test-writing subagent", }, }); - if (run.mode !== "created") throw new Error("Expected created run."); + if (run.mode !== "dispatched") throw new Error("Expected dispatched run."); const [approval] = await context.db .select() .from(approvals) @@ -158,7 +158,7 @@ describe("plan-review approval synchronization", () => { title: "Test-writing subagent", }, }); - if (run.mode !== "created") throw new Error("Expected created run."); + if (run.mode !== "dispatched") throw new Error("Expected dispatched run."); const plan = createPlanningAgentSeedPlan({ body: "## Acceptance Criteria\n- Planner output is pinned before review.", @@ -199,7 +199,7 @@ describe("plan-review approval synchronization", () => { title: "Test-writing subagent", }, }); - if (run.mode !== "created") throw new Error("Expected created run."); + if (run.mode !== "dispatched") throw new Error("Expected dispatched run."); const startedAt = new Date("2026-07-11T15:30:00.000Z"); await context.db .update(loopRuns) diff --git a/tests/unit/db/migrations.test.ts b/tests/unit/db/migrations.test.ts index 37498e1..65447a9 100644 --- a/tests/unit/db/migrations.test.ts +++ b/tests/unit/db/migrations.test.ts @@ -5,6 +5,7 @@ import { approvalTransitionEvents, artifacts, artifactTypeEnum, + idempotencyLocks, loopRuns, repositories, runTerminalReasonEnum, @@ -99,6 +100,17 @@ describe("Drizzle migrations", () => { expect(migrationSql).toContain('"terminal_reason" "run_terminal_reason"'); }); + it("tracks dispatch lease correlation and active issue uniqueness", () => { + expect(Object.keys(idempotencyLocks)).toEqual(expect.arrayContaining(["runId", "traceId"])); + + const migrationSql = readMigrationSql(); + expect(migrationSql).toContain('ALTER TABLE "idempotency_locks" ADD COLUMN "run_id" uuid'); + expect(migrationSql).toContain('ALTER TABLE "idempotency_locks" ADD COLUMN "trace_id" text'); + expect(migrationSql).toContain("loop_runs_active_repository_issue_idx"); + expect(migrationSql).toContain("'waiting_for_approval', 'blocked'"); + expect(migrationSql).toContain('"loop_runs"."completed_at" IS NULL'); + }); + it( "tracks screenshot artifacts in the schema and generated migrations", async () => { diff --git a/tests/unit/github/webhook-store.integration.test.ts b/tests/unit/github/webhook-store.integration.test.ts index d4610b4..a30050f 100644 --- a/tests/unit/github/webhook-store.integration.test.ts +++ b/tests/unit/github/webhook-store.integration.test.ts @@ -1,5 +1,6 @@ /** @vitest-environment node */ +import { createHmac } from "node:crypto"; import { context as otelContext, type Span, TraceFlags, trace } from "@opentelemetry/api"; import { eq } from "drizzle-orm"; @@ -309,7 +310,7 @@ describe("GitHub webhook delivery store (pglite integration)", () => { agentReadyTrigger: { shouldTrigger: true, workflow: "development" }, developmentRun: { artifactCount: 10, - mode: "created", + mode: "dispatched", stageCount: 8, }, nextAction: "queue_planning_agent", @@ -367,13 +368,68 @@ describe("GitHub webhook delivery store (pglite integration)", () => { duplicate: true, }); expect(await context.db.select().from(webhookDeliveries)).toHaveLength(1); - expect(await context.db.select().from(idempotencyLocks)).toHaveLength(1); + const replayLocks = await context.db.select().from(idempotencyLocks); + expect(replayLocks).toHaveLength(4); + expect(replayLocks).toEqual( + expect.arrayContaining([ + expect.objectContaining({ scope: "loop:dispatch:group-guard", status: "released" }), + expect.objectContaining({ scope: "loop:dispatch:issue-guard", status: "released" }), + expect.objectContaining({ + owner: runRows[0]?.id, + runId: runRows[0]?.id, + status: "acquired", + traceId: runRows[0]?.traceId, + }), + ]), + ); expect(await context.db.select().from(loopRuns)).toHaveLength(1); expect(await context.db.select().from(runSteps)).toHaveLength(8); expect(await context.db.select().from(artifacts)).toHaveLength(10); expect(await context.db.select().from(agentPlans)).toHaveLength(1); }); + it("reports deferred admission instead of telling a worker to start planning", async () => { + vi.stubEnv("GITHUB_WEBHOOK_SECRET", "dev-webhook-secret"); + await insertLoopworksRepository(); + const store = createStore(); + const send = async (deliveryId: string, issueNumber: number) => { + const fixture = createGithubWebhookFixture({ + deliveryId, + kind: "agent-ready", + secret: "dev-webhook-secret", + url: "https://loopworks.local/api/github/webhooks", + }); + const payloadText = JSON.stringify({ + ...fixture.payload, + issue: { + ...fixture.payload.issue, + html_url: `https://github.com/ncolesummers/loopworks/issues/${issueNumber}`, + number: issueNumber, + }, + }); + const signature = `sha256=${createHmac("sha256", "dev-webhook-secret").update(payloadText).digest("hex")}`; + return handleGithubWebhookPost( + new Request(fixture.url, { + body: payloadText, + headers: { ...fixture.headers, "x-hub-signature-256": signature }, + method: "POST", + }), + { + developmentRunDatabase: context.db as unknown as DevelopmentLoopRunDatabase, + now: () => new Date("2026-06-28T02:00:00.000Z"), + webhookDeliveryStore: store, + }, + ); + }; + + await send("dispatch-first", 11); + const deferred = await send("dispatch-deferred", 12); + await expect(deferred.json()).resolves.toMatchObject({ + developmentRun: { mode: "deferred", reason: "max_in_flight" }, + nextAction: "await_dispatch_capacity", + }); + }); + it("persists one idempotent research-loop run for an accepted spike delivery", async () => { vi.stubEnv("GITHUB_WEBHOOK_SECRET", "dev-webhook-secret"); await insertLoopworksRepository(); diff --git a/tests/unit/loops/development-run-dispatch.integration.test.ts b/tests/unit/loops/development-run-dispatch.integration.test.ts new file mode 100644 index 0000000..b20f3c2 --- /dev/null +++ b/tests/unit/loops/development-run-dispatch.integration.test.ts @@ -0,0 +1,436 @@ +/** @vitest-environment node */ + +import { and, eq } from "drizzle-orm"; + +import { idempotencyLocks, loopRuns, repositories, runSteps } from "@/db/schema"; +import { + createDevelopmentLoopRun, + dispatchDevelopmentLoopRun, + drainDevelopmentLoopDispatchQueue, + type DevelopmentLoopRunDatabase, +} from "@/lib/loops/development-run"; +import { + finalizeDevelopmentLoopRun, + retryDevelopmentLoopStep, + type DevelopmentLoopTransitionDatabase, +} from "@/lib/loops/development-run-transitions"; +import { defaultLoopManifest } from "@/lib/loops/manifest"; +import { createResearchLoopRun } from "@/lib/loops/research-run"; +import type { LoopManifest } from "../../../schemas/loop-manifest"; +import { createPgliteTestDatabase, type PgliteTestDatabase } from "../../helpers/pglite"; + +const traceId = "4bf92f3577b34da6a3ce929d0e0e4736"; + +function runDatabase(context: PgliteTestDatabase): DevelopmentLoopRunDatabase { + return context.db as unknown as DevelopmentLoopRunDatabase; +} + +function transitionDatabase(context: PgliteTestDatabase): DevelopmentLoopTransitionDatabase { + return context.db as unknown as DevelopmentLoopTransitionDatabase; +} + +function manifestWith(input: { group?: string; maxInFlight: number }): LoopManifest { + return { + ...defaultLoopManifest, + loops: defaultLoopManifest.loops.map((loop) => + loop.key === "development-loop" + ? { + ...loop, + concurrency: { + ...loop.concurrency, + group: input.group ?? loop.concurrency.group, + maxInFlight: input.maxInFlight, + }, + } + : loop, + ), + }; +} + +function trigger(issueNumber: number, deliveryId = `issue-${issueNumber}-delivery`) { + return { + body: `## Acceptance Criteria\n- Dispatch issue ${issueNumber} durably.`, + deliveryId, + issueNumber, + issueUrl: `https://github.com/ncolesummers/loopworks/issues/${issueNumber}`, + labels: ["agent-ready", "area:loops"], + milestone: "M3 Durable Loop MVP", + repositoryFullName: "ncolesummers/loopworks", + title: `Issue ${issueNumber}`, + }; +} + +async function insertRepository(context: PgliteTestDatabase) { + await context.db.insert(repositories).values({ + githubRepoId: 96_000_001, + owner: "ncolesummers", + name: "loopworks", + fullName: "ncolesummers/loopworks", + }); +} + +describe("development-loop dispatch admission", () => { + let context: PgliteTestDatabase; + + beforeEach(async () => { + context = await createPgliteTestDatabase(); + await insertRepository(context); + }); + + afterEach(async () => { + await context.close(); + }); + + it("defers over-cap work and leases it after the in-flight run terminates", async () => { + const manifest = manifestWith({ maxInFlight: 1 }); + const first = await dispatchDevelopmentLoopRun({ + clock: () => new Date("2026-07-24T16:00:00.000Z"), + database: runDatabase(context), + manifest, + traceId, + trigger: trigger(96), + }); + const second = await dispatchDevelopmentLoopRun({ + clock: () => new Date("2026-07-24T16:00:01.000Z"), + database: runDatabase(context), + manifest, + traceId, + trigger: trigger(97), + }); + + expect(first).toMatchObject({ mode: "dispatched" }); + expect(second).toMatchObject({ mode: "deferred", reason: "max_in_flight" }); + expect(await context.db.select().from(loopRuns)).toHaveLength(2); + expect( + await context.db + .select() + .from(idempotencyLocks) + .where(eq(idempotencyLocks.status, "acquired")), + ).toEqual([ + expect.objectContaining({ + owner: first.runId, + runId: first.runId, + scope: "repo:ncolesummers/loopworks:loop:development", + traceId, + }), + ]); + + await finalizeDevelopmentLoopRun({ + database: transitionDatabase(context), + manifest, + occurredAt: new Date("2026-07-24T16:05:00.000Z"), + reason: "succeeded", + runId: first.runId, + }); + + const acquired = await context.db + .select() + .from(idempotencyLocks) + .where(eq(idempotencyLocks.status, "acquired")); + expect(acquired).toEqual([ + expect.objectContaining({ owner: second.runId, runId: second.runId }), + ]); + }); + + it("allows exactly one concurrent run for the same issue across delivery ids", async () => { + const manifest = manifestWith({ maxInFlight: 2 }); + const outcomes = await Promise.all([ + dispatchDevelopmentLoopRun({ + clock: () => new Date("2026-07-24T16:00:00.000Z"), + database: runDatabase(context), + manifest, + traceId, + trigger: trigger(96, "delivery-a"), + }), + dispatchDevelopmentLoopRun({ + clock: () => new Date("2026-07-24T16:00:00.000Z"), + database: runDatabase(context), + manifest, + traceId, + trigger: trigger(96, "delivery-b"), + }), + ]); + + expect(outcomes.filter(({ mode }) => mode === "dispatched")).toHaveLength(1); + expect(outcomes.filter(({ mode }) => mode === "lease_contention")).toHaveLength(1); + expect(await context.db.select().from(loopRuns)).toHaveLength(1); + expect( + await context.db + .select() + .from(idempotencyLocks) + .where(eq(idempotencyLocks.status, "acquired")), + ).toHaveLength(1); + }); + + it("changes capacity and isolation when manifest values change", async () => { + const capTwo = manifestWith({ maxInFlight: 2 }); + const first = await dispatchDevelopmentLoopRun({ + clock: () => new Date("2026-07-24T16:00:00.000Z"), + database: runDatabase(context), + manifest: capTwo, + trigger: trigger(96), + }); + const second = await dispatchDevelopmentLoopRun({ + clock: () => new Date("2026-07-24T16:00:01.000Z"), + database: runDatabase(context), + manifest: capTwo, + trigger: trigger(97), + }); + + expect(first.mode).toBe("dispatched"); + expect(second.mode).toBe("dispatched"); + + const isolated = manifestWith({ + group: "repo:{repo}:loop:development:isolated", + maxInFlight: 1, + }); + const third = await dispatchDevelopmentLoopRun({ + clock: () => new Date("2026-07-24T16:00:02.000Z"), + database: runDatabase(context), + manifest: isolated, + trigger: trigger(98), + }); + expect(third.mode).toBe("dispatched"); + }); + + it("keeps expired but unreconciled leases in capacity and rejects unknown placeholders", async () => { + const manifest = manifestWith({ maxInFlight: 1 }); + await dispatchDevelopmentLoopRun({ + clock: () => new Date("2026-07-24T16:00:00.000Z"), + database: runDatabase(context), + manifest, + trigger: trigger(96), + }); + await expect( + dispatchDevelopmentLoopRun({ + clock: () => new Date("2026-07-24T18:00:00.000Z"), + database: runDatabase(context), + manifest, + trigger: trigger(97), + }), + ).resolves.toMatchObject({ mode: "deferred", reason: "max_in_flight" }); + + await expect( + dispatchDevelopmentLoopRun({ + clock: () => new Date("2026-07-24T18:00:00.000Z"), + database: runDatabase(context), + manifest: manifestWith({ group: "repo:{repo}:loop:{unknown}", maxInFlight: 1 }), + trigger: trigger(98), + }), + ).rejects.toThrow("Unsupported concurrency group placeholder: {unknown}"); + }); + + it("promotes only due queued retries and preserves the stored trace", async () => { + const manifest = manifestWith({ maxInFlight: 1 }); + const first = await dispatchDevelopmentLoopRun({ + clock: () => new Date("2026-07-24T16:00:00.000Z"), + database: runDatabase(context), + manifest, + traceId, + trigger: trigger(96), + }); + const deferred = await dispatchDevelopmentLoopRun({ + clock: () => new Date("2026-07-24T16:00:01.000Z"), + database: runDatabase(context), + manifest, + traceId, + trigger: trigger(97), + }); + if (first.mode !== "dispatched" || deferred.mode !== "deferred") { + throw new Error("Expected one dispatched and one deferred run."); + } + await context.db + .update(loopRuns) + .set({ queuedAt: new Date("2026-07-24T16:10:00.000Z") }) + .where(eq(loopRuns.id, deferred.runId)); + await finalizeDevelopmentLoopRun({ + database: transitionDatabase(context), + manifest, + occurredAt: new Date("2026-07-24T16:05:00.000Z"), + reason: "succeeded", + runId: first.runId, + }); + + expect( + await context.db + .select() + .from(idempotencyLocks) + .where( + and(eq(idempotencyLocks.runId, deferred.runId), eq(idempotencyLocks.status, "acquired")), + ), + ).toEqual([]); + + const promoted = await drainDevelopmentLoopDispatchQueue({ + clock: () => new Date("2026-07-24T16:10:00.000Z"), + database: runDatabase(context), + manifest, + }); + expect(promoted).toEqual([ + expect.objectContaining({ mode: "dispatched", runId: deferred.runId, traceId }), + ]); + const steps = await context.db + .select() + .from(runSteps) + .where(eq(runSteps.runId, deferred.runId)); + expect(steps.every((step) => step.traceId === traceId)).toBe(true); + }); + + it("preserves admission outcomes through the persistent compatibility entry point", async () => { + const first = await createDevelopmentLoopRun({ + database: runDatabase(context), + now: () => new Date("2026-07-24T16:00:00.000Z"), + trigger: trigger(96), + }); + const second = await createDevelopmentLoopRun({ + database: runDatabase(context), + now: () => new Date("2026-07-24T16:00:01.000Z"), + trigger: trigger(97), + }); + + expect(first.mode).toBe("dispatched"); + expect(second).toMatchObject({ mode: "deferred", reason: "max_in_flight" }); + }); + + it("refuses stage mutation for a deferred run without an execution lease", async () => { + await dispatchDevelopmentLoopRun({ + clock: () => new Date("2026-07-24T16:00:00.000Z"), + database: runDatabase(context), + manifest: defaultLoopManifest, + trigger: trigger(96), + }); + const deferred = await dispatchDevelopmentLoopRun({ + clock: () => new Date("2026-07-24T16:00:01.000Z"), + database: runDatabase(context), + manifest: defaultLoopManifest, + trigger: trigger(97), + }); + const [planning] = await context.db + .select() + .from(runSteps) + .where(eq(runSteps.runId, deferred.runId)); + await context.db.update(runSteps).set({ status: "failed" }).where(eq(runSteps.id, planning.id)); + await context.db + .update(loopRuns) + .set({ status: "failed" }) + .where(eq(loopRuns.id, deferred.runId)); + + await expect( + retryDevelopmentLoopStep({ + database: transitionDatabase(context), + reason: "operator_retry", + runId: deferred.runId, + stage: planning.stage, + }), + ).rejects.toThrow("does not own an acquired execution lease"); + }); + + it("enforces cross-loop issue exclusivity without surfacing a unique violation", async () => { + const development = await dispatchDevelopmentLoopRun({ + clock: () => new Date("2026-07-24T16:00:00.000Z"), + database: runDatabase(context), + manifest: defaultLoopManifest, + trigger: trigger(96), + }); + const research = await createResearchLoopRun({ + database: runDatabase(context), + now: () => new Date("2026-07-24T16:00:01.000Z"), + trigger: trigger(96, "research-delivery"), + }); + + expect(research).toMatchObject({ mode: "lease_contention", runId: development.runId }); + const researchFirst = await createResearchLoopRun({ + database: runDatabase(context), + now: () => new Date("2026-07-24T16:00:02.000Z"), + trigger: trigger(97, "research-first"), + }); + expect(researchFirst.mode).toBe("created"); + if (researchFirst.mode !== "created") throw new Error("Expected a research run."); + const developmentSecond = await dispatchDevelopmentLoopRun({ + clock: () => new Date("2026-07-24T16:00:03.000Z"), + database: runDatabase(context), + manifest: defaultLoopManifest, + trigger: trigger(97, "development-second"), + }); + expect(developmentSecond).toMatchObject({ + mode: "lease_contention", + runId: researchFirst.runId, + }); + expect(await context.db.select().from(loopRuns)).toHaveLength(2); + }); + + it("does not emit dispatch contention telemetry for a delivery replay", async () => { + const recordContention = vi.fn(); + const first = await dispatchDevelopmentLoopRun({ + clock: () => new Date("2026-07-24T16:00:00.000Z"), + database: runDatabase(context), + manifest: defaultLoopManifest, + observability: { recordLockContentionMetric: recordContention }, + trigger: trigger(96, "same-delivery"), + }); + const replay = await dispatchDevelopmentLoopRun({ + clock: () => new Date("2026-07-24T16:00:01.000Z"), + database: runDatabase(context), + manifest: defaultLoopManifest, + observability: { recordLockContentionMetric: recordContention }, + trigger: trigger(96, "same-delivery"), + }); + + expect(first.mode).toBe("dispatched"); + expect(replay).toMatchObject({ mode: "lease_contention", reason: "delivery_replay" }); + expect(recordContention).not.toHaveBeenCalled(); + }); + + it("does not let an orphaned audit lease consume capacity after its run is deleted", async () => { + const first = await dispatchDevelopmentLoopRun({ + clock: () => new Date("2026-07-24T16:00:00.000Z"), + database: runDatabase(context), + manifest: defaultLoopManifest, + trigger: trigger(96), + }); + await context.db.delete(loopRuns).where(eq(loopRuns.id, first.runId)); + + const second = await dispatchDevelopmentLoopRun({ + clock: () => new Date("2026-07-24T16:00:01.000Z"), + database: runDatabase(context), + manifest: defaultLoopManifest, + trigger: trigger(97), + }); + expect(second.mode).toBe("dispatched"); + expect( + await context.db + .select() + .from(idempotencyLocks) + .where(eq(idempotencyLocks.status, "acquired")), + ).toEqual([ + expect.objectContaining({ runId: null }), + expect.objectContaining({ runId: second.runId }), + ]); + }); + + it("repairs a leaked acquired lease when a terminal finalization is replayed", async () => { + const run = await dispatchDevelopmentLoopRun({ + clock: () => new Date("2026-07-24T16:00:00.000Z"), + database: runDatabase(context), + manifest: defaultLoopManifest, + trigger: trigger(96), + }); + const completedAt = new Date("2026-07-24T16:05:00.000Z"); + await context.db + .update(loopRuns) + .set({ completedAt, status: "failed", terminalReason: "failed" }) + .where(eq(loopRuns.id, run.runId)); + + await finalizeDevelopmentLoopRun({ + database: transitionDatabase(context), + manifest: defaultLoopManifest, + occurredAt: new Date("2026-07-24T16:06:00.000Z"), + reason: "failed", + runId: run.runId, + }); + const [lease] = await context.db + .select() + .from(idempotencyLocks) + .where(eq(idempotencyLocks.runId, run.runId)); + expect(lease).toMatchObject({ releasedAt: completedAt, status: "released" }); + }); +}); diff --git a/tests/unit/loops/development-run-reconciliation-store.integration.test.ts b/tests/unit/loops/development-run-reconciliation-store.integration.test.ts index b4e6f38..9c85e26 100644 --- a/tests/unit/loops/development-run-reconciliation-store.integration.test.ts +++ b/tests/unit/loops/development-run-reconciliation-store.integration.test.ts @@ -1,7 +1,13 @@ /** @vitest-environment node */ import { and, eq } from "drizzle-orm"; -import { loopRuns, observabilityEvents, repositories, runSteps } from "@/db/schema"; +import { + idempotencyLocks, + loopRuns, + observabilityEvents, + repositories, + runSteps, +} from "@/db/schema"; import { createDevelopmentLoopRun, type DevelopmentLoopRunDatabase, @@ -56,7 +62,7 @@ describe("development-loop reconciliation store", () => { traceId: "4bf92f3577b34da6a3ce929d0e0e4736", trigger, }); - if (created.mode !== "created") throw new Error("Expected a created run."); + if (created.mode !== "dispatched") throw new Error("Expected a dispatched run."); await context.db .update(loopRuns) .set({ status: "running" }) @@ -137,7 +143,7 @@ describe("development-loop reconciliation store", () => { repositoryFullName: "ncolesummers/loopworks-race", }, }); - if (created.mode !== "created") throw new Error("Expected a created run."); + if (created.mode !== "dispatched") throw new Error("Expected a dispatched run."); await context.db .update(loopRuns) .set({ status: "running" }) @@ -168,4 +174,73 @@ describe("development-loop reconciliation store", () => { const [run] = await context.db.select().from(loopRuns).where(eq(loopRuns.id, created.runId)); expect(run).toMatchObject({ status: "running", terminalReason: null }); }); + + it("uses the durable run lease as the default execution-liveness authority", async () => { + await context.db.insert(repositories).values({ + githubRepoId: 95_000_003, + installationId: 95_003, + owner: "ncolesummers", + name: "loopworks", + fullName: "ncolesummers/loopworks", + }); + const created = await createDevelopmentLoopRun({ + database: runDatabase(context), + now: () => new Date("2026-07-22T16:00:00.000Z"), + trigger: { ...trigger, deliveryId: "lease-liveness" }, + }); + if (created.mode !== "dispatched") throw new Error("Expected a dispatched run."); + await context.db + .update(loopRuns) + .set({ status: "running" }) + .where(eq(loopRuns.id, created.runId)); + const activeStore = createDevelopmentLoopRunStore({ + clock: () => new Date("2026-07-22T16:01:00.000Z"), + database: context.db as unknown as DevelopmentLoopReconciliationDatabase, + }); + const [run] = await activeStore.listActiveRuns(); + if (!run) throw new Error("Expected active run."); + await expect(activeStore.getExecutionLiveness(run)).resolves.toBe("active"); + + const expiredStore = createDevelopmentLoopRunStore({ + clock: () => new Date("2026-07-22T17:31:00.000Z"), + database: context.db as unknown as DevelopmentLoopReconciliationDatabase, + }); + await expect(expiredStore.getExecutionLiveness(run)).resolves.toBe("inactive"); + await context.db + .update(idempotencyLocks) + .set({ status: "released" }) + .where(eq(idempotencyLocks.runId, created.runId)); + await expect(activeStore.getExecutionLiveness(run)).resolves.toBe("inactive"); + + await context.db.delete(idempotencyLocks).where(eq(idempotencyLocks.runId, created.runId)); + await expect(activeStore.getExecutionLiveness(run)).resolves.toBe("unknown"); + }); + + it("includes an expired queued lease owner for reconciliation but excludes deferred queued work", async () => { + await context.db.insert(repositories).values({ + githubRepoId: 95_000_004, + installationId: 95_004, + owner: "ncolesummers", + name: "loopworks", + fullName: "ncolesummers/loopworks", + }); + const leased = await createDevelopmentLoopRun({ + database: runDatabase(context), + now: () => new Date("2026-07-22T16:00:00.000Z"), + trigger: { ...trigger, deliveryId: "queued-lease-owner" }, + }); + const deferred = await createDevelopmentLoopRun({ + database: runDatabase(context), + now: () => new Date("2026-07-22T16:00:01.000Z"), + trigger: { ...trigger, deliveryId: "queued-deferred", issueNumber: 96 }, + }); + expect(leased.mode).toBe("dispatched"); + expect(deferred.mode).toBe("deferred"); + + const store = createDevelopmentLoopRunStore({ + clock: () => new Date("2026-07-22T17:31:00.000Z"), + database: context.db as unknown as DevelopmentLoopReconciliationDatabase, + }); + expect((await store.listActiveRuns()).map(({ runId }) => runId)).toEqual([leased.runId]); + }); }); diff --git a/tests/unit/loops/development-run-retry.integration.test.ts b/tests/unit/loops/development-run-retry.integration.test.ts new file mode 100644 index 0000000..599690e --- /dev/null +++ b/tests/unit/loops/development-run-retry.integration.test.ts @@ -0,0 +1,412 @@ +/** @vitest-environment node */ + +import { eq } from "drizzle-orm"; + +import { idempotencyLocks, loopRuns, repositories, runSteps } from "@/db/schema"; +import { + calculateDevelopmentLoopRetryDelaySeconds, + dispatchDevelopmentLoopRun, + runDevelopmentLoopRetrySupervisorTick, + type DevelopmentLoopRunDatabase, +} from "@/lib/loops/development-run"; +import { + finalizeDevelopmentLoopRun, + retryDevelopmentLoopStep, + scheduleDevelopmentLoopStageRetry, + type DevelopmentLoopTransitionDatabase, +} from "@/lib/loops/development-run-transitions"; +import { defaultLoopManifest } from "@/lib/loops/manifest"; +import { createPgliteTestDatabase, type PgliteTestDatabase } from "../../helpers/pglite"; + +const traceId = "4bf92f3577b34da6a3ce929d0e0e4736"; + +function runDatabase(context: PgliteTestDatabase): DevelopmentLoopRunDatabase { + return context.db as unknown as DevelopmentLoopRunDatabase; +} + +function transitionDatabase(context: PgliteTestDatabase): DevelopmentLoopTransitionDatabase { + return context.db as unknown as DevelopmentLoopTransitionDatabase; +} + +describe("development-loop automatic retries", () => { + let context: PgliteTestDatabase; + + beforeEach(async () => { + context = await createPgliteTestDatabase(); + await context.db.insert(repositories).values({ + githubRepoId: 96_000_002, + owner: "ncolesummers", + name: "loopworks", + fullName: "ncolesummers/loopworks", + }); + }); + + afterEach(async () => { + await context.close(); + }); + + it("calculates fixed and capped exponential backoff with total-attempt semantics", () => { + expect( + calculateDevelopmentLoopRetryDelaySeconds({ + backoff: { initialSeconds: 30, maxSeconds: 300, strategy: "fixed" }, + completedAttempt: 4, + }), + ).toBe(30); + expect( + [1, 2, 3, 4, 5, 6].map((completedAttempt) => + calculateDevelopmentLoopRetryDelaySeconds({ + backoff: { initialSeconds: 30, maxSeconds: 300, strategy: "exponential" }, + completedAttempt, + }), + ), + ).toEqual([30, 60, 120, 240, 300, 300]); + }); + + it.each([ + "stalled", + "timed_out", + ] as const)("preserves a terminal %s run and creates one trace-linked retry from planning", async (reason) => { + const source = await dispatchDevelopmentLoopRun({ + clock: () => new Date("2026-07-24T16:00:00.000Z"), + database: runDatabase(context), + manifest: defaultLoopManifest, + traceId, + trigger: { + body: "## Acceptance Criteria\n- Preserve this exact trigger.", + deliveryId: `delivery-${reason}`, + issueNumber: 96, + issueUrl: "https://github.com/ncolesummers/loopworks/issues/96", + labels: ["agent-ready"], + milestone: "M3 Durable Loop MVP", + repositoryFullName: "ncolesummers/loopworks", + repositoryRevision: { commitSha: "a".repeat(40), ref: "main" }, + title: "Durable dispatch", + }, + }); + expect(source.mode).toBe("dispatched"); + + await finalizeDevelopmentLoopRun({ + database: transitionDatabase(context), + manifest: defaultLoopManifest, + occurredAt: new Date("2026-07-24T16:05:00.000Z"), + reason, + runId: source.runId, + }); + + const runs = await context.db.select().from(loopRuns); + expect(runs).toHaveLength(2); + const terminal = runs.find(({ id }) => id === source.runId); + const retry = runs.find(({ id }) => id !== source.runId); + expect(terminal).toMatchObject({ completedAt: new Date("2026-07-24T16:05:00.000Z") }); + expect(retry).toMatchObject({ + currentStage: "planning", + queuedAt: new Date("2026-07-24T16:05:30.000Z"), + status: "queued", + traceId, + }); + expect( + ( + await context.db + .select() + .from(runSteps) + .where(eq(runSteps.runId, retry?.id ?? "")) + ).find(({ stage }) => stage === "planning")?.queuedAt, + ).toEqual(new Date("2026-07-24T16:05:00.000Z")); + expect(retry?.metadata).toMatchObject({ + dispatchAttempt: 2, + retryOfRunId: source.runId, + triggerSnapshot: { body: "## Acceptance Criteria\n- Preserve this exact trigger." }, + }); + expect( + await context.db + .select() + .from(idempotencyLocks) + .where(eq(idempotencyLocks.status, "acquired")), + ).toEqual([]); + + expect( + await runDevelopmentLoopRetrySupervisorTick({ + clock: () => new Date("2026-07-24T16:05:29.999Z"), + database: runDatabase(context), + manifest: defaultLoopManifest, + }), + ).toEqual([]); + const due = await runDevelopmentLoopRetrySupervisorTick({ + clock: () => new Date("2026-07-24T16:05:30.000Z"), + database: runDatabase(context), + manifest: defaultLoopManifest, + }); + expect(due).toEqual([expect.objectContaining({ runId: retry?.id })]); + expect( + ( + await context.db + .select() + .from(runSteps) + .where(eq(runSteps.runId, retry?.id ?? "")) + ).every((step) => step.traceId === traceId), + ).toBe(true); + }); + + it("never links a retry for cancellation", async () => { + const source = await dispatchDevelopmentLoopRun({ + clock: () => new Date("2026-07-24T16:00:00.000Z"), + database: runDatabase(context), + manifest: defaultLoopManifest, + trigger: { + issueNumber: 96, + repositoryFullName: "ncolesummers/loopworks", + }, + }); + await finalizeDevelopmentLoopRun({ + database: transitionDatabase(context), + manifest: defaultLoopManifest, + occurredAt: new Date("2026-07-24T16:05:00.000Z"), + reason: "canceled_by_reconciliation", + runId: source.runId, + }); + expect(await context.db.select().from(loopRuns)).toHaveLength(1); + }); + + it("schedules a retryable failed stage and increments its attempt only when due", async () => { + const source = await dispatchDevelopmentLoopRun({ + clock: () => new Date("2026-07-24T16:00:00.000Z"), + database: runDatabase(context), + manifest: defaultLoopManifest, + traceId, + trigger: { issueNumber: 96, repositoryFullName: "ncolesummers/loopworks" }, + }); + const [planning] = await context.db + .select() + .from(runSteps) + .where(eq(runSteps.runId, source.runId)); + await context.db + .update(runSteps) + .set({ + completedAt: new Date("2026-07-24T16:05:00.000Z"), + metadata: { failure: { code: "provider_unavailable", retryable: true } }, + status: "failed", + }) + .where(eq(runSteps.id, planning.id)); + await context.db + .update(loopRuns) + .set({ currentStage: planning.stage, status: "failed" }) + .where(eq(loopRuns.id, source.runId)); + + const scheduled = await scheduleDevelopmentLoopStageRetry({ + database: transitionDatabase(context), + failure: { code: "provider_unavailable", retryable: true }, + manifest: defaultLoopManifest, + occurredAt: new Date("2026-07-24T16:05:00.000Z"), + runId: source.runId, + stage: planning.stage, + }); + expect(scheduled).toMatchObject({ + attempt: 1, + eligibleAt: new Date("2026-07-24T16:05:30.000Z"), + status: "scheduled", + }); + expect( + (await context.db.select().from(runSteps).where(eq(runSteps.id, planning.id)))[0], + ).toMatchObject({ attempt: 1, status: "failed" }); + expect( + await context.db + .select() + .from(idempotencyLocks) + .where(eq(idempotencyLocks.status, "acquired")), + ).toEqual([]); + + await runDevelopmentLoopRetrySupervisorTick({ + clock: () => new Date("2026-07-24T16:05:30.000Z"), + database: runDatabase(context), + manifest: defaultLoopManifest, + }); + expect( + (await context.db.select().from(runSteps).where(eq(runSteps.id, planning.id)))[0], + ).toMatchObject({ + attempt: 2, + completedAt: null, + metadata: { + attemptHistory: [expect.objectContaining({ attempt: 1, status: "failed" })], + }, + status: "queued", + traceId, + }); + + await finalizeDevelopmentLoopRun({ + database: transitionDatabase(context), + manifest: defaultLoopManifest, + occurredAt: new Date("2026-07-24T16:06:00.000Z"), + reason: "timed_out", + runId: source.runId, + }); + expect(await context.db.select().from(loopRuns)).toHaveLength(1); + }); + + it("finalizes exhausted stage work without erasing its failure evidence", async () => { + const oneAttemptManifest = { + ...defaultLoopManifest, + loops: defaultLoopManifest.loops.map((loop) => + loop.key === "development-loop" + ? { ...loop, retryPolicy: { ...loop.retryPolicy, maxAttempts: 1 } } + : loop, + ), + }; + const source = await dispatchDevelopmentLoopRun({ + clock: () => new Date("2026-07-24T16:00:00.000Z"), + database: runDatabase(context), + manifest: oneAttemptManifest, + trigger: { issueNumber: 96, repositoryFullName: "ncolesummers/loopworks" }, + }); + const [planning] = await context.db + .select() + .from(runSteps) + .where(eq(runSteps.runId, source.runId)); + const failure = { code: "provider_unavailable", retryable: true }; + await context.db + .update(runSteps) + .set({ metadata: { failure }, status: "failed" }) + .where(eq(runSteps.id, planning.id)); + await context.db + .update(loopRuns) + .set({ status: "failed" }) + .where(eq(loopRuns.id, source.runId)); + + expect( + await scheduleDevelopmentLoopStageRetry({ + database: transitionDatabase(context), + failure, + manifest: oneAttemptManifest, + occurredAt: new Date("2026-07-24T16:05:00.000Z"), + runId: source.runId, + stage: planning.stage, + }), + ).toMatchObject({ attempt: 1, status: "exhausted" }); + expect((await context.db.select().from(loopRuns))[0]).toMatchObject({ + completedAt: new Date("2026-07-24T16:05:00.000Z"), + status: "failed", + terminalReason: "failed", + }); + expect( + (await context.db.select().from(runSteps).where(eq(runSteps.id, planning.id)))[0], + ).toMatchObject({ metadata: { failure }, status: "failed" }); + }); + + it("does not finalize an ineligible retry request", async () => { + const source = await dispatchDevelopmentLoopRun({ + clock: () => new Date("2026-07-24T16:00:00.000Z"), + database: runDatabase(context), + manifest: defaultLoopManifest, + trigger: { issueNumber: 96, repositoryFullName: "ncolesummers/loopworks" }, + }); + const [planning] = await context.db + .select() + .from(runSteps) + .where(eq(runSteps.runId, source.runId)); + + await expect( + scheduleDevelopmentLoopStageRetry({ + database: transitionDatabase(context), + failure: { code: "provider_unavailable", retryable: true }, + manifest: defaultLoopManifest, + occurredAt: new Date("2026-07-24T16:05:00.000Z"), + runId: source.runId, + stage: planning.stage, + }), + ).resolves.toMatchObject({ status: "ineligible" }); + expect((await context.db.select().from(loopRuns))[0]).toMatchObject({ + completedAt: null, + status: "queued", + terminalReason: null, + }); + }); + + it("shares one total attempt budget across stage and linked retries", async () => { + const source = await dispatchDevelopmentLoopRun({ + clock: () => new Date("2026-07-24T16:00:00.000Z"), + database: runDatabase(context), + manifest: defaultLoopManifest, + trigger: { issueNumber: 96, repositoryFullName: "ncolesummers/loopworks" }, + }); + await finalizeDevelopmentLoopRun({ + database: transitionDatabase(context), + manifest: defaultLoopManifest, + occurredAt: new Date("2026-07-24T16:05:00.000Z"), + reason: "timed_out", + runId: source.runId, + }); + const retry = (await context.db.select().from(loopRuns)).find(({ id }) => id !== source.runId); + if (!retry) throw new Error("Expected linked retry."); + const [planning] = await context.db.select().from(runSteps).where(eq(runSteps.runId, retry.id)); + const failure = { code: "provider_unavailable", retryable: true }; + await context.db + .update(runSteps) + .set({ metadata: { failure }, status: "failed" }) + .where(eq(runSteps.id, planning.id)); + await context.db.update(loopRuns).set({ status: "failed" }).where(eq(loopRuns.id, retry.id)); + + await expect( + scheduleDevelopmentLoopStageRetry({ + database: transitionDatabase(context), + failure, + manifest: defaultLoopManifest, + occurredAt: new Date("2026-07-24T16:06:00.000Z"), + runId: retry.id, + stage: planning.stage, + }), + ).resolves.toMatchObject({ attempt: 2, status: "exhausted" }); + expect(await context.db.select().from(loopRuns)).toHaveLength(2); + }); + + it("prevents operator retry from racing a scheduled automatic retry or reviving a terminal run", async () => { + const source = await dispatchDevelopmentLoopRun({ + clock: () => new Date("2026-07-24T16:00:00.000Z"), + database: runDatabase(context), + manifest: defaultLoopManifest, + trigger: { issueNumber: 96, repositoryFullName: "ncolesummers/loopworks" }, + }); + const [planning] = await context.db + .select() + .from(runSteps) + .where(eq(runSteps.runId, source.runId)); + const failure = { code: "provider_unavailable", retryable: true }; + await context.db + .update(runSteps) + .set({ metadata: { failure }, status: "failed" }) + .where(eq(runSteps.id, planning.id)); + await context.db + .update(loopRuns) + .set({ status: "failed" }) + .where(eq(loopRuns.id, source.runId)); + await scheduleDevelopmentLoopStageRetry({ + database: transitionDatabase(context), + failure, + manifest: defaultLoopManifest, + occurredAt: new Date("2026-07-24T16:05:00.000Z"), + runId: source.runId, + stage: planning.stage, + }); + + await expect( + retryDevelopmentLoopStep({ + database: transitionDatabase(context), + occurredAt: new Date("2026-07-24T16:05:01.000Z"), + reason: "operator_retry", + runId: source.runId, + stage: planning.stage, + }), + ).resolves.toMatchObject({ attempt: 1, idempotent: true }); + + await context.db + .update(loopRuns) + .set({ completedAt: new Date("2026-07-24T16:05:02.000Z"), status: "failed" }) + .where(eq(loopRuns.id, source.runId)); + await expect( + retryDevelopmentLoopStep({ + database: transitionDatabase(context), + reason: "operator_retry", + runId: source.runId, + stage: planning.stage, + }), + ).rejects.toThrow("terminal run"); + }); +}); diff --git a/tests/unit/loops/development-run-transitions.test.ts b/tests/unit/loops/development-run-transitions.test.ts index 3c4ced8..fe30340 100644 --- a/tests/unit/loops/development-run-transitions.test.ts +++ b/tests/unit/loops/development-run-transitions.test.ts @@ -1,7 +1,14 @@ /** @vitest-environment node */ import { and, eq } from "drizzle-orm"; -import { artifacts, loopRuns, observabilityEvents, repositories, runSteps } from "@/db/schema"; +import { + artifacts, + idempotencyLocks, + loopRuns, + observabilityEvents, + repositories, + runSteps, +} from "@/db/schema"; import { createDevelopmentLoopRun, type DevelopmentLoopRunDatabase, @@ -62,7 +69,7 @@ async function createRun(context: PgliteTestDatabase) { trigger: issueTrigger, }); - if (run.mode !== "created") { + if (run.mode !== "dispatched") { throw new Error("Expected test run creation."); } @@ -620,6 +627,10 @@ describe("development-loop run transitions", () => { }); const [run] = await context.db.select().from(loopRuns).where(eq(loopRuns.id, runId)); + const [lease] = await context.db + .select() + .from(idempotencyLocks) + .where(eq(idempotencyLocks.runId, runId)); expect(run).toMatchObject({ status, @@ -628,6 +639,11 @@ describe("development-loop run transitions", () => { if (status === "canceled") { expect(run.canceledAt).toEqual(new Date("2026-07-08T16:10:00.000Z")); } + expect(lease).toMatchObject({ + owner: runId, + releasedAt: new Date("2026-07-08T16:10:00.000Z"), + status: "released", + }); expect(metrics.runCompleted).toHaveBeenCalledWith({ loopKey: "development-loop", repository: "ncolesummers/loopworks", diff --git a/tests/unit/loops/development-run.test.ts b/tests/unit/loops/development-run.test.ts index b9e220b..efc8e47 100644 --- a/tests/unit/loops/development-run.test.ts +++ b/tests/unit/loops/development-run.test.ts @@ -156,7 +156,7 @@ describe("agent-ready development loop run skeleton", () => { expect(result).toMatchObject({ artifactCount: 10, - mode: "created", + mode: "dispatched", stageCount: 8, }); @@ -271,7 +271,14 @@ describe("agent-ready development loop run skeleton", () => { trigger: issueTrigger, }); - expect(second).toEqual(first); + expect(first.mode).toBe("dispatched"); + expect(second).toMatchObject({ + artifactCount: first.artifactCount, + mode: "lease_contention", + reason: "delivery_replay", + runId: first.runId, + stageCount: first.stageCount, + }); expect(await context.db.select().from(loopRuns)).toHaveLength(1); expect(await context.db.select().from(runSteps)).toHaveLength(8); expect(await context.db.select().from(artifacts)).toHaveLength(10); diff --git a/tests/unit/loops/implementation-transition.test.ts b/tests/unit/loops/implementation-transition.test.ts index ffbe998..9b717d5 100644 --- a/tests/unit/loops/implementation-transition.test.ts +++ b/tests/unit/loops/implementation-transition.test.ts @@ -58,7 +58,7 @@ describe("implementation stage transition", () => { title: "Implementation subagent", }, }); - if (created.mode !== "created") throw new Error("Expected created run."); + if (created.mode !== "dispatched") throw new Error("Expected dispatched run."); const [approval] = await context.db .select() .from(approvals) diff --git a/tests/unit/loops/pr-preparation-transition.test.ts b/tests/unit/loops/pr-preparation-transition.test.ts index 98efeac..06ca1db 100644 --- a/tests/unit/loops/pr-preparation-transition.test.ts +++ b/tests/unit/loops/pr-preparation-transition.test.ts @@ -77,7 +77,7 @@ describe("PR preparation transition", () => { title: "PR preparation subagent for PR intent content", }, }); - if (created.mode !== "created") throw new Error("Expected a persisted run."); + if (created.mode !== "dispatched") throw new Error("Expected a persisted run."); const runId = created.runId; const [planRow] = await context.db.select().from(agentPlans).where(eq(agentPlans.runId, runId)); if (!planRow) throw new Error("Expected a plan row."); diff --git a/tests/unit/loops/pr-stage.test.ts b/tests/unit/loops/pr-stage.test.ts index 99ba578..4caf2c6 100644 --- a/tests/unit/loops/pr-stage.test.ts +++ b/tests/unit/loops/pr-stage.test.ts @@ -11,14 +11,15 @@ import { import { approvals, artifacts, deployments, loopRuns, repositories, runSteps } from "@/db/schema"; import { createDevelopmentLoopRun, + runDevelopmentLoopRetrySupervisorTick, type DevelopmentLoopRunDatabase, } from "@/lib/loops/development-run"; import { applyDevelopmentLoopValidationReport, executeDevelopmentLoopPrStage, type DevelopmentLoopTransitionDatabase, - retryDevelopmentLoopStep, } from "@/lib/loops/development-run-transitions"; +import { defaultLoopManifest } from "@/lib/loops/manifest"; import { composePrIntent, createPrIntentArtifactMetadata, @@ -169,7 +170,7 @@ describe("development-loop PR stage", () => { title: "PR creation path", }, }); - if (created.mode !== "created") { + if (created.mode !== "dispatched") { throw new Error("Expected a persisted development-loop run."); } @@ -632,13 +633,10 @@ describe("development-loop PR stage", () => { status: "failed", }); - await retryDevelopmentLoopStep({ - database: transitionDatabase(context), - metrics, - occurredAt: new Date("2026-07-09T20:11:00.000Z"), - reason: "github_pr_creation_failed", - runId, - stage: "pr", + await runDevelopmentLoopRetrySupervisorTick({ + clock: () => new Date("2026-07-09T20:10:30.000Z"), + database: context.db as unknown as DevelopmentLoopRunDatabase, + manifest: defaultLoopManifest, }); await expect( executeDevelopmentLoopPrStage({ diff --git a/tests/unit/loops/research-run.test.ts b/tests/unit/loops/research-run.test.ts index 632e441..3ea5804 100644 --- a/tests/unit/loops/research-run.test.ts +++ b/tests/unit/loops/research-run.test.ts @@ -211,13 +211,19 @@ describe("spike agent-ready research loop run skeleton", () => { expect(await context.db.select().from(loopRuns)).toHaveLength(1); expect(await context.db.select().from(runSteps)).toHaveLength(4); expect(await context.db.select().from(artifacts)).toHaveLength(4); - expect(await context.db.select().from(idempotencyLocks)).toEqual([ - expect.objectContaining({ - key: "research-loop:run:issue-43-delivery", - scope: "research-loop", - status: "released", - }), - ]); + expect(await context.db.select().from(idempotencyLocks)).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + key: "research-loop:run:issue-43-delivery", + scope: "research-loop", + status: "released", + }), + expect.objectContaining({ + scope: "loop:dispatch:issue-guard", + status: "released", + }), + ]), + ); await expect( createResearchLoopRun({ diff --git a/tests/unit/loops/test-writing-transition.test.ts b/tests/unit/loops/test-writing-transition.test.ts index 88a7546..918fc51 100644 --- a/tests/unit/loops/test-writing-transition.test.ts +++ b/tests/unit/loops/test-writing-transition.test.ts @@ -146,7 +146,7 @@ describe("test-writing stage transition", () => { title: "Test-writing subagent", }, }); - if (created.mode !== "created") throw new Error("Expected created run."); + if (created.mode !== "dispatched") throw new Error("Expected dispatched run."); const [approval] = await context.db .select() diff --git a/tests/unit/loops/validation-review-transition.test.ts b/tests/unit/loops/validation-review-transition.test.ts index 7dcac80..b72197e 100644 --- a/tests/unit/loops/validation-review-transition.test.ts +++ b/tests/unit/loops/validation-review-transition.test.ts @@ -71,7 +71,7 @@ describe("validation review transition", () => { title: "Validation review subagent", }, }); - if (created.mode !== "created") throw new Error("Expected created run."); + if (created.mode !== "dispatched") throw new Error("Expected dispatched run."); const [approval] = await context.db .select() .from(approvals) diff --git a/tests/unit/observability/trace-context.test.ts b/tests/unit/observability/trace-context.test.ts index 27150fe..3668e88 100644 --- a/tests/unit/observability/trace-context.test.ts +++ b/tests/unit/observability/trace-context.test.ts @@ -1,7 +1,11 @@ /** @vitest-environment node */ -import type { Span, Tracer } from "@opentelemetry/api"; +import { trace, type Context, type Span, type Tracer } from "@opentelemetry/api"; -import { startLoopworksSpan } from "@/lib/observability/trace-context"; +import { + startDevelopmentLoopDispatchSpan, + startDevelopmentLoopRetrySpan, + startLoopworksSpan, +} from "@/lib/observability/trace-context"; describe("Loopworks trace context helpers", () => { it("starts spans through the centralized Loopworks tracer helper", () => { @@ -36,4 +40,42 @@ describe("Loopworks trace context helpers", () => { }, ]); }); + + it("owns stable dispatch and retry span contracts centrally", () => { + const span = { setAttribute: vi.fn() } as unknown as Span; + const names: string[] = []; + const tracer = { + startSpan(name: string) { + names.push(name); + return span; + }, + } as unknown as Tracer; + + startDevelopmentLoopDispatchSpan(tracer).setOutcome("deferred"); + startDevelopmentLoopRetrySpan(tracer).setOutcome("scheduled"); + + expect(names).toEqual(["loopworks.run.dispatch", "loopworks.run.retry"]); + expect(span.setAttribute).toHaveBeenNthCalledWith(1, "loopworks.dispatch.outcome", "deferred"); + expect(span.setAttribute).toHaveBeenNthCalledWith(2, "loopworks.retry.outcome", "scheduled"); + }); + + it("binds dispatch and retry spans to a supplied durable trace id", () => { + const traceId = "4bf92f3577b34da6a3ce929d0e0e4736"; + const span = { setAttribute: vi.fn() } as unknown as Span; + const parents: Context[] = []; + const tracer = { + startSpan(_name: string, _options?: unknown, parent?: Context) { + if (parent) parents.push(parent); + return span; + }, + } as unknown as Tracer; + + startDevelopmentLoopDispatchSpan({ traceId, tracer }); + startDevelopmentLoopRetrySpan({ traceId, tracer }); + + expect(parents.map((parent) => trace.getSpanContext(parent)?.traceId)).toEqual([ + traceId, + traceId, + ]); + }); });