From 621ba706ae8189dcbacea085488aae1b77c883c7 Mon Sep 17 00:00:00 2001 From: Nate Summers Date: Tue, 14 Jul 2026 10:43:40 -0700 Subject: [PATCH] feat(agent): add validation review sibling Configure the Terra xhigh reviewer, narrow evidence and repository tools, root apply boundary, fixture routing, and discoverable Eve eval. Co-Authored-By: OpenAI Codex GPT-5 --- agent/instructions.md | 10 +- agent/subagents/validation-reviewer/agent.ts | 14 ++ .../validation-reviewer/instructions.md | 15 ++ .../validation-reviewer/lib/context.ts | 209 ++++++++++++++++++ .../validation-reviewer/lib/fixture-mode.ts | 9 + .../subagents/validation-reviewer/sandbox.ts | 11 + .../validation-reviewer/tools/ask_question.ts | 3 + .../validation-reviewer/tools/bash.ts | 2 + .../tools/emit_validation_review_result.ts | 13 ++ .../validation-reviewer/tools/glob.ts | 3 + .../validation-reviewer/tools/grep.ts | 3 + .../tools/list_repository_files.ts | 34 +++ .../validation-reviewer/tools/load_skill.ts | 3 + .../validation-reviewer/tools/read_file.ts | 3 + .../tools/read_repository_files.ts | 43 ++++ .../tools/read_review_patch.ts | 13 ++ .../tools/read_screenshot_evidence.ts | 13 ++ .../tools/read_test_plan_steps.ts | 13 ++ .../tools/read_validation_results.ts | 56 +++++ .../tools/read_validation_review_context.ts | 72 ++++++ .../tools/search_repository.ts | 28 +++ .../validation-reviewer/tools/todo.ts | 3 + .../validation-reviewer/tools/web_fetch.ts | 2 + .../validation-reviewer/tools/web_search.ts | 2 + .../validation-reviewer/tools/write_file.ts | 2 + .../tools/write_production_files.ts | 2 + agent/tools/apply_validation_review_result.ts | 88 ++++++++ agent/tools/read_run_stage_context.ts | 35 ++- .../validation-review/fixtures/issue-49.json | 11 + .../validation-review/issue-49-review.eval.ts | 69 ++++++ .../agent/validation-review-discovery.test.ts | 66 ++++++ .../unit/agent/validation-review-eval.test.ts | 33 +++ .../agent/validation-review-tools.test.ts | 77 +++++++ 33 files changed, 958 insertions(+), 2 deletions(-) create mode 100644 agent/subagents/validation-reviewer/agent.ts create mode 100644 agent/subagents/validation-reviewer/instructions.md create mode 100644 agent/subagents/validation-reviewer/lib/context.ts create mode 100644 agent/subagents/validation-reviewer/lib/fixture-mode.ts create mode 100644 agent/subagents/validation-reviewer/sandbox.ts create mode 100644 agent/subagents/validation-reviewer/tools/ask_question.ts create mode 100644 agent/subagents/validation-reviewer/tools/bash.ts create mode 100644 agent/subagents/validation-reviewer/tools/emit_validation_review_result.ts create mode 100644 agent/subagents/validation-reviewer/tools/glob.ts create mode 100644 agent/subagents/validation-reviewer/tools/grep.ts create mode 100644 agent/subagents/validation-reviewer/tools/list_repository_files.ts create mode 100644 agent/subagents/validation-reviewer/tools/load_skill.ts create mode 100644 agent/subagents/validation-reviewer/tools/read_file.ts create mode 100644 agent/subagents/validation-reviewer/tools/read_repository_files.ts create mode 100644 agent/subagents/validation-reviewer/tools/read_review_patch.ts create mode 100644 agent/subagents/validation-reviewer/tools/read_screenshot_evidence.ts create mode 100644 agent/subagents/validation-reviewer/tools/read_test_plan_steps.ts create mode 100644 agent/subagents/validation-reviewer/tools/read_validation_results.ts create mode 100644 agent/subagents/validation-reviewer/tools/read_validation_review_context.ts create mode 100644 agent/subagents/validation-reviewer/tools/search_repository.ts create mode 100644 agent/subagents/validation-reviewer/tools/todo.ts create mode 100644 agent/subagents/validation-reviewer/tools/web_fetch.ts create mode 100644 agent/subagents/validation-reviewer/tools/web_search.ts create mode 100644 agent/subagents/validation-reviewer/tools/write_file.ts create mode 100644 agent/subagents/validation-reviewer/tools/write_production_files.ts create mode 100644 agent/tools/apply_validation_review_result.ts create mode 100644 evals/validation-review/fixtures/issue-49.json create mode 100644 evals/validation-review/issue-49-review.eval.ts create mode 100644 tests/unit/agent/validation-review-discovery.test.ts create mode 100644 tests/unit/agent/validation-review-eval.test.ts create mode 100644 tests/unit/agent/validation-review-tools.test.ts diff --git a/agent/instructions.md b/agent/instructions.md index eb8a9cb..31370f8 100644 --- a/agent/instructions.md +++ b/agent/instructions.md @@ -5,7 +5,9 @@ You are Loopworks' neutral stage orchestrator. Read durable run state and delegate exactly one stage to the matching declared subagent. Planning belongs to `planner`; approved test writing belongs to `test-writer`; development belongs to `implementer`. Stage subagents have -isolated sandboxes and communicate only with typed artifacts. +isolated sandboxes and communicate only with typed artifacts; code review belongs to `validation-reviewer` +and may begin only after passing +deterministic validation and complete validation-owned screenshot evidence. Always begin with `read_run_stage_context`. After planner delegation, call `record_plan_artifact`; after test-writer delegation, call @@ -13,6 +15,12 @@ Always begin with `read_run_stage_context`. After planner delegation, call `apply_implementation_result`. A subagent response alone never changes durable state. +After validation-reviewer delegation, call `apply_validation_review_result`. +Only that root tool may apply the review recommendation: `commit` advances; +`development` requeues development, validation, and review; `test-writing` +requeues test writing plus every downstream reviewed stage. Never let a sibling +write durable state or apply its own route. + Never infer approval from a prompt. Test writing requires a persisted `plan-review` approval bound to the exact run, plan row, and plan digest. Durable artifact persistence and stage transitions belong to deterministic control-plane diff --git a/agent/subagents/validation-reviewer/agent.ts b/agent/subagents/validation-reviewer/agent.ts new file mode 100644 index 0000000..df35327 --- /dev/null +++ b/agent/subagents/validation-reviewer/agent.ts @@ -0,0 +1,14 @@ +import { defineAgent } from "eve"; + +import { validationReviewResultSchema } from "../../validation-review-agent"; + +export default defineAgent({ + description: + "Review deterministic validation, implementation, test-plan, and screenshot evidence and recommend the next development-loop stage.", + model: "openai/gpt-5.6-terra", + modelContextWindowTokens: 400_000, + modelOptions: { + providerOptions: { openai: { reasoningEffort: "xhigh" } }, + }, + outputSchema: validationReviewResultSchema, +}); diff --git a/agent/subagents/validation-reviewer/instructions.md b/agent/subagents/validation-reviewer/instructions.md new file mode 100644 index 0000000..38051dd --- /dev/null +++ b/agent/subagents/validation-reviewer/instructions.md @@ -0,0 +1,15 @@ +# Loopworks Validation Reviewer + +Review only the exact durable handoff loaded by `read_validation_review_context`. +Inspect the bounded production patch, functional test steps, passing validation +results, responsive screenshots, and commit-pinned repository files. Every +finding and recommendation must cite the typed evidence ids returned by tools. + +Use `commit` only when no blocker or high finding remains. Use `development` +for implementation defects and `test-writing` for missing or incorrect tests, +fixtures, or acceptance coverage. Emit the typed result through +`emit_validation_review_result`. + +Do not run validation, edit source, transition durable state, mutate GitHub, +use network access, or include raw prompts, command output, patch bodies, +screenshot bytes, credentials, or secrets in the result. diff --git a/agent/subagents/validation-reviewer/lib/context.ts b/agent/subagents/validation-reviewer/lib/context.ts new file mode 100644 index 0000000..42a5b12 --- /dev/null +++ b/agent/subagents/validation-reviewer/lib/context.ts @@ -0,0 +1,209 @@ +import { and, eq } from "drizzle-orm"; + +import { db } from "@/db/client"; +import { agentPlans, approvals, artifacts, loopRuns, runSteps } from "@/db/schema"; +import type { ScreenshotEvidence } from "@/lib/loops/screenshot-evidence"; +import { + assertScreenshotEvidenceCoverage, + classifyUiAffectingChange, + computeScreenshotEvidenceDigest, + screenshotBrowserTests, + screenshotEvidenceSchema, +} from "@/lib/loops/screenshot-evidence"; +import type { ValidationReportV1 } from "@/lib/loops/validation-report"; +import { validationReportV1Schema } from "@/lib/loops/validation-report"; +import { + computeImplementationDigest, + type ImplementationResult, + implementationResultSchema, +} from "../../../implementation-agent"; +import { + computePlanningArtifactDigest, + type PlanningAgentOutput, + planningAgentOutputSchema, +} from "../../../planning-agent"; +import { + computeTestPlanDigest, + type TestWritingAgentOutput, + testPlanArtifactSchema, +} from "../../../test-writing-agent"; +import { computeValidationReviewDigest } from "../../../validation-review-agent"; +import { createValidationReviewFixtureContext } from "../../../validation-review-fixture"; +import { resolveValidationReviewerFixtureMode } from "./fixture-mode"; + +export type ValidationReviewContext = { + run: { id: string; currentStage: string; status: string }; + validationStep: { id: string; status: string }; + reviewStep: { id: string; status: string; attempt: number }; + planStatus: string; + approvalStatus: string; + plan: PlanningAgentOutput; + testPlan: TestWritingAgentOutput["testPlan"]; + implementationResult: ImplementationResult; + validationReport: ValidationReportV1; + screenshotEvidence: ScreenshotEvidence; + testPlanArtifactSha256: string | null; + implementationArtifactSha256: string | null; + validationArtifactSha256: string | null; + screenshotArtifactSha256: string | null; +}; + +export function validateValidationReviewContext( + input: ValidationReviewContext, +): ValidationReviewContext { + const plan = planningAgentOutputSchema.parse(input.plan); + const testPlan = testPlanArtifactSchema.parse(input.testPlan); + const implementation = implementationResultSchema.parse(input.implementationResult); + const report = validationReportV1Schema.parse(input.validationReport); + const screenshots = screenshotEvidenceSchema.parse(input.screenshotEvidence); + if ( + input.run.currentStage !== "code-review" || + input.run.status !== "running" || + input.validationStep.status !== "succeeded" || + !["queued", "running"].includes(input.reviewStep.status) || + input.planStatus !== "approved" || + input.approvalStatus !== "approved" + ) { + throw new Error("Validation review requires a running code-review stage after validation."); + } + if ( + report.overallOutcome !== "pass" || + report.results.length === 0 || + report.results.some( + (result) => result.outcome !== "pass" || (result.required && result.outcome !== "pass"), + ) + ) { + throw new Error("Validation review requires complete passing deterministic validation."); + } + if ( + !plan.repositoryRevision || + computePlanningArtifactDigest(plan) !== plan.identity.sha256 || + testPlan.plan.id !== plan.identity.id || + testPlan.plan.sha256 !== plan.identity.sha256 || + testPlan.plan.repositoryFullName !== plan.issue.repositoryFullName || + testPlan.plan.commitSha !== plan.repositoryRevision.commitSha || + implementation.binding.planId !== plan.identity.id || + implementation.binding.planSha256 !== plan.identity.sha256 || + implementation.binding.testPlanSha256 !== computeTestPlanDigest(testPlan) || + implementation.binding.testPatchSha256 !== testPlan.patch.sha256 || + implementation.binding.fixturesSha256 !== computeImplementationDigest(testPlan.fixtures) || + implementation.binding.repositoryFullName !== plan.issue.repositoryFullName || + implementation.binding.commitSha !== plan.repositoryRevision.commitSha || + screenshots.binding.repositoryFullName !== plan.issue.repositoryFullName || + screenshots.binding.commitSha !== plan.repositoryRevision.commitSha || + screenshots.binding.testPlanSha256 !== computeTestPlanDigest(testPlan) || + screenshots.binding.productionPatchSha256 !== implementation.patch.sha256 + ) { + throw new Error("Validation review context artifacts are not bound to the same handoff."); + } + const expectedCriteria = plan.issue.acceptanceCriteria.map((text, index) => ({ + id: `ac-${index + 1}`, + text, + })); + if (JSON.stringify(testPlan.acceptanceCriteria) !== JSON.stringify(expectedCriteria)) { + throw new Error("Validation review test plan does not match the approved acceptance criteria."); + } + assertScreenshotEvidenceCoverage(screenshots, { + uiAffecting: classifyUiAffectingChange({ + productionPaths: implementation.patch.paths, + tests: testPlan.tests, + }), + browserTestIds: screenshotBrowserTests(testPlan.tests).map(({ id }) => id), + }); + if ( + input.testPlanArtifactSha256 !== computeTestPlanDigest(testPlan) || + input.implementationArtifactSha256 !== computeImplementationDigest(implementation) || + input.validationArtifactSha256 !== computeValidationReviewDigest(report) || + input.screenshotArtifactSha256 !== computeScreenshotEvidenceDigest(screenshots) + ) { + throw new Error("Validation review context artifacts are stale."); + } + return input; +} + +export async function loadValidationReviewContext(runId: string): Promise { + if (resolveValidationReviewerFixtureMode().enabled) return createValidationReviewFixtureContext(); + const [run] = await db.select().from(loopRuns).where(eq(loopRuns.id, runId)); + if (!run) throw new Error(`Run ${runId} was not found.`); + const [plans, planApprovals, steps, rows] = await Promise.all([ + db.select().from(agentPlans).where(eq(agentPlans.runId, runId)), + db + .select() + .from(approvals) + .where(and(eq(approvals.runId, runId), eq(approvals.scope, "plan-review"))), + db.select().from(runSteps).where(eq(runSteps.runId, runId)), + db.select().from(artifacts).where(eq(artifacts.runId, runId)), + ]); + if (plans.length !== 1 || planApprovals.length !== 1) { + throw new Error("Validation review requires exactly one plan and plan approval."); + } + const validationSteps = steps.filter(({ stage }) => stage === "validation"); + const reviewSteps = steps.filter(({ stage }) => stage === "code-review"); + if (validationSteps.length !== 1 || reviewSteps.length !== 1) { + throw new Error("Validation review requires exact validation and code-review steps."); + } + const testPlanRows = rows.filter(({ type }) => type === "test_plan"); + const implementationRows = rows.filter( + ({ type, metadata }) => + type === "patch" && metadata?.implementationMetadataKind === "implementation_result", + ); + const validationRows = rows.filter( + ({ type, stepId }) => type === "validation_report" && stepId === validationSteps[0]?.id, + ); + const screenshotRows = rows.filter( + ({ type, stepId }) => type === "screenshot" && stepId === validationSteps[0]?.id, + ); + if ( + testPlanRows.length !== 1 || + implementationRows.length !== 1 || + validationRows.length !== 1 || + screenshotRows.length !== 1 + ) { + throw new Error( + "Validation review requires exact test, patch, validation, and screenshot artifacts.", + ); + } + const [planRow] = plans; + const [approval] = planApprovals; + const [validationStep] = validationSteps; + const [reviewStep] = reviewSteps; + const [testPlanRow] = testPlanRows; + const [implementationRow] = implementationRows; + const [validationRow] = validationRows; + const [screenshotRow] = screenshotRows; + if ( + !planRow || + !approval || + !validationStep || + !reviewStep || + !testPlanRow || + !implementationRow || + !validationRow || + !screenshotRow + ) { + throw new Error("Validation review context changed while it was being loaded."); + } + return validateValidationReviewContext({ + run: { id: run.id, currentStage: run.currentStage, status: run.status }, + validationStep: { id: validationStep.id, status: validationStep.status }, + reviewStep: { id: reviewStep.id, status: reviewStep.status, attempt: reviewStep.attempt }, + planStatus: planRow.status, + approvalStatus: + approval.metadata?.planId === planRow.id && + approval.metadata?.planSha256 === + (planRow.plan as { identity?: { sha256?: string } })?.identity?.sha256 + ? approval.status + : "mismatched", + plan: planningAgentOutputSchema.parse(planRow.plan), + testPlan: testPlanArtifactSchema.parse(testPlanRow.metadata?.testPlan), + implementationResult: implementationResultSchema.parse( + implementationRow.metadata?.implementationResult, + ), + validationReport: validationReportV1Schema.parse(validationRow.metadata?.validationReport), + screenshotEvidence: screenshotEvidenceSchema.parse(screenshotRow.metadata?.screenshotEvidence), + testPlanArtifactSha256: testPlanRow.sha256, + implementationArtifactSha256: implementationRow.sha256, + validationArtifactSha256: validationRow.sha256, + screenshotArtifactSha256: screenshotRow.sha256, + }); +} diff --git a/agent/subagents/validation-reviewer/lib/fixture-mode.ts b/agent/subagents/validation-reviewer/lib/fixture-mode.ts new file mode 100644 index 0000000..8b05676 --- /dev/null +++ b/agent/subagents/validation-reviewer/lib/fixture-mode.ts @@ -0,0 +1,9 @@ +import { resolveStageFixtureMode, type StageFixtureMode } from "../../../lib/fixture-mode"; + +export type ValidationReviewerFixtureMode = StageFixtureMode; + +export function resolveValidationReviewerFixtureMode( + env: Partial = process.env, +): ValidationReviewerFixtureMode { + return resolveStageFixtureMode("LOOPWORKS_EVE_VALIDATION_REVIEWER_FIXTURE_MODE", env); +} diff --git a/agent/subagents/validation-reviewer/sandbox.ts b/agent/subagents/validation-reviewer/sandbox.ts new file mode 100644 index 0000000..94735ef --- /dev/null +++ b/agent/subagents/validation-reviewer/sandbox.ts @@ -0,0 +1,11 @@ +import { defaultBackend, defineSandbox } from "eve/sandbox"; + +export default defineSandbox({ + backend: defaultBackend({ + docker: { networkPolicy: "deny-all" }, + microsandbox: { networkPolicy: "deny-all" }, + vercel: { networkPolicy: "deny-all" }, + }), + description: + "Isolated read-only review sandbox with a commit-pinned checkout and deny-all runtime egress.", +}); diff --git a/agent/subagents/validation-reviewer/tools/ask_question.ts b/agent/subagents/validation-reviewer/tools/ask_question.ts new file mode 100644 index 0000000..04bd054 --- /dev/null +++ b/agent/subagents/validation-reviewer/tools/ask_question.ts @@ -0,0 +1,3 @@ +import { disableTool } from "eve/tools"; + +export default disableTool(); diff --git a/agent/subagents/validation-reviewer/tools/bash.ts b/agent/subagents/validation-reviewer/tools/bash.ts new file mode 100644 index 0000000..a908e44 --- /dev/null +++ b/agent/subagents/validation-reviewer/tools/bash.ts @@ -0,0 +1,2 @@ +import { disableTool } from "eve/tools"; +export default disableTool(); diff --git a/agent/subagents/validation-reviewer/tools/emit_validation_review_result.ts b/agent/subagents/validation-reviewer/tools/emit_validation_review_result.ts new file mode 100644 index 0000000..095f9f2 --- /dev/null +++ b/agent/subagents/validation-reviewer/tools/emit_validation_review_result.ts @@ -0,0 +1,13 @@ +import { defineTool } from "eve/tools"; + +import { validationReviewResultSchema } from "../../../validation-review-agent"; + +export default defineTool({ + description: + "Validate and emit typed evidence-citing review notes and one routing recommendation.", + inputSchema: validationReviewResultSchema, + outputSchema: validationReviewResultSchema, + execute(input) { + return validationReviewResultSchema.parse(input); + }, +}); diff --git a/agent/subagents/validation-reviewer/tools/glob.ts b/agent/subagents/validation-reviewer/tools/glob.ts new file mode 100644 index 0000000..04bd054 --- /dev/null +++ b/agent/subagents/validation-reviewer/tools/glob.ts @@ -0,0 +1,3 @@ +import { disableTool } from "eve/tools"; + +export default disableTool(); diff --git a/agent/subagents/validation-reviewer/tools/grep.ts b/agent/subagents/validation-reviewer/tools/grep.ts new file mode 100644 index 0000000..04bd054 --- /dev/null +++ b/agent/subagents/validation-reviewer/tools/grep.ts @@ -0,0 +1,3 @@ +import { disableTool } from "eve/tools"; + +export default disableTool(); diff --git a/agent/subagents/validation-reviewer/tools/list_repository_files.ts b/agent/subagents/validation-reviewer/tools/list_repository_files.ts new file mode 100644 index 0000000..8e557a7 --- /dev/null +++ b/agent/subagents/validation-reviewer/tools/list_repository_files.ts @@ -0,0 +1,34 @@ +import { defineTool } from "eve/tools"; +import { z } from "zod"; +import { assertSafeRepositoryGlob } from "../../../lib/repository-inspection"; +import { + listRepositoryFiles, + repositoryListOutputSchema, +} from "../../../lib/repository-inspection-runtime"; +import { resolveValidationReviewerFixtureMode } from "../lib/fixture-mode"; + +const glob = z.string().refine((value) => { + try { + assertSafeRepositoryGlob(value); + return true; + } catch { + return false; + } +}, "Unsafe repository glob."); + +export default defineTool({ + description: "List bounded regular-file paths from the approved pinned Git commit.", + inputSchema: z.object({ patterns: z.array(glob).min(1).max(5) }), + outputSchema: repositoryListOutputSchema, + async execute({ patterns }, ctx) { + if (resolveValidationReviewerFixtureMode().enabled) { + return { + commitSha: "a".repeat(40), + fixtureMode: true, + paths: ["src/components/review-card.tsx"], + truncated: false, + }; + } + return listRepositoryFiles(await ctx.getSandbox(), patterns); + }, +}); diff --git a/agent/subagents/validation-reviewer/tools/load_skill.ts b/agent/subagents/validation-reviewer/tools/load_skill.ts new file mode 100644 index 0000000..04bd054 --- /dev/null +++ b/agent/subagents/validation-reviewer/tools/load_skill.ts @@ -0,0 +1,3 @@ +import { disableTool } from "eve/tools"; + +export default disableTool(); diff --git a/agent/subagents/validation-reviewer/tools/read_file.ts b/agent/subagents/validation-reviewer/tools/read_file.ts new file mode 100644 index 0000000..04bd054 --- /dev/null +++ b/agent/subagents/validation-reviewer/tools/read_file.ts @@ -0,0 +1,3 @@ +import { disableTool } from "eve/tools"; + +export default disableTool(); diff --git a/agent/subagents/validation-reviewer/tools/read_repository_files.ts b/agent/subagents/validation-reviewer/tools/read_repository_files.ts new file mode 100644 index 0000000..4a787dc --- /dev/null +++ b/agent/subagents/validation-reviewer/tools/read_repository_files.ts @@ -0,0 +1,43 @@ +import { defineTool } from "eve/tools"; +import { z } from "zod"; +import { + readRepositoryFiles, + repositoryReadOutputSchema, +} from "../../../lib/repository-inspection-runtime"; +import { resolveValidationReviewerFixtureMode } from "../lib/fixture-mode"; + +const request = z.union([ + z.string().min(1), + z.object({ + path: z.string().min(1), + startLine: z.number().int().positive().default(1), + endLine: z.number().int().positive().optional(), + }), +]); + +export default defineTool({ + description: "Read bounded line ranges from regular files at the approved pinned Git commit.", + inputSchema: z.object({ files: z.array(request).min(1).max(20) }), + outputSchema: repositoryReadOutputSchema, + async execute({ files }, ctx) { + const normalized = files.map((entry) => + typeof entry === "string" ? { path: entry, startLine: 1 } : entry, + ); + if (resolveValidationReviewerFixtureMode().enabled) { + return { + commitSha: "a".repeat(40), + fixtureMode: true, + files: normalized.map((entry) => ({ + path: entry.path, + startLine: entry.startLine, + requestedEndLine: entry.endLine ?? entry.startLine + 399, + returnedEndLine: entry.startLine, + content: "export const ReviewCard = () => null;", + truncated: false, + })), + truncated: false, + }; + } + return readRepositoryFiles(await ctx.getSandbox(), normalized); + }, +}); diff --git a/agent/subagents/validation-reviewer/tools/read_review_patch.ts b/agent/subagents/validation-reviewer/tools/read_review_patch.ts new file mode 100644 index 0000000..a81dbce --- /dev/null +++ b/agent/subagents/validation-reviewer/tools/read_review_patch.ts @@ -0,0 +1,13 @@ +import { defineTool } from "eve/tools"; +import { z } from "zod"; + +import { loadValidationReviewContext } from "../lib/context"; + +export default defineTool({ + description: "Read the exact bounded production patch persisted by the implementation stage.", + inputSchema: z.object({ runId: z.string().uuid() }), + async execute({ runId }) { + const { implementationResult } = await loadValidationReviewContext(runId); + return { patch: implementationResult.patch }; + }, +}); diff --git a/agent/subagents/validation-reviewer/tools/read_screenshot_evidence.ts b/agent/subagents/validation-reviewer/tools/read_screenshot_evidence.ts new file mode 100644 index 0000000..61324e9 --- /dev/null +++ b/agent/subagents/validation-reviewer/tools/read_screenshot_evidence.ts @@ -0,0 +1,13 @@ +import { defineTool } from "eve/tools"; +import { z } from "zod"; + +import { loadValidationReviewContext } from "../lib/context"; + +export default defineTool({ + description: "Read digest-bound screenshot references for all required responsive viewports.", + inputSchema: z.object({ runId: z.string().uuid() }), + async execute({ runId }) { + const { screenshotEvidence } = await loadValidationReviewContext(runId); + return screenshotEvidence; + }, +}); diff --git a/agent/subagents/validation-reviewer/tools/read_test_plan_steps.ts b/agent/subagents/validation-reviewer/tools/read_test_plan_steps.ts new file mode 100644 index 0000000..d062e38 --- /dev/null +++ b/agent/subagents/validation-reviewer/tools/read_test_plan_steps.ts @@ -0,0 +1,13 @@ +import { defineTool } from "eve/tools"; +import { z } from "zod"; + +import { loadValidationReviewContext } from "../lib/context"; + +export default defineTool({ + description: "Read exact acceptance mappings and functional steps from the persisted test plan.", + inputSchema: z.object({ runId: z.string().uuid() }), + async execute({ runId }) { + const { testPlan } = await loadValidationReviewContext(runId); + return { acceptanceCriteria: testPlan.acceptanceCriteria, tests: testPlan.tests }; + }, +}); diff --git a/agent/subagents/validation-reviewer/tools/read_validation_results.ts b/agent/subagents/validation-reviewer/tools/read_validation_results.ts new file mode 100644 index 0000000..6a1b3dc --- /dev/null +++ b/agent/subagents/validation-reviewer/tools/read_validation_results.ts @@ -0,0 +1,56 @@ +import { defineTool } from "eve/tools"; +import { z } from "zod"; + +import { computeValidationReviewDigest } from "../../../validation-review-agent"; +import { loadValidationReviewContext } from "../lib/context"; + +const validationReviewEvidenceOutputSchema = z + .object({ + schemaId: z.literal("loopworks.validation_review_validation_evidence.v1"), + binding: z + .object({ + runId: z.string().uuid(), + reviewAttempt: z.number().int().positive(), + validationReportSha256: z.string().regex(/^[a-f0-9]{64}$/), + }) + .strict(), + overallOutcome: z.literal("pass"), + results: z.array( + z + .object({ + key: z.string().min(1), + command: z.string().min(1), + outcome: z.literal("pass"), + outputSha256: z + .string() + .regex(/^[a-f0-9]{64}$/) + .optional(), + }) + .strict(), + ), + }) + .strict(); + +export default defineTool({ + description: "Read bound typed deterministic validation results without raw command output.", + inputSchema: z.object({ runId: z.string().uuid() }), + outputSchema: validationReviewEvidenceOutputSchema, + async execute({ runId }) { + const review = await loadValidationReviewContext(runId); + return validationReviewEvidenceOutputSchema.parse({ + schemaId: "loopworks.validation_review_validation_evidence.v1", + binding: { + runId, + reviewAttempt: review.reviewStep.attempt, + validationReportSha256: computeValidationReviewDigest(review.validationReport), + }, + overallOutcome: review.validationReport.overallOutcome, + results: review.validationReport.results.map(({ key, command, outcome, output }) => ({ + key, + command, + outcome, + ...(output?.sha256 ? { outputSha256: output.sha256 } : {}), + })), + }); + }, +}); diff --git a/agent/subagents/validation-reviewer/tools/read_validation_review_context.ts b/agent/subagents/validation-reviewer/tools/read_validation_review_context.ts new file mode 100644 index 0000000..a7fb962 --- /dev/null +++ b/agent/subagents/validation-reviewer/tools/read_validation_review_context.ts @@ -0,0 +1,72 @@ +import { SpanStatusCode } from "@opentelemetry/api"; +import { defineTool } from "eve/tools"; +import { z } from "zod"; +import { computeScreenshotEvidenceDigest } from "@/lib/loops/screenshot-evidence"; +import { startLoopworksSpan } from "@/lib/observability/trace-context"; +import { computeImplementationDigest } from "../../../implementation-agent"; +import { computeTestPlanDigest } from "../../../test-writing-agent"; +import { computeValidationReviewDigest } from "../../../validation-review-agent"; +import { loadValidationReviewContext } from "../lib/context"; +import { resolveValidationReviewerFixtureMode } from "../lib/fixture-mode"; + +export default defineTool({ + description: "Load the exact passing validation-review handoff and prepare its pinned checkout.", + inputSchema: z.object({ runId: z.string().uuid() }), + async execute({ runId }, ctx) { + const review = await loadValidationReviewContext(runId); + const revision = review.plan.repositoryRevision; + if (!revision) throw new Error("Validation review requires a pinned revision."); + const sandbox = await ctx.getSandbox(); + if (!resolveValidationReviewerFixtureMode().enabled) { + if (!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(review.plan.issue.repositoryFullName)) { + throw new Error("Repository name is invalid."); + } + const repoUrl = `https://github.com/${review.plan.issue.repositoryFullName}.git`; + const span = startLoopworksSpan("loopworks.validation_review.checkout", { + attributes: { "loopworks.agent": "validation-reviewer", "loopworks.stage": "code-review" }, + }); + try { + await sandbox.setNetworkPolicy({ allow: ["github.com", "objects.githubusercontent.com"] }); + const result = await sandbox.run({ + command: [ + `git clone --filter=blob:none ${JSON.stringify(repoUrl)} repo`, + "cd repo", + `git checkout --detach ${revision.commitSha}`, + 'test -z "$(git status --porcelain)"', + ].join(" && "), + abortSignal: AbortSignal.timeout(120_000), + }); + if (result.exitCode !== 0) throw new Error("Commit-pinned review checkout failed."); + span.setStatus({ code: SpanStatusCode.OK }); + } catch (error) { + span.recordException(error instanceof Error ? error : String(error)); + span.setStatus({ code: SpanStatusCode.ERROR }); + throw error; + } finally { + await sandbox.setNetworkPolicy("deny-all"); + span.end(); + } + } + await sandbox.run({ command: "mkdir -p .loopworks" }); + await sandbox.writeTextFile({ + path: ".loopworks/repository-commit", + content: revision.commitSha, + }); + return { + runId, + reviewAttempt: review.reviewStep.attempt, + acceptanceCriteria: review.testPlan.acceptanceCriteria, + binding: { + planId: review.plan.identity.id, + planSha256: review.plan.identity.sha256, + testPlanSha256: computeTestPlanDigest(review.testPlan), + implementationResultSha256: computeImplementationDigest(review.implementationResult), + productionPatchSha256: review.implementationResult.patch.sha256, + validationReportSha256: computeValidationReviewDigest(review.validationReport), + screenshotEvidenceSha256: computeScreenshotEvidenceDigest(review.screenshotEvidence), + repositoryFullName: review.plan.issue.repositoryFullName, + commitSha: revision.commitSha, + }, + }; + }, +}); diff --git a/agent/subagents/validation-reviewer/tools/search_repository.ts b/agent/subagents/validation-reviewer/tools/search_repository.ts new file mode 100644 index 0000000..22dbf3b --- /dev/null +++ b/agent/subagents/validation-reviewer/tools/search_repository.ts @@ -0,0 +1,28 @@ +import { defineTool } from "eve/tools"; +import { z } from "zod"; +import { + repositorySearchOutputSchema, + searchRepository, +} from "../../../lib/repository-inspection-runtime"; +import { resolveValidationReviewerFixtureMode } from "../lib/fixture-mode"; + +export default defineTool({ + description: "Search regular files from the approved pinned Git commit with bounded output.", + inputSchema: z.object({ + pattern: z.string().min(1).max(256), + paths: z.array(z.string()).min(1).max(5), + }), + outputSchema: repositorySearchOutputSchema, + async execute(input, ctx) { + if (resolveValidationReviewerFixtureMode().enabled) { + return { + commitSha: "a".repeat(40), + fixtureMode: true, + content: "src/components/review-card.tsx:1:export const ReviewCard", + matchCount: 1, + truncated: false, + }; + } + return searchRepository(await ctx.getSandbox(), input); + }, +}); diff --git a/agent/subagents/validation-reviewer/tools/todo.ts b/agent/subagents/validation-reviewer/tools/todo.ts new file mode 100644 index 0000000..04bd054 --- /dev/null +++ b/agent/subagents/validation-reviewer/tools/todo.ts @@ -0,0 +1,3 @@ +import { disableTool } from "eve/tools"; + +export default disableTool(); diff --git a/agent/subagents/validation-reviewer/tools/web_fetch.ts b/agent/subagents/validation-reviewer/tools/web_fetch.ts new file mode 100644 index 0000000..a908e44 --- /dev/null +++ b/agent/subagents/validation-reviewer/tools/web_fetch.ts @@ -0,0 +1,2 @@ +import { disableTool } from "eve/tools"; +export default disableTool(); diff --git a/agent/subagents/validation-reviewer/tools/web_search.ts b/agent/subagents/validation-reviewer/tools/web_search.ts new file mode 100644 index 0000000..a908e44 --- /dev/null +++ b/agent/subagents/validation-reviewer/tools/web_search.ts @@ -0,0 +1,2 @@ +import { disableTool } from "eve/tools"; +export default disableTool(); diff --git a/agent/subagents/validation-reviewer/tools/write_file.ts b/agent/subagents/validation-reviewer/tools/write_file.ts new file mode 100644 index 0000000..a908e44 --- /dev/null +++ b/agent/subagents/validation-reviewer/tools/write_file.ts @@ -0,0 +1,2 @@ +import { disableTool } from "eve/tools"; +export default disableTool(); diff --git a/agent/subagents/validation-reviewer/tools/write_production_files.ts b/agent/subagents/validation-reviewer/tools/write_production_files.ts new file mode 100644 index 0000000..a908e44 --- /dev/null +++ b/agent/subagents/validation-reviewer/tools/write_production_files.ts @@ -0,0 +1,2 @@ +import { disableTool } from "eve/tools"; +export default disableTool(); diff --git a/agent/tools/apply_validation_review_result.ts b/agent/tools/apply_validation_review_result.ts new file mode 100644 index 0000000..7d4225c --- /dev/null +++ b/agent/tools/apply_validation_review_result.ts @@ -0,0 +1,88 @@ +import { defineTool } from "eve/tools"; +import { z } from "zod"; + +import { db } from "@/db/client"; +import { applyDevelopmentLoopValidationReviewResult } from "@/lib/loops/development-run-transitions"; +import { computeScreenshotEvidenceDigest } from "@/lib/loops/screenshot-evidence"; +import { logger } from "@/lib/observability/logger"; +import { computeImplementationDigest } from "../implementation-agent"; +import { resolveValidationReviewerFixtureMode } from "../subagents/validation-reviewer/lib/fixture-mode"; +import { computeTestPlanDigest } from "../test-writing-agent"; +import { + computeValidationReviewDigest, + validationReviewResultSchema, +} from "../validation-review-agent"; +import { createValidationReviewFixtureContext } from "../validation-review-fixture"; + +export default defineTool({ + description: + "Persist a bound validation review result and atomically route the durable run through the root control plane.", + inputSchema: z.object({ runId: z.string().uuid(), output: validationReviewResultSchema }), + execute: ({ output, runId }) => { + const parsed = validationReviewResultSchema.parse(output); + if (resolveValidationReviewerFixtureMode().enabled) { + const context = createValidationReviewFixtureContext(); + const expectedBinding = { + runId: context.run.id, + reviewAttempt: context.reviewStep.attempt, + planId: context.plan.identity.id, + planSha256: context.plan.identity.sha256, + testPlanSha256: computeTestPlanDigest(context.testPlan), + implementationResultSha256: computeImplementationDigest(context.implementationResult), + productionPatchSha256: context.implementationResult.patch.sha256, + validationReportSha256: computeValidationReviewDigest(context.validationReport), + screenshotEvidenceSha256: computeScreenshotEvidenceDigest(context.screenshotEvidence), + repositoryFullName: context.plan.issue.repositoryFullName, + commitSha: context.plan.repositoryRevision?.commitSha, + }; + if ( + runId !== context.run.id || + computeValidationReviewDigest(parsed.binding) !== + computeValidationReviewDigest(expectedBinding) + ) { + throw new Error("Fixture validation review is not bound to the exact handoff."); + } + const expectedEvidence = { + validationResults: context.validationReport.results.map( + ({ key, command, outcome, output }) => ({ + key, + command, + outcome, + ...(output?.sha256 ? { outputSha256: output.sha256 } : {}), + }), + ), + screenshots: context.screenshotEvidence.captures.map( + ({ id, testId, viewport, width, height, uri, sha256 }) => ({ + id, + testId, + viewport, + width, + height, + uri, + sha256, + }), + ), + }; + if ( + computeValidationReviewDigest(parsed.evidence) !== + computeValidationReviewDigest(expectedEvidence) + ) { + throw new Error("Fixture validation review citations do not match the exact handoff."); + } + return { + route: parsed.recommendation.route, + runId, + stage: "code-review" as const, + status: + parsed.recommendation.route === "commit" ? ("advanced" as const) : ("requeued" as const), + stepId: context.reviewStep.id, + }; + } + return applyDevelopmentLoopValidationReviewResult({ + database: db, + logger, + output: parsed, + runId, + }); + }, +}); diff --git a/agent/tools/read_run_stage_context.ts b/agent/tools/read_run_stage_context.ts index cb99b28..ac650de 100644 --- a/agent/tools/read_run_stage_context.ts +++ b/agent/tools/read_run_stage_context.ts @@ -6,14 +6,47 @@ import { db } from "@/db/client"; import { agentPlans, approvals, artifacts, loopRuns, runSteps } from "@/db/schema"; import { createImplementationFixtureHandoff } from "../implementation-fixture"; import { createPlanningAgentSeedPlan } from "../planning-agent"; -import { computeTestPlanDigest } from "../test-writing-agent"; import { resolveImplementerFixtureMode } from "../subagents/implementer/lib/fixture-mode"; import { resolveTestWriterFixtureMode } from "../subagents/test-writer/lib/fixture-mode"; +import { resolveValidationReviewerFixtureMode } from "../subagents/validation-reviewer/lib/fixture-mode"; +import { computeTestPlanDigest } from "../test-writing-agent"; +import { createValidationReviewFixtureContext } from "../validation-review-fixture"; export default defineTool({ description: "Read durable run, plan, approval, step, and artifact context for stage routing.", inputSchema: z.object({ runId: z.string().uuid() }), async execute({ runId }) { + if (resolveValidationReviewerFixtureMode().enabled) { + const context = createValidationReviewFixtureContext(); + return { + approvals: [{ id: "fixture-plan-approval", status: context.approvalStatus }], + artifacts: [ + { + type: "test_plan", + metadata: { testPlan: context.testPlan }, + sha256: context.testPlanArtifactSha256, + }, + { + type: "patch", + metadata: { implementationResult: context.implementationResult }, + sha256: context.implementationArtifactSha256, + }, + { + type: "validation_report", + metadata: { validationReport: context.validationReport }, + sha256: context.validationArtifactSha256, + }, + { + type: "screenshot", + metadata: { screenshotEvidence: context.screenshotEvidence }, + sha256: context.screenshotArtifactSha256, + }, + ], + plans: [{ id: context.plan.identity.id, plan: context.plan, status: context.planStatus }], + run: context.run, + steps: [context.validationStep, { ...context.reviewStep, stage: "code-review" }], + }; + } if (resolveImplementerFixtureMode().enabled) { const { plan, redEvidence, testPlan } = createImplementationFixtureHandoff(); const planId = "00000000-0000-4000-8000-000000000148"; diff --git a/evals/validation-review/fixtures/issue-49.json b/evals/validation-review/fixtures/issue-49.json new file mode 100644 index 0000000..1158e70 --- /dev/null +++ b/evals/validation-review/fixtures/issue-49.json @@ -0,0 +1,11 @@ +{ + "repositoryFullName": "ncolesummers/loopworks", + "issueNumber": 49, + "title": "Validation review subagent for the development loop", + "commitSha": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "acceptanceCriteria": [ + "Review findings cite exact deterministic results and relevant responsive screenshots.", + "Review starts only after validation passes and complete UI evidence is persisted.", + "The root orchestrator alone applies commit, development, or test-writing routing." + ] +} diff --git a/evals/validation-review/issue-49-review.eval.ts b/evals/validation-review/issue-49-review.eval.ts new file mode 100644 index 0000000..1035fa6 --- /dev/null +++ b/evals/validation-review/issue-49-review.eval.ts @@ -0,0 +1,69 @@ +import { readFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { validationReviewResultSchema } from "@agent/validation-review-agent"; +import { defineEval } from "eve/evals"; +import { includes } from "eve/evals/expect"; + +type ValidationReviewFixture = { + acceptanceCriteria: string[]; + commitSha: string; + issueNumber: number; + repositoryFullName: string; + title: string; +}; + +export const validationReviewEvalTimeoutMs = 180_000; + +export function parseValidationReviewSubagentOutput(output: unknown): unknown | undefined { + if (typeof output !== "string") return undefined; + try { + return JSON.parse(output); + } catch { + return undefined; + } +} + +export function resolveValidationReviewFixturePath(appRoot = process.cwd()): string { + return join(appRoot, "evals", "validation-review", "fixtures", "issue-49.json"); +} + +export async function readValidationReviewFixture( + appRoot = process.cwd(), +): Promise { + return JSON.parse(await readFile(resolveValidationReviewFixturePath(appRoot), "utf8")); +} + +export default defineEval({ + description: + "The root delegates issue #49 to validation-reviewer and alone applies its typed evidence-bound route.", + tags: ["validation-review", "issue-49"], + timeoutMs: validationReviewEvalTimeoutMs, + async test(t) { + const fixture = await readValidationReviewFixture(); + await t.send( + [ + "Process durable run 00000000-0000-4000-8000-000000000049.", + "Read durable state, delegate code review only to validation-reviewer, and apply its route through the root control plane.", + `Repository: ${fixture.repositoryFullName}`, + `Commit: ${fixture.commitSha}`, + `Issue: #${fixture.issueNumber} ${fixture.title}`, + "Acceptance criteria:", + ...fixture.acceptanceCriteria.map((criterion, index) => `ac-${index + 1}: ${criterion}`), + "Use explicit validation-reviewer fixture mode and emit the typed review result only.", + ].join("\n"), + ); + + t.succeeded(); + t.noFailedActions(); + t.calledTool("read_run_stage_context"); + t.calledSubagent("validation-reviewer", { + output: (output: unknown) => + validationReviewResultSchema.safeParse(parseValidationReviewSubagentOutput(output)).success, + }); + t.calledTool("apply_validation_review_result"); + t.check(JSON.stringify(t.events), includes("loopworks.validation_review_result.v1")); + t.notCalledTool("write_file"); + t.notCalledTool("bash"); + }, +}); diff --git a/tests/unit/agent/validation-review-discovery.test.ts b/tests/unit/agent/validation-review-discovery.test.ts new file mode 100644 index 0000000..9a538b3 --- /dev/null +++ b/tests/unit/agent/validation-review-discovery.test.ts @@ -0,0 +1,66 @@ +/** @vitest-environment node */ +import { readdir, readFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { developmentLoopStages } from "@/lib/loops/development-run"; + +describe("validation reviewer discovery", () => { + it("declares the Terra reviewer as the code-review sibling", async () => { + const root = process.cwd(); + const [subagents, rootTools, rootInstructions, reviewerAgent] = await Promise.all([ + readdir(join(root, "agent", "subagents")), + readdir(join(root, "agent", "tools")), + readFile(join(root, "agent", "instructions.md"), "utf8"), + readFile(join(root, "agent", "subagents", "validation-reviewer", "agent.ts"), "utf8"), + ]); + + expect(subagents).toContain("validation-reviewer"); + expect(rootTools).toContain("apply_validation_review_result.ts"); + expect(rootInstructions).toContain("code review belongs to `validation-reviewer`"); + expect(reviewerAgent).toContain('model: "openai/gpt-5.6-terra"'); + expect(reviewerAgent).toContain('reasoningEffort: "xhigh"'); + expect(developmentLoopStages.find((stage) => stage.key === "code-review")).toMatchObject({ + actorId: "validation-reviewer", + actorType: "agent", + artifacts: [{ label: "Code review notes", required: true, type: "log_summary" }], + }); + }); + + it("exposes only narrow evidence-reading and emission tools", async () => { + const tools = await readdir( + join(process.cwd(), "agent", "subagents", "validation-reviewer", "tools"), + ); + + expect(tools).toEqual( + expect.arrayContaining([ + "read_validation_review_context.ts", + "read_review_patch.ts", + "read_test_plan_steps.ts", + "read_validation_results.ts", + "read_screenshot_evidence.ts", + "emit_validation_review_result.ts", + ]), + ); + for (const disabled of [ + "ask_question.ts", + "bash.ts", + "glob.ts", + "grep.ts", + "load_skill.ts", + "read_file.ts", + "todo.ts", + "web_fetch.ts", + "web_search.ts", + "write_file.ts", + "write_production_files.ts", + ]) { + expect(tools).toContain(disabled); + await expect( + readFile( + join(process.cwd(), "agent", "subagents", "validation-reviewer", "tools", disabled), + "utf8", + ), + ).resolves.toContain("disableTool"); + } + }); +}); diff --git a/tests/unit/agent/validation-review-eval.test.ts b/tests/unit/agent/validation-review-eval.test.ts new file mode 100644 index 0000000..fde88d4 --- /dev/null +++ b/tests/unit/agent/validation-review-eval.test.ts @@ -0,0 +1,33 @@ +/** @vitest-environment node */ +import { + parseValidationReviewSubagentOutput, + readValidationReviewFixture, + resolveValidationReviewFixturePath, + validationReviewEvalTimeoutMs, +} from "../../../evals/validation-review/issue-49-review.eval"; + +describe("validation review Eve eval fixture", () => { + it("discovers the pinned issue #49 fixture without a live model call", async () => { + expect( + resolveValidationReviewFixturePath().endsWith( + "evals/validation-review/fixtures/issue-49.json", + ), + ).toBe(true); + await expect(readValidationReviewFixture()).resolves.toMatchObject({ + issueNumber: 49, + repositoryFullName: "ncolesummers/loopworks", + acceptanceCriteria: expect.arrayContaining([ + expect.stringContaining("deterministic results"), + expect.stringContaining("root orchestrator"), + ]), + }); + expect(validationReviewEvalTimeoutMs).toBeGreaterThanOrEqual(180_000); + }); + + it("decodes the serialized subagent output", () => { + const result = { schemaId: "loopworks.validation_review_result.v1", version: 1 }; + + expect(parseValidationReviewSubagentOutput(JSON.stringify(result))).toEqual(result); + expect(parseValidationReviewSubagentOutput("not json")).toBeUndefined(); + }); +}); diff --git a/tests/unit/agent/validation-review-tools.test.ts b/tests/unit/agent/validation-review-tools.test.ts new file mode 100644 index 0000000..97aa785 --- /dev/null +++ b/tests/unit/agent/validation-review-tools.test.ts @@ -0,0 +1,77 @@ +/** @vitest-environment node */ + +import { validateValidationReviewContext } from "@agent/subagents/validation-reviewer/lib/context"; +import { createValidationReviewFixtureContext } from "@agent/validation-review-fixture"; + +type FixtureContext = ReturnType; + +function firstValidationResult(value: FixtureContext) { + const [result] = value.validationReport.results; + if (!result) throw new Error("Expected fixture validation result."); + return result; +} + +describe("validation reviewer context policy", () => { + it("accepts only a completed passing validation handoff", () => { + const context = createValidationReviewFixtureContext(); + + expect(validateValidationReviewContext(context)).toEqual(context); + }); + + it.each([ + [ + "wrong stage", + (value: ReturnType) => { + value.run.currentStage = "validation"; + }, + ], + [ + "unfinished validation", + (value: ReturnType) => { + value.validationStep.status = "running"; + }, + ], + [ + "failed validation", + (value: ReturnType) => { + value.validationReport.overallOutcome = "fail"; + value.validationReport.counts = { failed: 1, passed: 0, skipped: 0, total: 1 }; + firstValidationResult(value).outcome = "fail"; + firstValidationResult(value).exitCode = 1; + }, + ], + [ + "required skipped gate", + (value: ReturnType) => { + value.validationReport.overallOutcome = "skipped"; + value.validationReport.counts = { failed: 0, passed: 0, skipped: 1, total: 1 }; + firstValidationResult(value).outcome = "skipped"; + firstValidationResult(value).exitCode = null; + firstValidationResult(value).skipReason = "not available"; + }, + ], + ])("rejects %s", (_label, mutate) => { + const context = createValidationReviewFixtureContext(); + mutate(context); + + expect(() => validateValidationReviewContext(context)).toThrow(); + }); + + it("rejects stale patch, report, and screenshot digests", () => { + for (const mutate of [ + (value: ReturnType) => { + value.implementationArtifactSha256 = "0".repeat(64); + }, + (value: ReturnType) => { + value.validationArtifactSha256 = "0".repeat(64); + }, + (value: ReturnType) => { + value.screenshotArtifactSha256 = "0".repeat(64); + }, + ]) { + const context = createValidationReviewFixtureContext(); + mutate(context); + expect(() => validateValidationReviewContext(context)).toThrow(); + } + }); +});