From 0e7f9b7a4fdcac9f0a2219c42063ac63c52d33f7 Mon Sep 17 00:00:00 2001 From: Nate Summers Date: Tue, 21 Jul 2026 21:26:40 -0700 Subject: [PATCH 1/5] feat(loops): add research loop contract Define the four-stage research skeleton, semantic artifacts, manifest policy, durable idempotency, and OTel run/no-op events with focused coverage. Co-Authored-By: OpenAI Codex GPT-5 --- src/components/portal/status-mapping.ts | 2 + src/lib/loops/manifest.ts | 107 +++++ src/lib/loops/research-run.ts | 483 +++++++++++++++++++++++ src/lib/observability/metrics.ts | 58 +++ src/lib/types.ts | 2 + tests/unit/loops/manifest.test.ts | 41 ++ tests/unit/loops/research-run.test.ts | 274 +++++++++++++ tests/unit/observability/metrics.test.ts | 86 +++- tests/unit/portal/status-mapping.test.ts | 9 + 9 files changed, 1061 insertions(+), 1 deletion(-) create mode 100644 src/lib/loops/research-run.ts create mode 100644 tests/unit/loops/research-run.test.ts diff --git a/src/components/portal/status-mapping.ts b/src/components/portal/status-mapping.ts index bfc0813..2223c31 100644 --- a/src/components/portal/status-mapping.ts +++ b/src/components/portal/status-mapping.ts @@ -165,6 +165,8 @@ export function getTimelineKindStatus(kind: TimelineKind): PortalStatusMeta { planning: { status: "pending", label: "Planning" }, test: { status: "running", label: "Test" }, development: { status: "running", label: "Development" }, + research: { status: "running", label: "Research" }, + authoring: { status: "running", label: "Authoring" }, validation: { status: "succeeded", label: "Validation" }, review: { status: "needsApproval", label: "Review" }, commit: { status: "done", label: "Commit" }, diff --git a/src/lib/loops/manifest.ts b/src/lib/loops/manifest.ts index ce41152..118e48a 100644 --- a/src/lib/loops/manifest.ts +++ b/src/lib/loops/manifest.ts @@ -164,6 +164,113 @@ export const defaultLoopManifest: LoopManifest = loopManifestSchema.parse({ requireApprovalForLabels: true, }, }, + { + key: "research-loop", + name: "Research loop", + description: + "Routes spike and agent-ready issues through fixture-backed planning, research, authoring, and completion contracts.", + enabled: true, + repoScope: { + repositories: ["ncolesummers/loopworks"], + branchPatterns: ["main", "codex/*"], + includeForks: false, + }, + triggers: { + issueLabels: ["agent-ready", "spike"], + blockedLabels: ["status:blocked"], + issueStates: ["opened", "reopened", "labeled"], + manual: false, + schedule: { enabled: false, timezone: "UTC" }, + }, + modelPolicy: { defaultModel: "codex-default" }, + toolPolicy: { + allowedToolCategories: ["repo-read", "browser", "validation"], + externalWritesRequireApproval: true, + }, + budgets: { + maxRunMinutes: 90, + maxModelUsd: 25, + maxToolCalls: 120, + }, + approvals: { + requiredFor: ["manifest_rollout"], + bypassPolicy: "none", + gates: [ + { + key: "manifest-review", + name: "Manifest review", + required: true, + reviewers: ["maintainer"], + evidence: ["plan"], + }, + ], + }, + artifacts: [ + { + type: "plan", + required: true, + description: "Issue-backed research plan placeholder for the planning stage.", + retention: "audit", + }, + { + type: "summary", + required: true, + description: + "Indexed findings placeholders, one per subquestion from isolated child sessions.", + retention: "audit", + }, + { + type: "summary", + required: true, + description: "Reviewable research document placeholder for the authoring stage.", + retention: "audit", + }, + { + type: "summary", + required: true, + description: "Durable completion summary placeholder for the terminal stage.", + retention: "run", + }, + ], + validationGates: [ + { + key: "focused-research-tests", + name: "Focused research loop tests", + command: "bun run test -- tests/unit/loops/research-run.test.ts", + required: true, + phase: "before_implementation", + produces: "validation_report", + }, + { + key: "aggregate-validation", + name: "Aggregate validation", + command: "bun run validate", + required: true, + phase: "before_rollout", + produces: "validation_report", + }, + ], + retryPolicy: { + maxAttempts: 2, + retryableStatuses: [...retryableStatusValues], + backoff: { strategy: "exponential", initialSeconds: 30, maxSeconds: 300 }, + }, + concurrency: { + group: "repo:{repo}:loop:research", + maxInFlight: 1, + cancelInProgress: false, + }, + cancellation: { + onSuperseded: "mark_canceled", + onDisabled: "skip_new_runs", + requiresReason: true, + }, + githubWriteback: { + enabled: false, + channels: [], + requireApprovalForLabels: true, + }, + }, ], milestones: [ { diff --git a/src/lib/loops/research-run.ts b/src/lib/loops/research-run.ts new file mode 100644 index 0000000..f9bd57a --- /dev/null +++ b/src/lib/loops/research-run.ts @@ -0,0 +1,483 @@ +import { randomUUID } from "node:crypto"; + +import { and, eq, sql } from "drizzle-orm"; + +import type { db } from "@/db/client"; +import { + artifacts, + idempotencyLocks, + loopRuns, + observabilityEvents, + repositories, + runSteps, +} from "@/db/schema"; +import { recordResearchLoopRunCreatedObservability } from "@/lib/observability/metrics"; +import { getActiveTraceId, isValidW3cTraceId } from "@/lib/observability/trace-context"; +import type { ArtifactRecord, TimelineEvent, TimelineKind } from "@/lib/types"; + +export const researchLoopKey = "research-loop"; +export const researchLoopNoopEventType = "research_loop_noop"; + +export type ResearchLoopStageKey = "planning" | "researching" | "authoring" | "done"; +export type ResearchArtifactKind = + | "research_plan" + | "findings" + | "research_document" + | "completion_summary"; +export type ResearchArtifactCardinality = "one" | "one_per_subquestion"; +export type ResearchArtifactIsolation = "run" | "child_session"; +export type ResearchArtifactPersistedType = "plan" | "other"; + +type ResearchArtifactContract = { + cardinality: ResearchArtifactCardinality; + isolation: ResearchArtifactIsolation; + kind: ResearchArtifactKind; + label: string; + required: true; + type: ResearchArtifactPersistedType; +}; + +type ResearchLoopStageContract = { + actorId: "research-planner" | "researcher" | "research-author" | "loopworks"; + actorType: "agent" | "system"; + artifact: ResearchArtifactContract; + key: ResearchLoopStageKey; + summary: string; + timelineKind: TimelineKind; + title: string; +}; + +export const researchLoopStages = [ + { + actorId: "research-planner", + actorType: "agent", + artifact: { + cardinality: "one", + isolation: "run", + kind: "research_plan", + label: "Research plan", + required: true, + type: "plan", + }, + key: "planning", + summary: "Define the research question, subquestions, sources, and completion criteria.", + timelineKind: "planning", + title: "Planning", + }, + { + actorId: "researcher", + actorType: "agent", + artifact: { + cardinality: "one_per_subquestion", + isolation: "child_session", + kind: "findings", + label: "Findings artifacts", + required: true, + type: "other", + }, + key: "researching", + summary: + "Index findings expected from isolated child sessions; live fan-out remains deferred to issue #45.", + timelineKind: "research", + title: "Researching", + }, + { + actorId: "research-author", + actorType: "agent", + artifact: { + cardinality: "one", + isolation: "run", + kind: "research_document", + label: "Research document", + required: true, + type: "other", + }, + key: "authoring", + summary: "Synthesize the indexed findings into a reviewable research document.", + timelineKind: "authoring", + title: "Authoring", + }, + { + actorId: "loopworks", + actorType: "system", + artifact: { + cardinality: "one", + isolation: "run", + kind: "completion_summary", + label: "Completion summary", + required: true, + type: "other", + }, + key: "done", + summary: "Record completion after the research document contract is satisfied.", + timelineKind: "done", + title: "Done", + }, +] as const satisfies readonly ResearchLoopStageContract[]; + +const nextResearchLoopStage = { + planning: "researching", + researching: "authoring", + authoring: "done", + done: null, +} as const satisfies Record; + +export function getNextResearchLoopStage(stage: ResearchLoopStageKey): ResearchLoopStageKey | null { + return nextResearchLoopStage[stage]; +} + +export type ResearchLoopTrigger = { + body?: string; + deliveryId?: string; + issueNumber: number; + issueUrl?: string; + labels?: readonly string[]; + milestone?: string | null; + repositoryFullName: string; + title?: string | null; +}; + +export type ResearchLoopRunDatabase = Pick; + +export type ResearchLoopRunMetadata = + | { artifactCount: number; mode: "created"; runId: string; stageCount: number } + | { artifactCount: number; mode: "simulated"; stageCount: number }; + +export type ResearchLoopNoopMetadata = { mode: "noop"; reason: "loop_disabled" }; + +type ResearchArtifactInstance = ResearchArtifactContract & { + detail: string; + stageKey: ResearchLoopStageKey; + uri: string; +}; + +type ResearchStageInstance = Omit & { + artifact: ResearchArtifactInstance; + queuedAt: Date; + status: "queued"; +}; + +export type ResearchLoopRunSkeleton = { + artifacts: ResearchArtifactInstance[]; + loopKey: typeof researchLoopKey; + mode: "created" | "simulated"; + runId?: string; + stages: ResearchStageInstance[]; + trigger: ResearchLoopTrigger; +}; + +function minutesAfter(date: Date, minutes: number): Date { + return new Date(date.getTime() + minutes * 60_000); +} + +function getIssueUrl(trigger: ResearchLoopTrigger): string { + return ( + trigger.issueUrl?.trim() || + `https://github.com/${trigger.repositoryFullName}/issues/${trigger.issueNumber}` + ); +} + +function getArtifactKind(type: ResearchArtifactPersistedType): ArtifactRecord["kind"] { + return type === "plan" ? "review" : "log"; +} + +export function createResearchLoopRunSkeleton(input: { + mode: "created" | "simulated"; + now: Date; + runId?: string; + trigger: ResearchLoopTrigger; +}): ResearchLoopRunSkeleton { + const stages = researchLoopStages.map((definition, index) => { + const stage: ResearchLoopStageContract = definition; + const artifact: ResearchArtifactInstance = { + ...stage.artifact, + detail: stage.summary, + stageKey: stage.key, + uri: `${getIssueUrl(input.trigger)}#research-loop-${stage.key}-${stage.artifact.kind.replaceAll("_", "-")}`, + }; + + return { + ...stage, + artifact, + queuedAt: minutesAfter(input.now, index), + status: "queued" as const, + }; + }); + + return { + artifacts: stages.map((stage) => stage.artifact), + loopKey: researchLoopKey, + mode: input.mode, + ...(input.runId ? { runId: input.runId } : {}), + stages, + trigger: input.trigger, + }; +} + +export function projectResearchLoopTimeline(skeleton: ResearchLoopRunSkeleton): TimelineEvent[] { + return skeleton.stages.map((stage) => ({ + actor: stage.actorId, + artifact: stage.artifact.label, + at: stage.queuedAt.toISOString().slice(11, 16), + detail: stage.summary, + kind: stage.timelineKind, + title: stage.title, + })); +} + +export function projectResearchLoopArtifacts(skeleton: ResearchLoopRunSkeleton): ArtifactRecord[] { + return skeleton.artifacts.map((artifact) => ({ + detail: artifact.detail, + href: artifact.uri, + kind: getArtifactKind(artifact.type), + label: artifact.label, + state: "available", + })); +} + +export function simulateResearchLoopRun(input: { + now: Date; + trigger: ResearchLoopTrigger; +}): ResearchLoopRunMetadata { + const skeleton = createResearchLoopRunSkeleton({ + mode: "simulated", + now: input.now, + trigger: input.trigger, + }); + return { + artifactCount: skeleton.artifacts.length, + mode: "simulated", + stageCount: skeleton.stages.length, + }; +} + +export async function createResearchLoopRun(input: { + database: ResearchLoopRunDatabase; + now?: () => Date; + traceId?: string; + trigger: ResearchLoopTrigger; +}): Promise { + const createdAt = input.now?.() ?? new Date(); + const traceId = + input.traceId === undefined + ? getActiveTraceId() + : isValidW3cTraceId(input.traceId) + ? input.traceId + : undefined; + + const result = await input.database.transaction(async (tx) => { + if (input.trigger.deliveryId) { + await tx + .insert(idempotencyLocks) + .values({ + acquiredAt: createdAt, + expiresAt: createdAt, + key: `${researchLoopKey}:run:${input.trigger.deliveryId}`, + metadata: { deliveryId: input.trigger.deliveryId, operation: "run" }, + owner: researchLoopKey, + releasedAt: createdAt, + scope: researchLoopKey, + status: "released", + }) + .onConflictDoNothing({ target: idempotencyLocks.key }); + } + + const existingRun = input.trigger.deliveryId + ? await tx + .select({ id: loopRuns.id }) + .from(loopRuns) + .where( + and( + eq(loopRuns.loopKey, researchLoopKey), + sql`${loopRuns.metadata}->>'deliveryId' = ${input.trigger.deliveryId}`, + ), + ) + .limit(1) + : []; + + if (existingRun[0]) { + const [existingArtifacts, existingSteps] = await Promise.all([ + tx + .select({ id: artifacts.id }) + .from(artifacts) + .where(eq(artifacts.runId, existingRun[0].id)), + tx.select({ id: runSteps.id }).from(runSteps).where(eq(runSteps.runId, existingRun[0].id)), + ]); + return { + metadata: { + artifactCount: existingArtifacts.length, + mode: "created" as const, + runId: existingRun[0].id, + stageCount: existingSteps.length, + }, + }; + } + + const [repository] = await tx + .select({ id: repositories.id }) + .from(repositories) + .where(eq(repositories.fullName, input.trigger.repositoryFullName)) + .limit(1); + if (!repository) { + throw new Error( + `Cannot create research loop run for unknown repository: ${input.trigger.repositoryFullName}`, + ); + } + + const runId = randomUUID(); + const skeleton = createResearchLoopRunSkeleton({ + mode: "created", + now: createdAt, + runId, + trigger: input.trigger, + }); + await tx.insert(loopRuns).values({ + id: runId, + currentStage: researchLoopStages[0].key, + githubIssueNumber: input.trigger.issueNumber, + githubIssueUrl: getIssueUrl(input.trigger), + loopKey: researchLoopKey, + metadata: { + deliveryId: input.trigger.deliveryId, + issueTitle: input.trigger.title ?? `Issue #${input.trigger.issueNumber}`, + labels: input.trigger.labels ?? [], + milestone: input.trigger.milestone ?? null, + source: "github_issue", + stageCount: skeleton.stages.length, + }, + queuedAt: createdAt, + repositoryId: repository.id, + status: "queued", + traceId, + }); + + const stepIds = new Map(); + for (const stage of skeleton.stages) { + const stepId = randomUUID(); + stepIds.set(stage.key, stepId); + await tx.insert(runSteps).values({ + actorId: stage.actorId, + actorType: stage.actorType, + id: stepId, + metadata: { + artifactKind: stage.artifact.kind, + requiredArtifact: stage.artifact.required, + }, + queuedAt: stage.queuedAt, + runId, + stage: stage.key, + status: stage.status, + summary: stage.summary, + traceId, + }); + } + + await tx.insert(artifacts).values( + skeleton.artifacts.map((artifact) => ({ + id: randomUUID(), + metadata: { + cardinality: artifact.cardinality, + isolation: artifact.isolation, + required: artifact.required, + researchArtifactKind: artifact.kind, + stage: artifact.stageKey, + }, + runId, + stepId: stepIds.get(artifact.stageKey), + title: artifact.label, + type: artifact.type, + uri: artifact.uri, + })), + ); + + const emitObservability = await recordResearchLoopRunCreatedObservability({ + artifactCount: skeleton.artifacts.length, + deliveryId: input.trigger.deliveryId, + issueNumber: input.trigger.issueNumber, + loopKey: researchLoopKey, + repositoryFullName: input.trigger.repositoryFullName, + repositoryId: repository.id, + runId, + stageCount: skeleton.stages.length, + traceId, + triggerLabel: "spike", + writer: tx, + }); + + return { + emitObservability, + metadata: { + artifactCount: skeleton.artifacts.length, + mode: "created" as const, + runId, + stageCount: skeleton.stages.length, + }, + }; + }); + + result.emitObservability?.(); + return result.metadata; +} + +export async function recordResearchLoopNoop(input: { + database: ResearchLoopRunDatabase; + now?: () => Date; + reason: "loop_disabled"; + trigger: ResearchLoopTrigger; +}): Promise { + const createdAt = input.now?.() ?? new Date(); + await input.database.transaction(async (tx) => { + if (input.trigger.deliveryId) { + await tx + .insert(idempotencyLocks) + .values({ + acquiredAt: createdAt, + expiresAt: createdAt, + key: `${researchLoopKey}:noop:${input.trigger.deliveryId}`, + metadata: { deliveryId: input.trigger.deliveryId, operation: "noop" }, + owner: researchLoopKey, + releasedAt: createdAt, + scope: researchLoopKey, + status: "released", + }) + .onConflictDoNothing({ target: idempotencyLocks.key }); + } + + const existing = input.trigger.deliveryId + ? await tx + .select({ id: observabilityEvents.id }) + .from(observabilityEvents) + .where( + and( + eq(observabilityEvents.eventType, researchLoopNoopEventType), + sql`${observabilityEvents.payload}->>'deliveryId' = ${input.trigger.deliveryId}`, + ), + ) + .limit(1) + : []; + if (existing[0]) return; + + const [repository] = await tx + .select({ id: repositories.id }) + .from(repositories) + .where(eq(repositories.fullName, input.trigger.repositoryFullName)) + .limit(1); + await tx.insert(observabilityEvents).values({ + ...(repository ? { repositoryId: repository.id } : {}), + correlationId: input.trigger.deliveryId, + createdAt, + eventType: researchLoopNoopEventType, + message: "Spike research loop trigger recorded as a no-op.", + payload: { + deliveryId: input.trigger.deliveryId, + issueNumber: input.trigger.issueNumber, + loopKey: researchLoopKey, + reason: input.reason, + repositoryFullName: input.trigger.repositoryFullName, + }, + severity: "info", + }); + }); + + return { mode: "noop", reason: input.reason }; +} diff --git a/src/lib/observability/metrics.ts b/src/lib/observability/metrics.ts index 14c23dc..c026c9a 100644 --- a/src/lib/observability/metrics.ts +++ b/src/lib/observability/metrics.ts @@ -141,6 +141,8 @@ export function resolveObservabilityMetricDefinition( export const developmentLoopRunCreatedEventType = "development_loop_run_created"; export const developmentLoopRunCreatedDurableMetricName = developmentLoopRunCreatedEventType; +export const researchLoopRunCreatedEventType = "research_loop_run_created"; +export const researchLoopRunCreatedDurableMetricName = researchLoopRunCreatedEventType; export type DurableObservabilityEventDefinition = { eventType: string; @@ -154,6 +156,11 @@ export const durableObservabilityEventContract = [ metricName: developmentLoopRunCreatedDurableMetricName, otelMetricName: "loopworks.run.started", }, + { + eventType: researchLoopRunCreatedEventType, + metricName: researchLoopRunCreatedDurableMetricName, + otelMetricName: "loopworks.run.started", + }, ] as const satisfies readonly DurableObservabilityEventDefinition[]; export type DurableObservabilityEventMetricName = @@ -808,3 +815,54 @@ export async function recordDevelopmentLoopRunCreatedObservability(input: { } }; } + +export async function recordResearchLoopRunCreatedObservability(input: { + artifactCount: number; + deliveryId?: string; + issueNumber: number; + loopKey: string; + meter?: RunStartedMeter; + repositoryFullName: string; + repositoryId: string; + runId: string; + stageCount: number; + traceId?: string; + triggerLabel: string; + writer: ObservabilityEventsWriter; +}): Promise<() => void> { + const event = resolveDurableObservabilityEventDefinition(researchLoopRunCreatedDurableMetricName); + + await input.writer.insert(observabilityEvents).values({ + correlationId: input.deliveryId, + eventType: event.eventType, + message: "Spike research loop run skeleton created.", + metricName: event.metricName, + metricValue: input.stageCount, + payload: { + artifactCount: input.artifactCount, + issueNumber: input.issueNumber, + loopKey: input.loopKey, + repositoryFullName: input.repositoryFullName, + stageCount: input.stageCount, + }, + repositoryId: input.repositoryId, + runId: input.runId, + severity: "info", + traceId: input.traceId, + }); + + return () => { + try { + recordDevelopmentLoopRunStartedMetric( + { + loopKey: input.loopKey, + repository: input.repositoryFullName, + triggerLabel: input.triggerLabel, + }, + input.meter, + ); + } catch { + // Durable event persistence is the source of truth; OTel emission must not roll back runs. + } + }; +} diff --git a/src/lib/types.ts b/src/lib/types.ts index 67c981a..43ef2d5 100644 --- a/src/lib/types.ts +++ b/src/lib/types.ts @@ -36,6 +36,8 @@ export type TimelineKind = | "planning" | "test" | "development" + | "research" + | "authoring" | "validation" | "review" | "commit" diff --git a/tests/unit/loops/manifest.test.ts b/tests/unit/loops/manifest.test.ts index 5965703..a4c3cfc 100644 --- a/tests/unit/loops/manifest.test.ts +++ b/tests/unit/loops/manifest.test.ts @@ -49,6 +49,47 @@ describe("loop manifest schema", () => { expect(manifest.loops.map((loop) => loop.enabled)).toEqual([true, false]); }); + it("declares an enabled read-only research loop with distinct trigger and concurrency policy", () => { + const manifest = parseLoopManifest(defaultLoopManifest); + const researchLoop = manifest.loops.find((loop) => loop.key === "research-loop"); + + expect(researchLoop).toMatchObject({ + enabled: true, + triggers: { + issueLabels: ["agent-ready", "spike"], + }, + toolPolicy: { + allowedToolCategories: ["repo-read", "browser", "validation"], + externalWritesRequireApproval: true, + }, + approvals: { + requiredFor: ["manifest_rollout"], + }, + concurrency: { + group: "repo:{repo}:loop:research", + }, + githubWriteback: { + enabled: false, + channels: [], + }, + }); + expect(researchLoop?.artifacts.map((artifact) => artifact.description)).toEqual([ + expect.stringContaining("research plan"), + expect.stringContaining("findings"), + expect.stringContaining("research document"), + expect.stringContaining("completion summary"), + ]); + expect(researchLoop?.validationGates).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + command: "bun run test -- tests/unit/loops/research-run.test.ts", + key: "focused-research-tests", + }), + expect.objectContaining({ command: "bun run validate", key: "aggregate-validation" }), + ]), + ); + }); + it("covers trigger labels, validation gates, approvals, retries, and concurrency", () => { const manifest = parseLoopManifest(defaultLoopManifest); const developmentLoop = manifest.loops[0]; diff --git a/tests/unit/loops/research-run.test.ts b/tests/unit/loops/research-run.test.ts new file mode 100644 index 0000000..632e441 --- /dev/null +++ b/tests/unit/loops/research-run.test.ts @@ -0,0 +1,274 @@ +/** @vitest-environment node */ + +import { context as otelContext, type Span, TraceFlags, trace } from "@opentelemetry/api"; +import { eq } from "drizzle-orm"; + +import { + agentPlans, + approvals, + artifacts, + idempotencyLocks, + loopRuns, + observabilityEvents, + repositories, + runSteps, +} from "@/db/schema"; +import { + createResearchLoopRun, + createResearchLoopRunSkeleton, + getNextResearchLoopStage, + projectResearchLoopArtifacts, + projectResearchLoopTimeline, + type ResearchLoopRunDatabase, + recordResearchLoopNoop, + researchLoopStages, + simulateResearchLoopRun, +} from "@/lib/loops/research-run"; +import { createPgliteTestDatabase, type PgliteTestDatabase } from "../../helpers/pglite"; + +const issueTrigger = { + body: "Prove that the neutral orchestrator can route a non-code loop.", + deliveryId: "issue-43-delivery", + issueNumber: 43, + issueUrl: "https://github.com/ncolesummers/loopworks/issues/43", + labels: ["agent-ready", "spike", "area:loops", "loop:research", "priority:p2"], + milestone: "M3 Durable Loop MVP", + repositoryFullName: "ncolesummers/loopworks", + title: "Research loop skeleton", +}; + +function testDatabase(context: PgliteTestDatabase): ResearchLoopRunDatabase { + return context.db as unknown as ResearchLoopRunDatabase; +} + +async function insertRepository(context: PgliteTestDatabase) { + await context.db.insert(repositories).values({ + githubRepoId: 43_000_001, + owner: "ncolesummers", + name: "loopworks", + fullName: "ncolesummers/loopworks", + enabledLoops: ["Research routing"], + validationGates: ["Focused tests", "Aggregate validation"], + }); +} + +function withTestTrace(callback: () => T): T { + const span = { + spanContext: () => ({ + traceId: "4bf92f3577b34da6a3ce929d0e0e4736", + spanId: "00f067aa0ba902b7", + traceFlags: TraceFlags.SAMPLED, + }), + } as Span; + + return otelContext.with(trace.setSpan(otelContext.active(), span), callback); +} + +describe("spike agent-ready research loop run skeleton", () => { + let context: PgliteTestDatabase; + + beforeEach(async () => { + context = await createPgliteTestDatabase(); + }); + + afterEach(async () => { + await context.close(); + }); + + it("defines a deterministic non-code stage and artifact contract", () => { + expect(researchLoopStages).toEqual([ + expect.objectContaining({ + actorId: "research-planner", + artifact: expect.objectContaining({ kind: "research_plan", type: "plan" }), + key: "planning", + timelineKind: "planning", + }), + expect.objectContaining({ + actorId: "researcher", + artifact: expect.objectContaining({ + cardinality: "one_per_subquestion", + isolation: "child_session", + kind: "findings", + type: "other", + }), + key: "researching", + timelineKind: "research", + }), + expect.objectContaining({ + actorId: "research-author", + artifact: expect.objectContaining({ kind: "research_document", type: "other" }), + key: "authoring", + timelineKind: "authoring", + }), + expect.objectContaining({ + actorId: "loopworks", + artifact: expect.objectContaining({ kind: "completion_summary", type: "other" }), + key: "done", + timelineKind: "done", + }), + ]); + expect(researchLoopStages.map((stage) => stage.key)).not.toEqual( + expect.arrayContaining(["test-writing", "development", "validation", "pr"]), + ); + expect(getNextResearchLoopStage("planning")).toBe("researching"); + expect(getNextResearchLoopStage("researching")).toBe("authoring"); + expect(getNextResearchLoopStage("authoring")).toBe("done"); + expect(getNextResearchLoopStage("done")).toBeNull(); + }); + + it("projects a simulated fixture into four visible timeline and artifact records", () => { + const skeleton = createResearchLoopRunSkeleton({ + mode: "simulated", + now: new Date("2026-07-22T02:00:00.000Z"), + trigger: issueTrigger, + }); + + expect( + simulateResearchLoopRun({ now: new Date("2026-07-22T02:00:00.000Z"), trigger: issueTrigger }), + ).toEqual({ + artifactCount: 4, + mode: "simulated", + stageCount: 4, + }); + expect(projectResearchLoopTimeline(skeleton).map((event) => event.title)).toEqual([ + "Planning", + "Researching", + "Authoring", + "Done", + ]); + expect(projectResearchLoopArtifacts(skeleton).map((artifact) => artifact.label)).toEqual([ + "Research plan", + "Findings artifacts", + "Research document", + "Completion summary", + ]); + }); + + it("creates one durable traced run with four steps and artifact placeholders", async () => { + await insertRepository(context); + + const result = await withTestTrace(() => + createResearchLoopRun({ + database: testDatabase(context), + now: () => new Date("2026-07-22T02:00:00.000Z"), + trigger: issueTrigger, + }), + ); + + expect(result).toMatchObject({ artifactCount: 4, mode: "created", stageCount: 4 }); + const [run] = await context.db.select().from(loopRuns); + const steps = await context.db.select().from(runSteps); + const artifactRows = await context.db.select().from(artifacts); + const [event] = await context.db + .select() + .from(observabilityEvents) + .where(eq(observabilityEvents.eventType, "research_loop_run_created")); + + expect(run).toMatchObject({ + currentStage: "planning", + githubIssueNumber: 43, + loopKey: "research-loop", + status: "queued", + traceId: "4bf92f3577b34da6a3ce929d0e0e4736", + }); + expect(steps.map((step) => step.stage)).toEqual([ + "planning", + "researching", + "authoring", + "done", + ]); + expect(steps.every((step) => step.traceId === run?.traceId)).toBe(true); + expect(artifactRows).toHaveLength(4); + expect(artifactRows.map((artifact) => artifact.metadata?.researchArtifactKind)).toEqual([ + "research_plan", + "findings", + "research_document", + "completion_summary", + ]); + expect(artifactRows[1]?.metadata).toMatchObject({ + cardinality: "one_per_subquestion", + isolation: "child_session", + }); + expect(event).toMatchObject({ + eventType: "research_loop_run_created", + metricName: "research_loop_run_created", + runId: run?.id, + traceId: run?.traceId, + }); + expect(await context.db.select().from(agentPlans)).toEqual([]); + expect(await context.db.select().from(approvals)).toEqual([]); + }); + + it("is idempotent for a repeated delivery and rejects unknown repositories", async () => { + await insertRepository(context); + + const [first, second] = await Promise.all([ + createResearchLoopRun({ database: testDatabase(context), trigger: issueTrigger }), + createResearchLoopRun({ database: testDatabase(context), trigger: issueTrigger }), + ]); + + expect(second).toEqual(first); + expect(await context.db.select().from(loopRuns)).toHaveLength(1); + expect(await context.db.select().from(runSteps)).toHaveLength(4); + expect(await context.db.select().from(artifacts)).toHaveLength(4); + expect(await context.db.select().from(idempotencyLocks)).toEqual([ + expect.objectContaining({ + key: "research-loop:run:issue-43-delivery", + scope: "research-loop", + status: "released", + }), + ]); + + await expect( + createResearchLoopRun({ + database: testDatabase(context), + trigger: { + ...issueTrigger, + deliveryId: "unknown-repository", + repositoryFullName: "ncolesummers/missing", + }, + }), + ).rejects.toThrow("Cannot create research loop run for unknown repository"); + }); + + it("records one research-specific disabled no-op without creating a run", async () => { + await insertRepository(context); + + const [first, second] = await Promise.all([ + recordResearchLoopNoop({ + database: testDatabase(context), + now: () => new Date("2026-07-22T02:01:00.000Z"), + reason: "loop_disabled", + trigger: issueTrigger, + }), + recordResearchLoopNoop({ + database: testDatabase(context), + now: () => new Date("2026-07-22T02:02:00.000Z"), + reason: "loop_disabled", + trigger: issueTrigger, + }), + ]); + + expect(second).toEqual(first); + expect(first).toEqual({ mode: "noop", reason: "loop_disabled" }); + expect(await context.db.select().from(loopRuns)).toEqual([]); + const events = await context.db.select().from(observabilityEvents); + expect(events).toHaveLength(1); + expect(events[0]).toMatchObject({ + correlationId: "issue-43-delivery", + eventType: "research_loop_noop", + payload: expect.objectContaining({ + issueNumber: 43, + loopKey: "research-loop", + reason: "loop_disabled", + }), + }); + expect(await context.db.select().from(idempotencyLocks)).toEqual([ + expect.objectContaining({ + key: "research-loop:noop:issue-43-delivery", + scope: "research-loop", + status: "released", + }), + ]); + }); +}); diff --git a/tests/unit/observability/metrics.test.ts b/tests/unit/observability/metrics.test.ts index d242e21..be9cea0 100644 --- a/tests/unit/observability/metrics.test.ts +++ b/tests/unit/observability/metrics.test.ts @@ -8,8 +8,8 @@ import { collectControlPlaneGaugeMeasurements, developmentLoopRunCreatedDurableMetricName, observabilityMetricNames, - recordDevelopmentLoopRunCompletedMetric, recordApprovalWaitTimeMetric, + recordDevelopmentLoopRunCompletedMetric, recordDevelopmentLoopRunCreatedObservability, recordDevelopmentLoopRunDurationMetric, recordDevelopmentLoopRunStartedMetric, @@ -19,7 +19,9 @@ import { recordDevelopmentLoopValidationOutcomeMetric, recordGithubWebhookOutcomeMetric, recordLockContentionMetric, + recordResearchLoopRunCreatedObservability, registerControlPlaneGaugeMetrics, + researchLoopRunCreatedDurableMetricName, resolveDurableObservabilityEventDefinition, resolveObservabilityMetricDefinition, } from "@/lib/observability/metrics"; @@ -540,12 +542,93 @@ describe("ADR 0012 observability metric contract", () => { metricName: "development_loop_run_created", otelMetricName: "loopworks.run.started", }); + expect( + resolveDurableObservabilityEventDefinition(researchLoopRunCreatedDurableMetricName), + ).toMatchObject({ + eventType: "research_loop_run_created", + metricName: "research_loop_run_created", + otelMetricName: "loopworks.run.started", + }); expect(() => resolveDurableObservabilityEventDefinition("development_loop_run_started"), ).toThrow("Unsupported durable observability event metric name: development_loop_run_started"); }); + it("emits the research-loop run-created durable event and existing OTel run-started metric", async () => { + const insertedRows: Record[] = []; + const recordings: { + attributes: Record | undefined; + name: string; + value: number; + }[] = []; + const writer = { + insert(table: unknown) { + expect(table).toBe(observabilityEvents); + return { + values(row: Record) { + insertedRows.push(row); + return Promise.resolve(); + }, + }; + }, + }; + const meter = { + createCounter(name: string) { + return { + add(value: number, attributes?: Record) { + recordings.push({ attributes, name, value }); + }, + }; + }, + }; + + const emitMetric = await recordResearchLoopRunCreatedObservability({ + artifactCount: 4, + deliveryId: "issue-43-delivery", + issueNumber: 43, + loopKey: "research-loop", + meter, + repositoryFullName: "ncolesummers/loopworks", + repositoryId: "64f8ca7a-1b5d-4b3f-8c5e-2e2d814a18aa", + runId: "9a8d379f-1d65-4fb0-bd91-4f82306a3159", + stageCount: 4, + traceId: "4bf92f3577b34da6a3ce929d0e0e4736", + triggerLabel: "spike", + writer, + }); + + expect(recordings).toEqual([]); + expect(insertedRows).toEqual([ + expect.objectContaining({ + correlationId: "issue-43-delivery", + eventType: "research_loop_run_created", + metricName: "research_loop_run_created", + metricValue: 4, + payload: { + artifactCount: 4, + issueNumber: 43, + loopKey: "research-loop", + repositoryFullName: "ncolesummers/loopworks", + stageCount: 4, + }, + traceId: "4bf92f3577b34da6a3ce929d0e0e4736", + }), + ]); + emitMetric(); + expect(recordings).toEqual([ + { + attributes: { + "loop.key": "research-loop", + repository: "ncolesummers/loopworks", + "trigger.label": "spike", + }, + name: "loopworks.run.started", + value: 1, + }, + ]); + }); + it("emits the development-loop run-created durable event and OTel metric from one helper", async () => { const insertedRows: Record[] = []; const recordings: { @@ -678,6 +761,7 @@ describe("ADR 0012 observability metric contract", () => { const forbiddenMetricLiterals = new Set([ ...observabilityMetricNames, developmentLoopRunCreatedDurableMetricName, + researchLoopRunCreatedDurableMetricName, ]); const violations: string[] = []; diff --git a/tests/unit/portal/status-mapping.test.ts b/tests/unit/portal/status-mapping.test.ts index bdc156c..705d969 100644 --- a/tests/unit/portal/status-mapping.test.ts +++ b/tests/unit/portal/status-mapping.test.ts @@ -9,6 +9,7 @@ import { getRepoHealthStatus, getRunStatus, getRunStepStatus, + getTimelineKindStatus, getValidationResultStatus, } from "@/components/portal/status-mapping"; @@ -111,5 +112,13 @@ describe("portal status mapping", () => { status: "succeeded", label: "Succeeded", }); + expect(getTimelineKindStatus("research")).toEqual({ + status: "running", + label: "Research", + }); + expect(getTimelineKindStatus("authoring")).toEqual({ + status: "running", + label: "Authoring", + }); }); }); From 3328cc430cc8f186b3deda59afaaf5a0b27ddfef Mon Sep 17 00:00:00 2001 From: Nate Summers Date: Tue, 21 Jul 2026 21:26:57 -0700 Subject: [PATCH 2/5] feat(webhooks): route research loop deliveries Return research-specific outcomes, persist idempotent runs and no-ops, align trigger actions with the manifest, and correct the signed issue 43 fixture. Co-Authored-By: OpenAI Codex GPT-5 --- scripts/github-webhook-fixture.ts | 21 +++-- src/app/api/github/webhooks/route.ts | 77 ++++++++++++++++++ src/lib/github/webhooks.ts | 13 ++- .../github/webhook-store.integration.test.ts | 80 ++++++++++++++++++- tests/unit/github/webhooks.test.ts | 36 ++++++++- .../scripts/github-webhook-fixture.test.ts | 4 +- 6 files changed, 212 insertions(+), 19 deletions(-) diff --git a/scripts/github-webhook-fixture.ts b/scripts/github-webhook-fixture.ts index 5ac850d..e33c0c5 100644 --- a/scripts/github-webhook-fixture.ts +++ b/scripts/github-webhook-fixture.ts @@ -51,26 +51,25 @@ function createSignature(secret: string, payloadText: string): string { } function createPayload(kind: GithubWebhookFixtureKind): GithubWebhookFixturePayload { - const labels = - kind === "spike-agent-ready" - ? ["agent-ready", "spike", "area:loops", "area:agents", "loop:development", "priority:p0"] - : ["agent-ready", "area:loops", "area:agents", "loop:development", "priority:p0"]; + const isResearch = kind === "spike-agent-ready"; + const labels = isResearch + ? ["agent-ready", "spike", "area:loops", "area:agents", "loop:research", "priority:p2"] + : ["agent-ready", "area:loops", "area:agents", "loop:development", "priority:p0"]; return { action: "labeled", issue: { - body: "Implement the first durable loop skeleton for issues labeled agent-ready.", - html_url: "https://github.com/ncolesummers/loopworks/issues/11", + body: isResearch + ? "Implement a durable, fixture-backed research loop skeleton." + : "Implement the first durable loop skeleton for issues labeled agent-ready.", + html_url: `https://github.com/ncolesummers/loopworks/issues/${isResearch ? 43 : 11}`, labels: labels.map((name) => ({ name })), milestone: { title: "M3 Durable Loop MVP", }, - number: 11, + number: isResearch ? 43 : 11, state: "open", - title: - kind === "spike-agent-ready" - ? "Research agent-ready development loop skeleton" - : "Agent-ready development loop skeleton", + title: isResearch ? "Research loop skeleton" : "Agent-ready development loop skeleton", }, repository: { full_name: defaultRepository, diff --git a/src/app/api/github/webhooks/route.ts b/src/app/api/github/webhooks/route.ts index eeae1ab..1691ac4 100644 --- a/src/app/api/github/webhooks/route.ts +++ b/src/app/api/github/webhooks/route.ts @@ -22,6 +22,14 @@ import { recordDevelopmentLoopNoop, simulateDevelopmentLoopRun, } from "@/lib/loops/development-run"; +import { + createResearchLoopRun, + type ResearchLoopNoopMetadata, + type ResearchLoopRunMetadata, + type ResearchLoopTrigger, + recordResearchLoopNoop, + simulateResearchLoopRun, +} from "@/lib/loops/research-run"; import { createRequestLogger } from "@/lib/observability/logger"; import { type GithubWebhookOutcomeMetricInput, @@ -48,6 +56,7 @@ type GithubWebhookPostDependencies = { type GithubWebhookDeliveryStoreMode = "drizzle" | "injected" | "memory"; type DevelopmentRunOutcome = DevelopmentLoopRunMetadata | DevelopmentLoopNoopMetadata; +type ResearchRunOutcome = ResearchLoopRunMetadata | ResearchLoopNoopMetadata; function asIssuesPayload(payload: unknown): GithubIssuesWebhookPayload | null { if (typeof payload !== "object" || payload === null) { @@ -187,6 +196,13 @@ function getDevelopmentLoopTrigger( }; } +function getResearchLoopTrigger( + payload: GithubIssuesWebhookPayload | null, + deliveryId: string, +): ResearchLoopTrigger | null { + return getDevelopmentLoopTrigger(payload, deliveryId); +} + async function resolveDevelopmentRunOutcome(input: { agentReadyTrigger: GithubAgentReadyTrigger; database: DevelopmentLoopRunDatabase; @@ -243,6 +259,54 @@ async function resolveDevelopmentRunOutcome(input: { return undefined; } +async function resolveResearchRunOutcome(input: { + agentReadyTrigger: GithubAgentReadyTrigger; + database: DevelopmentLoopRunDatabase; + issuesPayload: GithubIssuesWebhookPayload | null; + normalizedDeliveryId: string; + now: Date; + persist: boolean; + traceId?: string; +}): Promise { + const trigger = getResearchLoopTrigger(input.issuesPayload, input.normalizedDeliveryId); + + if ( + input.agentReadyTrigger.shouldTrigger && + input.agentReadyTrigger.workflow === "research" && + trigger + ) { + if (input.persist) { + return createResearchLoopRun({ + database: input.database, + now: () => input.now, + traceId: input.traceId, + trigger, + }); + } + return simulateResearchLoopRun({ now: input.now, trigger }); + } + + if ( + !input.agentReadyTrigger.shouldTrigger && + input.agentReadyTrigger.skipped && + input.agentReadyTrigger.reason === "loop_disabled" && + input.agentReadyTrigger.workflow === "research" && + trigger + ) { + if (input.persist) { + return recordResearchLoopNoop({ + database: input.database, + now: () => input.now, + reason: "loop_disabled", + trigger, + }); + } + return { mode: "noop", reason: "loop_disabled" }; + } + + return undefined; +} + export async function handleGithubWebhookPost( request: Request, dependencies: GithubWebhookPostDependencies = {}, @@ -421,11 +485,22 @@ export async function handleGithubWebhookPost( selectedDeliveryStore.mode === "drizzle" || Boolean(dependencies.developmentRunDatabase), traceId, }); + const researchRun = await resolveResearchRunOutcome({ + agentReadyTrigger, + database: developmentRunDatabase, + issuesPayload, + normalizedDeliveryId: claim.deliveryId, + now: processedAt, + persist: + selectedDeliveryStore.mode === "drizzle" || Boolean(dependencies.developmentRunDatabase), + traceId, + }); await webhookDeliveryStore.complete(claim.key, { deliveryId: claim.deliveryId, metadata: { ...(developmentRun ? { developmentRun } : {}), + ...(researchRun ? { researchRun } : {}), nextAction, triggerReason: agentReadyTrigger.reason, triggerWorkflow: agentReadyTrigger.workflow ?? "none", @@ -439,6 +514,7 @@ export async function handleGithubWebhookPost( idempotencyKey: claim.key, agentReadyTrigger, developmentRun, + researchRun, nextAction, triggerWorkflow: agentReadyTrigger.workflow ?? "none", }, @@ -459,6 +535,7 @@ export async function handleGithubWebhookPost( event, agentReadyTrigger, ...(developmentRun ? { developmentRun } : {}), + ...(researchRun ? { researchRun } : {}), nextAction, ...getFixtureFallbackResponse(selectedDeliveryStore.mode), }, diff --git a/src/lib/github/webhooks.ts b/src/lib/github/webhooks.ts index 6627665..a878133 100644 --- a/src/lib/github/webhooks.ts +++ b/src/lib/github/webhooks.ts @@ -1,10 +1,14 @@ import { createHmac, timingSafeEqual } from "node:crypto"; - +import { defaultLoopManifest } from "@/lib/loops/manifest"; import { evaluateLoopTriggerDecision } from "@/lib/loops/trigger-decision"; import { isProductionRuntime } from "@/lib/runtime"; const githubSignaturePrefix = "sha256="; const triggerableIssueActions = new Set(["edited", "labeled", "milestoned", "opened", "reopened"]); +const researchTriggerableIssueActions = new Set( + defaultLoopManifest.loops.find((loop) => loop.key === "research-loop")?.triggers.issueStates ?? + [], +); export type GithubWebhookDeliveryStore = { claim: ( @@ -297,11 +301,16 @@ export function getAgentReadyTriggerFromIssuesWebhook( }; } + const workflow = readiness.labels.includes("spike") ? "research" : "development"; + if (workflow === "research" && !researchTriggerableIssueActions.has(action)) { + return { shouldTrigger: false, reason: "unsupported_action" }; + } + return { shouldTrigger: true, issueNumber: payload.issue.number, repositoryFullName: payload.repository?.full_name ?? undefined, - workflow: readiness.labels.includes("spike") ? "research" : "development", + workflow, reason: "issue_became_agent_ready", }; } diff --git a/tests/unit/github/webhook-store.integration.test.ts b/tests/unit/github/webhook-store.integration.test.ts index 62dd3cd..d4610b4 100644 --- a/tests/unit/github/webhook-store.integration.test.ts +++ b/tests/unit/github/webhook-store.integration.test.ts @@ -6,6 +6,7 @@ import { eq } from "drizzle-orm"; import { handleGithubWebhookPost } from "@/app/api/github/webhooks/route"; import { agentPlans, + approvals, artifacts, idempotencyLocks, loopRuns, @@ -373,6 +374,73 @@ describe("GitHub webhook delivery store (pglite integration)", () => { expect(await context.db.select().from(agentPlans)).toHaveLength(1); }); + it("persists one idempotent research-loop run for an accepted spike delivery", async () => { + vi.stubEnv("GITHUB_WEBHOOK_SECRET", "dev-webhook-secret"); + await insertLoopworksRepository(); + const store = createStore(); + const fixture = createGithubWebhookFixture({ + deliveryId: "fixture-route-research-persistence-delivery", + kind: "spike-agent-ready", + secret: "dev-webhook-secret", + url: "https://loopworks.local/api/github/webhooks", + }); + const makeRequest = () => + new Request(fixture.url, { + body: fixture.payloadText, + headers: fixture.headers, + method: "POST", + }); + + const first = await withTestTrace(() => + handleGithubWebhookPost(makeRequest(), { + developmentRunDatabase: context.db as unknown as DevelopmentLoopRunDatabase, + now: () => new Date("2026-07-21T16:00:00.000Z"), + webhookDeliveryStore: store, + }), + ); + + expect(first.status).toBe(202); + await expect(first.json()).resolves.toMatchObject({ + accepted: true, + agentReadyTrigger: { shouldTrigger: true, workflow: "research" }, + nextAction: "queue_deep_research_loop", + researchRun: { artifactCount: 4, mode: "created", stageCount: 4 }, + }); + const runRows = await context.db.select().from(loopRuns); + expect(runRows).toHaveLength(1); + expect(runRows[0]).toMatchObject({ + githubIssueNumber: 43, + loopKey: "research-loop", + traceId: "4bf92f3577b34da6a3ce929d0e0e4736", + }); + const stepRows = await context.db.select().from(runSteps); + expect(stepRows.map((step) => step.stage)).toEqual([ + "planning", + "researching", + "authoring", + "done", + ]); + expect(stepRows.every((step) => step.traceId === runRows[0]?.traceId)).toBe(true); + expect(await context.db.select().from(artifacts)).toHaveLength(4); + expect(await context.db.select().from(agentPlans)).toHaveLength(0); + expect(await context.db.select().from(approvals)).toHaveLength(0); + const [event] = await context.db + .select() + .from(observabilityEvents) + .where(eq(observabilityEvents.eventType, "research_loop_run_created")); + expect(event.traceId).toBe(runRows[0]?.traceId); + + const replay = await handleGithubWebhookPost(makeRequest(), { + developmentRunDatabase: context.db as unknown as DevelopmentLoopRunDatabase, + now: () => new Date("2026-07-21T16:01:00.000Z"), + webhookDeliveryStore: store, + }); + await expect(replay.json()).resolves.toMatchObject({ accepted: false, duplicate: true }); + expect(await context.db.select().from(loopRuns)).toHaveLength(1); + expect(await context.db.select().from(runSteps)).toHaveLength(4); + expect(await context.db.select().from(artifacts)).toHaveLength(4); + }); + it("persists disabled-loop no-op state at the route boundary", async () => { vi.stubEnv("GITHUB_WEBHOOK_SECRET", "dev-webhook-secret"); vi.stubEnv("LOOPWORKS_DEVELOPMENT_LOOP_ENABLED", "false"); @@ -420,7 +488,7 @@ describe("GitHub webhook delivery store (pglite integration)", () => { ).toHaveLength(1); }); - it("does not persist a development no-op for disabled research loops", async () => { + it("persists a research-specific no-op for disabled research loops", async () => { vi.stubEnv("GITHUB_WEBHOOK_SECRET", "dev-webhook-secret"); vi.stubEnv("LOOPWORKS_RESEARCH_LOOP_ENABLED", "false"); await insertLoopworksRepository(); @@ -454,6 +522,10 @@ describe("GitHub webhook delivery store (pglite integration)", () => { skipped: true, workflow: "research", }, + researchRun: { + mode: "noop", + reason: "loop_disabled", + }, }); expect(responseBody).not.toHaveProperty("developmentRun"); expect(await context.db.select().from(loopRuns)).toHaveLength(0); @@ -463,5 +535,11 @@ describe("GitHub webhook delivery store (pglite integration)", () => { .from(observabilityEvents) .where(eq(observabilityEvents.eventType, "development_loop_noop")), ).toHaveLength(0); + expect( + await context.db + .select() + .from(observabilityEvents) + .where(eq(observabilityEvents.eventType, "research_loop_noop")), + ).toHaveLength(1); }); }); diff --git a/tests/unit/github/webhooks.test.ts b/tests/unit/github/webhooks.test.ts index 6dcf4ce..27ca917 100644 --- a/tests/unit/github/webhooks.test.ts +++ b/tests/unit/github/webhooks.test.ts @@ -190,6 +190,27 @@ describe("GitHub webhook helpers", () => { }); }); + it("does not start research runs for issue actions outside the manifest", () => { + const trigger = getAgentReadyTriggerFromIssuesWebhook({ + action: "edited", + repository: { full_name: "ncolesummers/loopworks" }, + issue: { + body: "An ordinary edit must not create another research run.", + labels: [ + { name: "agent-ready" }, + { name: "spike" }, + { name: "area:loops" }, + { name: "priority:p2" }, + ], + milestone: { title: "M3 Durable Loop MVP" }, + number: 43, + state: "open", + }, + }); + + expect(trigger).toEqual({ shouldTrigger: false, reason: "unsupported_action" }); + }); + it("skips an agent-ready issue before queueing when the target loop is disabled", () => { const resolveLoop = vi.fn(() => ({ enabled: false })); const trigger = getLoopAwareAgentReadyTriggerFromIssuesWebhook( @@ -350,7 +371,8 @@ describe("GitHub webhook helpers", () => { ); expect(response.status).toBe(202); - await expect(response.json()).resolves.toMatchObject({ + const responseBody = await response.json(); + expect(responseBody).toMatchObject({ accepted: true, duplicate: false, agentReadyTrigger: { @@ -477,7 +499,8 @@ describe("GitHub webhook helpers", () => { ); expect(response.status).toBe(202); - await expect(response.json()).resolves.toMatchObject({ + const responseBody = await response.json(); + expect(responseBody).toMatchObject({ accepted: true, agentReadyTrigger: { shouldTrigger: true, @@ -510,14 +533,21 @@ describe("GitHub webhook helpers", () => { ); expect(response.status).toBe(202); - await expect(response.json()).resolves.toMatchObject({ + const responseBody = await response.json(); + expect(responseBody).toMatchObject({ accepted: true, agentReadyTrigger: { shouldTrigger: true, workflow: "research", }, nextAction: "queue_deep_research_loop", + researchRun: { + artifactCount: 4, + mode: "simulated", + stageCount: 4, + }, }); + expect(responseBody).not.toHaveProperty("developmentRun"); }); it("records a failed outcome when accepted delivery processing throws", async () => { diff --git a/tests/unit/scripts/github-webhook-fixture.test.ts b/tests/unit/scripts/github-webhook-fixture.test.ts index 0c1ae7c..5bea16b 100644 --- a/tests/unit/scripts/github-webhook-fixture.test.ts +++ b/tests/unit/scripts/github-webhook-fixture.test.ts @@ -52,9 +52,9 @@ describe("GitHub webhook fixture script", () => { expect(result.stdout).toContain("x-github-delivery: dry-run-delivery"); expect(result.stdout).toContain("x-github-event: issues"); expect(result.stdout).toContain("x-hub-signature-256: sha256="); - expect(result.stdout).toContain("Issue: #11 Research agent-ready development loop skeleton"); + expect(result.stdout).toContain("Issue: #43 Research loop skeleton"); expect(result.stdout).toContain( - "Labels: agent-ready, spike, area:loops, area:agents, loop:development, priority:p0", + "Labels: agent-ready, spike, area:loops, area:agents, loop:research, priority:p2", ); expect(result.stdout).not.toContain("super-sensitive-fixture-secret"); expect(result.stderr).not.toContain("super-sensitive-fixture-secret"); From 4e186001bfee55547c3339d059c00289dd5ee854 Mon Sep 17 00:00:00 2001 From: Nate Summers Date: Tue, 21 Jul 2026 21:27:17 -0700 Subject: [PATCH 3/5] feat(agent): route research runs through root Expose loop identity in stage context and fail closed for undeclared research actors while preserving development fixture routing. Co-Authored-By: OpenAI Codex GPT-5 --- agent/instructions.md | 8 +++++ agent/tools/read_run_stage_context.ts | 27 +++++++++++---- .../agent/research-routing-discovery.test.ts | 34 +++++++++++++++++++ 3 files changed, 63 insertions(+), 6 deletions(-) create mode 100644 tests/unit/agent/research-routing-discovery.test.ts diff --git a/agent/instructions.md b/agent/instructions.md index 46868b4..26bb5cd 100644 --- a/agent/instructions.md +++ b/agent/instructions.md @@ -18,6 +18,14 @@ Always begin with `read_run_stage_context`. After planner delegation, call `apply_implementation_result`. A subagent response alone never changes durable state. +Route by the returned `run.loopKey` before considering the stage. The declared +development path remains `planning → test-writing → development → validation → +code-review → commit → pr → done`. The research skeleton declares +`planning → researching → authoring → done` for `research-loop`, but its research planner, +researcher fan-out, and research author are intentionally undeclared until +issues #44, #45, and #46. Fail closed for every research stage: do not delegate +to a development sibling, advance durable state, or fabricate an artifact. + After validation-reviewer delegation, call `apply_validation_review_result`. Only that root tool may apply the review recommendation: `commit` advances; `development` requeues development, validation, and review; `test-writing` diff --git a/agent/tools/read_run_stage_context.ts b/agent/tools/read_run_stage_context.ts index 2283534..459768e 100644 --- a/agent/tools/read_run_stage_context.ts +++ b/agent/tools/read_run_stage_context.ts @@ -6,13 +6,13 @@ import { db } from "@/db/client"; import { agentPlans, approvals, artifacts, loopRuns, runSteps } from "@/db/schema"; import { createImplementationFixtureHandoff } from "../implementation-fixture"; import { createPlanningAgentSeedPlan } from "../planning-agent"; +import { createPrPreparationFixtureContext } from "../pr-preparation-fixture"; import { resolveImplementerFixtureMode } from "../subagents/implementer/lib/fixture-mode"; import { resolvePrPreparerFixtureMode } from "../subagents/pr-preparer/lib/fixture-mode"; import { resolveTestWriterFixtureMode } from "../subagents/test-writer/lib/fixture-mode"; import { resolveValidationReviewerFixtureMode } from "../subagents/validation-reviewer/lib/fixture-mode"; import { computeTestPlanDigest } from "../test-writing-agent"; import { createValidationReviewFixtureContext } from "../validation-review-fixture"; -import { createPrPreparationFixtureContext } from "../pr-preparation-fixture"; export default defineTool({ description: "Read durable run, plan, approval, step, and artifact context for stage routing.", @@ -28,7 +28,7 @@ export default defineTool({ uri: artifact.uri, })), plans: [{ id: context.planId, plan: context.plan, status: context.planStatus }], - run: context.run, + run: { ...context.run, loopKey: "development-loop" }, steps: [ { ...context.validationStep, stage: "validation" }, { ...context.reviewStep, stage: "code-review" }, @@ -64,7 +64,7 @@ export default defineTool({ }, ], plans: [{ id: context.plan.identity.id, plan: context.plan, status: context.planStatus }], - run: context.run, + run: { ...context.run, loopKey: "development-loop" }, steps: [context.validationStep, { ...context.reviewStep, stage: "code-review" }], }; } @@ -101,7 +101,12 @@ export default defineTool({ }, ], plans: [{ id: planId, plan, status: "approved" }], - run: { currentStage: "development", id: runId, status: "running" }, + run: { + currentStage: "development", + id: runId, + loopKey: "development-loop", + status: "running", + }, steps: [ { id: "00000000-0000-4000-8000-000000000347", @@ -143,7 +148,12 @@ export default defineTool({ ], artifacts: [], plans: [{ id: planId, plan, status: "approved" }], - run: { currentStage: "test-writing", id: runId, status: "running" }, + run: { + currentStage: "test-writing", + id: runId, + loopKey: "development-loop", + status: "running", + }, steps: [ { id: "00000000-0000-4000-8000-000000000347", @@ -165,7 +175,12 @@ export default defineTool({ db.select().from(artifacts).where(eq(artifacts.runId, runId)), ]); return { - run: { currentStage: run.currentStage, id: run.id, status: run.status }, + run: { + currentStage: run.currentStage, + id: run.id, + loopKey: run.loopKey, + status: run.status, + }, steps: steps.map(({ id, stage, status }) => ({ id, stage, status })), plans: plans.map(({ id, plan, status }) => ({ id, plan, status })), approvals: planApprovals.map(({ id, metadata, status }) => ({ id, metadata, status })), diff --git a/tests/unit/agent/research-routing-discovery.test.ts b/tests/unit/agent/research-routing-discovery.test.ts new file mode 100644 index 0000000..5f197e4 --- /dev/null +++ b/tests/unit/agent/research-routing-discovery.test.ts @@ -0,0 +1,34 @@ +/** @vitest-environment node */ + +import { readdirSync, readFileSync } from "node:fs"; +import path from "node:path"; + +describe("neutral root research routing contract", () => { + it("returns loop identity and fails closed for the undeclared research actors", () => { + const instructions = readFileSync("agent/instructions.md", "utf8"); + const contextTool = readFileSync("agent/tools/read_run_stage_context.ts", "utf8"); + const declaredSubagents = readdirSync("agent/subagents", { withFileTypes: true }) + .filter((entry) => entry.isDirectory()) + .map((entry) => entry.name); + + expect(contextTool).toContain("loopKey"); + expect(instructions).toContain("research-loop"); + expect(instructions).toContain("planning → researching → authoring → done"); + expect(instructions).toMatch(/fail closed/i); + expect(instructions).toContain("#44"); + expect(instructions).toContain("#45"); + expect(instructions).toContain("#46"); + expect(declaredSubagents).not.toContain("research-planner"); + expect(declaredSubagents).not.toContain("researcher"); + expect(declaredSubagents).not.toContain("research-author"); + }); + + it("keeps every development fixture context explicitly scoped to development-loop", () => { + const contextTool = readFileSync( + path.join(process.cwd(), "agent/tools/read_run_stage_context.ts"), + "utf8", + ); + + expect(contextTool.match(/loopKey: "development-loop"/g)).toHaveLength(4); + }); +}); From fa11e4b090e79ddfd8bb4e6cebbc1b3cf7e9c3d0 Mon Sep 17 00:00:00 2001 From: Nate Summers Date: Tue, 21 Jul 2026 21:27:37 -0700 Subject: [PATCH 4/5] feat(portal): expose research run evidence Add the completed issue 43 fixture, research registry entry, Storybook story, persisted projections, responsive checks, and light/dark accessibility coverage. Co-Authored-By: OpenAI Codex GPT-5 --- .../portal/run-records-view.stories.tsx | 8 +++ src/lib/fixtures.ts | 10 ++- src/lib/runs/fixtures.ts | 38 ++++++++++ src/lib/runs/run-record.ts | 14 ++-- tests/e2e/portal.spec.ts | 62 +++++++++++++++++ tests/unit/portal/components.test.tsx | 61 +++++++++++++++- tests/unit/runs/run-record.test.ts | 69 ++++++++++++++++++- 7 files changed, 251 insertions(+), 11 deletions(-) diff --git a/src/components/portal/run-records-view.stories.tsx b/src/components/portal/run-records-view.stories.tsx index 709cf6b..8061f24 100644 --- a/src/components/portal/run-records-view.stories.tsx +++ b/src/components/portal/run-records-view.stories.tsx @@ -29,6 +29,14 @@ export const BlockedAndWaiting: Story = { }, }; +export const ResearchLoop: Story = { + args: { + initialRunId: "fixture-run-research", + runs: fixtureRuns.filter((run) => run.id === "fixture-run-research"), + sourceLabel: "Fixture fallback", + }, +}; + export const Empty: Story = { args: { runs: [], diff --git a/src/lib/fixtures.ts b/src/lib/fixtures.ts index 6175531..2fad60e 100644 --- a/src/lib/fixtures.ts +++ b/src/lib/fixtures.ts @@ -1,10 +1,10 @@ -import type { FixtureState, GitHubSettingRecord, LoopRegistryItem } from "@/lib/types"; import { createDevelopmentLoopRunSkeleton, projectDevelopmentLoopArtifacts, projectDevelopmentLoopTimeline, } from "@/lib/loops/development-run"; import { buildRunFixtureRecords } from "@/lib/runs/fixtures"; +import type { FixtureState, GitHubSettingRecord, LoopRegistryItem } from "@/lib/types"; const developmentLoopFixture = createDevelopmentLoopRunSkeleton({ mode: "simulated", @@ -202,6 +202,14 @@ export const portalFixture: FixtureState = { queueDepth: 2, risk: "high", }, + { + name: "Research routing", + state: "Planned", + enabled: true, + owner: "Loopworks", + queueDepth: 1, + risk: "low", + }, ] satisfies LoopRegistryItem[], timeline: developmentLoopTimeline, artifacts: [ diff --git a/src/lib/runs/fixtures.ts b/src/lib/runs/fixtures.ts index c99769e..ad15158 100644 --- a/src/lib/runs/fixtures.ts +++ b/src/lib/runs/fixtures.ts @@ -1,3 +1,8 @@ +import { + createResearchLoopRunSkeleton, + projectResearchLoopArtifacts, + projectResearchLoopTimeline, +} from "@/lib/loops/research-run"; import type { RunRecord, ValidationGateSummaryRecord } from "@/lib/types"; const emptyValidationSummary: ValidationGateSummaryRecord = { @@ -76,6 +81,19 @@ const invalidRawArtifactValidationSummary: ValidationGateSummaryRecord = { }; export function buildRunFixtureRecords(): RunRecord[] { + const researchSkeleton = createResearchLoopRunSkeleton({ + mode: "simulated", + now: new Date("2026-07-21T16:00:00.000Z"), + trigger: { + issueNumber: 43, + issueUrl: "https://github.com/ncolesummers/loopworks/issues/43", + labels: ["agent-ready", "spike", "loop:research"], + milestone: "M3 Durable Loop MVP", + repositoryFullName: "ncolesummers/loopworks", + title: "Research loop skeleton", + }, + }); + return [ { id: "fixture-run-waiting-approval", @@ -319,5 +337,25 @@ export function buildRunFixtureRecords(): RunRecord[] { }, ], }, + { + age: "Complete", + approvals: [], + artifacts: projectResearchLoopArtifacts(researchSkeleton), + currentStage: "done", + id: "fixture-run-research", + issue: "#43", + issueHref: "https://github.com/ncolesummers/loopworks/issues/43", + loopKey: "research-loop", + priorityLabel: "Succeeded", + queuedAt: "09:00", + repositoryFullName: "ncolesummers/loopworks", + status: "succeeded", + steps: projectResearchLoopTimeline(researchSkeleton).map((event, index) => ({ + ...event, + id: `fixture-step-research-${researchSkeleton.stages[index]?.key ?? index}`, + status: "succeeded" as const, + })), + validationSummary: emptyValidationSummary, + }, ]; } diff --git a/src/lib/runs/run-record.ts b/src/lib/runs/run-record.ts index 835dd14..47db9cd 100644 --- a/src/lib/runs/run-record.ts +++ b/src/lib/runs/run-record.ts @@ -2,6 +2,8 @@ import { asc, eq, inArray } from "drizzle-orm"; import type { db } from "@/db/client"; import { approvals, artifacts, loopRuns, repositories, runSteps } from "@/db/schema"; +import { validationReportArtifactMetadataSchema } from "@/lib/loops/validation-report"; +import type { LoopworksLogger } from "@/lib/observability/logger"; import { isProductionRuntime } from "@/lib/runtime"; import type { ArtifactKind, @@ -15,8 +17,6 @@ import type { ValidationGateRecord, ValidationGateSummaryRecord, } from "@/lib/types"; -import { validationReportArtifactMetadataSchema } from "@/lib/loops/validation-report"; -import type { LoopworksLogger } from "@/lib/observability/logger"; export type RunRecordDatabase = Pick; type ArtifactRow = typeof artifacts.$inferSelect; @@ -117,6 +117,8 @@ function timelineKindForStage(stage: string): TimelineKind { planning: "planning", "test-writing": "test", development: "development", + researching: "research", + authoring: "authoring", validation: "validation", review: "review", "code-review": "review", @@ -128,7 +130,7 @@ function timelineKindForStage(stage: string): TimelineKind { return kinds[stage] ?? "state"; } -function artifactKind(type: string): ArtifactKind { +function artifactKind(type: string, metadata?: Record | null): ArtifactKind { if (type === "validation_report") { return "validation"; } @@ -141,6 +143,10 @@ function artifactKind(type: string): ArtifactKind { return "review"; } + if (metadataString(metadata, "researchArtifactKind") === "research_plan") { + return "review"; + } + return "log"; } @@ -366,7 +372,7 @@ export async function readRunRecords(input: { const artifactsForRun = artifactRowsForRun.map((artifact) => ({ detail: metadataString(artifact.metadata, "detail") ?? artifact.title, href: artifact.uri, - kind: artifactKind(artifact.type), + kind: artifactKind(artifact.type, artifact.metadata), label: artifact.title, state: artifactState({ stepStatus: artifact.stepId ? stepStatusById.get(artifact.stepId) : undefined, diff --git a/tests/e2e/portal.spec.ts b/tests/e2e/portal.spec.ts index da9d990..03ecd23 100644 --- a/tests/e2e/portal.spec.ts +++ b/tests/e2e/portal.spec.ts @@ -182,6 +182,68 @@ test.describe("Loopworks portal", () => { await expect(page.getByText("Skipped: loop_disabled")).toBeVisible(); }); + test("Research routing records its own disabled trigger reason", async ({ page }) => { + await page.goto("/loops"); + + const researchLoop = page.getByRole("switch", { name: "Research routing" }); + await expect(researchLoop).toBeChecked(); + await researchLoop.click(); + + await expect(researchLoop).not.toBeChecked(); + await expect(page.getByText("Skipped: loop_disabled")).toBeVisible(); + }); + + test("research run shows ordered stages and accessible artifact links", async ({ page }) => { + await page.goto("/runs?run=fixture-run-research"); + + await expect(page.getByRole("button", { name: /#43 ncolesummers\/loopworks/ })).toHaveAttribute( + "aria-pressed", + "true", + ); + const stages = ["Planning", "Researching", "Authoring", "Done"]; + let previousY = -1; + for (const stage of stages) { + const item = page.getByText(stage, { exact: true }).last(); + await expect(item).toBeVisible(); + const box = await item.boundingBox(); + expect(box, stage).not.toBeNull(); + if (box) { + expect(box.y, `${stage} should appear after prior stage`).toBeGreaterThan(previousY); + previousY = box.y; + } + } + for (const artifact of [ + "Research plan", + "Findings artifacts", + "Research document", + "Completion summary", + ]) { + await expect(page.getByRole("link", { name: artifact })).toBeVisible(); + } + }); + + test("research run remains usable at mobile width", async ({ page }) => { + await page.setViewportSize({ width: 390, height: 844 }); + await page.goto("/runs?run=fixture-run-research"); + + await expect(page.getByText("Researching", { exact: true }).last()).toBeVisible(); + expect( + await page.evaluate(() => document.documentElement.scrollWidth <= window.innerWidth), + ).toBe(true); + }); + + for (const colorScheme of ["light", "dark"] as const) { + test.describe(`research run color scheme: ${colorScheme}`, () => { + test.use({ colorScheme }); + + test("research run has no accessibility violations", async ({ page }) => { + await page.goto("/runs?run=fixture-run-research"); + const results = await new AxeBuilder({ page }).analyze(); + expect(results.violations, `${colorScheme} research run`).toEqual([]); + }); + }); + } + // Persona A01/A03/R01: deterministic evidence is visible before PR/review stages. test("run timeline shows ordered stages and validation evidence before review", async ({ page, diff --git a/tests/unit/portal/components.test.tsx b/tests/unit/portal/components.test.tsx index e014ed4..0d3fc74 100644 --- a/tests/unit/portal/components.test.tsx +++ b/tests/unit/portal/components.test.tsx @@ -1,20 +1,25 @@ import { cleanup, fireEvent, render, screen, waitFor } from "@testing-library/react"; - -import { ArtifactListItem } from "@/components/portal/artifact-list-item"; import { ApprovalGatePanel } from "@/components/portal/approval-gate-panel"; -import { DeploymentSummary } from "@/components/portal/deployment-summary"; +import { ArtifactListItem } from "@/components/portal/artifact-list-item"; import { LoopRegistry } from "@/components/portal/dashboard-view"; +import { DeploymentSummary } from "@/components/portal/deployment-summary"; import { GitHubSettingsView } from "@/components/portal/github-settings-view"; import { RepoCatalog } from "@/components/portal/repo-catalog"; import { RunRecordsView } from "@/components/portal/run-records-view"; import { RunTimelineItem } from "@/components/portal/run-timeline-item"; import { ValidationGateSummary } from "@/components/portal/validation-gate-summary"; import { ValidationResultSummary } from "@/components/portal/validation-result-summary"; +import { portalFixture } from "@/lib/fixtures"; import { createDevelopmentLoopRunSkeleton, projectDevelopmentLoopArtifacts, projectDevelopmentLoopTimeline, } from "@/lib/loops/development-run"; +import { + createResearchLoopRunSkeleton, + projectResearchLoopArtifacts, + projectResearchLoopTimeline, +} from "@/lib/loops/research-run"; import type { ArtifactRecord, DeploymentRecord, @@ -46,6 +51,18 @@ describe("portal reusable components", () => { expect(screen.getByText("No validation results yet")).toBeTruthy(); }); + it("declares the enabled Research routing loop registry fixture", () => { + expect(portalFixture.loops).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + enabled: true, + name: "Research routing", + state: "Planned", + }), + ]), + ); + }); + it("does not render unsafe artifact or evidence hrefs as links", () => { const artifact: ArtifactRecord = { label: "Dangerous artifact", @@ -201,6 +218,44 @@ describe("portal reusable components", () => { expect(screen.getByRole("link", { name: "PR intent" })).toBeTruthy(); }); + it("renders every research stage and artifact contract through shared portal primitives", () => { + const skeleton = createResearchLoopRunSkeleton({ + mode: "simulated", + now: new Date("2026-07-21T16:00:00.000Z"), + trigger: { + issueNumber: 43, + issueUrl: "https://github.com/ncolesummers/loopworks/issues/43", + labels: ["agent-ready", "spike", "loop:research"], + milestone: "M3 Durable Loop MVP", + repositoryFullName: "ncolesummers/loopworks", + title: "Research loop skeleton", + }, + }); + + render( +
+ {projectResearchLoopTimeline(skeleton).map((event) => ( + + ))} + {projectResearchLoopArtifacts(skeleton).map((artifact) => ( + + ))} +
, + ); + + for (const stage of ["Planning", "Researching", "Authoring", "Done"]) { + expect(screen.getAllByText(stage, { exact: true }).length).toBeGreaterThan(0); + } + for (const artifact of [ + "Research plan", + "Findings artifacts", + "Research document", + "Completion summary", + ]) { + expect(screen.getByRole("link", { name: artifact })).toBeTruthy(); + } + }); + it("renders blocked and waiting-for-approval runs at a glance with detail evidence", () => { const runs: RunRecord[] = [ { diff --git a/tests/unit/runs/run-record.test.ts b/tests/unit/runs/run-record.test.ts index a90f4cf..7b54ebe 100644 --- a/tests/unit/runs/run-record.test.ts +++ b/tests/unit/runs/run-record.test.ts @@ -1,18 +1,19 @@ /** @vitest-environment node */ import { eq } from "drizzle-orm"; -import { artifacts, loopRuns } from "@/db/schema"; +import { artifacts, loopRuns, repositories } from "@/db/schema"; +import { createResearchLoopRun } from "@/lib/loops/research-run"; import { createValidationReportArtifactMetadata, type ValidationReportV1, } from "@/lib/loops/validation-report"; +import { buildRunFixtureRecords } from "@/lib/runs/fixtures"; import { getRunRecordsForPortal, getRunRecordsForResult, readRunRecords, } from "@/lib/runs/run-record"; -import { buildRunFixtureRecords } from "@/lib/runs/fixtures"; -import { demoSeedIds, seedDemoData, type SeedDatabase } from "@/lib/seed/demo-data"; +import { demoSeedIds, type SeedDatabase, seedDemoData } from "@/lib/seed/demo-data"; import { createPgliteTestDatabase, type PgliteTestDatabase } from "../../helpers/pglite"; @@ -377,6 +378,68 @@ describe("run records (pglite integration)", () => { ).toEqual([]); }); + it("includes a completed issue-43 research fixture with all stage and artifact contracts", () => { + const researchRun = buildRunFixtureRecords().find((run) => run.id === "fixture-run-research"); + + expect(researchRun).toMatchObject({ + currentStage: "done", + issue: "#43", + loopKey: "research-loop", + status: "succeeded", + }); + expect(researchRun?.steps.map((step) => step.title)).toEqual([ + "Planning", + "Researching", + "Authoring", + "Done", + ]); + expect(researchRun?.artifacts.map((artifact) => artifact.label)).toEqual([ + "Research plan", + "Findings artifacts", + "Research document", + "Completion summary", + ]); + }); + + it("projects persisted research stages and semantic artifacts without fixture-only fallbacks", async () => { + await context.db.insert(repositories).values({ + enabledLoops: ["Research routing"], + fullName: "ncolesummers/loopworks", + githubRepoId: 43_000_043, + name: "loopworks", + owner: "ncolesummers", + validationGates: ["Focused tests"], + }); + await createResearchLoopRun({ + database: context.db as never, + now: () => new Date("2026-07-21T16:00:00.000Z"), + trigger: { + deliveryId: "run-record-research-delivery", + issueNumber: 43, + repositoryFullName: "ncolesummers/loopworks", + }, + }); + + const result = await readRunRecords({ + database: context.db, + now: new Date("2026-07-21T16:10:00.000Z"), + }); + expect(result.source).toBe("db"); + const researchRun = result.runs.find((run) => run.loopKey === "research-loop"); + expect(researchRun?.steps.map((step) => step.kind)).toEqual([ + "planning", + "research", + "authoring", + "done", + ]); + expect(researchRun?.artifacts.map((artifact) => artifact.kind)).toEqual([ + "review", + "log", + "log", + "log", + ]); + }); + it("uses explicit non-production fixture mode without reading run records", async () => { const fixtureRuns = buildRunFixtureRecords(); const database = { From a4b9243917ae8caee1a9dde9d431c9fd76a7a18f Mon Sep 17 00:00:00 2001 From: Nate Summers Date: Tue, 21 Jul 2026 21:27:55 -0700 Subject: [PATCH 5/5] docs(architecture): document research loop skeleton Record the research actor and artifact boundaries, root orchestration contract, manifest policy, telemetry behavior, and operator expectations in the existing durable documentation. Co-Authored-By: OpenAI Codex GPT-5 --- ...012-telemetry-backend-and-metric-contract.md | 4 ++-- ...chestrator-and-isolated-subagent-handoffs.md | 16 +++++++++++++++- docs/architecture.md | 9 ++++++++- docs/loop-manifest.md | 17 +++++++++++++++++ docs/observability.md | 13 ++++++++----- docs/personas-and-test-scenarios.md | 4 ++-- 6 files changed, 52 insertions(+), 11 deletions(-) diff --git a/docs/adr/0012-telemetry-backend-and-metric-contract.md b/docs/adr/0012-telemetry-backend-and-metric-contract.md index 3e3829c..418c948 100644 --- a/docs/adr/0012-telemetry-backend-and-metric-contract.md +++ b/docs/adr/0012-telemetry-backend-and-metric-contract.md @@ -57,7 +57,7 @@ Run cancellations are counted as `loopworks.run.completed` with `status="cancell ### Relationship to `observability_events` -The `observability_events.metric_name` column keeps its snake_case event vocabulary (for example `development_loop_run_created`): those rows are durable control-plane records for audit and replay, per the "logs are not the event store" rule. OTel meters are operational telemetry. Where both exist, one instrumentation helper emits both from a single call site, and the DB event name maps to a meter plus attributes (`development_loop_run_created` → `loopworks.run.started{loop.key="development-loop"}`). Neither vocabulary may be written ad hoc outside `src/lib/observability/` helpers. +The `observability_events.metric_name` column keeps its snake_case event vocabulary (for example `development_loop_run_created` and `research_loop_run_created`): those rows are durable control-plane records for audit and replay, per the "logs are not the event store" rule. OTel meters are operational telemetry. Where both exist, one instrumentation helper emits both from a single call site. Development creation maps to `loopworks.run.started{loop.key="development-loop", trigger.label="agent-ready"}` and research creation maps to `loopworks.run.started{loop.key="research-loop", trigger.label="spike"}`. Neither vocabulary may be written ad hoc outside `src/lib/observability/` helpers. ### Correlation alignment @@ -71,7 +71,7 @@ The `observability_events.metric_name` column keeps its snake_case event vocabul | `repository` | `repositories` | `loopworks.repository` | | `traceId` | `loop_runs.trace_id`, `run_steps.trace_id`, `observability_events.trace_id` | W3C trace id from active span context | -The active W3C trace id is persisted into the `trace_id` columns at run and step creation, closing the loop between spans, logs, and durable records. Artifacts carry no `trace_id` of their own: `artifacts.run_id` and `artifacts.step_id` correlate them transitively through the owning run and step. +The active W3C trace id is persisted into the `trace_id` columns at development- and research-run creation, their steps, and their durable creation events, closing the loop between spans, logs, and durable records. Artifacts carry no `trace_id` of their own: `artifacts.run_id` and `artifacts.step_id` correlate them transitively through the owning run and step. ### Rollout diff --git a/docs/adr/0015-stage-orchestrator-and-isolated-subagent-handoffs.md b/docs/adr/0015-stage-orchestrator-and-isolated-subagent-handoffs.md index 147bab5..50d2b2a 100644 --- a/docs/adr/0015-stage-orchestrator-and-isolated-subagent-handoffs.md +++ b/docs/adr/0015-stage-orchestrator-and-isolated-subagent-handoffs.md @@ -6,7 +6,8 @@ Date: 2026-07-13 Driving issues: [#47](https://github.com/ncolesummers/loopworks/issues/47), [#48](https://github.com/ncolesummers/loopworks/issues/48), and [#49](https://github.com/ncolesummers/loopworks/issues/49), and -[#50](https://github.com/ncolesummers/loopworks/issues/50) +[#50](https://github.com/ncolesummers/loopworks/issues/50), with the +research-loop generality probe in [#43](https://github.com/ncolesummers/loopworks/issues/43) ## Context @@ -31,6 +32,16 @@ implementer, validation-reviewer, and PR-preparer use `openai/gpt-5.6-terra`. Ea independent `xhigh` reasoning configuration so model routing can evolve per stage without changing the shared topology. +The same neutral root routes by durable `loopKey`. Issue #43 declares the +research sequence `planning → researching → authoring → done`, actors +`research-planner`, `researcher`, `research-author`, and `loopworks`, and the +artifact boundaries for a research plan, per-subquestion findings index, +research document, and completion summary. It intentionally does not declare +research Eve subagents, live fan-out, provider evals, or versioned research +payload schemas. Until issues #44-#46 declare those siblings and transitions, +the root fails closed rather than routing research stages to development actors +or advancing durable state. + Planner, test-writer, and implementer receive repository-scoped discovery, text search, and line-range read tools against their isolated commit-pinned checkouts. The tools @@ -129,6 +140,9 @@ deterministic control-plane behavior rather than model judgment. 7. PR-preparation tests cover exact evidence binding, non-UI empty manifests, idempotent persistence, conflicting replay, and guarded-writer approval. 8. `bun run validate` and `bun run build` pass before review. +9. Research-routing discovery proves `loopKey` is returned, the root fails + closed for undeclared research siblings, and no research subagent directories + or versioned research payload schemas are introduced by #43. ## Follow-Ups diff --git a/docs/architecture.md b/docs/architecture.md index 09221a2..a819a81 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -177,6 +177,8 @@ stage subagent, validates the typed result, and invokes deterministic control-plane transitions. Stage subagents are siblings with independent model, tool, and isolated sandbox contracts; they do not own GitHub writes or workflow state transitions. +Routing begins with the durable `loopKey`, so development and research stage +names cannot accidentally share actors. The planner subagent should: @@ -188,7 +190,12 @@ The planner subagent should: Stage specialists use typed handoffs and independent capability boundaries: -1. Research loop skeleton, research planner, researcher, and research author agents — the `spike` plus `agent-ready` research loop, run in parallel to the development loop. +1. Research loop skeleton — issue #43 persists and projects `planning → + researching → authoring → done` with actor and artifact placeholders. The + findings contract is `one_per_subquestion` from isolated child sessions, but + no child sessions run yet. Research planner, fan-out, authoring agent, live + eval, and versioned payload work stays in #44-#46; the root fails closed until + those siblings exist. 2. Test-writer subagent — test-writing stage, red test evidence plus a reusable automated test plan, explicit seed data, and a bounded test-only patch. 3. Implementation subagent — development stage, digest-bound production patch plus signed focused and aggregate green evidence. It reuses the exact diff --git a/docs/loop-manifest.md b/docs/loop-manifest.md index fdb08fb..851907a 100644 --- a/docs/loop-manifest.md +++ b/docs/loop-manifest.md @@ -91,6 +91,23 @@ skipped/no-op reason such as `loop_disabled` so operators can explain why an `agent-ready` issue did not start. Research-loop disabled evidence is tracked separately from the development-loop skeleton. +The enabled `research-loop` is a parallel, fixture-backed generality probe. It +requires both `spike` and `agent-ready`, uses a separate +`repo:{repo}:loop:research` concurrency group, and permits only repository read, +browser, and validation tool categories. GitHub writeback is disabled; only +manifest rollout requires approval. Its deterministic sequence and persisted +placeholder contract are: + +1. Planning by `research-planner`: one research plan (`plan`). +2. Researching by `researcher`: findings (`other`), indexed + `one_per_subquestion` from isolated child sessions. +3. Authoring by `research-author`: one research document (`other`). +4. Done by `loopworks`: one completion summary (`other`). + +Issue #43 creates only the four steps and four placeholders. It does not create +an agent plan or approval row, run live fan-out, or define versioned research +payload schemas; those behaviors remain in #44-#46. + The planning-to-test-writing boundary requires an `approved` `plan-review` record tied to the exact run, plan row, and canonical plan digest. The test-writing stage succeeds only when every approved-plan acceptance criterion diff --git a/docs/observability.md b/docs/observability.md index fb43d47..a47080e 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -77,14 +77,17 @@ The internal control plane should eventually expose: ADR 0012 defines the concrete metric-name contract. The exported metric names live in `src/lib/observability/metrics.ts`; code should import helpers from that -module instead of naming OTel meters ad hoc. The first wired metric maps the -durable `development_loop_run_created` control-plane event to -`loopworks.run.started` with `loop.key`, `repository`, and `trigger.label` -attributes. +module instead of naming OTel meters ad hoc. The run-creation helpers map +durable `development_loop_run_created` and `research_loop_run_created` +control-plane events to `loopworks.run.started` with `loop.key`, `repository`, +and `trigger.label` attributes. Research creation uses +`loop.key=research-loop` and `trigger.label=spike`; disabled research triggers +persist `research_loop_noop` without fabricating a run or emitting a +run-started metric. ## Tracing Direction -The MVP logger is compatible with trace context. Development-loop run creation +The MVP logger is compatible with trace context. Development- and research-loop run creation persists the active W3C trace id into `loop_runs.trace_id`, `run_steps.trace_id`, and `observability_events.trace_id`, allowing Axiom traces, stdout logs, and durable run records to be correlated. diff --git a/docs/personas-and-test-scenarios.md b/docs/personas-and-test-scenarios.md index 0e5ebee..a9638ba 100644 --- a/docs/personas-and-test-scenarios.md +++ b/docs/personas-and-test-scenarios.md @@ -93,9 +93,9 @@ Risks: | P03 | Product Operator | A durable decision from planning links to an ADR proposal or accepted ADR. | Integration, docs review | | P04 | Product Operator | An operator switches between light and dark mode from the app shell; the choice persists across reloads and both themes meet contrast. | Playwright, a11y | | M01 | Maintainer | Catalog rows show owner, framework, CI commands, docs, observability, design-system, enabled loops, Vercel project links, and search/filter controls. | Playwright, Storybook | -| M02 | Maintainer | Turning a loop off prevents trigger execution and records a skipped reason. | Unit, Playwright | +| M02 | Maintainer | Turning either development or research routing off prevents trigger execution and records the loop-specific skipped reason without fabricating a run. | Unit, integration, Playwright | | M03 | Maintainer | Missing Vercel credentials in dev returns explicit fixture fallback metadata; production does not silently return fixtures. | Unit, integration | -| A01 | Agent Supervisor | Run detail shows planning, test-writing, development, validation, code review, commit, PR, and done stages, including separate red-evidence and automated-test-plan artifacts. | Unit, Storybook, Playwright | +| A01 | Agent Supervisor | Run detail shows the exact loop sequence and artifacts: all development stages with separate red/test-plan evidence, or research planning, researching, authoring, and done with four placeholder contracts. | Unit, Storybook, Playwright | | A02 | Agent Supervisor | Approval gates show requested, approved, rejected, bypassed, and expired states with actor and evidence; test writing requires an exact approved plan review. | Unit, Storybook, Playwright | | A03 | Agent Supervisor | AC-mapped expected-red evidence appears before implementation, and green deterministic validation appears before LLM review or judgment. | Integration, Playwright | | R01 | Reviewer | A PR intent or created PR links to the source issue, run, validation artifacts, and Vercel preview. | Integration, Playwright |