diff --git a/src/snapshot-tests/__tests__/json-harness-error-state.test.ts b/src/snapshot-tests/__tests__/json-harness-error-state.test.ts index 17e144d3..573b6634 100644 --- a/src/snapshot-tests/__tests__/json-harness-error-state.test.ts +++ b/src/snapshot-tests/__tests__/json-harness-error-state.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { resolveCliJsonSnapshotErrorState } from '../harness.ts'; +import { assertCliSnapshotProcessResult, resolveCliJsonSnapshotErrorState } from '../harness.ts'; import { resolveMcpSnapshotErrorState } from '../mcp-harness.ts'; import type { StructuredOutputEnvelope } from '../../types/structured-output.ts'; @@ -17,6 +17,42 @@ const errorEnvelope: StructuredOutputEnvelope = { error: 'Failed', }; +describe('CLI snapshot process result guard', () => { + it('accepts completed domain invocations without process stderr', () => { + expect(() => + assertCliSnapshotProcessResult( + { error: undefined, signal: null, status: 1, stderr: '' }, + 'tool', + ), + ).not.toThrow(); + }); + + it('rejects process stderr so domain snapshots cannot hide user-visible noise', () => { + expect(() => + assertCliSnapshotProcessResult( + { error: undefined, signal: null, status: 0, stderr: 'warning\n' }, + 'tool', + ), + ).toThrow('CLI process emitted unexpected stderr for tool:\nwarning'); + }); + + it('rejects failed process execution before snapshot matching', () => { + expect(() => + assertCliSnapshotProcessResult( + { error: new Error('spawn failed'), signal: null, status: null, stderr: '' }, + 'tool', + ), + ).toThrow('CLI process failed for tool: spawn failed'); + + expect(() => + assertCliSnapshotProcessResult( + { error: undefined, signal: 'SIGTERM', status: null, stderr: '' }, + 'tool', + ), + ).toThrow('CLI process for tool was terminated by signal SIGTERM.'); + }); +}); + describe('JSON snapshot harness error state', () => { it('uses CLI process status and envelope.didError when they agree', () => { expect(resolveCliJsonSnapshotErrorState(0, successEnvelope, 'tool')).toBe(false); diff --git a/src/snapshot-tests/harness.ts b/src/snapshot-tests/harness.ts index 42da3036..4f5b5114 100644 --- a/src/snapshot-tests/harness.ts +++ b/src/snapshot-tests/harness.ts @@ -56,6 +56,34 @@ function runSnapshotCli( }); } +function readProcessOutput(output: string | Buffer | null | undefined): string { + return typeof output === 'string' ? output : (output?.toString('utf8') ?? ''); +} + +export function assertCliSnapshotProcessResult( + result: Pick, 'error' | 'signal' | 'status' | 'stderr'>, + label: string, +): void { + if (result.error) { + throw new Error(`CLI process failed for ${label}: ${result.error.message}`); + } + + if (result.signal) { + throw new Error(`CLI process for ${label} was terminated by signal ${result.signal}.`); + } + + if (result.status === null) { + throw new Error( + `CLI process exit status was null for ${label}; the process may have timed out or been killed by a signal.`, + ); + } + + const stderr = readProcessOutput(result.stderr).trim(); + if (stderr.length > 0) { + throw new Error(`CLI process emitted unexpected stderr for ${label}:\n${stderr}`); + } +} + function parseStructuredEnvelope( stdout: string, label: string, @@ -107,9 +135,10 @@ export async function createSnapshotHarness( throw new Error(`Tool '${cliToolName}' in workflow '${workflow}' is not CLI-available`); } + const label = `${workflow}/${cliToolName}`; const result = runSnapshotCli(workflow, cliToolName, args, 'text', options); - const stdout = - typeof result.stdout === 'string' ? result.stdout : (result.stdout?.toString('utf8') ?? ''); + assertCliSnapshotProcessResult(result, label); + const stdout = readProcessOutput(result.stdout); return { text: normalizeSnapshotOutput(stdout), @@ -141,19 +170,16 @@ export async function createCliJsonSnapshotHarness( throw new Error(`Tool '${cliToolName}' in workflow '${workflow}' is not CLI-available`); } + const label = `${workflow}/${cliToolName}`; const result = runSnapshotCli(workflow, cliToolName, args, 'json', options); - const stdout = - typeof result.stdout === 'string' ? result.stdout : (result.stdout?.toString('utf8') ?? ''); - const envelope = parseStructuredEnvelope(stdout, `${workflow}/${cliToolName}`); + assertCliSnapshotProcessResult(result, label); + const stdout = readProcessOutput(result.stdout); + const envelope = parseStructuredEnvelope(stdout, label); return { text: formatStructuredEnvelopeFixture(envelope), rawText: stdout, - isError: resolveCliJsonSnapshotErrorState( - result.status, - envelope, - `${workflow}/${cliToolName}`, - ), + isError: resolveCliJsonSnapshotErrorState(result.status, envelope, label), structuredEnvelope: envelope, }; }