From 7320348e6fb96ac814db9db7edf7995f181a5bcd Mon Sep 17 00:00:00 2001
From: Cameron Cooke <web@cameroncooke.com>
Date: Sun, 17 May 2026 19:37:40 +0100
Subject: [PATCH] fix(snapshot-tests): Guard CLI snapshot process output

Fail CLI domain snapshot invocations when the CLI process itself fails,
is terminated, has no exit status, or emits unexpected stderr. This keeps
fixtures focused on domain tool output while preventing hidden process-level
noise from being ignored.

Co-Authored-By: OpenAI Codex <codex@openai.com>
---
 .../json-harness-error-state.test.ts          | 38 ++++++++++++++-
 src/snapshot-tests/harness.ts                 | 46 +++++++++++++++----
 2 files changed, 73 insertions(+), 11 deletions(-)

diff --git a/src/snapshot-tests/__tests__/json-harness-error-state.test.ts b/src/snapshot-tests/__tests__/json-harness-error-state.test.ts
index 17e144d3..573b6634 100644
--- a/src/snapshot-tests/__tests__/json-harness-error-state.test.ts
+++ b/src/snapshot-tests/__tests__/json-harness-error-state.test.ts
@@ -1,5 +1,5 @@
 import { describe, expect, it } from 'vitest';
-import { resolveCliJsonSnapshotErrorState } from '../harness.ts';
+import { assertCliSnapshotProcessResult, resolveCliJsonSnapshotErrorState } from '../harness.ts';
 import { resolveMcpSnapshotErrorState } from '../mcp-harness.ts';
 import type { StructuredOutputEnvelope } from '../../types/structured-output.ts';
 
@@ -17,6 +17,42 @@ const errorEnvelope: StructuredOutputEnvelope<null> = {
   error: 'Failed',
 };
 
+describe('CLI snapshot process result guard', () => {
+  it('accepts completed domain invocations without process stderr', () => {
+    expect(() =>
+      assertCliSnapshotProcessResult(
+        { error: undefined, signal: null, status: 1, stderr: '' },
+        'tool',
+      ),
+    ).not.toThrow();
+  });
+
+  it('rejects process stderr so domain snapshots cannot hide user-visible noise', () => {
+    expect(() =>
+      assertCliSnapshotProcessResult(
+        { error: undefined, signal: null, status: 0, stderr: 'warning\n' },
+        'tool',
+      ),
+    ).toThrow('CLI process emitted unexpected stderr for tool:\nwarning');
+  });
+
+  it('rejects failed process execution before snapshot matching', () => {
+    expect(() =>
+      assertCliSnapshotProcessResult(
+        { error: new Error('spawn failed'), signal: null, status: null, stderr: '' },
+        'tool',
+      ),
+    ).toThrow('CLI process failed for tool: spawn failed');
+
+    expect(() =>
+      assertCliSnapshotProcessResult(
+        { error: undefined, signal: 'SIGTERM', status: null, stderr: '' },
+        'tool',
+      ),
+    ).toThrow('CLI process for tool was terminated by signal SIGTERM.');
+  });
+});
+
 describe('JSON snapshot harness error state', () => {
   it('uses CLI process status and envelope.didError when they agree', () => {
     expect(resolveCliJsonSnapshotErrorState(0, successEnvelope, 'tool')).toBe(false);
diff --git a/src/snapshot-tests/harness.ts b/src/snapshot-tests/harness.ts
index 42da3036..4f5b5114 100644
--- a/src/snapshot-tests/harness.ts
+++ b/src/snapshot-tests/harness.ts
@@ -56,6 +56,34 @@ function runSnapshotCli(
   });
 }
 
+function readProcessOutput(output: string | Buffer | null | undefined): string {
+  return typeof output === 'string' ? output : (output?.toString('utf8') ?? '');
+}
+
+export function assertCliSnapshotProcessResult(
+  result: Pick<ReturnType<typeof spawnSync>, 'error' | 'signal' | 'status' | 'stderr'>,
+  label: string,
+): void {
+  if (result.error) {
+    throw new Error(`CLI process failed for ${label}: ${result.error.message}`);
+  }
+
+  if (result.signal) {
+    throw new Error(`CLI process for ${label} was terminated by signal ${result.signal}.`);
+  }
+
+  if (result.status === null) {
+    throw new Error(
+      `CLI process exit status was null for ${label}; the process may have timed out or been killed by a signal.`,
+    );
+  }
+
+  const stderr = readProcessOutput(result.stderr).trim();
+  if (stderr.length > 0) {
+    throw new Error(`CLI process emitted unexpected stderr for ${label}:\n${stderr}`);
+  }
+}
+
 function parseStructuredEnvelope(
   stdout: string,
   label: string,
@@ -107,9 +135,10 @@ export async function createSnapshotHarness(
       throw new Error(`Tool '${cliToolName}' in workflow '${workflow}' is not CLI-available`);
     }
 
+    const label = `${workflow}/${cliToolName}`;
     const result = runSnapshotCli(workflow, cliToolName, args, 'text', options);
-    const stdout =
-      typeof result.stdout === 'string' ? result.stdout : (result.stdout?.toString('utf8') ?? '');
+    assertCliSnapshotProcessResult(result, label);
+    const stdout = readProcessOutput(result.stdout);
 
     return {
       text: normalizeSnapshotOutput(stdout),
@@ -141,19 +170,16 @@ export async function createCliJsonSnapshotHarness(
       throw new Error(`Tool '${cliToolName}' in workflow '${workflow}' is not CLI-available`);
     }
 
+    const label = `${workflow}/${cliToolName}`;
     const result = runSnapshotCli(workflow, cliToolName, args, 'json', options);
-    const stdout =
-      typeof result.stdout === 'string' ? result.stdout : (result.stdout?.toString('utf8') ?? '');
-    const envelope = parseStructuredEnvelope(stdout, `${workflow}/${cliToolName}`);
+    assertCliSnapshotProcessResult(result, label);
+    const stdout = readProcessOutput(result.stdout);
+    const envelope = parseStructuredEnvelope(stdout, label);
 
     return {
       text: formatStructuredEnvelopeFixture(envelope),
       rawText: stdout,
-      isError: resolveCliJsonSnapshotErrorState(
-        result.status,
-        envelope,
-        `${workflow}/${cliToolName}`,
-      ),
+      isError: resolveCliJsonSnapshotErrorState(result.status, envelope, label),
       structuredEnvelope: envelope,
     };
   }