Fix nightly eval response extraction (Fixes #2651)

evals/README.md

@@ -72,7 +72,7 @@ describe('my_feature', () => {
   - `settings`: Override LLxprt settings for this test
   - `fakeResponsesPath`: Use canned responses instead of live model
 
-Note: Tool call logs are automatically saved to `evals/logs/` for all evals.
+Note: Versioned JSON diagnostic logs are automatically saved to `evals/logs/` for all evals. Each log contains the raw process capture and parsed tool calls.
 
 ### Available Assertions
 

evals/test-helper.ts

@@ -6,28 +6,102 @@
 
 import { it } from 'vitest';
 import fs from 'node:fs';
+import { z } from 'zod';
 import {
   TestRig,
   type TestRigSetupOptions,
+  type RunCapture,
 } from '@vybestack/llxprt-code-test-utils';
 
 export * from '@vybestack/llxprt-code-test-utils';
 
 export type EvalPolicy = 'ALWAYS_PASSES' | 'USUALLY_PASSES';
 
+const EvalOutputSchema = z.object({ response: z.string() });
+
+export function buildEvalArgs(prompt: string): string[] {
+  return [`--prompt=${prompt}`, '--output-format', 'json'];
+}
+
+export function extractModelResponse(output: string): string {
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(output);
+  } catch {
+    throw new Error('Expected valid JSON output from LLxprt CLI');
+  }
+
+  const result = EvalOutputSchema.safeParse(parsed);
+  if (!result.success) {
+    const details = result.error.issues
+      .map((issue) => {
+        const path = issue.path.join('.');
+        return `${path === '' ? '<root>' : path}: ${issue.message}`;
+      })
+      .join('; ');
+    throw new Error(
+      `Expected LLxprt CLI JSON output to include a string response: ${details}`,
+    );
+  }
+  return result.data.response;
+}
+
+/**
+ * Serialized shape of the eval artifact written to evals/logs. Contains the
+ * structured process-run capture (separate stdout/stderr/exitCode/timedOut)
+ * alongside the extracted tool-call records for post-run diagnosis.
+ */
+export interface EvalArtifact {
+  readonly schemaVersion: 1;
+  readonly capture: RunCapture | null;
+  readonly toolCalls: ReturnType<TestRig['readToolLogs']> | EvalArtifactError;
+}
+
+interface EvalArtifactError {
+  readonly error: string;
+}
+
+export function formatEvalLog(
+  capture: RunCapture | null,
+  toolCalls: ReturnType<TestRig['readToolLogs']> | EvalArtifactError,
+): string {
+  const artifact: EvalArtifact = { schemaVersion: 1, capture, toolCalls };
+  return JSON.stringify(artifact, null, 2);
+}
+
 export function evalTest(policy: EvalPolicy, evalCase: EvalCase): void {
   const fn = async (): Promise<void> => {
     const rig = new TestRig();
+    let primaryError: unknown;
+    let failed = false;
     try {
       rig.setup(evalCase.name, evalCase.params);
-      const result = await rig.run({ args: evalCase.prompt });
-      await evalCase.assert(rig, result);
-    } finally {
-      await logToFile(
-        evalCase.name,
-        JSON.stringify(rig.readToolLogs(), null, 2),
-      );
-      await rig.cleanup();
+      const cliOutput = await rig.run({ args: buildEvalArgs(evalCase.prompt) });
+      await evalCase.assert(rig, extractModelResponse(cliOutput));
+    } catch (error) {
+      failed = true;
+      primaryError = error;
+    }
+
+    let finalizationError: unknown;
+    try {
+      await finalizeEval(rig, evalCase.name);
+    } catch (error) {
+      finalizationError = error;
+    }
+
+    if (failed) {
+      if (finalizationError !== undefined) {
+        throw new AggregateError(
+          [primaryError, finalizationError],
+          'Eval failed and diagnostics finalization also failed',
+          { cause: primaryError },
+        );
+      }
+      throw primaryError;
+    }
+    if (finalizationError !== undefined) {
+      throw finalizationError;
     }
   };
 
@@ -49,6 +123,46 @@ export interface EvalCase {
   assert: (rig: TestRig, result: string) => Promise<void>;
 }
 
+async function finalizeEval(rig: TestRig, name: string): Promise<void> {
+  const capture = rig.getLastRunCapture();
+  let toolCalls: ReturnType<TestRig['readToolLogs']> | EvalArtifactError;
+  try {
+    toolCalls = rig.readToolLogs();
+  } catch (error) {
+    toolCalls = {
+      error: error instanceof Error ? error.message : String(error),
+    };
+  }
+
+  let artifactError: unknown;
+  try {
+    await logToFile(name, formatEvalLog(capture, toolCalls));
+  } catch (error) {
+    artifactError = error;
+  }
+
+  let cleanupError: unknown;
+  try {
+    await rig.cleanup();
+  } catch (error) {
+    cleanupError = error;
+  }
+
+  if (artifactError !== undefined && cleanupError !== undefined) {
+    throw new AggregateError(
+      [artifactError, cleanupError],
+      'Writing eval diagnostics and cleaning up both failed',
+      { cause: artifactError },
+    );
+  }
+  if (artifactError !== undefined) {
+    throw artifactError;
+  }
+  if (cleanupError !== undefined) {
+    throw cleanupError;
+  }
+}
+
 // Canonical deterministic contract for the save_memory eval: the prompt
 // instructs the model to answer exactly this value.
 const CANONICAL_ANSWER = '$blue$';

integration-tests/json-output.test.ts

@@ -39,6 +39,42 @@ describe('JSON output', () => {
     expect(typeof parsed.stats).toBe('object');
   });
 
+  it('should preserve an exact tool-using response and separate diagnostics', async () => {
+    await rig.setup('json-output-save-memory', {
+      settings: { tools: { core: ['save_memory'] } },
+      fakeResponsesPath: join(
+        import.meta.dirname,
+        'save-memory.responses.jsonl',
+      ),
+    });
+
+    const result = await rig.run({
+      args: [
+        'Save that my favorite color is blue, then answer with exactly $blue$.',
+        '--output-format',
+        'json',
+      ],
+    });
+
+    expect(JSON.parse(result).response).toBe('$blue$');
+    expect(
+      rig
+        .readToolLogs()
+        .some(
+          (entry) =>
+            entry.toolRequest.name === 'save_memory' &&
+            entry.toolRequest.success,
+        ),
+    ).toBe(true);
+
+    const capture = rig.getLastRunCapture();
+    expect(capture?.stdout).toContain('$blue$');
+    expect(capture?.exitCode).toBe(0);
+    expect(capture?.timedOut).toBe(false);
+    expect(capture?.stderr).toBeDefined();
+    expect(result).not.toContain('StdErr:');
+  });
+
   it('should return a valid JSON with a session ID', async () => {
     await rig.setup('json-output-session-id', {
       fakeResponsesPath: join(