From fb9ab7cec9c1ffe8b21a9edb577547c303ff4487 Mon Sep 17 00:00:00 2001
From: Walker Hughes <74113220+walkerhughes@users.noreply.github.com>
Date: Sat, 25 Jul 2026 08:07:05 -0700
Subject: [PATCH] test(evals): show captured output when a task verifier fails

Opened to confirm the tastytrade-mcp eval gate triggers on a change confined to
evals/, and to make the gate useful when it does fail.

validate_local.sh discarded both runs with >/dev/null, so a CI failure printed
only a task name and the two reward values. Diagnosing meant reproducing it
locally. The oracle, verifier, and negative-control output now go to files inside
the existing $work tempdir and are printed, indented and tail-limited, only on
failure. Passing runs are unchanged.

Verified by breaking one oracle on purpose: the run reported 11 passed, 1 failed
and surfaced the real cause plus the resulting cascade (oracle exited non-zero,
so no answer.json existed for the verifier to read). Restored, 12 passed.
---
 tastytrade-mcp/evals/validate_local.sh | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/tastytrade-mcp/evals/validate_local.sh b/tastytrade-mcp/evals/validate_local.sh
index cd1300a..617c66b 100755
--- a/tastytrade-mcp/evals/validate_local.sh
+++ b/tastytrade-mcp/evals/validate_local.sh
@@ -16,14 +16,17 @@ for task in */; do
   export MOCK_STATE_FILE="$APP_DIR/placed_orders.jsonl"
   mkdir -p "$APP_DIR" "$LOG_DIR"
 
-  bash "$task/solution/solve.sh" >/dev/null 2>&1 || true
-  bash "$task/tests/test.sh" >/dev/null 2>&1 || true
+  # Captured rather than discarded: on failure these logs are the only clue, and
+  # a CI run that prints just a task name means reproducing it locally to learn
+  # anything. Kept inside $work so they are cleaned up with it.
+  bash "$task/solution/solve.sh" >"$work/oracle.log" 2>&1 || true
+  bash "$task/tests/test.sh" >"$work/verify.log" 2>&1 || true
   reward="$(cat "$LOG_DIR/reward.txt" 2>/dev/null || echo missing)"
 
   # Negative control: an empty answer must NOT earn reward.
   rm -rf "$APP_DIR"/* "$LOG_DIR"/* 2>/dev/null || true
   echo '{}' > "$APP_DIR/answer.json"
-  bash "$task/tests/test.sh" >/dev/null 2>&1 || true
+  bash "$task/tests/test.sh" >"$work/negative.log" 2>&1 || true
   neg="$(cat "$LOG_DIR/reward.txt" 2>/dev/null || echo missing)"
 
   if [ "$reward" = "1" ] && [ "$neg" = "0" ]; then
@@ -31,6 +34,12 @@ for task in */; do
     pass=$((pass + 1))
   else
     echo "FAIL  $task  (oracle=$reward negative=$neg)"
+    for log in oracle verify negative; do
+      if [ -s "$work/$log.log" ]; then
+        echo "      --- $log ---"
+        sed 's/^/      /' "$work/$log.log" | tail -15
+      fi
+    done
     fail=$((fail + 1))
   fi
   rm -rf "$work"