diff --git a/tastytrade-mcp/evals/validate_local.sh b/tastytrade-mcp/evals/validate_local.sh index cd1300a..617c66b 100755 --- a/tastytrade-mcp/evals/validate_local.sh +++ b/tastytrade-mcp/evals/validate_local.sh @@ -16,14 +16,17 @@ for task in */; do export MOCK_STATE_FILE="$APP_DIR/placed_orders.jsonl" mkdir -p "$APP_DIR" "$LOG_DIR" - bash "$task/solution/solve.sh" >/dev/null 2>&1 || true - bash "$task/tests/test.sh" >/dev/null 2>&1 || true + # Captured rather than discarded: on failure these logs are the only clue, and + # a CI run that prints just a task name means reproducing it locally to learn + # anything. Kept inside $work so they are cleaned up with it. + bash "$task/solution/solve.sh" >"$work/oracle.log" 2>&1 || true + bash "$task/tests/test.sh" >"$work/verify.log" 2>&1 || true reward="$(cat "$LOG_DIR/reward.txt" 2>/dev/null || echo missing)" # Negative control: an empty answer must NOT earn reward. rm -rf "$APP_DIR"/* "$LOG_DIR"/* 2>/dev/null || true echo '{}' > "$APP_DIR/answer.json" - bash "$task/tests/test.sh" >/dev/null 2>&1 || true + bash "$task/tests/test.sh" >"$work/negative.log" 2>&1 || true neg="$(cat "$LOG_DIR/reward.txt" 2>/dev/null || echo missing)" if [ "$reward" = "1" ] && [ "$neg" = "0" ]; then @@ -31,6 +34,12 @@ for task in */; do pass=$((pass + 1)) else echo "FAIL $task (oracle=$reward negative=$neg)" + for log in oracle verify negative; do + if [ -s "$work/$log.log" ]; then + echo " --- $log ---" + sed 's/^/ /' "$work/$log.log" | tail -15 + fi + done fail=$((fail + 1)) fi rm -rf "$work"