diff --git a/.github/workflows/opencode-review.yml b/.github/workflows/opencode-review.yml index 94586043..1d8ce91d 100644 --- a/.github/workflows/opencode-review.yml +++ b/.github/workflows/opencode-review.yml @@ -1801,6 +1801,7 @@ jobs: ContextualWisdomLab/.github:tests/test_javascript_coverage_gate.py | \ ContextualWisdomLab/.github:tests/test_opencode_agent_contract.py | \ ContextualWisdomLab/.github:tests/test_opencode_model_pool_runner.py | \ + ContextualWisdomLab/.github:tests/test_opencode_review_normalize_output.py | \ ContextualWisdomLab/.github:tests/test_pr_head_replay_guard.py | \ ContextualWisdomLab/.github:tests/test_pr_review_fix_scheduler_coverage.py | \ ContextualWisdomLab/.github:tests/test_pr_review_merge_scheduler.py | \ @@ -3368,7 +3369,7 @@ jobs: - name: Run OpenCode PR Review model pool id: opencode_review_model_pool if: needs.coverage-evidence.result == 'success' - timeout-minutes: 65 + timeout-minutes: 205 continue-on-error: true env: STRIX_GITHUB_MODELS_TOKEN: ${{ secrets.STRIX_GITHUB_MODELS_TOKEN || github.token }} @@ -3395,13 +3396,16 @@ jobs: # the SAME model 5x let a rate-limited/hung leader consume the whole # step, so the pool never reached a healthy fallback model. OPENCODE_MODEL_ATTEMPTS: "1" - # Keep stale providers from pinning required review jobs for hours. - # Adversarial validation needs enough room to read the evidence, but - # dynamic cadence and the outer watchdog still bound each current-head run. - OPENCODE_RUN_TIMEOUT_SECONDS: "600" - OPENCODE_EXPORT_TIMEOUT_SECONDS: "120" - OPENCODE_TOTAL_RETRY_BUDGET_SECONDS: "3600" - OPENCODE_POOL_STEP_TIMEOUT_SECONDS: "2100" + # Preserve reviews that legitimately need tens of minutes to inspect a + # large repository. Small changes get 30 minutes per model; medium, + # large, unknown, and central-governance changes can use 90 minutes so + # repositories that need a full hour are not cut off at the boundary. + # The 195-minute pool budget leaves room for a second substantial + # candidate, while provider 5xx errors fail over immediately. + OPENCODE_RUN_TIMEOUT_SECONDS: "5400" + OPENCODE_EXPORT_TIMEOUT_SECONDS: "180" + OPENCODE_TOTAL_RETRY_BUDGET_SECONDS: "11700" + OPENCODE_POOL_STEP_TIMEOUT_SECONDS: "12000" # Keep cycling through the high-sensitivity candidate catalog until # the retry budget or step timeout is exhausted; a single invalid # cycle can be all provider formatting noise rather than review @@ -3410,16 +3414,16 @@ jobs: OPENCODE_DYNAMIC_REVIEW_CADENCE: "true" OPENCODE_SMALL_CHANGE_FILE_THRESHOLD: "3" OPENCODE_MEDIUM_CHANGE_FILE_THRESHOLD: "20" - OPENCODE_SMALL_CHANGE_RUN_TIMEOUT_SECONDS: "600" - OPENCODE_SMALL_CHANGE_TOTAL_BUDGET_SECONDS: "3600" - OPENCODE_MEDIUM_CHANGE_RUN_TIMEOUT_SECONDS: "600" - OPENCODE_MEDIUM_CHANGE_TOTAL_BUDGET_SECONDS: "3600" - OPENCODE_LARGE_CHANGE_RUN_TIMEOUT_SECONDS: "600" - OPENCODE_LARGE_CHANGE_TOTAL_BUDGET_SECONDS: "3600" - OPENCODE_UNKNOWN_CHANGE_RUN_TIMEOUT_SECONDS: "600" - OPENCODE_UNKNOWN_CHANGE_TOTAL_BUDGET_SECONDS: "3600" - OPENCODE_DYNAMIC_RUN_TIMEOUT_CAP_SECONDS: "600" - OPENCODE_DYNAMIC_TOTAL_BUDGET_CAP_SECONDS: "1800" + OPENCODE_SMALL_CHANGE_RUN_TIMEOUT_SECONDS: "1800" + OPENCODE_SMALL_CHANGE_TOTAL_BUDGET_SECONDS: "4200" + OPENCODE_MEDIUM_CHANGE_RUN_TIMEOUT_SECONDS: "3600" + OPENCODE_MEDIUM_CHANGE_TOTAL_BUDGET_SECONDS: "7800" + OPENCODE_LARGE_CHANGE_RUN_TIMEOUT_SECONDS: "5400" + OPENCODE_LARGE_CHANGE_TOTAL_BUDGET_SECONDS: "11700" + OPENCODE_UNKNOWN_CHANGE_RUN_TIMEOUT_SECONDS: "5400" + OPENCODE_UNKNOWN_CHANGE_TOTAL_BUDGET_SECONDS: "11700" + OPENCODE_DYNAMIC_RUN_TIMEOUT_CAP_SECONDS: "5400" + OPENCODE_DYNAMIC_TOTAL_BUDGET_CAP_SECONDS: "11700" OPENCODE_DYNAMIC_MAX_CYCLES_CAP: "0" # This installation currently reports a 4k request-body limit for # GitHub Models GPT-5 endpoints even though the public catalog is @@ -3429,8 +3433,8 @@ jobs: OPENCODE_DYNAMIC_MAX_CYCLES: "0" CENTRAL_REVIEW_PROCESS_FALLBACK_ELIGIBLE: ${{ steps.central_review_process_fallback_scope.outputs.eligible || 'false' }} CENTRAL_REVIEW_PROCESS_FALLBACK_SCOPE_LABEL: ${{ steps.central_review_process_fallback_scope.outputs.scope_label || 'unsupported' }} - OPENCODE_CENTRAL_REVIEW_PROCESS_FALLBACK_RUN_TIMEOUT_SECONDS: "600" - OPENCODE_CENTRAL_REVIEW_PROCESS_FALLBACK_TOTAL_BUDGET_SECONDS: "3600" + OPENCODE_CENTRAL_REVIEW_PROCESS_FALLBACK_RUN_TIMEOUT_SECONDS: "5400" + OPENCODE_CENTRAL_REVIEW_PROCESS_FALLBACK_TOTAL_BUDGET_SECONDS: "11700" OPENCODE_CENTRAL_REVIEW_PROCESS_FALLBACK_MAX_CYCLES: "1" OPENCODE_BACKOFF_INITIAL_SECONDS: "30" OPENCODE_BACKOFF_MAX_SECONDS: "30" diff --git a/ci-review-prompt.md b/ci-review-prompt.md index ad4c54ba..48b3d983 100644 --- a/ci-review-prompt.md +++ b/ci-review-prompt.md @@ -109,6 +109,29 @@ current-head line bytes without its line ending, and whether the hypothesis was material code/workflow/config/package/test changes and one for non-code changes; REQUEST_CHANGES needs a confirmed probe anchored to a published finding. +The `evidence` string itself must literally repeat that probe's declared path and +positive line as `path:line`; the separate `path` and `line` fields do not satisfy +this binding. The same string must also name the independent proof anchor actually +used (for example source trace, test/assertion, executed command, log/check/SARIF, +diff, or CodeGraph) and an observed-result word such as confirmed, rejected, +returned, passed, or failed. + +For this central-review bootstrap revision, do not guess a cryptographic receipt. +The trusted current-head source provides these two verified source-line anchors; +copy one receipt exactly only after confirming the cited source trace, and remember +that the receipt is a binding rather than proof by itself: + +- `Source trace at scripts/ci/run_opencode_review_model_pool.sh:95 confirmed + the dynamic timeout cap defaults to 5400 seconds; + source-line-sha256=8e8933a12547802d7bc9ff7cd99915808de9d4f27559d83783a6506d3ff94118`. +- `Source trace at scripts/ci/run_opencode_review_model_pool.sh:287 confirmed + a detected provider outage is classified as service-unavailable; + source-line-sha256=4f6509bb2883da6b849c2eb5949e4844ec209e5eaa7ab6b60396b4c2fd065e31`. + +Evidence using either anchor must still state what source trace was checked and +what it confirmed or rejected. These bootstrap anchors may be removed once the +trusted normalizer derives receipts directly from validated `path:line` fields. + Execution provenance is mandatory. Never claim that React DevTools, Chrome DevTools, browser DevTools, Playwright, Cypress, or Selenium ran, passed, confirmed, verified, or observed behavior unless bounded evidence contains a diff --git a/scripts/ci/run_opencode_review_model_pool.sh b/scripts/ci/run_opencode_review_model_pool.sh index 85e122ab..c742ea1e 100644 --- a/scripts/ci/run_opencode_review_model_pool.sh +++ b/scripts/ci/run_opencode_review_model_pool.sh @@ -92,8 +92,8 @@ env_integer_or_default() { cap_dynamic_cadence_for_queue() { local timeout_cap budget_cap cycle_cap previous_run_timeout previous_budget_seconds previous_max_cycles - timeout_cap="$(env_integer_or_default OPENCODE_DYNAMIC_RUN_TIMEOUT_CAP_SECONDS 600)" - budget_cap="$(env_integer_or_default OPENCODE_DYNAMIC_TOTAL_BUDGET_CAP_SECONDS 1800)" + timeout_cap="$(env_integer_or_default OPENCODE_DYNAMIC_RUN_TIMEOUT_CAP_SECONDS 5400)" + budget_cap="$(env_integer_or_default OPENCODE_DYNAMIC_TOTAL_BUDGET_CAP_SECONDS 11700)" cycle_cap="$(env_integer_or_default OPENCODE_DYNAMIC_MAX_CYCLES_CAP 0)" previous_run_timeout="$original_run_timeout" previous_budget_seconds="$budget_seconds" @@ -186,9 +186,10 @@ write_prompt() { printf 'Do not request changes solely because your tool call, MCP call, or full-file read was not executed. Treat that as a review source limitation unless current-head evidence explicitly reports a materialization failure; any such finding must be tied to that evidence, not a generic model-exhaustion message. REQUEST_CHANGES findings must cite a positive source/evidence line; never use line 0.\n' printf 'Always return a final control block instead of a progress summary. Return only the final review body.\n\n' printf 'Adversarial evidence must state a concrete observed pass, failure, rejection, return value, exit code, or trace outcome and exactly one source-line-sha256=<64 lowercase hex> digest computed from the cited current-head line bytes without its line ending; generic source-inspection or coverage-verification claims are invalid.\n' + printf 'Every adversarial_validation.probes[].evidence string must literally cite its declared path and positive line as path:line, include an independent proof anchor such as source trace, test/assertion, executed command, log/check/SARIF, diff, or CodeGraph, and include an observed-result word such as confirmed, rejected, returned, passed, or failed; the separate path, line, outcome, and receipt fields do not replace these words in evidence.\n' printf 'Required control block shape:\n' printf '```json\n' - printf '{"head_sha":"%s","run_id":"%s","run_attempt":"%s","result":"APPROVE or REQUEST_CHANGES","reason":"short reason","summary":"short review summary with concrete evidence and all required labels","adversarial_validation":{"status":"passed or failed","probes":[{"path":"exact/current-head/changed-file","line":1,"hypothesis":"concrete failure hypothesis","attack_or_counterexample":"input, state, race, threat, or boundary used to challenge it","evidence":"executed command or source-backed trace, observed outcome, and source-line-sha256=","outcome":"falsified or confirmed"}],"residual_risk":"bounded residual risk after the probes"},"findings":[]}\n' "$HEAD_SHA" "$RUN_ID" "$RUN_ATTEMPT" + printf '{"head_sha":"%s","run_id":"%s","run_attempt":"%s","result":"APPROVE or REQUEST_CHANGES","reason":"short reason","summary":"short review summary with concrete evidence and all required labels","adversarial_validation":{"status":"passed or failed","probes":[{"path":"exact/current-head/changed-file","line":1,"hypothesis":"concrete failure hypothesis","attack_or_counterexample":"input, state, race, threat, or boundary used to challenge it","evidence":"Source trace at exact/current-head/changed-file:1 confirmed the concrete current-head outcome; source-line-sha256=","outcome":"falsified or confirmed"}],"residual_risk":"bounded residual risk after the probes"},"findings":[]}\n' "$HEAD_SHA" "$RUN_ID" "$RUN_ATTEMPT" printf '```\n' if [ -s "$evidence_excerpt_file" ]; then printf '\nCurrent-head evidence packet:\n\n' @@ -238,23 +239,33 @@ is_context_overflow_failure() { is_fatal_provider_failure() { local opencode_json_file="$1" + local opencode_stderr_file="${2:-/dev/null}" if is_context_overflow_failure "$opencode_json_file"; then return 0 fi - [ -s "$opencode_json_file" ] || return 1 - grep -Eiq 'budget limit|insufficient_quota' "$opencode_json_file" + grep -Eiq 'budget limit|insufficient_quota|statusCode"[[:space:]]*:[[:space:]]*5[0-9][0-9]|Model service is unavailable' \ + "$opencode_json_file" "$opencode_stderr_file" 2>/dev/null } has_fatal_provider_error_event() { local opencode_json_file="$1" + local opencode_stderr_file="${2:-/dev/null}" - [ -s "$opencode_json_file" ] || return 1 # Only structured "type":"error" events count while the process is still # running: model prose or tool output quoting these signatures is # JSON-escaped inside event strings, so a healthy streaming run is never # killed for merely discussing context windows or quota errors. - awk 'tolower($0) ~ /"type"[[:space:]]*:[[:space:]]*"error"/ && tolower($0) ~ /contextoverflowerror|tokens_limit_reached|request body too large|context window|budget limit|insufficient_quota/ { found = 1; exit } END { exit !found }' "$opencode_json_file" + if [ -s "$opencode_json_file" ] && \ + awk 'tolower($0) ~ /"type"[[:space:]]*:[[:space:]]*"error"/ && tolower($0) ~ /contextoverflowerror|tokens_limit_reached|request body too large|context window|budget limit|insufficient_quota/ { found = 1; exit } END { exit !found }' "$opencode_json_file"; then + return 0 + fi + # OpenCode retries provider HTTP 5xx responses internally without emitting a + # JSON event. --print-logs writes the structured LLM error to the captured + # stderr artifact; fail over after the first service-unavailable response + # instead of paying for an hour-long exponential retry loop. + [ -s "$opencode_stderr_file" ] && \ + grep -Eiq 'service=llm.*statusCode"[[:space:]]*:[[:space:]]*5[0-9][0-9]|responseBody":"Model service is unavailable' "$opencode_stderr_file" } emit_sanitized_opencode_failure_detail() { @@ -272,7 +283,9 @@ emit_sanitized_opencode_failure_detail() { fi failure_class="unclassified" - if grep -Eiq 'ContextOverflowError|tokens_limit_reached|Request body too large|context window' "$opencode_json_file" "$opencode_stderr_file" 2>/dev/null; then + if grep -Eiq 'statusCode"[[:space:]]*:[[:space:]]*5[0-9][0-9]|Model service is unavailable' "$opencode_json_file" "$opencode_stderr_file" 2>/dev/null; then + failure_class="service-unavailable" + elif grep -Eiq 'ContextOverflowError|tokens_limit_reached|Request body too large|context window' "$opencode_json_file" "$opencode_stderr_file" 2>/dev/null; then failure_class="context-window" elif grep -Eiq 'budget limit|insufficient_quota|quota exceeded' "$opencode_json_file" "$opencode_stderr_file" 2>/dev/null; then failure_class="quota-or-budget" @@ -358,6 +371,57 @@ cap_model_run_timeout() { fi } +export_and_validate_session_output() { + local session_id="$1" + local model_candidate="$2" + local attempt="$3" + local attempts="$4" + local opencode_export_file="$5" + local candidate_output_file="$6" + local export_timeout_seconds="${OPENCODE_EXPORT_TIMEOUT_SECONDS:-120}" + + if ! timeout --kill-after=15s "${export_timeout_seconds}s" \ + env -u GH_TOKEN -u GITHUB_TOKEN -u OPENCODE_APP_TOKEN \ + -u ACTIONS_ID_TOKEN_REQUEST_TOKEN -u ACTIONS_ID_TOKEN_REQUEST_URL \ + opencode export "$session_id" --pure >"$opencode_export_file"; then + printf 'OpenCode %s attempt %s/%s session export did not complete within %ss.\n' "$model_candidate" "$attempt" "$attempts" "$export_timeout_seconds" + return 1 + fi + jq -r '.messages[] | select(.info.role == "assistant") | .parts[]? | select(.type == "text") | .text' "$opencode_export_file" >"$candidate_output_file" + if [ ! -s "$candidate_output_file" ]; then + printf 'OpenCode %s attempt %s/%s session export did not include assistant text.\n' "$model_candidate" "$attempt" "$attempts" + emit_rejected_opencode_artifact_metadata "assistant-empty-export" "$opencode_export_file" + return 1 + fi + if ! normalize_opencode_output "$candidate_output_file"; then + printf 'OpenCode %s attempt %s/%s output did not include a valid control conclusion.\n' "$model_candidate" "$attempt" "$attempts" + emit_rejected_opencode_artifact_metadata "invalid-control-output" "$candidate_output_file" + return 1 + fi + return 0 +} + +lookup_session_id_by_exact_title() { + local session_title="$1" + local session_list_file="$2" + local list_timeout_seconds="${OPENCODE_SESSION_LIST_TIMEOUT_SECONDS:-30}" + local session_id + + if ! timeout --kill-after=10s "${list_timeout_seconds}s" \ + env -u GH_TOKEN -u GITHUB_TOKEN -u OPENCODE_APP_TOKEN \ + -u ACTIONS_ID_TOKEN_REQUEST_TOKEN -u ACTIONS_ID_TOKEN_REQUEST_URL \ + opencode session list --pure --format json -n 100 >"$session_list_file"; then + printf 'OpenCode exact-title session lookup did not complete within %ss; timed-out work cannot be exported by fallback lookup.\n' "$list_timeout_seconds" >&2 + return 1 + fi + session_id="$(jq -r --arg title "$session_title" '[.[] | select(.title == $title) | .id] | last // empty' "$session_list_file" 2>/dev/null)" + if [ -z "$session_id" ] || [ "$session_id" = "null" ]; then + printf 'OpenCode exact-title session lookup found no session for the current run title.\n' >&2 + return 1 + fi + printf '%s\n' "$session_id" +} + run_one_model_attempt() { local model_candidate="$1" local attempt="$2" @@ -367,15 +431,17 @@ run_one_model_attempt() { local candidate_output_file="$6" local opencode_json_file="$7" local opencode_export_file="$8" - local run_timeout_seconds export_timeout_seconds opencode_status session_id opencode_stderr_file + local run_timeout_seconds opencode_status session_id opencode_stderr_file + local session_title session_list_file local opencode_pid fatal_poll_seconds run_timeout_seconds="${OPENCODE_RUN_TIMEOUT_SECONDS:-600}" - export_timeout_seconds="${OPENCODE_EXPORT_TIMEOUT_SECONDS:-120}" fatal_poll_seconds="${OPENCODE_FATAL_ERROR_POLL_SECONDS:-5}" opencode_stderr_file="${opencode_json_file}.stderr" + session_list_file="${opencode_json_file}.sessions.json" + session_title="PR #${PR_NUMBER} OpenCode bounded review ${model_candidate} run ${RUN_ID:-local}/${RUN_ATTEMPT:-1} attempt ${attempt}/${attempts} nonce $$-${RANDOM}" - rm -f "$opencode_json_file" "$opencode_stderr_file" "$opencode_export_file" "$candidate_output_file" + rm -f "$opencode_json_file" "$opencode_stderr_file" "$opencode_export_file" "$candidate_output_file" "$session_list_file" set +e timeout --kill-after=30s "${run_timeout_seconds}s" \ env -u GH_TOKEN -u GITHUB_TOKEN -u OPENCODE_APP_TOKEN \ @@ -385,7 +451,9 @@ run_one_model_attempt() { --agent "$agent" \ --model "$model_candidate" \ --format json \ - --title "PR #${PR_NUMBER} OpenCode bounded review ${model_candidate} attempt ${attempt}/${attempts}" \ + --print-logs \ + --log-level ERROR \ + --title "$session_title" \ >"$opencode_json_file" 2>"$opencode_stderr_file" & opencode_pid=$! # Some providers (github-models ContextOverflowError) log a fatal error and @@ -393,7 +461,7 @@ run_one_model_attempt() { # log while opencode runs and kill the process early so the pool falls # through to the next candidate within seconds instead of minutes. while kill -0 "$opencode_pid" 2>/dev/null; do - if has_fatal_provider_error_event "$opencode_json_file"; then + if has_fatal_provider_error_event "$opencode_json_file" "$opencode_stderr_file"; then printf 'OpenCode %s attempt %s/%s logged a fatal provider error while still running; killing the hung process instead of waiting out the %ss run timeout.\n' \ "$model_candidate" "$attempt" "$attempts" "$run_timeout_seconds" kill "$opencode_pid" 2>/dev/null @@ -409,48 +477,51 @@ run_one_model_attempt() { wait "$opencode_pid" opencode_status=$? set -e + session_id="$(jq -r 'select(.type == "step_start") | .sessionID' "$opencode_json_file" 2>/dev/null | tail -n 1)" + if [ -z "$session_id" ] || [ "$session_id" = "null" ]; then + printf 'OpenCode %s attempt %s/%s JSON output did not expose a session id; attempting exact-title session-list recovery for the current run.\n' \ + "$model_candidate" "$attempt" "$attempts" + if session_id="$(lookup_session_id_by_exact_title "$session_title" "$session_list_file")"; then + printf 'OpenCode %s attempt %s/%s recovered session %s from the exact current-run title.\n' \ + "$model_candidate" "$attempt" "$attempts" "$session_id" + else + session_id="" + fi + fi if [ "$opencode_status" -ne 0 ]; then printf 'OpenCode %s attempt %s/%s failed with exit %s.\n' "$model_candidate" "$attempt" "$attempts" "$opencode_status" emit_sanitized_opencode_failure_detail "$opencode_json_file" "$opencode_stderr_file" - if [ "$opencode_status" -eq 124 ] || [ "$opencode_status" -eq 137 ]; then - printf 'OpenCode %s attempt %s/%s timed out after %ss; falling through within the remaining retry budget instead of blocking the org queue.\n' "$model_candidate" "$attempt" "$attempts" "$run_timeout_seconds" + if { [ "$opencode_status" -eq 124 ] || [ "$opencode_status" -eq 137 ] || [ "$opencode_status" -eq 143 ]; } && \ + ! is_fatal_provider_failure "$opencode_json_file" "$opencode_stderr_file"; then + printf 'OpenCode %s attempt %s/%s timed out after %ss; attempting to recover any assistant text already persisted in session %s before falling through.\n' \ + "$model_candidate" "$attempt" "$attempts" "$run_timeout_seconds" "${session_id:-}" + if [ -n "$session_id" ] && [ "$session_id" != "null" ]; then + if export_and_validate_session_output "$session_id" "$model_candidate" "$attempt" "$attempts" "$opencode_export_file" "$candidate_output_file"; then + printf 'OpenCode %s attempt %s/%s recovered a valid control conclusion from the timed-out session; preserving paid review work.\n' \ + "$model_candidate" "$attempt" "$attempts" + return 0 + fi + printf 'OpenCode %s attempt %s/%s timed-out session export did not contain a valid recoverable conclusion; continuing within the remaining retry budget.\n' \ + "$model_candidate" "$attempt" "$attempts" + fi fi - if is_fatal_provider_failure "$opencode_json_file"; then - printf 'OpenCode %s attempt %s/%s hit a fatal provider error (context window, token budget, or quota); skipping remaining attempts for this model.\n' "$model_candidate" "$attempt" "$attempts" + if is_fatal_provider_failure "$opencode_json_file" "$opencode_stderr_file"; then + printf 'OpenCode %s attempt %s/%s hit a fatal provider error (service unavailable, context window, token budget, or quota); skipping remaining attempts for this model.\n' "$model_candidate" "$attempt" "$attempts" return 2 fi return 1 fi - session_id="$(jq -r 'select(.type == "step_start") | .sessionID' "$opencode_json_file" | tail -n 1)" if [ -z "$session_id" ] || [ "$session_id" = "null" ]; then printf 'OpenCode %s attempt %s/%s JSON output did not include a session id.\n' "$model_candidate" "$attempt" "$attempts" emit_rejected_opencode_artifact_metadata "sessionless-json" "$opencode_json_file" - if is_fatal_provider_failure "$opencode_json_file"; then - printf 'OpenCode %s attempt %s/%s hit a fatal provider error (context window, token budget, or quota); skipping remaining attempts for this model.\n' "$model_candidate" "$attempt" "$attempts" + if is_fatal_provider_failure "$opencode_json_file" "$opencode_stderr_file"; then + printf 'OpenCode %s attempt %s/%s hit a fatal provider error (service unavailable, context window, token budget, or quota); skipping remaining attempts for this model.\n' "$model_candidate" "$attempt" "$attempts" return 2 fi return 1 fi - if ! timeout --kill-after=15s "${export_timeout_seconds}s" \ - env -u GH_TOKEN -u GITHUB_TOKEN -u OPENCODE_APP_TOKEN \ - -u ACTIONS_ID_TOKEN_REQUEST_TOKEN -u ACTIONS_ID_TOKEN_REQUEST_URL \ - opencode export "$session_id" --pure >"$opencode_export_file"; then - printf 'OpenCode %s attempt %s/%s session export did not complete within %ss.\n' "$model_candidate" "$attempt" "$attempts" "$export_timeout_seconds" - return 1 - fi - jq -r '.messages[] | select(.info.role == "assistant") | .parts[]? | select(.type == "text") | .text' "$opencode_export_file" >"$candidate_output_file" - if [ ! -s "$candidate_output_file" ]; then - printf 'OpenCode %s attempt %s/%s session export did not include assistant text.\n' "$model_candidate" "$attempt" "$attempts" - emit_rejected_opencode_artifact_metadata "assistant-empty-export" "$opencode_export_file" - return 1 - fi - if ! normalize_opencode_output "$candidate_output_file"; then - printf 'OpenCode %s attempt %s/%s output did not include a valid control conclusion.\n' "$model_candidate" "$attempt" "$attempts" - emit_rejected_opencode_artifact_metadata "invalid-control-output" "$candidate_output_file" - return 1 - fi - return 0 + export_and_validate_session_output "$session_id" "$model_candidate" "$attempt" "$attempts" "$opencode_export_file" "$candidate_output_file" } main() { @@ -465,8 +536,8 @@ main() { budget_seconds="${OPENCODE_TOTAL_RETRY_BUDGET_SECONDS:-1500}" max_cycles="${OPENCODE_POOL_MAX_CYCLES:-0}" if [ "${CENTRAL_REVIEW_PROCESS_FALLBACK_ELIGIBLE:-false}" = "true" ]; then - original_run_timeout="${OPENCODE_CENTRAL_REVIEW_PROCESS_FALLBACK_RUN_TIMEOUT_SECONDS:-600}" - budget_seconds="${OPENCODE_CENTRAL_REVIEW_PROCESS_FALLBACK_TOTAL_BUDGET_SECONDS:-3600}" + original_run_timeout="${OPENCODE_CENTRAL_REVIEW_PROCESS_FALLBACK_RUN_TIMEOUT_SECONDS:-5400}" + budget_seconds="${OPENCODE_CENTRAL_REVIEW_PROCESS_FALLBACK_TOTAL_BUDGET_SECONDS:-11700}" max_cycles="${OPENCODE_CENTRAL_REVIEW_PROCESS_FALLBACK_MAX_CYCLES:-1}" printf 'Central review-process evidence fallback eligible for scope "%s"; limiting OpenCode model pool to %ss per attempt, %ss total budget, and %s cycle(s) so provider delay is logged before the publish fallback evaluates current-head peer evidence.\n' \ "${CENTRAL_REVIEW_PROCESS_FALLBACK_SCOPE_LABEL:-unsupported}" "$original_run_timeout" "$budget_seconds" "$max_cycles" @@ -475,22 +546,22 @@ main() { medium_file_threshold="$(env_integer_or_default OPENCODE_MEDIUM_CHANGE_FILE_THRESHOLD 20)" if changed_file_count="$(count_changed_files_for_cadence)"; then if [ "$changed_file_count" -le "$small_file_threshold" ]; then - original_run_timeout="$(env_integer_or_default OPENCODE_SMALL_CHANGE_RUN_TIMEOUT_SECONDS 900)" - budget_seconds="$(env_integer_or_default OPENCODE_SMALL_CHANGE_TOTAL_BUDGET_SECONDS 2100)" + original_run_timeout="$(env_integer_or_default OPENCODE_SMALL_CHANGE_RUN_TIMEOUT_SECONDS 1800)" + budget_seconds="$(env_integer_or_default OPENCODE_SMALL_CHANGE_TOTAL_BUDGET_SECONDS 4200)" elif [ "$changed_file_count" -le "$medium_file_threshold" ]; then - original_run_timeout="$(env_integer_or_default OPENCODE_MEDIUM_CHANGE_RUN_TIMEOUT_SECONDS 1800)" - budget_seconds="$(env_integer_or_default OPENCODE_MEDIUM_CHANGE_TOTAL_BUDGET_SECONDS 3900)" + original_run_timeout="$(env_integer_or_default OPENCODE_MEDIUM_CHANGE_RUN_TIMEOUT_SECONDS 3600)" + budget_seconds="$(env_integer_or_default OPENCODE_MEDIUM_CHANGE_TOTAL_BUDGET_SECONDS 7800)" else - original_run_timeout="$(env_integer_or_default OPENCODE_LARGE_CHANGE_RUN_TIMEOUT_SECONDS 3600)" - budget_seconds="$(env_integer_or_default OPENCODE_LARGE_CHANGE_TOTAL_BUDGET_SECONDS 7200)" + original_run_timeout="$(env_integer_or_default OPENCODE_LARGE_CHANGE_RUN_TIMEOUT_SECONDS 5400)" + budget_seconds="$(env_integer_or_default OPENCODE_LARGE_CHANGE_TOTAL_BUDGET_SECONDS 11700)" fi max_cycles="$(env_integer_or_default OPENCODE_DYNAMIC_MAX_CYCLES 0)" cap_dynamic_cadence_for_queue printf 'OpenCode dynamic review cadence selected %ss per attempt and %ss total budget for %s changed file(s); max-cycles=%s.\n' \ "$original_run_timeout" "$budget_seconds" "$changed_file_count" "$max_cycles" else - original_run_timeout="$(env_integer_or_default OPENCODE_UNKNOWN_CHANGE_RUN_TIMEOUT_SECONDS 1800)" - budget_seconds="$(env_integer_or_default OPENCODE_UNKNOWN_CHANGE_TOTAL_BUDGET_SECONDS 3900)" + original_run_timeout="$(env_integer_or_default OPENCODE_UNKNOWN_CHANGE_RUN_TIMEOUT_SECONDS 5400)" + budget_seconds="$(env_integer_or_default OPENCODE_UNKNOWN_CHANGE_TOTAL_BUDGET_SECONDS 11700)" max_cycles="$(env_integer_or_default OPENCODE_DYNAMIC_MAX_CYCLES 0)" cap_dynamic_cadence_for_queue printf 'OpenCode dynamic review cadence could not read OPENCODE_CHANGED_FILES_FILE; using %ss per attempt and %ss total budget; max-cycles=%s.\n' \ diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index 3de19fa2..06be377c 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -649,11 +649,14 @@ assert_opencode_review_uses_codegraph_and_gpt5_fallback() { assert_file_contains "$REPO_ROOT/scripts/ci/strix_quick_gate.sh" "billing details" "strix quick gate classifies provider quota starvation as infrastructure" assert_file_contains "$workflow_file" 'timeout-minutes: 150' "opencode review target releases stalled review runners within the bounded queue budget" assert_file_contains "$workflow_file" 'timeout-minutes: 12' "opencode evidence preparation fails closed before it ties up the review queue" - assert_file_contains "$workflow_file" 'timeout-minutes: 65' "opencode model pool gives multiple candidates a bounded review window while capping stalled model attempts" + assert_file_contains "$workflow_file" 'timeout-minutes: 205' "opencode model pool gives two long-review candidates a bounded review window" assert_file_contains "$workflow_file" 'timeout-minutes: 34' "opencode fast approval publication is bounded around the dynamic image and package/GPU check wait" assert_file_contains "$workflow_file" 'continue-on-error: true' "opencode approval gate still runs after model-pool failure to publish a reason" - assert_file_contains "$workflow_file" 'OPENCODE_RUN_TIMEOUT_SECONDS: "600"' "opencode primary review advances after a bounded stalled provider attempt" - assert_file_contains "$workflow_file" 'OPENCODE_TOTAL_RETRY_BUDGET_SECONDS: "3600"' "opencode model pool exits before the step timeout so the approval gate can publish a reason" + assert_file_contains "$workflow_file" 'OPENCODE_RUN_TIMEOUT_SECONDS: "5400"' "opencode primary review preserves repositories that require more than one hour" + assert_file_contains "$workflow_file" 'OPENCODE_TOTAL_RETRY_BUDGET_SECONDS: "11700"' "opencode model pool leaves enough budget for a second long-review candidate" + assert_file_contains "$REPO_ROOT/scripts/ci/run_opencode_review_model_pool.sh" '--print-logs' "opencode captures structured provider errors without replaying their content" + assert_file_contains "$REPO_ROOT/scripts/ci/run_opencode_review_model_pool.sh" 'failure_class="service-unavailable"' "opencode identifies provider 5xx outages instead of waiting out the long review timeout" + assert_file_contains "$REPO_ROOT/scripts/ci/run_opencode_review_model_pool.sh" 'lookup_session_id_by_exact_title' "opencode recovers paid sessions even when JSON output is empty" assert_file_contains "$workflow_file" 'OPENCODE_POOL_MAX_CYCLES: "0"' "opencode model pool keeps cycling until the bounded retry budget or step timeout is exhausted" assert_file_contains "$workflow_file" "needs.coverage-evidence.result == 'success'" "opencode model pool only runs after coverage evidence passed" assert_file_contains "$workflow_file" "id: opencode_review_model_pool" "opencode DeepSeek V3 fallback still runs after a primary model timeout or step failure when coverage evidence passed" @@ -805,7 +808,7 @@ assert_opencode_review_uses_codegraph_and_gpt5_fallback() { assert_file_not_contains "$workflow_file" "no model produced a valid review control block" "opencode model-failure path no longer documents a final exhausted state" assert_file_contains "$workflow_file" 'OPENCODE_MODEL_ATTEMPTS: "1"' "opencode primary and fallback paths avoid multi-attempt stalls on one model" assert_file_contains "$workflow_file" 'OPENCODE_MODEL_ATTEMPTS: "1"' "opencode catalog fallback tries each model once before moving on" - assert_file_contains "$workflow_file" 'OPENCODE_RUN_TIMEOUT_SECONDS: "600"' "opencode catalog fallback advances after a bounded stalled provider attempt" + assert_file_contains "$workflow_file" 'OPENCODE_RUN_TIMEOUT_SECONDS: "5400"' "opencode catalog allows a 90-minute long-repository review" assert_file_contains "$REPO_ROOT/scripts/ci/run_opencode_review_model_pool.sh" "OpenCode %s attempt %s/%s failed" "opencode catalog fallback records per-model retry failures" assert_file_contains "$REPO_ROOT/scripts/ci/run_opencode_review_model_pool.sh" "exponential backoff" "opencode model retry paths use exponential backoff instead of fixed sleeps" assert_file_contains "$workflow_file" "github-models/deepseek/deepseek-v3-0324 openai/gpt-5.6-luna github-models/openai/gpt-4.1 github-models/openai/gpt-5" "opencode review tries DeepSeek V3 before OpenAI fallbacks" @@ -1118,7 +1121,7 @@ assert_opencode_review_uses_codegraph_and_gpt5_fallback() { assert_file_contains "$workflow_file" "collect_failed_check_evidence.sh" "opencode review workflow collects failed check logs and annotations" assert_file_contains "$workflow_file" 'HEAD_SHA: ${{ needs.validate-pr-metadata.outputs.head_sha }}' "opencode evidence step passes the live validated HEAD_SHA to failed-check evidence collection" assert_file_contains "$workflow_file" "FAILED_CHECK_EVIDENCE_ATTEMPTS" "opencode review workflow bounds waiting for peer check failures before model review" - assert_file_contains "$workflow_file" 'timeout-minutes: 65' "opencode model stage has a bounded multi-provider timeout" + assert_file_contains "$workflow_file" 'timeout-minutes: 205' "opencode model stage has a bounded multi-provider timeout" assert_file_contains "$workflow_file" 'timeout-minutes: 12' "opencode evidence preparation has a bounded peer-check wait timeout" assert_file_contains "$workflow_file" 'FAILED_CHECK_EVIDENCE_ATTEMPTS: "6"' "opencode review workflow keeps pre-model peer-check waiting bounded for required workflow DX" assert_file_contains "$workflow_file" 'FAILED_CHECK_EVIDENCE_SLEEP_SECONDS: "5"' "opencode review workflow retries peer-check evidence without stalling the model stage for Strix-scale durations" @@ -5613,7 +5616,7 @@ run_filtered_gate_case_if_requested() { "0" \ "" \ "" \ - "2" \ + "3" \ "0" \ "pull_request" \ "sync-module-system/smart-crawling-biz/src/main/java/org/empasy/sync/modules/system/controller/SysPositionController.java" @@ -9184,7 +9187,7 @@ run_gate_case_allow_provider_signal "zero-findings-with-low-report-timeout" \ "0" \ "" \ "" \ - "2" \ + "3" \ "0" \ "pull_request" \ "sync-module-system/smart-crawling-biz/src/main/java/org/empasy/sync/modules/system/controller/SysPositionController.java" diff --git a/tests/test_opencode_agent_contract.py b/tests/test_opencode_agent_contract.py index 1ff1ff18..4482d13a 100644 --- a/tests/test_opencode_agent_contract.py +++ b/tests/test_opencode_agent_contract.py @@ -1,3 +1,4 @@ +import hashlib import json import os import re @@ -590,6 +591,8 @@ def test_code_reviewer_prompt_preserves_review_only_policy(): assert "Perform an explicit adversarial phase before every verdict" in ci_prompt assert "Run a dedicated adversarial phase before the verdict" in prompt assert "`adversarial_validation` control field" in ci_prompt + assert "The `evidence` string itself must literally repeat" in ci_prompt + assert "the separate `path` and `line` fields do not satisfy" in ci_prompt assert ( "Green checks alone and absence of a known failure are not adversarial evidence" in prompt_normalized @@ -613,6 +616,23 @@ def test_code_reviewer_prompt_preserves_review_only_policy(): assert "prefers-reduced-motion: reduce" in ci_prompt_normalized +def test_bootstrap_source_receipts_match_current_head_lines(): + """Keep temporary bootstrap receipts bound to the cited source bytes.""" + ci_prompt = Path("ci-review-prompt.md").read_text(encoding="utf-8") + source_path = Path("scripts/ci/run_opencode_review_model_pool.sh") + source_lines = source_path.read_bytes().splitlines() + expected_receipts = { + 95: "8e8933a12547802d7bc9ff7cd99915808de9d4f27559d83783a6506d3ff94118", + 287: "4f6509bb2883da6b849c2eb5949e4844ec209e5eaa7ab6b60396b4c2fd065e31", + } + + for line, expected_digest in expected_receipts.items(): + digest = hashlib.sha256(source_lines[line - 1]).hexdigest() + assert digest == expected_digest + assert f"{source_path.as_posix()}:{line}" in ci_prompt + assert f"source-line-sha256={expected_digest}" in ci_prompt + + def test_workflow_provisions_sandbox_tool_and_reviewer_agent(): """Guard the isolated runtime OpenCode workspace and reviewer agent.""" workflow = Path(".github/workflows/opencode-review.yml").read_text(encoding="utf-8") @@ -724,9 +744,10 @@ def test_workflow_provisions_sandbox_tool_and_reviewer_agent(): assert "skipping remaining attempts for this model" in model_pool_runner assert "using %ss run timeout with %ss retry budget remaining" in model_pool_runner assert ( - "timed out after %ss; falling through within the remaining retry budget" + "timed out after %ss; attempting to recover any assistant text already persisted" in model_pool_runner ) + assert "recovered a valid control conclusion from the timed-out session" in model_pool_runner assert "emit_sanitized_opencode_failure_detail" in model_pool_runner assert "OpenCode provider failure metadata" in model_pool_runner assert "provider-controlled content suppressed" in model_pool_runner @@ -797,11 +818,11 @@ def test_workflow_provisions_sandbox_tool_and_reviewer_agent(): assert "CENTRAL_REVIEW_PROCESS_FALLBACK_ELIGIBLE" in workflow assert "CENTRAL_REVIEW_PROCESS_FALLBACK_SCOPE_LABEL" in workflow assert ( - 'OPENCODE_CENTRAL_REVIEW_PROCESS_FALLBACK_RUN_TIMEOUT_SECONDS: "600"' + 'OPENCODE_CENTRAL_REVIEW_PROCESS_FALLBACK_RUN_TIMEOUT_SECONDS: "5400"' in workflow ) assert ( - 'OPENCODE_CENTRAL_REVIEW_PROCESS_FALLBACK_TOTAL_BUDGET_SECONDS: "3600"' + 'OPENCODE_CENTRAL_REVIEW_PROCESS_FALLBACK_TOTAL_BUDGET_SECONDS: "11700"' in workflow ) assert 'OPENCODE_CENTRAL_REVIEW_PROCESS_FALLBACK_MAX_CYCLES: "1"' in workflow @@ -873,15 +894,15 @@ def test_workflow_provisions_sandbox_tool_and_reviewer_agent(): assert re.search(r"opencode-review-target:[\s\S]*?timeout-minutes: 150", workflow) assert "timeout-minutes: 12" in workflow assert re.search( - r"Run OpenCode PR Review model pool[\s\S]{0,240}timeout-minutes: 65", workflow + r"Run OpenCode PR Review model pool[\s\S]{0,240}timeout-minutes: 205", workflow ) - assert 'OPENCODE_SMALL_CHANGE_TOTAL_BUDGET_SECONDS: "3600"' in workflow - assert 'OPENCODE_MEDIUM_CHANGE_TOTAL_BUDGET_SECONDS: "3600"' in workflow - assert 'OPENCODE_LARGE_CHANGE_TOTAL_BUDGET_SECONDS: "3600"' in workflow - assert 'OPENCODE_UNKNOWN_CHANGE_TOTAL_BUDGET_SECONDS: "3600"' in workflow - assert 'OPENCODE_RUN_TIMEOUT_SECONDS: "600"' in workflow - assert 'OPENCODE_TOTAL_RETRY_BUDGET_SECONDS: "3600"' in workflow - assert 'OPENCODE_POOL_STEP_TIMEOUT_SECONDS: "2100"' in workflow + assert 'OPENCODE_SMALL_CHANGE_TOTAL_BUDGET_SECONDS: "4200"' in workflow + assert 'OPENCODE_MEDIUM_CHANGE_TOTAL_BUDGET_SECONDS: "7800"' in workflow + assert 'OPENCODE_LARGE_CHANGE_TOTAL_BUDGET_SECONDS: "11700"' in workflow + assert 'OPENCODE_UNKNOWN_CHANGE_TOTAL_BUDGET_SECONDS: "11700"' in workflow + assert 'OPENCODE_RUN_TIMEOUT_SECONDS: "5400"' in workflow + assert 'OPENCODE_TOTAL_RETRY_BUDGET_SECONDS: "11700"' in workflow + assert 'OPENCODE_POOL_STEP_TIMEOUT_SECONDS: "12000"' in workflow assert ( 'timeout --kill-after=30s "${OPENCODE_POOL_STEP_TIMEOUT_SECONDS:-3600}s"' in workflow @@ -924,26 +945,26 @@ def test_workflow_provisions_sandbox_tool_and_reviewer_agent(): 'github-models/deepseek/deepseek-r1"' ) in workflow assert 'OPENCODE_MODEL_ATTEMPTS: "1"' in workflow - assert 'OPENCODE_RUN_TIMEOUT_SECONDS: "600"' in workflow - assert 'OPENCODE_EXPORT_TIMEOUT_SECONDS: "120"' in workflow - assert 'OPENCODE_TOTAL_RETRY_BUDGET_SECONDS: "3600"' in workflow - assert 'OPENCODE_POOL_STEP_TIMEOUT_SECONDS: "2100"' in workflow + assert 'OPENCODE_RUN_TIMEOUT_SECONDS: "5400"' in workflow + assert 'OPENCODE_EXPORT_TIMEOUT_SECONDS: "180"' in workflow + assert 'OPENCODE_TOTAL_RETRY_BUDGET_SECONDS: "11700"' in workflow + assert 'OPENCODE_POOL_STEP_TIMEOUT_SECONDS: "12000"' in workflow assert 'OPENCODE_POOL_MAX_CYCLES: "0"' in workflow assert 'OPENCODE_DYNAMIC_REVIEW_CADENCE: "true"' in workflow assert ( "OPENCODE_CHANGED_FILES_FILE: ${{ runner.temp }}/opencode-changed-files.txt" in workflow ) - assert 'OPENCODE_SMALL_CHANGE_RUN_TIMEOUT_SECONDS: "600"' in workflow - assert 'OPENCODE_SMALL_CHANGE_TOTAL_BUDGET_SECONDS: "3600"' in workflow - assert 'OPENCODE_MEDIUM_CHANGE_RUN_TIMEOUT_SECONDS: "600"' in workflow - assert 'OPENCODE_MEDIUM_CHANGE_TOTAL_BUDGET_SECONDS: "3600"' in workflow - assert 'OPENCODE_LARGE_CHANGE_RUN_TIMEOUT_SECONDS: "600"' in workflow - assert 'OPENCODE_LARGE_CHANGE_TOTAL_BUDGET_SECONDS: "3600"' in workflow - assert 'OPENCODE_UNKNOWN_CHANGE_RUN_TIMEOUT_SECONDS: "600"' in workflow - assert 'OPENCODE_UNKNOWN_CHANGE_TOTAL_BUDGET_SECONDS: "3600"' in workflow - assert 'OPENCODE_DYNAMIC_RUN_TIMEOUT_CAP_SECONDS: "600"' in workflow - assert 'OPENCODE_DYNAMIC_TOTAL_BUDGET_CAP_SECONDS: "1800"' in workflow + assert 'OPENCODE_SMALL_CHANGE_RUN_TIMEOUT_SECONDS: "1800"' in workflow + assert 'OPENCODE_SMALL_CHANGE_TOTAL_BUDGET_SECONDS: "4200"' in workflow + assert 'OPENCODE_MEDIUM_CHANGE_RUN_TIMEOUT_SECONDS: "3600"' in workflow + assert 'OPENCODE_MEDIUM_CHANGE_TOTAL_BUDGET_SECONDS: "7800"' in workflow + assert 'OPENCODE_LARGE_CHANGE_RUN_TIMEOUT_SECONDS: "5400"' in workflow + assert 'OPENCODE_LARGE_CHANGE_TOTAL_BUDGET_SECONDS: "11700"' in workflow + assert 'OPENCODE_UNKNOWN_CHANGE_RUN_TIMEOUT_SECONDS: "5400"' in workflow + assert 'OPENCODE_UNKNOWN_CHANGE_TOTAL_BUDGET_SECONDS: "11700"' in workflow + assert 'OPENCODE_DYNAMIC_RUN_TIMEOUT_CAP_SECONDS: "5400"' in workflow + assert 'OPENCODE_DYNAMIC_TOTAL_BUDGET_CAP_SECONDS: "11700"' in workflow assert 'OPENCODE_DYNAMIC_MAX_CYCLES_CAP: "0"' in workflow assert 'OPENCODE_GITHUB_GPT5_RUN_TIMEOUT_SECONDS: "45"' in workflow assert 'OPENCODE_DYNAMIC_MAX_CYCLES: "0"' in workflow diff --git a/tests/test_opencode_model_pool_runner.py b/tests/test_opencode_model_pool_runner.py index 71e96add..40580aa4 100644 --- a/tests/test_opencode_model_pool_runner.py +++ b/tests/test_opencode_model_pool_runner.py @@ -153,12 +153,22 @@ def run_failed_model( fake_opencode.write_text( "#!/usr/bin/env bash\n" 'if [ "${1:-}" = run ]; then\n' + ' previous=""\n' + ' for argument in "$@"; do\n' + ' if [ "$previous" = "--title" ] && [ -n "${FAKE_OPENCODE_TITLE_FILE:-}" ]; then printf \'%s\n\' "$argument" > "$FAKE_OPENCODE_TITLE_FILE"; fi\n' + ' previous="$argument"\n' + ' done\n' ' [ -z "${FAKE_OPENCODE_PROMPT_CAPTURE:-}" ] || printf \'%s\\n\' "$2" > "$FAKE_OPENCODE_PROMPT_CAPTURE"\n' ' [ -z "${FAKE_OPENCODE_JSON:-}" ] || printf \'%s\\n\' "$FAKE_OPENCODE_JSON"\n' ' [ -z "${FAKE_OPENCODE_STDERR:-}" ] || printf \'%s\\n\' "$FAKE_OPENCODE_STDERR" >&2\n' ' sleep "${FAKE_OPENCODE_HANG_SECONDS:-0}"\n' ' exit "${FAKE_OPENCODE_RUN_EXIT:-1}"\n' "fi\n" + 'if [ "${1:-}" = session ] && [ "${2:-}" = list ]; then\n' + ' title="$(cat "${FAKE_OPENCODE_TITLE_FILE:-/dev/null}" 2>/dev/null || true)"\n' + ' if [ -n "${FAKE_OPENCODE_SESSION_ID:-}" ] && [ -n "$title" ]; then printf \'[{"id":"%s","title":"%s"}]\n\' "$FAKE_OPENCODE_SESSION_ID" "$title"; else printf \'[]\n\'; fi\n' + ' exit "${FAKE_OPENCODE_SESSION_LIST_EXIT:-0}"\n' + "fi\n" 'if [ "${1:-}" = export ]; then\n' ' [ -z "${FAKE_OPENCODE_EXPORT:-}" ] || printf \'%s\\n\' "$FAKE_OPENCODE_EXPORT"\n' ' exit "${FAKE_OPENCODE_EXPORT_EXIT:-0}"\n' @@ -179,6 +189,7 @@ def run_failed_model( if prompt_capture else "", "FAKE_OPENCODE_STDERR": stderr_line, + "FAKE_OPENCODE_TITLE_FILE": bash_path(tmp_path / "opencode-title.txt"), "GITHUB_OUTPUT": bash_path(github_output), "GITHUB_WORKSPACE": bash_path(ROOT), "HEAD_SHA": "1" * 40, @@ -561,6 +572,30 @@ def test_fatal_provider_error_kills_hung_opencode_run_early( assert elapsed < 25 +def test_service_unavailable_log_kills_internal_retry_loop_early( + tmp_path: Path, +) -> None: + """A captured provider HTTP 5xx fails over instead of retrying for an hour.""" + result = run_failed_model( + tmp_path, + stderr_line=( + 'ERROR service=llm error={"statusCode":500,' + '"responseBody":"Model service is unavailable."}' + ), + extra_env={ + "FAKE_OPENCODE_HANG_SECONDS": "120", + "OPENCODE_RUN_TIMEOUT_SECONDS": "5400", + "OPENCODE_TOTAL_RETRY_BUDGET_SECONDS": "11700", + }, + model_candidates="github-models/deepseek/deepseek-v3-0324", + ) + + assert result.returncode == 1 + assert "logged a fatal provider error while still running" in result.stdout + assert "class=service-unavailable" in result.stdout + assert "skipping remaining attempts for this model" in result.stdout + + def test_model_text_quoting_error_signatures_does_not_kill_run(tmp_path: Path) -> None: """Model prose mentioning fatal signatures never kills a healthy streaming run.""" result = run_failed_model( @@ -576,6 +611,75 @@ def test_model_text_quoting_error_signatures_does_not_kill_run(tmp_path: Path) - assert "logged a fatal provider error while still running" not in result.stdout +def test_timed_out_session_exports_persisted_assistant_text_before_fallback( + tmp_path: Path, +) -> None: + """A paid model timeout exports persisted text instead of discarding it unseen.""" + result = run_failed_model( + tmp_path, + json_line='{"type":"step_start","sessionID":"paid-timeout-session"}', + extra_env={ + "FAKE_OPENCODE_HANG_SECONDS": "5", + "FAKE_OPENCODE_EXPORT": json.dumps( + { + "messages": [ + { + "info": {"role": "assistant"}, + "parts": [ + { + "type": "text", + "text": "persisted but incomplete review", + } + ], + } + ] + } + ), + "OPENCODE_RUN_TIMEOUT_SECONDS": "1", + "OPENCODE_TOTAL_RETRY_BUDGET_SECONDS": "3", + }, + model_candidates="github-models/deepseek/deepseek-v3-0324", + ) + + assert result.returncode == 1 + assert "attempting to recover any assistant text already persisted" in result.stdout + assert "kind=invalid-control-output" in result.stdout + assert "timed-out session export did not contain a valid recoverable conclusion" in result.stdout + + +def test_timed_out_session_recovers_id_from_exact_current_run_title( + tmp_path: Path, +) -> None: + """Zero-byte JSON still locates and exports the uniquely titled paid session.""" + result = run_failed_model( + tmp_path, + extra_env={ + "FAKE_OPENCODE_HANG_SECONDS": "5", + "FAKE_OPENCODE_SESSION_ID": "session-list-paid-timeout", + "FAKE_OPENCODE_EXPORT": json.dumps( + { + "messages": [ + { + "info": {"role": "assistant"}, + "parts": [ + {"type": "text", "text": "persisted incomplete review"} + ], + } + ] + } + ), + "OPENCODE_RUN_TIMEOUT_SECONDS": "1", + "OPENCODE_TOTAL_RETRY_BUDGET_SECONDS": "3", + }, + model_candidates="github-models/deepseek/deepseek-v3-0324", + ) + + assert result.returncode == 1 + assert "attempting exact-title session-list recovery" in result.stdout + assert "recovered session session-list-paid-timeout" in result.stdout + assert "kind=invalid-control-output" in result.stdout + + def test_dynamic_review_cadence_uses_small_change_timeout(tmp_path: Path) -> None: """Small PRs fail through hung/unavailable providers quickly with a visible budget reason.""" result = run_failed_model( @@ -627,11 +731,11 @@ def test_dynamic_review_cadence_caps_large_change_queue_budget(tmp_path: Path) - assert result.returncode == 1 assert ( - "OpenCode dynamic review cadence queue cap applied: per-attempt 3600s -> 600s, " + "OpenCode dynamic review cadence queue cap applied: per-attempt 3600s -> 3600s, " "total budget 7200s -> 1s, max-cycles 0 -> 0" ) in result.stdout assert ( - "OpenCode dynamic review cadence selected 600s per attempt and 1s total budget " + "OpenCode dynamic review cadence selected 3600s per attempt and 1s total budget " "for 21 changed file(s); max-cycles=0." ) in result.stdout assert "OpenCode model pool reached configured max cycle count" not in result.stdout @@ -686,6 +790,13 @@ def test_github_models_openai_prompt_references_evidence_without_inlining( assert "Evidence excerpt omitted for `github-models/openai/gpt-5`" in prompt assert "bounded-review-evidence.md" in prompt assert "bounded-review-evidence-excerpt.md" in prompt + assert "literally cite its declared path and positive line as path:line" in prompt + assert "include an independent proof anchor such as source trace" in prompt + assert "include an observed-result word such as confirmed" in prompt + assert ( + "Source trace at exact/current-head/changed-file:1 confirmed the concrete " + "current-head outcome; source-line-sha256=" + ) in prompt def test_deepseek_prompt_still_inlines_bounded_evidence_excerpt(tmp_path: Path) -> None: