diff --git a/.github/workflows/diagnose.yml b/.github/workflows/diagnose.yml new file mode 100644 index 0000000..07544e2 --- /dev/null +++ b/.github/workflows/diagnose.yml @@ -0,0 +1,137 @@ +name: Diagnose Gateway + +on: + workflow_dispatch: + +env: + GCP_PROJECT_ID: interactivebrokersquant + GCP_WORKLOAD_IDENTITY_PROVIDER: projects/303168642265/locations/global/workloadIdentityPools/github-actions/providers/github-ibkr-gateway-main + GCP_WORKLOAD_IDENTITY_SERVICE_ACCOUNT: ibkr-gateway-deploy@interactivebrokersquant.iam.gserviceaccount.com + +jobs: + diagnose: + runs-on: ubuntu-latest + timeout-minutes: 8 + permissions: + contents: read + id-token: write + env: + GCE_USER: ${{ vars.IB_GATEWAY_GCE_USER }} + GCE_INSTANCE_NAME: ${{ vars.IB_GATEWAY_INSTANCE_NAME }} + GCE_ZONE: ${{ vars.IB_GATEWAY_ZONE }} + DEPLOY_PATH: ${{ vars.IB_GATEWAY_DEPLOY_PATH }} + SSH_PRIVATE_KEY_SECRET_NAME: ${{ vars.IB_GATEWAY_SSH_PRIVATE_KEY_SECRET_NAME }} + steps: + - name: Authenticate to Google Cloud + id: auth + uses: google-github-actions/auth@v3 + with: + workload_identity_provider: ${{ env.GCP_WORKLOAD_IDENTITY_PROVIDER }} + service_account: ${{ env.GCP_WORKLOAD_IDENTITY_SERVICE_ACCOUNT }} + + - name: Set up gcloud + uses: google-github-actions/setup-gcloud@v3 + with: + project_id: ${{ env.GCP_PROJECT_ID }} + version: '>= 416.0.0' + + - name: Prepare SSH key + run: | + set -euo pipefail + + if [ -z "${SSH_PRIVATE_KEY_SECRET_NAME:-}" ]; then + echo "SSH_PRIVATE_KEY_SECRET_NAME is required." >&2 + exit 1 + fi + + install -d -m 700 "$RUNNER_TEMP/ssh" + SSH_KEY_FILE="$RUNNER_TEMP/ssh/google_compute_engine" + gcloud secrets versions access latest \ + --project "${GCP_PROJECT_ID}" \ + --secret "${SSH_PRIVATE_KEY_SECRET_NAME}" | tr -d '\r' > "$SSH_KEY_FILE" + chmod 600 "$SSH_KEY_FILE" + ssh-keygen -y -f "$SSH_KEY_FILE" > "$SSH_KEY_FILE.pub" + chmod 644 "$SSH_KEY_FILE.pub" + echo "SSH_KEY_FILE=$SSH_KEY_FILE" >> "$GITHUB_ENV" + + - name: Collect remote diagnostics + run: | + set -euo pipefail + + for var_name in GCE_USER GCE_INSTANCE_NAME GCE_ZONE DEPLOY_PATH; do + if [ -z "${!var_name:-}" ]; then + echo "${var_name} is required." >&2 + exit 1 + fi + done + + REMOTE_TARGET="${GCE_USER}@${GCE_INSTANCE_NAME}" + SSH_FLAGS=( + --project "${GCP_PROJECT_ID}" + --zone "${GCE_ZONE}" + --quiet + --tunnel-through-iap + --ssh-key-file "${SSH_KEY_FILE}" + --ssh-flag="-o ServerAliveInterval=30" + --ssh-flag="-o ServerAliveCountMax=4" + --ssh-flag="-o TCPKeepAlive=yes" + ) + + REMOTE_DIAG_COMMAND=$(cat <<'EOF' + set -euo pipefail + cd __DEPLOY_PATH__ + + section() { + printf '\n===== %s =====\n' "$1" + } + + section "safe env" + if [ -f .env ]; then + grep -E '^(TRADING_MODE|ACCEPT_API_FROM_IP|ALLOW_CONNECTIONS_FROM_LOCALHOST_ONLY|TWS_ACCEPT_INCOMING|READ_ONLY_API|TWOFA_TIMEOUT_ACTION|RELOGIN_AFTER_TWOFA_TIMEOUT|EXISTING_SESSION_DETECTED_ACTION|LOGIN_TIMEOUT|JAVA_HEAP_SIZE)=' .env || true + else + echo ".env missing" + fi + + section "docker compose ps" + sudo docker compose ps || true + + section "host listening ports" + sudo ss -ltnp | grep -E ':(4001|4002|4003|4004|5900)\b' || true + + section "2fa watcher systemd" + sudo systemctl status ibkr-2fa-bot.timer ibkr-2fa-bot.service --no-pager || true + + section "2fa watcher journal" + sudo journalctl -u ibkr-2fa-bot.service -n 100 --no-pager || true + + section "container processes" + sudo docker exec ib-gateway bash -lc 'pgrep -a -f "2fa_bot.py|ibgateway|socat|Xvfb|x11vnc|ibcstart" || true' || true + + section "container listening ports" + sudo docker exec ib-gateway bash -lc 'ss -ltnp 2>/dev/null | grep -E ":(4001|4002|4003|4004|5900)\b" || netstat -ltnp 2>/dev/null | grep -E ":(4001|4002|4003|4004|5900)\b" || true' || true + + section "x11 windows" + sudo docker exec ib-gateway bash -lc ' + export DISPLAY=:1 + ids=$(xdotool search --name ".*" 2>/dev/null || true) + if [ -z "$ids" ]; then + echo "no x11 windows found" + else + for id in $ids; do + printf "%s " "$id" + xdotool getwindowname "$id" 2>/dev/null || true + done + fi + ' || true + + section "2fa bot log" + sudo docker exec ib-gateway bash -lc 'tail -n 160 /home/ibgateway/2fa.log 2>/dev/null || echo "2fa.log missing"' || true + + section "ib gateway docker log" + sudo docker logs --tail 220 ib-gateway 2>&1 || true + EOF + ) + deploy_path_quoted="$(printf '%q' "${DEPLOY_PATH}")" + REMOTE_DIAG_COMMAND="${REMOTE_DIAG_COMMAND/__DEPLOY_PATH__/${deploy_path_quoted}}" + + gcloud compute ssh "${REMOTE_TARGET}" "${SSH_FLAGS[@]}" --command "${REMOTE_DIAG_COMMAND}"