From 43ccdb5cb49e1f4c0b79d0f2b3b00b7804fe588d Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Thu, 16 Apr 2026 04:54:07 +0800 Subject: [PATCH] Add IB Gateway API health recovery --- .github/workflows/main.yml | 6 ++ README.md | 32 ++++++++ scripts/install_gateway_health_watcher.sh | 72 +++++++++++++++++ scripts/restart_ib_gateway_daily.sh | 20 +++++ scripts/wait_for_ib_gateway_ready.sh | 95 ++++++++++++++++++++++- tests/test_gateway_recovery_scripts.sh | 18 +++++ tests/test_wait_for_ib_gateway_ready.sh | 6 +- tests/test_workflow_shared_config.sh | 3 + 8 files changed, 247 insertions(+), 5 deletions(-) create mode 100755 scripts/install_gateway_health_watcher.sh create mode 100755 scripts/restart_ib_gateway_daily.sh diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7593e34..277de6c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -350,9 +350,12 @@ jobs: echo "[\$(date -u +%FT%TZ)] Scheduled keepalive mode: skip docker build" sudo bash ./scripts/ensure_host_swap.sh sudo bash ./scripts/install_2fa_bot_watcher.sh + sudo bash ./scripts/install_gateway_health_watcher.sh sudo bash ./scripts/recover_ib_gateway_ready.sh '${IB_GATEWAY_MODE}' sudo docker compose ps sudo systemctl status ibkr-2fa-bot.timer --no-pager + sudo systemctl status ibkr-gateway-healthcheck.timer --no-pager + sudo systemctl status ibkr-gateway-daily-restart.timer --no-pager echo "[\$(date -u +%FT%TZ)] Keepalive complete." EOF ) @@ -367,9 +370,12 @@ jobs: sudo docker compose up -d sudo bash ./scripts/install_2fa_bot_watcher.sh + sudo bash ./scripts/install_gateway_health_watcher.sh sudo bash ./scripts/recover_ib_gateway_ready.sh '${IB_GATEWAY_MODE}' sudo docker compose ps sudo systemctl status ibkr-2fa-bot.timer --no-pager + sudo systemctl status ibkr-gateway-healthcheck.timer --no-pager + sudo systemctl status ibkr-gateway-daily-restart.timer --no-pager echo "Deployment complete. Bot is monitoring the login window." echo "Run 'docker exec ib-gateway tail -f /home/ibgateway/2fa.log' for real-time status." diff --git a/README.md b/README.md index 8e3beaf..565c21c 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,7 @@ An automated deployment solution for IBKR Gateway on Google Compute Engine (GCE) - **Containerized Deployment**: Docker-based setup. - **Automated 2FA**: Python bot (`pyotp` + `xdotool`) auto-fills TOTP. - **Daily Auto-Reconnect**: restart policy for long-running reliability. +- **API Handshake Recovery**: systemd health check validates the IB API handshake and restarts/recreates the container if the API is not ready. - **Private Network API Access**: IBKR API exposed on GCE private network for Cloud Run. --- @@ -103,6 +104,7 @@ This shared GitHub config is scoped to the **IBKR deployment pair only** (`Inter ```bash docker compose up -d --build sudo bash ./scripts/install_2fa_bot_watcher.sh +sudo bash ./scripts/install_gateway_health_watcher.sh ``` > If you use this repository's GitHub Actions workflow, pushing deploy-related changes to `main` triggers a full deployment to GCE. The daily scheduled run only does a lightweight keepalive start and watcher check; it does not rebuild the Docker image. @@ -118,6 +120,17 @@ ss -lntp | grep -E '4001|4002' Expected: host is listening on `0.0.0.0:4001` and/or `0.0.0.0:4002` (or VM private interface) and container is healthy. +The readiness script checks the actual IB API handshake, not just TCP connectivity: + +```bash +sudo bash ./scripts/wait_for_ib_gateway_ready.sh paper +``` + +It opens an IB protocol session, waits for the Gateway handshake, sends `StartApi`, +and only succeeds after `nextValidId` plus `managedAccounts` have arrived. This +catches the common failure mode where the Docker port is listening but Gateway is +blocked by a login/API prompt. + --- ## Cloud Run Connectivity Checklist @@ -215,6 +228,25 @@ docker exec ib-gateway pgrep -f 2fa_bot.py systemctl status ibkr-2fa-bot.timer --no-pager ``` +### Check Gateway API Health Timers + +```bash +systemctl status ibkr-gateway-healthcheck.timer --no-pager +systemctl status ibkr-gateway-daily-restart.timer --no-pager +``` + +`ibkr-gateway-healthcheck.timer` runs every 5 minutes by default. It calls +`recover_ib_gateway_ready.sh`, which first checks IB API handshake readiness, +then restarts and finally recreates the container if the API does not recover. + +`ibkr-gateway-daily-restart.timer` restarts the Gateway once per day at +`10:30 UTC` by default, then waits for the same API handshake readiness. Override +the schedule during install with: + +```bash +IB_GATEWAY_DAILY_RESTART_ON_CALENDAR='Mon..Fri 10:30:00 UTC' sudo bash ./scripts/install_gateway_health_watcher.sh +``` + ### Check API Port in Container ```bash diff --git a/scripts/install_gateway_health_watcher.sh b/scripts/install_gateway_health_watcher.sh new file mode 100755 index 0000000..33b73de --- /dev/null +++ b/scripts/install_gateway_health_watcher.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir="$(cd "$(dirname "$0")" && pwd)" +repo_dir="${REPO_DIR:-$(cd "${script_dir}/.." && pwd)}" +systemd_dir="${SYSTEMD_DIR:-/etc/systemd/system}" +systemctl_bin="${SYSTEMCTL_BIN:-systemctl}" +container_name="${CONTAINER_NAME:-${IB_GATEWAY_CONTAINER_NAME:-ib-gateway}}" +gateway_mode="${IB_GATEWAY_MODE:-paper}" +health_interval_seconds="${IB_GATEWAY_HEALTHCHECK_INTERVAL_SECONDS:-300}" +daily_restart_calendar="${IB_GATEWAY_DAILY_RESTART_ON_CALENDAR:-*-*-* 10:30:00 UTC}" + +install -d "$systemd_dir" + +cat >"$systemd_dir/ibkr-gateway-healthcheck.service" <"$systemd_dir/ibkr-gateway-healthcheck.timer" <"$systemd_dir/ibkr-gateway-daily-restart.service" <"$systemd_dir/ibkr-gateway-daily-restart.timer" < float: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise TimeoutError("IB API healthcheck timed out") + return remaining + + +def recv_exact(sock: socket.socket, size: int) -> bytes: + chunks = [] + remaining = size + while remaining > 0: + sock.settimeout(remaining_timeout()) + chunk = sock.recv(remaining) + if not chunk: + raise ConnectionError("IB API socket closed during healthcheck") + chunks.append(chunk) + remaining -= len(chunk) + return b"".join(chunks) + + +def read_fields(sock: socket.socket) -> list[str]: + raw_size = recv_exact(sock, 4) + size = struct.unpack(">I", raw_size)[0] + payload = recv_exact(sock, size).decode(errors="backslashreplace") + fields = payload.split("\0") + if fields and fields[-1] == "": + fields.pop() + return fields + + +def send_prefixed(sock: socket.socket, payload: bytes) -> None: + sock.sendall(struct.pack(">I", len(payload)) + payload) + + +with socket.create_connection((host, port), timeout=remaining_timeout()) as sock: + sock.settimeout(remaining_timeout()) + # Match the modern IB API v100+ handshake used by ib_insync. + hello = b"API\0" + struct.pack(">I", len(b"v157..176")) + b"v157..176" + sock.sendall(hello) + + handshake_fields = read_fields(sock) + if len(handshake_fields) != 2: + raise RuntimeError(f"unexpected IB handshake response: {handshake_fields!r}") + server_version = int(handshake_fields[0]) + if server_version < 157: + raise RuntimeError(f"IB server version too old for healthcheck: {server_version}") + + start_api_payload = b"71\0" + b"2\0" + str(client_id).encode() + b"\0\0" + send_prefixed(sock, start_api_payload) + + has_next_valid_id = False + has_managed_accounts = False + while not (has_next_valid_id and has_managed_accounts): + fields = read_fields(sock) + if not fields: + continue + msg_id = fields[0] + if msg_id == "9": + has_next_valid_id = True + elif msg_id == "15": + has_managed_accounts = True + +print(f"IB API handshake ready: server_version={server_version} client_id={client_id}") +PY +} + +echo "Waiting for ${container_name} IB API handshake readiness on internal port ${gateway_port} (mode=${gateway_mode}, client_id=${healthcheck_client_id})" while true; do if docker inspect --format '{{.State.Running}}' "${container_name}" 2>/dev/null | grep -Fxq 'true'; then - if timeout 3 docker exec "${container_name}" bash -lc "exec 3<>/dev/tcp/127.0.0.1/${gateway_port}" >/dev/null 2>&1; then - echo "IB gateway API is ready on internal port ${gateway_port} (mode=${gateway_mode})" + if check_api_handshake; then + echo "IB gateway API handshake is ready on internal port ${gateway_port} (mode=${gateway_mode})" exit 0 fi fi if [ "${SECONDS}" -ge "${deadline}" ]; then - echo "Timed out waiting for ${container_name} API readiness on internal port ${gateway_port}" >&2 + echo "Timed out waiting for ${container_name} IB API handshake readiness on internal port ${gateway_port}" >&2 echo "--- docker compose ps ---" >&2 docker compose ps >&2 || true echo "--- recent container logs ---" >&2 diff --git a/tests/test_gateway_recovery_scripts.sh b/tests/test_gateway_recovery_scripts.sh index a97ee8a..ab5d9d8 100644 --- a/tests/test_gateway_recovery_scripts.sh +++ b/tests/test_gateway_recovery_scripts.sh @@ -4,11 +4,17 @@ set -euo pipefail repo_dir="$(cd "$(dirname "$0")/.." && pwd)" recover_script="$repo_dir/scripts/recover_ib_gateway_ready.sh" swap_script="$repo_dir/scripts/ensure_host_swap.sh" +daily_restart_script="$repo_dir/scripts/restart_ib_gateway_daily.sh" +health_watcher_script="$repo_dir/scripts/install_gateway_health_watcher.sh" test -f "$recover_script" test -f "$swap_script" +test -f "$daily_restart_script" +test -f "$health_watcher_script" test -x "$recover_script" test -x "$swap_script" +test -x "$daily_restart_script" +test -x "$health_watcher_script" grep -Fq 'IB_GATEWAY_RECOVERY_INITIAL_WAIT_SECONDS:-60' "$recover_script" grep -Fq 'docker compose restart "${container_name}"' "$recover_script" @@ -20,3 +26,15 @@ grep -Fq 'fallocate -l "${swap_size_mib}M" "${swap_file}"' "$swap_script" grep -Fq 'swapon "${swap_file}"' "$swap_script" grep -Fq 'grep -Fq "${swap_file} none swap sw 0 0" /etc/fstab' "$swap_script" grep -Fq 'printf '\''%s none swap sw 0 0\n'\'' "${swap_file}" >>/etc/fstab' "$swap_script" + +grep -Fq 'docker compose restart "${container_name}"' "$daily_restart_script" +grep -Fq 'recover_ib_gateway_ready.sh" "${gateway_mode}"' "$daily_restart_script" + +grep -Fq 'ibkr-gateway-healthcheck.service' "$health_watcher_script" +grep -Fq 'ibkr-gateway-healthcheck.timer' "$health_watcher_script" +grep -Fq 'ibkr-gateway-daily-restart.service' "$health_watcher_script" +grep -Fq 'ibkr-gateway-daily-restart.timer' "$health_watcher_script" +grep -Fq 'IB_GATEWAY_HEALTHCHECK_INTERVAL_SECONDS:-300' "$health_watcher_script" +grep -Fq 'IB_GATEWAY_DAILY_RESTART_ON_CALENDAR:-*-*-* 10:30:00 UTC' "$health_watcher_script" +grep -Fq 'enable --now ibkr-gateway-healthcheck.timer' "$health_watcher_script" +grep -Fq 'enable --now ibkr-gateway-daily-restart.timer' "$health_watcher_script" diff --git a/tests/test_wait_for_ib_gateway_ready.sh b/tests/test_wait_for_ib_gateway_ready.sh index 0d9d7ff..2a17c3b 100644 --- a/tests/test_wait_for_ib_gateway_ready.sh +++ b/tests/test_wait_for_ib_gateway_ready.sh @@ -11,5 +11,9 @@ grep -Fq 'ready_timeout_seconds="${IB_GATEWAY_READY_TIMEOUT_SECONDS:-240}"' "$sc grep -Fq 'gateway_port=4002' "$script_file" grep -Fq 'gateway_port=4001' "$script_file" grep -Fq "docker inspect --format '{{.State.Running}}'" "$script_file" -grep -Fq 'timeout 3 docker exec "${container_name}" bash -lc "exec 3<>/dev/tcp/127.0.0.1/${gateway_port}"' "$script_file" +grep -Fq 'IB_GATEWAY_HEALTHCHECK_CLIENT_ID:-999' "$script_file" +grep -Fq 'check_api_handshake()' "$script_file" +grep -Fq 'b"API\0" + struct.pack(">I", len(b"v157..176")) + b"v157..176"' "$script_file" +grep -Fq 'has_next_valid_id and has_managed_accounts' "$script_file" +grep -Fq 'IB API handshake readiness' "$script_file" grep -Fq 'docker logs --tail 120 "${container_name}"' "$script_file" diff --git a/tests/test_workflow_shared_config.sh b/tests/test_workflow_shared_config.sh index bd98614..8fb617c 100644 --- a/tests/test_workflow_shared_config.sh +++ b/tests/test_workflow_shared_config.sh @@ -49,7 +49,10 @@ grep -Fq 'gcloud compute instances reset "${GCE_INSTANCE_NAME}"' "$workflow_file grep -Fq 'run_remote_ssh "Repository sync" "${REMOTE_SYNC_COMMAND}"' "$workflow_file" grep -Fq 'copy_remote_file "${ENV_FILE}" "${DEPLOY_PATH}/.env"' "$workflow_file" grep -Fq 'sudo bash ./scripts/ensure_host_swap.sh' "$workflow_file" +grep -Fq 'sudo bash ./scripts/install_gateway_health_watcher.sh' "$workflow_file" grep -Fq "sudo bash ./scripts/recover_ib_gateway_ready.sh '\${IB_GATEWAY_MODE}'" "$workflow_file" +grep -Fq 'sudo systemctl status ibkr-gateway-healthcheck.timer --no-pager' "$workflow_file" +grep -Fq 'sudo systemctl status ibkr-gateway-daily-restart.timer --no-pager' "$workflow_file" grep -Fq 'Full deploy mode: rebuilding container' "$workflow_file" grep -Fq '"TRADING_MODE": os.environ["IB_GATEWAY_MODE"]' "$workflow_file" grep -Fq '"ACCEPT_API_FROM_IP": os.environ["CLOUD_RUN_EGRESS_CIDR"]' "$workflow_file"