QuantStrategyLab · Pigbibi · Apr 17, 2026 · Apr 15, 2026
@@ -350,9 +350,12 @@ jobs:
           echo "[\$(date -u +%FT%TZ)] Scheduled keepalive mode: skip docker build"
           sudo bash ./scripts/ensure_host_swap.sh
           sudo bash ./scripts/install_2fa_bot_watcher.sh
+          sudo bash ./scripts/install_gateway_health_watcher.sh
           sudo bash ./scripts/recover_ib_gateway_ready.sh '${IB_GATEWAY_MODE}'
           sudo docker compose ps
           sudo systemctl status ibkr-2fa-bot.timer --no-pager
+          sudo systemctl status ibkr-gateway-healthcheck.timer --no-pager
+          sudo systemctl status ibkr-gateway-daily-restart.timer --no-pager
           echo "[\$(date -u +%FT%TZ)] Keepalive complete."
           EOF
             )
@@ -367,9 +370,12 @@ jobs:
           sudo docker compose up -d
 
           sudo bash ./scripts/install_2fa_bot_watcher.sh
+          sudo bash ./scripts/install_gateway_health_watcher.sh
           sudo bash ./scripts/recover_ib_gateway_ready.sh '${IB_GATEWAY_MODE}'
           sudo docker compose ps
           sudo systemctl status ibkr-2fa-bot.timer --no-pager
+          sudo systemctl status ibkr-gateway-healthcheck.timer --no-pager
+          sudo systemctl status ibkr-gateway-daily-restart.timer --no-pager
 
           echo "Deployment complete. Bot is monitoring the login window."
           echo "Run 'docker exec ib-gateway tail -f /home/ibgateway/2fa.log' for real-time status."

@@ -9,6 +9,7 @@ An automated deployment solution for IBKR Gateway on Google Compute Engine (GCE)
 - **Containerized Deployment**: Docker-based setup.
 - **Automated 2FA**: Python bot (`pyotp` + `xdotool`) auto-fills TOTP.
 - **Daily Auto-Reconnect**: restart policy for long-running reliability.
+- **API Handshake Recovery**: systemd health check validates the IB API handshake and restarts/recreates the container if the API is not ready.
 - **Private Network API Access**: IBKR API exposed on GCE private network for Cloud Run.
 
 ---
@@ -103,6 +104,7 @@ This shared GitHub config is scoped to the **IBKR deployment pair only** (`Inter
 ```bash
 docker compose up -d --build
 sudo bash ./scripts/install_2fa_bot_watcher.sh
+sudo bash ./scripts/install_gateway_health_watcher.sh
 ```
 
 > If you use this repository's GitHub Actions workflow, pushing deploy-related changes to `main` triggers a full deployment to GCE. The daily scheduled run only does a lightweight keepalive start and watcher check; it does not rebuild the Docker image.
@@ -118,6 +120,17 @@ ss -lntp | grep -E '4001|4002'
 
 Expected: host is listening on `0.0.0.0:4001` and/or `0.0.0.0:4002` (or VM private interface) and container is healthy.
 
+The readiness script checks the actual IB API handshake, not just TCP connectivity:
+
+```bash
+sudo bash ./scripts/wait_for_ib_gateway_ready.sh paper
+```
+
+It opens an IB protocol session, waits for the Gateway handshake, sends `StartApi`,
+and only succeeds after `nextValidId` plus `managedAccounts` have arrived. This
+catches the common failure mode where the Docker port is listening but Gateway is
+blocked by a login/API prompt.
+
 ---
 
 ## Cloud Run Connectivity Checklist
@@ -215,6 +228,25 @@ docker exec ib-gateway pgrep -f 2fa_bot.py
 systemctl status ibkr-2fa-bot.timer --no-pager
 ```
 
+### Check Gateway API Health Timers
+
+```bash
+systemctl status ibkr-gateway-healthcheck.timer --no-pager
+systemctl status ibkr-gateway-daily-restart.timer --no-pager
+```
+
+`ibkr-gateway-healthcheck.timer` runs every 5 minutes by default. It calls
+`recover_ib_gateway_ready.sh`, which first checks IB API handshake readiness,
+then restarts and finally recreates the container if the API does not recover.
+
+`ibkr-gateway-daily-restart.timer` restarts the Gateway once per day at
+`10:30 UTC` by default, then waits for the same API handshake readiness. Override
+the schedule during install with:
+
+```bash
+IB_GATEWAY_DAILY_RESTART_ON_CALENDAR='Mon..Fri 10:30:00 UTC' sudo bash ./scripts/install_gateway_health_watcher.sh
+```
+
 ### Check API Port in Container
 
 ```bash

@@ -0,0 +1,72 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+script_dir="$(cd "$(dirname "$0")" && pwd)"
+repo_dir="${REPO_DIR:-$(cd "${script_dir}/.." && pwd)}"
+systemd_dir="${SYSTEMD_DIR:-/etc/systemd/system}"
+systemctl_bin="${SYSTEMCTL_BIN:-systemctl}"
+container_name="${CONTAINER_NAME:-${IB_GATEWAY_CONTAINER_NAME:-ib-gateway}}"
+gateway_mode="${IB_GATEWAY_MODE:-paper}"
+health_interval_seconds="${IB_GATEWAY_HEALTHCHECK_INTERVAL_SECONDS:-300}"
+daily_restart_calendar="${IB_GATEWAY_DAILY_RESTART_ON_CALENDAR:-*-*-* 10:30:00 UTC}"
+
+install -d "$systemd_dir"
+
+cat >"$systemd_dir/ibkr-gateway-healthcheck.service" <<EOF
+[Unit]
+Description=Check and recover IBKR Gateway API readiness
+After=docker.service network-online.target
+Wants=docker.service network-online.target
+
+[Service]
+Type=oneshot
+WorkingDirectory=$repo_dir
+Environment=IB_GATEWAY_CONTAINER_NAME=$container_name
+Environment=IB_GATEWAY_MODE=$gateway_mode
+ExecStart=/bin/bash -lc 'cd "$repo_dir" && exec ./scripts/recover_ib_gateway_ready.sh "$gateway_mode"'
+EOF
+
+cat >"$systemd_dir/ibkr-gateway-healthcheck.timer" <<EOF
+[Unit]
+Description=Run IBKR Gateway API readiness recovery every $health_interval_seconds seconds
+
+[Timer]
+OnBootSec=5min
+OnUnitActiveSec=$health_interval_seconds
+Unit=ibkr-gateway-healthcheck.service
+
+[Install]
+WantedBy=timers.target
+EOF
+
+cat >"$systemd_dir/ibkr-gateway-daily-restart.service" <<EOF
+[Unit]
+Description=Scheduled IBKR Gateway container restart
+After=docker.service network-online.target
+Wants=docker.service network-online.target
+
+[Service]
+Type=oneshot
+WorkingDirectory=$repo_dir
+Environment=IB_GATEWAY_CONTAINER_NAME=$container_name
+Environment=IB_GATEWAY_MODE=$gateway_mode
+ExecStart=/bin/bash -lc 'cd "$repo_dir" && exec ./scripts/restart_ib_gateway_daily.sh "$gateway_mode"'
+EOF
+
+cat >"$systemd_dir/ibkr-gateway-daily-restart.timer" <<EOF
+[Unit]
+Description=Restart IBKR Gateway on a fixed daily schedule
+
+[Timer]
+OnCalendar=$daily_restart_calendar
+Persistent=true
+Unit=ibkr-gateway-daily-restart.service
+
+[Install]
+WantedBy=timers.target
+EOF
+
+"$systemctl_bin" daemon-reload
+"$systemctl_bin" enable --now ibkr-gateway-healthcheck.timer
+"$systemctl_bin" enable --now ibkr-gateway-daily-restart.timer
+"$systemctl_bin" start ibkr-gateway-healthcheck.service
@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+script_dir="$(cd "$(dirname "$0")" && pwd)"
+repo_dir="$(cd "${script_dir}/.." && pwd)"
+container_name="${IB_GATEWAY_CONTAINER_NAME:-ib-gateway}"
+gateway_mode="${1:-${IB_GATEWAY_MODE:-paper}}"
+ready_wait_seconds="${IB_GATEWAY_DAILY_RESTART_READY_WAIT_SECONDS:-240}"
+
+cd "${repo_dir}"
+
+echo "Restarting ${container_name} for scheduled IB Gateway refresh (mode=${gateway_mode})."
+docker compose up -d --no-build
+docker compose restart "${container_name}"
+
+IB_GATEWAY_CONTAINER_NAME="${container_name}" \
+IB_GATEWAY_RECOVERY_INITIAL_WAIT_SECONDS="${ready_wait_seconds}" \
+  bash "${script_dir}/recover_ib_gateway_ready.sh" "${gateway_mode}"
+
+echo "Scheduled IB Gateway refresh complete."
@@ -5,6 +5,8 @@ container_name="${IB_GATEWAY_CONTAINER_NAME:-ib-gateway}"
 gateway_mode="${1:-${IB_GATEWAY_MODE:-paper}}"
 ready_timeout_seconds="${IB_GATEWAY_READY_TIMEOUT_SECONDS:-240}"
 poll_interval_seconds="${IB_GATEWAY_READY_POLL_INTERVAL_SECONDS:-5}"
+handshake_timeout_seconds="${IB_GATEWAY_HANDSHAKE_TIMEOUT_SECONDS:-12}"
+healthcheck_client_id="${IB_GATEWAY_HEALTHCHECK_CLIENT_ID:-999}"
 
 case "${gateway_mode}" in
   paper)
@@ -21,18 +23,103 @@ esac
 
 deadline=$((SECONDS + ready_timeout_seconds))
 
-echo "Waiting for ${container_name} API to become ready on internal port ${gateway_port} (mode=${gateway_mode})"
+check_api_handshake() {
+  timeout "${handshake_timeout_seconds}" docker exec -i "${container_name}" \
+    env IB_GATEWAY_HEALTHCHECK_PORT="${gateway_port}" \
+      IB_GATEWAY_HEALTHCHECK_CLIENT_ID="${healthcheck_client_id}" \
+      IB_GATEWAY_HEALTHCHECK_TIMEOUT_SECONDS="${handshake_timeout_seconds}" \
+    python3 <<'PY'
+import os
+import socket
+import struct
+import time
+
+
+host = "127.0.0.1"
+port = int(os.environ["IB_GATEWAY_HEALTHCHECK_PORT"])
+client_id = int(os.environ["IB_GATEWAY_HEALTHCHECK_CLIENT_ID"])
+timeout_seconds = float(os.environ["IB_GATEWAY_HEALTHCHECK_TIMEOUT_SECONDS"])
+deadline = time.monotonic() + timeout_seconds
+
+
+def remaining_timeout() -> float:
+    remaining = deadline - time.monotonic()
+    if remaining <= 0:
+        raise TimeoutError("IB API healthcheck timed out")
+    return remaining
+
+
+def recv_exact(sock: socket.socket, size: int) -> bytes:
+    chunks = []
+    remaining = size
+    while remaining > 0:
+        sock.settimeout(remaining_timeout())
+        chunk = sock.recv(remaining)
+        if not chunk:
+            raise ConnectionError("IB API socket closed during healthcheck")
+        chunks.append(chunk)
+        remaining -= len(chunk)
+    return b"".join(chunks)
+
+
+def read_fields(sock: socket.socket) -> list[str]:
+    raw_size = recv_exact(sock, 4)
+    size = struct.unpack(">I", raw_size)[0]
+    payload = recv_exact(sock, size).decode(errors="backslashreplace")
+    fields = payload.split("\0")
+    if fields and fields[-1] == "":
+        fields.pop()
+    return fields
+
+
+def send_prefixed(sock: socket.socket, payload: bytes) -> None:
+    sock.sendall(struct.pack(">I", len(payload)) + payload)
+
+
+with socket.create_connection((host, port), timeout=remaining_timeout()) as sock:
+    sock.settimeout(remaining_timeout())
+    # Match the modern IB API v100+ handshake used by ib_insync.
+    hello = b"API\0" + struct.pack(">I", len(b"v157..176")) + b"v157..176"
+    sock.sendall(hello)
+
+    handshake_fields = read_fields(sock)
+    if len(handshake_fields) != 2:
+        raise RuntimeError(f"unexpected IB handshake response: {handshake_fields!r}")
+    server_version = int(handshake_fields[0])
+    if server_version < 157:
+        raise RuntimeError(f"IB server version too old for healthcheck: {server_version}")
+
+    start_api_payload = b"71\0" + b"2\0" + str(client_id).encode() + b"\0\0"
+    send_prefixed(sock, start_api_payload)
+
+    has_next_valid_id = False
+    has_managed_accounts = False
+    while not (has_next_valid_id and has_managed_accounts):
+        fields = read_fields(sock)
+        if not fields:
+            continue
+        msg_id = fields[0]
+        if msg_id == "9":
+            has_next_valid_id = True
+        elif msg_id == "15":
+            has_managed_accounts = True
+
+print(f"IB API handshake ready: server_version={server_version} client_id={client_id}")
+PY
+}
+
+echo "Waiting for ${container_name} IB API handshake readiness on internal port ${gateway_port} (mode=${gateway_mode}, client_id=${healthcheck_client_id})"
 
 while true; do
   if docker inspect --format '{{.State.Running}}' "${container_name}" 2>/dev/null | grep -Fxq 'true'; then
-    if timeout 3 docker exec "${container_name}" bash -lc "exec 3<>/dev/tcp/127.0.0.1/${gateway_port}" >/dev/null 2>&1; then
-      echo "IB gateway API is ready on internal port ${gateway_port} (mode=${gateway_mode})"
+    if check_api_handshake; then
+      echo "IB gateway API handshake is ready on internal port ${gateway_port} (mode=${gateway_mode})"
       exit 0
     fi
   fi
 
   if [ "${SECONDS}" -ge "${deadline}" ]; then
-    echo "Timed out waiting for ${container_name} API readiness on internal port ${gateway_port}" >&2
+    echo "Timed out waiting for ${container_name} IB API handshake readiness on internal port ${gateway_port}" >&2
     echo "--- docker compose ps ---" >&2
     docker compose ps >&2 || true
     echo "--- recent container logs ---" >&2

@@ -4,11 +4,17 @@ set -euo pipefail
 repo_dir="$(cd "$(dirname "$0")/.." && pwd)"
 recover_script="$repo_dir/scripts/recover_ib_gateway_ready.sh"
 swap_script="$repo_dir/scripts/ensure_host_swap.sh"
+daily_restart_script="$repo_dir/scripts/restart_ib_gateway_daily.sh"
+health_watcher_script="$repo_dir/scripts/install_gateway_health_watcher.sh"
 
 test -f "$recover_script"
 test -f "$swap_script"
+test -f "$daily_restart_script"
+test -f "$health_watcher_script"
 test -x "$recover_script"
 test -x "$swap_script"
+test -x "$daily_restart_script"
+test -x "$health_watcher_script"
 
 grep -Fq 'IB_GATEWAY_RECOVERY_INITIAL_WAIT_SECONDS:-60' "$recover_script"
 grep -Fq 'docker compose restart "${container_name}"' "$recover_script"
@@ -20,3 +26,15 @@ grep -Fq 'fallocate -l "${swap_size_mib}M" "${swap_file}"' "$swap_script"
 grep -Fq 'swapon "${swap_file}"' "$swap_script"
 grep -Fq 'grep -Fq "${swap_file} none swap sw 0 0" /etc/fstab' "$swap_script"
 grep -Fq 'printf '\''%s none swap sw 0 0\n'\'' "${swap_file}" >>/etc/fstab' "$swap_script"
+
+grep -Fq 'docker compose restart "${container_name}"' "$daily_restart_script"
+grep -Fq 'recover_ib_gateway_ready.sh" "${gateway_mode}"' "$daily_restart_script"
+
+grep -Fq 'ibkr-gateway-healthcheck.service' "$health_watcher_script"
+grep -Fq 'ibkr-gateway-healthcheck.timer' "$health_watcher_script"
+grep -Fq 'ibkr-gateway-daily-restart.service' "$health_watcher_script"
+grep -Fq 'ibkr-gateway-daily-restart.timer' "$health_watcher_script"
+grep -Fq 'IB_GATEWAY_HEALTHCHECK_INTERVAL_SECONDS:-300' "$health_watcher_script"
+grep -Fq 'IB_GATEWAY_DAILY_RESTART_ON_CALENDAR:-*-*-* 10:30:00 UTC' "$health_watcher_script"
+grep -Fq 'enable --now ibkr-gateway-healthcheck.timer' "$health_watcher_script"
+grep -Fq 'enable --now ibkr-gateway-daily-restart.timer' "$health_watcher_script"
@@ -11,5 +11,9 @@ grep -Fq 'ready_timeout_seconds="${IB_GATEWAY_READY_TIMEOUT_SECONDS:-240}"' "$sc
 grep -Fq 'gateway_port=4002' "$script_file"
 grep -Fq 'gateway_port=4001' "$script_file"
 grep -Fq "docker inspect --format '{{.State.Running}}'" "$script_file"
-grep -Fq 'timeout 3 docker exec "${container_name}" bash -lc "exec 3<>/dev/tcp/127.0.0.1/${gateway_port}"' "$script_file"
+grep -Fq 'IB_GATEWAY_HEALTHCHECK_CLIENT_ID:-999' "$script_file"
+grep -Fq 'check_api_handshake()' "$script_file"
+grep -Fq 'b"API\0" + struct.pack(">I", len(b"v157..176")) + b"v157..176"' "$script_file"
+grep -Fq 'has_next_valid_id and has_managed_accounts' "$script_file"
+grep -Fq 'IB API handshake readiness' "$script_file"
 grep -Fq 'docker logs --tail 120 "${container_name}"' "$script_file"
@@ -49,7 +49,10 @@ grep -Fq 'gcloud compute instances reset "${GCE_INSTANCE_NAME}"' "$workflow_file
 grep -Fq 'run_remote_ssh "Repository sync" "${REMOTE_SYNC_COMMAND}"' "$workflow_file"
 grep -Fq 'copy_remote_file "${ENV_FILE}" "${DEPLOY_PATH}/.env"' "$workflow_file"
 grep -Fq 'sudo bash ./scripts/ensure_host_swap.sh' "$workflow_file"
+grep -Fq 'sudo bash ./scripts/install_gateway_health_watcher.sh' "$workflow_file"
 grep -Fq "sudo bash ./scripts/recover_ib_gateway_ready.sh '\${IB_GATEWAY_MODE}'" "$workflow_file"
+grep -Fq 'sudo systemctl status ibkr-gateway-healthcheck.timer --no-pager' "$workflow_file"
+grep -Fq 'sudo systemctl status ibkr-gateway-daily-restart.timer --no-pager' "$workflow_file"
 grep -Fq 'Full deploy mode: rebuilding container' "$workflow_file"
 grep -Fq '"TRADING_MODE": os.environ["IB_GATEWAY_MODE"]' "$workflow_file"
 grep -Fq '"ACCEPT_API_FROM_IP": os.environ["CLOUD_RUN_EGRESS_CIDR"]' "$workflow_file"