Fix IBKR headless gateway startup

Dockerfile

@@ -14,4 +14,7 @@ RUN apt-get update && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 
+COPY --chown=1000:1000 ./container_overrides/run.sh /home/ibgateway/scripts/run.sh
+RUN chmod a+x /home/ibgateway/scripts/run.sh
+
 USER ibgateway

README.md

@@ -59,6 +59,9 @@ VNC_SERVER_PASSWORD=your_vnc_password
 TRADING_MODE=live
 TWS_ACCEPT_INCOMING=accept
 READ_ONLY_API=no
+TWOFA_TIMEOUT_ACTION=restart
+RELOGIN_AFTER_TWOFA_TIMEOUT=yes
+EXISTING_SESSION_DETECTED_ACTION=primary
 JAVA_HEAP_SIZE=512
 
 # Recommended: use the exact CIDR used by your Cloud Run egress path
@@ -202,6 +205,11 @@ the upstream gateway image's default `-Xmx768m` can leave too little memory for
 Docker, and the GCE guest agent. For better long-running stability, use at least
 `e2-small` / `e2-medium` instead of relying only on swap.
 
+This repository also overrides the upstream headless startup script so Gateway runs with
+`Xvfb :1 -screen 0 1024x768x24` and `x11vnc -noxdamage`. On the current GCE target, the
+upstream `16bpp` display frequently led to black VNC output and intermittent IBC window
+detection failures before login completed.
+
 For direct `docker compose` usage outside GitHub Actions, `ACCEPT_API_FROM_IP` must still be set explicitly in `.env`; there is no longer a silent default CIDR.
 
 These GitHub secrets are specific to this repository's deployment flow. They are not intended to be global secrets shared by every quant repository.

container_overrides/run.sh

@@ -0,0 +1,156 @@
+#!/bin/bash
+# shellcheck disable=SC2317
+# Don't warn about unreachable commands in this file
+
+set -Eeo pipefail
+
+echo "*************************************************************************"
+echo ".> Starting IBC/IB gateway"
+echo "*************************************************************************"
+
+source "${SCRIPT_PATH}/common.sh"
+
+stop_ibc() {
+	echo ".> 😘 Received SIGINT or SIGTERM. Shutting down IB Gateway."
+
+	if pgrep x11vnc >/dev/null; then
+		echo ".> Stopping x11vnc."
+		pkill x11vnc
+	fi
+
+	echo ".> Stopping Xvfb."
+	pkill Xvfb
+
+	if [ -n "$SSH_TUNNEL" ]; then
+		echo ".> Stopping ssh."
+		pkill run_ssh.sh
+		pkill ssh
+		echo ".> Stopping socat."
+		pkill run_socat.sh
+		pkill socat
+	else
+		echo ".> Stopping socat."
+		pkill run_socat.sh
+		pkill socat
+	fi
+
+	echo ".> Stopping IBC."
+	kill -SIGTERM "${pid[@]}"
+	wait "${pid[@]}"
+	echo ".> Done... $?"
+}
+
+start_xvfb() {
+	echo ".> Starting Xvfb server"
+	DISPLAY=:1
+	export DISPLAY
+	rm -f /tmp/.X1-lock
+	Xvfb "$DISPLAY" -ac -screen 0 "${IB_XVFB_SCREEN:-1024x768x24}" &
+}
+
+start_vnc() {
+	wait_x_socket
+	file_env 'VNC_SERVER_PASSWORD'
+	if [ -n "$VNC_SERVER_PASSWORD" ]; then
+		echo ".> Starting VNC server"
+		x11vnc -display "$DISPLAY" -forever -shared -bg -noipv6 \
+			-ncache_cr -noxdamage \
+			-passwd "$VNC_SERVER_PASSWORD" &
+		unset_env 'VNC_SERVER_PASSWORD'
+	else
+		echo ".> VNC server disabled"
+	fi
+}
+
+start_IBC() {
+	echo ".> Starting IBC in ${TRADING_MODE} mode, with params:"
+	echo ".>		Version: ${TWS_MAJOR_VRSN}"
+	echo ".>		program: ${IBC_COMMAND:-gateway}"
+	echo ".>		tws-path: ${TWS_PATH}"
+	echo ".>		ibc-path: ${IBC_PATH}"
+	echo ".>		ibc-init: ${IBC_INI}"
+	echo ".>		tws-settings-path: ${TWS_SETTINGS_PATH:-$TWS_PATH}"
+	echo ".>		on2fatimeout: ${TWOFA_TIMEOUT_ACTION}"
+	"${IBC_PATH}/scripts/ibcstart.sh" "${TWS_MAJOR_VRSN}" -g \
+		"--tws-path=${TWS_PATH}" \
+		"--ibc-path=${IBC_PATH}" "--ibc-ini=${IBC_INI}" \
+		"--on2fatimeout=${TWOFA_TIMEOUT_ACTION}" \
+		"--tws-settings-path=${TWS_SETTINGS_PATH:-}" &
+	_p="$!"
+	pid+=("$_p")
+	export pid
+	echo "$_p" >"/tmp/pid_${TRADING_MODE}"
+}
+
+start_process() {
+	set_ports
+	apply_settings
+	port_forwarding
+	start_IBC
+}
+
+if [ -n "$START_SCRIPTS" ]; then
+	run_scripts "$HOME/$START_SCRIPTS"
+fi
+
+start_xvfb
+setup_ssh
+set_java_heap
+start_vnc
+
+if [ -n "$X_SCRIPTS" ]; then
+	wait_x_socket
+	run_scripts "$HOME/$X_SCRIPTS"
+fi
+
+if [ "$TRADING_MODE" == "both" ] || [ "$DUAL_MODE" == "yes" ]; then
+	DUAL_MODE=yes
+	export DUAL_MODE
+	TRADING_MODE=live
+	_IBC_INI="${IBC_INI}"
+	export _IBC_INI
+	IBC_INI="${_IBC_INI}_${TRADING_MODE}"
+	if [ -n "$TWS_SETTINGS_PATH" ]; then
+		_TWS_SETTINGS_PATH="${TWS_SETTINGS_PATH}"
+		export _TWS_SETTINGS_PATH
+		TWS_SETTINGS_PATH="${_TWS_SETTINGS_PATH}_${TRADING_MODE}"
+	else
+		_TWS_SETTINGS_PATH="${TWS_PATH}"
+		export _TWS_SETTINGS_PATH
+		TWS_SETTINGS_PATH="${_TWS_SETTINGS_PATH}_${TRADING_MODE}"
+	fi
+fi
+
+start_process
+
+if [ "$DUAL_MODE" == "yes" ]; then
+	TRADING_MODE=paper
+	TWS_USERID="${TWS_USERID_PAPER}"
+	export TWS_USERID
+
+	if [ -n "${TWS_PASSWORD_PAPER_FILE}" ]; then
+		TWS_PASSWORD_FILE="${TWS_PASSWORD_PAPER_FILE}"
+		export TWS_PASSWORD_FILE
+	else
+		TWS_PASSWORD="${TWS_PASSWORD_PAPER}"
+		export TWS_PASSWORD
+	fi
+
+	SSH_VNC_PORT=
+	export SSH_VNC_PORT
+	SSH_REMOTE_PORT=
+	export SSH_REMOTE_PORT
+	IBC_INI="${_IBC_INI}_${TRADING_MODE}"
+	TWS_SETTINGS_PATH="${_TWS_SETTINGS_PATH}_${TRADING_MODE}"
+
+	sleep 15
+	start_process
+fi
+
+if [ -n "$IBC_SCRIPTS" ]; then
+	run_scripts "$HOME/$IBC_SCRIPTS"
+fi
+
+trap stop_ibc SIGINT SIGTERM
+wait "${pid[@]}"
+exit $?

docker-compose.yml

@@ -12,6 +12,11 @@ services:
       - TRADING_MODE=${TRADING_MODE:-paper}
       - TWS_ACCEPT_INCOMING=${TWS_ACCEPT_INCOMING:-accept}
       - READ_ONLY_API=${READ_ONLY_API:-no}
+      # Keep IBC aligned with the upstream image defaults so paper/live
+      # sessions recover after missed 2FA or stale-session prompts.
+      - TWOFA_TIMEOUT_ACTION=${TWOFA_TIMEOUT_ACTION:-restart}
+      - RELOGIN_AFTER_TWOFA_TIMEOUT=${RELOGIN_AFTER_TWOFA_TIMEOUT:-yes}
+      - EXISTING_SESSION_DETECTED_ACTION=${EXISTING_SESSION_DETECTED_ACTION:-primary}
       # e2-micro has less than 1 GiB RAM. The upstream image defaults to
       # -Xmx768m, which can starve sshd/Docker/guest-agent. Keep this
       # configurable, but use a safer default for the current VM.

tests/test_docker_compose_ports.sh

@@ -4,16 +4,24 @@ set -euo pipefail
 repo_dir="$(cd "$(dirname "$0")/.." && pwd)"
 compose_file="$repo_dir/docker-compose.yml"
 dockerfile="$repo_dir/Dockerfile"
+run_override="$repo_dir/container_overrides/run.sh"
 
 grep -Fq 'pip3 install pyotp ib_insync --break-system-packages' "$dockerfile"
 grep -Fq 'FROM gnzsnz/ib-gateway:10.37.1q' "$dockerfile"
+grep -Fq 'COPY --chown=1000:1000 ./container_overrides/run.sh /home/ibgateway/scripts/run.sh' "$dockerfile"
+grep -Fq 'chmod a+x /home/ibgateway/scripts/run.sh' "$dockerfile"
 grep -Fq 'libgtk-3-0' "$dockerfile"
 grep -Fq 'libglib2.0-0' "$dockerfile"
 grep -Fq 'libxtst6' "$dockerfile"
+grep -Fq 'Xvfb "$DISPLAY" -ac -screen 0 "${IB_XVFB_SCREEN:-1024x768x24}" &' "$run_override"
+grep -Fq -- '-ncache_cr -noxdamage' "$run_override"
 
 grep -Fq '      - "4001:4003"' "$compose_file"
 grep -Fq '      - "4002:4004"' "$compose_file"
 grep -Fq '      - TWS_ACCEPT_INCOMING=${TWS_ACCEPT_INCOMING:-accept}' "$compose_file"
 grep -Fq '      - READ_ONLY_API=${READ_ONLY_API:-no}' "$compose_file"
+grep -Fq '      - TWOFA_TIMEOUT_ACTION=${TWOFA_TIMEOUT_ACTION:-restart}' "$compose_file"
+grep -Fq '      - RELOGIN_AFTER_TWOFA_TIMEOUT=${RELOGIN_AFTER_TWOFA_TIMEOUT:-yes}' "$compose_file"
+grep -Fq '      - EXISTING_SESSION_DETECTED_ACTION=${EXISTING_SESSION_DETECTED_ACTION:-primary}' "$compose_file"
 grep -Fq '      - JAVA_HEAP_SIZE=${JAVA_HEAP_SIZE:-512}' "$compose_file"
 grep -Fq '      - ACCEPT_API_FROM_IP=${ACCEPT_API_FROM_IP:?Set ACCEPT_API_FROM_IP to your Cloud Run egress subnet or connector CIDR}' "$compose_file"