diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 277de6c..37296c7 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -348,6 +348,7 @@ jobs: set -euo pipefail cd '${DEPLOY_PATH}' echo "[\$(date -u +%FT%TZ)] Scheduled keepalive mode: skip docker build" + sudo systemctl stop ibkr-gateway-healthcheck.timer ibkr-gateway-healthcheck.service 2>/dev/null || true sudo bash ./scripts/ensure_host_swap.sh sudo bash ./scripts/install_2fa_bot_watcher.sh sudo bash ./scripts/install_gateway_health_watcher.sh @@ -364,6 +365,7 @@ jobs: set -euo pipefail cd '${DEPLOY_PATH}' echo "[\$(date -u +%FT%TZ)] Full deploy mode: rebuilding container" + sudo systemctl stop ibkr-gateway-healthcheck.timer ibkr-gateway-healthcheck.service 2>/dev/null || true sudo bash ./scripts/ensure_host_swap.sh sudo docker compose build sudo docker compose down diff --git a/scripts/install_gateway_health_watcher.sh b/scripts/install_gateway_health_watcher.sh index a40e412..d037b54 100755 --- a/scripts/install_gateway_health_watcher.sh +++ b/scripts/install_gateway_health_watcher.sh @@ -23,6 +23,7 @@ Type=oneshot WorkingDirectory=$repo_dir Environment=IB_GATEWAY_CONTAINER_NAME=$container_name Environment=IB_GATEWAY_MODE=$gateway_mode +Environment=IB_GATEWAY_RECOVERY_LOCK_WAIT_SECONDS=0 ExecStart=/bin/bash -lc 'cd "$repo_dir" && exec ./scripts/recover_ib_gateway_ready.sh "$gateway_mode"' EOF diff --git a/scripts/recover_ib_gateway_ready.sh b/scripts/recover_ib_gateway_ready.sh index a664a9d..bf687f6 100755 --- a/scripts/recover_ib_gateway_ready.sh +++ b/scripts/recover_ib_gateway_ready.sh @@ -8,9 +8,25 @@ gateway_mode="${1:-${IB_GATEWAY_MODE:-paper}}" initial_wait_seconds="${IB_GATEWAY_RECOVERY_INITIAL_WAIT_SECONDS:-240}" restart_wait_seconds="${IB_GATEWAY_RECOVERY_RESTART_WAIT_SECONDS:-300}" recreate_wait_seconds="${IB_GATEWAY_RECOVERY_RECREATE_WAIT_SECONDS:-360}" +lock_file="${IB_GATEWAY_RECOVERY_LOCK_FILE:-/var/lock/ib_gateway_recovery.lock}" +lock_wait_seconds="${IB_GATEWAY_RECOVERY_LOCK_WAIT_SECONDS:-900}" cd "${repo_dir}" +mkdir -p "$(dirname "${lock_file}")" 2>/dev/null || true +exec 9>"${lock_file}" +if [ "${lock_wait_seconds}" = "0" ]; then + if ! flock -n 9; then + echo "Another IB gateway recovery is already running; skipping this check." + exit 0 + fi +elif ! flock -w "${lock_wait_seconds}" 9; then + echo "Timed out waiting for IB gateway recovery lock: ${lock_file}" >&2 + exit 1 +fi + +echo "Acquired IB gateway recovery lock: ${lock_file}" + wait_for_ready() { local timeout_seconds="$1" IB_GATEWAY_CONTAINER_NAME="${container_name}" \ diff --git a/tests/test_gateway_recovery_scripts.sh b/tests/test_gateway_recovery_scripts.sh index c8dcbcd..f7ddf7c 100644 --- a/tests/test_gateway_recovery_scripts.sh +++ b/tests/test_gateway_recovery_scripts.sh @@ -19,6 +19,10 @@ test -x "$health_watcher_script" grep -Fq 'IB_GATEWAY_RECOVERY_INITIAL_WAIT_SECONDS:-240' "$recover_script" grep -Fq 'IB_GATEWAY_RECOVERY_RESTART_WAIT_SECONDS:-300' "$recover_script" grep -Fq 'IB_GATEWAY_RECOVERY_RECREATE_WAIT_SECONDS:-360' "$recover_script" +grep -Fq 'IB_GATEWAY_RECOVERY_LOCK_FILE:-/var/lock/ib_gateway_recovery.lock' "$recover_script" +grep -Fq 'IB_GATEWAY_RECOVERY_LOCK_WAIT_SECONDS:-900' "$recover_script" +grep -Fq 'flock -n 9' "$recover_script" +grep -Fq 'flock -w "${lock_wait_seconds}" 9' "$recover_script" grep -Fq 'docker compose restart "${container_name}"' "$recover_script" grep -Fq 'docker compose up -d --force-recreate --no-build "${container_name}"' "$recover_script" grep -Fq 'IB_GATEWAY_READY_TIMEOUT_SECONDS="${timeout_seconds}"' "$recover_script" @@ -38,6 +42,7 @@ grep -Fq 'ibkr-gateway-daily-restart.service' "$health_watcher_script" grep -Fq 'ibkr-gateway-daily-restart.timer' "$health_watcher_script" grep -Fq 'IB_GATEWAY_HEALTHCHECK_INTERVAL_SECONDS:-300' "$health_watcher_script" grep -Fq 'IB_GATEWAY_DAILY_RESTART_ON_CALENDAR:-*-*-* 10:30:00 UTC' "$health_watcher_script" +grep -Fq 'Environment=IB_GATEWAY_RECOVERY_LOCK_WAIT_SECONDS=0' "$health_watcher_script" grep -Fq 'enable --now ibkr-gateway-healthcheck.timer' "$health_watcher_script" grep -Fq 'enable --now ibkr-gateway-daily-restart.timer' "$health_watcher_script" ! grep -Fq 'start ibkr-gateway-healthcheck.service' "$health_watcher_script" diff --git a/tests/test_workflow_shared_config.sh b/tests/test_workflow_shared_config.sh index 8fb617c..401a422 100644 --- a/tests/test_workflow_shared_config.sh +++ b/tests/test_workflow_shared_config.sh @@ -49,6 +49,7 @@ grep -Fq 'gcloud compute instances reset "${GCE_INSTANCE_NAME}"' "$workflow_file grep -Fq 'run_remote_ssh "Repository sync" "${REMOTE_SYNC_COMMAND}"' "$workflow_file" grep -Fq 'copy_remote_file "${ENV_FILE}" "${DEPLOY_PATH}/.env"' "$workflow_file" grep -Fq 'sudo bash ./scripts/ensure_host_swap.sh' "$workflow_file" +grep -Fq 'sudo systemctl stop ibkr-gateway-healthcheck.timer ibkr-gateway-healthcheck.service 2>/dev/null || true' "$workflow_file" grep -Fq 'sudo bash ./scripts/install_gateway_health_watcher.sh' "$workflow_file" grep -Fq "sudo bash ./scripts/recover_ib_gateway_ready.sh '\${IB_GATEWAY_MODE}'" "$workflow_file" grep -Fq 'sudo systemctl status ibkr-gateway-healthcheck.timer --no-pager' "$workflow_file"