fix: harden IBKR runtime against historical failures

Co-Authored-By: Codex <noreply@openai.com>

Author: Pigbibi

.github/workflows/ci.yml

@@ -100,6 +100,9 @@ jobs:
           assert resolve_hk_canonical_profile("hk_global_etf_tactical_rotation") == "hk_global_etf_tactical_rotation"
           PY
 
+      - name: Validate production Cloud Run startup
+        run: uv run --no-sync python scripts/validate_cloud_run_startup.py
+
       - name: Install editable shared repositories
         run: |
           set -euo pipefail

Dockerfile

@@ -16,6 +16,7 @@ RUN apt-get update \
 COPY . .
 RUN python -m pip install --upgrade pip uv \
     && uv sync --frozen --no-dev \
+    && python scripts/validate_cloud_run_startup.py \
     && apt-get purge -y git \
     && apt-get autoremove -y --purge \
     && rm -rf /var/lib/apt/lists/* \

application/runtime_broker_adapters.py

@@ -82,9 +82,11 @@ def fetch_account_portfolio_snapshot(self, ib):
 
     def connect_ib(self):
         self.ensure_event_loop_fn()
-        host = self.host_resolver()
         last_error = None
         for attempt in range(1, self.connect_attempts + 1):
+            # Resolve on every retry. GCE-backed gateways can receive a new
+            # internal IP after restart, so retries must not pin the first one.
+            host = self.host_resolver()
             client_id = self.ib_client_id + ((attempt - 1) * self.client_id_retry_offset)
             self.printer(
                 "Connecting to IB gateway "

main.py

@@ -131,12 +131,16 @@ def resolve_gce_instance_ip(instance_name, zone):
 
 def get_ib_host():
     global IB_HOST
-    if IB_HOST:
-        return IB_HOST
     host = RUNTIME_SETTINGS.ib_gateway_instance_name
     zone = RUNTIME_SETTINGS.ib_gateway_zone
     if zone:
+        # A restarted GCE gateway can receive a new internal IP. The broker
+        # adapter calls this resolver for every retry, so refresh zoned hosts.
         host = resolve_gce_instance_ip(host, zone)
+        IB_HOST = host
+        return host
+    if IB_HOST:
+        return IB_HOST
     IB_HOST = host
     return host
 
@@ -1309,29 +1313,44 @@ def _handle_request(
             },
         )
         raise
-    except TimeoutError as exc:
+    except (ConnectionError, TimeoutError) as exc:
         append_runtime_report_error(
             report,
             stage="ibkr_connect",
             message=str(exc),
             error_type=type(exc).__name__,
+            failure_category="ibkr_gateway_unavailable",
+        )
+        finalize_runtime_report(
+            report,
+            status="error",
+            diagnostics={"failure_category": "ibkr_gateway_unavailable"},
+        )
+        event = (
+            "ibkr_gateway_connect_timeout"
+            if isinstance(exc, TimeoutError)
+            else "ibkr_gateway_connect_failed"
         )
-        finalize_runtime_report(report, status="error")
         log_runtime_event(
             log_context,
-            "ibkr_gateway_connect_timeout",
-            message="IBKR gateway handshake timed out",
+            event,
+            message=(
+                "IBKR gateway handshake timed out"
+                if isinstance(exc, TimeoutError)
+                else "IBKR gateway connection failed"
+            ),
             severity="ERROR",
             error_type=type(exc).__name__,
             error_message=str(exc),
+            failure_category="ibkr_gateway_unavailable",
         )
         error_msg = f"🚨 【IBKR 连接异常】\n{str(exc)}"
         _publish_runtime_failure_notification(
             detailed_text=error_msg,
             compact_text=error_msg,
             exc=exc,
         )
-        return "Error", 500
+        return "Error", 503
     except Exception as exc:
         append_runtime_report_error(
             report,

scripts/validate_cloud_run_startup.py

@@ -0,0 +1,72 @@
+#!/usr/bin/env python3
+"""Import the production WSGI app with an inert IBKR configuration.
+
+This check performs no network or broker operations. It catches dependency API
+drift, module import failures, and Flask route registration conflicts before a
+container can be deployed.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+
+
+def _install_smoke_environment() -> None:
+    account_group = "startup-smoke"
+    os.environ["ACCOUNT_GROUP"] = account_group
+    os.environ["IB_ACCOUNT_GROUP_CONFIG_JSON"] = json.dumps(
+        {
+            "groups": {
+                account_group: {
+                    "ib_gateway_instance_name": "127.0.0.1",
+                    "ib_gateway_mode": "paper",
+                    "ib_client_id": 1,
+                    "account_ids": ["DU000000"],
+                }
+            }
+        },
+        separators=(",", ":"),
+    )
+    os.environ["IB_ACCOUNT_GROUP_CONFIG_SECRET_NAME"] = ""
+    os.environ["RUNTIME_TARGET_JSON"] = json.dumps(
+        {
+            "platform_id": "ibkr",
+            "strategy_profile": "global_etf_rotation",
+            "dry_run_only": True,
+            "execution_mode": "paper",
+            "account_scope": account_group,
+        },
+        separators=(",", ":"),
+    )
+
+
+def validate_startup() -> None:
+    _install_smoke_environment()
+
+    import main
+
+    routes = {
+        rule.rule: set(rule.methods) - {"HEAD", "OPTIONS"}
+        for rule in main.app.url_map.iter_rules()
+    }
+    required_routes = {
+        "/health": {"GET"},
+        "/run": {"GET", "POST"},
+        "/dry-run": {"GET", "POST"},
+        "/probe": {"GET", "POST"},
+        "/monitor-dispatch": {"GET", "POST"},
+    }
+    missing_or_invalid = {
+        path: {"expected": sorted(methods), "actual": sorted(routes.get(path, set()))}
+        for path, methods in required_routes.items()
+        if routes.get(path) != methods
+    }
+    if missing_or_invalid:
+        raise RuntimeError(f"Cloud Run route contract is invalid: {missing_or_invalid}")
+
+    print(f"Cloud Run startup validation passed: routes={len(routes)}")
+
+
+if __name__ == "__main__":
+    validate_startup()

tests/test_cloud_run_startup_validation.py

@@ -0,0 +1,14 @@
+from pathlib import Path
+
+
+ROOT = Path(__file__).resolve().parents[1]
+
+
+def test_production_startup_validation_is_a_ci_and_image_build_gate():
+    command = "python scripts/validate_cloud_run_startup.py"
+    dockerfile = (ROOT / "Dockerfile").read_text()
+    ci_workflow = (ROOT / ".github" / "workflows" / "ci.yml").read_text()
+
+    assert command in dockerfile
+    assert "Validate production Cloud Run startup" in ci_workflow
+    assert f"uv run --no-sync {command}" in ci_workflow

tests/test_event_loop.py

@@ -138,6 +138,22 @@ def get(self, project, zone, instance):
     assert module.IB_HOST == "10.0.0.8"
 
 
+def test_get_ib_host_refreshes_zoned_gateway_ip(strategy_module_factory, monkeypatch):
+    module = strategy_module_factory(
+        IB_GATEWAY_ZONE="us-central1-a",
+        IB_ACCOUNT_GROUP_CONFIG_JSON=(
+            '{"groups":{"default":{"ib_gateway_instance_name":"ib-gateway",'
+            '"ib_gateway_mode":"live","ib_client_id":1}}}'
+        ),
+    )
+    resolved = iter(("10.0.0.8", "10.0.0.9"))
+    monkeypatch.setattr(module, "resolve_gce_instance_ip", lambda *_args: next(resolved))
+
+    assert module.get_ib_host() == "10.0.0.8"
+    assert module.get_ib_host() == "10.0.0.9"
+    assert module.IB_HOST == "10.0.0.9"
+
+
 def test_ib_gateway_mode_derives_paper_port(strategy_module_factory):
     module = strategy_module_factory(
         IB_ACCOUNT_GROUP_CONFIG_JSON=(

tests/test_request_handling.py

@@ -903,6 +903,47 @@ def test_handle_request_error_persists_machine_readable_report(strategy_module,
     assert len(observed["messages"]) == 1
 
 
+def test_handle_request_classifies_gateway_connection_failure_as_unavailable(strategy_module, monkeypatch):
+    observed = {"events": [], "notifications": []}
+
+    monkeypatch.setattr(strategy_module, "is_market_open_now", lambda **_kwargs: True)
+    monkeypatch.setattr(
+        strategy_module,
+        "run_strategy_core",
+        lambda **_kwargs: (_ for _ in ()).throw(
+            ConnectionRefusedError("TCP preflight failed: connection refused")
+        ),
+    )
+    monkeypatch.setattr(
+        strategy_module,
+        "persist_execution_report",
+        lambda report: observed.setdefault("report", dict(report)) or "/tmp/runtime-report.json",
+    )
+    monkeypatch.setattr(
+        strategy_module,
+        "log_runtime_event",
+        lambda _context, event, **fields: observed["events"].append((event, fields)),
+    )
+    monkeypatch.setattr(
+        strategy_module,
+        "publish_notification",
+        lambda **kwargs: observed["notifications"].append(kwargs),
+    )
+
+    with strategy_module.app.test_request_context("/run", method="POST"):
+        body, status = strategy_module.handle_request()
+
+    assert status == 503
+    assert body == "Error"
+    assert observed["report"]["status"] == "error"
+    assert observed["report"]["errors"][0]["stage"] == "ibkr_connect"
+    assert observed["report"]["errors"][0]["failure_category"] == "ibkr_gateway_unavailable"
+    assert observed["report"]["diagnostics"]["failure_category"] == "ibkr_gateway_unavailable"
+    assert observed["events"][-1][0] == "ibkr_gateway_connect_failed"
+    assert observed["events"][-1][1]["failure_category"] == "ibkr_gateway_unavailable"
+    assert len(observed["notifications"]) == 1
+
+
 def test_run_strategy_core_allows_multiple_runs_in_same_process(strategy_module, monkeypatch):
     observed = {"connect_calls": 0, "disconnect_calls": 0, "messages": []}
 

tests/test_runtime_broker_adapters.py

@@ -53,6 +53,39 @@ def test_connect_ib_accepts_configured_managed_account():
     assert ib.managedAccounts() == ["U1234567"]
 
 
+def test_connect_ib_resolves_gateway_host_again_for_each_retry():
+    observed = {"hosts": [], "resolved": []}
+    resolved_hosts = iter(("10.0.0.8", "10.0.0.9"))
+
+    def resolve_host():
+        host = next(resolved_hosts)
+        observed["resolved"].append(host)
+        return host
+
+    def connect(host, *_args, **_kwargs):
+        observed["hosts"].append(host)
+        if len(observed["hosts"]) == 1:
+            raise ConnectionRefusedError("gateway restarting")
+        return SimpleNamespace(managedAccounts=lambda: ["U1234567"])
+
+    adapters = _build_adapters()
+    adapters = adapters.__class__(
+        **{
+            **adapters.__dict__,
+            "host_resolver": resolve_host,
+            "connect_ib_fn": connect,
+            "connect_attempts": 2,
+        }
+    )
+
+    adapters.connect_ib()
+
+    assert observed == {
+        "hosts": ["10.0.0.8", "10.0.0.9"],
+        "resolved": ["10.0.0.8", "10.0.0.9"],
+    }
+
+
 def test_connect_ib_rejects_configured_account_not_visible_to_gateway_username():
     observed = {"disconnects": 0}