fix: harden precheck scheduler rollout

Co-Authored-By: Codex <noreply@openai.com>

Author: Pigbibi

.github/workflows/sync-cloud-run-env.yml

@@ -536,6 +536,32 @@ jobs:
             done
           done
 
+      - name: Pause legacy monitor dispatcher
+        if: steps.config.outputs.env_sync_enabled == 'true'
+        run: |
+          set -euo pipefail
+
+          if [ "${WORKFLOW_TARGET:-configured}" = "hk-verify" ]; then
+            echo "Skipping global dispatcher retirement for the isolated HK verify target."
+            exit 0
+          fi
+
+          scheduler_location="${CLOUD_SCHEDULER_LOCATION:-${CLOUD_RUN_REGION}}"
+          legacy_monitor_job="interactive-brokers-monitor-dispatcher-scheduler"
+          legacy_monitor_state="$(gcloud scheduler jobs describe "${legacy_monitor_job}" \
+            --project="${GCP_PROJECT_ID}" \
+            --location="${scheduler_location}" \
+            --format='value(state)' 2>/dev/null || true)"
+          if [ -z "${legacy_monitor_state}" ] || [ "${legacy_monitor_state}" = "PAUSED" ]; then
+            exit 0
+          fi
+
+          echo "Pausing legacy Cloud Scheduler job ${legacy_monitor_job} before removing its target configuration."
+          gcloud scheduler jobs pause "${legacy_monitor_job}" \
+            --project="${GCP_PROJECT_ID}" \
+            --location="${scheduler_location}" \
+            --quiet
+
       - name: Sync Cloud Run environment
         if: steps.config.outputs.env_sync_enabled == 'true'
         env:
@@ -1192,6 +1218,7 @@ jobs:
                     --oidc-token-audience="${service_url}" \
                     --attempt-deadline=120s \
                     --max-retry-attempts=0 \
+                    --max-retry-duration=0s \
                     --quiet
                 else
                   echo "Creating Cloud Scheduler precheck ${precheck_job_name} at ${desired_precheck_schedule}."
@@ -1206,6 +1233,7 @@ jobs:
                     --oidc-token-audience="${service_url}" \
                     --attempt-deadline=120s \
                     --max-retry-attempts=0 \
+                    --max-retry-duration=0s \
                     --quiet
                 fi
                 if [ "${precheck_state}" = "PAUSED" ]; then
@@ -1227,12 +1255,16 @@ jobs:
             esac
           done
 
-          python3 scripts/reconcile_cloud_runtime.py \
-            --platform=ibkr \
-            --project="${GCP_PROJECT_ID}" \
-            --region="${CLOUD_RUN_REGION}" \
-            --scheduler-location="${scheduler_location}" \
-            --delete-legacy-schedulers
+          reconcile_args=(
+            --platform=ibkr
+            --project="${GCP_PROJECT_ID}"
+            --region="${CLOUD_RUN_REGION}"
+            --scheduler-location="${scheduler_location}"
+          )
+          if [ "${WORKFLOW_TARGET:-configured}" != "hk-verify" ]; then
+            reconcile_args+=(--delete-legacy-schedulers)
+          fi
+          python3 scripts/reconcile_cloud_runtime.py "${reconcile_args[@]}"
 
       - name: Prune old Cloud Run revisions
         if: steps.config.outputs.enabled == 'true'

tests/test_scheduler_deadline_contract.py

@@ -52,6 +52,7 @@ def test_precheck_uses_per_service_scheduler_with_bounded_deadline() -> None:
     assert 'precheck_uri="${service_url}/dry-run"' in workflow
     assert workflow.count("--attempt-deadline=120s") == 2
     assert workflow.count("--max-retry-attempts=0") == 2
+    assert workflow.count("--max-retry-duration=0s") == 2
     assert 'gcloud scheduler jobs resume "${precheck_job_name}"' in workflow
     assert 'gcloud scheduler jobs pause "${precheck_job_name}"' in workflow
     assert 'monitor_job_name="interactive-brokers-monitor-dispatcher-scheduler"' not in workflow
@@ -62,3 +63,16 @@ def test_precheck_uses_per_service_scheduler_with_bounded_deadline() -> None:
     assert strategy_deadline is not None
     assert report_grace is not None
     assert 120 > int(strategy_deadline.group(1)) + int(report_grace.group(1))
+
+
+def test_legacy_dispatcher_is_paused_before_removing_its_targets() -> None:
+    workflow = WORKFLOW.read_text(encoding="utf-8")
+
+    pause_step = workflow.index("- name: Pause legacy monitor dispatcher")
+    remove_targets = workflow.index('"IBKR_MONITOR_DISPATCH_TARGETS_JSON"')
+
+    assert pause_step < remove_targets
+    assert 'gcloud scheduler jobs pause "${legacy_monitor_job}"' in workflow
+    assert 'if [ "${WORKFLOW_TARGET:-configured}" = "hk-verify" ]; then' in workflow
+    assert 'if [ "${WORKFLOW_TARGET:-configured}" != "hk-verify" ]; then' in workflow
+    assert 'reconcile_args+=(--delete-legacy-schedulers)' in workflow

tests/test_sync_cloud_run_env_workflow.sh

@@ -89,6 +89,11 @@ grep -Fq 'python3 scripts/reconcile_cloud_runtime.py' "$workflow_file"
 grep -Fq -- '--platform=ibkr' "$workflow_file"
 grep -Fq -- '--ensure-latest-traffic' "$workflow_file"
 grep -Fq -- '--delete-legacy-schedulers' "$workflow_file"
+grep -Fq 'Pause legacy monitor dispatcher' "$workflow_file"
+grep -Fq 'gcloud scheduler jobs pause "${legacy_monitor_job}"' "$workflow_file"
+grep -Fq 'if [ "${WORKFLOW_TARGET:-configured}" = "hk-verify" ]; then' "$workflow_file"
+grep -Fq 'if [ "${WORKFLOW_TARGET:-configured}" != "hk-verify" ]; then' "$workflow_file"
+grep -Fq 'reconcile_args+=(--delete-legacy-schedulers)' "$workflow_file"
 
 grep -Fq 'emit_target_env_pairs()' "$workflow_file"
 grep -Fq 'emit_target_remove_env_vars()' "$workflow_file"
@@ -162,6 +167,7 @@ grep -Fq 'gcloud scheduler jobs update http "${precheck_job_name}"' "$workflow_f
 grep -Fq 'gcloud scheduler jobs create http "${precheck_job_name}"' "$workflow_file"
 test "$(grep -Fc -- '--attempt-deadline=120s' "$workflow_file")" -eq 2
 test "$(grep -Fc -- '--max-retry-attempts=0' "$workflow_file")" -eq 2
+test "$(grep -Fc -- '--max-retry-duration=0s' "$workflow_file")" -eq 2
 grep -Fq 'gcloud scheduler jobs resume "${precheck_job_name}"' "$workflow_file"
 grep -Fq 'gcloud scheduler jobs pause "${precheck_job_name}"' "$workflow_file"
 test "$(grep -Fc 'monitor_job_name="interactive-brokers-monitor-dispatcher-scheduler"' "$workflow_file")" -eq 0