From 68ab2410554358bb494d477cf3314fe9d61a1f31 Mon Sep 17 00:00:00 2001 From: Hassan Abdel-Rahman Date: Mon, 6 Jul 2026 11:47:43 -0400 Subject: [PATCH 1/3] Fail the deploy when the DB migration crash-loops MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The `Wait for db-migration` step used to `sleep 180` and proceed, keying success on the pg-migration ECS service rollout rather than on the migration actually applying. A migration that exits non-zero crash-loops the service while the step still reports success, so the rest of the stack deploys against a DB missing the migration. Replace the blind sleep with a gate that watches the service's new deployment: succeed once it reaches rolloutState COMPLETED (the migrated task is up and stable, which the container's `... && sleep infinity` command only permits on a zero exit), and fail the moment a task on the new task-definition revision stops (the crash loop). The service has no ECS deployment circuit breaker, so rolloutState never flips to FAILED on its own — the stopped task's exit is read directly and fails the deploy. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/manual-deploy.yml | 112 +++++++++++++++++++++++++++- 1 file changed, 108 insertions(+), 4 deletions(-) diff --git a/.github/workflows/manual-deploy.yml b/.github/workflows/manual-deploy.yml index ac312f3317..21acb3b888 100644 --- a/.github/workflows/manual-deploy.yml +++ b/.github/workflows/manual-deploy.yml @@ -227,15 +227,119 @@ jobs: image: ${{ needs.build-pg-migration.outputs.image }} wait-for-service-stability: false - # the wait-for-service-stability flag doesn't seem to work in - # aws-actions/amazon-ecs-deploy-task-definition@v2. we keep getting timeouts - # waiting for service stability. So we are manually waiting here. + # Gate the rest of the deploy on the DB migration actually succeeding, so the + # app never rolls out ahead of its schema. `migrate-db` force-deploys the + # pg-migration service with the new image but doesn't wait + # (wait-for-service-stability: false), and the migration container's command + # is ` && sleep infinity` — so a task only reaches and stays + # RUNNING when the migration exits 0; a failing migration exits non-zero and + # ECS restarts it in a crash loop. This job watches the service's new + # deployment and asserts that outcome instead of blindly sleeping: it + # succeeds once the deployment reaches rolloutState COMPLETED (a migrated + # task is up and stable), and fails the moment a task on the new task + # definition revision STOPs (the crash loop). The pg-migration service has no + # ECS deployment circuit breaker, so rolloutState never flips to FAILED on + # its own — we read the stopped task's exit code directly. post-migrate-db: name: Wait for db-migration needs: [migrate-db] runs-on: ubuntu-latest + permissions: + contents: read + id-token: write + env: + CLUSTER: ${{ inputs.environment }} + SERVICE: boxel-pg-migration-${{ inputs.environment }} + CONTAINER: boxel-pg-migration + LOG_GROUP: ecs-boxel-pg-migration-${{ inputs.environment }} + # Generous cap: real failures are caught immediately via the stopped-task + # check, so this only bounds a migration that hangs (e.g. blocked on a + # lock) rather than fails. A long backfill migration must finish under it. + TIMEOUT_SECONDS: "1800" steps: - - run: sleep 180 + - name: Set up env + run: | + if [ "${{ inputs.environment }}" = "production" ]; then + echo "AWS_ROLE_ARN=arn:aws:iam::120317779495:role/github" >> "$GITHUB_ENV" + elif [ "${{ inputs.environment }}" = "staging" ]; then + echo "AWS_ROLE_ARN=arn:aws:iam::680542703984:role/github" >> "$GITHUB_ENV" + else + echo "unrecognized environment: ${{ inputs.environment }}" + exit 1 + fi + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.AWS_ROLE_ARN }} + aws-region: us-east-1 + + - name: Assert migration succeeded + run: | + set -euo pipefail + + # Scope every check to the task definition revision migrate-db just + # deployed (the new PRIMARY deployment) so a previous deploy's tasks + # can't be mistaken for this migration's. + target_td=$(aws ecs describe-services \ + --cluster "$CLUSTER" --services "$SERVICE" \ + --query "services[0].deployments[?status=='PRIMARY']|[0].taskDefinition" \ + --output text) + if [ -z "$target_td" ] || [ "$target_td" = "None" ]; then + echo "No PRIMARY deployment found for $SERVICE — cannot verify the migration." + exit 1 + fi + echo "Watching migration task definition: $target_td" + + region=us-east-1 + console_logs="https://${region}.console.aws.amazon.com/cloudwatch/home?region=${region}#logsV2:log-groups/log-group/${LOG_GROUP}" + + report_stopped() { + echo "Migration container logs: ${console_logs}" + aws ecs describe-tasks --cluster "$CLUSTER" --tasks "$@" \ + --query "tasks[].{task:taskArn,stopCode:stopCode,taskReason:stoppedReason,exit:containers[?name=='${CONTAINER}']|[0].exitCode,containerReason:containers[?name=='${CONTAINER}']|[0].reason}" \ + --output table || true + } + + deadline=$(( $(date +%s) + TIMEOUT_SECONDS )) + while :; do + # Fail fast on a crash loop: with `... && sleep infinity`, a task on + # the new revision can only be STOPPED if the migration exited + # non-zero (or the container otherwise died), never on success. + stopped=$(aws ecs list-tasks --cluster "$CLUSTER" --service-name "$SERVICE" \ + --desired-status STOPPED --query 'taskArns' --output text) + if [ -n "$stopped" ] && [ "$stopped" != "None" ]; then + failed=$(aws ecs describe-tasks --cluster "$CLUSTER" --tasks $stopped \ + --query "tasks[?taskDefinitionArn=='$target_td'].taskArn" --output text) + if [ -n "$failed" ] && [ "$failed" != "None" ]; then + echo "::error::DB migration failed — pg-migration task stopped instead of staying up." + report_stopped $failed + exit 1 + fi + fi + + state=$(aws ecs describe-services --cluster "$CLUSTER" --services "$SERVICE" \ + --query "services[0].deployments[?status=='PRIMARY']|[0].rolloutState" \ + --output text) + echo "pg-migration rollout state: ${state}" + case "$state" in + COMPLETED) + echo "DB migration succeeded — pg-migration task is running and stable." + exit 0 + ;; + FAILED) + echo "::error::pg-migration deployment reported rolloutState FAILED." + exit 1 + ;; + esac + + if [ "$(date +%s)" -ge "$deadline" ]; then + echo "::error::Timed out after ${TIMEOUT_SECONDS}s waiting for the DB migration to succeed." + echo "Migration container logs: ${console_logs}" + exit 1 + fi + sleep 15 + done deploy-prerender: name: Deploy prerender From 6e04fe4d4a137a8a41ef849639991824f20e5d23 Mon Sep 17 00:00:00 2001 From: Hassan Abdel-Rahman Date: Mon, 6 Jul 2026 12:08:19 -0400 Subject: [PATCH 2/3] Run migrations as a one-shot task and gate on its exit code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Gate the deploy on the migration container's actual exit code instead of on ECS service-rollout state. The pg-migration service has no health check, so a rollout reaches COMPLETED as soon as the container is RUNNING — which happens while node-pg-migrate is still executing — so rollout completion can't tell a finished migration from an in-progress one, and a slow migration would unblock the deploy before its schema landed. `migrate-db` now registers a task-definition revision pinned to the freshly built image and runs it as a one-shot Fargate task (in the pg-migration service's own subnets/security group, the one allowed to reach the DB), with the command overridden to run the migrations without the service's trailing `sleep infinity`. It waits for the task to stop and fails the deploy unless the migration container exited 0 — so a failed or crash-looping migration blocks every downstream service. The migrate command moves into packages/postgres/scripts/run-migrations.sh so the container CMD (service) and the one-shot command override share one source. Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/manual-deploy.yml | 176 ++++++++++---------- packages/postgres/Dockerfile | 13 +- packages/postgres/scripts/run-migrations.sh | 25 +++ 3 files changed, 114 insertions(+), 100 deletions(-) create mode 100755 packages/postgres/scripts/run-migrations.sh diff --git a/.github/workflows/manual-deploy.yml b/.github/workflows/manual-deploy.yml index 21acb3b888..eaecb2421e 100644 --- a/.github/workflows/manual-deploy.yml +++ b/.github/workflows/manual-deploy.yml @@ -107,7 +107,7 @@ jobs: dockerfile: "packages/ai-bot/Dockerfile" deploy-ai-bot: - needs: [build-ai-bot, post-migrate-db] + needs: [build-ai-bot, migrate-db] name: Deploy ai-bot to AWS ECS uses: cardstack/gh-actions/.github/workflows/ecs-deploy.yml@main secrets: inherit @@ -128,7 +128,7 @@ jobs: dockerfile: "packages/bot-runner/Dockerfile" deploy-bot-runner: - needs: [build-bot-runner, post-migrate-db] + needs: [build-bot-runner, migrate-db] name: Deploy bot-runner to AWS ECS uses: cardstack/gh-actions/.github/workflows/ecs-deploy.yml@main secrets: inherit @@ -211,38 +211,18 @@ jobs: environment: ${{ inputs.environment }} dockerfile: "packages/postgres/Dockerfile" + # Run the DB migrations as a one-shot ECS task and gate the rest of the deploy + # on its exit code, so the app never rolls out ahead of its schema. The + # pg-migration image applies migrations and exits with node-pg-migrate's + # status (packages/postgres/scripts/run-migrations.sh); run one-shot with the + # service's trailing `sleep infinity` overridden away, a failing or + # crash-looping migration exits non-zero and fails this job — blocking every + # downstream deploy. deploy-host / build-realm-server are deps so migrations + # run at the last possible moment, minimizing how long old code sees the new + # schema. migrate-db: - # use "deploy-host" and "build-realm-server" as deps so we can run - # migrations at last possible moment in order to reduce the amount of time - # that old code is pointing to new schema needs: [build-pg-migration, build-realm-server, deploy-host] - name: Deploy and run DB migrations - uses: cardstack/gh-actions/.github/workflows/ecs-deploy.yml@main - secrets: inherit - with: - container-name: "boxel-pg-migration" - environment: ${{ inputs.environment }} - cluster: ${{ inputs.environment }} - service-name: "boxel-pg-migration-${{ inputs.environment }}" - image: ${{ needs.build-pg-migration.outputs.image }} - wait-for-service-stability: false - - # Gate the rest of the deploy on the DB migration actually succeeding, so the - # app never rolls out ahead of its schema. `migrate-db` force-deploys the - # pg-migration service with the new image but doesn't wait - # (wait-for-service-stability: false), and the migration container's command - # is ` && sleep infinity` — so a task only reaches and stays - # RUNNING when the migration exits 0; a failing migration exits non-zero and - # ECS restarts it in a crash loop. This job watches the service's new - # deployment and asserts that outcome instead of blindly sleeping: it - # succeeds once the deployment reaches rolloutState COMPLETED (a migrated - # task is up and stable), and fails the moment a task on the new task - # definition revision STOPs (the crash loop). The pg-migration service has no - # ECS deployment circuit breaker, so rolloutState never flips to FAILED on - # its own — we read the stopped task's exit code directly. - post-migrate-db: - name: Wait for db-migration - needs: [migrate-db] + name: Run DB migrations runs-on: ubuntu-latest permissions: contents: read @@ -251,10 +231,10 @@ jobs: CLUSTER: ${{ inputs.environment }} SERVICE: boxel-pg-migration-${{ inputs.environment }} CONTAINER: boxel-pg-migration + IMAGE: ${{ needs.build-pg-migration.outputs.image }} LOG_GROUP: ecs-boxel-pg-migration-${{ inputs.environment }} - # Generous cap: real failures are caught immediately via the stopped-task - # check, so this only bounds a migration that hangs (e.g. blocked on a - # lock) rather than fails. A long backfill migration must finish under it. + # Bounds a migration that hangs (e.g. blocked on a lock); a long backfill + # migration must finish under it. Genuine failures exit immediately. TIMEOUT_SECONDS: "1800" steps: - name: Set up env @@ -274,73 +254,86 @@ jobs: role-to-assume: ${{ env.AWS_ROLE_ARN }} aws-region: us-east-1 - - name: Assert migration succeeded + - name: Run migrations and gate on exit code run: | set -euo pipefail - - # Scope every check to the task definition revision migrate-db just - # deployed (the new PRIMARY deployment) so a previous deploy's tasks - # can't be mistaken for this migration's. - target_td=$(aws ecs describe-services \ - --cluster "$CLUSTER" --services "$SERVICE" \ - --query "services[0].deployments[?status=='PRIMARY']|[0].taskDefinition" \ - --output text) - if [ -z "$target_td" ] || [ "$target_td" = "None" ]; then - echo "No PRIMARY deployment found for $SERVICE — cannot verify the migration." - exit 1 - fi - echo "Watching migration task definition: $target_td" - region=us-east-1 console_logs="https://${region}.console.aws.amazon.com/cloudwatch/home?region=${region}#logsV2:log-groups/log-group/${LOG_GROUP}" - report_stopped() { - echo "Migration container logs: ${console_logs}" - aws ecs describe-tasks --cluster "$CLUSTER" --tasks "$@" \ - --query "tasks[].{task:taskArn,stopCode:stopCode,taskReason:stoppedReason,exit:containers[?name=='${CONTAINER}']|[0].exitCode,containerReason:containers[?name=='${CONTAINER}']|[0].reason}" \ - --output table || true - } + # Register a new revision of the pg-migration task definition pinned to + # the image just built. Strip the read-only fields describe returns + # that register-task-definition rejects. + aws ecs describe-task-definition --task-definition "$SERVICE" \ + --query 'taskDefinition' --output json > td.json + jq --arg IMAGE "$IMAGE" --arg CONTAINER "$CONTAINER" ' + .containerDefinitions |= map(if .name == $CONTAINER then .image = $IMAGE else . end) + | del(.taskDefinitionArn, .revision, .status, .requiresAttributes, + .compatibilities, .registeredAt, .registeredBy, .deregisteredAt) + ' td.json > td-new.json + task_def=$(aws ecs register-task-definition --cli-input-json file://td-new.json \ + --query 'taskDefinition.taskDefinitionArn' --output text) + echo "Registered migration task definition: $task_def" + + # Place the one-shot task in the pg-migration service's own network + # config — its security group is the one allowed to reach the DB. + net=$(aws ecs describe-services --cluster "$CLUSTER" --services "$SERVICE" \ + --query 'services[0].networkConfiguration.awsvpcConfiguration' --output json) + subnets=$(echo "$net" | jq -r '.subnets | join(",")') + groups=$(echo "$net" | jq -r '.securityGroups | join(",")') + public=$(echo "$net" | jq -r '.assignPublicIp // "DISABLED"') + + # Override the command to run the migrations WITHOUT the service's + # trailing `sleep infinity`, so the task exits with the migration's + # status instead of staying up. + overrides=$(jq -cn --arg CONTAINER "$CONTAINER" \ + '{containerOverrides: [{name: $CONTAINER, command: ["./scripts/run-migrations.sh"]}]}') + run_out=$(aws ecs run-task \ + --cluster "$CLUSTER" \ + --task-definition "$task_def" \ + --launch-type FARGATE \ + --count 1 \ + --started-by "deploy-${GITHUB_SHA:0:7}" \ + --network-configuration "awsvpcConfiguration={subnets=[$subnets],securityGroups=[$groups],assignPublicIp=$public}" \ + --overrides "$overrides" \ + --output json) + task_arn=$(echo "$run_out" | jq -r '.tasks[0].taskArn // empty') + if [ -z "$task_arn" ]; then + echo "::error::Failed to start the migration task." + echo "$run_out" | jq '.failures' + exit 1 + fi + echo "Started migration task: $task_arn" + # Wait for the task to stop, then gate on the container's exit code. deadline=$(( $(date +%s) + TIMEOUT_SECONDS )) while :; do - # Fail fast on a crash loop: with `... && sleep infinity`, a task on - # the new revision can only be STOPPED if the migration exited - # non-zero (or the container otherwise died), never on success. - stopped=$(aws ecs list-tasks --cluster "$CLUSTER" --service-name "$SERVICE" \ - --desired-status STOPPED --query 'taskArns' --output text) - if [ -n "$stopped" ] && [ "$stopped" != "None" ]; then - failed=$(aws ecs describe-tasks --cluster "$CLUSTER" --tasks $stopped \ - --query "tasks[?taskDefinitionArn=='$target_td'].taskArn" --output text) - if [ -n "$failed" ] && [ "$failed" != "None" ]; then - echo "::error::DB migration failed — pg-migration task stopped instead of staying up." - report_stopped $failed - exit 1 - fi - fi - - state=$(aws ecs describe-services --cluster "$CLUSTER" --services "$SERVICE" \ - --query "services[0].deployments[?status=='PRIMARY']|[0].rolloutState" \ - --output text) - echo "pg-migration rollout state: ${state}" - case "$state" in - COMPLETED) - echo "DB migration succeeded — pg-migration task is running and stable." - exit 0 - ;; - FAILED) - echo "::error::pg-migration deployment reported rolloutState FAILED." - exit 1 - ;; - esac - + status=$(aws ecs describe-tasks --cluster "$CLUSTER" --tasks "$task_arn" \ + --query 'tasks[0].lastStatus' --output text) + echo "migration task status: $status" + [ "$status" = "STOPPED" ] && break if [ "$(date +%s)" -ge "$deadline" ]; then - echo "::error::Timed out after ${TIMEOUT_SECONDS}s waiting for the DB migration to succeed." - echo "Migration container logs: ${console_logs}" + echo "::error::Timed out after ${TIMEOUT_SECONDS}s waiting for the migration task to finish." + echo "Migration logs: ${console_logs}" + aws ecs stop-task --cluster "$CLUSTER" --task "$task_arn" \ + --reason "migration gate timeout" >/dev/null 2>&1 || true exit 1 fi - sleep 15 + sleep 10 done + exit_code=$(aws ecs describe-tasks --cluster "$CLUSTER" --tasks "$task_arn" \ + --query "tasks[0].containers[?name=='${CONTAINER}']|[0].exitCode" --output text) + echo "migration container exit code: ${exit_code}" + if [ "$exit_code" != "0" ]; then + echo "::error::DB migration failed (exit ${exit_code}) — deploy blocked so the app never runs ahead of its schema." + aws ecs describe-tasks --cluster "$CLUSTER" --tasks "$task_arn" \ + --query "tasks[0].{stopCode:stopCode,taskReason:stoppedReason,containerReason:containers[?name=='${CONTAINER}']|[0].reason}" \ + --output table || true + echo "Migration logs: ${console_logs}" + exit 1 + fi + echo "DB migration succeeded." + deploy-prerender: name: Deploy prerender needs: [build-prerender] @@ -372,7 +365,7 @@ jobs: deploy-worker: name: Deploy worker needs: - [build-worker, deploy-host, post-migrate-db, deploy-prerender-manager] + [build-worker, deploy-host, migrate-db, deploy-prerender-manager] uses: cardstack/gh-actions/.github/workflows/ecs-deploy.yml@main secrets: inherit with: @@ -396,7 +389,7 @@ jobs: deploy-realm-server: name: Deploy realm server needs: - [post-deploy-worker, build-realm-server, deploy-host, post-migrate-db] + [post-deploy-worker, build-realm-server, deploy-host, migrate-db] uses: cardstack/gh-actions/.github/workflows/ecs-deploy.yml@main secrets: inherit with: @@ -501,7 +494,6 @@ jobs: build-worker, build-pg-migration, migrate-db, - post-migrate-db, deploy-prerender, deploy-prerender-manager, deploy-worker, diff --git a/packages/postgres/Dockerfile b/packages/postgres/Dockerfile index 62fd66fc5e..4897028d0a 100644 --- a/packages/postgres/Dockerfile +++ b/packages/postgres/Dockerfile @@ -23,11 +23,8 @@ RUN CI=1 pnpm install -r --offline WORKDIR /boxel/packages/postgres -# --ignore-pattern keeps node-pg-migrate from treating non-migration files in -# the migrations directory (the `{ "type": "commonjs" }` package.json that -# pins migrations to CommonJS, plus .eslintrc.js) as migrations. Without it, -# package.json is loaded as a migration, has no `up` export, and the whole `up` -# run errors and rolls back — so no migrations apply. Mirrors the `migrate` -# script in package.json. (.eslintrc.js is a dotfile and ignored by default; -# package.json is not, so it must be listed explicitly.) -CMD node ./scripts/fix-migration-names.ts && ./node_modules/.bin/node-pg-migrate --check-order false --migrations-table migrations --ignore-pattern '.*\.eslintrc\.js|package\.json' up && sleep infinity +# Run the migrations (see scripts/run-migrations.sh), then keep the container +# alive so the long-lived ECS service stays up after a successful migrate. The +# deploy runs the same script as a one-shot task and gates on its exit code; +# `&& sleep infinity` runs only when the migrate exited 0. +CMD ./scripts/run-migrations.sh && sleep infinity diff --git a/packages/postgres/scripts/run-migrations.sh b/packages/postgres/scripts/run-migrations.sh new file mode 100755 index 0000000000..e5861d6ac1 --- /dev/null +++ b/packages/postgres/scripts/run-migrations.sh @@ -0,0 +1,25 @@ +#!/bin/sh +# Apply all pending DB migrations and exit with node-pg-migrate's status, so a +# failing migration surfaces as a non-zero exit. Run from the image's WORKDIR +# (/boxel/packages/postgres). Used two ways: +# - the pg-migration container's CMD wraps this in `... && sleep infinity` so +# the long-lived service stays up after a successful migrate +# - the deploy runs it as a one-shot task (command override, no sleep) and +# gates the rest of the rollout on its exit code +set -e + +# Normalize legacy migration filenames before running (see fix-migration-names). +node ./scripts/fix-migration-names.ts + +# --ignore-pattern keeps node-pg-migrate from treating non-migration files in +# the migrations directory (the `{ "type": "commonjs" }` package.json that pins +# migrations to CommonJS, plus .eslintrc.js) as migrations. Without it, +# package.json is loaded as a migration, has no `up` export, and the whole `up` +# run errors and rolls back — so no migrations apply. Mirrors the `migrate` +# script in package.json. (.eslintrc.js is a dotfile and ignored by default; +# package.json is not, so it must be listed explicitly.) +exec ./node_modules/.bin/node-pg-migrate \ + --check-order false \ + --migrations-table migrations \ + --ignore-pattern '.*\.eslintrc\.js|package\.json' \ + up From d0aea49838d74748f715aaeaac17c9ea27e4f3c8 Mon Sep 17 00:00:00 2001 From: Hassan Abdel-Rahman Date: Mon, 6 Jul 2026 12:18:07 -0400 Subject: [PATCH 3/3] Drop misleading package.json cross-reference from run-migrations comment The dev `migrate` script in package.json diverges (ensure-db-exists, PG* env, --no-check-order, --verbose=false), so "mirrors" was inaccurate. The comment now just explains why --ignore-pattern is needed. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/postgres/scripts/run-migrations.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/postgres/scripts/run-migrations.sh b/packages/postgres/scripts/run-migrations.sh index e5861d6ac1..f80b6fd267 100755 --- a/packages/postgres/scripts/run-migrations.sh +++ b/packages/postgres/scripts/run-migrations.sh @@ -15,9 +15,8 @@ node ./scripts/fix-migration-names.ts # the migrations directory (the `{ "type": "commonjs" }` package.json that pins # migrations to CommonJS, plus .eslintrc.js) as migrations. Without it, # package.json is loaded as a migration, has no `up` export, and the whole `up` -# run errors and rolls back — so no migrations apply. Mirrors the `migrate` -# script in package.json. (.eslintrc.js is a dotfile and ignored by default; -# package.json is not, so it must be listed explicitly.) +# run errors and rolls back — so no migrations apply. (.eslintrc.js is a dotfile +# and ignored by default; package.json is not, so it must be listed explicitly.) exec ./node_modules/.bin/node-pg-migrate \ --check-order false \ --migrations-table migrations \