Fail the deploy when the DB migration crash-loops

.github/workflows/manual-deploy.yml

@@ -107,7 +107,7 @@ jobs:
       dockerfile: "packages/ai-bot/Dockerfile"
 
   deploy-ai-bot:
-    needs: [build-ai-bot, post-migrate-db]
+    needs: [build-ai-bot, migrate-db]
     name: Deploy ai-bot to AWS ECS
     uses: cardstack/gh-actions/.github/workflows/ecs-deploy.yml@main
     secrets: inherit
@@ -128,7 +128,7 @@ jobs:
       dockerfile: "packages/bot-runner/Dockerfile"
 
   deploy-bot-runner:
-    needs: [build-bot-runner, post-migrate-db]
+    needs: [build-bot-runner, migrate-db]
     name: Deploy bot-runner to AWS ECS
     uses: cardstack/gh-actions/.github/workflows/ecs-deploy.yml@main
     secrets: inherit
@@ -211,31 +211,128 @@ jobs:
       environment: ${{ inputs.environment }}
       dockerfile: "packages/postgres/Dockerfile"
 
+  # Run the DB migrations as a one-shot ECS task and gate the rest of the deploy
+  # on its exit code, so the app never rolls out ahead of its schema. The
+  # pg-migration image applies migrations and exits with node-pg-migrate's
+  # status (packages/postgres/scripts/run-migrations.sh); run one-shot with the
+  # service's trailing `sleep infinity` overridden away, a failing or
+  # crash-looping migration exits non-zero and fails this job — blocking every
+  # downstream deploy. deploy-host / build-realm-server are deps so migrations
+  # run at the last possible moment, minimizing how long old code sees the new
+  # schema.
   migrate-db:
-    # use "deploy-host" and "build-realm-server" as deps so we can run
-    # migrations at last possible moment in order to reduce the amount of time
-    # that old code is pointing to new schema
     needs: [build-pg-migration, build-realm-server, deploy-host]
-    name: Deploy and run DB migrations
-    uses: cardstack/gh-actions/.github/workflows/ecs-deploy.yml@main
-    secrets: inherit
-    with:
-      container-name: "boxel-pg-migration"
-      environment: ${{ inputs.environment }}
-      cluster: ${{ inputs.environment }}
-      service-name: "boxel-pg-migration-${{ inputs.environment }}"
-      image: ${{ needs.build-pg-migration.outputs.image }}
-      wait-for-service-stability: false
-
-  # the wait-for-service-stability flag doesn't seem to work in
-  # aws-actions/amazon-ecs-deploy-task-definition@v2. we keep getting timeouts
-  # waiting for service stability. So we are manually waiting here.
-  post-migrate-db:
-    name: Wait for db-migration
-    needs: [migrate-db]
+    name: Run DB migrations
     runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      id-token: write
+    env:
+      CLUSTER: ${{ inputs.environment }}
+      SERVICE: boxel-pg-migration-${{ inputs.environment }}
+      CONTAINER: boxel-pg-migration
+      IMAGE: ${{ needs.build-pg-migration.outputs.image }}
+      LOG_GROUP: ecs-boxel-pg-migration-${{ inputs.environment }}
+      # Bounds a migration that hangs (e.g. blocked on a lock); a long backfill
+      # migration must finish under it. Genuine failures exit immediately.
+      TIMEOUT_SECONDS: "1800"
     steps:
-      - run: sleep 180
+      - name: Set up env
+        run: |
+          if [ "${{ inputs.environment }}" = "production" ]; then
+            echo "AWS_ROLE_ARN=arn:aws:iam::120317779495:role/github" >> "$GITHUB_ENV"
+          elif [ "${{ inputs.environment }}" = "staging" ]; then
+            echo "AWS_ROLE_ARN=arn:aws:iam::680542703984:role/github" >> "$GITHUB_ENV"
+          else
+            echo "unrecognized environment: ${{ inputs.environment }}"
+            exit 1
+          fi
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ env.AWS_ROLE_ARN }}
+          aws-region: us-east-1
+
+      - name: Run migrations and gate on exit code
+        run: |
+          set -euo pipefail
+          region=us-east-1
+          console_logs="https://${region}.console.aws.amazon.com/cloudwatch/home?region=${region}#logsV2:log-groups/log-group/${LOG_GROUP}"
+
+          # Register a new revision of the pg-migration task definition pinned to
+          # the image just built. Strip the read-only fields describe returns
+          # that register-task-definition rejects.
+          aws ecs describe-task-definition --task-definition "$SERVICE" \
+            --query 'taskDefinition' --output json > td.json
+          jq --arg IMAGE "$IMAGE" --arg CONTAINER "$CONTAINER" '
+            .containerDefinitions |= map(if .name == $CONTAINER then .image = $IMAGE else . end)
+            | del(.taskDefinitionArn, .revision, .status, .requiresAttributes,
+                  .compatibilities, .registeredAt, .registeredBy, .deregisteredAt)
+          ' td.json > td-new.json
+          task_def=$(aws ecs register-task-definition --cli-input-json file://td-new.json \
+            --query 'taskDefinition.taskDefinitionArn' --output text)
+          echo "Registered migration task definition: $task_def"
+
+          # Place the one-shot task in the pg-migration service's own network
+          # config — its security group is the one allowed to reach the DB.
+          net=$(aws ecs describe-services --cluster "$CLUSTER" --services "$SERVICE" \
+            --query 'services[0].networkConfiguration.awsvpcConfiguration' --output json)
+          subnets=$(echo "$net" | jq -r '.subnets | join(",")')
+          groups=$(echo "$net" | jq -r '.securityGroups | join(",")')
+          public=$(echo "$net" | jq -r '.assignPublicIp // "DISABLED"')
+
+          # Override the command to run the migrations WITHOUT the service's
+          # trailing `sleep infinity`, so the task exits with the migration's
+          # status instead of staying up.
+          overrides=$(jq -cn --arg CONTAINER "$CONTAINER" \
+            '{containerOverrides: [{name: $CONTAINER, command: ["./scripts/run-migrations.sh"]}]}')
+          run_out=$(aws ecs run-task \
+            --cluster "$CLUSTER" \
+            --task-definition "$task_def" \
+            --launch-type FARGATE \
+            --count 1 \
+            --started-by "deploy-${GITHUB_SHA:0:7}" \
+            --network-configuration "awsvpcConfiguration={subnets=[$subnets],securityGroups=[$groups],assignPublicIp=$public}" \
+            --overrides "$overrides" \
+            --output json)
+          task_arn=$(echo "$run_out" | jq -r '.tasks[0].taskArn // empty')
+          if [ -z "$task_arn" ]; then
+            echo "::error::Failed to start the migration task."
+            echo "$run_out" | jq '.failures'
+            exit 1
+          fi
+          echo "Started migration task: $task_arn"
+
+          # Wait for the task to stop, then gate on the container's exit code.
+          deadline=$(( $(date +%s) + TIMEOUT_SECONDS ))
+          while :; do
+            status=$(aws ecs describe-tasks --cluster "$CLUSTER" --tasks "$task_arn" \
+              --query 'tasks[0].lastStatus' --output text)
+            echo "migration task status: $status"
+            [ "$status" = "STOPPED" ] && break
+            if [ "$(date +%s)" -ge "$deadline" ]; then
+              echo "::error::Timed out after ${TIMEOUT_SECONDS}s waiting for the migration task to finish."
+              echo "Migration logs: ${console_logs}"
+              aws ecs stop-task --cluster "$CLUSTER" --task "$task_arn" \
+                --reason "migration gate timeout" >/dev/null 2>&1 || true
+              exit 1
+            fi
+            sleep 10
+          done
+
+          exit_code=$(aws ecs describe-tasks --cluster "$CLUSTER" --tasks "$task_arn" \
+            --query "tasks[0].containers[?name=='${CONTAINER}']|[0].exitCode" --output text)
+          echo "migration container exit code: ${exit_code}"
+          if [ "$exit_code" != "0" ]; then
+            echo "::error::DB migration failed (exit ${exit_code}) — deploy blocked so the app never runs ahead of its schema."
+            aws ecs describe-tasks --cluster "$CLUSTER" --tasks "$task_arn" \
+              --query "tasks[0].{stopCode:stopCode,taskReason:stoppedReason,containerReason:containers[?name=='${CONTAINER}']|[0].reason}" \
+              --output table || true
+            echo "Migration logs: ${console_logs}"
+            exit 1
+          fi
+          echo "DB migration succeeded."
 
   deploy-prerender:
     name: Deploy prerender
@@ -268,7 +365,7 @@ jobs:
   deploy-worker:
     name: Deploy worker
     needs:
-      [build-worker, deploy-host, post-migrate-db, deploy-prerender-manager]
+      [build-worker, deploy-host, migrate-db, deploy-prerender-manager]
     uses: cardstack/gh-actions/.github/workflows/ecs-deploy.yml@main
     secrets: inherit
     with:
@@ -292,7 +389,7 @@ jobs:
   deploy-realm-server:
     name: Deploy realm server
     needs:
-      [post-deploy-worker, build-realm-server, deploy-host, post-migrate-db]
+      [post-deploy-worker, build-realm-server, deploy-host, migrate-db]
     uses: cardstack/gh-actions/.github/workflows/ecs-deploy.yml@main
     secrets: inherit
     with:
@@ -397,7 +494,6 @@ jobs:
         build-worker,
         build-pg-migration,
         migrate-db,
-        post-migrate-db,
         deploy-prerender,
         deploy-prerender-manager,
         deploy-worker,

packages/postgres/Dockerfile

@@ -23,11 +23,8 @@
 
 WORKDIR /boxel/packages/postgres
 
-# --ignore-pattern keeps node-pg-migrate from treating non-migration files in
-# the migrations directory (the `{ "type": "commonjs" }` package.json that
-# pins migrations to CommonJS, plus .eslintrc.js) as migrations. Without it,
-# package.json is loaded as a migration, has no `up` export, and the whole `up`
-# run errors and rolls back — so no migrations apply. Mirrors the `migrate`
-# script in package.json. (.eslintrc.js is a dotfile and ignored by default;
-# package.json is not, so it must be listed explicitly.)
-CMD node ./scripts/fix-migration-names.ts && ./node_modules/.bin/node-pg-migrate --check-order false --migrations-table migrations --ignore-pattern '.*\.eslintrc\.js|package\.json' up && sleep infinity
+# Run the migrations (see scripts/run-migrations.sh), then keep the container
+# alive so the long-lived ECS service stays up after a successful migrate. The
+# deploy runs the same script as a one-shot task and gates on its exit code;
+# `&& sleep infinity` runs only when the migrate exited 0.
+CMD ./scripts/run-migrations.sh && sleep infinity

packages/postgres/scripts/run-migrations.sh

@@ -0,0 +1,24 @@
+#!/bin/sh
+# Apply all pending DB migrations and exit with node-pg-migrate's status, so a
+# failing migration surfaces as a non-zero exit. Run from the image's WORKDIR
+# (/boxel/packages/postgres). Used two ways:
+#   - the pg-migration container's CMD wraps this in `... && sleep infinity` so
+#     the long-lived service stays up after a successful migrate
+#   - the deploy runs it as a one-shot task (command override, no sleep) and
+#     gates the rest of the rollout on its exit code
+set -e
+
+# Normalize legacy migration filenames before running (see fix-migration-names).
+node ./scripts/fix-migration-names.ts
+
+# --ignore-pattern keeps node-pg-migrate from treating non-migration files in
+# the migrations directory (the `{ "type": "commonjs" }` package.json that pins
+# migrations to CommonJS, plus .eslintrc.js) as migrations. Without it,
+# package.json is loaded as a migration, has no `up` export, and the whole `up`
+# run errors and rolls back — so no migrations apply. (.eslintrc.js is a dotfile
+# and ignored by default; package.json is not, so it must be listed explicitly.)
+exec ./node_modules/.bin/node-pg-migrate \
+  --check-order false \
+  --migrations-table migrations \
+  --ignore-pattern '.*\.eslintrc\.js|package\.json' \
+  up