diff --git a/.github/workflows/manual-deploy.yml b/.github/workflows/manual-deploy.yml index ac312f3317..eaecb2421e 100644 --- a/.github/workflows/manual-deploy.yml +++ b/.github/workflows/manual-deploy.yml @@ -107,7 +107,7 @@ jobs: dockerfile: "packages/ai-bot/Dockerfile" deploy-ai-bot: - needs: [build-ai-bot, post-migrate-db] + needs: [build-ai-bot, migrate-db] name: Deploy ai-bot to AWS ECS uses: cardstack/gh-actions/.github/workflows/ecs-deploy.yml@main secrets: inherit @@ -128,7 +128,7 @@ jobs: dockerfile: "packages/bot-runner/Dockerfile" deploy-bot-runner: - needs: [build-bot-runner, post-migrate-db] + needs: [build-bot-runner, migrate-db] name: Deploy bot-runner to AWS ECS uses: cardstack/gh-actions/.github/workflows/ecs-deploy.yml@main secrets: inherit @@ -211,31 +211,128 @@ jobs: environment: ${{ inputs.environment }} dockerfile: "packages/postgres/Dockerfile" + # Run the DB migrations as a one-shot ECS task and gate the rest of the deploy + # on its exit code, so the app never rolls out ahead of its schema. The + # pg-migration image applies migrations and exits with node-pg-migrate's + # status (packages/postgres/scripts/run-migrations.sh); run one-shot with the + # service's trailing `sleep infinity` overridden away, a failing or + # crash-looping migration exits non-zero and fails this job — blocking every + # downstream deploy. deploy-host / build-realm-server are deps so migrations + # run at the last possible moment, minimizing how long old code sees the new + # schema. migrate-db: - # use "deploy-host" and "build-realm-server" as deps so we can run - # migrations at last possible moment in order to reduce the amount of time - # that old code is pointing to new schema needs: [build-pg-migration, build-realm-server, deploy-host] - name: Deploy and run DB migrations - uses: cardstack/gh-actions/.github/workflows/ecs-deploy.yml@main - secrets: inherit - with: - container-name: "boxel-pg-migration" - environment: ${{ inputs.environment }} - cluster: ${{ inputs.environment }} - service-name: "boxel-pg-migration-${{ inputs.environment }}" - image: ${{ needs.build-pg-migration.outputs.image }} - wait-for-service-stability: false - - # the wait-for-service-stability flag doesn't seem to work in - # aws-actions/amazon-ecs-deploy-task-definition@v2. we keep getting timeouts - # waiting for service stability. So we are manually waiting here. - post-migrate-db: - name: Wait for db-migration - needs: [migrate-db] + name: Run DB migrations runs-on: ubuntu-latest + permissions: + contents: read + id-token: write + env: + CLUSTER: ${{ inputs.environment }} + SERVICE: boxel-pg-migration-${{ inputs.environment }} + CONTAINER: boxel-pg-migration + IMAGE: ${{ needs.build-pg-migration.outputs.image }} + LOG_GROUP: ecs-boxel-pg-migration-${{ inputs.environment }} + # Bounds a migration that hangs (e.g. blocked on a lock); a long backfill + # migration must finish under it. Genuine failures exit immediately. + TIMEOUT_SECONDS: "1800" steps: - - run: sleep 180 + - name: Set up env + run: | + if [ "${{ inputs.environment }}" = "production" ]; then + echo "AWS_ROLE_ARN=arn:aws:iam::120317779495:role/github" >> "$GITHUB_ENV" + elif [ "${{ inputs.environment }}" = "staging" ]; then + echo "AWS_ROLE_ARN=arn:aws:iam::680542703984:role/github" >> "$GITHUB_ENV" + else + echo "unrecognized environment: ${{ inputs.environment }}" + exit 1 + fi + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.AWS_ROLE_ARN }} + aws-region: us-east-1 + + - name: Run migrations and gate on exit code + run: | + set -euo pipefail + region=us-east-1 + console_logs="https://${region}.console.aws.amazon.com/cloudwatch/home?region=${region}#logsV2:log-groups/log-group/${LOG_GROUP}" + + # Register a new revision of the pg-migration task definition pinned to + # the image just built. Strip the read-only fields describe returns + # that register-task-definition rejects. + aws ecs describe-task-definition --task-definition "$SERVICE" \ + --query 'taskDefinition' --output json > td.json + jq --arg IMAGE "$IMAGE" --arg CONTAINER "$CONTAINER" ' + .containerDefinitions |= map(if .name == $CONTAINER then .image = $IMAGE else . end) + | del(.taskDefinitionArn, .revision, .status, .requiresAttributes, + .compatibilities, .registeredAt, .registeredBy, .deregisteredAt) + ' td.json > td-new.json + task_def=$(aws ecs register-task-definition --cli-input-json file://td-new.json \ + --query 'taskDefinition.taskDefinitionArn' --output text) + echo "Registered migration task definition: $task_def" + + # Place the one-shot task in the pg-migration service's own network + # config — its security group is the one allowed to reach the DB. + net=$(aws ecs describe-services --cluster "$CLUSTER" --services "$SERVICE" \ + --query 'services[0].networkConfiguration.awsvpcConfiguration' --output json) + subnets=$(echo "$net" | jq -r '.subnets | join(",")') + groups=$(echo "$net" | jq -r '.securityGroups | join(",")') + public=$(echo "$net" | jq -r '.assignPublicIp // "DISABLED"') + + # Override the command to run the migrations WITHOUT the service's + # trailing `sleep infinity`, so the task exits with the migration's + # status instead of staying up. + overrides=$(jq -cn --arg CONTAINER "$CONTAINER" \ + '{containerOverrides: [{name: $CONTAINER, command: ["./scripts/run-migrations.sh"]}]}') + run_out=$(aws ecs run-task \ + --cluster "$CLUSTER" \ + --task-definition "$task_def" \ + --launch-type FARGATE \ + --count 1 \ + --started-by "deploy-${GITHUB_SHA:0:7}" \ + --network-configuration "awsvpcConfiguration={subnets=[$subnets],securityGroups=[$groups],assignPublicIp=$public}" \ + --overrides "$overrides" \ + --output json) + task_arn=$(echo "$run_out" | jq -r '.tasks[0].taskArn // empty') + if [ -z "$task_arn" ]; then + echo "::error::Failed to start the migration task." + echo "$run_out" | jq '.failures' + exit 1 + fi + echo "Started migration task: $task_arn" + + # Wait for the task to stop, then gate on the container's exit code. + deadline=$(( $(date +%s) + TIMEOUT_SECONDS )) + while :; do + status=$(aws ecs describe-tasks --cluster "$CLUSTER" --tasks "$task_arn" \ + --query 'tasks[0].lastStatus' --output text) + echo "migration task status: $status" + [ "$status" = "STOPPED" ] && break + if [ "$(date +%s)" -ge "$deadline" ]; then + echo "::error::Timed out after ${TIMEOUT_SECONDS}s waiting for the migration task to finish." + echo "Migration logs: ${console_logs}" + aws ecs stop-task --cluster "$CLUSTER" --task "$task_arn" \ + --reason "migration gate timeout" >/dev/null 2>&1 || true + exit 1 + fi + sleep 10 + done + + exit_code=$(aws ecs describe-tasks --cluster "$CLUSTER" --tasks "$task_arn" \ + --query "tasks[0].containers[?name=='${CONTAINER}']|[0].exitCode" --output text) + echo "migration container exit code: ${exit_code}" + if [ "$exit_code" != "0" ]; then + echo "::error::DB migration failed (exit ${exit_code}) — deploy blocked so the app never runs ahead of its schema." + aws ecs describe-tasks --cluster "$CLUSTER" --tasks "$task_arn" \ + --query "tasks[0].{stopCode:stopCode,taskReason:stoppedReason,containerReason:containers[?name=='${CONTAINER}']|[0].reason}" \ + --output table || true + echo "Migration logs: ${console_logs}" + exit 1 + fi + echo "DB migration succeeded." deploy-prerender: name: Deploy prerender @@ -268,7 +365,7 @@ jobs: deploy-worker: name: Deploy worker needs: - [build-worker, deploy-host, post-migrate-db, deploy-prerender-manager] + [build-worker, deploy-host, migrate-db, deploy-prerender-manager] uses: cardstack/gh-actions/.github/workflows/ecs-deploy.yml@main secrets: inherit with: @@ -292,7 +389,7 @@ jobs: deploy-realm-server: name: Deploy realm server needs: - [post-deploy-worker, build-realm-server, deploy-host, post-migrate-db] + [post-deploy-worker, build-realm-server, deploy-host, migrate-db] uses: cardstack/gh-actions/.github/workflows/ecs-deploy.yml@main secrets: inherit with: @@ -397,7 +494,6 @@ jobs: build-worker, build-pg-migration, migrate-db, - post-migrate-db, deploy-prerender, deploy-prerender-manager, deploy-worker, diff --git a/packages/postgres/Dockerfile b/packages/postgres/Dockerfile index 62fd66fc5e..4897028d0a 100644 --- a/packages/postgres/Dockerfile +++ b/packages/postgres/Dockerfile @@ -23,11 +23,8 @@ RUN CI=1 pnpm install -r --offline WORKDIR /boxel/packages/postgres -# --ignore-pattern keeps node-pg-migrate from treating non-migration files in -# the migrations directory (the `{ "type": "commonjs" }` package.json that -# pins migrations to CommonJS, plus .eslintrc.js) as migrations. Without it, -# package.json is loaded as a migration, has no `up` export, and the whole `up` -# run errors and rolls back — so no migrations apply. Mirrors the `migrate` -# script in package.json. (.eslintrc.js is a dotfile and ignored by default; -# package.json is not, so it must be listed explicitly.) -CMD node ./scripts/fix-migration-names.ts && ./node_modules/.bin/node-pg-migrate --check-order false --migrations-table migrations --ignore-pattern '.*\.eslintrc\.js|package\.json' up && sleep infinity +# Run the migrations (see scripts/run-migrations.sh), then keep the container +# alive so the long-lived ECS service stays up after a successful migrate. The +# deploy runs the same script as a one-shot task and gates on its exit code; +# `&& sleep infinity` runs only when the migrate exited 0. +CMD ./scripts/run-migrations.sh && sleep infinity diff --git a/packages/postgres/scripts/run-migrations.sh b/packages/postgres/scripts/run-migrations.sh new file mode 100755 index 0000000000..f80b6fd267 --- /dev/null +++ b/packages/postgres/scripts/run-migrations.sh @@ -0,0 +1,24 @@ +#!/bin/sh +# Apply all pending DB migrations and exit with node-pg-migrate's status, so a +# failing migration surfaces as a non-zero exit. Run from the image's WORKDIR +# (/boxel/packages/postgres). Used two ways: +# - the pg-migration container's CMD wraps this in `... && sleep infinity` so +# the long-lived service stays up after a successful migrate +# - the deploy runs it as a one-shot task (command override, no sleep) and +# gates the rest of the rollout on its exit code +set -e + +# Normalize legacy migration filenames before running (see fix-migration-names). +node ./scripts/fix-migration-names.ts + +# --ignore-pattern keeps node-pg-migrate from treating non-migration files in +# the migrations directory (the `{ "type": "commonjs" }` package.json that pins +# migrations to CommonJS, plus .eslintrc.js) as migrations. Without it, +# package.json is loaded as a migration, has no `up` export, and the whole `up` +# run errors and rolls back — so no migrations apply. (.eslintrc.js is a dotfile +# and ignored by default; package.json is not, so it must be listed explicitly.) +exec ./node_modules/.bin/node-pg-migrate \ + --check-order false \ + --migrations-table migrations \ + --ignore-pattern '.*\.eslintrc\.js|package\.json' \ + up