manual-deploy.yml

name: Manual Deploy [boxel]

on:
  workflow_dispatch:
    inputs:
      environment:
        description: Deployment environment
        required: false
        default: staging
  workflow_call:
    inputs:
      environment:
        required: true
        type: string

permissions:
  contents: read
  deployments: write
  id-token: write

# Serialize deploys per environment so overlapping runs can't land out of
# order. A deploy is a ~50-60 min serial chain that ends by pointing each ECS
# service at a :<git-sha> image; when two runs overlap, whichever finishes last
# wins, so an older commit's build can overwrite a newer one. For staging
# (auto-deployed on every merge to main) we cancel any in-flight deploy when a
# newer one starts, so the newest commit always wins — the superseding run is a
# full deploy of every service, so it reconciles any partially-updated state
# left by the cancelled run. Every other environment — production (manual only),
# and any value we don't explicitly recognize — queues instead of cancelling, to
# avoid interrupting a deliberate deploy mid-chain. Cancelling is opt-in per
# environment (== 'staging') rather than opt-out (!= 'production') so an
# unexpected env errs toward the safe behavior (queue), not the risky one.
#
# The environment is read with a fallback: `inputs.environment` for the
# workflow_call path (merge-to-main auto deploy), then
# `github.event.inputs.environment` for the workflow_dispatch path. The
# fallback is required because the `inputs` context is NOT populated in a
# workflow-level concurrency expression for workflow_dispatch — without it a
# manual deploy would land in an empty-env group with cancel-in-progress
# silently defaulting to false. The trailing 'staging' default keeps the group
# non-empty and matches every caller's default.
concurrency:
  group: deploy-${{ inputs.environment || github.event.inputs.environment || 'staging' }}
  cancel-in-progress: ${{ (inputs.environment || github.event.inputs.environment || 'staging') == 'staging' }}

jobs:
  create-deployment:
    name: Create GitHub deployment
    if: github.event_name == 'workflow_dispatch'
    runs-on: ubuntu-latest
    outputs:
      deployment-id: ${{ steps.create.outputs.deployment_id }}
      environment-url: ${{ steps.env.outputs.environment_url }}
    steps:
      - id: env
        run: |
          if [ "${{ inputs.environment }}" = "production" ]; then
            echo "environment_url=https://app.boxel.ai" >> "$GITHUB_OUTPUT"
          elif [ "${{ inputs.environment }}" = "staging" ]; then
            echo "environment_url=https://realms-staging.stack.cards" >> "$GITHUB_OUTPUT"
          else
            echo "environment_url=" >> "$GITHUB_OUTPUT"
          fi
      - id: create
        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
        with:
          script: |
            const environment = '${{ inputs.environment }}';
            const response = await github.rest.repos.createDeployment({
              owner: context.repo.owner,
              repo: context.repo.repo,
              ref: context.sha,
              required_contexts: [],
              auto_merge: false,
              environment,
              description: `Manual deploy to ${environment}`,
              transient_environment: false,
              production_environment: environment === 'production',
            });
            core.setOutput('deployment_id', response.data.id.toString());
      - name: Mark deployment in progress
        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
        with:
          script: |
            const deployment_id = Number('${{ steps.create.outputs.deployment_id }}');
            const environmentUrlValue = '${{ steps.env.outputs.environment_url }}';
            const environment_url =
              environmentUrlValue && environmentUrlValue.length > 0
                ? environmentUrlValue
                : undefined;
            await github.rest.repos.createDeploymentStatus({
              owner: context.repo.owner,
              repo: context.repo.repo,
              deployment_id,
              state: 'in_progress',
              environment_url,
              log_url: `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`,
              description: 'Deployment started',
            });

  build-ai-bot:
    name: Build ai-bot Docker image
    uses: cardstack/gh-actions/.github/workflows/docker-ecr.yml@main
    with:
      repository: "boxel-ai-bot-${{ inputs.environment }}"
      environment: ${{ inputs.environment }}
      dockerfile: "packages/ai-bot/Dockerfile"

  deploy-ai-bot:
    needs: [build-ai-bot, migrate-db]
    name: Deploy ai-bot to AWS ECS
    uses: cardstack/gh-actions/.github/workflows/ecs-deploy.yml@main
    secrets: inherit
    with:
      container-name: "boxel-ai-bot"
      environment: ${{ inputs.environment }}
      cluster: ${{ inputs.environment }}
      service-name: "boxel-ai-bot-${{ inputs.environment }}"
      image: ${{ needs.build-ai-bot.outputs.image }}
      wait-for-service-stability: false

  build-bot-runner:
    name: Build bot-runner Docker image
    uses: cardstack/gh-actions/.github/workflows/docker-ecr.yml@main
    with:
      repository: "boxel-bot-runner-${{ inputs.environment }}"
      environment: ${{ inputs.environment }}
      dockerfile: "packages/bot-runner/Dockerfile"

  deploy-bot-runner:
    needs: [build-bot-runner, migrate-db]
    name: Deploy bot-runner to AWS ECS
    uses: cardstack/gh-actions/.github/workflows/ecs-deploy.yml@main
    secrets: inherit
    with:
      container-name: "boxel-bot-runner"
      environment: ${{ inputs.environment }}
      cluster: ${{ inputs.environment }}
      service-name: "boxel-bot-runner-${{ inputs.environment }}"
      image: ${{ needs.build-bot-runner.outputs.image }}
      wait-for-service-stability: false

  build-host:
    name: Build host
    uses: ./.github/workflows/build-host.yml
    secrets: inherit
    with:
      environment: ${{ inputs.environment }}

  deploy-host:
    name: Deploy host
    needs: [build-host]
    uses: ./.github/workflows/deploy-host.yml
    secrets: inherit
    with:
      environment: ${{ inputs.environment }}

  deploy-ui:
    name: Deploy boxel-ui
    if: inputs.environment == 'staging'
    uses: ./.github/workflows/deploy-ui.yml
    secrets: inherit
    with:
      environment: staging

  build-realm-server:
    name: Build realm-server Docker image
    uses: cardstack/gh-actions/.github/workflows/docker-ecr.yml@main
    with:
      repository: "boxel-realm-server-${{ inputs.environment }}"
      environment: ${{ inputs.environment }}
      dockerfile: "packages/realm-server/realm-server.Dockerfile"
      build-args: |
        "realm_server_script=scripts/start-${{ inputs.environment }}.sh"

  build-prerender-manager:
    name: Build prerender manager Docker image
    uses: cardstack/gh-actions/.github/workflows/docker-ecr.yml@main
    with:
      repository: "boxel-prerender-manager-${{ inputs.environment }}"
      environment: ${{ inputs.environment }}
      dockerfile: "packages/realm-server/prerender-manager.Dockerfile"
      build-args: |
        "prerender_manager_script=scripts/start-prerender-manager.sh"

  build-prerender:
    name: Build prerender Docker image
    uses: cardstack/gh-actions/.github/workflows/docker-ecr.yml@main
    with:
      repository: "boxel-prerender-server-${{ inputs.environment }}"
      environment: ${{ inputs.environment }}
      dockerfile: "packages/realm-server/prerender.Dockerfile"
      build-args: |
        "prerender_script=scripts/start-prerender-${{ inputs.environment }}.sh"

  build-worker:
    name: Build worker Docker image
    uses: cardstack/gh-actions/.github/workflows/docker-ecr.yml@main
    with:
      repository: "boxel-worker-${{ inputs.environment }}"
      environment: ${{ inputs.environment }}
      dockerfile: "packages/realm-server/worker.Dockerfile"
      build-args: |
        "worker_script=scripts/start-worker-${{ inputs.environment }}.sh"

  build-pg-migration:
    name: Build pg-migration Docker image
    uses: cardstack/gh-actions/.github/workflows/docker-ecr.yml@main
    with:
      repository: "boxel-pg-migration-${{ inputs.environment }}"
      environment: ${{ inputs.environment }}
      dockerfile: "packages/postgres/Dockerfile"

  # Run the DB migrations as a one-shot ECS task and gate the rest of the deploy
  # on its exit code, so the app never rolls out ahead of its schema. The
  # pg-migration image applies migrations and exits with node-pg-migrate's
  # status (packages/postgres/scripts/run-migrations.sh); run one-shot with the
  # service's trailing `sleep infinity` overridden away, a failing or
  # crash-looping migration exits non-zero and fails this job — blocking every
  # downstream deploy. deploy-host / build-realm-server are deps so migrations
  # run at the last possible moment, minimizing how long old code sees the new
  # schema.
  migrate-db:
    needs: [build-pg-migration, build-realm-server, deploy-host]
    name: Run DB migrations
    runs-on: ubuntu-latest
    permissions:
      contents: read
      id-token: write
    env:
      CLUSTER: ${{ inputs.environment }}
      SERVICE: boxel-pg-migration-${{ inputs.environment }}
      CONTAINER: boxel-pg-migration
      IMAGE: ${{ needs.build-pg-migration.outputs.image }}
      LOG_GROUP: ecs-boxel-pg-migration-${{ inputs.environment }}
      # Bounds a migration that hangs (e.g. blocked on a lock); a long backfill
      # migration must finish under it. Genuine failures exit immediately.
      TIMEOUT_SECONDS: "1800"
    steps:
      - name: Check out
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - name: Set up env
        run: |
          if [ "${{ inputs.environment }}" = "production" ]; then
            echo "AWS_ROLE_ARN=arn:aws:iam::120317779495:role/github" >> "$GITHUB_ENV"
          elif [ "${{ inputs.environment }}" = "staging" ]; then
            echo "AWS_ROLE_ARN=arn:aws:iam::680542703984:role/github" >> "$GITHUB_ENV"
          else
            echo "unrecognized environment: ${{ inputs.environment }}"
            exit 1
          fi

      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: ${{ env.AWS_ROLE_ARN }}
          aws-region: us-east-1

      - name: Run migrations and gate on exit code
        run: .github/scripts/run-gated-migration-task.sh ./scripts/run-migrations.sh

  deploy-prerender:
    name: Deploy prerender
    needs: [build-prerender]
    uses: cardstack/gh-actions/.github/workflows/ecs-deploy.yml@main
    secrets: inherit
    with:
      container-name: "boxel-prerender-server"
      environment: ${{ inputs.environment }}
      cluster: ${{ inputs.environment }}
      service-name: "boxel-prerender-server-${{ inputs.environment }}"
      image: ${{ needs.build-prerender.outputs.image }}
      timeout-minutes: 10
      wait-for-service-stability: true

  deploy-prerender-manager:
    name: Deploy prerender manager
    needs: [build-prerender-manager, deploy-prerender]
    uses: cardstack/gh-actions/.github/workflows/ecs-deploy.yml@main
    secrets: inherit
    with:
      container-name: "boxel-prerender-manager"
      environment: ${{ inputs.environment }}
      cluster: ${{ inputs.environment }}
      service-name: "boxel-prerender-manager-${{ inputs.environment }}"
      image: ${{ needs.build-prerender-manager.outputs.image }}
      timeout-minutes: 10
      wait-for-service-stability: true

  deploy-worker:
    name: Deploy worker
    needs: [build-worker, deploy-host, migrate-db, deploy-prerender-manager]
    uses: cardstack/gh-actions/.github/workflows/ecs-deploy.yml@main
    secrets: inherit
    with:
      container-name: "boxel-worker"
      environment: ${{ inputs.environment }}
      cluster: ${{ inputs.environment }}
      service-name: "boxel-worker-${{ inputs.environment }}"
      image: ${{ needs.build-worker.outputs.image }}
      wait-for-service-stability: false

  # the wait-for-service-stability flag doesn't seem to work in
  # aws-actions/amazon-ecs-deploy-task-definition@v2. we keep getting timeouts
  # waiting for service stability. So we are manually waiting here.
  post-deploy-worker:
    name: Wait for worker
    needs: [deploy-worker]
    runs-on: ubuntu-latest
    steps:
      - run: sleep 180

  deploy-realm-server:
    name: Deploy realm server
    needs: [post-deploy-worker, build-realm-server, deploy-host, migrate-db]
    uses: cardstack/gh-actions/.github/workflows/ecs-deploy.yml@main
    secrets: inherit
    with:
      container-name: "boxel-realm-server"
      environment: ${{ inputs.environment }}
      cluster: ${{ inputs.environment }}
      service-name: "boxel-realm-server-${{ inputs.environment }}"
      image: ${{ needs.build-realm-server.outputs.image }}
      timeout-minutes: 10
      wait-for-service-stability: true

  post-deploy-realm-server:
    name: After realm server stable deployment
    needs: [deploy-realm-server]
    runs-on: ubuntu-latest
    steps:
      - name: Call post-deployment endpoint on realm server
        run: |
          if [ "${{ inputs.environment }}" = "production" ]; then
            URL="https://app.boxel.ai/_post-deployment"
            SECRET="${{ secrets.PRODUCTION_REALM_SERVER_SECRET }}"
          elif [ "${{ inputs.environment }}" = "staging" ]; then
            URL="https://realms-staging.stack.cards/_post-deployment"
            SECRET="${{ secrets.STAGING_REALM_SERVER_SECRET }}"
          else
            echo "Unknown environment: ${{ inputs.environment }}"
            exit 1
          fi

          response=$(curl -s -w "\n%{http_code}" -X POST \
            -H "Authorization: $SECRET" "$URL")
          response_body=$(echo "$response" | head -n -1)
          response_code=$(echo "$response" | tail -n 1)

          echo "Response body: $response_body"

          if [ "$response_code" != "200" ]; then
            echo "Post-deployment endpoint returned $response_code, expected 200"
            echo "Response body: $response_body"
            exit 1
          fi

  # Destructive ("removal") migrations — column/table drops and renames, held in
  # packages/postgres/migrations-removal/ — run HERE, after deploy-realm-server
  # has reached stability. `wait-for-service-stability: true` on that job means
  # the previous realm-server task set has fully drained (and, via its
  # post-deploy-worker dependency, the previous worker task set too), so no task
  # on the old code revision is left to query a column this phase drops. That is
  # the whole point: running drops before the old tasks drain is what takes the
  # published realms down (old code 500s on `column i.<x> does not exist`).
  #
  # A failure here is non-fatal to serving: the new code, by definition, no
  # longer reads the removed columns, so it is already live and healthy — the
  # drop simply hasn't happened yet and can be retried on the next deploy.
  #
  # Gate on deploy-realm-server, NOT post-deploy-realm-server: the latter is a
  # separate /_post-deployment hook that can fail independently of the only real
  # precondition here — the old tasks being gone.
  migrate-db-remove:
    needs: [build-pg-migration, deploy-realm-server]
    name: Run DB removal migrations
    runs-on: ubuntu-latest
    permissions:
      contents: read
      id-token: write
    env:
      CLUSTER: ${{ inputs.environment }}
      SERVICE: boxel-pg-migration-${{ inputs.environment }}
      CONTAINER: boxel-pg-migration
      IMAGE: ${{ needs.build-pg-migration.outputs.image }}
      LOG_GROUP: ecs-boxel-pg-migration-${{ inputs.environment }}
      # Bounds a migration that hangs (e.g. blocked on a lock); a long backfill
      # migration must finish under it. Genuine failures exit immediately.
      TIMEOUT_SECONDS: "1800"
    steps:
      - name: Check out
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - name: Set up env
        run: |
          if [ "${{ inputs.environment }}" = "production" ]; then
            echo "AWS_ROLE_ARN=arn:aws:iam::120317779495:role/github" >> "$GITHUB_ENV"
          elif [ "${{ inputs.environment }}" = "staging" ]; then
            echo "AWS_ROLE_ARN=arn:aws:iam::680542703984:role/github" >> "$GITHUB_ENV"
          else
            echo "unrecognized environment: ${{ inputs.environment }}"
            exit 1
          fi

      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: ${{ env.AWS_ROLE_ARN }}
          aws-region: us-east-1

      - name: Run removal migrations and gate on exit code
        run: .github/scripts/run-gated-migration-task.sh ./scripts/run-migrations.sh migrations-removal migrations_removal

  recycle-prerender:
    name: Recycle prerender after host is live
    # The prerender fleet deploys before the realm server (the manager,
    # worker, and realm server all depend on it being up for boot indexing),
    # so its tabs warm against the OLD host shell the realm server was still
    # serving at that point. Once the realm server is up serving the new
    # shell, re-deploy the prerender service so its tabs re-warm against it.
    # The reusable deploy passes force-new-deployment, so this recycles fresh
    # tasks even though the image is unchanged.
    #
    # Gate on `deploy-realm-server` (wait-for-service-stability: true → the
    # new realm server is up and serving the new shell), NOT on
    # `post-deploy-realm-server`: that job is a separate post-deploy hook
    # (it POSTs the realm server's `/_post-deployment` endpoint) and can fail
    # independently of the recycle's only real precondition — the new shell
    # being served. Coupling to it would skip this recycle on an unrelated
    # hook failure.
    needs: [build-prerender, deploy-realm-server]
    uses: cardstack/gh-actions/.github/workflows/ecs-deploy.yml@main
    secrets: inherit
    with:
      container-name: "boxel-prerender-server"
      environment: ${{ inputs.environment }}
      cluster: ${{ inputs.environment }}
      service-name: "boxel-prerender-server-${{ inputs.environment }}"
      image: ${{ needs.build-prerender.outputs.image }}
      timeout-minutes: 10
      wait-for-service-stability: true

  apply-observability:
    # Push the observability/ package's dashboards/folders/data sources/alerts
    # into the production self-host Grafana as part of the deploy. The
    # called workflow's `environment: observability-production` gives the run
    # a production badge and a Grafana URL link, but doesn't gate (no required
    # reviewers — dispatching the manual deploy IS the approval).
    #
    # Production-only. Staging applies happen automatically on merge to main
    # via observability-apply-staging.yml — no need to re-run during a
    # manual staging deploy.
    name: Apply observability to production
    if: inputs.environment == 'production'
    needs: [post-deploy-realm-server]
    uses: ./.github/workflows/observability-apply-production.yml
    secrets: inherit

  finalize-deployment:
    name: Update GitHub deployment status
    needs:
      [
        create-deployment,
        build-ai-bot,
        deploy-ai-bot,
        build-bot-runner,
        deploy-bot-runner,
        build-host,
        deploy-host,
        build-realm-server,
        build-prerender-manager,
        build-prerender,
        build-worker,
        build-pg-migration,
        migrate-db,
        deploy-prerender,
        deploy-prerender-manager,
        deploy-worker,
        post-deploy-worker,
        deploy-realm-server,
        post-deploy-realm-server,
        migrate-db-remove,
        recycle-prerender,
        apply-observability,
      ]
    if: github.event_name == 'workflow_dispatch' && always()
    runs-on: ubuntu-latest
    steps:
      - name: Set deployment status
        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
        env:
          NEEDS: ${{ toJson(needs) }}
          DEPLOYMENT_ID: ${{ needs.create-deployment.outputs.deployment-id }}
          ENVIRONMENT_URL: ${{ needs.create-deployment.outputs.environment-url }}
        with:
          script: |
            const deploymentIdRaw = process.env.DEPLOYMENT_ID;
            if (!deploymentIdRaw) {
              core.info('No deployment id found; skipping status update.');
              return;
            }

            const needs = JSON.parse(process.env.NEEDS);
            const results = Object.values(needs).map((job) => job.result);
            let state = 'success';
            if (results.includes('failure')) {
              state = 'failure';
            } else if (results.includes('cancelled')) {
              state = 'inactive';
            }

            const environment_url =
              process.env.ENVIRONMENT_URL &&
              process.env.ENVIRONMENT_URL.length > 0
                ? process.env.ENVIRONMENT_URL
                : undefined;
            await github.rest.repos.createDeploymentStatus({
              owner: context.repo.owner,
              repo: context.repo.repo,
              deployment_id: Number(deploymentIdRaw),
              state,
              environment_url,
              log_url: `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`,
              description:
                state === 'success'
                  ? 'Deployment finished'
                  : 'Deployment finished with errors',
            });