Gate API migrations before staging deployment

.github/workflows/build-images.yml

@@ -218,10 +218,136 @@ jobs:
           echo "Push enabled: ${{ steps.context.outputs.should_push }}"
           echo "Primary tag: ${{ steps.tags.outputs.image_tag }}"
 
-  deploy:
+  migrate:
+    name: Run API database migration
     runs-on: ubuntu-latest
     needs: build-and-publish
-    if: github.event_name != 'pull_request'
+    if: >-
+      ${{ (github.ref == 'refs/heads/staging' ||
+      github.event_name == 'workflow_dispatch') &&
+      (github.event_name != 'workflow_dispatch' ||
+      github.event.inputs.service == '' ||
+      github.event.inputs.service == 'all' ||
+      github.event.inputs.service == 'api') }}
+    permissions:
+      contents: read
+
+    steps:
+      - name: Decide migration context
+        id: context
+        shell: bash
+        run: |
+          set -euo pipefail
+
+          if [ "${{ github.event_name }}" = "release" ] || [[ "${{ github.ref }}" == refs/tags/* ]]; then
+            environment="prod"
+          else
+            environment="staging"
+          fi
+
+          if [ "$environment" = "prod" ]; then
+            namespace="knowhere-prod"
+          else
+            namespace="knowhere-staging"
+          fi
+
+          short_sha="${GITHUB_SHA::8}"
+          if [ "${{ github.event_name }}" = "release" ]; then
+            image_tag="${{ github.event.release.tag_name }}-${environment}"
+          elif [[ "${{ github.ref }}" == refs/tags/* ]]; then
+            image_tag="${GITHUB_REF#refs/tags/}-${environment}"
+          else
+            image_tag="${environment}-${short_sha}"
+          fi
+
+          echo "environment=$environment" >> "$GITHUB_OUTPUT"
+          echo "namespace=$namespace" >> "$GITHUB_OUTPUT"
+          echo "image_uri=${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}/knowhere-backend:${image_tag}" >> "$GITHUB_OUTPUT"
+
+      - name: Validate deployment configuration
+        shell: bash
+        run: |
+          set -euo pipefail
+          if [ -z "${{ secrets.AWS_ACCESS_KEY_ID }}" ] || \
+             [ -z "${{ secrets.AWS_SECRET_ACCESS_KEY }}" ] || \
+             [ -z "${{ env.AWS_EKS_PROD_CLUSTER_NAME }}" ] || \
+             [ -z "${{ env.AWS_EKS_PROD_REGION }}" ]; then
+            echo "::error::Migration requires AWS deployment credentials and cluster configuration."
+            exit 1
+          fi
+
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: ${{ env.AWS_EKS_PROD_REGION }}
+
+      - name: Setup kubectl
+        uses: azure/setup-kubectl@v3
+
+      - name: Update kubeconfig
+        shell: bash
+        run: |
+          aws eks update-kubeconfig \
+            --name "${AWS_EKS_PROD_CLUSTER_NAME}" \
+            --region "${AWS_EKS_PROD_REGION}"
+
+      - name: Run migration job
+        shell: bash
+        env:
+          IMAGE_URI: ${{ steps.context.outputs.image_uri }}
+          NAMESPACE: ${{ steps.context.outputs.namespace }}
+        run: |
+          set -euo pipefail
+
+          job_name="knowhere-api-migrate-${GITHUB_RUN_ID}"
+
+          migration_manifest="$(kubectl get deployment/knowhere-api --namespace "$NAMESPACE" -o json \
+            | jq --arg job_name "$job_name" --arg image_uri "$IMAGE_URI" '
+                if ((.spec.template.spec.containers // []) | length) == 0 then
+                  error("knowhere-api deployment has no containers")
+                else
+                {
+                  apiVersion: "batch/v1",
+                  kind: "Job",
+                  metadata: {name: $job_name, namespace: .metadata.namespace},
+                  spec: {
+                    backoffLimit: 0,
+                    ttlSecondsAfterFinished: 300,
+                    template: {
+                      metadata: {labels: {"app": "knowhere-api-migrate"}},
+                      spec: (
+                        .spec.template.spec
+                        | .restartPolicy = "Never"
+                        | .containers[0].image = $image_uri
+                        | .containers[0].command = ["python", "-m", "alembic", "upgrade", "heads"]
+                        | del(.containers[0].args)
+                      )
+                    }
+                  }
+                }
+                end
+              ')"
+
+          printf '%s\n' "$migration_manifest" | kubectl apply -f -
+
+          if ! kubectl wait --for=condition=complete "job/$job_name" \
+            --namespace "$NAMESPACE" --timeout=900s; then
+            kubectl describe job "$job_name" --namespace "$NAMESPACE" || true
+            kubectl logs "job/$job_name" --namespace "$NAMESPACE" --all-containers=true || true
+            exit 1
+          fi
+
+          kubectl delete job "$job_name" --namespace "$NAMESPACE" --ignore-not-found
+
+  deploy:
+    runs-on: ubuntu-latest
+    needs: [build-and-publish, migrate]
+    if: >-
+      ${{ always() && github.event_name != 'pull_request' &&
+      needs.build-and-publish.result == 'success' &&
+      (needs.migrate.result == 'success' || needs.migrate.result == 'skipped') }}
     permissions:
       contents: read
 

README.md

@@ -191,7 +191,12 @@ cd apps/api && uv run main.py
 cd apps/worker && uv run worker.py
 ```
 
-The API runs migrations during startup.
+Run API migrations explicitly before starting the API when the database schema needs updating:
+
+```bash
+cd apps/api
+uv run alembic upgrade heads
+```
 
 For API-only development without the dashboard, create an API-only user/key
 after the API service starts:

apps/api/main.py

@@ -1,5 +1,4 @@
 import os
-from pathlib import Path
 import uvicorn
 from fastapi import FastAPI
 from starlette.routing import Route
@@ -29,28 +28,6 @@ async def lifespan(app: FastAPI):
     """
     Application lifecycle management
     """
-    # Run database migrations
-    import subprocess
-    import sys
-
-    try:
-        logger.info("start running database migration...")
-        result = subprocess.run(
-            [sys.executable, "-m", "alembic", "upgrade", "heads"],
-            cwd=str(Path(__file__).parent),
-            capture_output=True,
-            text=True,
-        )
-
-        if result.returncode == 0:
-            logger.info("database migration completed")
-        else:
-            logger.error(f"database migration failed: {result.stderr}")
-            raise Exception(f"database migration failed: {result.stderr}")
-    except Exception as e:
-        logger.error(f"running database migration failed: {e}")
-        raise
-
     from shared.core.database import prewarm_connection_pool
 
     await prewarm_connection_pool()

apps/worker/tests/contract/test_profile_agent_protocol_contract.py

@@ -23,8 +23,7 @@
     ToolContext,
 )
 from app.services.document_agent.planner.planner import _parse_profile_and_decision
-from app.services.document_agent.registry import REGISTRY
-from app.services.document_agent import tools as _registered_tools  # noqa: F401
+from app.services.document_agent.tools import REGISTRY
 from app.services.document_agent.state import AgentBlackboard