diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml index 89c25bd0..baf95a9d 100644 --- a/.github/workflows/build-images.yml +++ b/.github/workflows/build-images.yml @@ -17,6 +17,14 @@ on: - all - api - worker + deployment_target: + description: Staging deployment target + required: false + type: choice + default: ecs-staging + options: + - ecs-staging + - eks-staging-rollback env: ACR_REGISTRY: ${{ secrets.ALIYUN_ACR_REGISTRY }} @@ -25,6 +33,14 @@ env: ECR_REPOSITORY: knowhere AWS_EKS_PROD_CLUSTER_NAME: ${{ secrets.AWS_EKS_PROD_CLUSTER_NAME }} AWS_EKS_PROD_REGION: ${{ secrets.AWS_EKS_PROD_REGION }} + AWS_ECS_STAGING_CLUSTER_NAME: knowhere-fargate + AWS_ECS_STAGING_REGION: us-east-1 + AWS_ECS_STAGING_EXECUTION_ROLE_ARN: arn:aws:iam::107424103509:role/knowhere-fargate-staging-execution-role + AWS_ECS_STAGING_API_TASK_ROLE_ARN: arn:aws:iam::107424103509:role/knowhere-api-staging-task-role + AWS_ECS_STAGING_WORKER_TASK_ROLE_ARN: arn:aws:iam::107424103509:role/knowhere-worker-staging-task-role + AWS_ECS_STAGING_SECRETS_ARN: ${{ secrets.AWS_ECS_STAGING_SECRETS_ARN }} + AWS_ECS_STAGING_API_SERVICE_NAME: knowhere-api-staging + AWS_ECS_STAGING_WORKER_SERVICE_NAME: knowhere-worker-staging jobs: build-and-publish: @@ -79,7 +95,7 @@ jobs: should_push="true" - if [ "${{ github.event_name }}" = "workflow_dispatch" ] && [ -n "${{ github.event.inputs.service }}" ] && [ "${{ github.event.inputs.service }}" != "all" ] && [ "${{ github.event.inputs.service }}" != "${{ matrix.service }}" ]; then + if [ "${{ github.event_name }}" = "workflow_dispatch" ] && [ "${{ github.event.inputs.deployment_target }}" != "ecs-staging" ] && [ -n "${{ github.event.inputs.service }}" ] && [ "${{ github.event.inputs.service }}" != "all" ] && [ "${{ github.event.inputs.service }}" != "${{ matrix.service }}" ]; then should_build="false" else should_build="true" @@ -218,14 +234,16 @@ jobs: echo "Push enabled: ${{ steps.context.outputs.should_push }}" echo "Primary tag: ${{ steps.tags.outputs.image_tag }}" - migrate: - name: Run API database migration + migrate-ecs-staging: + name: Run staging Neon database migration runs-on: ubuntu-latest needs: build-and-publish if: >- ${{ (github.ref == 'refs/heads/staging' || github.event_name == 'workflow_dispatch') && (github.event_name != 'workflow_dispatch' || + github.event.inputs.deployment_target == 'ecs-staging') && + (github.event_name != 'workflow_dispatch' || github.event.inputs.service == '' || github.event.inputs.service == 'all' || github.event.inputs.service == 'api') }} @@ -233,36 +251,70 @@ jobs: contents: read steps: - - name: Decide migration context - id: context + - name: Validate migration configuration shell: bash run: | set -euo pipefail - - if [ "${{ github.event_name }}" = "release" ] || [[ "${{ github.ref }}" == refs/tags/* ]]; then - environment="prod" - else - environment="staging" + if [ -z "${{ secrets.AWS_ACCESS_KEY_ID }}" ] || [ -z "${{ secrets.AWS_SECRET_ACCESS_KEY }}" ]; then + echo "::error::Staging Neon migration requires AWS ECR credentials." + exit 1 fi - - if [ "$environment" = "prod" ]; then - namespace="knowhere-prod" - else - namespace="knowhere-staging" + if [ -z "${{ secrets.STAGING_MIGRATION_DATABASE_URL }}" ]; then + echo "::error::STAGING_MIGRATION_DATABASE_URL is not configured. Add the direct Neon migration-role URL before the ECS staging deployment." + exit 1 fi + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ env.AWS_ECS_STAGING_REGION }} + + - name: Login to ECR + shell: bash + run: | + set -euo pipefail + aws ecr get-login-password --region "${AWS_ECS_STAGING_REGION}" \ + | docker login --username AWS --password-stdin "${ECR_REGISTRY}" + + - name: Run migration against Neon + shell: bash + env: + DATABASE_URL: ${{ secrets.STAGING_MIGRATION_DATABASE_URL }} + run: | + set -euo pipefail short_sha="${GITHUB_SHA::8}" - if [ "${{ github.event_name }}" = "release" ]; then - image_tag="${{ github.event.release.tag_name }}-${environment}" - elif [[ "${{ github.ref }}" == refs/tags/* ]]; then - image_tag="${GITHUB_REF#refs/tags/}-${environment}" - else - image_tag="${environment}-${short_sha}" - fi + image_uri="${ECR_REGISTRY}/${ECR_REPOSITORY}/knowhere-backend:staging-${short_sha}" + docker run --rm \ + --env DATABASE_URL \ + --env DB_SSL_MODE=require \ + --entrypoint python \ + "$image_uri" \ + -m alembic upgrade heads + + migrate-eks-staging: + name: Run EKS rollback database migration + runs-on: ubuntu-latest + needs: build-and-publish + if: >- + ${{ github.event_name == 'workflow_dispatch' && + github.event.inputs.deployment_target == 'eks-staging-rollback' && + (github.event.inputs.service == '' || + github.event.inputs.service == 'all' || + github.event.inputs.service == 'api') }} + permissions: + contents: read - echo "environment=$environment" >> "$GITHUB_OUTPUT" - echo "namespace=$namespace" >> "$GITHUB_OUTPUT" - echo "image_uri=${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}/knowhere-backend:${image_tag}" >> "$GITHUB_OUTPUT" + steps: + - name: Decide migration context + id: context + shell: bash + run: | + set -euo pipefail + short_sha="${GITHUB_SHA::8}" + echo "namespace=knowhere-staging" >> "$GITHUB_OUTPUT" + echo "image_uri=${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}/knowhere-backend:staging-${short_sha}" >> "$GITHUB_OUTPUT" - name: Validate deployment configuration shell: bash @@ -272,7 +324,7 @@ jobs: [ -z "${{ secrets.AWS_SECRET_ACCESS_KEY }}" ] || \ [ -z "${{ env.AWS_EKS_PROD_CLUSTER_NAME }}" ] || \ [ -z "${{ env.AWS_EKS_PROD_REGION }}" ]; then - echo "::error::Migration requires AWS deployment credentials and cluster configuration." + echo "::error::EKS rollback migration requires AWS deployment credentials and cluster configuration." exit 1 fi @@ -302,7 +354,6 @@ jobs: set -euo pipefail job_name="knowhere-api-migrate-${GITHUB_RUN_ID}" - migration_manifest="$(kubectl get deployment/knowhere-api --namespace "$NAMESPACE" -o json \ | jq --arg job_name "$job_name" --arg image_uri "$IMAGE_URI" ' if ((.spec.template.spec.containers // []) | length) == 0 then @@ -331,23 +382,256 @@ jobs: ')" printf '%s\n' "$migration_manifest" | kubectl apply -f - - - if ! kubectl wait --for=condition=complete "job/$job_name" \ - --namespace "$NAMESPACE" --timeout=900s; then + if ! kubectl wait --for=condition=complete "job/$job_name" --namespace "$NAMESPACE" --timeout=900s; then kubectl describe job "$job_name" --namespace "$NAMESPACE" || true kubectl logs "job/$job_name" --namespace "$NAMESPACE" --all-containers=true || true exit 1 fi - kubectl delete job "$job_name" --namespace "$NAMESPACE" --ignore-not-found - deploy: + deploy-ecs-staging: + name: Deploy staging services to ECS + runs-on: ubuntu-latest + needs: [build-and-publish, migrate-ecs-staging] + if: >- + ${{ always() && needs.build-and-publish.result == 'success' && + (needs['migrate-ecs-staging'].result == 'success' || needs['migrate-ecs-staging'].result == 'skipped') && + ((github.ref == 'refs/heads/staging' && github.event_name == 'push') || + (github.event_name == 'workflow_dispatch' && + github.event.inputs.deployment_target == 'ecs-staging')) }} + permissions: + contents: read + + steps: + - name: Checkout deployed source + uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Decide ECS services to update + id: deployment + shell: bash + run: | + set -euo pipefail + selected_service="${{ github.event.inputs.service }}" + if [ "${{ github.event_name }}" = "push" ] || [ -z "$selected_service" ] || [ "$selected_service" = "all" ]; then + deploy_api="true" + deploy_worker="true" + elif [ "$selected_service" = "api" ]; then + deploy_api="true" + deploy_worker="false" + elif [ "$selected_service" = "worker" ]; then + deploy_api="false" + deploy_worker="true" + else + echo "::error::Unsupported ECS staging service selection: ${selected_service}" + exit 1 + fi + echo "deploy_api=$deploy_api" >> "$GITHUB_OUTPUT" + echo "deploy_worker=$deploy_worker" >> "$GITHUB_OUTPUT" + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ env.AWS_ECS_STAGING_REGION }} + + - name: Validate ECS staging prerequisites + shell: bash + env: + STAGING_SECRETS_ARN: ${{ env.AWS_ECS_STAGING_SECRETS_ARN }} + run: | + set -euo pipefail + + if [ -z "${{ secrets.AWS_ACCESS_KEY_ID }}" ] || [ -z "${{ secrets.AWS_SECRET_ACCESS_KEY }}" ]; then + echo "::error::ECS staging deployment requires AWS deployment credentials." + exit 1 + fi + if [ -z "$STAGING_SECRETS_ARN" ]; then + echo "::error::AWS_ECS_STAGING_SECRETS_ARN is not configured. Create and approve the staging runtime secret before deploying ECS." + exit 1 + fi + + cluster_status="$(aws ecs describe-clusters \ + --clusters "${AWS_ECS_STAGING_CLUSTER_NAME}" \ + --query 'clusters[0].status' --output text)" + if [ "$cluster_status" != "ACTIVE" ]; then + echo "::error::ECS cluster ${AWS_ECS_STAGING_CLUSTER_NAME} is not ACTIVE (status: ${cluster_status})." + exit 1 + fi + + for role_arn in \ + "${AWS_ECS_STAGING_EXECUTION_ROLE_ARN}" \ + "${AWS_ECS_STAGING_API_TASK_ROLE_ARN}" \ + "${AWS_ECS_STAGING_WORKER_TASK_ROLE_ARN}"; do + aws iam get-role --role-name "${role_arn##*/}" --query 'Role.Arn' --output text >/dev/null + done + + aws secretsmanager describe-secret \ + --secret-id "$STAGING_SECRETS_ARN" \ + --query 'ARN' --output text >/dev/null + + for log_group in /ecs/knowhere-api-staging /ecs/knowhere-worker-staging; do + found="$(aws logs describe-log-groups \ + --log-group-name "$log_group" \ + --query 'logGroups[0].logGroupName' --output text)" + if [ "$found" != "$log_group" ]; then + echo "::error::Required CloudWatch log group ${log_group} does not exist." + exit 1 + fi + done + + services="" + if [ "${{ steps.deployment.outputs.deploy_api }}" = "true" ]; then + services="${AWS_ECS_STAGING_API_SERVICE_NAME}" + fi + if [ "${{ steps.deployment.outputs.deploy_worker }}" = "true" ]; then + services="${services} ${AWS_ECS_STAGING_WORKER_SERVICE_NAME}" + fi + for service in $services; do + service_json="$(aws ecs describe-services \ + --cluster "${AWS_ECS_STAGING_CLUSTER_NAME}" \ + --services "$service" --output json)" + status="$(jq -r '.services[0].status // "MISSING"' <<<"$service_json")" + if [ "$status" != "ACTIVE" ]; then + echo "::error::Required ECS service ${service} is not ACTIVE (status: ${status}). Create it before deploying." + exit 1 + fi + network_count="$(jq '[.services[0].networkConfiguration.awsvpcConfiguration.subnets // [] | length] | add' <<<"$service_json")" + security_group_count="$(jq '[.services[0].networkConfiguration.awsvpcConfiguration.securityGroups // [] | length] | add' <<<"$service_json")" + if [ "$network_count" -lt 1 ] || [ "$security_group_count" -lt 1 ]; then + echo "::error::ECS service ${service} has no usable awsvpc subnet/security-group configuration." + exit 1 + fi + done + + if [ "${{ steps.deployment.outputs.deploy_api }}" = "true" ]; then + api_load_balancer_count="$(aws ecs describe-services \ + --cluster "${AWS_ECS_STAGING_CLUSTER_NAME}" \ + --services "${AWS_ECS_STAGING_API_SERVICE_NAME}" \ + --query 'length(services[0].loadBalancers)' --output text)" + if [ "$api_load_balancer_count" -lt 1 ]; then + echo "::error::API ECS service has no load balancer target configured." + exit 1 + fi + fi + + - name: Resolve immutable ECR image digests + id: images + shell: bash + run: | + set -euo pipefail + short_sha="${GITHUB_SHA::8}" + image_tag="staging-${short_sha}" + + resolve_image() { + local repository="$1" + local output_name="$2" + local digest + digest="$(aws ecr describe-images \ + --repository-name "${ECR_REPOSITORY}/${repository}" \ + --image-ids "imageTag=${image_tag}" \ + --query 'imageDetails[0].imageDigest' --output text)" + if [ -z "$digest" ] || [ "$digest" = "None" ]; then + echo "::error::No ECR image found for ${repository}:${image_tag}." + exit 1 + fi + echo "${output_name}=${ECR_REGISTRY}/${ECR_REPOSITORY}/${repository}@${digest}" >> "$GITHUB_OUTPUT" + } + + resolve_image knowhere-backend api_image + resolve_image knowhere-worker worker_image + + - name: Render ECS task definitions + env: + API_IMAGE: ${{ steps.images.outputs.api_image }} + WORKER_IMAGE: ${{ steps.images.outputs.worker_image }} + EXECUTION_ROLE_ARN: ${{ env.AWS_ECS_STAGING_EXECUTION_ROLE_ARN }} + API_TASK_ROLE_ARN: ${{ env.AWS_ECS_STAGING_API_TASK_ROLE_ARN }} + WORKER_TASK_ROLE_ARN: ${{ env.AWS_ECS_STAGING_WORKER_TASK_ROLE_ARN }} + STAGING_SECRETS_ARN: ${{ env.AWS_ECS_STAGING_SECRETS_ARN }} + shell: bash + run: | + set -euo pipefail + python3 deploy/ecs/render_task_definitions.py --output-dir "$RUNNER_TEMP/ecs-task-definitions" + + - name: Register ECS task definitions + id: task-definitions + shell: bash + run: | + set -euo pipefail + if [ "${{ steps.deployment.outputs.deploy_api }}" = "true" ]; then + api_task_definition_arn="$(aws ecs register-task-definition \ + --cli-input-json "file://${RUNNER_TEMP}/ecs-task-definitions/knowhere-api-staging.json" \ + --query 'taskDefinition.taskDefinitionArn' --output text)" + echo "api_task_definition_arn=${api_task_definition_arn}" >> "$GITHUB_OUTPUT" + fi + if [ "${{ steps.deployment.outputs.deploy_worker }}" = "true" ]; then + worker_task_definition_arn="$(aws ecs register-task-definition \ + --cli-input-json "file://${RUNNER_TEMP}/ecs-task-definitions/knowhere-worker-staging.json" \ + --query 'taskDefinition.taskDefinitionArn' --output text)" + echo "worker_task_definition_arn=${worker_task_definition_arn}" >> "$GITHUB_OUTPUT" + fi + + - name: Update API ECS service on on-demand Fargate + if: steps.deployment.outputs.deploy_api == 'true' + shell: bash + run: | + set -euo pipefail + aws ecs update-service \ + --cluster "${AWS_ECS_STAGING_CLUSTER_NAME}" \ + --service "${AWS_ECS_STAGING_API_SERVICE_NAME}" \ + --task-definition "${{ steps.task-definitions.outputs.api_task_definition_arn }}" \ + --capacity-provider-strategy capacityProvider=FARGATE,weight=1 + + - name: Update worker ECS service on Fargate Spot + if: steps.deployment.outputs.deploy_worker == 'true' + shell: bash + run: | + set -euo pipefail + aws ecs update-service \ + --cluster "${AWS_ECS_STAGING_CLUSTER_NAME}" \ + --service "${AWS_ECS_STAGING_WORKER_SERVICE_NAME}" \ + --task-definition "${{ steps.task-definitions.outputs.worker_task_definition_arn }}" \ + --capacity-provider-strategy capacityProvider=FARGATE_SPOT,weight=1 + + - name: Wait for API ECS service to stabilize + if: steps.deployment.outputs.deploy_api == 'true' + shell: bash + run: | + set -euo pipefail + aws ecs wait services-stable \ + --cluster "${AWS_ECS_STAGING_CLUSTER_NAME}" \ + --services "${AWS_ECS_STAGING_API_SERVICE_NAME}" + + - name: Wait for worker ECS service to stabilize + if: steps.deployment.outputs.deploy_worker == 'true' + shell: bash + run: | + set -euo pipefail + aws ecs wait services-stable \ + --cluster "${AWS_ECS_STAGING_CLUSTER_NAME}" \ + --services "${AWS_ECS_STAGING_WORKER_SERVICE_NAME}" + + - name: Summarize ECS deployment + shell: bash + run: | + echo "Environment: staging" + echo "Cluster: ${AWS_ECS_STAGING_CLUSTER_NAME}" + echo "API image: ${{ steps.images.outputs.api_image }}" + echo "Worker image: ${{ steps.images.outputs.worker_image }}" + + deploy-eks: runs-on: ubuntu-latest - needs: [build-and-publish, migrate] + needs: [build-and-publish, migrate-eks-staging] if: >- ${{ always() && github.event_name != 'pull_request' && needs.build-and-publish.result == 'success' && - (needs.migrate.result == 'success' || needs.migrate.result == 'skipped') }} + (needs['migrate-eks-staging'].result == 'success' || needs['migrate-eks-staging'].result == 'skipped') && + (github.event_name == 'release' || + (github.event_name == 'workflow_dispatch' && + github.event.inputs.deployment_target == 'eks-staging-rollback')) }} permissions: contents: read @@ -471,7 +755,7 @@ jobs: release: name: Attach deployment release assets runs-on: ubuntu-latest - needs: deploy + needs: deploy-eks if: >- ${{ github.event_name == 'release' && github.event.action == 'published' }} diff --git a/deploy/ecs/README.md b/deploy/ecs/README.md index a8e561ed..74444776 100644 --- a/deploy/ecs/README.md +++ b/deploy/ecs/README.md @@ -22,3 +22,13 @@ python deploy/ecs/render_task_definitions.py --output-dir /tmp/knowhere-ecs-rend ``` The output directory is deployment-only and must not be committed. The renderer fails on missing inputs, unresolved placeholders, or either long-lived S3 credential variable. + +## Staging workflow prerequisites + +The staging workflow in `.github/workflows/build-images.yml` expects these GitHub Actions secrets: + +- `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`: the deployment credential used by the existing release workflow. +- `AWS_ECS_STAGING_SECRETS_ARN`: the ARN of the approved JSON runtime secret described above. +- `STAGING_MIGRATION_DATABASE_URL`: the direct Neon URL for the staging migration role. It is used only by the migration container and is never passed to the ECS runtime tasks. + +Before an ECS staging deployment, an operator must create and verify the ECS services, network configuration, API load-balancer target, CloudWatch log groups, and runtime secret. The workflow validates those resources and fails without registering or updating a service when any prerequisite is missing. It does not create or delete AWS resources.