From 3a9bc5c4e04d2b0bd9b3df0df8f5e8592203a58f Mon Sep 17 00:00:00 2001 From: suguanYang Date: Thu, 13 Aug 2026 09:18:20 +0800 Subject: [PATCH] feat: add staging ECS task definition templates --- deploy/ecs/README.md | 24 ++++ deploy/ecs/render_task_definitions.py | 119 ++++++++++++++++++ deploy/ecs/task-definition-api.staging.json | 92 ++++++++++++++ .../ecs/task-definition-worker.staging.json | 93 ++++++++++++++ deploy/ecs/test_render_task_definitions.py | 67 ++++++++++ 5 files changed, 395 insertions(+) create mode 100644 deploy/ecs/README.md create mode 100644 deploy/ecs/render_task_definitions.py create mode 100644 deploy/ecs/task-definition-api.staging.json create mode 100644 deploy/ecs/task-definition-worker.staging.json create mode 100644 deploy/ecs/test_render_task_definitions.py diff --git a/deploy/ecs/README.md b/deploy/ecs/README.md new file mode 100644 index 000000000..a8e561ed5 --- /dev/null +++ b/deploy/ecs/README.md @@ -0,0 +1,24 @@ +# ECS task definitions + +These files are deployment templates for the shared `knowhere-fargate` cluster. They do not contain secret values and are not registered automatically. + +The staging templates intentionally omit `S3_ACCESS_KEY_ID` and `S3_SECRET_ACCESS_KEY`. With `S3_TYPE=s3`, boto3 obtains temporary authenticated credentials from the ECS task role. + +The single staging Secrets Manager secret supplied to the renderer must be a JSON secret with these keys: + +- API: `DATABASE_URL`, `REDIS_HOST`, `REDIS_PORT`, `REDIS_PASSWORD`, `CELERY_REDIS_URL`, `SECRET_KEY`, `DS_KEY`, `ALI_API_KEYS`, `ARK_API_KEY`, `GPT_API_KEY`, `MINERU_API_KEYS`, `STRIPE_SECRET_KEY`, `STRIPE_WEBHOOK_SECRET`, `WEBHOOK_MASTER_KEY`, `LOGFIRE_TOKEN`, `QSTASH_TOKEN`, `QSTASH_CURRENT_SIGNING_KEY`, `QSTASH_NEXT_SIGNING_KEY` +- Worker: the API keys above plus `CELERY_REDIS_PASSWORD` and `ILOVEAPI_KEYS` + +Render only after the secret and log groups exist, substituting the exact immutable ECR image digests and IAM role/secret ARNs: + +```bash +API_IMAGE=107424103509.dkr.ecr.us-east-1.amazonaws.com/knowhere/knowhere-backend@sha256:... \ +WORKER_IMAGE=107424103509.dkr.ecr.us-east-1.amazonaws.com/knowhere/knowhere-worker@sha256:... \ +EXECUTION_ROLE_ARN=arn:aws:iam::107424103509:role/knowhere-fargate-staging-execution-role \ +API_TASK_ROLE_ARN=arn:aws:iam::107424103509:role/knowhere-api-staging-task-role \ +WORKER_TASK_ROLE_ARN=arn:aws:iam::107424103509:role/knowhere-worker-staging-task-role \ +STAGING_SECRETS_ARN=arn:aws:secretsmanager:us-east-1:107424103509:secret:knowhere/staging/runtime-... \ +python deploy/ecs/render_task_definitions.py --output-dir /tmp/knowhere-ecs-rendered +``` + +The output directory is deployment-only and must not be committed. The renderer fails on missing inputs, unresolved placeholders, or either long-lived S3 credential variable. diff --git a/deploy/ecs/render_task_definitions.py b/deploy/ecs/render_task_definitions.py new file mode 100644 index 000000000..4312533ce --- /dev/null +++ b/deploy/ecs/render_task_definitions.py @@ -0,0 +1,119 @@ +"""Render staging ECS task-definition templates without storing secrets.""" + +from __future__ import annotations + +import argparse +import json +import os +import re +from pathlib import Path +from typing import Final + + +PLACEHOLDER_PATTERN: Final[re.Pattern[str]] = re.compile(r"\$\{([A-Z0-9_]+)\}") +REQUIRED_VARIABLES: Final[tuple[str, ...]] = ( + "API_IMAGE", + "WORKER_IMAGE", + "EXECUTION_ROLE_ARN", + "API_TASK_ROLE_ARN", + "WORKER_TASK_ROLE_ARN", + "STAGING_SECRETS_ARN", +) +FORBIDDEN_ENVIRONMENT_NAMES: Final[frozenset[str]] = frozenset( + {"S3_ACCESS_KEY_ID", "S3_SECRET_ACCESS_KEY"} +) + + +def replace_placeholders(value: object, variables: dict[str, str]) -> object: + """Recursively replace required placeholders in a JSON-compatible value.""" + if isinstance(value, str): + return PLACEHOLDER_PATTERN.sub( + lambda match: variables.get(match.group(1), match.group(0)), value + ) + if isinstance(value, list): + return [replace_placeholders(item, variables) for item in value] + if isinstance(value, dict): + return { + str(key): replace_placeholders(item, variables) + for key, item in value.items() + } + return value + + +def collect_strings(value: object) -> list[str]: + """Collect all string values for unresolved-placeholder validation.""" + if isinstance(value, str): + return [value] + if isinstance(value, list): + return [item for child in value for item in collect_strings(child)] + if isinstance(value, dict): + return [item for child in value.values() for item in collect_strings(child)] + return [] + + +def validate_rendered_definition(definition: object) -> None: + """Reject unresolved placeholders and long-lived S3 credential variables.""" + strings = collect_strings(definition) + unresolved = [value for value in strings if PLACEHOLDER_PATTERN.search(value)] + if unresolved: + raise ValueError(f"Unresolved task-definition placeholders: {unresolved}") + + containers = definition.get("containerDefinitions") if isinstance(definition, dict) else None + if not isinstance(containers, list): + raise ValueError("Task definition must contain containerDefinitions") + environment_names = { + str(item.get("name")) + for container in containers + if isinstance(container, dict) + for item in [*(container.get("environment") or []), *(container.get("secrets") or [])] + if isinstance(item, dict) + } + forbidden = environment_names & FORBIDDEN_ENVIRONMENT_NAMES + if forbidden: + raise ValueError(f"Task definition contains forbidden S3 credentials: {sorted(forbidden)}") + + +def render_template(template_path: Path, output_path: Path, variables: dict[str, str]) -> None: + """Render one JSON template to a deployment-only output path.""" + definition = json.loads(template_path.read_text(encoding="utf-8")) + rendered = replace_placeholders(definition, variables) + validate_rendered_definition(rendered) + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(json.dumps(rendered, indent=2) + "\n", encoding="utf-8") + + +def parse_arguments() -> argparse.Namespace: + """Parse renderer CLI arguments.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output-dir", type=Path, required=True) + return parser.parse_args() + + +def load_variables() -> dict[str, str]: + """Load and validate deployment inputs from the environment.""" + variables = {name: os.environ.get(name, "") for name in REQUIRED_VARIABLES} + missing = [name for name, value in variables.items() if not value] + if missing: + raise ValueError(f"Missing required renderer variables: {', '.join(missing)}") + return variables + + +def main() -> None: + """Render both staging task definitions.""" + arguments = parse_arguments() + variables = load_variables() + template_directory = Path(__file__).parent + render_template( + template_directory / "task-definition-api.staging.json", + arguments.output_dir / "knowhere-api-staging.json", + variables, + ) + render_template( + template_directory / "task-definition-worker.staging.json", + arguments.output_dir / "knowhere-worker-staging.json", + variables, + ) + + +if __name__ == "__main__": + main() diff --git a/deploy/ecs/task-definition-api.staging.json b/deploy/ecs/task-definition-api.staging.json new file mode 100644 index 000000000..403f44dce --- /dev/null +++ b/deploy/ecs/task-definition-api.staging.json @@ -0,0 +1,92 @@ +{ + "family": "knowhere-api-staging", + "taskRoleArn": "${API_TASK_ROLE_ARN}", + "executionRoleArn": "${EXECUTION_ROLE_ARN}", + "networkMode": "awsvpc", + "requiresCompatibilities": ["FARGATE"], + "cpu": "256", + "memory": "1024", + "runtimePlatform": { + "cpuArchitecture": "X86_64", + "operatingSystemFamily": "LINUX" + }, + "containerDefinitions": [ + { + "name": "api", + "image": "${API_IMAGE}", + "essential": true, + "portMappings": [ + { + "name": "http", + "containerPort": 5005, + "hostPort": 5005, + "protocol": "tcp" + } + ], + "environment": [ + {"name": "ENVIRONMENT", "value": "staging"}, + {"name": "APP_ENV", "value": "staging"}, + {"name": "DB_SSL_MODE", "value": "require"}, + {"name": "TMP_PATH", "value": "/tmp/knowhere"}, + {"name": "S3_TYPE", "value": "s3"}, + {"name": "S3_BUCKET_NAME", "value": "knowhere-storage-staging"}, + {"name": "S3_RESULTS_BUCKET", "value": "knowhere-storage-staging"}, + {"name": "S3_REGION", "value": "us-east-1"}, + {"name": "S3_USE_SSL", "value": "true"}, + {"name": "S3_ADDRESSING_STYLE", "value": "virtual"}, + {"name": "REDIS_DATABASE", "value": "0"}, + {"name": "REDIS_SSL", "value": "true"}, + {"name": "RATE_LIMIT_ENABLED", "value": "true"}, + {"name": "TELEMETRY_ENABLED", "value": "false"}, + {"name": "API_STANDALONE_MODE_ENABLED", "value": "true"}, + {"name": "INTERNAL_DASHBOARD_ENDPOINT", "value": "https://staging.knowhereto.ai"}, + {"name": "FRONTEND_URL", "value": "https://staging.knowhereto.ai"}, + {"name": "BILLING_ENABLED", "value": "true"}, + {"name": "QSTASH_CALLBACK_BASE_URL", "value": "https://api-staging.knowhereto.ai/api/v1"}, + {"name": "AWS_REGION", "value": "us-east-1"}, + {"name": "AWS_ACCOUNT_ID", "value": "107424103509"} + ], + "secrets": [ + {"name": "DATABASE_URL", "valueFrom": "${STAGING_SECRETS_ARN}:DATABASE_URL::"}, + {"name": "REDIS_HOST", "valueFrom": "${STAGING_SECRETS_ARN}:REDIS_HOST::"}, + {"name": "REDIS_PORT", "valueFrom": "${STAGING_SECRETS_ARN}:REDIS_PORT::"}, + {"name": "REDIS_PASSWORD", "valueFrom": "${STAGING_SECRETS_ARN}:REDIS_PASSWORD::"}, + {"name": "CELERY_REDIS_URL", "valueFrom": "${STAGING_SECRETS_ARN}:CELERY_REDIS_URL::"}, + {"name": "SECRET_KEY", "valueFrom": "${STAGING_SECRETS_ARN}:SECRET_KEY::"}, + {"name": "DS_KEY", "valueFrom": "${STAGING_SECRETS_ARN}:DS_KEY::"}, + {"name": "ALI_API_KEYS", "valueFrom": "${STAGING_SECRETS_ARN}:ALI_API_KEYS::"}, + {"name": "ARK_API_KEY", "valueFrom": "${STAGING_SECRETS_ARN}:ARK_API_KEY::"}, + {"name": "GPT_API_KEY", "valueFrom": "${STAGING_SECRETS_ARN}:GPT_API_KEY::"}, + {"name": "MINERU_API_KEYS", "valueFrom": "${STAGING_SECRETS_ARN}:MINERU_API_KEYS::"}, + {"name": "STRIPE_SECRET_KEY", "valueFrom": "${STAGING_SECRETS_ARN}:STRIPE_SECRET_KEY::"}, + {"name": "STRIPE_WEBHOOK_SECRET", "valueFrom": "${STAGING_SECRETS_ARN}:STRIPE_WEBHOOK_SECRET::"}, + {"name": "WEBHOOK_MASTER_KEY", "valueFrom": "${STAGING_SECRETS_ARN}:WEBHOOK_MASTER_KEY::"}, + {"name": "LOGFIRE_TOKEN", "valueFrom": "${STAGING_SECRETS_ARN}:LOGFIRE_TOKEN::"}, + {"name": "QSTASH_TOKEN", "valueFrom": "${STAGING_SECRETS_ARN}:QSTASH_TOKEN::"}, + {"name": "QSTASH_CURRENT_SIGNING_KEY", "valueFrom": "${STAGING_SECRETS_ARN}:QSTASH_CURRENT_SIGNING_KEY::"}, + {"name": "QSTASH_NEXT_SIGNING_KEY", "valueFrom": "${STAGING_SECRETS_ARN}:QSTASH_NEXT_SIGNING_KEY::"} + ], + "healthCheck": { + "command": ["CMD-SHELL", "curl -f http://localhost:5005/health || exit 1"], + "interval": 30, + "timeout": 5, + "retries": 3, + "startPeriod": 30 + }, + "logConfiguration": { + "logDriver": "awslogs", + "options": { + "awslogs-group": "/ecs/knowhere-api-staging", + "awslogs-region": "us-east-1", + "awslogs-stream-prefix": "api" + } + } + } + ], + "tags": [ + {"key": "Project", "value": "knowhere"}, + {"key": "Environment", "value": "staging"}, + {"key": "Service", "value": "api"}, + {"key": "ManagedBy", "value": "knowhere-api-infra"} + ] +} diff --git a/deploy/ecs/task-definition-worker.staging.json b/deploy/ecs/task-definition-worker.staging.json new file mode 100644 index 000000000..e9781f2fe --- /dev/null +++ b/deploy/ecs/task-definition-worker.staging.json @@ -0,0 +1,93 @@ +{ + "family": "knowhere-worker-staging", + "taskRoleArn": "${WORKER_TASK_ROLE_ARN}", + "executionRoleArn": "${EXECUTION_ROLE_ARN}", + "networkMode": "awsvpc", + "requiresCompatibilities": ["FARGATE"], + "cpu": "512", + "memory": "4096", + "ephemeralStorage": {"sizeInGiB": 20}, + "runtimePlatform": { + "cpuArchitecture": "X86_64", + "operatingSystemFamily": "LINUX" + }, + "containerDefinitions": [ + { + "name": "worker", + "image": "${WORKER_IMAGE}", + "essential": true, + "stopTimeout": 120, + "environment": [ + {"name": "ENVIRONMENT", "value": "staging"}, + {"name": "APP_ENV", "value": "staging"}, + {"name": "DB_SSL_MODE", "value": "require"}, + {"name": "TMP_PATH", "value": "/tmp/knowhere"}, + {"name": "S3_TYPE", "value": "s3"}, + {"name": "S3_BUCKET_NAME", "value": "knowhere-storage-staging"}, + {"name": "S3_RESULTS_BUCKET", "value": "knowhere-storage-staging"}, + {"name": "S3_REGION", "value": "us-east-1"}, + {"name": "S3_USE_SSL", "value": "true"}, + {"name": "S3_ADDRESSING_STYLE", "value": "virtual"}, + {"name": "REDIS_DATABASE", "value": "0"}, + {"name": "REDIS_SSL", "value": "true"}, + {"name": "AWS_REGION", "value": "us-east-1"}, + {"name": "AWS_ACCOUNT_ID", "value": "107424103509"}, + {"name": "RATE_LIMIT_ENABLED", "value": "false"}, + {"name": "TELEMETRY_ENABLED", "value": "false"}, + {"name": "INTERNAL_DASHBOARD_ENDPOINT", "value": "https://staging.knowhereto.ai"}, + {"name": "API_WEBHOOK_ENDPOINT", "value": "https://api-staging.knowhereto.ai/v1/internal/s3-events"}, + {"name": "SNS_TOPIC_ARN", "value": "arn:aws:sns:us-east-1:107424103509:knowhere-staging-s3-events"}, + {"name": "QSTASH_CALLBACK_BASE_URL", "value": "https://api-staging.knowhereto.ai/api/v1"}, + {"name": "HF_HOME", "value": "/mnt/models/huggingface"}, + {"name": "TRANSFORMERS_CACHE", "value": "/mnt/models/huggingface"}, + {"name": "BILLING_ENABLED", "value": "true"}, + {"name": "SUMMARY_LLM_MAX_CONCURRENT", "value": "8"}, + {"name": "HIERARCHY_LLM_MODEL", "value": "deepseek-chat"}, + {"name": "NORMOL_MODEL", "value": "deepseek-v4-flash"}, + {"name": "IMAGE_MODEL", "value": "qwen3.6-flash"}, + {"name": "IMAGE_MODEL_MAX", "value": "qwen3.6-flash"} + ], + "secrets": [ + {"name": "DATABASE_URL", "valueFrom": "${STAGING_SECRETS_ARN}:DATABASE_URL::"}, + {"name": "REDIS_HOST", "valueFrom": "${STAGING_SECRETS_ARN}:REDIS_HOST::"}, + {"name": "REDIS_PORT", "valueFrom": "${STAGING_SECRETS_ARN}:REDIS_PORT::"}, + {"name": "REDIS_PASSWORD", "valueFrom": "${STAGING_SECRETS_ARN}:REDIS_PASSWORD::"}, + {"name": "CELERY_REDIS_URL", "valueFrom": "${STAGING_SECRETS_ARN}:CELERY_REDIS_URL::"}, + {"name": "CELERY_REDIS_PASSWORD", "valueFrom": "${STAGING_SECRETS_ARN}:CELERY_REDIS_PASSWORD::"}, + {"name": "SECRET_KEY", "valueFrom": "${STAGING_SECRETS_ARN}:SECRET_KEY::"}, + {"name": "WEBHOOK_MASTER_KEY", "valueFrom": "${STAGING_SECRETS_ARN}:WEBHOOK_MASTER_KEY::"}, + {"name": "DS_KEY", "valueFrom": "${STAGING_SECRETS_ARN}:DS_KEY::"}, + {"name": "ALI_API_KEYS", "valueFrom": "${STAGING_SECRETS_ARN}:ALI_API_KEYS::"}, + {"name": "ARK_API_KEY", "valueFrom": "${STAGING_SECRETS_ARN}:ARK_API_KEY::"}, + {"name": "GPT_API_KEY", "valueFrom": "${STAGING_SECRETS_ARN}:GPT_API_KEY::"}, + {"name": "MINERU_API_KEYS", "valueFrom": "${STAGING_SECRETS_ARN}:MINERU_API_KEYS::"}, + {"name": "ILOVEAPI_KEYS", "valueFrom": "${STAGING_SECRETS_ARN}:ILOVEAPI_KEYS::"}, + {"name": "LOGFIRE_TOKEN", "valueFrom": "${STAGING_SECRETS_ARN}:LOGFIRE_TOKEN::"}, + {"name": "QSTASH_TOKEN", "valueFrom": "${STAGING_SECRETS_ARN}:QSTASH_TOKEN::"}, + {"name": "QSTASH_CURRENT_SIGNING_KEY", "valueFrom": "${STAGING_SECRETS_ARN}:QSTASH_CURRENT_SIGNING_KEY::"}, + {"name": "QSTASH_NEXT_SIGNING_KEY", "valueFrom": "${STAGING_SECRETS_ARN}:QSTASH_NEXT_SIGNING_KEY::"} + ], + "healthCheck": { + "command": ["CMD-SHELL", "python -c \"from shared.services.worker_health import assert_worker_healthy; assert_worker_healthy()\""], + "interval": 30, + "timeout": 10, + "retries": 3, + "startPeriod": 60 + }, + "logConfiguration": { + "logDriver": "awslogs", + "options": { + "awslogs-group": "/ecs/knowhere-worker-staging", + "awslogs-region": "us-east-1", + "awslogs-stream-prefix": "worker" + } + } + } + ], + "tags": [ + {"key": "Project", "value": "knowhere"}, + {"key": "Environment", "value": "staging"}, + {"key": "Service", "value": "worker"}, + {"key": "ManagedBy", "value": "knowhere-api-infra"} + ] +} diff --git a/deploy/ecs/test_render_task_definitions.py b/deploy/ecs/test_render_task_definitions.py new file mode 100644 index 000000000..472d12528 --- /dev/null +++ b/deploy/ecs/test_render_task_definitions.py @@ -0,0 +1,67 @@ +"""Contracts for ECS task-definition rendering.""" + +import json +from pathlib import Path + +import pytest + +from render_task_definitions import ( + render_template, + validate_rendered_definition, +) + + +TEMPLATE_DIRECTORY: Path = Path(__file__).parent +RENDER_VARIABLES: dict[str, str] = { + "API_IMAGE": "api-image", + "WORKER_IMAGE": "worker-image", + "EXECUTION_ROLE_ARN": "execution-role", + "API_TASK_ROLE_ARN": "api-role", + "WORKER_TASK_ROLE_ARN": "worker-role", + "STAGING_SECRETS_ARN": "secrets-arn", +} + + +@pytest.mark.parametrize( + "template_name", + ["task-definition-api.staging.json", "task-definition-worker.staging.json"], +) +def test_staging_templates_render_without_long_lived_s3_keys( + tmp_path: Path, + template_name: str, +) -> None: + """Both templates render and exclude explicit S3 credential variables.""" + output_path: Path = tmp_path / template_name + render_template( + TEMPLATE_DIRECTORY / template_name, + output_path, + RENDER_VARIABLES, + ) + + rendered: dict[str, object] = json.loads(output_path.read_text(encoding="utf-8")) + validate_rendered_definition(rendered) + assert rendered["requiresCompatibilities"] == ["FARGATE"] + assert "${" not in output_path.read_text(encoding="utf-8") + + +def test_renderer_rejects_forbidden_s3_credential_variable() -> None: + """Task definitions must never inject long-lived S3 credentials.""" + definition: dict[str, object] = { + "containerDefinitions": [ + {"environment": [{"name": "S3_ACCESS_KEY_ID", "value": "bad"}]} + ] + } + + with pytest.raises(ValueError, match="forbidden S3 credentials"): + validate_rendered_definition(definition) + + +def test_renderer_rejects_unresolved_placeholders() -> None: + """Deployment cannot proceed with missing substitution values.""" + definition: dict[str, object] = { + "containerDefinitions": [{"environment": []}], + "image": "${API_IMAGE}", + } + + with pytest.raises(ValueError, match="Unresolved task-definition placeholders"): + validate_rendered_definition(definition)