feat: add staging ECS task definition templates

deploy/ecs/README.md

@@ -0,0 +1,24 @@
+# ECS task definitions
+
+These files are deployment templates for the shared `knowhere-fargate` cluster. They do not contain secret values and are not registered automatically.
+
+The staging templates intentionally omit `S3_ACCESS_KEY_ID` and `S3_SECRET_ACCESS_KEY`. With `S3_TYPE=s3`, boto3 obtains temporary authenticated credentials from the ECS task role.
+
+The single staging Secrets Manager secret supplied to the renderer must be a JSON secret with these keys:
+
+- API: `DATABASE_URL`, `REDIS_HOST`, `REDIS_PORT`, `REDIS_PASSWORD`, `CELERY_REDIS_URL`, `SECRET_KEY`, `DS_KEY`, `ALI_API_KEYS`, `ARK_API_KEY`, `GPT_API_KEY`, `MINERU_API_KEYS`, `STRIPE_SECRET_KEY`, `STRIPE_WEBHOOK_SECRET`, `WEBHOOK_MASTER_KEY`, `LOGFIRE_TOKEN`, `QSTASH_TOKEN`, `QSTASH_CURRENT_SIGNING_KEY`, `QSTASH_NEXT_SIGNING_KEY`
+- Worker: the API keys above plus `CELERY_REDIS_PASSWORD` and `ILOVEAPI_KEYS`
+
+Render only after the secret and log groups exist, substituting the exact immutable ECR image digests and IAM role/secret ARNs:
+
+```bash
+API_IMAGE=107424103509.dkr.ecr.us-east-1.amazonaws.com/knowhere/knowhere-backend@sha256:... \
+WORKER_IMAGE=107424103509.dkr.ecr.us-east-1.amazonaws.com/knowhere/knowhere-worker@sha256:... \
+EXECUTION_ROLE_ARN=arn:aws:iam::107424103509:role/knowhere-fargate-staging-execution-role \
+API_TASK_ROLE_ARN=arn:aws:iam::107424103509:role/knowhere-api-staging-task-role \
+WORKER_TASK_ROLE_ARN=arn:aws:iam::107424103509:role/knowhere-worker-staging-task-role \
+STAGING_SECRETS_ARN=arn:aws:secretsmanager:us-east-1:107424103509:secret:knowhere/staging/runtime-... \
+python deploy/ecs/render_task_definitions.py --output-dir /tmp/knowhere-ecs-rendered
+```
+
+The output directory is deployment-only and must not be committed. The renderer fails on missing inputs, unresolved placeholders, or either long-lived S3 credential variable.

deploy/ecs/render_task_definitions.py

@@ -0,0 +1,119 @@
+"""Render staging ECS task-definition templates without storing secrets."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import re
+from pathlib import Path
+from typing import Final
+
+
+PLACEHOLDER_PATTERN: Final[re.Pattern[str]] = re.compile(r"\$\{([A-Z0-9_]+)\}")
+REQUIRED_VARIABLES: Final[tuple[str, ...]] = (
+    "API_IMAGE",
+    "WORKER_IMAGE",
+    "EXECUTION_ROLE_ARN",
+    "API_TASK_ROLE_ARN",
+    "WORKER_TASK_ROLE_ARN",
+    "STAGING_SECRETS_ARN",
+)
+FORBIDDEN_ENVIRONMENT_NAMES: Final[frozenset[str]] = frozenset(
+    {"S3_ACCESS_KEY_ID", "S3_SECRET_ACCESS_KEY"}
+)
+
+
+def replace_placeholders(value: object, variables: dict[str, str]) -> object:
+    """Recursively replace required placeholders in a JSON-compatible value."""
+    if isinstance(value, str):
+        return PLACEHOLDER_PATTERN.sub(
+            lambda match: variables.get(match.group(1), match.group(0)), value
+        )
+    if isinstance(value, list):
+        return [replace_placeholders(item, variables) for item in value]
+    if isinstance(value, dict):
+        return {
+            str(key): replace_placeholders(item, variables)
+            for key, item in value.items()
+        }
+    return value
+
+
+def collect_strings(value: object) -> list[str]:
+    """Collect all string values for unresolved-placeholder validation."""
+    if isinstance(value, str):
+        return [value]
+    if isinstance(value, list):
+        return [item for child in value for item in collect_strings(child)]
+    if isinstance(value, dict):
+        return [item for child in value.values() for item in collect_strings(child)]
+    return []
+
+
+def validate_rendered_definition(definition: object) -> None:
+    """Reject unresolved placeholders and long-lived S3 credential variables."""
+    strings = collect_strings(definition)
+    unresolved = [value for value in strings if PLACEHOLDER_PATTERN.search(value)]
+    if unresolved:
+        raise ValueError(f"Unresolved task-definition placeholders: {unresolved}")
+
+    containers = definition.get("containerDefinitions") if isinstance(definition, dict) else None
+    if not isinstance(containers, list):
+        raise ValueError("Task definition must contain containerDefinitions")
+    environment_names = {
+        str(item.get("name"))
+        for container in containers
+        if isinstance(container, dict)
+        for item in [*(container.get("environment") or []), *(container.get("secrets") or [])]
+        if isinstance(item, dict)
+    }
+    forbidden = environment_names & FORBIDDEN_ENVIRONMENT_NAMES
+    if forbidden:
+        raise ValueError(f"Task definition contains forbidden S3 credentials: {sorted(forbidden)}")
+
+
+def render_template(template_path: Path, output_path: Path, variables: dict[str, str]) -> None:
+    """Render one JSON template to a deployment-only output path."""
+    definition = json.loads(template_path.read_text(encoding="utf-8"))
+    rendered = replace_placeholders(definition, variables)
+    validate_rendered_definition(rendered)
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    output_path.write_text(json.dumps(rendered, indent=2) + "\n", encoding="utf-8")
+
+
+def parse_arguments() -> argparse.Namespace:
+    """Parse renderer CLI arguments."""
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--output-dir", type=Path, required=True)
+    return parser.parse_args()
+
+
+def load_variables() -> dict[str, str]:
+    """Load and validate deployment inputs from the environment."""
+    variables = {name: os.environ.get(name, "") for name in REQUIRED_VARIABLES}
+    missing = [name for name, value in variables.items() if not value]
+    if missing:
+        raise ValueError(f"Missing required renderer variables: {', '.join(missing)}")
+    return variables
+
+
+def main() -> None:
+    """Render both staging task definitions."""
+    arguments = parse_arguments()
+    variables = load_variables()
+    template_directory = Path(__file__).parent
+    render_template(
+        template_directory / "task-definition-api.staging.json",
+        arguments.output_dir / "knowhere-api-staging.json",
+        variables,
+    )
+    render_template(
+        template_directory / "task-definition-worker.staging.json",
+        arguments.output_dir / "knowhere-worker-staging.json",
+        variables,
+    )
+
+
+if __name__ == "__main__":
+    main()

deploy/ecs/task-definition-api.staging.json

@@ -0,0 +1,92 @@
+{
+  "family": "knowhere-api-staging",
+  "taskRoleArn": "${API_TASK_ROLE_ARN}",
+  "executionRoleArn": "${EXECUTION_ROLE_ARN}",
+  "networkMode": "awsvpc",
+  "requiresCompatibilities": ["FARGATE"],
+  "cpu": "256",
+  "memory": "1024",
+  "runtimePlatform": {
+    "cpuArchitecture": "X86_64",
+    "operatingSystemFamily": "LINUX"
+  },
+  "containerDefinitions": [
+    {
+      "name": "api",
+      "image": "${API_IMAGE}",
+      "essential": true,
+      "portMappings": [
+        {
+          "name": "http",
+          "containerPort": 5005,
+          "hostPort": 5005,
+          "protocol": "tcp"
+        }
+      ],
+      "environment": [
+        {"name": "ENVIRONMENT", "value": "staging"},
+        {"name": "APP_ENV", "value": "staging"},
+        {"name": "DB_SSL_MODE", "value": "require"},
+        {"name": "TMP_PATH", "value": "/tmp/knowhere"},
+        {"name": "S3_TYPE", "value": "s3"},
+        {"name": "S3_BUCKET_NAME", "value": "knowhere-storage-staging"},
+        {"name": "S3_RESULTS_BUCKET", "value": "knowhere-storage-staging"},
+        {"name": "S3_REGION", "value": "us-east-1"},
+        {"name": "S3_USE_SSL", "value": "true"},
+        {"name": "S3_ADDRESSING_STYLE", "value": "virtual"},
+        {"name": "REDIS_DATABASE", "value": "0"},
+        {"name": "REDIS_SSL", "value": "true"},
+        {"name": "RATE_LIMIT_ENABLED", "value": "true"},
+        {"name": "TELEMETRY_ENABLED", "value": "false"},
+        {"name": "API_STANDALONE_MODE_ENABLED", "value": "true"},
+        {"name": "INTERNAL_DASHBOARD_ENDPOINT", "value": "https://staging.knowhereto.ai"},
+        {"name": "FRONTEND_URL", "value": "https://staging.knowhereto.ai"},
+        {"name": "BILLING_ENABLED", "value": "true"},
+        {"name": "QSTASH_CALLBACK_BASE_URL", "value": "https://api-staging.knowhereto.ai/api/v1"},
+        {"name": "AWS_REGION", "value": "us-east-1"},
+        {"name": "AWS_ACCOUNT_ID", "value": "107424103509"}
+      ],
+      "secrets": [
+        {"name": "DATABASE_URL", "valueFrom": "${STAGING_SECRETS_ARN}:DATABASE_URL::"},
+        {"name": "REDIS_HOST", "valueFrom": "${STAGING_SECRETS_ARN}:REDIS_HOST::"},
+        {"name": "REDIS_PORT", "valueFrom": "${STAGING_SECRETS_ARN}:REDIS_PORT::"},
+        {"name": "REDIS_PASSWORD", "valueFrom": "${STAGING_SECRETS_ARN}:REDIS_PASSWORD::"},
+        {"name": "CELERY_REDIS_URL", "valueFrom": "${STAGING_SECRETS_ARN}:CELERY_REDIS_URL::"},
+        {"name": "SECRET_KEY", "valueFrom": "${STAGING_SECRETS_ARN}:SECRET_KEY::"},
+        {"name": "DS_KEY", "valueFrom": "${STAGING_SECRETS_ARN}:DS_KEY::"},
+        {"name": "ALI_API_KEYS", "valueFrom": "${STAGING_SECRETS_ARN}:ALI_API_KEYS::"},
+        {"name": "ARK_API_KEY", "valueFrom": "${STAGING_SECRETS_ARN}:ARK_API_KEY::"},
+        {"name": "GPT_API_KEY", "valueFrom": "${STAGING_SECRETS_ARN}:GPT_API_KEY::"},
+        {"name": "MINERU_API_KEYS", "valueFrom": "${STAGING_SECRETS_ARN}:MINERU_API_KEYS::"},
+        {"name": "STRIPE_SECRET_KEY", "valueFrom": "${STAGING_SECRETS_ARN}:STRIPE_SECRET_KEY::"},
+        {"name": "STRIPE_WEBHOOK_SECRET", "valueFrom": "${STAGING_SECRETS_ARN}:STRIPE_WEBHOOK_SECRET::"},
+        {"name": "WEBHOOK_MASTER_KEY", "valueFrom": "${STAGING_SECRETS_ARN}:WEBHOOK_MASTER_KEY::"},
+        {"name": "LOGFIRE_TOKEN", "valueFrom": "${STAGING_SECRETS_ARN}:LOGFIRE_TOKEN::"},
+        {"name": "QSTASH_TOKEN", "valueFrom": "${STAGING_SECRETS_ARN}:QSTASH_TOKEN::"},
+        {"name": "QSTASH_CURRENT_SIGNING_KEY", "valueFrom": "${STAGING_SECRETS_ARN}:QSTASH_CURRENT_SIGNING_KEY::"},
+        {"name": "QSTASH_NEXT_SIGNING_KEY", "valueFrom": "${STAGING_SECRETS_ARN}:QSTASH_NEXT_SIGNING_KEY::"}
+      ],
+      "healthCheck": {
+        "command": ["CMD-SHELL", "curl -f http://localhost:5005/health || exit 1"],
+        "interval": 30,
+        "timeout": 5,
+        "retries": 3,
+        "startPeriod": 30
+      },
+      "logConfiguration": {
+        "logDriver": "awslogs",
+        "options": {
+          "awslogs-group": "/ecs/knowhere-api-staging",
+          "awslogs-region": "us-east-1",
+          "awslogs-stream-prefix": "api"
+        }
+      }
+    }
+  ],
+  "tags": [
+    {"key": "Project", "value": "knowhere"},
+    {"key": "Environment", "value": "staging"},
+    {"key": "Service", "value": "api"},
+    {"key": "ManagedBy", "value": "knowhere-api-infra"}
+  ]
+}

deploy/ecs/task-definition-worker.staging.json

@@ -0,0 +1,93 @@
+{
+  "family": "knowhere-worker-staging",
+  "taskRoleArn": "${WORKER_TASK_ROLE_ARN}",
+  "executionRoleArn": "${EXECUTION_ROLE_ARN}",
+  "networkMode": "awsvpc",
+  "requiresCompatibilities": ["FARGATE"],
+  "cpu": "512",
+  "memory": "4096",
+  "ephemeralStorage": {"sizeInGiB": 20},
+  "runtimePlatform": {
+    "cpuArchitecture": "X86_64",
+    "operatingSystemFamily": "LINUX"
+  },
+  "containerDefinitions": [
+    {
+      "name": "worker",
+      "image": "${WORKER_IMAGE}",
+      "essential": true,
+      "stopTimeout": 120,
+      "environment": [
+        {"name": "ENVIRONMENT", "value": "staging"},
+        {"name": "APP_ENV", "value": "staging"},
+        {"name": "DB_SSL_MODE", "value": "require"},
+        {"name": "TMP_PATH", "value": "/tmp/knowhere"},
+        {"name": "S3_TYPE", "value": "s3"},
+        {"name": "S3_BUCKET_NAME", "value": "knowhere-storage-staging"},
+        {"name": "S3_RESULTS_BUCKET", "value": "knowhere-storage-staging"},
+        {"name": "S3_REGION", "value": "us-east-1"},
+        {"name": "S3_USE_SSL", "value": "true"},
+        {"name": "S3_ADDRESSING_STYLE", "value": "virtual"},
+        {"name": "REDIS_DATABASE", "value": "0"},
+        {"name": "REDIS_SSL", "value": "true"},
+        {"name": "AWS_REGION", "value": "us-east-1"},
+        {"name": "AWS_ACCOUNT_ID", "value": "107424103509"},
+        {"name": "RATE_LIMIT_ENABLED", "value": "false"},
+        {"name": "TELEMETRY_ENABLED", "value": "false"},
+        {"name": "INTERNAL_DASHBOARD_ENDPOINT", "value": "https://staging.knowhereto.ai"},
+        {"name": "API_WEBHOOK_ENDPOINT", "value": "https://api-staging.knowhereto.ai/v1/internal/s3-events"},
+        {"name": "SNS_TOPIC_ARN", "value": "arn:aws:sns:us-east-1:107424103509:knowhere-staging-s3-events"},
+        {"name": "QSTASH_CALLBACK_BASE_URL", "value": "https://api-staging.knowhereto.ai/api/v1"},
+        {"name": "HF_HOME", "value": "/mnt/models/huggingface"},
+        {"name": "TRANSFORMERS_CACHE", "value": "/mnt/models/huggingface"},
+        {"name": "BILLING_ENABLED", "value": "true"},
+        {"name": "SUMMARY_LLM_MAX_CONCURRENT", "value": "8"},
+        {"name": "HIERARCHY_LLM_MODEL", "value": "deepseek-chat"},
+        {"name": "NORMOL_MODEL", "value": "deepseek-v4-flash"},
+        {"name": "IMAGE_MODEL", "value": "qwen3.6-flash"},
+        {"name": "IMAGE_MODEL_MAX", "value": "qwen3.6-flash"}
+      ],
+      "secrets": [
+        {"name": "DATABASE_URL", "valueFrom": "${STAGING_SECRETS_ARN}:DATABASE_URL::"},
+        {"name": "REDIS_HOST", "valueFrom": "${STAGING_SECRETS_ARN}:REDIS_HOST::"},
+        {"name": "REDIS_PORT", "valueFrom": "${STAGING_SECRETS_ARN}:REDIS_PORT::"},
+        {"name": "REDIS_PASSWORD", "valueFrom": "${STAGING_SECRETS_ARN}:REDIS_PASSWORD::"},
+        {"name": "CELERY_REDIS_URL", "valueFrom": "${STAGING_SECRETS_ARN}:CELERY_REDIS_URL::"},
+        {"name": "CELERY_REDIS_PASSWORD", "valueFrom": "${STAGING_SECRETS_ARN}:CELERY_REDIS_PASSWORD::"},
+        {"name": "SECRET_KEY", "valueFrom": "${STAGING_SECRETS_ARN}:SECRET_KEY::"},
+        {"name": "WEBHOOK_MASTER_KEY", "valueFrom": "${STAGING_SECRETS_ARN}:WEBHOOK_MASTER_KEY::"},
+        {"name": "DS_KEY", "valueFrom": "${STAGING_SECRETS_ARN}:DS_KEY::"},
+        {"name": "ALI_API_KEYS", "valueFrom": "${STAGING_SECRETS_ARN}:ALI_API_KEYS::"},
+        {"name": "ARK_API_KEY", "valueFrom": "${STAGING_SECRETS_ARN}:ARK_API_KEY::"},
+        {"name": "GPT_API_KEY", "valueFrom": "${STAGING_SECRETS_ARN}:GPT_API_KEY::"},
+        {"name": "MINERU_API_KEYS", "valueFrom": "${STAGING_SECRETS_ARN}:MINERU_API_KEYS::"},
+        {"name": "ILOVEAPI_KEYS", "valueFrom": "${STAGING_SECRETS_ARN}:ILOVEAPI_KEYS::"},
+        {"name": "LOGFIRE_TOKEN", "valueFrom": "${STAGING_SECRETS_ARN}:LOGFIRE_TOKEN::"},
+        {"name": "QSTASH_TOKEN", "valueFrom": "${STAGING_SECRETS_ARN}:QSTASH_TOKEN::"},
+        {"name": "QSTASH_CURRENT_SIGNING_KEY", "valueFrom": "${STAGING_SECRETS_ARN}:QSTASH_CURRENT_SIGNING_KEY::"},
+        {"name": "QSTASH_NEXT_SIGNING_KEY", "valueFrom": "${STAGING_SECRETS_ARN}:QSTASH_NEXT_SIGNING_KEY::"}
+      ],
+      "healthCheck": {
+        "command": ["CMD-SHELL", "python -c \"from shared.services.worker_health import assert_worker_healthy; assert_worker_healthy()\""],
+        "interval": 30,
+        "timeout": 10,
+        "retries": 3,
+        "startPeriod": 60
+      },
+      "logConfiguration": {
+        "logDriver": "awslogs",
+        "options": {
+          "awslogs-group": "/ecs/knowhere-worker-staging",
+          "awslogs-region": "us-east-1",
+          "awslogs-stream-prefix": "worker"
+        }
+      }
+    }
+  ],
+  "tags": [
+    {"key": "Project", "value": "knowhere"},
+    {"key": "Environment", "value": "staging"},
+    {"key": "Service", "value": "worker"},
+    {"key": "ManagedBy", "value": "knowhere-api-infra"}
+  ]
+}

deploy/ecs/test_render_task_definitions.py

@@ -0,0 +1,67 @@
+"""Contracts for ECS task-definition rendering."""
+
+import json
+from pathlib import Path
+
+import pytest
+
+from render_task_definitions import (
+    render_template,
+    validate_rendered_definition,
+)
+
+
+TEMPLATE_DIRECTORY: Path = Path(__file__).parent
+RENDER_VARIABLES: dict[str, str] = {
+    "API_IMAGE": "api-image",
+    "WORKER_IMAGE": "worker-image",
+    "EXECUTION_ROLE_ARN": "execution-role",
+    "API_TASK_ROLE_ARN": "api-role",
+    "WORKER_TASK_ROLE_ARN": "worker-role",
+    "STAGING_SECRETS_ARN": "secrets-arn",
+}
+
+
+@pytest.mark.parametrize(
+    "template_name",
+    ["task-definition-api.staging.json", "task-definition-worker.staging.json"],
+)
+def test_staging_templates_render_without_long_lived_s3_keys(
+    tmp_path: Path,
+    template_name: str,
+) -> None:
+    """Both templates render and exclude explicit S3 credential variables."""
+    output_path: Path = tmp_path / template_name
+    render_template(
+        TEMPLATE_DIRECTORY / template_name,
+        output_path,
+        RENDER_VARIABLES,
+    )
+
+    rendered: dict[str, object] = json.loads(output_path.read_text(encoding="utf-8"))
+    validate_rendered_definition(rendered)
+    assert rendered["requiresCompatibilities"] == ["FARGATE"]
+    assert "${" not in output_path.read_text(encoding="utf-8")
+
+
+def test_renderer_rejects_forbidden_s3_credential_variable() -> None:
+    """Task definitions must never inject long-lived S3 credentials."""
+    definition: dict[str, object] = {
+        "containerDefinitions": [
+            {"environment": [{"name": "S3_ACCESS_KEY_ID", "value": "bad"}]}
+        ]
+    }
+
+    with pytest.raises(ValueError, match="forbidden S3 credentials"):
+        validate_rendered_definition(definition)
+
+
+def test_renderer_rejects_unresolved_placeholders() -> None:
+    """Deployment cannot proceed with missing substitution values."""
+    definition: dict[str, object] = {
+        "containerDefinitions": [{"environment": []}],
+        "image": "${API_IMAGE}",
+    }
+
+    with pytest.raises(ValueError, match="Unresolved task-definition placeholders"):
+        validate_rendered_definition(definition)