Configure graceful worker shutdown for Fargate

apps/api/pyproject.toml

@@ -11,7 +11,7 @@ dependencies = [
     "sqlalchemy==2.0.42",
     "alembic==1.13.1",
     "redis==5.3.1",
-    "celery==5.4.0",
+    "celery==5.5.3",
     "stripe==13.0.1",
     "pydantic==2.13.4",
     "PyJWT==2.12.0",

apps/worker/requirements.txt

@@ -50,7 +50,7 @@ beautifulsoup4==4.13.4
     #   knowhere-worker-app
     #   markdownify
     #   markitdown
-billiard==4.2.0
+billiard==4.2.1
     # via
     #   celery
     #   knowhere-shared
@@ -63,7 +63,7 @@ botocore==1.38.46
     #   boto3
     #   knowhere-shared
     #   s3transfer
-celery==5.4.0
+celery==5.5.3
     # via
     #   celery-redbeat
     #   knowhere-shared
@@ -185,7 +185,7 @@ jmespath==0.10.0
     #   aliyun-python-sdk-core
     #   boto3
     #   botocore
-kombu==5.4.0
+kombu==5.5.4
     # via
     #   celery
     #   knowhere-shared
@@ -207,7 +207,9 @@ mako==1.3.11
 markdown-it-py==4.0.0
     # via rich
 markdownify==1.2.2
-    # via markitdown
+    # via
+    #   knowhere-worker-app
+    #   markitdown
 markitdown==0.1.2
     # via knowhere-worker-app
 markupsafe==3.0.3
@@ -301,6 +303,7 @@ oss2==2.19.1
     # via knowhere-worker-app
 packaging==26.1
     # via
+    #   kombu
     #   onnxruntime
     #   opentelemetry-instrumentation
     #   opentelemetry-instrumentation-sqlalchemy
@@ -379,7 +382,9 @@ pymupdf-layout==1.27.2
 pymupdf4llm==1.27.2.1
     # via knowhere-worker-app
 pypdf==6.10.2
-    # via knowhere-worker-app
+    # via
+    #   knowhere-shared
+    #   knowhere-worker-app
 pytest==9.0.3
     # via
     #   knowhere-shared
@@ -503,7 +508,7 @@ typing-inspection==0.4.2
     #   pydantic-settings
 tzdata==2026.1
     # via
-    #   celery
+    #   kombu
     #   pandas
 urllib3==2.6.3
     # via

apps/worker/tests/contract/test_worker_shutdown_contract.py

@@ -0,0 +1,35 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+
+def test_should_preserve_fargate_worker_soft_shutdown_contract(
+    worker_contract_environment: None,
+) -> None:
+    from shared.core.celery_app import celery_app
+
+    repository_root: Path = Path(__file__).resolve().parents[4]
+    task_definition_path: Path = (
+        repository_root / "deploy/ecs/task-definition-worker.staging.json"
+    )
+    task_definition: dict[str, object] = json.loads(
+        task_definition_path.read_text(encoding="utf-8")
+    )
+    container_definitions: list[dict[str, object]] = task_definition[
+        "containerDefinitions"
+    ]
+    worker_container: dict[str, object] = next(
+        container
+        for container in container_definitions
+        if container.get("name") == "worker"
+    )
+    environment: list[dict[str, str]] = worker_container["environment"]
+    environment_values: dict[str, str] = {
+        item["name"]: item["value"] for item in environment
+    }
+
+    assert celery_app.conf.worker_soft_shutdown_timeout == 90
+    assert celery_app.conf.worker_enable_soft_shutdown_on_idle is True
+    assert environment_values["REMAP_SIGTERM"] == "SIGQUIT"
+    assert worker_container["stopTimeout"] == 120

deploy/ecs/task-definition-worker.staging.json

@@ -20,6 +20,7 @@
       "environment": [
         {"name": "ENVIRONMENT", "value": "staging"},
         {"name": "APP_ENV", "value": "staging"},
+        {"name": "REMAP_SIGTERM", "value": "SIGQUIT"},
         {"name": "DB_SSL_MODE", "value": "require"},
         {"name": "TMP_PATH", "value": "/tmp/knowhere"},
         {"name": "S3_TYPE", "value": "s3"},

packages/shared-python/pyproject.toml

@@ -21,9 +21,9 @@ dependencies = [
     "redis==5.3.1",
 
     # Celery task queue
-    "celery==5.4.0",
-    "kombu==5.4.0",
-    "billiard==4.2.0",
+    "celery==5.5.3",
+    "kombu==5.5.4",
+    "billiard==4.2.1",
     "vine==5.1.0",
     "celery-redbeat==2.2.0",
 

packages/shared-python/shared/core/celery_app.py

@@ -60,6 +60,8 @@ def get_unique_node_name() -> str:
     worker_prefetch_multiplier=1,
     task_acks_late=True,
     worker_disable_rate_limits=True,
+    worker_soft_shutdown_timeout=90,
+    worker_enable_soft_shutdown_on_idle=True,
     # Redis serverless does not support the pidbox PSUBSCRIBE control channel.
     worker_enable_remote_control=False,
     task_reject_on_worker_lost=True,