From 5c55c6f7e9523872ebe48a6b8f081938d422499c Mon Sep 17 00:00:00 2001 From: pranayyelugam Date: Tue, 21 Jul 2026 21:45:27 -0700 Subject: [PATCH 1/3] Helm: Scope KEDA_DB_CONN to worker and triggerer pods --- chart/templates/_helpers.yaml | 3 +- .../triggerer/triggerer-deployment.yaml | 4 +- .../templates/workers/worker-deployment.yaml | 8 ++-- chart/tests/helm_tests/other/test_keda.py | 46 +++++++++++++++++++ 4 files changed, 54 insertions(+), 7 deletions(-) diff --git a/chart/templates/_helpers.yaml b/chart/templates/_helpers.yaml index 2fefc497905d1..deead035d5b4c 100644 --- a/chart/templates/_helpers.yaml +++ b/chart/templates/_helpers.yaml @@ -87,7 +87,8 @@ If release name contains chart name it will be used as a full name. {{- $triggererKedaEnabled := and .Values.triggerer.enabled .Values.triggerer.keda.enabled }} {{- $workersKedaNeedsDbConn := and .Values.workers.celery.keda.enabled (or (eq .Values.data.metadataConnection.protocol "mysql") (and .Values.pgbouncer.enabled (not .Values.workers.celery.keda.usePgbouncer))) }} {{- $triggererKedaNeedsDbConn := and $triggererKedaEnabled (or (eq .Values.data.metadataConnection.protocol "mysql") (and .Values.pgbouncer.enabled (not .Values.triggerer.keda.usePgbouncer))) }} - {{- if or $workersKedaNeedsDbConn $triggererKedaNeedsDbConn }} + {{- $component := .Component | default "" }} + {{- if or (and (eq $component "worker") $workersKedaNeedsDbConn) (and (eq $component "triggerer") $triggererKedaNeedsDbConn) }} - name: KEDA_DB_CONN valueFrom: secretKeyRef: diff --git a/chart/templates/triggerer/triggerer-deployment.yaml b/chart/templates/triggerer/triggerer-deployment.yaml index 591153d3e054c..aee3967d563bb 100644 --- a/chart/templates/triggerer/triggerer-deployment.yaml +++ b/chart/templates/triggerer/triggerer-deployment.yaml @@ -155,7 +155,7 @@ spec: envFrom: {{- include "custom_airflow_environment_from" . | default "\n []" | indent 10 }} env: {{- include "custom_airflow_environment" . | indent 10 }} - {{- include "standard_airflow_environment" (merge (dict "IncludeJwtSecret" false) .) | indent 10 }} + {{- include "standard_airflow_environment" (merge (dict "IncludeJwtSecret" false "Component" "triggerer") .) | indent 10 }} {{- if .Values.triggerer.waitForMigrations.env }} {{- tpl (toYaml .Values.triggerer.waitForMigrations.env) $ | nindent 12 }} {{- end }} @@ -200,7 +200,7 @@ spec: envFrom: {{- include "custom_airflow_environment_from" . | default "\n []" | indent 10 }} env: {{- include "custom_airflow_environment" . | indent 10 }} - {{- include "standard_airflow_environment" (merge (dict "IncludeJwtSecret" false) .) | indent 10 }} + {{- include "standard_airflow_environment" (merge (dict "IncludeJwtSecret" false "Component" "triggerer") .) | indent 10 }} {{- include "container_extra_envs" (list . .Values.triggerer.env) | nindent 10 }} livenessProbe: initialDelaySeconds: {{ .Values.triggerer.livenessProbe.initialDelaySeconds }} diff --git a/chart/templates/workers/worker-deployment.yaml b/chart/templates/workers/worker-deployment.yaml index d45a4b15485f3..17e03df69d166 100644 --- a/chart/templates/workers/worker-deployment.yaml +++ b/chart/templates/workers/worker-deployment.yaml @@ -217,7 +217,7 @@ spec: - name: KRB5CCNAME value: {{ include "kerberos_ccache_path" . | quote }} {{- include "custom_airflow_environment" . | indent 10 }} - {{- include "standard_airflow_environment" (merge (dict "IncludeJwtSecret" false) .) | indent 10 }} + {{- include "standard_airflow_environment" (merge (dict "IncludeJwtSecret" false "Component" "worker") .) | indent 10 }} {{- end }} {{- if .Values.workers.celery.waitForMigrations.enabled }} - name: wait-for-airflow-migrations @@ -242,7 +242,7 @@ spec: envFrom: {{- include "custom_airflow_environment_from" . | default "\n []" | indent 10 }} env: {{- include "custom_airflow_environment" . | indent 10 }} - {{- include "standard_airflow_environment" (merge (dict "IncludeJwtSecret" false) .) | indent 10 }} + {{- include "standard_airflow_environment" (merge (dict "IncludeJwtSecret" false "Component" "worker") .) | indent 10 }} {{- if .Values.workers.celery.waitForMigrations.env }} {{- tpl (toYaml .Values.workers.celery.waitForMigrations.env) $ | nindent 12 }} {{- end }} @@ -325,7 +325,7 @@ spec: - name: DUMB_INIT_SETSID value: "0" {{- include "custom_airflow_environment" . | indent 10 }} - {{- include "standard_airflow_environment" (merge (dict "IncludeJwtSecret" false) .) | indent 10 }} + {{- include "standard_airflow_environment" (merge (dict "IncludeJwtSecret" false "Component" "worker") .) | indent 10 }} {{- include "container_extra_envs" (list . .Values.workers.celery.env) | indent 10 }} {{- if .Values.workers.celery.kerberosSidecar.enabled }} - name: KRB5_CONFIG @@ -431,7 +431,7 @@ spec: - name: KRB5CCNAME value: {{ include "kerberos_ccache_path" . | quote }} {{- include "custom_airflow_environment" . | indent 10 }} - {{- include "standard_airflow_environment" (merge (dict "IncludeJwtSecret" false) .) | indent 10 }} + {{- include "standard_airflow_environment" (merge (dict "IncludeJwtSecret" false "Component" "worker") .) | indent 10 }} {{- end }} {{- if .Values.workers.celery.extraContainers }} {{- tpl (toYaml .Values.workers.celery.extraContainers) . | nindent 8 }} diff --git a/chart/tests/helm_tests/other/test_keda.py b/chart/tests/helm_tests/other/test_keda.py index 41b07021e0a60..25d61fb9b5fb6 100644 --- a/chart/tests/helm_tests/other/test_keda.py +++ b/chart/tests/helm_tests/other/test_keda.py @@ -406,3 +406,49 @@ def test_overwrite_keda_max_replica_count(self): ) assert jmespath.search("spec.maxReplicaCount", docs[0]) == 5 + + @staticmethod + def _env_names(doc): + return jmespath.search("spec.template.spec.containers[].env[].name", doc) or [] + + def test_worker_keda_db_conn_not_leaked_to_other_components(self): + """Worker KEDA_DB_CONN should render on the worker only, not on unrelated components.""" + docs = render_chart( + values={ + "workers": {"celery": {"keda": {"enabled": True, "usePgbouncer": False}}}, + "executor": "CeleryExecutor", + "pgbouncer": {"enabled": True}, + }, + show_only=[ + "templates/workers/worker-deployment.yaml", + "templates/scheduler/scheduler-deployment.yaml", + "templates/api-server/api-server-deployment.yaml", + "templates/dag-processor/dag-processor-deployment.yaml", + "templates/triggerer/triggerer-deployment.yaml", + ], + ) + worker, scheduler, api_server, dag_processor, triggerer = docs + assert "KEDA_DB_CONN" in self._env_names(worker) + assert "KEDA_DB_CONN" not in self._env_names(scheduler) + assert "KEDA_DB_CONN" not in self._env_names(api_server) + assert "KEDA_DB_CONN" not in self._env_names(dag_processor) + assert "KEDA_DB_CONN" not in self._env_names(triggerer) + + def test_triggerer_keda_db_conn_not_leaked_to_other_components(self): + """Triggerer KEDA_DB_CONN should render on the triggerer only, not on the worker or others.""" + docs = render_chart( + values={ + "triggerer": {"keda": {"enabled": True, "usePgbouncer": False}}, + "executor": "CeleryExecutor", + "pgbouncer": {"enabled": True}, + }, + show_only=[ + "templates/triggerer/triggerer-deployment.yaml", + "templates/workers/worker-deployment.yaml", + "templates/scheduler/scheduler-deployment.yaml", + ], + ) + triggerer, worker, scheduler = docs + assert "KEDA_DB_CONN" in self._env_names(triggerer) + assert "KEDA_DB_CONN" not in self._env_names(worker) + assert "KEDA_DB_CONN" not in self._env_names(scheduler) From 2d8bb090fcc5816b929a522f5c95e05a44e0742f Mon Sep 17 00:00:00 2001 From: pranayyelugam Date: Wed, 22 Jul 2026 16:20:08 -0700 Subject: [PATCH 2/3] Helm: Extract KEDA database connection into a dedicated env helper --- chart/templates/_helpers.yaml | 26 +++++++++++-------- .../triggerer/triggerer-deployment.yaml | 5 ++-- .../templates/workers/worker-deployment.yaml | 9 ++++--- 3 files changed, 23 insertions(+), 17 deletions(-) diff --git a/chart/templates/_helpers.yaml b/chart/templates/_helpers.yaml index deead035d5b4c..1011f54cd924f 100644 --- a/chart/templates/_helpers.yaml +++ b/chart/templates/_helpers.yaml @@ -84,17 +84,6 @@ If release name contains chart name it will be used as a full name. name: {{ template "airflow_metadata_secret" . }} key: connection {{- end }} - {{- $triggererKedaEnabled := and .Values.triggerer.enabled .Values.triggerer.keda.enabled }} - {{- $workersKedaNeedsDbConn := and .Values.workers.celery.keda.enabled (or (eq .Values.data.metadataConnection.protocol "mysql") (and .Values.pgbouncer.enabled (not .Values.workers.celery.keda.usePgbouncer))) }} - {{- $triggererKedaNeedsDbConn := and $triggererKedaEnabled (or (eq .Values.data.metadataConnection.protocol "mysql") (and .Values.pgbouncer.enabled (not .Values.triggerer.keda.usePgbouncer))) }} - {{- $component := .Component | default "" }} - {{- if or (and (eq $component "worker") $workersKedaNeedsDbConn) (and (eq $component "triggerer") $triggererKedaNeedsDbConn) }} - - name: KEDA_DB_CONN - valueFrom: - secretKeyRef: - name: {{ template "airflow_metadata_secret" . }} - key: kedaConnection - {{- end }} {{- if .Values.enableBuiltInSecretEnvVars.AIRFLOW__API__SECRET_KEY }} - name: AIRFLOW__API__SECRET_KEY valueFrom: @@ -159,6 +148,21 @@ If release name contains chart name it will be used as a full name. {{- end }} {{- end }} +{{/* KEDA scaler database connection for the worker and triggerer autoscalers */}} +{{- define "keda_airflow_environment" }} + {{- $triggererKedaEnabled := and .Values.triggerer.enabled .Values.triggerer.keda.enabled }} + {{- $workersKedaNeedsDbConn := and .Values.workers.celery.keda.enabled (or (eq .Values.data.metadataConnection.protocol "mysql") (and .Values.pgbouncer.enabled (not .Values.workers.celery.keda.usePgbouncer))) }} + {{- $triggererKedaNeedsDbConn := and $triggererKedaEnabled (or (eq .Values.data.metadataConnection.protocol "mysql") (and .Values.pgbouncer.enabled (not .Values.triggerer.keda.usePgbouncer))) }} + {{- $component := .Component | default "" }} + {{- if or (and (eq $component "worker") $workersKedaNeedsDbConn) (and (eq $component "triggerer") $triggererKedaNeedsDbConn) }} + - name: KEDA_DB_CONN + valueFrom: + secretKeyRef: + name: {{ template "airflow_metadata_secret" . }} + key: kedaConnection + {{- end }} +{{- end }} + {{/* User defined Airflow environment variables */}} {{- define "custom_airflow_environment" }} # Dynamically created environment variables diff --git a/chart/templates/triggerer/triggerer-deployment.yaml b/chart/templates/triggerer/triggerer-deployment.yaml index aee3967d563bb..3cb248728ad49 100644 --- a/chart/templates/triggerer/triggerer-deployment.yaml +++ b/chart/templates/triggerer/triggerer-deployment.yaml @@ -155,7 +155,7 @@ spec: envFrom: {{- include "custom_airflow_environment_from" . | default "\n []" | indent 10 }} env: {{- include "custom_airflow_environment" . | indent 10 }} - {{- include "standard_airflow_environment" (merge (dict "IncludeJwtSecret" false "Component" "triggerer") .) | indent 10 }} + {{- include "standard_airflow_environment" (merge (dict "IncludeJwtSecret" false) .) | indent 10 }} {{- if .Values.triggerer.waitForMigrations.env }} {{- tpl (toYaml .Values.triggerer.waitForMigrations.env) $ | nindent 12 }} {{- end }} @@ -200,7 +200,8 @@ spec: envFrom: {{- include "custom_airflow_environment_from" . | default "\n []" | indent 10 }} env: {{- include "custom_airflow_environment" . | indent 10 }} - {{- include "standard_airflow_environment" (merge (dict "IncludeJwtSecret" false "Component" "triggerer") .) | indent 10 }} + {{- include "standard_airflow_environment" (merge (dict "IncludeJwtSecret" false) .) | indent 10 }} + {{- include "keda_airflow_environment" (merge (dict "Component" "triggerer") .) | indent 10 }} {{- include "container_extra_envs" (list . .Values.triggerer.env) | nindent 10 }} livenessProbe: initialDelaySeconds: {{ .Values.triggerer.livenessProbe.initialDelaySeconds }} diff --git a/chart/templates/workers/worker-deployment.yaml b/chart/templates/workers/worker-deployment.yaml index 17e03df69d166..589ec0372b292 100644 --- a/chart/templates/workers/worker-deployment.yaml +++ b/chart/templates/workers/worker-deployment.yaml @@ -217,7 +217,7 @@ spec: - name: KRB5CCNAME value: {{ include "kerberos_ccache_path" . | quote }} {{- include "custom_airflow_environment" . | indent 10 }} - {{- include "standard_airflow_environment" (merge (dict "IncludeJwtSecret" false "Component" "worker") .) | indent 10 }} + {{- include "standard_airflow_environment" (merge (dict "IncludeJwtSecret" false) .) | indent 10 }} {{- end }} {{- if .Values.workers.celery.waitForMigrations.enabled }} - name: wait-for-airflow-migrations @@ -242,7 +242,7 @@ spec: envFrom: {{- include "custom_airflow_environment_from" . | default "\n []" | indent 10 }} env: {{- include "custom_airflow_environment" . | indent 10 }} - {{- include "standard_airflow_environment" (merge (dict "IncludeJwtSecret" false "Component" "worker") .) | indent 10 }} + {{- include "standard_airflow_environment" (merge (dict "IncludeJwtSecret" false) .) | indent 10 }} {{- if .Values.workers.celery.waitForMigrations.env }} {{- tpl (toYaml .Values.workers.celery.waitForMigrations.env) $ | nindent 12 }} {{- end }} @@ -325,7 +325,8 @@ spec: - name: DUMB_INIT_SETSID value: "0" {{- include "custom_airflow_environment" . | indent 10 }} - {{- include "standard_airflow_environment" (merge (dict "IncludeJwtSecret" false "Component" "worker") .) | indent 10 }} + {{- include "standard_airflow_environment" (merge (dict "IncludeJwtSecret" false) .) | indent 10 }} + {{- include "keda_airflow_environment" (merge (dict "Component" "worker") .) | indent 10 }} {{- include "container_extra_envs" (list . .Values.workers.celery.env) | indent 10 }} {{- if .Values.workers.celery.kerberosSidecar.enabled }} - name: KRB5_CONFIG @@ -431,7 +432,7 @@ spec: - name: KRB5CCNAME value: {{ include "kerberos_ccache_path" . | quote }} {{- include "custom_airflow_environment" . | indent 10 }} - {{- include "standard_airflow_environment" (merge (dict "IncludeJwtSecret" false "Component" "worker") .) | indent 10 }} + {{- include "standard_airflow_environment" (merge (dict "IncludeJwtSecret" false) .) | indent 10 }} {{- end }} {{- if .Values.workers.celery.extraContainers }} {{- tpl (toYaml .Values.workers.celery.extraContainers) . | nindent 8 }} From 575c925a0a3a1c2322e4b200fbd0e58dd82bd81b Mon Sep 17 00:00:00 2001 From: pranayyelugam Date: Fri, 24 Jul 2026 10:44:48 -0700 Subject: [PATCH 3/3] Helm: Simplify KEDA env helper by gating on keda at the call sites --- chart/templates/_helpers.yaml | 6 +----- chart/templates/triggerer/triggerer-deployment.yaml | 4 +++- chart/templates/workers/worker-deployment.yaml | 4 +++- chart/tests/helm_tests/other/test_keda.py | 2 -- 4 files changed, 7 insertions(+), 9 deletions(-) diff --git a/chart/templates/_helpers.yaml b/chart/templates/_helpers.yaml index 1011f54cd924f..bbc24fe53b5f3 100644 --- a/chart/templates/_helpers.yaml +++ b/chart/templates/_helpers.yaml @@ -150,11 +150,7 @@ If release name contains chart name it will be used as a full name. {{/* KEDA scaler database connection for the worker and triggerer autoscalers */}} {{- define "keda_airflow_environment" }} - {{- $triggererKedaEnabled := and .Values.triggerer.enabled .Values.triggerer.keda.enabled }} - {{- $workersKedaNeedsDbConn := and .Values.workers.celery.keda.enabled (or (eq .Values.data.metadataConnection.protocol "mysql") (and .Values.pgbouncer.enabled (not .Values.workers.celery.keda.usePgbouncer))) }} - {{- $triggererKedaNeedsDbConn := and $triggererKedaEnabled (or (eq .Values.data.metadataConnection.protocol "mysql") (and .Values.pgbouncer.enabled (not .Values.triggerer.keda.usePgbouncer))) }} - {{- $component := .Component | default "" }} - {{- if or (and (eq $component "worker") $workersKedaNeedsDbConn) (and (eq $component "triggerer") $triggererKedaNeedsDbConn) }} + {{- if or (eq .Values.data.metadataConnection.protocol "mysql") (and .Values.pgbouncer.enabled (not .UsePgbouncer)) }} - name: KEDA_DB_CONN valueFrom: secretKeyRef: diff --git a/chart/templates/triggerer/triggerer-deployment.yaml b/chart/templates/triggerer/triggerer-deployment.yaml index 3cb248728ad49..1c773d349ea9e 100644 --- a/chart/templates/triggerer/triggerer-deployment.yaml +++ b/chart/templates/triggerer/triggerer-deployment.yaml @@ -201,7 +201,9 @@ spec: env: {{- include "custom_airflow_environment" . | indent 10 }} {{- include "standard_airflow_environment" (merge (dict "IncludeJwtSecret" false) .) | indent 10 }} - {{- include "keda_airflow_environment" (merge (dict "Component" "triggerer") .) | indent 10 }} + {{- if $keda }} + {{- include "keda_airflow_environment" (merge (dict "UsePgbouncer" .Values.triggerer.keda.usePgbouncer) .) | indent 10 }} + {{- end }} {{- include "container_extra_envs" (list . .Values.triggerer.env) | nindent 10 }} livenessProbe: initialDelaySeconds: {{ .Values.triggerer.livenessProbe.initialDelaySeconds }} diff --git a/chart/templates/workers/worker-deployment.yaml b/chart/templates/workers/worker-deployment.yaml index 589ec0372b292..294b0d4101020 100644 --- a/chart/templates/workers/worker-deployment.yaml +++ b/chart/templates/workers/worker-deployment.yaml @@ -326,7 +326,9 @@ spec: value: "0" {{- include "custom_airflow_environment" . | indent 10 }} {{- include "standard_airflow_environment" (merge (dict "IncludeJwtSecret" false) .) | indent 10 }} - {{- include "keda_airflow_environment" (merge (dict "Component" "worker") .) | indent 10 }} + {{- if $keda }} + {{- include "keda_airflow_environment" (merge (dict "UsePgbouncer" .Values.workers.celery.keda.usePgbouncer) .) | indent 10 }} + {{- end }} {{- include "container_extra_envs" (list . .Values.workers.celery.env) | indent 10 }} {{- if .Values.workers.celery.kerberosSidecar.enabled }} - name: KRB5_CONFIG diff --git a/chart/tests/helm_tests/other/test_keda.py b/chart/tests/helm_tests/other/test_keda.py index 25d61fb9b5fb6..ff4c3cf49409c 100644 --- a/chart/tests/helm_tests/other/test_keda.py +++ b/chart/tests/helm_tests/other/test_keda.py @@ -412,7 +412,6 @@ def _env_names(doc): return jmespath.search("spec.template.spec.containers[].env[].name", doc) or [] def test_worker_keda_db_conn_not_leaked_to_other_components(self): - """Worker KEDA_DB_CONN should render on the worker only, not on unrelated components.""" docs = render_chart( values={ "workers": {"celery": {"keda": {"enabled": True, "usePgbouncer": False}}}, @@ -435,7 +434,6 @@ def test_worker_keda_db_conn_not_leaked_to_other_components(self): assert "KEDA_DB_CONN" not in self._env_names(triggerer) def test_triggerer_keda_db_conn_not_leaked_to_other_components(self): - """Triggerer KEDA_DB_CONN should render on the triggerer only, not on the worker or others.""" docs = render_chart( values={ "triggerer": {"keda": {"enabled": True, "usePgbouncer": False}},