Handle missed deadline alerts when DagRuns fail

airflow-core/docs/howto/deadline-alerts.rst

@@ -44,6 +44,7 @@ Creating a Deadline Alert requires three mandatory parameters:
 * Reference: When to start counting from
 * Interval: How far before or after the reference point to trigger the alert (either a timedelta or a dynamic interval such as VariableInterval)
 * Callback: A Callback object which contains a path to a callable and optional kwargs to pass to it if the deadline is exceeded
+* Fire on failure: Optional. If set to ``True``, a pending Dag run deadline alert fires immediately when the scheduler marks the Dag run failed. Manual UI/API mark-failed paths are unchanged. The default is ``False``.
 
 Here is how Deadlines are calculated:
 

airflow-core/docs/migrations-ref.rst

@@ -39,7 +39,9 @@ Here's the list of all the Database Migrations that are executed via when you ru
 +-------------------------+------------------+-------------------+--------------------------------------------------------------+
 | Revision ID             | Revises ID       | Airflow Version   | Description                                                  |
 +=========================+==================+===================+==============================================================+
-| ``c4e7a1f9b2d0`` (head) | ``436dc127462c`` | ``3.4.0``         | Add index on asset.uri.                                      |
+| ``b4f8c2a1d9e0`` (head) | ``c4e7a1f9b2d0`` | ``3.4.0``         | Add fire_on_failure to deadline_alert.                       |
++-------------------------+------------------+-------------------+--------------------------------------------------------------+
+| ``c4e7a1f9b2d0``        | ``436dc127462c`` | ``3.4.0``         | Add index on asset.uri.                                      |
 +-------------------------+------------------+-------------------+--------------------------------------------------------------+
 | ``436dc127462c``        | ``5a5d3253e946`` | ``3.4.0``         | Drop span_status column.                                     |
 +-------------------------+------------------+-------------------+--------------------------------------------------------------+

airflow-core/newsfragments/68961.feature.rst

@@ -0,0 +1 @@
+Add ``DeadlineAlert.fire_on_failure`` to optionally fire pending DagRun deadline alerts immediately when the scheduler marks a DagRun failed.

airflow-core/src/airflow/api_fastapi/core_api/datamodels/ui/deadline.py

@@ -53,6 +53,7 @@ class DeadlineAlertResponse(BaseModel):
     name: str | None = None
     reference_type: str = Field(validation_alias=AliasPath("reference", "reference_type"))
     interval: float = Field(description="Interval in seconds between deadline evaluations.")
+    fire_on_failure: bool
     created_at: datetime
 
 

airflow-core/src/airflow/api_fastapi/core_api/openapi/_private_ui.yaml

@@ -2851,6 +2851,9 @@ components:
           type: number
           title: Interval
           description: Interval in seconds between deadline evaluations.
+        fire_on_failure:
+          type: boolean
+          title: Fire On Failure
         created_at:
           type: string
           format: date-time
@@ -2860,6 +2863,7 @@ components:
       - id
       - reference_type
       - interval
+      - fire_on_failure
       - created_at
       title: DeadlineAlertResponse
       description: DeadlineAlert serializer for responses.

airflow-core/src/airflow/migrations/versions/0127_3_4_0_add_fire_on_failure_to_deadline_alert.py

@@ -0,0 +1,48 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""
+Add fire_on_failure to deadline_alert.
+
+Revision ID: b4f8c2a1d9e0
+Revises: c4e7a1f9b2d0
+Create Date: 2026-07-01 12:00:00.000000
+"""
+
+from __future__ import annotations
+
+import sqlalchemy as sa
+from alembic import op
+
+revision = "b4f8c2a1d9e0"
+down_revision = "c4e7a1f9b2d0"
+branch_labels = None
+depends_on = None
+airflow_version = "3.4.0"
+
+
+def upgrade():
+    """Add fire_on_failure column to deadline_alert."""
+    with op.batch_alter_table("deadline_alert", schema=None) as batch_op:
+        batch_op.add_column(sa.Column("fire_on_failure", sa.Boolean(), nullable=False, server_default="0"))
+
+
+def downgrade():
+    """Remove fire_on_failure column from deadline_alert."""
+    with op.batch_alter_table("deadline_alert", schema=None) as batch_op:
+        batch_op.drop_column("fire_on_failure")

airflow-core/src/airflow/models/dagrun.py

@@ -53,11 +53,20 @@
     update,
 )
 from sqlalchemy.dialects import postgresql
-from sqlalchemy.exc import IntegrityError
+from sqlalchemy.exc import DBAPIError, IntegrityError
 from sqlalchemy.ext.associationproxy import association_proxy
 from sqlalchemy.ext.hybrid import hybrid_property
 from sqlalchemy.ext.mutable import MutableDict
-from sqlalchemy.orm import Mapped, declared_attr, joinedload, mapped_column, relationship, synonym, validates
+from sqlalchemy.orm import (
+    Mapped,
+    declared_attr,
+    joinedload,
+    mapped_column,
+    relationship,
+    selectinload,
+    synonym,
+    validates,
+)
 from sqlalchemy.orm.exc import StaleDataError
 from sqlalchemy.sql.expression import false, select
 from sqlalchemy.sql.functions import coalesce
@@ -1160,6 +1169,42 @@ def _emit_dagrun_span(self, state: DagRunState):
             span.set_status(status_code)
             span.end()
 
+    def _handle_missed_deadlines(self, *, session: Session) -> None:
+        """Handle pending deadlines that opt in to firing when their DagRun fails."""
+        deadline_query = (
+            select(Deadline)
+            .join(DeadlineAlertModel, Deadline.deadline_alert_id == DeadlineAlertModel.id)
+            .where(Deadline.dagrun_id == self.id)
+            .where(~Deadline.missed)
+            .where(DeadlineAlertModel.fire_on_failure.is_(True))
+            .options(
+                selectinload(Deadline.callback),
+                selectinload(Deadline.dagrun),
+                selectinload(Deadline.deadline_alert),
+            )
+        )
+        for deadline in session.scalars(
+            with_row_locks(
+                deadline_query,
+                of=Deadline,
+                session=session,
+                skip_locked=True,
+                key_share=False,
+            )
+        ):
+            deadline_id = deadline.id
+            try:
+                deadline.handle_miss(session)
+            except DBAPIError:
+                raise
+            except Exception:
+                self.log.warning(
+                    "Failed to handle missed deadline %s for %s",
+                    deadline_id,
+                    self,
+                    exc_info=True,
+                )
+
     @provide_session
     def update_state(
         self, *, session: Session = NEW_SESSION, execute_callbacks: bool = True
@@ -1260,6 +1305,9 @@ def recalculate(self) -> _UnfinishedStates:
                 )
                 self._check_last_n_dagruns_failed(dag.dag_id, dag.max_consecutive_failed_dag_runs, session)
 
+            if dag.deadline:
+                self._handle_missed_deadlines(session=session)
+
         # if all leaves succeeded and no unfinished tasks, the run succeeded
         elif not unfinished.tis and all(x.state in State.success_states for x in tis_for_dagrun_state):
             self.log.info("Marking run %s successful", self)
@@ -1317,6 +1365,9 @@ def recalculate(self) -> _UnfinishedStates:
                     execute=execute_callbacks,
                 )
 
+            if dag.deadline:
+                self._handle_missed_deadlines(session=session)
+
         # finally, if the leaves aren't done, the dag is still running
         else:
             self.set_state(DagRunState.RUNNING)

airflow-core/src/airflow/models/deadline_alert.py

@@ -21,7 +21,7 @@
 from uuid import UUID
 
 import uuid6
-from sqlalchemy import JSON, ForeignKey, String, Text, Uuid, select
+from sqlalchemy import JSON, Boolean, ForeignKey, String, Text, Uuid, select
 from sqlalchemy.exc import NoResultFound
 from sqlalchemy.orm import Mapped, mapped_column
 
@@ -52,6 +52,7 @@ class DeadlineAlert(Base):
     reference: Mapped[dict] = mapped_column(JSON, nullable=False)
     interval: Mapped[dict] = mapped_column(JSON, nullable=False)
     callback_def: Mapped[dict] = mapped_column(JSON, nullable=False)
+    fire_on_failure: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, server_default="0")
 
     def __repr__(self):
 
@@ -79,6 +80,7 @@ def __repr__(self):
             f"name={self.name or 'Unnamed'}, "
             f"reference={self.reference}, "
             f"interval={interval_display}, "
+            f"fire_on_failure={self.fire_on_failure}, "
             f"callback={self.callback_def}"
         )
 
@@ -90,6 +92,7 @@ def matches_definition(self, other: DeadlineAlert) -> bool:
             self.reference == other.reference
             and self.interval == other.interval
             and self.callback_def == other.callback_def
+            and self.fire_on_failure == other.fire_on_failure
         )
 
     @property

airflow-core/src/airflow/models/serialized_dag.py

@@ -459,6 +459,7 @@ def _definitions_match(deadline_data: dict, existing: DeadlineAlertModel) -> boo
                 deadline_data[DeadlineAlertFields.REFERENCE] == existing.reference
                 and deadline_data[DeadlineAlertFields.INTERVAL] == existing.interval
                 and deadline_data[DeadlineAlertFields.CALLBACK] == existing.callback_def
+                and deadline_data.get(DeadlineAlertFields.FIRE_ON_FAILURE, False) == existing.fire_on_failure
             )
 
         if len(existing_deadline_uuids) != len(new_deadline_data):
@@ -525,6 +526,7 @@ def _create_deadline_alert_records(
                 reference=deadline_data[DeadlineAlertFields.REFERENCE],
                 interval=deadline_data[DeadlineAlertFields.INTERVAL],
                 callback_def=deadline_data[DeadlineAlertFields.CALLBACK],
+                fire_on_failure=deadline_data.get(DeadlineAlertFields.FIRE_ON_FAILURE, False),
             )
             serialized_dag.deadline_alerts.append(alert)
 

airflow-core/src/airflow/serialization/decoders.py

@@ -196,6 +196,7 @@ def decode_deadline_alert(encoded_data: dict):
         interval=interval,
         callback=deserialize(data[DeadlineAlertFields.CALLBACK]),
         name=data.get(DeadlineAlertFields.NAME),
+        fire_on_failure=data.get(DeadlineAlertFields.FIRE_ON_FAILURE, False),
     )
 
 

airflow-core/src/airflow/serialization/definitions/dag.py

@@ -724,6 +724,7 @@ def _process_dagrun_deadline_alerts(
                         DeadlineAlertFields.REFERENCE: deadline_alert.reference,
                         DeadlineAlertFields.INTERVAL: deadline_alert.interval,
                         DeadlineAlertFields.CALLBACK: deadline_alert.callback_def,
+                        DeadlineAlertFields.FIRE_ON_FAILURE: deadline_alert.fire_on_failure,
                     },
                 }
             )

airflow-core/src/airflow/serialization/definitions/deadline.py

@@ -55,6 +55,7 @@ class DeadlineAlertFields:
     REFERENCE = "reference"
     INTERVAL = "interval"
     CALLBACK = "callback"
+    FIRE_ON_FAILURE = "fire_on_failure"
 
 
 class SerializedReferenceModels:
@@ -380,3 +381,4 @@ class SerializedDeadlineAlert:
     interval: timedelta | VariableInterval
     callback: Any
     name: str | None = None
+    fire_on_failure: bool = False

airflow-core/src/airflow/serialization/encoders.py

@@ -251,6 +251,7 @@ def encode_deadline_alert(d: DeadlineAlert | SerializedDeadlineAlert) -> dict[st
         "reference": encode_deadline_reference(d.reference),
         "interval": serialize(d.interval),
         "callback": serialize(d.callback),
+        "fire_on_failure": d.fire_on_failure,
     }
 
 

airflow-core/src/airflow/ui/openapi-gen/requests/schemas.gen.ts

@@ -9426,14 +9426,18 @@ export const $DeadlineAlertResponse = {
             title: 'Interval',
             description: 'Interval in seconds between deadline evaluations.'
         },
+        fire_on_failure: {
+            type: 'boolean',
+            title: 'Fire On Failure'
+        },
         created_at: {
             type: 'string',
             format: 'date-time',
             title: 'Created At'
         }
     },
     type: 'object',
-    required: ['id', 'reference_type', 'interval', 'created_at'],
+    required: ['id', 'reference_type', 'interval', 'fire_on_failure', 'created_at'],
     title: 'DeadlineAlertResponse',
     description: 'DeadlineAlert serializer for responses.'
 } as const;

airflow-core/src/airflow/ui/openapi-gen/requests/types.gen.ts

@@ -2406,6 +2406,7 @@ export type DeadlineAlertResponse = {
      * Interval in seconds between deadline evaluations.
      */
     interval: number;
+    fire_on_failure: boolean;
     created_at: string;
 };
 

airflow-core/src/airflow/utils/db.py

@@ -117,7 +117,7 @@ class MappedClassProtocol(Protocol):
     "3.1.8": "509b94a1042d",
     "3.2.0": "1d6611b6ab7c",
     "3.3.0": "d2f4e1b3c5a7",
-    "3.4.0": "c4e7a1f9b2d0",
+    "3.4.0": "b4f8c2a1d9e0",
 }
 
 # Prefix used to identify tables holding data moved during migration.

airflow-core/tests/unit/api_fastapi/core_api/routes/ui/test_deadlines.py

@@ -489,6 +489,7 @@ def test_alert_response_fields(self, test_client):
         assert alert["name"] == ALERT_NAME
         assert alert["interval"] == 3600.0
         assert alert["reference_type"] == "DagRunQueuedAtDeadline"
+        assert alert["fire_on_failure"] is False
         assert "id" in alert
         assert "created_at" in alert