Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion airflow-core/docs/authoring-and-scheduling/assets.rst
Original file line number Diff line number Diff line change
Expand Up @@ -623,7 +623,9 @@ partition match can be produced, so the downstream Dag is not triggered for
that key.

Inside partitioned Dag runs, access the resolved partition through
``dag_run.partition_key``.
``dag_run.partition_key``. For date-shaped partitions, the underlying
``datetime`` is also available as ``dag_run.partition_date``, so
templates can use ``{{ partition_date | ds }}``.

You can also trigger a DagRun manually with a partition key (for example,
through the Trigger Dag window in the UI, or through the REST API by
Expand Down
5 changes: 4 additions & 1 deletion airflow-core/docs/migrations-ref.rst
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,10 @@ Here's the list of all the Database Migrations that are executed via when you ru
+-------------------------+------------------+-------------------+--------------------------------------------------------------+
| Revision ID | Revises ID | Airflow Version | Description |
+=========================+==================+===================+==============================================================+
| ``acc215baed80`` (head) | ``a1b2c3d4e5f6`` | ``3.3.0`` | Add team_name to trigger table. |
| ``d2f4e1b3c5a7`` (head) | ``acc215baed80`` | ``3.3.0`` | Add partition_date to asset_event and |
| | | | asset_partition_dag_run. |
+-------------------------+------------------+-------------------+--------------------------------------------------------------+
| ``acc215baed80`` | ``a1b2c3d4e5f6`` | ``3.3.0`` | Add team_name to trigger table. |
+-------------------------+------------------+-------------------+--------------------------------------------------------------+
| ``a1b2c3d4e5f6`` | ``a7f3b2c1d4e5`` | ``3.3.0`` | Add version_data to dag_version. |
+-------------------------+------------------+-------------------+--------------------------------------------------------------+
Expand Down
3 changes: 3 additions & 0 deletions airflow-core/docs/templates-ref.rst
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,9 @@ Variable Type Description
| is enabled in ``airflow.cfg``.
``{{ partition_key }}`` str | None | The partition key from the current :class:`~airflow.models.dagrun.DagRun`.
| Returns ``None`` if no partition key was set. Added in version 3.3.0.
``{{ partition_date }}`` datetime | None | The partition datetime from the current :class:`~airflow.models.dagrun.DagRun`.
| Use ``{{ partition_date | ds }}`` and related filters for formatting.
| Returns ``None`` if no partition date was set. Added in version 3.3.0.
``{{ var.value }}`` Airflow variables. See `Airflow Variables in Templates`_ below.
``{{ var.json }}`` Airflow variables. See `Airflow Variables in Templates`_ below.
``{{ conn }}`` Airflow connections. See `Airflow Connections in Templates`_ below.
Expand Down
1 change: 1 addition & 0 deletions airflow-core/newsfragments/67285.feature.rst
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Propagate ``partition_date`` from producer DagRuns to consumers of partitioned assets, so date-shaped partitions are available in consumer task templates.
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,7 @@ class DAGRunResponse(BaseModel):
bundle_version: str | None
dag_display_name: str = Field(validation_alias=AliasPath("dag_model", "dag_display_name"))
partition_key: str | None
partition_date: datetime | None


class DAGRunCollectionResponse(BaseModel):
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13189,6 +13189,12 @@ components:
- type: string
- type: 'null'
title: Partition Key
partition_date:
anyOf:
- type: string
format: date-time
- type: 'null'
title: Partition Date
type: object
required:
- dag_run_id
Expand All @@ -13212,6 +13218,7 @@ components:
- bundle_version
- dag_display_name
- partition_key
- partition_date
title: DAGRunResponse
description: Dag Run serializer for responses.
DAGRunsBatchBody:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ class DagRunAssetReference(StrictBaseModel):
data_interval_start: datetime | None
data_interval_end: datetime | None
partition_key: str | None
partition_date: datetime | None


class AssetEventResponse(BaseModel):
Expand All @@ -54,6 +55,7 @@ class AssetEventResponse(BaseModel):
source_run_id: str | None = None
source_map_index: int | None = None
partition_key: str | None = None
partition_date: datetime | None = None


class AssetEventsResponse(BaseModel):
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -307,6 +307,7 @@ class DagRun(StrictBaseModel):
triggering_user_name: str | None = None
consumed_asset_events: list[AssetEventDagRunReference]
partition_key: str | None
partition_date: UtcDateTime | None = None
note: str | None = None
team_name: str | None = None

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@ def _get_asset_events_through_sql_clauses(
source_run_id=event.source_run_id,
source_map_index=event.source_map_index,
partition_key=event.partition_key,
partition_date=event.partition_date,
)
for event in asset_events
]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -46,11 +46,14 @@
AddStateEndpoints,
AddTeamNameField,
)
from airflow.api_fastapi.execution_api.versions.v2026_06_30 import AddVariableKeysEndpoint
from airflow.api_fastapi.execution_api.versions.v2026_06_30 import (
AddPartitionDateField,
AddVariableKeysEndpoint,
)

bundle = VersionBundle(
HeadVersion(),
Version("2026-06-30", AddVariableKeysEndpoint),
Version("2026-06-30", AddVariableKeysEndpoint, AddPartitionDateField),
Version(
"2026-06-16",
AddRetryPolicyFields,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,17 @@

from __future__ import annotations

from cadwyn import VersionChange, endpoint
from cadwyn import ResponseInfo, VersionChange, convert_response_to_previous_version_for, endpoint, schema

from airflow.api_fastapi.execution_api.datamodels.asset_event import (
AssetEventResponse,
AssetEventsResponse,
DagRunAssetReference,
)
from airflow.api_fastapi.execution_api.datamodels.taskinstance import (
DagRun,
TIRunContext,
)


class AddVariableKeysEndpoint(VersionChange):
Expand All @@ -26,3 +36,30 @@ class AddVariableKeysEndpoint(VersionChange):
description = __doc__

instructions_to_migrate_to_previous_version = (endpoint("/variables/keys", ["GET"]).didnt_exist,)


class AddPartitionDateField(VersionChange):
"""Expose the producer's partition datetime on the execution API so consumer tasks can template it."""

description = __doc__

instructions_to_migrate_to_previous_version = (
schema(DagRun).field("partition_date").didnt_exist,
schema(AssetEventResponse).field("partition_date").didnt_exist,
schema(DagRunAssetReference).field("partition_date").didnt_exist,
)

@convert_response_to_previous_version_for(TIRunContext) # type: ignore[arg-type]
def remove_partition_date_from_dag_run(response: ResponseInfo) -> None: # type: ignore[misc]
"""Strip ``partition_date`` from the nested ``dag_run`` payload for older clients."""
if "dag_run" in response.body and isinstance(response.body["dag_run"], dict):
response.body["dag_run"].pop("partition_date", None)

@convert_response_to_previous_version_for(AssetEventsResponse) # type: ignore[arg-type]
def remove_partition_date_from_asset_events(response: ResponseInfo) -> None: # type: ignore[misc]
"""Strip ``partition_date`` from each asset event and its ``created_dagruns`` references."""
events = response.body["asset_events"]
for elem in events:
elem.pop("partition_date", None)
for dag_ref in elem.get("created_dagruns", []):
dag_ref.pop("partition_date", None)
74 changes: 71 additions & 3 deletions airflow-core/src/airflow/assets/manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -47,11 +47,14 @@
from airflow.utils.sqlalchemy import get_dialect_name, with_row_locks

if TYPE_CHECKING:
from datetime import datetime

from sqlalchemy.orm.session import Session

from airflow.models.dag import DagModel
from airflow.models.serialized_dag import SerializedDagModel
from airflow.models.taskinstance import TaskInstance
from airflow.partition_mappers.base import PartitionMapper
from airflow.serialization.definitions.assets import (
SerializedAsset,
SerializedAssetAlias,
Expand All @@ -62,6 +65,41 @@
log = structlog.get_logger(__name__)


def _compute_target_partition_date(
*,
mapper: PartitionMapper,
source_partition_key: str,
source_partition_date: datetime | None,
) -> datetime | None:
"""
Derive the consumer's ``partition_date`` from the partition mapper.

Computed once at APDR creation and stored on the row, so the consumer
DagRun's ``partition_date`` is locked to the mapper output at the time
the source event was queued — later mapper code or config changes do
not retroactively shift the date.

- ``IdentityMapper``: passes the source ``partition_date`` through.
- ``_BaseTemporalMapper`` subclasses (``StartOf*Mapper``): re-parse the
source key with the mapper's ``input_format`` and apply ``normalize``.
- All other mappers: ``None``.
"""
from airflow.partition_mappers.identity import IdentityMapper
from airflow.partition_mappers.temporal import _BaseTemporalMapper

if isinstance(mapper, IdentityMapper):
return source_partition_date
if isinstance(mapper, _BaseTemporalMapper):
try:
return mapper.to_downstream_normalized(source_partition_key)
except Exception:
# to_downstream() already succeeded for the same input at the
# call site, so a failure here would indicate a custom subclass
# raising in `normalize`. Stay defensive.
return None
return None


@contextmanager
def _lock_asset_model(
*,
Expand Down Expand Up @@ -253,6 +291,7 @@ def register_asset_change(
source_alias_names: Collection[str] = (),
session: Session,
partition_key: str | None = None,
partition_date: datetime | None = None,
source_is_api: bool = False,
api_user_teams: set[str] | None = None,
**kwargs,
Expand Down Expand Up @@ -301,6 +340,7 @@ def register_asset_change(
"asset_id": asset_model.id,
"extra": extra or {},
"partition_key": partition_key,
"partition_date": partition_date,
}
if task_instance:
event_kwargs.update(
Expand Down Expand Up @@ -365,6 +405,7 @@ def register_asset_change(
source_map_index=asset_event.source_map_index,
source_aliases=[aam.to_serialized() for aam in asset_alias_models],
partition_key=partition_key,
partition_date=partition_date,
)
)

Expand Down Expand Up @@ -524,11 +565,10 @@ def _queue_partitioned_dags(
if (asset_model := session.scalar(select(AssetModel).where(AssetModel.id == asset_id))) is None:
raise RuntimeError(f"Could not find asset for asset_id={asset_id}")

mapper = timetable.get_partition_mapper(name=asset_model.name, uri=asset_model.uri)
try:
# We'll need to catch every possible exception happen when mapping partition_key.
target_key = timetable.get_partition_mapper(
name=asset_model.name, uri=asset_model.uri
).to_downstream(partition_key)
target_key = mapper.to_downstream(partition_key)
except Exception as err:
log.exception(
"Could not map partition key for asset in target Dag. "
Expand Down Expand Up @@ -564,9 +604,19 @@ def _queue_partitioned_dags(
target_keys = [target_key]
del target_key

# Compute the target partition_date once per (mapper, source_key).
# to_downstream already succeeded above, so to_downstream_normalized
# on the same input is expected to succeed for temporal mappers.
target_partition_date: datetime | None = _compute_target_partition_date(
mapper=mapper,
source_partition_key=partition_key,
source_partition_date=event.partition_date,
)

for target_key in target_keys:
apdr = cls._get_or_create_apdr(
target_key=target_key,
target_partition_date=target_partition_date,
target_dag=target_dag,
asset_id=asset_id,
session=session,
Expand All @@ -586,6 +636,7 @@ def _get_or_create_apdr(
cls,
*,
target_key: str,
target_partition_date: datetime | None,
target_dag: SerializedDagModel,
asset_id: int,
session: Session,
Expand All @@ -598,6 +649,13 @@ def _get_or_create_apdr(
This leads to the unintended outcome of having two APDRs created instead of one.
To resolve this, we add a mutex lock to AssetModel for PostgreSQL and MySQL and use
AssetPartitionDagRunMutexLock table for SQLite.

When an existing pending APDR is returned, its stored ``partition_date`` (set by the first
event that queued it) is kept. If a later event resolves to the same ``target_key`` with a
different ``target_partition_date`` — possible when two source assets use different
timezone-configured mappers that happen to format to the same string — we log a warning
and let the first event win, since the consumer DagRun has already been semantically
committed to the first datetime.
"""
with _lock_asset_model(session=session, asset_id=asset_id):
latest_apdr: AssetPartitionDagRun | None = session.scalar(
Expand All @@ -610,6 +668,15 @@ def _get_or_create_apdr(
.limit(1)
)
if latest_apdr and latest_apdr.created_dag_run_id is None:
if latest_apdr.partition_date != target_partition_date:
log.warning(
"Existing pending APDR has partition_date that differs from "
"the newly computed one; keeping the first value (first-event-wins).",
target_dag_id=target_dag.dag_id,
target_key=target_key,
existing_partition_date=latest_apdr.partition_date,
incoming_partition_date=target_partition_date,
)
cls.logger().debug(
"Existing APDR found for key %s dag_id %s",
target_key,
Expand All @@ -622,6 +689,7 @@ def _get_or_create_apdr(
target_dag_id=target_dag.dag_id,
created_dag_run_id=None,
partition_key=target_key,
partition_date=target_partition_date,
)
session.add(apdr)
session.flush()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ def combine_player_stats(dag_run=None):
"""Merge the aligned hourly partitions into a combined dataset."""
if TYPE_CHECKING:
assert dag_run
print(dag_run.partition_key)
print(dag_run.partition_key, dag_run.partition_date)

combine_player_stats()

Expand Down
1 change: 1 addition & 0 deletions airflow-core/src/airflow/jobs/scheduler_job_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -1908,6 +1908,7 @@ def _create_dagruns_for_partitioned_asset_dags(self, session: Session) -> set[st
logical_date=None,
data_interval=None,
partition_key=apdr.partition_key,
partition_date=apdr.partition_date,
run_after=run_after,
run_type=DagRunType.ASSET_TRIGGERED,
triggered_by=DagRunTriggeredByType.ASSET,
Expand Down
3 changes: 3 additions & 0 deletions airflow-core/src/airflow/listeners/types.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@
import attrs

if TYPE_CHECKING:
from datetime import datetime

from pydantic import JsonValue

from airflow.serialization.definitions.assets import SerializedAsset, SerializedAssetAlias
Expand All @@ -40,3 +42,4 @@ class AssetEvent:
source_map_index: int | None
source_aliases: list[SerializedAssetAlias]
partition_key: str | None
partition_date: datetime | None = None
Loading
Loading