Skip to content

Add a DagRun and TaskInstance event listener under the Kafka provider #68082

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
shahar1 merged 1 commit into from
Jul 13, 2026
Merged

Add a DagRun and TaskInstance event listener under the Kafka provider #68082

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
200 changes: 19 additions & 181 deletions airflow-core/tests/integration/otel/test_otel.py
View file
Open in desktop
Comment thread
jscheffl marked this conversation as resolved.
Original file line number Diff line number Diff line change
Expand Up @@ -16,34 +16,35 @@
# under the License.
from __future__ import annotations

import json
import logging
import os
import socket
import subprocess
import time
from typing import Any
from typing import TYPE_CHECKING, Any

import pytest
import requests
from sqlalchemy import func, select
from sqlalchemy import select

from airflow._shared.timezones import timezone
from airflow.dag_processing.bundles.manager import DagBundlesManager
from airflow.dag_processing.dagbag import DagBag
from airflow.models import DagRun
from airflow.models.serialized_dag import SerializedDagModel
from airflow.models.taskinstance import TaskInstance
from airflow.serialization.definitions.dag import SerializedDAG
from airflow.utils.session import create_session
from airflow.utils.state import State

from tests_common.test_utils.dag import create_scheduler_dag
from tests_common.test_utils.integration_setup import (
serialize_and_get_dags,
start_scheduler,
terminate_process,
unpause_trigger_dag_and_get_run_id,
wait_for_dag_run,
)
from tests_common.test_utils.otel_utils import (
dump_airflow_metadata_db,
extract_metrics_from_output,
)
from tests_common.test_utils.version_compat import AIRFLOW_V_3_0_PLUS, AIRFLOW_V_3_1_PLUS

if TYPE_CHECKING:
from airflow.serialization.definitions.dag import SerializedDAG

log = logging.getLogger("integration.otel.test_otel")

Expand Down Expand Up @@ -86,62 +87,6 @@ def wait_for_otel_collector(host: str, port: int, timeout: int = 120) -> None:
)


def unpause_trigger_dag_and_get_run_id(dag_id: str, conf: dict | None = None) -> str:
unpause_command = ["airflow", "dags", "unpause", dag_id]

# Unpause the dag using the cli.
subprocess.run(unpause_command, check=True, env=os.environ.copy())

execution_date = timezone.utcnow()
run_id = f"manual__{execution_date.isoformat()}"

trigger_command = [
"airflow",
"dags",
"trigger",
dag_id,
"--run-id",
run_id,
"--logical-date",
execution_date.isoformat(),
]

if conf:
import json

trigger_command += ["--conf", json.dumps(conf)]

# Trigger the dag using the cli.
subprocess.run(trigger_command, check=True, env=os.environ.copy())

return run_id


def wait_for_dag_run(dag_id: str, run_id: str, max_wait_time: int):
# max_wait_time, is the timeout for the DAG run to complete. The value is in seconds.
start_time = timezone.utcnow().timestamp()

while timezone.utcnow().timestamp() - start_time < max_wait_time:
with create_session() as session:
dag_run = session.scalar(
select(DagRun).where(
DagRun.dag_id == dag_id,
DagRun.run_id == run_id,
)
)

if dag_run is None:
time.sleep(5)
continue

dag_run_state = dag_run.state
log.debug("DAG Run state: %s.", dag_run_state)

if dag_run_state in [State.SUCCESS, State.FAILED]:
break
return dag_run_state


def print_ti_output_for_dag_run(dag_id: str, run_id: str):
breeze_logs_dir = "/root/airflow/logs"

Expand Down Expand Up @@ -189,19 +134,6 @@ class TestOtelIntegration:
use_otel = os.getenv("use_otel", default="false")
log_level = os.getenv("log_level", default="none")

scheduler_command_args = [
"airflow",
"scheduler",
]

apiserver_command_args = [
"airflow",
"api-server",
"--port",
"8080",
"--daemon",
]

dags: dict[str, SerializedDAG] = {}

@classmethod
Expand Down Expand Up @@ -251,67 +183,7 @@ def setup_class(cls):
migrate_command = ["airflow", "db", "migrate"]
subprocess.run(migrate_command, check=True, env=os.environ.copy())

cls.dags = cls.serialize_and_get_dags()

@classmethod
def serialize_and_get_dags(cls) -> dict[str, SerializedDAG]:
log.info("Serializing Dags from directory %s", cls.dag_folder)
# Load DAGs from the dag directory.
dag_bag = DagBag(dag_folder=cls.dag_folder)

dag_ids = dag_bag.dag_ids
assert len(dag_ids) == 1

dag_dict: dict[str, SerializedDAG] = {}
with create_session() as session:
for dag_id in dag_ids:
dag = dag_bag.get_dag(dag_id)
assert dag is not None, f"DAG with ID {dag_id} not found."
# Sync the DAG to the database.
if AIRFLOW_V_3_0_PLUS:
from airflow.models.dagbundle import DagBundleModel

count = session.scalar(
select(func.count())
.select_from(DagBundleModel)
.where(DagBundleModel.name == "testing")
)
if count == 0:
session.add(DagBundleModel(name="testing"))
session.commit()
SerializedDAG.bulk_write_to_db(
bundle_name="testing", bundle_version=None, dags=[dag], session=session
)
dag_dict[dag_id] = create_scheduler_dag(dag)
else:
dag.sync_to_db(session=session)
dag_dict[dag_id] = dag
# Manually serialize the dag and write it to the db to avoid a db error.
if AIRFLOW_V_3_1_PLUS:
from airflow.serialization.serialized_objects import LazyDeserializedDAG

SerializedDagModel.write_dag(
LazyDeserializedDAG.from_dag(dag), bundle_name="testing", session=session
)
else:
SerializedDagModel.write_dag(dag, bundle_name="testing", session=session)

session.commit()

TESTING_BUNDLE_CONFIG = [
{
"name": "testing",
"classpath": "airflow.dag_processing.bundles.local.LocalDagBundle",
"kwargs": {"path": f"{cls.dag_folder}", "refresh_interval": 1},
}
]

os.environ["AIRFLOW__DAG_PROCESSOR__DAG_BUNDLE_CONFIG_LIST"] = json.dumps(TESTING_BUNDLE_CONFIG)
# Initial add
manager = DagBundlesManager()
manager.sync_bundles_to_db()

return dag_dict
cls.dags = serialize_and_get_dags(dag_folder=cls.dag_folder)

def dag_execution_for_testing_metrics(self, capfd):
# Metrics.
Expand All @@ -327,7 +199,7 @@ def dag_execution_for_testing_metrics(self, capfd):
try:
# Start the processes here and not as fixtures or in a common setup,
# so that the test can capture their output.
scheduler_process, apiserver_process = self.start_scheduler(capture_output=True)
scheduler_process, apiserver_process = start_scheduler(capture_output=True)

dag_id = "otel_test_dag"

Expand Down Expand Up @@ -357,13 +229,13 @@ def dag_execution_for_testing_metrics(self, capfd):
finally:
# Terminate the processes.

self._terminate_process(scheduler_process)
terminate_process(scheduler_process)
scheduler_status = scheduler_process.poll()
assert scheduler_status is not None, (
"The scheduler_1 process status is None, which means that it hasn't terminated as expected."
)

self._terminate_process(apiserver_process)
terminate_process(apiserver_process)
apiserver_status = apiserver_process.poll()
assert apiserver_status is not None, (
"The apiserver process status is None, which means that it hasn't terminated as expected."
Expand Down Expand Up @@ -491,7 +363,7 @@ def test_dag_execution_succeeds(self, capfd, task_span_detail_level, expected_hi
try:
# Start the processes here and not as fixtures or in a common setup,
# so that the test can capture their output.
scheduler_process, apiserver_process = self.start_scheduler()
scheduler_process, apiserver_process = start_scheduler()

dag_id = "otel_test_dag"

Expand Down Expand Up @@ -521,13 +393,13 @@ def test_dag_execution_succeeds(self, capfd, task_span_detail_level, expected_hi
dump_airflow_metadata_db(session)

# Terminate the processes.
self._terminate_process(scheduler_process)
terminate_process(scheduler_process)
scheduler_status = scheduler_process.poll()
assert scheduler_status is not None, (
"The scheduler_1 process status is None, which means that it hasn't terminated as expected."
)

self._terminate_process(apiserver_process)
terminate_process(apiserver_process)
apiserver_status = apiserver_process.poll()
assert apiserver_status is not None, (
"The apiserver process status is None, which means that it hasn't terminated as expected."
Expand Down Expand Up @@ -568,37 +440,3 @@ def get_parent_span_id(span):

nested = get_span_hierarchy()
assert nested == expected_hierarchy

@staticmethod
def _terminate_process(proc: subprocess.Popen, timeout: int = 30) -> None:
# Grace period covers OTel atexit flush (force_flush default: 10s);
# SIGKILL is the fallback if the process is still alive after timeout.
proc.terminate()
try:
proc.wait(timeout=timeout)
except subprocess.TimeoutExpired:
proc.kill()
proc.wait()

def start_scheduler(self, capture_output: bool = False):
stdout = None if capture_output else subprocess.DEVNULL
stderr = None if capture_output else subprocess.DEVNULL

scheduler_process = subprocess.Popen(
self.scheduler_command_args,
env=os.environ.copy(),
stdout=stdout,
stderr=stderr,
)

apiserver_process = subprocess.Popen(
self.apiserver_command_args,
env=os.environ.copy(),
stdout=stdout,
stderr=stderr,
)

# Wait to ensure both processes have started.
time.sleep(10)

return scheduler_process, apiserver_process
15 changes: 8 additions & 7 deletions airflow-core/tests/unit/api_fastapi/core_api/routes/public/test_plugins.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ class TestGetPlugins:
# Filters
(
{},
18,
19,
[
"InformaticaProviderPlugin",
"MetadataCollectionPlugin",
Expand All @@ -45,6 +45,7 @@ class TestGetPlugins:
"edge_executor",
"hitl_review",
"hive",
"kafka_listener",
"plugin-a",
"plugin-b",
"plugin-c",
Expand All @@ -58,14 +59,14 @@ class TestGetPlugins:
),
(
{"limit": 3, "offset": 3},
18,
19,
[
"business_day_window_plugin",
"databricks_workflow",
"decreasing_priority_weight_strategy_plugin",
],
),
({"limit": 1}, 18, ["InformaticaProviderPlugin"]),
({"limit": 1}, 19, ["InformaticaProviderPlugin"]),
],
)
def test_should_respond_200(
Expand Down Expand Up @@ -158,17 +159,17 @@ def test_invalid_external_view_destination_should_log_warning_and_continue(self,
# Verify warning was logged
assert any("Skipping invalid plugin due to error" in rec.message for rec in caplog.records)

response = test_client.get("/plugins", params={"limit": 7, "offset": 9})
response = test_client.get("/plugins", params={"limit": 8, "offset": 9})
assert response.status_code == 200

body = response.json()
plugins_page = body["plugins"]

# Even though limit=7, only 6 valid plugins should come back
assert len(plugins_page) == 7
# Invalid plugins are filtered before pagination, so the page is full.
assert len(plugins_page) == 8
assert "test_plugin_invalid" not in [p["name"] for p in plugins_page]

assert body["total_entries"] == 18
assert body["total_entries"] == 19


@skip_if_force_lowest_dependencies_marker
Expand Down
2 changes: 1 addition & 1 deletion airflow-core/tests/unit/plugins/test_plugins_manager.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -408,7 +408,7 @@ def test_does_not_double_import_entrypoint_provider_plugins(self):
# Mock/skip loading from plugin dir
with mock.patch("airflow.plugins_manager._load_plugins_from_plugin_directory", return_value=([], [])):
plugins = plugins_manager._get_plugins()[0]
assert len(plugins) == 6
assert len(plugins) == 7


class TestWindowPluginRegistration:
Expand Down
Loading
Loading