From ff31388704b2e43a7f68de80295f0986e064859e Mon Sep 17 00:00:00 2001 From: Shahar Epstein <60007259+shahar1@users.noreply.github.com> Date: Thu, 9 Jul 2026 22:57:35 +0300 Subject: [PATCH 1/5] Consolidate OpenLineage e2e tests into airflow-e2e-tests The OpenLineage provider e2e suite was introduced as a standalone `providers-e2e-tests/` framework with its own docker-compose stack, harness, Dockerfile, breeze command and CI workflow. Review feedback on the introducing PR pointed out that `airflow-e2e-tests/` already hosts deployed-stack suites for providers (Elasticsearch/OpenSearch remote logging, Kafka event-driven), so a second parallel framework duplicates setup that already exists. Fold OpenLineage in as an `openlineage` E2E test mode: it reuses the shared docker-compose base, the testcontainers conftest, and the `AirflowClient`, adding only an overlay for the OpenLineage env config and a small runner that drives the provider's system-test DAGs. The separate framework, its breeze command, workspace member and dedicated workflow are removed; the older-Airflow compatibility matrix is dropped along with them. --- .dockerignore | 1 - .../workflows/additional-prod-image-tests.yml | 35 +--- .github/workflows/ci-amd.yml | 6 +- .github/workflows/ci-arm.yml | 6 +- .github/workflows/providers-e2e-tests.yml | 157 -------------- .rat-excludes | 1 - airflow-e2e-tests/docker/openlineage.yml | 83 ++++++++ .../tests/airflow_e2e_tests/conftest.py | 40 +++- .../tests/airflow_e2e_tests/constants.py | 4 + .../openlineage_tests/__init__.py | 17 -- .../dags_extra/e2e_versioned_bundle.py | 0 .../dags_extra/openlineage_warmup_dag.py | 0 .../openlineage_tests}/harness.py | 93 +++------ .../openlineage_tests}/prepare_dags.py | 61 +----- .../openlineage_tests}/task_logs.py | 0 .../test_openlineage_e2e.py | 27 ++- dev/breeze/doc/05_test_commands.rst | 28 +-- dev/breeze/doc/ci/04_selective_checks.md | 7 +- ...utput_setup_check-all-params-in-groups.svg | 2 +- ...utput_setup_check-all-params-in-groups.txt | 2 +- ...output_setup_regenerate-command-images.svg | 8 +- ...output_setup_regenerate-command-images.txt | 2 +- dev/breeze/doc/images/output_testing.svg | 18 +- dev/breeze/doc/images/output_testing.txt | 2 +- .../output_testing_airflow-e2e-tests.svg | 2 +- .../output_testing_airflow-e2e-tests.txt | 2 +- .../output_testing_providers-e2e-tests.svg | 182 ---------------- .../output_testing_providers-e2e-tests.txt | 1 - .../commands/testing_commands.py | 170 +-------------- .../commands/testing_commands_config.py | 20 -- .../src/airflow_breeze/utils/run_tests.py | 6 - .../airflow_breeze/utils/selective_checks.py | 13 +- dev/breeze/tests/test_selective_checks.py | 12 +- providers-e2e-tests/README.md | 111 ---------- providers-e2e-tests/openlineage/.gitignore | 7 - providers-e2e-tests/openlineage/Dockerfile | 28 --- providers-e2e-tests/openlineage/README.md | 75 ------- .../openlineage/docker-compose-local.yaml | 56 ----- .../openlineage/docker-compose.yaml | 181 ---------------- .../openlineage/pyproject.toml | 85 -------- .../openlineage/tests/conftest.py | 194 ------------------ .../openlineage/tests/constants.py | 42 ---- pyproject.toml | 7 - ...py_full_dist_local_venv_or_breeze_in_ci.py | 4 - uv.lock | 29 --- 45 files changed, 229 insertions(+), 1598 deletions(-) delete mode 100644 .github/workflows/providers-e2e-tests.yml create mode 100644 airflow-e2e-tests/docker/openlineage.yml rename providers-e2e-tests/openlineage/.pre-commit-config.yaml => airflow-e2e-tests/tests/airflow_e2e_tests/openlineage_tests/__init__.py (61%) rename {providers-e2e-tests/openlineage => airflow-e2e-tests/tests/airflow_e2e_tests/openlineage_tests}/dags_extra/e2e_versioned_bundle.py (100%) rename {providers-e2e-tests/openlineage => airflow-e2e-tests/tests/airflow_e2e_tests/openlineage_tests}/dags_extra/openlineage_warmup_dag.py (100%) rename {providers-e2e-tests/openlineage/tests => airflow-e2e-tests/tests/airflow_e2e_tests/openlineage_tests}/harness.py (71%) rename {providers-e2e-tests/openlineage => airflow-e2e-tests/tests/airflow_e2e_tests/openlineage_tests}/prepare_dags.py (61%) rename {providers-e2e-tests/openlineage/tests => airflow-e2e-tests/tests/airflow_e2e_tests/openlineage_tests}/task_logs.py (100%) rename {providers-e2e-tests/openlineage/tests => airflow-e2e-tests/tests/airflow_e2e_tests/openlineage_tests}/test_openlineage_e2e.py (78%) delete mode 100644 dev/breeze/doc/images/output_testing_providers-e2e-tests.svg delete mode 100644 dev/breeze/doc/images/output_testing_providers-e2e-tests.txt delete mode 100644 providers-e2e-tests/README.md delete mode 100644 providers-e2e-tests/openlineage/.gitignore delete mode 100644 providers-e2e-tests/openlineage/Dockerfile delete mode 100644 providers-e2e-tests/openlineage/README.md delete mode 100644 providers-e2e-tests/openlineage/docker-compose-local.yaml delete mode 100644 providers-e2e-tests/openlineage/docker-compose.yaml delete mode 100644 providers-e2e-tests/openlineage/pyproject.toml delete mode 100644 providers-e2e-tests/openlineage/tests/conftest.py delete mode 100644 providers-e2e-tests/openlineage/tests/constants.py diff --git a/.dockerignore b/.dockerignore index a7e55a8f4e78e..401e8fff9fb40 100644 --- a/.dockerignore +++ b/.dockerignore @@ -51,7 +51,6 @@ !airflow-ctl-tests !shared/ !airflow-e2e-tests -!providers-e2e-tests # Add scripts so that we can use them inside the container !scripts diff --git a/.github/workflows/additional-prod-image-tests.yml b/.github/workflows/additional-prod-image-tests.yml index 27861ccce5165..aef55b9d890af 100644 --- a/.github/workflows/additional-prod-image-tests.yml +++ b/.github/workflows/additional-prod-image-tests.yml @@ -60,14 +60,10 @@ on: # yamllint disable-line rule:truthy description: "Whether to run Go SDK e2e tests (true/false)" required: true type: string - run-providers-e2e-tests-openlineage: + run-openlineage-e2e-tests: description: "Whether to run OpenLineage e2e tests (true/false)" required: true type: string - providers-compatibility-tests-matrix: - description: "JSON matrix of released Airflow versions for provider e2e compat tests." - required: true - type: string constraints-branch: description: "Branch used to construct constraints URL from." required: true @@ -329,34 +325,17 @@ jobs: e2e_test_mode: "go_sdk" if: inputs.canary-run == 'true' || inputs.run-go-sdk-e2e-tests == 'true' - test-providers-e2e-tests-openlineage: - name: "Provider E2E tests openlineage" - uses: ./.github/workflows/providers-e2e-tests.yml - with: - provider: "openlineage" - provider-display-name: "OpenLineage" - runners: ${{ inputs.runners }} - platform: ${{ inputs.platform }} - default-python-version: "${{ inputs.default-python-version }}" - use-uv: ${{ inputs.use-uv }} - if: inputs.canary-run == 'true' || inputs.run-providers-e2e-tests-openlineage == 'true' - - test-providers-e2e-tests-openlineage-compat: - name: "Provider E2E tests openlineage compat" - strategy: - fail-fast: false - matrix: - compat: ${{ fromJSON(inputs.providers-compatibility-tests-matrix) }} - uses: ./.github/workflows/providers-e2e-tests.yml + test-e2e-integration-tests-openlineage: + name: "OpenLineage e2e tests with PROD image" + uses: ./.github/workflows/airflow-e2e-tests.yml with: - provider: "openlineage" - provider-display-name: "OpenLineage" + workflow-name: "OpenLineage e2e test" runners: ${{ inputs.runners }} platform: ${{ inputs.platform }} default-python-version: "${{ inputs.default-python-version }}" use-uv: ${{ inputs.use-uv }} - airflow-version: ${{ matrix.compat.airflow-version }} - if: inputs.canary-run == 'true' || inputs.run-providers-e2e-tests-openlineage == 'true' + e2e_test_mode: "openlineage" + if: inputs.canary-run == 'true' || inputs.run-openlineage-e2e-tests == 'true' test-ui-e2e-chromium: name: "Chromium UI e2e tests with PROD image" diff --git a/.github/workflows/ci-amd.yml b/.github/workflows/ci-amd.yml index 4b321374542c9..94218264d5f29 100644 --- a/.github/workflows/ci-amd.yml +++ b/.github/workflows/ci-amd.yml @@ -147,7 +147,7 @@ jobs: run-event-driven-e2e-tests: ${{ steps.selective-checks.outputs.run-event-driven-e2e-tests }} run-java-sdk-e2e-tests: ${{ steps.selective-checks.outputs.run-java-sdk-e2e-tests }} run-go-sdk-e2e-tests: ${{ steps.selective-checks.outputs.run-go-sdk-e2e-tests }} - run-providers-e2e-tests-openlineage: ${{ steps.selective-checks.outputs.run-providers-e2e-tests-openlineage }} + run-openlineage-e2e-tests: ${{ steps.selective-checks.outputs.run-openlineage-e2e-tests }} run-system-tests: ${{ steps.selective-checks.outputs.run-system-tests }} run-task-sdk-tests: ${{ steps.selective-checks.outputs.run-task-sdk-tests }} run-task-sdk-integration-tests: ${{ steps.selective-checks.outputs.run-task-sdk-integration-tests }} @@ -920,9 +920,7 @@ jobs: run-event-driven-e2e-tests: ${{ needs.build-info.outputs.run-event-driven-e2e-tests }} run-java-sdk-e2e-tests: ${{ needs.build-info.outputs.run-java-sdk-e2e-tests }} run-go-sdk-e2e-tests: ${{ needs.build-info.outputs.run-go-sdk-e2e-tests }} - run-providers-e2e-tests-openlineage: ${{ needs.build-info.outputs.run-providers-e2e-tests-openlineage }} - providers-compatibility-tests-matrix: > - ${{ needs.build-info.outputs.providers-compatibility-tests-matrix }} + run-openlineage-e2e-tests: ${{ needs.build-info.outputs.run-openlineage-e2e-tests }} use-uv: ${{ needs.build-info.outputs.use-uv }} run-ui-e2e-tests: ${{ needs.build-info.outputs.run-ui-e2e-tests }} run-airflow-ctl-integration-tests: ${{ needs.build-info.outputs.run-airflow-ctl-integration-tests }} diff --git a/.github/workflows/ci-arm.yml b/.github/workflows/ci-arm.yml index 9bbc72bcc1ba3..418bf101a5c4e 100644 --- a/.github/workflows/ci-arm.yml +++ b/.github/workflows/ci-arm.yml @@ -136,7 +136,7 @@ jobs: run-event-driven-e2e-tests: ${{ steps.selective-checks.outputs.run-event-driven-e2e-tests }} run-java-sdk-e2e-tests: ${{ steps.selective-checks.outputs.run-java-sdk-e2e-tests }} run-go-sdk-e2e-tests: ${{ steps.selective-checks.outputs.run-go-sdk-e2e-tests }} - run-providers-e2e-tests-openlineage: ${{ steps.selective-checks.outputs.run-providers-e2e-tests-openlineage }} + run-openlineage-e2e-tests: ${{ steps.selective-checks.outputs.run-openlineage-e2e-tests }} run-system-tests: ${{ steps.selective-checks.outputs.run-system-tests }} run-task-sdk-tests: ${{ steps.selective-checks.outputs.run-task-sdk-tests }} run-task-sdk-integration-tests: ${{ steps.selective-checks.outputs.run-task-sdk-integration-tests }} @@ -909,9 +909,7 @@ jobs: run-event-driven-e2e-tests: ${{ needs.build-info.outputs.run-event-driven-e2e-tests }} run-java-sdk-e2e-tests: ${{ needs.build-info.outputs.run-java-sdk-e2e-tests }} run-go-sdk-e2e-tests: ${{ needs.build-info.outputs.run-go-sdk-e2e-tests }} - run-providers-e2e-tests-openlineage: ${{ needs.build-info.outputs.run-providers-e2e-tests-openlineage }} - providers-compatibility-tests-matrix: > - ${{ needs.build-info.outputs.providers-compatibility-tests-matrix }} + run-openlineage-e2e-tests: ${{ needs.build-info.outputs.run-openlineage-e2e-tests }} use-uv: ${{ needs.build-info.outputs.use-uv }} run-ui-e2e-tests: ${{ needs.build-info.outputs.run-ui-e2e-tests }} run-airflow-ctl-integration-tests: ${{ needs.build-info.outputs.run-airflow-ctl-integration-tests }} diff --git a/.github/workflows/providers-e2e-tests.yml b/.github/workflows/providers-e2e-tests.yml deleted file mode 100644 index 1ed6d77eb754e..0000000000000 --- a/.github/workflows/providers-e2e-tests.yml +++ /dev/null @@ -1,157 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# ---- - -name: Providers E2E Tests - -permissions: - contents: read -on: # yamllint disable-line rule:truthy - workflow_dispatch: - inputs: - provider: - description: "The provider whose e2e suite to run (a directory under providers-e2e-tests/)." - type: string - default: 'openlineage' - provider-display-name: - description: "Display name for the provider, used only in the job name." - type: string - default: 'OpenLineage' - runners: - description: "The array of labels (in json form) determining runners." - type: string - default: '["ubuntu-24.04"]' - platform: - description: "Platform for the build - 'linux/amd64' or 'linux/arm64'" - type: string - default: 'linux/amd64' - default-python-version: - description: "Which version of python should be used by default" - type: string - default: '3.10' - use-uv: - description: "Whether to use uv to build the image (true/false)" - type: string - default: 'true' - docker-image-tag: - description: "Tag of the Docker image to test" - type: string - required: true - airflow-version: - description: "Released Airflow version to test with current providers (empty = default PROD image)" - type: string - default: "" - - workflow_call: - inputs: - provider: - description: "The provider whose e2e suite to run (a directory under providers-e2e-tests/)." - type: string - default: 'openlineage' - provider-display-name: - description: "Display name for the provider, used only in the job name." - type: string - default: 'OpenLineage' - runners: - description: "The array of labels (in json form) determining runners." - required: true - type: string - platform: - description: "Platform for the build - 'linux/amd64' or 'linux/arm64'" - required: true - type: string - default-python-version: - description: "Which version of python should be used by default" - required: true - type: string - use-uv: - description: "Whether to use uv to build the image (true/false)" - required: true - type: string - docker-image-tag: - description: "Tag of the Docker image to test" - type: string - default: "" - airflow-version: - description: "Released Airflow version to test with current providers (empty = default PROD image)" - type: string - default: "" - -jobs: - test-providers-e2e-tests: - timeout-minutes: 60 - name: "Provider E2E tests \ - ${{ inputs.provider-display-name }}\ - ${{ inputs.airflow-version != '' && format(' Compat {0}', inputs.airflow-version) || '' }}" - runs-on: ${{ fromJSON(inputs.runners) }} - env: - PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" - PROVIDER: ${{ inputs.provider }} - GITHUB_REPOSITORY: ${{ github.repository }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_USERNAME: ${{ github.actor }} - VERBOSE: "true" - steps: - # Gating on job-level `if:` instead of per-step would skip the whole job — and GitHub Actions - # does not evaluate a dynamic `name:` (the one above, using `inputs.*`) for a skipped job, so it - # renders the raw unevaluated template instead. Gating each step keeps the job itself "running" - # (and its name properly resolved), ending in success with every real step skipped for AF2. - - name: "Skip notice for Airflow 2.x" - if: ${{ startsWith(inputs.airflow-version, '2.') }} - run: echo "Skipping $PROVIDER e2e tests — not supported against Airflow 2.x." - - name: "Cleanup repo" - if: ${{ !startsWith(inputs.airflow-version, '2.') }} - shell: bash - run: sudo rm -rf ${GITHUB_WORKSPACE}/* - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - if: ${{ !startsWith(inputs.airflow-version, '2.') }} - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - with: - fetch-depth: 2 - persist-credentials: false - - name: "Prepare breeze & PROD image: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}" - if: ${{ !startsWith(inputs.airflow-version, '2.') }} - uses: ./.github/actions/prepare_breeze_and_image - with: - platform: ${{ inputs.platform }} - image-type: "prod" - python: ${{ env.PYTHON_MAJOR_MINOR_VERSION }} - use-uv: ${{ inputs.use-uv }} - make-mnt-writeable-and-cleanup: true - id: breeze - - name: "Run ${{ inputs.provider }} e2e tests" - if: ${{ !startsWith(inputs.airflow-version, '2.') }} - run: > - breeze testing providers-e2e-tests "$PROVIDER" - ${AIRFLOW_VERSION:+--airflow-version ${AIRFLOW_VERSION}} - --skip-mounting-local-volumes - env: - DOCKER_IMAGE: "${{ inputs.docker-image-tag }}" - AIRFLOW_VERSION: "${{ inputs.airflow-version }}" - - name: Zip logs - if: ${{ always() && !startsWith(inputs.airflow-version, '2.') }} - run: | - cd "./providers-e2e-tests/$PROVIDER" && zip -r logs.zip logs - - name: "Upload logs" - if: ${{ always() && !startsWith(inputs.airflow-version, '2.') }} - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: "${{ inputs.provider }}-e2e-test-logs-${{ inputs.airflow-version || 'prod' }}" - path: './providers-e2e-tests/${{ inputs.provider }}/logs.zip' - retention-days: '7' - if-no-files-found: 'error' diff --git a/.rat-excludes b/.rat-excludes index 0510ce24783f9..f0cd2910f5b27 100644 --- a/.rat-excludes +++ b/.rat-excludes @@ -45,7 +45,6 @@ venv clients/* files/* dags/* -providers-e2e-tests/openlineage/dags/dag_doc.md generated/* .gitmodules prod_image_installed_providers.txt diff --git a/airflow-e2e-tests/docker/openlineage.yml b/airflow-e2e-tests/docker/openlineage.yml new file mode 100644 index 0000000000000..79ea5f2dafeed --- /dev/null +++ b/airflow-e2e-tests/docker/openlineage.yml @@ -0,0 +1,83 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Docker Compose override for the openlineage E2E test mode. +# +# The OpenLineage system-test DAGs are sourced into the dags folder by +# openlineage_tests/prepare_dags.py (conftest._setup_openlineage_integration). They emit through the +# test VariableTransport (events land in Airflow Variables) and each DAG's terminal +# OpenLineageTestOperator validates them, so a run ending "success" means the lineage matched. +--- +# yamllint disable rule:line-length +x-ol-common-env: &ol-common-env + # These DAGs are the only ones the harness drives; keep the stock example DAGs out of the way. + AIRFLOW__CORE__LOAD_EXAMPLES: 'false' + # The OpenLineage system-test DAGs (and the custom VariableTransport / OpenLineageTestOperator + # they import) live under /system/openlineage; the dags folder is added to PYTHONPATH so + # both DAG parsing and the dotted-path transport class resolve in every component. + PYTHONPATH: '/opt/airflow/dags' + # Route OpenLineage events to the test VariableTransport (stores events in Airflow Variables, + # read back and validated by the in-DAG OpenLineageTestOperator). The transport type is the + # fully-qualified class path; the OpenLineage client imports it directly. + AIRFLOW__OPENLINEAGE__TRANSPORT: '{"type": "system.openlineage.transport.variable.VariableTransport"}' + AIRFLOW__OPENLINEAGE__DISABLED: 'false' + # Deployment-wide policy required by the two conf-driven policy DAGs. Each rule is scoped by + # dag_id, so both rules coexist and do not affect any other DAG. + AIRFLOW__OPENLINEAGE__EMISSION_POLICY: '[{"scope": {"dag_id": "openlineage_policy_conf_locked_dag"}, "locked": true, "controls": {"include_source_code": false}}, {"scope": {"dag_id": "openlineage_policy_conf_source_code_dag"}, "controls": {"include_source_code": false}}]' + AIRFLOW__OPENLINEAGE__EXECUTION_TIMEOUT: '60' + # Both control how quickly the dag-processor picks up dags: REFRESH_INTERVAL is how often it + # re-scans the folder for new/removed files, MIN_FILE_PROCESS_INTERVAL is how often it re-parses + # a file it already knows about. Tightened from the 300s/30s defaults so the e2e run doesn't wait + # on the dag-processor's own cadence on top of everything else. + AIRFLOW__DAG_PROCESSOR__REFRESH_INTERVAL: '3' + AIRFLOW__DAG_PROCESSOR__MIN_FILE_PROCESS_INTERVAL: '3' + # openlineage_defer_simple_dag's TimeDeltaSensor defaults to a 180s delta so it reliably defers at + # least once even before a worker picks up the task; tighten it here since we do not need to wait. + SYSTEM_TESTS_OL_TIMEDELTA: '60' + # Every DAG's dagrun_timeout (system.openlineage.constants.DEFAULT_DAGRUN_TIMEOUT) defaults to 10 + # minutes; tighten it here since we do not need to wait for worker. + SYSTEM_TESTS_OL_DAGRUN_TIMEOUT_MINUTES: '5' + AIRFLOW__CORE__TASK_SUCCESS_OVERTIME: '70' + # Serve the dags folder through a versioned bundle so example_openlineage_versioned_dag gets a + # non-null dag_bundle_version. MockVersionedLocalDagBundle still serves from /opt/airflow/dags + # (keeping PYTHONPATH valid); it is copied into the dags folder by prepare_dags.py. + AIRFLOW__DAG_PROCESSOR__DAG_BUNDLE_CONFIG_LIST: '[{"name": "dags-folder", "classpath": "e2e_versioned_bundle.MockVersionedLocalDagBundle", "kwargs": {}}]' + # The breeze PROD image installs every provider from source, including informatica, whose listener + # fires on every task and errors without its connection — suppressing OL events. A curated + # deployment (e.g. Astro) does not ship it; disable its listener to match. + AIRFLOW__INFORMATICA__LISTENER_DISABLED: 'true' + +x-ol-common: &ol-common + environment: + <<: *ol-common-env + volumes: + # example_openlineage_docs_file_dag sets doc_md="dag_doc.md", which Airflow resolves with a + # CWD-relative open(); components run with CWD=/opt/airflow. prepare_dags.py generates a clean, + # license-free dag_doc.md (matching the expected event) in the dags folder; expose it here. + - ./dags/dag_doc.md:/opt/airflow/dag_doc.md + +services: + airflow-apiserver: + <<: *ol-common + airflow-scheduler: + <<: *ol-common + airflow-dag-processor: + <<: *ol-common + airflow-triggerer: + <<: *ol-common + airflow-worker: + <<: *ol-common diff --git a/airflow-e2e-tests/tests/airflow_e2e_tests/conftest.py b/airflow-e2e-tests/tests/airflow_e2e_tests/conftest.py index eb6ed32e9a63c..af72b97e7cf62 100644 --- a/airflow-e2e-tests/tests/airflow_e2e_tests/conftest.py +++ b/airflow-e2e-tests/tests/airflow_e2e_tests/conftest.py @@ -52,6 +52,7 @@ KAFKA_DIR_PATH, LOCALSTACK_PATH, LOGS_FOLDER, + OPENLINEAGE_COMPOSE_PATH, OPENSEARCH_PATH, PROVIDERS_MOUNT_CONTAINER_PATH, PROVIDERS_ROOT_PATH, @@ -69,6 +70,7 @@ class _E2ETestState: compose_instance: DockerCompose | None = None airflow_logs_path: Path | None = None + airflow_dags_path: Path | None = None def _copy_localstack_files(tmp_dir): @@ -566,6 +568,21 @@ def _setup_go_sdk_integration(dot_env_file, tmp_dir): os.environ["ENV_FILE_PATH"] = str(dot_env_file) +def _setup_openlineage_integration(dot_env_file, tmp_dir): + """Set up the openlineage E2E test mode. + + The OpenLineage system-test DAGs are the single source of truth; ``prepare_dags`` copies them + into the stack's dags folder (stripping the pytest-only footer) alongside the harness-only warmup + DAG and versioned bundle. The ``openlineage.yml`` overlay carries the OpenLineage env config and + mounts the generated ``dag_doc.md`` where the docs DAG resolves it. + """ + from airflow_e2e_tests.openlineage_tests.prepare_dags import prepare_dags + + console.print("[yellow]Preparing OpenLineage DAGs from the provider system tests...") + prepare_dags(tmp_dir / "dags") + copyfile(OPENLINEAGE_COMPOSE_PATH, tmp_dir / "openlineage.yml") + + def spin_up_airflow_environment(tmp_path_factory: pytest.TempPathFactory): tmp_dir = tmp_path_factory.mktemp("breeze-airflow-e2e-tests") @@ -580,9 +597,13 @@ def spin_up_airflow_environment(tmp_path_factory: pytest.TempPathFactory): (tmp_dir / subdir).mkdir() _E2ETestState.airflow_logs_path = tmp_dir / "logs" + _E2ETestState.airflow_dags_path = tmp_dir / "dags" - console.print(f"[yellow]Copying dags to:[/ {tmp_dir / 'dags'}") - copytree(E2E_DAGS_FOLDER, tmp_dir / "dags", dirs_exist_ok=True) + # openlineage sources its dags from the provider system tests (via _setup_openlineage_integration), + # so it must not also load the stock e2e dags — the harness triggers every dag it finds. + if E2E_TEST_MODE != "openlineage": + console.print(f"[yellow]Copying dags to:[/ {tmp_dir / 'dags'}") + copytree(E2E_DAGS_FOLDER, tmp_dir / "dags", dirs_exist_ok=True) dot_env_file = tmp_dir / ".env" dot_env_file.write_text(f"AIRFLOW_UID={os.getuid()}\n") @@ -613,6 +634,9 @@ def spin_up_airflow_environment(tmp_path_factory: pytest.TempPathFactory): elif E2E_TEST_MODE == "go_sdk": compose_file_names.append("go.yml") _setup_go_sdk_integration(dot_env_file, tmp_dir) + elif E2E_TEST_MODE == "openlineage": + compose_file_names.append("openlineage.yml") + _setup_openlineage_integration(dot_env_file, tmp_dir) # # Please Do not use this Fernet key in any deployments! Please generate your own key. @@ -708,6 +732,18 @@ def compose_instance(): return _E2ETestState.compose_instance +@pytest.fixture(scope="session") +def airflow_logs_path(): + """Live host path of the stack's task logs (bind-mounted), readable while tests run.""" + return _E2ETestState.airflow_logs_path + + +@pytest.fixture(scope="session") +def airflow_dags_path(): + """Host path of the dags served to the stack.""" + return _E2ETestState.airflow_dags_path + + def generate_test_report(results): """Generate test report with json summary.""" report = { diff --git a/airflow-e2e-tests/tests/airflow_e2e_tests/constants.py b/airflow-e2e-tests/tests/airflow_e2e_tests/constants.py index f3a67137608a5..b0392d96a8a7c 100644 --- a/airflow-e2e-tests/tests/airflow_e2e_tests/constants.py +++ b/airflow-e2e-tests/tests/airflow_e2e_tests/constants.py @@ -50,6 +50,10 @@ KAFKA_DIR_PATH = AIRFLOW_ROOT_PATH / "airflow-e2e-tests" / "docker" / "kafka" +# OpenLineage E2E test paths. The DAGs are sourced from the provider system tests at runtime by +# openlineage_tests/prepare_dags.py; the overlay carries the OpenLineage-specific env + dag_doc mount. +OPENLINEAGE_COMPOSE_PATH = AIRFLOW_ROOT_PATH / "airflow-e2e-tests" / "docker" / "openlineage.yml" + # Java SDK E2E test paths JAVA_SDK_ROOT_PATH = AIRFLOW_ROOT_PATH / "java-sdk" JAVA_SDK_EXAMPLE_DAGS_PATH = JAVA_SDK_ROOT_PATH / "example" / "src" / "resources" / "dags" diff --git a/providers-e2e-tests/openlineage/.pre-commit-config.yaml b/airflow-e2e-tests/tests/airflow_e2e_tests/openlineage_tests/__init__.py similarity index 61% rename from providers-e2e-tests/openlineage/.pre-commit-config.yaml rename to airflow-e2e-tests/tests/airflow_e2e_tests/openlineage_tests/__init__.py index 85fd7040301b9..13a83393a9124 100644 --- a/providers-e2e-tests/openlineage/.pre-commit-config.yaml +++ b/airflow-e2e-tests/tests/airflow_e2e_tests/openlineage_tests/__init__.py @@ -14,20 +14,3 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. ---- -default_stages: [pre-commit, pre-push] -minimum_prek_version: '0.3.4' -default_language_version: - python: python3 -repos: - - repo: local - hooks: - - id: mypy-openlineage-e2e-tests - name: Run mypy for providers-e2e-tests/openlineage - language: python - entry: >- - ../../scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py - providers-e2e-tests/openlineage - pass_filenames: false - files: ^.*\.py$ - require_serial: true diff --git a/providers-e2e-tests/openlineage/dags_extra/e2e_versioned_bundle.py b/airflow-e2e-tests/tests/airflow_e2e_tests/openlineage_tests/dags_extra/e2e_versioned_bundle.py similarity index 100% rename from providers-e2e-tests/openlineage/dags_extra/e2e_versioned_bundle.py rename to airflow-e2e-tests/tests/airflow_e2e_tests/openlineage_tests/dags_extra/e2e_versioned_bundle.py diff --git a/providers-e2e-tests/openlineage/dags_extra/openlineage_warmup_dag.py b/airflow-e2e-tests/tests/airflow_e2e_tests/openlineage_tests/dags_extra/openlineage_warmup_dag.py similarity index 100% rename from providers-e2e-tests/openlineage/dags_extra/openlineage_warmup_dag.py rename to airflow-e2e-tests/tests/airflow_e2e_tests/openlineage_tests/dags_extra/openlineage_warmup_dag.py diff --git a/providers-e2e-tests/openlineage/tests/harness.py b/airflow-e2e-tests/tests/airflow_e2e_tests/openlineage_tests/harness.py similarity index 71% rename from providers-e2e-tests/openlineage/tests/harness.py rename to airflow-e2e-tests/tests/airflow_e2e_tests/openlineage_tests/harness.py index 05c2b23674273..e1972a6006df7 100644 --- a/providers-e2e-tests/openlineage/tests/harness.py +++ b/airflow-e2e-tests/tests/airflow_e2e_tests/openlineage_tests/harness.py @@ -17,9 +17,8 @@ """ Drive all OpenLineage Dags in the deployed stack and collect their final run state. -Adapted from the OpenLineage dags-dashboard ``trigger_dag.py`` integration harness: it talks to a -local Airflow REST API v2 (rather than an Astro deployment), authenticates with a SimpleAuthManager -bearer token, and reports through logs only. The actual OpenLineage event validation happens inside +Talks to the local Airflow REST API v2 through the shared ``AirflowClient`` (same auth and retry +setup as the other airflow-e2e-tests suites). The actual OpenLineage event validation happens inside each Dag's terminal ``OpenLineageTestOperator`` task, so a run that ends ``success`` means its emitted events matched the expected templates. """ @@ -30,11 +29,15 @@ import re import time from pathlib import Path +from typing import TYPE_CHECKING import requests -from constants import console -from requests.adapters import HTTPAdapter -from urllib3.util.retry import Retry +from rich.console import Console + +if TYPE_CHECKING: + from airflow_e2e_tests.e2e_test_utils.clients import AirflowClient + +console = Console(width=400, color_system="standard") WARMUP_DAG_ID = "openlineage_warmup_dag" TRIGGER_DELAY_SECONDS = 2 @@ -67,59 +70,25 @@ def discover_expected_dag_ids(dags_folder: Path) -> set[str]: return dag_ids -def _session_with_retries() -> requests.Session: - retry = Retry( - total=10, - connect=5, - read=10, - backoff_factor=2, - raise_on_status=False, - allowed_methods=["GET", "POST", "PATCH", "DELETE"], - status_forcelist=[429, 500, 502, 503, 504], - ) - session = requests.Session() - session.mount("http://", HTTPAdapter(max_retries=retry)) - session.mount("https://", HTTPAdapter(max_retries=retry)) - return session - - class OpenLineageE2ERunner: """Triggers all OpenLineage Dags against a running deployment and collects their final states.""" - def __init__(self, api_base_url: str, headers: dict[str, str]): - self.api_url = f"{api_base_url}/api/v2" - self.headers = headers - self.session = _session_with_retries() + def __init__(self, client: AirflowClient): + self.client = client now = dt.datetime.now(tz=dt.timezone.utc) self.run_id = f"ci_triggered_{now.isoformat()}" self.retry_run_id = f"{self.run_id}_retry1" # dag_id -> whether its final state came from retry_run_id rather than run_id; populated by run(). self.retried_dag_ids: set[str] = set() - def wait_for_airflow_api(self, max_attempts: int = 30, poll_interval: int = 10) -> None: - for attempt in range(1, max_attempts + 1): - try: - response = requests.get(f"{self.api_url}/dags?limit=1", headers=self.headers, timeout=15) - if response.status_code < 500: - console.print(f"[green]Airflow API ready after {attempt} attempt(s)") - return - except requests.exceptions.RequestException as exc: - console.print(f"[yellow]Airflow API not ready (attempt {attempt}/{max_attempts}): {exc}") - time.sleep(poll_interval) - raise RuntimeError(f"Airflow API did not become ready after {max_attempts} attempts.") - def list_dags(self) -> list[str]: - response = self.session.get(f"{self.api_url}/dags?limit=500", headers=self.headers) - response.raise_for_status() - return [dag["dag_id"] for dag in response.json()["dags"]] + response = self.client._make_request(method="GET", endpoint="dags?limit=500") + return [dag["dag_id"] for dag in response["dags"]] def get_task_states(self, dag_id: str, run_id: str) -> dict[str, str]: """task_id -> state for every task instance in a dag run.""" - response = self.session.get( - f"{self.api_url}/dags/{dag_id}/dagRuns/{run_id}/taskInstances", headers=self.headers - ) - response.raise_for_status() - return {ti["task_id"]: ti["state"] for ti in response.json()["task_instances"]} + response = self.client.get_task_instances(dag_id, run_id) + return {ti["task_id"]: ti["state"] for ti in response["task_instances"]} def wait_for_dags_loaded(self, timeout: int = 60, poll_interval: int = 3) -> list[str]: """Poll until the dag-processor has parsed the Dags (the warmup Dag is the readiness marker).""" @@ -135,31 +104,27 @@ def wait_for_dags_loaded(self, timeout: int = 60, poll_interval: int = 3) -> lis raise RuntimeError(f"Dags did not load within {timeout}s (warmup Dag missing; have {dag_ids}).") def unpause_dag(self, dag_id: str) -> None: - response = self.session.patch( - f"{self.api_url}/dags/{dag_id}", headers=self.headers, json={"is_paused": False} - ) - if response.status_code != 200: - console.print(f"[red]Failed to unpause Dag `{dag_id}`: {response.text}") + try: + self.client._make_request(method="PATCH", endpoint=f"dags/{dag_id}", json={"is_paused": False}) + except requests.HTTPError as exc: + console.print(f"[red]Failed to unpause Dag `{dag_id}`: {exc}") def trigger_dag_run(self, dag_id: str, run_id: str) -> bool: now = dt.datetime.now(tz=dt.timezone.utc).isoformat() payload = {"dag_run_id": run_id, "logical_date": now, "conf": {}} - response = self.session.post( - f"{self.api_url}/dags/{dag_id}/dagRuns", headers=self.headers, json=payload - ) - if response.status_code not in (200, 201): - console.print(f"[red]Failed to trigger Dag `{dag_id}`: {response.text}") + try: + self.client._make_request(method="POST", endpoint=f"dags/{dag_id}/dagRuns", json=payload) + except requests.HTTPError as exc: + console.print(f"[red]Failed to trigger Dag `{dag_id}`: {exc}") return False return True def wait_for_dag_run_to_complete(self, dag_id: str, run_id: str, timeout: int = 300) -> str: - url = f"{self.api_url}/dags/{dag_id}/dagRuns/{run_id}" deadline = time.monotonic() + timeout state = "unknown" while time.monotonic() < deadline: - response = self.session.get(url, headers=self.headers) - response.raise_for_status() - state = response.json()["state"] + response = self.client._make_request(method="GET", endpoint=f"dags/{dag_id}/dagRuns/{run_id}") + state = response["state"] if state not in ("running", "queued"): break time.sleep(5) @@ -167,11 +132,10 @@ def wait_for_dag_run_to_complete(self, dag_id: str, run_id: str, timeout: int = return state def clear_airflow_variables(self) -> None: - response = self.session.get(f"{self.api_url}/variables?limit=500", headers=self.headers) - response.raise_for_status() - keys = [variable["key"] for variable in response.json()["variables"]] + response = self.client._make_request(method="GET", endpoint="variables?limit=500") + keys = [variable["key"] for variable in response["variables"]] for key in keys: - self.session.delete(f"{self.api_url}/variables/{key}", headers=self.headers) + self.client._make_request(method="DELETE", endpoint=f"variables/{key}") def warmup(self, dag_ids: list[str]) -> None: """Unpause all Dags and run the warmup Dag so the worker is confirmed ready.""" @@ -210,7 +174,6 @@ def _trigger_and_wait( def run(self, expected_dag_ids: set[str]) -> dict[str, str]: """Run the full cycle and return ``{dag_id: final_state}`` for every triggered Dag.""" - self.wait_for_airflow_api() dag_ids = self.wait_for_dags_loaded() if not dag_ids: raise ValueError("No Dags found in the deployment.") diff --git a/providers-e2e-tests/openlineage/prepare_dags.py b/airflow-e2e-tests/tests/airflow_e2e_tests/openlineage_tests/prepare_dags.py similarity index 61% rename from providers-e2e-tests/openlineage/prepare_dags.py rename to airflow-e2e-tests/tests/airflow_e2e_tests/openlineage_tests/prepare_dags.py index 7d6b51f45f855..81bb46ab165d1 100644 --- a/providers-e2e-tests/openlineage/prepare_dags.py +++ b/airflow-e2e-tests/tests/airflow_e2e_tests/openlineage_tests/prepare_dags.py @@ -20,8 +20,8 @@ The OpenLineage system-test DAGs in ``providers/openlineage/tests/system/openlineage`` are the single source of truth. They are *designed* to run in-process via pytest, so each one ends with a ``get_test_run`` block that imports ``tests_common`` — which is not present in the PROD image. This -script copies the whole ``system`` package into the e2e dags folder and strips that pytest-only -footer so the DAGs parse and run in a real deployment. +copies the whole ``system`` package into the e2e dags folder and strips that pytest-only footer so +the DAGs parse and run in a real deployment. The package layout is preserved as ``/system/openlineage/...`` and the dags folder is added to ``PYTHONPATH`` in docker-compose, so the DAGs' ``from system.openlineage... import ...`` and the @@ -30,23 +30,13 @@ from __future__ import annotations -import os import shutil from pathlib import Path HERE = Path(__file__).resolve().parent -AIRFLOW_ROOT_PATH = HERE.parents[1] - -# Example DAGs that need a minimum Airflow version. When running against an older version, the DAG -# file is removed entirely — a module-level import (e.g. the HITL operators) would otherwise raise at -# parse time and take the whole dag-processor down, not just skip that one DAG. -MIN_AIRFLOW_VERSION_FOR_DAG: dict[str, tuple[int, int]] = { - # HITL operators import-raise on Airflow < 3.1 (taking the dag-processor down at parse time). - "example_openlineage_hitl_dag.py": (3, 1), -} +AIRFLOW_ROOT_PATH = HERE.parents[4] SYSTEM_TESTS_SOURCE = AIRFLOW_ROOT_PATH / "providers" / "openlineage" / "tests" / "system" -DAGS_DEST = HERE / "dags" # Harness-only modules (warmup DAG, versioned bundle) that are not part of the provider system tests. DAGS_EXTRA_SOURCE = HERE / "dags_extra" @@ -54,17 +44,6 @@ PYTEST_FOOTER_MARKER = "from tests_common.test_utils.system_tests import get_test_run" -def _target_airflow_version() -> tuple[int, int] | None: - """Parse the (major, minor) of the targeted Airflow version, or None for the default/prod run.""" - raw = os.environ.get("E2E_TARGET_AIRFLOW_VERSION", "").strip() - if not raw: - return None - parts = raw.split(".") - if len(parts) < 2 or not (parts[0].isdigit() and parts[1].isdigit()): - return None - return int(parts[0]), int(parts[1]) - - def _strip_pytest_footer(dag_file: Path) -> None: lines = dag_file.read_text().splitlines(keepends=True) kept: list[str] = [] @@ -75,52 +54,34 @@ def _strip_pytest_footer(dag_file: Path) -> None: dag_file.write_text("".join(kept)) -def prepare_dags() -> Path: - """Populate the e2e dags folder from the provider system tests. Returns the dags folder path.""" +def prepare_dags(dest: Path) -> Path: + """Populate ``dest`` with the OpenLineage e2e dags from the provider system tests.""" if not SYSTEM_TESTS_SOURCE.is_dir(): raise FileNotFoundError(f"OpenLineage system tests not found at {SYSTEM_TESTS_SOURCE}") - if DAGS_DEST.exists(): - for child in DAGS_DEST.iterdir(): - if child.is_dir(): - shutil.rmtree(child) - else: - child.unlink() - else: - DAGS_DEST.mkdir(parents=True) + dest.mkdir(parents=True, exist_ok=True) # Copy the whole `system` package so `system.openlineage.{operator,transport,expected_events}` # imports resolve from the dags folder. - shutil.copytree(SYSTEM_TESTS_SOURCE, DAGS_DEST / "system") + shutil.copytree(SYSTEM_TESTS_SOURCE, dest / "system", dirs_exist_ok=True) - openlineage_dir = DAGS_DEST / "system" / "openlineage" + openlineage_dir = dest / "system" / "openlineage" # The pytest conftest imports `pytest`, which is absent from the PROD image; drop it so the # dag-processor does not choke on it. (openlineage_dir / "conftest.py").unlink(missing_ok=True) - target_version = _target_airflow_version() - if target_version is not None: - for dag_file_name, min_version in MIN_AIRFLOW_VERSION_FOR_DAG.items(): - if target_version < min_version: - (openlineage_dir / dag_file_name).unlink(missing_ok=True) - for dag_file in openlineage_dir.glob("example_openlineage_*.py"): _strip_pytest_footer(dag_file) for extra_file in DAGS_EXTRA_SOURCE.glob("*.py"): - shutil.copy2(extra_file, DAGS_DEST / extra_file.name) + shutil.copy2(extra_file, dest / extra_file.name) # example_openlineage_docs_file_dag uses doc_md="dag_doc.md" and the expected event's # documentation.description is exactly "# MD doc file". The provider's own dag_doc.md carries an # Apache license header (required for repo files), so OpenLineage would emit the header too. # Generate a clean, license-free copy here (it lives only in the gitignored dags folder, never in # the repo); docker-compose mounts it into the parsing CWD so doc_md resolves to this content. - (DAGS_DEST / "dag_doc.md").write_text("# MD doc file") - - return DAGS_DEST - + (dest / "dag_doc.md").write_text("# MD doc file") -if __name__ == "__main__": - dest = prepare_dags() - print(f"Prepared OpenLineage e2e dags in {dest}") + return dest diff --git a/providers-e2e-tests/openlineage/tests/task_logs.py b/airflow-e2e-tests/tests/airflow_e2e_tests/openlineage_tests/task_logs.py similarity index 100% rename from providers-e2e-tests/openlineage/tests/task_logs.py rename to airflow-e2e-tests/tests/airflow_e2e_tests/openlineage_tests/task_logs.py diff --git a/providers-e2e-tests/openlineage/tests/test_openlineage_e2e.py b/airflow-e2e-tests/tests/airflow_e2e_tests/openlineage_tests/test_openlineage_e2e.py similarity index 78% rename from providers-e2e-tests/openlineage/tests/test_openlineage_e2e.py rename to airflow-e2e-tests/tests/airflow_e2e_tests/openlineage_tests/test_openlineage_e2e.py index 277ce2a942214..081d2f9dcbe04 100644 --- a/providers-e2e-tests/openlineage/tests/test_openlineage_e2e.py +++ b/airflow-e2e-tests/tests/airflow_e2e_tests/openlineage_tests/test_openlineage_e2e.py @@ -16,14 +16,21 @@ # under the License. from __future__ import annotations +from pathlib import Path + import pytest import requests -from constants import API_BASE_URL, DAGS_FOLDER, LOGS_FOLDER, console -from harness import OpenLineageE2ERunner, discover_expected_dag_ids -from task_logs import find_log_paths, print_task_log + +from airflow_e2e_tests.e2e_test_utils.clients import AirflowClient +from airflow_e2e_tests.openlineage_tests.harness import ( + OpenLineageE2ERunner, + console, + discover_expected_dag_ids, +) +from airflow_e2e_tests.openlineage_tests.task_logs import find_log_paths, print_task_log -def _print_failure_logs(runner: OpenLineageE2ERunner, failed: dict[str, str]) -> None: +def _print_failure_logs(runner: OpenLineageE2ERunner, logs_folder: Path, failed: dict[str, str]) -> None: """Print the relevant task logs for each failed DAG, so CI output doesn't require the log artifact.""" for dag_id, state in failed.items(): if state == "missing": @@ -44,29 +51,29 @@ def _print_failure_logs(runner: OpenLineageE2ERunner, failed: dict[str, str]) -> for task_id, ti_state in task_states.items(): if ti_state == "success" or task_id in tasks_to_show: continue - if ti_state == "skipped" and not find_log_paths(LOGS_FOLDER, dag_id, run_id, task_id): + if ti_state == "skipped" and not find_log_paths(logs_folder, dag_id, run_id, task_id): continue tasks_to_show.add(task_id) for task_id in sorted(tasks_to_show): - print_task_log(console, LOGS_FOLDER, dag_id, run_id, task_id) + print_task_log(console, logs_folder, dag_id, run_id, task_id) @pytest.mark.execution_timeout(900) # 15 min -def test_all_openlineage_dags_succeed(auth_headers): +def test_all_openlineage_dags_succeed(compose_instance, airflow_dags_path, airflow_logs_path): """ Trigger every OpenLineage system-test DAG in the deployment and require each run to succeed. The terminal ``OpenLineageTestOperator`` task inside each DAG validates the emitted OpenLineage events against the expected templates, so a ``success`` run state means the lineage matched. """ - expected_dag_ids = discover_expected_dag_ids(DAGS_FOLDER) + expected_dag_ids = discover_expected_dag_ids(airflow_dags_path) assert expected_dag_ids, "No expected OpenLineage DAG ids were discovered from the prepared dags" - runner = OpenLineageE2ERunner(API_BASE_URL, auth_headers) + runner = OpenLineageE2ERunner(AirflowClient()) statuses = runner.run(expected_dag_ids) console.print(f"[blue]OpenLineage e2e results ({len(statuses)} DAGs): {statuses}") failed = {dag_id: state for dag_id, state in sorted(statuses.items()) if state != "success"} if failed: - _print_failure_logs(runner, failed) + _print_failure_logs(runner, airflow_logs_path, failed) assert not failed, f"OpenLineage e2e DAG runs were not successful: {failed}" diff --git a/dev/breeze/doc/05_test_commands.rst b/dev/breeze/doc/05_test_commands.rst index 52256c930fbfb..0415431434419 100644 --- a/dev/breeze/doc/05_test_commands.rst +++ b/dev/breeze/doc/05_test_commands.rst @@ -371,28 +371,12 @@ a local venv. You can build the prod image with breeze and that will be used by You can override the ``DOCKER_IMAGE`` environment variable to point to the image to test using the ``breeze testing airflow-e2e-tests`` command. -The Airflow E2E tests are in ``airflow-e2e-tests/`` folder in the main repo. - -Running provider E2E tests -.......................... - -You can use Breeze to run provider end-to-end tests, run against a real built Airflow image -(``$DOCKER_IMAGE``) rather than the in-process ``dag.test()`` path used by provider system tests. -What "e2e" means, and how the stack under test gets stood up, is entirely up to the provider — the -OpenLineage suite, for example, deploys a real Airflow via docker-compose and runs its system-test -DAGs against it, but a different provider could drive it a completely different way. Those tests run -using the Production image by default; pass ``--airflow-version`` to run against an older released -Airflow version with the current providers installed from main. - -.. image:: ./images/output_testing_providers-e2e-tests.svg - :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/images/output_testing_providers-e2e-tests.svg - :width: 100% - :alt: Breeze testing providers-e2e-tests - -The provider E2E tests are in the ``providers-e2e-tests/`` folder in the main repo; see -``providers-e2e-tests/README.md`` for how to run any provider's suite, how it's wired into CI, and -what's involved in adding a new provider (``providers-e2e-tests/openlineage/`` is one -example implementation). +The Airflow E2E tests are in ``airflow-e2e-tests/`` folder in the main repo. Each suite is a +``--e2e-test-mode`` (``basic``, ``remote_log``, ``event_driven``, ``java_sdk``, ``go_sdk``, +``openlineage``, ...); the mode selects a docker-compose overlay and the matching test package under +``airflow-e2e-tests/tests/airflow_e2e_tests/_tests``. For example, the ``openlineage`` mode +deploys a real Airflow and runs the OpenLineage provider's system-test DAGs against it, asserting the +emitted lineage events match — run it with ``breeze testing airflow-e2e-tests --e2e-test-mode openlineage``. Running Airflow UI E2E tests ............................. diff --git a/dev/breeze/doc/ci/04_selective_checks.md b/dev/breeze/doc/ci/04_selective_checks.md index bc131cdd066b7..293b1c0cca66d 100644 --- a/dev/breeze/doc/ci/04_selective_checks.md +++ b/dev/breeze/doc/ci/04_selective_checks.md @@ -427,9 +427,10 @@ together using `pytest-xdist` (pytest-xdist distributes the tests among parallel of affected providers (but not recursively - only direct dependencies are added) * if there are any changes to "common" provider code not belonging to any provider (usually system tests or tests), then tests for all Providers are run -* `OpenLineage E2E tests` (the deployed-stack tests under `providers-e2e-tests/openlineage`, exposed as - the `run-providers-e2e-tests-openlineage` output) run when the `openlineage` or `common` providers or the - `providers-e2e-tests` harness change — and always on `canary` runs (where `full tests needed` also +* `OpenLineage E2E tests` (the `openlineage` mode of the deployed-stack tests under + `airflow-e2e-tests/tests/airflow_e2e_tests/openlineage_tests`, exposed as the + `run-openlineage-e2e-tests` output) run when the `openlineage` or `common` providers or the + openlineage e2e suite change — and always on `canary` runs (where `full tests needed` also covers core/task-sdk changes). Like the other deployed e2e suites, enabling them forces `PROD Image building`. * The specific unit test type is enabled only if changed files match the expected patterns for each type diff --git a/dev/breeze/doc/images/output_setup_check-all-params-in-groups.svg b/dev/breeze/doc/images/output_setup_check-all-params-in-groups.svg index 22be2e9030b39..202eba652be36 100644 --- a/dev/breeze/doc/images/output_setup_check-all-params-in-groups.svg +++ b/dev/breeze/doc/images/output_setup_check-all-params-in-groups.svg @@ -240,7 +240,7 @@ │setup:config | setup:regenerate-command-images | setup:self-upgrade | setup:synchronize-local-mounts | │ │setup:version | shell | start-airflow | testing | testing:airflow-ctl-integration-tests | │ │testing:airflow-ctl-tests | testing:airflow-e2e-tests | testing:core-integration-tests | │ -│testing:core-tests | testing:docker-compose-tests | testing:helm-tests | testing:providers-e2e-tests | │ +│testing:core-tests | testing:docker-compose-tests | testing:helm-tests | │ │testing:providers-integration-tests | testing:providers-tests | testing:python-api-client-tests | │ │testing:system-tests | testing:task-sdk-integration-tests | testing:task-sdk-tests | testing:ui-e2e-tests│ │| ui | ui:check-translation-completeness | ui:compile-assets | workflow-run | workflow-run:publish-docs)│ diff --git a/dev/breeze/doc/images/output_setup_check-all-params-in-groups.txt b/dev/breeze/doc/images/output_setup_check-all-params-in-groups.txt index ef361b8db62cf..92e75f17fd456 100644 --- a/dev/breeze/doc/images/output_setup_check-all-params-in-groups.txt +++ b/dev/breeze/doc/images/output_setup_check-all-params-in-groups.txt @@ -1 +1 @@ -ebfe38ba1a4b30b85d55382436ef1834 +fc5f65d76377357873c6b3ea200f2ec8 diff --git a/dev/breeze/doc/images/output_setup_regenerate-command-images.svg b/dev/breeze/doc/images/output_setup_regenerate-command-images.svg index 88faae5eb3574..de56522e8180b 100644 --- a/dev/breeze/doc/images/output_setup_regenerate-command-images.svg +++ b/dev/breeze/doc/images/output_setup_regenerate-command-images.svg @@ -258,10 +258,10 @@ │| setup:synchronize-local-mounts | setup:version | shell | start-airflow | testing | │ │testing:airflow-ctl-integration-tests | testing:airflow-ctl-tests | testing:airflow-e2e-tests | │ │testing:core-integration-tests | testing:core-tests | testing:docker-compose-tests | │ -│testing:helm-tests | testing:providers-e2e-tests | testing:providers-integration-tests | │ -│testing:providers-tests | testing:python-api-client-tests | testing:system-tests | │ -│testing:task-sdk-integration-tests | testing:task-sdk-tests | testing:ui-e2e-tests | ui | │ -│ui:check-translation-completeness | ui:compile-assets | workflow-run | workflow-run:publish-docs)│ +│testing:helm-tests | testing:providers-integration-tests | testing:providers-tests | │ +│testing:python-api-client-tests | testing:system-tests | testing:task-sdk-integration-tests | │ +│testing:task-sdk-tests | testing:ui-e2e-tests | ui | ui:check-translation-completeness | │ +│ui:compile-assets | workflow-run | workflow-run:publish-docs)│ │--check-onlyOnly check if some images need to be regenerated. Return 0 if no need or 1 if needed. Cannot be used  │ │together with --command flag or --force.                                                              │ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/dev/breeze/doc/images/output_setup_regenerate-command-images.txt b/dev/breeze/doc/images/output_setup_regenerate-command-images.txt index 81d6cfb046ac2..fe2e502097d32 100644 --- a/dev/breeze/doc/images/output_setup_regenerate-command-images.txt +++ b/dev/breeze/doc/images/output_setup_regenerate-command-images.txt @@ -1 +1 @@ -021adb1e353dab626f2092d96b2e7b45 +141b4bc2fbf13eddd1301de1064bf9fe diff --git a/dev/breeze/doc/images/output_testing.svg b/dev/breeze/doc/images/output_testing.svg index a05e3458ee1dd..60192bbfea0d5 100644 --- a/dev/breeze/doc/images/output_testing.svg +++ b/dev/breeze/doc/images/output_testing.svg @@ -1,4 +1,4 @@ - + ╭─ UI Tests ───────────────────────────────────────────────────────────────────────────────────────────────────────────╮ │ui-e2e-tests               Run UI End-to-End tests using Playwright.                                              │ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Commands ───────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│providers-e2e-tests        Run provider end-to-end tests against a deployed Airflow stack.                        │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/dev/breeze/doc/images/output_testing.txt b/dev/breeze/doc/images/output_testing.txt index bee169a1b6781..e80be8356c537 100644 --- a/dev/breeze/doc/images/output_testing.txt +++ b/dev/breeze/doc/images/output_testing.txt @@ -1 +1 @@ -f7645849511cf80e1747ddb411a881f5 +5c361e73595950fbdbe6d3aa8fd3855b diff --git a/dev/breeze/doc/images/output_testing_airflow-e2e-tests.svg b/dev/breeze/doc/images/output_testing_airflow-e2e-tests.svg index 0ba82f2e91353..e650495eb5410 100644 --- a/dev/breeze/doc/images/output_testing_airflow-e2e-tests.svg +++ b/dev/breeze/doc/images/output_testing_airflow-e2e-tests.svg @@ -135,7 +135,7 @@ │(TEXT)│ │--e2e-test-mode               Specify the mode to use for E2E tests. [default: basic]│ │(basic|remote_log|remote_log_elasticsearch|remote_log_opensearch|xcom_object_sto│ -│rage|event_driven|java_sdk|go_sdk)│ +│rage|event_driven|java_sdk|go_sdk|openlineage)│ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ │--verbose-vPrint verbose information about performed steps.│ diff --git a/dev/breeze/doc/images/output_testing_airflow-e2e-tests.txt b/dev/breeze/doc/images/output_testing_airflow-e2e-tests.txt index 58ae5ef183273..c68f420f5600f 100644 --- a/dev/breeze/doc/images/output_testing_airflow-e2e-tests.txt +++ b/dev/breeze/doc/images/output_testing_airflow-e2e-tests.txt @@ -1 +1 @@ -518a65f4380756c2c7b0f9c79aa3e3bc +712ca37b265a35cb79a70e42f189859b diff --git a/dev/breeze/doc/images/output_testing_providers-e2e-tests.svg b/dev/breeze/doc/images/output_testing_providers-e2e-tests.svg deleted file mode 100644 index e727fbeb032e4..0000000000000 --- a/dev/breeze/doc/images/output_testing_providers-e2e-tests.svg +++ /dev/null @@ -1,182 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Command: testing providers-e2e-tests - - - - - - - - - - -Usage:breeze testing providers-e2e-tests[OPTIONS] {openlineage} [EXTRA_PYTEST_ARGS]... - -Run provider end-to-end tests against a deployed Airflow stack. -Defaults to the PROD image (current Airflow + providers from sources). Pass --airflow-version to run against an older  -released Airflow version with the current providers installed from main. - -╭─ Docker-compose tests flag ──────────────────────────────────────────────────────────────────────────────────────────╮ -│--image-name                  -nName of the image to verify (overrides --python). (TEXT)│ -│--python                      -pPython major/minor version used in Airflow image for images. [default: 3.10]│ -│(>3.10< | 3.11 | 3.12 | 3.13 | 3.14)│ -│--airflow-version             Run against a released Airflow version (e.g. 3.0.6) with current providers from │ -│main, instead of the default PROD image. Builds a lightweight image from        │ -│apache/airflow:<version>. (TEXT)│ -│--skip-docker-compose-deletionSkip deletion of docker-compose instance after the test│ -│--skip-mounting-local-volumes Skip mounting local volumes - useful when we do not want to iterate with        │ -│modified local files. For example, when the PROD image is built from packages   │ -│rather than from local sources.                                                 │ -│--down                        Shuts down the docker-compose setup without running any tests. Useful to make   │ -│sure to free resources.                                                         │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common CI options ──────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--include-success-outputsWhether to include outputs of successful runs (not shown by default).│ -│--github-repository      -gGitHub repository used to pull, push run images. [default: apache/airflow](TEXT)│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--help   -hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ - - - - diff --git a/dev/breeze/doc/images/output_testing_providers-e2e-tests.txt b/dev/breeze/doc/images/output_testing_providers-e2e-tests.txt deleted file mode 100644 index 806839eb74ae3..0000000000000 --- a/dev/breeze/doc/images/output_testing_providers-e2e-tests.txt +++ /dev/null @@ -1 +0,0 @@ -236c2c2140bb30b55a1c7a74f6044453 diff --git a/dev/breeze/src/airflow_breeze/commands/testing_commands.py b/dev/breeze/src/airflow_breeze/commands/testing_commands.py index 504ac0f8a5757..662b502806264 100644 --- a/dev/breeze/src/airflow_breeze/commands/testing_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/testing_commands.py @@ -115,7 +115,6 @@ ) from airflow_breeze.utils.path_utils import AIRFLOW_CTL_ROOT_PATH, FILES_PATH, cleanup_python_generated_files from airflow_breeze.utils.run_tests import ( - PROVIDERS_E2E_TESTS_ROOT_PATH, TASK_SDK_INTEGRATION_TESTS_ROOT_PATH, are_all_test_paths_excluded, file_name_from_test_type, @@ -923,174 +922,6 @@ def task_sdk_integration_tests( sys.exit(return_code) -def _available_e2e_providers() -> list[str]: - if not PROVIDERS_E2E_TESTS_ROOT_PATH.exists(): - return [] - return sorted( - d.name - for d in PROVIDERS_E2E_TESTS_ROOT_PATH.iterdir() - if d.is_dir() and (d / "pyproject.toml").exists() - ) - - -def _build_providers_e2e_compat_image(provider: str, airflow_version: str, python: str) -> str: - """Build a lightweight image: released ``apache/airflow:`` + current providers from main. - - Replicates the provider-compatibility approach (current provider code on an older Airflow core) - without a full PROD image build — the released image is pulled and the providers are reinstalled - from wheels built from main. - - The list of providers to install is read from ``providers-e2e-tests//pyproject.toml`` - under ``[tool.e2e-tests] required-providers``. - """ - import shutil - from pathlib import Path - - try: - import tomllib - except ImportError: - import tomli as tomllib # type: ignore[no-redef] - - from airflow_breeze.utils.path_utils import AIRFLOW_ROOT_PATH - - provider_root = PROVIDERS_E2E_TESTS_ROOT_PATH / provider - pyproject_path = provider_root / "pyproject.toml" - required_providers: list[str] = tomllib.loads(pyproject_path.read_text())["tool"]["e2e-tests"][ - "required-providers" - ] - console_print(f"[info]Building provider wheels from main: {required_providers}[/]") - run_command( - [ - "breeze", - "release-management", - "prepare-provider-distributions", - *required_providers, - "--distribution-format", - "wheel", - "--skip-tag-check", - "--include-not-ready-providers", - "--clean-dist", - ], - check=True, - cwd=AIRFLOW_ROOT_PATH, - ) - provider_dist = provider_root / "provider_dist" - if provider_dist.exists(): - shutil.rmtree(provider_dist) - provider_dist.mkdir(parents=True) - for wheel in (AIRFLOW_ROOT_PATH / "dist").glob("*.whl"): # With --clean-dist above we can copy all - shutil.copy2(wheel, provider_dist / wheel.name) - - base_image = f"apache/airflow:{airflow_version}-python{python}" - image_name = f"{provider}-e2e/airflow:{airflow_version}-python{python}" - console_print(f"[info]Building {image_name} from {base_image}[/]") - run_command( - [ - "docker", - "build", - "--build-arg", - f"AIRFLOW_BASE_IMAGE={base_image}", - "-t", - image_name, - "-f", - (provider_root / "Dockerfile").as_posix(), - Path(provider_root).as_posix(), - ], - check=True, - ) - return image_name - - -@testing_group.command( - name="providers-e2e-tests", - context_settings=dict( - ignore_unknown_options=True, - allow_extra_args=True, - ), -) -@click.argument("provider", type=click.Choice(_available_e2e_providers())) -@option_python -@option_image_name -@click.option( - "--airflow-version", - help="Run against a released Airflow version (e.g. 3.0.6) with current providers from main, " - "instead of the default PROD image. Builds a lightweight image from apache/airflow:.", - default=None, -) -@option_skip_docker_compose_deletion -@option_skip_mounting_local_volumes -@click.option( - "--down", - help="Shuts down the docker-compose setup without running any tests. " - "Useful to make sure to free resources.", - is_flag=True, -) -@option_github_repository -@option_include_success_outputs -@option_verbose -@option_dry_run -@click.argument("extra_pytest_args", nargs=-1, type=click.Path(path_type=str)) -def providers_e2e_tests( - provider: str, - python: str, - image_name: str | None, - airflow_version: str | None, - down: bool, - skip_docker_compose_deletion: bool, - skip_mounting_local_volumes: bool, - github_repository: str, - include_success_outputs: bool, - extra_pytest_args: tuple, -): - """Run provider end-to-end tests against a deployed Airflow stack. - - Defaults to the PROD image (current Airflow + providers from sources). Pass --airflow-version to - run against an older released Airflow version with the current providers installed from main. - """ - perform_environment_checks() - - if airflow_version: - # prepare_dags.py reads this to drop DAGs that need a newer Airflow than the target. - os.environ["E2E_TARGET_AIRFLOW_VERSION"] = airflow_version - if image_name is None: - image_name = _build_providers_e2e_compat_image(provider, airflow_version, python) - else: - image_name = image_name or os.environ.get("DOCKER_IMAGE") - if not image_name: - build_params = BuildProdParams(python=python, github_repository=github_repository) - image_name = build_params.airflow_image_name - - if down: - env = { - **os.environ, - "DOCKER_IMAGE": image_name, - } - down_cmd = [ - "docker", - "compose", - "down", - "--remove-orphans", - "--volumes", - ] - console_print("[info]Running docker-compose down[/]") - run_command(down_cmd, output=None, check=False, env=env, cwd=PROVIDERS_E2E_TESTS_ROOT_PATH / provider) - sys.exit(0) - - console_print(f"[info]Running {provider} e2e tests with image: {image_name}[/]") - return_code, info = run_docker_compose_tests( - image_name=image_name, - python_version=python, - include_success_outputs=include_success_outputs, - extra_pytest_args=extra_pytest_args, - skip_docker_compose_deletion=skip_docker_compose_deletion, - skip_mounting_local_volumes=skip_mounting_local_volumes, - test_type="providers-e2e-tests", - provider=provider, - skip_image_check=bool(airflow_version), - ) - sys.exit(return_code) - - @testing_group.command( name="airflow-ctl-integration-tests", context_settings=dict( @@ -1616,6 +1447,7 @@ def python_api_client_tests( "event_driven", "java_sdk", "go_sdk", + "openlineage", ], case_sensitive=False, ), diff --git a/dev/breeze/src/airflow_breeze/commands/testing_commands_config.py b/dev/breeze/src/airflow_breeze/commands/testing_commands_config.py index 4e4c1601a7443..56a4b17450239 100644 --- a/dev/breeze/src/airflow_breeze/commands/testing_commands_config.py +++ b/dev/breeze/src/airflow_breeze/commands/testing_commands_config.py @@ -215,26 +215,6 @@ ], }, ], - "breeze testing providers-e2e-tests": [ - { - "name": "Docker-compose tests flag", - "options": [ - "--image-name", - "--python", - "--airflow-version", - "--skip-docker-compose-deletion", - "--skip-mounting-local-volumes", - "--down", - ], - }, - { - "name": "Common CI options", - "options": [ - "--include-success-outputs", - "--github-repository", - ], - }, - ], "breeze testing airflow-ctl-tests": [ { "name": "Test environment", diff --git a/dev/breeze/src/airflow_breeze/utils/run_tests.py b/dev/breeze/src/airflow_breeze/utils/run_tests.py index d90b5f63dad93..ef87dd1f6e3a7 100644 --- a/dev/breeze/src/airflow_breeze/utils/run_tests.py +++ b/dev/breeze/src/airflow_breeze/utils/run_tests.py @@ -49,8 +49,6 @@ AIRFLOW_E2E_TESTS_ROOT_PATH = AIRFLOW_ROOT_PATH / "airflow-e2e-tests" -PROVIDERS_E2E_TESTS_ROOT_PATH = AIRFLOW_ROOT_PATH / "providers-e2e-tests" - AIRFLOW_CTL_TESTS_ROOT_PATH = AIRFLOW_ROOT_PATH / "airflow-ctl-tests" IGNORE_DB_INIT_FOR_TEST_GROUPS = [ @@ -125,7 +123,6 @@ def run_docker_compose_tests( test_type: str = "docker-compose", skip_image_check: bool = False, test_mode: str = "basic", - provider: str = "", ) -> tuple[int, str]: if not skip_image_check: command_result = run_command( @@ -168,9 +165,6 @@ def run_docker_compose_tests( elif test_type == "airflow-e2e-tests": test_path = Path("tests") / "airflow_e2e_tests" / f"{test_mode}_tests" cwd = AIRFLOW_E2E_TESTS_ROOT_PATH.as_posix() - elif test_type == "providers-e2e-tests": - test_path = Path("tests") - cwd = (PROVIDERS_E2E_TESTS_ROOT_PATH / provider).as_posix() elif test_type == "airflow-ctl-integration": test_path = Path("tests") / "airflowctl_tests" cwd = AIRFLOW_CTL_TESTS_ROOT_PATH.as_posix() diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py b/dev/breeze/src/airflow_breeze/utils/selective_checks.py index 5c05642c65a24..3a758a2160b4f 100644 --- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py +++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py @@ -132,7 +132,7 @@ class FileGroupForCi(Enum): EVENT_DRIVEN_E2E_FILES = auto() JAVA_SDK_E2E_FILES = auto() GO_SDK_E2E_FILES = auto() - PROVIDERS_E2E_OPENLINEAGE_FILES = auto() + OPENLINEAGE_E2E_FILES = auto() ALL_PYPROJECT_TOML_FILES = auto() ALL_PYTHON_FILES = auto() ALL_SOURCE_FILES = auto() @@ -251,8 +251,9 @@ def __hash__(self): r"^task-sdk/src/airflow/sdk/coordinators/_subprocess\.py$", r"^task-sdk/src/airflow/sdk/coordinators/executable/.*", ], - FileGroupForCi.PROVIDERS_E2E_OPENLINEAGE_FILES: [ - r"^providers-e2e-tests/openlineage/.*", + FileGroupForCi.OPENLINEAGE_E2E_FILES: [ + r"^airflow-e2e-tests/tests/airflow_e2e_tests/openlineage_tests/.*", + r"^airflow-e2e-tests/docker/openlineage\.yml$", r"^providers/openlineage/.*", r"^providers/common/compat/.*", r"^providers/common/io/.*", @@ -1045,8 +1046,8 @@ def run_go_sdk_e2e_tests(self) -> bool: return self._should_be_run(FileGroupForCi.GO_SDK_E2E_FILES) @cached_property - def run_providers_e2e_tests_openlineage(self) -> bool: - return self._should_be_run(FileGroupForCi.PROVIDERS_E2E_OPENLINEAGE_FILES) + def run_openlineage_e2e_tests(self) -> bool: + return self._should_be_run(FileGroupForCi.OPENLINEAGE_E2E_FILES) @cached_property def run_amazon_tests(self) -> bool: @@ -1190,7 +1191,7 @@ def prod_image_build(self) -> bool: or self.run_event_driven_e2e_tests or self.run_java_sdk_e2e_tests or self.run_go_sdk_e2e_tests - or self.run_providers_e2e_tests_openlineage + or self.run_openlineage_e2e_tests or self.run_ui_e2e_tests ) diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index 1e0e2644ffd5d..973ec115ca75d 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -1203,7 +1203,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): "ci-image-build": "true", # common providers feed OpenLineage e2e tests, which need the PROD image. "prod-image-build": "true", - "run-providers-e2e-tests-openlineage": "true", + "run-openlineage-e2e-tests": "true", "run-helm-tests": "false", "run-unit-tests": "true", "run-amazon-tests": "false", @@ -1477,7 +1477,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): pytest.param( ("providers/openlineage/src/airflow/providers/openlineage/plugins/adapter.py",), { - "run-providers-e2e-tests-openlineage": "true", + "run-openlineage-e2e-tests": "true", "prod-image-build": "true", }, id="Run OpenLineage e2e tests for openlineage provider change", @@ -1485,15 +1485,15 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): pytest.param( ("providers/common/compat/src/airflow/providers/common/compat/sdk.py",), { - "run-providers-e2e-tests-openlineage": "true", + "run-openlineage-e2e-tests": "true", "prod-image-build": "true", }, id="Run OpenLineage e2e tests for common provider change", ), pytest.param( - ("providers-e2e-tests/openlineage/tests/harness.py",), + ("airflow-e2e-tests/tests/airflow_e2e_tests/openlineage_tests/harness.py",), { - "run-providers-e2e-tests-openlineage": "true", + "run-openlineage-e2e-tests": "true", "prod-image-build": "true", }, id="Run OpenLineage e2e tests when the e2e harness changes", @@ -1501,7 +1501,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): pytest.param( ("providers/ftp/src/airflow/providers/ftp/hooks/ftp.py",), { - "run-providers-e2e-tests-openlineage": "false", + "run-openlineage-e2e-tests": "false", "prod-image-build": "false", }, id="Do not run OpenLineage e2e tests for unrelated provider change", diff --git a/providers-e2e-tests/README.md b/providers-e2e-tests/README.md deleted file mode 100644 index 588e13be45943..0000000000000 --- a/providers-e2e-tests/README.md +++ /dev/null @@ -1,111 +0,0 @@ - - -# Provider end-to-end tests - -Each subdirectory here is a self-contained e2e suite for one provider, that can be run against a real built -Airflow image (`$DOCKER_IMAGE`) rather than the in-process `dag.test()` path used by provider system tests. - -What "e2e" means, and how the stack under test gets stood up, is entirely up to the provider — see that provider's -own `README.md` for what its suite tests, how it works, and how to interpret its results. For an example implementation -(a docker-compose deployed stack), see [`openlineage`](openlineage) and its [README](openlineage/README.md). - -## Running locally - -Docker must be running. Every provider is driven by the same breeze command: - -```bash -# Default: PROD image (current Airflow from sources + providers from sources) -breeze testing providers-e2e-tests - -# An older released Airflow version, with the current providers installed from main -# (builds apache/airflow: + current provider wheels — no full PROD build) -breeze testing providers-e2e-tests --airflow-version 3.1.8 - -# Tear the stack down -breeze testing providers-e2e-tests --down -``` - -`` is any subdirectory of `providers-e2e-tests/` with a `pyproject.toml` — run -`breeze testing providers-e2e-tests --help` to see the current choices. - -Useful flags, passed to every provider's pytest run as env vars for its own harness to honor: - -- `--skip-docker-compose-deletion` (`SKIP_DOCKER_COMPOSE_DELETION`) — keep the stack up after the run to inspect it. -- `--skip-mounting-local-volumes` (`SKIP_MOUNTING_LOCAL_VOLUMES`) — run against the image only (what CI does). -Omit it during dev to mount and hot-reload local sources. -- Anything after the provider is passed through to pytest (e.g. `... -- -s`). - -The first run builds the PROD image; subsequent runs are much faster. -When running non-interactively without a pre-built image, prefix `ANSWER=yes` so the image-build prompt auto-confirms. - -## How this is wired into CI - -```text -ci-amd.yml / ci-arm.yml - └─ additional-prod-image-tests (job) ──uses──▶ additional-prod-image-tests.yml - └─ test-providers-e2e-tests-[-compat] (job) ──uses──▶ providers-e2e-tests.yml - └─ runs: breeze testing providers-e2e-tests [--airflow-version ] -``` - -`additional-prod-image-tests.yml` is a single reusable workflow called identically from both -`ci-amd.yml` and `ci-arm.yml`, so any job defined there — including each provider's e2e tests — -automatically gets both ARM and AMD coverage from one job definition. -[`scripts/ci/prek/check_ci_workflows_in_sync.py`](../scripts/ci/prek/check_ci_workflows_in_sync.py) -enforces that the two `ci-*.yml` files stay identical outside a small documented allowlist, so -don't add provider-e2e jobs directly to `ci-amd.yml`/`ci-arm.yml` — they belong in -`additional-prod-image-tests.yml`. - -Each provider's suite is gated by its own `run-providers-e2e-tests-` selective-checks output -(computed in `dev/breeze/src/airflow_breeze/utils/selective_checks.py` from the files changed in a -PR) — it always runs on canary/`main` runs, and on a PR only when that provider (or a file group it -depends on, e.g. `common` providers) changed. - -## Adding a new provider - -1. Create `providers-e2e-tests//`. The breeze harness (`breeze testing - providers-e2e-tests`, see `dev/breeze/src/airflow_breeze/commands/testing_commands.py` and - `run_docker_compose_tests` in `dev/breeze/src/airflow_breeze/utils/run_tests.py`) only requires: - - `pyproject.toml` — its presence is what makes `` a valid choice - (`_available_e2e_providers()`). It only needs a `[tool.e2e-tests] required-providers = [...]` - entry if the provider supports `--airflow-version` compat mode — that lists the provider - distributions to build from `main` for that mode (see `_build_providers_e2e_compat_image`). - - `tests/` — a pytest suite. Breeze runs `pytest tests/` from this directory with `$DOCKER_IMAGE` - (and the flags above) set as env vars; what it does with that image is entirely up to the - provider. - - `Dockerfile` — only needed if the provider supports `--airflow-version` compat mode: breeze - builds `apache/airflow:` + the provider wheels from `main` using it. -2. Wire the trigger into selective checks - (`dev/breeze/src/airflow_breeze/utils/selective_checks.py`): add a `FileGroupForCi` member and - its regex patterns, a `run_providers_e2e_tests_` cached property, and expose it as a - `run-providers-e2e-tests-` output — mirror the existing `PROVIDERS_E2E_OPENLINEAGE_FILES` / - `run_providers_e2e_tests_openlineage` pair. Update - [`dev/breeze/tests/test_selective_checks.py`](../dev/breeze/tests/test_selective_checks.py) and - [`dev/breeze/doc/ci/04_selective_checks.md`](../dev/breeze/doc/ci/04_selective_checks.md) in the - same PR. -3. Thread the new output through **both** `ci-amd.yml` and `ci-arm.yml` identically: add it to the - `build-info` job's `outputs:` and to the `additional-prod-image-tests` job's `with:` block. -4. Add the job(s) to `.github/workflows/additional-prod-image-tests.yml`, named - `test-providers-e2e-tests-` (and `test-providers-e2e-tests--compat` for the - compat matrix job, if any) — mirror the existing `test-providers-e2e-tests-openlineage` pair. - Call `providers-e2e-tests.yml` with `provider: ""` and - `provider-display-name: ""`, gated on - `inputs.canary-run == 'true' || inputs.run-providers-e2e-tests- == 'true'`. The compat - job additionally needs a `strategy.matrix` over `inputs.providers-compatibility-tests-matrix` and - `airflow-version: ${{ matrix.compat.airflow-version }}`. diff --git a/providers-e2e-tests/openlineage/.gitignore b/providers-e2e-tests/openlineage/.gitignore deleted file mode 100644 index fdf48551bf27f..0000000000000 --- a/providers-e2e-tests/openlineage/.gitignore +++ /dev/null @@ -1,7 +0,0 @@ -.env -*.iml -# DAGs are sourced from providers/openlineage/tests/system at runtime by prepare_dags.py -dags/ -logs/ -# Provider wheels built from main for older-version (compat) image builds -provider_dist/ diff --git a/providers-e2e-tests/openlineage/Dockerfile b/providers-e2e-tests/openlineage/Dockerfile deleted file mode 100644 index 3c1d772cf4b06..0000000000000 --- a/providers-e2e-tests/openlineage/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Lightweight image for running the OpenLineage e2e tests against an OLDER, released Airflow version. -# Instead of building a full PROD image, it starts from the published apache/airflow: image -# and reinstalls the providers under test (OpenLineage + the providers the system-test DAGs rely on) -# from wheels built from main — so the current provider code runs on an older Airflow core, the same -# idea as the provider compatibility tests. The default (prod) run does NOT use this Dockerfile; it -# uses the breeze PROD image with current sources. -ARG AIRFLOW_BASE_IMAGE -FROM ${AIRFLOW_BASE_IMAGE} - -COPY --chown=airflow:0 provider_dist/ /tmp/provider_dist/ -RUN pip install --no-cache-dir --upgrade /tmp/provider_dist/*.whl diff --git a/providers-e2e-tests/openlineage/README.md b/providers-e2e-tests/openlineage/README.md deleted file mode 100644 index 76c1bc5a56485..0000000000000 --- a/providers-e2e-tests/openlineage/README.md +++ /dev/null @@ -1,75 +0,0 @@ - - -# OpenLineage provider end-to-end tests - -These tests deploy a **real, multi-process Airflow** (via docker-compose) and run the OpenLineage -provider's system-test DAGs against it — exercising the integration the way it actually runs in a -deployment (separate scheduler / dag-processor / triggerer / api-server / workers), rather than the -in-process `dag.test()` path. - -See the [top-level README](../README.md) for how to run any provider's e2e suite with breeze, how -this is wired into CI, and what's involved in adding a new provider. - -## How it works - -- The DAGs, `OpenLineageTestOperator`, `VariableTransport`, and expected-event JSON are the ones in - [`providers/openlineage/tests/system/openlineage`](../../providers/openlineage/tests/system/openlineage) — - that directory stays the single source of truth. `prepare_dags.py` copies them into a (gitignored) - `dags/` folder at runtime and strips the pytest-only `get_test_run` footer so they parse in a - deployment. -- OpenLineage is configured to emit through the test `VariableTransport` (events land in Airflow - Variables); each DAG's terminal `OpenLineageTestOperator` task validates the emitted events against - the expected templates. **A DAG run that ends `success` means its lineage matched.** -- `harness.py` triggers every DAG through the REST API, waits for completion, retries a failed DAG - once, and asserts all expected DAGs succeeded. -- A small `MockVersionedLocalDagBundle` provides a versioned bundle (so `dag_bundle_version` is - populated) while still serving DAGs from the local dags folder. - -## Checking results - -- **pytest summary:** `1 passed` means every expected DAG is green. On failure it prints the - offenders, e.g. `... not successful: {'openlineage_docs_file_dag': 'failed'}`, and logs - `⚠ DAGs that passed only on retry (flaky first run): [...]`. -- **On failure, the relevant task logs are printed right in the pytest output** (`check_events` plus - any task that didn't succeed, filtered to warning/error lines — see `tests/task_logs.py`) — no - need to open the log artifact for the common case, e.g. the exact mismatch - (``Path `job > facets > ...`: expected X but got Y``). -- **Per-DAG logs** are still written in full to `logs/dag_id=/.../task_id=/attempt=1.log` - for deeper digging beyond what gets printed. -- With `--skip-docker-compose-deletion`, the stack stays up: get a token from - `curl -s http://localhost:8080/auth/token` and check `GET /api/v2/importErrors` for DAG parse - errors, or browse the runs in the UI at (login `airflow` / `airflow`). - -## Version-specific DAGs - -Some DAGs require a newer Airflow than the compat targets and are dropped for older versions in -`prepare_dags.py` (`MIN_AIRFLOW_VERSION_FOR_DAG`) — e.g. `example_openlineage_hitl_dag` needs 3.1+ -(its operators import-raise on older cores). Add an entry there when a new DAG is version-gated. - -## Layout - -```text -docker-compose.yaml # the deployed stack (postgres + apiserver + scheduler + dag-processor + triggerer) -docker-compose-local.yaml # overlay that mounts local sources (used unless --skip-mounting-local-volumes) -Dockerfile # lightweight image for --airflow-version: apache/airflow: + current providers -prepare_dags.py # sources DAGs from the provider system tests into dags/ at runtime -dags_extra/ # harness-only DAGs/modules (warmup DAG, versioned bundle) copied into dags/ -tests/ # conftest (compose lifecycle), harness, constants, and the pytest entrypoint -``` diff --git a/providers-e2e-tests/openlineage/docker-compose-local.yaml b/providers-e2e-tests/openlineage/docker-compose-local.yaml deleted file mode 100644 index 70d543f809a96..0000000000000 --- a/providers-e2e-tests/openlineage/docker-compose-local.yaml +++ /dev/null @@ -1,56 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# ---- -# If the image is locally built from the repo we can mount sources of Airflow and the -# OpenLineage provider so we do not have to rebuild the image on every change. -services: - airflow-init: - volumes: - - ../../airflow-core/src:/opt/airflow/airflow-core/src - - ../../task-sdk/src:/opt/airflow/task-sdk/src - - ../../providers/openlineage/src:/opt/airflow/providers/openlineage/src - environment: - - DEV_MODE=true - airflow-apiserver: - volumes: - - ../../airflow-core/src:/opt/airflow/airflow-core/src - - ../../task-sdk/src:/opt/airflow/task-sdk/src - - ../../providers/openlineage/src:/opt/airflow/providers/openlineage/src - environment: - - DEV_MODE=true - airflow-scheduler: - volumes: - - ../../airflow-core/src:/opt/airflow/airflow-core/src - - ../../task-sdk/src:/opt/airflow/task-sdk/src - - ../../providers/openlineage/src:/opt/airflow/providers/openlineage/src - environment: - - DEV_MODE=true - airflow-dag-processor: - volumes: - - ../../airflow-core/src:/opt/airflow/airflow-core/src - - ../../task-sdk/src:/opt/airflow/task-sdk/src - - ../../providers/openlineage/src:/opt/airflow/providers/openlineage/src - environment: - - DEV_MODE=true - airflow-triggerer: - volumes: - - ../../airflow-core/src:/opt/airflow/airflow-core/src - - ../../task-sdk/src:/opt/airflow/task-sdk/src - - ../../providers/openlineage/src:/opt/airflow/providers/openlineage/src - environment: - - DEV_MODE=true diff --git a/providers-e2e-tests/openlineage/docker-compose.yaml b/providers-e2e-tests/openlineage/docker-compose.yaml deleted file mode 100644 index 8adcc95e80884..0000000000000 --- a/providers-e2e-tests/openlineage/docker-compose.yaml +++ /dev/null @@ -1,181 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# ---- -x-airflow-common: - &airflow-common - image: ${AIRFLOW_IMAGE_NAME} - env_file: - - ${ENV_FILE_PATH:-.env} - environment: - &airflow-common-env - AIRFLOW__CORE__EXECUTOR: LocalExecutor - # yamllint disable rule:line-length - AIRFLOW__CORE__AUTH_MANAGER: 'airflow.api_fastapi.auth.managers.simple.simple_auth_manager.SimpleAuthManager' - AIRFLOW__CORE__SIMPLE_AUTH_MANAGER_ALL_ADMINS: 'true' - AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://airflow:airflow@postgres/airflow - AIRFLOW__CORE__FERNET_KEY: ${FERNET_KEY} - AIRFLOW__CORE__LOAD_EXAMPLES: 'false' - AIRFLOW__CORE__DAGS_FOLDER: '/opt/airflow/dags' - AIRFLOW__CORE__EXECUTION_API_SERVER_URL: 'http://airflow-apiserver:8080/execution/' - AIRFLOW__API__BASE_URL: 'http://airflow-apiserver:8080/' - # Do not reuse this secret in any deployment — it exists only for this ephemeral CI/test stack. - AIRFLOW__API_AUTH__JWT_SECRET: 'test-secret-key-for-testing' - # The OpenLineage system-test DAGs (and the custom VariableTransport / OpenLineageTestOperator - # they import) live under /system/openlineage; the dags folder is added to PYTHONPATH so - # both DAG parsing and the dotted-path transport class resolve in every component. - PYTHONPATH: '/opt/airflow/dags' - # Route OpenLineage events to the test VariableTransport (stores events in Airflow Variables, - # read back and validated by the in-DAG OpenLineageTestOperator). The transport type is the - # fully-qualified class path; the OpenLineage client imports it directly. - AIRFLOW__OPENLINEAGE__TRANSPORT: '{"type": "system.openlineage.transport.variable.VariableTransport"}' - AIRFLOW__OPENLINEAGE__DISABLED: 'false' - # Deployment-wide policy required by the two conf-driven policy DAGs. Each rule is scoped by - # dag_id, so both rules coexist and do not affect any other DAG. - AIRFLOW__OPENLINEAGE__EMISSION_POLICY: '[{"scope": {"dag_id": "openlineage_policy_conf_locked_dag"}, "locked": true, "controls": {"include_source_code": false}}, {"scope": {"dag_id": "openlineage_policy_conf_source_code_dag"}, "controls": {"include_source_code": false}}]' - AIRFLOW__OPENLINEAGE__EXECUTION_TIMEOUT: '60' - # Both control how quickly the dag-processor picks up dags: REFRESH_INTERVAL is how often it - # re-scans the folder for new/removed files, MIN_FILE_PROCESS_INTERVAL is how often it re-parses - # a file it already knows about. Tightened from the 300s/30s defaults so the e2e run doesn't wait - # on the dag-processor's own cadence on top of everything else. - AIRFLOW__DAG_PROCESSOR__REFRESH_INTERVAL: '3' - AIRFLOW__DAG_PROCESSOR__MIN_FILE_PROCESS_INTERVAL: '3' - # openlineage_defer_simple_dag's TimeDeltaSensor defaults to a 180s delta so it reliably defers at least - # once even before a worker picks up the task; tighten it here since we do not need to wait for worker. - SYSTEM_TESTS_OL_TIMEDELTA: '60' - # Every DAG's dagrun_timeout (system.openlineage.constants.DEFAULT_DAGRUN_TIMEOUT) - # defaults to 10 minutes; tighten it here since we do not need to wait for worker. - SYSTEM_TESTS_OL_DAGRUN_TIMEOUT_MINUTES: '5' - AIRFLOW__CORE__TASK_SUCCESS_OVERTIME: '70' - # Serve the dags folder through a versioned bundle so example_openlineage_versioned_dag gets a - # non-null dag_bundle_version. MockVersionedLocalDagBundle still serves from /opt/airflow/dags - # (keeping PYTHONPATH valid); it is copied into the dags folder by prepare_dags.py. - AIRFLOW__DAG_PROCESSOR__DAG_BUNDLE_CONFIG_LIST: '[{"name": "dags-folder", "classpath": "e2e_versioned_bundle.MockVersionedLocalDagBundle", "kwargs": {}}]' - # The breeze PROD image installs every provider from source, including informatica, whose - # listener fires on every task and errors without its connection — suppressing OL events. - # A curated deployment (e.g. Astro) does not ship it; disable its listener to match. - AIRFLOW__INFORMATICA__LISTENER_DISABLED: 'true' - HOST_OS: ${HOST_OS:-linux} - user: "${AIRFLOW_UID:-50000}:0" - volumes: - - ./dags:/opt/airflow/dags - - ./logs:/opt/airflow/logs - # example_openlineage_docs_file_dag sets doc_md="dag_doc.md", which Airflow resolves with a - # CWD-relative open(); components run with CWD=/opt/airflow. prepare_dags.py generates a clean, - # license-free dag_doc.md (matching the expected event) in the dags folder; expose it here. - - ./dags/dag_doc.md:/opt/airflow/dag_doc.md - depends_on: - &airflow-common-depends-on - postgres: - condition: service_healthy - -services: - postgres: - image: postgres:13 - environment: - POSTGRES_USER: airflow - POSTGRES_PASSWORD: airflow - POSTGRES_DB: airflow - healthcheck: - test: ["CMD", "pg_isready", "-U", "airflow"] - interval: 10s - retries: 5 - start_period: 5s - restart: always - - airflow-init: - <<: *airflow-common - entrypoint: /bin/bash - command: - - -c - - | - echo "Creating missing opt dirs if missing:" - mkdir -v -p /opt/airflow/{logs,dags,plugins,config} - echo "Airflow version:" - /entrypoint airflow version - echo "Running airflow config list to create default config file if missing." - /entrypoint airflow config list >/dev/null - if [ "${HOST_OS}" == "linux" ]; then - echo "Change ownership of files in /opt/airflow to ${AIRFLOW_UID}:0" - chown -R "${AIRFLOW_UID}:0" /opt/airflow/ - else - echo "Skipping changing ownership as platform is not linux (${HOST_OS})" - fi - environment: - <<: *airflow-common-env - _AIRFLOW_DB_MIGRATE: 'true' - _AIRFLOW_WWW_USER_CREATE: 'true' - _AIRFLOW_WWW_USER_USERNAME: ${_AIRFLOW_WWW_USER_USERNAME:-airflow} - _AIRFLOW_WWW_USER_PASSWORD: ${_AIRFLOW_WWW_USER_PASSWORD:-airflow} - _PIP_ADDITIONAL_REQUIREMENTS: '' - user: "0:0" - depends_on: - <<: *airflow-common-depends-on - - airflow-apiserver: - <<: *airflow-common - command: api-server - ports: - - "8080:8080" - healthcheck: - test: ["CMD", "curl", "--fail", "http://localhost:8080/api/v2/monitor/health"] - interval: 30s - timeout: 10s - retries: 5 - start_period: 30s - restart: always - depends_on: - <<: *airflow-common-depends-on - airflow-init: - condition: service_completed_successfully - - airflow-scheduler: - <<: *airflow-common - command: scheduler - healthcheck: - test: ["CMD", "airflow", "jobs", "check", "--job-type", "SchedulerJob"] - interval: 30s - timeout: 10s - retries: 5 - restart: always - depends_on: - postgres: - condition: service_healthy - airflow-init: - condition: service_completed_successfully - - airflow-dag-processor: - <<: *airflow-common - command: dag-processor - depends_on: - <<: *airflow-common-depends-on - airflow-init: - condition: service_completed_successfully - - airflow-triggerer: - <<: *airflow-common - command: triggerer - healthcheck: - test: ["CMD", "airflow", "jobs", "check", "--job-type", "TriggererJob"] - interval: 30s - timeout: 10s - retries: 5 - restart: always - depends_on: - <<: *airflow-common-depends-on - airflow-init: - condition: service_completed_successfully diff --git a/providers-e2e-tests/openlineage/pyproject.toml b/providers-e2e-tests/openlineage/pyproject.toml deleted file mode 100644 index 981a424dfab15..0000000000000 --- a/providers-e2e-tests/openlineage/pyproject.toml +++ /dev/null @@ -1,85 +0,0 @@ - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -[build-system] -requires = [ - "hatchling==1.30.1", - "packaging==26.2", - "pathspec==1.1.1", - "pluggy==1.6.0", - "tomli==2.4.1; python_version < '3.11'", - "trove-classifiers==2026.6.1.19", -] -build-backend = "hatchling.build" - -[project] -name = "apache-airflow-openlineage-e2e-tests" -description = "OpenLineage provider end-to-end tests for Apache Airflow" -classifiers = [ - "Private :: Do Not Upload", -] -requires-python = ">=3.10,!=3.15" -authors = [ - { name = "Apache Software Foundation", email = "dev@airflow.apache.org" }, -] -maintainers = [ - { name = "Apache Software Foundation", email="dev@airflow.apache.org" }, -] -version = "0.0.1" - -dependencies = [ - "apache-airflow-core", - "apache-airflow-devel-common", -] - -[tool.pytest] -addopts = [ - "-rasl", - "--verbosity=2", - "-p", "no:flaky", - "-p", "no:nose", - "-p", "no:legacypath", -] -norecursedirs = [ - ".eggs", -] -log_level = "INFO" -filterwarnings = [ - "error::pytest.PytestCollectionWarning", -] -python_files = [ - "*.py", -] - -# Keep temporary directories (created by `tmp_path`) for 2 recent runs only failed tests. -tmp_path_retention_count = "2" -tmp_path_retention_policy = "failed" - -[tool.e2e-tests] -required-providers = ["openlineage", "standard", "common.compat", "common.sql", "common.io"] - -[tool.hatch.build.targets.sdist] -exclude = ["*"] - -[tool.hatch.build.targets.wheel] -bypass-selection = true - -[dependency-groups] -mypy = [ - "apache-airflow-devel-common[mypy]", -] diff --git a/providers-e2e-tests/openlineage/tests/conftest.py b/providers-e2e-tests/openlineage/tests/conftest.py deleted file mode 100644 index 917ac4a677283..0000000000000 --- a/providers-e2e-tests/openlineage/tests/conftest.py +++ /dev/null @@ -1,194 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -from __future__ import annotations - -import os -import platform -import re -import subprocess -import sys -import time - -import pytest -import requests -from constants import ( - API_BASE_URL, - DOCKER_COMPOSE_FILE_PATH, - DOCKER_IMAGE, - ENV_FILE, - LOCAL_DOCKER_COMPOSE_FILE_PATH, - MIN_DOCKER_COMPOSE_VERSION, - MIN_DOCKER_VERSION, - OPENLINEAGE_E2E_TESTS_ROOT, - console, -) -from packaging import version - -from tests_common.test_utils.fernet import generate_fernet_key_string - -COMPOSE_PROJECT_NAME = "breeze-openlineage-e2e" -FALSEY_VALUES = ("false", "0") - - -def _env_flag(name: str) -> bool: - """True when the environment variable is set to a non-falsey value.""" - value = os.environ.get(name, "") - return bool(value) and value.lower() not in FALSEY_VALUES - - -def _compare_gte(a: str, b: str) -> bool: - try: - return version.parse(a) >= version.parse(b) - except Exception: - return False - - -def _docker_is_running() -> bool: - try: - return ( - subprocess.run(["docker", "info"], capture_output=True, check=False, timeout=10).returncode == 0 - ) - except Exception: - return False - - -def _tool_version(command: list[str]) -> str: - try: - result = subprocess.run(command, capture_output=True, text=True, check=False, timeout=10) - if result.returncode != 0: - return "unknown" - match = re.search(r"(\d+)\.(\d+)\.(\d+)", result.stdout.strip()) - return ".".join(match.groups()) if match else "unknown" - except Exception: - return "unknown" - - -def check_docker_requirements() -> None: - """Verify Docker is running and Docker / Docker Compose meet the minimum versions.""" - console.print("[yellow]Checking Docker requirements...[/]") - if not _docker_is_running(): - console.print("[red]Docker is not running. Please start Docker and try again.[/]") - sys.exit(1) - - docker_version = _tool_version(["docker", "version", "--format", "{{.Client.Version}}"]) - if not _compare_gte(docker_version, MIN_DOCKER_VERSION): - console.print(f"[red]Docker version {docker_version} is too old (min {MIN_DOCKER_VERSION}).[/]") - sys.exit(1) - console.print(f"[green]Docker version {docker_version} meets requirements[/]") - - compose_version = _tool_version(["docker", "compose", "version"]) - if not _compare_gte(compose_version, MIN_DOCKER_COMPOSE_VERSION): - console.print( - f"[red]Docker Compose version {compose_version} is too old (min {MIN_DOCKER_COMPOSE_VERSION}).[/]" - ) - sys.exit(1) - console.print(f"[green]Docker Compose version {compose_version} meets requirements[/]") - - -def _print_diagnostics(compose) -> None: - console.print("[red]=== DIAGNOSTIC INFORMATION ===[/]") - try: - for container in compose.compose.ps(): - console.print(f" {container.name}: {container.state}") - console.print("\n[yellow]Container Logs:[/]") - console.print(compose.compose.logs()) - except Exception as exc: - console.print(f" Error collecting diagnostics: {exc}") - - -def pytest_sessionstart(session): - """Check Docker and source the OpenLineage DAGs into the dags folder before the stack starts.""" - check_docker_requirements() - - # prepare_dags lives at the project root (next to docker-compose.yaml), not in the test package. - sys.path.insert(0, OPENLINEAGE_E2E_TESTS_ROOT.as_posix()) - from prepare_dags import prepare_dags - - console.print("[yellow]Preparing OpenLineage DAGs from the provider system tests...[/]") - dags_folder = prepare_dags() - console.print(f"[green]Prepared dags in {dags_folder}[/]") - - -@pytest.fixture(scope="session") -def docker_compose_setup(): - """Start docker-compose once per session and tear it down at the end.""" - from python_on_whales import DockerClient - - debugging_on = _env_flag("VERBOSE") - mount_volumes = not _env_flag("SKIP_MOUNTING_LOCAL_VOLUMES") - delete_compose = not _env_flag("SKIP_DOCKER_COMPOSE_DELETION") - if mount_volumes: - delete_compose = False - - with open(ENV_FILE, "w") as f: - print(f"AIRFLOW_IMAGE_NAME={DOCKER_IMAGE}", file=f) - print(f"AIRFLOW_UID={os.getuid()}", file=f) - print(f"HOST_OS={platform.system().lower()}", file=f) - # Please do not reuse this Fernet key in any deployment — it is generated only for these tests. - print(f"FERNET_KEY={generate_fernet_key_string()}", file=f) - - docker_compose_files = [DOCKER_COMPOSE_FILE_PATH.as_posix()] - if mount_volumes: - docker_compose_files.append(LOCAL_DOCKER_COMPOSE_FILE_PATH.as_posix()) - log_level = "debug" if debugging_on else "info" - - compose = DockerClient( - compose_files=docker_compose_files, - debug=debugging_on, - log_level=log_level, - compose_project_name=COMPOSE_PROJECT_NAME, - ) - - start_new_compose = True - processes = compose.compose.ps(["airflow-apiserver"]) - if processes and processes[0].state.status == "running": - if mount_volumes: - console.print( - "[yellow]Docker compose already running. Reusing it. Run `docker compose down` to reset.[/]" - ) - start_new_compose = False - else: - compose.compose.down(remove_orphans=True, volumes=True, quiet=True) - - try: - if start_new_compose: - console.print("[yellow]Starting docker-compose for session...[/]") - files = " ".join(f'-f "{file}"' for file in docker_compose_files) - console.print(f"[info]Equivalent manual command: docker compose {files} up --detach[/]") - compose.compose.up(detach=True, wait=True) - console.print("[green]Docker compose started successfully![/]") - yield compose - except Exception as exc: - console.print(f"[red]Docker compose failed to start: {exc}[/]") - _print_diagnostics(compose) - raise - finally: - if delete_compose: - console.print("[yellow]Cleaning up docker-compose...[/]") - compose.compose.down(remove_orphans=True, volumes=True, quiet=True) - console.print("[green]Docker compose cleaned up[/]") - - -@pytest.fixture(scope="session") -def auth_headers(docker_compose_setup) -> dict[str, str]: - """Wait for Airflow and return REST API auth headers (SimpleAuthManager grants an admin token).""" - time.sleep(15) - auth_response = requests.get(f"{API_BASE_URL}/auth/token", timeout=30) - auth_response.raise_for_status() - token = auth_response.json()["access_token"] - console.print("[green]✅ Got Airflow auth token[/]") - return {"Authorization": f"Bearer {token}", "Content-Type": "application/json"} diff --git a/providers-e2e-tests/openlineage/tests/constants.py b/providers-e2e-tests/openlineage/tests/constants.py deleted file mode 100644 index cce0aa92e91bd..0000000000000 --- a/providers-e2e-tests/openlineage/tests/constants.py +++ /dev/null @@ -1,42 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -from __future__ import annotations - -import os -from pathlib import Path - -from rich.console import Console - -console = Console(width=400, color_system="standard") - -AIRFLOW_ROOT_PATH = Path(__file__).resolve().parents[3] -OPENLINEAGE_E2E_TESTS_ROOT = AIRFLOW_ROOT_PATH / "providers-e2e-tests" / "openlineage" - -DEFAULT_PYTHON_MAJOR_MINOR_VERSION = "3.10" -MIN_DOCKER_VERSION = "25.0.0" -MIN_DOCKER_COMPOSE_VERSION = "2.20.2" -DEFAULT_DOCKER_IMAGE = f"ghcr.io/apache/airflow/main/prod/python{DEFAULT_PYTHON_MAJOR_MINOR_VERSION}:latest" -DOCKER_IMAGE = os.environ.get("DOCKER_IMAGE") or DEFAULT_DOCKER_IMAGE - -API_HOST_PORT = os.environ.get("HOST_PORT", "localhost:8080") -API_BASE_URL = f"http://{API_HOST_PORT}" - -DOCKER_COMPOSE_FILE_PATH = OPENLINEAGE_E2E_TESTS_ROOT / "docker-compose.yaml" -LOCAL_DOCKER_COMPOSE_FILE_PATH = OPENLINEAGE_E2E_TESTS_ROOT / "docker-compose-local.yaml" -DAGS_FOLDER = OPENLINEAGE_E2E_TESTS_ROOT / "dags" -LOGS_FOLDER = OPENLINEAGE_E2E_TESTS_ROOT / "logs" -ENV_FILE = OPENLINEAGE_E2E_TESTS_ROOT / ".env" diff --git a/pyproject.toml b/pyproject.toml index 42a9e93a80130..7e28ca58b1c1b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -819,7 +819,6 @@ testing = ["dev", "providers.tests", "tests_common", "tests", "system", "unit", "airflow-core/tests/*" = ["D", "TID253", "S101", "TRY002"] "airflow-ctl-tests/tests/*" = ["D", "TID253", "S101", "TRY002"] "airflow-e2e-tests/tests/*" = ["D", "TID253", "S101", "TRY002"] -"providers-e2e-tests/*" = ["D", "TID253", "S101", "TRY002"] "docker-tests/*" = ["D", "TID253", "S101", "TRY002"] "task-sdk-integration-tests/*" = ["D", "TID253", "S101", "TRY002"] "kubernetes-tests/*" = ["D", "TID253", "S101", "TRY002"] @@ -1086,7 +1085,6 @@ mypy_path = [ "$MYPY_CONFIG_FILE_DIR/kubernetes-tests/tests", "$MYPY_CONFIG_FILE_DIR/docker-tests/tests", "$MYPY_CONFIG_FILE_DIR/task-sdk-integration-tests/tests", - "$MYPY_CONFIG_FILE_DIR/providers-e2e-tests/openlineage/tests", # Automatically generated mypy paths (update_airflow_pyproject_toml.py) "$MYPY_CONFIG_FILE_DIR/providers/airbyte/src", "$MYPY_CONFIG_FILE_DIR/providers/airbyte/tests", @@ -1346,7 +1344,6 @@ dev = [ "apache-airflow-devel-common[no-doc]", "apache-airflow-docker-tests", "apache-airflow-task-sdk-integration-tests", - "apache-airflow-openlineage-e2e-tests", "apache-airflow-helm-chart", "apache-airflow-kubernetes-tests", "apache-airflow-task-sdk[all]", @@ -1431,7 +1428,6 @@ apache-airflow-e2e-tests = false apache-airflow-helm-chart = false apache-airflow-kubernetes-tests = false apache-airflow-mypy = false -apache-airflow-openlineage-e2e-tests = false apache-airflow-providers = false apache-airflow-providers-airbyte = false apache-airflow-providers-akeyless = false @@ -1577,7 +1573,6 @@ apache-airflow-e2e-tests = false apache-airflow-helm-chart = false apache-airflow-kubernetes-tests = false apache-airflow-mypy = false -apache-airflow-openlineage-e2e-tests = false apache-airflow-providers = false apache-airflow-providers-airbyte = false apache-airflow-providers-akeyless = false @@ -1720,7 +1715,6 @@ apache-airflow-devel-common = { workspace = true } apache-airflow-docker-tests = { workspace = true } apache-airflow-e2e-tests = { workspace = true } apache-airflow-task-sdk-integration-tests = { workspace = true } -apache-airflow-openlineage-e2e-tests = { workspace = true } apache-airflow-helm-chart = { workspace = true } apache-airflow-kubernetes-tests = { workspace = true } apache-airflow-providers = { workspace = true } @@ -1860,7 +1854,6 @@ members = [ "scripts", "docker-tests", "task-sdk-integration-tests", - "providers-e2e-tests/openlineage", "chart", "kubernetes-tests", "task-sdk", diff --git a/scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py b/scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py index ce1f79af0ac4e..ca9f09adae6c7 100755 --- a/scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py +++ b/scripts/ci/prek/run_mypy_full_dist_local_venv_or_breeze_in_ci.py @@ -68,7 +68,6 @@ "chart/tests", "airflow-e2e-tests", "task-sdk-integration-tests", - "providers-e2e-tests/openlineage", "docker-tests", "kubernetes-tests", ] @@ -128,9 +127,6 @@ r"^.*/node_modules/.*", r"^.*\\..*", r"^.*/src/airflow/__init__.py$", - # Generated-at-runtime artifacts of the provider e2e tests (gitignored; not source to check). - r"^.*/providers-e2e-tests/[^/]+/dags/.*", - r"^.*/providers-e2e-tests/[^/]+/provider_dist/.*", ] ] diff --git a/uv.lock b/uv.lock index 43b6db586d290..b225a3520e6fe 100644 --- a/uv.lock +++ b/uv.lock @@ -21,7 +21,6 @@ exclude-newer = "0001-01-01T00:00:00Z" # This has no effect and is included for exclude-newer-span = "P4D" [options.exclude-newer-package] -apache-airflow-openlineage-e2e-tests = false apache-airflow-providers-informatica = false apache-airflow-providers-clickhousedb = false apache-airflow-providers-amazon = false @@ -172,7 +171,6 @@ members = [ "apache-airflow-helm-chart", "apache-airflow-kubernetes-tests", "apache-airflow-mypy", - "apache-airflow-openlineage-e2e-tests", "apache-airflow-providers", "apache-airflow-providers-airbyte", "apache-airflow-providers-akeyless", @@ -1495,7 +1493,6 @@ ci-image = [ { name = "apache-airflow-docker-tests" }, { name = "apache-airflow-helm-chart" }, { name = "apache-airflow-kubernetes-tests" }, - { name = "apache-airflow-openlineage-e2e-tests" }, { name = "apache-airflow-scripts" }, { name = "apache-airflow-shared-configuration" }, { name = "apache-airflow-shared-dagnode" }, @@ -1525,7 +1522,6 @@ dev = [ { name = "apache-airflow-docker-tests" }, { name = "apache-airflow-helm-chart" }, { name = "apache-airflow-kubernetes-tests" }, - { name = "apache-airflow-openlineage-e2e-tests" }, { name = "apache-airflow-scripts" }, { name = "apache-airflow-shared-configuration" }, { name = "apache-airflow-shared-dagnode" }, @@ -1807,7 +1803,6 @@ ci-image = [ { name = "apache-airflow-docker-tests", editable = "docker-tests" }, { name = "apache-airflow-helm-chart", editable = "chart" }, { name = "apache-airflow-kubernetes-tests", editable = "kubernetes-tests" }, - { name = "apache-airflow-openlineage-e2e-tests", editable = "providers-e2e-tests/openlineage" }, { name = "apache-airflow-scripts", editable = "scripts" }, { name = "apache-airflow-shared-configuration", editable = "shared/configuration" }, { name = "apache-airflow-shared-dagnode", editable = "shared/dagnode" }, @@ -1837,7 +1832,6 @@ dev = [ { name = "apache-airflow-docker-tests", editable = "docker-tests" }, { name = "apache-airflow-helm-chart", editable = "chart" }, { name = "apache-airflow-kubernetes-tests", editable = "kubernetes-tests" }, - { name = "apache-airflow-openlineage-e2e-tests", editable = "providers-e2e-tests/openlineage" }, { name = "apache-airflow-scripts", editable = "scripts" }, { name = "apache-airflow-shared-configuration", editable = "shared/configuration" }, { name = "apache-airflow-shared-dagnode", editable = "shared/dagnode" }, @@ -2880,29 +2874,6 @@ dependencies = [ [package.metadata] requires-dist = [{ name = "mypy", specifier = ">=1.0.0" }] -[[package]] -name = "apache-airflow-openlineage-e2e-tests" -version = "0.0.1" -source = { editable = "providers-e2e-tests/openlineage" } -dependencies = [ - { name = "apache-airflow-core" }, - { name = "apache-airflow-devel-common" }, -] - -[package.dev-dependencies] -mypy = [ - { name = "apache-airflow-devel-common", extra = ["mypy"] }, -] - -[package.metadata] -requires-dist = [ - { name = "apache-airflow-core", editable = "airflow-core" }, - { name = "apache-airflow-devel-common", editable = "devel-common" }, -] - -[package.metadata.requires-dev] -mypy = [{ name = "apache-airflow-devel-common", extras = ["mypy"], editable = "devel-common" }] - [[package]] name = "apache-airflow-providers" version = "0.0.1" From feb205051b012d6727715d61fc691fb23285ad84 Mon Sep 17 00:00:00 2001 From: Shahar Epstein <60007259+shahar1@users.noreply.github.com> Date: Fri, 10 Jul 2026 07:30:25 +0300 Subject: [PATCH 2/5] Fix OpenLineage e2e system-tests path resolution The repo root was computed with an off-by-one parents index, so the system-test DAGs source resolved outside the repository and the e2e session failed at startup in CI. Reuse AIRFLOW_ROOT_PATH from the e2e framework constants instead of re-deriving it. Generated-by: Claude Code (Fable 5) --- .../tests/airflow_e2e_tests/openlineage_tests/prepare_dags.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/airflow-e2e-tests/tests/airflow_e2e_tests/openlineage_tests/prepare_dags.py b/airflow-e2e-tests/tests/airflow_e2e_tests/openlineage_tests/prepare_dags.py index 81bb46ab165d1..14c04a79ab268 100644 --- a/airflow-e2e-tests/tests/airflow_e2e_tests/openlineage_tests/prepare_dags.py +++ b/airflow-e2e-tests/tests/airflow_e2e_tests/openlineage_tests/prepare_dags.py @@ -33,8 +33,9 @@ import shutil from pathlib import Path +from airflow_e2e_tests.constants import AIRFLOW_ROOT_PATH + HERE = Path(__file__).resolve().parent -AIRFLOW_ROOT_PATH = HERE.parents[4] SYSTEM_TESTS_SOURCE = AIRFLOW_ROOT_PATH / "providers" / "openlineage" / "tests" / "system" # Harness-only modules (warmup DAG, versioned bundle) that are not part of the provider system tests. From 68ee59e2dc81f291a97d5a9717eddfdd62be2ada Mon Sep 17 00:00:00 2001 From: Shahar Epstein <60007259+shahar1@users.noreply.github.com> Date: Fri, 10 Jul 2026 08:38:03 +0300 Subject: [PATCH 3/5] Fix OpenLineage e2e crash on 204 responses from variable deletion The shared AirflowClient parsed every response body as JSON, but the variable-deletion cleanup in the OpenLineage e2e harness issues DELETE requests that return 204 No Content, whose empty body is not valid JSON. --- .../tests/airflow_e2e_tests/e2e_test_utils/clients.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/airflow-e2e-tests/tests/airflow_e2e_tests/e2e_test_utils/clients.py b/airflow-e2e-tests/tests/airflow_e2e_tests/e2e_test_utils/clients.py index ff98eb18e6d3e..0c1166cfc8fc4 100644 --- a/airflow-e2e-tests/tests/airflow_e2e_tests/e2e_test_utils/clients.py +++ b/airflow-e2e-tests/tests/airflow_e2e_tests/e2e_test_utils/clients.py @@ -99,7 +99,8 @@ def _make_request( **kwargs, ) response.raise_for_status() - return response.json() + # DELETE and other no-content responses (204) have an empty body; json() would choke on it. + return response.json() if response.content else None def get_dag(self, dag_id: str): return self._make_request(method="GET", endpoint=f"dags/{dag_id}") From 65b671dbf04421b65e795021f58ed58f2b8dce0a Mon Sep 17 00:00:00 2001 From: Shahar Epstein <60007259+shahar1@users.noreply.github.com> Date: Fri, 10 Jul 2026 17:19:06 +0300 Subject: [PATCH 4/5] Restore OpenLineage e2e older-Airflow compat matrix and README MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review feedback on the consolidation asked to keep the compatibility coverage that ran the OpenLineage suite against older released Airflow versions with current provider code — it catches core-vs-provider breakages early, which matters for a provider as tightly coupled to core as OpenLineage. To keep PR CI cheap, the compat matrix now runs on canary (scheduled / main) only; PRs still run the PROD-image job. Also restores the suite's README next to the tests for debugging reference. --- .../workflows/additional-prod-image-tests.yml | 21 ++++ .github/workflows/ci-amd.yml | 3 + .github/workflows/ci-arm.yml | 3 + .../openlineage-e2e-compat-tests.yml | 105 ++++++++++++++++++ .../docker/openlineage-compat.Dockerfile | 28 +++++ .../openlineage_tests/README.md | 97 ++++++++++++++++ .../openlineage_tests/prepare_dags.py | 25 +++++ dev/breeze/doc/05_test_commands.rst | 3 + .../output_testing_airflow-e2e-tests.svg | 35 ++++-- .../output_testing_airflow-e2e-tests.txt | 2 +- .../commands/testing_commands.py | 91 ++++++++++++++- .../commands/testing_commands_config.py | 1 + 12 files changed, 399 insertions(+), 15 deletions(-) create mode 100644 .github/workflows/openlineage-e2e-compat-tests.yml create mode 100644 airflow-e2e-tests/docker/openlineage-compat.Dockerfile create mode 100644 airflow-e2e-tests/tests/airflow_e2e_tests/openlineage_tests/README.md diff --git a/.github/workflows/additional-prod-image-tests.yml b/.github/workflows/additional-prod-image-tests.yml index aef55b9d890af..bf237936ba178 100644 --- a/.github/workflows/additional-prod-image-tests.yml +++ b/.github/workflows/additional-prod-image-tests.yml @@ -64,6 +64,10 @@ on: # yamllint disable-line rule:truthy description: "Whether to run OpenLineage e2e tests (true/false)" required: true type: string + providers-compatibility-tests-matrix: + description: "JSON matrix of released Airflow versions for the OpenLineage e2e compat tests." + required: true + type: string constraints-branch: description: "Branch used to construct constraints URL from." required: true @@ -337,6 +341,23 @@ jobs: e2e_test_mode: "openlineage" if: inputs.canary-run == 'true' || inputs.run-openlineage-e2e-tests == 'true' + # Older-Airflow compatibility: run only on canary (scheduled / main) to catch core-vs-provider + # breakages early without adding the matrix cost to every PR — PRs run the PROD job above only. + test-e2e-integration-tests-openlineage-compat: + name: "OpenLineage e2e compat tests with PROD image" + strategy: + fail-fast: false + matrix: + compat: ${{ fromJSON(inputs.providers-compatibility-tests-matrix) }} + uses: ./.github/workflows/openlineage-e2e-compat-tests.yml + with: + runners: ${{ inputs.runners }} + platform: ${{ inputs.platform }} + default-python-version: "${{ inputs.default-python-version }}" + use-uv: ${{ inputs.use-uv }} + airflow-version: ${{ matrix.compat.airflow-version }} + if: inputs.canary-run == 'true' + test-ui-e2e-chromium: name: "Chromium UI e2e tests with PROD image" uses: ./.github/workflows/ui-e2e-tests.yml diff --git a/.github/workflows/ci-amd.yml b/.github/workflows/ci-amd.yml index 94218264d5f29..00f34000bdc14 100644 --- a/.github/workflows/ci-amd.yml +++ b/.github/workflows/ci-amd.yml @@ -921,6 +921,9 @@ jobs: run-java-sdk-e2e-tests: ${{ needs.build-info.outputs.run-java-sdk-e2e-tests }} run-go-sdk-e2e-tests: ${{ needs.build-info.outputs.run-go-sdk-e2e-tests }} run-openlineage-e2e-tests: ${{ needs.build-info.outputs.run-openlineage-e2e-tests }} + # yamllint disable rule:line-length + providers-compatibility-tests-matrix: > + ${{ needs.build-info.outputs.providers-compatibility-tests-matrix }} use-uv: ${{ needs.build-info.outputs.use-uv }} run-ui-e2e-tests: ${{ needs.build-info.outputs.run-ui-e2e-tests }} run-airflow-ctl-integration-tests: ${{ needs.build-info.outputs.run-airflow-ctl-integration-tests }} diff --git a/.github/workflows/ci-arm.yml b/.github/workflows/ci-arm.yml index 418bf101a5c4e..f8e81203ec8f7 100644 --- a/.github/workflows/ci-arm.yml +++ b/.github/workflows/ci-arm.yml @@ -910,6 +910,9 @@ jobs: run-java-sdk-e2e-tests: ${{ needs.build-info.outputs.run-java-sdk-e2e-tests }} run-go-sdk-e2e-tests: ${{ needs.build-info.outputs.run-go-sdk-e2e-tests }} run-openlineage-e2e-tests: ${{ needs.build-info.outputs.run-openlineage-e2e-tests }} + # yamllint disable rule:line-length + providers-compatibility-tests-matrix: > + ${{ needs.build-info.outputs.providers-compatibility-tests-matrix }} use-uv: ${{ needs.build-info.outputs.use-uv }} run-ui-e2e-tests: ${{ needs.build-info.outputs.run-ui-e2e-tests }} run-airflow-ctl-integration-tests: ${{ needs.build-info.outputs.run-airflow-ctl-integration-tests }} diff --git a/.github/workflows/openlineage-e2e-compat-tests.yml b/.github/workflows/openlineage-e2e-compat-tests.yml new file mode 100644 index 0000000000000..cfd31f8f19d09 --- /dev/null +++ b/.github/workflows/openlineage-e2e-compat-tests.yml @@ -0,0 +1,105 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +--- + +name: OpenLineage E2E Compat Tests + +permissions: + contents: read +on: # yamllint disable-line rule:truthy + workflow_call: + inputs: + runners: + description: "The array of labels (in json form) determining runners." + required: true + type: string + platform: + description: "Platform for the build - 'linux/amd64' or 'linux/arm64'" + required: true + type: string + default-python-version: + description: "Which version of python should be used by default" + required: true + type: string + use-uv: + description: "Whether to use uv to build the image (true/false)" + required: true + type: string + airflow-version: + description: "Released Airflow version to test with current providers from main." + required: true + type: string + +jobs: + test-openlineage-e2e-compat: + timeout-minutes: 60 + name: "OpenLineage e2e compat ${{ inputs.airflow-version }}" + runs-on: ${{ fromJSON(inputs.runners) }} + env: + PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_USERNAME: ${{ github.actor }} + VERBOSE: "true" + steps: + # OpenLineage requires Airflow 3.x. Gating each step (rather than the whole job) keeps the job + # itself "running" so its dynamic name resolves, ending in success with every real step skipped + # for the Airflow 2.x matrix entry. + - name: "Skip notice for Airflow 2.x" + if: ${{ startsWith(inputs.airflow-version, '2.') }} + run: echo "Skipping OpenLineage e2e tests — not supported against Airflow 2.x." + - name: "Cleanup repo" + if: ${{ !startsWith(inputs.airflow-version, '2.') }} + shell: bash + run: sudo rm -rf ${GITHUB_WORKSPACE}/* + - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" + if: ${{ !startsWith(inputs.airflow-version, '2.') }} + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + fetch-depth: 2 + persist-credentials: false + - name: "Prepare breeze & PROD image: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}" + if: ${{ !startsWith(inputs.airflow-version, '2.') }} + uses: ./.github/actions/prepare_breeze_and_image + with: + platform: ${{ inputs.platform }} + image-type: "prod" + python: ${{ env.PYTHON_MAJOR_MINOR_VERSION }} + use-uv: ${{ inputs.use-uv }} + make-mnt-writeable-and-cleanup: true + id: breeze + - name: "Run OpenLineage e2e compat tests against Airflow ${{ inputs.airflow-version }}" + if: ${{ !startsWith(inputs.airflow-version, '2.') }} + run: > + breeze testing airflow-e2e-tests + --e2e-test-mode openlineage + --airflow-version "${AIRFLOW_VERSION}" + env: + AIRFLOW_VERSION: "${{ inputs.airflow-version }}" + - name: Zip logs + if: ${{ always() && !startsWith(inputs.airflow-version, '2.') }} + run: | + cd ./airflow-e2e-tests && zip -r logs.zip logs + - name: "Upload logs" + if: ${{ always() && !startsWith(inputs.airflow-version, '2.') }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: "openlineage-e2e-compat-logs-${{ inputs.airflow-version }}" + path: './airflow-e2e-tests/logs.zip' + retention-days: '7' + if-no-files-found: 'error' diff --git a/airflow-e2e-tests/docker/openlineage-compat.Dockerfile b/airflow-e2e-tests/docker/openlineage-compat.Dockerfile new file mode 100644 index 0000000000000..3c1d772cf4b06 --- /dev/null +++ b/airflow-e2e-tests/docker/openlineage-compat.Dockerfile @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Lightweight image for running the OpenLineage e2e tests against an OLDER, released Airflow version. +# Instead of building a full PROD image, it starts from the published apache/airflow: image +# and reinstalls the providers under test (OpenLineage + the providers the system-test DAGs rely on) +# from wheels built from main — so the current provider code runs on an older Airflow core, the same +# idea as the provider compatibility tests. The default (prod) run does NOT use this Dockerfile; it +# uses the breeze PROD image with current sources. +ARG AIRFLOW_BASE_IMAGE +FROM ${AIRFLOW_BASE_IMAGE} + +COPY --chown=airflow:0 provider_dist/ /tmp/provider_dist/ +RUN pip install --no-cache-dir --upgrade /tmp/provider_dist/*.whl diff --git a/airflow-e2e-tests/tests/airflow_e2e_tests/openlineage_tests/README.md b/airflow-e2e-tests/tests/airflow_e2e_tests/openlineage_tests/README.md new file mode 100644 index 0000000000000..09c1938b0e714 --- /dev/null +++ b/airflow-e2e-tests/tests/airflow_e2e_tests/openlineage_tests/README.md @@ -0,0 +1,97 @@ + + +# OpenLineage provider end-to-end tests + +These tests deploy a **real, multi-process Airflow** (via docker-compose) and run the OpenLineage +provider's system-test DAGs against it — exercising the integration the way it actually runs in a +deployment (separate scheduler / dag-processor / triggerer / api-server / workers), rather than the +in-process `dag.test()` path. + +They run as the `openlineage` mode of the shared `airflow-e2e-tests` suite, reusing its +docker-compose base, testcontainers conftest, and `AirflowClient`: + +```bash +breeze testing airflow-e2e-tests --e2e-test-mode openlineage +``` + +## How it works + +- The DAGs, `OpenLineageTestOperator`, `VariableTransport`, and expected-event JSON are the ones in + [`providers/openlineage/tests/system/openlineage`](../../../../providers/openlineage/tests/system/openlineage) — + that directory stays the single source of truth. `prepare_dags.py` copies them into a (gitignored) + `dags/` folder at runtime and strips the pytest-only `get_test_run` footer so they parse in a + deployment. +- The `openlineage` mode adds only the [`docker/openlineage.yml`](../../../docker/openlineage.yml) + overlay (OpenLineage env config + `dag_doc.md` mount) on top of the shared compose base; the + conftest's `_setup_openlineage_integration` wires it up. +- OpenLineage is configured to emit through the test `VariableTransport` (events land in Airflow + Variables); each DAG's terminal `OpenLineageTestOperator` task validates the emitted events against + the expected templates. **A DAG run that ends `success` means its lineage matched.** +- `harness.py` triggers every DAG through the REST API, waits for completion, retries a failed DAG + once, and asserts all expected DAGs succeeded. +- A small `MockVersionedLocalDagBundle` (in `dags_extra/`) provides a versioned bundle (so + `dag_bundle_version` is populated) while still serving DAGs from the local dags folder. + +## Checking results + +- **pytest summary:** `1 passed` means every expected DAG is green. On failure it prints the + offenders, e.g. `... not successful: {'openlineage_docs_file_dag': 'failed'}`, and logs + `⚠ DAGs that passed only on retry (flaky first run): [...]`. +- **On failure, the relevant task logs are printed right in the pytest output** (`check_events` plus + any task that didn't succeed, filtered to warning/error lines — see `task_logs.py`) — no need to + open the log artifact for the common case, e.g. the exact mismatch + (``Path `job > facets > ...`: expected X but got Y``). +- **Per-DAG logs** are still written in full under `airflow-e2e-tests/logs/` for deeper digging + beyond what gets printed. +- With `--skip-docker-compose-deletion`, the stack stays up: get a token from + `curl -s http://localhost:8080/auth/token` and check `GET /api/v2/importErrors` for DAG parse + errors, or browse the runs in the UI at (login `airflow` / `airflow`). + +## Compatibility with older Airflow + +On canary (scheduled / main) runs, a compat matrix reruns the suite against older *released* Airflow +versions with the current provider code from main — the same idea as the provider compatibility +tests, catching core-vs-provider breakages early. It builds a lightweight image +(`apache/airflow:` + current OpenLineage-related provider wheels, see +[`docker/openlineage-compat.Dockerfile`](../../../docker/openlineage-compat.Dockerfile)) and runs: + +```bash +breeze testing airflow-e2e-tests --e2e-test-mode openlineage --airflow-version 3.1.8 +``` + +PRs run the PROD-image job only; the compat matrix is canary-only to keep PR CI cheap. + +Some DAGs require a newer Airflow than the compat targets and are dropped for older versions in +`prepare_dags.py` (`MIN_AIRFLOW_VERSION_FOR_DAG`) — e.g. `example_openlineage_hitl_dag` needs 3.1+ +(its operators import-raise on older cores). Add an entry there when a new DAG is version-gated. + +## Layout + +```text +prepare_dags.py # sources DAGs from the provider system tests into dags/ at runtime +harness.py # triggers every DAG via the REST API and collects final run states +task_logs.py # locates and prints the relevant task logs on failure +test_openlineage_e2e.py # the pytest entrypoint +dags_extra/ # harness-only DAGs/modules (warmup DAG, versioned bundle) copied into dags/ +``` + +The OpenLineage-specific compose overlay lives at +[`airflow-e2e-tests/docker/openlineage.yml`](../../../docker/openlineage.yml); the compose lifecycle +and the shared `AirflowClient` come from the parent `airflow-e2e-tests` suite. diff --git a/airflow-e2e-tests/tests/airflow_e2e_tests/openlineage_tests/prepare_dags.py b/airflow-e2e-tests/tests/airflow_e2e_tests/openlineage_tests/prepare_dags.py index 14c04a79ab268..7d99e83b09ef4 100644 --- a/airflow-e2e-tests/tests/airflow_e2e_tests/openlineage_tests/prepare_dags.py +++ b/airflow-e2e-tests/tests/airflow_e2e_tests/openlineage_tests/prepare_dags.py @@ -30,6 +30,7 @@ from __future__ import annotations +import os import shutil from pathlib import Path @@ -44,6 +45,24 @@ # The marker after which everything is Airflow's in-process test harness (pytest-only). PYTEST_FOOTER_MARKER = "from tests_common.test_utils.system_tests import get_test_run" +# Example DAGs that need a minimum Airflow version. When running the compat matrix against an older +# version, the DAG file is removed entirely — a module-level import (e.g. the HITL operators) would +# otherwise raise at parse time and take the whole dag-processor down, not just skip that one DAG. +MIN_AIRFLOW_VERSION_FOR_DAG: dict[str, tuple[int, int]] = { + "example_openlineage_hitl_dag.py": (3, 1), +} + + +def _target_airflow_version() -> tuple[int, int] | None: + """(major, minor) of the compat-targeted Airflow version, or None for the default/prod run.""" + raw = os.environ.get("E2E_TARGET_AIRFLOW_VERSION", "").strip() + if not raw: + return None + parts = raw.split(".") + if len(parts) < 2 or not (parts[0].isdigit() and parts[1].isdigit()): + return None + return int(parts[0]), int(parts[1]) + def _strip_pytest_footer(dag_file: Path) -> None: lines = dag_file.read_text().splitlines(keepends=True) @@ -72,6 +91,12 @@ def prepare_dags(dest: Path) -> Path: # dag-processor does not choke on it. (openlineage_dir / "conftest.py").unlink(missing_ok=True) + target_version = _target_airflow_version() + if target_version is not None: + for dag_file_name, min_version in MIN_AIRFLOW_VERSION_FOR_DAG.items(): + if target_version < min_version: + (openlineage_dir / dag_file_name).unlink(missing_ok=True) + for dag_file in openlineage_dir.glob("example_openlineage_*.py"): _strip_pytest_footer(dag_file) diff --git a/dev/breeze/doc/05_test_commands.rst b/dev/breeze/doc/05_test_commands.rst index 0415431434419..1992940a919f9 100644 --- a/dev/breeze/doc/05_test_commands.rst +++ b/dev/breeze/doc/05_test_commands.rst @@ -377,6 +377,9 @@ The Airflow E2E tests are in ``airflow-e2e-tests/`` folder in the main repo. Eac ``airflow-e2e-tests/tests/airflow_e2e_tests/_tests``. For example, the ``openlineage`` mode deploys a real Airflow and runs the OpenLineage provider's system-test DAGs against it, asserting the emitted lineage events match — run it with ``breeze testing airflow-e2e-tests --e2e-test-mode openlineage``. +Pass ``--airflow-version`` (openlineage mode only) to run against an older released Airflow version +with the current providers from main, instead of the default PROD image; CI runs this compat matrix +on canary only. See ``airflow-e2e-tests/tests/airflow_e2e_tests/openlineage_tests/README.md``. Running Airflow UI E2E tests ............................. diff --git a/dev/breeze/doc/images/output_testing_airflow-e2e-tests.svg b/dev/breeze/doc/images/output_testing_airflow-e2e-tests.svg index e650495eb5410..7f2f3b4a26aa7 100644 --- a/dev/breeze/doc/images/output_testing_airflow-e2e-tests.svg +++ b/dev/breeze/doc/images/output_testing_airflow-e2e-tests.svg @@ -1,4 +1,4 @@ - + - + @@ -108,9 +109,21 @@ + + + + + + + + + + + + - Command: testing airflow-e2e-tests + Command: testing airflow-e2e-tests @@ -136,12 +149,16 @@ │--e2e-test-mode               Specify the mode to use for E2E tests. [default: basic]│ │(basic|remote_log|remote_log_elasticsearch|remote_log_opensearch|xcom_object_sto│ │rage|event_driven|java_sdk|go_sdk|openlineage)│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--help   -hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +│--airflow-version             Run the openlineage mode against a released Airflow version (e.g. 3.1.8) with   │ +│current providers from main, instead of the default PROD image. Builds a        │ +│lightweight image from apache/airflow:<version>. Only used by the openlineage   │ +│compat matrix. (TEXT)│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--help   -hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/dev/breeze/doc/images/output_testing_airflow-e2e-tests.txt b/dev/breeze/doc/images/output_testing_airflow-e2e-tests.txt index c68f420f5600f..123969fa33710 100644 --- a/dev/breeze/doc/images/output_testing_airflow-e2e-tests.txt +++ b/dev/breeze/doc/images/output_testing_airflow-e2e-tests.txt @@ -1 +1 @@ -712ca37b265a35cb79a70e42f189859b +45750ff1a736395d4f563fdca433886f diff --git a/dev/breeze/src/airflow_breeze/commands/testing_commands.py b/dev/breeze/src/airflow_breeze/commands/testing_commands.py index 662b502806264..fd3f0444bd895 100644 --- a/dev/breeze/src/airflow_breeze/commands/testing_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/testing_commands.py @@ -113,7 +113,12 @@ check_async_run_results, run_with_pool, ) -from airflow_breeze.utils.path_utils import AIRFLOW_CTL_ROOT_PATH, FILES_PATH, cleanup_python_generated_files +from airflow_breeze.utils.path_utils import ( + AIRFLOW_CTL_ROOT_PATH, + AIRFLOW_ROOT_PATH, + FILES_PATH, + cleanup_python_generated_files, +) from airflow_breeze.utils.run_tests import ( TASK_SDK_INTEGRATION_TESTS_ROOT_PATH, are_all_test_paths_excluded, @@ -1431,6 +1436,66 @@ def python_api_client_tests( sys.exit(returncode) +# Providers reinstalled from main onto the released Airflow base for the OpenLineage compat image: +# OpenLineage plus the providers its system-test DAGs import. +OPENLINEAGE_E2E_COMPAT_PROVIDERS = ["openlineage", "standard", "common.compat", "common.sql", "common.io"] + + +def _build_openlineage_e2e_compat_image(airflow_version: str, python: str) -> str: + """Build a lightweight image: released ``apache/airflow:`` + current OL providers from main. + + Replicates the provider-compatibility approach (current provider code on an older Airflow core) + without a full PROD image build — the released image is pulled and the providers are reinstalled + from wheels built from main. + """ + import shutil + import tempfile + from pathlib import Path + + console_print(f"[info]Building provider wheels from main: {OPENLINEAGE_E2E_COMPAT_PROVIDERS}[/]") + run_command( + [ + "breeze", + "release-management", + "prepare-provider-distributions", + *OPENLINEAGE_E2E_COMPAT_PROVIDERS, + "--distribution-format", + "wheel", + "--skip-tag-check", + "--include-not-ready-providers", + "--clean-dist", + ], + check=True, + cwd=AIRFLOW_ROOT_PATH, + ) + build_context = Path(tempfile.mkdtemp(prefix="ol-e2e-compat-")) + provider_dist = build_context / "provider_dist" + provider_dist.mkdir() + for wheel in (AIRFLOW_ROOT_PATH / "dist").glob("*.whl"): # --clean-dist above leaves only our wheels + shutil.copy2(wheel, provider_dist / wheel.name) + shutil.copy2( + AIRFLOW_ROOT_PATH / "airflow-e2e-tests" / "docker" / "openlineage-compat.Dockerfile", + build_context / "Dockerfile", + ) + + base_image = f"apache/airflow:{airflow_version}-python{python}" + image_name = f"openlineage-e2e/airflow:{airflow_version}-python{python}" + console_print(f"[info]Building {image_name} from {base_image}[/]") + run_command( + [ + "docker", + "build", + "--build-arg", + f"AIRFLOW_BASE_IMAGE={base_image}", + "-t", + image_name, + build_context.as_posix(), + ], + check=True, + ) + return image_name + + option_e2e_test_mode = click.option( "--e2e-test-mode", help="Specify the mode to use for E2E tests.", @@ -1469,6 +1534,13 @@ def python_api_client_tests( @option_verbose @option_dry_run @option_e2e_test_mode +@click.option( + "--airflow-version", + help="Run the openlineage mode against a released Airflow version (e.g. 3.1.8) with current " + "providers from main, instead of the default PROD image. Builds a lightweight image from " + "apache/airflow:. Only used by the openlineage compat matrix.", + default=None, +) @click.argument("extra_pytest_args", nargs=-1, type=click.Path(path_type=str)) def airflow_e2e_tests( python: str, @@ -1477,19 +1549,28 @@ def airflow_e2e_tests( github_repository: str, include_success_outputs: bool, e2e_test_mode: str, + airflow_version: str | None, extra_pytest_args: tuple, ): """Run Airflow E2E tests.""" perform_environment_checks() image_name = image_name or os.environ.get("DOCKER_IMAGE") - if image_name is None or image_name.strip() == "": + if image_name and image_name.strip() == "": + image_name = None + if airflow_version: + # prepare_dags.py reads this to drop DAGs that need a newer Airflow than the target. + os.environ["E2E_TARGET_AIRFLOW_VERSION"] = airflow_version + if not image_name: + image_name = _build_openlineage_e2e_compat_image(airflow_version, python) + elif not image_name: build_params = BuildProdParams(python=python, github_repository=github_repository) image_name = build_params.airflow_image_name - console_print(f"[info]Running Airflow E2E tests with PROD image: {image_name}[/]") - # If the image is used from docker hub, test container will pull that part of test. - skip_image_check = bool(image_name and image_name.startswith("apache/airflow")) + console_print(f"[info]Running Airflow E2E tests with image: {image_name}[/]") + # If the image is used from docker hub, test container will pull that part of test. A locally + # built compat image (--airflow-version) is present on the host, so skip the check for it too. + skip_image_check = bool(image_name and image_name.startswith("apache/airflow")) or bool(airflow_version) return_code, info = run_docker_compose_tests( image_name=image_name, python_version=python, diff --git a/dev/breeze/src/airflow_breeze/commands/testing_commands_config.py b/dev/breeze/src/airflow_breeze/commands/testing_commands_config.py index 56a4b17450239..b24c9b2d2a977 100644 --- a/dev/breeze/src/airflow_breeze/commands/testing_commands_config.py +++ b/dev/breeze/src/airflow_breeze/commands/testing_commands_config.py @@ -313,6 +313,7 @@ "--include-success-outputs", "--github-repository", "--e2e-test-mode", + "--airflow-version", ], } ], From 8b6f0d938499e3cf562631b0e9cf8423b6deee42 Mon Sep 17 00:00:00 2001 From: Shahar Epstein <60007259+shahar1@users.noreply.github.com> Date: Fri, 10 Jul 2026 18:02:07 +0300 Subject: [PATCH 5/5] Allow triggering OpenLineage e2e compat matrix via full-tests-needed label The compat matrix was gated to canary-only, giving no way to exercise it on a PR for a larger OpenLineage change. Add a dedicated run-openlineage-e2e-compat-tests selective-check output (canary or the explicit 'full tests needed' label) so maintainers can opt a PR in, while keeping it off ordinary OpenLineage PRs. Deliberately not tied to derived full_tests_needed, matching the run-ui-e2e-tests convention. --- .github/workflows/additional-prod-image-tests.yml | 11 ++++++++--- .github/workflows/ci-amd.yml | 2 ++ .github/workflows/ci-arm.yml | 2 ++ .../airflow_e2e_tests/openlineage_tests/README.md | 12 +++++++----- dev/breeze/doc/ci/04_selective_checks.md | 5 +++++ .../src/airflow_breeze/utils/selective_checks.py | 10 ++++++++++ dev/breeze/tests/test_selective_checks.py | 3 +++ 7 files changed, 37 insertions(+), 8 deletions(-) diff --git a/.github/workflows/additional-prod-image-tests.yml b/.github/workflows/additional-prod-image-tests.yml index bf237936ba178..e81bfb7256610 100644 --- a/.github/workflows/additional-prod-image-tests.yml +++ b/.github/workflows/additional-prod-image-tests.yml @@ -64,6 +64,10 @@ on: # yamllint disable-line rule:truthy description: "Whether to run OpenLineage e2e tests (true/false)" required: true type: string + run-openlineage-e2e-compat-tests: + description: "Whether to run the OpenLineage older-Airflow compat matrix (true/false)" + required: true + type: string providers-compatibility-tests-matrix: description: "JSON matrix of released Airflow versions for the OpenLineage e2e compat tests." required: true @@ -341,8 +345,9 @@ jobs: e2e_test_mode: "openlineage" if: inputs.canary-run == 'true' || inputs.run-openlineage-e2e-tests == 'true' - # Older-Airflow compatibility: run only on canary (scheduled / main) to catch core-vs-provider - # breakages early without adding the matrix cost to every PR — PRs run the PROD job above only. + # Older-Airflow compatibility: costly, so it does not run on every OpenLineage PR — only on canary + # (scheduled / main) or when a maintainer sets the "full tests needed" label (both folded into the + # run-openlineage-e2e-compat-tests selective-check output). PRs otherwise run the PROD job only. test-e2e-integration-tests-openlineage-compat: name: "OpenLineage e2e compat tests with PROD image" strategy: @@ -356,7 +361,7 @@ jobs: default-python-version: "${{ inputs.default-python-version }}" use-uv: ${{ inputs.use-uv }} airflow-version: ${{ matrix.compat.airflow-version }} - if: inputs.canary-run == 'true' + if: inputs.run-openlineage-e2e-compat-tests == 'true' test-ui-e2e-chromium: name: "Chromium UI e2e tests with PROD image" diff --git a/.github/workflows/ci-amd.yml b/.github/workflows/ci-amd.yml index 00f34000bdc14..ad0b70da87130 100644 --- a/.github/workflows/ci-amd.yml +++ b/.github/workflows/ci-amd.yml @@ -148,6 +148,7 @@ jobs: run-java-sdk-e2e-tests: ${{ steps.selective-checks.outputs.run-java-sdk-e2e-tests }} run-go-sdk-e2e-tests: ${{ steps.selective-checks.outputs.run-go-sdk-e2e-tests }} run-openlineage-e2e-tests: ${{ steps.selective-checks.outputs.run-openlineage-e2e-tests }} + run-openlineage-e2e-compat-tests: ${{ steps.selective-checks.outputs.run-openlineage-e2e-compat-tests }} run-system-tests: ${{ steps.selective-checks.outputs.run-system-tests }} run-task-sdk-tests: ${{ steps.selective-checks.outputs.run-task-sdk-tests }} run-task-sdk-integration-tests: ${{ steps.selective-checks.outputs.run-task-sdk-integration-tests }} @@ -921,6 +922,7 @@ jobs: run-java-sdk-e2e-tests: ${{ needs.build-info.outputs.run-java-sdk-e2e-tests }} run-go-sdk-e2e-tests: ${{ needs.build-info.outputs.run-go-sdk-e2e-tests }} run-openlineage-e2e-tests: ${{ needs.build-info.outputs.run-openlineage-e2e-tests }} + run-openlineage-e2e-compat-tests: ${{ needs.build-info.outputs.run-openlineage-e2e-compat-tests }} # yamllint disable rule:line-length providers-compatibility-tests-matrix: > ${{ needs.build-info.outputs.providers-compatibility-tests-matrix }} diff --git a/.github/workflows/ci-arm.yml b/.github/workflows/ci-arm.yml index f8e81203ec8f7..ee7ffdd916dae 100644 --- a/.github/workflows/ci-arm.yml +++ b/.github/workflows/ci-arm.yml @@ -137,6 +137,7 @@ jobs: run-java-sdk-e2e-tests: ${{ steps.selective-checks.outputs.run-java-sdk-e2e-tests }} run-go-sdk-e2e-tests: ${{ steps.selective-checks.outputs.run-go-sdk-e2e-tests }} run-openlineage-e2e-tests: ${{ steps.selective-checks.outputs.run-openlineage-e2e-tests }} + run-openlineage-e2e-compat-tests: ${{ steps.selective-checks.outputs.run-openlineage-e2e-compat-tests }} run-system-tests: ${{ steps.selective-checks.outputs.run-system-tests }} run-task-sdk-tests: ${{ steps.selective-checks.outputs.run-task-sdk-tests }} run-task-sdk-integration-tests: ${{ steps.selective-checks.outputs.run-task-sdk-integration-tests }} @@ -910,6 +911,7 @@ jobs: run-java-sdk-e2e-tests: ${{ needs.build-info.outputs.run-java-sdk-e2e-tests }} run-go-sdk-e2e-tests: ${{ needs.build-info.outputs.run-go-sdk-e2e-tests }} run-openlineage-e2e-tests: ${{ needs.build-info.outputs.run-openlineage-e2e-tests }} + run-openlineage-e2e-compat-tests: ${{ needs.build-info.outputs.run-openlineage-e2e-compat-tests }} # yamllint disable rule:line-length providers-compatibility-tests-matrix: > ${{ needs.build-info.outputs.providers-compatibility-tests-matrix }} diff --git a/airflow-e2e-tests/tests/airflow_e2e_tests/openlineage_tests/README.md b/airflow-e2e-tests/tests/airflow_e2e_tests/openlineage_tests/README.md index 09c1938b0e714..cadf25c399b9c 100644 --- a/airflow-e2e-tests/tests/airflow_e2e_tests/openlineage_tests/README.md +++ b/airflow-e2e-tests/tests/airflow_e2e_tests/openlineage_tests/README.md @@ -66,17 +66,19 @@ breeze testing airflow-e2e-tests --e2e-test-mode openlineage ## Compatibility with older Airflow -On canary (scheduled / main) runs, a compat matrix reruns the suite against older *released* Airflow -versions with the current provider code from main — the same idea as the provider compatibility -tests, catching core-vs-provider breakages early. It builds a lightweight image -(`apache/airflow:` + current OpenLineage-related provider wheels, see +A compat matrix reruns the suite against older *released* Airflow versions with the current provider +code from main — the same idea as the provider compatibility tests, catching core-vs-provider +breakages early. It builds a lightweight image (`apache/airflow:` + current +OpenLineage-related provider wheels, see [`docker/openlineage-compat.Dockerfile`](../../../docker/openlineage-compat.Dockerfile)) and runs: ```bash breeze testing airflow-e2e-tests --e2e-test-mode openlineage --airflow-version 3.1.8 ``` -PRs run the PROD-image job only; the compat matrix is canary-only to keep PR CI cheap. +The matrix is costly, so it does **not** run on every OpenLineage PR: it runs on canary (scheduled / +main) or when a maintainer sets the **`full tests needed`** label on the PR (both folded into the +`run-openlineage-e2e-compat-tests` selective-check output). Ordinary PRs run the PROD-image job only. Some DAGs require a newer Airflow than the compat targets and are dropped for older versions in `prepare_dags.py` (`MIN_AIRFLOW_VERSION_FOR_DAG`) — e.g. `example_openlineage_hitl_dag` needs 3.1+ diff --git a/dev/breeze/doc/ci/04_selective_checks.md b/dev/breeze/doc/ci/04_selective_checks.md index 293b1c0cca66d..99c384ef8748d 100644 --- a/dev/breeze/doc/ci/04_selective_checks.md +++ b/dev/breeze/doc/ci/04_selective_checks.md @@ -433,6 +433,11 @@ together using `pytest-xdist` (pytest-xdist distributes the tests among parallel openlineage e2e suite change — and always on `canary` runs (where `full tests needed` also covers core/task-sdk changes). Like the other deployed e2e suites, enabling them forces `PROD Image building`. +* `OpenLineage E2E compat tests` (the same suite rerun against older released Airflow versions with + current provider code, exposed as the `run-openlineage-e2e-compat-tests` output) are costly, so + they do NOT run on every OpenLineage PR: only on `canary` runs or when the `full tests needed` + label is explicitly set. Deliberately not tied to *derived* `full tests needed` (large PRs, env + changes, pushes) — the same rationale as `run-ui-e2e-tests`. * The specific unit test type is enabled only if changed files match the expected patterns for each type (`API`, `CLI`, `WWW`, `Providers` etc.). The `Always` test type is added always if any unit tests are run. `Providers` tests are removed if current branch is different than `main` diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py b/dev/breeze/src/airflow_breeze/utils/selective_checks.py index 3a758a2160b4f..b9c41a23c0933 100644 --- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py +++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py @@ -1049,6 +1049,16 @@ def run_go_sdk_e2e_tests(self) -> bool: def run_openlineage_e2e_tests(self) -> bool: return self._should_be_run(FileGroupForCi.OPENLINEAGE_E2E_FILES) + @cached_property + def run_openlineage_e2e_compat_tests(self) -> bool: + # The older-Airflow compat matrix is costly, so it does not run on every OpenLineage PR: + # only on canary (scheduled / main) or when a maintainer explicitly asks via the label. + # Deliberately not tied to derived full_tests_needed (large PRs, env changes, pushes) — same + # rationale as run_ui_e2e_tests. + if self._is_canary_run() or FULL_TESTS_NEEDED_LABEL in self._pr_labels: + return True + return False + @cached_property def run_amazon_tests(self) -> bool: if self.providers_test_types_list_as_strings_in_json == "[]": diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index 973ec115ca75d..6bb5056e9e23c 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -1204,6 +1204,8 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): # common providers feed OpenLineage e2e tests, which need the PROD image. "prod-image-build": "true", "run-openlineage-e2e-tests": "true", + # ordinary OL PR (no canary, no "full tests needed" label): compat matrix stays off. + "run-openlineage-e2e-compat-tests": "false", "run-helm-tests": "false", "run-unit-tests": "true", "run-amazon-tests": "false", @@ -2076,6 +2078,7 @@ def test_non_test_workflows_do_not_force_full_tests(files: tuple[str, ...], expe "providers-test-types-list-as-strings-in-json": ALL_PROVIDERS_SELECTIVE_TEST_TYPES_AS_JSON, "run-mypy-providers": "true", "run-ui-e2e-tests": "true", + "run-openlineage-e2e-compat-tests": "true", }, id="Everything should run including all providers when full tests are needed " "but with single python and kubernetes if no version label is set",