From 633a342ad517cc00dd61f9ea214c7564fb815b35 Mon Sep 17 00:00:00 2001 From: David Blain Date: Thu, 23 Jul 2026 07:20:51 +0200 Subject: [PATCH] [v3-3-test] Run check-provider-yaml-valid when prek scripts or providers config change (#70230) (cherry picked from commit ff0ac28565815a155925ffd304a6b2c2dbe8ed77) Co-authored-by: David Blain --- dev/breeze/doc/ci/04_selective_checks.md | 13 +++++--- .../airflow_breeze/utils/selective_checks.py | 8 +++-- dev/breeze/tests/test_selective_checks.py | 31 +++++++++++++++++++ 3 files changed, 45 insertions(+), 7 deletions(-) diff --git a/dev/breeze/doc/ci/04_selective_checks.md b/dev/breeze/doc/ci/04_selective_checks.md index f16d8773ff18e..7bb69dde134dc 100644 --- a/dev/breeze/doc/ci/04_selective_checks.md +++ b/dev/breeze/doc/ci/04_selective_checks.md @@ -19,7 +19,7 @@ -**Table of Contents** *generated with [DocToc](https://github.com/thlorenz/doctoc)* +**Table of Contents** *generated with [DocToc](https://github.com/thlorenz/doctoc)* - [Selective CI Checks](#selective-ci-checks) - [Why selective checks exist (the optimisation goal)](#why-selective-checks-exist-the-optimisation-goal) @@ -355,8 +355,8 @@ We have the following Groups of files for CI that determine which tests are run: * `API tests files` and `Codegen test files` - those are OpenAPI definition files that impact Open API specification and determine that we should run dedicated API tests. * `Helm files` - change in those files impacts helm "rendering" tests - `chart` folder (which contains the chart sources and tests under `chart/tests/`). -* `Build files` - change in the files indicates that we should run `upgrade to newer dependencies` - - build dependencies in `pyproject.toml` and generated dependencies files in `generated` folder. +* `Build files` - change in the files indicates that we should run `upgrade to newer dependencies` - + build dependencies in `pyproject.toml` and generated dependencies files in `generated` folder. The dependencies are automatically generated from the `provider.yaml` files in provider by the `hatch_build.py` build hook. The provider.yaml is a single source of truth for each provider and `hatch_build.py` for all regular dependencies. @@ -480,8 +480,11 @@ when some files are not changed. Those are the rules implemented: * if no `Java SDK files` changed - `ktlint` check is skipped (it runs the java-sdk Gradle wrapper, which downloads the Gradle distribution, so we avoid that download on PRs that do not touch `java-sdk/`) - * if no `All Providers Python files` and no `All Providers Yaml files` are changed - - `check-provider-yaml-valid` check is skipped + * `check-provider-yaml-valid` is skipped unless at least one of these changed: + `All Providers Python files`, `All Providers Distribution Config files` + (which includes `provider.yaml`, `pyproject.toml`, and `providers/.pre-commit-config.yaml`), + or `Prek files` (`scripts/ci/prek/`). The last condition ensures the check runs + when the check script itself is modified. ## Suspended providers diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py b/dev/breeze/src/airflow_breeze/utils/selective_checks.py index 8221da3776416..09d9bc4d9970e 100644 --- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py +++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py @@ -356,6 +356,7 @@ def __hash__(self): FileGroupForCi.ALL_PROVIDERS_DISTRIBUTION_CONFIG_FILES: [ r"^providers/.*/pyproject\.toml$", r"^providers/.*/provider\.yaml$", + r"^providers/\.pre-commit-config\.yaml$", ], FileGroupForCi.ALL_DEV_PYTHON_FILES: [ r"^dev/.*\.py$", @@ -1606,9 +1607,12 @@ def skip_prek_hooks(self) -> str: FileGroupForCi.ALL_PROVIDERS_DISTRIBUTION_CONFIG_FILES, CI_FILE_GROUP_MATCHES ) or self._matching_files(FileGroupForCi.ALL_PROVIDERS_PYTHON_FILES, CI_FILE_GROUP_MATCHES) + or self._matching_files(FileGroupForCi.PREK_FILES, CI_FILE_GROUP_MATCHES) ): - # only skip provider validation if none of the provider.yaml and provider - # python files changed because validation also walks through all the provider python files + # Skip provider validation only when none of these changed: + # - provider.yaml / pyproject.toml / providers/.pre-commit-config.yaml + # - provider Python files (validation walks all provider Python files) + # - prek scripts (the check script itself may have changed) prek_hooks_to_skip.add("check-provider-yaml-valid") # Non-provider mypy checks run as prek hooks in static checks. # Skip them when their relevant files haven't changed, unless devel-common diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index 4185fc1dc1273..b0ff2703d469e 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -1856,6 +1856,37 @@ def test_full_test_needed_when_scripts_changes(files: tuple[str, ...], expected_ assert_outputs_are_printed(expected_outputs, str(stderr)) +@pytest.mark.parametrize( + "files", + [ + pytest.param( + ("scripts/ci/prek/check_provider_yaml_files.py",), + id="provider yaml check script changed", + ), + pytest.param( + ("providers/.pre-commit-config.yaml",), + id="providers prek config changed", + ), + pytest.param( + ( + "scripts/ci/prek/check_provider_yaml_files.py", + "providers/.pre-commit-config.yaml", + ), + id="provider yaml check script and providers prek config changed together", + ), + ], +) +def test_provider_yaml_check_not_skipped_when_check_scripts_change(files: tuple[str, ...]): + stderr = SelectiveChecks( + files=files, + github_event=GithubEvents.PULL_REQUEST, + commit_ref=NEUTRAL_COMMIT, + default_branch="main", + ) + skip_prek_hooks = str(stderr).split("skip-prek-hooks=")[1].split("\n")[0] + assert "check-provider-yaml-valid" not in skip_prek_hooks.split(",") + + @pytest.mark.parametrize( ("files", "expected_outputs"), [