[v3-3-test] Run check-provider-yaml-valid when prek scripts or providers config change (#70230)

dev/breeze/doc/ci/04_selective_checks.md

@@ -19,7 +19,7 @@
 
 <!-- START doctoc generated TOC please keep comment here to allow auto update -->
 <!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
-**Table of Contents**  *generated with [DocToc](https://github.com/thlorenz/doctoc)*
+**Table of Contents** *generated with [DocToc](https://github.com/thlorenz/doctoc)*
 
 - [Selective CI Checks](#selective-ci-checks)
   - [Why selective checks exist (the optimisation goal)](#why-selective-checks-exist-the-optimisation-goal)
@@ -355,8 +355,8 @@ We have the following Groups of files for CI that determine which tests are run:
 * `API tests files` and `Codegen test files` - those are OpenAPI definition files that impact
   Open API specification and determine that we should run dedicated API tests.
 * `Helm files` - change in those files impacts helm "rendering" tests - `chart` folder (which contains the chart sources and tests under `chart/tests/`).
-* `Build files` - change in the files indicates that we should run  `upgrade to newer dependencies` -
-  build dependencies in `pyproject.toml` and  generated dependencies files in `generated` folder.
+* `Build files` - change in the files indicates that we should run `upgrade to newer dependencies` -
+  build dependencies in `pyproject.toml` and generated dependencies files in `generated` folder.
   The dependencies are automatically generated from the `provider.yaml` files in provider by
   the `hatch_build.py` build hook. The provider.yaml is a single source of truth for each
   provider and `hatch_build.py` for all regular dependencies.
@@ -480,8 +480,11 @@ when some files are not changed. Those are the rules implemented:
   * if no `Java SDK files` changed - `ktlint` check is skipped (it runs the java-sdk Gradle
     wrapper, which downloads the Gradle distribution, so we avoid that download on PRs that do
     not touch `java-sdk/`)
-  * if no `All Providers Python files` and no `All Providers Yaml files` are changed -
-    `check-provider-yaml-valid` check is skipped
+  * `check-provider-yaml-valid` is skipped unless at least one of these changed:
+    `All Providers Python files`, `All Providers Distribution Config files`
+    (which includes `provider.yaml`, `pyproject.toml`, and `providers/.pre-commit-config.yaml`),
+    or `Prek files` (`scripts/ci/prek/`). The last condition ensures the check runs
+    when the check script itself is modified.
 
 ## Suspended providers
 

dev/breeze/src/airflow_breeze/utils/selective_checks.py

@@ -356,6 +356,7 @@ def __hash__(self):
         FileGroupForCi.ALL_PROVIDERS_DISTRIBUTION_CONFIG_FILES: [
             r"^providers/.*/pyproject\.toml$",
             r"^providers/.*/provider\.yaml$",
+            r"^providers/\.pre-commit-config\.yaml$",
         ],
         FileGroupForCi.ALL_DEV_PYTHON_FILES: [
             r"^dev/.*\.py$",
@@ -1606,9 +1607,12 @@ def skip_prek_hooks(self) -> str:
                 FileGroupForCi.ALL_PROVIDERS_DISTRIBUTION_CONFIG_FILES, CI_FILE_GROUP_MATCHES
             )
             or self._matching_files(FileGroupForCi.ALL_PROVIDERS_PYTHON_FILES, CI_FILE_GROUP_MATCHES)
+            or self._matching_files(FileGroupForCi.PREK_FILES, CI_FILE_GROUP_MATCHES)
         ):
-            # only skip provider validation if none of the provider.yaml and provider
-            # python files changed because validation also walks through all the provider python files
+            # Skip provider validation only when none of these changed:
+            # - provider.yaml / pyproject.toml / providers/.pre-commit-config.yaml
+            # - provider Python files (validation walks all provider Python files)
+            # - prek scripts (the check script itself may have changed)
             prek_hooks_to_skip.add("check-provider-yaml-valid")
         # Non-provider mypy checks run as prek hooks in static checks.
         # Skip them when their relevant files haven't changed, unless devel-common

dev/breeze/tests/test_selective_checks.py

@@ -1856,6 +1856,37 @@ def test_full_test_needed_when_scripts_changes(files: tuple[str, ...], expected_
     assert_outputs_are_printed(expected_outputs, str(stderr))
 
 
+@pytest.mark.parametrize(
+    "files",
+    [
+        pytest.param(
+            ("scripts/ci/prek/check_provider_yaml_files.py",),
+            id="provider yaml check script changed",
+        ),
+        pytest.param(
+            ("providers/.pre-commit-config.yaml",),
+            id="providers prek config changed",
+        ),
+        pytest.param(
+            (
+                "scripts/ci/prek/check_provider_yaml_files.py",
+                "providers/.pre-commit-config.yaml",
+            ),
+            id="provider yaml check script and providers prek config changed together",
+        ),
+    ],
+)
+def test_provider_yaml_check_not_skipped_when_check_scripts_change(files: tuple[str, ...]):
+    stderr = SelectiveChecks(
+        files=files,
+        github_event=GithubEvents.PULL_REQUEST,
+        commit_ref=NEUTRAL_COMMIT,
+        default_branch="main",
+    )
+    skip_prek_hooks = str(stderr).split("skip-prek-hooks=")[1].split("\n")[0]
+    assert "check-provider-yaml-valid" not in skip_prek_hooks.split(",")
+
+
 @pytest.mark.parametrize(
     ("files", "expected_outputs"),
     [