From 7c4c2c924b1850cc3fc237af350d8141462943c3 Mon Sep 17 00:00:00 2001 From: Wei Lee Date: Fri, 24 Jul 2026 15:48:54 +0800 Subject: [PATCH] [v3-3-test] Resolve nested provider paths for conn-fields check from the real directory tree (#70261) (cherry picked from commit 030efa6f74a50c54e4d4e28df6d8d98eb227c7c4) Co-authored-by: Wei Lee --- scripts/ci/prek/check_provider_yaml_files.py | 53 +++++++++--- .../ci/prek/test_check_provider_yaml_files.py | 84 +++++++++++++++++++ 2 files changed, 127 insertions(+), 10 deletions(-) create mode 100644 scripts/tests/ci/prek/test_check_provider_yaml_files.py diff --git a/scripts/ci/prek/check_provider_yaml_files.py b/scripts/ci/prek/check_provider_yaml_files.py index 0c74234e06406..5e8d8da7a9817 100755 --- a/scripts/ci/prek/check_provider_yaml_files.py +++ b/scripts/ci/prek/check_provider_yaml_files.py @@ -16,28 +16,61 @@ # specific language governing permissions and limitations # under the License. # /// script -# requires-python = ">=3.10,<3.11" +# requires-python = ">=3.10" # dependencies = [ # "rich>=13.6.0", # ] # /// from __future__ import annotations +import pathlib import sys from common_prek_utils import ( + get_provider_base_dir_from_path, initialize_breeze_prek, run_command_via_breeze_run, validate_cmd_result, ) -initialize_breeze_prek(__name__, __file__) -files_to_test = sys.argv[1:] -cmd_result = run_command_via_breeze_run( - ["python3", "/opt/airflow/scripts/in_container/run_provider_yaml_files_check.py", *files_to_test], - backend="sqlite", - warn_image_upgrade_needed=True, - extra_env={"PYTHONWARNINGS": "default"}, -) -validate_cmd_result(cmd_result, include_ci_env_check=True) +def _resolve_provider_yaml_files(raw_files: list[str]) -> list[str]: + """ + Accept a mix of provider.yaml paths and Python source files. + + When a Python source file is passed (e.g. a hook whose + ``get_connection_form_widgets()`` was edited), map it to the + ``provider.yaml`` at the root of the same provider package so the + conn-fields check runs even when only the hook changes. + + All paths are relative to the ``providers/`` directory, as supplied by + prek. Rather than guessing how many path segments make up the provider + package name, this walks up the real directory tree (via + ``get_provider_base_dir_from_path``) until it finds the actual + ``provider.yaml`` file, so nested/namespace provider packages + (e.g. ``apache/beam``, ``ibm/mq``) resolve correctly without maintaining + a list of known namespace prefixes. + """ + result: set[str] = set() + for f in raw_files: + p = pathlib.PurePosixPath(f) + if p.name == "provider.yaml": + result.add(f) + else: + provider_dir = get_provider_base_dir_from_path(pathlib.Path(f)) + if provider_dir is not None: + result.add((provider_dir / "provider.yaml").as_posix()) + return sorted(result) + + +if __name__ == "__main__": + initialize_breeze_prek(__name__, __file__) + + files_to_test = _resolve_provider_yaml_files(sys.argv[1:]) + cmd_result = run_command_via_breeze_run( + ["python3", "/opt/airflow/scripts/in_container/run_provider_yaml_files_check.py", *files_to_test], + backend="sqlite", + warn_image_upgrade_needed=True, + extra_env={"PYTHONWARNINGS": "default"}, + ) + validate_cmd_result(cmd_result, include_ci_env_check=True) diff --git a/scripts/tests/ci/prek/test_check_provider_yaml_files.py b/scripts/tests/ci/prek/test_check_provider_yaml_files.py new file mode 100644 index 0000000000000..590ad73efb2ef --- /dev/null +++ b/scripts/tests/ci/prek/test_check_provider_yaml_files.py @@ -0,0 +1,84 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import pytest +from ci.prek.check_provider_yaml_files import _resolve_provider_yaml_files + + +def _touch(path): + path.parent.mkdir(parents=True, exist_ok=True) + path.touch() + + +class TestResolveProviderYamlFiles: + @pytest.mark.parametrize( + "file_names, raw_files, expected", + ( + ( + ["samba/provider.yaml"], + ["samba/provider.yaml"], + ["samba/provider.yaml"], + ), + ( + [ + "samba/provider.yaml", + "samba/src/airflow/providers/samba/hooks/samba.py", + ], + ["samba/src/airflow/providers/samba/hooks/samba.py"], + ["samba/provider.yaml"], + ), + ( + [ + "ibm/mq/provider.yaml", + "ibm/mq/src/airflow/providers/ibm/mq/hooks/mq.py", + ], + ["ibm/mq/src/airflow/providers/ibm/mq/hooks/mq.py"], + ["ibm/mq/provider.yaml"], + ), + ( + [ + "samba/provider.yaml", + "samba/src/airflow/providers/samba/hooks/samba.py", + "ibm/mq/provider.yaml", + "ibm/mq/src/airflow/providers/ibm/mq/hooks/mq.py", + ], + [ + "samba/provider.yaml", + "samba/src/airflow/providers/samba/hooks/samba.py", + "ibm/mq/src/airflow/providers/ibm/mq/hooks/mq.py", + ], + ["ibm/mq/provider.yaml", "samba/provider.yaml"], + ), + (["unrelated/hooks/hook.py"], ["unrelated/hooks/hook.py"], []), + ), + ids=[ + "provider_yaml_path_preserved", + "top_level_provider_hook_file_resolves", + "nested_namespace_provider_hook_file_resolves_without_known_list", + "mixed_input_dedups_and_sorts", + "no_provider_found", + ], + ) + def test__resolve_provider_yaml_files(self, tmp_path, monkeypatch, file_names, raw_files, expected): + monkeypatch.chdir(tmp_path) + for file_name in file_names: + _touch(tmp_path / file_name) + + result = _resolve_provider_yaml_files(raw_files) + + assert result == expected