From 6ff6ada1ba9da2eaf73d4fd5da72365420cc5736 Mon Sep 17 00:00:00 2001 From: bibi samina Date: Fri, 17 Jul 2026 12:37:13 +0530 Subject: [PATCH 1/2] Strip connection headers on cross-host redirect in HttpHook An HTTP connection's extra field is documented as a place to carry credentials, and requests only protects the Authorization name, so a secret under any other header name reached whatever host a redirect named. --- .../src/airflow/providers/http/hooks/http.py | 24 +++++++++++++++- .../http/tests/unit/http/hooks/test_http.py | 28 +++++++++++++++++++ .../http/tests/unit/http/sensors/test_http.py | 8 +++--- 3 files changed, 55 insertions(+), 5 deletions(-) diff --git a/providers/http/src/airflow/providers/http/hooks/http.py b/providers/http/src/airflow/providers/http/hooks/http.py index 0c5067743f090..061a0d8930134 100644 --- a/providers/http/src/airflow/providers/http/hooks/http.py +++ b/providers/http/src/airflow/providers/http/hooks/http.py @@ -120,6 +120,25 @@ def _retryable_error_async(exception: BaseException) -> bool: return exception.status >= 500 +class _ConnectionSession(Session): + """ + Session that drops Connection-supplied headers when a redirect leaves the original host. + + ``requests`` only strips ``Authorization``, but an HTTP Connection may carry its + credentials under any header name, so those need the same treatment. + """ + + def __init__(self) -> None: + super().__init__() + self.connection_headers: set[str] = set() + + def rebuild_auth(self, prepared_request: PreparedRequest, response: Response) -> None: + super().rebuild_auth(prepared_request, response) + if self.should_strip_auth(response.request.url, prepared_request.url): + for header in self.connection_headers: + prepared_request.headers.pop(header, None) + + class HttpHook(BaseHook): """ Interact with HTTP servers. @@ -196,7 +215,7 @@ def get_conn( :param extra_options: additional options to be used when executing the request :return: A configured requests.Session object. """ - session = Session() + session: Session = _ConnectionSession() connection = self.get_connection(self.http_conn_id) self._set_base_url(connection) session = self._configure_session_from_auth(session, connection) # type: ignore[arg-type] @@ -268,6 +287,9 @@ def _configure_session_from_extra( session.headers.update(conn_extra_options) except TypeError: self.log.warning("Connection to %s has invalid extra field.", connection.host) + else: + if isinstance(session, _ConnectionSession): + session.connection_headers.update(conn_extra_options) return session diff --git a/providers/http/tests/unit/http/hooks/test_http.py b/providers/http/tests/unit/http/hooks/test_http.py index f89532d7e2756..0c92a8bd7a02f 100644 --- a/providers/http/tests/unit/http/hooks/test_http.py +++ b/providers/http/tests/unit/http/hooks/test_http.py @@ -337,6 +337,34 @@ def test_post_request_raises_error_when_redirects_with_max_redirects_set_to_0(se assert history[0].url == "http://test:8080/v1/test" assert history[0].method == "POST" + @pytest.mark.parametrize( + ("location", "expected_forwarded_key"), + [ + pytest.param("http://another.host/v1/redirected", None, id="cross-host"), + pytest.param("http://test:8080/v1/redirected", "secret-key", id="same-host"), + ], + ) + def test_connection_header_is_only_forwarded_on_a_same_host_redirect( + self, requests_mock, create_connection_without_db, location, expected_forwarded_key + ): + create_connection_without_db( + Connection( + conn_id="http_conn_with_api_key", + conn_type="http", + host="test:8080/", + extra='{"X-API-Key": "secret-key"}', + ) + ) + requests_mock.get("http://test:8080/v1/test", status_code=302, headers={"Location": location}) + requests_mock.get(location, status_code=200, text='{"message": "OK"}') + + HttpHook(method="GET", http_conn_id="http_conn_with_api_key").run("v1/test") + + history = requests_mock.request_history + assert len(history) == 2 + assert history[0].headers["X-API-Key"] == "secret-key" + assert history[1].headers.get("X-API-Key") == expected_forwarded_key + @pytest.mark.parametrize( "setup_connections_with_extras", [{"bearer": "test", "check_response": False}], indirect=True ) diff --git a/providers/http/tests/unit/http/sensors/test_http.py b/providers/http/tests/unit/http/sensors/test_http.py index 9ce92a5661215..33213a54b5663 100644 --- a/providers/http/tests/unit/http/sensors/test_http.py +++ b/providers/http/tests/unit/http/sensors/test_http.py @@ -286,7 +286,7 @@ def mount(self, prefix, adapter): class TestHttpOpSensor: - @mock.patch("airflow.providers.http.hooks.http.Session", FakeSession) + @mock.patch("airflow.providers.http.hooks.http._ConnectionSession", FakeSession) def test_get(self): op = HttpOperator( task_id="get_op", @@ -297,7 +297,7 @@ def test_get(self): ) op.execute({}) - @mock.patch("airflow.providers.http.hooks.http.Session", FakeSession) + @mock.patch("airflow.providers.http.hooks.http._ConnectionSession", FakeSession) def test_get_response_check(self): op = HttpOperator( task_id="get_op", @@ -310,7 +310,7 @@ def test_get_response_check(self): op.execute({}) @pytest.mark.skipif(not AIRFLOW_V_3_0_PLUS, reason="Test only for Airflow 3.0+") - @mock.patch("airflow.providers.http.hooks.http.Session", FakeSession) + @mock.patch("airflow.providers.http.hooks.http._ConnectionSession", FakeSession) def test_sensor(self, run_task): sensor = HttpSensor( task_id="http_sensor_check", @@ -328,7 +328,7 @@ def test_sensor(self, run_task): assert run_task.error is None @pytest.mark.skipif(AIRFLOW_V_3_0_PLUS, reason="Test only for Airflow < 3.0") - @mock.patch("airflow.providers.http.hooks.http.Session", FakeSession) + @mock.patch("airflow.providers.http.hooks.http._ConnectionSession", FakeSession) def test_sensor_af2(self): dag = DAG(TEST_DAG_ID, schedule=None) sensor = HttpSensor( From 7dd7cff9a003486fd45ca391b1eea846d2280790 Mon Sep 17 00:00:00 2001 From: bibi samina Date: Tue, 21 Jul 2026 11:38:46 +0530 Subject: [PATCH 2/2] Document HttpHook redirect header change and link async tracking issue --- providers/http/docs/changelog.rst | 7 ++ .../src/airflow/providers/http/hooks/http.py | 3 + uv.lock | 70 ++++++++++--------- 3 files changed, 48 insertions(+), 32 deletions(-) diff --git a/providers/http/docs/changelog.rst b/providers/http/docs/changelog.rst index bf5c0747fd6db..ae1e7ea33ce57 100644 --- a/providers/http/docs/changelog.rst +++ b/providers/http/docs/changelog.rst @@ -27,6 +27,13 @@ Changelog --------- +Breaking changes +~~~~~~~~~~~~~~~~ + +* ``HttpHook no longer forwards Connection extra headers across a cross-host redirect (#70000)`` + + Headers defined in an HTTP Connection's ``extra`` field are documented as a way to carry credentials, and ``requests`` only strips ``Authorization`` when a redirect leaves the original host. ``HttpHook`` now gives those Connection-supplied headers the same treatment, so a secret held under any header name (``X-API-Key``, ``Private-Token``, ...) is no longer replayed to the redirect target. Same-host redirects are unaffected. If you relied on those headers reaching a different host - for example an API that redirects downloads to a CDN or object store that needs the same key - either configure a Connection whose host matches the final destination, or follow the redirect explicitly by passing ``allow_redirects=False`` in ``extra_options`` and issuing the second request yourself. ``HttpAsyncHook`` still forwards the headers; that is tracked in https://github.com/apache/airflow/issues/70164. + 6.0.4 ..... diff --git a/providers/http/src/airflow/providers/http/hooks/http.py b/providers/http/src/airflow/providers/http/hooks/http.py index 061a0d8930134..41368fc8024f7 100644 --- a/providers/http/src/airflow/providers/http/hooks/http.py +++ b/providers/http/src/airflow/providers/http/hooks/http.py @@ -652,6 +652,9 @@ async def config(self) -> SessionConfig: auth = self.auth_type(conn.login, conn.password) if conn.extra: + # Unlike HttpHook these headers survive a cross-host redirect, because aiohttp + # has no per-redirect hook to drop them from; + # tracked at https://github.com/apache/airflow/issues/70164 conn_extra_options, extra_options = _process_extra_options_from_connection( conn=conn, extra_options={} ) diff --git a/uv.lock b/uv.lock index dc07848533e42..3efa30f388798 100644 --- a/uv.lock +++ b/uv.lock @@ -64,9 +64,9 @@ apache-airflow-providers-apache-cassandra = false apache-airflow-providers-asana = false apache-airflow-providers-oracle = false apache-airflow-providers-mysql = false +apache-airflow-providers-teradata = false apache-airflow-providers-alibaba = false apache-airflow-providers-microsoft-mssql = false -apache-airflow-providers-teradata = false apache-airflow-providers-jdbc = false apache-airflow-helm-chart = false apache-airflow-providers-anthropic = false @@ -647,7 +647,7 @@ name = "aiohttp-cors" version = "0.8.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "aiohttp" }, + { name = "aiohttp", marker = "python_full_version < '3.15'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/6f/6d/d89e846a5444b3d5eb8985a6ddb0daef3774928e1bfbce8e84ec97b0ffa7/aiohttp_cors-0.8.1.tar.gz", hash = "sha256:ccacf9cb84b64939ea15f859a146af1f662a6b1d68175754a07315e305fb1403", size = 38626, upload-time = "2025-03-31T14:16:20.048Z" } wheels = [ @@ -4813,6 +4813,9 @@ dependencies = [ ] [package.optional-dependencies] +amazon = [ + { name = "apache-airflow-providers-amazon" }, +] avro = [ { name = "fastavro" }, ] @@ -4842,6 +4845,7 @@ standard = [ dev = [ { name = "apache-airflow" }, { name = "apache-airflow-devel-common" }, + { name = "apache-airflow-providers-amazon" }, { name = "apache-airflow-providers-common-compat" }, { name = "apache-airflow-providers-common-sql", extra = ["pandas", "polars"] }, { name = "apache-airflow-providers-databricks", extra = ["sqlalchemy"] }, @@ -4860,6 +4864,7 @@ docs = [ requires-dist = [ { name = "aiohttp", specifier = ">=3.14.0,<4" }, { name = "apache-airflow", editable = "." }, + { name = "apache-airflow-providers-amazon", marker = "extra == 'amazon'", editable = "providers/amazon" }, { name = "apache-airflow-providers-common-compat", editable = "providers/common/compat" }, { name = "apache-airflow-providers-common-sql", editable = "providers/common/sql" }, { name = "apache-airflow-providers-fab", marker = "extra == 'fab'", editable = "providers/fab" }, @@ -4882,12 +4887,13 @@ requires-dist = [ { name = "pyarrow", marker = "python_full_version >= '3.14'", specifier = ">=22.0.0" }, { name = "requests", specifier = ">=2.32.0,<3" }, ] -provides-extras = ["avro", "azure-identity", "fab", "google", "sdk", "standard", "openlineage", "sqlalchemy"] +provides-extras = ["avro", "amazon", "azure-identity", "fab", "google", "sdk", "standard", "openlineage", "sqlalchemy"] [package.metadata.requires-dev] dev = [ { name = "apache-airflow", editable = "." }, { name = "apache-airflow-devel-common", editable = "devel-common" }, + { name = "apache-airflow-providers-amazon", editable = "providers/amazon" }, { name = "apache-airflow-providers-common-compat", editable = "providers/common/compat" }, { name = "apache-airflow-providers-common-sql", editable = "providers/common/sql" }, { name = "apache-airflow-providers-common-sql", extras = ["pandas", "polars"], editable = "providers/common/sql" }, @@ -10982,7 +10988,7 @@ name = "colorful" version = "0.5.8" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "colorama", marker = "python_full_version < '3.15' and sys_platform == 'win32'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/82/31/109ef4bedeb32b4202e02ddb133162457adc4eb890a9ed9c05c9dd126ed0/colorful-0.5.8.tar.gz", hash = "sha256:bb16502b198be2f1c42ba3c52c703d5f651d826076817185f0294c1a549a7445", size = 209361, upload-time = "2025-10-29T11:53:21.663Z" } wheels = [ @@ -17521,9 +17527,9 @@ name = "opencensus" version = "0.11.4" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "google-api-core" }, - { name = "opencensus-context" }, - { name = "six" }, + { name = "google-api-core", marker = "python_full_version < '3.15'" }, + { name = "opencensus-context", marker = "python_full_version < '3.15'" }, + { name = "six", marker = "python_full_version < '3.15'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/15/a7/a46dcffa1b63084f9f17fe3c8cb20724c4c8f91009fd0b2cfdb27d5d2b35/opencensus-0.11.4.tar.gz", hash = "sha256:cbef87d8b8773064ab60e5c2a1ced58bbaa38a6d052c41aec224958ce544eff2", size = 64966, upload-time = "2024-01-03T18:04:07.085Z" } wheels = [ @@ -20822,14 +20828,14 @@ name = "ray" version = "2.56.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "click" }, - { name = "filelock" }, - { name = "jsonschema" }, - { name = "msgpack" }, - { name = "packaging" }, - { name = "protobuf" }, - { name = "pyyaml" }, - { name = "requests" }, + { name = "click", marker = "python_full_version < '3.15'" }, + { name = "filelock", marker = "python_full_version < '3.15'" }, + { name = "jsonschema", marker = "python_full_version < '3.15'" }, + { name = "msgpack", marker = "python_full_version < '3.15'" }, + { name = "packaging", marker = "python_full_version < '3.15'" }, + { name = "protobuf", marker = "python_full_version < '3.15'" }, + { name = "pyyaml", marker = "python_full_version < '3.15'" }, + { name = "requests", marker = "python_full_version < '3.15'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/9e/c0/d6ffbe8ae2e2e10ea07aa08ea2dd35597239f0b3faaa29b5d7bbc405ab32/ray-2.56.0-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:f34b2345a47ad144292c1b34eeba2ed8d556078f7bd118d1adf2090d5199c843", size = 66362009, upload-time = "2026-06-29T20:50:14.442Z" }, @@ -20854,20 +20860,20 @@ wheels = [ [package.optional-dependencies] default = [ - { name = "aiohttp" }, - { name = "aiohttp-cors" }, - { name = "colorful" }, - { name = "grpcio" }, - { name = "opencensus" }, - { name = "opentelemetry-exporter-prometheus" }, - { name = "opentelemetry-proto" }, - { name = "opentelemetry-sdk" }, - { name = "prometheus-client" }, - { name = "py-spy" }, - { name = "pydantic" }, - { name = "requests" }, - { name = "smart-open" }, - { name = "virtualenv" }, + { name = "aiohttp", marker = "python_full_version < '3.15'" }, + { name = "aiohttp-cors", marker = "python_full_version < '3.15'" }, + { name = "colorful", marker = "python_full_version < '3.15'" }, + { name = "grpcio", marker = "python_full_version < '3.15'" }, + { name = "opencensus", marker = "python_full_version < '3.15'" }, + { name = "opentelemetry-exporter-prometheus", marker = "python_full_version < '3.15'" }, + { name = "opentelemetry-proto", marker = "python_full_version < '3.15'" }, + { name = "opentelemetry-sdk", marker = "python_full_version < '3.15'" }, + { name = "prometheus-client", marker = "python_full_version < '3.15'" }, + { name = "py-spy", marker = "python_full_version < '3.15'" }, + { name = "pydantic", marker = "python_full_version < '3.15'" }, + { name = "requests", marker = "python_full_version < '3.15'" }, + { name = "smart-open", marker = "python_full_version < '3.15'" }, + { name = "virtualenv", marker = "python_full_version < '3.15'" }, ] [[package]] @@ -21964,8 +21970,8 @@ name = "secretstorage" version = "3.5.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cryptography", marker = "python_full_version >= '3.14' or platform_machine != 'arm64' or sys_platform != 'darwin'" }, - { name = "jeepney", marker = "python_full_version >= '3.14' or platform_machine != 'arm64' or sys_platform != 'darwin'" }, + { name = "cryptography", marker = "(python_full_version >= '3.14' and sys_platform == 'darwin') or (python_full_version < '3.15' and sys_platform == 'emscripten') or (python_full_version < '3.15' and sys_platform == 'win32') or (platform_machine != 'arm64' and sys_platform == 'darwin') or (sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32')" }, + { name = "jeepney", marker = "(python_full_version >= '3.14' and sys_platform == 'darwin') or (python_full_version < '3.15' and sys_platform == 'emscripten') or (python_full_version < '3.15' and sys_platform == 'win32') or (platform_machine != 'arm64' and sys_platform == 'darwin') or (sys_platform != 'darwin' and sys_platform != 'emscripten' and sys_platform != 'win32')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/1c/03/e834bcd866f2f8a49a85eaff47340affa3bfa391ee9912a952a1faa68c7b/secretstorage-3.5.0.tar.gz", hash = "sha256:f04b8e4689cbce351744d5537bf6b1329c6fc68f91fa666f60a380edddcd11be", size = 19884, upload-time = "2025-11-23T19:02:53.191Z" } wheels = [ @@ -22164,7 +22170,7 @@ name = "smart-open" version = "8.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "wrapt" }, + { name = "wrapt", marker = "python_full_version < '3.15'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/29/3e/79fd5fd2375a8a500b9ec2f6a0762fc1ac33e35582d4a87483a78d19408f/smart_open-8.0.0.tar.gz", hash = "sha256:5a2008d60688bd3b33c52e2ef666d3c60cf956e73e215de8c7b242cf56fdd1b2", size = 61520, upload-time = "2026-06-27T16:28:11.894Z" } wheels = [