From 6c96bdf83928131ae42e18a866191ef18dcef454 Mon Sep 17 00:00:00 2001 From: Sam Sahbaei Razavi Date: Sun, 28 Jun 2026 21:22:14 +0200 Subject: [PATCH] fix: suppress pyarrow nullability warning for primary keys --- dlt/extract/extractors.py | 8 +++++ .../sql_database/test_sql_database_source.py | 29 +++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/dlt/extract/extractors.py b/dlt/extract/extractors.py index bdaebca50a..3fef7e0465 100644 --- a/dlt/extract/extractors.py +++ b/dlt/extract/extractors.py @@ -541,6 +541,14 @@ def _compute_tables( for hint_name, hint in column.items(): if (src_hint := src_column.get(hint_name)) is not None: if src_hint != hint: + if ( + hint_name == "nullable" + and src_column.get("primary_key") is True + and src_hint is False + and hint is True + ): + # PyArrow has no primary-key metadata, so dlt PK fields may look nullable. + continue override_warn = True logger.info( f"In resource: {resource.name}, when merging arrow schema" diff --git a/tests/load/sources/sql_database/test_sql_database_source.py b/tests/load/sources/sql_database/test_sql_database_source.py index 2f788feac2..03b55152ac 100644 --- a/tests/load/sources/sql_database/test_sql_database_source.py +++ b/tests/load/sources/sql_database/test_sql_database_source.py @@ -708,6 +708,35 @@ def make_subquery(table): assert len(data) == postgres_db.table_infos["chat_message"]["row_count"] +def test_remove_nullability_pyarrow_primary_key_does_not_warn( + postgres_db: PostgresSourceDB, caplog: Any +) -> None: + read_table = sql_table( + table="chat_message", + credentials=postgres_db.credentials, + schema=postgres_db.schema, + backend="pyarrow", + reflection_level="full_with_precision", + table_adapter_callback=remove_nullability_adapter, + ) + + table_schema = read_table.compute_table_schema() + id_column = table_schema["columns"]["id"] + assert id_column["primary_key"] is True + assert id_column["nullable"] is False + + pipeline = make_pipeline("duckdb") + with capture_dlt_logger(caplog) as caplog: + pipeline.run(read_table) + + warning_messages = [ + record.message + for record in caplog.records + if "when merging arrow schema" in record.message + ] + assert warning_messages == [] + + @pytest.mark.parametrize("backend", ["sqlalchemy", "pandas", "pyarrow", "connectorx"]) @pytest.mark.parametrize("row_order", ["asc", "desc", None]) @pytest.mark.parametrize("last_value_func", [min, max, lambda x: max(x)])