From a99911c710ede23b84490253e8f81cf0e39da685 Mon Sep 17 00:00:00 2001 From: Faisal Dosani Date: Thu, 11 Jun 2026 13:19:53 -0400 Subject: [PATCH 1/3] fix: handle empty Arrow-backed join keys to prevent merge crashes --- datacompy/pandas.py | 15 ++++++++++----- tests/test_pandas.py | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 5 deletions(-) diff --git a/datacompy/pandas.py b/datacompy/pandas.py index 24a42f1b..256accdc 100644 --- a/datacompy/pandas.py +++ b/datacompy/pandas.py @@ -363,11 +363,16 @@ def _dataframe_merge(self, ignore_spaces: bool) -> None: else: params = {"on": self.join_columns} - for column in self.join_columns: - self.df1[column] = pandas_normalize_string_column( - self.df1[column], ignore_spaces=ignore_spaces, ignore_case=False - ) - self.df2[column] = pandas_normalize_string_column( + # Skip normalization for empty frames: str.strip() on an empty + # Arrow-backed column drops its chunks to zero, which causes pandas + # merge to call pa.chunked_array([]) and raise ArrowInvalid when + # multiple join columns are present. See issue #514. + if len(self.df1) > 0 or len(self.df2) > 0: + for column in self.join_columns: + self.df1[column] = pandas_normalize_string_column( + self.df1[column], ignore_spaces=ignore_spaces, ignore_case=False + ) + self.df2[column] = pandas_normalize_string_column( self.df2[column], ignore_spaces=ignore_spaces, ignore_case=False ) diff --git a/tests/test_pandas.py b/tests/test_pandas.py index 0fe64dee..b7ee1963 100644 --- a/tests/test_pandas.py +++ b/tests/test_pandas.py @@ -2645,6 +2645,40 @@ def test_columns_with_mismatches_empty_dataframes(): assert result == [] +def test_compare_empty_arrow_backed_join_keys(): + """Regression for #514: empty Arrow-backed join keys must not crash merge.""" + import pyarrow as pa + + for arrow_type in (pa.string(), pa.int64()): + dtype = pd.ArrowDtype(arrow_type) + df1 = pd.DataFrame( + { + "key1": pd.Series([], dtype=dtype), + "key2": pd.Series([], dtype=dtype), + "value": pd.Series([], dtype=dtype), + } + ) + df2 = pd.DataFrame( + { + "key1": pd.Series([], dtype=dtype), + "key2": pd.Series([], dtype=dtype), + "value": pd.Series([], dtype=dtype), + } + ) + + compare = PandasCompare( + df1, + df2, + join_columns=["key1", "key2"], + ignore_spaces=True, + ignore_case=False + ) + + assert len(compare.intersect_rows) == 0 + assert len(compare.df1_unq_rows) == 0 + assert len(compare.df2_unq_rows) == 0 + + def test_columns_with_mismatches_on_index(): """Test columns_with_mismatches when comparing on index.""" df1 = pd.DataFrame( From 076ae5f6cb98f32a82fadea2dff80bbe1cf04d36 Mon Sep 17 00:00:00 2001 From: Faisal Dosani Date: Thu, 11 Jun 2026 13:44:27 -0400 Subject: [PATCH 2/3] lint: ruff files --- tests/test_pandas.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_pandas.py b/tests/test_pandas.py index b7ee1963..69fc4e2c 100644 --- a/tests/test_pandas.py +++ b/tests/test_pandas.py @@ -2667,11 +2667,11 @@ def test_compare_empty_arrow_backed_join_keys(): ) compare = PandasCompare( - df1, - df2, - join_columns=["key1", "key2"], + df1, + df2, + join_columns=["key1", "key2"], ignore_spaces=True, - ignore_case=False + ignore_case=False, ) assert len(compare.intersect_rows) == 0 From 5baede4f77b93bccf5a663411e6b56189787167d Mon Sep 17 00:00:00 2001 From: Faisal Dosani Date: Thu, 11 Jun 2026 13:50:01 -0400 Subject: [PATCH 3/3] fix: correct indentation in string normalization within PandasCompare class --- datacompy/pandas.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/datacompy/pandas.py b/datacompy/pandas.py index 256accdc..496e0d49 100644 --- a/datacompy/pandas.py +++ b/datacompy/pandas.py @@ -373,8 +373,8 @@ def _dataframe_merge(self, ignore_spaces: bool) -> None: self.df1[column], ignore_spaces=ignore_spaces, ignore_case=False ) self.df2[column] = pandas_normalize_string_column( - self.df2[column], ignore_spaces=ignore_spaces, ignore_case=False - ) + self.df2[column], ignore_spaces=ignore_spaces, ignore_case=False + ) outer_join = self.df1.merge( self.df2,