diff --git a/datacompy/pandas.py b/datacompy/pandas.py index 24a42f1b..496e0d49 100644 --- a/datacompy/pandas.py +++ b/datacompy/pandas.py @@ -363,13 +363,18 @@ def _dataframe_merge(self, ignore_spaces: bool) -> None: else: params = {"on": self.join_columns} - for column in self.join_columns: - self.df1[column] = pandas_normalize_string_column( - self.df1[column], ignore_spaces=ignore_spaces, ignore_case=False - ) - self.df2[column] = pandas_normalize_string_column( - self.df2[column], ignore_spaces=ignore_spaces, ignore_case=False - ) + # Skip normalization for empty frames: str.strip() on an empty + # Arrow-backed column drops its chunks to zero, which causes pandas + # merge to call pa.chunked_array([]) and raise ArrowInvalid when + # multiple join columns are present. See issue #514. + if len(self.df1) > 0 or len(self.df2) > 0: + for column in self.join_columns: + self.df1[column] = pandas_normalize_string_column( + self.df1[column], ignore_spaces=ignore_spaces, ignore_case=False + ) + self.df2[column] = pandas_normalize_string_column( + self.df2[column], ignore_spaces=ignore_spaces, ignore_case=False + ) outer_join = self.df1.merge( self.df2, diff --git a/tests/test_pandas.py b/tests/test_pandas.py index 0fe64dee..69fc4e2c 100644 --- a/tests/test_pandas.py +++ b/tests/test_pandas.py @@ -2645,6 +2645,40 @@ def test_columns_with_mismatches_empty_dataframes(): assert result == [] +def test_compare_empty_arrow_backed_join_keys(): + """Regression for #514: empty Arrow-backed join keys must not crash merge.""" + import pyarrow as pa + + for arrow_type in (pa.string(), pa.int64()): + dtype = pd.ArrowDtype(arrow_type) + df1 = pd.DataFrame( + { + "key1": pd.Series([], dtype=dtype), + "key2": pd.Series([], dtype=dtype), + "value": pd.Series([], dtype=dtype), + } + ) + df2 = pd.DataFrame( + { + "key1": pd.Series([], dtype=dtype), + "key2": pd.Series([], dtype=dtype), + "value": pd.Series([], dtype=dtype), + } + ) + + compare = PandasCompare( + df1, + df2, + join_columns=["key1", "key2"], + ignore_spaces=True, + ignore_case=False, + ) + + assert len(compare.intersect_rows) == 0 + assert len(compare.df1_unq_rows) == 0 + assert len(compare.df2_unq_rows) == 0 + + def test_columns_with_mismatches_on_index(): """Test columns_with_mismatches when comparing on index.""" df1 = pd.DataFrame(