From e616856a0c304491dad18322f3309890a6355cbc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Thu, 9 Jul 2026 10:44:05 +0200 Subject: [PATCH 1/8] GH-48808: [Python] Investigate what entails bumping pandas >=2 --- python/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/pyproject.toml b/python/pyproject.toml index 9ceb5da92778..94a9e5368668 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -63,7 +63,7 @@ test = [ 'hypothesis', 'cffi', 'pytz', - 'pandas' + 'pandas>=2' ] [tool.numpydoc_validation] From 7a046528d7648bf3d0e04d68862ee52f1020fe8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Thu, 9 Jul 2026 11:24:38 +0200 Subject: [PATCH 2/8] Drop a bunch of pd<2 cases --- python/pyarrow/array.pxi | 4 - python/pyarrow/pandas-shim.pxi | 14 +-- python/pyarrow/pandas_compat.py | 2 - python/pyarrow/table.pxi | 4 - .../tests/interchange/test_conversion.py | 33 +------ python/pyarrow/tests/parquet/test_pandas.py | 3 - python/pyarrow/tests/test_compute.py | 18 +--- python/pyarrow/tests/test_pandas.py | 92 ++----------------- python/pyarrow/tests/test_schema.py | 8 -- python/pyarrow/types.pxi | 4 +- 10 files changed, 21 insertions(+), 161 deletions(-) diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi index 5fc74969abfd..e259a5e4fbaf 100644 --- a/python/pyarrow/array.pxi +++ b/python/pyarrow/array.pxi @@ -2383,10 +2383,6 @@ cdef _array_like_to_pandas(obj, options, types_mapper): arr = dtype.__from_arrow__(obj) return pandas_api.series(arr, name=name, copy=False) - if pandas_api.is_v1(): - # ARROW-3789: Coerce date/timestamp types to datetime64[ns] - c_options.coerce_temporal_nanoseconds = True - if isinstance(obj, Array): with nogil: check_status(ConvertArrayToPandas(c_options, diff --git a/python/pyarrow/pandas-shim.pxi b/python/pyarrow/pandas-shim.pxi index 3dbb97cd8028..ba7a2385ba74 100644 --- a/python/pyarrow/pandas-shim.pxi +++ b/python/pyarrow/pandas-shim.pxi @@ -38,7 +38,7 @@ cdef class _PandasAPIShim(object): object _array_like_types, _is_extension_array_dtype, _lock bint has_sparse bint _pd024 - bint _is_v1, _is_ge_v21, _is_ge_v23, _is_ge_v3, _is_ge_v3_strict + bint _is_ge_v21, _is_ge_v23, _is_ge_v3, _is_ge_v3_strict def __init__(self): self._lock = Lock() @@ -61,25 +61,23 @@ cdef class _PandasAPIShim(object): self._pd = pd self._version = pd.__version__ self._loose_version = Version(pd.__version__) - self._is_v1 = False - if self._loose_version < Version('1.0.0'): + if self._loose_version < Version('2.0.0'): self._have_pandas = False if raise_: raise ImportError( - f"pyarrow requires pandas 1.0.0 or above, pandas {self._version} is " + f"pyarrow requires pandas 2.0.0 or above, pandas {self._version} is " "installed" ) else: warnings.warn( - f"pyarrow requires pandas 1.0.0 or above, pandas {self._version} is " + f"pyarrow requires pandas 2.0.0 or above, pandas {self._version} is " "installed. Therefore, pandas-specific integration is not " "used.", stacklevel=2 ) return - self._is_v1 = self._loose_version < Version('2.0.0') self._is_ge_v21 = self._loose_version >= Version('2.1.0') self._is_ge_v23 = self._loose_version >= Version('2.3.0.dev0') self._is_ge_v3 = self._loose_version >= Version('3.0.0.dev0') @@ -166,10 +164,6 @@ cdef class _PandasAPIShim(object): self._check_import() return self._version - def is_v1(self): - self._check_import() - return self._is_v1 - def is_ge_v21(self): self._check_import() return self._is_ge_v21 diff --git a/python/pyarrow/pandas_compat.py b/python/pyarrow/pandas_compat.py index d8fd383d31d1..ccb89fc05da5 100644 --- a/python/pyarrow/pandas_compat.py +++ b/python/pyarrow/pandas_compat.py @@ -790,8 +790,6 @@ def _reconstruct_block(item, columns=None, extension_columns=None, return_block= def make_datetimetz(unit, tz): - if _pandas_api.is_v1(): - unit = 'ns' # ARROW-3789: Coerce date/timestamp types to datetime64[ns] tz = pa.lib.string_to_tzinfo(tz, prefer_zoneinfo=_pandas_api.is_ge_v3()) return _pandas_api.datetimetz_type(unit, tz=tz) diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi index fc7c4fcfc84f..17ca35e476d8 100644 --- a/python/pyarrow/table.pxi +++ b/python/pyarrow/table.pxi @@ -4070,10 +4070,6 @@ def table_to_blocks(options, Table table, categories, extension_columns): c_options.extension_columns = make_shared[unordered_set[c_string]]( unordered_set[c_string]({tobytes(col) for col in extension_columns})) - if pandas_api.is_v1(): - # ARROW-3789: Coerce date/timestamp types to datetime64[ns] - c_options.coerce_temporal_nanoseconds = True - if c_options.self_destruct: # Move the shared_ptr, table is now unsafe to use further c_table = move(table.sp_table) diff --git a/python/pyarrow/tests/interchange/test_conversion.py b/python/pyarrow/tests/interchange/test_conversion.py index 81713b94e6db..4218694d3418 100644 --- a/python/pyarrow/tests/interchange/test_conversion.py +++ b/python/pyarrow/tests/interchange/test_conversion.py @@ -122,9 +122,6 @@ def test_offset_of_sliced_array(): ] ) def test_pandas_roundtrip(uint, int, float, np_float_str): - if Version(pd.__version__) < Version("1.5.0"): - pytest.skip("__dataframe__ added to pandas in 1.5.0") - arr = [1, 2, 3] table = pa.table( { @@ -153,9 +150,6 @@ def test_pandas_roundtrip(uint, int, float, np_float_str): @pytest.mark.pandas def test_pandas_roundtrip_string(): # See https://github.com/pandas-dev/pandas/issues/50554 - if Version(pd.__version__) < Version("1.6"): - pytest.skip("Column.size() bug in pandas") - arr = ["a", "", "c"] table = pa.table({"a": pa.array(arr)}) @@ -182,9 +176,6 @@ def test_pandas_roundtrip_string(): @pytest.mark.pandas def test_pandas_roundtrip_large_string(): # See https://github.com/pandas-dev/pandas/issues/50554 - if Version(pd.__version__) < Version("1.6"): - pytest.skip("Column.size() bug in pandas") - arr = ["a", "", "c"] table = pa.table({"a_large": pa.array(arr, type=pa.large_string())}) @@ -219,9 +210,6 @@ def test_pandas_roundtrip_large_string(): @pytest.mark.pandas def test_pandas_roundtrip_string_with_missing(): # See https://github.com/pandas-dev/pandas/issues/50554 - if Version(pd.__version__) < Version("1.6"): - pytest.skip("Column.size() bug in pandas") - arr = ["a", "", "c", None] table = pa.table({"a": pa.array(arr), "a_large": pa.array(arr, type=pa.large_string())}) @@ -299,8 +287,6 @@ def test_pandas_roundtrip_categorical(): @pytest.mark.pandas @pytest.mark.parametrize("unit", ['s', 'ms', 'us', 'ns']) def test_pandas_roundtrip_datetime(unit): - if Version(pd.__version__) < Version("1.5.0"): - pytest.skip("__dataframe__ added to pandas in 1.5.0") from datetime import datetime as dt # timezones not included as they are not yet supported in @@ -308,12 +294,7 @@ def test_pandas_roundtrip_datetime(unit): dt_arr = [dt(2007, 7, 13), dt(2007, 7, 14), dt(2007, 7, 15)] table = pa.table({"a": pa.array(dt_arr, type=pa.timestamp(unit))}) - if Version(pd.__version__) < Version("1.6"): - # pandas < 2.0 always creates datetime64 in "ns" - # resolution - expected = pa.table({"a": pa.array(dt_arr, type=pa.timestamp('ns'))}) - else: - expected = table + expected = table from pandas.api.interchange import ( from_dataframe as pandas_from_dataframe @@ -337,9 +318,6 @@ def test_pandas_roundtrip_datetime(unit): "np_float_str", ["float32", "float64"] ) def test_pandas_to_pyarrow_with_missing(np_float_str): - if Version(pd.__version__) < Version("1.5.0"): - pytest.skip("__dataframe__ added to pandas in 1.5.0") - np_array = np.array([0, np.nan, 2], dtype=np.dtype(np_float_str)) datetime_array = [None, dt(2007, 7, 14), dt(2007, 7, 15)] df = pd.DataFrame({ @@ -359,9 +337,6 @@ def test_pandas_to_pyarrow_with_missing(np_float_str): @pytest.mark.pandas def test_pandas_to_pyarrow_float16_with_missing(): - if Version(pd.__version__) < Version("1.5.0"): - pytest.skip("__dataframe__ added to pandas in 1.5.0") - # np.float16 errors if ps.is_nan is used # pyarrow.lib.ArrowNotImplementedError: Function 'is_nan' has no kernel # matching input types (halffloat) @@ -482,9 +457,6 @@ def test_nan_as_null(): @pytest.mark.pandas def test_allow_copy_false(): - if Version(pd.__version__) < Version("1.5.0"): - pytest.skip("__dataframe__ added to pandas in 1.5.0") - # Test that an error is raised when a copy is needed # to create a bitmask @@ -501,9 +473,6 @@ def test_allow_copy_false(): @pytest.mark.pandas def test_allow_copy_false_bool_categorical(): - if Version(pd.__version__) < Version("1.5.0"): - pytest.skip("__dataframe__ added to pandas in 1.5.0") - # Test that an error is raised for boolean # and categorical dtype (copy is always made) diff --git a/python/pyarrow/tests/parquet/test_pandas.py b/python/pyarrow/tests/parquet/test_pandas.py index 53864ff15ea2..8dbd5d71b9bc 100644 --- a/python/pyarrow/tests/parquet/test_pandas.py +++ b/python/pyarrow/tests/parquet/test_pandas.py @@ -524,9 +524,6 @@ def test_pandas_categorical_roundtrip(): @pytest.mark.pandas def test_categories_with_string_pyarrow_dtype(tempdir): # gh-33727: writing to parquet should not fail - if Version(pd.__version__) < Version("1.3.0"): - pytest.skip("PyArrow backed string data type introduced in pandas 1.3.0") - df1 = pd.DataFrame({"x": ["foo", "bar", "foo"]}, dtype="string[pyarrow]") df1 = df1.astype("category") diff --git a/python/pyarrow/tests/test_compute.py b/python/pyarrow/tests/test_compute.py index 39167e76c8f6..1e08e73668e5 100644 --- a/python/pyarrow/tests/test_compute.py +++ b/python/pyarrow/tests/test_compute.py @@ -2612,8 +2612,6 @@ def test_strftime(): def _check_datetime_components(timestamps, timezone=None): - from pyarrow.vendored.version import Version - ts = pd.to_datetime(timestamps).tz_localize( "UTC").tz_convert(timezone).to_series() tsa = pa.array(ts, pa.timestamp("ns", tz=timezone)) @@ -2626,17 +2624,11 @@ def _check_datetime_components(timestamps, timezone=None): pa.field('iso_day_of_week', pa.int64()) ] - if Version(pd.__version__) < Version("1.1.0"): - # https://github.com/pandas-dev/pandas/issues/33206 - iso_year = ts.map(lambda x: x.isocalendar()[0]).astype("int64") - iso_week = ts.map(lambda x: x.isocalendar()[1]).astype("int64") - iso_day = ts.map(lambda x: x.isocalendar()[2]).astype("int64") - else: - # Casting is required because pandas isocalendar returns int32 - # while arrow isocalendar returns int64. - iso_year = ts.dt.isocalendar()["year"].astype("int64") - iso_week = ts.dt.isocalendar()["week"].astype("int64") - iso_day = ts.dt.isocalendar()["day"].astype("int64") + # Casting is required because pandas isocalendar returns int32 + # while arrow isocalendar returns int64. + iso_year = ts.dt.isocalendar()["year"].astype("int64") + iso_week = ts.dt.isocalendar()["week"].astype("int64") + iso_day = ts.dt.isocalendar()["day"].astype("int64") iso_calendar = pa.StructArray.from_arrays( [iso_year, iso_week, iso_day], diff --git a/python/pyarrow/tests/test_pandas.py b/python/pyarrow/tests/test_pandas.py index c42dc04cd196..7572cee404ee 100644 --- a/python/pyarrow/tests/test_pandas.py +++ b/python/pyarrow/tests/test_pandas.py @@ -488,14 +488,10 @@ def test_string_column_index(self): @pytest.mark.parametrize('unit', ['us', 'ns']) def test_datetimetz_column_index(self, unit): - ext_kwargs = {} - if Version(pd.__version__) >= Version("2.0.0"): - # unit argument not supported on date_range for pandas < 2.0.0 - ext_kwargs = {'unit': unit} df = pd.DataFrame( [(1, 'a', 2.0), (2, 'b', 3.0), (3, 'c', 4.0)], columns=pd.date_range( - start='2017-01-01', periods=3, tz='America/New_York', **ext_kwargs + start='2017-01-01', periods=3, tz='America/New_York', unit=unit ) ) t = pa.Table.from_pandas(df, preserve_index=True) @@ -504,10 +500,7 @@ def test_datetimetz_column_index(self, unit): column_indexes, = js['column_indexes'] assert column_indexes['name'] is None assert column_indexes['pandas_type'] == 'datetimetz' - if ext_kwargs: - assert column_indexes['numpy_type'] == f'datetime64[{unit}]' - else: - assert column_indexes['numpy_type'] == 'datetime64[ns]' + assert column_indexes['numpy_type'] == f'datetime64[{unit}]' md = column_indexes['metadata'] assert md['timezone'] == 'America/New_York' @@ -751,12 +744,8 @@ def test_mismatch_metadata_schema(self): # It is possible that the metadata and actual schema is not fully # matching (eg no timezone information for tz-aware column) # -> to_pandas() conversion should not fail on that - ext_kwargs = {} - if Version(pd.__version__) >= Version("2.0.0"): - # unit argument not supported on date_range for pandas < 2.0.0 - ext_kwargs = {'unit': 'ns'} df = pd.DataFrame({"datetime": pd.date_range( - "2020-01-01", periods=3, **ext_kwargs)}) + "2020-01-01", periods=3, unit='ns')}) # OPTION 1: casting after conversion table = pa.Table.from_pandas(df) @@ -1156,8 +1145,6 @@ def test_timestamps_notimezone_nulls(self): @pytest.mark.parametrize('unit', ['s', 'ms', 'us', 'ns']) def test_timestamps_with_timezone(self, unit): - if Version(pd.__version__) < Version("2.0.0") and unit != 'ns': - pytest.skip("pandas < 2.0 only supports nanosecond datetime64") df = pd.DataFrame({ 'datetime64': np.array([ '2007-07-13T01:23:34.123', @@ -1347,10 +1334,6 @@ def test_array_types_date_as_object(self, coerce_to_ns, expected_dtype): expected_days = np.array(['2000-01-01', None, '1970-01-01', '2040-02-26'], dtype='datetime64[D]') - if Version(pd.__version__) < Version("2.0.0"): - # ARROW-3789: Coerce date/timestamp types to datetime64[ns] - expected_dtype = 'datetime64[ns]' - expected = np.array(['2000-01-01', None, '1970-01-01', '2040-02-26'], dtype=expected_dtype) @@ -1648,9 +1631,6 @@ def test_numpy_datetime64_columns(self): def test_timestamp_to_pandas_coerces_to_ns(self): # non-ns timestamp gets cast to ns on conversion to pandas - if Version(pd.__version__) >= Version("2.0.0"): - pytest.skip("pandas >= 2.0 supports non-nanosecond datetime64") - arr = pa.array([1, 2, 3], pa.timestamp('ms')) expected = pd.Series(pd.to_datetime([1, 2, 3], unit='ms')) s = arr.to_pandas() @@ -1735,8 +1715,6 @@ def test_fixed_offset_timezone(self): @pytest.mark.parametrize("unit", ['s', 'ms', 'us', 'ns']) def test_timedeltas_no_nulls(self, unit): - if Version(pd.__version__) < Version("2.0.0"): - unit = 'ns' df = pd.DataFrame({ 'timedelta64': np.array([0, 3600000000000, 7200000000000], dtype=f'timedelta64[{unit}]') @@ -1750,8 +1728,6 @@ def test_timedeltas_no_nulls(self, unit): @pytest.mark.parametrize("unit", ['s', 'ms', 'us', 'ns']) def test_timedeltas_nulls(self, unit): - if Version(pd.__version__) < Version("2.0.0"): - unit = 'ns' df = pd.DataFrame({ 'timedelta64': np.array([0, None, 7200000000000], dtype=f'timedelta64[{unit}]') @@ -2419,11 +2395,6 @@ def test_to_list_of_structs_pandas(self): tm.assert_series_equal(series, expected) def test_to_list_of_maps_pandas(self): - if ((Version(np.__version__) >= Version("1.25.0.dev0")) and - (Version(pd.__version__) < Version("2.0.0"))): - # TODO: regression in pandas with numpy 1.25dev - # https://github.com/pandas-dev/pandas/issues/50360 - pytest.skip("Regression in pandas with numpy 1.25") data = [ [[('foo', ['a', 'b']), ('bar', ['c', 'd'])]], [[('baz', []), ('qux', None), ('quux', [None, 'e'])], [('quz', ['f', 'g'])]] @@ -2443,13 +2414,6 @@ def test_to_list_of_maps_pandas_sliced(self): """ A slightly more rigorous test for chunk/slice combinations """ - - if ((Version(np.__version__) >= Version("1.25.0.dev0")) and - (Version(pd.__version__) < Version("2.0.0"))): - # TODO: regression in pandas with numpy 1.25dev - # https://github.com/pandas-dev/pandas/issues/50360 - pytest.skip("Regression in pandas with numpy 1.25") - keys = pa.array(['ignore', 'foo', 'bar', 'baz', 'qux', 'quux', 'ignore']).slice(1, 5) items = pa.array( @@ -3799,9 +3763,7 @@ def test_table_from_pandas_schema_field_order_metadata(): result = table.to_pandas() coerce_cols_to_types = {"float": "float32"} - if Version(pd.__version__) >= Version("2.0.0"): - # Pandas v2 now support non-nanosecond time units - coerce_cols_to_types["datetime"] = "datetime64[s, UTC]" + coerce_cols_to_types["datetime"] = "datetime64[s, UTC]" expected = df[["float", "datetime"]].astype(coerce_cols_to_types) tm.assert_frame_equal(result, expected) @@ -3969,10 +3931,7 @@ def test_to_pandas_split_blocks(): def _get_mgr(df): - if Version(pd.__version__) < Version("1.1.0"): - return df._data - else: - return df._mgr + return df._mgr def _check_blocks_created(t, number): @@ -4383,9 +4342,6 @@ def test_dictionary_from_pandas_specified_type(): def test_convert_categories_to_array_with_string_pyarrow_dtype(): # gh-33727: categories should be converted to pa.Array - if Version(pd.__version__) < Version("1.3.0"): - pytest.skip("PyArrow backed string data type introduced in pandas 1.3.0") - df = pd.DataFrame({"x": ["foo", "bar", "foo"]}, dtype="string[pyarrow]") df = df.astype("category") indices = pa.array(df['x'].cat.codes) @@ -4524,12 +4480,8 @@ def test_convert_to_extension_array(monkeypatch): tm.assert_frame_equal(result, df2) # monkeypatch pandas Int64Dtype to *not* have the protocol method - if Version(pd.__version__) < Version("1.3.0.dev"): - monkeypatch.delattr( - pd.core.arrays.integer._IntegerDtype, "__from_arrow__") - else: - monkeypatch.delattr( - pd.core.arrays.integer.NumericDtype, "__from_arrow__") + monkeypatch.delattr( + pd.core.arrays.integer.NumericDtype, "__from_arrow__") # Int64Dtype has no __from_arrow__ -> use normal conversion result = table.to_pandas() assert len(_get_mgr(result).blocks) == 1 @@ -4569,12 +4521,8 @@ def test_conversion_extensiontype_to_extensionarray(monkeypatch): # monkeypatch pandas Int64Dtype to *not* have the protocol method # (remove the version added above and the actual version for recent pandas) - if Version(pd.__version__) < Version("1.3.0.dev"): - monkeypatch.delattr( - pd.core.arrays.integer._IntegerDtype, "__from_arrow__") - else: - monkeypatch.delattr( - pd.core.arrays.integer.NumericDtype, "__from_arrow__") + monkeypatch.delattr( + pd.core.arrays.integer.NumericDtype, "__from_arrow__") result = arr.to_pandas() assert _get_mgr(result).blocks[0].values.dtype == np.dtype("int64") @@ -4612,8 +4560,6 @@ def test_to_pandas_extension_dtypes_mapping(): def test_to_pandas_extension_dtypes_mapping_complex_type(): # https://github.com/apache/arrow/pull/44720 - if Version(pd.__version__) < Version("1.5.2"): - pytest.skip("Test relies on pd.ArrowDtype") pa_type = pa.struct( [ pa.field("bar", pa.bool_(), nullable=False), @@ -4636,9 +4582,6 @@ def test_to_pandas_extension_dtypes_mapping_complex_type(): def test_array_to_pandas(): - if Version(pd.__version__) < Version("1.1"): - pytest.skip("ExtensionDtype to_pandas method missing") - for arr in [pd.period_range("2012-01-01", periods=3, freq="D").array, pd.interval_range(1, 4).array]: result = pa.array(arr).to_pandas() @@ -4653,20 +4596,11 @@ def test_array_to_pandas(): def test_roundtrip_empty_table_with_extension_dtype_index(): df = pd.DataFrame(index=pd.interval_range(start=0, end=3)) table = pa.table(df) - if Version(pd.__version__) > Version("1.0"): - tm.assert_index_equal(table.to_pandas().index, df.index) - else: - tm.assert_index_equal(table.to_pandas().index, - pd.Index([{'left': 0, 'right': 1}, - {'left': 1, 'right': 2}, - {'left': 2, 'right': 3}], - dtype='object')) + tm.assert_index_equal(table.to_pandas().index, df.index) @pytest.mark.parametrize("index", ["a", ["a", "b"]]) def test_to_pandas_types_mapper_index(index): - if Version(pd.__version__) < Version("1.5.0"): - pytest.skip("ArrowDtype missing") df = pd.DataFrame( { "a": [1, 2], @@ -4683,9 +4617,6 @@ def test_to_pandas_types_mapper_index(index): def test_array_to_pandas_types_mapper(): # https://issues.apache.org/jira/browse/ARROW-9664 - if Version(pd.__version__) < Version("1.2.0"): - pytest.skip("Float64Dtype extension dtype missing") - data = pa.array([1, 2, 3], pa.int64()) # Test with mapper function @@ -4707,9 +4638,6 @@ def test_array_to_pandas_types_mapper(): @pytest.mark.pandas def test_chunked_array_to_pandas_types_mapper(): # https://issues.apache.org/jira/browse/ARROW-9664 - if Version(pd.__version__) < Version("1.2.0"): - pytest.skip("Float64Dtype extension dtype missing") - data = pa.chunked_array([pa.array([1, 2, 3], pa.int64())]) assert isinstance(data, pa.ChunkedArray) diff --git a/python/pyarrow/tests/test_schema.py b/python/pyarrow/tests/test_schema.py index 029e14ca1628..db44ad32114b 100644 --- a/python/pyarrow/tests/test_schema.py +++ b/python/pyarrow/tests/test_schema.py @@ -27,12 +27,6 @@ import pyarrow as pa import pyarrow.tests.util as test_util -from pyarrow.vendored.version import Version - -try: - import pandas as pd -except ImportError: - pass def test_schema_constructor_errors(): @@ -55,8 +49,6 @@ def test_type_integers(): @pytest.mark.pandas def test_type_to_pandas_dtype(): M8 = np.dtype('datetime64[ms]') - if Version(pd.__version__) < Version("2.0.0"): - M8 = np.dtype('datetime64[ns]') cases = [ (pa.null(), np.object_), (pa.bool_(), np.bool_), diff --git a/python/pyarrow/types.pxi b/python/pyarrow/types.pxi index ec1a5a2ba9a3..007f007dc1b6 100644 --- a/python/pyarrow/types.pxi +++ b/python/pyarrow/types.pxi @@ -186,9 +186,7 @@ def _get_pandas_tz_type(arrow_type, coerce_to_ns=False): def _to_pandas_dtype(arrow_type, options=None): - coerce_to_ns = (options and options.get('coerce_temporal_nanoseconds', False)) or ( - _pandas_api.is_v1() and arrow_type.id in - [_Type_DATE32, _Type_DATE64, _Type_TIMESTAMP, _Type_DURATION]) + coerce_to_ns = (options and options.get('coerce_temporal_nanoseconds', False)) if getattr(arrow_type, 'tz', None): dtype = _get_pandas_tz_type(arrow_type, coerce_to_ns) From 4546c607cdfa401eb0a12aeee32c97438818ee65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Thu, 9 Jul 2026 12:37:01 +0200 Subject: [PATCH 3/8] Bump minimum Pandas CI to 2.0.0 --- .github/workflows/python.yml | 8 ++++---- dev/tasks/tasks.yml | 2 +- docs/source/python/install.rst | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 30e6272fb31b..4a91e3ba2393 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -67,7 +67,7 @@ jobs: name: - conda-python-docs - conda-python-3.12-nopandas - - conda-python-3.11-pandas-1.5.2 + - conda-python-3.11-pandas-2.0.0 - conda-python-3.14-pandas-latest - conda-python-3.13-no-numpy include: @@ -79,11 +79,11 @@ jobs: image: conda-python title: AMD64 Conda Python 3.12 Without Pandas python: "3.12" - - name: conda-python-3.11-pandas-1.5.2 + - name: conda-python-3.11-pandas-2.0.0 image: conda-python-pandas - title: AMD64 Conda Python 3.11 Pandas 1.5.2 + title: AMD64 Conda Python 3.11 Pandas 2.0.0 python: "3.11" - pandas: "1.5.2" + pandas: "2.0.0" numpy: "1.23.2" - name: conda-python-3.14-pandas-latest image: conda-python-pandas diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index c443c5a7e7c7..9404b3c6a9da 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -727,7 +727,7 @@ tasks: ############################## Integration tests ############################ -{% for python_version, pandas_version, numpy_version, cache_leaf in [("3.11", "1.5.2", "1.23.2", True), +{% for python_version, pandas_version, numpy_version, cache_leaf in [("3.11", "2.0.0", "1.23.2", True), ("3.12", "latest", "latest", False), ("3.13", "latest", "1.26.2", False), ("3.13", "latest", "latest", False), diff --git a/docs/source/python/install.rst b/docs/source/python/install.rst index 5fa9ab6c778b..40c328512a91 100644 --- a/docs/source/python/install.rst +++ b/docs/source/python/install.rst @@ -74,7 +74,7 @@ Dependencies Optional dependencies * **NumPy 1.23.2** or higher. -* **pandas 1.5.2** or higher, +* **pandas 2.0.0** or higher, * **cffi**. Additional packages PyArrow is compatible with are :ref:`fsspec ` From e2ae2d8c839db2c2a7398d5b6de1c2489270f623 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Thu, 9 Jul 2026 13:11:23 +0200 Subject: [PATCH 4/8] Minor typo and remove unnecessary pin from pyproject.toml --- python/pyarrow/src/arrow/python/arrow_to_pandas.cc | 2 +- python/pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/python/pyarrow/src/arrow/python/arrow_to_pandas.cc b/python/pyarrow/src/arrow/python/arrow_to_pandas.cc index f163266f3b87..348d352a0482 100644 --- a/python/pyarrow/src/arrow/python/arrow_to_pandas.cc +++ b/python/pyarrow/src/arrow/python/arrow_to_pandas.cc @@ -1293,7 +1293,7 @@ struct ObjectWriterVisitor { PyObject** out) { ARROW_DCHECK(internal::BorrowPandasDataOffsetType() != nullptr); // DateOffset objects do not add nanoseconds component to pd.Timestamp. - // as of Pandas 1.3.3 + // as of Pandas 1.3.3 // (https://github.com/pandas-dev/pandas/issues/43892). // So convert microseconds and remainder to preserve data // but give users more expected results. diff --git a/python/pyproject.toml b/python/pyproject.toml index 94a9e5368668..9ceb5da92778 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -63,7 +63,7 @@ test = [ 'hypothesis', 'cffi', 'pytz', - 'pandas>=2' + 'pandas' ] [tool.numpydoc_validation] From 59da22da5c0496dd9d9d773525456fb5ec466e88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Thu, 9 Jul 2026 13:49:24 +0200 Subject: [PATCH 5/8] Remove old test only testing Pandas < 2 --- python/pyarrow/tests/test_pandas.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/python/pyarrow/tests/test_pandas.py b/python/pyarrow/tests/test_pandas.py index 7572cee404ee..ec6ceb553044 100644 --- a/python/pyarrow/tests/test_pandas.py +++ b/python/pyarrow/tests/test_pandas.py @@ -1629,16 +1629,6 @@ def test_numpy_datetime64_columns(self): dtype='datetime64[s]') _check_array_from_pandas_roundtrip(datetime64_s) - def test_timestamp_to_pandas_coerces_to_ns(self): - # non-ns timestamp gets cast to ns on conversion to pandas - arr = pa.array([1, 2, 3], pa.timestamp('ms')) - expected = pd.Series(pd.to_datetime([1, 2, 3], unit='ms')) - s = arr.to_pandas() - tm.assert_series_equal(s, expected) - arr = pa.chunked_array([arr]) - s = arr.to_pandas() - tm.assert_series_equal(s, expected) - def test_timestamp_to_pandas_out_of_bounds(self): # ARROW-7758 check for out of bounds timestamps for non-ns timestamps # that end up getting coerced into ns timestamps. From 419d3446c857f4d3d2311adca9071f686b066261 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Thu, 9 Jul 2026 14:45:04 +0200 Subject: [PATCH 6/8] Add check for Version < 2.0.2 on Pandas test --- python/pyarrow/tests/test_pandas.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/python/pyarrow/tests/test_pandas.py b/python/pyarrow/tests/test_pandas.py index ec6ceb553044..251d8dc4a1c4 100644 --- a/python/pyarrow/tests/test_pandas.py +++ b/python/pyarrow/tests/test_pandas.py @@ -4550,6 +4550,10 @@ def test_to_pandas_extension_dtypes_mapping(): def test_to_pandas_extension_dtypes_mapping_complex_type(): # https://github.com/apache/arrow/pull/44720 + # For the pandas issue, see backport commit to 2.0.2: + # https://github.com/pandas-dev/pandas/commit/c8cd0277f8f889c5db7463ef7f36b495b5c9de69 + if Version(pd.__version__) < Version("2.0.2"): + pytest.skip("ArrowDtype.type doesn't support struct before pandas 2.0.2") pa_type = pa.struct( [ pa.field("bar", pa.bool_(), nullable=False), From 9b80eded4ddb5c9b5e65fd289fd65b3f4695c4aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Thu, 9 Jul 2026 16:24:05 +0200 Subject: [PATCH 7/8] Bump minimum pandas to 2.0.3 instead of 2.0.0 --- .github/workflows/python.yml | 8 +-- dev/tasks/tasks.yml | 2 +- docs/source/python/install.rst | 2 +- python/pyarrow/pandas-shim.pxi | 6 +- .../tests/interchange/test_conversion.py | 56 +++++++------------ python/pyarrow/tests/test_pandas.py | 5 -- 6 files changed, 28 insertions(+), 51 deletions(-) diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 4a91e3ba2393..713168545fb5 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -67,7 +67,7 @@ jobs: name: - conda-python-docs - conda-python-3.12-nopandas - - conda-python-3.11-pandas-2.0.0 + - conda-python-3.11-pandas-2.0.3 - conda-python-3.14-pandas-latest - conda-python-3.13-no-numpy include: @@ -79,11 +79,11 @@ jobs: image: conda-python title: AMD64 Conda Python 3.12 Without Pandas python: "3.12" - - name: conda-python-3.11-pandas-2.0.0 + - name: conda-python-3.11-pandas-2.0.3 image: conda-python-pandas - title: AMD64 Conda Python 3.11 Pandas 2.0.0 + title: AMD64 Conda Python 3.11 Pandas 2.0.3 python: "3.11" - pandas: "2.0.0" + pandas: "2.0.3" numpy: "1.23.2" - name: conda-python-3.14-pandas-latest image: conda-python-pandas diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index 9404b3c6a9da..89d239a0a266 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -727,7 +727,7 @@ tasks: ############################## Integration tests ############################ -{% for python_version, pandas_version, numpy_version, cache_leaf in [("3.11", "2.0.0", "1.23.2", True), +{% for python_version, pandas_version, numpy_version, cache_leaf in [("3.11", "2.0.3", "1.23.2", True), ("3.12", "latest", "latest", False), ("3.13", "latest", "1.26.2", False), ("3.13", "latest", "latest", False), diff --git a/docs/source/python/install.rst b/docs/source/python/install.rst index 40c328512a91..d076ca96436f 100644 --- a/docs/source/python/install.rst +++ b/docs/source/python/install.rst @@ -74,7 +74,7 @@ Dependencies Optional dependencies * **NumPy 1.23.2** or higher. -* **pandas 2.0.0** or higher, +* **pandas 2.0.3** or higher, * **cffi**. Additional packages PyArrow is compatible with are :ref:`fsspec ` diff --git a/python/pyarrow/pandas-shim.pxi b/python/pyarrow/pandas-shim.pxi index ba7a2385ba74..94802da6b060 100644 --- a/python/pyarrow/pandas-shim.pxi +++ b/python/pyarrow/pandas-shim.pxi @@ -62,16 +62,16 @@ cdef class _PandasAPIShim(object): self._version = pd.__version__ self._loose_version = Version(pd.__version__) - if self._loose_version < Version('2.0.0'): + if self._loose_version < Version('2.0.3'): self._have_pandas = False if raise_: raise ImportError( - f"pyarrow requires pandas 2.0.0 or above, pandas {self._version} is " + f"pyarrow requires pandas 2.0.3 or above, pandas {self._version} is " "installed" ) else: warnings.warn( - f"pyarrow requires pandas 2.0.0 or above, pandas {self._version} is " + f"pyarrow requires pandas 2.0.3 or above, pandas {self._version} is " "installed. Therefore, pandas-specific integration is not " "used.", stacklevel=2 diff --git a/python/pyarrow/tests/interchange/test_conversion.py b/python/pyarrow/tests/interchange/test_conversion.py index 4218694d3418..cc105178ad87 100644 --- a/python/pyarrow/tests/interchange/test_conversion.py +++ b/python/pyarrow/tests/interchange/test_conversion.py @@ -17,7 +17,6 @@ from datetime import datetime as dt import pyarrow as pa -from pyarrow.vendored.version import Version import pytest try: @@ -183,28 +182,20 @@ def test_pandas_roundtrip_large_string(): from_dataframe as pandas_from_dataframe ) - if Version(pd.__version__) >= Version("2.0.1"): - pandas_df = pandas_from_dataframe(table) - result = pi.from_dataframe(pandas_df) - - assert result["a_large"].to_pylist() == table["a_large"].to_pylist() - assert pa.types.is_large_string(table["a_large"].type) - assert pa.types.is_large_string(result["a_large"].type) + pandas_df = pandas_from_dataframe(table) + result = pi.from_dataframe(pandas_df) - table_protocol = table.__dataframe__() - result_protocol = result.__dataframe__() + assert result["a_large"].to_pylist() == table["a_large"].to_pylist() + assert pa.types.is_large_string(table["a_large"].type) + assert pa.types.is_large_string(result["a_large"].type) - assert table_protocol.num_columns() == result_protocol.num_columns() - assert table_protocol.num_rows() == result_protocol.num_rows() - assert table_protocol.num_chunks() == result_protocol.num_chunks() - assert table_protocol.column_names() == result_protocol.column_names() + table_protocol = table.__dataframe__() + result_protocol = result.__dataframe__() - else: - # large string not supported by pandas implementation for - # older versions of pandas - # https://github.com/pandas-dev/pandas/issues/52795 - with pytest.raises(AssertionError): - pandas_from_dataframe(table) + assert table_protocol.num_columns() == result_protocol.num_columns() + assert table_protocol.num_rows() == result_protocol.num_rows() + assert table_protocol.num_chunks() == result_protocol.num_chunks() + assert table_protocol.column_names() == result_protocol.column_names() @pytest.mark.pandas @@ -218,29 +209,20 @@ def test_pandas_roundtrip_string_with_missing(): from_dataframe as pandas_from_dataframe ) - if Version(pd.__version__) >= Version("2.0.2"): - pandas_df = pandas_from_dataframe(table) - result = pi.from_dataframe(pandas_df) + pandas_df = pandas_from_dataframe(table) + result = pi.from_dataframe(pandas_df) - assert result["a"].to_pylist() == table["a"].to_pylist() - assert pa.types.is_string(table["a"].type) - assert pa.types.is_large_string(result["a"].type) + assert result["a"].to_pylist() == table["a"].to_pylist() + assert pa.types.is_string(table["a"].type) + assert pa.types.is_large_string(result["a"].type) - assert result["a_large"].to_pylist() == table["a_large"].to_pylist() - assert pa.types.is_large_string(table["a_large"].type) - assert pa.types.is_large_string(result["a_large"].type) - else: - # older versions of pandas do not have bitmask support - # https://github.com/pandas-dev/pandas/issues/49888 - with pytest.raises(NotImplementedError): - pandas_from_dataframe(table) + assert result["a_large"].to_pylist() == table["a_large"].to_pylist() + assert pa.types.is_large_string(table["a_large"].type) + assert pa.types.is_large_string(result["a_large"].type) @pytest.mark.pandas def test_pandas_roundtrip_categorical(): - if Version(pd.__version__) < Version("2.0.2"): - pytest.skip("Bitmasks not supported in pandas interchange implementation") - arr = ["Mon", "Tue", "Mon", "Wed", "Mon", "Thu", "Fri", "Sat", None] table = pa.table( {"weekday": pa.array(arr).dictionary_encode()} diff --git a/python/pyarrow/tests/test_pandas.py b/python/pyarrow/tests/test_pandas.py index 251d8dc4a1c4..2a782de1648c 100644 --- a/python/pyarrow/tests/test_pandas.py +++ b/python/pyarrow/tests/test_pandas.py @@ -4549,11 +4549,6 @@ def test_to_pandas_extension_dtypes_mapping(): def test_to_pandas_extension_dtypes_mapping_complex_type(): - # https://github.com/apache/arrow/pull/44720 - # For the pandas issue, see backport commit to 2.0.2: - # https://github.com/pandas-dev/pandas/commit/c8cd0277f8f889c5db7463ef7f36b495b5c9de69 - if Version(pd.__version__) < Version("2.0.2"): - pytest.skip("ArrowDtype.type doesn't support struct before pandas 2.0.2") pa_type = pa.struct( [ pa.field("bar", pa.bool_(), nullable=False), From 5dad2a21ec7dcaac667de964e31b78060cc1887f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Thu, 9 Jul 2026 16:35:01 +0200 Subject: [PATCH 8/8] Keep coerce_to_ns as boolean --- python/pyarrow/types.pxi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/pyarrow/types.pxi b/python/pyarrow/types.pxi index 007f007dc1b6..8eea34224acc 100644 --- a/python/pyarrow/types.pxi +++ b/python/pyarrow/types.pxi @@ -186,7 +186,7 @@ def _get_pandas_tz_type(arrow_type, coerce_to_ns=False): def _to_pandas_dtype(arrow_type, options=None): - coerce_to_ns = (options and options.get('coerce_temporal_nanoseconds', False)) + coerce_to_ns = bool(options and options.get('coerce_temporal_nanoseconds', False)) if getattr(arrow_type, 'tz', None): dtype = _get_pandas_tz_type(arrow_type, coerce_to_ns)