diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 30e6272fb31b..713168545fb5 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -67,7 +67,7 @@ jobs: name: - conda-python-docs - conda-python-3.12-nopandas - - conda-python-3.11-pandas-1.5.2 + - conda-python-3.11-pandas-2.0.3 - conda-python-3.14-pandas-latest - conda-python-3.13-no-numpy include: @@ -79,11 +79,11 @@ jobs: image: conda-python title: AMD64 Conda Python 3.12 Without Pandas python: "3.12" - - name: conda-python-3.11-pandas-1.5.2 + - name: conda-python-3.11-pandas-2.0.3 image: conda-python-pandas - title: AMD64 Conda Python 3.11 Pandas 1.5.2 + title: AMD64 Conda Python 3.11 Pandas 2.0.3 python: "3.11" - pandas: "1.5.2" + pandas: "2.0.3" numpy: "1.23.2" - name: conda-python-3.14-pandas-latest image: conda-python-pandas diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index c443c5a7e7c7..89d239a0a266 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -727,7 +727,7 @@ tasks: ############################## Integration tests ############################ -{% for python_version, pandas_version, numpy_version, cache_leaf in [("3.11", "1.5.2", "1.23.2", True), +{% for python_version, pandas_version, numpy_version, cache_leaf in [("3.11", "2.0.3", "1.23.2", True), ("3.12", "latest", "latest", False), ("3.13", "latest", "1.26.2", False), ("3.13", "latest", "latest", False), diff --git a/docs/source/python/install.rst b/docs/source/python/install.rst index 5fa9ab6c778b..d076ca96436f 100644 --- a/docs/source/python/install.rst +++ b/docs/source/python/install.rst @@ -74,7 +74,7 @@ Dependencies Optional dependencies * **NumPy 1.23.2** or higher. -* **pandas 1.5.2** or higher, +* **pandas 2.0.3** or higher, * **cffi**. Additional packages PyArrow is compatible with are :ref:`fsspec ` diff --git a/python/pyarrow/array.pxi b/python/pyarrow/array.pxi index 5fc74969abfd..e259a5e4fbaf 100644 --- a/python/pyarrow/array.pxi +++ b/python/pyarrow/array.pxi @@ -2383,10 +2383,6 @@ cdef _array_like_to_pandas(obj, options, types_mapper): arr = dtype.__from_arrow__(obj) return pandas_api.series(arr, name=name, copy=False) - if pandas_api.is_v1(): - # ARROW-3789: Coerce date/timestamp types to datetime64[ns] - c_options.coerce_temporal_nanoseconds = True - if isinstance(obj, Array): with nogil: check_status(ConvertArrayToPandas(c_options, diff --git a/python/pyarrow/pandas-shim.pxi b/python/pyarrow/pandas-shim.pxi index 3dbb97cd8028..94802da6b060 100644 --- a/python/pyarrow/pandas-shim.pxi +++ b/python/pyarrow/pandas-shim.pxi @@ -38,7 +38,7 @@ cdef class _PandasAPIShim(object): object _array_like_types, _is_extension_array_dtype, _lock bint has_sparse bint _pd024 - bint _is_v1, _is_ge_v21, _is_ge_v23, _is_ge_v3, _is_ge_v3_strict + bint _is_ge_v21, _is_ge_v23, _is_ge_v3, _is_ge_v3_strict def __init__(self): self._lock = Lock() @@ -61,25 +61,23 @@ cdef class _PandasAPIShim(object): self._pd = pd self._version = pd.__version__ self._loose_version = Version(pd.__version__) - self._is_v1 = False - if self._loose_version < Version('1.0.0'): + if self._loose_version < Version('2.0.3'): self._have_pandas = False if raise_: raise ImportError( - f"pyarrow requires pandas 1.0.0 or above, pandas {self._version} is " + f"pyarrow requires pandas 2.0.3 or above, pandas {self._version} is " "installed" ) else: warnings.warn( - f"pyarrow requires pandas 1.0.0 or above, pandas {self._version} is " + f"pyarrow requires pandas 2.0.3 or above, pandas {self._version} is " "installed. Therefore, pandas-specific integration is not " "used.", stacklevel=2 ) return - self._is_v1 = self._loose_version < Version('2.0.0') self._is_ge_v21 = self._loose_version >= Version('2.1.0') self._is_ge_v23 = self._loose_version >= Version('2.3.0.dev0') self._is_ge_v3 = self._loose_version >= Version('3.0.0.dev0') @@ -166,10 +164,6 @@ cdef class _PandasAPIShim(object): self._check_import() return self._version - def is_v1(self): - self._check_import() - return self._is_v1 - def is_ge_v21(self): self._check_import() return self._is_ge_v21 diff --git a/python/pyarrow/pandas_compat.py b/python/pyarrow/pandas_compat.py index d8fd383d31d1..ccb89fc05da5 100644 --- a/python/pyarrow/pandas_compat.py +++ b/python/pyarrow/pandas_compat.py @@ -790,8 +790,6 @@ def _reconstruct_block(item, columns=None, extension_columns=None, return_block= def make_datetimetz(unit, tz): - if _pandas_api.is_v1(): - unit = 'ns' # ARROW-3789: Coerce date/timestamp types to datetime64[ns] tz = pa.lib.string_to_tzinfo(tz, prefer_zoneinfo=_pandas_api.is_ge_v3()) return _pandas_api.datetimetz_type(unit, tz=tz) diff --git a/python/pyarrow/src/arrow/python/arrow_to_pandas.cc b/python/pyarrow/src/arrow/python/arrow_to_pandas.cc index f163266f3b87..348d352a0482 100644 --- a/python/pyarrow/src/arrow/python/arrow_to_pandas.cc +++ b/python/pyarrow/src/arrow/python/arrow_to_pandas.cc @@ -1293,7 +1293,7 @@ struct ObjectWriterVisitor { PyObject** out) { ARROW_DCHECK(internal::BorrowPandasDataOffsetType() != nullptr); // DateOffset objects do not add nanoseconds component to pd.Timestamp. - // as of Pandas 1.3.3 + // as of Pandas 1.3.3 // (https://github.com/pandas-dev/pandas/issues/43892). // So convert microseconds and remainder to preserve data // but give users more expected results. diff --git a/python/pyarrow/table.pxi b/python/pyarrow/table.pxi index fc7c4fcfc84f..17ca35e476d8 100644 --- a/python/pyarrow/table.pxi +++ b/python/pyarrow/table.pxi @@ -4070,10 +4070,6 @@ def table_to_blocks(options, Table table, categories, extension_columns): c_options.extension_columns = make_shared[unordered_set[c_string]]( unordered_set[c_string]({tobytes(col) for col in extension_columns})) - if pandas_api.is_v1(): - # ARROW-3789: Coerce date/timestamp types to datetime64[ns] - c_options.coerce_temporal_nanoseconds = True - if c_options.self_destruct: # Move the shared_ptr, table is now unsafe to use further c_table = move(table.sp_table) diff --git a/python/pyarrow/tests/interchange/test_conversion.py b/python/pyarrow/tests/interchange/test_conversion.py index 81713b94e6db..cc105178ad87 100644 --- a/python/pyarrow/tests/interchange/test_conversion.py +++ b/python/pyarrow/tests/interchange/test_conversion.py @@ -17,7 +17,6 @@ from datetime import datetime as dt import pyarrow as pa -from pyarrow.vendored.version import Version import pytest try: @@ -122,9 +121,6 @@ def test_offset_of_sliced_array(): ] ) def test_pandas_roundtrip(uint, int, float, np_float_str): - if Version(pd.__version__) < Version("1.5.0"): - pytest.skip("__dataframe__ added to pandas in 1.5.0") - arr = [1, 2, 3] table = pa.table( { @@ -153,9 +149,6 @@ def test_pandas_roundtrip(uint, int, float, np_float_str): @pytest.mark.pandas def test_pandas_roundtrip_string(): # See https://github.com/pandas-dev/pandas/issues/50554 - if Version(pd.__version__) < Version("1.6"): - pytest.skip("Column.size() bug in pandas") - arr = ["a", "", "c"] table = pa.table({"a": pa.array(arr)}) @@ -182,9 +175,6 @@ def test_pandas_roundtrip_string(): @pytest.mark.pandas def test_pandas_roundtrip_large_string(): # See https://github.com/pandas-dev/pandas/issues/50554 - if Version(pd.__version__) < Version("1.6"): - pytest.skip("Column.size() bug in pandas") - arr = ["a", "", "c"] table = pa.table({"a_large": pa.array(arr, type=pa.large_string())}) @@ -192,36 +182,25 @@ def test_pandas_roundtrip_large_string(): from_dataframe as pandas_from_dataframe ) - if Version(pd.__version__) >= Version("2.0.1"): - pandas_df = pandas_from_dataframe(table) - result = pi.from_dataframe(pandas_df) - - assert result["a_large"].to_pylist() == table["a_large"].to_pylist() - assert pa.types.is_large_string(table["a_large"].type) - assert pa.types.is_large_string(result["a_large"].type) + pandas_df = pandas_from_dataframe(table) + result = pi.from_dataframe(pandas_df) - table_protocol = table.__dataframe__() - result_protocol = result.__dataframe__() + assert result["a_large"].to_pylist() == table["a_large"].to_pylist() + assert pa.types.is_large_string(table["a_large"].type) + assert pa.types.is_large_string(result["a_large"].type) - assert table_protocol.num_columns() == result_protocol.num_columns() - assert table_protocol.num_rows() == result_protocol.num_rows() - assert table_protocol.num_chunks() == result_protocol.num_chunks() - assert table_protocol.column_names() == result_protocol.column_names() + table_protocol = table.__dataframe__() + result_protocol = result.__dataframe__() - else: - # large string not supported by pandas implementation for - # older versions of pandas - # https://github.com/pandas-dev/pandas/issues/52795 - with pytest.raises(AssertionError): - pandas_from_dataframe(table) + assert table_protocol.num_columns() == result_protocol.num_columns() + assert table_protocol.num_rows() == result_protocol.num_rows() + assert table_protocol.num_chunks() == result_protocol.num_chunks() + assert table_protocol.column_names() == result_protocol.column_names() @pytest.mark.pandas def test_pandas_roundtrip_string_with_missing(): # See https://github.com/pandas-dev/pandas/issues/50554 - if Version(pd.__version__) < Version("1.6"): - pytest.skip("Column.size() bug in pandas") - arr = ["a", "", "c", None] table = pa.table({"a": pa.array(arr), "a_large": pa.array(arr, type=pa.large_string())}) @@ -230,29 +209,20 @@ def test_pandas_roundtrip_string_with_missing(): from_dataframe as pandas_from_dataframe ) - if Version(pd.__version__) >= Version("2.0.2"): - pandas_df = pandas_from_dataframe(table) - result = pi.from_dataframe(pandas_df) + pandas_df = pandas_from_dataframe(table) + result = pi.from_dataframe(pandas_df) - assert result["a"].to_pylist() == table["a"].to_pylist() - assert pa.types.is_string(table["a"].type) - assert pa.types.is_large_string(result["a"].type) + assert result["a"].to_pylist() == table["a"].to_pylist() + assert pa.types.is_string(table["a"].type) + assert pa.types.is_large_string(result["a"].type) - assert result["a_large"].to_pylist() == table["a_large"].to_pylist() - assert pa.types.is_large_string(table["a_large"].type) - assert pa.types.is_large_string(result["a_large"].type) - else: - # older versions of pandas do not have bitmask support - # https://github.com/pandas-dev/pandas/issues/49888 - with pytest.raises(NotImplementedError): - pandas_from_dataframe(table) + assert result["a_large"].to_pylist() == table["a_large"].to_pylist() + assert pa.types.is_large_string(table["a_large"].type) + assert pa.types.is_large_string(result["a_large"].type) @pytest.mark.pandas def test_pandas_roundtrip_categorical(): - if Version(pd.__version__) < Version("2.0.2"): - pytest.skip("Bitmasks not supported in pandas interchange implementation") - arr = ["Mon", "Tue", "Mon", "Wed", "Mon", "Thu", "Fri", "Sat", None] table = pa.table( {"weekday": pa.array(arr).dictionary_encode()} @@ -299,8 +269,6 @@ def test_pandas_roundtrip_categorical(): @pytest.mark.pandas @pytest.mark.parametrize("unit", ['s', 'ms', 'us', 'ns']) def test_pandas_roundtrip_datetime(unit): - if Version(pd.__version__) < Version("1.5.0"): - pytest.skip("__dataframe__ added to pandas in 1.5.0") from datetime import datetime as dt # timezones not included as they are not yet supported in @@ -308,12 +276,7 @@ def test_pandas_roundtrip_datetime(unit): dt_arr = [dt(2007, 7, 13), dt(2007, 7, 14), dt(2007, 7, 15)] table = pa.table({"a": pa.array(dt_arr, type=pa.timestamp(unit))}) - if Version(pd.__version__) < Version("1.6"): - # pandas < 2.0 always creates datetime64 in "ns" - # resolution - expected = pa.table({"a": pa.array(dt_arr, type=pa.timestamp('ns'))}) - else: - expected = table + expected = table from pandas.api.interchange import ( from_dataframe as pandas_from_dataframe @@ -337,9 +300,6 @@ def test_pandas_roundtrip_datetime(unit): "np_float_str", ["float32", "float64"] ) def test_pandas_to_pyarrow_with_missing(np_float_str): - if Version(pd.__version__) < Version("1.5.0"): - pytest.skip("__dataframe__ added to pandas in 1.5.0") - np_array = np.array([0, np.nan, 2], dtype=np.dtype(np_float_str)) datetime_array = [None, dt(2007, 7, 14), dt(2007, 7, 15)] df = pd.DataFrame({ @@ -359,9 +319,6 @@ def test_pandas_to_pyarrow_with_missing(np_float_str): @pytest.mark.pandas def test_pandas_to_pyarrow_float16_with_missing(): - if Version(pd.__version__) < Version("1.5.0"): - pytest.skip("__dataframe__ added to pandas in 1.5.0") - # np.float16 errors if ps.is_nan is used # pyarrow.lib.ArrowNotImplementedError: Function 'is_nan' has no kernel # matching input types (halffloat) @@ -482,9 +439,6 @@ def test_nan_as_null(): @pytest.mark.pandas def test_allow_copy_false(): - if Version(pd.__version__) < Version("1.5.0"): - pytest.skip("__dataframe__ added to pandas in 1.5.0") - # Test that an error is raised when a copy is needed # to create a bitmask @@ -501,9 +455,6 @@ def test_allow_copy_false(): @pytest.mark.pandas def test_allow_copy_false_bool_categorical(): - if Version(pd.__version__) < Version("1.5.0"): - pytest.skip("__dataframe__ added to pandas in 1.5.0") - # Test that an error is raised for boolean # and categorical dtype (copy is always made) diff --git a/python/pyarrow/tests/parquet/test_pandas.py b/python/pyarrow/tests/parquet/test_pandas.py index 53864ff15ea2..8dbd5d71b9bc 100644 --- a/python/pyarrow/tests/parquet/test_pandas.py +++ b/python/pyarrow/tests/parquet/test_pandas.py @@ -524,9 +524,6 @@ def test_pandas_categorical_roundtrip(): @pytest.mark.pandas def test_categories_with_string_pyarrow_dtype(tempdir): # gh-33727: writing to parquet should not fail - if Version(pd.__version__) < Version("1.3.0"): - pytest.skip("PyArrow backed string data type introduced in pandas 1.3.0") - df1 = pd.DataFrame({"x": ["foo", "bar", "foo"]}, dtype="string[pyarrow]") df1 = df1.astype("category") diff --git a/python/pyarrow/tests/test_compute.py b/python/pyarrow/tests/test_compute.py index 39167e76c8f6..1e08e73668e5 100644 --- a/python/pyarrow/tests/test_compute.py +++ b/python/pyarrow/tests/test_compute.py @@ -2612,8 +2612,6 @@ def test_strftime(): def _check_datetime_components(timestamps, timezone=None): - from pyarrow.vendored.version import Version - ts = pd.to_datetime(timestamps).tz_localize( "UTC").tz_convert(timezone).to_series() tsa = pa.array(ts, pa.timestamp("ns", tz=timezone)) @@ -2626,17 +2624,11 @@ def _check_datetime_components(timestamps, timezone=None): pa.field('iso_day_of_week', pa.int64()) ] - if Version(pd.__version__) < Version("1.1.0"): - # https://github.com/pandas-dev/pandas/issues/33206 - iso_year = ts.map(lambda x: x.isocalendar()[0]).astype("int64") - iso_week = ts.map(lambda x: x.isocalendar()[1]).astype("int64") - iso_day = ts.map(lambda x: x.isocalendar()[2]).astype("int64") - else: - # Casting is required because pandas isocalendar returns int32 - # while arrow isocalendar returns int64. - iso_year = ts.dt.isocalendar()["year"].astype("int64") - iso_week = ts.dt.isocalendar()["week"].astype("int64") - iso_day = ts.dt.isocalendar()["day"].astype("int64") + # Casting is required because pandas isocalendar returns int32 + # while arrow isocalendar returns int64. + iso_year = ts.dt.isocalendar()["year"].astype("int64") + iso_week = ts.dt.isocalendar()["week"].astype("int64") + iso_day = ts.dt.isocalendar()["day"].astype("int64") iso_calendar = pa.StructArray.from_arrays( [iso_year, iso_week, iso_day], diff --git a/python/pyarrow/tests/test_pandas.py b/python/pyarrow/tests/test_pandas.py index c42dc04cd196..2a782de1648c 100644 --- a/python/pyarrow/tests/test_pandas.py +++ b/python/pyarrow/tests/test_pandas.py @@ -488,14 +488,10 @@ def test_string_column_index(self): @pytest.mark.parametrize('unit', ['us', 'ns']) def test_datetimetz_column_index(self, unit): - ext_kwargs = {} - if Version(pd.__version__) >= Version("2.0.0"): - # unit argument not supported on date_range for pandas < 2.0.0 - ext_kwargs = {'unit': unit} df = pd.DataFrame( [(1, 'a', 2.0), (2, 'b', 3.0), (3, 'c', 4.0)], columns=pd.date_range( - start='2017-01-01', periods=3, tz='America/New_York', **ext_kwargs + start='2017-01-01', periods=3, tz='America/New_York', unit=unit ) ) t = pa.Table.from_pandas(df, preserve_index=True) @@ -504,10 +500,7 @@ def test_datetimetz_column_index(self, unit): column_indexes, = js['column_indexes'] assert column_indexes['name'] is None assert column_indexes['pandas_type'] == 'datetimetz' - if ext_kwargs: - assert column_indexes['numpy_type'] == f'datetime64[{unit}]' - else: - assert column_indexes['numpy_type'] == 'datetime64[ns]' + assert column_indexes['numpy_type'] == f'datetime64[{unit}]' md = column_indexes['metadata'] assert md['timezone'] == 'America/New_York' @@ -751,12 +744,8 @@ def test_mismatch_metadata_schema(self): # It is possible that the metadata and actual schema is not fully # matching (eg no timezone information for tz-aware column) # -> to_pandas() conversion should not fail on that - ext_kwargs = {} - if Version(pd.__version__) >= Version("2.0.0"): - # unit argument not supported on date_range for pandas < 2.0.0 - ext_kwargs = {'unit': 'ns'} df = pd.DataFrame({"datetime": pd.date_range( - "2020-01-01", periods=3, **ext_kwargs)}) + "2020-01-01", periods=3, unit='ns')}) # OPTION 1: casting after conversion table = pa.Table.from_pandas(df) @@ -1156,8 +1145,6 @@ def test_timestamps_notimezone_nulls(self): @pytest.mark.parametrize('unit', ['s', 'ms', 'us', 'ns']) def test_timestamps_with_timezone(self, unit): - if Version(pd.__version__) < Version("2.0.0") and unit != 'ns': - pytest.skip("pandas < 2.0 only supports nanosecond datetime64") df = pd.DataFrame({ 'datetime64': np.array([ '2007-07-13T01:23:34.123', @@ -1347,10 +1334,6 @@ def test_array_types_date_as_object(self, coerce_to_ns, expected_dtype): expected_days = np.array(['2000-01-01', None, '1970-01-01', '2040-02-26'], dtype='datetime64[D]') - if Version(pd.__version__) < Version("2.0.0"): - # ARROW-3789: Coerce date/timestamp types to datetime64[ns] - expected_dtype = 'datetime64[ns]' - expected = np.array(['2000-01-01', None, '1970-01-01', '2040-02-26'], dtype=expected_dtype) @@ -1646,19 +1629,6 @@ def test_numpy_datetime64_columns(self): dtype='datetime64[s]') _check_array_from_pandas_roundtrip(datetime64_s) - def test_timestamp_to_pandas_coerces_to_ns(self): - # non-ns timestamp gets cast to ns on conversion to pandas - if Version(pd.__version__) >= Version("2.0.0"): - pytest.skip("pandas >= 2.0 supports non-nanosecond datetime64") - - arr = pa.array([1, 2, 3], pa.timestamp('ms')) - expected = pd.Series(pd.to_datetime([1, 2, 3], unit='ms')) - s = arr.to_pandas() - tm.assert_series_equal(s, expected) - arr = pa.chunked_array([arr]) - s = arr.to_pandas() - tm.assert_series_equal(s, expected) - def test_timestamp_to_pandas_out_of_bounds(self): # ARROW-7758 check for out of bounds timestamps for non-ns timestamps # that end up getting coerced into ns timestamps. @@ -1735,8 +1705,6 @@ def test_fixed_offset_timezone(self): @pytest.mark.parametrize("unit", ['s', 'ms', 'us', 'ns']) def test_timedeltas_no_nulls(self, unit): - if Version(pd.__version__) < Version("2.0.0"): - unit = 'ns' df = pd.DataFrame({ 'timedelta64': np.array([0, 3600000000000, 7200000000000], dtype=f'timedelta64[{unit}]') @@ -1750,8 +1718,6 @@ def test_timedeltas_no_nulls(self, unit): @pytest.mark.parametrize("unit", ['s', 'ms', 'us', 'ns']) def test_timedeltas_nulls(self, unit): - if Version(pd.__version__) < Version("2.0.0"): - unit = 'ns' df = pd.DataFrame({ 'timedelta64': np.array([0, None, 7200000000000], dtype=f'timedelta64[{unit}]') @@ -2419,11 +2385,6 @@ def test_to_list_of_structs_pandas(self): tm.assert_series_equal(series, expected) def test_to_list_of_maps_pandas(self): - if ((Version(np.__version__) >= Version("1.25.0.dev0")) and - (Version(pd.__version__) < Version("2.0.0"))): - # TODO: regression in pandas with numpy 1.25dev - # https://github.com/pandas-dev/pandas/issues/50360 - pytest.skip("Regression in pandas with numpy 1.25") data = [ [[('foo', ['a', 'b']), ('bar', ['c', 'd'])]], [[('baz', []), ('qux', None), ('quux', [None, 'e'])], [('quz', ['f', 'g'])]] @@ -2443,13 +2404,6 @@ def test_to_list_of_maps_pandas_sliced(self): """ A slightly more rigorous test for chunk/slice combinations """ - - if ((Version(np.__version__) >= Version("1.25.0.dev0")) and - (Version(pd.__version__) < Version("2.0.0"))): - # TODO: regression in pandas with numpy 1.25dev - # https://github.com/pandas-dev/pandas/issues/50360 - pytest.skip("Regression in pandas with numpy 1.25") - keys = pa.array(['ignore', 'foo', 'bar', 'baz', 'qux', 'quux', 'ignore']).slice(1, 5) items = pa.array( @@ -3799,9 +3753,7 @@ def test_table_from_pandas_schema_field_order_metadata(): result = table.to_pandas() coerce_cols_to_types = {"float": "float32"} - if Version(pd.__version__) >= Version("2.0.0"): - # Pandas v2 now support non-nanosecond time units - coerce_cols_to_types["datetime"] = "datetime64[s, UTC]" + coerce_cols_to_types["datetime"] = "datetime64[s, UTC]" expected = df[["float", "datetime"]].astype(coerce_cols_to_types) tm.assert_frame_equal(result, expected) @@ -3969,10 +3921,7 @@ def test_to_pandas_split_blocks(): def _get_mgr(df): - if Version(pd.__version__) < Version("1.1.0"): - return df._data - else: - return df._mgr + return df._mgr def _check_blocks_created(t, number): @@ -4383,9 +4332,6 @@ def test_dictionary_from_pandas_specified_type(): def test_convert_categories_to_array_with_string_pyarrow_dtype(): # gh-33727: categories should be converted to pa.Array - if Version(pd.__version__) < Version("1.3.0"): - pytest.skip("PyArrow backed string data type introduced in pandas 1.3.0") - df = pd.DataFrame({"x": ["foo", "bar", "foo"]}, dtype="string[pyarrow]") df = df.astype("category") indices = pa.array(df['x'].cat.codes) @@ -4524,12 +4470,8 @@ def test_convert_to_extension_array(monkeypatch): tm.assert_frame_equal(result, df2) # monkeypatch pandas Int64Dtype to *not* have the protocol method - if Version(pd.__version__) < Version("1.3.0.dev"): - monkeypatch.delattr( - pd.core.arrays.integer._IntegerDtype, "__from_arrow__") - else: - monkeypatch.delattr( - pd.core.arrays.integer.NumericDtype, "__from_arrow__") + monkeypatch.delattr( + pd.core.arrays.integer.NumericDtype, "__from_arrow__") # Int64Dtype has no __from_arrow__ -> use normal conversion result = table.to_pandas() assert len(_get_mgr(result).blocks) == 1 @@ -4569,12 +4511,8 @@ def test_conversion_extensiontype_to_extensionarray(monkeypatch): # monkeypatch pandas Int64Dtype to *not* have the protocol method # (remove the version added above and the actual version for recent pandas) - if Version(pd.__version__) < Version("1.3.0.dev"): - monkeypatch.delattr( - pd.core.arrays.integer._IntegerDtype, "__from_arrow__") - else: - monkeypatch.delattr( - pd.core.arrays.integer.NumericDtype, "__from_arrow__") + monkeypatch.delattr( + pd.core.arrays.integer.NumericDtype, "__from_arrow__") result = arr.to_pandas() assert _get_mgr(result).blocks[0].values.dtype == np.dtype("int64") @@ -4611,9 +4549,6 @@ def test_to_pandas_extension_dtypes_mapping(): def test_to_pandas_extension_dtypes_mapping_complex_type(): - # https://github.com/apache/arrow/pull/44720 - if Version(pd.__version__) < Version("1.5.2"): - pytest.skip("Test relies on pd.ArrowDtype") pa_type = pa.struct( [ pa.field("bar", pa.bool_(), nullable=False), @@ -4636,9 +4571,6 @@ def test_to_pandas_extension_dtypes_mapping_complex_type(): def test_array_to_pandas(): - if Version(pd.__version__) < Version("1.1"): - pytest.skip("ExtensionDtype to_pandas method missing") - for arr in [pd.period_range("2012-01-01", periods=3, freq="D").array, pd.interval_range(1, 4).array]: result = pa.array(arr).to_pandas() @@ -4653,20 +4585,11 @@ def test_array_to_pandas(): def test_roundtrip_empty_table_with_extension_dtype_index(): df = pd.DataFrame(index=pd.interval_range(start=0, end=3)) table = pa.table(df) - if Version(pd.__version__) > Version("1.0"): - tm.assert_index_equal(table.to_pandas().index, df.index) - else: - tm.assert_index_equal(table.to_pandas().index, - pd.Index([{'left': 0, 'right': 1}, - {'left': 1, 'right': 2}, - {'left': 2, 'right': 3}], - dtype='object')) + tm.assert_index_equal(table.to_pandas().index, df.index) @pytest.mark.parametrize("index", ["a", ["a", "b"]]) def test_to_pandas_types_mapper_index(index): - if Version(pd.__version__) < Version("1.5.0"): - pytest.skip("ArrowDtype missing") df = pd.DataFrame( { "a": [1, 2], @@ -4683,9 +4606,6 @@ def test_to_pandas_types_mapper_index(index): def test_array_to_pandas_types_mapper(): # https://issues.apache.org/jira/browse/ARROW-9664 - if Version(pd.__version__) < Version("1.2.0"): - pytest.skip("Float64Dtype extension dtype missing") - data = pa.array([1, 2, 3], pa.int64()) # Test with mapper function @@ -4707,9 +4627,6 @@ def test_array_to_pandas_types_mapper(): @pytest.mark.pandas def test_chunked_array_to_pandas_types_mapper(): # https://issues.apache.org/jira/browse/ARROW-9664 - if Version(pd.__version__) < Version("1.2.0"): - pytest.skip("Float64Dtype extension dtype missing") - data = pa.chunked_array([pa.array([1, 2, 3], pa.int64())]) assert isinstance(data, pa.ChunkedArray) diff --git a/python/pyarrow/tests/test_schema.py b/python/pyarrow/tests/test_schema.py index 029e14ca1628..db44ad32114b 100644 --- a/python/pyarrow/tests/test_schema.py +++ b/python/pyarrow/tests/test_schema.py @@ -27,12 +27,6 @@ import pyarrow as pa import pyarrow.tests.util as test_util -from pyarrow.vendored.version import Version - -try: - import pandas as pd -except ImportError: - pass def test_schema_constructor_errors(): @@ -55,8 +49,6 @@ def test_type_integers(): @pytest.mark.pandas def test_type_to_pandas_dtype(): M8 = np.dtype('datetime64[ms]') - if Version(pd.__version__) < Version("2.0.0"): - M8 = np.dtype('datetime64[ns]') cases = [ (pa.null(), np.object_), (pa.bool_(), np.bool_), diff --git a/python/pyarrow/types.pxi b/python/pyarrow/types.pxi index ec1a5a2ba9a3..8eea34224acc 100644 --- a/python/pyarrow/types.pxi +++ b/python/pyarrow/types.pxi @@ -186,9 +186,7 @@ def _get_pandas_tz_type(arrow_type, coerce_to_ns=False): def _to_pandas_dtype(arrow_type, options=None): - coerce_to_ns = (options and options.get('coerce_temporal_nanoseconds', False)) or ( - _pandas_api.is_v1() and arrow_type.id in - [_Type_DATE32, _Type_DATE64, _Type_TIMESTAMP, _Type_DURATION]) + coerce_to_ns = bool(options and options.get('coerce_temporal_nanoseconds', False)) if getattr(arrow_type, 'tz', None): dtype = _get_pandas_tz_type(arrow_type, coerce_to_ns)