From 512a95d30e77b4056633416da6d521ab89d047a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Fri, 10 Jul 2026 09:05:26 +0200 Subject: [PATCH 1/4] GH-50464: [C++][Python] Simplify arrow_to_pandas DateOffset handling for nanoseconds/milliseconds --- .../pyarrow/src/arrow/python/arrow_to_pandas.cc | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/python/pyarrow/src/arrow/python/arrow_to_pandas.cc b/python/pyarrow/src/arrow/python/arrow_to_pandas.cc index 348d352a0482..85ab9b5251ed 100644 --- a/python/pyarrow/src/arrow/python/arrow_to_pandas.cc +++ b/python/pyarrow/src/arrow/python/arrow_to_pandas.cc @@ -1292,24 +1292,11 @@ struct ObjectWriterVisitor { auto to_date_offset = [&](const MonthDayNanoIntervalType::MonthDayNanos& interval, PyObject** out) { ARROW_DCHECK(internal::BorrowPandasDataOffsetType() != nullptr); - // DateOffset objects do not add nanoseconds component to pd.Timestamp. - // as of Pandas 1.3.3 - // (https://github.com/pandas-dev/pandas/issues/43892). - // So convert microseconds and remainder to preserve data - // but give users more expected results. - int64_t microseconds = interval.nanoseconds / 1000; - int64_t nanoseconds; - if (interval.nanoseconds >= 0) { - nanoseconds = interval.nanoseconds % 1000; - } else { - nanoseconds = -((-interval.nanoseconds) % 1000); - } PyDict_SetItemString(kwargs.obj(), "months", PyLong_FromLong(interval.months)); PyDict_SetItemString(kwargs.obj(), "days", PyLong_FromLong(interval.days)); - PyDict_SetItemString(kwargs.obj(), "microseconds", - PyLong_FromLongLong(microseconds)); - PyDict_SetItemString(kwargs.obj(), "nanoseconds", PyLong_FromLongLong(nanoseconds)); + PyDict_SetItemString(kwargs.obj(), "nanoseconds", + PyLong_FromLongLong(interval.nanoseconds)); *out = PyObject_Call(internal::BorrowPandasDataOffsetType(), args.obj(), kwargs.obj()); RETURN_IF_PYERROR(); From fee35dae532a8dde9b273ddc96b2cf6fb2e251a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Fri, 10 Jul 2026 15:13:59 +0200 Subject: [PATCH 2/4] Changes on roundtripping, we roundtrip Month Day Nanoseconds but not Year, seconds, microseconds, minutes, hours, weeks --- python/pyarrow/tests/test_array.py | 12 +++++------- python/pyarrow/tests/test_pandas.py | 3 +-- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/python/pyarrow/tests/test_array.py b/python/pyarrow/tests/test_array.py index adc3e097b54a..b2165b819830 100644 --- a/python/pyarrow/tests/test_array.py +++ b/python/pyarrow/tests/test_array.py @@ -2762,11 +2762,10 @@ def test_interval_array_from_relativedelta(): assert arr.equals(expected) assert arr.to_pandas().tolist() == [ None, DateOffset(months=13, days=8, - microseconds=( + nanoseconds=( datetime.timedelta(seconds=1, microseconds=1, minutes=1, hours=1) // - datetime.timedelta(microseconds=1)), - nanoseconds=0)] + datetime.timedelta(microseconds=1)) * 1000)] with pytest.raises(ValueError): pa.array([DateOffset(years=((1 << 32) // 12), months=100)]) with pytest.raises(ValueError): @@ -2814,12 +2813,11 @@ def test_interval_array_from_dateoffset(): assert arr.equals(expected) expected_from_pandas = [ None, DateOffset(months=13, days=8, - microseconds=( + nanoseconds=( datetime.timedelta(seconds=1, microseconds=1, minutes=1, hours=1) // - datetime.timedelta(microseconds=1)), - nanoseconds=1), - DateOffset(months=0, days=0, microseconds=0, nanoseconds=0)] + datetime.timedelta(microseconds=1) * 1000) + 1), + DateOffset(months=0, days=0, nanoseconds=0)] assert arr.to_pandas().tolist() == expected_from_pandas diff --git a/python/pyarrow/tests/test_pandas.py b/python/pyarrow/tests/test_pandas.py index 2a782de1648c..19b64e45a1db 100644 --- a/python/pyarrow/tests/test_pandas.py +++ b/python/pyarrow/tests/test_pandas.py @@ -1733,8 +1733,7 @@ def test_month_day_nano_interval(self): from pandas.tseries.offsets import DateOffset df = pd.DataFrame({ 'date_offset': [None, - DateOffset(days=3600, months=3600, microseconds=3, - nanoseconds=600)] + DateOffset(days=3600, months=3600, nanoseconds=3600)] }) schema = pa.schema([('date_offset', pa.month_day_nano_interval())]) _check_pandas_roundtrip( From b30917087a280b248a7051afda5c1be350101d4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 22 Jul 2026 11:25:37 +0200 Subject: [PATCH 3/4] Wrap PyLong_FromLong and PyLong_FromLongLong in OwnedRef and check for Error on PyDict_SetItemString --- python/pyarrow/src/arrow/python/arrow_to_pandas.cc | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/python/pyarrow/src/arrow/python/arrow_to_pandas.cc b/python/pyarrow/src/arrow/python/arrow_to_pandas.cc index 85ab9b5251ed..a6a8cdeb352b 100644 --- a/python/pyarrow/src/arrow/python/arrow_to_pandas.cc +++ b/python/pyarrow/src/arrow/python/arrow_to_pandas.cc @@ -1293,10 +1293,16 @@ struct ObjectWriterVisitor { PyObject** out) { ARROW_DCHECK(internal::BorrowPandasDataOffsetType() != nullptr); - PyDict_SetItemString(kwargs.obj(), "months", PyLong_FromLong(interval.months)); - PyDict_SetItemString(kwargs.obj(), "days", PyLong_FromLong(interval.days)); - PyDict_SetItemString(kwargs.obj(), "nanoseconds", - PyLong_FromLongLong(interval.nanoseconds)); + OwnedRef months(PyLong_FromLong(interval.months)); + OwnedRef days(PyLong_FromLong(interval.days)); + OwnedRef nanoseconds(PyLong_FromLongLong(interval.nanoseconds)); + RETURN_IF_PYERROR(); + PyDict_SetItemString(kwargs.obj(), "months", months.obj()); + RETURN_IF_PYERROR(); + PyDict_SetItemString(kwargs.obj(), "days", days.obj()); + RETURN_IF_PYERROR(); + PyDict_SetItemString(kwargs.obj(), "nanoseconds", nanoseconds.obj()); + RETURN_IF_PYERROR(); *out = PyObject_Call(internal::BorrowPandasDataOffsetType(), args.obj(), kwargs.obj()); RETURN_IF_PYERROR(); From 31bec7b0386f773f4ef625ea76e39e896622333b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Cumplido?= Date: Wed, 22 Jul 2026 12:02:15 +0200 Subject: [PATCH 4/4] Use a single RETURN_IF_PYERROR after all PyDict_SetItemString --- python/pyarrow/src/arrow/python/arrow_to_pandas.cc | 2 -- 1 file changed, 2 deletions(-) diff --git a/python/pyarrow/src/arrow/python/arrow_to_pandas.cc b/python/pyarrow/src/arrow/python/arrow_to_pandas.cc index a6a8cdeb352b..3cf6da31a867 100644 --- a/python/pyarrow/src/arrow/python/arrow_to_pandas.cc +++ b/python/pyarrow/src/arrow/python/arrow_to_pandas.cc @@ -1298,9 +1298,7 @@ struct ObjectWriterVisitor { OwnedRef nanoseconds(PyLong_FromLongLong(interval.nanoseconds)); RETURN_IF_PYERROR(); PyDict_SetItemString(kwargs.obj(), "months", months.obj()); - RETURN_IF_PYERROR(); PyDict_SetItemString(kwargs.obj(), "days", days.obj()); - RETURN_IF_PYERROR(); PyDict_SetItemString(kwargs.obj(), "nanoseconds", nanoseconds.obj()); RETURN_IF_PYERROR(); *out =