Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 7 additions & 2 deletions python/pyarrow/pandas_compat.py
Original file line number Diff line number Diff line change
Expand Up @@ -378,7 +378,10 @@ def _index_level_name(index, i, column_names):
if index.name is not None and index.name not in column_names:
return _column_name_to_strings(index.name)
else:
return f'__index_level_{i:d}__'
j = i
while f'__index_level_{j:d}__' in column_names:
j += 1
return f'__index_level_{j:d}__'
Comment on lines +381 to +384

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Isn't schema based conversion already buggy without this change when it comes to the index levels? It probably silently ignores the duplicated level 0 currently?

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

OK, getting used to this :) Copilot can't answer. Well, I think the change suggested can be a possible follow-up if we see this would be needed. But I do not think it is in the scope of this PR.



def _get_columns_to_convert(df, schema, preserve_index, columns):
Expand Down Expand Up @@ -419,7 +422,9 @@ def _get_columns_to_convert(df, schema, preserve_index, columns):
index_descriptors = []
index_column_names = []
for i, index_level in enumerate(index_levels):
name = _index_level_name(index_level, i, column_names)
name = _index_level_name(
index_level, i, column_names + index_column_names
)
if (isinstance(index_level, _pandas_api.pd.RangeIndex) and
preserve_index is None):
descr = _get_range_index_descriptor(index_level)
Expand Down
38 changes: 37 additions & 1 deletion python/pyarrow/tests/test_pandas.py
Original file line number Diff line number Diff line change
Expand Up @@ -404,14 +404,50 @@ def test_index_metadata_field_name(self):
assert col3['name'] == col3['field_name']

idx0_descr, foo_descr = js['index_columns']
assert idx0_descr == '__index_level_0__'
# __index_level_0__ exists, unnamed bumped to __index_level_1__
assert idx0_descr == '__index_level_1__'
assert idx0['field_name'] == idx0_descr
assert idx0['name'] is None

assert foo_descr == 'foo'
assert foo['field_name'] == foo_descr
assert foo['name'] == foo_descr

def test_index_level_name_bump(self):
# GH-46179
df = pd.DataFrame(
{"col": [1, 2, 3], "__index_level_0__": [4, 5, 6]},
index=[10, 20, 30],
)
_check_pandas_roundtrip(df, preserve_index=True)

# Explicit test
t = pa.table(df)
expected_schema = pa.schema([
("col", pa.int64()),
("__index_level_0__", pa.int64()),
("__index_level_1__", pa.int64())
])
assert t.schema.equals(expected_schema)

df2 = t.to_pandas()
assert df2.index.equals(pd.Index([10, 20, 30]))
Comment thread
AlenkaF marked this conversation as resolved.
assert df2.ndim == df.ndim == 2

def test_index_level_name_bump_multiindex(self):
# GH-46179
df = pd.DataFrame(
{"col": [1, 2], "__index_level_0__": [3, 4]},
index=pd.MultiIndex.from_arrays(
[[10, 20], [100, 200]], names=[None, None]
),
)
_check_pandas_roundtrip(df, preserve_index=True)

t = pa.Table.from_pandas(df, preserve_index=True)
assert t.schema.names == ['col', '__index_level_0__',
'__index_level_1__', '__index_level_2__']

def test_categorical_column_index(self):
df = pd.DataFrame(
[(1, 'a', 2.0), (2, 'b', 3.0), (3, 'c', 4.0)],
Expand Down