From 25e49e7cdb9d0ad9d37c0ca7d30083a9317dc8c0 Mon Sep 17 00:00:00 2001 From: mkzung <103102868+mkzung@users.noreply.github.com> Date: Sat, 11 Jul 2026 18:17:14 +0300 Subject: [PATCH 1/3] GH-37476: [C++][Python] Preserve unsigned dictionary index types when building from values --- cpp/src/arrow/array/array_dict_test.cc | 171 +++++++++++++++++++++++++ cpp/src/arrow/array/builder_dict.h | 69 ++++++++-- cpp/src/arrow/builder.cc | 8 +- python/pyarrow/tests/test_array.py | 45 +++++++ 4 files changed, 277 insertions(+), 16 deletions(-) diff --git a/cpp/src/arrow/array/array_dict_test.cc b/cpp/src/arrow/array/array_dict_test.cc index 5f4335bcbc92..8dd76ddf8541 100644 --- a/cpp/src/arrow/array/array_dict_test.cc +++ b/cpp/src/arrow/array/array_dict_test.cc @@ -1040,6 +1040,177 @@ TYPED_TEST(TestDictionaryBuilderIndexByteWidth, MakeBuilder) { AssertIndexByteWidth(); } +// ---------------------------------------------------------------------- +// GH-37476: the requested index type's signedness must be preserved + +TEST(TestDictionaryBuilderIndexType, TypePreservesUnsignedIndexType) { + for (auto index_type : {uint8(), uint16(), uint32(), uint64()}) { + ARROW_SCOPED_TRACE("index_type = ", index_type->ToString()); + auto dict_type = dictionary(index_type, utf8()); + ASSERT_OK_AND_ASSIGN(auto boxed_builder, MakeBuilder(dict_type)); + + auto builder_type = boxed_builder->type(); + const auto& dict_builder_type = checked_cast(*builder_type); + AssertTypeEqual(*index_type, *dict_builder_type.index_type()); + } +} + +TEST(TestDictionaryBuilderIndexType, TypePreservesSignedIndexType) { + for (auto index_type : {int8(), int16(), int32(), int64()}) { + ARROW_SCOPED_TRACE("index_type = ", index_type->ToString()); + auto dict_type = dictionary(index_type, utf8()); + ASSERT_OK_AND_ASSIGN(auto boxed_builder, MakeBuilder(dict_type)); + + auto builder_type = boxed_builder->type(); + const auto& dict_builder_type = checked_cast(*builder_type); + AssertTypeEqual(*index_type, *dict_builder_type.index_type()); + } +} + +TEST(TestDictionaryBuilderIndexType, FinishPreservesUnsignedIndexType) { + for (auto index_type : {uint8(), uint16(), uint32(), uint64()}) { + ARROW_SCOPED_TRACE("index_type = ", index_type->ToString()); + auto dict_type = dictionary(index_type, utf8()); + ASSERT_OK_AND_ASSIGN(auto boxed_builder, MakeBuilder(dict_type)); + auto& builder = checked_cast&>(*boxed_builder); + + ASSERT_OK(builder.Append("a")); + ASSERT_OK(builder.Append("b")); + ASSERT_OK(builder.AppendNull()); + ASSERT_OK(builder.Append("a")); + + std::shared_ptr result; + ASSERT_OK(builder.Finish(&result)); + ASSERT_OK(result->ValidateFull()); + + auto ex_dict = ArrayFromJSON(utf8(), R"(["a", "b"])"); + auto ex_indices = ArrayFromJSON(index_type, "[0, 1, null, 0]"); + DictionaryArray expected(dict_type, ex_indices, ex_dict); + AssertTypeEqual(*dict_type, *result->type()); + AssertArraysEqual(expected, *result); + } +} + +TEST(TestDictionaryBuilderIndexType, UnsignedIndexWidthStillAdapts) { + // The index width remains adaptive, exactly as it is for signed index types: + // only the signedness of the requested type is preserved. + auto dict_type = dictionary(uint8(), utf8()); + ASSERT_OK_AND_ASSIGN(auto boxed_builder, MakeBuilder(dict_type)); + auto& builder = checked_cast&>(*boxed_builder); + + // More distinct values than a uint8 index start width accommodates. + for (int i = 0; i < 200; ++i) { + ASSERT_OK(builder.Append(std::to_string(i))); + } + + std::shared_ptr result; + ASSERT_OK(builder.Finish(&result)); + ASSERT_OK(result->ValidateFull()); + + const auto& result_type = checked_cast(*result->type()); + // Widened, but still unsigned rather than falling back to a signed type. + AssertTypeEqual(*uint16(), *result_type.index_type()); +} + +TEST(TestDictionaryBuilderIndexType, NullValueTypeUnsignedIndexType) { + auto dict_type = dictionary(uint32(), null()); + ASSERT_OK_AND_ASSIGN(auto boxed_builder, MakeBuilder(dict_type)); + auto& builder = checked_cast&>(*boxed_builder); + + ASSERT_OK(builder.AppendNull()); + + std::shared_ptr result; + ASSERT_OK(builder.Finish(&result)); + ASSERT_OK(result->ValidateFull()); + + const auto& result_type = checked_cast(*result->type()); + AssertTypeEqual(*uint32(), *result_type.index_type()); +} + +TEST(TestDictionaryBuilderIndexType, FinishDeltaPreservesUnsignedIndexType) { + auto dict_type = dictionary(uint32(), utf8()); + ASSERT_OK_AND_ASSIGN(auto boxed_builder, MakeBuilder(dict_type)); + auto& builder = checked_cast&>(*boxed_builder); + + ASSERT_OK(builder.Append("a")); + ASSERT_OK(builder.Append("b")); + + std::shared_ptr result_indices, result_delta; + ASSERT_OK(builder.FinishDelta(&result_indices, &result_delta)); + ASSERT_OK(result_indices->ValidateFull()); + + // The delta indices must carry the requested index type too, not the signed + // type that the adaptive indices builder produces internally. + AssertTypeEqual(*uint32(), *result_indices->type()); + AssertArraysEqual(*ArrayFromJSON(uint32(), "[0, 1]"), *result_indices); +} + +TEST(TestDictionaryBuilderIndexType, SuppliedDictionaryPreservesUnsignedIndexType) { + // The supplied-dictionary constructor starts the adaptive indices builder at its + // default width rather than at the requested one, so the requested *width* is not + // honoured on this path. That predates this change: a requested int32 reports int8 + // too. What must hold is that the *signedness* survives, i.e. uint8 and not int8. + auto dict_values = ArrayFromJSON(utf8(), R"(["a", "b"])"); + + ASSERT_OK_AND_ASSIGN(auto unsigned_builder, + MakeDictionaryBuilder(dictionary(uint32(), utf8()), dict_values)); + auto unsigned_type = unsigned_builder->type(); + AssertTypeEqual(*uint8(), + *checked_cast(*unsigned_type).index_type()); + + ASSERT_OK_AND_ASSIGN(auto signed_builder, + MakeDictionaryBuilder(dictionary(int32(), utf8()), dict_values)); + auto signed_type = signed_builder->type(); + AssertTypeEqual(*int8(), + *checked_cast(*signed_type).index_type()); +} + +TEST(TestDictionaryBuilderIndexType, FixedSizeBinaryUnsignedIndexType) { + auto dict_type = dictionary(uint16(), fixed_size_binary(2)); + ASSERT_OK_AND_ASSIGN(auto boxed_builder, MakeBuilder(dict_type)); + auto& builder = checked_cast&>(*boxed_builder); + + ASSERT_OK(builder.Append("ab")); + ASSERT_OK(builder.Append("cd")); + ASSERT_OK(builder.Append("ab")); + + std::shared_ptr result; + ASSERT_OK(builder.Finish(&result)); + ASSERT_OK(result->ValidateFull()); + + const auto& result_type = checked_cast(*result->type()); + AssertTypeEqual(*uint16(), *result_type.index_type()); +} + +TEST(TestDictionaryBuilderIndexType, OrderedAndUnsignedIndexType) { + // The ordered flag (GH-49689) and the index type must both survive. + auto dict_type = dictionary(uint32(), utf8(), /*ordered=*/true); + ASSERT_OK_AND_ASSIGN(auto boxed_builder, MakeBuilder(dict_type)); + auto& builder = checked_cast&>(*boxed_builder); + + ASSERT_OK(builder.Append("a")); + ASSERT_OK(builder.Append("b")); + + std::shared_ptr result; + ASSERT_OK(builder.Finish(&result)); + ASSERT_OK(result->ValidateFull()); + + const auto& result_type = checked_cast(*result->type()); + AssertTypeEqual(*uint32(), *result_type.index_type()); + ASSERT_TRUE(result_type.ordered()); +} + +TEST(TestDictionaryBuilderIndexType, ExactIndexTypeStillHonoursUnsigned) { + // The exact-index builder already honoured unsigned index types; guard it. + auto dict_type = dictionary(uint16(), utf8()); + std::unique_ptr boxed_builder; + ASSERT_OK(MakeBuilderExactIndex(default_memory_pool(), dict_type, &boxed_builder)); + + auto builder_type = boxed_builder->type(); + const auto& dict_builder_type = checked_cast(*builder_type); + AssertTypeEqual(*uint16(), *dict_builder_type.index_type()); +} + // ---------------------------------------------------------------------- // DictionaryArray tests diff --git a/cpp/src/arrow/array/builder_dict.h b/cpp/src/arrow/array/builder_dict.h index 269cdeee646d..dd178eef826e 100644 --- a/cpp/src/arrow/array/builder_dict.h +++ b/cpp/src/arrow/array/builder_dict.h @@ -134,6 +134,33 @@ class ARROW_EXPORT DictionaryMemoTable { namespace internal { +/// \brief Return the unsigned integer type of the same width when an unsigned +/// dictionary index type was requested. +/// +/// The adaptive indices builder only ever produces signed integer types. Dictionary +/// indices are non-negative, so the signed and unsigned integer types of a given +/// width have identical memory layout and reporting one as the other is +/// value-preserving (GH-37476). The width itself stays adaptive, exactly as it is +/// for signed index types; only the signedness of the requested type is preserved. +inline std::shared_ptr MaybeUnsignedIndexType( + const std::shared_ptr& index_type, bool unsigned_index) { + if (!unsigned_index) { + return index_type; + } + switch (index_type->id()) { + case Type::INT8: + return ::arrow::uint8(); + case Type::INT16: + return ::arrow::uint16(); + case Type::INT32: + return ::arrow::uint32(); + case Type::INT64: + return ::arrow::uint64(); + default: + return index_type; + } +} + /// \brief Array builder for created encoded DictionaryArray from /// dense array /// @@ -154,14 +181,16 @@ class DictionaryBuilderBase : public ArrayBuilder { const std::shared_ptr&> value_type, MemoryPool* pool = default_memory_pool(), - int64_t alignment = kDefaultBufferAlignment, bool ordered = false) + int64_t alignment = kDefaultBufferAlignment, bool ordered = false, + bool unsigned_index = false) : ArrayBuilder(pool, alignment), memo_table_(new internal::DictionaryMemoTable(pool, value_type)), delta_offset_(0), byte_width_(-1), indices_builder_(start_int_size, pool, alignment), value_type_(value_type), - ordered_(ordered) {} + ordered_(ordered), + unsigned_index_(unsigned_index) {} template explicit DictionaryBuilderBase( @@ -199,14 +228,16 @@ class DictionaryBuilderBase : public ArrayBuilder { const std::shared_ptr&> value_type, MemoryPool* pool = default_memory_pool(), - int64_t alignment = kDefaultBufferAlignment, bool ordered = false) + int64_t alignment = kDefaultBufferAlignment, bool ordered = false, + bool unsigned_index = false) : ArrayBuilder(pool, alignment), memo_table_(new internal::DictionaryMemoTable(pool, value_type)), delta_offset_(0), byte_width_(static_cast(*value_type).byte_width()), indices_builder_(start_int_size, pool, alignment), value_type_(value_type), - ordered_(ordered) {} + ordered_(ordered), + unsigned_index_(unsigned_index) {} template explicit DictionaryBuilderBase( @@ -244,14 +275,15 @@ class DictionaryBuilderBase : public ArrayBuilder { explicit DictionaryBuilderBase(const std::shared_ptr& dictionary, MemoryPool* pool = default_memory_pool(), int64_t alignment = kDefaultBufferAlignment, - bool ordered = false) + bool ordered = false, bool unsigned_index = false) : ArrayBuilder(pool, alignment), memo_table_(new internal::DictionaryMemoTable(pool, dictionary)), delta_offset_(0), byte_width_(-1), indices_builder_(pool, alignment), value_type_(dictionary->type()), - ordered_(ordered) {} + ordered_(ordered), + unsigned_index_(unsigned_index) {} ~DictionaryBuilderBase() override = default; @@ -486,6 +518,7 @@ class DictionaryBuilderBase : public ArrayBuilder { std::shared_ptr indices_data; std::shared_ptr delta_data; ARROW_RETURN_NOT_OK(FinishWithDictOffset(delta_offset_, &indices_data, &delta_data)); + indices_data->type = MaybeUnsignedIndexType(indices_data->type, unsigned_index_); *out_indices = MakeArray(indices_data); *out_delta = MakeArray(delta_data); return Status::OK(); @@ -498,7 +531,9 @@ class DictionaryBuilderBase : public ArrayBuilder { Status Finish(std::shared_ptr* out) { return FinishTyped(out); } std::shared_ptr type() const override { - return ::arrow::dictionary(indices_builder_.type(), value_type_, ordered_); + return ::arrow::dictionary( + MaybeUnsignedIndexType(indices_builder_.type(), unsigned_index_), value_type_, + ordered_); } protected: @@ -570,6 +605,7 @@ class DictionaryBuilderBase : public ArrayBuilder { BuilderType indices_builder_; std::shared_ptr value_type_; bool ordered_ = false; + bool unsigned_index_ = false; }; template @@ -581,10 +617,12 @@ class DictionaryBuilderBase : public ArrayBuilder { start_int_size, const std::shared_ptr& value_type, MemoryPool* pool = default_memory_pool(), - int64_t alignment = kDefaultBufferAlignment, bool ordered = false) + int64_t alignment = kDefaultBufferAlignment, bool ordered = false, + bool unsigned_index = false) : ArrayBuilder(pool, alignment), indices_builder_(start_int_size, pool, alignment), - ordered_(ordered) {} + ordered_(ordered), + unsigned_index_(unsigned_index) {} explicit DictionaryBuilderBase(const std::shared_ptr& value_type, MemoryPool* pool = default_memory_pool(), @@ -623,10 +661,11 @@ class DictionaryBuilderBase : public ArrayBuilder { explicit DictionaryBuilderBase(const std::shared_ptr& dictionary, MemoryPool* pool = default_memory_pool(), int64_t alignment = kDefaultBufferAlignment, - bool ordered = false) + bool ordered = false, bool unsigned_index = false) : ArrayBuilder(pool, alignment), indices_builder_(pool, alignment), - ordered_(ordered) {} + ordered_(ordered), + unsigned_index_(unsigned_index) {} /// \brief Append a scalar null value Status AppendNull() final { @@ -678,7 +717,8 @@ class DictionaryBuilderBase : public ArrayBuilder { Status FinishInternal(std::shared_ptr* out) override { ARROW_RETURN_NOT_OK(indices_builder_.FinishInternal(out)); - (*out)->type = dictionary((*out)->type, null(), ordered_); + (*out)->type = dictionary(MaybeUnsignedIndexType((*out)->type, unsigned_index_), + null(), ordered_); (*out)->dictionary = NullArray(0).data(); return Status::OK(); } @@ -690,12 +730,15 @@ class DictionaryBuilderBase : public ArrayBuilder { Status Finish(std::shared_ptr* out) { return FinishTyped(out); } std::shared_ptr type() const override { - return ::arrow::dictionary(indices_builder_.type(), null(), ordered_); + return ::arrow::dictionary( + MaybeUnsignedIndexType(indices_builder_.type(), unsigned_index_), null(), + ordered_); } protected: BuilderType indices_builder_; bool ordered_ = false; + bool unsigned_index_ = false; }; } // namespace internal diff --git a/cpp/src/arrow/builder.cc b/cpp/src/arrow/builder.cc index 2190f1ce04a9..d4269f57d959 100644 --- a/cpp/src/arrow/builder.cc +++ b/cpp/src/arrow/builder.cc @@ -170,9 +170,10 @@ struct DictionaryBuilderCase { using AdaptiveBuilderType = DictionaryBuilder; using ExactBuilderType = internal::DictionaryBuilderBase; + const bool unsigned_index = is_unsigned_integer(index_type->id()); if (dictionary != nullptr) { - out->reset( - new AdaptiveBuilderType(dictionary, pool, kDefaultBufferAlignment, ordered)); + out->reset(new AdaptiveBuilderType(dictionary, pool, kDefaultBufferAlignment, + ordered, unsigned_index)); } else if (exact_index_type) { if (!is_integer(index_type->id())) { return Status::TypeError("MakeBuilder: invalid index type ", *index_type); @@ -182,7 +183,8 @@ struct DictionaryBuilderCase { } else { auto start_int_size = index_type->byte_width(); out->reset(new AdaptiveBuilderType(start_int_size, value_type, pool, - kDefaultBufferAlignment, ordered)); + kDefaultBufferAlignment, ordered, + unsigned_index)); } return Status::OK(); } diff --git a/python/pyarrow/tests/test_array.py b/python/pyarrow/tests/test_array.py index adc3e097b54a..fe3f6c39cec0 100644 --- a/python/pyarrow/tests/test_array.py +++ b/python/pyarrow/tests/test_array.py @@ -4527,3 +4527,48 @@ def test_dunders_checked_overflow(): arr ** pa.scalar(2, type=pa.int8()) with pytest.raises(pa.ArrowInvalid, match=error_match): arr / (-arr) + + +@pytest.mark.parametrize("index_type", [pa.uint8(), pa.uint16(), + pa.uint32(), pa.uint64()]) +def test_dictionary_array_preserves_unsigned_index_type(index_type): + # GH-37476: an unsigned dictionary index type used to be silently replaced + # by the signed integer type of the same width. + dict_type = pa.dictionary(index_type, pa.string()) + + arr = pa.array(["a", "b", None, "a"], type=dict_type) + assert arr.type == dict_type + assert arr.to_pylist() == ["a", "b", None, "a"] + arr.validate(full=True) + + chunked = pa.chunked_array([["a", "b", "a"]], dict_type) + assert chunked.type == dict_type + + +@pytest.mark.parametrize("index_type", [pa.int8(), pa.int16(), + pa.int32(), pa.int64()]) +def test_dictionary_array_signed_index_type_unchanged(index_type): + dict_type = pa.dictionary(index_type, pa.string()) + arr = pa.array(["a", "b", "a"], type=dict_type) + assert arr.type == dict_type + + +def test_dictionary_array_index_width_still_adapts(): + # The index width remains adaptive, as it is for signed index types; only + # the signedness of the requested index type is preserved. + values = [str(i) for i in range(200)] + arr = pa.array(values, type=pa.dictionary(pa.uint8(), pa.string())) + assert arr.type == pa.dictionary(pa.uint16(), pa.string()) + assert arr.to_pylist() == values + + +@pytest.mark.pandas +def test_dictionary_uint64_index_to_pandas_not_supported(): + # GH-37476: uint64 dictionary indices are now preserved instead of being + # silently downgraded to int64. Arrow deliberately does not support converting + # them to pandas (pandas categorical codes are signed), so that pre-existing + # limitation is now reachable, where the silent downgrade used to mask it. + # uint8/uint16/uint32 dictionary indices convert to pandas as before. + arr = pa.array(["a", "b"], type=pa.dictionary(pa.uint64(), pa.string())) + with pytest.raises(pa.ArrowTypeError, match="UInt64 dictionary indices"): + arr.to_pandas() From 6a67659066224b9f5a5342099e19bd14d178a472 Mon Sep 17 00:00:00 2001 From: mkzung <103102868+mkzung@users.noreply.github.com> Date: Fri, 24 Jul 2026 19:25:51 +0500 Subject: [PATCH 2/3] GH-37476: address review comments Consolidate the dictionary index-type tests down to the distinct code paths and reword the context-dependent comments. Move MaybeUnsignedIndexType out of the header into builder.cc and rename the flag to use_unsigned_index. Support converting uint64 dictionary indices to pandas: they map to int64 categorical codes, which is safe because the indices are bounds-checked below the dictionary length. This removes the previous TypeError. --- cpp/src/arrow/array/array_dict_test.cc | 155 +++++------------- cpp/src/arrow/array/builder_dict.h | 61 +++---- cpp/src/arrow/builder.cc | 32 ++++ .../src/arrow/python/arrow_to_pandas.cc | 14 +- python/pyarrow/tests/test_array.py | 48 +++--- python/pyarrow/tests/test_pandas.py | 13 +- 6 files changed, 128 insertions(+), 195 deletions(-) diff --git a/cpp/src/arrow/array/array_dict_test.cc b/cpp/src/arrow/array/array_dict_test.cc index 8dd76ddf8541..497745180921 100644 --- a/cpp/src/arrow/array/array_dict_test.cc +++ b/cpp/src/arrow/array/array_dict_test.cc @@ -1041,46 +1041,25 @@ TYPED_TEST(TestDictionaryBuilderIndexByteWidth, MakeBuilder) { } // ---------------------------------------------------------------------- -// GH-37476: the requested index type's signedness must be preserved +// GH-37476: the requested dictionary index type's signedness must be preserved. -TEST(TestDictionaryBuilderIndexType, TypePreservesUnsignedIndexType) { - for (auto index_type : {uint8(), uint16(), uint32(), uint64()}) { +TEST(TestDictionaryBuilderIndexType, PreservesRequestedIndexType) { + // Both the builder's reported type and the finished array must carry the requested + // index type, for signed and unsigned widths alike. + for (auto index_type : + {int8(), int16(), int32(), int64(), uint8(), uint16(), uint32(), uint64()}) { ARROW_SCOPED_TRACE("index_type = ", index_type->ToString()); auto dict_type = dictionary(index_type, utf8()); - ASSERT_OK_AND_ASSIGN(auto boxed_builder, MakeBuilder(dict_type)); - - auto builder_type = boxed_builder->type(); - const auto& dict_builder_type = checked_cast(*builder_type); - AssertTypeEqual(*index_type, *dict_builder_type.index_type()); - } -} - -TEST(TestDictionaryBuilderIndexType, TypePreservesSignedIndexType) { - for (auto index_type : {int8(), int16(), int32(), int64()}) { - ARROW_SCOPED_TRACE("index_type = ", index_type->ToString()); - auto dict_type = dictionary(index_type, utf8()); - ASSERT_OK_AND_ASSIGN(auto boxed_builder, MakeBuilder(dict_type)); - - auto builder_type = boxed_builder->type(); - const auto& dict_builder_type = checked_cast(*builder_type); - AssertTypeEqual(*index_type, *dict_builder_type.index_type()); - } -} - -TEST(TestDictionaryBuilderIndexType, FinishPreservesUnsignedIndexType) { - for (auto index_type : {uint8(), uint16(), uint32(), uint64()}) { - ARROW_SCOPED_TRACE("index_type = ", index_type->ToString()); - auto dict_type = dictionary(index_type, utf8()); - ASSERT_OK_AND_ASSIGN(auto boxed_builder, MakeBuilder(dict_type)); - auto& builder = checked_cast&>(*boxed_builder); - - ASSERT_OK(builder.Append("a")); - ASSERT_OK(builder.Append("b")); - ASSERT_OK(builder.AppendNull()); - ASSERT_OK(builder.Append("a")); + ASSERT_OK_AND_ASSIGN(auto builder, MakeBuilder(dict_type)); + AssertTypeEqual(*index_type, + *checked_cast(*builder->type()).index_type()); - std::shared_ptr result; - ASSERT_OK(builder.Finish(&result)); + auto& dict_builder = checked_cast&>(*builder); + ASSERT_OK(dict_builder.Append("a")); + ASSERT_OK(dict_builder.Append("b")); + ASSERT_OK(dict_builder.AppendNull()); + ASSERT_OK(dict_builder.Append("a")); + ASSERT_OK_AND_ASSIGN(auto result, dict_builder.Finish()); ASSERT_OK(result->ValidateFull()); auto ex_dict = ArrayFromJSON(utf8(), R"(["a", "b"])"); @@ -1091,43 +1070,43 @@ TEST(TestDictionaryBuilderIndexType, FinishPreservesUnsignedIndexType) { } } -TEST(TestDictionaryBuilderIndexType, UnsignedIndexWidthStillAdapts) { - // The index width remains adaptive, exactly as it is for signed index types: - // only the signedness of the requested type is preserved. +TEST(TestDictionaryBuilderIndexType, WidthStillAdaptsWhenUnsigned) { + // The width stays adaptive, as it does for signed indices. The underlying builder is + // signed, so it widens after 128 distinct values rather than the 256 a uint8 could + // hold, but the widened type stays unsigned rather than falling back to a signed type. auto dict_type = dictionary(uint8(), utf8()); ASSERT_OK_AND_ASSIGN(auto boxed_builder, MakeBuilder(dict_type)); auto& builder = checked_cast&>(*boxed_builder); - // More distinct values than a uint8 index start width accommodates. for (int i = 0; i < 200; ++i) { ASSERT_OK(builder.Append(std::to_string(i))); } - std::shared_ptr result; - ASSERT_OK(builder.Finish(&result)); + ASSERT_OK_AND_ASSIGN(auto result, builder.Finish()); ASSERT_OK(result->ValidateFull()); - - const auto& result_type = checked_cast(*result->type()); - // Widened, but still unsigned rather than falling back to a signed type. - AssertTypeEqual(*uint16(), *result_type.index_type()); + AssertTypeEqual(*uint16(), + *checked_cast(*result->type()).index_type()); } -TEST(TestDictionaryBuilderIndexType, NullValueTypeUnsignedIndexType) { +TEST(TestDictionaryBuilderIndexType, NullValueTypePreservesUnsignedIndexType) { + // The NullType value builder is a separate specialization with its own type() and + // FinishInternal, so it needs its own guard. auto dict_type = dictionary(uint32(), null()); ASSERT_OK_AND_ASSIGN(auto boxed_builder, MakeBuilder(dict_type)); auto& builder = checked_cast&>(*boxed_builder); + AssertTypeEqual(*uint32(), + *checked_cast(*builder.type()).index_type()); ASSERT_OK(builder.AppendNull()); - - std::shared_ptr result; - ASSERT_OK(builder.Finish(&result)); + ASSERT_OK_AND_ASSIGN(auto result, builder.Finish()); ASSERT_OK(result->ValidateFull()); - - const auto& result_type = checked_cast(*result->type()); - AssertTypeEqual(*uint32(), *result_type.index_type()); + AssertTypeEqual(*uint32(), + *checked_cast(*result->type()).index_type()); } TEST(TestDictionaryBuilderIndexType, FinishDeltaPreservesUnsignedIndexType) { + // FinishDelta is a distinct path from Finish and must carry the requested index type + // too, not the signed type the adaptive builder produces internally. auto dict_type = dictionary(uint32(), utf8()); ASSERT_OK_AND_ASSIGN(auto boxed_builder, MakeBuilder(dict_type)); auto& builder = checked_cast&>(*boxed_builder); @@ -1138,77 +1117,31 @@ TEST(TestDictionaryBuilderIndexType, FinishDeltaPreservesUnsignedIndexType) { std::shared_ptr result_indices, result_delta; ASSERT_OK(builder.FinishDelta(&result_indices, &result_delta)); ASSERT_OK(result_indices->ValidateFull()); - - // The delta indices must carry the requested index type too, not the signed - // type that the adaptive indices builder produces internally. AssertTypeEqual(*uint32(), *result_indices->type()); AssertArraysEqual(*ArrayFromJSON(uint32(), "[0, 1]"), *result_indices); } TEST(TestDictionaryBuilderIndexType, SuppliedDictionaryPreservesUnsignedIndexType) { - // The supplied-dictionary constructor starts the adaptive indices builder at its - // default width rather than at the requested one, so the requested *width* is not - // honoured on this path. That predates this change: a requested int32 reports int8 - // too. What must hold is that the *signedness* survives, i.e. uint8 and not int8. + // The supplied-dictionary constructor starts the adaptive builder at its default width + // rather than the requested one, so a requested uint32 reports uint8. The width is not + // honoured on this path (a signed request behaves the same way), but the signedness + // must survive regardless: uint8, not int8. auto dict_values = ArrayFromJSON(utf8(), R"(["a", "b"])"); - - ASSERT_OK_AND_ASSIGN(auto unsigned_builder, + ASSERT_OK_AND_ASSIGN(auto builder, MakeDictionaryBuilder(dictionary(uint32(), utf8()), dict_values)); - auto unsigned_type = unsigned_builder->type(); AssertTypeEqual(*uint8(), - *checked_cast(*unsigned_type).index_type()); - - ASSERT_OK_AND_ASSIGN(auto signed_builder, - MakeDictionaryBuilder(dictionary(int32(), utf8()), dict_values)); - auto signed_type = signed_builder->type(); - AssertTypeEqual(*int8(), - *checked_cast(*signed_type).index_type()); -} - -TEST(TestDictionaryBuilderIndexType, FixedSizeBinaryUnsignedIndexType) { - auto dict_type = dictionary(uint16(), fixed_size_binary(2)); - ASSERT_OK_AND_ASSIGN(auto boxed_builder, MakeBuilder(dict_type)); - auto& builder = checked_cast&>(*boxed_builder); - - ASSERT_OK(builder.Append("ab")); - ASSERT_OK(builder.Append("cd")); - ASSERT_OK(builder.Append("ab")); - - std::shared_ptr result; - ASSERT_OK(builder.Finish(&result)); - ASSERT_OK(result->ValidateFull()); - - const auto& result_type = checked_cast(*result->type()); - AssertTypeEqual(*uint16(), *result_type.index_type()); + *checked_cast(*builder->type()).index_type()); } -TEST(TestDictionaryBuilderIndexType, OrderedAndUnsignedIndexType) { - // The ordered flag (GH-49689) and the index type must both survive. - auto dict_type = dictionary(uint32(), utf8(), /*ordered=*/true); - ASSERT_OK_AND_ASSIGN(auto boxed_builder, MakeBuilder(dict_type)); - auto& builder = checked_cast&>(*boxed_builder); - - ASSERT_OK(builder.Append("a")); - ASSERT_OK(builder.Append("b")); - - std::shared_ptr result; - ASSERT_OK(builder.Finish(&result)); - ASSERT_OK(result->ValidateFull()); - - const auto& result_type = checked_cast(*result->type()); - AssertTypeEqual(*uint32(), *result_type.index_type()); - ASSERT_TRUE(result_type.ordered()); -} - -TEST(TestDictionaryBuilderIndexType, ExactIndexTypeStillHonoursUnsigned) { - // The exact-index builder already honoured unsigned index types; guard it. +TEST(TestDictionaryBuilderIndexType, ExactIndexBuilderPreservesUnsignedIndexType) { + // MakeBuilderExactIndex is a separate builder that honours unsigned index types on its + // own; guard that it keeps doing so. auto dict_type = dictionary(uint16(), utf8()); std::unique_ptr boxed_builder; ASSERT_OK(MakeBuilderExactIndex(default_memory_pool(), dict_type, &boxed_builder)); - - auto builder_type = boxed_builder->type(); - const auto& dict_builder_type = checked_cast(*builder_type); - AssertTypeEqual(*uint16(), *dict_builder_type.index_type()); + AssertTypeEqual( + *uint16(), + *checked_cast(*boxed_builder->type()).index_type()); } // ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/array/builder_dict.h b/cpp/src/arrow/array/builder_dict.h index dd178eef826e..4dd4e6ea44fd 100644 --- a/cpp/src/arrow/array/builder_dict.h +++ b/cpp/src/arrow/array/builder_dict.h @@ -135,31 +135,10 @@ class ARROW_EXPORT DictionaryMemoTable { namespace internal { /// \brief Return the unsigned integer type of the same width when an unsigned -/// dictionary index type was requested. -/// -/// The adaptive indices builder only ever produces signed integer types. Dictionary -/// indices are non-negative, so the signed and unsigned integer types of a given -/// width have identical memory layout and reporting one as the other is -/// value-preserving (GH-37476). The width itself stays adaptive, exactly as it is -/// for signed index types; only the signedness of the requested type is preserved. -inline std::shared_ptr MaybeUnsignedIndexType( - const std::shared_ptr& index_type, bool unsigned_index) { - if (!unsigned_index) { - return index_type; - } - switch (index_type->id()) { - case Type::INT8: - return ::arrow::uint8(); - case Type::INT16: - return ::arrow::uint16(); - case Type::INT32: - return ::arrow::uint32(); - case Type::INT64: - return ::arrow::uint64(); - default: - return index_type; - } -} +/// dictionary index type was requested. Defined in builder.cc; see there for why +/// this is value-preserving and why the width widens on the signed threshold. +ARROW_EXPORT std::shared_ptr MaybeUnsignedIndexType( + const std::shared_ptr& index_type, bool use_unsigned_index); /// \brief Array builder for created encoded DictionaryArray from /// dense array @@ -182,7 +161,7 @@ class DictionaryBuilderBase : public ArrayBuilder { value_type, MemoryPool* pool = default_memory_pool(), int64_t alignment = kDefaultBufferAlignment, bool ordered = false, - bool unsigned_index = false) + bool use_unsigned_index = false) : ArrayBuilder(pool, alignment), memo_table_(new internal::DictionaryMemoTable(pool, value_type)), delta_offset_(0), @@ -190,7 +169,7 @@ class DictionaryBuilderBase : public ArrayBuilder { indices_builder_(start_int_size, pool, alignment), value_type_(value_type), ordered_(ordered), - unsigned_index_(unsigned_index) {} + use_unsigned_index_(use_unsigned_index) {} template explicit DictionaryBuilderBase( @@ -229,7 +208,7 @@ class DictionaryBuilderBase : public ArrayBuilder { value_type, MemoryPool* pool = default_memory_pool(), int64_t alignment = kDefaultBufferAlignment, bool ordered = false, - bool unsigned_index = false) + bool use_unsigned_index = false) : ArrayBuilder(pool, alignment), memo_table_(new internal::DictionaryMemoTable(pool, value_type)), delta_offset_(0), @@ -237,7 +216,7 @@ class DictionaryBuilderBase : public ArrayBuilder { indices_builder_(start_int_size, pool, alignment), value_type_(value_type), ordered_(ordered), - unsigned_index_(unsigned_index) {} + use_unsigned_index_(use_unsigned_index) {} template explicit DictionaryBuilderBase( @@ -275,7 +254,7 @@ class DictionaryBuilderBase : public ArrayBuilder { explicit DictionaryBuilderBase(const std::shared_ptr& dictionary, MemoryPool* pool = default_memory_pool(), int64_t alignment = kDefaultBufferAlignment, - bool ordered = false, bool unsigned_index = false) + bool ordered = false, bool use_unsigned_index = false) : ArrayBuilder(pool, alignment), memo_table_(new internal::DictionaryMemoTable(pool, dictionary)), delta_offset_(0), @@ -283,7 +262,7 @@ class DictionaryBuilderBase : public ArrayBuilder { indices_builder_(pool, alignment), value_type_(dictionary->type()), ordered_(ordered), - unsigned_index_(unsigned_index) {} + use_unsigned_index_(use_unsigned_index) {} ~DictionaryBuilderBase() override = default; @@ -518,7 +497,7 @@ class DictionaryBuilderBase : public ArrayBuilder { std::shared_ptr indices_data; std::shared_ptr delta_data; ARROW_RETURN_NOT_OK(FinishWithDictOffset(delta_offset_, &indices_data, &delta_data)); - indices_data->type = MaybeUnsignedIndexType(indices_data->type, unsigned_index_); + indices_data->type = MaybeUnsignedIndexType(indices_data->type, use_unsigned_index_); *out_indices = MakeArray(indices_data); *out_delta = MakeArray(delta_data); return Status::OK(); @@ -532,7 +511,7 @@ class DictionaryBuilderBase : public ArrayBuilder { std::shared_ptr type() const override { return ::arrow::dictionary( - MaybeUnsignedIndexType(indices_builder_.type(), unsigned_index_), value_type_, + MaybeUnsignedIndexType(indices_builder_.type(), use_unsigned_index_), value_type_, ordered_); } @@ -605,7 +584,7 @@ class DictionaryBuilderBase : public ArrayBuilder { BuilderType indices_builder_; std::shared_ptr value_type_; bool ordered_ = false; - bool unsigned_index_ = false; + bool use_unsigned_index_ = false; }; template @@ -618,11 +597,11 @@ class DictionaryBuilderBase : public ArrayBuilder { const std::shared_ptr& value_type, MemoryPool* pool = default_memory_pool(), int64_t alignment = kDefaultBufferAlignment, bool ordered = false, - bool unsigned_index = false) + bool use_unsigned_index = false) : ArrayBuilder(pool, alignment), indices_builder_(start_int_size, pool, alignment), ordered_(ordered), - unsigned_index_(unsigned_index) {} + use_unsigned_index_(use_unsigned_index) {} explicit DictionaryBuilderBase(const std::shared_ptr& value_type, MemoryPool* pool = default_memory_pool(), @@ -661,11 +640,11 @@ class DictionaryBuilderBase : public ArrayBuilder { explicit DictionaryBuilderBase(const std::shared_ptr& dictionary, MemoryPool* pool = default_memory_pool(), int64_t alignment = kDefaultBufferAlignment, - bool ordered = false, bool unsigned_index = false) + bool ordered = false, bool use_unsigned_index = false) : ArrayBuilder(pool, alignment), indices_builder_(pool, alignment), ordered_(ordered), - unsigned_index_(unsigned_index) {} + use_unsigned_index_(use_unsigned_index) {} /// \brief Append a scalar null value Status AppendNull() final { @@ -717,7 +696,7 @@ class DictionaryBuilderBase : public ArrayBuilder { Status FinishInternal(std::shared_ptr* out) override { ARROW_RETURN_NOT_OK(indices_builder_.FinishInternal(out)); - (*out)->type = dictionary(MaybeUnsignedIndexType((*out)->type, unsigned_index_), + (*out)->type = dictionary(MaybeUnsignedIndexType((*out)->type, use_unsigned_index_), null(), ordered_); (*out)->dictionary = NullArray(0).data(); return Status::OK(); @@ -731,14 +710,14 @@ class DictionaryBuilderBase : public ArrayBuilder { std::shared_ptr type() const override { return ::arrow::dictionary( - MaybeUnsignedIndexType(indices_builder_.type(), unsigned_index_), null(), + MaybeUnsignedIndexType(indices_builder_.type(), use_unsigned_index_), null(), ordered_); } protected: BuilderType indices_builder_; bool ordered_ = false; - bool unsigned_index_ = false; + bool use_unsigned_index_ = false; }; } // namespace internal diff --git a/cpp/src/arrow/builder.cc b/cpp/src/arrow/builder.cc index d4269f57d959..d333946bcf10 100644 --- a/cpp/src/arrow/builder.cc +++ b/cpp/src/arrow/builder.cc @@ -38,6 +38,38 @@ class MemoryPool; using arrow::internal::checked_cast; +namespace internal { + +/// Return the unsigned integer type of the same width when an unsigned dictionary +/// index type was requested (GH-37476). +/// +/// The adaptive indices builder only ever produces signed integer types. Dictionary +/// indices are non-negative, so the signed and unsigned integer types of a given width +/// have identical memory layout and reporting one as the other is value-preserving. The +/// width stays adaptive, as it is for signed index types, and it widens on the signed +/// threshold: a uint8 index widens after 128 distinct values rather than the 256 a real +/// uint8 could hold, so the extra bit does not delay widening. +std::shared_ptr MaybeUnsignedIndexType( + const std::shared_ptr& index_type, bool use_unsigned_index) { + if (!use_unsigned_index) { + return index_type; + } + switch (index_type->id()) { + case Type::INT8: + return ::arrow::uint8(); + case Type::INT16: + return ::arrow::uint16(); + case Type::INT32: + return ::arrow::uint32(); + case Type::INT64: + return ::arrow::uint64(); + default: + return index_type; + } +} + +} // namespace internal + // Generic int builder that delegates to the builder for a specific // type. Used to reduce the number of template instantiations in the // exact_index_type case below, to reduce build time and memory usage. diff --git a/python/pyarrow/src/arrow/python/arrow_to_pandas.cc b/python/pyarrow/src/arrow/python/arrow_to_pandas.cc index 348d352a0482..ebf151ed6086 100644 --- a/python/pyarrow/src/arrow/python/arrow_to_pandas.cc +++ b/python/pyarrow/src/arrow/python/arrow_to_pandas.cc @@ -1863,13 +1863,17 @@ class CategoricalWriter } Status WriteIndicesUniform(const ChunkedArray& data) { - // For unsigned types, upcast to signed since pandas uses -1 for nulls - // uint8 to int16, uint16 to int32, uint32 to int64, signed types unchanged + // For unsigned types, use a signed output since pandas uses -1 for nulls: + // uint8 to int16, uint16 to int32, uint32 to int64. uint64 also maps to int64, + // which is safe because the indices are bounds-checked below the dictionary length + // and so never reach the int64 range limit. Signed types are unchanged. using OutputType = std::conditional_t< std::is_same::value, int16_t, std::conditional_t< std::is_same::value, int32_t, - std::conditional_t::value, int64_t, T>>>; + std::conditional_t< + std::is_same::value, int64_t, + std::conditional_t::value, int64_t, T>>>>; const int npy_output_type = std::is_same::value ? NPY_INT16 : std::is_same::value ? NPY_INT32 : std::is_same::value @@ -2044,9 +2048,7 @@ Status MakeWriter(const PandasOptions& options, PandasWriter::type writer_type, CATEGORICAL_CASE(UInt8Type); CATEGORICAL_CASE(UInt16Type); CATEGORICAL_CASE(UInt32Type); - case Type::UINT64: - return Status::TypeError( - "Converting UInt64 dictionary indices to pandas is not supported."); + CATEGORICAL_CASE(UInt64Type); default: // Unreachable ARROW_DCHECK(false); diff --git a/python/pyarrow/tests/test_array.py b/python/pyarrow/tests/test_array.py index fe3f6c39cec0..6caf9d2b5550 100644 --- a/python/pyarrow/tests/test_array.py +++ b/python/pyarrow/tests/test_array.py @@ -4529,11 +4529,12 @@ def test_dunders_checked_overflow(): arr / (-arr) -@pytest.mark.parametrize("index_type", [pa.uint8(), pa.uint16(), - pa.uint32(), pa.uint64()]) -def test_dictionary_array_preserves_unsigned_index_type(index_type): - # GH-37476: an unsigned dictionary index type used to be silently replaced - # by the signed integer type of the same width. +@pytest.mark.parametrize("index_type", [pa.int8(), pa.int16(), pa.int32(), pa.int64(), + pa.uint8(), pa.uint16(), pa.uint32(), + pa.uint64()]) +def test_dictionary_array_preserves_index_type(index_type): + # GH-37476: an unsigned dictionary index type must be preserved, not replaced by the + # signed integer type of the same width. Signed index types are kept as-is. dict_type = pa.dictionary(index_type, pa.string()) arr = pa.array(["a", "b", None, "a"], type=dict_type) @@ -4545,30 +4546,23 @@ def test_dictionary_array_preserves_unsigned_index_type(index_type): assert chunked.type == dict_type -@pytest.mark.parametrize("index_type", [pa.int8(), pa.int16(), - pa.int32(), pa.int64()]) -def test_dictionary_array_signed_index_type_unchanged(index_type): - dict_type = pa.dictionary(index_type, pa.string()) - arr = pa.array(["a", "b", "a"], type=dict_type) - assert arr.type == dict_type - - -def test_dictionary_array_index_width_still_adapts(): - # The index width remains adaptive, as it is for signed index types; only - # the signedness of the requested index type is preserved. +@pytest.mark.parametrize("start_type, widened_type", [(pa.int8(), pa.int16()), + (pa.uint8(), pa.uint16())]) +def test_dictionary_array_index_width_adapts(start_type, widened_type): + # The index width adapts to the number of distinct values, as it does for signed + # indices; only the signedness of the requested type is preserved. values = [str(i) for i in range(200)] - arr = pa.array(values, type=pa.dictionary(pa.uint8(), pa.string())) - assert arr.type == pa.dictionary(pa.uint16(), pa.string()) + arr = pa.array(values, type=pa.dictionary(start_type, pa.string())) + assert arr.type == pa.dictionary(widened_type, pa.string()) assert arr.to_pylist() == values @pytest.mark.pandas -def test_dictionary_uint64_index_to_pandas_not_supported(): - # GH-37476: uint64 dictionary indices are now preserved instead of being - # silently downgraded to int64. Arrow deliberately does not support converting - # them to pandas (pandas categorical codes are signed), so that pre-existing - # limitation is now reachable, where the silent downgrade used to mask it. - # uint8/uint16/uint32 dictionary indices convert to pandas as before. - arr = pa.array(["a", "b"], type=pa.dictionary(pa.uint64(), pa.string())) - with pytest.raises(pa.ArrowTypeError, match="UInt64 dictionary indices"): - arr.to_pandas() +def test_dictionary_uint64_index_to_pandas(): + # GH-37476: uint64 dictionary indices are preserved, and converting them to pandas + # maps the indices to int64 categorical codes, which is safe because the indices are + # bounded by the dictionary length. + arr = pa.array(["a", "b", None, "a"], type=pa.dictionary(pa.uint64(), pa.string())) + result = arr.to_pandas() + assert list(result.cat.categories) == ["a", "b"] + assert result.cat.codes.tolist() == [0, 1, -1, 0] diff --git a/python/pyarrow/tests/test_pandas.py b/python/pyarrow/tests/test_pandas.py index 2a782de1648c..d2ad7ac7f3e4 100644 --- a/python/pyarrow/tests/test_pandas.py +++ b/python/pyarrow/tests/test_pandas.py @@ -4142,21 +4142,14 @@ def test_dictionary_with_pandas(): d1 = pa.DictionaryArray.from_arrays(indices, dictionary) d2 = pa.DictionaryArray.from_arrays(indices, dictionary, mask=mask) - if index_type == 'uint64': - # uint64 is not supported due to overflow risk (values > 2^63-1) - with pytest.raises(TypeError, - match="UInt64 dictionary indices"): - d1.to_pandas() - continue - pandas1 = d1.to_pandas() # Pandas Categorical uses signed int codes. Arrow converts: - # uint8 to int16, uint16 to int32, uint32 to int64, signed types unchanged + # uint8 to int16, uint16 to int32, uint32 and uint64 to int64, signed unchanged if index_type == 'uint8': compare_indices = indices.astype('int16') elif index_type == 'uint16': compare_indices = indices.astype('int32') - elif index_type == 'uint32': + elif index_type in ('uint32', 'uint64'): compare_indices = indices.astype('int64') else: compare_indices = indices @@ -4172,7 +4165,7 @@ def test_dictionary_with_pandas(): signed_indices = indices.astype('int16') elif index_type == 'uint16': signed_indices = indices.astype('int32') - elif index_type == 'uint32': + elif index_type in ('uint32', 'uint64'): signed_indices = indices.astype('int64') else: signed_indices = indices From a1836dbe8e9911a4ea2236ad47fa781442dd042d Mon Sep 17 00:00:00 2001 From: mkzung <103102868+mkzung@users.noreply.github.com> Date: Tue, 28 Jul 2026 15:59:33 +0500 Subject: [PATCH 3/3] GH-37476: address review round 2 - MaybeUnsignedIndexType: the default switch case is unreachable (the adaptive index builder only produces signed int8/16/32/64), so mark it with arrow::Unreachable instead of returning the argument - SuppliedDictionary/ExactIndexBuilder index-type tests: build the array and assert the finished array's index type, not just the builder's reported type - rename WidthStillAdaptsWhenUnsigned -> WidthAdaptsWhenUnsigned (drop the context-dependent "Still") --- cpp/src/arrow/array/array_dict_test.cc | 15 ++++++++++++++- cpp/src/arrow/builder.cc | 5 ++++- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/cpp/src/arrow/array/array_dict_test.cc b/cpp/src/arrow/array/array_dict_test.cc index 497745180921..23335ebb008a 100644 --- a/cpp/src/arrow/array/array_dict_test.cc +++ b/cpp/src/arrow/array/array_dict_test.cc @@ -1070,7 +1070,7 @@ TEST(TestDictionaryBuilderIndexType, PreservesRequestedIndexType) { } } -TEST(TestDictionaryBuilderIndexType, WidthStillAdaptsWhenUnsigned) { +TEST(TestDictionaryBuilderIndexType, WidthAdaptsWhenUnsigned) { // The width stays adaptive, as it does for signed indices. The underlying builder is // signed, so it widens after 128 distinct values rather than the 256 a uint8 could // hold, but the widened type stays unsigned rather than falling back to a signed type. @@ -1131,6 +1131,14 @@ TEST(TestDictionaryBuilderIndexType, SuppliedDictionaryPreservesUnsignedIndexTyp MakeDictionaryBuilder(dictionary(uint32(), utf8()), dict_values)); AssertTypeEqual(*uint8(), *checked_cast(*builder->type()).index_type()); + + auto& dict_builder = checked_cast&>(*builder); + ASSERT_OK(dict_builder.Append("a")); + ASSERT_OK(dict_builder.Append("b")); + ASSERT_OK_AND_ASSIGN(auto result, dict_builder.Finish()); + ASSERT_OK(result->ValidateFull()); + AssertTypeEqual(*uint8(), + *checked_cast(*result->type()).index_type()); } TEST(TestDictionaryBuilderIndexType, ExactIndexBuilderPreservesUnsignedIndexType) { @@ -1142,6 +1150,11 @@ TEST(TestDictionaryBuilderIndexType, ExactIndexBuilderPreservesUnsignedIndexType AssertTypeEqual( *uint16(), *checked_cast(*boxed_builder->type()).index_type()); + + ASSERT_OK_AND_ASSIGN(auto result, boxed_builder->Finish()); + ASSERT_OK(result->ValidateFull()); + AssertTypeEqual(*uint16(), + *checked_cast(*result->type()).index_type()); } // ---------------------------------------------------------------------- diff --git a/cpp/src/arrow/builder.cc b/cpp/src/arrow/builder.cc index d333946bcf10..1d0a42099b3e 100644 --- a/cpp/src/arrow/builder.cc +++ b/cpp/src/arrow/builder.cc @@ -27,6 +27,7 @@ #include "arrow/util/checked_cast.h" #include "arrow/util/hashing.h" #include "arrow/util/logging_internal.h" +#include "arrow/util/unreachable.h" #include "arrow/visit_type_inline.h" namespace arrow { @@ -64,7 +65,9 @@ std::shared_ptr MaybeUnsignedIndexType( case Type::INT64: return ::arrow::uint64(); default: - return index_type; + // The adaptive index builder only ever produces signed int8/16/32/64, so no + // other type reaches this point when an unsigned index was requested. + Unreachable("MaybeUnsignedIndexType: adaptive dictionary index type is not signed"); } }