Skip to content

GH-44183: [C++] Support run-end encoded struct, list (view), large list (view) and map values #50534

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
pitrou merged 4 commits into from
Jul 30, 2026
+365 −0
Merged

GH-44183: [C++] Support run-end encoded struct, list (view), large list (view) and map values #50534

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
7ee5256
support list and struct values for run-end encoding
BenMagyar Jul 17, 2026
5610e61
support list and struct values for run-end decoding
BenMagyar Jul 17, 2026
cf9bbc6
extend to large list, list view, large list view, map
BenMagyar Jul 28, 2026
22fc11b
drop python tests, unnecessary coverage
BenMagyar Jul 29, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
150 changes: 150 additions & 0 deletions cpp/src/arrow/compute/kernels/vector_run_end_encode.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,10 @@

#include <utility>

#include "arrow/array/array_run_end.h"
#include "arrow/array/builder_base.h"
#include "arrow/array/builder_primitive.h"
#include "arrow/compare.h"
#include "arrow/compute/api_vector.h"
#include "arrow/compute/kernel.h"
#include "arrow/compute/kernels/common_internal.h"
Expand Down Expand Up @@ -289,6 +293,68 @@ struct RunEndEncodeExec {
return Status::Invalid("Invalid run end type: ", *state->run_end_type);
}

template <typename RunEndType>
static Status DoExecNested(KernelContext* ctx, const ArraySpan& input_array,
ExecResult* result) {
using RunEndCType = typename RunEndType::c_type;
const int64_t input_length = input_array.length;
auto run_end_type = TypeTraits<RunEndType>::type_singleton();
RETURN_NOT_OK(ValidateRunEndType(run_end_type, input_length));

NumericBuilder<RunEndType> run_ends_builder(ctx->memory_pool());
if (input_length > 0) {
auto input = input_array.ToArray();
// Avoid merging floating-point values whose representations may differ. Signed
// zeros compare unequal, and NaNs remain in separate runs to preserve payloads.
const auto equal_options =
EqualOptions::Defaults().nans_equal(false).signed_zeros_equal(false);
for (int64_t i = 1; i < input_length; ++i) {
if (!ArrayRangeEquals(*input, *input, i - 1, i, i, equal_options)) {
RETURN_NOT_OK(run_ends_builder.Append(static_cast<RunEndCType>(i)));
}
}
RETURN_NOT_OK(run_ends_builder.Append(static_cast<RunEndCType>(input_length)));
}

ARROW_ASSIGN_OR_RAISE(
auto values_builder,
MakeBuilderExactIndex(input_array.type->GetSharedPtr(), ctx->memory_pool()));
RETURN_NOT_OK(values_builder->Reserve(run_ends_builder.length()));
int64_t run_start = 0;
for (int64_t i = 0; i < run_ends_builder.length(); ++i) {
if (input_array.IsNull(run_start)) {
RETURN_NOT_OK(values_builder->AppendNull());
} else {
RETURN_NOT_OK(values_builder->AppendArraySlice(input_array, run_start, 1));
}
run_start = run_ends_builder.GetValue(i);
}

ARROW_ASSIGN_OR_RAISE(auto run_ends, run_ends_builder.Finish());
ARROW_ASSIGN_OR_RAISE(auto values, values_builder->Finish());
ARROW_ASSIGN_OR_RAISE(auto output,
RunEndEncodedArray::Make(input_length, run_ends, values));
result->value = output->data();
return Status::OK();
}

static Status ExecNested(KernelContext* ctx, const ExecSpan& span, ExecResult* result) {
DCHECK(span.values[0].is_array());
const auto& input_array = span.values[0].array;
const auto* state = checked_cast<const RunEndEncodingState*>(ctx->state());
switch (state->run_end_type->id()) {
case Type::INT16:
return DoExecNested<Int16Type>(ctx, input_array, result);
case Type::INT32:
return DoExecNested<Int32Type>(ctx, input_array, result);
case Type::INT64:
return DoExecNested<Int64Type>(ctx, input_array, result);
default:
break;
}
return Status::Invalid("Invalid run end type: ", *state->run_end_type);
}

/// \brief The OutputType::Resolver of the "run_end_decode" function.
static Result<TypeHolder> ResolveOutputType(
KernelContext* ctx, const std::vector<TypeHolder>& input_types) {
Expand Down Expand Up @@ -470,6 +536,56 @@ struct RunEndDecodeExec {
return Status::Invalid("Invalid run end type: ", *ree_type->run_end_type());
}

template <typename RunEndType>
static Status DoExecNested(KernelContext* ctx, const ArraySpan& input_array,
ExecResult* result) {
using RunEndCType = typename RunEndType::c_type;
const auto& input_values = arrow::ree_util::ValuesArray(input_array);

ARROW_ASSIGN_OR_RAISE(
auto output_builder,
MakeBuilderExactIndex(input_values.type->GetSharedPtr(), ctx->memory_pool()));
RETURN_NOT_OK(output_builder->Reserve(input_array.length));

const arrow::ree_util::RunEndEncodedArraySpan<RunEndCType> ree_array_span(
input_array);
if (input_array.length > 0) {
for (auto it = ree_array_span.begin(); !it.is_end(ree_array_span); ++it) {
const int64_t physical_index = it.index_into_array();
const int64_t run_length = it.run_length();
if (input_values.IsNull(physical_index)) {
RETURN_NOT_OK(output_builder->AppendNulls(run_length));
continue;
}
for (int64_t i = 0; i < run_length; ++i) {
RETURN_NOT_OK(
output_builder->AppendArraySlice(input_values, physical_index, 1));
}
Comment on lines +560 to +563

@pitrou pitrou Jul 29, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I wonder if it would be more performant to implement this as:

Suggested change
for (int64_t i = 0; i < run_length; ++i) {
RETURN_NOT_OK(
output_builder->AppendArraySlice(input_values, physical_index, 1));
}
ARROW_ASSIGN_OR_RAISE(
auto scalar, input_values.GetScalar(physical_index));
RETURN_NOT_OK(output_builder->AppendScalar(scalar, run_length));

GetScalar adds its own cost, but it's amortized over the entire run.

@felipecrv What do you think?

@BenMagyar BenMagyar Jul 29, 2026 •
edited
Loading

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I ran some some mini benchmarks to test this:

Value type Run length 1 Run length 64 Run length 4096 Run length 16384
List<Int32>[4] 32.5x slower 12.1x slower 11.5x slower 11.6x slower
List<Int32>[64] 126.3x slower 116.3x slower 115.9x slower 115.2x slower
LargeList<Int32>[8] 41.9x slower 22.3x slower 21.8x slower 22.1x slower
ListView<Int32>[8] 41.7x slower 22.1x slower 21.7x slower 21.7x slower
LargeListView<Int32>[8] 42.3x slower 22.2x slower 21.9x slower 21.6x slower
FixedSizeList<Int32>[8] 53.4x slower 26.2x slower 25.7x slower 25.6x slower
Map<Utf8, Int32>[4] 130.5x slower 107.9x slower 109.6x slower 107.0x slower
Struct mixed 58.9x slower 16.5x slower 15.6x slower 15.7x slower
Empty List 24.2x slower 0.75x 0.35x 0.35x
List<Int32>[1] 23.7x slower 3.5x slower 3.1x slower 3.1x slower
Empty Map 28.0x slower 1.0x 0.52x 0.51x
Struct<Int32> 34.9x slower 1.4x slower 0.84x 0.77x
Empty Struct 30.6x slower 1.1x slower 0.56x 0.56x

@pitrou pitrou Jul 30, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I see, sorry for the bad idea then :)

}
}

ARROW_ASSIGN_OR_RAISE(auto output, output_builder->Finish());
result->value = output->data();
return Status::OK();
}

static Status ExecNested(KernelContext* ctx, const ExecSpan& span, ExecResult* result) {
DCHECK(span.values[0].is_array());
const auto& input_array = span.values[0].array;
const auto& ree_type = checked_cast<const RunEndEncodedType*>(input_array.type);
switch (ree_type->run_end_type()->id()) {
case Type::INT16:
return DoExecNested<Int16Type>(ctx, input_array, result);
case Type::INT32:
return DoExecNested<Int32Type>(ctx, input_array, result);
case Type::INT64:
return DoExecNested<Int64Type>(ctx, input_array, result);
default:
break;
}
return Status::Invalid("Invalid run end type: ", *ree_type->run_end_type());
}

/// \brief The OutputType::Resolver of the "run_end_decode" function.
static Result<TypeHolder> ResolveOutputType(KernelContext*,
const std::vector<TypeHolder>& in_types) {
Expand Down Expand Up @@ -562,6 +678,15 @@ void RegisterVectorRunEndEncode(FunctionRegistry* registry) {
DCHECK_OK(function->AddKernel(std::move(kernel)));
};

auto add_nested_kernel = [&function](Type::type type_id) {
auto sig = KernelSignature::Make({InputType(match::SameTypeId(type_id))},
OutputType(RunEndEncodeExec::ResolveOutputType));
VectorKernel kernel(sig, RunEndEncodeExec::ExecNested, RunEndEncodeInit);
// A REE has null_count=0, so no need to allocate a validity bitmap for them.
kernel.null_handling = NullHandling::OUTPUT_NOT_NULL;
DCHECK_OK(function->AddKernel(std::move(kernel)));
};

add_kernel(Type::NA);
add_kernel(Type::BOOL);
for (const auto& ty : NumericTypes()) {
Expand All @@ -585,6 +710,13 @@ void RegisterVectorRunEndEncode(FunctionRegistry* registry) {
add_kernel(Type::BINARY);
add_kernel(Type::LARGE_STRING);
add_kernel(Type::LARGE_BINARY);
add_nested_kernel(Type::FIXED_SIZE_LIST);
add_nested_kernel(Type::LIST);
Comment thread
pitrou marked this conversation as resolved.
add_nested_kernel(Type::LARGE_LIST);
add_nested_kernel(Type::LIST_VIEW);
add_nested_kernel(Type::LARGE_LIST_VIEW);
add_nested_kernel(Type::MAP);
add_nested_kernel(Type::STRUCT);

DCHECK_OK(registry->AddFunction(std::move(function)));
}
Expand All @@ -605,6 +737,17 @@ void RegisterVectorRunEndDecode(FunctionRegistry* registry) {
}
};

auto add_nested_kernel = [&function](Type::type type_id) {
for (const auto& run_end_type_id : {Type::INT16, Type::INT32, Type::INT64}) {
auto input_type_matcher = match::RunEndEncoded(match::SameTypeId(run_end_type_id),
match::SameTypeId(type_id));
auto sig = KernelSignature::Make({InputType(std::move(input_type_matcher))},
OutputType(RunEndDecodeExec::ResolveOutputType));
VectorKernel kernel(sig, RunEndDecodeExec::ExecNested);
DCHECK_OK(function->AddKernel(std::move(kernel)));
}
};

add_kernel(Type::NA);
add_kernel(Type::BOOL);
for (const auto& ty : NumericTypes()) {
Expand All @@ -628,6 +771,13 @@ void RegisterVectorRunEndDecode(FunctionRegistry* registry) {
add_kernel(Type::BINARY);
add_kernel(Type::LARGE_STRING);
add_kernel(Type::LARGE_BINARY);
add_nested_kernel(Type::FIXED_SIZE_LIST);
add_nested_kernel(Type::LIST);
add_nested_kernel(Type::LARGE_LIST);
add_nested_kernel(Type::LIST_VIEW);
add_nested_kernel(Type::LARGE_LIST_VIEW);
add_nested_kernel(Type::MAP);
add_nested_kernel(Type::STRUCT);

DCHECK_OK(registry->AddFunction(std::move(function)));
}
Expand Down
Loading
Loading