diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt index 7211bf5476dc..04b6eb396446 100644 --- a/cpp/src/arrow/CMakeLists.txt +++ b/cpp/src/arrow/CMakeLists.txt @@ -819,6 +819,7 @@ if(ARROW_COMPUTE) compute/kernels/scalar_arithmetic.cc compute/kernels/scalar_boolean.cc compute/kernels/scalar_compare.cc + compute/kernels/scalar_hash.cc compute/kernels/scalar_if_else.cc compute/kernels/scalar_nested.cc compute/kernels/scalar_random.cc diff --git a/cpp/src/arrow/compute/CMakeLists.txt b/cpp/src/arrow/compute/CMakeLists.txt index 88f9ae900647..f959a0d6a861 100644 --- a/cpp/src/arrow/compute/CMakeLists.txt +++ b/cpp/src/arrow/compute/CMakeLists.txt @@ -185,6 +185,7 @@ add_arrow_compute_test(row_test arrow_compute_testing) add_arrow_compute_benchmark(function_benchmark) +add_arrow_compute_benchmark(key_hash_benchmark) add_subdirectory(kernels) diff --git a/cpp/src/arrow/compute/api_scalar.cc b/cpp/src/arrow/compute/api_scalar.cc index 0aa8fd757a98..ba1f20f6eb09 100644 --- a/cpp/src/arrow/compute/api_scalar.cc +++ b/cpp/src/arrow/compute/api_scalar.cc @@ -957,6 +957,16 @@ Result MapLookup(const Datum& arg, MapLookupOptions options, ExecContext* return CallFunction("map_lookup", {arg}, &options, ctx); } +// ---------------------------------------------------------------------- +// Hash functions +Result Hash32(const Datum& input_array, ExecContext* ctx) { + return CallFunction("hash32", {input_array}, ctx); +} + +Result Hash64(const Datum& input_array, ExecContext* ctx) { + return CallFunction("hash64", {input_array}, ctx); +} + // ---------------------------------------------------------------------- } // namespace compute diff --git a/cpp/src/arrow/compute/api_scalar.h b/cpp/src/arrow/compute/api_scalar.h index c4238b956c9c..a2a68635f34f 100644 --- a/cpp/src/arrow/compute/api_scalar.h +++ b/cpp/src/arrow/compute/api_scalar.h @@ -1807,5 +1807,46 @@ ARROW_EXPORT Result NanosecondsBetween(const Datum& left, const Datum& ri /// \note API not yet finalized ARROW_EXPORT Result MapLookup(const Datum& map, MapLookupOptions options, ExecContext* ctx = NULLPTR); + +/// \brief Construct a hash value for each row of the input. +/// +/// The result has the same length and shape as the input (Array in, Array out; +/// ChunkedArray in, ChunkedArray out), but with element type UInt32. For a nested +/// input type (struct, list, map, etc.), each row's child values are combined into a +/// single hash for that row, recursively. A null input row produces a null output row; +/// within a struct, a null field makes that whole row null, while within a list or map a +/// null element does not (only the row's own validity matters there). Hash values are not +/// guaranteed to be stable across different versions of the library, and this function +/// does not currently take options, though these may be added in the future. +/// +/// \param[in] input_array input data to hash +/// \param[in] ctx function execution context, optional +/// \return elementwise hash values +/// +/// \since 26.0.0 +/// \note API not yet finalized +ARROW_EXPORT +Result Hash32(const Datum& input_array, ExecContext* ctx = NULLPTR); + +/// \brief Construct a hash value for each row of the input. +/// +/// The result has the same length and shape as the input (Array in, Array out; +/// ChunkedArray in, ChunkedArray out), but with element type UInt64. For a nested +/// input type (struct, list, map, etc.), each row's child values are combined into a +/// single hash for that row, recursively. A null input row produces a null output row; +/// within a struct, a null field makes that whole row null, while within a list or map a +/// null element does not (only the row's own validity matters there). Hash values are not +/// guaranteed to be stable across different versions of the library, and this function +/// does not currently take options, though these may be added in the future. +/// +/// \param[in] input_array input data to hash +/// \param[in] ctx function execution context, optional +/// \return elementwise hash values +/// +/// \since 26.0.0 +/// \note API not yet finalized +ARROW_EXPORT +Result Hash64(const Datum& input_array, ExecContext* ctx = NULLPTR); + } // namespace compute } // namespace arrow diff --git a/cpp/src/arrow/compute/initialize.cc b/cpp/src/arrow/compute/initialize.cc index d88835da04ac..ec386aa5e3ee 100644 --- a/cpp/src/arrow/compute/initialize.cc +++ b/cpp/src/arrow/compute/initialize.cc @@ -31,6 +31,7 @@ Status RegisterComputeKernels() { internal::RegisterScalarArithmetic(registry); internal::RegisterScalarBoolean(registry); internal::RegisterScalarComparison(registry); + internal::RegisterScalarHash(registry); internal::RegisterScalarIfElse(registry); internal::RegisterScalarNested(registry); internal::RegisterScalarRandom(registry); // Nullary diff --git a/cpp/src/arrow/compute/kernels/CMakeLists.txt b/cpp/src/arrow/compute/kernels/CMakeLists.txt index 15955b5ef883..3fc21a91bdcc 100644 --- a/cpp/src/arrow/compute/kernels/CMakeLists.txt +++ b/cpp/src/arrow/compute/kernels/CMakeLists.txt @@ -77,6 +77,7 @@ add_arrow_compute_test(scalar_math_test add_arrow_compute_test(scalar_utility_test SOURCES + scalar_hash_test.cc scalar_random_test.cc scalar_set_lookup_test.cc scalar_validity_test.cc @@ -89,6 +90,7 @@ add_arrow_benchmark(scalar_cast_benchmark PREFIX "arrow-compute") add_arrow_compute_benchmark(scalar_arithmetic_benchmark) add_arrow_compute_benchmark(scalar_boolean_benchmark) add_arrow_compute_benchmark(scalar_compare_benchmark) +add_arrow_compute_benchmark(scalar_hash_benchmark) add_arrow_compute_benchmark(scalar_if_else_benchmark) add_arrow_compute_benchmark(scalar_list_benchmark) add_arrow_compute_benchmark(scalar_random_benchmark) diff --git a/cpp/src/arrow/compute/kernels/scalar_hash.cc b/cpp/src/arrow/compute/kernels/scalar_hash.cc new file mode 100644 index 000000000000..9e9960cf73c6 --- /dev/null +++ b/cpp/src/arrow/compute/kernels/scalar_hash.cc @@ -0,0 +1,454 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include + +#include "arrow/array/array_base.h" +#include "arrow/array/util.h" +#include "arrow/compute/cast.h" +#include "arrow/compute/kernels/common_internal.h" +#include "arrow/compute/key_hash_internal.h" +#include "arrow/compute/light_array_internal.h" +#include "arrow/compute/registry_internal.h" +#include "arrow/compute/util.h" +#include "arrow/result.h" +#include "arrow/util/bit_run_reader.h" +#include "arrow/util/bit_util.h" +#include "arrow/util/bitmap_generate.h" +#include "arrow/util/bitmap_ops.h" + +namespace arrow { +namespace compute { +namespace internal { + +// Define symbols visible within `arrow::compute::internal` in this file; +// these symbols are not visible outside of this file. +namespace { + +// ------------------------------ +// Kernel implementations +// It is expected that HashArrowType is either UInt32Type or UInt64Type (default) + +// Free function (not dependent on ArrowType/Hasher) to avoid codegen per instantiation. +// Only called with a plain column; HashArray routes everything else (see +// NeedsRecursiveHash) elsewhere first. +Result ToColumnArray(const ArraySpan& array) { + KeyColumnMetadata metadata; + const uint8_t* validity_buffer = nullptr; + const uint8_t* fixed_length_buffer = nullptr; + const uint8_t* var_length_buffer = nullptr; + + if (array.GetBuffer(0) != nullptr) { + validity_buffer = array.GetBuffer(0)->data(); + } + if (array.GetBuffer(1) != nullptr) { + fixed_length_buffer = array.GetBuffer(1)->data(); + } + + auto type = array.type; + auto type_id = type->id(); + if (type_id == Type::NA) { + metadata = KeyColumnMetadata(true, 0, true); + } else if (type_id == Type::BOOL) { + metadata = KeyColumnMetadata(true, 0); + } else if (is_fixed_width(type_id)) { + metadata = KeyColumnMetadata(true, type->bit_width() / 8); + } else if (is_binary_like(type_id)) { + metadata = KeyColumnMetadata(false, sizeof(uint32_t)); + if (array.GetBuffer(2) != nullptr) { + var_length_buffer = array.GetBuffer(2)->data(); + } + } else if (is_large_binary_like(type_id)) { + metadata = KeyColumnMetadata(false, sizeof(uint64_t)); + if (array.GetBuffer(2) != nullptr) { + var_length_buffer = array.GetBuffer(2)->data(); + } + } else { + return Status::TypeError("Unsupported column data type ", type->name(), + " used with hash32/hash64 compute kernel"); + } + + return KeyColumnArray(metadata, array.length, validity_buffer, fixed_length_buffer, + var_length_buffer); +} + +// Whether HashArray must handle `type_id` itself rather than passing it to +// ToColumnArray/HashMultiColumn. Broader than is_nested(): EXTENSION and DICTIONARY +// aren't nested, but ToColumnArray has no case for either (and hashing a dictionary's +// raw indices would be wrong anyway). +bool NeedsRecursiveHash(Type::type type_id) { + return type_id == Type::EXTENSION || type_id == Type::DICTIONARY || is_nested(type_id); +} + +// Writes `array`'s own validity into `out_validity` (a fresh 0-offset bitmap), rebasing +// off array.offset. Only called for types whose validity really is a plain bitmap; union +// and run-end-encoded, which ArraySpan::IsValid computes specially, never reach here. +void WriteOwnValidity(const ArraySpan& array, uint8_t* out_validity) { + if (array.GetBuffer(0) == nullptr) { + // No bitmap: every row shares one answer, all valid or (for NA, whose null_count + // SetSlice keeps equal to length) all null. + bit_util::SetBitsTo(out_validity, 0, array.length, + /*bits_are_set=*/array.null_count != array.length); + return; + } + ::arrow::internal::CopyBitmap(array.GetBuffer(0)->data(), array.offset, array.length, + out_validity, /*dest_offset=*/0); +} + +// Folds one row's child hashes into a single hash. Seeded with CombineHashes(0, 0) rather +// than 0 just so an empty list doesn't hash to a bare 0 -- a hash-quality nicety, not a +// requirement, since a list row's validity is independent of its hash value. +template +c_type CombineRange(const c_type* value_hashes, int64_t start, int64_t end) { + c_type combined = Hasher::CombineHashes(0, 0); + for (int64_t j = start; j < end; j++) { + combined = Hasher::CombineHashes(combined, value_hashes[j]); + } + return combined; +} + +template +struct FastHashScalar { + using c_type = typename ArrowType::c_type; + + // Hashes the [offset, offset + length) slice of `child` into hash values plus real + // validity, always based at offset 0 whatever `child`'s own offset (callers read the + // buffers row-0-based). Only a null row's validity bit is meaningful, not its hash + // value; callers folding these into a parent hash must handle that (see + // HashListArray). + static Result> HashChild(const ArraySpan& child, + int64_t offset, int64_t length, + LightContext* hash_ctx, + ExecContext* exec_ctx) { + auto sliced = child; + sliced.SetSlice(offset, length); + auto arrow_type = TypeTraits::type_singleton(); + ARROW_ASSIGN_OR_RAISE(auto buffer, AllocateBuffer(sliced.length * sizeof(c_type), + exec_ctx->memory_pool())); + ARROW_ASSIGN_OR_RAISE(auto validity, + AllocateBitmap(sliced.length, exec_ctx->memory_pool())); + ARROW_RETURN_NOT_OK(HashArray(sliced, hash_ctx, exec_ctx, + buffer->mutable_data_as(), + validity->mutable_data())); + return ArrayData::Make(arrow_type, sliced.length, + {std::move(validity), std::move(buffer)}, kUnknownNullCount); + } + + static Status HashStructArray(const ArraySpan& array, LightContext* hash_ctx, + ExecContext* exec_ctx, c_type* out, + uint8_t* out_validity) { + // Row validity is the struct's own ANDed with every field's (in place, as + // swiss_join.cc does for multi-column nulls): an independently-null field makes the + // row invalid too (GH-17211), just like the struct row being null. + WriteOwnValidity(array, out_validity); + + if (array.child_data.empty()) { + // struct<>: HashMultiColumn needs >=1 column, so give every row one fixed hash; + // validity is already fully set above. + c_type empty_struct_hash = Hasher::CombineHashes(0, 0); + for (int64_t i = 0; i < array.length; i++) { + out[i] = empty_struct_hash; + } + return Status::OK(); + } + + std::vector> child_hashes(array.child_data.size()); + std::vector columns(array.child_data.size()); + for (size_t i = 0; i < array.child_data.size(); i++) { + // By reference: ArraySpan owns a child_data vector, so copying one heap-allocates. + const ArraySpan& child = array.child_data[i]; + // `child` may have its own offset independent of the struct's (see + // StructArray::GetFlattenedField): struct row r reads child row + // (child.offset + array.offset + r). + if (NeedsRecursiveHash(child.type->id())) { + // StructArray::Slice() doesn't reslice child_data, so `child` may be larger + // than this slice of `array` references -- hash only the referenced range. + ARROW_ASSIGN_OR_RAISE(child_hashes[i], + HashChild(child, child.offset + array.offset, array.length, + hash_ctx, exec_ctx)); + ::arrow::internal::BitmapAnd(out_validity, 0, child_hashes[i]->buffers[0]->data(), + 0, array.length, 0, out_validity); + ARROW_ASSIGN_OR_RAISE(auto column, ToColumnArray(*child_hashes[i])); + // child_hashes[i] already covers exactly [0, array.length): no further slice. + columns[i] = column.Slice(0, array.length); + } else { + if (child.GetBuffer(0) != nullptr) { + ::arrow::internal::BitmapAnd(out_validity, 0, child.GetBuffer(0)->data(), + child.offset + array.offset, array.length, 0, + out_validity); + } + ARROW_ASSIGN_OR_RAISE(auto column, ToColumnArray(child)); + columns[i] = column.Slice(child.offset + array.offset, array.length); + } + } + Hasher::HashMultiColumn(columns, hash_ctx, out); + return Status::OK(); + } + + // Handles FIXED_SIZE_LIST, LARGE_LIST, LIST, and MAP. `offsets` is null for + // FIXED_SIZE_LIST, which uses `list_size` as a constant stride instead. + template + static Status HashListArray(const ArraySpan& array, int64_t list_size, + const OffsetT* offsets, LightContext* hash_ctx, + ExecContext* exec_ctx, c_type* out, uint8_t* out_validity) { + // The range of `values` this array actually references, as logical indices relative + // to values.offset. Needed because ArraySpan::SetSlice() doesn't reslice child_data, + // so `values` can be far larger than what this (possibly sliced) array covers. + // offsets[] are already such logical indices; FIXED_SIZE_LIST derives them from its + // constant stride instead. + int64_t rel_start = 0, rel_end = 0; + if (array.length > 0) { + if (offsets != nullptr) { + rel_start = offsets[0]; + rel_end = offsets[array.length]; + } else { + rel_start = array.offset * list_size; + rel_end = (array.offset + array.length) * list_size; + } + } + + // By reference: ArraySpan owns a child_data vector, so copying one heap-allocates. + const ArraySpan& values = array.child_data[0]; + // Element k of the result is original values row (values.offset + rel_start + k). + ARROW_ASSIGN_OR_RAISE(auto value_hashes, + HashChild(values, values.offset + rel_start, + rel_end - rel_start, hash_ctx, exec_ctx)); + // Zero the null elements' hashes: CombineRange folds values blind, and a null slot's + // bytes are undefined per the columnar spec, so otherwise a null element contributes + // whatever garbage it sat on and list> rows [{f0: 7}] and [null] + // (whose f0 slot also holds 7) hash alike. Filling only the gaps between runs of + // valid rows leaves the common all-valid case free. HashStructArray needs no + // equivalent: HashMultiColumn gets its fields' validity and already fixes each null + // row's contribution. + c_type* value_hash_data = value_hashes->buffers[1]->mutable_data_as(); + int64_t valid_end = 0; + ::arrow::internal::VisitSetBitRunsVoid( + value_hashes->buffers[0]->data(), /*offset=*/0, value_hashes->length, + [&](int64_t position, int64_t run_length) { + std::fill(value_hash_data + valid_end, value_hash_data + position, c_type{0}); + valid_end = position + run_length; + }); + std::fill(value_hash_data + valid_end, value_hash_data + value_hashes->length, + c_type{0}); + + if (offsets != nullptr) { + // offsets[] index the values child; value_hash_data starts at rel_start. + for (int64_t i = 0; i < array.length; i++) { + out[i] = CombineRange(value_hash_data, offsets[i] - rel_start, + offsets[i + 1] - rel_start); + } + } else { + // rel_start is array.offset * list_size, so row i starts at i * list_size. + for (int64_t i = 0; i < array.length; i++) { + int64_t start = i * list_size; + out[i] = CombineRange(value_hash_data, start, start + list_size); + } + } + // A list/map row's validity is its own only -- what's inside it (even a null + // element, or a null row's non-empty offset range) never changes that. value_hashes' + // validity buffer is deliberately not consulted here. + WriteOwnValidity(array, out_validity); + return Status::OK(); + } + + // Routes to the per-shape hashing routine for `array`'s type, writing both hash + // values (`out`) and real per-row validity (`out_validity`, a fresh 0-offset bitmap, + // same convention `out` has via ArraySpan::GetValues). + static Status HashArray(const ArraySpan& array, LightContext* hash_ctx, + ExecContext* exec_ctx, c_type* out, uint8_t* out_validity) { + auto type_id = array.type->id(); + if (type_id == Type::FIXED_SIZE_BINARY && array.type->byte_width() == 0) { + // Zero-width values carry no data, so every row holds the same empty byte string + // and must hash identically. ToColumnArray can only describe this as a fixed-width + // column of length 0, exactly how a bit-packed boolean is encoded too, so + // HashMultiColumn would call HashBit and take each row's hash from a bit that + // doesn't exist -- uninitialized garbage, differing per row and per slice. + std::fill(out, out + array.length, Hasher::CombineHashes(0, 0)); + WriteOwnValidity(array, out_validity); + return Status::OK(); + } else if (!NeedsRecursiveHash(type_id)) { + ARROW_ASSIGN_OR_RAISE(auto column, ToColumnArray(array)); + std::vector columns{column.Slice(array.offset, array.length)}; + Hasher::HashMultiColumn(columns, hash_ctx, out); + // A plain column's own validity is the whole story, and HashMultiColumn has + // already folded it into the hash values via ToColumnArray's buffer. + WriteOwnValidity(array, out_validity); + return Status::OK(); + } else if (type_id == Type::EXTENSION) { + auto extension_type = checked_cast(array.type); + auto storage_array = array; + storage_array.type = extension_type->storage_type().get(); + return HashArray(storage_array, hash_ctx, exec_ctx, out, out_validity); + } else if (type_id == Type::DICTIONARY) { + // Hash the logical values, not the indices -- otherwise two dictionaries + // encoding the same values differently would hash differently, and a valid + // index pointing at a null dictionary entry would be missed. Cast's decode + // (Take under the hood) already produces a correct validity buffer for both, so + // recursing into the decoded array handles validity for free. Reuse the + // caller's ExecContext rather than a synthesized default one, same as other + // kernels' dictionary-decode path (see EnsureDictionaryDecoded). + auto dict_type = checked_cast(array.type); + ARROW_ASSIGN_OR_RAISE(auto decoded, + Cast(*MakeArray(array.ToArrayData()), dict_type->value_type(), + CastOptions::Safe(dict_type->value_type()), exec_ctx)); + return HashArray(*decoded->data(), hash_ctx, exec_ctx, out, out_validity); + } else if (type_id == Type::STRUCT) { + return HashStructArray(array, hash_ctx, exec_ctx, out, out_validity); + } else if (type_id == Type::FIXED_SIZE_LIST) { + auto list_size = checked_cast(array.type)->list_size(); + return HashListArray(array, list_size, /*offsets=*/nullptr, hash_ctx, + exec_ctx, out, out_validity); + } else if (type_id == Type::LARGE_LIST) { + return HashListArray(array, /*list_size=*/0, array.GetValues(1), + hash_ctx, exec_ctx, out, out_validity); + } else if (is_list_like(type_id)) { + // LIST and MAP both use 32-bit offsets. + return HashListArray(array, /*list_size=*/0, array.GetValues(1), + hash_ctx, exec_ctx, out, out_validity); + } else { + // NeedsRecursiveHash claims this type needs recursive handling, but no branch + // above knows how (e.g. a union or run-end-encoded type that somehow slipped + // past HashableMatcher's rejection) -- fail loudly and locally rather than + // silently falling through to a mismatched case. + return Status::NotImplemented("Unsupported column data type ", array.type->name(), + " used with hash32/hash64 compute kernel"); + } + } + + static Status Exec(KernelContext* ctx, const ExecSpan& input_arg, ExecResult* out) { + ARROW_DCHECK_EQ(input_arg.num_values(), 1); + ARROW_DCHECK(input_arg[0].is_array()); + ArraySpan hash_input = input_arg[0].array; + + auto exec_ctx = default_exec_context(); + if (ctx && ctx->exec_context()) { + exec_ctx = ctx->exec_context(); + } + + // Initialize stack-based memory allocator used by Hashing32 and Hashing64 + util::TempVectorStack stack_memallocator; + ARROW_RETURN_NOT_OK(stack_memallocator.Init(exec_ctx->memory_pool(), + Hasher::kHashBatchTempStackUsage)); + + // Prepare context used by Hashing32 and Hashing64 + LightContext hash_ctx; + hash_ctx.hardware_flags = exec_ctx->cpu_info()->hardware_flags(); + hash_ctx.stack = &stack_memallocator; + + // Call the hashing function, overloaded based on OutputCType + ArraySpan* result_span = out->array_span_mutable(); + c_type* result_ptr = result_span->GetValues(1); + + // HashArray writes validity into a fresh, 0-offset bitmap (matching `result_ptr`'s + // own 0-based convention). The kernel's real output buffer may start at a nonzero + // bit offset (e.g. contiguous chunked preallocation), so translate in one final + // pass below instead of threading an offset through the whole recursive engine. + ARROW_ASSIGN_OR_RAISE(auto validity, + AllocateBitmap(hash_input.length, exec_ctx->memory_pool())); + ARROW_RETURN_NOT_OK( + HashArray(hash_input, &hash_ctx, exec_ctx, result_ptr, validity->mutable_data())); + + const uint8_t* validity_data = validity->data(); + int64_t out_null_count = 0; + int64_t row = 0; + ::arrow::internal::GenerateBitsUnrolled( + result_span->buffers[0].data, result_span->offset, hash_input.length, [&] { + bool is_valid = bit_util::GetBit(validity_data, row++); + out_null_count += !is_valid; + return is_valid; + }); + result_span->null_count = out_null_count; + + return Status::OK(); + } +}; + +class HashableMatcher : public TypeMatcher { + public: + HashableMatcher() {} + + bool Matches(const DataType& type) const override { + // Unwrap extension/dictionary types (recursively, either nesting order) so an + // unsupported storage/value type is rejected here, not with a raw TypeError deep + // inside HashArray/ToColumnArray/Cast. + const DataType* physical_type = &type; + while (true) { + if (physical_type->id() == Type::EXTENSION) { + physical_type = + checked_cast(*physical_type).storage_type().get(); + } else if (physical_type->id() == Type::DICTIONARY) { + physical_type = + checked_cast(*physical_type).value_type().get(); + } else { + break; + } + } + return !(is_union(*physical_type) || is_binary_view_like(*physical_type) || + is_list_view(*physical_type) || + physical_type->id() == Type::RUN_END_ENCODED); + } + + bool Equals(const TypeMatcher& other) const override { + if (this == &other) { + return true; + } + auto casted = dynamic_cast(&other); + return casted != nullptr; + } + + std::string ToString() const override { return "hashable"; } +}; + +const FunctionDoc hash32_doc{ + "Construct a hash for every element of the input argument", + ("This function is not suitable for cryptographic purposes.\n" + "Hash results are 32-bit. A null input row produces a null in the output."), + {"hash_input"}}; + +const FunctionDoc hash64_doc{ + "Construct a hash for every element of the input argument", + ("This function is not suitable for cryptographic purposes.\n" + "Hash results are 64-bit. A null input row produces a null in the output."), + {"hash_input"}}; + +} // namespace + +void RegisterScalarHash(FunctionRegistry* registry) { + // Create hash32 and hash64 function instances + auto hash32 = std::make_shared("hash32", Arity::Unary(), hash32_doc); + auto hash64 = std::make_shared("hash64", Arity::Unary(), hash64_doc); + + // Add 32-bit and 64-bit kernels to hash32 and hash64 functions + auto type_matcher = std::make_shared(); + ScalarKernel kernel32({InputType(type_matcher)}, OutputType(uint32()), + FastHashScalar::Exec); + ScalarKernel kernel64({InputType(type_matcher)}, OutputType(uint64()), + FastHashScalar::Exec); + kernel32.null_handling = NullHandling::COMPUTED_PREALLOCATE; + kernel64.null_handling = NullHandling::COMPUTED_PREALLOCATE; + ARROW_DCHECK_OK(hash32->AddKernel(std::move(kernel32))); + ARROW_DCHECK_OK(hash64->AddKernel(std::move(kernel64))); + + // Register hash32 and hash64 functions + ARROW_DCHECK_OK(registry->AddFunction(std::move(hash32))); + ARROW_DCHECK_OK(registry->AddFunction(std::move(hash64))); +} + +} // namespace internal +} // namespace compute +} // namespace arrow diff --git a/cpp/src/arrow/compute/kernels/scalar_hash_benchmark.cc b/cpp/src/arrow/compute/kernels/scalar_hash_benchmark.cc new file mode 100644 index 000000000000..965fd743841d --- /dev/null +++ b/cpp/src/arrow/compute/kernels/scalar_hash_benchmark.cc @@ -0,0 +1,207 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +#include +#include +#include +#include + +#include "benchmark/benchmark.h" + +#include "arrow/testing/gtest_util.h" +#include "arrow/testing/random.h" +#include "arrow/util/hashing.h" + +#include "arrow/array/array_nested.h" +#include "arrow/compute/exec.h" + +namespace arrow { +namespace internal { + +// ------------------------------ +// Anonymous namespace with global params + +namespace { +// copied from scalar_string_benchmark +constexpr auto kSeed = 0x94378165; +constexpr double null_prob = 0.2; + +static random::RandomArrayGenerator hashing_rng(kSeed); +} // namespace + +// ------------------------------ +// Convenience functions + +static Result> MakeStructArray(int64_t n_values, + int32_t min_strlen, + int32_t max_strlen) { + auto vals_first = hashing_rng.Int64(n_values, 0, std::numeric_limits::max()); + auto vals_second = hashing_rng.String(n_values, min_strlen, max_strlen, null_prob); + auto vals_third = hashing_rng.Int64(n_values, 0, std::numeric_limits::max()); + + return arrow::StructArray::Make( + arrow::ArrayVector{vals_first, vals_second, vals_third}, + arrow::FieldVector{arrow::field("first", arrow::int64()), + arrow::field("second", arrow::utf8()), + arrow::field("third", arrow::int64())}); +} + +// ------------------------------ +// Benchmark implementations + +static void Hash64Int64(benchmark::State& state) { // NOLINT non-const reference + auto test_vals = hashing_rng.Int64(10000, 0, std::numeric_limits::max()); + + while (state.KeepRunning()) { + Datum hash_result = compute::CallFunction("hash64", {test_vals}).ValueOrDie(); + benchmark::DoNotOptimize(hash_result); + } + + state.SetBytesProcessed(state.iterations() * test_vals->length() * sizeof(int64_t)); + state.SetItemsProcessed(state.iterations() * test_vals->length()); +} + +// Shared by the Hash64StructWithStrings benchmarks below, which only differ in the +// string column's length range (args: min_strlen, max_strlen). +static void Hash64StructWithStrings( + benchmark::State& state) { // NOLINT non-const reference + ASSERT_OK_AND_ASSIGN(std::shared_ptr values_array, + MakeStructArray(10000, static_cast(state.range(0)), + static_cast(state.range(1)))); + + // 2nd column (index 1) is a string column, which has offset type of int32_t + ASSERT_OK_AND_ASSIGN(std::shared_ptr values_second, + values_array->GetFlattenedField(1)); + auto str_vals = std::static_pointer_cast(values_second); + int32_t total_string_size = str_vals->total_values_length(); + + while (state.KeepRunning()) { + Datum hash_result = compute::CallFunction("hash64", {values_array}).ValueOrDie(); + benchmark::DoNotOptimize(hash_result); + } + + state.SetBytesProcessed(state.iterations() * + ((values_array->length() * sizeof(int64_t)) + + (total_string_size) + + (values_array->length() * sizeof(int64_t)))); + state.SetItemsProcessed(state.iterations() * 3 * values_array->length()); +} + +static void Hash64ListInt64(benchmark::State& state) { // NOLINT non-const reference + constexpr int64_t test_size = 10000; + auto test_vals = hashing_rng.ArrayOf(list(int64()), test_size, null_prob); + + while (state.KeepRunning()) { + Datum hash_result = compute::CallFunction("hash64", {test_vals}).ValueOrDie(); + benchmark::DoNotOptimize(hash_result); + } + + state.SetItemsProcessed(state.iterations() * test_size); +} + +// Hashing a small slice of a much larger list array; child_data isn't sliced by +// ArrayData::Slice(), so this used to cost proportionally to the whole array. +static void Hash64ListInt64HeavilySliced( + benchmark::State& state) { // NOLINT non-const reference + constexpr int64_t total_size = 1000000; + constexpr int64_t slice_size = 100; + auto test_vals = hashing_rng.ArrayOf(list(int64()), total_size, null_prob); + auto sliced = test_vals->Slice(total_size / 2, slice_size); + + while (state.KeepRunning()) { + Datum hash_result = compute::CallFunction("hash64", {sliced}).ValueOrDie(); + benchmark::DoNotOptimize(hash_result); + } + + state.SetItemsProcessed(state.iterations() * slice_size); +} + +// Unlike lists (see Hash64ListInt64HeavilySliced), binary-like arrays scale with the +// slice, not the underlying array: KeyColumnArray::Slice() narrows the offsets buffer +// itself, so there's no unsliced "child" to worry about. +static void Hash64StringHeavilySliced( + benchmark::State& state) { // NOLINT non-const reference + constexpr int64_t total_size = 1000000; + constexpr int64_t slice_size = 100; + auto test_vals = hashing_rng.String(total_size, 2, 20, null_prob); + auto sliced = test_vals->Slice(total_size / 2, slice_size); + + while (state.KeepRunning()) { + Datum hash_result = compute::CallFunction("hash64", {sliced}).ValueOrDie(); + benchmark::DoNotOptimize(hash_result); + } + + state.SetItemsProcessed(state.iterations() * slice_size); +} + +// Same idea as Hash64ListInt64HeavilySliced, but for a nested field within a struct: +// StructArray::Slice() also doesn't reslice child_data, so hashing a small slice of a +// struct with a nested list field used to cost proportionally to the whole array. +static void Hash64StructWithNestedListHeavilySliced( + benchmark::State& state) { // NOLINT non-const reference + constexpr int64_t total_size = 1000000; + constexpr int64_t slice_size = 100; + auto lists = hashing_rng.ArrayOf(list(int64()), total_size, null_prob); + ASSERT_OK_AND_ASSIGN(auto struct_arr, + StructArray::Make({lists}, {field("f0", list(int64()))})); + auto sliced = struct_arr->Slice(total_size / 2, slice_size); + + while (state.KeepRunning()) { + Datum hash_result = compute::CallFunction("hash64", {sliced}).ValueOrDie(); + benchmark::DoNotOptimize(hash_result); + } + + state.SetItemsProcessed(state.iterations() * slice_size); +} + +static void Hash64Map(benchmark::State& state) { // NOLINT non-const reference + constexpr int64_t test_size = 10000; + auto test_keys = hashing_rng.String(test_size, 2, 20, /*null_probability=*/0); + auto test_vals = hashing_rng.Int64(test_size, 0, std::numeric_limits::max()); + auto test_keyvals = hashing_rng.Map(test_keys, test_vals, test_size); + + auto key_arr = std::static_pointer_cast(test_keys); + int32_t total_key_size = key_arr->total_values_length(); + int32_t total_val_size = test_size * sizeof(int64_t); + + while (state.KeepRunning()) { + Datum hash_result = compute::CallFunction("hash64", {test_keyvals}).ValueOrDie(); + benchmark::DoNotOptimize(hash_result); + } + + state.SetBytesProcessed(state.iterations() * (total_key_size + total_val_size)); + state.SetItemsProcessed(state.iterations() * 2 * test_size); +} + +// ------------------------------ +// Benchmark declarations + +// Uses "FastHash" compute functions (wraps KeyHash functions) +BENCHMARK(Hash64Int64); + +BENCHMARK(Hash64StructWithStrings)->Args({2, 20})->Args({20, 120})->Args({120, 2000}); + +BENCHMARK(Hash64Map); +BENCHMARK(Hash64ListInt64); +BENCHMARK(Hash64ListInt64HeavilySliced); +BENCHMARK(Hash64StringHeavilySliced); +BENCHMARK(Hash64StructWithNestedListHeavilySliced); + +} // namespace internal +} // namespace arrow diff --git a/cpp/src/arrow/compute/kernels/scalar_hash_test.cc b/cpp/src/arrow/compute/kernels/scalar_hash_test.cc new file mode 100644 index 000000000000..95a5bf8210a2 --- /dev/null +++ b/cpp/src/arrow/compute/kernels/scalar_hash_test.cc @@ -0,0 +1,1362 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include + +#include "arrow/array/builder_nested.h" +#include "arrow/array/builder_primitive.h" +#include "arrow/chunked_array.h" +#include "arrow/compute/api.h" +#include "arrow/compute/kernels/test_util_internal.h" +#include "arrow/compute/key_hash_internal.h" +#include "arrow/compute/util.h" +#include "arrow/result.h" +#include "arrow/status.h" +#include "arrow/testing/extension_type.h" +#include "arrow/testing/gtest_util.h" +#include "arrow/testing/matchers.h" +#include "arrow/testing/random.h" +#include "arrow/testing/util.h" +#include "arrow/util/bit_util.h" +#include "arrow/util/cpu_info.h" +#include "arrow/util/key_value_metadata.h" + +namespace arrow { +namespace compute { + +constexpr auto kSeed = 0x94378165; +constexpr auto kArrayLengths = {0, 50, 100}; +constexpr auto kNullProbabilities = {0.0, 0.5, 1.0}; + +class TestScalarHash : public ::testing::Test { + public: + template + void AssertHashesEqual(const std::shared_ptr& arr, Datum res, + std::vector exp) { + auto res_array = res.array(); + for (int64_t val_ndx = 0; val_ndx < arr->length(); ++val_ndx) { + if (arr->IsNull(val_ndx)) { + ASSERT_TRUE(res_array->IsNull(val_ndx)) + << "row " << val_ndx << " is null and should produce a null hash"; + } else { + ASSERT_TRUE(res_array->IsValid(val_ndx)) + << "row " << val_ndx << " is valid and should not produce a null hash"; + c_type actual_hash = res_array->GetValues(1)[val_ndx]; + ASSERT_EQ(exp[val_ndx], actual_hash); + } + } + } + + // Reference hash for valid rows only -- AssertHashesEqual never reads this vector's + // null-row entries, since a null row's validity (not its value) is what's checked + // there, and this raw HashFixed call doesn't handle nulls at all. + template + std::vector HashPrimitive(const std::shared_ptr& arr) { + std::vector hashes(arr->length()); + // Choose the Hasher type conditionally based on c_type + + if constexpr (std::is_same_v) { + Hashing64::HashFixed(false, static_cast(arr->length()), + arr->type()->bit_width() / 8, + arr->data()->GetValues(1), hashes.data()); + } else { + Hashing32::HashFixed(::arrow::internal::CpuInfo::GetInstance()->hardware_flags(), + false, static_cast(arr->length()), + arr->type()->bit_width() / 8, + arr->data()->GetValues(1), hashes.data(), nullptr); + } + + return hashes; + } + + template + std::vector HashBinaryLike(const std::shared_ptr& arr) { + std::vector hashes(arr->length()); + auto length = static_cast(arr->length()); + auto values = arr->data()->GetValues(2); + if constexpr (std::is_same_v) { + if (arr->type_id() == Type::LARGE_BINARY || arr->type_id() == Type::LARGE_STRING) { + Hashing64::HashVarLen(false, length, arr->data()->GetValues(1), values, + hashes.data()); + } else { + Hashing64::HashVarLen(false, length, arr->data()->GetValues(1), values, + hashes.data()); + } + } else { + auto hw_flags = ::arrow::internal::CpuInfo::GetInstance()->hardware_flags(); + if (arr->type_id() == Type::LARGE_BINARY || arr->type_id() == Type::LARGE_STRING) { + Hashing32::HashVarLen(hw_flags, false, length, + arr->data()->GetValues(1), values, hashes.data(), + nullptr); + } else { + Hashing32::HashVarLen(hw_flags, false, length, + arr->data()->GetValues(1), values, hashes.data(), + nullptr); + } + } + return hashes; + } + + void CheckDeterministic(const std::string& func, const std::shared_ptr& arr) { + // Check that the hash is deterministic between different runs + ASSERT_OK_AND_ASSIGN(Datum res1, CallFunction(func, {arr})); + ASSERT_OK_AND_ASSIGN(Datum res2, CallFunction(func, {arr})); + ValidateOutput(res1); + ValidateOutput(res2); + ASSERT_EQ(res1.length(), arr->length()); + ASSERT_EQ(res2.length(), arr->length()); + if (func == "hash64") { + ASSERT_EQ(res1.type()->id(), Type::UINT64); + } else if (func == "hash32") { + ASSERT_EQ(res1.type()->id(), Type::UINT32); + } else { + FAIL() << "Unknown function: " << func; + } + AssertDatumsEqual(res1, res2); + + // Check that slicing the array does not affect the hash + auto hashes = res1.make_array(); + if (arr->length() >= 1) { + auto in1 = arr->Slice(1); + ASSERT_OK_AND_ASSIGN(Datum out1, CallFunction(func, {in1})); + ValidateOutput(out1); + AssertArraysEqual(*out1.make_array(), *hashes->Slice(1)); + } + if (arr->length() >= 4) { + auto in2 = arr->Slice(2, 2); + ASSERT_OK_AND_ASSIGN(Datum out2, CallFunction(func, {in2})); + ValidateOutput(out2); + AssertArraysEqual(*out2.make_array(), *hashes->Slice(2, 2)); + } + } + + void CheckHashQuality(const std::string& func, const std::shared_ptr& arr, + double tolerance = 1.0) { + ASSERT_OK_AND_ASSIGN(Datum result, CallFunction(func, {arr})); + auto hashes = result.make_array(); + + auto expected = arr->length(); + if (arr->null_count()) { + expected -= (arr->null_count() - 1); + } + if (func == "hash64") { + auto hashes64 = dynamic_cast(hashes.get()); + std::unordered_set hash_set; + for (int64_t i = 0; i < hashes64->length(); ++i) { + hash_set.insert(hashes64->Value(i)); + } + ASSERT_LE(hash_set.size(), expected); + ASSERT_GE(hash_set.size(), expected * tolerance); + } else if (func == "hash32") { + auto hashes32 = dynamic_cast(hashes.get()); + std::unordered_set hash_set; + for (int64_t i = 0; i < hashes32->length(); ++i) { + // Read the raw value regardless of validity: every null row still stores a + // deterministic 0 internally, so nulls collapse into exactly one shared bucket + // here, matching `expected`'s `null_count - 1` above (same as hash64 below). + hash_set.insert(hashes32->Value(i)); + } + ASSERT_LE(hash_set.size(), expected); + ASSERT_GE(hash_set.size(), expected * tolerance); + } else { + FAIL() << "Unknown function: " << func; + } + } + + void CheckPrimitive(const std::string& func, const std::shared_ptr& arr) { + ASSERT_OK_AND_ASSIGN(Datum hash_result, CallFunction(func, {arr})); + CheckDeterministic(func, arr); + if (func == "hash64") { + AssertHashesEqual(arr, hash_result, HashPrimitive(arr)); + } else if (func == "hash32") { + AssertHashesEqual(arr, hash_result, HashPrimitive(arr)); + } else { + FAIL() << "Unknown function: " << func; + } + } + + void CheckBinary(const std::string& func, const std::shared_ptr& arr) { + ASSERT_OK_AND_ASSIGN(Datum hash_result, CallFunction(func, {arr})); + CheckDeterministic(func, arr); + if (func == "hash64") { + AssertHashesEqual(arr, hash_result, HashBinaryLike(arr)); + } else if (func == "hash32") { + AssertHashesEqual(arr, hash_result, HashBinaryLike(arr)); + } else { + FAIL() << "Unknown function: " << func; + } + } + + // hash32/hash64 decode dictionaries to their logical values before hashing (rather + // than hashing the index buffer directly), so the result must match hashing the + // plain decoded array. + void CheckDictionary(const std::string& func, const std::shared_ptr& dict) { + CheckDeterministic(func, dict); + ASSERT_OK_AND_ASSIGN(Datum decoded, CallFunction("dictionary_decode", {dict})); + ASSERT_OK_AND_ASSIGN(Datum dict_hash, CallFunction(func, {dict})); + ASSERT_OK_AND_ASSIGN(Datum decoded_hash, CallFunction(func, {decoded})); + AssertDatumsEqual(dict_hash, decoded_hash); + } +}; + +TEST_F(TestScalarHash, Null) { + Datum res; + std::shared_ptr arr; + std::shared_ptr exp; + + arr = ArrayFromJSON(null(), R"([])"); + exp = ArrayFromJSON(uint32(), "[]"); + ASSERT_OK_AND_ASSIGN(res, CallFunction("hash32", {arr})); + AssertArraysEqual(*res.make_array(), *exp); + CheckDeterministic("hash32", arr); + + arr = ArrayFromJSON(null(), R"([])"); + exp = ArrayFromJSON(uint64(), "[]"); + ASSERT_OK_AND_ASSIGN(res, CallFunction("hash64", {arr})); + AssertArraysEqual(*res.make_array(), *exp); + CheckDeterministic("hash64", arr); + + arr = ArrayFromJSON(null(), R"([null, null, null])"); + exp = ArrayFromJSON(uint32(), "[null, null, null]"); + ASSERT_OK_AND_ASSIGN(res, CallFunction("hash32", {arr})); + AssertArraysEqual(*res.make_array(), *exp); + CheckDeterministic("hash32", arr); + + arr = ArrayFromJSON(null(), R"([null, null, null])"); + exp = ArrayFromJSON(uint64(), "[null, null, null]"); + ASSERT_OK_AND_ASSIGN(res, CallFunction("hash64", {arr})); + AssertArraysEqual(*res.make_array(), *exp); + CheckDeterministic("hash64", arr); +} + +TEST_F(TestScalarHash, NullProducesNull) { + auto arr1 = ArrayFromJSON(int32(), R"([null, 0, 1])"); + ASSERT_OK_AND_ASSIGN(auto res1, CallFunction("hash64", {arr1})); + auto res1_array = res1.array(); + auto buf1 = res1_array->GetValues(1); + ASSERT_TRUE(res1_array->IsNull(0)); + ASSERT_TRUE(res1_array->IsValid(1)); + ASSERT_TRUE(res1_array->IsValid(2)); + ASSERT_NE(buf1[1], buf1[2]); + + auto arr2 = ArrayFromJSON(int8(), R"([null, 0, 1])"); + ASSERT_OK_AND_ASSIGN(auto res2, CallFunction("hash32", {arr2})); + auto res2_array = res2.array(); + auto buf2 = res2_array->GetValues(1); + ASSERT_TRUE(res2_array->IsNull(0)); + ASSERT_TRUE(res2_array->IsValid(1)); + ASSERT_TRUE(res2_array->IsValid(2)); + ASSERT_NE(buf2[1], buf2[2]); +} + +// HashIntImp (used for any fixed-width type whose byte width is a power of 2 up to 8: +// ints, floats, dates, times, timestamps, durations) doesn't special-case an +// all-zero-bits key, so a legitimately valid "zero" value hashes to a raw 0 -- same as +// HashMultiColumn's own null handling would produce for an actually-null row. That's +// fine: nullness is tracked via real, independent validity (see HashArray), not by +// avoiding any particular hash value, so a valid row landing on 0 is just an ordinary +// (if slightly more likely) hash collision, not a correctness problem. What must still +// hold is that such a row is reported valid, not null. Checked across every affected +// byte width, not just int8/int32 (see NullProducesNull). +TEST_F(TestScalarHash, ZeroValueIsValid) { + std::vector, std::string>> cases{ + {int8(), R"([null, 0, 1])"}, + {int16(), R"([null, 0, 1])"}, + {int32(), R"([null, 0, 1])"}, + {int64(), R"([null, 0, 1])"}, + {uint8(), R"([null, 0, 1])"}, + {uint16(), R"([null, 0, 1])"}, + {uint32(), R"([null, 0, 1])"}, + {uint64(), R"([null, 0, 1])"}, + {float32(), R"([null, 0.0, 1.0])"}, + {float64(), R"([null, 0.0, 1.0])"}, + {date32(), R"([null, 0, 1])"}, + {date64(), R"([null, 0, 86400000])"}, + {time32(TimeUnit::SECOND), R"([null, 0, 1])"}, + {time64(TimeUnit::NANO), R"([null, 0, 1])"}, + {timestamp(TimeUnit::SECOND), R"([null, 0, 1])"}, + {duration(TimeUnit::MILLI), R"([null, 0, 1])"}, + }; + for (const std::string func : {"hash32", "hash64"}) { + for (const auto& type_and_json : cases) { + auto arr = ArrayFromJSON(type_and_json.first, type_and_json.second); + ASSERT_OK_AND_ASSIGN(Datum result, CallFunction(func, {arr})); + auto hashes = result.make_array(); + ASSERT_OK_AND_ASSIGN(auto null_hash, hashes->GetScalar(0)); + ASSERT_OK_AND_ASSIGN(auto zero_hash, hashes->GetScalar(1)); + ASSERT_OK_AND_ASSIGN(auto one_hash, hashes->GetScalar(2)); + ASSERT_FALSE(null_hash->is_valid) << type_and_json.first->ToString(); + ASSERT_TRUE(zero_hash->is_valid) << type_and_json.first->ToString(); + ASSERT_TRUE(one_hash->is_valid) << type_and_json.first->ToString(); + ASSERT_FALSE(zero_hash->Equals(*one_hash)) << type_and_json.first->ToString(); + } + } +} + +TEST_F(TestScalarHash, Boolean) { + Datum result; + std::shared_ptr array; + auto input = ArrayFromJSON(boolean(), R"([true, false, null, true, null, false])"); + CheckDeterministic("hash32", input); + CheckDeterministic("hash64", input); + + ASSERT_OK_AND_ASSIGN(result, CallFunction("hash32", {input})); + + array = result.make_array(); + auto array32 = checked_cast(array.get()); + ASSERT_TRUE(array32->IsValid(0)); + ASSERT_TRUE(array32->IsValid(1)); + ASSERT_TRUE(array32->IsNull(2)); + ASSERT_NE(array32->Value(0), array32->Value(1)); + ASSERT_NE(array32->Value(0), array32->Value(2)); + ASSERT_NE(array32->Value(1), array32->Value(2)); + ASSERT_EQ(array32->Value(0), array32->Value(3)); + ASSERT_EQ(array32->Value(2), array32->Value(4)); + ASSERT_EQ(array32->Value(1), array32->Value(5)); + + ASSERT_OK_AND_ASSIGN(result, CallFunction("hash64", {input})); + array = result.make_array(); + auto array64 = checked_cast(array.get()); + ASSERT_TRUE(array64->IsValid(0)); + ASSERT_TRUE(array64->IsValid(1)); + ASSERT_TRUE(array64->IsNull(2)); + ASSERT_NE(array64->Value(0), array64->Value(1)); + ASSERT_NE(array64->Value(0), array64->Value(2)); + ASSERT_NE(array64->Value(1), array64->Value(2)); + ASSERT_EQ(array64->Value(0), array64->Value(3)); + ASSERT_EQ(array64->Value(2), array64->Value(4)); + ASSERT_EQ(array64->Value(1), array64->Value(5)); +} + +TEST_F(TestScalarHash, Primitive) { + auto types = {int8(), + int16(), + int32(), + int64(), + uint8(), + uint16(), + uint32(), + uint64(), + float16(), + float32(), + float64(), + time32(TimeUnit::SECOND), + time64(TimeUnit::NANO), + date32(), + date64(), + timestamp(TimeUnit::SECOND), + duration(TimeUnit::MILLI)}; + + for (auto func : {"hash32", "hash64"}) { + for (auto type : types) { + CheckPrimitive(func, ArrayFromJSON(type, R"([])")); + CheckPrimitive(func, ArrayFromJSON(type, R"([null])")); + CheckPrimitive(func, ArrayFromJSON(type, R"([1])")); + CheckPrimitive(func, ArrayFromJSON(type, R"([1, 2])")); + CheckPrimitive(func, ArrayFromJSON(type, R"([1, 2, null])")); + CheckPrimitive(func, ArrayFromJSON(type, R"([null, 2, 3])")); + CheckPrimitive(func, ArrayFromJSON(type, R"([1, 2, 3, 4])")); + } + } +} + +TEST_F(TestScalarHash, BinaryLike) { + auto types = {binary(), utf8(), large_binary(), large_utf8()}; + for (auto func : {"hash32", "hash64"}) { + for (auto type : types) { + CheckBinary(func, ArrayFromJSON(type, R"([])")); + CheckBinary(func, ArrayFromJSON(type, R"([null])")); + CheckBinary(func, ArrayFromJSON(type, R"([""])")); + CheckBinary(func, ArrayFromJSON(type, R"(["first", "second", null])")); + CheckBinary(func, ArrayFromJSON(type, R"(["first", "second", "third"])")); + CheckBinary(func, ArrayFromJSON(type, R"(["first", "second", "third"])")); + } + } + for (auto func : {"hash32", "hash64"}) { + auto type = fixed_size_binary(1); + CheckPrimitive(func, ArrayFromJSON(type, R"([])")); + CheckPrimitive(func, ArrayFromJSON(type, R"([null])")); + CheckPrimitive(func, ArrayFromJSON(type, R"(["a", "b"])")); + CheckPrimitive(func, ArrayFromJSON(type, R"([null, "b"])")); + + type = fixed_size_binary(3); + CheckPrimitive(func, ArrayFromJSON(type, R"([])")); + CheckPrimitive(func, ArrayFromJSON(type, R"([null])")); + CheckPrimitive(func, ArrayFromJSON(type, R"(["alt", "blt"])")); + CheckPrimitive(func, ArrayFromJSON(type, R"([null, "blt"])")); + } +} + +TEST_F(TestScalarHash, ExtensionType) { + auto storage = ArrayFromJSON(int16(), R"([1, 2, 3, 4, null])"); + auto extension = ExtensionType::WrapArray(smallint(), storage); + CheckPrimitive("hash32", extension); + CheckPrimitive("hash64", extension); +} + +TEST_F(TestScalarHash, DictionaryType) { + auto dict_type = dictionary(int8(), utf8()); + auto dict = DictArrayFromJSON(dict_type, "[1, 2, null, 3, 0]", + "[\"A0\", \"A1\", \"C2\", \"C3\"]"); + CheckDictionary("hash32", dict); + CheckDictionary("hash64", dict); +} + +TEST_F(TestScalarHash, DictionaryNullValueProducesNull) { + // A valid index pointing at a null dictionary entry (legal -- see the comment on + // ArrayData::IsNull) must produce a null in the output like any other null row, even + // though the index's own validity bit is set. + auto dict_type = dictionary(int8(), utf8()); + auto dict = DictArrayFromJSON(dict_type, "[0, 1]", "[null, \"A1\"]"); + + for (const std::string func : {"hash32", "hash64"}) { + ASSERT_OK_AND_ASSIGN(Datum result, CallFunction(func, {dict})); + auto result_array = result.array(); + ASSERT_TRUE(result_array->IsNull(0)); + ASSERT_TRUE(result_array->IsValid(1)); + } +} + +TEST_F(TestScalarHash, DictionaryHashIndependentOfDictionaryLayout) { + // Two dictionary arrays encoding the same logical values via differently-ordered + // dictionaries must hash identically -- the hash reflects logical value, not index. + auto dict_type = dictionary(int8(), utf8()); + auto dict1 = DictArrayFromJSON(dict_type, "[0, 1, 2]", "[\"A\", \"B\", \"C\"]"); + auto dict2 = DictArrayFromJSON(dict_type, "[2, 1, 0]", "[\"C\", \"B\", \"A\"]"); + + ASSERT_OK_AND_ASSIGN(Datum hash1, CallFunction("hash64", {dict1})); + ASSERT_OK_AND_ASSIGN(Datum hash2, CallFunction("hash64", {dict2})); + AssertDatumsEqual(hash1, hash2); +} + +TEST_F(TestScalarHash, RandomBinaryLike) { + auto rand = random::RandomArrayGenerator(kSeed); + auto types = {binary(), utf8(), large_binary(), large_utf8()}; + + for (auto length : kArrayLengths) { + for (auto null_probability : kNullProbabilities) { + for (auto type : types) { + auto arr = rand.ArrayOf(type, length, null_probability); + CheckBinary("hash32", arr); + CheckBinary("hash64", arr); + } + for (auto type : {fixed_size_binary(1), fixed_size_binary(3)}) { + auto arr = rand.ArrayOf(type, length, null_probability); + CheckPrimitive("hash32", arr); + CheckPrimitive("hash64", arr); + } + auto arr = rand.ArrayOf(fixed_size_binary(0), length, null_probability); + CheckDeterministic("hash32", arr); + CheckDeterministic("hash64", arr); + } + } +} + +// A zero-width fixed_size_binary holds no data, so every value is the same empty byte +// string and every row must hash identically. ToColumnArray can only describe it as a +// fixed-width column of length 0 -- indistinguishable from a bit-packed boolean -- so +// HashMultiColumn used to hash each row from a nonexistent bit, producing uninitialized +// garbage that varied per row and per slice. Only reachable via a dictionary once +// dictionaries started being decoded, but broken for the plain type all along. +TEST_F(TestScalarHash, ZeroWidthFixedSizeBinaryRowsHashEqually) { + auto type = fixed_size_binary(0); + auto arr = ArrayFromJSON(type, R"(["", "", "", ""])"); + auto dict = DictArrayFromJSON(dictionary(int8(), type), "[0, 0, 0, 0]", R"([""])"); + + for (const std::string func : {"hash32", "hash64"}) { + for (const auto& input : {arr, dict}) { + ASSERT_OK_AND_ASSIGN(Datum result, CallFunction(func, {input})); + auto hashes = result.make_array(); + ASSERT_OK_AND_ASSIGN(auto first, hashes->GetScalar(0)); + for (int64_t i = 1; i < hashes->length(); i++) { + ASSERT_OK_AND_ASSIGN(auto other, hashes->GetScalar(i)); + ASSERT_TRUE(first->Equals(*other)) + << "row " << i << " of " << input->type()->ToString() + << " holds the same empty value as row 0 and must hash the same"; + } + // Hashing a slice must agree with slicing the hash (the garbage-bit read above + // depended on the row's absolute bit offset, so it did not). + auto sliced = input->Slice(2, 2); + ASSERT_OK_AND_ASSIGN(Datum sliced_result, CallFunction(func, {sliced})); + AssertArraysEqual(*sliced_result.make_array(), *hashes->Slice(2, 2)); + } + } +} + +TEST_F(TestScalarHash, RandomPrimitive) { + auto rand = random::RandomArrayGenerator(kSeed); + auto types = {int8(), + int16(), + int32(), + int64(), + uint8(), + uint16(), + uint32(), + uint64(), + float16(), + float32(), + float64(), + decimal128(18, 5), + decimal256(38, 5), + time32(TimeUnit::SECOND), + time64(TimeUnit::NANO), + date32(), + date64(), + timestamp(TimeUnit::SECOND), + duration(TimeUnit::MILLI)}; + + for (auto type : types) { + for (auto length : kArrayLengths) { + for (auto null_probability : kNullProbabilities) { + auto arr = rand.ArrayOf(type, length, null_probability); + CheckPrimitive("hash32", arr); + CheckPrimitive("hash64", arr); + if (type->bit_width() >= 16) { + // The generated arrays are usually all-unique at these lengths, but + // RandomArrayGenerator's std::uniform_int_distribution is platform-defined + // (not just seed-defined), so an occasional incidental duplicate value -- + // and thus a duplicate hash, correctly -- is expected on some platforms + // (e.g. MinGW). A tighter tolerance would make this test flaky rather than + // meaningful; HashQuality below already covers hash quality rigorously + // using inputs that are unique by construction. + CheckHashQuality("hash32", arr, 0.9); + CheckHashQuality("hash64", arr, 0.9); + } + } + } + } +} + +TEST_F(TestScalarHash, RandomList) { + auto rand = random::RandomArrayGenerator(kSeed); + auto types = { + list(int32()), + list(float64()), + list(utf8()), + list(large_binary()), + large_list(int64()), + large_list(utf8()), + large_list(large_binary()), + list(boolean()), + list(list(int16())), + list(list(list(uint8()))), + fixed_size_list(int32(), 3), + }; + for (auto type : types) { + for (auto length : kArrayLengths) { + for (auto null_probability : kNullProbabilities) { + auto arr = rand.ArrayOf(type, length, null_probability); + CheckDeterministic("hash32", arr); + CheckDeterministic("hash64", arr); + } + } + } +} + +// GH-17211: hashing nested (list-like) child values reused the parent's element +// offsets directly as byte offsets into the hashed-child buffer, without +// rescaling by the width of the hashed code (4 bytes for hash32, 8 for hash64). +// This corrupted results in a way that depended on row position, so two +// occurrences of the exact same nested value at different rows would hash +// differently. +void CheckIdenticalRowsHashEqually(const std::string& func, + const std::shared_ptr& arr, int64_t row_a, + int64_t row_b) { + ASSERT_OK_AND_ASSIGN(Datum result, CallFunction(func, {arr})); + ASSERT_OK_AND_ASSIGN(auto scalar_a, result.make_array()->GetScalar(row_a)); + ASSERT_OK_AND_ASSIGN(auto scalar_b, result.make_array()->GetScalar(row_b)); + ASSERT_TRUE(scalar_a->Equals(*scalar_b)) + << "row " << row_a << " and row " << row_b << " have the same value in " + << arr->ToString() << " and should hash identically"; +} + +TEST_F(TestScalarHash, ListLikeDuplicateRowsHashEqually) { + for (const std::string func : {"hash32", "hash64"}) { + CheckIdenticalRowsHashEqually( + func, + ArrayFromJSON(fixed_size_list(int32(), 3), + "[[7, 8, 9], [100, 101, 102], [7, 8, 9], [200, 201, 202]]"), + 0, 2); + CheckIdenticalRowsHashEqually( + func, + ArrayFromJSON(list(int32()), + "[[7, 8, 9], [100, 101], [7, 8, 9], [200, 201, 202, 203]]"), + 0, 2); + CheckIdenticalRowsHashEqually( + func, + ArrayFromJSON(large_list(int32()), + "[[7, 8, 9], [100, 101], [7, 8, 9], [200, 201, 202, 203]]"), + 0, 2); + CheckIdenticalRowsHashEqually( + func, + ArrayFromJSON(list(list(int16())), + "[[[7, 8], [9]], [[1], [2, 3]], [[7, 8], [9]], [[4]]]"), + 0, 2); + CheckIdenticalRowsHashEqually( + func, + ArrayFromJSON( + map(utf8(), int32()), + R"([[["a", 1], ["b", 2]], [["c", 3]], [["a", 1], ["b", 2]], [["d", 4]]])"), + 0, 2); + CheckIdenticalRowsHashEqually( + func, + ArrayFromJSON( + struct_({field("f0", list(int32()))}), + R"([{"f0": [7, 8, 9]}, {"f0": [1, 2]}, {"f0": [7, 8, 9]}, {"f0": [4]}])"), + 0, 2); + } +} + +// Same as above, but with a large array and the duplicated rows far apart, as a +// stress test of the row-folding loop in HashArray's is_list_like branch beyond +// the handful of rows exercised above. +TEST_F(TestScalarHash, ListLikeDuplicateRowsFarApartHashEqually) { + constexpr int64_t kRowA = 10; + constexpr int64_t kRowB = 2 * util::MiniBatch::kMiniBatchLength + 10; + constexpr int64_t kLength = kRowB + 100; + + Int32Builder value_builder; + ListBuilder list_builder(default_memory_pool(), std::make_shared()); + auto* values = checked_cast(list_builder.value_builder()); + for (int64_t row = 0; row < kLength; row++) { + ASSERT_OK(list_builder.Append()); + int64_t content = row == kRowB ? kRowA : row; + ASSERT_OK(values->Append(static_cast(content))); + ASSERT_OK(values->Append(static_cast(content + 1))); + } + ASSERT_OK_AND_ASSIGN(auto arr, list_builder.Finish()); + + for (const std::string func : {"hash32", "hash64"}) { + CheckIdenticalRowsHashEqually(func, arr, kRowA, kRowB); + } +} + +// Guards against HashChild hashing the entire (unsliced) child values array instead +// of only the range referenced by this slice of the parent list/map array: since +// ArrayData::Slice() doesn't slice child_data, a small slice of a much larger list +// array must still hash identically to an equivalent, independently-built array. +TEST_F(TestScalarHash, ListLikeSliceOfLargerArrayMatchesIndependentArray) { + constexpr int64_t kTotalRows = 1000; + constexpr int64_t kSliceOffset = 137; + constexpr int64_t kSliceLength = 10; + + Int32Builder value_builder; + ListBuilder list_builder(default_memory_pool(), std::make_shared()); + auto* values = checked_cast(list_builder.value_builder()); + for (int64_t row = 0; row < kTotalRows; row++) { + ASSERT_OK(list_builder.Append()); + ASSERT_OK(values->Append(static_cast(row))); + ASSERT_OK(values->Append(static_cast(row + 1))); + } + ASSERT_OK_AND_ASSIGN(auto large_arr, list_builder.Finish()); + auto sliced = large_arr->Slice(kSliceOffset, kSliceLength); + + ListBuilder independent_builder(default_memory_pool(), + std::make_shared()); + auto* independent_values = + checked_cast(independent_builder.value_builder()); + for (int64_t row = kSliceOffset; row < kSliceOffset + kSliceLength; row++) { + ASSERT_OK(independent_builder.Append()); + ASSERT_OK(independent_values->Append(static_cast(row))); + ASSERT_OK(independent_values->Append(static_cast(row + 1))); + } + ASSERT_OK_AND_ASSIGN(auto independent_arr, independent_builder.Finish()); + + for (const std::string func : {"hash32", "hash64"}) { + ASSERT_OK_AND_ASSIGN(Datum sliced_result, CallFunction(func, {sliced})); + ASSERT_OK_AND_ASSIGN(Datum independent_result, CallFunction(func, {independent_arr})); + AssertDatumsEqual(sliced_result, independent_result); + } +} + +// Same as ListLikeSliceOfLargerArrayMatchesIndependentArray, but for FIXED_SIZE_LIST, +// which computes its referenced range via arithmetic (offset * list_size) rather than +// reading an offsets buffer, so it's a genuinely different code path worth covering +// on its own. +TEST_F(TestScalarHash, FixedSizeListSliceOfLargerArrayMatchesIndependentArray) { + constexpr int64_t kTotalRows = 1000; + constexpr int64_t kSliceOffset = 137; + constexpr int64_t kSliceLength = 10; + constexpr int32_t kListSize = 2; + + FixedSizeListBuilder list_builder(default_memory_pool(), + std::make_shared(), kListSize); + auto* values = checked_cast(list_builder.value_builder()); + for (int64_t row = 0; row < kTotalRows; row++) { + ASSERT_OK(list_builder.Append()); + ASSERT_OK(values->Append(static_cast(row))); + ASSERT_OK(values->Append(static_cast(row + 1))); + } + ASSERT_OK_AND_ASSIGN(auto large_arr, list_builder.Finish()); + auto sliced = large_arr->Slice(kSliceOffset, kSliceLength); + + FixedSizeListBuilder independent_builder(default_memory_pool(), + std::make_shared(), kListSize); + auto* independent_values = + checked_cast(independent_builder.value_builder()); + for (int64_t row = kSliceOffset; row < kSliceOffset + kSliceLength; row++) { + ASSERT_OK(independent_builder.Append()); + ASSERT_OK(independent_values->Append(static_cast(row))); + ASSERT_OK(independent_values->Append(static_cast(row + 1))); + } + ASSERT_OK_AND_ASSIGN(auto independent_arr, independent_builder.Finish()); + + for (const std::string func : {"hash32", "hash64"}) { + ASSERT_OK_AND_ASSIGN(Datum sliced_result, CallFunction(func, {sliced})); + ASSERT_OK_AND_ASSIGN(Datum independent_result, CallFunction(func, {independent_arr})); + AssertDatumsEqual(sliced_result, independent_result); + } +} + +// Guards against a real bug: LIST/LARGE_LIST/FIXED_SIZE_LIST/MAP computed rel_start as +// `offsets[0] - values.offset` and then passed `values.offset + rel_start` to +// HashChild -- the values.offset term canceled itself out, so it was never actually +// applied. This only manifests when `values` (or MAP's items) itself has a +// pre-existing nonzero offset independent of the parent array -- as opposed to the +// slicing tests above, which slice the *parent* and leave `values` at offset 0. A +// values/items child having its own offset is ordinary: e.g. ListArray::FromArrays +// called with an already-sliced values array. +TEST_F(TestScalarHash, ValuesChildWithOwnOffsetHashesCorrectly) { + auto base_values = ArrayFromJSON(int32(), "[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14]"); + auto sliced_values = base_values->Slice(5, 6); // offset=5, content [5,6,7,8,9,10] + ASSERT_GT(sliced_values->offset(), 0); + auto independent_values = ArrayFromJSON(int32(), "[5,6,7,8,9,10]"); + + auto offsets32 = ArrayFromJSON(int32(), "[0, 2, 4, 6]"); + auto offsets64 = ArrayFromJSON(int64(), "[0, 2, 4, 6]"); + + ASSERT_OK_AND_ASSIGN(auto list_with_offset, + ListArray::FromArrays(*offsets32, *sliced_values)); + ASSERT_OK_AND_ASSIGN(auto independent_list, + ListArray::FromArrays(*offsets32, *independent_values)); + + ASSERT_OK_AND_ASSIGN(auto large_list_with_offset, + LargeListArray::FromArrays(*offsets64, *sliced_values)); + ASSERT_OK_AND_ASSIGN(auto independent_large_list, + LargeListArray::FromArrays(*offsets64, *independent_values)); + + ASSERT_OK_AND_ASSIGN(auto fsl_with_offset, + FixedSizeListArray::FromArrays(sliced_values, 2)); + ASSERT_OK_AND_ASSIGN(auto independent_fsl, + FixedSizeListArray::FromArrays(independent_values, 2)); + + auto keys = ArrayFromJSON(utf8(), R"(["a", "b", "c", "d", "e", "f"])"); + ASSERT_OK_AND_ASSIGN(auto map_with_offset, + MapArray::FromArrays(offsets32, keys, sliced_values)); + ASSERT_OK_AND_ASSIGN(auto independent_map, + MapArray::FromArrays(offsets32, keys, independent_values)); + + std::vector, std::shared_ptr>> cases{ + {list_with_offset, independent_list}, + {large_list_with_offset, independent_large_list}, + {fsl_with_offset, independent_fsl}, + {map_with_offset, independent_map}, + }; + for (const std::string func : {"hash32", "hash64"}) { + for (const auto& with_offset_and_independent : cases) { + ASSERT_OK_AND_ASSIGN(Datum with_offset_result, + CallFunction(func, {with_offset_and_independent.first})); + ASSERT_OK_AND_ASSIGN(Datum independent_result, + CallFunction(func, {with_offset_and_independent.second})); + AssertDatumsEqual(with_offset_result, independent_result); + } + } +} + +void CheckRowsHashDifferently(const std::string& func, const std::shared_ptr& arr, + int64_t row_a, int64_t row_b) { + ASSERT_OK_AND_ASSIGN(Datum result, CallFunction(func, {arr})); + ASSERT_OK_AND_ASSIGN(auto scalar_a, result.make_array()->GetScalar(row_a)); + ASSERT_OK_AND_ASSIGN(auto scalar_b, result.make_array()->GetScalar(row_b)); + ASSERT_FALSE(scalar_a->Equals(*scalar_b)) + << "row " << row_a << " and row " << row_b << " have different values in " + << arr->ToString() << " and should (in practice) hash differently"; +} + +// Guards against a degenerate fold (e.g. one that ignores element order, or only +// looks at the first/last element) that would satisfy the "identical content hashes +// identically" tests above while still being a broken hash function. +TEST_F(TestScalarHash, ListLikeDistinctContentHashesDifferently) { + for (const std::string func : {"hash32", "hash64"}) { + // Reordering elements should (in practice) change the hash. + CheckRowsHashDifferently(func, ArrayFromJSON(list(int32()), "[[1, 2, 3], [3, 2, 1]]"), + 0, 1); + // Changing one element's value should (in practice) change the hash. + CheckRowsHashDifferently(func, ArrayFromJSON(list(int32()), "[[1, 2, 3], [1, 2, 4]]"), + 0, 1); + // A shorter list shouldn't be a prefix-consistent truncation of a longer one. + CheckRowsHashDifferently(func, ArrayFromJSON(list(int32()), "[[1, 2], [1, 2, 3]]"), 0, + 1); + // Swapping map values between keys should (in practice) change the hash. + CheckRowsHashDifferently( + func, + ArrayFromJSON(map(utf8(), int32()), + R"([[["a", 1], ["b", 2]], [["a", 2], ["b", 1]]])"), + 0, 1); + } +} + +// The seed used to fold a list-like row's child hashes together (see +// FastHashScalar::CombineRange) is deliberately not 0, so that an empty (but +// non-null) list doesn't collide with a null list, which produces a null in the +// output (see NullProducesNull). +TEST_F(TestScalarHash, ListLikeEmptyDiffersFromNull) { + for (const std::string func : {"hash32", "hash64"}) { + for (auto arr : { + ArrayFromJSON(list(int32()), "[[], null]"), + ArrayFromJSON(large_list(int32()), "[[], null]"), + ArrayFromJSON(map(utf8(), int32()), "[[], null]"), + }) { + ASSERT_OK_AND_ASSIGN(Datum result, CallFunction(func, {arr})); + auto hashes = result.make_array(); + ASSERT_TRUE(hashes->IsValid(0)) + << "hash of an empty " << arr->type()->ToString() << " should not be null"; + ASSERT_TRUE(hashes->IsNull(1)); + ASSERT_OK_AND_ASSIGN(auto empty_hash, hashes->GetScalar(0)); + ASSERT_OK_AND_ASSIGN(auto null_hash, hashes->GetScalar(1)); + ASSERT_FALSE(empty_hash->Equals(*null_hash)) + << "hash of an empty " << arr->type()->ToString() + << " should not collide with hash of a null one"; + } + } +} + +// Mirrors NullProducesNull, but for list-like types, whose null handling is a +// dedicated masking pass in HashArray's is_list_like branch rather than the +// generic path the other types go through. +TEST_F(TestScalarHash, ListLikeNullProducesNull) { + for (const std::string func : {"hash32", "hash64"}) { + for (auto arr : { + ArrayFromJSON(fixed_size_list(int32(), 2), "[null, [1, 2]]"), + ArrayFromJSON(list(int32()), "[null, [1, 2]]"), + ArrayFromJSON(large_list(int32()), "[null, [1, 2]]"), + ArrayFromJSON(map(utf8(), int32()), R"([null, [["a", 1]]])"), + }) { + ASSERT_OK_AND_ASSIGN(Datum result, CallFunction(func, {arr})); + auto hashes = result.make_array(); + ASSERT_TRUE(hashes->IsNull(0)) + << "null " << arr->type()->ToString() << " should produce a null hash"; + ASSERT_TRUE(hashes->IsValid(1)) + << "non-null " << arr->type()->ToString() << " should not produce a null hash"; + } + } +} + +// Per the columnar format spec, a null slot may have a positive slot length over +// undefined memory. Build a LIST array where the null row's offsets span 3 real +// (non-garbage, but logically "don't care") values instead of the canonical empty +// range, to make sure CombineRange's output for that row is still discarded by the +// masking pass rather than leaking into the result. +TEST_F(TestScalarHash, ListNullWithNonEmptyOffsetRangeProducesNull) { + auto offsets = ArrayFromJSON(int32(), "[0, 2, 5, 6]"); + auto values = ArrayFromJSON(int32(), "[10, 20, 30, 40, 50, 60]"); + ASSERT_OK_AND_ASSIGN(auto validity, AllocateEmptyBitmap(3)); + bit_util::SetBit(validity->mutable_data(), 0); + // Row 1 is null but its offset range [2, 5) is non-empty. + bit_util::SetBit(validity->mutable_data(), 2); + ASSERT_OK_AND_ASSIGN( + auto arr, ListArray::FromArrays(*offsets, *values, default_memory_pool(), validity, + /*null_count=*/1)); + ASSERT_TRUE(arr->IsNull(1)); + + for (const std::string func : {"hash32", "hash64"}) { + ASSERT_OK_AND_ASSIGN(Datum result, CallFunction(func, {arr})); + auto hashes = result.make_array(); + ASSERT_TRUE(hashes->IsNull(1)) + << "null row with a non-empty offset range should still produce a null hash"; + } +} + +// A null row's own validity bit makes it null in *this* array's output, but when the +// array is nested inside a parent list/struct the parent folds the child's hash VALUES +// into its combined hash -- and per the columnar spec a null slot's underlying bytes are +// undefined, so they may hold real-looking leftover data. Unless a null child row's hash +// value is canonicalized (see CanonicalizeInvalidHashes), a null element contributes that +// garbage and becomes indistinguishable from an element genuinely holding that data. +TEST_F(TestScalarHash, NestedNullElementDoesNotCollideWithRealContent) { + // list>: row 0 = [{f0: 7}], row 1 = [null], where the null struct's + // f0 slot also holds 7. The two rows are logically distinct and must not collide. + auto f0 = ArrayFromJSON(int32(), "[7, 7]"); + ASSERT_OK_AND_ASSIGN(auto struct_validity, AllocateEmptyBitmap(2)); + bit_util::SetBit(struct_validity->mutable_data(), 0); // row 0 valid, row 1 null + auto struct_data = ArrayData::Make(struct_({field("f0", int32())}), 2, + {struct_validity}, {f0->data()}, /*null_count=*/1); + auto structs = MakeArray(struct_data); + ASSERT_TRUE(structs->IsNull(1)); + ASSERT_OK_AND_ASSIGN( + auto struct_lists, + ListArray::FromArrays(*ArrayFromJSON(int32(), "[0, 1, 2]"), *structs)); + + // list>: same idea one level deeper -- the null inner list's offset range + // covers a real [10, 20] identical to the valid row's contents. + auto values = ArrayFromJSON(int32(), "[10, 20, 10, 20]"); + ASSERT_OK_AND_ASSIGN(auto inner_validity, AllocateEmptyBitmap(2)); + bit_util::SetBit(inner_validity->mutable_data(), 0); // inner row 0 valid, row 1 null + ASSERT_OK_AND_ASSIGN( + auto inner, ListArray::FromArrays(*ArrayFromJSON(int32(), "[0, 2, 4]"), *values, + default_memory_pool(), inner_validity, + /*null_count=*/1)); + ASSERT_TRUE(inner->IsNull(1)); + ASSERT_OK_AND_ASSIGN( + auto nested_lists, + ListArray::FromArrays(*ArrayFromJSON(int32(), "[0, 1, 2]"), *inner)); + + for (const std::string func : {"hash32", "hash64"}) { + for (const auto& arr : {struct_lists, nested_lists}) { + ASSERT_OK_AND_ASSIGN(Datum result, CallFunction(func, {arr})); + auto hashes = result.make_array(); + // Both outer rows are valid: a null *element* doesn't null out its container. + ASSERT_TRUE(hashes->IsValid(0)) << arr->type()->ToString(); + ASSERT_TRUE(hashes->IsValid(1)) << arr->type()->ToString(); + ASSERT_OK_AND_ASSIGN(auto real_content_hash, hashes->GetScalar(0)); + ASSERT_OK_AND_ASSIGN(auto null_element_hash, hashes->GetScalar(1)); + ASSERT_FALSE(real_content_hash->Equals(*null_element_hash)) + << "a " << arr->type()->ToString() << " row holding a null element should not " + << "hash the same as one holding that null slot's undefined leftover data"; + } + } +} + +// The generic path (bool, int, string, ...) zeroes nulls via HashMultiColumn, while +// list-like types are zeroed by HashArray's own is_list_like branch (see +// ListLikeNullProducesNull) and struct by recursing into per-field columns fed back +// into HashMultiColumn. Check they all agree (a null in the output), not just each +// individually hashing null to *something* self-consistent. +TEST_F(TestScalarHash, NullProducesNullAcrossTypes) { + for (const std::string func : {"hash32", "hash64"}) { + for (auto arr : { + ArrayFromJSON(boolean(), "[null]"), + ArrayFromJSON(int32(), "[null]"), + ArrayFromJSON(utf8(), "[null]"), + ArrayFromJSON(list(int32()), "[null]"), + ArrayFromJSON(struct_({field("f0", int32())}), "[null]"), + ArrayFromJSON(map(utf8(), int32()), "[null]"), + }) { + ASSERT_OK_AND_ASSIGN(Datum result, CallFunction(func, {arr})); + ASSERT_TRUE(result.make_array()->IsNull(0)) + << "null " << arr->type()->ToString() << " should produce a null hash, " + << "same as every other type"; + } + } +} + +// GH-17211: a nested (list-like or struct) field that is independently null within +// an otherwise-valid struct row must still produce a null in the output, same as a +// plain field. HashChild used to attach the *parent* struct's validity to the child +// hash buffer instead of the field's own, so an independently-null nested field's +// already-zeroed hash data got re-hashed via HashFixed as if it were ordinary +// (non-null) data, silently producing a non-null result instead. +TEST_F(TestScalarHash, NestedNullFieldWithinValidStructProducesNull) { + for (const std::string func : {"hash32", "hash64"}) { + // Plain (non-nested) null field, for comparison: already correct beforehand. + auto plain = ArrayFromJSON(struct_({field("f0", int32())}), R"([{"f0": null}])"); + ASSERT_OK_AND_ASSIGN(Datum plain_result, CallFunction(func, {plain})); + ASSERT_TRUE(plain_result.make_array()->IsNull(0)); + + for (auto nested : { + ArrayFromJSON(struct_({field("f0", list(int32()))}), R"([{"f0": null}])"), + ArrayFromJSON(struct_({field("f0", struct_({field("g0", int32())}))}), + R"([{"f0": null}])"), + }) { + ASSERT_OK_AND_ASSIGN(Datum nested_result, CallFunction(func, {nested})); + ASSERT_TRUE(nested_result.make_array()->IsNull(0)) + << "independently-null " << nested->type()->ToString() + << " field should produce a null hash, same as a plain null field"; + } + + // Same invariant, but for a struct with more than one field: HashMultiColumn only + // zeroes column 0's null rows outright, so this also exercises a null in a + // non-first column (see HashStructArray). + auto multi_field = struct_({field("f0", int64()), field("f1", int64())}); + for (auto row : {R"([{"f0": 5, "f1": null}])", R"([{"f0": null, "f1": 5}])"}) { + auto multi = ArrayFromJSON(multi_field, row); + ASSERT_OK_AND_ASSIGN(Datum multi_result, CallFunction(func, {multi})); + ASSERT_TRUE(multi_result.make_array()->IsNull(0)) + << "independently-null field of a multi-field struct (" << row + << ") should produce a null hash, same as a plain null field"; + } + } +} + +// A struct field of DICTIONARY type needs the same recursive (decode + validity) +// treatment HashArray gives dictionaries anywhere else -- before this was routed via +// NeedsRecursiveHash (rather than is_nested(), which DICTIONARY doesn't satisfy), a +// dictionary-typed field took the flat ToColumnArray path instead, which has no +// DICTIONARY case: it silently hashed the raw index buffer and only ever saw the +// index's own validity, missing a valid index pointing at a null dictionary value +// (legal -- see the comment on ArrayData::IsNull). Same bug class this session already +// fixed at the top level (see DictionaryNullValueProducesNull), but for a nested field. +TEST_F(TestScalarHash, StructFieldDictionaryNullValueProducesNull) { + auto dict_type = dictionary(int8(), utf8()); + auto dict = DictArrayFromJSON(dict_type, "[0, 1]", "[null, \"A1\"]"); + ASSERT_OK_AND_ASSIGN(auto struct_array, + StructArray::Make({dict}, {field("f0", dict_type)})); + + for (const std::string func : {"hash32", "hash64"}) { + ASSERT_OK_AND_ASSIGN(Datum result, CallFunction(func, {struct_array})); + auto result_array = result.array(); + ASSERT_TRUE(result_array->IsNull(0)) + << "a struct row whose only field is a valid index pointing at a null " + << "dictionary value should be null, same as any other independently-null field"; + ASSERT_TRUE(result_array->IsValid(1)); + } +} + +// HashStructArray's combined hash VALUE for an all-zero-bits field can legitimately be +// a raw 0 (see ZeroValueIsValid) -- but that must never affect the struct row's real, +// independently-computed validity (own validity AND every field's, see HashStructArray), +// which is what actually decides null-vs-valid here. A field that's independently null +// must still make the row invalid (see NestedNullFieldWithinValidStructProducesNull), +// so this checks both behaviors hold side by side rather than one regressing the other. +TEST_F(TestScalarHash, StructOfAllValidZerosDoesNotCollideWithNull) { + for (const std::string func : {"hash32", "hash64"}) { + // A single all-zero-bits field is exactly where HashMultiColumn's underlying + // fixed-width hash would otherwise produce a literal 0 for a valid row. + auto valid_zero = ArrayFromJSON(struct_({field("f0", int64())}), R"([{"f0": 0}])"); + ASSERT_OK_AND_ASSIGN(Datum valid_result, CallFunction(func, {valid_zero})); + ASSERT_TRUE(valid_result.make_array()->IsValid(0)) + << "a struct whose only field is a valid zero should not be indistinguishable " + << "from a null struct"; + + // A null struct and a null field still produce a null hash, unaffected by the above. + auto null_struct = ArrayFromJSON(struct_({field("f0", int64())}), R"([null])"); + ASSERT_OK_AND_ASSIGN(Datum null_result, CallFunction(func, {null_struct})); + ASSERT_TRUE(null_result.make_array()->IsNull(0)); + + auto null_field = ArrayFromJSON(struct_({field("f0", int64())}), R"([{"f0": null}])"); + ASSERT_OK_AND_ASSIGN(Datum null_field_result, CallFunction(func, {null_field})); + ASSERT_TRUE(null_field_result.make_array()->IsNull(0)); + } +} + +// Guards against HashChild reusing a nested field's raw (unshifted) validity buffer +// without rebasing it: the buffer requires bit `child.offset + i` to read logical row +// i, but the returned ArrayData has offset 0 and its buffer is read directly (bit 0 = +// row 0) once wrapped in a KeyColumnArray. If a struct's nested field is itself an +// offset slice of a larger array (e.g. GH-17211), this misreads validity by +// `child.offset` bits -- here, a valid row would be misread as null (or vice versa) +// unless the buffer is rebased to be self-consistent with the fresh hash values. +TEST_F(TestScalarHash, NestedFieldWithOwnOffsetHashesCorrectly) { + ListBuilder list_builder(default_memory_pool(), std::make_shared()); + auto* values = checked_cast(list_builder.value_builder()); + ASSERT_OK(list_builder.AppendNull()); + for (int32_t row = 1; row < 10; row++) { + ASSERT_OK(list_builder.Append()); + ASSERT_OK(values->Append(row)); + ASSERT_OK(values->Append(row + 1)); + } + ASSERT_OK_AND_ASSIGN(auto base, list_builder.Finish()); + auto sliced_field = base->Slice(3, 5); // offset=3, length=5; logical row 0 = valid + + ASSERT_OK_AND_ASSIGN(auto struct_with_offset_field, + StructArray::Make({sliced_field}, {field("f0", list(int32()))})); + + ListBuilder independent_builder(default_memory_pool(), + std::make_shared()); + auto* independent_values = + checked_cast(independent_builder.value_builder()); + for (int32_t row = 3; row < 8; row++) { + ASSERT_OK(independent_builder.Append()); + ASSERT_OK(independent_values->Append(row)); + ASSERT_OK(independent_values->Append(row + 1)); + } + ASSERT_OK_AND_ASSIGN(auto independent_field, independent_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto independent_struct, + StructArray::Make({independent_field}, {field("f0", list(int32()))})); + + for (const std::string func : {"hash32", "hash64"}) { + ASSERT_OK_AND_ASSIGN(Datum offset_result, + CallFunction(func, {struct_with_offset_field})); + ASSERT_OK_AND_ASSIGN(Datum independent_result, + CallFunction(func, {independent_struct})); + AssertDatumsEqual(offset_result, independent_result); + } +} + +// Same idea as ListLikeSliceOfLargerArrayMatchesIndependentArray, but for a nested +// field within a struct: StructArray::Slice() also doesn't reslice child_data, so a +// small slice of a struct with a large nested list field must still hash identically +// to an equivalent, independently-built struct. +TEST_F(TestScalarHash, StructWithNestedFieldSliceOfLargerArrayMatchesIndependentArray) { + constexpr int64_t kTotalRows = 1000; + constexpr int64_t kSliceOffset = 137; + constexpr int64_t kSliceLength = 10; + + ListBuilder list_builder(default_memory_pool(), std::make_shared()); + auto* values = checked_cast(list_builder.value_builder()); + for (int64_t row = 0; row < kTotalRows; row++) { + ASSERT_OK(list_builder.Append()); + ASSERT_OK(values->Append(static_cast(row))); + ASSERT_OK(values->Append(static_cast(row + 1))); + } + ASSERT_OK_AND_ASSIGN(auto large_list, list_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto large_struct, + StructArray::Make({large_list}, {field("f0", list(int32()))})); + auto sliced = large_struct->Slice(kSliceOffset, kSliceLength); + + ListBuilder independent_builder(default_memory_pool(), + std::make_shared()); + auto* independent_values = + checked_cast(independent_builder.value_builder()); + for (int64_t row = kSliceOffset; row < kSliceOffset + kSliceLength; row++) { + ASSERT_OK(independent_builder.Append()); + ASSERT_OK(independent_values->Append(static_cast(row))); + ASSERT_OK(independent_values->Append(static_cast(row + 1))); + } + ASSERT_OK_AND_ASSIGN(auto independent_list, independent_builder.Finish()); + ASSERT_OK_AND_ASSIGN( + auto independent_struct, + StructArray::Make({independent_list}, {field("f0", list(int32()))})); + + for (const std::string func : {"hash32", "hash64"}) { + ASSERT_OK_AND_ASSIGN(Datum sliced_result, CallFunction(func, {sliced})); + ASSERT_OK_AND_ASSIGN(Datum independent_result, + CallFunction(func, {independent_struct})); + AssertDatumsEqual(sliced_result, independent_result); + } +} + +// The EXTENSION unwrapping at the top of HashArray should compose with the +// is_list_like recursion; this combination was otherwise untested (ExtensionType +// above only wraps a primitive). +TEST_F(TestScalarHash, ExtensionTypeWrappingList) { + auto storage = ArrayFromJSON(list(int32()), "[[7, 8, 9], [1, 2], [7, 8, 9]]"); + auto extension = ExtensionType::WrapArray(list_extension_type(), storage); + CheckIdenticalRowsHashEqually("hash32", extension, 0, 2); + CheckIdenticalRowsHashEqually("hash64", extension, 0, 2); +} + +// HashArray unwraps an extension by copying the ArraySpan and swapping only its `type` +// for the storage type, relying on the two having identical physical layout. That's +// subtlest when the storage is a DICTIONARY, because the dictionary branch then rebuilds +// an ArrayData via ArraySpan::ToArrayData(), which relocates child_data[0] into the +// ArrayData's dedicated `dictionary` field. Check the swap composes with that, and that +// unwrapping is fully transparent. +TEST_F(TestScalarHash, ExtensionTypeWrappingDictionary) { + auto storage = + DictArrayFromJSON(dictionary(int8(), utf8()), "[0, 1, null, 1]", R"(["a", "b"])"); + auto extension = ExtensionType::WrapArray(dict_extension_type(), storage); + + for (const std::string func : {"hash32", "hash64"}) { + ASSERT_OK_AND_ASSIGN(Datum extension_result, CallFunction(func, {extension})); + ASSERT_OK_AND_ASSIGN(Datum storage_result, CallFunction(func, {storage})); + AssertDatumsEqual(extension_result, storage_result); + + auto hashes = extension_result.make_array(); + ASSERT_TRUE(hashes->IsValid(0)); + ASSERT_TRUE(hashes->IsNull(2)); + // Rows 1 and 3 share a dictionary index, so they must hash equally. + CheckIdenticalRowsHashEqually(func, extension, 1, 3); + } +} + +TEST_F(TestScalarHash, RandomStruct) { + auto rand = random::RandomArrayGenerator(kSeed); + auto types = { + struct_({field("f0", int32())}), + struct_({field("f0", int32()), field("f1", utf8())}), + struct_({field("f0", list(int32()))}), + struct_({field("f0", struct_({field("f0", int32()), field("f1", utf8())}))}), + }; + for (auto type : types) { + for (auto length : kArrayLengths) { + for (auto null_probability : kNullProbabilities) { + auto arr = rand.ArrayOf(type, length, null_probability); + CheckDeterministic("hash32", arr); + CheckDeterministic("hash64", arr); + } + } + } +} + +// Guards against a struct field's own pre-existing offset (independent of the struct +// array's own offset) being silently ignored. StructArray::Slice() only touches the +// struct's own top-level offset -- child fields are not resliced (see +// StructArray::GetFlattenedField, which composes the struct's offset with each child's +// own offset) -- so a struct built from an already-offset field (e.g. a slice of a +// larger array) must still hash identically to an equivalent, independently-built +// struct with a zero-offset field. +TEST_F(TestScalarHash, StructFieldWithOwnOffsetHashesCorrectly) { + Int32Builder base_builder; + for (int32_t v = 0; v < 10; v++) { + ASSERT_OK(base_builder.Append(v)); + } + ASSERT_OK_AND_ASSIGN(auto base, base_builder.Finish()); + auto sliced_field = base->Slice(3, 4); // offset=3, length=4, content [3, 4, 5, 6] + ASSERT_GT(sliced_field->offset(), 0); + + Int32Builder second_builder; + for (int32_t v : {100, 101, 102, 103}) { + ASSERT_OK(second_builder.Append(v)); + } + ASSERT_OK_AND_ASSIGN(auto second_field, second_builder.Finish()); + + ASSERT_OK_AND_ASSIGN(auto struct_with_offset_field, + StructArray::Make({sliced_field, second_field}, + {field("f0", int32()), field("f1", int32())})); + + Int32Builder independent_builder; + for (int32_t v : {3, 4, 5, 6}) { + ASSERT_OK(independent_builder.Append(v)); + } + ASSERT_OK_AND_ASSIGN(auto independent_field, independent_builder.Finish()); + ASSERT_OK_AND_ASSIGN(auto independent_struct, + StructArray::Make({independent_field, second_field}, + {field("f0", int32()), field("f1", int32())})); + + for (const std::string func : {"hash32", "hash64"}) { + ASSERT_OK_AND_ASSIGN(Datum offset_result, + CallFunction(func, {struct_with_offset_field})); + ASSERT_OK_AND_ASSIGN(Datum independent_result, + CallFunction(func, {independent_struct})); + AssertDatumsEqual(offset_result, independent_result); + } +} + +// Guards against a crash on a zero-field struct: HashMultiColumn requires at least +// one column (it reads cols[0] unconditionally), so this type needs its own path in +// HashStructArray rather than falling through to HashMultiColumn with an empty list. +TEST_F(TestScalarHash, EmptyFieldStructHashesWithoutCrashing) { + auto type = struct_({}); + ASSERT_OK_AND_ASSIGN(auto validity, AllocateEmptyBitmap(2)); + bit_util::SetBit(validity->mutable_data(), 0); // row 0 valid, row 1 null + auto array_data = ArrayData::Make(type, 2, {validity}, /*null_count=*/1); + auto arr = MakeArray(array_data); + ASSERT_TRUE(arr->IsValid(0)); + ASSERT_TRUE(arr->IsNull(1)); + + for (const std::string func : {"hash32", "hash64"}) { + CheckDeterministic(func, arr); + ASSERT_OK_AND_ASSIGN(Datum result, CallFunction(func, {arr})); + auto hashes = result.make_array(); + ASSERT_TRUE(hashes->IsValid(0)); + ASSERT_TRUE(hashes->IsNull(1)); + } +} + +TEST_F(TestScalarHash, RandomMap) { + auto rand = random::RandomArrayGenerator(kSeed); + auto types = { + map(int32(), int32()), + map(int32(), utf8()), + map(utf8(), list(int16())), + map(utf8(), map(int32(), int32())), + }; + for (auto type : types) { + for (auto length : kArrayLengths) { + for (auto null_probability : kNullProbabilities) { + auto arr = rand.ArrayOf(type, length, null_probability); + CheckDeterministic("hash32", arr); + CheckDeterministic("hash64", arr); + } + } + } +} + +TEST_F(TestScalarHash, UnsupportedTypes) { + auto rand = random::RandomArrayGenerator(kSeed); + auto types = {list_view(int64()), + large_list_view(int64()), + binary_view(), + utf8_view(), + dense_union({field("a", int64()), field("b", binary())}), + sparse_union({field("a", int64()), field("b", binary())}), + run_end_encoded(int16(), utf8())}; + for (auto type : types) { + auto arr = rand.ArrayOf(type, 1, 0); + ASSERT_RAISES(NotImplemented, CallFunction("hash32", {arr})); + ASSERT_RAISES(NotImplemented, CallFunction("hash64", {arr})); + } +} + +// HashableMatcher only saw the top-level EXTENSION type id, so an extension wrapping +// an unsupported storage type (e.g. binary_view) passed dispatch and only failed +// later with a raw TypeError from ToColumnArray instead of a clean NotImplemented. +TEST_F(TestScalarHash, UnsupportedExtensionStorageType) { + auto storage = ArrayFromJSON(binary_view(), R"(["a", "b"])"); + auto extension = ExtensionType::WrapArray(binary_view_extension_type(), storage); + ASSERT_RAISES(NotImplemented, CallFunction("hash32", {extension})); + ASSERT_RAISES(NotImplemented, CallFunction("hash64", {extension})); +} + +// Same bug pattern as UnsupportedExtensionStorageType, but for dictionary support: +// HashableMatcher only saw the top-level DICTIONARY type id, so a dictionary wrapping +// an unsupported value type (e.g. binary_view) passed dispatch and only failed later +// with a raw TypeError from deep inside Cast/ToColumnArray instead of a clean +// NotImplemented. +TEST_F(TestScalarHash, UnsupportedDictionaryValueType) { + auto dict_type = dictionary(int8(), binary_view()); + auto dict = DictArrayFromJSON(dict_type, "[0, 1]", R"(["a", "b"])"); + ASSERT_RAISES(NotImplemented, CallFunction("hash32", {dict})); + ASSERT_RAISES(NotImplemented, CallFunction("hash64", {dict})); +} + +// copied from cpp/src/arrow/util/hashing_test.cc +template +static std::unordered_set MakeSequentialIntegers(int32_t n_values) { + std::unordered_set values; + values.reserve(n_values); + + for (int32_t i = 0; i < n_values; ++i) { + values.insert(static_cast(i)); + } + ARROW_DCHECK_EQ(values.size(), static_cast(n_values)); + return values; +} + +// copied from cpp/src/arrow/util/hashing_test.cc +static std::unordered_set MakeDistinctStrings(int32_t n_values) { + std::unordered_set values; + values.reserve(n_values); + + // Generate strings between 0 and 24 bytes, with ASCII characters + std::default_random_engine gen(42); + std::uniform_int_distribution length_dist(0, 24); + std::uniform_int_distribution char_dist('0', 'z'); + + while (values.size() < static_cast(n_values)) { + auto length = length_dist(gen); + std::string s(length, 'X'); + for (int32_t i = 0; i < length; ++i) { + s[i] = static_cast(char_dist(gen)); + } + values.insert(std::move(s)); + } + return values; +} + +TEST_F(TestScalarHash, HashQuality) { + for (auto& func : {"hash32", "hash64"}) { + std::shared_ptr arr; + auto integer_values = MakeSequentialIntegers(100000); + auto integer_vector = + std::vector(integer_values.begin(), integer_values.end()); + arrow::ArrayFromVector(integer_vector, &arr); + CheckHashQuality(func, arr); + + auto string_values = MakeDistinctStrings(10000); + auto string_vector = + std::vector(string_values.begin(), string_values.end()); + arrow::ArrayFromVector(string_vector, &arr); + CheckHashQuality(func, arr); + } +} + +} // namespace compute +} // namespace arrow diff --git a/cpp/src/arrow/compute/key_hash_benchmark.cc b/cpp/src/arrow/compute/key_hash_benchmark.cc new file mode 100644 index 000000000000..31b18b79c9ae --- /dev/null +++ b/cpp/src/arrow/compute/key_hash_benchmark.cc @@ -0,0 +1,119 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +#include +#include +#include +#include + +#include "benchmark/benchmark.h" + +#include "arrow/testing/gtest_util.h" +#include "arrow/testing/random.h" +#include "arrow/util/hashing.h" + +#include "arrow/array/builder_primitive.h" +#include "arrow/compute/key_hash_internal.h" +#include "arrow/compute/util_internal.h" + +namespace arrow { +namespace internal { + +namespace { +// copied from scalar_string_benchmark +constexpr auto kSeed = 0x94378165; + +static random::RandomArrayGenerator hashing_rng(kSeed); +} // namespace + +static void KeyHashIntegers32(benchmark::State& state) { // NOLINT non-const reference + auto test_vals = hashing_rng.Int32(10000, 0, std::numeric_limits::max()); + + // initialize the stack allocator + util::TempVectorStack stack_memallocator; + ASSERT_OK(stack_memallocator.Init(compute::default_exec_context()->memory_pool(), + compute::Hashing32::kHashBatchTempStackUsage)); + + // prepare the execution context for Hashing32 + compute::LightContext hash_ctx; + hash_ctx.hardware_flags = compute::default_exec_context()->cpu_info()->hardware_flags(); + hash_ctx.stack = &stack_memallocator; + + // allocate memory for results + ASSERT_OK_AND_ASSIGN(std::unique_ptr hash_buffer, + AllocateBuffer(test_vals->length() * sizeof(int32_t))); + + // Prepare input data structure for propagation to hash function + ASSERT_OK_AND_ASSIGN( + compute::KeyColumnArray input_keycol, + compute::ColumnArrayFromArrayData(test_vals->data(), 0, test_vals->length())); + std::vector columns{input_keycol}; + + // run the benchmark + while (state.KeepRunning()) { + compute::Hashing32::HashMultiColumn( + columns, &hash_ctx, reinterpret_cast(hash_buffer->mutable_data())); + } + + state.SetBytesProcessed(state.iterations() * test_vals->length() * sizeof(int32_t)); + state.SetItemsProcessed(state.iterations() * test_vals->length()); +} + +static void KeyHashIntegers64(benchmark::State& state) { // NOLINT non-const reference + auto test_vals = hashing_rng.Int64(10000, 0, std::numeric_limits::max()); + + // initialize the stack allocator + util::TempVectorStack stack_memallocator; + ASSERT_OK(stack_memallocator.Init(compute::default_exec_context()->memory_pool(), + compute::Hashing64::kHashBatchTempStackUsage)); + + // prepare the execution context for Hashing64 + compute::LightContext hash_ctx; + hash_ctx.hardware_flags = compute::default_exec_context()->cpu_info()->hardware_flags(); + hash_ctx.stack = &stack_memallocator; + + // allocate memory for results + ASSERT_OK_AND_ASSIGN(std::unique_ptr hash_buffer, + AllocateBuffer(test_vals->length() * sizeof(int64_t))); + + // Prepare input data structure for propagation to hash function + ASSERT_OK_AND_ASSIGN( + compute::KeyColumnArray input_keycol, + compute::ColumnArrayFromArrayData(test_vals->data(), 0, test_vals->length())); + std::vector columns{input_keycol}; + + // run the benchmark + while (state.KeepRunning()) { + compute::Hashing64::HashMultiColumn( + columns, &hash_ctx, reinterpret_cast(hash_buffer->mutable_data())); + } + + state.SetBytesProcessed(state.iterations() * test_vals->length() * sizeof(int64_t)); + state.SetItemsProcessed(state.iterations() * test_vals->length()); +} + +// ---------------------------------------------------------------------- +// Benchmark declarations + +// Directly uses "KeyHash" hash functions from key_hash.h (xxHash-like) +BENCHMARK(KeyHashIntegers32); +BENCHMARK(KeyHashIntegers64); + +} // namespace internal +} // namespace arrow diff --git a/cpp/src/arrow/compute/key_hash_internal.h b/cpp/src/arrow/compute/key_hash_internal.h index d141603ce0f6..54b709f1989b 100644 --- a/cpp/src/arrow/compute/key_hash_internal.h +++ b/cpp/src/arrow/compute/key_hash_internal.h @@ -37,6 +37,7 @@ enum class BloomFilterBuildStrategy; // class ARROW_COMPUTE_EXPORT Hashing32 { friend class TestVectorHash; + friend class TestScalarHash; template friend void TestBloomLargeHashHelper(int64_t, int64_t, const std::vector&, int64_t, int, T*); @@ -46,6 +47,13 @@ class ARROW_COMPUTE_EXPORT Hashing32 { static void HashMultiColumn(const std::vector& cols, LightContext* ctx, uint32_t* out_hash); + // Combine two hash values into one, e.g. to fold together the hashes of a nested + // column's child elements, or of a row's separate key columns (as HashMultiColumn + // does internally). + static uint32_t CombineHashes(uint32_t previous_hash, uint32_t hash) { + return CombineHashesImp(previous_hash, hash); + } + // Clarify the max temp stack usage for HashBatch, which might be necessary for the // caller to be aware of at compile time to reserve enough stack size in advance. The // HashBatch implementation uses one uint32 temp vector as a buffer for hash, one uint16 @@ -160,6 +168,7 @@ class ARROW_COMPUTE_EXPORT Hashing32 { class ARROW_COMPUTE_EXPORT Hashing64 { friend class TestVectorHash; + friend class TestScalarHash; template friend void TestBloomLargeHashHelper(int64_t, int64_t, const std::vector&, int64_t, int, T*); @@ -169,6 +178,13 @@ class ARROW_COMPUTE_EXPORT Hashing64 { static void HashMultiColumn(const std::vector& cols, LightContext* ctx, uint64_t* hashes); + // Combine two hash values into one, e.g. to fold together the hashes of a nested + // column's child elements, or of a row's separate key columns (as HashMultiColumn + // does internally). + static uint64_t CombineHashes(uint64_t previous_hash, uint64_t hash) { + return CombineHashesImp(previous_hash, hash); + } + // Clarify the max temp stack usage for HashBatch, which might be necessary for the // caller to be aware of at compile time to reserve enough stack size in advance. The // HashBatch implementation uses one uint16 temp vector as a buffer for null indices and diff --git a/cpp/src/arrow/compute/registry_internal.h b/cpp/src/arrow/compute/registry_internal.h index 5b9d7f8d608f..90165abba69c 100644 --- a/cpp/src/arrow/compute/registry_internal.h +++ b/cpp/src/arrow/compute/registry_internal.h @@ -30,6 +30,7 @@ void RegisterScalarBoolean(FunctionRegistry* registry); void RegisterScalarCast(FunctionRegistry* registry); void RegisterDictionaryDecode(FunctionRegistry* registry); void RegisterScalarComparison(FunctionRegistry* registry); +void RegisterScalarHash(FunctionRegistry* registry); void RegisterScalarIfElse(FunctionRegistry* registry); void RegisterScalarNested(FunctionRegistry* registry); void RegisterScalarRandom(FunctionRegistry* registry); // Nullary diff --git a/cpp/src/arrow/util/hashing_benchmark.cc b/cpp/src/arrow/util/hashing_benchmark.cc index c7051d1a3515..4d322cc1ecb3 100644 --- a/cpp/src/arrow/util/hashing_benchmark.cc +++ b/cpp/src/arrow/util/hashing_benchmark.cc @@ -24,6 +24,7 @@ #include "benchmark/benchmark.h" +#include "arrow/array/builder_primitive.h" #include "arrow/testing/gtest_util.h" #include "arrow/util/hashing.h" @@ -62,7 +63,22 @@ static std::vector MakeStrings(int32_t n_values, int32_t min_length return values; } -static void HashIntegers(benchmark::State& state) { // NOLINT non-const reference +static void HashIntegers32(benchmark::State& state) { // NOLINT non-const reference + const std::vector values = MakeIntegers(10000); + + while (state.KeepRunning()) { + hash_t total = 0; + for (const int32_t v : values) { + total += ScalarHelper::ComputeHash(v); + total += ScalarHelper::ComputeHash(v); + } + benchmark::DoNotOptimize(total); + } + state.SetBytesProcessed(2 * state.iterations() * values.size() * sizeof(int32_t)); + state.SetItemsProcessed(2 * state.iterations() * values.size()); +} + +static void HashIntegers64(benchmark::State& state) { // NOLINT non-const reference const std::vector values = MakeIntegers(10000); while (state.KeepRunning()) { @@ -114,7 +130,9 @@ static void HashLargeStrings(benchmark::State& state) { // NOLINT non-const ref // ---------------------------------------------------------------------- // Benchmark declarations -BENCHMARK(HashIntegers); +// Directly uses "Hashing" hash functions from hashing.h (xxHash) +BENCHMARK(HashIntegers32); +BENCHMARK(HashIntegers64); BENCHMARK(HashSmallStrings); BENCHMARK(HashMediumStrings); BENCHMARK(HashLargeStrings); diff --git a/docs/source/cpp/compute.rst b/docs/source/cpp/compute.rst index 1e067c52188d..5ad163802218 100644 --- a/docs/source/cpp/compute.rst +++ b/docs/source/cpp/compute.rst @@ -1282,6 +1282,27 @@ Containment tests * \(8) Output is true iff :member:`MatchSubstringOptions::pattern` matches the corresponding input element at any position. +Hash Functions +~~~~~~~~~~~~~~ + +Not to be confused with the "group by" functions, Hash functions produce an array of hash +values corresponding to the length of the input. Currently, these functions take a single +array as input. + ++---------------+-------+-------------+-------------+---------------+-------+ +| Function name | Arity | Input types | Output type | Options class | Notes | ++===============+=======+=============+=============+===============+=======+ +| hash32 | Unary | Any | UInt32 | | \(1) | ++---------------+-------+-------------+-------------+---------------+-------+ +| hash64 | Unary | Any | UInt64 | | \(1) | ++---------------+-------+-------------+-------------+---------------+-------+ + +* \(1) The implementation doesn't guarantee hash stability across different versions of + the library. Union, view and run end encoded types are not supported yet. A null + input value produces a null output value. For a struct, a field that is null makes + the whole struct row's output null; for a list or map, by contrast, a null element + does not, since only the row's own validity matters there. + Categorizations ~~~~~~~~~~~~~~~ diff --git a/python/pyarrow/tests/strategies.py b/python/pyarrow/tests/strategies.py index cb96f71e2627..65395a5a801c 100644 --- a/python/pyarrow/tests/strategies.py +++ b/python/pyarrow/tests/strategies.py @@ -213,18 +213,22 @@ def fields(draw, type_strategy=primitive_types, name_strategy=None): return pa.field(name, type=typ, nullable=nullable, metadata=meta) -def list_types(item_strategy=primitive_types): - return ( +def list_types(item_strategy=primitive_types, include_views=True): + types = ( st.builds(pa.list_, item_strategy) | st.builds(pa.large_list, item_strategy) | st.builds( pa.list_, item_strategy, st.integers(min_value=0, max_value=16) - ) | - st.builds(pa.list_view, item_strategy) | - st.builds(pa.large_list_view, item_strategy) + ) ) + if include_views: + types |= ( + st.builds(pa.list_view, item_strategy) | + st.builds(pa.large_list_view, item_strategy) + ) + return types @st.composite @@ -347,8 +351,7 @@ def arrays(draw, type, size=None, nullable=True): elif pa.types.is_timestamp(ty): if zoneinfo is None: pytest.skip('no module named zoneinfo (or tzdata on Windows)') - if ty.tz is None: - pytest.skip('requires timezone not None') + h.assume(ty.tz is not None) min_int64 = -(2**63) max_int64 = 2**63 - 1 min_datetime = datetime.datetime.fromtimestamp( @@ -421,7 +424,8 @@ def arrays(draw, type, size=None, nullable=True): value = st.one_of(st.none(), value) values = st.lists(value, min_size=size, max_size=size) - return pa.array(draw(values), type=ty) + actual_values = draw(values) + return pa.array(actual_values, type=ty) @st.composite diff --git a/python/pyarrow/tests/test_compute.py b/python/pyarrow/tests/test_compute.py index 1e08e73668e5..f31b1b05bd81 100644 --- a/python/pyarrow/tests/test_compute.py +++ b/python/pyarrow/tests/test_compute.py @@ -27,6 +27,8 @@ import random import sys import textwrap +import hypothesis as h +import hypothesis.strategies as st try: import numpy as np @@ -41,6 +43,7 @@ import pyarrow as pa import pyarrow.compute as pc from pyarrow.lib import ArrowNotImplementedError, ArrowIndexError +import pyarrow.tests.strategies as past try: import pyarrow.substrait as pas @@ -4338,3 +4341,92 @@ def test_winsorize(): result = pc.winsorize( arr, options=pc.WinsorizeOptions(lower_limit=0.1, upper_limit=0.8)) assert result.to_pylist() == [8, 4, 8, 8, 5, 3, 7, 2, 2, 6] + + +hash_types = st.deferred( + lambda: ( + past.primitive_types | + past.list_types(include_views=False) | + past.struct_types() | + past.dictionary_types() | + past.map_types() | + past.list_types(hash_types, include_views=False) | + past.struct_types(hash_types) + ) +) + + +def _contains_null(value): + # Whether a Python value produced by Array.as_py() is None, or (for a nested + # list/struct/map value) contains a None anywhere within it. + if value is None: + return True + if isinstance(value, dict): + return any(_contains_null(v) for v in value.values()) + if isinstance(value, (list, tuple)): + return any(_contains_null(v) for v in value) + return False + + +def _check_hash_quality(func, arr): + result1 = func(arr) + result2 = func(arr) + assert result1.equals(result2), "hashing must be deterministic" + + # Nullness is carried by the output's validity bitmap, not by any reserved hash + # value (see NullProducesNullAcrossTypes in scalar_hash_test.cc), so the hash of a + # valid row is unconstrained here -- 0 is a perfectly legal hash for one. + for i in range(len(arr)): + valid = result1[i].is_valid + if not arr[i].is_valid: + assert not valid, f"row {i} is null, so its hash must be null" + elif not _contains_null(arr[i].as_py()): + # No null anywhere in the row's content, so nothing can make it null. + assert valid, ( + f"row {i} ({arr[i].as_py()!r}) is fully valid but hashed to null") + # Otherwise the row is valid but holds a null somewhere: whether that nulls the + # hash depends on where (a null struct field does, a null list element does + # not), so this leaves it unconstrained -- scalar_hash_test.cc pins down both. + + +@pytest.mark.numpy +@h.given(past.arrays(hash_types)) +def test_hash32(arr): + result = pc.hash32(arr) + assert result.type == pa.uint32() + _check_hash_quality(pc.hash32, arr) + + +@pytest.mark.numpy +@h.given(past.arrays(hash_types)) +def test_hash64(arr): + result = pc.hash64(arr) + assert result.type == pa.uint64() + _check_hash_quality(pc.hash64, arr) + + +@st.composite +def hash_arrays_with_slice(draw): + arr = draw(past.arrays(hash_types)) + offset = draw(st.integers(min_value=0, max_value=len(arr))) + length = draw(st.integers(min_value=0, max_value=len(arr) - offset)) + return arr, offset, length + + +# Hashing a slice must match slicing the hash of the unsliced array: nested child +# arrays aren't resliced by Array.slice, so this exercises the offset handling for +# list/map/struct fields, not just plain values. +@pytest.mark.numpy +@h.given(hash_arrays_with_slice()) +def test_hash32_slice_consistency(args): + arr, offset, length = args + sliced = arr.slice(offset, length) + assert pc.hash32(sliced).equals(pc.hash32(arr).slice(offset, length)) + + +@pytest.mark.numpy +@h.given(hash_arrays_with_slice()) +def test_hash64_slice_consistency(args): + arr, offset, length = args + sliced = arr.slice(offset, length) + assert pc.hash64(sliced).equals(pc.hash64(arr).slice(offset, length)) diff --git a/r/tests/testthat/test-compute-aggregate.R b/r/tests/testthat/test-compute-aggregate.R index dd79682361bf..d624e35c3bdb 100644 --- a/r/tests/testthat/test-compute-aggregate.R +++ b/r/tests/testthat/test-compute-aggregate.R @@ -21,8 +21,11 @@ test_that("list_compute_functions", { justmins <- list_compute_functions("^min") expect_true(length(justmins) > 0) expect_all_true(grepl("min", justmins)) - no_hash_funcs <- list_compute_functions("^hash") + no_hash_funcs <- list_compute_functions("^hash_") expect_true(length(no_hash_funcs) == 0) + # hash32/hash64 are scalar functions (no underscore), unlike the hash_* + # group-by aggregations filtered out above, so they remain discoverable + expect_true(all(c("hash32", "hash64") %in% allfuncs)) }) test_that("sum.Array", {