From 43931b50eee0d282994c8d0114d0f92724181f50 Mon Sep 17 00:00:00 2001 From: likun Date: Sun, 5 Jul 2026 21:39:17 +0800 Subject: [PATCH 1/5] GH-50371: [C++][Gandiva] Fold common subexpressions --- cpp/src/gandiva/CMakeLists.txt | 1 + cpp/src/gandiva/engine.cc | 10 + cpp/src/gandiva/engine.h | 4 + cpp/src/gandiva/expr_cse.cc | 273 ++++++++++++++++++++++++ cpp/src/gandiva/expr_cse.h | 35 +++ cpp/src/gandiva/expr_decomposer.cc | 84 ++++++-- cpp/src/gandiva/expr_decomposer.h | 13 +- cpp/src/gandiva/filter.cc | 11 +- cpp/src/gandiva/llvm_generator.h | 1 + cpp/src/gandiva/projector.cc | 19 +- cpp/src/gandiva/projector.h | 2 + cpp/src/gandiva/tests/projector_test.cc | 224 +++++++++++++++++++ 12 files changed, 644 insertions(+), 33 deletions(-) create mode 100644 cpp/src/gandiva/expr_cse.cc create mode 100644 cpp/src/gandiva/expr_cse.h diff --git a/cpp/src/gandiva/CMakeLists.txt b/cpp/src/gandiva/CMakeLists.txt index aabe4ec8bf70..349e15e77995 100644 --- a/cpp/src/gandiva/CMakeLists.txt +++ b/cpp/src/gandiva/CMakeLists.txt @@ -65,6 +65,7 @@ set(SRC_FILES engine.cc date_utils.cc encrypt_utils.cc + expr_cse.cc expr_decomposer.cc expr_validator.cc expression.cc diff --git a/cpp/src/gandiva/engine.cc b/cpp/src/gandiva/engine.cc index 901421c86cb3..4cdc9238f678 100644 --- a/cpp/src/gandiva/engine.cc +++ b/cpp/src/gandiva/engine.cc @@ -543,6 +543,10 @@ Status Engine::FinalizeModule() { if (!cached_) { ARROW_RETURN_NOT_OK(RemoveUnusedFunctions()); + if (conf_->dump_ir()) { + unoptimized_module_ir_ = DumpModuleIR(*module_); + } + if (optimize_) { auto target_analysis = target_machine_->getTargetIRAnalysis(); // misc passes to allow for inlining, vectorization, .. @@ -615,4 +619,10 @@ const std::string& Engine::ir() { return module_ir_; } +const std::string& Engine::unoptimized_ir() { + DCHECK(!unoptimized_module_ir_.empty()) + << "dump_ir in Configuration must be set for dumping IR"; + return unoptimized_module_ir_; +} + } // namespace gandiva diff --git a/cpp/src/gandiva/engine.h b/cpp/src/gandiva/engine.h index 20165787cb66..2d4035db5b35 100644 --- a/cpp/src/gandiva/engine.h +++ b/cpp/src/gandiva/engine.h @@ -87,6 +87,9 @@ class GANDIVA_EXPORT Engine { /// Return the generated IR for the module. const std::string& ir(); + /// Return the generated IR before the optimizer pipeline runs. + const std::string& unoptimized_ir(); + /// Load the function IRs that can be accessed in the module. Status LoadFunctionIRs(); @@ -129,6 +132,7 @@ class GANDIVA_EXPORT Engine { bool cached_; bool functions_loaded_ = false; std::shared_ptr function_registry_; + std::string unoptimized_module_ir_; std::string module_ir_; // The lifetime of the TargetMachine is shared with LLJIT. This prevents unnecessary // duplication of this expensive object. diff --git a/cpp/src/gandiva/expr_cse.cc b/cpp/src/gandiva/expr_cse.cc new file mode 100644 index 000000000000..bf8524b0230a --- /dev/null +++ b/cpp/src/gandiva/expr_cse.cc @@ -0,0 +1,273 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "gandiva/expr_cse.h" + +#include +#include +#include +#include +#include +#include + +#include "gandiva/condition.h" +#include "gandiva/function_registry.h" +#include "gandiva/function_signature.h" +#include "gandiva/node.h" + +namespace gandiva { + +namespace { + +struct FoldedNode { + NodePtr node; + std::string key; + bool can_eliminate; +}; + +class CommonSubexpressionFolder { + public: + explicit CommonSubexpressionFolder(const FunctionRegistry& registry) + : registry_(registry) {} + + ExpressionPtr FoldExpression(const ExpressionPtr& expression) { + auto folded = Fold(expression->root()); + if (folded.node == expression->root()) { + return expression; + } + return std::make_shared(folded.node, expression->result()); + } + + ConditionPtr FoldCondition(const ConditionPtr& condition) { + auto folded = Fold(condition->root()); + if (folded.node == condition->root()) { + return condition; + } + return std::make_shared(folded.node); + } + + private: + FoldedNode Fold(const NodePtr& node) { + if (node == nullptr) { + return {nullptr, "null", false}; + } + + if (auto field_node = std::dynamic_pointer_cast(node)) { + auto key = "field:" + field_node->field()->ToString(); + return {Intern(key, node), std::move(key), true}; + } + + if (auto literal_node = std::dynamic_pointer_cast(node)) { + auto key = "literal:" + literal_node->ToString(); + return {Intern(key, node), std::move(key), true}; + } + + if (auto function_node = std::dynamic_pointer_cast(node)) { + return FoldFunction(node, *function_node); + } + + if (auto boolean_node = std::dynamic_pointer_cast(node)) { + return FoldBoolean(node, *boolean_node); + } + + if (auto if_node = std::dynamic_pointer_cast(node)) { + return FoldIf(node, *if_node); + } + + // InExpressionNode stores constants in unordered_sets, so ToString() is not a + // stable structural key. Keep it opaque in this conservative pass. + std::stringstream ss; + ss << "opaque:" << node.get(); + return {node, ss.str(), false}; + } + + FoldedNode FoldFunction(const NodePtr& original, const FunctionNode& function_node) { + NodeVector children; + children.reserve(function_node.children().size()); + + std::vector child_keys; + child_keys.reserve(function_node.children().size()); + + bool children_unchanged = true; + bool children_can_eliminate = true; + for (const auto& child : function_node.children()) { + auto folded = Fold(child); + children_unchanged = children_unchanged && folded.node == child; + children_can_eliminate = children_can_eliminate && folded.can_eliminate; + children.push_back(folded.node); + child_keys.push_back(std::move(folded.key)); + } + + auto desc = function_node.descriptor(); + auto return_type = desc->return_type() == NULLPTR ? std::string("untyped") + : desc->return_type()->ToString(); + auto key = JoinKey("function", desc->name(), return_type, child_keys); + auto folded_node = + children_unchanged + ? original + : std::make_shared(desc->name(), children, desc->return_type()); + bool can_eliminate = children_can_eliminate && IsFunctionSafe(function_node); + return {can_eliminate ? Intern(key, folded_node) : folded_node, std::move(key), + can_eliminate}; + } + + FoldedNode FoldBoolean(const NodePtr& original, const BooleanNode& boolean_node) { + NodeVector folded_children; + std::vector folded_keys; + std::unordered_set seen_eliminable_children; + bool children_unchanged = true; + bool children_can_eliminate = true; + + for (const auto& child : boolean_node.children()) { + auto folded = Fold(child); + children_unchanged = children_unchanged && folded.node == child; + AppendBooleanChild(boolean_node.expr_type(), std::move(folded), + &seen_eliminable_children, &folded_children, &folded_keys, + &children_unchanged, &children_can_eliminate); + } + + if (boolean_node.children().size() > 1 && folded_children.size() == 1 && + children_can_eliminate) { + return {folded_children[0], folded_keys[0], true}; + } + + auto op = boolean_node.expr_type() == BooleanNode::AND ? "and" : "or"; + auto key = JoinKey("boolean", op, "bool", folded_keys); + auto folded_node = + children_unchanged && folded_children.size() == boolean_node.children().size() + ? original + : std::make_shared(boolean_node.expr_type(), folded_children); + return {children_can_eliminate ? Intern(key, folded_node) : folded_node, + std::move(key), children_can_eliminate}; + } + + void AppendBooleanChild(BooleanNode::ExprType expr_type, FoldedNode folded, + std::unordered_set* seen_eliminable_children, + NodeVector* folded_children, + std::vector* folded_keys, bool* children_unchanged, + bool* children_can_eliminate) { + auto nested_boolean = std::dynamic_pointer_cast(folded.node); + if (nested_boolean != nullptr && nested_boolean->expr_type() == expr_type && + folded.can_eliminate) { + *children_unchanged = false; + for (const auto& nested_child : nested_boolean->children()) { + auto nested_folded = Fold(nested_child); + AppendBooleanChild(expr_type, std::move(nested_folded), seen_eliminable_children, + folded_children, folded_keys, children_unchanged, + children_can_eliminate); + } + return; + } + + if (folded.can_eliminate) { + if (!seen_eliminable_children->insert(folded.key).second) { + *children_unchanged = false; + return; + } + } else { + *children_can_eliminate = false; + } + + folded_children->push_back(folded.node); + folded_keys->push_back(std::move(folded.key)); + } + + FoldedNode FoldIf(const NodePtr& original, const IfNode& if_node) { + auto condition = Fold(if_node.condition()); + auto then_node = Fold(if_node.then_node()); + auto else_node = Fold(if_node.else_node()); + + if (condition.can_eliminate && then_node.can_eliminate && else_node.can_eliminate && + then_node.key == else_node.key) { + return then_node; + } + + std::vector child_keys{condition.key, then_node.key, else_node.key}; + auto key = JoinKey("if", "", if_node.return_type()->ToString(), child_keys); + bool children_unchanged = condition.node == if_node.condition() && + then_node.node == if_node.then_node() && + else_node.node == if_node.else_node(); + auto folded_node = children_unchanged ? original + : std::make_shared( + condition.node, then_node.node, + else_node.node, if_node.return_type()); + + return {folded_node, std::move(key), false}; + } + + bool IsFunctionSafe(const FunctionNode& node) const { + auto desc = node.descriptor(); + FunctionSignature signature(desc->name(), desc->params(), desc->return_type()); + const NativeFunction* native_function = registry_.LookupSignature(signature); + if (native_function == nullptr) { + return false; + } + return native_function->result_nullable_type() != kResultNullInternal && + !native_function->NeedsContext() && !native_function->NeedsFunctionHolder() && + !native_function->CanReturnErrors(); + } + + NodePtr Intern(const std::string& key, const NodePtr& node) { + auto it = canonical_nodes_.find(key); + if (it != canonical_nodes_.end()) { + return it->second; + } + canonical_nodes_.emplace(key, node); + return node; + } + + std::string JoinKey(const std::string& kind, const std::string& name, + const std::string& return_type, + const std::vector& child_keys) const { + std::stringstream ss; + ss << kind << ":" << name << ":" << return_type << "("; + bool first = true; + for (const auto& child_key : child_keys) { + if (!first) { + ss << ","; + } + ss << child_key.size() << ":" << child_key; + first = false; + } + ss << ")"; + return ss.str(); + } + + const FunctionRegistry& registry_; + std::unordered_map canonical_nodes_; +}; + +} // namespace + +ExpressionVector FoldCommonSubexpressions(const FunctionRegistry& registry, + const ExpressionVector& expressions) { + CommonSubexpressionFolder folder(registry); + ExpressionVector folded_expressions; + folded_expressions.reserve(expressions.size()); + for (const auto& expression : expressions) { + folded_expressions.push_back(folder.FoldExpression(expression)); + } + return folded_expressions; +} + +ConditionPtr FoldCommonSubexpressions(const FunctionRegistry& registry, + const ConditionPtr& condition) { + CommonSubexpressionFolder folder(registry); + return folder.FoldCondition(condition); +} + +} // namespace gandiva diff --git a/cpp/src/gandiva/expr_cse.h b/cpp/src/gandiva/expr_cse.h new file mode 100644 index 000000000000..69664cfecddd --- /dev/null +++ b/cpp/src/gandiva/expr_cse.h @@ -0,0 +1,35 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "gandiva/expression.h" +#include "gandiva/gandiva_aliases.h" +#include "gandiva/visibility.h" + +namespace gandiva { + +class Condition; +class FunctionRegistry; + +GANDIVA_EXPORT ExpressionVector FoldCommonSubexpressions( + const FunctionRegistry& registry, const ExpressionVector& expressions); + +GANDIVA_EXPORT ConditionPtr FoldCommonSubexpressions(const FunctionRegistry& registry, + const ConditionPtr& condition); + +} // namespace gandiva diff --git a/cpp/src/gandiva/expr_decomposer.cc b/cpp/src/gandiva/expr_decomposer.cc index 921829db6a95..eaf44993416e 100644 --- a/cpp/src/gandiva/expr_decomposer.cc +++ b/cpp/src/gandiva/expr_decomposer.cc @@ -61,6 +61,52 @@ const FunctionNode ExprDecomposer::TryOptimize(const FunctionNode& node) { } } +Status ExprDecomposer::DecomposeNode(const Node& node, ValueValidityPairPtr* out) { + bool can_reuse = CanReuseDecomposition(node); + if (can_reuse) { + auto it = decomposed_cache_.find(&node); + if (it != decomposed_cache_.end()) { + *out = it->second; + return Status::OK(); + } + } + + ARROW_RETURN_NOT_OK(node.Accept(*this)); + *out = std::move(result_); + if (can_reuse) { + decomposed_cache_.emplace(&node, *out); + } + return Status::OK(); +} + +bool ExprDecomposer::CanReuseDecomposition(const Node& node) { + auto cached = reusable_node_cache_.find(&node); + if (cached != reusable_node_cache_.end()) { + return cached->second; + } + + bool reusable = false; + if (dynamic_cast(&node) != nullptr || + dynamic_cast(&node) != nullptr) { + reusable = true; + } else if (auto function_node = dynamic_cast(&node)) { + auto desc = function_node->descriptor(); + FunctionSignature signature(desc->name(), desc->params(), desc->return_type()); + const NativeFunction* native_function = registry_.LookupSignature(signature); + reusable = native_function != nullptr && + native_function->result_nullable_type() != kResultNullInternal && + !native_function->NeedsContext() && + !native_function->NeedsFunctionHolder() && + !native_function->CanReturnErrors(); + for (const auto& child : function_node->children()) { + reusable = reusable && CanReuseDecomposition(*child); + } + } + + reusable_node_cache_.emplace(&node, reusable); + return reusable; +} + // Decompose a field node - wherever possible, merge the validity vectors of the // child nodes. Status ExprDecomposer::Visit(const FunctionNode& in_node) { @@ -73,10 +119,9 @@ Status ExprDecomposer::Visit(const FunctionNode& in_node) { // decompose the children. std::vector args; for (auto& child : node.children()) { - auto status = child->Accept(*this); - ARROW_RETURN_NOT_OK(status); - - args.push_back(result()); + ValueValidityPairPtr child_vv; + ARROW_RETURN_NOT_OK(DecomposeNode(*child, &child_vv)); + args.push_back(child_vv); } // Make a function holder, if required. @@ -130,24 +175,24 @@ Status ExprDecomposer::Visit(const IfNode& node) { nested_if_else_ = false; PushConditionEntry(node); - auto status = node.condition()->Accept(*this); + ValueValidityPairPtr condition_vv; + auto status = DecomposeNode(*node.condition(), &condition_vv); ARROW_RETURN_NOT_OK(status); - auto condition_vv = result(); PopConditionEntry(node); // Add a local bitmap to track the output validity. int local_bitmap_idx = PushThenEntry(node, svd_nested_if_else); - status = node.then_node()->Accept(*this); + ValueValidityPairPtr then_vv; + status = DecomposeNode(*node.then_node(), &then_vv); ARROW_RETURN_NOT_OK(status); - auto then_vv = result(); PopThenEntry(node); PushElseEntry(node, local_bitmap_idx); nested_if_else_ = (dynamic_cast(node.else_node().get()) != nullptr); - status = node.else_node()->Accept(*this); + ValueValidityPairPtr else_vv; + status = DecomposeNode(*node.else_node(), &else_vv); ARROW_RETURN_NOT_OK(status); - auto else_vv = result(); bool is_terminal_else = PopElseEntry(node); auto validity_dex = std::make_shared(local_bitmap_idx); @@ -164,10 +209,9 @@ Status ExprDecomposer::Visit(const BooleanNode& node) { // decompose the children. std::vector args; for (auto& child : node.children()) { - auto status = child->Accept(*this); - ARROW_RETURN_NOT_OK(status); - - args.push_back(result()); + ValueValidityPairPtr child_vv; + ARROW_RETURN_NOT_OK(DecomposeNode(*child, &child_vv)); + args.push_back(child_vv); } // Add a local bitmap to track the output validity. @@ -190,9 +234,9 @@ Status ExprDecomposer::Visit(const BooleanNode& node) { Status ExprDecomposer::Visit(const InExpressionNode& node) { /* decompose the children. */ std::vector args; - auto status = node.eval_expr()->Accept(*this); - ARROW_RETURN_NOT_OK(status); - args.push_back(result()); + ValueValidityPairPtr eval_vv; + ARROW_RETURN_NOT_OK(DecomposeNode(*node.eval_expr(), &eval_vv)); + args.push_back(eval_vv); /* In always outputs valid results, so no validity dex */ auto value_dex = std::make_shared>( args, node.values(), node.get_precision(), node.get_scale()); @@ -206,9 +250,9 @@ template Status ExprDecomposer::VisitInGeneric(const InExpressionNode& node) { /* decompose the children. */ std::vector args; - auto status = node.eval_expr()->Accept(*this); - ARROW_RETURN_NOT_OK(status); - args.push_back(result()); + ValueValidityPairPtr eval_vv; + ARROW_RETURN_NOT_OK(DecomposeNode(*node.eval_expr(), &eval_vv)); + args.push_back(eval_vv); /* In always outputs valid results, so no validity dex */ auto value_dex = std::make_shared>(args, node.values()); int holder_idx = annotator_.AddHolderPointer(value_dex->in_holder().get()); diff --git a/cpp/src/gandiva/expr_decomposer.h b/cpp/src/gandiva/expr_decomposer.h index 90a27744b362..e1d399709055 100644 --- a/cpp/src/gandiva/expr_decomposer.h +++ b/cpp/src/gandiva/expr_decomposer.h @@ -21,6 +21,7 @@ #include #include #include +#include #include #include "gandiva/arrow.h" @@ -42,11 +43,7 @@ class GANDIVA_EXPORT ExprDecomposer : public NodeVisitor { : registry_(registry), annotator_(annotator), nested_if_else_(false) {} Status Decompose(const Node& root, ValueValidityPairPtr* out) { - auto status = root.Accept(*this); - if (status.ok()) { - *out = std::move(result_); - } - return status; + return DecomposeNode(root, out); } private: @@ -75,6 +72,10 @@ class GANDIVA_EXPORT ExprDecomposer : public NodeVisitor { template Status VisitInGeneric(const InExpressionNode& node); + Status DecomposeNode(const Node& node, ValueValidityPairPtr* out); + + bool CanReuseDecomposition(const Node& node); + // Optimize a function node, if possible. const FunctionNode TryOptimize(const FunctionNode& node); @@ -125,6 +126,8 @@ class GANDIVA_EXPORT ExprDecomposer : public NodeVisitor { Annotator& annotator_; std::stack> if_entries_stack_; ValueValidityPairPtr result_; + std::unordered_map decomposed_cache_; + std::unordered_map reusable_node_cache_; bool nested_if_else_; }; diff --git a/cpp/src/gandiva/filter.cc b/cpp/src/gandiva/filter.cc index 8a270cfdc06f..ed49ec37acf1 100644 --- a/cpp/src/gandiva/filter.cc +++ b/cpp/src/gandiva/filter.cc @@ -23,6 +23,7 @@ #include "gandiva/bitmap_accumulator.h" #include "gandiva/cache.h" #include "gandiva/condition.h" +#include "gandiva/expr_cse.h" #include "gandiva/expr_validator.h" #include "gandiva/llvm_generator.h" #include "gandiva/selection_vector_impl.h" @@ -45,10 +46,13 @@ Status Filter::Make(SchemaPtr schema, ConditionPtr condition, ARROW_RETURN_IF(configuration == nullptr, Status::Invalid("Configuration cannot be null")); + auto folded_condition = + FoldCommonSubexpressions(*configuration->function_registry(), condition); + std::shared_ptr>> cache = LLVMGenerator::GetCache(); - Condition conditionToKey = *(condition.get()); + Condition conditionToKey = *(folded_condition.get()); ExpressionCacheKey cache_key(schema, configuration, conditionToKey); @@ -73,13 +77,14 @@ Status Filter::Make(SchemaPtr schema, ConditionPtr condition, // Return if the expression is invalid since we will not be able to process further. ExprValidator expr_validator(llvm_gen->types(), schema, configuration->function_registry()); - ARROW_RETURN_NOT_OK(expr_validator.Validate(condition)); + ARROW_RETURN_NOT_OK(expr_validator.Validate(folded_condition)); } // Set the object cache for LLVM ARROW_RETURN_NOT_OK(llvm_gen->SetLLVMObjectCache(obj_cache)); - ARROW_RETURN_NOT_OK(llvm_gen->Build({condition}, SelectionVector::Mode::MODE_NONE)); + ARROW_RETURN_NOT_OK( + llvm_gen->Build({folded_condition}, SelectionVector::Mode::MODE_NONE)); // Instantiate the filter with the completely built llvm generator *filter = std::make_shared(std::move(llvm_gen), schema, configuration); diff --git a/cpp/src/gandiva/llvm_generator.h b/cpp/src/gandiva/llvm_generator.h index a60e2bf6b29e..485b2466e494 100644 --- a/cpp/src/gandiva/llvm_generator.h +++ b/cpp/src/gandiva/llvm_generator.h @@ -84,6 +84,7 @@ class GANDIVA_EXPORT LLVMGenerator { LLVMTypes* types() { return engine_->types(); } llvm::Module* module() { return engine_->module(); } const std::string& ir() { return engine_->ir(); } + const std::string& unoptimized_ir() { return engine_->unoptimized_ir(); } private: explicit LLVMGenerator(bool cached, diff --git a/cpp/src/gandiva/projector.cc b/cpp/src/gandiva/projector.cc index ec0302146fff..5f5419852ea5 100644 --- a/cpp/src/gandiva/projector.cc +++ b/cpp/src/gandiva/projector.cc @@ -24,6 +24,7 @@ #include "arrow/util/logging.h" #include "gandiva/cache.h" +#include "gandiva/expr_cse.h" #include "gandiva/expr_validator.h" #include "gandiva/llvm_generator.h" @@ -61,11 +62,15 @@ Status Projector::Make(SchemaPtr schema, const ExpressionVector& exprs, ARROW_RETURN_IF(configuration == nullptr, Status::Invalid("Configuration cannot be null")); + auto folded_exprs = + FoldCommonSubexpressions(*configuration->function_registry(), exprs); + // see if equivalent projector was already built std::shared_ptr>> cache = LLVMGenerator::GetCache(); - ExpressionCacheKey cache_key(schema, configuration, exprs, selection_vector_mode); + ExpressionCacheKey cache_key(schema, configuration, folded_exprs, + selection_vector_mode); bool is_cached = false; @@ -89,7 +94,7 @@ Status Projector::Make(SchemaPtr schema, const ExpressionVector& exprs, if (!is_cached) { ExprValidator expr_validator(llvm_gen->types(), schema, configuration->function_registry()); - for (auto& expr : exprs) { + for (auto& expr : folded_exprs) { ARROW_RETURN_NOT_OK(expr_validator.Validate(expr)); } } @@ -97,12 +102,12 @@ Status Projector::Make(SchemaPtr schema, const ExpressionVector& exprs, // Set the object cache for LLVM ARROW_RETURN_NOT_OK(llvm_gen->SetLLVMObjectCache(obj_cache)); - ARROW_RETURN_NOT_OK(llvm_gen->Build(exprs, selection_vector_mode)); + ARROW_RETURN_NOT_OK(llvm_gen->Build(folded_exprs, selection_vector_mode)); // save the output field types. Used for validation at Evaluate() time. std::vector output_fields; - output_fields.reserve(exprs.size()); - for (auto& expr : exprs) { + output_fields.reserve(folded_exprs.size()); + for (auto& expr : folded_exprs) { output_fields.push_back(expr->result()); } @@ -283,6 +288,10 @@ Status Projector::ValidateArrayDataCapacity(const arrow::ArrayData& array_data, const std::string& Projector::DumpIR() { return llvm_generator_->ir(); } +const std::string& Projector::DumpUnoptimizedIR() { + return llvm_generator_->unoptimized_ir(); +} + void Projector::SetBuiltFromCache(bool flag) { built_from_cache_ = flag; } bool Projector::GetBuiltFromCache() { return built_from_cache_; } diff --git a/cpp/src/gandiva/projector.h b/cpp/src/gandiva/projector.h index f1ae7e4dc8cc..2fa1e56b35a8 100644 --- a/cpp/src/gandiva/projector.h +++ b/cpp/src/gandiva/projector.h @@ -120,6 +120,8 @@ class GANDIVA_EXPORT Projector { const std::string& DumpIR(); + const std::string& DumpUnoptimizedIR(); + void SetBuiltFromCache(bool flag); bool GetBuiltFromCache(); diff --git a/cpp/src/gandiva/tests/projector_test.cc b/cpp/src/gandiva/tests/projector_test.cc index 268cb55a6422..c92b50950b58 100644 --- a/cpp/src/gandiva/tests/projector_test.cc +++ b/cpp/src/gandiva/tests/projector_test.cc @@ -24,6 +24,7 @@ #include #include +#include #include "arrow/memory_pool.h" #include "gandiva/function_registry.h" @@ -39,6 +40,40 @@ using arrow::float32; using arrow::int32; using arrow::int64; +namespace { + +int CountOccurrences(const std::string& text, const std::string& needle) { + int count = 0; + std::string::size_type pos = 0; + while ((pos = text.find(needle, pos)) != std::string::npos) { + ++count; + pos += needle.size(); + } + return count; +} + +int CountInt32AddInstructions(const std::string& ir) { + return CountOccurrences(ir, " add i32 ") + CountOccurrences(ir, " add nsw i32 ") + + CountOccurrences(ir, " add nuw i32 ") + + CountOccurrences(ir, " add nuw nsw i32 "); +} + +std::string ExtractFunctionIR(const std::string& ir, const std::string& function_name) { + const auto name_pos = ir.find("@" + function_name + "("); + if (name_pos == std::string::npos) { + return ""; + } + const auto function_start = ir.rfind("\ndefine ", name_pos); + const auto start = function_start == std::string::npos ? 0 : function_start + 1; + const auto function_end = ir.find("\n}\n", name_pos); + if (function_end == std::string::npos) { + return ir.substr(start); + } + return ir.substr(start, function_end + 3 - start); +} + +} // namespace + class TestProjector : public ::testing::Test { public: void SetUp() { @@ -388,6 +423,195 @@ TEST_F(TestProjector, TestAllIntTypes) { TestArithmeticOpsForType(pool_); } +TEST_F(TestProjector, TestCommonSubexpressionEliminationIR) { + auto field0 = arrow::field("cse_f0", arrow::int32()); + auto field1 = arrow::field("cse_f1", arrow::int32()); + auto schema = arrow::schema({field0, field1}); + + auto left_sum = TreeExprBuilder::MakeFunction( + "add", {TreeExprBuilder::MakeField(field0), TreeExprBuilder::MakeField(field1)}, + arrow::int32()); + auto right_sum = TreeExprBuilder::MakeFunction( + "add", {TreeExprBuilder::MakeField(field0), TreeExprBuilder::MakeField(field1)}, + arrow::int32()); + auto square = + TreeExprBuilder::MakeFunction("multiply", {left_sum, right_sum}, arrow::int32()); + auto expr = + TreeExprBuilder::MakeExpression(square, arrow::field("cse_out", arrow::int32())); + + auto configuration = std::make_shared( + true, gandiva::default_function_registry(), /*dump_ir=*/true); + std::shared_ptr projector; + ASSERT_OK(Projector::Make(schema, {expr}, configuration, &projector)); + + const auto unoptimized_expr_ir = + ExtractFunctionIR(projector->DumpUnoptimizedIR(), "expr_0_0"); + const auto optimized_expr_ir = ExtractFunctionIR(projector->DumpIR(), "expr_0_0"); + ASSERT_FALSE(unoptimized_expr_ir.empty()); + ASSERT_FALSE(optimized_expr_ir.empty()); + + EXPECT_EQ(2, CountOccurrences(unoptimized_expr_ir, "call i32 @add_int32_int32")); + EXPECT_EQ(1, CountOccurrences(unoptimized_expr_ir, "call i32 @multiply_int32_int32")); + EXPECT_EQ(0, CountOccurrences(optimized_expr_ir, "call i32 @add_int32_int32")); + EXPECT_EQ(0, CountOccurrences(optimized_expr_ir, "call i32 @multiply_int32_int32")); + EXPECT_EQ(1, CountInt32AddInstructions(optimized_expr_ir)); +} + +TEST_F(TestProjector, TestNestedCommonSubexpressionEliminationIR) { + auto field0 = arrow::field("nested_cse_f0", arrow::int32()); + auto field1 = arrow::field("nested_cse_f1", arrow::int32()); + auto schema = arrow::schema({field0, field1}); + + auto make_sum = [&]() { + return TreeExprBuilder::MakeFunction( + "add", {TreeExprBuilder::MakeField(field0), TreeExprBuilder::MakeField(field1)}, + arrow::int32()); + }; + auto make_square = [&]() { + return TreeExprBuilder::MakeFunction("multiply", {make_sum(), make_sum()}, + arrow::int32()); + }; + auto nested = TreeExprBuilder::MakeFunction("add", {make_square(), make_square()}, + arrow::int32()); + auto expr = TreeExprBuilder::MakeExpression( + nested, arrow::field("nested_cse_out", arrow::int32())); + + auto configuration = std::make_shared( + true, gandiva::default_function_registry(), /*dump_ir=*/true); + std::shared_ptr projector; + ASSERT_OK(Projector::Make(schema, {expr}, configuration, &projector)); + + const auto unoptimized_expr_ir = + ExtractFunctionIR(projector->DumpUnoptimizedIR(), "expr_0_0"); + const auto optimized_expr_ir = ExtractFunctionIR(projector->DumpIR(), "expr_0_0"); + ASSERT_FALSE(unoptimized_expr_ir.empty()); + ASSERT_FALSE(optimized_expr_ir.empty()); + + EXPECT_EQ(5, CountOccurrences(unoptimized_expr_ir, "call i32 @add_int32_int32")); + EXPECT_EQ(2, CountOccurrences(unoptimized_expr_ir, "call i32 @multiply_int32_int32")); + EXPECT_EQ(0, CountOccurrences(optimized_expr_ir, "call i32 @add_int32_int32")); + EXPECT_EQ(0, CountOccurrences(optimized_expr_ir, "call i32 @multiply_int32_int32")); +} + +TEST_F(TestProjector, TestGeneratedIfCommonSubexpressionEliminationIR) { + auto condition_field = arrow::field("generated_if_cond", arrow::boolean()); + auto value_field = arrow::field("generated_if_value", arrow::int32()); + auto schema = arrow::schema({condition_field, value_field}); + + auto condition = TreeExprBuilder::MakeField(condition_field); + auto then_node = TreeExprBuilder::MakeField(value_field); + auto else_node = TreeExprBuilder::MakeField(value_field); + auto if_node = TreeExprBuilder::MakeIf(condition, then_node, else_node, arrow::int32()); + auto expr = TreeExprBuilder::MakeExpression( + if_node, arrow::field("generated_if_out", arrow::int32())); + + auto configuration = std::make_shared( + true, gandiva::default_function_registry(), /*dump_ir=*/true); + std::shared_ptr projector; + ASSERT_OK(Projector::Make(schema, {expr}, configuration, &projector)); + + const auto unoptimized_expr_ir = + ExtractFunctionIR(projector->DumpUnoptimizedIR(), "expr_0_0"); + const auto optimized_expr_ir = ExtractFunctionIR(projector->DumpIR(), "expr_0_0"); + ASSERT_FALSE(unoptimized_expr_ir.empty()); + ASSERT_FALSE(optimized_expr_ir.empty()); + + EXPECT_NE(std::string::npos, unoptimized_expr_ir.find("generated_if_value")); + EXPECT_EQ(std::string::npos, unoptimized_expr_ir.find("generated_if_cond")); + EXPECT_EQ(std::string::npos, unoptimized_expr_ir.find("then:")); + EXPECT_EQ(std::string::npos, unoptimized_expr_ir.find("else:")); + EXPECT_EQ(std::string::npos, unoptimized_expr_ir.find("validAndMatch")); + EXPECT_EQ(std::string::npos, unoptimized_expr_ir.find("res_value = phi")); + EXPECT_NE(std::string::npos, optimized_expr_ir.find("generated_if_value")); + EXPECT_EQ(std::string::npos, optimized_expr_ir.find("generated_if_cond")); + EXPECT_EQ(std::string::npos, optimized_expr_ir.find("validAndMatch")); + EXPECT_EQ(std::string::npos, optimized_expr_ir.find("res_value = phi")); +} + +TEST_F(TestProjector, TestGeneratedBooleanCommonSubexpressionEliminationIR) { + auto field0 = arrow::field("generated_bool_f0", arrow::boolean()); + auto field1 = arrow::field("generated_bool_f1", arrow::boolean()); + auto schema = arrow::schema({field0, field1}); + + auto make_and = [&]() { + return TreeExprBuilder::MakeAnd( + {TreeExprBuilder::MakeField(field0), TreeExprBuilder::MakeField(field1)}); + }; + auto bool_expr = TreeExprBuilder::MakeOr({make_and(), make_and()}); + auto expr = TreeExprBuilder::MakeExpression( + bool_expr, arrow::field("generated_bool_out", arrow::boolean())); + + auto configuration = std::make_shared( + true, gandiva::default_function_registry(), /*dump_ir=*/true); + std::shared_ptr projector; + ASSERT_OK(Projector::Make(schema, {expr}, configuration, &projector)); + + const auto unoptimized_expr_ir = + ExtractFunctionIR(projector->DumpUnoptimizedIR(), "expr_0_0"); + const auto optimized_expr_ir = ExtractFunctionIR(projector->DumpIR(), "expr_0_0"); + ASSERT_FALSE(unoptimized_expr_ir.empty()); + ASSERT_FALSE(optimized_expr_ir.empty()); + + EXPECT_GT(CountOccurrences(unoptimized_expr_ir, "short_circuit"), 0); + EXPECT_GT(CountOccurrences(unoptimized_expr_ir, "non_short_circuit"), 0); + EXPECT_GT(CountOccurrences(unoptimized_expr_ir, "res_value = phi"), 0); + EXPECT_GT(CountOccurrences(unoptimized_expr_ir, "\"0_lbmap\""), 0); + EXPECT_EQ(0, CountOccurrences(unoptimized_expr_ir, "\"1_lbmap\"")); + EXPECT_EQ(0, CountOccurrences(unoptimized_expr_ir, "\"2_lbmap\"")); + EXPECT_GT(CountOccurrences(optimized_expr_ir, "generated_bool_f0"), 0); + EXPECT_GT(CountOccurrences(optimized_expr_ir, "generated_bool_f1"), 0); + EXPECT_GT(CountOccurrences(optimized_expr_ir, "\"0_lbmap\""), 0); + EXPECT_EQ(0, CountOccurrences(optimized_expr_ir, "\"1_lbmap\"")); + EXPECT_EQ(0, CountOccurrences(optimized_expr_ir, "\"2_lbmap\"")); +} + +TEST_F(TestProjector, TestNestedBetweenCommonSubexpressionFoldIR) { + auto value = arrow::field("nested_between_value", arrow::int32()); + auto lower = arrow::field("nested_between_lower", arrow::int32()); + auto upper = arrow::field("nested_between_upper", arrow::int32()); + auto schema = arrow::schema({value, lower, upper}); + + auto make_between = [&]() { + auto ge_lower = TreeExprBuilder::MakeFunction( + "greater_than_or_equal_to", + {TreeExprBuilder::MakeField(value), TreeExprBuilder::MakeField(lower)}, + arrow::boolean()); + auto le_upper = TreeExprBuilder::MakeFunction( + "less_than_or_equal_to", + {TreeExprBuilder::MakeField(value), TreeExprBuilder::MakeField(upper)}, + arrow::boolean()); + return TreeExprBuilder::MakeAnd({ge_lower, le_upper}); + }; + + auto nested_between = TreeExprBuilder::MakeAnd( + {make_between(), TreeExprBuilder::MakeOr({make_between(), make_between()})}); + auto expr = TreeExprBuilder::MakeExpression( + nested_between, arrow::field("nested_between_out", arrow::boolean())); + + auto configuration = std::make_shared( + true, gandiva::default_function_registry(), /*dump_ir=*/true); + std::shared_ptr projector; + ASSERT_OK(Projector::Make(schema, {expr}, configuration, &projector)); + + const auto unoptimized_expr_ir = + ExtractFunctionIR(projector->DumpUnoptimizedIR(), "expr_0_0"); + const auto optimized_expr_ir = ExtractFunctionIR(projector->DumpIR(), "expr_0_0"); + ASSERT_FALSE(unoptimized_expr_ir.empty()); + ASSERT_FALSE(optimized_expr_ir.empty()); + + EXPECT_EQ(1, CountOccurrences(unoptimized_expr_ir, + "call i1 @greater_than_or_equal_to_int32_int32")); + EXPECT_EQ(1, CountOccurrences(unoptimized_expr_ir, + "call i1 @less_than_or_equal_to_int32_int32")); + EXPECT_GT(CountOccurrences(unoptimized_expr_ir, "\"0_lbmap\""), 0); + EXPECT_EQ(0, CountOccurrences(unoptimized_expr_ir, "\"1_lbmap\"")); + EXPECT_EQ(0, CountOccurrences(unoptimized_expr_ir, "\"2_lbmap\"")); + EXPECT_EQ(0, CountOccurrences(optimized_expr_ir, + "call i1 @greater_than_or_equal_to_int32_int32")); + EXPECT_EQ(0, CountOccurrences(optimized_expr_ir, + "call i1 @less_than_or_equal_to_int32_int32")); +} + TEST_F(TestProjector, TestExtendedMath) { #ifdef __aarch64__ GTEST_SKIP() << "Failed on aarch64 with 'JIT session error: Symbols not found: [ " From 473a9e5b1c57b96939eef7fbd253cd74213ac85b Mon Sep 17 00:00:00 2001 From: likun Date: Tue, 14 Jul 2026 17:36:38 +0800 Subject: [PATCH 2/5] GH-50371: [C++][Gandiva] Address review comments --- cpp/src/gandiva/expr_cse.cc | 4 +-- cpp/src/gandiva/expr_cse.h | 16 +++++++--- cpp/src/gandiva/expr_decomposer.cc | 7 ++--- cpp/src/gandiva/tests/projector_test.cc | 42 +++++++++++++++++++++++++ 4 files changed, 56 insertions(+), 13 deletions(-) diff --git a/cpp/src/gandiva/expr_cse.cc b/cpp/src/gandiva/expr_cse.cc index bf8524b0230a..95c8dd5f9e0b 100644 --- a/cpp/src/gandiva/expr_cse.cc +++ b/cpp/src/gandiva/expr_cse.cc @@ -216,9 +216,7 @@ class CommonSubexpressionFolder { if (native_function == nullptr) { return false; } - return native_function->result_nullable_type() != kResultNullInternal && - !native_function->NeedsContext() && !native_function->NeedsFunctionHolder() && - !native_function->CanReturnErrors(); + return CanReuseNativeFunction(*native_function); } NodePtr Intern(const std::string& key, const NodePtr& node) { diff --git a/cpp/src/gandiva/expr_cse.h b/cpp/src/gandiva/expr_cse.h index 69664cfecddd..30ed9dbb3d8a 100644 --- a/cpp/src/gandiva/expr_cse.h +++ b/cpp/src/gandiva/expr_cse.h @@ -19,17 +19,23 @@ #include "gandiva/expression.h" #include "gandiva/gandiva_aliases.h" -#include "gandiva/visibility.h" +#include "gandiva/native_function.h" namespace gandiva { class Condition; class FunctionRegistry; -GANDIVA_EXPORT ExpressionVector FoldCommonSubexpressions( - const FunctionRegistry& registry, const ExpressionVector& expressions); +inline bool CanReuseNativeFunction(const NativeFunction& native_function) { + return native_function.result_nullable_type() != kResultNullInternal && + !native_function.NeedsContext() && !native_function.NeedsFunctionHolder() && + !native_function.CanReturnErrors(); +} -GANDIVA_EXPORT ConditionPtr FoldCommonSubexpressions(const FunctionRegistry& registry, - const ConditionPtr& condition); +ExpressionVector FoldCommonSubexpressions(const FunctionRegistry& registry, + const ExpressionVector& expressions); + +ConditionPtr FoldCommonSubexpressions(const FunctionRegistry& registry, + const ConditionPtr& condition); } // namespace gandiva diff --git a/cpp/src/gandiva/expr_decomposer.cc b/cpp/src/gandiva/expr_decomposer.cc index eaf44993416e..4007966b0059 100644 --- a/cpp/src/gandiva/expr_decomposer.cc +++ b/cpp/src/gandiva/expr_decomposer.cc @@ -26,6 +26,7 @@ #include "arrow/util/logging_internal.h" #include "gandiva/annotator.h" #include "gandiva/dex.h" +#include "gandiva/expr_cse.h" #include "gandiva/function_holder_maker_registry.h" #include "gandiva/function_registry.h" #include "gandiva/function_signature.h" @@ -93,11 +94,7 @@ bool ExprDecomposer::CanReuseDecomposition(const Node& node) { auto desc = function_node->descriptor(); FunctionSignature signature(desc->name(), desc->params(), desc->return_type()); const NativeFunction* native_function = registry_.LookupSignature(signature); - reusable = native_function != nullptr && - native_function->result_nullable_type() != kResultNullInternal && - !native_function->NeedsContext() && - !native_function->NeedsFunctionHolder() && - !native_function->CanReturnErrors(); + reusable = native_function != nullptr && CanReuseNativeFunction(*native_function); for (const auto& child : function_node->children()) { reusable = reusable && CanReuseDecomposition(*child); } diff --git a/cpp/src/gandiva/tests/projector_test.cc b/cpp/src/gandiva/tests/projector_test.cc index c92b50950b58..f04f7cf6fccc 100644 --- a/cpp/src/gandiva/tests/projector_test.cc +++ b/cpp/src/gandiva/tests/projector_test.cc @@ -72,6 +72,17 @@ std::string ExtractFunctionIR(const std::string& ir, const std::string& function return ir.substr(start, function_end + 3 - start); } +void ExpectProjectorOutput(const std::shared_ptr& projector, + const SchemaPtr& schema, const arrow::ArrayVector& inputs, + const ArrayPtr& expected, arrow::MemoryPool* pool) { + ASSERT_FALSE(inputs.empty()); + auto batch = arrow::RecordBatch::Make(schema, inputs[0]->length(), inputs); + arrow::ArrayVector outputs; + ASSERT_OK(projector->Evaluate(*batch, pool, &outputs)); + ASSERT_EQ(1, outputs.size()); + EXPECT_ARROW_ARRAY_EQUALS(expected, outputs[0]); +} + } // namespace class TestProjector : public ::testing::Test { @@ -455,6 +466,11 @@ TEST_F(TestProjector, TestCommonSubexpressionEliminationIR) { EXPECT_EQ(0, CountOccurrences(optimized_expr_ir, "call i32 @add_int32_int32")); EXPECT_EQ(0, CountOccurrences(optimized_expr_ir, "call i32 @multiply_int32_int32")); EXPECT_EQ(1, CountInt32AddInstructions(optimized_expr_ir)); + + auto input0 = MakeArrowArrayInt32({1, 2, 3, 4}, {true, true, false, true}); + auto input1 = MakeArrowArrayInt32({10, -2, 3, 5}, {true, true, true, false}); + auto expected = MakeArrowArrayInt32({121, 0, 0, 0}, {true, true, false, false}); + ExpectProjectorOutput(projector, schema, {input0, input1}, expected, pool_); } TEST_F(TestProjector, TestNestedCommonSubexpressionEliminationIR) { @@ -491,6 +507,11 @@ TEST_F(TestProjector, TestNestedCommonSubexpressionEliminationIR) { EXPECT_EQ(2, CountOccurrences(unoptimized_expr_ir, "call i32 @multiply_int32_int32")); EXPECT_EQ(0, CountOccurrences(optimized_expr_ir, "call i32 @add_int32_int32")); EXPECT_EQ(0, CountOccurrences(optimized_expr_ir, "call i32 @multiply_int32_int32")); + + auto input0 = MakeArrowArrayInt32({1, 2, 3, 4}, {true, true, false, true}); + auto input1 = MakeArrowArrayInt32({10, -2, 3, 5}, {true, true, true, false}); + auto expected = MakeArrowArrayInt32({242, 0, 0, 0}, {true, true, false, false}); + ExpectProjectorOutput(projector, schema, {input0, input1}, expected, pool_); } TEST_F(TestProjector, TestGeneratedIfCommonSubexpressionEliminationIR) { @@ -526,6 +547,12 @@ TEST_F(TestProjector, TestGeneratedIfCommonSubexpressionEliminationIR) { EXPECT_EQ(std::string::npos, optimized_expr_ir.find("generated_if_cond")); EXPECT_EQ(std::string::npos, optimized_expr_ir.find("validAndMatch")); EXPECT_EQ(std::string::npos, optimized_expr_ir.find("res_value = phi")); + + auto conditions = + MakeArrowArrayBool({true, false, false, true}, {true, true, false, true}); + auto values = MakeArrowArrayInt32({7, 8, 9, 10}, {true, false, true, true}); + auto expected = MakeArrowArrayInt32({7, 0, 9, 10}, {true, false, true, true}); + ExpectProjectorOutput(projector, schema, {conditions, values}, expected, pool_); } TEST_F(TestProjector, TestGeneratedBooleanCommonSubexpressionEliminationIR) { @@ -563,6 +590,14 @@ TEST_F(TestProjector, TestGeneratedBooleanCommonSubexpressionEliminationIR) { EXPECT_GT(CountOccurrences(optimized_expr_ir, "\"0_lbmap\""), 0); EXPECT_EQ(0, CountOccurrences(optimized_expr_ir, "\"1_lbmap\"")); EXPECT_EQ(0, CountOccurrences(optimized_expr_ir, "\"2_lbmap\"")); + + auto input0 = MakeArrowArrayBool({true, true, false, false, false}, + {true, true, true, false, false}); + auto input1 = MakeArrowArrayBool({true, false, true, true, false}, + {true, false, false, true, true}); + auto expected = MakeArrowArrayBool({true, false, false, false, false}, + {true, false, true, false, true}); + ExpectProjectorOutput(projector, schema, {input0, input1}, expected, pool_); } TEST_F(TestProjector, TestNestedBetweenCommonSubexpressionFoldIR) { @@ -610,6 +645,13 @@ TEST_F(TestProjector, TestNestedBetweenCommonSubexpressionFoldIR) { "call i1 @greater_than_or_equal_to_int32_int32")); EXPECT_EQ(0, CountOccurrences(optimized_expr_ir, "call i1 @less_than_or_equal_to_int32_int32")); + + auto values = MakeArrowArrayInt32({5, 1, 10, 7, 4}, {true, true, true, true, false}); + auto lowers = MakeArrowArrayInt32({1, 2, 10, 6, 0}, {true, true, true, false, true}); + auto uppers = MakeArrowArrayInt32({10, 5, 10, 9, 10}); + auto expected = MakeArrowArrayBool({true, false, true, false, false}, + {true, true, true, false, false}); + ExpectProjectorOutput(projector, schema, {values, lowers, uppers}, expected, pool_); } TEST_F(TestProjector, TestExtendedMath) { From 0e4e513dc69c4b882e024a30839a5a551e36ed35 Mon Sep 17 00:00:00 2001 From: likun Date: Thu, 16 Jul 2026 14:22:16 +0800 Subject: [PATCH 3/5] GH-50371: [C++][Gandiva] Address cache and test feedback --- cpp/src/gandiva/projector.cc | 7 ++- cpp/src/gandiva/projector.h | 4 +- cpp/src/gandiva/tests/filter_test.cc | 43 ++++++++++++++ cpp/src/gandiva/tests/projector_test.cc | 77 +++++++++++-------------- 4 files changed, 85 insertions(+), 46 deletions(-) diff --git a/cpp/src/gandiva/projector.cc b/cpp/src/gandiva/projector.cc index 5f5419852ea5..fa7f772a3179 100644 --- a/cpp/src/gandiva/projector.cc +++ b/cpp/src/gandiva/projector.cc @@ -288,7 +288,12 @@ Status Projector::ValidateArrayDataCapacity(const arrow::ArrayData& array_data, const std::string& Projector::DumpIR() { return llvm_generator_->ir(); } -const std::string& Projector::DumpUnoptimizedIR() { +Result Projector::DumpUnoptimizedIR() { + ARROW_RETURN_IF(!configuration_->dump_ir(), + Status::Invalid("IR dumping is not enabled for this projector")); + ARROW_RETURN_IF( + built_from_cache_, + Status::Invalid("Unoptimized IR is unavailable for projectors built from cache")); return llvm_generator_->unoptimized_ir(); } diff --git a/cpp/src/gandiva/projector.h b/cpp/src/gandiva/projector.h index 2fa1e56b35a8..8352f3c662ca 100644 --- a/cpp/src/gandiva/projector.h +++ b/cpp/src/gandiva/projector.h @@ -120,7 +120,9 @@ class GANDIVA_EXPORT Projector { const std::string& DumpIR(); - const std::string& DumpUnoptimizedIR(); + /// Return the generated IR before the optimizer pipeline runs. + /// Unavailable when IR dumping is disabled or the projector is built from cache. + Result DumpUnoptimizedIR(); void SetBuiltFromCache(bool flag); diff --git a/cpp/src/gandiva/tests/filter_test.cc b/cpp/src/gandiva/tests/filter_test.cc index 749000aa0cf2..42e6264211d3 100644 --- a/cpp/src/gandiva/tests/filter_test.cc +++ b/cpp/src/gandiva/tests/filter_test.cc @@ -87,6 +87,49 @@ TEST_F(TestFilter, TestFilterCache) { EXPECT_FALSE(should_be_new_filter->GetBuiltFromCache()); } +TEST_F(TestFilter, TestCommonSubexpressionFoldAndCache) { + auto field0 = field("filter_cse_f0", int32()); + auto field1 = field("filter_cse_f1", int32()); + auto schema = arrow::schema({field0, field1}); + + auto make_less_than = [&]() { + auto sum = TreeExprBuilder::MakeFunction( + "add", {TreeExprBuilder::MakeField(field0), TreeExprBuilder::MakeField(field1)}, + arrow::int32()); + return TreeExprBuilder::MakeFunction( + "less_than", {sum, TreeExprBuilder::MakeLiteral(static_cast(10))}, + arrow::boolean()); + }; + auto repeated_condition = TreeExprBuilder::MakeCondition( + TreeExprBuilder::MakeAnd({make_less_than(), make_less_than(), make_less_than()})); + auto configuration = TestConfiguration(); + + std::shared_ptr filter; + ASSERT_OK(Filter::Make(schema, repeated_condition, configuration, &filter)); + ASSERT_FALSE(filter->GetBuiltFromCache()); + + auto input0 = MakeArrowArrayInt32({1, 2, 3, 4, 6}, {true, true, true, false, true}); + auto input1 = MakeArrowArrayInt32({5, 9, 6, 17, 3}, {true, true, false, true, true}); + auto batch = arrow::RecordBatch::Make(schema, 5, {input0, input1}); + auto expected = MakeArrowArrayUint16({0, 4}); + + std::shared_ptr selection_vector; + ASSERT_OK(SelectionVector::MakeInt16(batch->num_rows(), pool_, &selection_vector)); + ASSERT_OK(filter->Evaluate(*batch, selection_vector)); + EXPECT_ARROW_ARRAY_EQUALS(expected, selection_vector->ToArray()); + + auto equivalent_condition = TreeExprBuilder::MakeCondition(make_less_than()); + std::shared_ptr cached_filter; + ASSERT_OK(Filter::Make(schema, equivalent_condition, configuration, &cached_filter)); + ASSERT_TRUE(cached_filter->GetBuiltFromCache()); + + std::shared_ptr cached_selection_vector; + ASSERT_OK( + SelectionVector::MakeInt16(batch->num_rows(), pool_, &cached_selection_vector)); + ASSERT_OK(cached_filter->Evaluate(*batch, cached_selection_vector)); + EXPECT_ARROW_ARRAY_EQUALS(expected, cached_selection_vector->ToArray()); +} + TEST_F(TestFilter, TestFilterCacheNullTreatment) { // schema for input fields auto field0 = field("f0", utf8()); diff --git a/cpp/src/gandiva/tests/projector_test.cc b/cpp/src/gandiva/tests/projector_test.cc index f04f7cf6fccc..6fb9c2ffd812 100644 --- a/cpp/src/gandiva/tests/projector_test.cc +++ b/cpp/src/gandiva/tests/projector_test.cc @@ -52,12 +52,6 @@ int CountOccurrences(const std::string& text, const std::string& needle) { return count; } -int CountInt32AddInstructions(const std::string& ir) { - return CountOccurrences(ir, " add i32 ") + CountOccurrences(ir, " add nsw i32 ") + - CountOccurrences(ir, " add nuw i32 ") + - CountOccurrences(ir, " add nuw nsw i32 "); -} - std::string ExtractFunctionIR(const std::string& ir, const std::string& function_name) { const auto name_pos = ir.find("@" + function_name + "("); if (name_pos == std::string::npos) { @@ -455,17 +449,12 @@ TEST_F(TestProjector, TestCommonSubexpressionEliminationIR) { std::shared_ptr projector; ASSERT_OK(Projector::Make(schema, {expr}, configuration, &projector)); - const auto unoptimized_expr_ir = - ExtractFunctionIR(projector->DumpUnoptimizedIR(), "expr_0_0"); - const auto optimized_expr_ir = ExtractFunctionIR(projector->DumpIR(), "expr_0_0"); + ASSERT_OK_AND_ASSIGN(auto unoptimized_ir, projector->DumpUnoptimizedIR()); + const auto unoptimized_expr_ir = ExtractFunctionIR(unoptimized_ir, "expr_0_0"); ASSERT_FALSE(unoptimized_expr_ir.empty()); - ASSERT_FALSE(optimized_expr_ir.empty()); EXPECT_EQ(2, CountOccurrences(unoptimized_expr_ir, "call i32 @add_int32_int32")); EXPECT_EQ(1, CountOccurrences(unoptimized_expr_ir, "call i32 @multiply_int32_int32")); - EXPECT_EQ(0, CountOccurrences(optimized_expr_ir, "call i32 @add_int32_int32")); - EXPECT_EQ(0, CountOccurrences(optimized_expr_ir, "call i32 @multiply_int32_int32")); - EXPECT_EQ(1, CountInt32AddInstructions(optimized_expr_ir)); auto input0 = MakeArrowArrayInt32({1, 2, 3, 4}, {true, true, false, true}); auto input1 = MakeArrowArrayInt32({10, -2, 3, 5}, {true, true, true, false}); @@ -473,6 +462,29 @@ TEST_F(TestProjector, TestCommonSubexpressionEliminationIR) { ExpectProjectorOutput(projector, schema, {input0, input1}, expected, pool_); } +TEST_F(TestProjector, TestUnoptimizedIRUnavailableForCachedProjector) { + auto field0 = arrow::field("cached_ir_f0", arrow::int32()); + auto field1 = arrow::field("cached_ir_f1", arrow::int32()); + auto schema = arrow::schema({field0, field1}); + auto expr = TreeExprBuilder::MakeExpression( + "add", {field0, field1}, arrow::field("cached_ir_out", arrow::int32())); + auto configuration = std::make_shared( + true, gandiva::default_function_registry(), /*dump_ir=*/true); + + std::shared_ptr projector; + ASSERT_OK(Projector::Make(schema, {expr}, configuration, &projector)); + ASSERT_FALSE(projector->GetBuiltFromCache()); + ASSERT_OK_AND_ASSIGN(auto unoptimized_ir, projector->DumpUnoptimizedIR()); + ASSERT_FALSE(unoptimized_ir.empty()); + + std::shared_ptr cached_projector; + ASSERT_OK(Projector::Make(schema, {expr}, configuration, &cached_projector)); + ASSERT_TRUE(cached_projector->GetBuiltFromCache()); + ASSERT_RAISES_WITH_MESSAGE( + Invalid, "Invalid: Unoptimized IR is unavailable for projectors built from cache", + cached_projector->DumpUnoptimizedIR()); +} + TEST_F(TestProjector, TestNestedCommonSubexpressionEliminationIR) { auto field0 = arrow::field("nested_cse_f0", arrow::int32()); auto field1 = arrow::field("nested_cse_f1", arrow::int32()); @@ -497,16 +509,12 @@ TEST_F(TestProjector, TestNestedCommonSubexpressionEliminationIR) { std::shared_ptr projector; ASSERT_OK(Projector::Make(schema, {expr}, configuration, &projector)); - const auto unoptimized_expr_ir = - ExtractFunctionIR(projector->DumpUnoptimizedIR(), "expr_0_0"); - const auto optimized_expr_ir = ExtractFunctionIR(projector->DumpIR(), "expr_0_0"); + ASSERT_OK_AND_ASSIGN(auto unoptimized_ir, projector->DumpUnoptimizedIR()); + const auto unoptimized_expr_ir = ExtractFunctionIR(unoptimized_ir, "expr_0_0"); ASSERT_FALSE(unoptimized_expr_ir.empty()); - ASSERT_FALSE(optimized_expr_ir.empty()); EXPECT_EQ(5, CountOccurrences(unoptimized_expr_ir, "call i32 @add_int32_int32")); EXPECT_EQ(2, CountOccurrences(unoptimized_expr_ir, "call i32 @multiply_int32_int32")); - EXPECT_EQ(0, CountOccurrences(optimized_expr_ir, "call i32 @add_int32_int32")); - EXPECT_EQ(0, CountOccurrences(optimized_expr_ir, "call i32 @multiply_int32_int32")); auto input0 = MakeArrowArrayInt32({1, 2, 3, 4}, {true, true, false, true}); auto input1 = MakeArrowArrayInt32({10, -2, 3, 5}, {true, true, true, false}); @@ -531,11 +539,9 @@ TEST_F(TestProjector, TestGeneratedIfCommonSubexpressionEliminationIR) { std::shared_ptr projector; ASSERT_OK(Projector::Make(schema, {expr}, configuration, &projector)); - const auto unoptimized_expr_ir = - ExtractFunctionIR(projector->DumpUnoptimizedIR(), "expr_0_0"); - const auto optimized_expr_ir = ExtractFunctionIR(projector->DumpIR(), "expr_0_0"); + ASSERT_OK_AND_ASSIGN(auto unoptimized_ir, projector->DumpUnoptimizedIR()); + const auto unoptimized_expr_ir = ExtractFunctionIR(unoptimized_ir, "expr_0_0"); ASSERT_FALSE(unoptimized_expr_ir.empty()); - ASSERT_FALSE(optimized_expr_ir.empty()); EXPECT_NE(std::string::npos, unoptimized_expr_ir.find("generated_if_value")); EXPECT_EQ(std::string::npos, unoptimized_expr_ir.find("generated_if_cond")); @@ -543,10 +549,6 @@ TEST_F(TestProjector, TestGeneratedIfCommonSubexpressionEliminationIR) { EXPECT_EQ(std::string::npos, unoptimized_expr_ir.find("else:")); EXPECT_EQ(std::string::npos, unoptimized_expr_ir.find("validAndMatch")); EXPECT_EQ(std::string::npos, unoptimized_expr_ir.find("res_value = phi")); - EXPECT_NE(std::string::npos, optimized_expr_ir.find("generated_if_value")); - EXPECT_EQ(std::string::npos, optimized_expr_ir.find("generated_if_cond")); - EXPECT_EQ(std::string::npos, optimized_expr_ir.find("validAndMatch")); - EXPECT_EQ(std::string::npos, optimized_expr_ir.find("res_value = phi")); auto conditions = MakeArrowArrayBool({true, false, false, true}, {true, true, false, true}); @@ -573,11 +575,9 @@ TEST_F(TestProjector, TestGeneratedBooleanCommonSubexpressionEliminationIR) { std::shared_ptr projector; ASSERT_OK(Projector::Make(schema, {expr}, configuration, &projector)); - const auto unoptimized_expr_ir = - ExtractFunctionIR(projector->DumpUnoptimizedIR(), "expr_0_0"); - const auto optimized_expr_ir = ExtractFunctionIR(projector->DumpIR(), "expr_0_0"); + ASSERT_OK_AND_ASSIGN(auto unoptimized_ir, projector->DumpUnoptimizedIR()); + const auto unoptimized_expr_ir = ExtractFunctionIR(unoptimized_ir, "expr_0_0"); ASSERT_FALSE(unoptimized_expr_ir.empty()); - ASSERT_FALSE(optimized_expr_ir.empty()); EXPECT_GT(CountOccurrences(unoptimized_expr_ir, "short_circuit"), 0); EXPECT_GT(CountOccurrences(unoptimized_expr_ir, "non_short_circuit"), 0); @@ -585,11 +585,6 @@ TEST_F(TestProjector, TestGeneratedBooleanCommonSubexpressionEliminationIR) { EXPECT_GT(CountOccurrences(unoptimized_expr_ir, "\"0_lbmap\""), 0); EXPECT_EQ(0, CountOccurrences(unoptimized_expr_ir, "\"1_lbmap\"")); EXPECT_EQ(0, CountOccurrences(unoptimized_expr_ir, "\"2_lbmap\"")); - EXPECT_GT(CountOccurrences(optimized_expr_ir, "generated_bool_f0"), 0); - EXPECT_GT(CountOccurrences(optimized_expr_ir, "generated_bool_f1"), 0); - EXPECT_GT(CountOccurrences(optimized_expr_ir, "\"0_lbmap\""), 0); - EXPECT_EQ(0, CountOccurrences(optimized_expr_ir, "\"1_lbmap\"")); - EXPECT_EQ(0, CountOccurrences(optimized_expr_ir, "\"2_lbmap\"")); auto input0 = MakeArrowArrayBool({true, true, false, false, false}, {true, true, true, false, false}); @@ -628,11 +623,9 @@ TEST_F(TestProjector, TestNestedBetweenCommonSubexpressionFoldIR) { std::shared_ptr projector; ASSERT_OK(Projector::Make(schema, {expr}, configuration, &projector)); - const auto unoptimized_expr_ir = - ExtractFunctionIR(projector->DumpUnoptimizedIR(), "expr_0_0"); - const auto optimized_expr_ir = ExtractFunctionIR(projector->DumpIR(), "expr_0_0"); + ASSERT_OK_AND_ASSIGN(auto unoptimized_ir, projector->DumpUnoptimizedIR()); + const auto unoptimized_expr_ir = ExtractFunctionIR(unoptimized_ir, "expr_0_0"); ASSERT_FALSE(unoptimized_expr_ir.empty()); - ASSERT_FALSE(optimized_expr_ir.empty()); EXPECT_EQ(1, CountOccurrences(unoptimized_expr_ir, "call i1 @greater_than_or_equal_to_int32_int32")); @@ -641,10 +634,6 @@ TEST_F(TestProjector, TestNestedBetweenCommonSubexpressionFoldIR) { EXPECT_GT(CountOccurrences(unoptimized_expr_ir, "\"0_lbmap\""), 0); EXPECT_EQ(0, CountOccurrences(unoptimized_expr_ir, "\"1_lbmap\"")); EXPECT_EQ(0, CountOccurrences(unoptimized_expr_ir, "\"2_lbmap\"")); - EXPECT_EQ(0, CountOccurrences(optimized_expr_ir, - "call i1 @greater_than_or_equal_to_int32_int32")); - EXPECT_EQ(0, CountOccurrences(optimized_expr_ir, - "call i1 @less_than_or_equal_to_int32_int32")); auto values = MakeArrowArrayInt32({5, 1, 10, 7, 4}, {true, true, true, true, false}); auto lowers = MakeArrowArrayInt32({1, 2, 10, 6, 0}, {true, true, true, false, true}); From 2458a837c418bc61479aa6108674f0f98739544c Mon Sep 17 00:00:00 2001 From: likun Date: Sat, 18 Jul 2026 17:31:16 +0800 Subject: [PATCH 4/5] GH-50371: [C++][Gandiva] Narrow CSE safety and add benchmarks --- cpp/src/gandiva/CMakeLists.txt | 1 + cpp/src/gandiva/engine.h | 2 + cpp/src/gandiva/expr_cse.cc | 160 ++++++--------- cpp/src/gandiva/expr_cse.h | 9 +- cpp/src/gandiva/expr_cse_test.cc | 226 ++++++++++++++++++++++ cpp/src/gandiva/expr_decomposer.cc | 3 +- cpp/src/gandiva/function_registry.cc | 11 +- cpp/src/gandiva/function_registry.h | 7 +- cpp/src/gandiva/llvm_generator.cc | 30 ++- cpp/src/gandiva/llvm_generator.h | 10 + cpp/src/gandiva/projector.cc | 6 +- cpp/src/gandiva/tests/filter_test.cc | 4 +- cpp/src/gandiva/tests/micro_benchmarks.cc | 212 ++++++++++++++++++++ cpp/src/gandiva/tests/projector_test.cc | 88 +++++++-- 14 files changed, 629 insertions(+), 140 deletions(-) create mode 100644 cpp/src/gandiva/expr_cse_test.cc diff --git a/cpp/src/gandiva/CMakeLists.txt b/cpp/src/gandiva/CMakeLists.txt index 349e15e77995..f2d61a248a5e 100644 --- a/cpp/src/gandiva/CMakeLists.txt +++ b/cpp/src/gandiva/CMakeLists.txt @@ -260,6 +260,7 @@ add_gandiva_test(internals-test annotator_test.cc tree_expr_test.cc encrypt_utils_test.cc + expr_cse_test.cc expr_decomposer_test.cc exported_funcs_registry_test.cc expression_registry_test.cc diff --git a/cpp/src/gandiva/engine.h b/cpp/src/gandiva/engine.h index 2d4035db5b35..6e8aefc39e0f 100644 --- a/cpp/src/gandiva/engine.h +++ b/cpp/src/gandiva/engine.h @@ -90,6 +90,8 @@ class GANDIVA_EXPORT Engine { /// Return the generated IR before the optimizer pipeline runs. const std::string& unoptimized_ir(); + bool has_unoptimized_ir() const { return !unoptimized_module_ir_.empty(); } + /// Load the function IRs that can be accessed in the module. Status LoadFunctionIRs(); diff --git a/cpp/src/gandiva/expr_cse.cc b/cpp/src/gandiva/expr_cse.cc index 95c8dd5f9e0b..23bc2ce64dd6 100644 --- a/cpp/src/gandiva/expr_cse.cc +++ b/cpp/src/gandiva/expr_cse.cc @@ -17,13 +17,13 @@ #include "gandiva/expr_cse.h" -#include +#include #include #include -#include #include #include +#include "arrow/util/hash_util.h" #include "gandiva/condition.h" #include "gandiva/function_registry.h" #include "gandiva/function_signature.h" @@ -33,12 +33,47 @@ namespace gandiva { namespace { +enum class NodeKind { + kField, + kLiteral, + kFunction, +}; + +struct NodeKey { + NodeKind kind; + std::string name; + std::string type; + std::vector children; + + bool operator==(const NodeKey& other) const { + return kind == other.kind && name == other.name && type == other.type && + children == other.children; + } +}; + +struct NodeKeyHash { + size_t operator()(const NodeKey& key) const { + size_t hash = static_cast(key.kind); + arrow::internal::hash_combine(hash, key.name); + arrow::internal::hash_combine(hash, key.type); + for (auto child : key.children) { + arrow::internal::hash_combine(hash, child); + } + return hash; + } +}; + struct FoldedNode { NodePtr node; - std::string key; + size_t id; bool can_eliminate; }; +struct CanonicalNode { + NodePtr node; + size_t id; +}; + class CommonSubexpressionFolder { public: explicit CommonSubexpressionFolder(const FunctionRegistry& registry) @@ -63,17 +98,15 @@ class CommonSubexpressionFolder { private: FoldedNode Fold(const NodePtr& node) { if (node == nullptr) { - return {nullptr, "null", false}; + return Fresh(nullptr); } if (auto field_node = std::dynamic_pointer_cast(node)) { - auto key = "field:" + field_node->field()->ToString(); - return {Intern(key, node), std::move(key), true}; + return Intern({NodeKind::kField, field_node->field()->ToString(), "", {}}, node); } if (auto literal_node = std::dynamic_pointer_cast(node)) { - auto key = "literal:" + literal_node->ToString(); - return {Intern(key, node), std::move(key), true}; + return Intern({NodeKind::kLiteral, literal_node->ToString(), "", {}}, node); } if (auto function_node = std::dynamic_pointer_cast(node)) { @@ -88,19 +121,17 @@ class CommonSubexpressionFolder { return FoldIf(node, *if_node); } - // InExpressionNode stores constants in unordered_sets, so ToString() is not a - // stable structural key. Keep it opaque in this conservative pass. - std::stringstream ss; - ss << "opaque:" << node.get(); - return {node, ss.str(), false}; + // InExpressionNode stores constants in unordered_sets, so keep it opaque in this + // conservative pass. + return Fresh(node); } FoldedNode FoldFunction(const NodePtr& original, const FunctionNode& function_node) { NodeVector children; children.reserve(function_node.children().size()); - std::vector child_keys; - child_keys.reserve(function_node.children().size()); + std::vector child_ids; + child_ids.reserve(function_node.children().size()); bool children_unchanged = true; bool children_can_eliminate = true; @@ -109,81 +140,38 @@ class CommonSubexpressionFolder { children_unchanged = children_unchanged && folded.node == child; children_can_eliminate = children_can_eliminate && folded.can_eliminate; children.push_back(folded.node); - child_keys.push_back(std::move(folded.key)); + child_ids.push_back(folded.id); } auto desc = function_node.descriptor(); auto return_type = desc->return_type() == NULLPTR ? std::string("untyped") : desc->return_type()->ToString(); - auto key = JoinKey("function", desc->name(), return_type, child_keys); + NodeKey key{NodeKind::kFunction, desc->name(), std::move(return_type), + std::move(child_ids)}; auto folded_node = children_unchanged ? original : std::make_shared(desc->name(), children, desc->return_type()); bool can_eliminate = children_can_eliminate && IsFunctionSafe(function_node); - return {can_eliminate ? Intern(key, folded_node) : folded_node, std::move(key), - can_eliminate}; + return can_eliminate ? Intern(std::move(key), folded_node) : Fresh(folded_node); } FoldedNode FoldBoolean(const NodePtr& original, const BooleanNode& boolean_node) { NodeVector folded_children; - std::vector folded_keys; - std::unordered_set seen_eliminable_children; + folded_children.reserve(boolean_node.children().size()); bool children_unchanged = true; - bool children_can_eliminate = true; for (const auto& child : boolean_node.children()) { auto folded = Fold(child); children_unchanged = children_unchanged && folded.node == child; - AppendBooleanChild(boolean_node.expr_type(), std::move(folded), - &seen_eliminable_children, &folded_children, &folded_keys, - &children_unchanged, &children_can_eliminate); + folded_children.push_back(std::move(folded.node)); } - if (boolean_node.children().size() > 1 && folded_children.size() == 1 && - children_can_eliminate) { - return {folded_children[0], folded_keys[0], true}; - } - - auto op = boolean_node.expr_type() == BooleanNode::AND ? "and" : "or"; - auto key = JoinKey("boolean", op, "bool", folded_keys); auto folded_node = - children_unchanged && folded_children.size() == boolean_node.children().size() + children_unchanged ? original : std::make_shared(boolean_node.expr_type(), folded_children); - return {children_can_eliminate ? Intern(key, folded_node) : folded_node, - std::move(key), children_can_eliminate}; - } - - void AppendBooleanChild(BooleanNode::ExprType expr_type, FoldedNode folded, - std::unordered_set* seen_eliminable_children, - NodeVector* folded_children, - std::vector* folded_keys, bool* children_unchanged, - bool* children_can_eliminate) { - auto nested_boolean = std::dynamic_pointer_cast(folded.node); - if (nested_boolean != nullptr && nested_boolean->expr_type() == expr_type && - folded.can_eliminate) { - *children_unchanged = false; - for (const auto& nested_child : nested_boolean->children()) { - auto nested_folded = Fold(nested_child); - AppendBooleanChild(expr_type, std::move(nested_folded), seen_eliminable_children, - folded_children, folded_keys, children_unchanged, - children_can_eliminate); - } - return; - } - - if (folded.can_eliminate) { - if (!seen_eliminable_children->insert(folded.key).second) { - *children_unchanged = false; - return; - } - } else { - *children_can_eliminate = false; - } - - folded_children->push_back(folded.node); - folded_keys->push_back(std::move(folded.key)); + return Fresh(folded_node); } FoldedNode FoldIf(const NodePtr& original, const IfNode& if_node) { @@ -191,13 +179,6 @@ class CommonSubexpressionFolder { auto then_node = Fold(if_node.then_node()); auto else_node = Fold(if_node.else_node()); - if (condition.can_eliminate && then_node.can_eliminate && else_node.can_eliminate && - then_node.key == else_node.key) { - return then_node; - } - - std::vector child_keys{condition.key, then_node.key, else_node.key}; - auto key = JoinKey("if", "", if_node.return_type()->ToString(), child_keys); bool children_unchanged = condition.node == if_node.condition() && then_node.node == if_node.then_node() && else_node.node == if_node.else_node(); @@ -206,7 +187,7 @@ class CommonSubexpressionFolder { condition.node, then_node.node, else_node.node, if_node.return_type()); - return {folded_node, std::move(key), false}; + return Fresh(folded_node); } bool IsFunctionSafe(const FunctionNode& node) const { @@ -216,37 +197,24 @@ class CommonSubexpressionFolder { if (native_function == nullptr) { return false; } - return CanReuseNativeFunction(*native_function); + return CanReuseNativeFunction(registry_, *native_function); } - NodePtr Intern(const std::string& key, const NodePtr& node) { + FoldedNode Intern(NodeKey key, const NodePtr& node) { auto it = canonical_nodes_.find(key); if (it != canonical_nodes_.end()) { - return it->second; + return {it->second.node, it->second.id, true}; } - canonical_nodes_.emplace(key, node); - return node; + auto id = next_id_++; + canonical_nodes_.emplace(std::move(key), CanonicalNode{node, id}); + return {node, id, true}; } - std::string JoinKey(const std::string& kind, const std::string& name, - const std::string& return_type, - const std::vector& child_keys) const { - std::stringstream ss; - ss << kind << ":" << name << ":" << return_type << "("; - bool first = true; - for (const auto& child_key : child_keys) { - if (!first) { - ss << ","; - } - ss << child_key.size() << ":" << child_key; - first = false; - } - ss << ")"; - return ss.str(); - } + FoldedNode Fresh(const NodePtr& node) { return {node, next_id_++, false}; } const FunctionRegistry& registry_; - std::unordered_map canonical_nodes_; + std::unordered_map canonical_nodes_; + size_t next_id_ = 1; }; } // namespace diff --git a/cpp/src/gandiva/expr_cse.h b/cpp/src/gandiva/expr_cse.h index 30ed9dbb3d8a..93ed1e9788e2 100644 --- a/cpp/src/gandiva/expr_cse.h +++ b/cpp/src/gandiva/expr_cse.h @@ -18,16 +18,17 @@ #pragma once #include "gandiva/expression.h" +#include "gandiva/function_registry.h" #include "gandiva/gandiva_aliases.h" #include "gandiva/native_function.h" namespace gandiva { class Condition; -class FunctionRegistry; - -inline bool CanReuseNativeFunction(const NativeFunction& native_function) { - return native_function.result_nullable_type() != kResultNullInternal && +inline bool CanReuseNativeFunction(const FunctionRegistry& registry, + const NativeFunction& native_function) { + return registry.IsBuiltIn(native_function) && + native_function.result_nullable_type() != kResultNullInternal && !native_function.NeedsContext() && !native_function.NeedsFunctionHolder() && !native_function.CanReturnErrors(); } diff --git a/cpp/src/gandiva/expr_cse_test.cc b/cpp/src/gandiva/expr_cse_test.cc new file mode 100644 index 000000000000..9fc7578a75e5 --- /dev/null +++ b/cpp/src/gandiva/expr_cse_test.cc @@ -0,0 +1,226 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "gandiva/expr_cse.h" + +#include + +#include +#include +#include +#include + +#include "arrow/testing/gtest_util.h" +#include "gandiva/function_registry.h" +#include "gandiva/node.h" +#include "gandiva/tree_expr_builder.h" + +namespace gandiva { +namespace { + +ExpressionPtr MakeExpression(NodePtr root, const std::string& name = "out") { + auto return_type = root->return_type(); + return TreeExprBuilder::MakeExpression(std::move(root), + arrow::field(name, return_type)); +} + +NodePtr MakeBinary(const std::string& name, const FieldPtr& left, const FieldPtr& right, + const DataTypePtr& return_type) { + return TreeExprBuilder::MakeFunction( + name, {TreeExprBuilder::MakeField(left), TreeExprBuilder::MakeField(right)}, + return_type); +} + +void ExpectDuplicateRootsNotFolded(const NodePtr& left, const NodePtr& right) { + auto registry = default_function_registry(); + auto folded = FoldCommonSubexpressions( + *registry, {MakeExpression(left, "left"), MakeExpression(right, "right")}); + ASSERT_EQ(2, folded.size()); + EXPECT_NE(folded[0]->root(), folded[1]->root()); +} + +int32_t VolatileIdentity(int32_t value) { return value; } + +TEST(CommonSubexpressionFolderTest, FoldsRepeatedSafeSubtrees) { + auto left = arrow::field("left", arrow::int32()); + auto right = arrow::field("right", arrow::int32()); + auto root = + TreeExprBuilder::MakeFunction("multiply", + {MakeBinary("add", left, right, arrow::int32()), + MakeBinary("add", left, right, arrow::int32())}, + arrow::int32()); + + auto folded = + FoldCommonSubexpressions(*default_function_registry(), {MakeExpression(root)}); + auto folded_root = std::dynamic_pointer_cast(folded[0]->root()); + ASSERT_NE(nullptr, folded_root); + ASSERT_EQ(2, folded_root->children().size()); + EXPECT_EQ(folded_root->children()[0], folded_root->children()[1]); +} + +TEST(CommonSubexpressionFolderTest, FoldsAcrossMultipleOutputExpressions) { + auto left = arrow::field("left", arrow::int32()); + auto right = arrow::field("right", arrow::int32()); + auto folded = FoldCommonSubexpressions( + *default_function_registry(), + {MakeExpression(MakeBinary("add", left, right, arrow::int32()), "out1"), + MakeExpression(MakeBinary("add", left, right, arrow::int32()), "out2")}); + + ASSERT_EQ(2, folded.size()); + EXPECT_EQ(folded[0]->root(), folded[1]->root()); +} + +TEST(CommonSubexpressionFolderTest, DoesNotApplyBooleanAlgebra) { + auto left = arrow::field("left", arrow::boolean()); + auto right = arrow::field("right", arrow::boolean()); + auto make_and = [&] { + return TreeExprBuilder::MakeAnd( + {TreeExprBuilder::MakeField(left), TreeExprBuilder::MakeField(right)}); + }; + auto root = TreeExprBuilder::MakeOr({make_and(), make_and()}); + + auto folded = + FoldCommonSubexpressions(*default_function_registry(), {MakeExpression(root)}); + auto folded_or = std::dynamic_pointer_cast(folded[0]->root()); + ASSERT_NE(nullptr, folded_or); + ASSERT_EQ(2, folded_or->children().size()); + EXPECT_NE(folded_or->children()[0], folded_or->children()[1]); + + auto first_and = std::dynamic_pointer_cast(folded_or->children()[0]); + auto second_and = std::dynamic_pointer_cast(folded_or->children()[1]); + ASSERT_NE(nullptr, first_and); + ASSERT_NE(nullptr, second_and); + EXPECT_EQ(first_and->children()[0], second_and->children()[0]); + EXPECT_EQ(first_and->children()[1], second_and->children()[1]); +} + +TEST(CommonSubexpressionFolderTest, DoesNotApplyIfAlgebra) { + auto condition = arrow::field("condition", arrow::boolean()); + auto value = arrow::field("value", arrow::int32()); + auto root = TreeExprBuilder::MakeIf(TreeExprBuilder::MakeField(condition), + TreeExprBuilder::MakeField(value), + TreeExprBuilder::MakeField(value), arrow::int32()); + + auto folded = + FoldCommonSubexpressions(*default_function_registry(), {MakeExpression(root)}); + auto folded_if = std::dynamic_pointer_cast(folded[0]->root()); + ASSERT_NE(nullptr, folded_if); + EXPECT_EQ(folded_if->then_node(), folded_if->else_node()); +} + +TEST(CommonSubexpressionFolderTest, DoesNotFoldNullInternalFunctions) { + auto type = arrow::decimal128(38, 0); + auto value = arrow::field("value", type); + auto make_node = [&] { + return TreeExprBuilder::MakeFunction("castDECIMALNullOnOverflow", + {TreeExprBuilder::MakeField(value)}, type); + }; + ExpectDuplicateRootsNotFolded(make_node(), make_node()); +} + +TEST(CommonSubexpressionFolderTest, DoesNotFoldFunctionsNeedingContext) { + auto value = arrow::field("value", arrow::int32()); + auto make_node = [&] { + return TreeExprBuilder::MakeFunction("chr", {TreeExprBuilder::MakeField(value)}, + arrow::utf8()); + }; + ExpectDuplicateRootsNotFolded(make_node(), make_node()); +} + +TEST(CommonSubexpressionFolderTest, DoesNotFoldFunctionsNeedingHolder) { + auto make_node = [] { + return TreeExprBuilder::MakeFunction("random", {}, arrow::float64()); + }; + ExpectDuplicateRootsNotFolded(make_node(), make_node()); +} + +TEST(CommonSubexpressionFolderTest, DoesNotFoldFunctionsThatCanReturnErrors) { + auto left = arrow::field("left", arrow::int32()); + auto right = arrow::field("right", arrow::int32()); + ExpectDuplicateRootsNotFolded(MakeBinary("divide", left, right, arrow::int32()), + MakeBinary("divide", left, right, arrow::int32())); +} + +TEST(CommonSubexpressionFolderTest, DoesNotFoldSafeParentWithUnsafeChild) { + auto left = arrow::field("left", arrow::int32()); + auto right = arrow::field("right", arrow::int32()); + auto make_node = [&] { + auto divide = MakeBinary("divide", left, right, arrow::int32()); + return TreeExprBuilder::MakeFunction( + "add", {divide, TreeExprBuilder::MakeLiteral(int32_t{1})}, arrow::int32()); + }; + ExpectDuplicateRootsNotFolded(make_node(), make_node()); +} + +TEST(CommonSubexpressionFolderTest, DoesNotFoldUnknownFunctions) { + auto value = arrow::field("value", arrow::int32()); + auto make_node = [&] { + return TreeExprBuilder::MakeFunction( + "unknown_function", {TreeExprBuilder::MakeField(value)}, arrow::int32()); + }; + ExpectDuplicateRootsNotFolded(make_node(), make_node()); +} + +TEST(CommonSubexpressionFolderTest, DoesNotFoldRegisteredFunctionsByDefault) { + auto registry = std::make_shared(); + NativeFunction function("volatile_identity", {}, {arrow::int32()}, arrow::int32(), + kResultNullIfNull, "volatile_identity_int32"); + ASSERT_OK(registry->Register(std::move(function), + reinterpret_cast(&VolatileIdentity))); + + auto value = arrow::field("value", arrow::int32()); + auto make_node = [&] { + return TreeExprBuilder::MakeFunction( + "volatile_identity", {TreeExprBuilder::MakeField(value)}, arrow::int32()); + }; + auto folded = FoldCommonSubexpressions( + *registry, + {MakeExpression(make_node(), "left"), MakeExpression(make_node(), "right")}); + + ASSERT_EQ(2, folded.size()); + EXPECT_NE(folded[0]->root(), folded[1]->root()); +} + +TEST(CommonSubexpressionFolderTest, KeepsInExpressionsOpaque) { + auto value = arrow::field("value", arrow::int32()); + auto make_node = [&] { + return TreeExprBuilder::MakeInExpressionInt32(TreeExprBuilder::MakeField(value), + std::unordered_set{1, 2, 3}); + }; + ExpectDuplicateRootsNotFolded(make_node(), make_node()); +} + +TEST(CommonSubexpressionFolderTest, DistinguishesStructuralDifferences) { + auto left = arrow::field("left", arrow::int32()); + auto right = arrow::field("right", arrow::int32()); + auto registry = default_function_registry(); + auto folded = FoldCommonSubexpressions( + *registry, + {MakeExpression(MakeBinary("add", left, right, arrow::int32()), "ordered"), + MakeExpression(MakeBinary("add", right, left, arrow::int32()), "reversed"), + MakeExpression(TreeExprBuilder::MakeLiteral(int32_t{1}), "int32_literal"), + MakeExpression(TreeExprBuilder::MakeLiteral(int32_t{2}), "other_value"), + MakeExpression(TreeExprBuilder::MakeLiteral(int64_t{1}), "int64_literal")}); + + ASSERT_EQ(5, folded.size()); + EXPECT_NE(folded[0]->root(), folded[1]->root()); + EXPECT_NE(folded[2]->root(), folded[3]->root()); + EXPECT_NE(folded[2]->root(), folded[4]->root()); +} + +} // namespace +} // namespace gandiva diff --git a/cpp/src/gandiva/expr_decomposer.cc b/cpp/src/gandiva/expr_decomposer.cc index 4007966b0059..b3e356eda780 100644 --- a/cpp/src/gandiva/expr_decomposer.cc +++ b/cpp/src/gandiva/expr_decomposer.cc @@ -94,7 +94,8 @@ bool ExprDecomposer::CanReuseDecomposition(const Node& node) { auto desc = function_node->descriptor(); FunctionSignature signature(desc->name(), desc->params(), desc->return_type()); const NativeFunction* native_function = registry_.LookupSignature(signature); - reusable = native_function != nullptr && CanReuseNativeFunction(*native_function); + reusable = + native_function != nullptr && CanReuseNativeFunction(registry_, *native_function); for (const auto& child : function_node->children()) { reusable = reusable && CanReuseDecomposition(*child); } diff --git a/cpp/src/gandiva/function_registry.cc b/cpp/src/gandiva/function_registry.cc index 7ef9178d8d8a..d62d746968e9 100644 --- a/cpp/src/gandiva/function_registry.cc +++ b/cpp/src/gandiva/function_registry.cc @@ -84,13 +84,20 @@ const NativeFunction* FunctionRegistry::LookupSignature( return got == pc_registry_map_.end() ? nullptr : got->second; } -Status FunctionRegistry::Add(NativeFunction func) { +bool FunctionRegistry::IsBuiltIn(const NativeFunction& function) const { + return built_in_functions_.find(&function) != built_in_functions_.end(); +} + +Status FunctionRegistry::Add(NativeFunction func, bool is_built_in) { if (pc_registry_.size() == kMaxFunctionSignatures) { return Status::CapacityError("Exceeded max function signatures limit of ", kMaxFunctionSignatures); } pc_registry_.emplace_back(std::move(func)); const auto& last_func = pc_registry_.back(); + if (is_built_in) { + built_in_functions_.insert(&last_func); + } for (const auto& func_signature : last_func.signatures()) { pc_registry_map_.emplace(&func_signature, &last_func); } @@ -148,7 +155,7 @@ arrow::Result> MakeDefaultFunctionRegistry() { GetHashFunctionRegistry(), GetMathOpsFunctionRegistry(), GetStringFunctionRegistry(), GetDateTimeArithmeticFunctionRegistry()}) { for (const auto& func_signature : funcs) { - ARROW_RETURN_NOT_OK(registry->Add(func_signature)); + ARROW_RETURN_NOT_OK(registry->Add(func_signature, true)); } } return registry; diff --git a/cpp/src/gandiva/function_registry.h b/cpp/src/gandiva/function_registry.h index 24b64fac5f3f..ce1d59b304fb 100644 --- a/cpp/src/gandiva/function_registry.h +++ b/cpp/src/gandiva/function_registry.h @@ -20,6 +20,7 @@ #include #include #include +#include #include #include "arrow/buffer.h" @@ -48,6 +49,9 @@ class GANDIVA_EXPORT FunctionRegistry { /// Lookup a pre-compiled function by its signature. const NativeFunction* LookupSignature(const FunctionSignature& signature) const; + /// Return whether a function is part of Gandiva's built-in registry. + bool IsBuiltIn(const NativeFunction& function) const; + /// \brief register a set of functions into the function registry from a given bitcode /// file arrow::Status Register(const std::vector& funcs, @@ -85,11 +89,12 @@ class GANDIVA_EXPORT FunctionRegistry { private: std::vector pc_registry_; SignatureMap pc_registry_map_; + std::unordered_set built_in_functions_; std::vector> bitcode_memory_buffers_; std::vector> c_functions_; FunctionHolderMakerRegistry holder_maker_registry_; - Status Add(NativeFunction func); + Status Add(NativeFunction func, bool is_built_in = false); }; /// \brief get the default function registry diff --git a/cpp/src/gandiva/llvm_generator.cc b/cpp/src/gandiva/llvm_generator.cc index a42f71a1f7f1..3268fd5e83b3 100644 --- a/cpp/src/gandiva/llvm_generator.cc +++ b/cpp/src/gandiva/llvm_generator.cc @@ -1241,15 +1241,27 @@ LValuePtr LLVMGenerator::Visitor::BuildIfElse(llvm::Value* condition, LValuePtr LLVMGenerator::Visitor::BuildValueAndValidity(const ValueValidityPair& pair) { // generate code for value - auto value_expr = pair.value_expr(); - value_expr->Accept(*this); - auto value = result()->data(); - auto length = result()->length(); + auto value = BuildDex(pair.value_expr()); // generate code for validity auto validity = BuildCombinedValidity(pair.validity_exprs()); - return std::make_shared(value, length, validity); + return std::make_shared(value->data(), value->length(), validity); +} + +LValuePtr LLVMGenerator::Visitor::BuildDex(const DexPtr& dex) { + auto* block = ir_builder()->GetInsertBlock(); + auto cached = dex_cache_.find(dex.get()); + if (cached != dex_cache_.end() && cached->second.block == block) { + return cached->second.value; + } + + dex->Accept(*this); + auto value = result(); + if (ir_builder()->GetInsertBlock() == block) { + dex_cache_.insert_or_assign(dex.get(), CachedDexValue{block, value}); + } + return value; } LValuePtr LLVMGenerator::Visitor::BuildFunctionCall(const NativeFunction* func, @@ -1335,9 +1347,7 @@ std::vector LLVMGenerator::Visitor::BuildParams( // build the function params, along with the validities. for (auto& pair : args) { // build value. - DexPtr value_expr = pair->value_expr(); - value_expr->Accept(*this); - LValue& result_ref = *result(); + LValue& result_ref = *BuildDex(pair->value_expr()); // append all the parameters corresponding to this LValue. result_ref.AppendFunctionParams(¶ms); @@ -1359,8 +1369,8 @@ llvm::Value* LLVMGenerator::Visitor::BuildCombinedValidity(const DexVector& vali llvm::Value* isValid = types->true_constant(); for (auto& dex : validities) { - dex->Accept(*this); - isValid = builder->CreateAnd(isValid, result()->data(), "validityBitAnd"); + auto value = BuildDex(dex); + isValid = builder->CreateAnd(isValid, value->data(), "validityBitAnd"); } ADD_VISITOR_TRACE("combined validity is %T", isValid); return isValid; diff --git a/cpp/src/gandiva/llvm_generator.h b/cpp/src/gandiva/llvm_generator.h index 485b2466e494..e9099f56d924 100644 --- a/cpp/src/gandiva/llvm_generator.h +++ b/cpp/src/gandiva/llvm_generator.h @@ -22,6 +22,7 @@ #include #include #include +#include #include #include "arrow/util/macros.h" @@ -85,6 +86,7 @@ class GANDIVA_EXPORT LLVMGenerator { llvm::Module* module() { return engine_->module(); } const std::string& ir() { return engine_->ir(); } const std::string& unoptimized_ir() { return engine_->unoptimized_ir(); } + bool has_unoptimized_ir() const { return engine_->has_unoptimized_ir(); } private: explicit LLVMGenerator(bool cached, @@ -152,6 +154,9 @@ class GANDIVA_EXPORT LLVMGenerator { // Generate the code to build the validity and the value for the given pair. LValuePtr BuildValueAndValidity(const ValueValidityPair& pair); + // Generate or reuse code for a decomposed value in the current basic block. + LValuePtr BuildDex(const DexPtr& dex); + // Generate code to build the params. std::vector BuildParams(int holder_idx, const ValueValidityPairVector& args, @@ -189,6 +194,11 @@ class GANDIVA_EXPORT LLVMGenerator { llvm::Value* arg_context_ptr_; llvm::Value* loop_var_; bool has_arena_allocs_; + struct CachedDexValue { + llvm::BasicBlock* block; + LValuePtr value; + }; + std::unordered_map dex_cache_; }; // Generate the code for one expression for default mode, with the output of diff --git a/cpp/src/gandiva/projector.cc b/cpp/src/gandiva/projector.cc index fa7f772a3179..27b94654d93a 100644 --- a/cpp/src/gandiva/projector.cc +++ b/cpp/src/gandiva/projector.cc @@ -289,11 +289,9 @@ Status Projector::ValidateArrayDataCapacity(const arrow::ArrayData& array_data, const std::string& Projector::DumpIR() { return llvm_generator_->ir(); } Result Projector::DumpUnoptimizedIR() { - ARROW_RETURN_IF(!configuration_->dump_ir(), - Status::Invalid("IR dumping is not enabled for this projector")); ARROW_RETURN_IF( - built_from_cache_, - Status::Invalid("Unoptimized IR is unavailable for projectors built from cache")); + !llvm_generator_->has_unoptimized_ir(), + Status::Invalid("Unoptimized IR was not captured when this projector was built")); return llvm_generator_->unoptimized_ir(); } diff --git a/cpp/src/gandiva/tests/filter_test.cc b/cpp/src/gandiva/tests/filter_test.cc index 42e6264211d3..eef97f76fff6 100644 --- a/cpp/src/gandiva/tests/filter_test.cc +++ b/cpp/src/gandiva/tests/filter_test.cc @@ -87,7 +87,7 @@ TEST_F(TestFilter, TestFilterCache) { EXPECT_FALSE(should_be_new_filter->GetBuiltFromCache()); } -TEST_F(TestFilter, TestCommonSubexpressionFoldAndCache) { +TEST_F(TestFilter, TestCommonSubexpressionSafety) { auto field0 = field("filter_cse_f0", int32()); auto field1 = field("filter_cse_f1", int32()); auto schema = arrow::schema({field0, field1}); @@ -121,7 +121,7 @@ TEST_F(TestFilter, TestCommonSubexpressionFoldAndCache) { auto equivalent_condition = TreeExprBuilder::MakeCondition(make_less_than()); std::shared_ptr cached_filter; ASSERT_OK(Filter::Make(schema, equivalent_condition, configuration, &cached_filter)); - ASSERT_TRUE(cached_filter->GetBuiltFromCache()); + ASSERT_FALSE(cached_filter->GetBuiltFromCache()); std::shared_ptr cached_selection_vector; ASSERT_OK( diff --git a/cpp/src/gandiva/tests/micro_benchmarks.cc b/cpp/src/gandiva/tests/micro_benchmarks.cc index 450e691323ca..4da9099ca0c2 100644 --- a/cpp/src/gandiva/tests/micro_benchmarks.cc +++ b/cpp/src/gandiva/tests/micro_benchmarks.cc @@ -17,12 +17,20 @@ #include +#include +#include +#include +#include +#include + #include "arrow/memory_pool.h" #include "arrow/status.h" #include "arrow/testing/gtest_util.h" #include "arrow/type_fwd.h" #include "benchmark/benchmark.h" #include "gandiva/decimal_type_util.h" +#include "gandiva/expr_cse.h" +#include "gandiva/filter.h" #include "gandiva/projector.h" #include "gandiva/tests/test_util.h" #include "gandiva/tests/timed_evaluate.h" @@ -35,6 +43,190 @@ using arrow::int32; using arrow::int64; using arrow::utf8; +namespace { + +enum class CsePattern : int64_t { + kDeepUnique, + kRepeatedSafe, + kRepeatedUnsafe, +}; + +struct CseBenchmarkExpression { + SchemaPtr schema; + ExpressionPtr expression; + ConditionPtr condition; +}; + +NodePtr MakeBalancedTree(int64_t leaves, const std::function& make_leaf) { + if (leaves <= 1) { + return make_leaf(); + } + auto left = MakeBalancedTree(leaves / 2, make_leaf); + auto right = MakeBalancedTree(leaves - leaves / 2, make_leaf); + return TreeExprBuilder::MakeFunction("add", {left, right}, arrow::float64()); +} + +CseBenchmarkExpression MakeCseBenchmarkExpression(CsePattern pattern, int64_t size, + int64_t salt) { + auto left = field("cse_left", arrow::float64()); + auto right = field("cse_right", arrow::float64()); + auto schema = arrow::schema({left, right}); + + NodePtr root; + switch (pattern) { + case CsePattern::kDeepUnique: { + root = TreeExprBuilder::MakeField(left); + for (int64_t i = 0; i < size; ++i) { + root = TreeExprBuilder::MakeFunction( + "add", {root, TreeExprBuilder::MakeLiteral(static_cast(i + 1))}, + arrow::float64()); + } + break; + } + case CsePattern::kRepeatedSafe: { + auto make_leaf = [&] { + auto make_sum = [&] { + return TreeExprBuilder::MakeFunction( + "add", + {TreeExprBuilder::MakeField(left), TreeExprBuilder::MakeField(right)}, + arrow::float64()); + }; + return TreeExprBuilder::MakeFunction("multiply", {make_sum(), make_sum()}, + arrow::float64()); + }; + root = MakeBalancedTree(size, make_leaf); + break; + } + case CsePattern::kRepeatedUnsafe: { + auto make_leaf = [&] { + return TreeExprBuilder::MakeFunction( + "divide", + {TreeExprBuilder::MakeField(left), TreeExprBuilder::MakeField(right)}, + arrow::float64()); + }; + root = MakeBalancedTree(size, make_leaf); + break; + } + } + + root = TreeExprBuilder::MakeFunction( + "add", {root, TreeExprBuilder::MakeLiteral(static_cast(salt) + 0.125)}, + arrow::float64()); + auto condition_root = TreeExprBuilder::MakeFunction( + "greater_than", {root, TreeExprBuilder::MakeLiteral(0.0)}, arrow::boolean()); + return {schema, + TreeExprBuilder::MakeExpression( + root, field("cse_result_" + std::to_string(salt), arrow::float64())), + TreeExprBuilder::MakeCondition(condition_root)}; +} + +void CseFoldOnly(benchmark::State& state) { + auto pattern = static_cast(state.range(0)); + auto size = state.range(1); + int64_t iteration = 0; + for (auto _ : state) { + state.PauseTiming(); + auto input = MakeCseBenchmarkExpression(pattern, size, iteration++); + state.ResumeTiming(); + auto folded = FoldCommonSubexpressions(*default_function_registry(), + ExpressionVector{input.expression}); + benchmark::DoNotOptimize(folded); + } +} + +void CseProjectorBuild(benchmark::State& state) { + auto pattern = static_cast(state.range(0)); + auto size = state.range(1); + int64_t iteration = 0; + int64_t total_ir_bytes = 0; + for (auto _ : state) { + state.PauseTiming(); + auto input = MakeCseBenchmarkExpression(pattern, size, iteration++); + auto configuration = std::make_shared( + true, default_function_registry(), /*dump_ir=*/true); + state.ResumeTiming(); + + std::shared_ptr projector; + ASSERT_OK( + Projector::Make(input.schema, {input.expression}, configuration, &projector)); + + state.PauseTiming(); + ASSERT_OK_AND_ASSIGN(auto ir, projector->DumpUnoptimizedIR()); + total_ir_bytes += static_cast(ir.size()); + benchmark::DoNotOptimize(projector); + state.ResumeTiming(); + } + state.counters["ir_bytes"] = + benchmark::Counter(total_ir_bytes, benchmark::Counter::kAvgIterations); +} + +void CseFilterBuild(benchmark::State& state) { + auto pattern = static_cast(state.range(0)); + auto size = state.range(1); + int64_t iteration = 0; + for (auto _ : state) { + state.PauseTiming(); + auto input = MakeCseBenchmarkExpression(pattern, size, iteration++); + auto configuration = std::make_shared( + true, default_function_registry(), /*dump_ir=*/false); + state.ResumeTiming(); + + std::shared_ptr filter; + ASSERT_OK(Filter::Make(input.schema, input.condition, configuration, &filter)); + benchmark::DoNotOptimize(filter); + } +} + +void CseProjectorEvaluate(benchmark::State& state) { + auto pattern = static_cast(state.range(0)); + auto size = state.range(1); + auto input = MakeCseBenchmarkExpression(pattern, size, 0); + std::shared_ptr projector; + ASSERT_OK( + Projector::Make(input.schema, {input.expression}, TestConfiguration(), &projector)); + auto left = MakeArrowArrayFloat64(std::vector(1024, 12.0)); + auto right = MakeArrowArrayFloat64(std::vector(1024, 3.0)); + auto batch = arrow::RecordBatch::Make(input.schema, 1024, {left, right}); + + for (auto _ : state) { + arrow::ArrayVector outputs; + ASSERT_OK(projector->Evaluate(*batch, arrow::default_memory_pool(), &outputs)); + benchmark::DoNotOptimize(outputs); + } + state.SetItemsProcessed(state.iterations() * batch->num_rows()); +} + +void CseFilterEvaluate(benchmark::State& state) { + auto pattern = static_cast(state.range(0)); + auto size = state.range(1); + auto input = MakeCseBenchmarkExpression(pattern, size, 0); + std::shared_ptr filter; + ASSERT_OK(Filter::Make(input.schema, input.condition, TestConfiguration(), &filter)); + auto left = MakeArrowArrayFloat64(std::vector(1024, 12.0)); + auto right = MakeArrowArrayFloat64(std::vector(1024, 3.0)); + auto batch = arrow::RecordBatch::Make(input.schema, 1024, {left, right}); + std::shared_ptr selection; + ASSERT_OK(SelectionVector::MakeInt32(batch->num_rows(), arrow::default_memory_pool(), + &selection)); + + for (auto _ : state) { + ASSERT_OK(filter->Evaluate(*batch, selection)); + benchmark::DoNotOptimize(selection); + } + state.SetItemsProcessed(state.iterations() * batch->num_rows()); +} + +void CseBenchmarkArguments(benchmark::internal::Benchmark* benchmark) { + for (auto pattern : {CsePattern::kDeepUnique, CsePattern::kRepeatedSafe, + CsePattern::kRepeatedUnsafe}) { + for (auto size : {10, 100, 1000}) { + benchmark->Args({static_cast(pattern), size}); + } + } +} + +} // namespace + static void TimedTestAdd3(benchmark::State& state) { // schema for input fields auto field0 = field("f0", int64()); @@ -491,6 +683,26 @@ static void DecimalAdd3Large(benchmark::State& state) { } BENCHMARK(TimedTestExprCompilation)->Unit(benchmark::kMicrosecond); +BENCHMARK(CseFoldOnly) + ->Apply(CseBenchmarkArguments) + ->ArgNames({"pattern", "size"}) + ->Unit(benchmark::kMicrosecond); +BENCHMARK(CseProjectorBuild) + ->Apply(CseBenchmarkArguments) + ->ArgNames({"pattern", "size"}) + ->Unit(benchmark::kMillisecond); +BENCHMARK(CseFilterBuild) + ->Apply(CseBenchmarkArguments) + ->ArgNames({"pattern", "size"}) + ->Unit(benchmark::kMillisecond); +BENCHMARK(CseProjectorEvaluate) + ->Apply(CseBenchmarkArguments) + ->ArgNames({"pattern", "size"}) + ->Unit(benchmark::kMicrosecond); +BENCHMARK(CseFilterEvaluate) + ->Apply(CseBenchmarkArguments) + ->ArgNames({"pattern", "size"}) + ->Unit(benchmark::kMicrosecond); BENCHMARK(TimedTestAdd3)->Unit(benchmark::kMicrosecond); BENCHMARK(TimedTestBigNested)->Unit(benchmark::kMicrosecond); BENCHMARK(TimedTestExtractYear)->Unit(benchmark::kMicrosecond); diff --git a/cpp/src/gandiva/tests/projector_test.cc b/cpp/src/gandiva/tests/projector_test.cc index 6fb9c2ffd812..3d890af7a186 100644 --- a/cpp/src/gandiva/tests/projector_test.cc +++ b/cpp/src/gandiva/tests/projector_test.cc @@ -453,7 +453,7 @@ TEST_F(TestProjector, TestCommonSubexpressionEliminationIR) { const auto unoptimized_expr_ir = ExtractFunctionIR(unoptimized_ir, "expr_0_0"); ASSERT_FALSE(unoptimized_expr_ir.empty()); - EXPECT_EQ(2, CountOccurrences(unoptimized_expr_ir, "call i32 @add_int32_int32")); + EXPECT_EQ(1, CountOccurrences(unoptimized_expr_ir, "call i32 @add_int32_int32")); EXPECT_EQ(1, CountOccurrences(unoptimized_expr_ir, "call i32 @multiply_int32_int32")); auto input0 = MakeArrowArrayInt32({1, 2, 3, 4}, {true, true, false, true}); @@ -481,10 +481,28 @@ TEST_F(TestProjector, TestUnoptimizedIRUnavailableForCachedProjector) { ASSERT_OK(Projector::Make(schema, {expr}, configuration, &cached_projector)); ASSERT_TRUE(cached_projector->GetBuiltFromCache()); ASSERT_RAISES_WITH_MESSAGE( - Invalid, "Invalid: Unoptimized IR is unavailable for projectors built from cache", + Invalid, "Invalid: Unoptimized IR was not captured when this projector was built", cached_projector->DumpUnoptimizedIR()); } +TEST_F(TestProjector, TestUnoptimizedIRAvailabilityUsesCapturedState) { + auto field0 = arrow::field("mutable_config_f0", arrow::int32()); + auto schema = arrow::schema({field0}); + auto expr = + TreeExprBuilder::MakeExpression(TreeExprBuilder::MakeField(field0), + arrow::field("mutable_config_out", arrow::int32())); + auto configuration = std::make_shared( + true, gandiva::default_function_registry(), /*dump_ir=*/false); + + std::shared_ptr projector; + ASSERT_OK(Projector::Make(schema, {expr}, configuration, &projector)); + configuration->set_dump_ir(true); + + ASSERT_RAISES_WITH_MESSAGE( + Invalid, "Invalid: Unoptimized IR was not captured when this projector was built", + projector->DumpUnoptimizedIR()); +} + TEST_F(TestProjector, TestNestedCommonSubexpressionEliminationIR) { auto field0 = arrow::field("nested_cse_f0", arrow::int32()); auto field1 = arrow::field("nested_cse_f1", arrow::int32()); @@ -513,8 +531,8 @@ TEST_F(TestProjector, TestNestedCommonSubexpressionEliminationIR) { const auto unoptimized_expr_ir = ExtractFunctionIR(unoptimized_ir, "expr_0_0"); ASSERT_FALSE(unoptimized_expr_ir.empty()); - EXPECT_EQ(5, CountOccurrences(unoptimized_expr_ir, "call i32 @add_int32_int32")); - EXPECT_EQ(2, CountOccurrences(unoptimized_expr_ir, "call i32 @multiply_int32_int32")); + EXPECT_EQ(2, CountOccurrences(unoptimized_expr_ir, "call i32 @add_int32_int32")); + EXPECT_EQ(1, CountOccurrences(unoptimized_expr_ir, "call i32 @multiply_int32_int32")); auto input0 = MakeArrowArrayInt32({1, 2, 3, 4}, {true, true, false, true}); auto input1 = MakeArrowArrayInt32({10, -2, 3, 5}, {true, true, true, false}); @@ -522,7 +540,43 @@ TEST_F(TestProjector, TestNestedCommonSubexpressionEliminationIR) { ExpectProjectorOutput(projector, schema, {input0, input1}, expected, pool_); } -TEST_F(TestProjector, TestGeneratedIfCommonSubexpressionEliminationIR) { +TEST_F(TestProjector, TestCommonSubexpressionNotReusedAcrossIfBranches) { + auto condition_field = arrow::field("branch_cse_cond", arrow::boolean()); + auto left = arrow::field("branch_cse_left", arrow::int32()); + auto right = arrow::field("branch_cse_right", arrow::int32()); + auto schema = arrow::schema({condition_field, left, right}); + + auto make_sum = [&] { + return TreeExprBuilder::MakeFunction( + "add", {TreeExprBuilder::MakeField(left), TreeExprBuilder::MakeField(right)}, + arrow::int32()); + }; + auto else_node = TreeExprBuilder::MakeFunction( + "multiply", {make_sum(), TreeExprBuilder::MakeLiteral(int32_t{2})}, arrow::int32()); + auto if_node = TreeExprBuilder::MakeIf(TreeExprBuilder::MakeField(condition_field), + make_sum(), else_node, arrow::int32()); + auto expr = TreeExprBuilder::MakeExpression( + if_node, arrow::field("branch_cse_out", arrow::int32())); + + auto configuration = std::make_shared( + true, gandiva::default_function_registry(), /*dump_ir=*/true); + std::shared_ptr projector; + ASSERT_OK(Projector::Make(schema, {expr}, configuration, &projector)); + + ASSERT_OK_AND_ASSIGN(auto unoptimized_ir, projector->DumpUnoptimizedIR()); + const auto unoptimized_expr_ir = ExtractFunctionIR(unoptimized_ir, "expr_0_0"); + ASSERT_FALSE(unoptimized_expr_ir.empty()); + EXPECT_EQ(2, CountOccurrences(unoptimized_expr_ir, "call i32 @add_int32_int32")); + + auto conditions = MakeArrowArrayBool({true, false, true, false}); + auto left_values = MakeArrowArrayInt32({1, 2, 3, 4}); + auto right_values = MakeArrowArrayInt32({10, 20, 30, 40}); + auto expected = MakeArrowArrayInt32({11, 44, 33, 88}); + ExpectProjectorOutput(projector, schema, {conditions, left_values, right_values}, + expected, pool_); +} + +TEST_F(TestProjector, TestIfAlgebraicFoldIsNotApplied) { auto condition_field = arrow::field("generated_if_cond", arrow::boolean()); auto value_field = arrow::field("generated_if_value", arrow::int32()); auto schema = arrow::schema({condition_field, value_field}); @@ -544,11 +598,11 @@ TEST_F(TestProjector, TestGeneratedIfCommonSubexpressionEliminationIR) { ASSERT_FALSE(unoptimized_expr_ir.empty()); EXPECT_NE(std::string::npos, unoptimized_expr_ir.find("generated_if_value")); - EXPECT_EQ(std::string::npos, unoptimized_expr_ir.find("generated_if_cond")); - EXPECT_EQ(std::string::npos, unoptimized_expr_ir.find("then:")); - EXPECT_EQ(std::string::npos, unoptimized_expr_ir.find("else:")); - EXPECT_EQ(std::string::npos, unoptimized_expr_ir.find("validAndMatch")); - EXPECT_EQ(std::string::npos, unoptimized_expr_ir.find("res_value = phi")); + EXPECT_NE(std::string::npos, unoptimized_expr_ir.find("generated_if_cond")); + EXPECT_NE(std::string::npos, unoptimized_expr_ir.find("then:")); + EXPECT_NE(std::string::npos, unoptimized_expr_ir.find("else:")); + EXPECT_NE(std::string::npos, unoptimized_expr_ir.find("validAndMatch")); + EXPECT_NE(std::string::npos, unoptimized_expr_ir.find("res_value = phi")); auto conditions = MakeArrowArrayBool({true, false, false, true}, {true, true, false, true}); @@ -557,7 +611,7 @@ TEST_F(TestProjector, TestGeneratedIfCommonSubexpressionEliminationIR) { ExpectProjectorOutput(projector, schema, {conditions, values}, expected, pool_); } -TEST_F(TestProjector, TestGeneratedBooleanCommonSubexpressionEliminationIR) { +TEST_F(TestProjector, TestBooleanAlgebraicFoldIsNotApplied) { auto field0 = arrow::field("generated_bool_f0", arrow::boolean()); auto field1 = arrow::field("generated_bool_f1", arrow::boolean()); auto schema = arrow::schema({field0, field1}); @@ -582,9 +636,6 @@ TEST_F(TestProjector, TestGeneratedBooleanCommonSubexpressionEliminationIR) { EXPECT_GT(CountOccurrences(unoptimized_expr_ir, "short_circuit"), 0); EXPECT_GT(CountOccurrences(unoptimized_expr_ir, "non_short_circuit"), 0); EXPECT_GT(CountOccurrences(unoptimized_expr_ir, "res_value = phi"), 0); - EXPECT_GT(CountOccurrences(unoptimized_expr_ir, "\"0_lbmap\""), 0); - EXPECT_EQ(0, CountOccurrences(unoptimized_expr_ir, "\"1_lbmap\"")); - EXPECT_EQ(0, CountOccurrences(unoptimized_expr_ir, "\"2_lbmap\"")); auto input0 = MakeArrowArrayBool({true, true, false, false, false}, {true, true, true, false, false}); @@ -595,7 +646,7 @@ TEST_F(TestProjector, TestGeneratedBooleanCommonSubexpressionEliminationIR) { ExpectProjectorOutput(projector, schema, {input0, input1}, expected, pool_); } -TEST_F(TestProjector, TestNestedBetweenCommonSubexpressionFoldIR) { +TEST_F(TestProjector, TestNestedBetweenDoesNotReuseAcrossBooleanControlFlow) { auto value = arrow::field("nested_between_value", arrow::int32()); auto lower = arrow::field("nested_between_lower", arrow::int32()); auto upper = arrow::field("nested_between_upper", arrow::int32()); @@ -627,13 +678,10 @@ TEST_F(TestProjector, TestNestedBetweenCommonSubexpressionFoldIR) { const auto unoptimized_expr_ir = ExtractFunctionIR(unoptimized_ir, "expr_0_0"); ASSERT_FALSE(unoptimized_expr_ir.empty()); - EXPECT_EQ(1, CountOccurrences(unoptimized_expr_ir, + EXPECT_EQ(3, CountOccurrences(unoptimized_expr_ir, "call i1 @greater_than_or_equal_to_int32_int32")); - EXPECT_EQ(1, CountOccurrences(unoptimized_expr_ir, + EXPECT_EQ(3, CountOccurrences(unoptimized_expr_ir, "call i1 @less_than_or_equal_to_int32_int32")); - EXPECT_GT(CountOccurrences(unoptimized_expr_ir, "\"0_lbmap\""), 0); - EXPECT_EQ(0, CountOccurrences(unoptimized_expr_ir, "\"1_lbmap\"")); - EXPECT_EQ(0, CountOccurrences(unoptimized_expr_ir, "\"2_lbmap\"")); auto values = MakeArrowArrayInt32({5, 1, 10, 7, 4}, {true, true, true, true, false}); auto lowers = MakeArrowArrayInt32({1, 2, 10, 6, 0}, {true, true, true, false, true}); From a0981500d5623797e9e347b199e649f2aee849c8 Mon Sep 17 00:00:00 2001 From: likun Date: Mon, 20 Jul 2026 17:05:59 +0800 Subject: [PATCH 5/5] GH-50371: [C++][Gandiva] Harden CSE shared-DAG handling --- cpp/src/gandiva/expr_cse.cc | 12 ++ cpp/src/gandiva/expr_cse_test.cc | 21 ++ cpp/src/gandiva/expr_decomposer.cc | 4 + cpp/src/gandiva/expr_decomposer_test.cc | 18 ++ cpp/src/gandiva/function_registry_test.cc | 42 ++++ cpp/src/gandiva/tests/micro_benchmarks.cc | 19 +- cpp/src/gandiva/tests/projector_test.cc | 237 ++++++++++++++++++++++ 7 files changed, 352 insertions(+), 1 deletion(-) diff --git a/cpp/src/gandiva/expr_cse.cc b/cpp/src/gandiva/expr_cse.cc index 23bc2ce64dd6..5867947191fd 100644 --- a/cpp/src/gandiva/expr_cse.cc +++ b/cpp/src/gandiva/expr_cse.cc @@ -101,6 +101,17 @@ class CommonSubexpressionFolder { return Fresh(nullptr); } + auto cached = folded_nodes_.find(node.get()); + if (cached != folded_nodes_.end()) { + return cached->second; + } + + auto folded = FoldUncached(node); + folded_nodes_.emplace(node.get(), folded); + return folded; + } + + FoldedNode FoldUncached(const NodePtr& node) { if (auto field_node = std::dynamic_pointer_cast(node)) { return Intern({NodeKind::kField, field_node->field()->ToString(), "", {}}, node); } @@ -213,6 +224,7 @@ class CommonSubexpressionFolder { FoldedNode Fresh(const NodePtr& node) { return {node, next_id_++, false}; } const FunctionRegistry& registry_; + std::unordered_map folded_nodes_; std::unordered_map canonical_nodes_; size_t next_id_ = 1; }; diff --git a/cpp/src/gandiva/expr_cse_test.cc b/cpp/src/gandiva/expr_cse_test.cc index 9fc7578a75e5..fe713c827ff8 100644 --- a/cpp/src/gandiva/expr_cse_test.cc +++ b/cpp/src/gandiva/expr_cse_test.cc @@ -72,6 +72,27 @@ TEST(CommonSubexpressionFolderTest, FoldsRepeatedSafeSubtrees) { EXPECT_EQ(folded_root->children()[0], folded_root->children()[1]); } +TEST(CommonSubexpressionFolderTest, FoldsDeepSharedDagOncePerNode) { + auto value = arrow::field("value", arrow::int32()); + NodePtr root = TreeExprBuilder::MakeField(value); + constexpr int kDepth = 24; + for (int depth = 0; depth < kDepth; ++depth) { + root = TreeExprBuilder::MakeFunction("add", {root, root}, arrow::int32()); + } + + auto folded = + FoldCommonSubexpressions(*default_function_registry(), {MakeExpression(root)}); + auto current = folded[0]->root(); + for (int depth = 0; depth < kDepth; ++depth) { + auto function = std::dynamic_pointer_cast(current); + ASSERT_NE(nullptr, function); + ASSERT_EQ(2, function->children().size()); + EXPECT_EQ(function->children()[0], function->children()[1]); + current = function->children()[0]; + } + EXPECT_NE(nullptr, std::dynamic_pointer_cast(current)); +} + TEST(CommonSubexpressionFolderTest, FoldsAcrossMultipleOutputExpressions) { auto left = arrow::field("left", arrow::int32()); auto right = arrow::field("right", arrow::int32()); diff --git a/cpp/src/gandiva/expr_decomposer.cc b/cpp/src/gandiva/expr_decomposer.cc index b3e356eda780..2a37cdde82c2 100644 --- a/cpp/src/gandiva/expr_decomposer.cc +++ b/cpp/src/gandiva/expr_decomposer.cc @@ -136,7 +136,11 @@ Status ExprDecomposer::Visit(const FunctionNode& in_node) { // These functions are decomposable, merge the validity bits of the children. std::vector merged_validity; + std::unordered_set merged_args; for (auto& decomposed : args) { + if (!merged_args.insert(decomposed.get()).second) { + continue; + } // Merge the validity_expressions of the children to build a combined validity // expression. merged_validity.insert(merged_validity.end(), decomposed->validity_exprs().begin(), diff --git a/cpp/src/gandiva/expr_decomposer_test.cc b/cpp/src/gandiva/expr_decomposer_test.cc index 194c13bc82c8..8aa961861643 100644 --- a/cpp/src/gandiva/expr_decomposer_test.cc +++ b/cpp/src/gandiva/expr_decomposer_test.cc @@ -19,11 +19,13 @@ #include +#include "arrow/testing/gtest_util.h" #include "gandiva/annotator.h" #include "gandiva/dex.h" #include "gandiva/function_registry.h" #include "gandiva/gandiva_aliases.h" #include "gandiva/node.h" +#include "gandiva/tree_expr_builder.h" namespace gandiva { @@ -34,6 +36,22 @@ class TestExprDecomposer : public ::testing::Test { std::shared_ptr registry_ = default_function_registry(); }; +TEST_F(TestExprDecomposer, TestSharedDagValidityIsMergedOnce) { + auto value = arrow::field("value", arrow::int32()); + NodePtr root = TreeExprBuilder::MakeField(value); + constexpr int kDepth = 24; + for (int depth = 0; depth < kDepth; ++depth) { + root = TreeExprBuilder::MakeFunction("add", {root, root}, arrow::int32()); + } + + Annotator annotator; + ExprDecomposer decomposer(*registry_, annotator); + ValueValidityPairPtr decomposed; + ASSERT_OK(decomposer.Decompose(*root, &decomposed)); + ASSERT_NE(nullptr, decomposed); + EXPECT_EQ(1, decomposed->validity_exprs().size()); +} + TEST_F(TestExprDecomposer, TestStackSimple) { Annotator annotator; ExprDecomposer decomposer(*registry_, annotator); diff --git a/cpp/src/gandiva/function_registry_test.cc b/cpp/src/gandiva/function_registry_test.cc index bbe72c0ee970..514bb3f4ecc3 100644 --- a/cpp/src/gandiva/function_registry_test.cc +++ b/cpp/src/gandiva/function_registry_test.cc @@ -20,6 +20,7 @@ #include #include #include +#include #include #include @@ -27,6 +28,16 @@ namespace gandiva { +arrow::Result> MakeDefaultFunctionRegistry(); + +namespace { + +int32_t CustomIdentity(int32_t value) { return value; } + +int32_t CustomAdd(int32_t left, int32_t right) { return left + right; } + +} // namespace + class TestFunctionRegistry : public ::testing::Test { protected: std::shared_ptr registry_ = gandiva::default_function_registry(); @@ -122,4 +133,35 @@ TEST_F(TestFunctionRegistry, TestNoDuplicates) { "different precompiled functions:\n" << stream.str(); } + +TEST_F(TestFunctionRegistry, TestBuiltInIdentitySurvivesCustomRegistrations) { + ASSERT_OK_AND_ASSIGN(auto registry, MakeDefaultFunctionRegistry()); + FunctionSignature add_signature("add", {arrow::int32(), arrow::int32()}, + arrow::int32()); + const NativeFunction* built_in_add = registry->LookupSignature(add_signature); + ASSERT_NE(nullptr, built_in_add); + ASSERT_TRUE(registry->IsBuiltIn(*built_in_add)); + + for (int i = 0; i < 64; ++i) { + auto name = "custom_identity_" + std::to_string(i); + NativeFunction function(name, {}, {arrow::int32()}, arrow::int32(), kResultNullIfNull, + name + "_int32"); + ASSERT_OK(registry->Register(std::move(function), + reinterpret_cast(&CustomIdentity))); + } + + EXPECT_EQ(built_in_add, registry->LookupSignature(add_signature)); + EXPECT_TRUE(registry->IsBuiltIn(*built_in_add)); + ASSERT_NE(registry->begin(), registry->end()); + EXPECT_FALSE(registry->IsBuiltIn(*registry->back())); + + NativeFunction shadow_add("add", {}, {arrow::int32(), arrow::int32()}, arrow::int32(), + kResultNullIfNull, "custom_add_int32_int32"); + ASSERT_OK( + registry->Register(std::move(shadow_add), reinterpret_cast(&CustomAdd))); + + EXPECT_EQ(built_in_add, registry->LookupSignature(add_signature)); + EXPECT_TRUE(registry->IsBuiltIn(*built_in_add)); + EXPECT_FALSE(registry->IsBuiltIn(*registry->back())); +} } // namespace gandiva diff --git a/cpp/src/gandiva/tests/micro_benchmarks.cc b/cpp/src/gandiva/tests/micro_benchmarks.cc index 4da9099ca0c2..54bbc96221b4 100644 --- a/cpp/src/gandiva/tests/micro_benchmarks.cc +++ b/cpp/src/gandiva/tests/micro_benchmarks.cc @@ -49,6 +49,7 @@ enum class CsePattern : int64_t { kDeepUnique, kRepeatedSafe, kRepeatedUnsafe, + kSharedDag, }; struct CseBenchmarkExpression { @@ -107,6 +108,15 @@ CseBenchmarkExpression MakeCseBenchmarkExpression(CsePattern pattern, int64_t si root = MakeBalancedTree(size, make_leaf); break; } + case CsePattern::kSharedDag: { + root = TreeExprBuilder::MakeFunction( + "add", {TreeExprBuilder::MakeField(left), TreeExprBuilder::MakeField(right)}, + arrow::float64()); + for (int64_t i = 0; i < size; ++i) { + root = TreeExprBuilder::MakeFunction("add", {root, root}, arrow::float64()); + } + break; + } } root = TreeExprBuilder::MakeFunction( @@ -225,6 +235,13 @@ void CseBenchmarkArguments(benchmark::internal::Benchmark* benchmark) { } } +void CseFoldBenchmarkArguments(benchmark::internal::Benchmark* benchmark) { + CseBenchmarkArguments(benchmark); + for (auto depth : {10, 20, 24}) { + benchmark->Args({static_cast(CsePattern::kSharedDag), depth}); + } +} + } // namespace static void TimedTestAdd3(benchmark::State& state) { @@ -684,7 +701,7 @@ static void DecimalAdd3Large(benchmark::State& state) { BENCHMARK(TimedTestExprCompilation)->Unit(benchmark::kMicrosecond); BENCHMARK(CseFoldOnly) - ->Apply(CseBenchmarkArguments) + ->Apply(CseFoldBenchmarkArguments) ->ArgNames({"pattern", "size"}) ->Unit(benchmark::kMicrosecond); BENCHMARK(CseProjectorBuild) diff --git a/cpp/src/gandiva/tests/projector_test.cc b/cpp/src/gandiva/tests/projector_test.cc index 3d890af7a186..8f369af68afe 100644 --- a/cpp/src/gandiva/tests/projector_test.cc +++ b/cpp/src/gandiva/tests/projector_test.cc @@ -540,6 +540,103 @@ TEST_F(TestProjector, TestNestedCommonSubexpressionEliminationIR) { ExpectProjectorOutput(projector, schema, {input0, input1}, expected, pool_); } +TEST_F(TestProjector, TestCommonSubexpressionEliminationForVarLenInput) { + auto value = arrow::field("varlen_cse_value", arrow::utf8()); + auto schema = arrow::schema({value}); + + auto make_length = [&] { + return TreeExprBuilder::MakeFunction( + "octet_length", {TreeExprBuilder::MakeField(value)}, arrow::int32()); + }; + auto root = TreeExprBuilder::MakeFunction("add", {make_length(), make_length()}, + arrow::int32()); + auto expr = TreeExprBuilder::MakeExpression( + root, arrow::field("varlen_cse_out", arrow::int32())); + + auto configuration = std::make_shared( + true, gandiva::default_function_registry(), /*dump_ir=*/true); + std::shared_ptr projector; + ASSERT_OK(Projector::Make(schema, {expr}, configuration, &projector)); + + ASSERT_OK_AND_ASSIGN(auto unoptimized_ir, projector->DumpUnoptimizedIR()); + const auto unoptimized_expr_ir = ExtractFunctionIR(unoptimized_ir, "expr_0_0"); + ASSERT_FALSE(unoptimized_expr_ir.empty()); + EXPECT_EQ(1, CountOccurrences(unoptimized_expr_ir, "call i32 @octet_length_utf8")); + + auto values = + MakeArrowArrayUtf8({"", "abc", "路学", "ignored"}, {true, true, true, false}); + auto expected = MakeArrowArrayInt32({0, 6, 12, 0}, {true, true, true, false}); + ExpectProjectorOutput(projector, schema, {values}, expected, pool_); +} + +TEST_F(TestProjector, TestCommonSubexpressionEliminationForNullNeverFunction) { + auto value = arrow::field("null_never_cse_value", arrow::utf8()); + auto schema = arrow::schema({value}); + + auto make_is_null = [&] { + return TreeExprBuilder::MakeFunction("isnull", {TreeExprBuilder::MakeField(value)}, + arrow::boolean()); + }; + auto root = TreeExprBuilder::MakeFunction("equal", {make_is_null(), make_is_null()}, + arrow::boolean()); + auto expr = TreeExprBuilder::MakeExpression( + root, arrow::field("null_never_cse_out", arrow::boolean())); + + auto configuration = std::make_shared( + true, gandiva::default_function_registry(), /*dump_ir=*/true); + std::shared_ptr projector; + ASSERT_OK(Projector::Make(schema, {expr}, configuration, &projector)); + + ASSERT_OK_AND_ASSIGN(auto unoptimized_ir, projector->DumpUnoptimizedIR()); + const auto unoptimized_expr_ir = ExtractFunctionIR(unoptimized_ir, "expr_0_0"); + ASSERT_FALSE(unoptimized_expr_ir.empty()); + EXPECT_EQ(1, CountOccurrences(unoptimized_expr_ir, "call i1 @isnull_utf8")); + + auto values = MakeArrowArrayUtf8({"present", "ignored", "", "also present"}, + {true, false, true, true}); + auto expected = MakeArrowArrayBool({true, true, true, true}); + ExpectProjectorOutput(projector, schema, {values}, expected, pool_); +} + +TEST_F(TestProjector, TestCommonSubexpressionEliminationAcrossMultipleOutputs) { + auto left = arrow::field("multi_output_cse_left", arrow::int32()); + auto right = arrow::field("multi_output_cse_right", arrow::int32()); + auto schema = arrow::schema({left, right}); + + auto make_sum = [&] { + return TreeExprBuilder::MakeFunction( + "add", {TreeExprBuilder::MakeField(left), TreeExprBuilder::MakeField(right)}, + arrow::int32()); + }; + auto first = TreeExprBuilder::MakeExpression( + make_sum(), arrow::field("multi_output_cse_first", arrow::int32())); + auto second = TreeExprBuilder::MakeExpression( + make_sum(), arrow::field("multi_output_cse_second", arrow::int32())); + + auto configuration = std::make_shared( + true, gandiva::default_function_registry(), /*dump_ir=*/true); + std::shared_ptr projector; + ASSERT_OK(Projector::Make(schema, {first, second}, configuration, &projector)); + + ASSERT_OK_AND_ASSIGN(auto unoptimized_ir, projector->DumpUnoptimizedIR()); + const auto first_expr_ir = ExtractFunctionIR(unoptimized_ir, "expr_0_0"); + const auto second_expr_ir = ExtractFunctionIR(unoptimized_ir, "expr_1_0"); + ASSERT_FALSE(first_expr_ir.empty()); + ASSERT_FALSE(second_expr_ir.empty()); + EXPECT_EQ(1, CountOccurrences(first_expr_ir, "call i32 @add_int32_int32")); + EXPECT_EQ(1, CountOccurrences(second_expr_ir, "call i32 @add_int32_int32")); + + auto left_values = MakeArrowArrayInt32({1, 2, 3}, {true, true, false}); + auto right_values = MakeArrowArrayInt32({10, 20, 30}); + auto expected = MakeArrowArrayInt32({11, 22, 0}, {true, true, false}); + auto batch = arrow::RecordBatch::Make(schema, 3, {left_values, right_values}); + arrow::ArrayVector outputs; + ASSERT_OK(projector->Evaluate(*batch, pool_, &outputs)); + ASSERT_EQ(2, outputs.size()); + EXPECT_ARROW_ARRAY_EQUALS(expected, outputs[0]); + EXPECT_ARROW_ARRAY_EQUALS(expected, outputs[1]); +} + TEST_F(TestProjector, TestCommonSubexpressionNotReusedAcrossIfBranches) { auto condition_field = arrow::field("branch_cse_cond", arrow::boolean()); auto left = arrow::field("branch_cse_left", arrow::int32()); @@ -576,6 +673,146 @@ TEST_F(TestProjector, TestCommonSubexpressionNotReusedAcrossIfBranches) { expected, pool_); } +TEST_F(TestProjector, TestCommonSubexpressionRebuiltAfterIfMerge) { + auto condition = arrow::field("merge_cse_condition", arrow::boolean()); + auto left = arrow::field("merge_cse_left", arrow::int32()); + auto right = arrow::field("merge_cse_right", arrow::int32()); + auto schema = arrow::schema({condition, left, right}); + + auto make_sum = [&] { + return TreeExprBuilder::MakeFunction( + "add", {TreeExprBuilder::MakeField(left), TreeExprBuilder::MakeField(right)}, + arrow::int32()); + }; + auto else_value = TreeExprBuilder::MakeFunction( + "add", {make_sum(), TreeExprBuilder::MakeLiteral(int32_t{1})}, arrow::int32()); + auto conditional = TreeExprBuilder::MakeIf(TreeExprBuilder::MakeField(condition), + make_sum(), else_value, arrow::int32()); + auto root = + TreeExprBuilder::MakeFunction("add", {conditional, make_sum()}, arrow::int32()); + auto expr = TreeExprBuilder::MakeExpression( + root, arrow::field("merge_cse_out", arrow::int32())); + + auto configuration = std::make_shared( + true, gandiva::default_function_registry(), /*dump_ir=*/true); + std::shared_ptr projector; + ASSERT_OK(Projector::Make(schema, {expr}, configuration, &projector)); + + ASSERT_OK_AND_ASSIGN(auto unoptimized_ir, projector->DumpUnoptimizedIR()); + const auto unoptimized_expr_ir = ExtractFunctionIR(unoptimized_ir, "expr_0_0"); + ASSERT_FALSE(unoptimized_expr_ir.empty()); + EXPECT_EQ(5, CountOccurrences(unoptimized_expr_ir, "call i32 @add_int32_int32")); + EXPECT_NE(std::string::npos, unoptimized_expr_ir.find("res_value = phi")); + + auto conditions = + MakeArrowArrayBool({true, false, false, true}, {true, true, false, true}); + auto left_values = MakeArrowArrayInt32({1, 2, 3, 4}, {true, true, true, false}); + auto right_values = MakeArrowArrayInt32({10, 20, 30, 40}); + auto expected = MakeArrowArrayInt32({22, 45, 67, 0}, {true, true, true, false}); + ExpectProjectorOutput(projector, schema, {conditions, left_values, right_values}, + expected, pool_); +} + +TEST_F(TestProjector, TestUnsafeFunctionsAreNotCommoned) { + auto dividend = arrow::field("unsafe_cse_dividend", arrow::int32()); + auto divisor = arrow::field("unsafe_cse_divisor", arrow::int32()); + auto schema = arrow::schema({dividend, divisor}); + + auto make_divide = [&] { + return TreeExprBuilder::MakeFunction( + "divide", + {TreeExprBuilder::MakeField(dividend), TreeExprBuilder::MakeField(divisor)}, + arrow::int32()); + }; + auto root = TreeExprBuilder::MakeFunction("add", {make_divide(), make_divide()}, + arrow::int32()); + auto expr = TreeExprBuilder::MakeExpression( + root, arrow::field("unsafe_cse_divide_out", arrow::int32())); + + auto configuration = std::make_shared( + true, gandiva::default_function_registry(), /*dump_ir=*/true); + std::shared_ptr projector; + ASSERT_OK(Projector::Make(schema, {expr}, configuration, &projector)); + + ASSERT_OK_AND_ASSIGN(auto unoptimized_ir, projector->DumpUnoptimizedIR()); + const auto unoptimized_expr_ir = ExtractFunctionIR(unoptimized_ir, "expr_0_0"); + ASSERT_FALSE(unoptimized_expr_ir.empty()); + EXPECT_EQ(2, CountOccurrences(unoptimized_expr_ir, "call i32 @divide_int32_int32")); + + auto dividends = MakeArrowArrayInt32({4, 8}); + auto divisors = MakeArrowArrayInt32({2, 0}); + auto batch = arrow::RecordBatch::Make(schema, 2, {dividends, divisors}); + arrow::ArrayVector outputs; + auto status = projector->Evaluate(*batch, pool_, &outputs); + EXPECT_EQ(StatusCode::ExecutionError, status.code()); + EXPECT_NE(std::string::npos, status.message().find("divide by zero error")); +} + +TEST_F(TestProjector, TestContextFunctionsAreNotCommoned) { + auto value = arrow::field("context_cse_value", arrow::int64()); + auto schema = arrow::schema({value}); + + auto make_chr = [&] { + return TreeExprBuilder::MakeFunction("chr", {TreeExprBuilder::MakeField(value)}, + arrow::utf8()); + }; + auto root = + TreeExprBuilder::MakeFunction("equal", {make_chr(), make_chr()}, arrow::boolean()); + auto expr = TreeExprBuilder::MakeExpression( + root, arrow::field("context_cse_out", arrow::boolean())); + + auto configuration = std::make_shared( + true, gandiva::default_function_registry(), /*dump_ir=*/true); + std::shared_ptr projector; + ASSERT_OK(Projector::Make(schema, {expr}, configuration, &projector)); + + ASSERT_OK_AND_ASSIGN(auto unoptimized_ir, projector->DumpUnoptimizedIR()); + const auto unoptimized_expr_ir = ExtractFunctionIR(unoptimized_ir, "expr_0_0"); + ASSERT_FALSE(unoptimized_expr_ir.empty()); + EXPECT_EQ(2, CountOccurrences(unoptimized_expr_ir, "@chr_int64(")); + + auto values = MakeArrowArrayInt64({65, 340, -5, 84}, {true, true, true, false}); + auto expected = + MakeArrowArrayBool({true, true, true, false}, {true, true, true, false}); + ExpectProjectorOutput(projector, schema, {values}, expected, pool_); +} + +TEST_F(TestProjector, TestHolderFunctionsAreNotCommoned) { + auto dummy = arrow::field("holder_cse_dummy", arrow::int32()); + auto schema = arrow::schema({dummy}); + + auto make_random = [] { + return TreeExprBuilder::MakeFunction("random", {}, arrow::float64()); + }; + auto root = TreeExprBuilder::MakeFunction("subtract", {make_random(), make_random()}, + arrow::float64()); + auto expr = TreeExprBuilder::MakeExpression( + root, arrow::field("holder_cse_out", arrow::float64())); + + auto configuration = std::make_shared( + true, gandiva::default_function_registry(), /*dump_ir=*/true); + std::shared_ptr projector; + ASSERT_OK(Projector::Make(schema, {expr}, configuration, &projector)); + + ASSERT_OK_AND_ASSIGN(auto unoptimized_ir, projector->DumpUnoptimizedIR()); + const auto unoptimized_expr_ir = ExtractFunctionIR(unoptimized_ir, "expr_0_0"); + ASSERT_FALSE(unoptimized_expr_ir.empty()); + EXPECT_EQ(2, CountOccurrences(unoptimized_expr_ir, "call double @gdv_fn_random")); + + constexpr int kNumRecords = 64; + auto dummy_values = MakeArrowArrayInt32(std::vector(kNumRecords, 0)); + auto batch = arrow::RecordBatch::Make(schema, kNumRecords, {dummy_values}); + arrow::ArrayVector outputs; + ASSERT_OK(projector->Evaluate(*batch, pool_, &outputs)); + auto result = std::dynamic_pointer_cast(outputs[0]); + ASSERT_NE(nullptr, result); + ASSERT_EQ(0, result->null_count()); + for (int64_t i = 0; i < result->length(); ++i) { + EXPECT_GT(result->Value(i), -1.0); + EXPECT_LT(result->Value(i), 1.0); + } +} + TEST_F(TestProjector, TestIfAlgebraicFoldIsNotApplied) { auto condition_field = arrow::field("generated_if_cond", arrow::boolean()); auto value_field = arrow::field("generated_if_value", arrow::int32());