From 981dfb1366e433bc08252e98b6b4ed37d86a3ffb Mon Sep 17 00:00:00 2001 From: Jialiang Liang Date: Wed, 29 Jul 2026 11:27:34 -0700 Subject: [PATCH 1/4] Fail fast when relevance functions cannot be pushed down Relevance search functions (match, match_phrase, query_string, ...) are Calcite marker UDFs with no executable implementation -- they exist only to be rewritten into an OpenSearch query during push down. Nothing guaranteed that rewrite happened, and the two script fallbacks in PredicateAnalyzer would serialize whatever they could not analyze into a Calcite script, including a relevance call. That script was then shipped to every data node, where code generation hit RelevanceQueryImplementor and failed, surfacing to the user as: QueryShardException[failed to create query: Failed to compile inline script [...]] -> 500, all shards failed Guard both fallbacks so a node containing a relevance function is never serialized into a script, and report it on the coordinator instead. Also: - Reject a non-indexed field or non-literal query operand in visitRelevanceFunc as PredicateAnalyzerException rather than letting an unchecked ClassCastException escape to the top-level Throwable catch, which was one route into the script fallback. - Give the last-resort UnsupportedOperationException an actionable message naming the function and the constraint, instead of only "only supported when they are pushed down". - Hoist the relevance-function detection out of RelevanceFunctionPushdownRule into UserDefinedFunctionUtils so the analyzer and the rule share one definition. Queries that legitimately push down are unaffected; the reported combination of match() with a like() filter keeps working, with the LIKE going down as a script and the relevance call as a native match query. Signed-off-by: Jialiang Liang --- .../utils/UserDefinedFunctionUtils.java | 72 +++++++++ .../function/udf/RelevanceQueryFunction.java | 12 +- ...iteRelevanceFunctionPushdownFailureIT.java | 150 ++++++++++++++++++ .../rules/RelevanceFunctionPushdownRule.java | 48 +----- .../opensearch/request/PredicateAnalyzer.java | 49 ++++++ 5 files changed, 283 insertions(+), 48 deletions(-) create mode 100644 integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteRelevanceFunctionPushdownFailureIT.java diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/UserDefinedFunctionUtils.java b/core/src/main/java/org/opensearch/sql/calcite/utils/UserDefinedFunctionUtils.java index f619d966cc8..72a87a3ee48 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/utils/UserDefinedFunctionUtils.java +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/UserDefinedFunctionUtils.java @@ -17,6 +17,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Locale; import java.util.Objects; import java.util.Set; import javax.annotation.Nullable; @@ -29,6 +30,7 @@ import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rex.RexCall; import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexVisitorImpl; import org.apache.calcite.schema.impl.AggregateFunctionImpl; import org.apache.calcite.sql.SqlAggFunction; import org.apache.calcite.sql.SqlIdentifier; @@ -81,6 +83,76 @@ public class UserDefinedFunctionUtils { ImmutableSet.of("simple_query_string", "query_string", "multi_match"); public static String IP_FUNCTION_NAME = "IP"; + /** Returns true if the given operator name is a relevance search query function. */ + public static boolean isRelevanceFunction(String operatorName) { + String lowerCased = operatorName.toLowerCase(Locale.ROOT); + return SINGLE_FIELD_RELEVANCE_FUNCTION_SET.contains(lowerCased) + || MULTI_FIELDS_RELEVANCE_FUNCTION_SET.contains(lowerCased); + } + + /** + * Checks whether a {@link RexNode} tree contains a relevance search query function such as {@code + * match} or {@code query_string}. + * + *

Relevance functions have no executable implementation: they exist only as markers to be + * rewritten into an OpenSearch query during push down (see {@code RelevanceQueryFunction}). + * Callers must use this to avoid handing such a node to any execution path that would try to + * generate code for it -- notably script push down, where the failure would surface as a shard + * side "Failed to compile inline script" error instead of a local planning error. + * + * @param node the node to inspect + * @return true if the tree contains a relevance function + */ + public static boolean containsRelevanceFunction(RexNode node) { + RelevanceFunctionFinder finder = new RelevanceFunctionFinder(); + node.accept(finder); + return finder.found; + } + + /** Visitor that detects relevance functions anywhere in a {@link RexNode} tree. */ + private static class RelevanceFunctionFinder extends RexVisitorImpl { + private boolean found = false; + + RelevanceFunctionFinder() { + super(true); + } + + @Override + public Void visitCall(RexCall call) { + if (isRelevanceFunction(call.getOperator().getName())) { + found = true; + return null; // stop descending once found + } + return super.visitCall(call); + } + } + + /** + * Returns the name of the first relevance function found in the tree, or null if there is none. + */ + public static String findRelevanceFunctionName(RexNode node) { + RelevanceFunctionNameFinder finder = new RelevanceFunctionNameFinder(); + node.accept(finder); + return finder.name; + } + + private static class RelevanceFunctionNameFinder extends RexVisitorImpl { + private String name = null; + + RelevanceFunctionNameFinder() { + super(true); + } + + @Override + public Void visitCall(RexCall call) { + if (name == null && isRelevanceFunction(call.getOperator().getName())) { + name = call.getOperator().getName().toLowerCase(Locale.ROOT); + return null; + } + return super.visitCall(call); + } + } + /** * Creates a SqlUserDefinedAggFunction that wraps a Java class implementing an aggregate function. * diff --git a/core/src/main/java/org/opensearch/sql/expression/function/udf/RelevanceQueryFunction.java b/core/src/main/java/org/opensearch/sql/expression/function/udf/RelevanceQueryFunction.java index d8e53704804..aa63fbcfff4 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/udf/RelevanceQueryFunction.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/udf/RelevanceQueryFunction.java @@ -7,6 +7,7 @@ import com.google.common.collect.ImmutableList; import java.util.List; +import java.util.Locale; import org.apache.calcite.adapter.enumerable.NotNullImplementor; import org.apache.calcite.adapter.enumerable.NullPolicy; import org.apache.calcite.adapter.enumerable.RexToLixTranslator; @@ -94,8 +95,17 @@ public static class RelevanceQueryImplementor implements NotNullImplementor { @Override public Expression implement( RexToLixTranslator translator, RexCall call, List translatedOperands) { + // Reaching code generation means the call was not rewritten into an OpenSearch query. There + // is no row-by-row implementation to fall back to, so report why rather than just that. throw new UnsupportedOperationException( - "Relevance search query functions are only supported when they are pushed down"); + String.format( + Locale.ROOT, + "Relevance search function [%s] could not be pushed down to OpenSearch, and it has no" + + " other execution path. Apply it directly to an indexed field before any" + + " command that transforms rows (eval, parse, rex, stats, top, rare, sort, head," + + " lookup), and not to a column computed by the query itself. Expression: %s", + call.getOperator().getName().toLowerCase(Locale.ROOT), + call)); } } } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteRelevanceFunctionPushdownFailureIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteRelevanceFunctionPushdownFailureIT.java new file mode 100644 index 00000000000..72baa622686 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteRelevanceFunctionPushdownFailureIT.java @@ -0,0 +1,150 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.remote; + +import static org.opensearch.sql.util.MatcherUtils.rows; +import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; +import static org.opensearch.sql.util.TestUtils.createIndexByRestClient; +import static org.opensearch.sql.util.TestUtils.getResponseBody; +import static org.opensearch.sql.util.TestUtils.isIndexExist; +import static org.opensearch.sql.util.TestUtils.performRequest; + +import java.io.IOException; +import java.util.Locale; +import org.json.JSONObject; +import org.junit.Test; +import org.opensearch.client.Request; +import org.opensearch.client.ResponseException; +import org.opensearch.sql.ppl.PPLIntegTestCase; + +/** + * Tests that relevance search functions which cannot be pushed down fail fast on the coordinator + * with an actionable message, instead of being serialized into a script that only fails when + * compiled on the shards. + * + *

Relevance functions have no row-by-row implementation -- they exist only to be rewritten into + * an OpenSearch query during push down. Before this fix, a query the analyzer could not handle was + * wrapped in a script and shipped to every data node, surfacing as {@code QueryShardException: + * Failed to compile inline script} with all shards failing. + */ +public class CalciteRelevanceFunctionPushdownFailureIT extends PPLIntegTestCase { + + private static final String TEST_INDEX = "relevance_pushdown_failure"; + + @Override + public void init() throws Exception { + super.init(); + enableCalcite(); + createTestIndex(); + } + + private void createTestIndex() throws IOException { + if (isIndexExist(client(), TEST_INDEX)) { + return; + } + createIndexByRestClient( + client(), + TEST_INDEX, + "{\"mappings\":{\"properties\":{\"body\":{\"type\":\"text\"}," + + "\"idx\":{\"type\":\"integer\"}}}}"); + Request bulk = new Request("POST", "/" + TEST_INDEX + "/_bulk?refresh=true"); + bulk.setJsonEntity( + "{\"index\":{\"_id\":\"1\"}}\n" + + "{\"body\":\"ERROR something bad happened\",\"idx\":1}\n" + + "{\"index\":{\"_id\":\"2\"}}\n" + + "{\"body\":\"INFO all good\",\"idx\":2}\n"); + performRequest(client(), bulk); + } + + private String errorOf(String query) throws IOException { + ResponseException e = + assertThrows(ResponseException.class, () -> executeQuery(query)); + return getResponseBody(e.getResponse()); + } + + private void assertFailsCleanly(String query, String expectedFunctionName) + throws IOException { + String message = errorOf(query); + assertFalse( + "Relevance function must not be pushed down as a script; it cannot compile on the shard." + + " Error was: " + + message, + message.contains("Failed to compile inline script")); + assertFalse( + "Query must fail on the coordinator, not as a shard failure. Error was: " + message, + message.contains("all shards failed")); + assertTrue( + String.format( + Locale.ROOT, "Error should name the function [%s]. Error was: %s", "match", message), + message.toLowerCase(Locale.ROOT).contains(expectedFunctionName)); + } + + /** A relevance function over a column computed by the query cannot be pushed down. */ + @Test + public void relevanceOnEvalDerivedColumnFailsCleanly() throws IOException { + assertFailsCleanly( + String.format( + Locale.ROOT, + "source=%s | eval b2 = upper(body) | where match(b2, 'ERROR') | fields idx", + TEST_INDEX), + "match"); + } + + /** Same for a column produced by parse. */ + @Test + public void relevanceOnParseDerivedColumnFailsCleanly() throws IOException { + assertFailsCleanly( + String.format( + Locale.ROOT, + "source=%s | parse body '(?\\\\w+)' | where match(lvl, 'ERROR') | fields idx", + TEST_INDEX), + "match"); + } + + /** A relevance filter above an aggregation has no scan to be pushed onto. */ + @Test + public void relevanceAboveAggregationFailsCleanly() throws IOException { + assertFailsCleanly( + String.format(Locale.ROOT, "source=%s | top 1 body | where match(body, 'ERROR')", TEST_INDEX), + "match"); + } + + /** A relevance function in a projection is never rewritten into a query. */ + @Test + public void relevanceInProjectionFailsCleanly() throws IOException { + assertFailsCleanly( + String.format( + Locale.ROOT, "source=%s | eval m = match(body, 'ERROR') | fields idx, m", TEST_INDEX), + "match"); + } + + /** + * Regression guard: the pattern from the customer report -- a relevance filter combined with a + * LIKE filter on a pure text field -- must keep working. The LIKE goes down as a script while the + * relevance function goes down as a native match query. + */ + @Test + public void relevanceCombinedWithLikeStillWorks() throws IOException { + JSONObject result = + executeQuery( + String.format( + Locale.ROOT, + "source=%s | where match(body, 'ERROR') | where like(body, '%%error%%') | fields" + + " idx", + TEST_INDEX)); + verifyDataRows(result, rows(1)); + } + + /** Regression guard: a plain relevance filter on an indexed field is unaffected. */ + @Test + public void relevanceOnIndexedFieldStillWorks() throws IOException { + JSONObject result = + executeQuery( + String.format( + Locale.ROOT, "source=%s | where match(body, 'ERROR') | fields idx", TEST_INDEX)); + verifyDataRows(result, rows(1)); + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/RelevanceFunctionPushdownRule.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/RelevanceFunctionPushdownRule.java index b49b62e0163..e2019203a33 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/RelevanceFunctionPushdownRule.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/rules/RelevanceFunctionPushdownRule.java @@ -5,18 +5,13 @@ package org.opensearch.sql.opensearch.planner.rules; -import static org.opensearch.sql.calcite.utils.UserDefinedFunctionUtils.MULTI_FIELDS_RELEVANCE_FUNCTION_SET; -import static org.opensearch.sql.calcite.utils.UserDefinedFunctionUtils.SINGLE_FIELD_RELEVANCE_FUNCTION_SET; +import static org.opensearch.sql.calcite.utils.UserDefinedFunctionUtils.containsRelevanceFunction; import org.apache.calcite.plan.RelOptRuleCall; import org.apache.calcite.rel.AbstractRelNode; import org.apache.calcite.rel.core.Filter; import org.apache.calcite.rel.logical.LogicalFilter; import org.apache.calcite.rel.rules.SubstitutionRule; -import org.apache.calcite.rex.RexCall; -import org.apache.calcite.rex.RexNode; -import org.apache.calcite.rex.RexVisitorImpl; -import org.apache.calcite.sql.SqlOperator; import org.immutables.value.Value; import org.opensearch.sql.calcite.plan.rule.OpenSearchRuleConfig; import org.opensearch.sql.calcite.utils.PlanUtils; @@ -61,47 +56,6 @@ protected void apply(RelOptRuleCall call, Filter filter, CalciteLogicalIndexScan } } - /** - * Checks if a RexNode contains any relevance functions. - * - * @param node The RexNode to check - * @return true if the node contains relevance functions, false otherwise - */ - private boolean containsRelevanceFunction(RexNode node) { - RelevanceFunctionVisitor visitor = new RelevanceFunctionVisitor(); - node.accept(visitor); - return visitor.hasRelevanceFunction(); - } - - /** Visitor to detect relevance functions in a RexNode tree. */ - private static class RelevanceFunctionVisitor extends RexVisitorImpl { - private boolean foundRelevanceFunction = false; - - RelevanceFunctionVisitor() { - super(true); - } - - @Override - public Void visitCall(RexCall call) { - SqlOperator operator = call.getOperator(); - String operatorName = operator.getName().toLowerCase(); - - // Check if this is a relevance function - if (SINGLE_FIELD_RELEVANCE_FUNCTION_SET.contains(operatorName) - || MULTI_FIELDS_RELEVANCE_FUNCTION_SET.contains(operatorName)) { - foundRelevanceFunction = true; - return null; // Stop traversing once we find a relevance function - } - - // Continue traversing the tree - return super.visitCall(call); - } - - boolean hasRelevanceFunction() { - return foundRelevanceFunction; - } - } - /** Rule configuration. */ @Value.Immutable public interface Config extends OpenSearchRuleConfig { diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java index 476e0018fcd..087901142db 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java @@ -43,6 +43,8 @@ import static org.opensearch.script.Script.DEFAULT_SCRIPT_TYPE; import static org.opensearch.sql.calcite.utils.UserDefinedFunctionUtils.MULTI_FIELDS_RELEVANCE_FUNCTION_SET; import static org.opensearch.sql.calcite.utils.UserDefinedFunctionUtils.SINGLE_FIELD_RELEVANCE_FUNCTION_SET; +import static org.opensearch.sql.calcite.utils.UserDefinedFunctionUtils.containsRelevanceFunction; +import static org.opensearch.sql.calcite.utils.UserDefinedFunctionUtils.findRelevanceFunctionName; import static org.opensearch.sql.opensearch.storage.script.CompoundedScriptEngine.COMPOUNDED_LANG_NAME; import com.google.common.base.Strings; @@ -240,6 +242,12 @@ public static QueryExpression analyzeExpression( if (e instanceof UnsupportedScriptException) { throw new ExpressionNotAnalyzableException("Can't convert " + expression, e); } + // A relevance function has no executable implementation, so it must never be serialized into + // a script: the failure would only surface on the shards as "Failed to compile inline + // script". + if (containsRelevanceFunction(expression)) { + throw new ExpressionNotAnalyzableException(relevanceNotPushableMessage(expression), e); + } try { return new ScriptQueryExpression( expression, rowType, fieldTypes, cluster, Collections.emptyMap()); @@ -249,6 +257,22 @@ public static QueryExpression analyzeExpression( } } + /** + * Builds an actionable message for a relevance function that cannot be pushed down. Reported at + * planning time on the coordinator rather than as a shard failure. + */ + static String relevanceNotPushableMessage(RexNode expression) { + String funcName = findRelevanceFunctionName(expression); + return String.format( + Locale.ROOT, + "Relevance search function [%s] cannot be pushed down to OpenSearch in this query, so it" + + " cannot be executed. It must be applied directly to an indexed field, before any" + + " command that transforms rows (eval, parse, rex, stats, top, rare, sort, head," + + " lookup). Condition: %s", + funcName == null ? "relevance" : funcName, + expression); + } + /** Traverses {@link RexNode} tree and builds OpenSearch query. */ static class Visitor extends RexVisitorImpl { @@ -424,6 +448,26 @@ private QueryExpression visitRelevanceFunc(RexCall call) { List.of( AliasPair.from(ops.get(0), funcName).value, AliasPair.from(ops.get(1), funcName).value)); + // The field operand must resolve to a real indexed field. A computed column (eval, parse, + // rex, ...) resolves to something else -- reject it as a PredicateAnalyzerException so the + // caller reports it at planning time instead of failing later on the shards. + if (!(fieldQueryOperands.get(0) instanceof NamedFieldExpression)) { + throw new PredicateAnalyzerException( + format( + Locale.ROOT, + "Relevance search function [%s] requires an indexed field as its first argument," + + " but got [%s]. Computed columns cannot be searched.", + funcName, + ops.get(0))); + } + if (!(fieldQueryOperands.get(1) instanceof LiteralExpression)) { + throw new PredicateAnalyzerException( + format( + Locale.ROOT, + "Relevance search function [%s] requires a literal query string, but got [%s].", + funcName, + ops.get(1))); + } NamedFieldExpression namedFieldExpression = (NamedFieldExpression) fieldQueryOperands.get(0); String queryLiteralOperand = ((LiteralExpression) fieldQueryOperands.get(1)).stringValue(); @@ -913,6 +957,11 @@ public Expression tryAnalyzeOperand(RexNode node) { } return qe; } catch (PredicateAnalyzerException firstFailed) { + // Never fall back to a script for a relevance function: it has no executable + // implementation, so the script would only fail when compiled on the shards. + if (containsRelevanceFunction(node)) { + throw firstFailed; + } try { QueryExpression qe = new ScriptQueryExpression(node, rowType, fieldTypes, cluster, Collections.emptyMap()); From b55674cb6d8379fc776d41019fefd2bf5ea8c7a2 Mon Sep 17 00:00:00 2001 From: Jialiang Liang Date: Wed, 29 Jul 2026 12:03:49 -0700 Subject: [PATCH 2/4] Name the offending column when relevance push down fails Follows the "reject early, not at runtime" proposal in #5491, which asks for a planning-time failure whose message names the column. The previous commit stopped the shard-side crash, but the message the user actually saw still came from RelevanceQueryImplementor and carried only positional references: Expression: match($t4, $t7) The message built in PredicateAnalyzer never reached the user at all, because CalciteLogicalIndexScan.pushDownFilter swallows the exception and returns null, leaving the filter in the plan for code generation to reject. Resolve the column against the pre-compilation plan in the planning error path instead, where the row type is still available, so the report names the column as the user wrote it: Relevance search function [match] cannot be applied to column [b2]. It must run against a field indexed by OpenSearch, and cannot be evaluated on a value the query computes (eval, parse, rex), on aggregated output (stats, top, rare), or after a row limit (head). Move the match filter so it directly follows `source=` and targets an indexed field, or use a non-relevance predicate such as `like` which can be evaluated per row. The error is also now coded UNSUPPORTED_OPERATION with the function and column exposed as context, and carries a suggestion. Extends the IT to assert the column name is present for each shape: b2 (eval), lvl (parse), body (top, and relevance in a projection). Signed-off-by: Jialiang Liang --- .../sql/calcite/utils/CalciteToolsHelper.java | 49 ++++++++++ .../utils/UserDefinedFunctionUtils.java | 89 +++++++++++++++++++ ...iteRelevanceFunctionPushdownFailureIT.java | 32 +++++-- 3 files changed, 161 insertions(+), 9 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/CalciteToolsHelper.java b/core/src/main/java/org/opensearch/sql/calcite/utils/CalciteToolsHelper.java index 54b9d4ffbaf..9dec492bd16 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/utils/CalciteToolsHelper.java +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/CalciteToolsHelper.java @@ -28,6 +28,7 @@ package org.opensearch.sql.calcite.utils; import static java.util.Objects.requireNonNull; +import static org.opensearch.sql.calcite.utils.UserDefinedFunctionUtils.findRelevanceUsage; import static org.opensearch.sql.monitor.profile.MetricName.OPTIMIZE; import com.google.common.collect.ImmutableList; @@ -37,6 +38,7 @@ import java.sql.SQLException; import java.time.Instant; import java.util.List; +import java.util.Locale; import java.util.Properties; import java.util.function.Consumer; import org.apache.calcite.adapter.enumerable.EnumerableConvention; @@ -106,6 +108,7 @@ import org.opensearch.sql.calcite.plan.rule.OpenSearchRules; import org.opensearch.sql.calcite.plan.rule.PPLSimplifyDedupRule; import org.opensearch.sql.calcite.profile.PlanProfileBuilder; +import org.opensearch.sql.calcite.utils.UserDefinedFunctionUtils.RelevanceUsage; import org.opensearch.sql.common.error.ErrorCode; import org.opensearch.sql.common.error.ErrorReport; import org.opensearch.sql.expression.function.PPLBuiltinOperators; @@ -487,6 +490,51 @@ private static String rootCauseMessage(Throwable e) { return rc != null ? rc : e.getMessage(); } + /** + * Reports a relevance function that survived push down in terms the user can act on. + * + *

Relevance functions have no executable implementation; reaching code generation means the + * planner could not rewrite the call into an OpenSearch query. The generated expression only + * carries positional references by then, so the column is resolved from the pre-compilation + * plan to name it as the user wrote it. + */ + private static void enrichRelevanceNotPushedDown( + ErrorReport.Builder report, SQLException e, RelNode rel) { + String rootCause = rootCauseMessage(e); + if (rootCause == null || !rootCause.contains("Relevance search function")) { + return; + } + RelevanceUsage usage = findRelevanceUsage(rel); + if (usage == null) { + return; + } + String target = + usage.columnName() == null + ? "the column it is applied to" + : String.format(Locale.ROOT, "column [%s]", usage.columnName()); + report + .details( + String.format( + Locale.ROOT, + "Relevance search function [%s] cannot be applied to %s. It must run against a" + + " field indexed by OpenSearch, and cannot be evaluated on a value the query" + + " computes (eval, parse, rex), on aggregated output (stats, top, rare), or" + + " after a row limit (head). Move the %s filter so it directly follows" + + " `source=` and targets an indexed field, or use a non-relevance predicate" + + " such as `like` which can be evaluated per row.", + usage.functionName(), + target, + usage.functionName())) + .code(ErrorCode.UNSUPPORTED_OPERATION) + .context("relevance_function", usage.functionName()) + .context("relevance_column", String.valueOf(usage.columnName())) + .suggestion( + String.format( + Locale.ROOT, + "apply %s directly to an indexed field before any command that transforms rows", + usage.functionName())); + } + private static void enrichErrorsForSpecialCases(ErrorReport.Builder report, SQLException e) { if (e.getMessage().contains("Error while preparing plan [") && e.getCause() != null) { // Generic 'something went wrong' planning error, try to get the cause @@ -546,6 +594,7 @@ public RelNode visit(TableScan scan) { .location("while compiling the optimized query plan for physical execution") .code(ErrorCode.PLANNING_ERROR); enrichErrorsForSpecialCases(report, e); + enrichRelevanceNotPushedDown(report, e, rel); throw report.build(); } } diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/UserDefinedFunctionUtils.java b/core/src/main/java/org/opensearch/sql/calcite/utils/UserDefinedFunctionUtils.java index 72a87a3ee48..d86b56b123f 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/utils/UserDefinedFunctionUtils.java +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/UserDefinedFunctionUtils.java @@ -27,9 +27,12 @@ import org.apache.calcite.adapter.enumerable.RexToLixTranslator; import org.apache.calcite.linq4j.tree.Expression; import org.apache.calcite.linq4j.tree.Expressions; +import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexInputRef; import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexShuttle; import org.apache.calcite.rex.RexVisitorImpl; import org.apache.calcite.schema.impl.AggregateFunctionImpl; import org.apache.calcite.sql.SqlAggFunction; @@ -127,6 +130,92 @@ public Void visitCall(RexCall call) { } } + /** + * A relevance function found in a plan, together with the column it targets as the user wrote it. + */ + public record RelevanceUsage(String functionName, String columnName) {} + + /** + * Finds the first relevance function left in a plan and resolves the column it targets back to + * its user-facing name. + * + *

Used to report a relevance function that survived push down. By the time code generation + * fails, the expression only carries positional references such as {@code $t4}; resolving them + * against the owning operator's input row type recovers the name the user actually wrote (for + * example {@code b2} for {@code eval b2 = upper(body)}). + * + * @param root the plan to inspect + * @return the first usage found, or null if the plan contains no relevance function + */ + public static RelevanceUsage findRelevanceUsage(RelNode root) { + if (root == null) { + return null; + } + for (RelNode input : root.getInputs()) { + RelevanceUsage fromInput = findRelevanceUsage(input); + if (fromInput != null) { + return fromInput; + } + } + List inputFieldNames = new ArrayList<>(); + for (RelNode input : root.getInputs()) { + inputFieldNames.addAll(input.getRowType().getFieldNames()); + } + RelevanceUsageFinder finder = new RelevanceUsageFinder(inputFieldNames); + root.accept(finder); + return finder.usage; + } + + /** + * Locates a relevance call and resolves the field operand against the supplied input field names. + * Relevance calls are shaped as {@code match(MAP('field', ), MAP('query', ), + * ...)}, so the column lives inside the first map argument. + */ + private static class RelevanceUsageFinder extends RexShuttle { + private final List inputFieldNames; + private RelevanceUsage usage; + + RelevanceUsageFinder(List inputFieldNames) { + this.inputFieldNames = inputFieldNames; + } + + @Override + public RexNode visitCall(RexCall call) { + if (usage == null && isRelevanceFunction(call.getOperator().getName())) { + usage = + new RelevanceUsage( + call.getOperator().getName().toLowerCase(Locale.ROOT), resolveColumnName(call)); + return call; + } + return super.visitCall(call); + } + + private String resolveColumnName(RexCall relevanceCall) { + if (relevanceCall.getOperands().isEmpty()) { + return null; + } + ColumnRefCollector collector = new ColumnRefCollector(); + relevanceCall.getOperands().get(0).accept(collector); + if (collector.index == null) { + return null; + } + return collector.index < inputFieldNames.size() ? inputFieldNames.get(collector.index) : null; + } + } + + /** Grabs the first input reference index inside an expression. */ + private static class ColumnRefCollector extends RexShuttle { + private Integer index; + + @Override + public RexNode visitInputRef(RexInputRef inputRef) { + if (index == null) { + index = inputRef.getIndex(); + } + return inputRef; + } + } + /** * Returns the name of the first relevance function found in the tree, or null if there is none. */ diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteRelevanceFunctionPushdownFailureIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteRelevanceFunctionPushdownFailureIT.java index 72baa622686..e536c090636 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteRelevanceFunctionPushdownFailureIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteRelevanceFunctionPushdownFailureIT.java @@ -60,12 +60,11 @@ private void createTestIndex() throws IOException { } private String errorOf(String query) throws IOException { - ResponseException e = - assertThrows(ResponseException.class, () -> executeQuery(query)); + ResponseException e = assertThrows(ResponseException.class, () -> executeQuery(query)); return getResponseBody(e.getResponse()); } - private void assertFailsCleanly(String query, String expectedFunctionName) + private void assertFailsCleanly(String query, String expectedFunctionName, String expectedColumn) throws IOException { String message = errorOf(query); assertFalse( @@ -78,8 +77,18 @@ private void assertFailsCleanly(String query, String expectedFunctionName) message.contains("all shards failed")); assertTrue( String.format( - Locale.ROOT, "Error should name the function [%s]. Error was: %s", "match", message), + Locale.ROOT, + "Error should name the function [%s]. Error was: %s", + expectedFunctionName, + message), message.toLowerCase(Locale.ROOT).contains(expectedFunctionName)); + assertTrue( + String.format( + Locale.ROOT, + "Error should name the column [%s]. Error was: %s", + expectedColumn, + message), + message.contains(String.format(Locale.ROOT, "column [%s]", expectedColumn))); } /** A relevance function over a column computed by the query cannot be pushed down. */ @@ -90,7 +99,8 @@ public void relevanceOnEvalDerivedColumnFailsCleanly() throws IOException { Locale.ROOT, "source=%s | eval b2 = upper(body) | where match(b2, 'ERROR') | fields idx", TEST_INDEX), - "match"); + "match", + "b2"); } /** Same for a column produced by parse. */ @@ -101,15 +111,18 @@ public void relevanceOnParseDerivedColumnFailsCleanly() throws IOException { Locale.ROOT, "source=%s | parse body '(?\\\\w+)' | where match(lvl, 'ERROR') | fields idx", TEST_INDEX), - "match"); + "match", + "lvl"); } /** A relevance filter above an aggregation has no scan to be pushed onto. */ @Test public void relevanceAboveAggregationFailsCleanly() throws IOException { assertFailsCleanly( - String.format(Locale.ROOT, "source=%s | top 1 body | where match(body, 'ERROR')", TEST_INDEX), - "match"); + String.format( + Locale.ROOT, "source=%s | top 1 body | where match(body, 'ERROR')", TEST_INDEX), + "match", + "body"); } /** A relevance function in a projection is never rewritten into a query. */ @@ -118,7 +131,8 @@ public void relevanceInProjectionFailsCleanly() throws IOException { assertFailsCleanly( String.format( Locale.ROOT, "source=%s | eval m = match(body, 'ERROR') | fields idx, m", TEST_INDEX), - "match"); + "match", + "body"); } /** From 3bb6701726ea545498a927709226673ded82dd4e Mon Sep 17 00:00:00 2001 From: Jialiang Liang Date: Wed, 29 Jul 2026 12:08:10 -0700 Subject: [PATCH 3/4] Document usage constraints for relevance functions Adds a "Usage constraints" section to the relevance function docs explaining that these functions are executed by the OpenSearch search engine against the inverted index built at write time, so they must be applied to an indexed field before any command that transforms rows. Each unsupported pattern is listed with the reason it cannot work, and the planning-time error the user will see. Every claim was verified against a running cluster rather than inferred: - stats on the grouping key followed by a relevance filter DOES work (the filter transposes below the aggregation), so it is documented as supported rather than listed as a limitation. - sort + head followed by a relevance filter fails at planning time. - top / rare followed by a relevance filter fails at planning time. Also documents the known issue where a relevance filter placed directly after `head` silently returns rows drawn from the whole index instead of from the intended sample, since that reordering is not yet guarded. Signed-off-by: Jialiang Liang --- docs/user/ppl/functions/relevance.md | 55 ++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/docs/user/ppl/functions/relevance.md b/docs/user/ppl/functions/relevance.md index a0bcfe59cd2..a48d4e1b814 100644 --- a/docs/user/ppl/functions/relevance.md +++ b/docs/user/ppl/functions/relevance.md @@ -4,6 +4,61 @@ Relevance-based functions enable users to search an index for documents based on You can use these functions for global query filtering, such as in condition expressions within `WHERE` or `HAVING` clauses. For more details about relevance-based search, see [Relevance Based Search With SQL/PPL Query Engine](https://github.com/opensearch-project/sql/issues/182). +## Usage constraints + +Relevance functions are executed by the OpenSearch search engine, not by the plugin. A relevance function matches against the inverted index that OpenSearch builds when a document is written, so it can only be applied to a field that exists in the index mapping, and only at a point in the pipeline where the query can still be handed to the search engine. + +In practice this means a relevance function must be applied **directly to an indexed field, before any command that transforms rows**. Place it immediately after `source=`: + +``` +source=logs | where match(body, 'ERROR') | eval level = upper(body) | fields level +``` + +The following are not supported, because there is no indexed field left to search against: + +| Pattern | Why it cannot work | +|---|---| +| `... \| eval b = upper(body) \| where match(b, 'ERROR')` | `b` is computed while the query runs; it was never indexed. The same applies to columns produced by `parse` and `rex`. | +| `... \| top 1 body \| where match(body, 'ERROR')`
`... \| rare 1 body \| where match(body, 'ERROR')` | These commands produce summary rows after the search phase completes; there are no documents left to search. | +| `... \| sort body \| head 1 \| where match(body, 'ERROR')` | Once a sort and a row limit have been applied, the filter can no longer be handed to the search engine. | +| `... \| eval m = match(body, 'ERROR')` | A relevance function is a search condition, not a per-row value, so it cannot be assigned to a column. | + +A query in this category fails at planning time, before any data is read, with an error naming the function and the column: + +``` +Relevance search function [match] cannot be applied to column [b]. It must run against a +field indexed by OpenSearch, and cannot be evaluated on a value the query computes (eval, +parse, rex), on aggregated output (stats, top, rare), or after a row limit (head). +``` + +A relevance filter placed after a `stats` aggregation on the grouping key is supported, because the filter can be applied to the underlying documents before grouping: + +``` +source=logs | stats count() by body | where match(body, 'ERROR') +``` + +If you need to filter on a computed column, use a predicate that can be evaluated per row, such as `like`: + +``` +source=logs | eval b = upper(body) | where like(b, '%ERROR%') +``` + +Note that `like` performs substring matching rather than analyzed text matching, so it does not apply tokenization, stemming, or relevance scoring. + +### Known issue: relevance filters after `head` + +Avoid placing a relevance filter directly after `head`: + +``` +source=logs | head 100 | where match(body, 'ERROR') +``` + +A search request applies its query before limiting results, so this is evaluated as "search the whole index, then take 100 matches" rather than "take 100 rows, then search them". The query returns results without an error, but they are drawn from the entire index rather than from the intended sample. Apply the relevance filter before `head` instead: + +``` +source=logs | where match(body, 'ERROR') | head 100 +``` + ## MATCH **Usage**: `MATCH(, [,