From e058b971a10f0d845a571c079821f8066625b25b Mon Sep 17 00:00:00 2001 From: Uwe Schindler Date: Mon, 27 Jul 2026 14:11:46 +0200 Subject: [PATCH 1/5] Add (configurable) guards for complexity of expressions to fail with ParseException instead of StackOverflowError or LinkageErrors --- .../expressions/js/JavascriptCompiler.java | 72 ++++++++++++++++--- .../js/JavascriptNestingDepthListener.java | 60 ++++++++++++++++ .../expressions/js/CompilerTestCase.java | 3 +- .../js/TestJavascriptCompiler.java | 29 ++++++++ 4 files changed, 155 insertions(+), 9 deletions(-) create mode 100644 lucene/expressions/src/java/org/apache/lucene/expressions/js/JavascriptNestingDepthListener.java diff --git a/lucene/expressions/src/java/org/apache/lucene/expressions/js/JavascriptCompiler.java b/lucene/expressions/src/java/org/apache/lucene/expressions/js/JavascriptCompiler.java index d8c1c6a663eb..7a04bdf7b718 100644 --- a/lucene/expressions/src/java/org/apache/lucene/expressions/js/JavascriptCompiler.java +++ b/lucene/expressions/src/java/org/apache/lucene/expressions/js/JavascriptCompiler.java @@ -62,6 +62,7 @@ import org.antlr.v4.runtime.Recognizer; import org.antlr.v4.runtime.atn.PredictionMode; import org.antlr.v4.runtime.tree.ParseTree; +import org.antlr.v4.runtime.tree.ParseTreeWalker; import org.apache.lucene.expressions.Expression; import org.apache.lucene.expressions.js.JavascriptParser.ExpressionContext; import org.apache.lucene.search.DoubleValues; @@ -129,8 +130,12 @@ public final class JavascriptCompiler { private static final ExceptionsAttribute ATTR_THROWS_IOEXCEPTION = ExceptionsAttribute.ofSymbols(IOException.class.describeConstable().orElseThrow()); + /** The default maximum depth of nesting (function calls, precedence) */ + public static final int DEFAULT_MAX_NESTING_DEPTH = 250; + final String sourceText; final Map functions; + final int maxNestingDepth; final boolean picky; /** @@ -139,6 +144,8 @@ public final class JavascriptCompiler { * @param sourceText The expression to compile * @return A new compiled expression * @throws ParseException on failure to compile + * @throws IllegalStateException if the resulting expression class fails to link (e.g., + * complexity) */ public static Expression compile(String sourceText) throws ParseException { return compile(sourceText, DEFAULT_FUNCTIONS); @@ -155,10 +162,35 @@ public static Expression compile(String sourceText) throws ParseException { * @param functions map of String names to {@link MethodHandle}s * @return A new compiled expression * @throws ParseException on failure to compile + * @throws IllegalArgumentException if any of the functions does not have correct signature + * @throws IllegalStateException if the resulting expression class fails to link (e.g., + * complexity) */ public static Expression compile(String sourceText, Map functions) throws ParseException { - return compile(sourceText, functions, false); + return compile(sourceText, functions, DEFAULT_MAX_NESTING_DEPTH); + } + + /** + * Compiles the given expression with the supplied custom functions using a custom maximum nesting + * depth. + * + *

Functions must be {@code public static}, return {@code double} and can take from zero to 256 + * {@code double} parameters. + * + * @param sourceText The expression to compile + * @param functions map of String names to {@link MethodHandle}s + * @param maxNestingDepth the maximum depth of nesting (function calls, precedence) + * @return A new compiled expression + * @throws ParseException on failure to compile + * @throws IllegalArgumentException if any of the functions does not have correct signature + * @throws IllegalStateException if the resulting expression class fails to link (e.g., + * complexity) + */ + public static Expression compile( + String sourceText, Map functions, int maxNestingDepth) + throws ParseException { + return compile(sourceText, functions, maxNestingDepth, false); } /** @@ -169,17 +201,23 @@ public static Expression compile(String sourceText, Map fu * * @param sourceText The expression to compile * @param functions map of String names to {@link MethodHandle}s + * @param maxNestingDepth the maximum depth of nesting (function calls, precedence) * @param picky whether to throw exception on ambiguity or other internal parsing issues (this * option makes things slower too, it is only for debugging). * @return A new compiled expression * @throws ParseException on failure to compile + * @throws IllegalArgumentException if any of the functions does not have correct signature + * @throws IllegalStateException if the resulting expression class fails to link (e.g., + * complexity) */ - static Expression compile(String sourceText, Map functions, boolean picky) + static Expression compile( + String sourceText, Map functions, int maxNestingDepth, boolean picky) throws ParseException { for (MethodHandle m : functions.values()) { checkFunction(m); } - return new JavascriptCompiler(sourceText, functions, picky).compileExpression(); + return new JavascriptCompiler(sourceText, functions, maxNestingDepth, picky) + .compileExpression(); } /** @@ -188,9 +226,10 @@ static Expression compile(String sourceText, Map functions * @param sourceText The expression to compile */ private JavascriptCompiler( - String sourceText, Map functions, boolean picky) { + String sourceText, Map functions, int maxNestingDepth, boolean picky) { this.sourceText = Objects.requireNonNull(sourceText, "sourceText"); this.functions = Map.copyOf(functions); + this.maxNestingDepth = maxNestingDepth; this.picky = picky; } @@ -218,6 +257,12 @@ private Expression compileExpression() throws ParseException { throw cause; } throw re; + } catch (StackOverflowError soe) { + // we should catch this before in the visitor, but too high limits may cause this. + final var e = + new ParseException("Invalid expression '" + sourceText + "': Nesting level too deep", 0); + e.initCause(soe); + throw e; } try { @@ -225,10 +270,12 @@ private Expression compileExpression() throws ParseException { LOOKUP.defineHiddenClassWithClassData( classFile, constantsMap.keySet().stream().map(functions::get).toList(), true); return invokeConstructor(lookup, lookup.lookupClass(), externalsMap); - } catch (ReflectiveOperationException exception) { + } catch (ReflectiveOperationException | LinkageError e) { throw new IllegalStateException( - "An internal error occurred attempting to compile the expression (" + sourceText + ").", - exception); + "An internal error occurred attempting to compile the expression ('" + + sourceText + + "'). This may be caused by too complex code triggering limits inside the Java VM.", + e); } } @@ -261,7 +308,16 @@ private ParseTree getAntlrParseTree() { setupPicky(javascriptParser); } javascriptParser.setErrorHandler(new JavascriptParserErrorStrategy()); - return javascriptParser.compile(); + // add listener to detect too deep nesting early, this may not catch all occurrences... (this + // will count root node, so add one recursion extra). + javascriptParser.addParseListener( + new JavascriptNestingDepthListener(sourceText, maxNestingDepth + 1)); + final ParseTree tree = javascriptParser.compile(); + // we do an extra check on the built parse tree to make sure the final structure is not too + // deeply nested. + ParseTreeWalker.DEFAULT.walk( + new JavascriptNestingDepthListener(sourceText, maxNestingDepth), tree); + return tree; } private void setupPicky(JavascriptParser parser) { diff --git a/lucene/expressions/src/java/org/apache/lucene/expressions/js/JavascriptNestingDepthListener.java b/lucene/expressions/src/java/org/apache/lucene/expressions/js/JavascriptNestingDepthListener.java new file mode 100644 index 000000000000..cc74a32f6607 --- /dev/null +++ b/lucene/expressions/src/java/org/apache/lucene/expressions/js/JavascriptNestingDepthListener.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.expressions.js; + +import org.antlr.v4.runtime.ParserRuleContext; +import org.antlr.v4.runtime.tree.ErrorNode; +import org.antlr.v4.runtime.tree.ParseTreeListener; +import org.antlr.v4.runtime.tree.TerminalNode; + +final class JavascriptNestingDepthListener implements ParseTreeListener { + private final String sourceText; + private final int maxNestingDepth; + + private int depth = 0; + + public JavascriptNestingDepthListener(String sourceText, int maxNestingDepth) { + super(); + this.sourceText = sourceText; + this.maxNestingDepth = maxNestingDepth; + } + + @Override + public void visitTerminal(TerminalNode node) {} + + @Override + public void visitErrorNode(ErrorNode node) {} + + @Override + public void enterEveryRule(ParserRuleContext ctx) { + depth++; + if (depth > maxNestingDepth) { + throw JavascriptCompiler.newWrappedParseException( + "Invalid expression '" + + sourceText + + "': Nesting level too deep (>" + + maxNestingDepth + + ")", + ctx.start != null ? ctx.start.getStartIndex() : -1); + } + } + + @Override + public void exitEveryRule(ParserRuleContext ctx) { + depth--; + } +} diff --git a/lucene/expressions/src/test/org/apache/lucene/expressions/js/CompilerTestCase.java b/lucene/expressions/src/test/org/apache/lucene/expressions/js/CompilerTestCase.java index d7cc843efdac..b28eb480b197 100644 --- a/lucene/expressions/src/test/org/apache/lucene/expressions/js/CompilerTestCase.java +++ b/lucene/expressions/src/test/org/apache/lucene/expressions/js/CompilerTestCase.java @@ -34,6 +34,7 @@ protected Expression compile(String sourceText) throws ParseException { /** compiles expression for sourceText with custom functions list */ protected Expression compile(String sourceText, Map functions) throws ParseException { - return JavascriptCompiler.compile(sourceText, functions, true); + return JavascriptCompiler.compile( + sourceText, functions, JavascriptCompiler.DEFAULT_MAX_NESTING_DEPTH, true); } } diff --git a/lucene/expressions/src/test/org/apache/lucene/expressions/js/TestJavascriptCompiler.java b/lucene/expressions/src/test/org/apache/lucene/expressions/js/TestJavascriptCompiler.java index 94d180df5f9d..497537f4f30b 100644 --- a/lucene/expressions/src/test/org/apache/lucene/expressions/js/TestJavascriptCompiler.java +++ b/lucene/expressions/src/test/org/apache/lucene/expressions/js/TestJavascriptCompiler.java @@ -17,6 +17,8 @@ package org.apache.lucene.expressions.js; import java.text.ParseException; +import java.util.stream.Collectors; +import java.util.stream.IntStream; import org.apache.lucene.expressions.Expression; import org.junit.jupiter.api.Test; @@ -269,4 +271,31 @@ public void testVariableNormalization() throws Exception { x = compile("foo['\\\\'][\"\\\\\"]"); assertEquals("foo['\\\\']['\\\\']", x.variables[0]); } + + @Test + public void testRecursionDepth1() throws Exception { + int depth = 20000; + String src = "(".repeat(depth) + "1" + ")".repeat(depth); + assertEquals(JavascriptCompiler.DEFAULT_MAX_NESTING_DEPTH, assertRecursionLimit(src)); + } + + @Test + public void testRecursionDepth2() throws Exception { + String src = "-".repeat(20000) + "1"; + assertEquals(JavascriptCompiler.DEFAULT_MAX_NESTING_DEPTH, assertRecursionLimit(src)); + } + + @Test + public void testRecursionDepth3() throws Exception { + String src = + IntStream.range(0, 20000).mapToObj(Integer::toString).collect(Collectors.joining("+")); + assertRecursionLimit(src); + } + + /** returns the error offset for further checks */ + private int assertRecursionLimit(String src) { + ParseException expected = expectThrows(ParseException.class, () -> compile(src)); + assertTrue(expected.getMessage(), expected.getMessage().contains("Nesting level too deep")); + return expected.getErrorOffset(); + } } From 9dd255217e96ffaa98a922e4c4f8db1436ddde47 Mon Sep 17 00:00:00 2001 From: Uwe Schindler Date: Mon, 27 Jul 2026 16:12:39 +0200 Subject: [PATCH 2/5] Raise nesting limit to 1024, add documentation that implicit precedence also counts --- .../expressions/js/JavascriptCompiler.java | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/lucene/expressions/src/java/org/apache/lucene/expressions/js/JavascriptCompiler.java b/lucene/expressions/src/java/org/apache/lucene/expressions/js/JavascriptCompiler.java index 7a04bdf7b718..a58d43529394 100644 --- a/lucene/expressions/src/java/org/apache/lucene/expressions/js/JavascriptCompiler.java +++ b/lucene/expressions/src/java/org/apache/lucene/expressions/js/JavascriptCompiler.java @@ -101,6 +101,13 @@ * be resolved correctly using a private {@link Lookup} instance. Ideally the methods should be * {@code static}, but you can use {@link MethodHandle#bindTo(Object)} to bind it to a receiver. * + *

The parser has a default maximum nesting limit of 1024 (see {@link + * #DEFAULT_MAX_NESTING_DEPTH}) which is enforced during parsing. Violating it will cause a {@link + * ParseException}. Please keep in mind that also code like + * a + b + c + d + ... + z is treated as if it would look like this: + * (((((a + b) + c) + d) + ...) + z) (because of the nature of + * the JVM as a stack-based machine). + * * @lucene.experimental */ public final class JavascriptCompiler { @@ -130,8 +137,8 @@ public final class JavascriptCompiler { private static final ExceptionsAttribute ATTR_THROWS_IOEXCEPTION = ExceptionsAttribute.ofSymbols(IOException.class.describeConstable().orElseThrow()); - /** The default maximum depth of nesting (function calls, precedence) */ - public static final int DEFAULT_MAX_NESTING_DEPTH = 250; + /** The default maximum depth of nesting (function calls, precedence - also implicit) */ + public static final int DEFAULT_MAX_NESTING_DEPTH = 1024; final String sourceText; final Map functions; @@ -180,7 +187,8 @@ public static Expression compile(String sourceText, Map fu * * @param sourceText The expression to compile * @param functions map of String names to {@link MethodHandle}s - * @param maxNestingDepth the maximum depth of nesting (function calls, precedence) + * @param maxNestingDepth the maximum depth of nesting (function calls, precedence - also + * implicit) * @return A new compiled expression * @throws ParseException on failure to compile * @throws IllegalArgumentException if any of the functions does not have correct signature @@ -201,7 +209,8 @@ public static Expression compile( * * @param sourceText The expression to compile * @param functions map of String names to {@link MethodHandle}s - * @param maxNestingDepth the maximum depth of nesting (function calls, precedence) + * @param maxNestingDepth the maximum depth of nesting (function calls, precedence - also + * implicit) * @param picky whether to throw exception on ambiguity or other internal parsing issues (this * option makes things slower too, it is only for debugging). * @return A new compiled expression From d705ffd04ca6ca39caf472b99a74ebcc60641152 Mon Sep 17 00:00:00 2001 From: Uwe Schindler Date: Mon, 27 Jul 2026 18:03:44 +0200 Subject: [PATCH 3/5] Avoid 2nd recursive tree walking by doing it while building classfile --- .../expressions/js/JavascriptCompiler.java | 19 +++++++++++-------- .../js/TestJavascriptCompiler.java | 17 ++++++++++++++++- 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/lucene/expressions/src/java/org/apache/lucene/expressions/js/JavascriptCompiler.java b/lucene/expressions/src/java/org/apache/lucene/expressions/js/JavascriptCompiler.java index a58d43529394..ab956d987d9b 100644 --- a/lucene/expressions/src/java/org/apache/lucene/expressions/js/JavascriptCompiler.java +++ b/lucene/expressions/src/java/org/apache/lucene/expressions/js/JavascriptCompiler.java @@ -58,11 +58,11 @@ import org.antlr.v4.runtime.CharStreams; import org.antlr.v4.runtime.CommonTokenStream; import org.antlr.v4.runtime.DiagnosticErrorListener; +import org.antlr.v4.runtime.ParserRuleContext; import org.antlr.v4.runtime.RecognitionException; import org.antlr.v4.runtime.Recognizer; import org.antlr.v4.runtime.atn.PredictionMode; import org.antlr.v4.runtime.tree.ParseTree; -import org.antlr.v4.runtime.tree.ParseTreeWalker; import org.apache.lucene.expressions.Expression; import org.apache.lucene.expressions.js.JavascriptParser.ExpressionContext; import org.apache.lucene.search.DoubleValues; @@ -269,7 +269,7 @@ private Expression compileExpression() throws ParseException { } catch (StackOverflowError soe) { // we should catch this before in the visitor, but too high limits may cause this. final var e = - new ParseException("Invalid expression '" + sourceText + "': Nesting level too deep", 0); + new ParseException("Invalid expression '" + sourceText + "': Nesting level too deep", -1); e.initCause(soe); throw e; } @@ -321,12 +321,7 @@ private ParseTree getAntlrParseTree() { // will count root node, so add one recursion extra). javascriptParser.addParseListener( new JavascriptNestingDepthListener(sourceText, maxNestingDepth + 1)); - final ParseTree tree = javascriptParser.compile(); - // we do an extra check on the built parse tree to make sure the final structure is not too - // deeply nested. - ParseTreeWalker.DEFAULT.walk( - new JavascriptNestingDepthListener(sourceText, maxNestingDepth), tree); - return tree; + return javascriptParser.compile(); } private void setupPicky(JavascriptParser parser) { @@ -408,10 +403,18 @@ private JavascriptBaseVisitor newClassFileGeneratorVisitor( CodeBuilder gen, final Map externalsMap, final Map constantsMap) { + // we do an extra check on the tree to make sure the final structure is not too deeply nested: + final var nestingChecker = new JavascriptNestingDepthListener(sourceText, maxNestingDepth); // to completely hide the ANTLR visitor we use an anonymous impl: return new JavascriptBaseVisitor<>() { private final Deque typeStack = new ArrayDeque<>(); + private void visit(ParserRuleContext ctx) { + nestingChecker.enterEveryRule(ctx); + ctx.accept(this); + nestingChecker.exitEveryRule(ctx); + } + @Override public Void visitCompile(JavascriptParser.CompileContext ctx) { typeStack.push(TypeKind.DOUBLE); diff --git a/lucene/expressions/src/test/org/apache/lucene/expressions/js/TestJavascriptCompiler.java b/lucene/expressions/src/test/org/apache/lucene/expressions/js/TestJavascriptCompiler.java index 497537f4f30b..cf6e02ef8f52 100644 --- a/lucene/expressions/src/test/org/apache/lucene/expressions/js/TestJavascriptCompiler.java +++ b/lucene/expressions/src/test/org/apache/lucene/expressions/js/TestJavascriptCompiler.java @@ -287,9 +287,24 @@ public void testRecursionDepth2() throws Exception { @Test public void testRecursionDepth3() throws Exception { + String src = + IntStream.range(0, JavascriptCompiler.DEFAULT_MAX_NESTING_DEPTH + 1) + .mapToObj(Integer::toString) + .collect(Collectors.joining("+")); + assertEquals( + "error offset needs to be 0 due to recursion with depth first", + 0, + assertRecursionLimit(src)); + } + + @Test + public void testRecursionDepth4() throws Exception { String src = IntStream.range(0, 20000).mapToObj(Integer::toString).collect(Collectors.joining("+")); - assertRecursionLimit(src); + assertEquals( + "error offset needs to be 0 due to recursion with depth first", + 0, + assertRecursionLimit(src)); } /** returns the error offset for further checks */ From 3066a5f569c35e5a14b665c777c835ffc1771215 Mon Sep 17 00:00:00 2001 From: Uwe Schindler Date: Mon, 27 Jul 2026 18:15:53 +0200 Subject: [PATCH 4/5] add changes entry --- lucene/CHANGES.txt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index d67e7742b625..84ea31b9a741 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -337,6 +337,12 @@ Improvements * GITHUB#16271: Add detail to KNN no-match explanations. (Jakub Slowinski) +* GITHUB#16423: Add (configurable) guards for complexity of expressions. Previously + the expressions parser could throw StackOverflowError on deeply nested expressions + or also cause VerifyError and other LinkageErrors when the expression was to complex. + The new code allows to add a hard limit for nesting (default is 1024) and no longer + throws virtual machine errors. (Uwe Schindler, Seth Kraft) + Optimizations --------------------- * GITHUG#16280: Single-pass writeString fast path for short strings in ByteBuffersDataOutput (neoremind) From eb3d2119c5f01856d915b7e90670b622f35f7970 Mon Sep 17 00:00:00 2001 From: Uwe Schindler Date: Mon, 27 Jul 2026 18:24:22 +0200 Subject: [PATCH 5/5] add extra test (on boundary) --- .../lucene/expressions/js/TestJavascriptCompiler.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lucene/expressions/src/test/org/apache/lucene/expressions/js/TestJavascriptCompiler.java b/lucene/expressions/src/test/org/apache/lucene/expressions/js/TestJavascriptCompiler.java index cf6e02ef8f52..e9e9a3d093fa 100644 --- a/lucene/expressions/src/test/org/apache/lucene/expressions/js/TestJavascriptCompiler.java +++ b/lucene/expressions/src/test/org/apache/lucene/expressions/js/TestJavascriptCompiler.java @@ -279,6 +279,13 @@ public void testRecursionDepth1() throws Exception { assertEquals(JavascriptCompiler.DEFAULT_MAX_NESTING_DEPTH, assertRecursionLimit(src)); } + @Test + public void testRecursionDepth1a() throws Exception { + int depth = JavascriptCompiler.DEFAULT_MAX_NESTING_DEPTH; + String src = "(".repeat(depth) + "1" + ")".repeat(depth); + assertEquals(JavascriptCompiler.DEFAULT_MAX_NESTING_DEPTH, assertRecursionLimit(src)); + } + @Test public void testRecursionDepth2() throws Exception { String src = "-".repeat(20000) + "1"; @@ -298,7 +305,7 @@ public void testRecursionDepth3() throws Exception { } @Test - public void testRecursionDepth4() throws Exception { + public void testRecursionDepth3a() throws Exception { String src = IntStream.range(0, 20000).mapToObj(Integer::toString).collect(Collectors.joining("+")); assertEquals(