From e6c028f34da1333362e6b97f7e50943a788476e2 Mon Sep 17 00:00:00 2001 From: vismaytiwari Date: Thu, 9 Jul 2026 22:50:19 +0530 Subject: [PATCH 1/6] Collapse repeated wildcard asterisks --- .../src/java/org/apache/lucene/search/WildcardQuery.java | 9 ++++++++- .../test/org/apache/lucene/search/TestWildcardQuery.java | 9 +++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/lucene/core/src/java/org/apache/lucene/search/WildcardQuery.java b/lucene/core/src/java/org/apache/lucene/search/WildcardQuery.java index 9c1e8c223ed5..34fb39f11a65 100644 --- a/lucene/core/src/java/org/apache/lucene/search/WildcardQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/WildcardQuery.java @@ -70,15 +70,20 @@ public static Automaton toAutomaton(Term wildcardquery) { String wildcardText = wildcardquery.text(); + boolean lastWasWildcardString = false; for (int i = 0; i < wildcardText.length(); ) { final int c = wildcardText.codePointAt(i); int length = Character.charCount(c); switch (c) { case WILDCARD_STRING: - automata.add(Automata.makeAnyString()); + if (!lastWasWildcardString) { + automata.add(Automata.makeAnyString()); + lastWasWildcardString = true; + } break; case WILDCARD_CHAR: automata.add(Automata.makeAnyChar()); + lastWasWildcardString = false; break; case WILDCARD_ESCAPE: // add the next codepoint instead, if it exists @@ -86,10 +91,12 @@ public static Automaton toAutomaton(Term wildcardquery) { final int nextChar = wildcardText.codePointAt(i + length); length += Character.charCount(nextChar); automata.add(Automata.makeChar(nextChar)); + lastWasWildcardString = false; break; } // else fallthru, lenient parsing with a trailing \ default: automata.add(Automata.makeChar(c)); + lastWasWildcardString = false; } i += length; } diff --git a/lucene/core/src/test/org/apache/lucene/search/TestWildcardQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestWildcardQuery.java index c076367ce5b7..f0c9211eefcf 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestWildcardQuery.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestWildcardQuery.java @@ -39,6 +39,7 @@ import org.apache.lucene.tests.index.RandomIndexWriter; import org.apache.lucene.tests.util.LuceneTestCase; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.automaton.Automaton; import org.apache.lucene.util.automaton.CompiledAutomaton; /** TestWildcardQuery tests the '*' and '?' wildcard characters. */ @@ -235,6 +236,14 @@ public void testEscapes() throws Exception { indexStore.close(); } + public void testRepeatedAsterisksCollapse() { + Automaton single = WildcardQuery.toAutomaton(new Term("field", "*")); + Automaton repeated = WildcardQuery.toAutomaton(new Term("field", "*".repeat(1000))); + + assertEquals(single.getNumStates(), repeated.getNumStates()); + assertEquals(single.getNumTransitions(), repeated.getNumTransitions()); + } + private Directory getIndexStore(String field, String[] contents) throws IOException { Directory indexStore = newDirectory(); RandomIndexWriter writer = new RandomIndexWriter(random(), indexStore); From 9dc9ac5e8c6c284193d2b3431b6ca6146a3b846c Mon Sep 17 00:00:00 2001 From: vismaytiwari Date: Fri, 10 Jul 2026 14:49:29 +0530 Subject: [PATCH 2/6] Modernize toAutomaton switch and strengthen the collapse test Convert the wildcard switch to an arrow switch (dropping the fallthrough and its @SuppressWarnings), per review. Compare automaton language equality (AutomatonTestUtil.sameLanguage) instead of only state/transition counts, add a variant mixing an escaped \* with literals and '?', and add a '*?' x5000 case asserting it stays linearly sized (it does not collapse but must not explode either). --- .../apache/lucene/search/WildcardQuery.java | 26 +++++++----- .../lucene/search/TestWildcardQuery.java | 42 ++++++++++++++++++- 2 files changed, 55 insertions(+), 13 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/search/WildcardQuery.java b/lucene/core/src/java/org/apache/lucene/search/WildcardQuery.java index 34fb39f11a65..7896cb769965 100644 --- a/lucene/core/src/java/org/apache/lucene/search/WildcardQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/WildcardQuery.java @@ -64,7 +64,6 @@ public WildcardQuery(Term term, RewriteMethod rewriteMethod) { * * @lucene.internal */ - @SuppressWarnings("fallthrough") public static Automaton toAutomaton(Term wildcardquery) { List automata = new ArrayList<>(); @@ -75,28 +74,33 @@ public static Automaton toAutomaton(Term wildcardquery) { final int c = wildcardText.codePointAt(i); int length = Character.charCount(c); switch (c) { - case WILDCARD_STRING: + case WILDCARD_STRING -> { + // collapse consecutive '*' into a single any-string automaton if (!lastWasWildcardString) { automata.add(Automata.makeAnyString()); lastWasWildcardString = true; } - break; - case WILDCARD_CHAR: + } + case WILDCARD_CHAR -> { automata.add(Automata.makeAnyChar()); lastWasWildcardString = false; - break; - case WILDCARD_ESCAPE: - // add the next codepoint instead, if it exists + } + case WILDCARD_ESCAPE -> { + // add the escaped codepoint as a literal if it exists; a trailing + // '\' is parsed leniently as a literal '\' if (i + length < wildcardText.length()) { final int nextChar = wildcardText.codePointAt(i + length); length += Character.charCount(nextChar); automata.add(Automata.makeChar(nextChar)); - lastWasWildcardString = false; - break; - } // else fallthru, lenient parsing with a trailing \ - default: + } else { + automata.add(Automata.makeChar(c)); + } + lastWasWildcardString = false; + } + default -> { automata.add(Automata.makeChar(c)); lastWasWildcardString = false; + } } i += length; } diff --git a/lucene/core/src/test/org/apache/lucene/search/TestWildcardQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestWildcardQuery.java index f0c9211eefcf..8f0abbc482ad 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestWildcardQuery.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestWildcardQuery.java @@ -38,9 +38,11 @@ import org.apache.lucene.tests.analysis.MockAnalyzer; import org.apache.lucene.tests.index.RandomIndexWriter; import org.apache.lucene.tests.util.LuceneTestCase; +import org.apache.lucene.tests.util.automaton.AutomatonTestUtil; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.automaton.Automaton; import org.apache.lucene.util.automaton.CompiledAutomaton; +import org.apache.lucene.util.automaton.Operations; /** TestWildcardQuery tests the '*' and '?' wildcard characters. */ public class TestWildcardQuery extends LuceneTestCase { @@ -236,14 +238,50 @@ public void testEscapes() throws Exception { indexStore.close(); } + private static Automaton det(Automaton a) { + return Operations.determinize(a, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT); + } + public void testRepeatedAsterisksCollapse() { - Automaton single = WildcardQuery.toAutomaton(new Term("field", "*")); - Automaton repeated = WildcardQuery.toAutomaton(new Term("field", "*".repeat(1000))); + // Consecutive '*' must collapse to a single any-string automaton: same + // language as a single '*', and the same (tiny) shape, not a huge one. + Automaton single = det(WildcardQuery.toAutomaton(new Term("field", "*"))); + Automaton repeated = det(WildcardQuery.toAutomaton(new Term("field", "*".repeat(1000)))); + assertTrue(AutomatonTestUtil.sameLanguage(single, repeated)); assertEquals(single.getNumStates(), repeated.getNumStates()); assertEquals(single.getNumTransitions(), repeated.getNumTransitions()); } + public void testRepeatedAsterisksCollapseWithEscapesAndLiterals() { + // Collapsing applies only to unescaped, consecutive '*'. An escaped '\*' + // stays a literal and the surrounding literals / '?' are untouched, so + // "a\*b***c?" matches the same language as its single-'*' form "a\*b*c?". + Automaton collapsed = det(WildcardQuery.toAutomaton(new Term("field", "a\\*b***c?"))); + Automaton reference = det(WildcardQuery.toAutomaton(new Term("field", "a\\*b*c?"))); + + assertTrue(AutomatonTestUtil.sameLanguage(reference, collapsed)); + assertEquals(reference.getNumStates(), collapsed.getNumStates()); + assertEquals(reference.getNumTransitions(), collapsed.getNumTransitions()); + + // Escaped '\*' is a literal, so "\*\*\*" matches the string "***", not the + // any-string '*' — it must NOT be collapsed. + Automaton single = det(WildcardQuery.toAutomaton(new Term("field", "*"))); + Automaton literalStars = det(WildcardQuery.toAutomaton(new Term("field", "\\*\\*\\*"))); + assertFalse(AutomatonTestUtil.sameLanguage(single, literalStars)); + } + + public void testInterleavedStarQuestionDoesNotBlowUp() { + // '*?' repeated does not collapse (the '*' are not consecutive), but it + // must still build a linearly-sized automaton rather than exploding the way + // repeated '*' used to before this fix (gh-16134). + int n = 5000; + Automaton a = WildcardQuery.toAutomaton(new Term("field", "*?".repeat(n))); + assertTrue( + "expected roughly linear size, got " + a.getNumTransitions() + " transitions for " + n, + a.getNumTransitions() < 100L * n); + } + private Directory getIndexStore(String field, String[] contents) throws IOException { Directory indexStore = newDirectory(); RandomIndexWriter writer = new RandomIndexWriter(random(), indexStore); From 276e3c782e77394a6ce4fe9c476a45ec995296d7 Mon Sep 17 00:00:00 2001 From: vismaytiwari Date: Fri, 10 Jul 2026 16:04:02 +0530 Subject: [PATCH 3/6] Add CHANGES.txt entry for the wildcard repeated-asterisk collapse --- lucene/CHANGES.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 5ae094c6ed64..b3114ad9e995 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -151,6 +151,10 @@ Improvements Optimizations --------------------- +* GITHUB#16382: WildcardQuery.toAutomaton now collapses consecutive unescaped '*' + wildcards into a single any-string automaton, avoiding a quadratic blow-up in + automaton size for patterns with many repeated asterisks. (Vismay Tiwari) + * GITHUB#16307: ReqExclBulkScorer now skips runs of excluded docs via TwoPhaseIterator#docIDRunEnd for two-phase excluded clauses (e.g. a doc-values not-equals filter), instead of advancing one doc at a time. (Jim Ferenczi) From 53dcb542289584c5f78cd55acd83a6b0e3790552 Mon Sep 17 00:00:00 2001 From: Vismay <89745751+vismaytiwari@users.noreply.github.com> Date: Fri, 10 Jul 2026 21:40:11 +0530 Subject: [PATCH 4/6] Update lucene/core/src/test/org/apache/lucene/search/TestWildcardQuery.java Co-authored-by: Namgyu Kim --- .../src/test/org/apache/lucene/search/TestWildcardQuery.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lucene/core/src/test/org/apache/lucene/search/TestWildcardQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestWildcardQuery.java index 8f0abbc482ad..d7f81ed71cd2 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestWildcardQuery.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestWildcardQuery.java @@ -274,7 +274,7 @@ public void testRepeatedAsterisksCollapseWithEscapesAndLiterals() { public void testInterleavedStarQuestionDoesNotBlowUp() { // '*?' repeated does not collapse (the '*' are not consecutive), but it // must still build a linearly-sized automaton rather than exploding the way - // repeated '*' used to before this fix (gh-16134). + // repeated '*' used to before this fix (GITHUB#16134). int n = 5000; Automaton a = WildcardQuery.toAutomaton(new Term("field", "*?".repeat(n))); assertTrue( From be2e99140a48a925b0a967593f3bc1cd4de639ff Mon Sep 17 00:00:00 2001 From: vismaytiwari Date: Sat, 11 Jul 2026 19:54:41 +0530 Subject: [PATCH 5/6] Simplify the repeated-asterisk collapse: drop the state flag MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the lastWasWildcardString boolean (which had to be set in every switch branch) with a check on whether the previous segment is already a total ("any string") automaton via Operations.isTotal. Behavior is unchanged — consecutive '*' still collapse to a single any-string — and the other branches become plain one-liners. --- .../apache/lucene/search/WildcardQuery.java | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/search/WildcardQuery.java b/lucene/core/src/java/org/apache/lucene/search/WildcardQuery.java index 7896cb769965..15153294f39c 100644 --- a/lucene/core/src/java/org/apache/lucene/search/WildcardQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/WildcardQuery.java @@ -69,22 +69,19 @@ public static Automaton toAutomaton(Term wildcardquery) { String wildcardText = wildcardquery.text(); - boolean lastWasWildcardString = false; for (int i = 0; i < wildcardText.length(); ) { final int c = wildcardText.codePointAt(i); int length = Character.charCount(c); switch (c) { case WILDCARD_STRING -> { - // collapse consecutive '*' into a single any-string automaton - if (!lastWasWildcardString) { + // Collapse consecutive '*' ("**" is equivalent to "*"): only append an + // any-string automaton when the previous segment isn't already one. + if (automata.isEmpty() + || Operations.isTotal(automata.get(automata.size() - 1)) == false) { automata.add(Automata.makeAnyString()); - lastWasWildcardString = true; } } - case WILDCARD_CHAR -> { - automata.add(Automata.makeAnyChar()); - lastWasWildcardString = false; - } + case WILDCARD_CHAR -> automata.add(Automata.makeAnyChar()); case WILDCARD_ESCAPE -> { // add the escaped codepoint as a literal if it exists; a trailing // '\' is parsed leniently as a literal '\' @@ -95,12 +92,8 @@ public static Automaton toAutomaton(Term wildcardquery) { } else { automata.add(Automata.makeChar(c)); } - lastWasWildcardString = false; - } - default -> { - automata.add(Automata.makeChar(c)); - lastWasWildcardString = false; } + default -> automata.add(Automata.makeChar(c)); } i += length; } From 8353efd96133f9aebfad05baaf1b577efe54d27d Mon Sep 17 00:00:00 2001 From: vismaytiwari Date: Sun, 12 Jul 2026 00:56:52 +0530 Subject: [PATCH 6/6] Use List.getLast() for the previous-segment check --- .../core/src/java/org/apache/lucene/search/WildcardQuery.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/search/WildcardQuery.java b/lucene/core/src/java/org/apache/lucene/search/WildcardQuery.java index 15153294f39c..b92bdb9d0827 100644 --- a/lucene/core/src/java/org/apache/lucene/search/WildcardQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/WildcardQuery.java @@ -76,8 +76,7 @@ public static Automaton toAutomaton(Term wildcardquery) { case WILDCARD_STRING -> { // Collapse consecutive '*' ("**" is equivalent to "*"): only append an // any-string automaton when the previous segment isn't already one. - if (automata.isEmpty() - || Operations.isTotal(automata.get(automata.size() - 1)) == false) { + if (automata.isEmpty() || Operations.isTotal(automata.getLast()) == false) { automata.add(Automata.makeAnyString()); } }