diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 5ae094c6ed64..b3114ad9e995 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -151,6 +151,10 @@ Improvements Optimizations --------------------- +* GITHUB#16382: WildcardQuery.toAutomaton now collapses consecutive unescaped '*' + wildcards into a single any-string automaton, avoiding a quadratic blow-up in + automaton size for patterns with many repeated asterisks. (Vismay Tiwari) + * GITHUB#16307: ReqExclBulkScorer now skips runs of excluded docs via TwoPhaseIterator#docIDRunEnd for two-phase excluded clauses (e.g. a doc-values not-equals filter), instead of advancing one doc at a time. (Jim Ferenczi) diff --git a/lucene/core/src/java/org/apache/lucene/search/WildcardQuery.java b/lucene/core/src/java/org/apache/lucene/search/WildcardQuery.java index 9c1e8c223ed5..b92bdb9d0827 100644 --- a/lucene/core/src/java/org/apache/lucene/search/WildcardQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/WildcardQuery.java @@ -64,7 +64,6 @@ public WildcardQuery(Term term, RewriteMethod rewriteMethod) { * * @lucene.internal */ - @SuppressWarnings("fallthrough") public static Automaton toAutomaton(Term wildcardquery) { List automata = new ArrayList<>(); @@ -74,22 +73,26 @@ public static Automaton toAutomaton(Term wildcardquery) { final int c = wildcardText.codePointAt(i); int length = Character.charCount(c); switch (c) { - case WILDCARD_STRING: - automata.add(Automata.makeAnyString()); - break; - case WILDCARD_CHAR: - automata.add(Automata.makeAnyChar()); - break; - case WILDCARD_ESCAPE: - // add the next codepoint instead, if it exists + case WILDCARD_STRING -> { + // Collapse consecutive '*' ("**" is equivalent to "*"): only append an + // any-string automaton when the previous segment isn't already one. + if (automata.isEmpty() || Operations.isTotal(automata.getLast()) == false) { + automata.add(Automata.makeAnyString()); + } + } + case WILDCARD_CHAR -> automata.add(Automata.makeAnyChar()); + case WILDCARD_ESCAPE -> { + // add the escaped codepoint as a literal if it exists; a trailing + // '\' is parsed leniently as a literal '\' if (i + length < wildcardText.length()) { final int nextChar = wildcardText.codePointAt(i + length); length += Character.charCount(nextChar); automata.add(Automata.makeChar(nextChar)); - break; - } // else fallthru, lenient parsing with a trailing \ - default: - automata.add(Automata.makeChar(c)); + } else { + automata.add(Automata.makeChar(c)); + } + } + default -> automata.add(Automata.makeChar(c)); } i += length; } diff --git a/lucene/core/src/test/org/apache/lucene/search/TestWildcardQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestWildcardQuery.java index c076367ce5b7..d7f81ed71cd2 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestWildcardQuery.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestWildcardQuery.java @@ -38,8 +38,11 @@ import org.apache.lucene.tests.analysis.MockAnalyzer; import org.apache.lucene.tests.index.RandomIndexWriter; import org.apache.lucene.tests.util.LuceneTestCase; +import org.apache.lucene.tests.util.automaton.AutomatonTestUtil; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.automaton.Automaton; import org.apache.lucene.util.automaton.CompiledAutomaton; +import org.apache.lucene.util.automaton.Operations; /** TestWildcardQuery tests the '*' and '?' wildcard characters. */ public class TestWildcardQuery extends LuceneTestCase { @@ -235,6 +238,50 @@ public void testEscapes() throws Exception { indexStore.close(); } + private static Automaton det(Automaton a) { + return Operations.determinize(a, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT); + } + + public void testRepeatedAsterisksCollapse() { + // Consecutive '*' must collapse to a single any-string automaton: same + // language as a single '*', and the same (tiny) shape, not a huge one. + Automaton single = det(WildcardQuery.toAutomaton(new Term("field", "*"))); + Automaton repeated = det(WildcardQuery.toAutomaton(new Term("field", "*".repeat(1000)))); + + assertTrue(AutomatonTestUtil.sameLanguage(single, repeated)); + assertEquals(single.getNumStates(), repeated.getNumStates()); + assertEquals(single.getNumTransitions(), repeated.getNumTransitions()); + } + + public void testRepeatedAsterisksCollapseWithEscapesAndLiterals() { + // Collapsing applies only to unescaped, consecutive '*'. An escaped '\*' + // stays a literal and the surrounding literals / '?' are untouched, so + // "a\*b***c?" matches the same language as its single-'*' form "a\*b*c?". + Automaton collapsed = det(WildcardQuery.toAutomaton(new Term("field", "a\\*b***c?"))); + Automaton reference = det(WildcardQuery.toAutomaton(new Term("field", "a\\*b*c?"))); + + assertTrue(AutomatonTestUtil.sameLanguage(reference, collapsed)); + assertEquals(reference.getNumStates(), collapsed.getNumStates()); + assertEquals(reference.getNumTransitions(), collapsed.getNumTransitions()); + + // Escaped '\*' is a literal, so "\*\*\*" matches the string "***", not the + // any-string '*' — it must NOT be collapsed. + Automaton single = det(WildcardQuery.toAutomaton(new Term("field", "*"))); + Automaton literalStars = det(WildcardQuery.toAutomaton(new Term("field", "\\*\\*\\*"))); + assertFalse(AutomatonTestUtil.sameLanguage(single, literalStars)); + } + + public void testInterleavedStarQuestionDoesNotBlowUp() { + // '*?' repeated does not collapse (the '*' are not consecutive), but it + // must still build a linearly-sized automaton rather than exploding the way + // repeated '*' used to before this fix (GITHUB#16134). + int n = 5000; + Automaton a = WildcardQuery.toAutomaton(new Term("field", "*?".repeat(n))); + assertTrue( + "expected roughly linear size, got " + a.getNumTransitions() + " transitions for " + n, + a.getNumTransitions() < 100L * n); + } + private Directory getIndexStore(String field, String[] contents) throws IOException { Directory indexStore = newDirectory(); RandomIndexWriter writer = new RandomIndexWriter(random(), indexStore);