From e06492ad549ed54ad5215e07b840efd3122986d8 Mon Sep 17 00:00:00 2001 From: kkewwei Date: Sun, 19 Jul 2026 18:31:34 +0800 Subject: [PATCH] Using prefixQuery to speedup wildcardQuery®expQuery in wildcard type when no TermQuery is present Signed-off-by: kkewwei Signed-off-by: kkewwei --- .../search/270_wildcard_fieldtype_queries.yml | 114 ++++++++++++++++++ .../index/mapper/WildcardFieldMapper.java | 91 ++++++++++---- .../index/mapper/WildcardFieldTypeTests.java | 59 +++++++++ 3 files changed, 241 insertions(+), 23 deletions(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/270_wildcard_fieldtype_queries.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/270_wildcard_fieldtype_queries.yml index cfad1f3a00165..0086a055404a1 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/270_wildcard_fieldtype_queries.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/270_wildcard_fieldtype_queries.yml @@ -226,6 +226,113 @@ setup: value: ".*06-08.*cluster-manager node.*" - match: { hits.total.value: 2 } +--- +"wildcard field short terms query matches": + - do: + indices.create: + index: test2 + body: + mappings: + properties: + my_field: + type: wildcard + + - do: + index: + index: test2 + id: 1 + body: + my_field: "ab" + - do: + index: + index: test2 + id: 2 + body: + my_field: "axxb" + - do: + index: + index: test2 + id: 3 + body: + my_field: "ba" + - do: + index: + index: test2 + id: 4 + body: + my_field: "ac" + - do: + index: + index: test2 + id: 5 + body: + my_field: "a-cde" + - do: + index: + index: test2 + id: 6 + body: + my_field: "cde-a" + - do: + index: + index: test2 + id: 7 + body: + my_field: "cde" + - do: + index: + index: test2 + id: 8 + body: + my_field: "a-cd" + - do: + indices.refresh: + index: test2 + + - do: + search: + index: test2 + body: + size: 0 + query: + wildcard: + my_field: + value: "*a*b*" + - match: { hits.total.value: 2 } + + - do: + search: + index: test2 + body: + size: 0 + query: + regexp: + my_field: + value: ".*a.*b.*" + - match: { hits.total.value: 2 } + + - do: + search: + index: test2 + body: + size: 0 + query: + wildcard: + my_field: + value: "*a*cde*" + - match: { hits.total.value: 1 } + + - do: + search: + index: test2 + body: + size: 0 + query: + regexp: + my_field: + value: ".*a.*cde.*" + - match: { hits.total.value: 1 } + --- "regexp query matches lowercase-normalized field": - do: @@ -435,3 +542,10 @@ setup: size: 4 - match: { hits.total.value: 8 } - match: { hits.hits.0._id: "8" } + +--- +teardown: + - do: + indices.delete: + index: test,test1,test2 + ignore: 404 diff --git a/server/src/main/java/org/opensearch/index/mapper/WildcardFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/WildcardFieldMapper.java index f29d8263a063a..ee2793b212240 100644 --- a/server/src/main/java/org/opensearch/index/mapper/WildcardFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/WildcardFieldMapper.java @@ -25,6 +25,7 @@ import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.MultiTermQuery; +import org.apache.lucene.search.PrefixQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.QueryVisitor; import org.apache.lucene.search.ScoreMode; @@ -465,6 +466,7 @@ static Set getRequiredNGrams(String value, boolean regexpMode) { String rawSequence = null; String currentSequence = null; char[] buffer = new char[NGRAM_SIZE]; + List possiblePrefixTerms = new ArrayList<>(); if (!value.startsWith("?") && !value.startsWith("*")) { // Can add prefix term rawSequence = getNonWildcardSequence(value, 0); @@ -486,8 +488,12 @@ static Set getRequiredNGrams(String value, boolean regexpMode) { boolean isEndOfValue = pos + rawSequence.length() == value.length(); currentSequence = performEscape(rawSequence, regexpMode); - for (int i = 0; i < currentSequence.length() - NGRAM_SIZE + 1; i++) { - terms.add(currentSequence.substring(i, i + 3)); + if (currentSequence.length() >= NGRAM_SIZE) { + for (int i = 0; i < currentSequence.length() - NGRAM_SIZE + 1; i++) { + terms.add(currentSequence.substring(i, i + 3)); + } + } else if (isEndOfValue != true && currentSequence.isEmpty() == false) { + possiblePrefixTerms.add(currentSequence); } if (isEndOfValue) { // This is the end of the input. We can attach a suffix anchor. @@ -510,6 +516,12 @@ static Set getRequiredNGrams(String value, boolean regexpMode) { pos = findNonWildcardSequence(value, pos + rawSequence.length()); rawSequence = getNonWildcardSequence(value, pos); } + + if (terms.isEmpty()) { + // If there are 3-gram terms, keep the first phase selective with shorter prefix terms. + terms.addAll(possiblePrefixTerms); + } + return terms; } @@ -616,9 +628,25 @@ public Query regexpQuery( }; } - Query approximation = regexpToQuery(name(), regExp, caseInsensitive); + List possiblePrefixTerms = new ArrayList<>(); + Query approximation = regexpToQuery(name(), regExp, caseInsensitive, possiblePrefixTerms); if (approximation instanceof MatchAllDocsQuery) { - approximation = existsQuery(context); + if (possiblePrefixTerms.isEmpty()) { + approximation = existsQuery(context); + } else { + String prefixTerm = possiblePrefixTerms.get(0); + for (int i = 1; i < possiblePrefixTerms.size(); i++) { + String candidate = possiblePrefixTerms.get(i); + if (candidate.length() > prefixTerm.length()) { + prefixTerm = candidate; + } + } + if (caseInsensitive) { + approximation = AutomatonQueries.caseInsensitivePrefixQuery(new Term(name(), prefixTerm)); + } else { + approximation = new PrefixQuery(new Term(name(), prefixTerm)); + } + } } return new WildcardMatchingQuery(name(), approximation, regexpPredicate, "/" + finalValue + "/", context, this); } @@ -630,16 +658,16 @@ public Query regexpQuery( * @param regExp a parsed node in the {@link RegExp} tree * @return a query that matches on the known required parts of the given regular expression */ - private static Query regexpToQuery(String fieldName, RegExp regExp, boolean caseInsensitive) { + private static Query regexpToQuery(String fieldName, RegExp regExp, boolean caseInsensitive, List possiblePrefixTerms) { BooleanQuery query; if (Objects.requireNonNull(regExp.kind) == RegExp.Kind.REGEXP_UNION) { List clauses = new ArrayList<>(); while (regExp.exp1.kind == RegExp.Kind.REGEXP_UNION) { - clauses.add(regexpToQuery(fieldName, regExp.exp2, caseInsensitive)); + clauses.add(regexpToQuery(fieldName, regExp.exp2, caseInsensitive, null)); regExp = regExp.exp1; } - clauses.add(regexpToQuery(fieldName, regExp.exp2, caseInsensitive)); - clauses.add(regexpToQuery(fieldName, regExp.exp1, caseInsensitive)); + clauses.add(regexpToQuery(fieldName, regExp.exp2, caseInsensitive, null)); + clauses.add(regexpToQuery(fieldName, regExp.exp1, caseInsensitive, null)); BooleanQuery.Builder builder = new BooleanQuery.Builder(); for (int i = clauses.size() - 1; i >= 0; i--) { Query clause = clauses.get(i); @@ -652,23 +680,28 @@ private static Query regexpToQuery(String fieldName, RegExp regExp, boolean case } else if (regExp.kind == RegExp.Kind.REGEXP_STRING) { BooleanQuery.Builder builder = new BooleanQuery.Builder(); for (String string : getRequiredNGrams("*" + regExp.s + "*", true)) { - final Query subQuery; - if (caseInsensitive) { - subQuery = AutomatonQueries.caseInsensitiveTermQuery(new Term(fieldName, string)); - } else { - subQuery = new TermQuery(new Term(fieldName, string)); + if (string.length() == NGRAM_SIZE) { + final Query subQuery; + if (caseInsensitive) { + subQuery = AutomatonQueries.caseInsensitiveTermQuery(new Term(fieldName, string)); + } else { + subQuery = new TermQuery(new Term(fieldName, string)); + } + builder.add(subQuery, BooleanClause.Occur.FILTER); + } else if (possiblePrefixTerms != null) { + assert string.length() < NGRAM_SIZE; + possiblePrefixTerms.add(string); } - builder.add(subQuery, BooleanClause.Occur.FILTER); } query = builder.build(); } else if (regExp.kind == RegExp.Kind.REGEXP_CONCATENATION) { List clauses = new ArrayList<>(); while (regExp.exp1.kind == RegExp.Kind.REGEXP_CONCATENATION) { - clauses.add(regexpToQuery(fieldName, regExp.exp2, caseInsensitive)); + clauses.add(regexpToQuery(fieldName, regExp.exp2, caseInsensitive, possiblePrefixTerms)); regExp = regExp.exp1; } - clauses.add(regexpToQuery(fieldName, regExp.exp2, caseInsensitive)); - clauses.add(regexpToQuery(fieldName, regExp.exp1, caseInsensitive)); + clauses.add(regexpToQuery(fieldName, regExp.exp2, caseInsensitive, possiblePrefixTerms)); + clauses.add(regexpToQuery(fieldName, regExp.exp1, caseInsensitive, possiblePrefixTerms)); BooleanQuery.Builder builder = new BooleanQuery.Builder(); for (int i = clauses.size() - 1; i >= 0; i--) { Query clause = clauses.get(i); @@ -679,14 +712,17 @@ private static Query regexpToQuery(String fieldName, RegExp regExp, boolean case query = builder.build(); } else if ((regExp.kind == RegExp.Kind.REGEXP_REPEAT_MIN || regExp.kind == RegExp.Kind.REGEXP_REPEAT_MINMAX) && regExp.min > 0) { - return regexpToQuery(fieldName, regExp.exp1, caseInsensitive); + return regexpToQuery(fieldName, regExp.exp1, caseInsensitive, possiblePrefixTerms); + } else if (regExp.kind == RegExp.Kind.REGEXP_CHAR && possiblePrefixTerms != null) { + possiblePrefixTerms.add(String.valueOf((char) regExp.c)); + return MatchAllDocsQuery.INSTANCE; } else { - return new MatchAllDocsQuery(); + return MatchAllDocsQuery.INSTANCE; } if (query.clauses().size() == 1) { return query.iterator().next().query(); } else if (query.clauses().size() == 0) { - return new MatchAllDocsQuery(); + return MatchAllDocsQuery.INSTANCE; } return query; } @@ -730,10 +766,19 @@ private static BooleanQuery matchAllTermsQuery(String fieldName, Set ter BooleanQuery.Builder matchAllTermsBuilder = new BooleanQuery.Builder(); Query query; for (String term : terms) { - if (caseInsensitive) { - query = AutomatonQueries.caseInsensitiveTermQuery(new Term(fieldName, term)); + if (term.length() == NGRAM_SIZE) { + if (caseInsensitive) { + query = AutomatonQueries.caseInsensitiveTermQuery(new Term(fieldName, term)); + } else { + query = new TermQuery(new Term(fieldName, term)); + } } else { - query = new TermQuery(new Term(fieldName, term)); + assert term.length() < NGRAM_SIZE; + if (caseInsensitive) { + query = AutomatonQueries.caseInsensitivePrefixQuery(new Term(fieldName, term)); + } else { + query = new PrefixQuery(new Term(fieldName, term)); + } } matchAllTermsBuilder.add(query, BooleanClause.Occur.FILTER); } diff --git a/server/src/test/java/org/opensearch/index/mapper/WildcardFieldTypeTests.java b/server/src/test/java/org/opensearch/index/mapper/WildcardFieldTypeTests.java index 496451f2aeaa2..e23f550b42d3f 100644 --- a/server/src/test/java/org/opensearch/index/mapper/WildcardFieldTypeTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/WildcardFieldTypeTests.java @@ -11,6 +11,7 @@ import org.apache.lucene.index.Term; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.PrefixQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; @@ -142,6 +143,31 @@ public void testMultipleWildcardsInQuery() { assertFalse(actualMatchingQuery.getSecondPhaseMatcher().test("abcdzzzefgqqh")); } + // Test wildcard queries uses prefix queries Without three ngram Terms query + public void testWildcardQueryUsesPrefixQuery() { + MappedFieldType ft = new WildcardFieldMapper.WildcardFieldType("field"); + BooleanQuery.Builder builder = new BooleanQuery.Builder(); + builder.add(new PrefixQuery(new Term("field", "a")), BooleanClause.Occur.FILTER); + builder.add(new PrefixQuery(new Term("field", "b")), BooleanClause.Occur.FILTER); + + String pattern = "*a*b*"; + Query actual = ft.wildcardQuery(pattern, null, null); + assertEquals(new WildcardFieldMapper.WildcardMatchingQuery("field", builder.build(), pattern), actual); + WildcardFieldMapper.WildcardMatchingQuery actualMatchingQuery = (WildcardFieldMapper.WildcardMatchingQuery) actual; + assertTrue(actualMatchingQuery.getSecondPhaseMatcher().test("zzazbzz")); + assertFalse(actualMatchingQuery.getSecondPhaseMatcher().test("zzbza")); + + pattern = "*ab*cde*"; + builder = new BooleanQuery.Builder(); + builder.add(new TermQuery(new Term("field", "cde")), BooleanClause.Occur.FILTER); + + actual = ft.wildcardQuery(pattern, null, null); + assertEquals(new WildcardFieldMapper.WildcardMatchingQuery("field", builder.build(), pattern), actual); + actualMatchingQuery = (WildcardFieldMapper.WildcardMatchingQuery) actual; + assertTrue(actualMatchingQuery.getSecondPhaseMatcher().test("zzabzzcdezz")); + assertFalse(actualMatchingQuery.getSecondPhaseMatcher().test("zzcdezzabzz")); + } + public void testEscapedBackslashFollowedByWildcard() { MappedFieldType ft = new WildcardFieldMapper.WildcardFieldType("field"); @@ -214,6 +240,39 @@ public void testRegexpQuery() { assertTrue(actualMatchingQuery.getSecondPhaseMatcher().test("abcghiqwertyjkl")); } + // Test regexp queries uses prefix queries Without three ngram Terms query + public void testRegexpQueryUsesPrefixQuery() { + MappedFieldType ft = new WildcardFieldMapper.WildcardFieldType("field"); + String pattern = ".*ab.*a.*"; + + Query actual = ft.regexpQuery(pattern, 0, 0, 1000, null, null); + assertEquals( + new WildcardFieldMapper.WildcardMatchingQuery("field", new PrefixQuery(new Term("field", "ab")), "/" + pattern + "/"), + actual + ); + WildcardFieldMapper.WildcardMatchingQuery actualMatchingQuery = (WildcardFieldMapper.WildcardMatchingQuery) actual; + assertTrue(actualMatchingQuery.getSecondPhaseMatcher().test("foo_ab_a")); + assertFalse(actualMatchingQuery.getSecondPhaseMatcher().test("foo_a_ab")); + + pattern = ".*(ab|cd).*"; + actual = ft.regexpQuery(pattern, 0, 0, 1000, null, null); + assertEquals(new WildcardFieldMapper.WildcardMatchingQuery("field", ft.existsQuery(null), "/" + pattern + "/"), actual); + actualMatchingQuery = (WildcardFieldMapper.WildcardMatchingQuery) actual; + assertTrue(actualMatchingQuery.getSecondPhaseMatcher().test("xxabxx")); + assertTrue(actualMatchingQuery.getSecondPhaseMatcher().test("xxcdxx")); + assertFalse(actualMatchingQuery.getSecondPhaseMatcher().test("xxefxx")); + + pattern = ".*a.*cde.*"; + actual = ft.regexpQuery(pattern, 0, 0, 1000, null, null); + assertEquals( + new WildcardFieldMapper.WildcardMatchingQuery("field", new TermQuery(new Term("field", "cde")), "/" + pattern + "/"), + actual + ); + actualMatchingQuery = (WildcardFieldMapper.WildcardMatchingQuery) actual; + assertTrue(actualMatchingQuery.getSecondPhaseMatcher().test("foo_a_cde")); + assertFalse(actualMatchingQuery.getSecondPhaseMatcher().test("foo_cde_a")); + } + public void testWildcardMatchAll() { String pattern = "???"; MappedFieldType ft = new WildcardFieldMapper.WildcardFieldType("field");