Using prefixQuery to speedup wildcardQuery&regexpQuery in wildcard type when no TermQuery is present

rest-api-spec/src/main/resources/rest-api-spec/test/search/270_wildcard_fieldtype_queries.yml

@@ -226,6 +226,113 @@ setup:
                 value: ".*06-08.*cluster-manager node.*"
   - match: { hits.total.value: 2 }
 
+---
+"wildcard field short terms query matches":
+  - do:
+      indices.create:
+        index: test2
+        body:
+          mappings:
+            properties:
+              my_field:
+                type: wildcard
+
+  - do:
+      index:
+        index: test2
+        id: 1
+        body:
+          my_field: "ab"
+  - do:
+      index:
+        index: test2
+        id: 2
+        body:
+          my_field: "axxb"
+  - do:
+      index:
+        index: test2
+        id: 3
+        body:
+          my_field: "ba"
+  - do:
+      index:
+        index: test2
+        id: 4
+        body:
+          my_field: "ac"
+  - do:
+      index:
+        index: test2
+        id: 5
+        body:
+          my_field: "a-cde"
+  - do:
+      index:
+        index: test2
+        id: 6
+        body:
+          my_field: "cde-a"
+  - do:
+      index:
+        index: test2
+        id: 7
+        body:
+          my_field: "cde"
+  - do:
+      index:
+        index: test2
+        id: 8
+        body:
+          my_field: "a-cd"
+  - do:
+      indices.refresh:
+        index: test2
+
+  - do:
+      search:
+        index: test2
+        body:
+          size: 0
+          query:
+            wildcard:
+              my_field:
+                value: "*a*b*"
+  - match: { hits.total.value: 2 }
+
+  - do:
+      search:
+        index: test2
+        body:
+          size: 0
+          query:
+            regexp:
+              my_field:
+                value: ".*a.*b.*"
+  - match: { hits.total.value: 2 }
+
+  - do:
+      search:
+        index: test2
+        body:
+          size: 0
+          query:
+            wildcard:
+              my_field:
+                value: "*a*cde*"
+  - match: { hits.total.value: 1 }
+
+  - do:
+      search:
+        index: test2
+        body:
+          size: 0
+          query:
+            regexp:
+              my_field:
+                value: ".*a.*cde.*"
+  - match: { hits.total.value: 1 }
+
 ---
 "regexp query matches lowercase-normalized field":
   - do:
@@ -435,3 +542,10 @@ setup:
           size: 4
   - match: { hits.total.value: 8 }
   - match: { hits.hits.0._id: "8" }
+
+---
+teardown:
+  - do:
+      indices.delete:
+        index: test,test1,test2
+        ignore: 404

server/src/main/java/org/opensearch/index/mapper/WildcardFieldMapper.java

@@ -25,6 +25,7 @@
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.MultiTermQuery;
+import org.apache.lucene.search.PrefixQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.QueryVisitor;
 import org.apache.lucene.search.ScoreMode;
@@ -465,6 +466,7 @@ static Set<String> getRequiredNGrams(String value, boolean regexpMode) {
             String rawSequence = null;
             String currentSequence = null;
             char[] buffer = new char[NGRAM_SIZE];
+            List<String> possiblePrefixTerms = new ArrayList<>();
             if (!value.startsWith("?") && !value.startsWith("*")) {
                 // Can add prefix term
                 rawSequence = getNonWildcardSequence(value, 0);
@@ -486,8 +488,12 @@ static Set<String> getRequiredNGrams(String value, boolean regexpMode) {
                 boolean isEndOfValue = pos + rawSequence.length() == value.length();
                 currentSequence = performEscape(rawSequence, regexpMode);
 
-                for (int i = 0; i < currentSequence.length() - NGRAM_SIZE + 1; i++) {
-                    terms.add(currentSequence.substring(i, i + 3));
+                if (currentSequence.length() >= NGRAM_SIZE) {
+                    for (int i = 0; i < currentSequence.length() - NGRAM_SIZE + 1; i++) {
+                        terms.add(currentSequence.substring(i, i + 3));
+                    }
+                } else if (isEndOfValue != true && currentSequence.isEmpty() == false) {
+                    possiblePrefixTerms.add(currentSequence);
                 }
                 if (isEndOfValue) {
                     // This is the end of the input. We can attach a suffix anchor.
@@ -510,6 +516,12 @@ static Set<String> getRequiredNGrams(String value, boolean regexpMode) {
                 pos = findNonWildcardSequence(value, pos + rawSequence.length());
                 rawSequence = getNonWildcardSequence(value, pos);
             }
+
+            if (terms.isEmpty()) {
+                // If there are 3-gram terms, keep the first phase selective with shorter prefix terms.
+                terms.addAll(possiblePrefixTerms);
+            }
+
             return terms;
         }
 
@@ -616,9 +628,25 @@ public Query regexpQuery(
                 };
             }
 
-            Query approximation = regexpToQuery(name(), regExp, caseInsensitive);
+            List<String> possiblePrefixTerms = new ArrayList<>();
+            Query approximation = regexpToQuery(name(), regExp, caseInsensitive, possiblePrefixTerms);
             if (approximation instanceof MatchAllDocsQuery) {
-                approximation = existsQuery(context);
+                if (possiblePrefixTerms.isEmpty()) {
+                    approximation = existsQuery(context);
+                } else {
+                    String prefixTerm = possiblePrefixTerms.get(0);
+                    for (int i = 1; i < possiblePrefixTerms.size(); i++) {
+                        String candidate = possiblePrefixTerms.get(i);
+                        if (candidate.length() > prefixTerm.length()) {
+                            prefixTerm = candidate;
+                        }
+                    }
+                    if (caseInsensitive) {
+                        approximation = AutomatonQueries.caseInsensitivePrefixQuery(new Term(name(), prefixTerm));
+                    } else {
+                        approximation = new PrefixQuery(new Term(name(), prefixTerm));
+                    }
+                }
             }
             return new WildcardMatchingQuery(name(), approximation, regexpPredicate, "/" + finalValue + "/", context, this);
         }
@@ -630,16 +658,16 @@ public Query regexpQuery(
          * @param regExp a parsed node in the {@link RegExp} tree
          * @return a query that matches on the known required parts of the given regular expression
          */
-        private static Query regexpToQuery(String fieldName, RegExp regExp, boolean caseInsensitive) {
+        private static Query regexpToQuery(String fieldName, RegExp regExp, boolean caseInsensitive, List<String> possiblePrefixTerms) {
             BooleanQuery query;
             if (Objects.requireNonNull(regExp.kind) == RegExp.Kind.REGEXP_UNION) {
                 List<Query> clauses = new ArrayList<>();
                 while (regExp.exp1.kind == RegExp.Kind.REGEXP_UNION) {
-                    clauses.add(regexpToQuery(fieldName, regExp.exp2, caseInsensitive));
+                    clauses.add(regexpToQuery(fieldName, regExp.exp2, caseInsensitive, null));
                     regExp = regExp.exp1;
                 }
-                clauses.add(regexpToQuery(fieldName, regExp.exp2, caseInsensitive));
-                clauses.add(regexpToQuery(fieldName, regExp.exp1, caseInsensitive));
+                clauses.add(regexpToQuery(fieldName, regExp.exp2, caseInsensitive, null));
+                clauses.add(regexpToQuery(fieldName, regExp.exp1, caseInsensitive, null));
                 BooleanQuery.Builder builder = new BooleanQuery.Builder();
                 for (int i = clauses.size() - 1; i >= 0; i--) {
                     Query clause = clauses.get(i);
@@ -652,23 +680,28 @@ private static Query regexpToQuery(String fieldName, RegExp regExp, boolean case
             } else if (regExp.kind == RegExp.Kind.REGEXP_STRING) {
                 BooleanQuery.Builder builder = new BooleanQuery.Builder();
                 for (String string : getRequiredNGrams("*" + regExp.s + "*", true)) {
-                    final Query subQuery;
-                    if (caseInsensitive) {
-                        subQuery = AutomatonQueries.caseInsensitiveTermQuery(new Term(fieldName, string));
-                    } else {
-                        subQuery = new TermQuery(new Term(fieldName, string));
+                    if (string.length() == NGRAM_SIZE) {
+                        final Query subQuery;
+                        if (caseInsensitive) {
+                            subQuery = AutomatonQueries.caseInsensitiveTermQuery(new Term(fieldName, string));
+                        } else {
+                            subQuery = new TermQuery(new Term(fieldName, string));
+                        }
+                        builder.add(subQuery, BooleanClause.Occur.FILTER);
+                    } else if (possiblePrefixTerms != null) {
+                        assert string.length() < NGRAM_SIZE;
+                        possiblePrefixTerms.add(string);
                     }
-                    builder.add(subQuery, BooleanClause.Occur.FILTER);
                 }
                 query = builder.build();
             } else if (regExp.kind == RegExp.Kind.REGEXP_CONCATENATION) {
                 List<Query> clauses = new ArrayList<>();
                 while (regExp.exp1.kind == RegExp.Kind.REGEXP_CONCATENATION) {
-                    clauses.add(regexpToQuery(fieldName, regExp.exp2, caseInsensitive));
+                    clauses.add(regexpToQuery(fieldName, regExp.exp2, caseInsensitive, possiblePrefixTerms));
                     regExp = regExp.exp1;
                 }
-                clauses.add(regexpToQuery(fieldName, regExp.exp2, caseInsensitive));
-                clauses.add(regexpToQuery(fieldName, regExp.exp1, caseInsensitive));
+                clauses.add(regexpToQuery(fieldName, regExp.exp2, caseInsensitive, possiblePrefixTerms));
+                clauses.add(regexpToQuery(fieldName, regExp.exp1, caseInsensitive, possiblePrefixTerms));
                 BooleanQuery.Builder builder = new BooleanQuery.Builder();
                 for (int i = clauses.size() - 1; i >= 0; i--) {
                     Query clause = clauses.get(i);
@@ -679,14 +712,17 @@ private static Query regexpToQuery(String fieldName, RegExp regExp, boolean case
                 query = builder.build();
             } else if ((regExp.kind == RegExp.Kind.REGEXP_REPEAT_MIN || regExp.kind == RegExp.Kind.REGEXP_REPEAT_MINMAX)
                 && regExp.min > 0) {
-                    return regexpToQuery(fieldName, regExp.exp1, caseInsensitive);
+                    return regexpToQuery(fieldName, regExp.exp1, caseInsensitive, possiblePrefixTerms);
+                } else if (regExp.kind == RegExp.Kind.REGEXP_CHAR && possiblePrefixTerms != null) {
+                    possiblePrefixTerms.add(String.valueOf((char) regExp.c));
+                    return MatchAllDocsQuery.INSTANCE;
                 } else {
-                    return new MatchAllDocsQuery();
+                    return MatchAllDocsQuery.INSTANCE;
                 }
             if (query.clauses().size() == 1) {
                 return query.iterator().next().query();
             } else if (query.clauses().size() == 0) {
-                return new MatchAllDocsQuery();
+                return MatchAllDocsQuery.INSTANCE;
             }
             return query;
         }
@@ -730,10 +766,19 @@ private static BooleanQuery matchAllTermsQuery(String fieldName, Set<String> ter
             BooleanQuery.Builder matchAllTermsBuilder = new BooleanQuery.Builder();
             Query query;
             for (String term : terms) {
-                if (caseInsensitive) {
-                    query = AutomatonQueries.caseInsensitiveTermQuery(new Term(fieldName, term));
+                if (term.length() == NGRAM_SIZE) {
+                    if (caseInsensitive) {
+                        query = AutomatonQueries.caseInsensitiveTermQuery(new Term(fieldName, term));
+                    } else {
+                        query = new TermQuery(new Term(fieldName, term));
+                    }
                 } else {
-                    query = new TermQuery(new Term(fieldName, term));
+                    assert term.length() < NGRAM_SIZE;
+                    if (caseInsensitive) {
+                        query = AutomatonQueries.caseInsensitivePrefixQuery(new Term(fieldName, term));
+                    } else {
+                        query = new PrefixQuery(new Term(fieldName, term));
+                    }
                 }
                 matchAllTermsBuilder.add(query, BooleanClause.Occur.FILTER);
             }

server/src/test/java/org/opensearch/index/mapper/WildcardFieldTypeTests.java

@@ -11,6 +11,7 @@
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.PrefixQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermQuery;
 
@@ -142,6 +143,31 @@ public void testMultipleWildcardsInQuery() {
         assertFalse(actualMatchingQuery.getSecondPhaseMatcher().test("abcdzzzefgqqh"));
     }
 
+    // Test wildcard queries uses prefix queries Without three ngram Terms query
+    public void testWildcardQueryUsesPrefixQuery() {
+        MappedFieldType ft = new WildcardFieldMapper.WildcardFieldType("field");
+        BooleanQuery.Builder builder = new BooleanQuery.Builder();
+        builder.add(new PrefixQuery(new Term("field", "a")), BooleanClause.Occur.FILTER);
+        builder.add(new PrefixQuery(new Term("field", "b")), BooleanClause.Occur.FILTER);
+
+        String pattern = "*a*b*";
+        Query actual = ft.wildcardQuery(pattern, null, null);
+        assertEquals(new WildcardFieldMapper.WildcardMatchingQuery("field", builder.build(), pattern), actual);
+        WildcardFieldMapper.WildcardMatchingQuery actualMatchingQuery = (WildcardFieldMapper.WildcardMatchingQuery) actual;
+        assertTrue(actualMatchingQuery.getSecondPhaseMatcher().test("zzazbzz"));
+        assertFalse(actualMatchingQuery.getSecondPhaseMatcher().test("zzbza"));
+
+        pattern = "*ab*cde*";
+        builder = new BooleanQuery.Builder();
+        builder.add(new TermQuery(new Term("field", "cde")), BooleanClause.Occur.FILTER);
+
+        actual = ft.wildcardQuery(pattern, null, null);
+        assertEquals(new WildcardFieldMapper.WildcardMatchingQuery("field", builder.build(), pattern), actual);
+        actualMatchingQuery = (WildcardFieldMapper.WildcardMatchingQuery) actual;
+        assertTrue(actualMatchingQuery.getSecondPhaseMatcher().test("zzabzzcdezz"));
+        assertFalse(actualMatchingQuery.getSecondPhaseMatcher().test("zzcdezzabzz"));
+    }
+
     public void testEscapedBackslashFollowedByWildcard() {
         MappedFieldType ft = new WildcardFieldMapper.WildcardFieldType("field");
 
@@ -214,6 +240,39 @@ public void testRegexpQuery() {
         assertTrue(actualMatchingQuery.getSecondPhaseMatcher().test("abcghiqwertyjkl"));
     }
 
+    // Test regexp queries uses prefix queries Without three ngram Terms query
+    public void testRegexpQueryUsesPrefixQuery() {
+        MappedFieldType ft = new WildcardFieldMapper.WildcardFieldType("field");
+        String pattern = ".*ab.*a.*";
+
+        Query actual = ft.regexpQuery(pattern, 0, 0, 1000, null, null);
+        assertEquals(
+            new WildcardFieldMapper.WildcardMatchingQuery("field", new PrefixQuery(new Term("field", "ab")), "/" + pattern + "/"),
+            actual
+        );
+        WildcardFieldMapper.WildcardMatchingQuery actualMatchingQuery = (WildcardFieldMapper.WildcardMatchingQuery) actual;
+        assertTrue(actualMatchingQuery.getSecondPhaseMatcher().test("foo_ab_a"));
+        assertFalse(actualMatchingQuery.getSecondPhaseMatcher().test("foo_a_ab"));
+
+        pattern = ".*(ab|cd).*";
+        actual = ft.regexpQuery(pattern, 0, 0, 1000, null, null);
+        assertEquals(new WildcardFieldMapper.WildcardMatchingQuery("field", ft.existsQuery(null), "/" + pattern + "/"), actual);
+        actualMatchingQuery = (WildcardFieldMapper.WildcardMatchingQuery) actual;
+        assertTrue(actualMatchingQuery.getSecondPhaseMatcher().test("xxabxx"));
+        assertTrue(actualMatchingQuery.getSecondPhaseMatcher().test("xxcdxx"));
+        assertFalse(actualMatchingQuery.getSecondPhaseMatcher().test("xxefxx"));
+
+        pattern = ".*a.*cde.*";
+        actual = ft.regexpQuery(pattern, 0, 0, 1000, null, null);
+        assertEquals(
+            new WildcardFieldMapper.WildcardMatchingQuery("field", new TermQuery(new Term("field", "cde")), "/" + pattern + "/"),
+            actual
+        );
+        actualMatchingQuery = (WildcardFieldMapper.WildcardMatchingQuery) actual;
+        assertTrue(actualMatchingQuery.getSecondPhaseMatcher().test("foo_a_cde"));
+        assertFalse(actualMatchingQuery.getSecondPhaseMatcher().test("foo_cde_a"));
+    }
+
     public void testWildcardMatchAll() {
         String pattern = "???";
         MappedFieldType ft = new WildcardFieldMapper.WildcardFieldType("field");