Smarter use of sort-field optimizations in DV range queries

lucene/CHANGES.txt

@@ -377,6 +377,9 @@ Optimizations
 * GITHUB#16380: Speed up filtered disjunction queries when the filter is part of the
   index sort by advancing the filter before calculating impacts. (Alan Woodward)
 
+* GITHUB#16424: Use secondary or lower sorts to accelerate XXDocValuesRangeQueries if the primary
+  sort field is empty or only contains a single value. (Alan Woodward)
+
 Bug Fixes
 ---------------------
 * GITHUB#16350: Disable bulk-scoring in monitor queries. (Alan Woodward)

lucene/core/src/java/org/apache/lucene/document/SortedNumericDocValuesRangeQuery.java

@@ -138,11 +138,13 @@ public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOExcepti
         final NumericDocValues singleton = DocValues.unwrapSingleton(values);
         final DocValuesSkipper skipper = context.reader().getDocValuesSkipper(field);
 
-        final SortField primarySortField;
         if (singleton != null) {
-          if (skipper != null
-              && (primarySortField = densePrimarySort(context.reader(), skipper)) != null) {
-            return getScorerSupplierFromDensePrimarySort(singleton, skipper, primarySortField);
+          if (skipper != null) {
+            SortField primarySortField =
+                canUsePrimarySortShortcut(context.reader(), field, skipper);
+            if (primarySortField != null) {
+              return getScorerSupplierFromDensePrimarySort(singleton, skipper, primarySortField);
+            }
           }
           // A single two-phase iterator covers every density: its approximation rides the skipper
           // (no over-scan), its intoBitSet bulk-evaluates dense blocks, and YES runs collect as
@@ -224,16 +226,23 @@ protected int nextDoc(int startDocId, LongPredicate predicate) throws IOExceptio
     };
   }
 
-  private SortField densePrimarySort(LeafReader reader, DocValuesSkipper skipper) {
-    if (skipper.docCount() != reader.maxDoc()) {
+  /**
+   * Returns the primary sort field if it is safe to use the {@link SortedSkipperScorerSupplier}
+   * range shortcuts for this query, or {@code null} if the shortcut cannot be applied.
+   *
+   * <p>The shortcuts ({@code skipperMinDocId=0} and {@code skipperMaxDocId=docCount}) assume that
+   * docs without a value do not appear within or before the matching doc-ID range. This requires
+   * that the field is dense (every doc has a value, {@code docCount == reader.maxDoc()}).
+   */
+  private static SortField canUsePrimarySortShortcut(
+      LeafReader reader, String field, DocValuesSkipper skipper) throws IOException {
+    SortField sf = Sort.getPrimarySortField(reader);
+    if (sf == null || sf.getField().equals(field) == false) {
       return null;
     }
-    final Sort indexSort = reader.getMetaData().sort();
-    if (indexSort == null
-        || indexSort.getSort().length == 0
-        || indexSort.getSort()[0].getField().equals(field) == false) {
+    if (skipper.docCount() != reader.maxDoc()) {
       return null;
     }
-    return indexSort.getSort()[0];
+    return sf;
   }
 }

lucene/core/src/java/org/apache/lucene/document/SortedSetDocValuesRangeQuery.java

@@ -121,12 +121,13 @@ public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOExcepti
         DocValuesSkipper skipper = context.reader().getDocValuesSkipper(field);
         SortedSetDocValues values = DocValues.getSortedSet(context.reader(), field);
         final SortedDocValues singleton = DocValues.unwrapSingleton(values);
-        final SortField primarySortField;
-        if (singleton != null
-            && skipper != null
-            && (primarySortField = densePrimarySort(context.reader(), skipper)) != null) {
-          return getScorerSupplierFromDensePrimarySort(
-              singleton, values, skipper, primarySortField);
+        if (singleton != null && skipper != null) {
+          SortField primarySortField =
+              canUsePrimarySortShortcut(context.reader(), field, skipper.docCount());
+          if (primarySortField != null) {
+            return getScorerSupplierFromDensePrimarySort(
+                singleton, values, skipper, primarySortField);
+          }
         }
         // implement ScorerSupplier, since we do some expensive stuff to make a scorer
         return new ConstantScoreScorerSupplier(score(), scoreMode, context.reader().maxDoc()) {
@@ -276,16 +277,23 @@ private long maxOrd(SortedSetDocValues values) throws IOException {
     return maxOrd;
   }
 
-  private SortField densePrimarySort(LeafReader reader, DocValuesSkipper skipper) {
-    if (skipper.docCount() != reader.maxDoc()) {
+  /**
+   * Returns the primary sort field if it is safe to use the {@link SortedSkipperScorerSupplier}
+   * range shortcuts for this query, or {@code null} if the shortcut cannot be applied.
+   *
+   * <p>The shortcuts assume that docs without a value do not appear before or within the matching
+   * doc-ID range. This requires that the field is dense (every doc has a value, {@code docCount ==
+   * reader.maxDoc()}).
+   */
+  private static SortField canUsePrimarySortShortcut(LeafReader reader, String field, int docCount)
+      throws IOException {
+    SortField sf = Sort.getPrimarySortField(reader);
+    if (sf == null || sf.getField().equals(field) == false) {
       return null;
     }
-    final Sort indexSort = reader.getMetaData().sort();
-    if (indexSort == null
-        || indexSort.getSort().length == 0
-        || indexSort.getSort()[0].getField().equals(field) == false) {
+    if (docCount != reader.maxDoc()) {
       return null;
     }
-    return indexSort.getSort()[0];
+    return sf;
   }
 }

lucene/core/src/java/org/apache/lucene/search/IndexSortSortedNumericDocValuesRangeQuery.java

@@ -190,21 +190,19 @@ public int count(LeafReaderContext context) throws IOException {
           }
 
           // use index sort optimization if possible
-          Sort indexSort = reader.getMetaData().sort();
-          if (indexSort != null
-              && indexSort.getSort().length > 0
-              && indexSort.getSort()[0].getField().equals(field)) {
-            final SortField sortField = indexSort.getSort()[0];
-            final SortField.Type sortFieldType = getSortFieldType(sortField);
+          SortField primarySortField = Sort.getPrimarySortField(reader);
+          if (primarySortField != null && primarySortField.getField().equals(field)) {
+            final SortField.Type sortFieldType = getSortFieldType(primarySortField);
             // The index sort optimization is only supported for Type.INT and Type.LONG
             if (sortFieldType == Type.INT || sortFieldType == Type.LONG) {
-              Object missingValue = sortField.getMissingValue();
+              Object missingValue = primarySortField.getMissingValue();
               final long missingLongValue =
                   missingValue == null ? 0L : ((Number) missingValue).longValue();
               // all documents have docValues or missing value falls outside the range
               if ((pointValues != null && pointValues.getDocCount() == reader.maxDoc())
                   || (missingLongValue < lowerValue || missingLongValue > upperValue)) {
-                itAndCount = getDocIdSetIterator(sortField, sortFieldType, context, numericValues);
+                itAndCount =
+                    getDocIdSetIterator(primarySortField, sortFieldType, context, numericValues);
               }
               if (itAndCount != null && itAndCount.count != -1) {
                 return itAndCount.count;
@@ -419,14 +417,12 @@ private boolean matchAll(PointValues points, byte[] queryLowerPoint, byte[] quer
 
   private IteratorAndCount getDocIdSetIteratorOrNullFromBkd(
       LeafReaderContext context, DocIdSetIterator delegate) throws IOException {
-    Sort indexSort = context.reader().getMetaData().sort();
-    if (indexSort == null
-        || indexSort.getSort().length == 0
-        || indexSort.getSort()[0].getField().equals(field) == false) {
+    SortField primarySortField = Sort.getPrimarySortField(context.reader());
+    if (primarySortField == null || primarySortField.getField().equals(field) == false) {
       return null;
     }
 
-    final boolean reverse = indexSort.getSort()[0].getReverse();
+    final boolean reverse = primarySortField.getReverse();
 
     PointValues points = context.reader().getPointValues(field);
     if (points == null) {
@@ -519,16 +515,12 @@ private IteratorAndCount getDocIdSetIteratorOrNull(LeafReaderContext context) th
       if (itAndCount != null) {
         return itAndCount;
       }
-      Sort indexSort = context.reader().getMetaData().sort();
-      if (indexSort != null
-          && indexSort.getSort().length > 0
-          && indexSort.getSort()[0].getField().equals(field)) {
-
-        final SortField sortField = indexSort.getSort()[0];
-        final SortField.Type sortFieldType = getSortFieldType(sortField);
+      SortField primarySortField = Sort.getPrimarySortField(context.reader());
+      if (primarySortField != null && primarySortField.getField().equals(field)) {
+        final SortField.Type sortFieldType = getSortFieldType(primarySortField);
         // The index sort optimization is only supported for Type.INT and Type.LONG
         if (sortFieldType == Type.INT || sortFieldType == Type.LONG) {
-          return getDocIdSetIterator(sortField, sortFieldType, context, numericValues);
+          return getDocIdSetIterator(primarySortField, sortFieldType, context, numericValues);
         }
       }
     }

lucene/core/src/java/org/apache/lucene/search/Sort.java

@@ -18,6 +18,8 @@
 
 import java.io.IOException;
 import java.util.Arrays;
+import org.apache.lucene.index.DocValuesSkipper;
+import org.apache.lucene.index.LeafReader;
 
 /**
  * Encapsulates sort criteria for returned hits.
@@ -132,4 +134,37 @@ public boolean needsScores() {
     }
     return false;
   }
+
+  /**
+   * Returns the primary index sort field for the given LeafReader, or null if none is configured.
+   *
+   * <p>If sorting against a field would have no effect then that SortField is skipped over and the
+   * next SortField in the list is returned. This can happen for fields with no values in the
+   * segment, or fields that have only a single distinct value.
+   */
+  public static SortField getPrimarySortField(LeafReader reader) {
+    Sort sort = reader.getMetaData().sort();
+    if (sort == null) {
+      return null;
+    }
+    for (SortField sf : sort.fields) {
+      String field = sf.getField();
+      if (field == null) {
+        // Custom field that we don't know anything about, so return it as primary
+        return sf;
+      }
+      if (reader.getFieldInfos().fieldInfo(field) == null) {
+        // Field has no values in this segment, so sorting by it has no effect.
+        continue;
+      }
+      // If the field has a skip index, check whether all values are identical,
+      // in which case sorting by this field is a no-op for this segment.
+      DocValuesSkipper skipper = reader.getDocValuesSkipper(field);
+      if (skipper != null && skipper.minValue() == skipper.maxValue()) {
+        continue;
+      }
+      return sf;
+    }
+    return null;
+  }
 }