GITHUB#12370: early-terminate string sort when field is missing from the whole index

lucene/CHANGES.txt

@@ -160,6 +160,10 @@ Optimizations
   wildcards into a single any-string automaton, avoiding a quadratic blow-up in
   automaton size for patterns with many repeated asterisks. (Vismay Tiwari)
 
+* GITHUB#12370: Sorting on a string field that is missing from the whole index no longer scans every
+  document. When the sort has no tie breaker, all documents tie on the missing value, so once the
+  top-N queue is full the remaining documents can be skipped. (Serhiy Bzhezytskyy)
+
 * GITHUB#16307: ReqExclBulkScorer now skips runs of excluded docs via
   TwoPhaseIterator#docIDRunEnd for two-phase excluded clauses (e.g. a doc-values
   not-equals filter), instead of advancing one doc at a time. (Jim Ferenczi)

lucene/core/src/java/org/apache/lucene/search/comparators/TermOrdValComparator.java

@@ -434,6 +434,13 @@ private void updateCompetitiveIterator() throws IOException {
           }
         } else if (sortMissingLast || dense) {
           minOrd = 0;
+        } else if (bottomValue == null && singleSort) {
+          // The worst entry in the queue is itself a missing value and there is no tie breaker, so
+          // another missing value can no longer compete: missing values are no longer competitive.
+          // (bottomValue==null is stronger than bottomOrd==missingOrd, which can also be produced
+          // by
+          // a real value that maps before ord 0 in this segment.)
+          minOrd = 0;
         } else {
           // Missing values are still competitive.
           minOrd = -1;

lucene/core/src/test/org/apache/lucene/search/TestSortOptimization.java

@@ -1239,6 +1239,47 @@ private void testStringSortOptimizationFieldMissingInSegment(
     dir.close();
   }
 
+  /**
+   * GITHUB#12370: when the sort field is missing from the whole index (not just one segment) and
+   * the sort has no tie breaker, all documents tie on the missing value, so once the top-N queue is
+   * full the remaining documents are non-competitive and must be skipped rather than fully
+   * collected.
+   */
+  public void testStringSortOptimizationFieldMissingInWholeIndex() throws IOException {
+    final Directory dir = newDirectory();
+    final IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig());
+
+    // None of the documents index the sort field, so it is absent from every segment's FieldInfos.
+    final int numDocs = atLeast(10000);
+    for (int i = 0; i < numDocs; i++) {
+      final Document doc = new Document();
+      doc.add(new StringField("other", Integer.toString(i), Field.Store.NO));
+      writer.addDocument(doc);
+    }
+
+    final DirectoryReader reader = DirectoryReader.open(writer);
+    writer.close();
+
+    final int numHits = 5;
+
+    { // ascending, sort-missing-first (the default for SortedSetSortField): missing is the best
+      // value, but with no tie breaker a second missing value can never displace one already in the
+      // queue, so the tail should be skipped.
+      SortField sortField = new SortedSetSortField("field_does_not_exist", false);
+      TopDocs topDocs = assertSearchHits(reader, new Sort(sortField), numHits, null);
+      assertNonCompetitiveHitsAreSkipped(topDocs.totalHits.value(), numDocs);
+    }
+
+    { // descending, sort-missing-first
+      SortField sortField = new SortedSetSortField("field_does_not_exist", true);
+      TopDocs topDocs = assertSearchHits(reader, new Sort(sortField), numHits, null);
+      assertNonCompetitiveHitsAreSkipped(topDocs.totalHits.value(), numDocs);
+    }
+
+    reader.close();
+    dir.close();
+  }
+
   private void doTestStringSortOptimization(DirectoryReader reader) throws IOException {
     final int numDocs = reader.numDocs();
     final int numHits = 5;