Merge branch 'main' into lucene/case-folding-table

Author: costin

lucene/CHANGES.txt

@@ -126,6 +126,8 @@ New Features
 
 * GITHUB#15818: Add BM25 k3 query-term frequency saturation to BM25Similarity. (Sagar Upadhyaya)
 
+* GITHUB#16383: Add fp16 vector encoding support. (Pulkit Gupta)
+
 Improvements
 ---------------------
 * GITHUB#15704: Replace LinkedList with more efficient data structure. (Renato Haeberli)
@@ -360,6 +362,8 @@ Optimizations
 
 * GITHUG#16280: Single-pass writeString fast path for short strings in ByteBuffersDataOutput (neoremind)
 
+* GITHUB#16220: Implement WANDScorer.advanceShallow() to enable block-max optimizations when nested in conjunctions. (Prithvi S)
+
 * GITHUB#16146: Speed up no-score filtered-optional Boolean queries by routing dense conjunctions with cached FixedBitSet filters
   through ConstantScoreBulkScorer, which batches matches in bulk via DocIdStream. (Costin Leau)
 
@@ -392,6 +396,8 @@ Optimizations
 * GITHUB#16380: Speed up filtered disjunction queries when the filter is part of the
   index sort by advancing the filter before calculating impacts. (Alan Woodward)
 
+* GITHUB#16180: Add bulk intoBitSet for SortedDocValues ordinal ranges. (Prithvi S)
+
 Bug Fixes
 ---------------------
 * GITHUB#16350: Disable bulk-scoring in monitor queries. (Alan Woodward)

lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesProducer.java

@@ -1485,6 +1485,15 @@ public void intoBitSet(int upTo, FixedBitSet bitSet, int offset) throws IOExcept
               advance(upTo);
             }
           }
+
+          @Override
+          public void ordinalRangeIntoBitSet(
+              int fromDoc, int toDoc, long minOrd, long maxOrd, FixedBitSet bitSet, int offset)
+              throws IOException {
+            // Dense packed ordinals: bulk evaluate via the same SIMD path as NumericDocValues
+            Lucene90DocValuesProducer.rangeIntoBitSet(
+                values, fromDoc, toDoc, minOrd, maxOrd, bitSet, offset);
+          }
         };
       } else if (ordsEntry.docsWithFieldOffset >= 0) { // sparse but non-empty
         final IndexedDISI disi =
@@ -1583,6 +1592,15 @@ public int docIDRunEnd() throws IOException {
       public void intoBitSet(int upTo, FixedBitSet bitSet, int offset) throws IOException {
         ords.intoBitSet(upTo, bitSet, offset);
       }
+
+      @Override
+      public void ordinalRangeIntoBitSet(
+          int fromDoc, int toDoc, long minOrd, long maxOrd, FixedBitSet bitSet, int offset)
+          throws IOException {
+        // Delegate to the underlying NumericDocValues which already has a (possibly SIMD)
+        // rangeIntoBitSet override.
+        ords.rangeIntoBitSet(fromDoc, toDoc, minOrd, maxOrd, bitSet, offset);
+      }
     };
   }
 

lucene/core/src/java/org/apache/lucene/index/SortedDocValues.java

@@ -18,6 +18,7 @@
 
 import java.io.IOException;
 import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.FixedBitSet;
 import org.apache.lucene.util.automaton.CompiledAutomaton;
 
 /**
@@ -95,6 +96,32 @@ public TermsEnum termsEnum() throws IOException {
     return new SortedDocValuesTermsEnum(this);
   }
 
+  /**
+   * Fills {@code bitSet} with the doc IDs in {@code [fromDoc, toDoc)} whose ordinals are in {@code
+   * [minOrd, maxOrd]}. This is a bulk operation that avoids per-doc virtual dispatch overhead.
+   *
+   * <p>The default implementation falls back to per-doc evaluation via {@link #advanceExact} and
+   * {@link #ordValue}. Subclasses with random-access storage (e.g., dense fixed-bitsPerValue
+   * fields) can override this for significantly better performance.
+   *
+   * @param fromDoc first doc ID to evaluate (inclusive)
+   * @param toDoc last doc ID to evaluate (exclusive)
+   * @param minOrd lower bound of the ordinal range (inclusive)
+   * @param maxOrd upper bound of the ordinal range (inclusive)
+   * @param bitSet the bitset to fill
+   * @param offset subtracted from each doc ID before setting the bit
+   */
+  public void ordinalRangeIntoBitSet(
+      int fromDoc, int toDoc, long minOrd, long maxOrd, FixedBitSet bitSet, int offset)
+      throws IOException {
+    for (int doc = docID() >= fromDoc ? docID() : advance(fromDoc); doc < toDoc; doc = nextDoc()) {
+      long ord = ordValue();
+      if (ord >= minOrd && ord <= maxOrd) {
+        bitSet.set(doc - offset);
+      }
+    }
+  }
+
   /**
    * Returns a {@link TermsEnum} over the values, filtered by a {@link CompiledAutomaton} The enum
    * supports {@link TermsEnum#ord()}.

lucene/core/src/java/org/apache/lucene/search/DocValuesRangeIterator.java

@@ -99,8 +99,8 @@ public static DocValuesRangeIterator forOrdinalRange(
         };
     return skipper == null
         ? new DocValuesValueRangeIterator(values, check, 2)
-        : new BulkOrdinalRangeIterator(
-            values, new SkipBlockRangeIterator(skipper, min, max), check, 2);
+        : new BulkSortedRangeIterator(
+            values, new SkipBlockRangeIterator(skipper, min, max), check, 2, min, max);
   }
 
   /**
@@ -474,9 +474,43 @@ void intoMaybeBlock(int blockStart, int blockEnd, FixedBitSet bitSet, int offset
   }
 
   /**
-   * Bulk range iterator over ordinal (sorted / sorted-set) doc values. There is no columnar
-   * range-decode like the numeric path, so MAYBE blocks confirm the ordinal predicate one doc at a
-   * time, visiting only docs that have a value.
+   * Bulk range iterator over single-valued sorted (ordinal) doc values. Delegates MAYBE blocks to
+   * {@link SortedDocValues#ordinalRangeIntoBitSet} so that dense packed implementations can use
+   * direct (SIMD) range evaluation over the ordinal array.
+   */
+  private static final class BulkSortedRangeIterator extends BulkBlockRangeIterator {
+
+    private final SortedDocValues sortedValues;
+    private final long minOrd;
+    private final long maxOrd;
+
+    private BulkSortedRangeIterator(
+        SortedDocValues values,
+        SkipBlockRangeIterator blockIterator,
+        IOBooleanSupplier predicate,
+        float matchCost,
+        long minOrd,
+        long maxOrd) {
+      super(values, blockIterator, predicate, matchCost);
+      this.sortedValues = values;
+      this.minOrd = minOrd;
+      this.maxOrd = maxOrd;
+    }
+
+    @Override
+    void intoMaybeBlock(int blockStart, int blockEnd, FixedBitSet bitSet, int offset)
+        throws IOException {
+      // sortedValues is the same instance as disi, so a preceding matches() call may have
+      // moved it beyond blockStart. Adjust the starting point.
+      int from = Math.max(blockStart, sortedValues.docID());
+      sortedValues.ordinalRangeIntoBitSet(from, blockEnd, minOrd, maxOrd, bitSet, offset);
+    }
+  }
+
+  /**
+   * Bulk range iterator over ordinal (sorted-set) doc values. There is no columnar range-decode
+   * like the numeric path, and for multi-valued we must check the predicate (which walks the
+   * per-doc ords), so MAYBE blocks confirm by visiting only docs that have a value.
    */
   private static final class BulkOrdinalRangeIterator extends BulkBlockRangeIterator {
 

lucene/core/src/java/org/apache/lucene/search/WANDScorer.java

@@ -566,17 +566,18 @@ public float score() throws IOException {
 
   @Override
   public int advanceShallow(int target) throws IOException {
-    // Propagate to improve score bounds
+    // Propagate to sub-scorers. Scorers past target constrain the boundary to docID - 1.
+    int newUpTo = DocIdSetIterator.NO_MORE_DOCS;
     for (Scorer scorer : allScorers) {
-      if (scorer.docID() < target) {
-        scorer.advanceShallow(target);
+      // Use <= instead of < so we still propagate and use the block boundary when already at
+      // target.
+      if (scorer.docID() <= target) {
+        newUpTo = Math.min(newUpTo, scorer.advanceShallow(target));
+      } else if (scorer.docID() != DocIdSetIterator.NO_MORE_DOCS) {
+        newUpTo = Math.min(newUpTo, scorer.docID() - 1);
       }
     }
-    if (target <= upTo) {
-      return upTo;
-    }
-    // TODO: implement
-    return DocIdSetIterator.NO_MORE_DOCS;
+    return newUpTo;
   }
 
   @Override

lucene/core/src/test/org/apache/lucene/search/TestConstantScoreScorer.java

@@ -318,7 +318,7 @@ public void testNextDocsAndScoresRandomClusteredDeletions() throws IOException {
       // scattered deletions.
       FixedBitSet liveDocs = new FixedBitSet(maxDoc);
       liveDocs.set(0, maxDoc);
-      int runLength = TestUtil.nextInt(random(), 2 * 4096, 3 * 4096);
+      int runLength = TestUtil.nextInt(random(), 2 * 4096, Math.min(3 * 4096, maxDoc));
       int runStart = TestUtil.nextInt(random(), 0, maxDoc - runLength);
       liveDocs.clear(runStart, runStart + runLength);
       for (int i = 0, scattered = random().nextInt(100); i < scattered; ++i) {

lucene/core/src/test/org/apache/lucene/search/TestDocValuesOrdinalRangeIterator.java

@@ -134,7 +134,12 @@ public int getValueCount() {
 
       @Override
       public boolean advanceExact(int target) {
-        throw new UnsupportedOperationException();
+        if (docHasValue(target)) {
+          doc = target;
+          return true;
+        }
+        doc = target;
+        return false;
       }
 
       @Override
@@ -349,7 +354,12 @@ public long getValueCount() {
 
       @Override
       public boolean advanceExact(int target) {
-        throw new UnsupportedOperationException();
+        if (docHasValue(target)) {
+          doc = target;
+          return true;
+        }
+        doc = target;
+        return false;
       }
 
       @Override

lucene/core/src/test/org/apache/lucene/search/TestDocValuesOrdinalSetIterator.java

@@ -209,7 +209,12 @@ public int getValueCount() {
 
       @Override
       public boolean advanceExact(int target) {
-        throw new UnsupportedOperationException();
+        if (docHasValue(target)) {
+          doc = target;
+          return true;
+        }
+        doc = target;
+        return false;
       }
 
       @Override
@@ -500,7 +505,12 @@ public long getValueCount() {
 
       @Override
       public boolean advanceExact(int target) {
-        throw new UnsupportedOperationException();
+        if (docHasValue(target)) {
+          doc = target;
+          return true;
+        }
+        doc = target;
+        return false;
       }
 
       @Override