Improve allEqual suffix length detection in Lucene103BlockTreeTermsWriter.

lucene/CHANGES.txt

@@ -115,6 +115,8 @@ New Features
 
 Improvements
 ---------------------
+* GITHUB#16244: Improve allEqual suffix length detection in Lucene103BlockTreeTermsWriter. (Zhou Hui)
+
 * GITHUB#15704: Replace LinkedList with more efficient data structure. (Renato Haeberli)
 
 * GITHUB#15682: Use ArrayDeque instead of LinkedList in CompoundWordTokenFilterBase.java. (Renato Haeberli)

lucene/core/src/java/org/apache/lucene/codecs/lucene103/blocktree/Lucene103BlockTreeTermsWriter.java

@@ -20,7 +20,6 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
-import java.util.Objects;
 import org.apache.lucene.codecs.BlockTermState;
 import org.apache.lucene.codecs.CodecUtil;
 import org.apache.lucene.codecs.FieldsConsumer;
@@ -690,16 +689,6 @@ void writeBlocks(int prefixLength, int count) throws IOException {
       newBlocks.clear();
     }
 
-    private boolean allEqual(byte[] b, int startOffset, int endOffset, byte value) {
-      Objects.checkFromToIndex(startOffset, endOffset, b.length);
-      for (int i = startOffset; i < endOffset; ++i) {
-        if (b[i] != value) {
-          return false;
-        }
-      }
-      return true;
-    }
-
     /**
      * Writes the specified slice (start is inclusive, end is exclusive) from pending stack as a new
      * block. If isFloor is true, there were too many (more than maxItemsInBlock) entries sharing
@@ -762,6 +751,9 @@ private PendingBlock writeBlock(
 
       boolean absolute = true;
 
+      boolean allEqual = true;
+      int lastSuffixLength = -1;
+
       if (isLeafBlock) {
         // Block contains only ordinary terms:
         subIndices = null;
@@ -786,6 +778,16 @@ private PendingBlock writeBlock(
 
           // For leaf block we write suffixLength straight
           suffixLengthsWriter.writeVInt(suffixLength);
+
+          // Check if all suffixes in this block have the same length.
+          if (allEqual) {
+            if (i == start) {
+              lastSuffixLength = suffixLength;
+            } else if (lastSuffixLength != suffixLength) {
+              allEqual = false;
+            }
+          }
+
           suffixWriter.append(term.termBytes, prefixLength, suffixLength);
           assert floorLeadLabel == -1 || (term.termBytes[prefixLength] & 0xff) >= floorLeadLabel;
 
@@ -798,6 +800,12 @@ private PendingBlock writeBlock(
         }
         statsWriter.finish();
       } else {
+        // We need to write sub-block's start file pointer after writing a floor block term's
+        // length.
+        // TODO: If all suffixes have the same length, we can write the suffix length first, then
+        // write a term/sub-block fixBit, and write all sub-blocks' file pointers.
+        allEqual = false;
+
         // Block has at least one prefix term or a sub block:
         subIndices = new ArrayList<>();
         StatsWriter statsWriter =
@@ -936,7 +944,9 @@ private PendingBlock writeBlock(
       spareBytes = ArrayUtil.growNoCopy(spareBytes, numSuffixBytes);
       suffixLengthsWriter.copyTo(new ByteArrayDataOutput(spareBytes));
       suffixLengthsWriter.reset();
-      if (allEqual(spareBytes, 1, numSuffixBytes, spareBytes[0])) {
+      // TODO：Maybe release the constraint of Byte.MAX_VALUE and write this VInt length, then we
+      // need to fill suffixLengthBytes and set it in suffixLengthsReader(ByteArrayDataInput).
+      if (allEqual && lastSuffixLength <= Byte.MAX_VALUE) {
         // Structured fields like IDs often have most values of the same length
         termsOut.writeVInt((numSuffixBytes << 1) | 1);
         termsOut.writeByte(spareBytes[0]);

lucene/test-framework/src/java/org/apache/lucene/tests/index/BasePostingsFormatTestCase.java

@@ -464,6 +464,62 @@ public void testBinarySearchTermLeaf() throws Exception {
     dir.close();
   }
 
+  /**
+   * Ensures terms with long suffixes (>127 bytes) are readable via seekExact/seekCeil. This
+   * exercises multi-byte VInt encoding for suffix lengths in block tree term dictionaries.
+   */
+  public void testLongSuffixLengthVIntEncoding() throws Exception {
+    Directory dir = newDirectory();
+
+    IndexWriterConfig iwc = newIndexWriterConfig(null);
+    iwc.setCodec(getCodec());
+    iwc.setMergePolicy(newTieredMergePolicy());
+    IndexWriter iw = new IndexWriter(dir, iwc);
+
+    final int termLength = 130; // requires at least 2 bytes in VInt length encoding
+    final int termCount = 40;
+    for (int i = 0; i < termCount; i++) {
+      Document document = new Document();
+      document.add(new StringField("id", longIdTerm(i, termLength), Field.Store.NO));
+      iw.addDocument(document);
+    }
+    iw.commit();
+    iw.forceMerge(1);
+
+    DirectoryReader reader = DirectoryReader.open(iw);
+    TermsEnum termsEnum = getOnlyLeafReader(reader).terms("id").iterator();
+
+    for (int i = 0; i < termCount; i++) {
+      BytesRef target = new BytesRef(longIdTerm(i, termLength));
+      assertTrue(termsEnum.seekExact(target));
+      assertEquals(target, termsEnum.term());
+      assertEquals(SeekStatus.FOUND, termsEnum.seekCeil(target));
+      assertEquals(target, termsEnum.term());
+    }
+
+    BytesRef missing = new BytesRef(longIdTerm(termCount + 1000, termLength));
+    assertFalse(termsEnum.seekExact(missing));
+
+    reader.close();
+    iw.close();
+    dir.close();
+  }
+
+  private static String longIdTerm(int id, int totalLength) {
+    StringBuilder sb = new StringBuilder(totalLength);
+    sb.append('t');
+    if (id < 10) {
+      sb.append("00");
+    } else if (id < 100) {
+      sb.append('0');
+    }
+    sb.append(id);
+    while (sb.length() < totalLength) {
+      sb.append('x');
+    }
+    return sb.toString();
+  }
+
   // tests that level 2 ghost fields still work
   public void testLevel2Ghosts() throws Exception {
     Directory dir = newDirectory();