From 9e00d961c22ebe6b4b2480ca8658279c668e7a79 Mon Sep 17 00:00:00 2001 From: Zhou Hui Date: Thu, 11 Jun 2026 15:06:02 +0800 Subject: [PATCH 1/2] Improve all-equal suffix length detection in Lucene103BlockTreeTermsWriter --- .../Lucene103BlockTreeTermsWriter.java | 34 +++++++---- .../index/BasePostingsFormatTestCase.java | 56 +++++++++++++++++++ 2 files changed, 78 insertions(+), 12 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene103/blocktree/Lucene103BlockTreeTermsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene103/blocktree/Lucene103BlockTreeTermsWriter.java index 4e85a69b881f..84586cc34b60 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene103/blocktree/Lucene103BlockTreeTermsWriter.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene103/blocktree/Lucene103BlockTreeTermsWriter.java @@ -20,7 +20,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import java.util.Objects; import org.apache.lucene.codecs.BlockTermState; import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.FieldsConsumer; @@ -690,16 +689,6 @@ void writeBlocks(int prefixLength, int count) throws IOException { newBlocks.clear(); } - private boolean allEqual(byte[] b, int startOffset, int endOffset, byte value) { - Objects.checkFromToIndex(startOffset, endOffset, b.length); - for (int i = startOffset; i < endOffset; ++i) { - if (b[i] != value) { - return false; - } - } - return true; - } - /** * Writes the specified slice (start is inclusive, end is exclusive) from pending stack as a new * block. If isFloor is true, there were too many (more than maxItemsInBlock) entries sharing @@ -762,6 +751,9 @@ private PendingBlock writeBlock( boolean absolute = true; + boolean allEqual = true; + int lastSuffixLength = -1; + if (isLeafBlock) { // Block contains only ordinary terms: subIndices = null; @@ -786,6 +778,16 @@ private PendingBlock writeBlock( // For leaf block we write suffixLength straight suffixLengthsWriter.writeVInt(suffixLength); + + // Check if all suffixes in this block have the same length. + if (allEqual) { + if (i == start) { + lastSuffixLength = suffixLength; + } else if (lastSuffixLength != suffixLength) { + allEqual = false; + } + } + suffixWriter.append(term.termBytes, prefixLength, suffixLength); assert floorLeadLabel == -1 || (term.termBytes[prefixLength] & 0xff) >= floorLeadLabel; @@ -798,6 +800,12 @@ private PendingBlock writeBlock( } statsWriter.finish(); } else { + // We need to write sub-block's start file pointer after writing a floor block term's + // length. + // TODO: If all suffixes have the same length, we can write the suffix length first, then + // write a term/sub-block fixBit, and write all sub-blocks' file pointers. + allEqual = false; + // Block has at least one prefix term or a sub block: subIndices = new ArrayList<>(); StatsWriter statsWriter = @@ -936,7 +944,9 @@ private PendingBlock writeBlock( spareBytes = ArrayUtil.growNoCopy(spareBytes, numSuffixBytes); suffixLengthsWriter.copyTo(new ByteArrayDataOutput(spareBytes)); suffixLengthsWriter.reset(); - if (allEqual(spareBytes, 1, numSuffixBytes, spareBytes[0])) { + // TODO：Maybe release the constraint of Byte.MAX_VALUE and write this VInt length, then we + // need to fill suffixLengthBytes and set it in suffixLengthsReader(ByteArrayDataInput). + if (allEqual && lastSuffixLength <= Byte.MAX_VALUE) { // Structured fields like IDs often have most values of the same length termsOut.writeVInt((numSuffixBytes << 1) | 1); termsOut.writeByte(spareBytes[0]); diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BasePostingsFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BasePostingsFormatTestCase.java index 385a52b193e8..f0cac0cfc8a6 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BasePostingsFormatTestCase.java +++ b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BasePostingsFormatTestCase.java @@ -464,6 +464,62 @@ public void testBinarySearchTermLeaf() throws Exception { dir.close(); } + /** + * Ensures terms with long suffixes (>127 bytes) are readable via seekExact/seekCeil. This + * exercises multi-byte VInt encoding for suffix lengths in block tree term dictionaries. + */ + public void testLongSuffixLengthVIntEncoding() throws Exception { + Directory dir = newDirectory(); + + IndexWriterConfig iwc = newIndexWriterConfig(null); + iwc.setCodec(getCodec()); + iwc.setMergePolicy(newTieredMergePolicy()); + IndexWriter iw = new IndexWriter(dir, iwc); + + final int termLength = 130; // requires at least 2 bytes in VInt length encoding + final int termCount = 40; + for (int i = 0; i < termCount; i++) { + Document document = new Document(); + document.add(new StringField("id", longIdTerm(i, termLength), Field.Store.NO)); + iw.addDocument(document); + } + iw.commit(); + iw.forceMerge(1); + + DirectoryReader reader = DirectoryReader.open(iw); + TermsEnum termsEnum = getOnlyLeafReader(reader).terms("id").iterator(); + + for (int i = 0; i < termCount; i++) { + BytesRef target = new BytesRef(longIdTerm(i, termLength)); + assertTrue(termsEnum.seekExact(target)); + assertEquals(target, termsEnum.term()); + assertEquals(SeekStatus.FOUND, termsEnum.seekCeil(target)); + assertEquals(target, termsEnum.term()); + } + + BytesRef missing = new BytesRef(longIdTerm(termCount + 1000, termLength)); + assertFalse(termsEnum.seekExact(missing)); + + reader.close(); + iw.close(); + dir.close(); + } + + private static String longIdTerm(int id, int totalLength) { + StringBuilder sb = new StringBuilder(totalLength); + sb.append('t'); + if (id < 10) { + sb.append("00"); + } else if (id < 100) { + sb.append('0'); + } + sb.append(id); + while (sb.length() < totalLength) { + sb.append('x'); + } + return sb.toString(); + } + // tests that level 2 ghost fields still work public void testLevel2Ghosts() throws Exception { Directory dir = newDirectory(); From 1a0653d0c2b3ddadb570e02452d1eb55dd039119 Mon Sep 17 00:00:00 2001 From: Zhou Hui Date: Thu, 11 Jun 2026 15:23:12 +0800 Subject: [PATCH 2/2] Add changes entry. --- lucene/CHANGES.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index fd9092a1a7de..6beb1ea98fb2 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -115,6 +115,8 @@ New Features Improvements --------------------- +* GITHUB#16244: Improve allEqual suffix length detection in Lucene103BlockTreeTermsWriter. (Zhou Hui) + * GITHUB#15704: Replace LinkedList with more efficient data structure. (Renato Haeberli) * GITHUB#15682: Use ArrayDeque instead of LinkedList in CompoundWordTokenFilterBase.java. (Renato Haeberli)