From 67150f19888c5ab0f1f6ea14efd5d28840798da5 Mon Sep 17 00:00:00 2001 From: dungba88 Date: Tue, 26 Dec 2023 15:32:14 +0900 Subject: [PATCH 01/12] Make FSTPostingFormat to build FST off-heap --- .../lucene/codecs/memory/FSTTermsReader.java | 55 +++++++++------ .../lucene/codecs/memory/FSTTermsWriter.java | 67 ++++++++++++------- 2 files changed, 80 insertions(+), 42 deletions(-) diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java index e63c85a30509..26d5e113ddda 100644 --- a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java +++ b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java @@ -65,29 +65,28 @@ public class FSTTermsReader extends FieldsProducer { private final TreeMap fields = new TreeMap<>(); private final PostingsReaderBase postingsReader; + private final IndexInput fstMetaInput; private final IndexInput fstTermsInput; public FSTTermsReader(SegmentReadState state, PostingsReaderBase postingsReader) throws IOException { - final String termsFileName = + final String termsMetaFileName = IndexFileNames.segmentFileName( - state.segmentInfo.name, state.segmentSuffix, FSTTermsWriter.TERMS_EXTENSION); + state.segmentInfo.name, state.segmentSuffix, FSTTermsWriter.TERMS_META_EXTENSION); + final String termsDataFileName = + IndexFileNames.segmentFileName( + state.segmentInfo.name, state.segmentSuffix, FSTTermsWriter.TERMS_DATA_EXTENSION); this.postingsReader = postingsReader; - this.fstTermsInput = state.directory.openInput(termsFileName, IOContext.LOAD); + this.fstMetaInput = state.directory.openInput(termsMetaFileName, IOContext.LOAD); + this.fstTermsInput = state.directory.openInput(termsDataFileName, IOContext.LOAD); - IndexInput in = this.fstTermsInput; + IndexInput in = this.fstMetaInput; boolean success = false; try { - CodecUtil.checkIndexHeader( - in, - FSTTermsWriter.TERMS_CODEC_NAME, - FSTTermsWriter.TERMS_VERSION_START, - FSTTermsWriter.TERMS_VERSION_CURRENT, - state.segmentInfo.getId(), - state.segmentSuffix); - CodecUtil.checksumEntireFile(in); + verifyInput(state, in); + verifyInput(state, fstTermsInput); this.postingsReader.init(in, state); seekDir(in); @@ -102,19 +101,32 @@ public FSTTermsReader(SegmentReadState state, PostingsReaderBase postingsReader) long sumDocFreq = fieldInfo.getIndexOptions() == IndexOptions.DOCS ? sumTotalTermFreq : in.readVLong(); int docCount = in.readVInt(); + long startFP = in.readVLong(); + fstTermsInput.seek(startFP); TermsReader current = - new TermsReader(fieldInfo, in, numTerms, sumTotalTermFreq, sumDocFreq, docCount); + new TermsReader(fieldInfo, in, fstTermsInput, numTerms, sumTotalTermFreq, sumDocFreq, docCount); TermsReader previous = fields.put(fieldInfo.name, current); checkFieldSummary(state.segmentInfo, in, current, previous); } success = true; } finally { if (success == false) { - IOUtils.closeWhileHandlingException(in); + IOUtils.closeWhileHandlingException(in, fstTermsInput); } } } + private static void verifyInput(SegmentReadState state, IndexInput in) throws IOException { + CodecUtil.checkIndexHeader( + in, + FSTTermsWriter.TERMS_CODEC_NAME, + FSTTermsWriter.TERMS_VERSION_START, + FSTTermsWriter.TERMS_VERSION_CURRENT, + state.segmentInfo.getId(), + state.segmentSuffix); + CodecUtil.checksumEntireFile(in); + } + private void seekDir(IndexInput in) throws IOException { in.seek(in.length() - CodecUtil.footerLength() - 8); in.seek(in.readLong()); @@ -165,7 +177,7 @@ public int size() { @Override public void close() throws IOException { try { - IOUtils.close(postingsReader, fstTermsInput); + IOUtils.close(postingsReader, fstMetaInput, fstTermsInput); } finally { fields.clear(); } @@ -182,7 +194,8 @@ final class TermsReader extends Terms { TermsReader( FieldInfo fieldInfo, - IndexInput in, + IndexInput metaIn, + IndexInput dataIn, long numTerms, long sumTotalTermFreq, long sumDocFreq, @@ -195,8 +208,8 @@ final class TermsReader extends Terms { this.docCount = docCount; OffHeapFSTStore offHeapFSTStore = new OffHeapFSTStore(); FSTTermOutputs outputs = new FSTTermOutputs(fieldInfo); - this.dict = new FST<>(FST.readMetadata(in, outputs), in, offHeapFSTStore); - in.skipBytes(offHeapFSTStore.size()); + this.dict = new FST<>(FST.readMetadata(metaIn, outputs), dataIn, offHeapFSTStore); + dataIn.skipBytes(offHeapFSTStore.size()); } @Override @@ -508,7 +521,11 @@ void decodeMetaData() throws IOException { if (meta.bytes != null) { bytesReader.reset(meta.bytes, 0, meta.bytes.length); } - postingsReader.decodeTerm(bytesReader, fieldInfo, state, true); + try { + postingsReader.decodeTerm(bytesReader, fieldInfo, state, true); + } catch (Exception ex) { + System.out.println("bingo"); + } decoded = true; } } diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsWriter.java b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsWriter.java index fa46f6451dad..49d5c4e4e4ac 100644 --- a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsWriter.java +++ b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsWriter.java @@ -107,42 +107,55 @@ * @lucene.experimental */ public class FSTTermsWriter extends FieldsConsumer { - static final String TERMS_EXTENSION = "tfp"; + static final String TERMS_META_EXTENSION = "tfp.meta"; + static final String TERMS_DATA_EXTENSION = "tfp.data"; static final String TERMS_CODEC_NAME = "FSTTerms"; public static final int TERMS_VERSION_START = 2; public static final int TERMS_VERSION_CURRENT = TERMS_VERSION_START; final PostingsWriterBase postingsWriter; final FieldInfos fieldInfos; - IndexOutput out; + IndexOutput dataOut; + IndexOutput metaOut; final int maxDoc; final List fields = new ArrayList<>(); public FSTTermsWriter(SegmentWriteState state, PostingsWriterBase postingsWriter) throws IOException { - final String termsFileName = + final String termsMetaFileName = IndexFileNames.segmentFileName( - state.segmentInfo.name, state.segmentSuffix, TERMS_EXTENSION); + state.segmentInfo.name, state.segmentSuffix, TERMS_META_EXTENSION); + final String termsDataFileName = + IndexFileNames.segmentFileName( + state.segmentInfo.name, state.segmentSuffix, TERMS_DATA_EXTENSION); this.postingsWriter = postingsWriter; this.fieldInfos = state.fieldInfos; - this.out = state.directory.createOutput(termsFileName, state.context); + this.metaOut = state.directory.createOutput(termsMetaFileName, state.context); + this.dataOut = state.directory.createOutput(termsDataFileName, state.context); this.maxDoc = state.segmentInfo.maxDoc(); boolean success = false; try { CodecUtil.writeIndexHeader( - out, + metaOut, + TERMS_CODEC_NAME, + TERMS_VERSION_CURRENT, + state.segmentInfo.getId(), + state.segmentSuffix); + + CodecUtil.writeIndexHeader( + dataOut, TERMS_CODEC_NAME, TERMS_VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix); - this.postingsWriter.init(out, state); + this.postingsWriter.init(metaOut, state); success = true; } finally { if (!success) { - IOUtils.closeWhileHandlingException(out); + IOUtils.closeWhileHandlingException(metaOut, dataOut); } } } @@ -187,33 +200,38 @@ public void write(Fields fields, NormsProducer norms) throws IOException { @Override public void close() throws IOException { - if (out != null) { + if (metaOut != null) { + assert dataOut != null; boolean success = false; try { // write field summary - final long dirStart = out.getFilePointer(); + final long dirStart = metaOut.getFilePointer(); - out.writeVInt(fields.size()); + metaOut.writeVInt(fields.size()); for (FieldMetaData field : fields) { - out.writeVInt(field.fieldInfo.number); - out.writeVLong(field.numTerms); + metaOut.writeVInt(field.fieldInfo.number); + metaOut.writeVLong(field.numTerms); if (field.fieldInfo.getIndexOptions() != IndexOptions.DOCS) { - out.writeVLong(field.sumTotalTermFreq); + metaOut.writeVLong(field.sumTotalTermFreq); } - out.writeVLong(field.sumDocFreq); - out.writeVInt(field.docCount); - field.dict.save(out, out); + metaOut.writeVLong(field.sumDocFreq); + metaOut.writeVInt(field.docCount); + // write the starting file pointer + metaOut.writeVLong(dataOut.getFilePointer() - field.dict.numBytes()); + field.dict.saveMetadata(metaOut); } - writeTrailer(out, dirStart); - CodecUtil.writeFooter(out); + writeTrailer(metaOut, dirStart); + CodecUtil.writeFooter(metaOut); + CodecUtil.writeFooter(dataOut); success = true; } finally { if (success) { - IOUtils.close(out, postingsWriter); + IOUtils.close(metaOut, dataOut, postingsWriter); } else { - IOUtils.closeWhileHandlingException(out, postingsWriter); + IOUtils.closeWhileHandlingException(metaOut, dataOut, postingsWriter); } - out = null; + metaOut = null; + dataOut = null; } } } @@ -256,7 +274,9 @@ final class TermsWriter { this.fieldInfo = fieldInfo; postingsWriter.setField(fieldInfo); this.outputs = new FSTTermOutputs(fieldInfo); - this.fstCompiler = new FSTCompiler.Builder<>(FST.INPUT_TYPE.BYTE1, outputs).build(); + this.fstCompiler = new FSTCompiler.Builder<>(FST.INPUT_TYPE.BYTE1, outputs) + .dataOutput(dataOut) + .build(); } public void finishTerm(BytesRef text, BlockTermState state) throws IOException { @@ -278,6 +298,7 @@ public void finish(long sumTotalTermFreq, long sumDocFreq, int docCount) throws // save FST dict if (numTerms > 0) { final FST fst = fstCompiler.compile(); + fst.saveMetadata(metaOut); fields.add( new FieldMetaData(fieldInfo, numTerms, sumTotalTermFreq, sumDocFreq, docCount, fst)); } From 65480754c8d383ada5fe2a3850c7665d3afeb5fd Mon Sep 17 00:00:00 2001 From: dungba88 Date: Tue, 26 Dec 2023 15:43:35 +0900 Subject: [PATCH 02/12] remove sysout --- .../apache/lucene/codecs/memory/FSTTermsReader.java | 12 ++++-------- .../apache/lucene/codecs/memory/FSTTermsWriter.java | 8 ++------ 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java index 26d5e113ddda..f705b78c4236 100644 --- a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java +++ b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java @@ -87,6 +87,7 @@ public FSTTermsReader(SegmentReadState state, PostingsReaderBase postingsReader) try { verifyInput(state, in); verifyInput(state, fstTermsInput); + this.postingsReader.init(in, state); seekDir(in); @@ -101,10 +102,9 @@ public FSTTermsReader(SegmentReadState state, PostingsReaderBase postingsReader) long sumDocFreq = fieldInfo.getIndexOptions() == IndexOptions.DOCS ? sumTotalTermFreq : in.readVLong(); int docCount = in.readVInt(); - long startFP = in.readVLong(); - fstTermsInput.seek(startFP); TermsReader current = - new TermsReader(fieldInfo, in, fstTermsInput, numTerms, sumTotalTermFreq, sumDocFreq, docCount); + new TermsReader( + fieldInfo, in, fstTermsInput, numTerms, sumTotalTermFreq, sumDocFreq, docCount); TermsReader previous = fields.put(fieldInfo.name, current); checkFieldSummary(state.segmentInfo, in, current, previous); } @@ -521,11 +521,7 @@ void decodeMetaData() throws IOException { if (meta.bytes != null) { bytesReader.reset(meta.bytes, 0, meta.bytes.length); } - try { - postingsReader.decodeTerm(bytesReader, fieldInfo, state, true); - } catch (Exception ex) { - System.out.println("bingo"); - } + postingsReader.decodeTerm(bytesReader, fieldInfo, state, true); decoded = true; } } diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsWriter.java b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsWriter.java index 49d5c4e4e4ac..b81741dbd3a5 100644 --- a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsWriter.java +++ b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsWriter.java @@ -216,8 +216,6 @@ public void close() throws IOException { } metaOut.writeVLong(field.sumDocFreq); metaOut.writeVInt(field.docCount); - // write the starting file pointer - metaOut.writeVLong(dataOut.getFilePointer() - field.dict.numBytes()); field.dict.saveMetadata(metaOut); } writeTrailer(metaOut, dirStart); @@ -274,9 +272,8 @@ final class TermsWriter { this.fieldInfo = fieldInfo; postingsWriter.setField(fieldInfo); this.outputs = new FSTTermOutputs(fieldInfo); - this.fstCompiler = new FSTCompiler.Builder<>(FST.INPUT_TYPE.BYTE1, outputs) - .dataOutput(dataOut) - .build(); + this.fstCompiler = + new FSTCompiler.Builder<>(FST.INPUT_TYPE.BYTE1, outputs).dataOutput(dataOut).build(); } public void finishTerm(BytesRef text, BlockTermState state) throws IOException { @@ -298,7 +295,6 @@ public void finish(long sumTotalTermFreq, long sumDocFreq, int docCount) throws // save FST dict if (numTerms > 0) { final FST fst = fstCompiler.compile(); - fst.saveMetadata(metaOut); fields.add( new FieldMetaData(fieldInfo, numTerms, sumTotalTermFreq, sumDocFreq, docCount, fst)); } From 83e3fac2bec1ef318d372052fb48d2d803ce437b Mon Sep 17 00:00:00 2001 From: dungba88 Date: Tue, 26 Dec 2023 18:11:44 +0900 Subject: [PATCH 03/12] lazily write the FST padding byte --- .../java/org/apache/lucene/util/fst/FSTCompiler.java | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/util/fst/FSTCompiler.java b/lucene/core/src/java/org/apache/lucene/util/fst/FSTCompiler.java index 961511616abb..2a04a575ef7c 100644 --- a/lucene/core/src/java/org/apache/lucene/util/fst/FSTCompiler.java +++ b/lucene/core/src/java/org/apache/lucene/util/fst/FSTCompiler.java @@ -164,14 +164,12 @@ private FSTCompiler( boolean allowFixedLengthArcs, DataOutput dataOutput, float directAddressingMaxOversizingFactor, - int version) - throws IOException { + int version) { this.allowFixedLengthArcs = allowFixedLengthArcs; this.directAddressingMaxOversizingFactor = directAddressingMaxOversizingFactor; this.version = version; // pad: ensure no node gets address 0 which is reserved to mean - // the stop state w/ no arcs - dataOutput.writeByte((byte) 0); + // the stop state w/ no arcs. the actual byte will be written lazily numBytesWritten++; this.dataOutput = dataOutput; fst = @@ -344,7 +342,7 @@ public Builder setVersion(int version) { } /** Creates a new {@link FSTCompiler}. */ - public FSTCompiler build() throws IOException { + public FSTCompiler build() { // create a default DataOutput if not specified if (dataOutput == null) { dataOutput = getOnHeapReaderWriter(15); @@ -552,6 +550,10 @@ long addNode(FSTCompiler.UnCompiledNode nodeIn) throws IOException { } reverseScratchBytes(); + if (numBytesWritten == 1) { + // first time, write the padding byte + dataOutput.writeByte((byte) 0); + } scratchBytes.writeTo(dataOutput); numBytesWritten += scratchBytes.getPosition(); From 2d8f270ee01fbe74890d7a6c4ab98baaa6ec88f2 Mon Sep 17 00:00:00 2001 From: dungba88 Date: Tue, 26 Dec 2023 18:11:44 +0900 Subject: [PATCH 04/12] lazily write the FST padding byte --- .../java/org/apache/lucene/util/fst/FSTCompiler.java | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/util/fst/FSTCompiler.java b/lucene/core/src/java/org/apache/lucene/util/fst/FSTCompiler.java index 961511616abb..2a04a575ef7c 100644 --- a/lucene/core/src/java/org/apache/lucene/util/fst/FSTCompiler.java +++ b/lucene/core/src/java/org/apache/lucene/util/fst/FSTCompiler.java @@ -164,14 +164,12 @@ private FSTCompiler( boolean allowFixedLengthArcs, DataOutput dataOutput, float directAddressingMaxOversizingFactor, - int version) - throws IOException { + int version) { this.allowFixedLengthArcs = allowFixedLengthArcs; this.directAddressingMaxOversizingFactor = directAddressingMaxOversizingFactor; this.version = version; // pad: ensure no node gets address 0 which is reserved to mean - // the stop state w/ no arcs - dataOutput.writeByte((byte) 0); + // the stop state w/ no arcs. the actual byte will be written lazily numBytesWritten++; this.dataOutput = dataOutput; fst = @@ -344,7 +342,7 @@ public Builder setVersion(int version) { } /** Creates a new {@link FSTCompiler}. */ - public FSTCompiler build() throws IOException { + public FSTCompiler build() { // create a default DataOutput if not specified if (dataOutput == null) { dataOutput = getOnHeapReaderWriter(15); @@ -552,6 +550,10 @@ long addNode(FSTCompiler.UnCompiledNode nodeIn) throws IOException { } reverseScratchBytes(); + if (numBytesWritten == 1) { + // first time, write the padding byte + dataOutput.writeByte((byte) 0); + } scratchBytes.writeTo(dataOutput); numBytesWritten += scratchBytes.getPosition(); From b4a349404810ff5ac15aa6131f0d2cb220f73598 Mon Sep 17 00:00:00 2001 From: dungba88 Date: Tue, 26 Dec 2023 18:25:21 +0900 Subject: [PATCH 05/12] Also write the pad byte when there is emptyOutput --- .../java/org/apache/lucene/util/fst/FSTCompiler.java | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/util/fst/FSTCompiler.java b/lucene/core/src/java/org/apache/lucene/util/fst/FSTCompiler.java index 2a04a575ef7c..16581d4096b6 100644 --- a/lucene/core/src/java/org/apache/lucene/util/fst/FSTCompiler.java +++ b/lucene/core/src/java/org/apache/lucene/util/fst/FSTCompiler.java @@ -551,8 +551,7 @@ long addNode(FSTCompiler.UnCompiledNode nodeIn) throws IOException { reverseScratchBytes(); if (numBytesWritten == 1) { - // first time, write the padding byte - dataOutput.writeByte((byte) 0); + writePaddingByte(); } scratchBytes.writeTo(dataOutput); numBytesWritten += scratchBytes.getPosition(); @@ -561,6 +560,10 @@ long addNode(FSTCompiler.UnCompiledNode nodeIn) throws IOException { return numBytesWritten - 1; } + private void writePaddingByte() throws IOException { + dataOutput.writeByte((byte) 0); + } + private void writeLabel(DataOutput out, int v) throws IOException { assert v >= 0 : "v=" + v; if (fst.metadata.inputType == INPUT_TYPE.BYTE1) { @@ -970,6 +973,10 @@ public FST compile() throws IOException { if (root.numArcs == 0) { if (fst.metadata.emptyOutput == null) { return null; + } else { + // we haven't written the pad byte so far, but the FST is still valid + assert numBytesWritten == 1; + writePaddingByte(); } } From 601a6e17ad2328f120d0bb5ec7d45ddf7ca4f1c0 Mon Sep 17 00:00:00 2001 From: dungba88 Date: Tue, 26 Dec 2023 18:27:32 +0900 Subject: [PATCH 06/12] add comment --- .../core/src/java/org/apache/lucene/util/fst/FSTCompiler.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lucene/core/src/java/org/apache/lucene/util/fst/FSTCompiler.java b/lucene/core/src/java/org/apache/lucene/util/fst/FSTCompiler.java index 16581d4096b6..7f67fb2a38bc 100644 --- a/lucene/core/src/java/org/apache/lucene/util/fst/FSTCompiler.java +++ b/lucene/core/src/java/org/apache/lucene/util/fst/FSTCompiler.java @@ -550,6 +550,7 @@ long addNode(FSTCompiler.UnCompiledNode nodeIn) throws IOException { } reverseScratchBytes(); + // write the padding byte if needed if (numBytesWritten == 1) { writePaddingByte(); } @@ -561,6 +562,7 @@ long addNode(FSTCompiler.UnCompiledNode nodeIn) throws IOException { } private void writePaddingByte() throws IOException { + assert numBytesWritten == 1; dataOutput.writeByte((byte) 0); } @@ -975,7 +977,6 @@ public FST compile() throws IOException { return null; } else { // we haven't written the pad byte so far, but the FST is still valid - assert numBytesWritten == 1; writePaddingByte(); } } From ce3837df06c191a4c0a869d29702e2158c2b270d Mon Sep 17 00:00:00 2001 From: dungba88 Date: Wed, 27 Dec 2023 12:42:37 +0900 Subject: [PATCH 07/12] Move openInput and createOutput to try-catch --- .../lucene/codecs/memory/FSTTermsReader.java | 41 +++++++++++-------- .../lucene/codecs/memory/FSTTermsWriter.java | 8 +++- 2 files changed, 29 insertions(+), 20 deletions(-) diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java index f705b78c4236..a5b26b77b6a6 100644 --- a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java +++ b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java @@ -66,7 +66,7 @@ public class FSTTermsReader extends FieldsProducer { private final TreeMap fields = new TreeMap<>(); private final PostingsReaderBase postingsReader; private final IndexInput fstMetaInput; - private final IndexInput fstTermsInput; + private final IndexInput fstDataInput; public FSTTermsReader(SegmentReadState state, PostingsReaderBase postingsReader) throws IOException { @@ -78,40 +78,45 @@ public FSTTermsReader(SegmentReadState state, PostingsReaderBase postingsReader) state.segmentInfo.name, state.segmentSuffix, FSTTermsWriter.TERMS_DATA_EXTENSION); this.postingsReader = postingsReader; - this.fstMetaInput = state.directory.openInput(termsMetaFileName, IOContext.LOAD); - this.fstTermsInput = state.directory.openInput(termsDataFileName, IOContext.LOAD); - IndexInput in = this.fstMetaInput; + IndexInput metaIn = null, dataIn = null; boolean success = false; try { - verifyInput(state, in); - verifyInput(state, fstTermsInput); + metaIn = state.directory.openInput(termsMetaFileName, IOContext.LOAD); + dataIn = state.directory.openInput(termsDataFileName, IOContext.LOAD); - this.postingsReader.init(in, state); - seekDir(in); + verifyInput(state, metaIn); + verifyInput(state, dataIn); + + this.postingsReader.init(metaIn, state); + seekDir(metaIn); final FieldInfos fieldInfos = state.fieldInfos; - final int numFields = in.readVInt(); + final int numFields = metaIn.readVInt(); for (int i = 0; i < numFields; i++) { - int fieldNumber = in.readVInt(); + int fieldNumber = metaIn.readVInt(); FieldInfo fieldInfo = fieldInfos.fieldInfo(fieldNumber); - long numTerms = in.readVLong(); - long sumTotalTermFreq = in.readVLong(); + long numTerms = metaIn.readVLong(); + long sumTotalTermFreq = metaIn.readVLong(); // if frequencies are omitted, sumTotalTermFreq=sumDocFreq and we only write one value long sumDocFreq = - fieldInfo.getIndexOptions() == IndexOptions.DOCS ? sumTotalTermFreq : in.readVLong(); - int docCount = in.readVInt(); + fieldInfo.getIndexOptions() == IndexOptions.DOCS + ? sumTotalTermFreq + : metaIn.readVLong(); + int docCount = metaIn.readVInt(); TermsReader current = new TermsReader( - fieldInfo, in, fstTermsInput, numTerms, sumTotalTermFreq, sumDocFreq, docCount); + fieldInfo, metaIn, dataIn, numTerms, sumTotalTermFreq, sumDocFreq, docCount); TermsReader previous = fields.put(fieldInfo.name, current); - checkFieldSummary(state.segmentInfo, in, current, previous); + checkFieldSummary(state.segmentInfo, metaIn, current, previous); } + this.fstMetaInput = metaIn; + this.fstDataInput = dataIn; success = true; } finally { if (success == false) { - IOUtils.closeWhileHandlingException(in, fstTermsInput); + IOUtils.closeWhileHandlingException(metaIn, dataIn); } } } @@ -177,7 +182,7 @@ public int size() { @Override public void close() throws IOException { try { - IOUtils.close(postingsReader, fstMetaInput, fstTermsInput); + IOUtils.close(postingsReader, fstMetaInput, fstDataInput); } finally { fields.clear(); } diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsWriter.java b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsWriter.java index b81741dbd3a5..e86ec33c323f 100644 --- a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsWriter.java +++ b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsWriter.java @@ -131,12 +131,14 @@ public FSTTermsWriter(SegmentWriteState state, PostingsWriterBase postingsWriter this.postingsWriter = postingsWriter; this.fieldInfos = state.fieldInfos; - this.metaOut = state.directory.createOutput(termsMetaFileName, state.context); - this.dataOut = state.directory.createOutput(termsDataFileName, state.context); this.maxDoc = state.segmentInfo.maxDoc(); + IndexOutput metaOut = null, dataOut = null; boolean success = false; try { + metaOut = state.directory.createOutput(termsMetaFileName, state.context); + dataOut = state.directory.createOutput(termsDataFileName, state.context); + CodecUtil.writeIndexHeader( metaOut, TERMS_CODEC_NAME, @@ -152,6 +154,8 @@ public FSTTermsWriter(SegmentWriteState state, PostingsWriterBase postingsWriter state.segmentSuffix); this.postingsWriter.init(metaOut, state); + this.metaOut = metaOut; + this.dataOut = dataOut; success = true; } finally { if (!success) { From cc249e17b1e6b2ebd38b0dff48d97b09046605a6 Mon Sep 17 00:00:00 2001 From: dungba88 Date: Thu, 28 Dec 2023 16:58:20 +0900 Subject: [PATCH 08/12] Add change logs --- lucene/CHANGES.txt | 3 +++ .../org/apache/lucene/codecs/memory/FSTPostingsFormat.java | 4 ++-- .../java/org/apache/lucene/codecs/memory/FSTTermsReader.java | 2 ++ .../java/org/apache/lucene/codecs/memory/FSTTermsWriter.java | 4 +++- 4 files changed, 10 insertions(+), 3 deletions(-) diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index a3dc9d595f1f..8266f400b8cb 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -77,6 +77,9 @@ API Changes * GITHUB#12855: Remove deprecated DrillSideways#createDrillDownFacetsCollector extension method. (Greg Miller) +* GITHUB#12980: Make FSTPostingsFormat to build FST off-heap. This PostingsFormat will now + create 2 FST files (tfp.meta and tfp.data) instead of a single one. (Anh Dung Bui) + New Features --------------------- diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTPostingsFormat.java b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTPostingsFormat.java index 0a18b75ac003..6fbbafd9ad8b 100644 --- a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTPostingsFormat.java +++ b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTPostingsFormat.java @@ -28,10 +28,10 @@ import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.util.IOUtils; -/** FST term dict + Lucene50PBF */ +/** FST term dict + Lucene99PBF */ public final class FSTPostingsFormat extends PostingsFormat { public FSTPostingsFormat() { - super("FST50"); + super("FST99"); } @Override diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java index a5b26b77b6a6..22891d4b9f8d 100644 --- a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java +++ b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java @@ -65,7 +65,9 @@ public class FSTTermsReader extends FieldsProducer { private final TreeMap fields = new TreeMap<>(); private final PostingsReaderBase postingsReader; + // IndexInput for FST metadata private final IndexInput fstMetaInput; + // IndexInput for FST data private final IndexInput fstDataInput; public FSTTermsReader(SegmentReadState state, PostingsReaderBase postingsReader) diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsWriter.java b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsWriter.java index e86ec33c323f..3addb4875a53 100644 --- a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsWriter.java +++ b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsWriter.java @@ -115,8 +115,10 @@ public class FSTTermsWriter extends FieldsConsumer { final PostingsWriterBase postingsWriter; final FieldInfos fieldInfos; - IndexOutput dataOut; + // IndexOutput for FST metadata IndexOutput metaOut; + // IndexOutput for FST data + IndexOutput dataOut; final int maxDoc; final List fields = new ArrayList<>(); From 58900851aa3bb6b543366542ea6a461c945f794b Mon Sep 17 00:00:00 2001 From: Anh Dung Bui Date: Sat, 13 Sep 2025 15:37:45 -0500 Subject: [PATCH 09/12] rebase --- .../lucene/codecs/memory/FSTTermsReader.java | 23 +++++++++---------- .../lucene/codecs/memory/FSTTermsWriter.java | 23 ++++--------------- 2 files changed, 16 insertions(+), 30 deletions(-) diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java index 90eba13c0bef..c9eb4605a3fc 100644 --- a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java +++ b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java @@ -81,15 +81,16 @@ public FSTTermsReader(SegmentReadState state, PostingsReaderBase postingsReader) state.segmentInfo.name, state.segmentSuffix, FSTTermsWriter.TERMS_DATA_EXTENSION); this.postingsReader = postingsReader; - this.fstTermsInput = - state.directory.openInput( - termsFileName, state.context.withHints(FileTypeHint.DATA, PreloadHint.INSTANCE)); IndexInput metaIn = null, dataIn = null; try { - metaIn = state.directory.openInput(termsMetaFileName, IOContext.LOAD); - dataIn = state.directory.openInput(termsDataFileName, IOContext.LOAD); + metaIn = + state.directory.openInput( + termsMetaFileName, state.context.withHints(FileTypeHint.DATA, PreloadHint.INSTANCE)); + dataIn = + state.directory.openInput( + termsDataFileName, state.context.withHints(FileTypeHint.DATA, PreloadHint.INSTANCE)); verifyInput(state, metaIn); verifyInput(state, dataIn); @@ -118,14 +119,9 @@ public FSTTermsReader(SegmentReadState state, PostingsReaderBase postingsReader) } this.fstMetaInput = metaIn; this.fstDataInput = dataIn; - success = true; } catch (Throwable t) { - IOUtils.closeWhileSuppressingExceptions(t, in); + IOUtils.closeWhileSuppressingExceptions(t, metaIn, dataIn); throw t; - } finally { - if (success == false) { - IOUtils.closeWhileHandlingException(metaIn, dataIn); - } } } @@ -220,7 +216,10 @@ final class TermsReader extends Terms { this.sumDocFreq = sumDocFreq; this.docCount = docCount; FSTTermOutputs outputs = new FSTTermOutputs(fieldInfo); - this.dict = new FST<>(FST.readMetadata(metaIn, outputs), dataIn, offHeapFSTStore); + final var fstMetadata = FST.readMetadata(metaIn, outputs); + OffHeapFSTStore offHeapFSTStore = + new OffHeapFSTStore(dataIn, dataIn.getFilePointer(), fstMetadata); + this.dict = FST.fromFSTReader(fstMetadata, offHeapFSTStore); dataIn.skipBytes(offHeapFSTStore.size()); } diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsWriter.java b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsWriter.java index bdeae8fd9735..82e99493745a 100644 --- a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsWriter.java +++ b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsWriter.java @@ -136,7 +136,6 @@ public FSTTermsWriter(SegmentWriteState state, PostingsWriterBase postingsWriter this.maxDoc = state.segmentInfo.maxDoc(); IndexOutput metaOut = null, dataOut = null; - boolean success = false; try { metaOut = state.directory.createOutput(termsMetaFileName, state.context); dataOut = state.directory.createOutput(termsDataFileName, state.context); @@ -158,15 +157,9 @@ public FSTTermsWriter(SegmentWriteState state, PostingsWriterBase postingsWriter this.postingsWriter.init(metaOut, state); this.metaOut = metaOut; this.dataOut = dataOut; - success = true; } catch (Throwable t) { - IOUtils.closeWhileSuppressingExceptions(t, out); + IOUtils.closeWhileSuppressingExceptions(t, metaOut, dataOut); throw t; - } finally { - if (!success) { - IOUtils.closeWhileHandlingException(metaOut, dataOut); - } - this.postingsWriter.init(out, state); } } @@ -212,9 +205,9 @@ public void write(Fields fields, NormsProducer norms) throws IOException { public void close() throws IOException { if (metaOut != null) { assert dataOut != null; - boolean success = false; - try { - // write field summary + try (IndexOutput _ = metaOut; + IndexOutput __ = dataOut; + postingsWriter) { // write field summary final long dirStart = metaOut.getFilePointer(); metaOut.writeVInt(fields.size()); @@ -226,18 +219,12 @@ public void close() throws IOException { } metaOut.writeVLong(field.sumDocFreq); metaOut.writeVInt(field.docCount); - field.dict.saveMetadata(metaOut); + field.dict.getMetadata().save(metaOut); } writeTrailer(metaOut, dirStart); CodecUtil.writeFooter(metaOut); CodecUtil.writeFooter(dataOut); - success = true; } finally { - if (success) { - IOUtils.close(metaOut, dataOut, postingsWriter); - } else { - IOUtils.closeWhileHandlingException(metaOut, dataOut, postingsWriter); - } metaOut = null; dataOut = null; } From 09768fc2bcf0007a56d3d3a80be483fd964d943a Mon Sep 17 00:00:00 2001 From: Anh Dung Bui Date: Sat, 13 Sep 2025 15:46:33 -0500 Subject: [PATCH 10/12] change version --- lucene/CHANGES.txt | 6 +++--- .../org/apache/lucene/codecs/memory/FSTPostingsFormat.java | 2 +- .../org/apache/lucene/codecs/memory/FSTTermsWriter.java | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index f48af72dfbcb..fc8b014d41f2 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -31,6 +31,9 @@ API Changes * GITHUB#14844: Change IndexInput.updateReadAdvice to take an IOContext instead (Simon Cooper) +* GITHUB#12980: Make FSTPostingsFormat to build FST off-heap. This PostingsFormat will now + create 2 FST files (tfp.meta and tfp.data) instead of a single one. (Anh Dung Bui) + New Features --------------------- * GITHUB#14097: Binary partitioning merge policy over float-valued vector field. (Mike Sokolov) @@ -780,9 +783,6 @@ API Changes * GITHUB#12855: Remove deprecated DrillSideways#createDrillDownFacetsCollector extension method. (Greg Miller) -* GITHUB#12980: Make FSTPostingsFormat to build FST off-heap. This PostingsFormat will now - create 2 FST files (tfp.meta and tfp.data) instead of a single one. (Anh Dung Bui) - * GITHUB#12875: Ensure token position is always increased in PathHierarchyTokenizer and ReversePathHierarchyTokenizer and resulting tokens do not overlap. (Michael Froh, Lukáš Vlček) diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTPostingsFormat.java b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTPostingsFormat.java index 5b03237e3cc6..8e87d911849a 100644 --- a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTPostingsFormat.java +++ b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTPostingsFormat.java @@ -31,7 +31,7 @@ /** FST term dict + Lucene99PBF */ public final class FSTPostingsFormat extends PostingsFormat { public FSTPostingsFormat() { - super("FST99"); + super("FST110"); } @Override diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsWriter.java b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsWriter.java index 82e99493745a..23821a454a53 100644 --- a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsWriter.java +++ b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsWriter.java @@ -206,7 +206,7 @@ public void close() throws IOException { if (metaOut != null) { assert dataOut != null; try (IndexOutput _ = metaOut; - IndexOutput __ = dataOut; + IndexOutput _ = dataOut; postingsWriter) { // write field summary final long dirStart = metaOut.getFilePointer(); From ad9c18ccab907b2bf24ba518fd808972d7f1bd1b Mon Sep 17 00:00:00 2001 From: Anh Dung Bui Date: Sun, 14 Sep 2025 04:45:49 -0500 Subject: [PATCH 11/12] only store the FST metadata --- .../apache/lucene/codecs/memory/FSTTermsWriter.java | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsWriter.java b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsWriter.java index 23821a454a53..7cfa5db57d60 100644 --- a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsWriter.java +++ b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsWriter.java @@ -219,7 +219,7 @@ public void close() throws IOException { } metaOut.writeVLong(field.sumDocFreq); metaOut.writeVInt(field.docCount); - field.dict.getMetadata().save(metaOut); + field.fstMetadata.save(metaOut); } writeTrailer(metaOut, dirStart); CodecUtil.writeFooter(metaOut); @@ -237,7 +237,7 @@ private static class FieldMetaData { public final long sumTotalTermFreq; public final long sumDocFreq; public final int docCount; - public final FST dict; + public final FST.FSTMetadata fstMetadata; public FieldMetaData( FieldInfo fieldInfo, @@ -245,13 +245,13 @@ public FieldMetaData( long sumTotalTermFreq, long sumDocFreq, int docCount, - FST fst) { + FST.FSTMetadata fstMetadata) { this.fieldInfo = fieldInfo; this.numTerms = numTerms; this.sumTotalTermFreq = sumTotalTermFreq; this.sumDocFreq = sumDocFreq; this.docCount = docCount; - this.dict = fst; + this.fstMetadata = fstMetadata; } } @@ -291,10 +291,9 @@ public void finishTerm(BytesRef text, BlockTermState state) throws IOException { public void finish(long sumTotalTermFreq, long sumDocFreq, int docCount) throws IOException { // save FST dict if (numTerms > 0) { - final FST fst = - FST.fromFSTReader(fstCompiler.compile(), fstCompiler.getFSTReader()); fields.add( - new FieldMetaData(fieldInfo, numTerms, sumTotalTermFreq, sumDocFreq, docCount, fst)); + new FieldMetaData( + fieldInfo, numTerms, sumTotalTermFreq, sumDocFreq, docCount, fstCompiler.compile())); } } } From 72c7f32cbe8e7a80b56e8e0c8920637861556fc4 Mon Sep 17 00:00:00 2001 From: Anh Dung Bui Date: Sun, 14 Sep 2025 05:13:25 -0500 Subject: [PATCH 12/12] Tidy up --- .../org/apache/lucene/codecs/memory/FSTTermsWriter.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsWriter.java b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsWriter.java index 7cfa5db57d60..c9ff83d38253 100644 --- a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsWriter.java +++ b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsWriter.java @@ -293,7 +293,12 @@ public void finish(long sumTotalTermFreq, long sumDocFreq, int docCount) throws if (numTerms > 0) { fields.add( new FieldMetaData( - fieldInfo, numTerms, sumTotalTermFreq, sumDocFreq, docCount, fstCompiler.compile())); + fieldInfo, + numTerms, + sumTotalTermFreq, + sumDocFreq, + docCount, + fstCompiler.compile())); } } }