From 2d8f270ee01fbe74890d7a6c4ab98baaa6ec88f2 Mon Sep 17 00:00:00 2001 From: dungba88 Date: Tue, 26 Dec 2023 18:11:44 +0900 Subject: [PATCH 1/4] lazily write the FST padding byte --- .../java/org/apache/lucene/util/fst/FSTCompiler.java | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/util/fst/FSTCompiler.java b/lucene/core/src/java/org/apache/lucene/util/fst/FSTCompiler.java index 961511616abb..2a04a575ef7c 100644 --- a/lucene/core/src/java/org/apache/lucene/util/fst/FSTCompiler.java +++ b/lucene/core/src/java/org/apache/lucene/util/fst/FSTCompiler.java @@ -164,14 +164,12 @@ private FSTCompiler( boolean allowFixedLengthArcs, DataOutput dataOutput, float directAddressingMaxOversizingFactor, - int version) - throws IOException { + int version) { this.allowFixedLengthArcs = allowFixedLengthArcs; this.directAddressingMaxOversizingFactor = directAddressingMaxOversizingFactor; this.version = version; // pad: ensure no node gets address 0 which is reserved to mean - // the stop state w/ no arcs - dataOutput.writeByte((byte) 0); + // the stop state w/ no arcs. the actual byte will be written lazily numBytesWritten++; this.dataOutput = dataOutput; fst = @@ -344,7 +342,7 @@ public Builder setVersion(int version) { } /** Creates a new {@link FSTCompiler}. */ - public FSTCompiler build() throws IOException { + public FSTCompiler build() { // create a default DataOutput if not specified if (dataOutput == null) { dataOutput = getOnHeapReaderWriter(15); @@ -552,6 +550,10 @@ long addNode(FSTCompiler.UnCompiledNode nodeIn) throws IOException { } reverseScratchBytes(); + if (numBytesWritten == 1) { + // first time, write the padding byte + dataOutput.writeByte((byte) 0); + } scratchBytes.writeTo(dataOutput); numBytesWritten += scratchBytes.getPosition(); From b4a349404810ff5ac15aa6131f0d2cb220f73598 Mon Sep 17 00:00:00 2001 From: dungba88 Date: Tue, 26 Dec 2023 18:25:21 +0900 Subject: [PATCH 2/4] Also write the pad byte when there is emptyOutput --- .../java/org/apache/lucene/util/fst/FSTCompiler.java | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/util/fst/FSTCompiler.java b/lucene/core/src/java/org/apache/lucene/util/fst/FSTCompiler.java index 2a04a575ef7c..16581d4096b6 100644 --- a/lucene/core/src/java/org/apache/lucene/util/fst/FSTCompiler.java +++ b/lucene/core/src/java/org/apache/lucene/util/fst/FSTCompiler.java @@ -551,8 +551,7 @@ long addNode(FSTCompiler.UnCompiledNode nodeIn) throws IOException { reverseScratchBytes(); if (numBytesWritten == 1) { - // first time, write the padding byte - dataOutput.writeByte((byte) 0); + writePaddingByte(); } scratchBytes.writeTo(dataOutput); numBytesWritten += scratchBytes.getPosition(); @@ -561,6 +560,10 @@ long addNode(FSTCompiler.UnCompiledNode nodeIn) throws IOException { return numBytesWritten - 1; } + private void writePaddingByte() throws IOException { + dataOutput.writeByte((byte) 0); + } + private void writeLabel(DataOutput out, int v) throws IOException { assert v >= 0 : "v=" + v; if (fst.metadata.inputType == INPUT_TYPE.BYTE1) { @@ -970,6 +973,10 @@ public FST compile() throws IOException { if (root.numArcs == 0) { if (fst.metadata.emptyOutput == null) { return null; + } else { + // we haven't written the pad byte so far, but the FST is still valid + assert numBytesWritten == 1; + writePaddingByte(); } } From 601a6e17ad2328f120d0bb5ec7d45ddf7ca4f1c0 Mon Sep 17 00:00:00 2001 From: dungba88 Date: Tue, 26 Dec 2023 18:27:32 +0900 Subject: [PATCH 3/4] add comment --- .../core/src/java/org/apache/lucene/util/fst/FSTCompiler.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lucene/core/src/java/org/apache/lucene/util/fst/FSTCompiler.java b/lucene/core/src/java/org/apache/lucene/util/fst/FSTCompiler.java index 16581d4096b6..7f67fb2a38bc 100644 --- a/lucene/core/src/java/org/apache/lucene/util/fst/FSTCompiler.java +++ b/lucene/core/src/java/org/apache/lucene/util/fst/FSTCompiler.java @@ -550,6 +550,7 @@ long addNode(FSTCompiler.UnCompiledNode nodeIn) throws IOException { } reverseScratchBytes(); + // write the padding byte if needed if (numBytesWritten == 1) { writePaddingByte(); } @@ -561,6 +562,7 @@ long addNode(FSTCompiler.UnCompiledNode nodeIn) throws IOException { } private void writePaddingByte() throws IOException { + assert numBytesWritten == 1; dataOutput.writeByte((byte) 0); } @@ -975,7 +977,6 @@ public FST compile() throws IOException { return null; } else { // we haven't written the pad byte so far, but the FST is still valid - assert numBytesWritten == 1; writePaddingByte(); } } From c2e4c52404822476f1ec1a5aaf2a9282d3db4c98 Mon Sep 17 00:00:00 2001 From: dungba88 Date: Thu, 11 Jan 2024 11:37:28 +0900 Subject: [PATCH 4/4] Add more comments --- .../org/apache/lucene/util/fst/FSTCompiler.java | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/util/fst/FSTCompiler.java b/lucene/core/src/java/org/apache/lucene/util/fst/FSTCompiler.java index 7f67fb2a38bc..620837357657 100644 --- a/lucene/core/src/java/org/apache/lucene/util/fst/FSTCompiler.java +++ b/lucene/core/src/java/org/apache/lucene/util/fst/FSTCompiler.java @@ -109,6 +109,9 @@ public class FSTCompiler { private final IntsRefBuilder lastInput = new IntsRefBuilder(); + // indicates whether we are not yet to write the padding byte + private boolean paddingBytePending; + // NOTE: cutting this over to ArrayList instead loses ~6% // in build performance on 9.8M Wikipedia terms; so we // left this as an array: @@ -171,6 +174,7 @@ private FSTCompiler( // pad: ensure no node gets address 0 which is reserved to mean // the stop state w/ no arcs. the actual byte will be written lazily numBytesWritten++; + paddingBytePending = true; this.dataOutput = dataOutput; fst = new FST<>( @@ -551,7 +555,7 @@ long addNode(FSTCompiler.UnCompiledNode nodeIn) throws IOException { reverseScratchBytes(); // write the padding byte if needed - if (numBytesWritten == 1) { + if (paddingBytePending) { writePaddingByte(); } scratchBytes.writeTo(dataOutput); @@ -561,9 +565,14 @@ long addNode(FSTCompiler.UnCompiledNode nodeIn) throws IOException { return numBytesWritten - 1; } + /** + * Write the padding byte, ensure no node gets address 0 which is reserved to mean the stop state + * w/ no arcs + */ private void writePaddingByte() throws IOException { - assert numBytesWritten == 1; + assert paddingBytePending; dataOutput.writeByte((byte) 0); + paddingBytePending = false; } private void writeLabel(DataOutput out, int v) throws IOException { @@ -974,9 +983,10 @@ public FST compile() throws IOException { freezeTail(0); if (root.numArcs == 0) { if (fst.metadata.emptyOutput == null) { + // return null for completely empty FST which accepts nothing return null; } else { - // we haven't written the pad byte so far, but the FST is still valid + // we haven't written the padding byte so far, but the FST is still valid writePaddingByte(); } }