diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 3eba5cc772f6..d24cac10af75 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -314,6 +314,8 @@ Optimizations * GITHUB#16316: Lazily allocate buffers in MaxScoreBulkScorer. These are now only allocated when going via code paths that actually use them. (Alan Woodward) +* GITHUB#16160: Improve numeric doc values range query performance for dense fields that use GCD or delta encoding. (Costin Leau) + Bug Fixes --------------------- (No changes) diff --git a/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/GcdDeltaRangeIntoBitSetBenchmark.java b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/GcdDeltaRangeIntoBitSetBenchmark.java new file mode 100644 index 000000000000..2cba7a0d688e --- /dev/null +++ b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/GcdDeltaRangeIntoBitSetBenchmark.java @@ -0,0 +1,215 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.benchmark.jmh; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Comparator; +import java.util.Random; +import java.util.concurrent.TimeUnit; +import java.util.stream.Stream; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.NumericDocValuesField; +import org.apache.lucene.document.SortedNumericDocValuesField; +import org.apache.lucene.document.StringField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.BooleanClause.Occur; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.MMapDirectory; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Warmup; + +/** Benchmarks range queries over dense numeric doc values encoded as raw, delta, GCD, or both. */ +@State(Scope.Thread) +@BenchmarkMode(Mode.Throughput) +@OutputTimeUnit(TimeUnit.SECONDS) +@Warmup(iterations = 3, time = 3) +@Measurement(iterations = 5, time = 5) +public class GcdDeltaRangeIntoBitSetBenchmark { + + private static final String FIELD = "val"; + private static final String LEAD_FIELD = "lead"; + private static final String LEAD_VALUE = "yes"; + private static final String NONE = "none"; + private static final String DELTA_ONLY = "delta_only"; + private static final String GCD_1000 = "gcd_1000"; + private static final String GCD_100_DELTA = "gcd_100_delta"; + private static final long DOMAIN = 10_000_000L; + private static final long DELTA = 1_700_000_000_000L; + + private Directory dir; + private DirectoryReader reader; + private IndexSearcher searcher; + private Path path; + private Query query; + + @Param({"1000000"}) + public int numDocs; + + @Param({NONE, DELTA_ONLY, GCD_1000, GCD_100_DELTA}) + public String encoding; + + @Param({"0.01", "0.1", "0.5"}) + public double selectivity; + + @Setup(Level.Trial) + public void setup() throws Exception { + path = Files.createTempDirectory("gcdDeltaRangeIntoBitSet"); + dir = MMapDirectory.open(path); + + Random random = new Random(0); + try (IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig())) { + for (int i = 0; i < numDocs; i++) { + Document doc = new Document(); + doc.add(NumericDocValuesField.indexedField(FIELD, valueForDoc(encoding, i, random))); + doc.add(new StringField(LEAD_FIELD, LEAD_VALUE, Field.Store.NO)); + writer.addDocument(doc); + } + writer.forceMerge(1); + } + + reader = DirectoryReader.open(dir); + searcher = new IndexSearcher(reader); + query = rangeQuery(encoding, selectivity); + } + + private static long valueForDoc(String encoding, int doc, Random random) { + if (doc == 0) { + return minimumValue(encoding); + } else if (doc == 1 && encoding.equals(GCD_100_DELTA)) { + // Anchor entry.gcd to exactly 100 for the GCD_100_DELTA encoding: random multiples of 100 + // could otherwise share a larger common factor under some seeds, which would change the + // shape of the encoded values and what the benchmark measures. + return DELTA + 100L; + } + + long value = random.nextLong(0, DOMAIN); + switch (encoding) { + case NONE: + return value; + case DELTA_ONLY: + return DELTA + value; + case GCD_1000: + return value * 1_000L; + case GCD_100_DELTA: + return DELTA + value * 100L; + default: + throw new IllegalArgumentException("Unknown encoding: " + encoding); + } + } + + private static long minimumValue(String encoding) { + switch (encoding) { + case NONE: + case GCD_1000: + return 0; + case DELTA_ONLY: + case GCD_100_DELTA: + return DELTA; + default: + throw new IllegalArgumentException("Unknown encoding: " + encoding); + } + } + + private static Query rangeQuery(String encoding, double selectivity) { + long range = Math.max(1, (long) (DOMAIN * selectivity)); + long min = (DOMAIN - range) / 2; + long max = min + range; + Query rangeQuery = + SortedNumericDocValuesField.newSlowRangeQuery( + FIELD, actualValue(encoding, min), actualValue(encoding, max)); + return new BooleanQuery.Builder() + .add(new TermQuery(new Term(LEAD_FIELD, LEAD_VALUE)), Occur.FILTER) + .add(rangeQuery, Occur.FILTER) + .build(); + } + + private static long actualValue(String encoding, long value) { + switch (encoding) { + case NONE: + return value; + case DELTA_ONLY: + return DELTA + value; + case GCD_1000: + return value * 1_000L; + case GCD_100_DELTA: + return DELTA + value * 100L; + default: + throw new IllegalArgumentException("Unknown encoding: " + encoding); + } + } + + @TearDown(Level.Trial) + public void tearDown() throws Exception { + reader.close(); + dir.close(); + if (Files.exists(path)) { + try (Stream walk = Files.walk(path)) { + walk.sorted(Comparator.reverseOrder()) + .forEach( + p -> { + try { + Files.delete(p); + } catch (IOException _) { + } + }); + } + } + } + + @Benchmark + @Fork( + value = 1, + jvmArgsAppend = {"-Xmx2g", "-Xms2g", "-XX:+AlwaysPreTouch"}) + public int rangeQueryDefaultProvider() throws IOException { + return searcher.count(query); + } + + @Benchmark + @Fork( + value = 1, + jvmArgsAppend = { + "--add-modules", + "jdk.incubator.vector", + "-Xmx2g", + "-Xms2g", + "-XX:+AlwaysPreTouch" + }) + public int rangeQueryPanamaProvider() throws IOException { + return searcher.count(query); + } +} diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesProducer.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesProducer.java index 568b0ef01a51..811f11c53436 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesProducer.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesProducer.java @@ -481,6 +481,72 @@ static void rangeIntoBitSet( values, fromDoc, toDoc, minValue, maxValue, bitSet, offset); } + /** + * Maps the raw query bounds {@code [minValue, maxValue]} into the encoded domain so the SIMD + * kernel in {@link org.apache.lucene.internal.vectorization.DocValuesRangeSupport} can run + * directly on packed values: {@code [encodedMin, encodedMax] = [ceil((min - delta) / mul), + * floor((max - delta) / mul)]}. Open bounds (e.g. {@code Long.MIN_VALUE} or {@code + * Long.MAX_VALUE}) are saturated to the encoded domain so they keep the SIMD path even when + * {@code min - delta} or {@code max - delta} would overflow. + */ + private static void rangeGcdDeltaIntoBitSet( + LongValues values, + int fromDoc, + int toDoc, + long minValue, + long maxValue, + long mul, + long delta, + FixedBitSet bitSet, + int offset) { + assert mul > 0; + long encodedMin = saturatingShiftLower(minValue, delta); + long encodedMax = saturatingShiftUpper(maxValue, delta); + if (mul != 1) { + // Math.ceilDiv / Math.floorDiv never overflow for mul > 0 (only Long.MIN_VALUE / -1 does), + // so the SIMD path is always taken; no fallback to the per-doc decoded loop is required. + encodedMin = Math.ceilDiv(encodedMin, mul); + encodedMax = Math.floorDiv(encodedMax, mul); + } + encodedMin = Math.max(0, encodedMin); + if (encodedMin <= encodedMax) { + rangeIntoBitSet(values, fromDoc, toDoc, encodedMin, encodedMax, bitSet, offset); + } + } + + /** + * Returns {@code minValue - delta}, saturating to {@code Long.MIN_VALUE} when the real value + * would underflow (every non-negative stored value satisfies the lower bound) or to {@code + * Long.MAX_VALUE} when it would overflow (no stored value can satisfy the lower bound). Stored + * values are non-negative, so the caller can keep using the SIMD path with these saturated + * sentinels. + */ + private static long saturatingShiftLower(long minValue, long delta) { + try { + return Math.subtractExact(minValue, delta); + } catch ( + @SuppressWarnings("unused") + ArithmeticException overflow) { + return delta > 0 ? Long.MIN_VALUE : Long.MAX_VALUE; + } + } + + /** + * Symmetric counterpart of {@link #saturatingShiftLower}: returns {@code maxValue - delta}, + * saturating to {@code Long.MAX_VALUE} when the real value would overflow (every stored value + * satisfies the upper bound) or to {@code Long.MIN_VALUE} when it would underflow (no stored + * value can satisfy the upper bound). + */ + private static long saturatingShiftUpper(long maxValue, long delta) { + try { + return Math.subtractExact(maxValue, delta); + } catch ( + @SuppressWarnings("unused") + ArithmeticException overflow) { + return delta < 0 ? Long.MAX_VALUE : Long.MIN_VALUE; + } + } + private static int fixedCardinality( SortedNumericEntry entry, DocValuesSkipperEntry skipperEntry) { if (skipperEntry == null @@ -960,13 +1026,8 @@ public void rangeIntoBitSet( long maxValue, FixedBitSet bitSet, int offset) { - // Per-doc evaluation for gcd/delta encoded fields - for (int d = fromDoc; d < toDoc; d++) { - long v = mul * values.get(d) + delta; - if (v >= minValue && v <= maxValue) { - bitSet.set(d - offset); - } - } + Lucene90DocValuesProducer.rangeGcdDeltaIntoBitSet( + values, fromDoc, toDoc, minValue, maxValue, mul, delta, bitSet, offset); } }; } diff --git a/lucene/core/src/test/org/apache/lucene/search/TestSkipBlockRangeIteratorIntoBitSet.java b/lucene/core/src/test/org/apache/lucene/search/TestSkipBlockRangeIteratorIntoBitSet.java index 76e01cae13c3..ccadb56b3afa 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestSkipBlockRangeIteratorIntoBitSet.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestSkipBlockRangeIteratorIntoBitSet.java @@ -19,6 +19,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Random; +import java.util.function.LongUnaryOperator; import org.apache.lucene.codecs.lucene104.Lucene104Codec; import org.apache.lucene.document.Document; import org.apache.lucene.document.NumericDocValuesField; @@ -634,6 +635,117 @@ public void testRangeIntoBitSetMatchesPerDocEvaluation() throws Exception { } } + public void testRangeIntoBitSetMatchesPerDocEvaluationWithDeltaEncoding() throws Exception { + long delta = 1_000_000L; + long[] values = rangeValues(DOC_COUNT, doc -> delta + doc); + assertRangeIntoBitSetMatchesPerDocEvaluation( + "delta-only encoded range must match decoded evaluation", + values, + delta + 127, + delta + 4097); + } + + public void testRangeIntoBitSetMatchesPerDocEvaluationWithGcdEncoding() throws Exception { + long[] values = rangeValues(DOC_COUNT, doc -> doc * 1_000L); + assertRangeIntoBitSetMatchesPerDocEvaluation( + "gcd encoded range must match decoded evaluation", values, 123_456L, 4_567_890L); + } + + public void testRangeIntoBitSetMatchesPerDocEvaluationWithGcdAndDeltaEncoding() throws Exception { + long delta = 1_000_000L; + long[] values = rangeValues(DOC_COUNT, doc -> delta + doc * 100L); + assertRangeIntoBitSetMatchesPerDocEvaluation( + "gcd+delta encoded range must match decoded evaluation", + values, + delta + 123, + delta + 456_789); + } + + public void testRangeIntoBitSetMatchesPerDocEvaluationWhenGcdRangeFallsBetweenValues() + throws Exception { + long[] values = rangeValues(DOC_COUNT, doc -> 10L + doc * 5L); + assertRangeIntoBitSetMatchesPerDocEvaluation( + "gcd encoded gap range must match no docs", values, 11, 14); + } + + public void testRangeIntoBitSetMatchesPerDocEvaluationWithOpenLowerBound() throws Exception { + long delta = 1_000_000L; + long[] values = rangeValues(DOC_COUNT, doc -> delta + doc); + assertRangeIntoBitSetMatchesPerDocEvaluation( + "open Long.MIN_VALUE lower bound must saturate and match all docs up to the upper bound", + values, + Long.MIN_VALUE, + delta + 127); + } + + public void testRangeIntoBitSetMatchesPerDocEvaluationWithOpenUpperBound() throws Exception { + long delta = 1_000_000L; + long[] values = rangeValues(DOC_COUNT, doc -> delta + doc * 100L); + assertRangeIntoBitSetMatchesPerDocEvaluation( + "open Long.MAX_VALUE upper bound must saturate and match all docs from the lower bound", + values, + delta + 50L * 100L, + Long.MAX_VALUE); + } + + public void testRangeIntoBitSetMatchesPerDocEvaluationWithBothOpenBounds() throws Exception { + long delta = 1_000_000L; + long[] values = rangeValues(DOC_COUNT, doc -> delta + doc * 100L); + assertRangeIntoBitSetMatchesPerDocEvaluation( + "fully open range must match every doc", values, Long.MIN_VALUE, Long.MAX_VALUE); + } + + public void testRangeIntoBitSetMatchesPerDocEvaluationWithNegativeEncodedLowerBound() + throws Exception { + // Stored values are non-negative (min == delta), but the query lower bound is below delta so + // (minValue - delta) is negative even though Math.subtractExact succeeds. This exercises the + // Math.max(0, encodedMin) clamp. + long delta = 1_000_000L; + long[] values = rangeValues(DOC_COUNT, doc -> delta + doc); + assertRangeIntoBitSetMatchesPerDocEvaluation( + "negative encoded lower bound must clamp to 0 and match docs up to the upper bound", + values, + delta - 50, + delta + 100); + } + + public void testRangeIntoBitSetMatchesPerDocEvaluationWithRangeBelowStoredValues() + throws Exception { + // The whole query range is below the stored minimum, so encodedMin > encodedMax after the + // bound transformation and the SIMD path is skipped without iterating. + long delta = 1_000_000L; + long[] values = rangeValues(DOC_COUNT, doc -> delta + doc * 100L); + assertRangeIntoBitSetMatchesPerDocEvaluation( + "query range below stored values must match no docs", values, delta - 1_000L, delta - 1L); + } + + public void testRangeIntoBitSetMatchesPerDocEvaluationWithRangeAboveStoredValues() + throws Exception { + long delta = 1_000_000L; + long[] values = rangeValues(DOC_COUNT, doc -> delta + doc * 100L); + long max = delta + (DOC_COUNT - 1L) * 100L; + assertRangeIntoBitSetMatchesPerDocEvaluation( + "query range above stored values must match no docs", values, max + 1L, max + 1_000L); + } + + public void testRangeIntoBitSetMatchesPerDocEvaluationWithFullGcdDeltaRange() throws Exception { + long delta = 1_000_000L; + long[] values = rangeValues(DOC_COUNT, doc -> delta + doc * 100L); + assertRangeIntoBitSetMatchesPerDocEvaluation( + "full gcd+delta encoded range must match all docs", + values, + delta, + delta + (DOC_COUNT - 1L) * 100L); + } + + public void testRangeIntoBitSetMatchesPerDocEvaluationWithSingleGcdDeltaValue() throws Exception { + long delta = 1_000_000L; + long value = delta + 123L * 100L; + long[] values = rangeValues(DOC_COUNT, doc -> delta + doc * 100L); + assertRangeIntoBitSetMatchesPerDocEvaluation( + "single gcd+delta encoded value range must match one doc", values, value, value); + } + public void testSortedNumericRangeIntoBitSetDenseFixedCardinality() throws Exception { doTestSortedNumericRangeIntoBitSet(true, true); } @@ -694,4 +806,47 @@ private void doTestSortedNumericRangeIntoBitSet(boolean dense, boolean fixedCard } } } + + private static long[] rangeValues(int numDocs, LongUnaryOperator valueFunction) { + long[] values = new long[numDocs]; + for (int i = 0; i < numDocs; i++) { + values[i] = valueFunction.applyAsLong(i); + } + return values; + } + + private void assertRangeIntoBitSetMatchesPerDocEvaluation( + String message, long[] values, long rangeMin, long rangeMax) throws Exception { + try (Directory dir = newDirectory()) { + IndexWriterConfig iwc = new IndexWriterConfig().setCodec(new Lucene104Codec()); + try (IndexWriter w = new IndexWriter(dir, iwc)) { + for (long value : values) { + Document doc = new Document(); + doc.add(NumericDocValuesField.indexedField("val", value)); + w.addDocument(doc); + } + w.forceMerge(1); + } + + try (DirectoryReader reader = DirectoryReader.open(dir)) { + LeafReaderContext ctx = reader.leaves().get(0); + FixedBitSet expected = new FixedBitSet(values.length); + NumericDocValues slowDv = ctx.reader().getNumericDocValues("val"); + for (int d = 0; d < values.length; d++) { + if (slowDv.advanceExact(d)) { + long value = slowDv.longValue(); + if (value >= rangeMin && value <= rangeMax) { + expected.set(d); + } + } + } + + FixedBitSet actual = new FixedBitSet(values.length); + NumericDocValues fastDv = ctx.reader().getNumericDocValues("val"); + fastDv.rangeIntoBitSet(0, values.length, rangeMin, rangeMax, actual, 0); + + assertEquals(message, expected, actual); + } + } + } }