Optimize encoded numeric range bitsets

lucene/CHANGES.txt

@@ -314,6 +314,8 @@ Optimizations
 * GITHUB#16316: Lazily allocate buffers in MaxScoreBulkScorer.  These are now only allocated when going
   via code paths that actually use them. (Alan Woodward)
 
+* GITHUB#16160: Improve numeric doc values range query performance for dense fields that use GCD or delta encoding. (Costin Leau)
+
 Bug Fixes
 ---------------------
 (No changes)

lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/GcdDeltaRangeIntoBitSetBenchmark.java

@@ -0,0 +1,215 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.benchmark.jmh;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Comparator;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Stream;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.NumericDocValuesField;
+import org.apache.lucene.document.SortedNumericDocValuesField;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.BooleanClause.Occur;
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.MMapDirectory;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.annotations.Warmup;
+
+/** Benchmarks range queries over dense numeric doc values encoded as raw, delta, GCD, or both. */
+@State(Scope.Thread)
+@BenchmarkMode(Mode.Throughput)
+@OutputTimeUnit(TimeUnit.SECONDS)
+@Warmup(iterations = 3, time = 3)
+@Measurement(iterations = 5, time = 5)
+public class GcdDeltaRangeIntoBitSetBenchmark {
+
+  private static final String FIELD = "val";
+  private static final String LEAD_FIELD = "lead";
+  private static final String LEAD_VALUE = "yes";
+  private static final String NONE = "none";
+  private static final String DELTA_ONLY = "delta_only";
+  private static final String GCD_1000 = "gcd_1000";
+  private static final String GCD_100_DELTA = "gcd_100_delta";
+  private static final long DOMAIN = 10_000_000L;
+  private static final long DELTA = 1_700_000_000_000L;
+
+  private Directory dir;
+  private DirectoryReader reader;
+  private IndexSearcher searcher;
+  private Path path;
+  private Query query;
+
+  @Param({"1000000"})
+  public int numDocs;
+
+  @Param({NONE, DELTA_ONLY, GCD_1000, GCD_100_DELTA})
+  public String encoding;
+
+  @Param({"0.01", "0.1", "0.5"})
+  public double selectivity;
+
+  @Setup(Level.Trial)
+  public void setup() throws Exception {
+    path = Files.createTempDirectory("gcdDeltaRangeIntoBitSet");
+    dir = MMapDirectory.open(path);
+
+    Random random = new Random(0);
+    try (IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig())) {
+      for (int i = 0; i < numDocs; i++) {
+        Document doc = new Document();
+        doc.add(NumericDocValuesField.indexedField(FIELD, valueForDoc(encoding, i, random)));
+        doc.add(new StringField(LEAD_FIELD, LEAD_VALUE, Field.Store.NO));
+        writer.addDocument(doc);
+      }
+      writer.forceMerge(1);
+    }
+
+    reader = DirectoryReader.open(dir);
+    searcher = new IndexSearcher(reader);
+    query = rangeQuery(encoding, selectivity);
+  }
+
+  private static long valueForDoc(String encoding, int doc, Random random) {
+    if (doc == 0) {
+      return minimumValue(encoding);
+    } else if (doc == 1 && encoding.equals(GCD_100_DELTA)) {
+      // Anchor entry.gcd to exactly 100 for the GCD_100_DELTA encoding: random multiples of 100
+      // could otherwise share a larger common factor under some seeds, which would change the
+      // shape of the encoded values and what the benchmark measures.
+      return DELTA + 100L;
+    }
+
+    long value = random.nextLong(0, DOMAIN);
+    switch (encoding) {
+      case NONE:
+        return value;
+      case DELTA_ONLY:
+        return DELTA + value;
+      case GCD_1000:
+        return value * 1_000L;
+      case GCD_100_DELTA:
+        return DELTA + value * 100L;
+      default:
+        throw new IllegalArgumentException("Unknown encoding: " + encoding);
+    }
+  }
+
+  private static long minimumValue(String encoding) {
+    switch (encoding) {
+      case NONE:
+      case GCD_1000:
+        return 0;
+      case DELTA_ONLY:
+      case GCD_100_DELTA:
+        return DELTA;
+      default:
+        throw new IllegalArgumentException("Unknown encoding: " + encoding);
+    }
+  }
+
+  private static Query rangeQuery(String encoding, double selectivity) {
+    long range = Math.max(1, (long) (DOMAIN * selectivity));
+    long min = (DOMAIN - range) / 2;
+    long max = min + range;
+    Query rangeQuery =
+        SortedNumericDocValuesField.newSlowRangeQuery(
+            FIELD, actualValue(encoding, min), actualValue(encoding, max));
+    return new BooleanQuery.Builder()
+        .add(new TermQuery(new Term(LEAD_FIELD, LEAD_VALUE)), Occur.FILTER)
+        .add(rangeQuery, Occur.FILTER)
+        .build();
+  }
+
+  private static long actualValue(String encoding, long value) {
+    switch (encoding) {
+      case NONE:
+        return value;
+      case DELTA_ONLY:
+        return DELTA + value;
+      case GCD_1000:
+        return value * 1_000L;
+      case GCD_100_DELTA:
+        return DELTA + value * 100L;
+      default:
+        throw new IllegalArgumentException("Unknown encoding: " + encoding);
+    }
+  }
+
+  @TearDown(Level.Trial)
+  public void tearDown() throws Exception {
+    reader.close();
+    dir.close();
+    if (Files.exists(path)) {
+      try (Stream<Path> walk = Files.walk(path)) {
+        walk.sorted(Comparator.reverseOrder())
+            .forEach(
+                p -> {
+                  try {
+                    Files.delete(p);
+                  } catch (IOException _) {
+                  }
+                });
+      }
+    }
+  }
+
+  @Benchmark
+  @Fork(
+      value = 1,
+      jvmArgsAppend = {"-Xmx2g", "-Xms2g", "-XX:+AlwaysPreTouch"})
+  public int rangeQueryDefaultProvider() throws IOException {
+    return searcher.count(query);
+  }
+
+  @Benchmark
+  @Fork(
+      value = 1,
+      jvmArgsAppend = {
+        "--add-modules",
+        "jdk.incubator.vector",
+        "-Xmx2g",
+        "-Xms2g",
+        "-XX:+AlwaysPreTouch"
+      })
+  public int rangeQueryPanamaProvider() throws IOException {
+    return searcher.count(query);
+  }
+}

lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesProducer.java

@@ -481,6 +481,72 @@ static void rangeIntoBitSet(
         values, fromDoc, toDoc, minValue, maxValue, bitSet, offset);
   }
 
+  /**
+   * Maps the raw query bounds {@code [minValue, maxValue]} into the encoded domain so the SIMD
+   * kernel in {@link org.apache.lucene.internal.vectorization.DocValuesRangeSupport} can run
+   * directly on packed values: {@code [encodedMin, encodedMax] = [ceil((min - delta) / mul),
+   * floor((max - delta) / mul)]}. Open bounds (e.g. {@code Long.MIN_VALUE} or {@code
+   * Long.MAX_VALUE}) are saturated to the encoded domain so they keep the SIMD path even when
+   * {@code min - delta} or {@code max - delta} would overflow.
+   */
+  private static void rangeGcdDeltaIntoBitSet(
+      LongValues values,
+      int fromDoc,
+      int toDoc,
+      long minValue,
+      long maxValue,
+      long mul,
+      long delta,
+      FixedBitSet bitSet,
+      int offset) {
+    assert mul > 0;
+    long encodedMin = saturatingShiftLower(minValue, delta);
+    long encodedMax = saturatingShiftUpper(maxValue, delta);
+    if (mul != 1) {
+      // Math.ceilDiv / Math.floorDiv never overflow for mul > 0 (only Long.MIN_VALUE / -1 does),
+      // so the SIMD path is always taken; no fallback to the per-doc decoded loop is required.
+      encodedMin = Math.ceilDiv(encodedMin, mul);
+      encodedMax = Math.floorDiv(encodedMax, mul);
+    }
+    encodedMin = Math.max(0, encodedMin);
+    if (encodedMin <= encodedMax) {
+      rangeIntoBitSet(values, fromDoc, toDoc, encodedMin, encodedMax, bitSet, offset);
+    }
+  }
+
+  /**
+   * Returns {@code minValue - delta}, saturating to {@code Long.MIN_VALUE} when the real value
+   * would underflow (every non-negative stored value satisfies the lower bound) or to {@code
+   * Long.MAX_VALUE} when it would overflow (no stored value can satisfy the lower bound). Stored
+   * values are non-negative, so the caller can keep using the SIMD path with these saturated
+   * sentinels.
+   */
+  private static long saturatingShiftLower(long minValue, long delta) {
+    try {
+      return Math.subtractExact(minValue, delta);
+    } catch (
+        @SuppressWarnings("unused")
+        ArithmeticException overflow) {
+      return delta > 0 ? Long.MIN_VALUE : Long.MAX_VALUE;
+    }
+  }
+
+  /**
+   * Symmetric counterpart of {@link #saturatingShiftLower}: returns {@code maxValue - delta},
+   * saturating to {@code Long.MAX_VALUE} when the real value would overflow (every stored value
+   * satisfies the upper bound) or to {@code Long.MIN_VALUE} when it would underflow (no stored
+   * value can satisfy the upper bound).
+   */
+  private static long saturatingShiftUpper(long maxValue, long delta) {
+    try {
+      return Math.subtractExact(maxValue, delta);
+    } catch (
+        @SuppressWarnings("unused")
+        ArithmeticException overflow) {
+      return delta < 0 ? Long.MAX_VALUE : Long.MIN_VALUE;
+    }
+  }
+
   private static int fixedCardinality(
       SortedNumericEntry entry, DocValuesSkipperEntry skipperEntry) {
     if (skipperEntry == null
@@ -960,13 +1026,8 @@ public void rangeIntoBitSet(
                   long maxValue,
                   FixedBitSet bitSet,
                   int offset) {
-                // Per-doc evaluation for gcd/delta encoded fields
-                for (int d = fromDoc; d < toDoc; d++) {
-                  long v = mul * values.get(d) + delta;
-                  if (v >= minValue && v <= maxValue) {
-                    bitSet.set(d - offset);
-                  }
-                }
+                Lucene90DocValuesProducer.rangeGcdDeltaIntoBitSet(
+                    values, fromDoc, toDoc, minValue, maxValue, mul, delta, bitSet, offset);
               }
             };
           }