The shortcuts ({@code skipperMinDocId=0} and {@code skipperMaxDocId=docCount}) assume that + * docs without a value do not appear within or before the matching doc-ID range. This requires + * that the field is dense (every doc has a value, {@code docCount == reader.maxDoc()}). + */ + private static SortField canUsePrimarySortShortcut( + LeafReader reader, String field, DocValuesSkipper skipper) throws IOException { + SortField sf = Sort.getPrimarySortField(reader); + if (sf == null || sf.getField().equals(field) == false) { return null; } - final Sort indexSort = reader.getMetaData().sort(); - if (indexSort == null - || indexSort.getSort().length == 0 - || indexSort.getSort()[0].getField().equals(field) == false) { + if (skipper.docCount() != reader.maxDoc()) { return null; } - return indexSort.getSort()[0]; + return sf; } } diff --git a/lucene/core/src/java/org/apache/lucene/document/SortedSetDocValuesRangeQuery.java b/lucene/core/src/java/org/apache/lucene/document/SortedSetDocValuesRangeQuery.java index 3022ade0f851..a4228bd34536 100644 --- a/lucene/core/src/java/org/apache/lucene/document/SortedSetDocValuesRangeQuery.java +++ b/lucene/core/src/java/org/apache/lucene/document/SortedSetDocValuesRangeQuery.java @@ -121,12 +121,13 @@ public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOExcepti DocValuesSkipper skipper = context.reader().getDocValuesSkipper(field); SortedSetDocValues values = DocValues.getSortedSet(context.reader(), field); final SortedDocValues singleton = DocValues.unwrapSingleton(values); - final SortField primarySortField; - if (singleton != null - && skipper != null - && (primarySortField = densePrimarySort(context.reader(), skipper)) != null) { - return getScorerSupplierFromDensePrimarySort( - singleton, values, skipper, primarySortField); + if (singleton != null && skipper != null) { + SortField primarySortField = + canUsePrimarySortShortcut(context.reader(), field, skipper.docCount()); + if (primarySortField != null) { + return getScorerSupplierFromDensePrimarySort( + singleton, values, skipper, primarySortField); + } } // implement ScorerSupplier, since we do some expensive stuff to make a scorer return new ConstantScoreScorerSupplier(score(), scoreMode, context.reader().maxDoc()) { @@ -276,16 +277,23 @@ private long maxOrd(SortedSetDocValues values) throws IOException { return maxOrd; } - private SortField densePrimarySort(LeafReader reader, DocValuesSkipper skipper) { - if (skipper.docCount() != reader.maxDoc()) { + /** + * Returns the primary sort field if it is safe to use the {@link SortedSkipperScorerSupplier} + * range shortcuts for this query, or {@code null} if the shortcut cannot be applied. + * + *
The shortcuts assume that docs without a value do not appear before or within the matching + * doc-ID range. This requires that the field is dense (every doc has a value, {@code docCount == + * reader.maxDoc()}). + */ + private static SortField canUsePrimarySortShortcut(LeafReader reader, String field, int docCount) + throws IOException { + SortField sf = Sort.getPrimarySortField(reader); + if (sf == null || sf.getField().equals(field) == false) { return null; } - final Sort indexSort = reader.getMetaData().sort(); - if (indexSort == null - || indexSort.getSort().length == 0 - || indexSort.getSort()[0].getField().equals(field) == false) { + if (docCount != reader.maxDoc()) { return null; } - return indexSort.getSort()[0]; + return sf; } } diff --git a/lucene/core/src/java/org/apache/lucene/search/IndexSortSortedNumericDocValuesRangeQuery.java b/lucene/core/src/java/org/apache/lucene/search/IndexSortSortedNumericDocValuesRangeQuery.java index fb4afd4c795c..83e907dcd50f 100644 --- a/lucene/core/src/java/org/apache/lucene/search/IndexSortSortedNumericDocValuesRangeQuery.java +++ b/lucene/core/src/java/org/apache/lucene/search/IndexSortSortedNumericDocValuesRangeQuery.java @@ -190,21 +190,19 @@ public int count(LeafReaderContext context) throws IOException { } // use index sort optimization if possible - Sort indexSort = reader.getMetaData().sort(); - if (indexSort != null - && indexSort.getSort().length > 0 - && indexSort.getSort()[0].getField().equals(field)) { - final SortField sortField = indexSort.getSort()[0]; - final SortField.Type sortFieldType = getSortFieldType(sortField); + SortField primarySortField = Sort.getPrimarySortField(reader); + if (primarySortField != null && primarySortField.getField().equals(field)) { + final SortField.Type sortFieldType = getSortFieldType(primarySortField); // The index sort optimization is only supported for Type.INT and Type.LONG if (sortFieldType == Type.INT || sortFieldType == Type.LONG) { - Object missingValue = sortField.getMissingValue(); + Object missingValue = primarySortField.getMissingValue(); final long missingLongValue = missingValue == null ? 0L : ((Number) missingValue).longValue(); // all documents have docValues or missing value falls outside the range if ((pointValues != null && pointValues.getDocCount() == reader.maxDoc()) || (missingLongValue < lowerValue || missingLongValue > upperValue)) { - itAndCount = getDocIdSetIterator(sortField, sortFieldType, context, numericValues); + itAndCount = + getDocIdSetIterator(primarySortField, sortFieldType, context, numericValues); } if (itAndCount != null && itAndCount.count != -1) { return itAndCount.count; @@ -419,14 +417,12 @@ private boolean matchAll(PointValues points, byte[] queryLowerPoint, byte[] quer private IteratorAndCount getDocIdSetIteratorOrNullFromBkd( LeafReaderContext context, DocIdSetIterator delegate) throws IOException { - Sort indexSort = context.reader().getMetaData().sort(); - if (indexSort == null - || indexSort.getSort().length == 0 - || indexSort.getSort()[0].getField().equals(field) == false) { + SortField primarySortField = Sort.getPrimarySortField(context.reader()); + if (primarySortField == null || primarySortField.getField().equals(field) == false) { return null; } - final boolean reverse = indexSort.getSort()[0].getReverse(); + final boolean reverse = primarySortField.getReverse(); PointValues points = context.reader().getPointValues(field); if (points == null) { @@ -519,16 +515,12 @@ private IteratorAndCount getDocIdSetIteratorOrNull(LeafReaderContext context) th if (itAndCount != null) { return itAndCount; } - Sort indexSort = context.reader().getMetaData().sort(); - if (indexSort != null - && indexSort.getSort().length > 0 - && indexSort.getSort()[0].getField().equals(field)) { - - final SortField sortField = indexSort.getSort()[0]; - final SortField.Type sortFieldType = getSortFieldType(sortField); + SortField primarySortField = Sort.getPrimarySortField(context.reader()); + if (primarySortField != null && primarySortField.getField().equals(field)) { + final SortField.Type sortFieldType = getSortFieldType(primarySortField); // The index sort optimization is only supported for Type.INT and Type.LONG if (sortFieldType == Type.INT || sortFieldType == Type.LONG) { - return getDocIdSetIterator(sortField, sortFieldType, context, numericValues); + return getDocIdSetIterator(primarySortField, sortFieldType, context, numericValues); } } } diff --git a/lucene/core/src/java/org/apache/lucene/search/Sort.java b/lucene/core/src/java/org/apache/lucene/search/Sort.java index f46515e8ea3f..b55b13d2a37f 100644 --- a/lucene/core/src/java/org/apache/lucene/search/Sort.java +++ b/lucene/core/src/java/org/apache/lucene/search/Sort.java @@ -18,6 +18,8 @@ import java.io.IOException; import java.util.Arrays; +import org.apache.lucene.index.DocValuesSkipper; +import org.apache.lucene.index.LeafReader; /** * Encapsulates sort criteria for returned hits. @@ -132,4 +134,37 @@ public boolean needsScores() { } return false; } + + /** + * Returns the primary index sort field for the given LeafReader, or null if none is configured. + * + *
If sorting against a field would have no effect then that SortField is skipped over and the + * next SortField in the list is returned. This can happen for fields with no values in the + * segment, or fields that have only a single distinct value. + */ + public static SortField getPrimarySortField(LeafReader reader) { + Sort sort = reader.getMetaData().sort(); + if (sort == null) { + return null; + } + for (SortField sf : sort.fields) { + String field = sf.getField(); + if (field == null) { + // Custom field that we don't know anything about, so return it as primary + return sf; + } + if (reader.getFieldInfos().fieldInfo(field) == null) { + // Field has no values in this segment, so sorting by it has no effect. + continue; + } + // If the field has a skip index, check whether all values are identical, + // in which case sorting by this field is a no-op for this segment. + DocValuesSkipper skipper = reader.getDocValuesSkipper(field); + if (skipper != null && skipper.minValue() == skipper.maxValue()) { + continue; + } + return sf; + } + return null; + } } diff --git a/lucene/core/src/test/org/apache/lucene/search/TestDocValuesQueries.java b/lucene/core/src/test/org/apache/lucene/search/TestDocValuesQueries.java index a932dc0061c5..cb70682fece8 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestDocValuesQueries.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestDocValuesQueries.java @@ -1122,4 +1122,252 @@ public void doTestPrimarySortDenseExactMatch( reader.close(); dir.close(); } + + /** + * Verifies that SortedNumericDocValuesRangeQuery can use a secondary sort field to accelerate + * querying if the primary sort field is single-valued and therefore a no-op + */ + @Test + public void testSortedNumericRangeQueryOptimizesWithConstantPrimary(Random random) + throws IOException { + Directory dir = newDirectory(); + IndexWriterConfig config = new IndexWriterConfig().setCodec(getCodec(random)); + config.setIndexSort( + new Sort( + new SortField("primary", SortField.Type.LONG), // constant → no-op + new SortField("secondary", SortField.Type.LONG))); // varying → effective primary + IndexWriter iw = new IndexWriter(dir, config); + for (int i = 0; i < 10; i++) { + Document doc = new Document(); + doc.add(NumericDocValuesField.indexedField("primary", 42)); // constant, skip index present + doc.add(NumericDocValuesField.indexedField("secondary", i)); // varying + iw.addDocument(doc); + } + iw.forceMerge(1); + iw.close(); + DirectoryReader reader = DirectoryReader.open(dir); + IndexSearcher searcher = new IndexSearcher(reader); + + // primary has a constant value, so getPrimarySortField skips it and returns "secondary". + // The query on "secondary" should therefore use SortedSkipperScorerSupplier, + // which yields a plain range iterator with no two-phase iterator. + Query query = SortedNumericDocValuesField.newSlowRangeQuery("secondary", 3, 7); + Weight weight = query.createWeight(searcher, ScoreMode.COMPLETE, 1.0f); + Scorer scorer = weight.scorer(reader.leaves().get(0)); + assertNotNull(scorer); + assertNull( + "SortedSkipperScorerSupplier should be used when primary sort is a no-op", + scorer.twoPhaseIterator()); + + reader.close(); + dir.close(); + } + + /** + * Verifies that SortedNumericDocValuesRangeQuery can use a secondary sort field to accelerate + * querying if the primary sort field is empty and therefore a no-op + */ + @Test + public void testSortedNumericRangeQueryOptimizesWithEmptyPrimary(Random random) + throws IOException { + Directory dir = newDirectory(); + IndexWriterConfig config = new IndexWriterConfig().setCodec(getCodec(random)); + config.setIndexSort( + new Sort( + new SortField("primary", SortField.Type.LONG), // empty → no-op + new SortField("secondary", SortField.Type.LONG))); // varying → effective primary + IndexWriter iw = new IndexWriter(dir, config); + for (int i = 0; i < 10; i++) { + Document doc = new Document(); + // nothing added for the primary sort field + doc.add(NumericDocValuesField.indexedField("secondary", i)); // varying + iw.addDocument(doc); + } + iw.forceMerge(1); + iw.close(); + DirectoryReader reader = DirectoryReader.open(dir); + IndexSearcher searcher = new IndexSearcher(reader); + + // primary has a constant value, so getPrimarySortField skips it and returns "secondary". + // The query on "secondary" should therefore use SortedSkipperScorerSupplier, + // which yields a plain range iterator with no two-phase iterator. + Query query = SortedNumericDocValuesField.newSlowRangeQuery("secondary", 3, 7); + Weight weight = query.createWeight(searcher, ScoreMode.COMPLETE, 1.0f); + Scorer scorer = weight.scorer(reader.leaves().get(0)); + assertNotNull(scorer); + assertNull( + "SortedSkipperScorerSupplier should be used when primary sort is a no-op", + scorer.twoPhaseIterator()); + + reader.close(); + dir.close(); + } + + /** + * Verifies that SortedNumericDocValuesRangeQuery does not use a secondary sort field to + * accelerate querying if the primary sort field is a genuine sort field. + */ + @Test + public void testSortedNumericRangeQueryDoesNotOptimizeWithActivePrimary(Random random) + throws IOException { + Directory dir = newDirectory(); + IndexWriterConfig config = new IndexWriterConfig().setCodec(getCodec(random)); + config.setIndexSort( + new Sort( + new SortField("primary", SortField.Type.LONG), // multiple values → effective primary + new SortField("secondary", SortField.Type.LONG))); + IndexWriter iw = new IndexWriter(dir, config); + for (int i = 0; i < 10; i++) { + Document doc = new Document(); + doc.add(NumericDocValuesField.indexedField("primary", i)); // varying + doc.add(NumericDocValuesField.indexedField("secondary", i)); // varying + iw.addDocument(doc); + } + iw.forceMerge(1); + iw.close(); + DirectoryReader reader = DirectoryReader.open(dir); + IndexSearcher searcher = new IndexSearcher(reader); + + // primary is active, so getPrimarySortField returns "primary", not "secondary". + // The query on "secondary" must fall back to the TwoPhaseIterator path. + Query query = SortedNumericDocValuesField.newSlowRangeQuery("secondary", 3, 7); + Weight weight = query.createWeight(searcher, ScoreMode.COMPLETE, 1.0f); + Scorer scorer = weight.scorer(reader.leaves().get(0)); + assertNotNull(scorer); + assertNotNull( + "TwoPhaseIterator fallback should be used when secondary is not the effective primary", + scorer.twoPhaseIterator()); + + reader.close(); + dir.close(); + } + + /** + * Verifies that SortedSetDocValuesRangeQuery can use a secondary sort field to accelerate + * querying if the primary sort field is single-valued and therefore a no-op + */ + @Test + public void testSortedSetRangeQueryOptimizesWithConstantPrimary(Random random) + throws IOException { + Directory dir = newDirectory(); + IndexWriterConfig config = new IndexWriterConfig().setCodec(getCodec(random)); + config.setIndexSort( + new Sort( + new SortField("primary", SortField.Type.STRING), // constant → no-op + new SortField("secondary", SortField.Type.STRING))); // varying → effective primary + IndexWriter iw = new IndexWriter(dir, config); + for (int i = 0; i < 10; i++) { + Document doc = new Document(); + doc.add(SortedDocValuesField.indexedField("primary", newBytesRef("constant"))); + doc.add( + SortedDocValuesField.indexedField( + "secondary", newBytesRef(String.format(Locale.ROOT, "%03d", i)))); + iw.addDocument(doc); + } + iw.forceMerge(1); + iw.close(); + DirectoryReader reader = DirectoryReader.open(dir); + IndexSearcher searcher = new IndexSearcher(reader); + + // primary has a constant ordinal, so getPrimarySortField skips it and returns "secondary". + // The query on "secondary" should therefore use SortedSkipperScorerSupplier, + // which yields a plain range iterator with no two-phase iterator. + Query query = + SortedSetDocValuesField.newSlowRangeQuery( + "secondary", newBytesRef("003"), newBytesRef("007"), true, true); + Weight weight = query.createWeight(searcher, ScoreMode.COMPLETE, 1.0f); + Scorer scorer = weight.scorer(reader.leaves().get(0)); + assertNotNull(scorer); + assertNull( + "SortedSkipperScorerSupplier should be used when primary sort is a no-op", + scorer.twoPhaseIterator()); + + reader.close(); + dir.close(); + } + + /** + * Verifies that SortedSetDocValuesRangeQuery can use a secondary sort field to accelerate + * querying if the primary sort field is empty and therefore a no-op + */ + @Test + public void testSortedSetRangeQueryOptimizesWithEmptyPrimary(Random random) throws IOException { + Directory dir = newDirectory(); + IndexWriterConfig config = new IndexWriterConfig().setCodec(getCodec(random)); + config.setIndexSort( + new Sort( + new SortField("primary", SortField.Type.STRING), // constant → no-op + new SortField("secondary", SortField.Type.STRING))); // varying → effective primary + IndexWriter iw = new IndexWriter(dir, config); + for (int i = 0; i < 10; i++) { + Document doc = new Document(); + // no values added for primary field + doc.add( + SortedDocValuesField.indexedField( + "secondary", newBytesRef(String.format(Locale.ROOT, "%03d", i)))); + iw.addDocument(doc); + } + iw.forceMerge(1); + iw.close(); + DirectoryReader reader = DirectoryReader.open(dir); + IndexSearcher searcher = new IndexSearcher(reader); + + // primary has a constant ordinal, so getPrimarySortField skips it and returns "secondary". + // The query on "secondary" should therefore use SortedSkipperScorerSupplier, + // which yields a plain range iterator with no two-phase iterator. + Query query = + SortedSetDocValuesField.newSlowRangeQuery( + "secondary", newBytesRef("003"), newBytesRef("007"), true, true); + Weight weight = query.createWeight(searcher, ScoreMode.COMPLETE, 1.0f); + Scorer scorer = weight.scorer(reader.leaves().get(0)); + assertNotNull(scorer); + assertNull( + "SortedSkipperScorerSupplier should be used when primary sort is a no-op", + scorer.twoPhaseIterator()); + + reader.close(); + dir.close(); + } + + /** + * Verifies that SortedSetDocValuesRangeQuery does not use a secondary sort field to accelerate + * querying if the primary sort field is a genuine sort field. + */ + @Test + public void testSortedSetRangeQueryDoesNotOptimizeWithActivePrimary(Random random) + throws IOException { + Directory dir = newDirectory(); + IndexWriterConfig config = new IndexWriterConfig().setCodec(getCodec(random)); + config.setIndexSort( + new Sort( + new SortField("primary", SortField.Type.STRING), // multiple values → effective primary + new SortField("secondary", SortField.Type.STRING))); + IndexWriter iw = new IndexWriter(dir, config); + for (int i = 0; i < 10; i++) { + String val = String.format(Locale.ROOT, "%03d", i); + Document doc = new Document(); + doc.add(SortedDocValuesField.indexedField("primary", newBytesRef(val))); // varying + doc.add(SortedDocValuesField.indexedField("secondary", newBytesRef(val))); // varying + iw.addDocument(doc); + } + iw.forceMerge(1); + iw.close(); + DirectoryReader reader = DirectoryReader.open(dir); + IndexSearcher searcher = new IndexSearcher(reader); + + // primary is active, so getPrimarySortField returns "primary", not "secondary". + // The query on "secondary" must fall back to the TwoPhaseIterator path. + Query query = + SortedSetDocValuesField.newSlowRangeQuery( + "secondary", newBytesRef("003"), newBytesRef("007"), true, true); + Weight weight = query.createWeight(searcher, ScoreMode.COMPLETE, 1.0f); + Scorer scorer = weight.scorer(reader.leaves().get(0)); + assertNotNull(scorer); + assertNotNull( + "TwoPhaseIterator fallback should be used when secondary is not the effective primary", + scorer.twoPhaseIterator()); + + reader.close(); + dir.close(); + } } diff --git a/lucene/core/src/test/org/apache/lucene/search/TestGetPrimarySortField.java b/lucene/core/src/test/org/apache/lucene/search/TestGetPrimarySortField.java new file mode 100644 index 000000000000..932b3af6a201 --- /dev/null +++ b/lucene/core/src/test/org/apache/lucene/search/TestGetPrimarySortField.java @@ -0,0 +1,282 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.search; + +import java.io.IOException; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.NumericDocValuesField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.NoMergePolicy; +import org.apache.lucene.store.Directory; +import org.apache.lucene.tests.util.LuceneTestCase; + +/** + * Tests for {@link Sort#getPrimarySortField(LeafReader)}, which returns the effective primary sort + * field for a segment, skipping fields that are no-ops because they have no values in the segment + * or have only a single distinct value according to their skip index. + */ +public class TestGetPrimarySortField extends LuceneTestCase { + + /** A reader with no index sort configured always returns null. */ + public void testNoIndexSort() throws IOException { + Directory dir = newDirectory(); + IndexWriter iw = new IndexWriter(dir, new IndexWriterConfig()); + Document doc = new Document(); + doc.add(NumericDocValuesField.indexedField("field", 42)); + iw.addDocument(doc); + iw.forceMerge(1); + iw.close(); + DirectoryReader reader = DirectoryReader.open(dir); + LeafReader leafReader = reader.leaves().get(0).reader(); + assertNull(Sort.getPrimarySortField(leafReader)); + reader.close(); + dir.close(); + } + + /** + * When the sort field has no DocValues skip index (plain DV field), there is no skipper to detect + * a single-value no-op, but the field exists in FieldInfos so it is returned as-is. + */ + public void testSortFieldWithoutSkipperIsReturned() throws IOException { + Directory dir = newDirectory(); + SortField sortField = new SortField("field", SortField.Type.LONG); + IndexWriter iw = + new IndexWriter(dir, new IndexWriterConfig().setIndexSort(new Sort(sortField))); + for (int i = 0; i < 10; i++) { + Document doc = new Document(); + doc.add(new NumericDocValuesField("field", i)); // plain DV, no skip index + iw.addDocument(doc); + } + iw.forceMerge(1); + iw.close(); + DirectoryReader reader = DirectoryReader.open(dir); + LeafReader leafReader = reader.leaves().get(0).reader(); + SortField result = Sort.getPrimarySortField(leafReader); + assertNotNull(result); + assertEquals("field", result.getField()); + reader.close(); + dir.close(); + } + + /** + * A sort field with a skip index and multiple distinct values is returned as-is: it is not a + * no-op because the skipper's min and max values differ. + */ + public void testSortFieldWithMultipleDistinctValuesIsReturned() throws IOException { + Directory dir = newDirectory(); + SortField sortField = new SortField("field", SortField.Type.LONG); + IndexWriter iw = + new IndexWriter(dir, new IndexWriterConfig().setIndexSort(new Sort(sortField))); + for (int i = 0; i < 10; i++) { + Document doc = new Document(); + doc.add(NumericDocValuesField.indexedField("field", i)); // indexed DV, skipper present + iw.addDocument(doc); + } + iw.forceMerge(1); + iw.close(); + DirectoryReader reader = DirectoryReader.open(dir); + LeafReader leafReader = reader.leaves().get(0).reader(); + SortField result = Sort.getPrimarySortField(leafReader); + assertNotNull(result); + assertEquals("field", result.getField()); + reader.close(); + dir.close(); + } + + /** + * When the primary sort field has no values at all in a segment, its FieldInfo is absent and it + * is treated as a no-op. The next sort field becomes the effective primary. + */ + public void testPrimaryWithNoValuesInSegmentIsSkipped() throws IOException { + Directory dir = newDirectory(); + SortField primary = new SortField("field1", SortField.Type.LONG); + SortField secondary = new SortField("field2", SortField.Type.LONG); + IndexWriter iw = + new IndexWriter(dir, new IndexWriterConfig().setIndexSort(new Sort(primary, secondary))); + for (int i = 0; i < 10; i++) { + Document doc = new Document(); + // field1: no doc values → FieldInfos has no entry for "field1" → skipped + doc.add(NumericDocValuesField.indexedField("field2", i)); + iw.addDocument(doc); + } + iw.forceMerge(1); + iw.close(); + DirectoryReader reader = DirectoryReader.open(dir); + LeafReader leafReader = reader.leaves().get(0).reader(); + SortField result = Sort.getPrimarySortField(leafReader); + assertNotNull(result); + assertEquals("field2", result.getField()); + reader.close(); + dir.close(); + } + + /** + * When the primary sort field has a skip index with a single distinct value (skipper min == max), + * it is a no-op and the next sort field becomes the effective primary. + */ + public void testSingleValuePrimaryIsSkippedToSecondary() throws IOException { + Directory dir = newDirectory(); + SortField primary = new SortField("field1", SortField.Type.LONG); + SortField secondary = new SortField("field2", SortField.Type.LONG); + IndexWriter iw = + new IndexWriter(dir, new IndexWriterConfig().setIndexSort(new Sort(primary, secondary))); + for (int i = 0; i < 10; i++) { + Document doc = new Document(); + doc.add(NumericDocValuesField.indexedField("field1", 42)); // constant → no-op + doc.add(NumericDocValuesField.indexedField("field2", i)); // multiple distinct values + iw.addDocument(doc); + } + iw.forceMerge(1); + iw.close(); + DirectoryReader reader = DirectoryReader.open(dir); + LeafReader leafReader = reader.leaves().get(0).reader(); + SortField result = Sort.getPrimarySortField(leafReader); + assertNotNull(result); + assertEquals("field2", result.getField()); + reader.close(); + dir.close(); + } + + /** + * When all sort fields have only a single distinct value in the segment, every field is a no-op + * and null is returned. + */ + public void testAllSingleValueFieldsReturnNull() throws IOException { + Directory dir = newDirectory(); + SortField primary = new SortField("field1", SortField.Type.LONG); + SortField secondary = new SortField("field2", SortField.Type.LONG); + IndexWriter iw = + new IndexWriter(dir, new IndexWriterConfig().setIndexSort(new Sort(primary, secondary))); + for (int i = 0; i < 10; i++) { + Document doc = new Document(); + doc.add(NumericDocValuesField.indexedField("field1", 42)); // constant + doc.add(NumericDocValuesField.indexedField("field2", 7)); // constant + iw.addDocument(doc); + } + iw.forceMerge(1); + iw.close(); + DirectoryReader reader = DirectoryReader.open(dir); + LeafReader leafReader = reader.leaves().get(0).reader(); + assertNull(Sort.getPrimarySortField(leafReader)); + reader.close(); + dir.close(); + } + + /** + * With NoMergePolicy ensuring a fixed three-segment layout, each leaf independently returns its + * own effective primary sort field: segment 0 has a real primary, segment 1 has an absent primary + * (no FieldInfo), and segment 2 has a constant primary (single distinct value). + */ + public void testPerSegmentEffectivePrimaryWithNoMergePolicy() throws IOException { + Directory dir = newDirectory(); + IndexWriterConfig iwc = new IndexWriterConfig(); + iwc.setMergePolicy(NoMergePolicy.INSTANCE); + iwc.setIndexSort( + new Sort( + new SortField("field1", SortField.Type.LONG), // primary + new SortField("field2", SortField.Type.LONG))); // secondary + IndexWriter iw = new IndexWriter(dir, iwc); + + // Segment 0: field1 has two distinct values → effective primary is field1 + Document doc = new Document(); + doc.add(NumericDocValuesField.indexedField("field1", 1)); + doc.add(NumericDocValuesField.indexedField("field2", 10)); + iw.addDocument(doc); + doc = new Document(); + doc.add(NumericDocValuesField.indexedField("field1", 2)); + doc.add(NumericDocValuesField.indexedField("field2", 20)); + iw.addDocument(doc); + iw.commit(); + + // Segment 1: field1 absent (no FieldInfo) → effective primary is field2 + doc = new Document(); + doc.add(NumericDocValuesField.indexedField("field2", 30)); + iw.addDocument(doc); + doc = new Document(); + doc.add(NumericDocValuesField.indexedField("field2", 31)); + iw.addDocument(doc); + iw.commit(); + + // Segment 2: field1 is constant (skipper min == max) → no-op, effective primary is field2 + doc = new Document(); + doc.add(NumericDocValuesField.indexedField("field1", 42)); + doc.add(NumericDocValuesField.indexedField("field2", 40)); + iw.addDocument(doc); + doc = new Document(); + doc.add(NumericDocValuesField.indexedField("field1", 42)); + doc.add(NumericDocValuesField.indexedField("field2", 50)); + iw.addDocument(doc); + iw.commit(); + + iw.close(); + + DirectoryReader reader = DirectoryReader.open(dir); + assertEquals(3, reader.leaves().size()); + + // Segment 0: field1 has multiple distinct values → effective primary is field1 + LeafReader leaf0 = reader.leaves().get(0).reader(); + SortField sf0 = Sort.getPrimarySortField(leaf0); + assertNotNull(sf0); + assertEquals("field1", sf0.getField()); + + // Segment 1: field1 absent → effective primary is field2 + LeafReader leaf1 = reader.leaves().get(1).reader(); + SortField sf1 = Sort.getPrimarySortField(leaf1); + assertNotNull(sf1); + assertEquals("field2", sf1.getField()); + + // Segment 2: field1 constant → no-op, effective primary is field2 + LeafReader leaf2 = reader.leaves().get(2).reader(); + SortField sf2 = Sort.getPrimarySortField(leaf2); + assertNotNull(sf2); + assertEquals("field2", sf2.getField()); + + reader.close(); + dir.close(); + } + + /** + * The reverse attribute of the effective primary sort field is correctly preserved when the + * initial primary sort field is skipped as a no-op. + */ + public void testSecondaryReverseAttributeIsPreserved() throws IOException { + Directory dir = newDirectory(); + SortField primary = new SortField("field1", SortField.Type.LONG, false); // ascending + SortField secondary = new SortField("field2", SortField.Type.LONG, true); // descending + IndexWriter iw = + new IndexWriter(dir, new IndexWriterConfig().setIndexSort(new Sort(primary, secondary))); + for (int i = 0; i < 10; i++) { + Document doc = new Document(); + doc.add(NumericDocValuesField.indexedField("field1", 42)); // constant → no-op + doc.add(NumericDocValuesField.indexedField("field2", i)); + iw.addDocument(doc); + } + iw.forceMerge(1); + iw.close(); + DirectoryReader reader = DirectoryReader.open(dir); + LeafReader leafReader = reader.leaves().get(0).reader(); + SortField result = Sort.getPrimarySortField(leafReader); + assertNotNull(result); + assertEquals("field2", result.getField()); + assertTrue("secondary sort field should be reversed", result.getReverse()); + reader.close(); + dir.close(); + } +} diff --git a/lucene/core/src/test/org/apache/lucene/search/TestIndexSortSortedNumericDocValuesRangeQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestIndexSortSortedNumericDocValuesRangeQuery.java index 4adb4a3cb3c1..69b667241c00 100644 --- a/lucene/core/src/test/org/apache/lucene/search/TestIndexSortSortedNumericDocValuesRangeQuery.java +++ b/lucene/core/src/test/org/apache/lucene/search/TestIndexSortSortedNumericDocValuesRangeQuery.java @@ -24,6 +24,7 @@ import org.apache.lucene.document.Field; import org.apache.lucene.document.IntPoint; import org.apache.lucene.document.LongPoint; +import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.document.SortedNumericDocValuesField; import org.apache.lucene.document.StringField; import org.apache.lucene.index.DirectoryReader; @@ -855,6 +856,91 @@ private void doTestRandomCountWithBkd(boolean reverse) throws Exception { dir.close(); } + /** + * Verifies that the binary-search optimization activates when the primary index sort field has a + * single constant value (detected as a no-op via its skip index), promoting the secondary field + * to effective primary and enabling direct doc-ID range iteration without a two-phase iterator. + */ + public void testNoOpPrimarySort() throws IOException { + Directory dir = newDirectory(); + IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random())); + Sort indexSort = + new Sort( + new SortField("field1", SortField.Type.LONG), // constant → no-op via skipper + new SortField("field2", SortField.Type.LONG)); // varying → effective primary + iwc.setIndexSort(indexSort); + RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc); + for (int i = 0; i < 10; i++) { + Document doc = new Document(); + doc.add(NumericDocValuesField.indexedField("field1", 42)); // constant; skip index present + doc.add(NumericDocValuesField.indexedField("field2", i)); // varying + iw.addDocument(doc); + } + iw.forceMerge(1); + DirectoryReader reader = iw.getReader(); + IndexSearcher searcher = newSearcher(reader); + iw.close(); + + // field1 has a single distinct value, so getPrimarySortField skips it and returns field2. + // The binary-search optimization should activate for the query on field2, producing a plain + // range iterator with no two-phase iterator. + Query fallback = SortedNumericDocValuesField.newSlowRangeQuery("field2", 3, 7); + Query query = new IndexSortSortedNumericDocValuesRangeQuery("field2", 3, 7, fallback); + Weight weight = query.createWeight(searcher, ScoreMode.TOP_SCORES, 1.0F); + for (LeafReaderContext context : searcher.getIndexReader().leaves()) { + Scorer scorer = weight.scorer(context); + assertNotNull(scorer); + assertNull( + "binary-search optimization should be active when primary sort is a no-op", + scorer.twoPhaseIterator()); + } + + reader.close(); + dir.close(); + } + + /** + * Verifies that the binary-search optimization activates when the primary index sort field has no + * values in the segment at all (detected as a no-op via FieldInfos), promoting the secondary + * field to effective primary. + */ + public void testMissingPrimarySort() throws IOException { + Directory dir = newDirectory(); + IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random())); + Sort indexSort = + new Sort( + new SortField("field1", SortField.Type.LONG), // absent → no-op via FieldInfos check + new SortField("field2", SortField.Type.LONG)); // varying → effective primary + iwc.setIndexSort(indexSort); + RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc); + for (int i = 0; i < 10; i++) { + Document doc = new Document(); + // field1 intentionally absent: no FieldInfo in segment → detected as no-op + doc.add(NumericDocValuesField.indexedField("field2", i)); + iw.addDocument(doc); + } + iw.forceMerge(1); + DirectoryReader reader = iw.getReader(); + IndexSearcher searcher = newSearcher(reader); + iw.close(); + + // field1 has no FieldInfo, so getPrimarySortField skips it and returns field2. + // The binary-search optimization should activate for the query on field2. + Query fallback = SortedNumericDocValuesField.newSlowRangeQuery("field2", 3, 7); + Query query = new IndexSortSortedNumericDocValuesRangeQuery("field2", 3, 7, fallback); + Weight weight = query.createWeight(searcher, ScoreMode.TOP_SCORES, 1.0F); + for (LeafReaderContext context : searcher.getIndexReader().leaves()) { + Scorer scorer = weight.scorer(context); + assertNotNull(scorer); + assertNull( + "binary-search optimization should be active when primary sort field has no values", + scorer.twoPhaseIterator()); + } + + reader.close(); + dir.close(); + } + private void addDocWithBkd(RandomIndexWriter indexWriter, String field, long value, int repeat) throws IOException { for (int i = 0; i < repeat; i++) {