Disable doc values

CHANGELOG.md

@@ -17,6 +17,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 * Add index operation listener to update translog source [#2629](https://github.com/opensearch-project/k-NN/pull/2629)
 * Add parent join support for faiss hnsw cagra [#2647](https://github.com/opensearch-project/k-NN/pull/2647)
 * [Remote Vector Index Build] Fix bug to support `COSINESIMIL` space type [#2627](https://github.com/opensearch-project/k-NN/pull/2627)
+* Disable doc value storage for vector field storage [#2646](https://github.com/opensearch-project/k-NN/pull/2646)
 ### Infrastructure
 * Add github action to run ITs against remote index builder [2620](https://github.com/opensearch-project/k-NN/pull/2620)
 ### Documentation
@@ -26,6 +27,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ### Refactoring
 * Switch derived source from field attributes to segment attribute [#2606](https://github.com/opensearch-project/k-NN/pull/2606)
 * Migrate derived source from filter to mask [#2612](https://github.com/opensearch-project/k-NN/pull/2612)
+* Consolidate MethodFieldMapper and LuceneFieldMapper into EngineFieldMapper [#2646](https://github.com/opensearch-project/k-NN/pull/2646)
 
 ## [Unreleased 2.x](https://github.com/opensearch-project/k-NN/compare/2.19...2.x)
 ### Features

src/main/java/org/opensearch/knn/index/mapper/EngineFieldMapper.java

@@ -0,0 +1,286 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.knn.index.mapper;
+
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.FieldType;
+import org.apache.lucene.index.DocValuesType;
+import org.apache.lucene.index.VectorEncoding;
+import org.opensearch.Version;
+import org.opensearch.common.Explicit;
+import org.opensearch.common.xcontent.XContentFactory;
+import org.opensearch.knn.index.DerivedKnnByteVectorField;
+import org.opensearch.knn.index.DerivedKnnFloatVectorField;
+import org.opensearch.knn.index.KNNVectorSimilarityFunction;
+import org.opensearch.knn.index.SpaceType;
+import org.opensearch.knn.index.VectorDataType;
+import org.opensearch.knn.index.VectorField;
+import org.opensearch.knn.index.engine.KNNEngine;
+import org.opensearch.knn.index.engine.KNNLibraryIndexingContext;
+import org.opensearch.knn.index.engine.KNNMethodConfigContext;
+import org.opensearch.knn.index.engine.KNNMethodContext;
+import org.opensearch.knn.index.engine.qframe.QuantizationConfig;
+import org.opensearch.knn.index.engine.qframe.QuantizationConfigParser;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+import static org.opensearch.knn.common.KNNConstants.DIMENSION;
+import static org.opensearch.knn.common.KNNConstants.KNN_ENGINE;
+import static org.opensearch.knn.common.KNNConstants.PARAMETERS;
+import static org.opensearch.knn.common.KNNConstants.QFRAMEWORK_CONFIG;
+import static org.opensearch.knn.common.KNNConstants.SPACE_TYPE;
+import static org.opensearch.knn.common.KNNConstants.VECTOR_DATA_TYPE_FIELD;
+import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.buildDocValuesFieldType;
+import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.createStoredFieldForByteVector;
+import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.createStoredFieldForFloatVector;
+
+/**
+ *  Field mapper for all supported engines.
+ */
+public class EngineFieldMapper extends KNNVectorFieldMapper {
+
+    private final FieldType vectorFieldType;
+    private final PerDimensionProcessor perDimensionProcessor;
+    private final PerDimensionValidator perDimensionValidator;
+    private final VectorValidator vectorValidator;
+    private final VectorTransformer vectorTransformer;
+    private final boolean isLuceneEngine;
+
+    public static EngineFieldMapper createFieldMapper(
+        String fullname,
+        String simpleName,
+        Map<String, String> metaValue,
+        KNNMethodConfigContext knnMethodConfigContext,
+        MultiFields multiFields,
+        CopyTo copyTo,
+        Explicit<Boolean> ignoreMalformed,
+        boolean stored,
+        boolean hasDocValues,
+        OriginalMappingParameters originalMappingParameters
+    ) {
+        KNNMethodContext methodContext = originalMappingParameters.getResolvedKnnMethodContext();
+        KNNLibraryIndexingContext libraryContext = methodContext.getKnnEngine()
+            .getKNNLibraryIndexingContext(methodContext, knnMethodConfigContext);
+        boolean isLuceneEngine = KNNEngine.LUCENE.equals(methodContext.getKnnEngine());
+
+        KNNVectorFieldType mappedFieldType = new KNNVectorFieldType(
+            fullname,
+            metaValue,
+            knnMethodConfigContext.getVectorDataType(),
+            new KNNMappingConfig() {
+                @Override
+                public Optional<KNNMethodContext> getKnnMethodContext() {
+                    return Optional.of(methodContext);
+                }
+
+                @Override
+                public int getDimension() {
+                    return knnMethodConfigContext.getDimension();
+                }
+
+                @Override
+                public Mode getMode() {
+                    return Mode.fromName(originalMappingParameters.getMode());
+                }
+
+                @Override
+                public CompressionLevel getCompressionLevel() {
+                    return knnMethodConfigContext.getCompressionLevel();
+                }
+
+                @Override
+                public Version getIndexCreatedVersion() {
+                    return knnMethodConfigContext.getVersionCreated();
+                }
+
+                @Override
+                public QuantizationConfig getQuantizationConfig() {
+                    return Optional.ofNullable(libraryContext)
+                        .map(KNNLibraryIndexingContext::getQuantizationConfig)
+                        .orElse(QuantizationConfig.EMPTY);
+                }
+
+                @Override
+                public KNNLibraryIndexingContext getKnnLibraryIndexingContext() {
+                    return libraryContext;
+                }
+            }
+        );
+
+        return new EngineFieldMapper(
+            simpleName,
+            mappedFieldType,
+            multiFields,
+            copyTo,
+            ignoreMalformed,
+            stored,
+            hasDocValues,
+            knnMethodConfigContext,
+            originalMappingParameters,
+            isLuceneEngine
+        );
+    }
+
+    private EngineFieldMapper(
+        String name,
+        KNNVectorFieldType mappedFieldType,
+        MultiFields multiFields,
+        CopyTo copyTo,
+        Explicit<Boolean> ignoreMalformed,
+        boolean stored,
+        boolean hasDocValues,
+        KNNMethodConfigContext knnMethodConfigContext,
+        OriginalMappingParameters originalMappingParameters,
+        boolean isLuceneEngine
+    ) {
+        super(
+            name,
+            mappedFieldType,
+            multiFields,
+            copyTo,
+            ignoreMalformed,
+            stored,
+            hasDocValues,
+            knnMethodConfigContext.getVersionCreated(),
+            originalMappingParameters
+        );
+        this.isLuceneEngine = isLuceneEngine;
+        updateEngineStats();
+        KNNMappingConfig knnMappingConfig = mappedFieldType.getKnnMappingConfig();
+        VectorDataType vectorDataType = mappedFieldType.getVectorDataType();
+        KNNMethodContext resolvedKnnMethodContext = originalMappingParameters.getResolvedKnnMethodContext();
+
+        final KNNVectorSimilarityFunction knnVectorSimilarityFunction = resolvedKnnMethodContext.getSpaceType()
+            .getKnnVectorSimilarityFunction();
+        KNNLibraryIndexingContext knnLibraryIndexingContext = resolvedKnnMethodContext.getKnnEngine()
+            .getKNNLibraryIndexingContext(resolvedKnnMethodContext, knnMethodConfigContext);
+
+        // LuceneFieldMapper attributes
+        if (this.isLuceneEngine) {
+            this.fieldType = vectorDataType.createKnnVectorFieldType(knnMappingConfig.getDimension(), knnVectorSimilarityFunction);
+
+            if (this.hasDocValues) {
+                this.vectorFieldType = buildDocValuesFieldType(resolvedKnnMethodContext.getKnnEngine());
+            } else {
+                this.vectorFieldType = null;
+            }
+            this.vectorTransformer = null;
+        } else {
+            // MethodFieldMapper attributes
+            this.vectorFieldType = null;
+            this.useLuceneBasedVectorField = KNNVectorFieldMapperUtil.useLuceneKNNVectorsFormat(indexCreatedVersion);
+            KNNEngine knnEngine = resolvedKnnMethodContext.getKnnEngine();
+            QuantizationConfig quantizationConfig = knnLibraryIndexingContext.getQuantizationConfig();
+            this.fieldType = new FieldType(KNNVectorFieldMapper.Defaults.FIELD_TYPE);
+            this.fieldType.putAttribute(DIMENSION, String.valueOf(knnMappingConfig.getDimension()));
+            this.fieldType.putAttribute(SPACE_TYPE, resolvedKnnMethodContext.getSpaceType().getValue());
+            // Conditionally add quantization config
+            if (quantizationConfig != null && quantizationConfig != QuantizationConfig.EMPTY) {
+                this.fieldType.putAttribute(QFRAMEWORK_CONFIG, QuantizationConfigParser.toCsv(quantizationConfig));
+            }
+
+            this.fieldType.putAttribute(VECTOR_DATA_TYPE_FIELD, vectorDataType.getValue());
+            this.fieldType.putAttribute(KNN_ENGINE, knnEngine.getName());
+            try {
+                this.fieldType.putAttribute(
+                    PARAMETERS,
+                    XContentFactory.jsonBuilder().map(knnLibraryIndexingContext.getLibraryParameters()).toString()
+                );
+            } catch (IOException ioe) {
+                throw new RuntimeException(String.format("Unable to create KNNVectorFieldMapper: %s", ioe), ioe);
+            }
+
+            if (useLuceneBasedVectorField) {
+                int adjustedDimension = mappedFieldType.vectorDataType == VectorDataType.BINARY
+                    ? knnMappingConfig.getDimension() / 8
+                    : knnMappingConfig.getDimension();
+                final VectorEncoding encoding = mappedFieldType.vectorDataType == VectorDataType.FLOAT
+                    ? VectorEncoding.FLOAT32
+                    : VectorEncoding.BYTE;
+                fieldType.setVectorAttributes(
+                    adjustedDimension,
+                    encoding,
+                    SpaceType.DEFAULT.getKnnVectorSimilarityFunction().getVectorSimilarityFunction()
+                );
+            } else {
+                fieldType.setDocValuesType(DocValuesType.BINARY);
+            }
+
+            this.fieldType.freeze();
+            this.vectorTransformer = knnLibraryIndexingContext.getVectorTransformer();
+        }
+
+        // Common Attributes
+        this.perDimensionProcessor = knnLibraryIndexingContext.getPerDimensionProcessor();
+        this.perDimensionValidator = knnLibraryIndexingContext.getPerDimensionValidator();
+        this.vectorValidator = knnLibraryIndexingContext.getVectorValidator();
+    }
+
+    @Override
+    protected List<Field> getFieldsForFloatVector(final float[] array, boolean isDerivedSourceEnabled) {
+        if (this.isLuceneEngine) {
+            final List<Field> fields = new ArrayList<>();
+            fields.add(new DerivedKnnFloatVectorField(name(), array, fieldType, isDerivedSourceEnabled));
+            if (hasDocValues && vectorFieldType != null) {
+                fields.add(new VectorField(name(), array, vectorFieldType));
+            }
+            if (stored) {
+                fields.add(createStoredFieldForFloatVector(name(), array));
+            }
+            return fields;
+        }
+        return super.getFieldsForFloatVector(array, isDerivedSourceEnabled);
+    }
+
+    @Override
+    protected List<Field> getFieldsForByteVector(final byte[] array, boolean isDerivedSourceEnabled) {
+        if (this.isLuceneEngine) {
+            final List<Field> fields = new ArrayList<>();
+            fields.add(new DerivedKnnByteVectorField(name(), array, fieldType, isDerivedSourceEnabled));
+            if (hasDocValues && vectorFieldType != null) {
+                fields.add(new VectorField(name(), array, vectorFieldType));
+            }
+            if (stored) {
+                fields.add(createStoredFieldForByteVector(name(), array));
+            }
+            return fields;
+        }
+        return super.getFieldsForByteVector(array, isDerivedSourceEnabled);
+    }
+
+    @Override
+    protected VectorValidator getVectorValidator() {
+        return vectorValidator;
+    }
+
+    @Override
+    protected PerDimensionValidator getPerDimensionValidator() {
+        return perDimensionValidator;
+    }
+
+    @Override
+    protected PerDimensionProcessor getPerDimensionProcessor() {
+        return perDimensionProcessor;
+    }
+
+    @Override
+    protected VectorTransformer getVectorTransformer() {
+        if (isLuceneEngine) {
+            return super.getVectorTransformer();
+        }
+        return vectorTransformer;
+    }
+
+    @Override
+    void updateEngineStats() {
+        Optional.ofNullable(originalMappingParameters)
+            .ifPresent(params -> params.getResolvedKnnMethodContext().getKnnEngine().setInitialized(true));
+    }
+}

src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java

@@ -95,7 +95,7 @@ public static class Builder extends ParametrizedFieldMapper.Builder {
         protected Boolean ignoreMalformed;
 
         protected final Parameter<Boolean> stored = Parameter.storeParam(m -> toType(m).stored, false);
-        protected final Parameter<Boolean> hasDocValues = Parameter.docValuesParam(m -> toType(m).hasDocValues, true);
+        protected Parameter<Boolean> hasDocValues;
         protected final Parameter<Integer> dimension = new Parameter<>(
             KNNConstants.DIMENSION,
             false,
@@ -216,6 +216,22 @@ public Builder(
             this.indexCreatedVersion = indexCreatedVersion;
             this.knnMethodConfigContext = knnMethodConfigContext;
             this.originalParameters = originalParameters;
+            /*
+             * For indices created on or after OpenSearch 3.0.0, docValues
+             * defaults to false when not explicitly configured. This reduces storage
+             * overhead and improves indexing performance for k-NN vector fields.
+             * Changing the default value breaks BwC for existing indices on a cluster.
+             *
+             * Behavior matrix:
+             * - Index < 3.0.0: Uses original default value
+             * - Index >= 3.0.0, docValues not configured: Sets to false
+             * - Any version, docValues explicitly configured: Respects configured value
+             */
+            if (indexCreatedVersion.before(Version.V_3_0_0)) {
+                hasDocValues = Parameter.docValuesParam(m -> toType(m).hasDocValues, true);
+            } else {
+                hasDocValues = Parameter.docValuesParam(m -> toType(m).hasDocValues, false);
+            }
         }
 
         @Override
@@ -273,10 +289,16 @@ public KNNVectorFieldMapper build(BuilderContext context) {
                 );
             }
 
-            // return FlatVectorFieldMapper only for indices that are created on or after 2.17.0, for others, use either LuceneFieldMapper
-            // or
-            // MethodFieldMapper to maintain backwards compatibility
+            // return FlatVectorFieldMapper only for indices that are created on or after 2.17.0, for others, use
+            // EngineFieldMapper to maintain backwards compatibility
             if (originalParameters.getResolvedKnnMethodContext() == null && indexCreatedVersion.onOrAfter(Version.V_2_17_0)) {
+                // Prior to 3.0.0, hasDocValues defaulted to false. However, FlatVectorFieldMapper requires
+                // hasDocValues to be true to maintain proper functionality for vector search operations.
+                // For indices created on or after 3.0.0, we automatically set hasDocValues to true if not
+                // explicitly configured to ensure consistent behavior.
+                if (indexCreatedVersion.onOrAfter(Version.V_3_0_0) && hasDocValues.isConfigured() == false) {
+                    hasDocValues = Parameter.docValuesParam(m -> toType(m).hasDocValues, true);
+                }
                 return FlatVectorFieldMapper.createFieldMapper(
                     buildFullName(context),
                     name,
@@ -295,28 +317,7 @@ public KNNVectorFieldMapper build(BuilderContext context) {
                 );
             }
 
-            if (originalParameters.getResolvedKnnMethodContext().getKnnEngine() == KNNEngine.LUCENE) {
-                log.debug(String.format(Locale.ROOT, "Use [LuceneFieldMapper] mapper for field [%s]", name));
-                LuceneFieldMapper.CreateLuceneFieldMapperInput createLuceneFieldMapperInput = LuceneFieldMapper.CreateLuceneFieldMapperInput
-                    .builder()
-                    .name(name)
-                    .multiFields(multiFieldsBuilder)
-                    .copyTo(copyToBuilder)
-                    .ignoreMalformed(ignoreMalformed)
-                    .stored(stored.getValue())
-                    .hasDocValues(hasDocValues.getValue())
-                    .originalKnnMethodContext(knnMethodContext.get())
-                    .build();
-                return LuceneFieldMapper.createFieldMapper(
-                    buildFullName(context),
-                    metaValue,
-                    knnMethodConfigContext,
-                    createLuceneFieldMapperInput,
-                    originalParameters
-                );
-            }
-
-            return MethodFieldMapper.createFieldMapper(
+            return EngineFieldMapper.createFieldMapper(
                 buildFullName(context),
                 name,
                 metaValue,
@@ -325,7 +326,7 @@ public KNNVectorFieldMapper build(BuilderContext context) {
                 copyToBuilder,
                 ignoreMalformed,
                 stored.getValue(),
-                hasDocValues.getValue(),
+                hasDocValues.get(),
                 originalParameters
             );
         }