diff --git a/CHANGELOG.md b/CHANGELOG.md index 9b7957dbc7..d686c3b14b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), * Add index operation listener to update translog source [#2629](https://github.com/opensearch-project/k-NN/pull/2629) * Add parent join support for faiss hnsw cagra [#2647](https://github.com/opensearch-project/k-NN/pull/2647) * [Remote Vector Index Build] Fix bug to support `COSINESIMIL` space type [#2627](https://github.com/opensearch-project/k-NN/pull/2627) +* Disable doc value storage for vector field storage [#2646](https://github.com/opensearch-project/k-NN/pull/2646) ### Infrastructure * Add github action to run ITs against remote index builder [2620](https://github.com/opensearch-project/k-NN/pull/2620) ### Documentation @@ -26,6 +27,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Refactoring * Switch derived source from field attributes to segment attribute [#2606](https://github.com/opensearch-project/k-NN/pull/2606) * Migrate derived source from filter to mask [#2612](https://github.com/opensearch-project/k-NN/pull/2612) +* Consolidate MethodFieldMapper and LuceneFieldMapper into EngineFieldMapper [#2646](https://github.com/opensearch-project/k-NN/pull/2646) ## [Unreleased 2.x](https://github.com/opensearch-project/k-NN/compare/2.19...2.x) ### Features diff --git a/src/main/java/org/opensearch/knn/index/mapper/EngineFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/EngineFieldMapper.java new file mode 100644 index 0000000000..baec510d84 --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/mapper/EngineFieldMapper.java @@ -0,0 +1,286 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index.mapper; + +import org.apache.lucene.document.Field; +import org.apache.lucene.document.FieldType; +import org.apache.lucene.index.DocValuesType; +import org.apache.lucene.index.VectorEncoding; +import org.opensearch.Version; +import org.opensearch.common.Explicit; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.knn.index.DerivedKnnByteVectorField; +import org.opensearch.knn.index.DerivedKnnFloatVectorField; +import org.opensearch.knn.index.KNNVectorSimilarityFunction; +import org.opensearch.knn.index.SpaceType; +import org.opensearch.knn.index.VectorDataType; +import org.opensearch.knn.index.VectorField; +import org.opensearch.knn.index.engine.KNNEngine; +import org.opensearch.knn.index.engine.KNNLibraryIndexingContext; +import org.opensearch.knn.index.engine.KNNMethodConfigContext; +import org.opensearch.knn.index.engine.KNNMethodContext; +import org.opensearch.knn.index.engine.qframe.QuantizationConfig; +import org.opensearch.knn.index.engine.qframe.QuantizationConfigParser; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static org.opensearch.knn.common.KNNConstants.DIMENSION; +import static org.opensearch.knn.common.KNNConstants.KNN_ENGINE; +import static org.opensearch.knn.common.KNNConstants.PARAMETERS; +import static org.opensearch.knn.common.KNNConstants.QFRAMEWORK_CONFIG; +import static org.opensearch.knn.common.KNNConstants.SPACE_TYPE; +import static org.opensearch.knn.common.KNNConstants.VECTOR_DATA_TYPE_FIELD; +import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.buildDocValuesFieldType; +import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.createStoredFieldForByteVector; +import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.createStoredFieldForFloatVector; + +/** + * Field mapper for all supported engines. + */ +public class EngineFieldMapper extends KNNVectorFieldMapper { + + private final FieldType vectorFieldType; + private final PerDimensionProcessor perDimensionProcessor; + private final PerDimensionValidator perDimensionValidator; + private final VectorValidator vectorValidator; + private final VectorTransformer vectorTransformer; + private final boolean isLuceneEngine; + + public static EngineFieldMapper createFieldMapper( + String fullname, + String simpleName, + Map metaValue, + KNNMethodConfigContext knnMethodConfigContext, + MultiFields multiFields, + CopyTo copyTo, + Explicit ignoreMalformed, + boolean stored, + boolean hasDocValues, + OriginalMappingParameters originalMappingParameters + ) { + KNNMethodContext methodContext = originalMappingParameters.getResolvedKnnMethodContext(); + KNNLibraryIndexingContext libraryContext = methodContext.getKnnEngine() + .getKNNLibraryIndexingContext(methodContext, knnMethodConfigContext); + boolean isLuceneEngine = KNNEngine.LUCENE.equals(methodContext.getKnnEngine()); + + KNNVectorFieldType mappedFieldType = new KNNVectorFieldType( + fullname, + metaValue, + knnMethodConfigContext.getVectorDataType(), + new KNNMappingConfig() { + @Override + public Optional getKnnMethodContext() { + return Optional.of(methodContext); + } + + @Override + public int getDimension() { + return knnMethodConfigContext.getDimension(); + } + + @Override + public Mode getMode() { + return Mode.fromName(originalMappingParameters.getMode()); + } + + @Override + public CompressionLevel getCompressionLevel() { + return knnMethodConfigContext.getCompressionLevel(); + } + + @Override + public Version getIndexCreatedVersion() { + return knnMethodConfigContext.getVersionCreated(); + } + + @Override + public QuantizationConfig getQuantizationConfig() { + return Optional.ofNullable(libraryContext) + .map(KNNLibraryIndexingContext::getQuantizationConfig) + .orElse(QuantizationConfig.EMPTY); + } + + @Override + public KNNLibraryIndexingContext getKnnLibraryIndexingContext() { + return libraryContext; + } + } + ); + + return new EngineFieldMapper( + simpleName, + mappedFieldType, + multiFields, + copyTo, + ignoreMalformed, + stored, + hasDocValues, + knnMethodConfigContext, + originalMappingParameters, + isLuceneEngine + ); + } + + private EngineFieldMapper( + String name, + KNNVectorFieldType mappedFieldType, + MultiFields multiFields, + CopyTo copyTo, + Explicit ignoreMalformed, + boolean stored, + boolean hasDocValues, + KNNMethodConfigContext knnMethodConfigContext, + OriginalMappingParameters originalMappingParameters, + boolean isLuceneEngine + ) { + super( + name, + mappedFieldType, + multiFields, + copyTo, + ignoreMalformed, + stored, + hasDocValues, + knnMethodConfigContext.getVersionCreated(), + originalMappingParameters + ); + this.isLuceneEngine = isLuceneEngine; + updateEngineStats(); + KNNMappingConfig knnMappingConfig = mappedFieldType.getKnnMappingConfig(); + VectorDataType vectorDataType = mappedFieldType.getVectorDataType(); + KNNMethodContext resolvedKnnMethodContext = originalMappingParameters.getResolvedKnnMethodContext(); + + final KNNVectorSimilarityFunction knnVectorSimilarityFunction = resolvedKnnMethodContext.getSpaceType() + .getKnnVectorSimilarityFunction(); + KNNLibraryIndexingContext knnLibraryIndexingContext = resolvedKnnMethodContext.getKnnEngine() + .getKNNLibraryIndexingContext(resolvedKnnMethodContext, knnMethodConfigContext); + + // LuceneFieldMapper attributes + if (this.isLuceneEngine) { + this.fieldType = vectorDataType.createKnnVectorFieldType(knnMappingConfig.getDimension(), knnVectorSimilarityFunction); + + if (this.hasDocValues) { + this.vectorFieldType = buildDocValuesFieldType(resolvedKnnMethodContext.getKnnEngine()); + } else { + this.vectorFieldType = null; + } + this.vectorTransformer = null; + } else { + // MethodFieldMapper attributes + this.vectorFieldType = null; + this.useLuceneBasedVectorField = KNNVectorFieldMapperUtil.useLuceneKNNVectorsFormat(indexCreatedVersion); + KNNEngine knnEngine = resolvedKnnMethodContext.getKnnEngine(); + QuantizationConfig quantizationConfig = knnLibraryIndexingContext.getQuantizationConfig(); + this.fieldType = new FieldType(KNNVectorFieldMapper.Defaults.FIELD_TYPE); + this.fieldType.putAttribute(DIMENSION, String.valueOf(knnMappingConfig.getDimension())); + this.fieldType.putAttribute(SPACE_TYPE, resolvedKnnMethodContext.getSpaceType().getValue()); + // Conditionally add quantization config + if (quantizationConfig != null && quantizationConfig != QuantizationConfig.EMPTY) { + this.fieldType.putAttribute(QFRAMEWORK_CONFIG, QuantizationConfigParser.toCsv(quantizationConfig)); + } + + this.fieldType.putAttribute(VECTOR_DATA_TYPE_FIELD, vectorDataType.getValue()); + this.fieldType.putAttribute(KNN_ENGINE, knnEngine.getName()); + try { + this.fieldType.putAttribute( + PARAMETERS, + XContentFactory.jsonBuilder().map(knnLibraryIndexingContext.getLibraryParameters()).toString() + ); + } catch (IOException ioe) { + throw new RuntimeException(String.format("Unable to create KNNVectorFieldMapper: %s", ioe), ioe); + } + + if (useLuceneBasedVectorField) { + int adjustedDimension = mappedFieldType.vectorDataType == VectorDataType.BINARY + ? knnMappingConfig.getDimension() / 8 + : knnMappingConfig.getDimension(); + final VectorEncoding encoding = mappedFieldType.vectorDataType == VectorDataType.FLOAT + ? VectorEncoding.FLOAT32 + : VectorEncoding.BYTE; + fieldType.setVectorAttributes( + adjustedDimension, + encoding, + SpaceType.DEFAULT.getKnnVectorSimilarityFunction().getVectorSimilarityFunction() + ); + } else { + fieldType.setDocValuesType(DocValuesType.BINARY); + } + + this.fieldType.freeze(); + this.vectorTransformer = knnLibraryIndexingContext.getVectorTransformer(); + } + + // Common Attributes + this.perDimensionProcessor = knnLibraryIndexingContext.getPerDimensionProcessor(); + this.perDimensionValidator = knnLibraryIndexingContext.getPerDimensionValidator(); + this.vectorValidator = knnLibraryIndexingContext.getVectorValidator(); + } + + @Override + protected List getFieldsForFloatVector(final float[] array, boolean isDerivedSourceEnabled) { + if (this.isLuceneEngine) { + final List fields = new ArrayList<>(); + fields.add(new DerivedKnnFloatVectorField(name(), array, fieldType, isDerivedSourceEnabled)); + if (hasDocValues && vectorFieldType != null) { + fields.add(new VectorField(name(), array, vectorFieldType)); + } + if (stored) { + fields.add(createStoredFieldForFloatVector(name(), array)); + } + return fields; + } + return super.getFieldsForFloatVector(array, isDerivedSourceEnabled); + } + + @Override + protected List getFieldsForByteVector(final byte[] array, boolean isDerivedSourceEnabled) { + if (this.isLuceneEngine) { + final List fields = new ArrayList<>(); + fields.add(new DerivedKnnByteVectorField(name(), array, fieldType, isDerivedSourceEnabled)); + if (hasDocValues && vectorFieldType != null) { + fields.add(new VectorField(name(), array, vectorFieldType)); + } + if (stored) { + fields.add(createStoredFieldForByteVector(name(), array)); + } + return fields; + } + return super.getFieldsForByteVector(array, isDerivedSourceEnabled); + } + + @Override + protected VectorValidator getVectorValidator() { + return vectorValidator; + } + + @Override + protected PerDimensionValidator getPerDimensionValidator() { + return perDimensionValidator; + } + + @Override + protected PerDimensionProcessor getPerDimensionProcessor() { + return perDimensionProcessor; + } + + @Override + protected VectorTransformer getVectorTransformer() { + if (isLuceneEngine) { + return super.getVectorTransformer(); + } + return vectorTransformer; + } + + @Override + void updateEngineStats() { + Optional.ofNullable(originalMappingParameters) + .ifPresent(params -> params.getResolvedKnnMethodContext().getKnnEngine().setInitialized(true)); + } +} diff --git a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java index 8506294966..ab62d9527a 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java +++ b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapper.java @@ -95,7 +95,7 @@ public static class Builder extends ParametrizedFieldMapper.Builder { protected Boolean ignoreMalformed; protected final Parameter stored = Parameter.storeParam(m -> toType(m).stored, false); - protected final Parameter hasDocValues = Parameter.docValuesParam(m -> toType(m).hasDocValues, true); + protected Parameter hasDocValues; protected final Parameter dimension = new Parameter<>( KNNConstants.DIMENSION, false, @@ -216,6 +216,22 @@ public Builder( this.indexCreatedVersion = indexCreatedVersion; this.knnMethodConfigContext = knnMethodConfigContext; this.originalParameters = originalParameters; + /* + * For indices created on or after OpenSearch 3.0.0, docValues + * defaults to false when not explicitly configured. This reduces storage + * overhead and improves indexing performance for k-NN vector fields. + * Changing the default value breaks BwC for existing indices on a cluster. + * + * Behavior matrix: + * - Index < 3.0.0: Uses original default value + * - Index >= 3.0.0, docValues not configured: Sets to false + * - Any version, docValues explicitly configured: Respects configured value + */ + if (indexCreatedVersion.before(Version.V_3_0_0)) { + hasDocValues = Parameter.docValuesParam(m -> toType(m).hasDocValues, true); + } else { + hasDocValues = Parameter.docValuesParam(m -> toType(m).hasDocValues, false); + } } @Override @@ -273,10 +289,16 @@ public KNNVectorFieldMapper build(BuilderContext context) { ); } - // return FlatVectorFieldMapper only for indices that are created on or after 2.17.0, for others, use either LuceneFieldMapper - // or - // MethodFieldMapper to maintain backwards compatibility + // return FlatVectorFieldMapper only for indices that are created on or after 2.17.0, for others, use + // EngineFieldMapper to maintain backwards compatibility if (originalParameters.getResolvedKnnMethodContext() == null && indexCreatedVersion.onOrAfter(Version.V_2_17_0)) { + // Prior to 3.0.0, hasDocValues defaulted to false. However, FlatVectorFieldMapper requires + // hasDocValues to be true to maintain proper functionality for vector search operations. + // For indices created on or after 3.0.0, we automatically set hasDocValues to true if not + // explicitly configured to ensure consistent behavior. + if (indexCreatedVersion.onOrAfter(Version.V_3_0_0) && hasDocValues.isConfigured() == false) { + hasDocValues = Parameter.docValuesParam(m -> toType(m).hasDocValues, true); + } return FlatVectorFieldMapper.createFieldMapper( buildFullName(context), name, @@ -295,28 +317,7 @@ public KNNVectorFieldMapper build(BuilderContext context) { ); } - if (originalParameters.getResolvedKnnMethodContext().getKnnEngine() == KNNEngine.LUCENE) { - log.debug(String.format(Locale.ROOT, "Use [LuceneFieldMapper] mapper for field [%s]", name)); - LuceneFieldMapper.CreateLuceneFieldMapperInput createLuceneFieldMapperInput = LuceneFieldMapper.CreateLuceneFieldMapperInput - .builder() - .name(name) - .multiFields(multiFieldsBuilder) - .copyTo(copyToBuilder) - .ignoreMalformed(ignoreMalformed) - .stored(stored.getValue()) - .hasDocValues(hasDocValues.getValue()) - .originalKnnMethodContext(knnMethodContext.get()) - .build(); - return LuceneFieldMapper.createFieldMapper( - buildFullName(context), - metaValue, - knnMethodConfigContext, - createLuceneFieldMapperInput, - originalParameters - ); - } - - return MethodFieldMapper.createFieldMapper( + return EngineFieldMapper.createFieldMapper( buildFullName(context), name, metaValue, @@ -325,7 +326,7 @@ public KNNVectorFieldMapper build(BuilderContext context) { copyToBuilder, ignoreMalformed, stored.getValue(), - hasDocValues.getValue(), + hasDocValues.get(), originalParameters ); } diff --git a/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java deleted file mode 100644 index 4a2561785d..0000000000 --- a/src/main/java/org/opensearch/knn/index/mapper/LuceneFieldMapper.java +++ /dev/null @@ -1,192 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.knn.index.mapper; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Optional; - -import lombok.AllArgsConstructor; -import lombok.Getter; -import lombok.NonNull; -import org.apache.lucene.document.Field; -import org.apache.lucene.document.FieldType; -import org.opensearch.Version; -import org.opensearch.common.Explicit; -import org.opensearch.knn.index.DerivedKnnByteVectorField; -import org.opensearch.knn.index.DerivedKnnFloatVectorField; -import org.opensearch.knn.index.KNNVectorSimilarityFunction; -import org.opensearch.knn.index.VectorDataType; -import org.opensearch.knn.index.VectorField; -import org.opensearch.knn.index.engine.KNNEngine; -import org.opensearch.knn.index.engine.KNNLibraryIndexingContext; -import org.opensearch.knn.index.engine.KNNMethodConfigContext; -import org.opensearch.knn.index.engine.KNNMethodContext; - -import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.createStoredFieldForByteVector; -import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.createStoredFieldForFloatVector; -import static org.opensearch.knn.index.mapper.KNNVectorFieldMapperUtil.buildDocValuesFieldType; - -/** - * Field mapper for case when Lucene has been set as an engine. - */ -public class LuceneFieldMapper extends KNNVectorFieldMapper { - - /** FieldType used for initializing VectorField, which is used for creating binary doc values. **/ - private final FieldType vectorFieldType; - - private final PerDimensionProcessor perDimensionProcessor; - private final PerDimensionValidator perDimensionValidator; - private final VectorValidator vectorValidator; - - static LuceneFieldMapper createFieldMapper( - String fullname, - Map metaValue, - KNNMethodConfigContext knnMethodConfigContext, - CreateLuceneFieldMapperInput createLuceneFieldMapperInput, - OriginalMappingParameters originalMappingParameters - ) { - final KNNVectorFieldType mappedFieldType = new KNNVectorFieldType( - fullname, - metaValue, - knnMethodConfigContext.getVectorDataType(), - new KNNMappingConfig() { - @Override - public Optional getKnnMethodContext() { - return Optional.of(originalMappingParameters.getResolvedKnnMethodContext()); - } - - @Override - public int getDimension() { - return knnMethodConfigContext.getDimension(); - } - - @Override - public Mode getMode() { - return knnMethodConfigContext.getMode(); - } - - @Override - public CompressionLevel getCompressionLevel() { - return knnMethodConfigContext.getCompressionLevel(); - } - - @Override - public Version getIndexCreatedVersion() { - return knnMethodConfigContext.getVersionCreated(); - } - } - ); - - return new LuceneFieldMapper(mappedFieldType, createLuceneFieldMapperInput, knnMethodConfigContext, originalMappingParameters); - } - - private LuceneFieldMapper( - final KNNVectorFieldType mappedFieldType, - final CreateLuceneFieldMapperInput input, - KNNMethodConfigContext knnMethodConfigContext, - OriginalMappingParameters originalMappingParameters - ) { - super( - input.getName(), - mappedFieldType, - input.getMultiFields(), - input.getCopyTo(), - input.getIgnoreMalformed(), - input.isStored(), - input.isHasDocValues(), - knnMethodConfigContext.getVersionCreated(), - originalMappingParameters - ); - KNNMappingConfig knnMappingConfig = mappedFieldType.getKnnMappingConfig(); - KNNMethodContext resolvedKnnMethodContext = originalMappingParameters.getResolvedKnnMethodContext(); - VectorDataType vectorDataType = mappedFieldType.getVectorDataType(); - - final KNNVectorSimilarityFunction knnVectorSimilarityFunction = resolvedKnnMethodContext.getSpaceType() - .getKnnVectorSimilarityFunction(); - - this.fieldType = vectorDataType.createKnnVectorFieldType(knnMappingConfig.getDimension(), knnVectorSimilarityFunction); - - if (this.hasDocValues) { - this.vectorFieldType = buildDocValuesFieldType(resolvedKnnMethodContext.getKnnEngine()); - } else { - this.vectorFieldType = null; - } - KNNLibraryIndexingContext knnLibraryIndexingContext = resolvedKnnMethodContext.getKnnEngine() - .getKNNLibraryIndexingContext(resolvedKnnMethodContext, knnMethodConfigContext); - this.perDimensionProcessor = knnLibraryIndexingContext.getPerDimensionProcessor(); - this.perDimensionValidator = knnLibraryIndexingContext.getPerDimensionValidator(); - this.vectorValidator = knnLibraryIndexingContext.getVectorValidator(); - } - - @Override - protected List getFieldsForFloatVector(final float[] array, boolean isDerivedSourceEnabled) { - final List fieldsToBeAdded = new ArrayList<>(); - fieldsToBeAdded.add(new DerivedKnnFloatVectorField(name(), array, fieldType, isDerivedSourceEnabled)); - - if (hasDocValues && vectorFieldType != null) { - fieldsToBeAdded.add(new VectorField(name(), array, vectorFieldType)); - } - - if (this.stored) { - fieldsToBeAdded.add(createStoredFieldForFloatVector(name(), array)); - } - return fieldsToBeAdded; - } - - @Override - protected List getFieldsForByteVector(final byte[] array, boolean isDerivedSourceEnabled) { - final List fieldsToBeAdded = new ArrayList<>(); - fieldsToBeAdded.add(new DerivedKnnByteVectorField(name(), array, fieldType, isDerivedSourceEnabled)); - - if (hasDocValues && vectorFieldType != null) { - fieldsToBeAdded.add(new VectorField(name(), array, vectorFieldType)); - } - - if (this.stored) { - fieldsToBeAdded.add(createStoredFieldForByteVector(name(), array)); - } - return fieldsToBeAdded; - } - - @Override - protected VectorValidator getVectorValidator() { - return vectorValidator; - } - - @Override - protected PerDimensionValidator getPerDimensionValidator() { - return perDimensionValidator; - } - - @Override - protected PerDimensionProcessor getPerDimensionProcessor() { - return perDimensionProcessor; - } - - @Override - void updateEngineStats() { - KNNEngine.LUCENE.setInitialized(true); - } - - @AllArgsConstructor - @lombok.Builder - @Getter - static class CreateLuceneFieldMapperInput { - @NonNull - String name; - @NonNull - MultiFields multiFields; - @NonNull - CopyTo copyTo; - @NonNull - Explicit ignoreMalformed; - boolean stored; - boolean hasDocValues; - KNNMethodContext originalKnnMethodContext; - } -} diff --git a/src/main/java/org/opensearch/knn/index/mapper/MethodFieldMapper.java b/src/main/java/org/opensearch/knn/index/mapper/MethodFieldMapper.java deleted file mode 100644 index 484fd64577..0000000000 --- a/src/main/java/org/opensearch/knn/index/mapper/MethodFieldMapper.java +++ /dev/null @@ -1,212 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.knn.index.mapper; - -import org.apache.lucene.document.FieldType; -import org.apache.lucene.index.DocValuesType; -import org.apache.lucene.index.VectorEncoding; -import org.opensearch.Version; -import org.opensearch.common.Explicit; -import org.opensearch.common.xcontent.XContentFactory; -import org.opensearch.knn.index.SpaceType; -import org.opensearch.knn.index.VectorDataType; -import org.opensearch.knn.index.engine.KNNEngine; -import org.opensearch.knn.index.engine.KNNLibraryIndexingContext; -import org.opensearch.knn.index.engine.KNNMethodConfigContext; -import org.opensearch.knn.index.engine.KNNMethodContext; -import org.opensearch.knn.index.engine.qframe.QuantizationConfig; -import org.opensearch.knn.index.engine.qframe.QuantizationConfigParser; - -import java.io.IOException; -import java.util.Map; -import java.util.Optional; - -import static org.opensearch.knn.common.KNNConstants.DIMENSION; -import static org.opensearch.knn.common.KNNConstants.KNN_ENGINE; -import static org.opensearch.knn.common.KNNConstants.PARAMETERS; -import static org.opensearch.knn.common.KNNConstants.QFRAMEWORK_CONFIG; -import static org.opensearch.knn.common.KNNConstants.SPACE_TYPE; -import static org.opensearch.knn.common.KNNConstants.VECTOR_DATA_TYPE_FIELD; - -/** - * Field mapper for method definition in mapping - */ -public class MethodFieldMapper extends KNNVectorFieldMapper { - - private final PerDimensionProcessor perDimensionProcessor; - private final PerDimensionValidator perDimensionValidator; - private final VectorValidator vectorValidator; - private final VectorTransformer vectorTransformer; - - public static MethodFieldMapper createFieldMapper( - String fullname, - String simpleName, - Map metaValue, - KNNMethodConfigContext knnMethodConfigContext, - MultiFields multiFields, - CopyTo copyTo, - Explicit ignoreMalformed, - boolean stored, - boolean hasDocValues, - OriginalMappingParameters originalMappingParameters - ) { - - KNNMethodContext knnMethodContext = originalMappingParameters.getResolvedKnnMethodContext(); - QuantizationConfig quantizationConfig = knnMethodContext.getKnnEngine() - .getKNNLibraryIndexingContext(knnMethodContext, knnMethodConfigContext) - .getQuantizationConfig(); - KNNLibraryIndexingContext libraryIndexingContext = knnMethodContext.getKnnEngine() - .getKNNLibraryIndexingContext(knnMethodContext, knnMethodConfigContext); - - final KNNVectorFieldType mappedFieldType = new KNNVectorFieldType( - fullname, - metaValue, - knnMethodConfigContext.getVectorDataType(), - new KNNMappingConfig() { - @Override - public Optional getKnnMethodContext() { - return Optional.of(originalMappingParameters.getResolvedKnnMethodContext()); - } - - @Override - public int getDimension() { - return knnMethodConfigContext.getDimension(); - } - - @Override - public Mode getMode() { - return Mode.fromName(originalMappingParameters.getMode()); - } - - @Override - public CompressionLevel getCompressionLevel() { - return knnMethodConfigContext.getCompressionLevel(); - } - - @Override - public QuantizationConfig getQuantizationConfig() { - return quantizationConfig; - } - - @Override - public Version getIndexCreatedVersion() { - return knnMethodConfigContext.getVersionCreated(); - } - - @Override - public KNNLibraryIndexingContext getKnnLibraryIndexingContext() { - return libraryIndexingContext; - } - } - ); - return new MethodFieldMapper( - simpleName, - mappedFieldType, - multiFields, - copyTo, - ignoreMalformed, - stored, - hasDocValues, - knnMethodConfigContext, - originalMappingParameters - ); - } - - private MethodFieldMapper( - String simpleName, - KNNVectorFieldType mappedFieldType, - MultiFields multiFields, - CopyTo copyTo, - Explicit ignoreMalformed, - boolean stored, - boolean hasDocValues, - KNNMethodConfigContext knnMethodConfigContext, - OriginalMappingParameters originalMappingParameters - ) { - - super( - simpleName, - mappedFieldType, - multiFields, - copyTo, - ignoreMalformed, - stored, - hasDocValues, - knnMethodConfigContext.getVersionCreated(), - originalMappingParameters - ); - this.useLuceneBasedVectorField = KNNVectorFieldMapperUtil.useLuceneKNNVectorsFormat(indexCreatedVersion); - KNNMappingConfig knnMappingConfig = mappedFieldType.getKnnMappingConfig(); - KNNMethodContext resolvedKnnMethodContext = originalMappingParameters.getResolvedKnnMethodContext(); - KNNEngine knnEngine = resolvedKnnMethodContext.getKnnEngine(); - KNNLibraryIndexingContext knnLibraryIndexingContext = knnEngine.getKNNLibraryIndexingContext( - resolvedKnnMethodContext, - knnMethodConfigContext - ); - QuantizationConfig quantizationConfig = knnLibraryIndexingContext.getQuantizationConfig(); - - this.fieldType = new FieldType(KNNVectorFieldMapper.Defaults.FIELD_TYPE); - this.fieldType.putAttribute(DIMENSION, String.valueOf(knnMappingConfig.getDimension())); - this.fieldType.putAttribute(SPACE_TYPE, resolvedKnnMethodContext.getSpaceType().getValue()); - // Conditionally add quantization config - if (quantizationConfig != null && quantizationConfig != QuantizationConfig.EMPTY) { - this.fieldType.putAttribute(QFRAMEWORK_CONFIG, QuantizationConfigParser.toCsv(quantizationConfig)); - } - - this.fieldType.putAttribute(VECTOR_DATA_TYPE_FIELD, vectorDataType.getValue()); - this.fieldType.putAttribute(KNN_ENGINE, knnEngine.getName()); - try { - this.fieldType.putAttribute( - PARAMETERS, - XContentFactory.jsonBuilder().map(knnLibraryIndexingContext.getLibraryParameters()).toString() - ); - } catch (IOException ioe) { - throw new RuntimeException(String.format("Unable to create KNNVectorFieldMapper: %s", ioe)); - } - - if (useLuceneBasedVectorField) { - int adjustedDimension = mappedFieldType.vectorDataType == VectorDataType.BINARY - ? knnMappingConfig.getDimension() / 8 - : knnMappingConfig.getDimension(); - final VectorEncoding encoding = mappedFieldType.vectorDataType == VectorDataType.FLOAT - ? VectorEncoding.FLOAT32 - : VectorEncoding.BYTE; - fieldType.setVectorAttributes( - adjustedDimension, - encoding, - SpaceType.DEFAULT.getKnnVectorSimilarityFunction().getVectorSimilarityFunction() - ); - } else { - fieldType.setDocValuesType(DocValuesType.BINARY); - } - - this.fieldType.freeze(); - this.perDimensionProcessor = knnLibraryIndexingContext.getPerDimensionProcessor(); - this.perDimensionValidator = knnLibraryIndexingContext.getPerDimensionValidator(); - this.vectorValidator = knnLibraryIndexingContext.getVectorValidator(); - this.vectorTransformer = knnLibraryIndexingContext.getVectorTransformer(); - } - - @Override - protected VectorValidator getVectorValidator() { - return vectorValidator; - } - - @Override - protected PerDimensionValidator getPerDimensionValidator() { - return perDimensionValidator; - } - - @Override - protected PerDimensionProcessor getPerDimensionProcessor() { - return perDimensionProcessor; - } - - @Override - protected VectorTransformer getVectorTransformer() { - return vectorTransformer; - } -} diff --git a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java index 2559bee619..5b55b31f8b 100644 --- a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java +++ b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldMapperTests.java @@ -176,7 +176,7 @@ public void testTypeParser_build_fromKnnMethodContext() throws IOException { Mapper.BuilderContext builderContext = new Mapper.BuilderContext(settings, new ContentPath()); KNNVectorFieldMapper knnVectorFieldMapper = builder.build(builderContext); - assertTrue(knnVectorFieldMapper instanceof MethodFieldMapper); + assertTrue(knnVectorFieldMapper instanceof EngineFieldMapper); assertTrue(knnVectorFieldMapper.fieldType().getKnnMappingConfig().getKnnMethodContext().isPresent()); assertEquals(spaceType, knnVectorFieldMapper.fieldType().getKnnMappingConfig().getKnnMethodContext().get().getSpaceType()); assertEquals( @@ -261,6 +261,95 @@ public void testKNNVectorFieldMapper_withBlockedKNNEngine() throws IOException { assertNotNull(builderWithFaiss); } + public void testKNNVectorFieldMapperLucene_docValueDefaults() throws IOException { + String fieldName = "test-field-name"; + String indexName = "test-index"; + + Settings settings = Settings.builder().put(settings(CURRENT).build()).put(KNN_INDEX, true).build(); + ModelDao modelDao = mock(ModelDao.class); + KNNVectorFieldMapper.TypeParser typeParser = new KNNVectorFieldMapper.TypeParser(() -> modelDao); + + // Creating a mapping before version 3.0.0 (doc values should be true) + XContentBuilder legacyDocValuesContentBuilder = XContentFactory.jsonBuilder() + .startObject() + .field(TYPE_FIELD_NAME, KNN_VECTOR_TYPE) + .field(DIMENSION_FIELD_NAME, 128) + .startObject(KNN_METHOD) + .field(NAME, METHOD_HNSW) + .field(KNN_ENGINE, KNNEngine.LUCENE.getName()) + .endObject() + .endObject(); + + // Should be true for versions before 3.0.0 + KNNVectorFieldMapper.Builder builderBeforeV3 = (KNNVectorFieldMapper.Builder) typeParser.parse( + fieldName, + xContentBuilderToMap(legacyDocValuesContentBuilder), + buildLegacyParserContext(indexName, settings, Version.V_2_19_0) // Version < 3.0.0 + ); + assertNotNull(builderBeforeV3); + assertTrue(builderBeforeV3.hasDocValues.getValue()); + + // Creating a mapping with Lucene on or after version 3.0.0 (doc values should default to false) + XContentBuilder currentDocValuesContentBuilder = XContentFactory.jsonBuilder() + .startObject() + .field(TYPE_FIELD_NAME, KNN_VECTOR_TYPE) + .field(DIMENSION_FIELD_NAME, 128) + .startObject(KNN_METHOD) + .field(NAME, METHOD_HNSW) + .field(KNN_ENGINE, KNNEngine.LUCENE.getName()) + .endObject() + .endObject(); + + KNNVectorFieldMapper.Builder builderAfterV3 = (KNNVectorFieldMapper.Builder) typeParser.parse( + fieldName, + xContentBuilderToMap(currentDocValuesContentBuilder), + buildParserContext(indexName, settings) // Version >= 3.0.0 + ); + assertNotNull(builderAfterV3); + assertFalse(builderAfterV3.hasDocValues.getValue()); + } + + public void testKNNVectorFieldMapperModel_docValueDefaults() throws IOException { + String fieldName = "test-field-name"; + String indexName = "test-index"; + String modelId = "test-model-id"; + + Settings settings = Settings.builder().put(settings(CURRENT).build()).put(KNN_INDEX, true).build(); + ModelDao modelDao = mock(ModelDao.class); + KNNVectorFieldMapper.TypeParser typeParser = new KNNVectorFieldMapper.TypeParser(() -> modelDao); + + // Creating a model mapping before version 3.0.0 (doc values should be true) + XContentBuilder legacyDocValuesContentBuilder = XContentFactory.jsonBuilder() + .startObject() + .field(TYPE_FIELD_NAME, KNN_VECTOR_TYPE) + .field(MODEL_ID, modelId) + .endObject(); + + // Should be true for versions before 3.0.0 + KNNVectorFieldMapper.Builder builderBeforeV3 = (KNNVectorFieldMapper.Builder) typeParser.parse( + fieldName, + xContentBuilderToMap(legacyDocValuesContentBuilder), + buildLegacyParserContext(indexName, settings, Version.V_2_19_0) // Version < 3.0.0 + ); + assertNotNull(builderBeforeV3); + assertTrue(builderBeforeV3.hasDocValues.getValue()); + + // Creating a mapping with model on or after version 3.0.0 (doc values should default to false) + XContentBuilder currentDocValuesContentBuilder = XContentFactory.jsonBuilder() + .startObject() + .field(TYPE_FIELD_NAME, KNN_VECTOR_TYPE) + .field(MODEL_ID, modelId) + .endObject(); + + KNNVectorFieldMapper.Builder builderAfterV3 = (KNNVectorFieldMapper.Builder) typeParser.parse( + fieldName, + xContentBuilderToMap(currentDocValuesContentBuilder), + buildParserContext(indexName, settings) // Version >= 3.0.0 + ); + assertNotNull(builderAfterV3); + assertFalse(builderAfterV3.hasDocValues.getValue()); + } + public void testTypeParser_withDifferentSpaceTypeCombinations_thenSuccess() throws IOException { // Check that knnMethodContext takes precedent over both model and legacy ModelDao modelDao = mock(ModelDao.class); @@ -282,7 +371,7 @@ public void testTypeParser_withDifferentSpaceTypeCombinations_thenSuccess() thro Mapper.BuilderContext builderContext = new Mapper.BuilderContext(settings, new ContentPath()); KNNVectorFieldMapper knnVectorFieldMapper = builder.build(builderContext); - assertTrue(knnVectorFieldMapper instanceof MethodFieldMapper); + assertTrue(knnVectorFieldMapper instanceof EngineFieldMapper); assertTrue(knnVectorFieldMapper.fieldType().getKnnMappingConfig().getKnnMethodContext().isPresent()); assertEquals(topLevelSpaceType, knnVectorFieldMapper.fieldType().getKnnMappingConfig().getKnnMethodContext().get().getSpaceType()); assertTrue(knnVectorFieldMapper.fieldType().getKnnMappingConfig().getModelId().isEmpty()); @@ -298,7 +387,7 @@ public void testTypeParser_withDifferentSpaceTypeCombinations_thenSuccess() thro builderContext = new Mapper.BuilderContext(settings, new ContentPath()); knnVectorFieldMapper = builder.build(builderContext); - assertTrue(knnVectorFieldMapper instanceof MethodFieldMapper); + assertTrue(knnVectorFieldMapper instanceof EngineFieldMapper); assertTrue(knnVectorFieldMapper.fieldType().getKnnMappingConfig().getKnnMethodContext().isPresent()); assertEquals(SpaceType.DEFAULT, knnVectorFieldMapper.fieldType().getKnnMappingConfig().getKnnMethodContext().get().getSpaceType()); assertTrue(knnVectorFieldMapper.fieldType().getKnnMappingConfig().getModelId().isEmpty()); @@ -313,7 +402,7 @@ public void testTypeParser_withDifferentSpaceTypeCombinations_thenSuccess() thro builderContext = new Mapper.BuilderContext(settings, new ContentPath()); knnVectorFieldMapper = builder.build(builderContext); - assertTrue(knnVectorFieldMapper instanceof MethodFieldMapper); + assertTrue(knnVectorFieldMapper instanceof EngineFieldMapper); assertTrue(knnVectorFieldMapper.fieldType().getKnnMappingConfig().getKnnMethodContext().isPresent()); assertEquals(topLevelSpaceType, knnVectorFieldMapper.fieldType().getKnnMappingConfig().getKnnMethodContext().get().getSpaceType()); assertTrue(knnVectorFieldMapper.fieldType().getKnnMappingConfig().getModelId().isEmpty()); @@ -337,7 +426,7 @@ public void testTypeParser_withDifferentSpaceTypeCombinations_thenSuccess() thro builderContext = new Mapper.BuilderContext(settings, new ContentPath()); knnVectorFieldMapper = builder.build(builderContext); - assertTrue(knnVectorFieldMapper instanceof MethodFieldMapper); + assertTrue(knnVectorFieldMapper instanceof EngineFieldMapper); assertTrue(knnVectorFieldMapper.fieldType().getKnnMappingConfig().getKnnMethodContext().isPresent()); assertEquals( SpaceType.DEFAULT_BINARY, @@ -363,7 +452,7 @@ public void testTypeParser_withDifferentSpaceTypeCombinations_thenSuccess() thro builderContext = new Mapper.BuilderContext(settings, new ContentPath()); knnVectorFieldMapper = builder.build(builderContext); - assertTrue(knnVectorFieldMapper instanceof MethodFieldMapper); + assertTrue(knnVectorFieldMapper instanceof EngineFieldMapper); assertTrue(knnVectorFieldMapper.fieldType().getKnnMappingConfig().getKnnMethodContext().isPresent()); assertEquals( topLevelSpaceType, @@ -408,7 +497,7 @@ public void testTypeParser_withSpaceTypeAndMode_thenSuccess() throws IOException Mapper.BuilderContext builderContext = new Mapper.BuilderContext(settings, new ContentPath()); KNNVectorFieldMapper knnVectorFieldMapper = builder.build(builderContext); - assertTrue(knnVectorFieldMapper instanceof MethodFieldMapper); + assertTrue(knnVectorFieldMapper instanceof EngineFieldMapper); assertTrue(knnVectorFieldMapper.fieldType().getKnnMappingConfig().getKnnMethodContext().isPresent()); assertEquals(topLevelSpaceType, knnVectorFieldMapper.fieldType().getKnnMappingConfig().getKnnMethodContext().get().getSpaceType()); assertTrue(knnVectorFieldMapper.fieldType().getKnnMappingConfig().getModelId().isEmpty()); @@ -477,7 +566,7 @@ public void testSpaceType_build_fromLegacy() throws IOException { // Setup settings Mapper.BuilderContext builderContext = new Mapper.BuilderContext(settings, new ContentPath()); KNNVectorFieldMapper knnVectorFieldMapper = builder.build(builderContext); - assertTrue(knnVectorFieldMapper instanceof MethodFieldMapper); + assertTrue(knnVectorFieldMapper instanceof EngineFieldMapper); assertTrue(knnVectorFieldMapper.fieldType().getKnnMappingConfig().getKnnMethodContext().isPresent()); assertTrue(knnVectorFieldMapper.fieldType().getKnnMappingConfig().getModelId().isEmpty()); assertEquals(SpaceType.L2, knnVectorFieldMapper.fieldType().getKnnMappingConfig().getKnnMethodContext().get().getSpaceType()); @@ -507,7 +596,7 @@ public void testBuilder_build_fromLegacy() throws IOException { // Setup settings Mapper.BuilderContext builderContext = new Mapper.BuilderContext(settings, new ContentPath()); KNNVectorFieldMapper knnVectorFieldMapper = builder.build(builderContext); - assertTrue(knnVectorFieldMapper instanceof MethodFieldMapper); + assertTrue(knnVectorFieldMapper instanceof EngineFieldMapper); assertTrue(knnVectorFieldMapper.fieldType().getKnnMappingConfig().getKnnMethodContext().isPresent()); assertTrue(knnVectorFieldMapper.fieldType().getKnnMappingConfig().getModelId().isEmpty()); assertEquals(SpaceType.L2, knnVectorFieldMapper.fieldType().getKnnMappingConfig().getKnnMethodContext().get().getSpaceType()); @@ -1226,7 +1315,7 @@ public void testMethodFieldMapperParseCreateField_validInput_thenDifferentFieldT SpaceType.UNDEFINED.getValue() ); originalMappingParameters.setResolvedKnnMethodContext(knnMethodContext); - MethodFieldMapper methodFieldMapper = MethodFieldMapper.createFieldMapper( + EngineFieldMapper methodFieldMapper = EngineFieldMapper.createFieldMapper( TEST_FIELD_NAME, TEST_FIELD_NAME, Collections.emptyMap(), @@ -1267,7 +1356,7 @@ public void testMethodFieldMapperParseCreateField_validInput_thenDifferentFieldT when(parseContext.path()).thenReturn(contentPath); when(parseContext.parser()).thenReturn(createXContentParser(dataType)); when(parseContext.indexSettings()).thenReturn(indexSettingsMock); - methodFieldMapper = MethodFieldMapper.createFieldMapper( + methodFieldMapper = EngineFieldMapper.createFieldMapper( TEST_FIELD_NAME, TEST_FIELD_NAME, Collections.emptyMap(), @@ -1412,9 +1501,6 @@ public void testModelFieldMapperParseCreateField_validInput_thenDifferentFieldTy @SneakyThrows public void testLuceneFieldMapper_parseCreateField_docValues_withFloats() { - // Create a lucene field mapper that creates a binary doc values field as well as KnnVectorField - LuceneFieldMapper.CreateLuceneFieldMapperInput.CreateLuceneFieldMapperInputBuilder inputBuilder = - createLuceneFieldMapperInputBuilder(); IndexSettings indexSettingsMock = mock(IndexSettings.class); when(indexSettingsMock.getSettings()).thenReturn(Settings.EMPTY); ParseContext.Document document = new ParseContext.Document(); @@ -1430,10 +1516,16 @@ public void testLuceneFieldMapper_parseCreateField_docValues_withFloats() { .dimension(TEST_DIMENSION) .build(); + KNNMethodContext luceneMethodContext = new KNNMethodContext( + KNNEngine.LUCENE, + SpaceType.DEFAULT, + new MethodComponentContext(METHOD_HNSW, Collections.emptyMap()) + ); + OriginalMappingParameters originalMappingParameters = new OriginalMappingParameters( VectorDataType.FLOAT, TEST_DIMENSION, - getDefaultKNNMethodContext(), + luceneMethodContext, Mode.NOT_CONFIGURED.getName(), CompressionLevel.NOT_CONFIGURED.getName(), null, @@ -1441,11 +1533,16 @@ public void testLuceneFieldMapper_parseCreateField_docValues_withFloats() { ); originalMappingParameters.setResolvedKnnMethodContext(originalMappingParameters.getKnnMethodContext()); - LuceneFieldMapper luceneFieldMapper = LuceneFieldMapper.createFieldMapper( + EngineFieldMapper luceneFieldMapper = EngineFieldMapper.createFieldMapper( + TEST_FIELD_NAME, TEST_FIELD_NAME, Collections.emptyMap(), knnMethodConfigContext, - inputBuilder.build(), + FieldMapper.MultiFields.empty(), + FieldMapper.CopyTo.empty(), + new Explicit<>(true, true), + false, + true, originalMappingParameters ); luceneFieldMapper.parseCreateField(parseContext, TEST_DIMENSION, VectorDataType.FLOAT); @@ -1482,30 +1579,31 @@ public void testLuceneFieldMapper_parseCreateField_docValues_withFloats() { when(parseContext.parser()).thenReturn(createXContentParser(VectorDataType.FLOAT)); when(parseContext.indexSettings()).thenReturn(indexSettingsMock); - inputBuilder.hasDocValues(false); - knnMethodConfigContext = KNNMethodConfigContext.builder() .vectorDataType(VectorDataType.FLOAT) .versionCreated(CURRENT) .dimension(TEST_DIMENSION) .build(); - MethodComponentContext methodComponentContext = new MethodComponentContext(METHOD_HNSW, Collections.emptyMap()); - KNNMethodContext knnMethodContext = new KNNMethodContext(KNNEngine.LUCENE, SpaceType.DEFAULT, methodComponentContext); originalMappingParameters = new OriginalMappingParameters( VectorDataType.FLOAT, TEST_DIMENSION, - knnMethodContext, + luceneMethodContext, Mode.NOT_CONFIGURED.getName(), CompressionLevel.NOT_CONFIGURED.getName(), null, SpaceType.UNDEFINED.getValue() ); originalMappingParameters.setResolvedKnnMethodContext(originalMappingParameters.getKnnMethodContext()); - luceneFieldMapper = LuceneFieldMapper.createFieldMapper( + luceneFieldMapper = EngineFieldMapper.createFieldMapper( + TEST_FIELD_NAME, TEST_FIELD_NAME, Collections.emptyMap(), knnMethodConfigContext, - inputBuilder.build(), + FieldMapper.MultiFields.empty(), + FieldMapper.CopyTo.empty(), + new Explicit<>(true, true), + false, + false, originalMappingParameters ); luceneFieldMapper.parseCreateField(parseContext, TEST_DIMENSION, VectorDataType.FLOAT); @@ -1521,10 +1619,6 @@ public void testLuceneFieldMapper_parseCreateField_docValues_withFloats() { @SneakyThrows public void testLuceneFieldMapper_parseCreateField_docValues_withBytes() { - // Create a lucene field mapper that creates a binary doc values field as well as KnnByteVectorField - - LuceneFieldMapper.CreateLuceneFieldMapperInput.CreateLuceneFieldMapperInputBuilder inputBuilder = - createLuceneFieldMapperInputBuilder(); IndexSettings indexSettingsMock = mock(IndexSettings.class); when(indexSettingsMock.getSettings()).thenReturn(Settings.EMPTY); ParseContext.Document document = new ParseContext.Document(); @@ -1534,10 +1628,13 @@ public void testLuceneFieldMapper_parseCreateField_docValues_withBytes() { when(parseContext.path()).thenReturn(contentPath); when(parseContext.indexSettings()).thenReturn(indexSettingsMock); + MethodComponentContext methodComponentContext = new MethodComponentContext(METHOD_HNSW, Collections.emptyMap()); + KNNMethodContext luceneByteKnnMethodContext = new KNNMethodContext(KNNEngine.LUCENE, SpaceType.DEFAULT, methodComponentContext); + OriginalMappingParameters originalMappingParameters = new OriginalMappingParameters( VectorDataType.BYTE, TEST_DIMENSION, - getDefaultByteKNNMethodContext(), + luceneByteKnnMethodContext, Mode.NOT_CONFIGURED.getName(), CompressionLevel.NOT_CONFIGURED.getName(), null, @@ -1545,8 +1642,9 @@ public void testLuceneFieldMapper_parseCreateField_docValues_withBytes() { ); originalMappingParameters.setResolvedKnnMethodContext(originalMappingParameters.getKnnMethodContext()); - LuceneFieldMapper luceneFieldMapper = Mockito.spy( - LuceneFieldMapper.createFieldMapper( + EngineFieldMapper luceneFieldMapper = Mockito.spy( + EngineFieldMapper.createFieldMapper( + TEST_FIELD_NAME, TEST_FIELD_NAME, Collections.emptyMap(), KNNMethodConfigContext.builder() @@ -1554,7 +1652,11 @@ public void testLuceneFieldMapper_parseCreateField_docValues_withBytes() { .versionCreated(CURRENT) .dimension(TEST_DIMENSION) .build(), - inputBuilder.build(), + FieldMapper.MultiFields.empty(), + FieldMapper.CopyTo.empty(), + new Explicit<>(true, true), + false, + true, originalMappingParameters ) ); @@ -1594,10 +1696,9 @@ public void testLuceneFieldMapper_parseCreateField_docValues_withBytes() { when(parseContext.path()).thenReturn(contentPath); when(parseContext.indexSettings()).thenReturn(indexSettingsMock); - inputBuilder.hasDocValues(false); - luceneFieldMapper = Mockito.spy( - LuceneFieldMapper.createFieldMapper( + EngineFieldMapper.createFieldMapper( + TEST_FIELD_NAME, TEST_FIELD_NAME, Collections.emptyMap(), KNNMethodConfigContext.builder() @@ -1605,7 +1706,11 @@ public void testLuceneFieldMapper_parseCreateField_docValues_withBytes() { .versionCreated(CURRENT) .dimension(TEST_DIMENSION) .build(), - inputBuilder.build(), + FieldMapper.MultiFields.empty(), + FieldMapper.CopyTo.empty(), + new Explicit<>(true, true), + false, + false, originalMappingParameters ) ); @@ -2213,16 +2318,6 @@ private void validateBuilderAfterParsing( } } - private LuceneFieldMapper.CreateLuceneFieldMapperInput.CreateLuceneFieldMapperInputBuilder createLuceneFieldMapperInputBuilder() { - return LuceneFieldMapper.CreateLuceneFieldMapperInput.builder() - .name(TEST_FIELD_NAME) - .multiFields(FieldMapper.MultiFields.empty()) - .copyTo(FieldMapper.CopyTo.empty()) - .hasDocValues(true) - .ignoreMalformed(new Explicit<>(true, true)) - .originalKnnMethodContext(getDefaultKNNMethodContext()); - } - private XContentBuilder createXContentForFieldMapping( SpaceType topLevelSpaceType, SpaceType methodSpaceType,