diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/AbstractSemanticMapperTestCase.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/AbstractSemanticMapperTestCase.java index bd29376461d6e..8e7f74d35d560 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/AbstractSemanticMapperTestCase.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/AbstractSemanticMapperTestCase.java @@ -14,13 +14,13 @@ import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.core.CheckedRunnable; import org.elasticsearch.core.Nullable; import org.elasticsearch.features.FeatureService; import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; -import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.InferenceMetadataFieldsMapper; import org.elasticsearch.index.mapper.LuceneDocument; @@ -37,8 +37,6 @@ import org.elasticsearch.index.mapper.vectors.SparseVectorFieldMapper; import org.elasticsearch.inference.ChunkingSettings; import org.elasticsearch.inference.MinimalServiceSettings; -import org.elasticsearch.inference.Model; -import org.elasticsearch.inference.ServiceSettings; import org.elasticsearch.inference.SimilarityMeasure; import org.elasticsearch.inference.TaskType; import org.elasticsearch.license.License; @@ -70,9 +68,11 @@ import java.util.function.Supplier; import static java.util.Objects.requireNonNull; +import static org.elasticsearch.index.IndexSettings.DENSE_VECTOR_EXPERIMENTAL_FEATURES_SETTING; import static org.elasticsearch.index.IndexVersions.NEW_SPARSE_VECTOR; import static org.elasticsearch.index.IndexVersions.SEMANTIC_TEXT_DEFAULTS_TO_BFLOAT16; import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.BBQ_MIN_DIMS; +import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapperTestUtils.defaultDenseVectorIndexOptions; import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapperTestUtils.getSupportedSimilarities; import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapperTestUtils.randomCompatibleDimensions; import static org.elasticsearch.xpack.inference.mapper.SemanticFieldMapper.INDEX_OPTIONS_FIELD; @@ -87,7 +87,6 @@ import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; -import static org.hamcrest.Matchers.is; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; @@ -265,34 +264,28 @@ public void testUpdateInferenceId_GivenNoModelSettings() throws IOException { String fieldName = randomAlphaOfLengthBetween(5, 15); String oldInferenceId = randomAlphaOfLengthBetween(5, 15); - TestModel oldModel = null; + MinimalServiceSettings oldModelSettings = null; if (randomBoolean()) { - oldModel = createRandomSupportedModel(); - givenModelSettings(oldInferenceId, new MinimalServiceSettings(oldModel)); + oldModelSettings = createRandomModelSettings(); + givenModelSettings(oldInferenceId, oldModelSettings); } var mapperService = createSemanticMapperService(semanticMapping(fieldName, oldInferenceId)); assertInferenceEndpoints(mapperService, fieldName, oldInferenceId, oldInferenceId); - assertSemanticField(mapperService, fieldName, false, null, null); - if (oldModel != null) { - assertEmbeddingsFieldMapperMatchesModel(mapperService, fieldName, oldModel); - } + assertSemanticField(mapperService, fieldName, false, oldModelSettings, null, null); String newInferenceId = randomValueOtherThan(oldInferenceId, () -> randomAlphaOfLengthBetween(5, 15)); - TestModel newModel = null; + MinimalServiceSettings newModelSettings = null; if (randomBoolean()) { - newModel = createRandomSupportedModel(); - givenModelSettings(newInferenceId, new MinimalServiceSettings(newModel)); + newModelSettings = createRandomModelSettings(); + givenModelSettings(newInferenceId, newModelSettings); } merge(mapperService, semanticMapping(fieldName, newInferenceId)); assertInferenceEndpoints(mapperService, fieldName, newInferenceId, newInferenceId); - assertSemanticField(mapperService, fieldName, false, null, null); - if (newModel != null) { - assertEmbeddingsFieldMapperMatchesModel(mapperService, fieldName, newModel); - } + assertSemanticField(mapperService, fieldName, false, newModelSettings, null, null); } } @@ -302,34 +295,29 @@ public void testUpdateInferenceId_GivenModelSettings() throws IOException { final String oldInferenceId = randomAlphaOfLengthBetween(5, 15); final String newInferenceId = randomValueOtherThan(oldInferenceId, () -> randomAlphaOfLengthBetween(5, 15)); - final TestModel oldModel = createRandomSupportedModel(); - final MinimalServiceSettings previousModelSettings = new MinimalServiceSettings(oldModel); + final MinimalServiceSettings previousModelSettings = createRandomModelSettings(); givenModelSettings(oldInferenceId, previousModelSettings); final MapperService mapperService = createSemanticMapperService( semanticMapping(fieldName, oldInferenceId, previousModelSettings) ); - final SemanticIndexOptions currentIndexOptions = extractCurrentIndexOptions(mapperService, fieldName); assertInferenceEndpoints(mapperService, fieldName, oldInferenceId, oldInferenceId); - assertSemanticField(mapperService, fieldName, true, null, currentIndexOptions); - assertEmbeddingsFieldMapperMatchesModel(mapperService, fieldName, oldModel); + assertSemanticField(mapperService, fieldName, true, previousModelSettings, null, null); final CheckedRunnable mergeRunner = () -> merge(mapperService, semanticMapping(fieldName, newInferenceId)); if (randomBoolean()) { // Compatible: new endpoint has identical task type / dimensions / similarity / element type - TestModel newModel = createCompatibleModel(newInferenceId, oldModel); - MinimalServiceSettings newModelSettings = new MinimalServiceSettings(newModel); + MinimalServiceSettings newModelSettings = createCompatibleModelSettings(previousModelSettings); givenModelSettings(newInferenceId, newModelSettings); mergeRunner.run(); assertInferenceEndpoints(mapperService, fieldName, newInferenceId, newInferenceId); - assertSemanticField(mapperService, fieldName, true, null, currentIndexOptions); - assertEmbeddingsFieldMapperMatchesModel(mapperService, fieldName, newModel); + assertSemanticField(mapperService, fieldName, true, newModelSettings, null, null); } else { - final TestModel incompatibleModel = createIncompatibleModel(newInferenceId, oldModel); + final MinimalServiceSettings incompatibleModelSettings = createIncompatibleModelSettings(previousModelSettings); final String expectedErrorMessage; - if (incompatibleModel == null) { + if (incompatibleModelSettings == null) { // Incompatible: new endpoint does not exist expectedErrorMessage = "Cannot update [" + contentType() @@ -340,7 +328,6 @@ public void testUpdateInferenceId_GivenModelSettings() throws IOException { + "] does not exist."; } else { // Incompatible: new endpoint exists but its model settings are incompatible - MinimalServiceSettings incompatibleModelSettings = new MinimalServiceSettings(incompatibleModel); givenModelSettings(newInferenceId, incompatibleModelSettings); expectedErrorMessage = "Cannot update [" @@ -364,44 +351,84 @@ public void testUpdateInferenceId_GivenModelSettings() throws IOException { } } - protected XContentBuilder semanticMapping(String fieldName, String inferenceId) throws IOException { + protected XContentBuilder semanticMapping(String fieldName, @Nullable String inferenceId) throws IOException { return semanticMapping(fieldName, inferenceId, null, null, null, null); } - protected XContentBuilder semanticMapping(String fieldName, String inferenceId, @Nullable String searchInferenceId) throws IOException { + protected XContentBuilder semanticMapping(String fieldName, @Nullable String inferenceId, @Nullable String searchInferenceId) + throws IOException { return semanticMapping(fieldName, inferenceId, searchInferenceId, null, null, null); } - protected XContentBuilder semanticMapping(String fieldName, String inferenceId, @Nullable MinimalServiceSettings modelSettings) - throws IOException { + protected XContentBuilder semanticMapping( + String fieldName, + @Nullable String inferenceId, + @Nullable MinimalServiceSettings modelSettings + ) throws IOException { return semanticMapping(fieldName, inferenceId, null, modelSettings, null, null); } protected XContentBuilder semanticMapping( String fieldName, - String inferenceId, + @Nullable String inferenceId, @Nullable String searchInferenceId, @Nullable MinimalServiceSettings modelSettings, @Nullable ChunkingSettings chunkingSettings, @Nullable SemanticIndexOptions indexOptions + ) throws IOException { + return semanticMapping(fieldName, inferenceId, searchInferenceId, modelSettings, chunkingSettings, indexOptions, null); + } + + protected XContentBuilder semanticMapping( + String fieldName, + @Nullable String inferenceId, + @Nullable String searchInferenceId, + @Nullable MinimalServiceSettings modelSettings, + @Nullable ChunkingSettings chunkingSettings, + @Nullable SemanticIndexOptions indexOptions, + @Nullable CheckedConsumer additionalFields ) throws IOException { return mapping( - b -> addSemanticMapping(b, fieldName, inferenceId, searchInferenceId, modelSettings, chunkingSettings, indexOptions) + b -> addSemanticMapping( + b, + fieldName, + inferenceId, + searchInferenceId, + modelSettings, + chunkingSettings, + indexOptions, + additionalFields + ) ); } protected void addSemanticMapping( XContentBuilder mappingBuilder, String fieldName, - String inferenceId, + @Nullable String inferenceId, @Nullable String searchInferenceId, @Nullable MinimalServiceSettings modelSettings, @Nullable ChunkingSettings chunkingSettings, @Nullable SemanticIndexOptions indexOptions + ) throws IOException { + addSemanticMapping(mappingBuilder, fieldName, inferenceId, searchInferenceId, modelSettings, chunkingSettings, indexOptions, null); + } + + protected void addSemanticMapping( + XContentBuilder mappingBuilder, + String fieldName, + @Nullable String inferenceId, + @Nullable String searchInferenceId, + @Nullable MinimalServiceSettings modelSettings, + @Nullable ChunkingSettings chunkingSettings, + @Nullable SemanticIndexOptions indexOptions, + @Nullable CheckedConsumer additionalFields ) throws IOException { mappingBuilder.startObject(fieldName); mappingBuilder.field("type", contentType()); - mappingBuilder.field(INFERENCE_ID_FIELD, inferenceId); + if (inferenceId != null) { + mappingBuilder.field(INFERENCE_ID_FIELD, inferenceId); + } if (searchInferenceId != null) { mappingBuilder.field(SEARCH_INFERENCE_ID_FIELD, searchInferenceId); } @@ -414,6 +441,9 @@ protected void addSemanticMapping( if (indexOptions != null) { mappingBuilder.field(INDEX_OPTIONS_FIELD, indexOptions); } + if (additionalFields != null) { + additionalFields.accept(mappingBuilder); + } mappingBuilder.endObject(); } @@ -423,10 +453,7 @@ protected MapperService createSemanticMapperService(XContentBuilder mappings) th } protected MapperService createSemanticMapperService(XContentBuilder mappings, IndexVersion minIndexVersion) throws IOException { - IndexVersion maxIndexVersion = useLegacyFormat() - ? IndexVersionUtils.getPreviousVersion(IndexVersions.SEMANTIC_TEXT_LEGACY_FORMAT_FORBIDDEN) - : IndexVersion.current(); - return createSemanticMapperService(mappings, minIndexVersion, maxIndexVersion); + return createSemanticMapperService(mappings, minIndexVersion, IndexVersion.current()); } protected MapperService createSemanticMapperService( @@ -434,18 +461,13 @@ protected MapperService createSemanticMapperService( IndexVersion minIndexVersion, IndexVersion maxIndexVersion ) throws IOException { - validateIndexVersion(minIndexVersion, useLegacyFormat()); - validateIndexVersion(maxIndexVersion, useLegacyFormat()); IndexVersion indexVersion = IndexVersionUtils.randomVersionBetween(minIndexVersion, maxIndexVersion); return createSemanticMapperServiceWithIndexVersion(mappings, indexVersion); } protected MapperService createSemanticMapperServiceWithIndexVersion(XContentBuilder mappings, IndexVersion indexVersion) throws IOException { - var settings = Settings.builder() - .put(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), indexVersion) - .put(InferenceMetadataFieldsMapper.USE_LEGACY_SEMANTIC_TEXT_FORMAT.getKey(), useLegacyFormat()) - .build(); + var settings = Settings.builder().put(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), indexVersion).build(); return createMapperService(indexVersion, settings, mappings); } @@ -484,6 +506,14 @@ protected TestModel createRandomSupportedModel() { return TestModel.createRandomInstance(randomFrom(supportedTaskTypes())); } + protected MinimalServiceSettings createRandomModelSettings() { + return new MinimalServiceSettings(createRandomSupportedModel()); + } + + protected MinimalServiceSettings createRandomModelSettings(TaskType taskType) { + return new MinimalServiceSettings(TestModel.createRandomInstance(taskType)); + } + protected T getSemanticFieldMapper(MapperService mapperService, String fieldName) { Mapper mapper = mapperService.mappingLookup().getMapper(fieldName); assertThat(mapper, instanceOf(expectedMapperClass())); @@ -524,15 +554,26 @@ protected void performDynamicUpdate( /** * Asserts the structural invariants of a semantic field mapping: the concrete mapper and field type classes, the nested chunks - * field, the chunks text/offsets sub-fields (see {@link #assertChunksTextField}), the embeddings sub-mapper when model settings - * are expected (see {@link #assertEmbeddingsField}), and the chunking settings. + * field, the chunks text/offsets sub-fields, the embeddings sub-mapper, model settings, chunking settings, and index options. + * + * @param mapperService The mapper service. + * @param fieldName The name of the field to check. + * @param modelSettingsSetOnFieldType Whether model settings should be set on the field type. If true, {@code modelSettings} is + * required. + * @param modelSettings The model settings used to validate the embeddings sub-mapper. When null, verifies that the embeddings + * sub-mapper is not configured. + * @param expectedChunkingSettings The expected chunking settings. + * @param expectedIndexOptions The expected index options. When null and {@code modelSettings} is provided, the expected default index + * options are automatically determined and validated. When non-null, they must specify the complete index + * options expected, including any element type override. */ protected void assertSemanticField( MapperService mapperService, String fieldName, - boolean expectedModelSettings, - ChunkingSettings expectedChunkingSettings, - SemanticIndexOptions expectedIndexOptions + boolean modelSettingsSetOnFieldType, + @Nullable MinimalServiceSettings modelSettings, + @Nullable ChunkingSettings expectedChunkingSettings, + @Nullable SemanticIndexOptions expectedIndexOptions ) { T semanticFieldMapper = getSemanticFieldMapper(mapperService, fieldName); @@ -550,15 +591,28 @@ protected void assertSemanticField( assertThat(chunksMapper.fullPath(), equalTo(getChunksFieldName(fieldName))); assertChunksTextField(semanticFieldType, chunksMapper); - if (expectedModelSettings) { - assertNotNull(semanticFieldType.getModelSettings()); - Mapper embeddingsMapper = chunksMapper.getMapper(CHUNKED_EMBEDDINGS_FIELD); + Mapper embeddingsMapper = chunksMapper.getMapper(CHUNKED_EMBEDDINGS_FIELD); + if (modelSettings != null) { + SemanticIndexOptions resolvedExpectedIndexOptions = expectedIndexOptions; + if (resolvedExpectedIndexOptions == null) { + resolvedExpectedIndexOptions = getDefaultIndexOptions(modelSettings, mapperService); + } + assertNotNull(embeddingsMapper); assertThat(embeddingsMapper, instanceOf(FieldMapper.class)); FieldMapper embeddingsFieldMapper = (FieldMapper) embeddingsMapper; assertSame(embeddingsFieldMapper.fieldType(), mapperService.mappingLookup().getFieldType(getEmbeddingsFieldName(fieldName))); assertThat(embeddingsMapper.fullPath(), equalTo(getEmbeddingsFieldName(fieldName))); - assertEmbeddingsField(mapperService, semanticFieldType, embeddingsFieldMapper, expectedIndexOptions); + assertEmbeddingsField(mapperService, embeddingsFieldMapper, modelSettings, resolvedExpectedIndexOptions); + } else { + assertNull(embeddingsMapper); + } + + if (modelSettingsSetOnFieldType) { + if (modelSettings == null) { + throw new AssertionError("modelSettings must be provided to check model settings on the field type"); + } + assertModelSettingsOnFieldType(semanticFieldType, modelSettings); } else { assertNull(semanticFieldType.getModelSettings()); } @@ -583,39 +637,37 @@ protected void assertChunksTextField(U fieldType, NestedObjectMapper chunksMappe } /** - * Asserts the embeddings sub-mapper against the field's model settings and expected index options. The base implementation - * covers the dense task types ({@code text_embedding}, {@code embedding}); {@code semantic_text} overrides this to also cover - * {@code sparse_embedding}. + * Asserts the embeddings sub-mapper against the referenced {@link MinimalServiceSettings} and expected index options. The base + * implementation covers the dense task types ({@code text_embedding}, {@code embedding}); {@code semantic_text} overrides this to + * also cover {@code sparse_embedding}. */ protected void assertEmbeddingsField( MapperService mapperService, - U fieldType, FieldMapper embeddingsMapper, + MinimalServiceSettings modelSettings, @Nullable SemanticIndexOptions expectedIndexOptions ) { IndexVersion indexVersion = mapperService.getIndexSettings().getIndexVersionCreated(); - MinimalServiceSettings modelSettings = fieldType.getModelSettings(); TaskType taskType = modelSettings.taskType(); if (taskType == TaskType.TEXT_EMBEDDING || taskType == TaskType.EMBEDDING) { assertThat(embeddingsMapper, instanceOf(DenseVectorFieldMapper.class)); DenseVectorFieldMapper denseVectorFieldMapper = (DenseVectorFieldMapper) embeddingsMapper; + IndexOptions expectedBaseIndexOptions = null; + DenseVectorFieldMapper.ElementType expectedElementType = modelSettings.elementType(); if (expectedIndexOptions != null) { IndexOptions expectedEmbeddingFieldIndexOptions = expectedIndexOptions.indexOptions(); if (expectedEmbeddingFieldIndexOptions instanceof ExtendedDenseVectorIndexOptions edvio) { - assertEquals(edvio.getBaseIndexOptions(), denseVectorFieldMapper.fieldType().getIndexOptions()); + expectedBaseIndexOptions = edvio.getBaseIndexOptions(); + if (edvio.getElementType() != null) { + expectedElementType = edvio.getElementType(); + } } else { - assertEquals(expectedEmbeddingFieldIndexOptions, denseVectorFieldMapper.fieldType().getIndexOptions()); + expectedBaseIndexOptions = expectedEmbeddingFieldIndexOptions; } - } else { - assertNull(denseVectorFieldMapper.fieldType().getIndexOptions()); } - DenseVectorFieldMapper.ElementType expectedElementType = getExpectedElementType( - indexVersion, - modelSettings.elementType(), - expectedIndexOptions - ); + assertEquals(expectedBaseIndexOptions, denseVectorFieldMapper.fieldType().getIndexOptions()); assertEquals(expectedElementType, denseVectorFieldMapper.fieldType().getElementType()); assertEquals(modelSettings.dimensions().intValue(), denseVectorFieldMapper.fieldType().getVectorDimensions()); if (modelSettings.similarity() != null && indexVersion.onOrAfter(NEW_SPARSE_VECTOR)) { @@ -623,55 +675,20 @@ protected void assertEmbeddingsField( assertEquals(modelSettings.similarity().vectorSimilarity(), denseVectorFieldMapper.fieldType().getSimilarity()); } } else { - throw new AssertionError("Invalid task type [" + modelSettings.taskType() + "]"); + throw new AssertionError("Invalid task type [" + taskType + "]"); } } - protected static DenseVectorFieldMapper.ElementType getExpectedElementType( - IndexVersion indexVersion, - DenseVectorFieldMapper.ElementType modelElementType, - @Nullable SemanticIndexOptions semanticIndexOptions - ) { - if (semanticIndexOptions != null && semanticIndexOptions.indexOptions() instanceof ExtendedDenseVectorIndexOptions edvio) { - if (edvio.getElementType() != null) { - return edvio.getElementType(); - } - } + protected void assertModelSettingsOnFieldType(U fieldType, MinimalServiceSettings modelSettings) { + MinimalServiceSettings actual = fieldType.getModelSettings(); + assertNotNull(actual); - DenseVectorFieldMapper.ElementType expectedElementType = modelElementType; - if (indexVersion.onOrAfter(SEMANTIC_TEXT_DEFAULTS_TO_BFLOAT16) && expectedElementType == DenseVectorFieldMapper.ElementType.FLOAT) { - expectedElementType = DenseVectorFieldMapper.ElementType.BFLOAT16; - } - return expectedElementType; - } + assertEquals(modelSettings.taskType(), actual.taskType()); + assertEquals(modelSettings.dimensions(), actual.dimensions()); + assertEquals(modelSettings.similarity(), actual.similarity()); + assertEquals(modelSettings.elementType(), actual.elementType()); - /** - * Asserts that the generated embeddings sub-mapper (sparse or dense) matches the task type and service settings of the - * referenced {@link Model}, including dimensions, element type, and similarity for dense models. - */ - protected void assertEmbeddingsFieldMapperMatchesModel(MapperService mapperService, String fieldName, Model model) { - Mapper embeddingsFieldMapper = mapperService.mappingLookup().getMapper(getEmbeddingsFieldName(fieldName)); - switch (model.getTaskType()) { - case SPARSE_EMBEDDING -> assertThat(embeddingsFieldMapper, is(instanceOf(SparseVectorFieldMapper.class))); - case TEXT_EMBEDDING, EMBEDDING -> { - T semanticFieldMapper = getSemanticFieldMapper(mapperService, fieldName); - DenseVectorFieldMapper.ElementType expectedElementType = getExpectedElementType( - mapperService.getIndexSettings().getIndexVersionCreated(), - model.getServiceSettings().elementType(), - semanticFieldMapper.fieldType().getIndexOptions() - ); - assertThat(embeddingsFieldMapper, is(instanceOf(DenseVectorFieldMapper.class))); - DenseVectorFieldMapper denseVectorFieldMapper = (DenseVectorFieldMapper) embeddingsFieldMapper; - ServiceSettings modelServiceSettings = model.getConfigurations().getServiceSettings(); - assertThat(denseVectorFieldMapper.fieldType().getVectorDimensions(), equalTo(modelServiceSettings.dimensions())); - assertThat(denseVectorFieldMapper.fieldType().getElementType(), equalTo(expectedElementType)); - assertThat( - denseVectorFieldMapper.fieldType().getSimilarity(), - equalTo(modelServiceSettings.similarity().vectorSimilarity()) - ); - } - default -> throw new AssertionError("Unexpected task type [" + model.getTaskType() + "]"); - } + assertTrue(modelSettings.canMergeWith(actual)); } protected static void assertInferenceEndpoints( @@ -688,18 +705,53 @@ protected static void assertInferenceEndpoints( assertEquals(expectedSearchInferenceId, semanticFieldType.getSearchInferenceId()); } - protected SemanticIndexOptions extractCurrentIndexOptions(MapperService mapperService, String fieldName) { - SemanticIndexOptions currentIndexOptions = null; - T sfm = getSemanticFieldMapper(mapperService, fieldName); - FieldMapper embeddingsMapper = sfm.fieldType().getEmbeddingsField(); - if (embeddingsMapper instanceof DenseVectorFieldMapper dvm) { - IndexOptions denseIndexOptions = dvm.fieldType().getIndexOptions(); - if (denseIndexOptions != null) { - currentIndexOptions = new SemanticIndexOptions(SemanticIndexOptions.SupportedIndexOptions.DENSE_VECTOR, denseIndexOptions); + protected SemanticIndexOptions getDefaultIndexOptions(MinimalServiceSettings modelSettings, MapperService mapperService) { + IndexVersion indexVersion = mapperService.getIndexSettings().getIndexVersionCreated(); + boolean experimentalFeatures = DENSE_VECTOR_EXPERIMENTAL_FEATURES_SETTING.get(mapperService.getIndexSettings().getSettings()); + + TaskType taskType = modelSettings.taskType(); + DenseVectorFieldMapper.ElementType elementType = modelSettings.elementType(); + Integer dimensions = modelSettings.dimensions(); + + return switch (taskType) { + case TEXT_EMBEDDING, EMBEDDING -> { + var denseDefaults = getExplicitDenseVectorIndexOptions(modelSettings, indexVersion); + if (denseDefaults == null) { + denseDefaults = defaultDenseVectorIndexOptions( + indexVersion, + operationMode == License.OperationMode.ENTERPRISE, + true, + dimensions, + elementType, + experimentalFeatures + ); + } + + DenseVectorFieldMapper.ElementType elementTypeOverride = indexVersion.onOrAfter(SEMANTIC_TEXT_DEFAULTS_TO_BFLOAT16) + && elementType == DenseVectorFieldMapper.ElementType.FLOAT ? DenseVectorFieldMapper.ElementType.BFLOAT16 : null; + + yield denseDefaults == null && elementTypeOverride == null + ? null + : new SemanticIndexOptions( + SemanticIndexOptions.SupportedIndexOptions.DENSE_VECTOR, + new ExtendedDenseVectorIndexOptions(denseDefaults, elementTypeOverride) + ); } - } + case SPARSE_EMBEDDING -> { + var sparseDefaults = SparseVectorFieldMapper.SparseVectorIndexOptions.getDefaultIndexOptions(indexVersion); + yield sparseDefaults == null + ? null + : new SemanticIndexOptions(SemanticIndexOptions.SupportedIndexOptions.SPARSE_VECTOR, sparseDefaults); + } + default -> throw new AssertionError("Unexpected task type [" + taskType + "]"); + }; + } - return currentIndexOptions; + protected DenseVectorFieldMapper.DenseVectorIndexOptions getExplicitDenseVectorIndexOptions( + MinimalServiceSettings modelSettings, + IndexVersion indexVersion + ) { + return null; } protected void givenModelSettings(String inferenceId, MinimalServiceSettings modelSettings) { @@ -707,45 +759,38 @@ protected void givenModelSettings(String inferenceId, MinimalServiceSettings mod } /** - * Creates a {@link TestModel} that is compatible with {@code baseModel} (same task type, dimensions, - * similarity, and element type) but with a distinct service, task settings, and secrets, registered - * under the given inference ID. Compatible models can be substituted via an inference-ID update. + * Creates a {@link MinimalServiceSettings} that is compatible with {@code base} (same task type, + * dimensions, similarity, and element type) but with a distinct service name. Compatible settings + * can be substituted via an inference-ID update. */ - protected TestModel createCompatibleModel(String inferenceId, TestModel baseModel) { - return new TestModel( - inferenceId, - baseModel.getTaskType(), + protected MinimalServiceSettings createCompatibleModelSettings(MinimalServiceSettings base) { + return new MinimalServiceSettings( randomAlphaOfLength(4), - new TestModel.TestServiceSettings( - randomAlphaOfLength(4), - baseModel.getServiceSettings().dimensions(), - baseModel.getServiceSettings().similarity(), - baseModel.getServiceSettings().elementType() - ), - new TestModel.TestTaskSettings(randomInt(3)), - new TestModel.TestSecretSettings(randomAlphaOfLength(4)) + base.taskType(), + base.dimensions(), + base.similarity(), + base.elementType() ); } /** - * Creates a {@link TestModel} that is NOT compatible with {@code baseModel}, choosing uniformly among the + * Creates a {@link MinimalServiceSettings} that is NOT compatible with {@code base}, choosing uniformly among the * applicable kinds of incompatibility: task type, dimensions, similarity, element type, or the endpoint not * existing. Setting-based perturbations (dimensions/similarity/element type) only apply to dense base models; * a sparse base model has null dimensions/similarity/element type, so only task-type and does-not-exist apply. * - * @return an incompatible model to register under {@code inferenceId}, or {@code null} to indicate the - * caller should NOT register an endpoint (the does-not-exist case). + * @return new incompatible settings, or {@code null} to indicate the caller should NOT register an endpoint (the does-not-exist case). */ - protected TestModel createIncompatibleModel(String inferenceId, TestModel baseModel) { - final TestModel.TestServiceSettings baseServiceSettings = baseModel.getServiceSettings(); - final DenseVectorFieldMapper.ElementType baseElementType = baseServiceSettings.elementType(); + @Nullable + protected MinimalServiceSettings createIncompatibleModelSettings(MinimalServiceSettings base) { + final DenseVectorFieldMapper.ElementType baseElementType = base.elementType(); List applicable = new ArrayList<>(); if (supportedTaskTypes().size() > 1) { applicable.add(IncompatibilityKind.TASK_TYPE); } applicable.add(IncompatibilityKind.DOES_NOT_EXIST); - if (baseModel.getTaskType() != TaskType.SPARSE_EMBEDDING) { + if (base.taskType() != TaskType.SPARSE_EMBEDDING) { applicable.add(IncompatibilityKind.DIMENSIONS); applicable.add(IncompatibilityKind.ELEMENT_TYPE); // SIMILARITY only when the element type supports more than one option to perturb to @@ -754,17 +799,18 @@ protected TestModel createIncompatibleModel(String inferenceId, TestModel baseMo } } - TaskType taskType = baseModel.getTaskType(); - Integer dimensions = baseServiceSettings.dimensions(); - SimilarityMeasure similarity = baseServiceSettings.similarity(); + TaskType taskType = base.taskType(); + Integer dimensions = base.dimensions(); + SimilarityMeasure similarity = base.similarity(); DenseVectorFieldMapper.ElementType elementType = baseElementType; - boolean returnNull = false; switch (randomFrom(applicable)) { - case DOES_NOT_EXIST -> returnNull = true; + case DOES_NOT_EXIST -> { + return null; + } case TASK_TYPE -> { taskType = randomValueOtherThan(taskType, () -> randomFrom(supportedTaskTypes())); - if (taskType != TaskType.SPARSE_EMBEDDING && baseModel.getTaskType() == TaskType.SPARSE_EMBEDDING) { + if (taskType != TaskType.SPARSE_EMBEDDING && base.taskType() == TaskType.SPARSE_EMBEDDING) { // Sparse -> dense: populate required dense settings from a fresh random dense model TestModel randomDenseModel = TestModel.createRandomInstance(taskType); dimensions = randomDenseModel.getServiceSettings().dimensions(); @@ -797,29 +843,7 @@ protected TestModel createIncompatibleModel(String inferenceId, TestModel baseMo } } - if (returnNull) { - return null; - } - - return buildModel(baseModel, inferenceId, taskType, elementType, dimensions, similarity); - } - - protected static TestModel buildModel( - TestModel baseModel, - String inferenceId, - TaskType taskType, - @Nullable DenseVectorFieldMapper.ElementType elementType, - @Nullable Integer dimensions, - @Nullable SimilarityMeasure similarity - ) { - return new TestModel( - inferenceId, - taskType, - baseModel.getConfigurations().getService(), - new TestModel.TestServiceSettings(baseModel.getServiceSettings().model(), dimensions, similarity, elementType), - baseModel.getTaskSettings(), - baseModel.getSecretSettings() - ); + return new MinimalServiceSettings(base.service(), taskType, dimensions, similarity, elementType); } protected static String randomFieldName(int numLevel) { @@ -832,16 +856,4 @@ protected static String randomFieldName(int numLevel) { } return builder.toString(); } - - private static void validateIndexVersion(IndexVersion indexVersion, boolean useLegacyFormat) { - if (useLegacyFormat == false - && indexVersion.before(IndexVersions.INFERENCE_METADATA_FIELDS) - && indexVersion.between(IndexVersions.INFERENCE_METADATA_FIELDS_BACKPORT, IndexVersions.UPGRADE_TO_LUCENE_10_0_0) == false) { - throw new IllegalArgumentException("Index version " + indexVersion + " does not support new semantic text format"); - } - - if (useLegacyFormat && indexVersion.onOrAfter(IndexVersions.SEMANTIC_TEXT_LEGACY_FORMAT_FORBIDDEN)) { - throw new IllegalArgumentException("Index version " + indexVersion + " does not support legacy semantic text format"); - } - } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticFieldMapperTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticFieldMapperTests.java index 0406419cf9cd6..f6a1f12199efd 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticFieldMapperTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticFieldMapperTests.java @@ -15,7 +15,6 @@ import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.ParsedDocument; @@ -38,6 +37,7 @@ import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xcontent.json.JsonXContent; import org.elasticsearch.xpack.inference.highlight.SemanticTextHighlighter; +import org.elasticsearch.xpack.inference.model.TestModel; import org.elasticsearch.xpack.inference.services.elastic.ElasticInferenceService; import java.io.IOException; @@ -58,49 +58,35 @@ import static org.mockito.Mockito.mock; public class SemanticFieldMapperTests extends AbstractSemanticMapperTestCase { - private static final String INFERENCE_ID = "inference-id"; + private static final TestModel TEST_MODEL = new TestModel( + "test_inference_id", + EMBEDDING, + ElasticInferenceService.NAME, + new TestModel.TestServiceSettings("test_model", 1024, SimilarityMeasure.COSINE, DenseVectorFieldMapper.ElementType.FLOAT), + new TestModel.TestTaskSettings((Integer) null), + new TestModel.TestSecretSettings("secret") + ); public SemanticFieldMapperTests(License.OperationMode operationMode) { super(operationMode); } - @Override - protected void registerDefaultEndpoints() { - registerMultiModalEisEndpoint(); - } - @ParametersFactory public static Iterable parameters() throws Exception { return List.of(new Object[] { License.OperationMode.BASIC }, new Object[] { License.OperationMode.ENTERPRISE }); } - private void registerMultiModalEisEndpoint() { + @Override + protected void registerDefaultEndpoints() { globalModelRegistry.putDefaultIdIfAbsent( new InferenceService.DefaultConfigId( - INFERENCE_ID, - new MinimalServiceSettings( - ElasticInferenceService.NAME, - EMBEDDING, - 1024, - SimilarityMeasure.COSINE, - DenseVectorFieldMapper.ElementType.FLOAT - ), + TEST_MODEL.getInferenceEntityId(), + new MinimalServiceSettings(TEST_MODEL), mock(InferenceService.class) ) ); } - private void minimalMappingWithModelSettings(XContentBuilder b) throws IOException { - MinimalServiceSettings modelSettings = new MinimalServiceSettings( - "test_service", - TaskType.EMBEDDING, - 128, - SimilarityMeasure.COSINE, - DenseVectorFieldMapper.ElementType.FLOAT - ); - addSemanticMapping(b, "my_field", "test_model", null, modelSettings, null, null); - } - /** * In synthetic-source (and columnar) indices, a {@code semantic} field rebuilds {@code _source} from its internal binary doc * values store. Text round-trips as a string; an image (base64 data URI) round-trips as a {@code {type, format, value}} object, @@ -108,7 +94,7 @@ private void minimalMappingWithModelSettings(XContentBuilder b) throws IOExcepti */ public void testOriginalValueRoundTripFromDocValues() throws IOException { MapperService mapperService = createSemanticMapperServiceWithSourceMode( - mapping(this::minimalMappingWithModelSettings), + semanticMapping("my_field", TEST_MODEL.getInferenceEntityId()), IndexVersion.current(), SourceFieldMapper.Mode.SYNTHETIC ); @@ -144,7 +130,7 @@ public void testSemanticFieldAcceptedInColumnar() throws IOException { for (IndexMode indexMode : List.of(IndexMode.COLUMNAR, IndexMode.LOGSDB_COLUMNAR)) { // Mapping creation succeeding is the assertion: the columnar "every field reconstructable from doc values" check passes. MapperService mapperService = createSemanticMapperServiceWithIndexMode( - mapping(this::minimalMappingWithModelSettings), + semanticMapping("my_field", TEST_MODEL.getInferenceEntityId()), IndexVersion.current(), indexMode ); @@ -168,7 +154,7 @@ public void testSemanticFieldAcceptedInColumnar() throws IOException { */ public void testOriginalValueFetchedFromDocValues() throws IOException { MapperService mapperService = createSemanticMapperServiceWithSourceMode( - mapping(this::minimalMappingWithModelSettings), + semanticMapping("my_field", TEST_MODEL.getInferenceEntityId()), IndexVersion.current(), SourceFieldMapper.Mode.SYNTHETIC ); @@ -237,7 +223,7 @@ private Object expectedFetchedValue(Object input) throws IOException { */ public void testBooleanAndNumericValuesRoundTripAsStrings() throws IOException { MapperService mapperService = createSemanticMapperServiceWithSourceMode( - mapping(this::minimalMappingWithModelSettings), + semanticMapping("my_field", TEST_MODEL.getInferenceEntityId()), IndexVersion.current(), SourceFieldMapper.Mode.SYNTHETIC ); @@ -267,7 +253,7 @@ public void testHighlighterAvoidsSourceWhenValuesInDocValues() throws IOExceptio SearchExecutionContext syntheticContext = createSearchExecutionContext( createSemanticMapperServiceWithSourceMode( - mapping(this::minimalMappingWithModelSettings), + semanticMapping("my_field", TEST_MODEL.getInferenceEntityId()), version, SourceFieldMapper.Mode.SYNTHETIC ) @@ -276,7 +262,7 @@ public void testHighlighterAvoidsSourceWhenValuesInDocValues() throws IOExceptio SearchExecutionContext storedContext = createSearchExecutionContext( createSemanticMapperServiceWithSourceMode( - mapping(this::minimalMappingWithModelSettings), + semanticMapping("my_field", TEST_MODEL.getInferenceEntityId()), version, SourceFieldMapper.Mode.STORED ) @@ -294,25 +280,32 @@ private static String imageObject(String dataUri) throws IOException { return Strings.toString(b); } - public void testSemanticFieldNotSupportedOnOldIndices() throws IOException { - IndexVersion oldVersion = IndexVersionUtils.randomPreviousCompatibleVersion(IndexVersions.SEMANTIC_FIELD_TYPE); + public void testSemanticFieldNotSupportedOnOldIndices() { + for (int i = 0; i < 20; i++) { + IndexVersion oldVersion = IndexVersionUtils.randomPreviousCompatibleVersion(IndexVersions.SEMANTIC_FIELD_TYPE); - var ex = expectThrows( - MapperParsingException.class, - () -> createSemanticMapperServiceWithIndexVersion(semanticMapping("my_field", "test_model"), oldVersion) - ); - assertThat(ex.getMessage(), containsString("[" + SemanticFieldMapper.CONTENT_TYPE + "]")); - assertThat(ex.getMessage(), containsString("is not supported on indices created before version")); - assertThat(ex.getMessage(), containsString(IndexVersions.SEMANTIC_FIELD_TYPE.toString())); + var ex = expectThrows( + MapperParsingException.class, + () -> createSemanticMapperServiceWithIndexVersion(semanticMapping("my_field", "test_model"), oldVersion) + ); + assertThat(ex.getMessage(), containsString("[" + SemanticFieldMapper.CONTENT_TYPE + "]")); + assertThat(ex.getMessage(), containsString("is not supported on indices created before version")); + assertThat(ex.getMessage(), containsString(IndexVersions.SEMANTIC_FIELD_TYPE.toString())); + } } public void testSemanticFieldSupportedOnNewIndices() throws IOException { - IndexVersion newVersion = IndexVersionUtils.randomVersionOnOrAfter(IndexVersions.SEMANTIC_FIELD_TYPE); + for (int i = 0; i < 20; i++) { + IndexVersion newVersion = IndexVersionUtils.randomVersionOnOrAfter(IndexVersions.SEMANTIC_FIELD_TYPE); - // Should not throw; model_settings provided to avoid consulting the model registry - var mapperService = createSemanticMapperServiceWithIndexVersion(mapping(this::minimalMappingWithModelSettings), newVersion); - assertNotNull(mapperService); - assertSemanticFieldMapper(mapperService, "my_field"); + // Should not throw; model_settings provided to avoid consulting the model registry + var mapperService = createSemanticMapperServiceWithIndexVersion( + semanticMapping("my_field", TEST_MODEL.getInferenceEntityId()), + newVersion + ); + assertNotNull(mapperService); + assertSemanticField(mapperService, "my_field", false, new MinimalServiceSettings(TEST_MODEL), null, null); + } } public void testSemanticFieldMappingUpdateNotSupportedOnOldIndices() throws IOException { @@ -332,14 +325,9 @@ public void testSemanticFieldMappingUpdateSupportedOnNewIndices() throws IOExcep var mapperService = createSemanticMapperServiceWithIndexVersion(mapping(b -> {}), newVersion); assertNotNull(mapperService); // Should not throw; model_settings provided to avoid consulting the model registry - merge(mapperService, mapping(this::minimalMappingWithModelSettings)); - - assertSemanticFieldMapper(mapperService, "my_field"); - } + merge(mapperService, semanticMapping("my_field", TEST_MODEL.getInferenceEntityId())); - private static void assertSemanticFieldMapper(MapperService mapperService, String fieldName) { - Mapper mapper = mapperService.mappingLookup().getMapper(fieldName); - assertThat(mapper, instanceOf(SemanticFieldMapper.class)); + assertSemanticField(mapperService, "my_field", false, new MinimalServiceSettings(TEST_MODEL), null, null); } @Override @@ -364,13 +352,14 @@ protected Set supportedTaskTypes() { @Override protected IndexVersion getRandomCompatibleIndexVersion() { + // TODO: Bias towards IndexVersion.current() return IndexVersionUtils.randomVersionOnOrAfter(IndexVersions.SEMANTIC_FIELD_TYPE); } @Override protected void minimalMapping(XContentBuilder b) throws IOException { - b.field("type", "semantic"); - b.field("inference_id", INFERENCE_ID); + b.field("type", contentType()); + b.field("inference_id", TEST_MODEL.getInferenceEntityId()); } @Override @@ -422,9 +411,7 @@ public void testInvalidInferenceEndpoints() { Exception e = expectThrows( MapperParsingException.class, () -> createMapperService( - fieldMapping( - b -> b.field("type", "semantic").field(INFERENCE_ID_FIELD, INFERENCE_ID).field(SEARCH_INFERENCE_ID_FIELD, "") - ) + fieldMapping(b -> b.field("type", "semantic").field(INFERENCE_ID_FIELD, "foo").field(SEARCH_INFERENCE_ID_FIELD, "")) ) ); assertThat(e.getMessage(), containsString("[search_inference_id] on mapper [field] of type [semantic] must not be empty")); diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java index e2154555fb97b..5f05c825221a7 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java @@ -28,7 +28,6 @@ import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.CheckedBiConsumer; -import org.elasticsearch.common.CheckedBiFunction; import org.elasticsearch.common.Strings; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.lucene.search.Queries; @@ -57,8 +56,8 @@ import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; import org.elasticsearch.index.mapper.vectors.DenseVectorFieldTypeTests; +import org.elasticsearch.index.mapper.vectors.IndexOptions; import org.elasticsearch.index.mapper.vectors.SparseVectorFieldMapper; -import org.elasticsearch.index.mapper.vectors.SparseVectorFieldMapperTests; import org.elasticsearch.index.mapper.vectors.SparseVectorFieldTypeTests; import org.elasticsearch.index.mapper.vectors.TokenPruningConfig; import org.elasticsearch.index.query.SearchExecutionContext; @@ -85,6 +84,7 @@ import org.elasticsearch.xpack.inference.highlight.SemanticTextHighlighter; import org.elasticsearch.xpack.inference.model.TestModel; import org.elasticsearch.xpack.inference.services.elastic.ElasticInferenceService; +import org.elasticsearch.xpack.inference.services.elasticsearch.ElasticsearchInternalService; import java.io.IOException; import java.util.ArrayList; @@ -98,15 +98,12 @@ import java.util.function.BiConsumer; import java.util.stream.Collectors; -import static org.elasticsearch.index.IndexSettings.DENSE_VECTOR_EXPERIMENTAL_FEATURES_SETTING; -import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapperTestUtils.defaultDenseVectorIndexOptions; import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldTypeTests.randomIndexOptionsAll; import static org.elasticsearch.index.mapper.vectors.SparseVectorFieldTypeTests.randomSparseVectorIndexOptions; import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.CHUNKS_FIELD; import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.INFERENCE_FIELD; import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.INFERENCE_ID_FIELD; import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.MODEL_SETTINGS_FIELD; -import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.SEARCH_INFERENCE_ID_FIELD; import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.TEXT_FIELD; import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.getChunksFieldName; import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.getEmbeddingsFieldName; @@ -132,6 +129,37 @@ public class SemanticTextFieldMapperTests extends AbstractSemanticMapperTestCase SemanticTextFieldMapper, SemanticTextFieldMapper.SemanticTextFieldType> { + private static final TestModel JINA_V5_TEXT_MODEL = new TestModel( + DEFAULT_EIS_JINA_V5_INFERENCE_ID, + TaskType.TEXT_EMBEDDING, + ElasticInferenceService.NAME, + new TestModel.TestServiceSettings("jina", 1024, SimilarityMeasure.COSINE, DenseVectorFieldMapper.ElementType.FLOAT), + new TestModel.TestTaskSettings((Integer) null), + new TestModel.TestSecretSettings("foo") + ); + private static final TestModel DEFAULT_EIS_ELSER_MODEL = new TestModel( + DEFAULT_EIS_ELSER_INFERENCE_ID, + TaskType.SPARSE_EMBEDDING, + ElasticInferenceService.NAME, + new TestModel.TestServiceSettings("elser", null, null, null), + new TestModel.TestTaskSettings((Integer) null), + new TestModel.TestSecretSettings("foo") + ); + private static final TestModel DEFAULT_FALLBACK_ELSER_MODEL = new TestModel( + DEFAULT_FALLBACK_ELSER_INFERENCE_ID, + TaskType.SPARSE_EMBEDDING, + ElasticsearchInternalService.NAME, + new TestModel.TestServiceSettings("elser", null, null, null), + new TestModel.TestTaskSettings((Integer) null), + new TestModel.TestSecretSettings("foo") + ); + + private static final List DEFAULT_MODELS = List.of( + JINA_V5_TEXT_MODEL, + DEFAULT_EIS_ELSER_MODEL, + DEFAULT_FALLBACK_ELSER_MODEL + ); + private final boolean useLegacyFormat; private final IndexVersion testIndexVersion; @@ -147,7 +175,21 @@ public SemanticTextFieldMapperTests(boolean useLegacyFormat, License.OperationMo @Override protected void registerDefaultEndpoints() { - registerDefaultEisEndpoint(); + for (TestModel model : DEFAULT_MODELS) { + globalModelRegistry.putDefaultIdIfAbsent( + new InferenceService.DefaultConfigId( + model.getInferenceEntityId(), + new MinimalServiceSettings(model), + mock(InferenceService.class) + ) + ); + } + } + + private void removeDefaultEndpoints(Set inferenceIds) { + PlainActionFuture removalFuture = new PlainActionFuture<>(); + globalModelRegistry.removeDefaultConfigs(inferenceIds, removalFuture); + safeGet(removalFuture); } @ParametersFactory @@ -167,21 +209,6 @@ protected String minimalIsInvalidRoutingPathErrorMessage(Mapper mapper) { return super.minimalIsInvalidRoutingPathErrorMessage(mapper); } - private void registerDefaultEisEndpoint() { - globalModelRegistry.putDefaultIdIfAbsent( - new InferenceService.DefaultConfigId( - DEFAULT_EIS_JINA_V5_INFERENCE_ID, - MinimalServiceSettings.textEmbedding( - ElasticInferenceService.NAME, - 1024, - SimilarityMeasure.COSINE, - DenseVectorFieldMapper.ElementType.FLOAT - ), - mock(InferenceService.class) - ) - ); - } - @Override protected boolean useLegacyFormat() { return useLegacyFormat; @@ -235,18 +262,13 @@ protected void metaMapping(XContentBuilder b) throws IOException { b.field(INFERENCE_ID_FIELD, DEFAULT_EIS_JINA_V5_INFERENCE_ID); } - protected void extendedMapping(XContentBuilder b, Map extensions) throws IOException { - minimalMapping(b); - b.mapContents(extensions); - } - @Override protected Object getSampleObjectForDocument() { // Only consulted by testSupportsParsingObject, and only for the legacy format (supportsParsingObject() is true only then). A // legacy value is the {text, inference} object; a minimal one with the resolved inference id and no inference results parses. final String expectedInferenceId = testIndexVersion.onOrAfter(IndexVersions.SEMANTIC_TEXT_DEFAULTS_TO_JINA_V5) ? DEFAULT_EIS_JINA_V5_INFERENCE_ID - : DEFAULT_FALLBACK_ELSER_INFERENCE_ID; + : DEFAULT_EIS_ELSER_INFERENCE_ID; return Map.of( SemanticTextField.TEXT_FIELD, randomAlphaOfLength(10), @@ -277,6 +299,31 @@ protected void assertSearchable(MappedFieldType fieldType) { assertTrue(fieldType.isSearchable()); } + @Override + protected DenseVectorFieldMapper.DenseVectorIndexOptions getExplicitDenseVectorIndexOptions( + MinimalServiceSettings modelSettings, + IndexVersion indexVersion + ) { + TaskType taskType = modelSettings.taskType(); + DenseVectorFieldMapper.ElementType elementType = modelSettings.elementType(); + Integer dimensions = modelSettings.dimensions(); + + if (taskType == TaskType.SPARSE_EMBEDDING) { + return null; + } + + DenseVectorFieldMapper.DenseVectorIndexOptions indexOptions = null; + boolean floatFamilyElementType = elementType == DenseVectorFieldMapper.ElementType.FLOAT + || elementType == DenseVectorFieldMapper.ElementType.BFLOAT16; + if (floatFamilyElementType + && SemanticTextFieldMapper.setExplicitIndexOptionsForSemanticText(indexVersion) + && dimensions >= DenseVectorFieldMapper.BBQ_MIN_DIMS) { + indexOptions = defaultBbqHnswDenseVectorIndexOptions(); + } + + return indexOptions; + } + /** * Randomized sweep over all combinations of endpoint availability and * index version to confirm correct default inference ID. @@ -289,7 +336,7 @@ public void testDefaults() throws Exception { MapperService mapperService = createSemanticMapperService(fieldMapping, IndexVersions.SEMANTIC_TEXT_DEFAULTS_TO_JINA_V5); DocumentMapper mapper = mapperService.documentMapper(); assertEquals(Strings.toString(expectedMapping), mapper.mappingSource().toString()); - assertSemanticField(mapperService, fieldName, false, null, null); + assertSemanticField(mapperService, fieldName, false, new MinimalServiceSettings(JINA_V5_TEXT_MODEL), null, null); assertInferenceEndpoints(mapperService, fieldName, DEFAULT_EIS_JINA_V5_INFERENCE_ID, DEFAULT_EIS_JINA_V5_INFERENCE_ID); ParsedDocument doc1 = mapper.parse(source(this::writeField)); @@ -299,32 +346,66 @@ public void testDefaults() throws Exception { assertTrue(fields.isEmpty()); for (int i = 0; i < 20; i++) { + registerDefaultEndpoints(); + boolean jinaAvailable = randomBoolean(); boolean eisElserAvailable = randomBoolean(); boolean postJinaV5 = randomBoolean(); - setDefaultEisEndpoints(jinaAvailable, eisElserAvailable); - - IndexVersion indexVersion = postJinaV5 - ? IndexVersionUtils.randomVersionBetween( + final IndexVersion indexVersion; + if (postJinaV5) { + indexVersion = IndexVersionUtils.randomVersionBetween( IndexVersions.SEMANTIC_TEXT_DEFAULTS_TO_JINA_V5, useLegacyFormat ? IndexVersionUtils.getPreviousVersion(IndexVersions.SEMANTIC_TEXT_LEGACY_FORMAT_FORBIDDEN) : IndexVersion.current() - ) - : IndexVersionUtils.randomPreviousCompatibleWriteVersion(IndexVersions.SEMANTIC_TEXT_DEFAULTS_TO_JINA_V5); + ); + } else if (useLegacyFormat) { + // Legacy format: any write-compatible version below JinaV5 + indexVersion = IndexVersionUtils.randomVersionBetween( + IndexVersions.MINIMUM_COMPATIBLE, + IndexVersionUtils.getPreviousVersion(IndexVersions.SEMANTIC_TEXT_DEFAULTS_TO_JINA_V5) + ); + } else if (randomBoolean()) { + // New format, 9.x, pre-JinaV5 + indexVersion = IndexVersionUtils.randomVersionBetween( + IndexVersions.INFERENCE_METADATA_FIELDS, + IndexVersionUtils.getPreviousVersion(IndexVersions.SEMANTIC_TEXT_DEFAULTS_TO_JINA_V5) + ); + } else { + // New format, 8.x backport window + indexVersion = IndexVersionUtils.randomVersionBetween( + IndexVersions.INFERENCE_METADATA_FIELDS_BACKPORT, + IndexVersionUtils.getPreviousVersion(IndexVersions.UPGRADE_TO_LUCENE_10_0_0) + ); + } - String expectedId; + Set unavailableEndpoints = new HashSet<>(); + if (jinaAvailable == false) { + unavailableEndpoints.add(DEFAULT_EIS_JINA_V5_INFERENCE_ID); + } + if (eisElserAvailable == false) { + unavailableEndpoints.add(DEFAULT_EIS_ELSER_INFERENCE_ID); + } + removeDefaultEndpoints(unavailableEndpoints); + + TestModel expectedModel; if (jinaAvailable && postJinaV5) { - expectedId = DEFAULT_EIS_JINA_V5_INFERENCE_ID; + expectedModel = JINA_V5_TEXT_MODEL; } else if (eisElserAvailable) { - expectedId = DEFAULT_EIS_ELSER_INFERENCE_ID; + expectedModel = DEFAULT_EIS_ELSER_MODEL; } else { - expectedId = DEFAULT_FALLBACK_ELSER_INFERENCE_ID; + expectedModel = DEFAULT_FALLBACK_ELSER_MODEL; } MapperService iterMapperService = createSemanticMapperServiceWithIndexVersion(fieldMapping, indexVersion); - assertInferenceEndpoints(iterMapperService, fieldName, expectedId, expectedId); + assertSemanticField(iterMapperService, fieldName, false, new MinimalServiceSettings(expectedModel), null, null); + assertInferenceEndpoints( + iterMapperService, + fieldName, + expectedModel.getInferenceEntityId(), + expectedModel.getInferenceEntityId() + ); } } @@ -466,7 +547,7 @@ public void testHighlighterRequiresSourceWhenCopyToSourceFieldNeedsIt() throws I assumeFalse("the legacy format keeps the original value in _source", useLegacyFormat); MapperService mapperService = createSemanticMapperServiceWithSourceMode(mapping(b -> { b.startObject("body").field("type", "text").field("copy_to", "field").endObject(); - b.startObject("field").field("type", "semantic_text").field("inference_id", "test_model").endObject(); + addSemanticMapping(b, "field", "test_model", null, null, null, null); }), IndexVersion.current(), SourceFieldMapper.Mode.SYNTHETIC); SearchExecutionContext context = createSearchExecutionContext(mapperService); assertFalse(new SemanticTextHighlighter().canHighlightWithoutSource(context.getFieldType("field"), context)); @@ -516,29 +597,6 @@ public void testEmptyDefaultInferenceIdSettingThrows() throws Exception { } } - /** - * Resets the model registry to exactly the given endpoint availability for each iteration of the - * randomized sweep in {@link #testDefaults()}. - */ - private void setDefaultEisEndpoints(boolean jinaEnabled, boolean eisElserEnabled) { - PlainActionFuture removalFuture = new PlainActionFuture<>(); - globalModelRegistry.removeDefaultConfigs(Set.of(DEFAULT_EIS_JINA_V5_INFERENCE_ID, DEFAULT_EIS_ELSER_INFERENCE_ID), removalFuture); - removalFuture.actionGet(TEST_REQUEST_TIMEOUT); - - if (jinaEnabled) { - registerDefaultEisEndpoint(); - } - if (eisElserEnabled) { - globalModelRegistry.putDefaultIdIfAbsent( - new InferenceService.DefaultConfigId( - DEFAULT_EIS_ELSER_INFERENCE_ID, - MinimalServiceSettings.sparseEmbedding(ElasticInferenceService.NAME), - mock(InferenceService.class) - ) - ); - } - } - @Override public void testFieldHasValue() { MappedFieldType fieldType = getMappedFieldType(); @@ -559,24 +617,22 @@ public void testSetInferenceEndpoints() throws IOException { { final XContentBuilder fieldMapping = semanticMapping("field", inferenceId); final MapperService mapperService = createSemanticMapperService(fieldMapping); - assertSemanticField(mapperService, fieldName, false, null, null); + assertSemanticField(mapperService, fieldName, false, null, null, null); assertInferenceEndpoints(mapperService, fieldName, inferenceId, inferenceId); assertSerialization.accept(fieldMapping, mapperService); } { - final XContentBuilder fieldMapping = fieldMapping( - b -> b.field("type", "semantic_text").field(SEARCH_INFERENCE_ID_FIELD, searchInferenceId) - ); + final XContentBuilder fieldMapping = semanticMapping("field", null, searchInferenceId); final XContentBuilder expectedMapping = semanticMapping("field", DEFAULT_EIS_JINA_V5_INFERENCE_ID, searchInferenceId); final MapperService mapperService = createSemanticMapperService(fieldMapping, IndexVersions.SEMANTIC_TEXT_DEFAULTS_TO_JINA_V5); - assertSemanticField(mapperService, fieldName, false, null, null); + assertSemanticField(mapperService, fieldName, false, new MinimalServiceSettings(JINA_V5_TEXT_MODEL), null, null); assertInferenceEndpoints(mapperService, fieldName, DEFAULT_EIS_JINA_V5_INFERENCE_ID, searchInferenceId); assertSerialization.accept(expectedMapping, mapperService); } { final XContentBuilder fieldMapping = semanticMapping("field", inferenceId, searchInferenceId); MapperService mapperService = createSemanticMapperService(fieldMapping); - assertSemanticField(mapperService, fieldName, false, null, null); + assertSemanticField(mapperService, fieldName, false, null, null, null); assertInferenceEndpoints(mapperService, fieldName, inferenceId, searchInferenceId); assertSerialization.accept(fieldMapping, mapperService); } @@ -596,41 +652,28 @@ public void testInvalidInferenceEndpoints() { ); } { - Exception e = expectThrows( - MapperParsingException.class, - () -> createSemanticMapperService(fieldMapping(b -> b.field("type", "semantic_text").field(INFERENCE_ID_FIELD, ""))) - ); + Exception e = expectThrows(MapperParsingException.class, () -> createSemanticMapperService(semanticMapping("field", ""))); assertThat(e.getMessage(), containsString("[inference_id] on mapper [field] of type [semantic_text] must not be empty")); } { - Exception e = expectThrows( - MapperParsingException.class, - () -> createSemanticMapperService(fieldMapping(b -> b.field("type", "semantic_text").field(SEARCH_INFERENCE_ID_FIELD, ""))) - ); + Exception e = expectThrows(MapperParsingException.class, () -> createSemanticMapperService(semanticMapping("field", null, ""))); assertThat(e.getMessage(), containsString("[search_inference_id] on mapper [field] of type [semantic_text] must not be empty")); } } - private SemanticIndexOptions getDefaultSparseVectorIndexOptionsForMapper(MapperService mapperService) { - var mapperIndexVersion = mapperService.getIndexSettings().getIndexVersionCreated(); - var defaultSparseVectorIndexOptions = SparseVectorFieldMapper.SparseVectorIndexOptions.getDefaultIndexOptions(mapperIndexVersion); - return defaultSparseVectorIndexOptions == null - ? null - : new SemanticIndexOptions(SemanticIndexOptions.SupportedIndexOptions.SPARSE_VECTOR, defaultSparseVectorIndexOptions); - } - public void testInvalidTaskTypes() { for (var taskType : TaskType.values()) { if (taskType == TaskType.TEXT_EMBEDDING || taskType == TaskType.SPARSE_EMBEDDING || taskType == TaskType.EMBEDDING) { continue; } - Exception e = expectThrows(MapperParsingException.class, () -> createSemanticMapperService(fieldMapping(b -> { - b.field("type", "semantic_text") - .field(INFERENCE_ID_FIELD, "test1") - .startObject("model_settings") - .field("task_type", taskType); - b.endObject(); - }))); + Exception e = expectThrows( + MapperParsingException.class, + () -> createSemanticMapperService(semanticMapping("field", "test1", null, null, null, null, b -> { + b.startObject("model_settings"); + b.field("task_type", taskType); + b.endObject(); + })) + ); assertThat(e.getMessage(), containsString("Wrong [task_type], expected text_embedding, embedding, or sparse_embedding")); } } @@ -643,13 +686,14 @@ public void testMultiFieldNamedInputIsRejected() throws IOException { IndexVersions.SEMANTIC_TEXT_ORIGINAL_VALUES_DOC_VALUES, IndexVersion.current() ); - Exception e = expectThrows(MapperParsingException.class, () -> createSemanticMapperServiceWithIndexVersion(fieldMapping(b -> { - b.field("type", "semantic_text"); - b.field("inference_id", "my_inference_id"); - b.startObject("fields"); - b.startObject("input").field("type", "keyword").endObject(); - b.endObject(); - }), indexVersion)); + Exception e = expectThrows( + MapperParsingException.class, + () -> createSemanticMapperServiceWithIndexVersion(semanticMapping("field", "my_inference_id", null, null, null, null, b -> { + b.startObject("fields"); + b.startObject("input").field("type", "keyword").endObject(); + b.endObject(); + }), indexVersion) + ); assertThat(e.getMessage(), containsString("cannot have a multi-field named [input]")); } @@ -658,13 +702,14 @@ public void testMultiFieldNamedInputIsAccepted() throws IOException { // a multi-field named [input] stays valid. This verifies the gating does not invalidate indices created at an older version. assumeFalse("multi-fields are rejected entirely in the legacy format", useLegacyFormat); IndexVersion indexVersion = IndexVersionUtils.getPreviousVersion(IndexVersions.SEMANTIC_TEXT_ORIGINAL_VALUES_DOC_VALUES); - MapperService mapperService = createSemanticMapperServiceWithIndexVersion(fieldMapping(b -> { - b.field("type", "semantic_text"); - b.field("inference_id", "my_inference_id"); - b.startObject("fields"); - b.startObject("input").field("type", "keyword").endObject(); - b.endObject(); - }), indexVersion); + MapperService mapperService = createSemanticMapperServiceWithIndexVersion( + semanticMapping("field", "my_inference_id", null, null, null, null, b -> { + b.startObject("fields"); + b.startObject("input").field("type", "keyword").endObject(); + b.endObject(); + }), + indexVersion + ); assertNotNull(mapperService.mappingLookup().getMapper("field.input")); } @@ -673,76 +718,47 @@ public void testMultiFieldsSupport() throws IOException { Exception e = expectThrows(MapperParsingException.class, () -> createSemanticMapperService(fieldMapping(b -> { b.field("type", "text"); b.startObject("fields"); - b.startObject("semantic"); - b.field("type", "semantic_text"); - b.field("inference_id", "my_inference_id"); - b.endObject(); + addSemanticMapping(b, "semantic", "my_inference_id", null, null, null, null); b.endObject(); }))); assertThat(e.getMessage(), containsString("Field [semantic] of type [semantic_text] can't be used in multifields")); } else { - IndexVersion indexVersion = SparseVectorFieldMapperTests.getIndexOptionsCompatibleIndexVersion(); - SparseVectorFieldMapper.SparseVectorIndexOptions expectedIndexOptions = SparseVectorFieldMapper.SparseVectorIndexOptions - .getDefaultIndexOptions(indexVersion); - SemanticIndexOptions semanticIndexOptions = expectedIndexOptions == null - ? null - : new SemanticIndexOptions(SemanticIndexOptions.SupportedIndexOptions.SPARSE_VECTOR, expectedIndexOptions); - var mapperService = createSemanticMapperServiceWithIndexVersion(fieldMapping(b -> { - b.field("type", "text"); - b.startObject("fields"); - b.startObject("semantic"); - b.field("type", "semantic_text"); - b.field("inference_id", "my_inference_id"); - b.startObject("model_settings"); - b.field("task_type", "sparse_embedding"); - b.endObject(); - b.endObject(); - b.endObject(); - }), indexVersion); - assertSemanticField(mapperService, "field.semantic", true, null, semanticIndexOptions); - - mapperService = createSemanticMapperServiceWithIndexVersion(fieldMapping(b -> { - b.field("type", "semantic_text"); - b.field("inference_id", "my_inference_id"); - b.startObject("model_settings"); - b.field("task_type", "sparse_embedding"); - b.endObject(); - b.startObject("fields"); - b.startObject("text"); + MinimalServiceSettings sparseModelSettings = createRandomModelSettings(TaskType.SPARSE_EMBEDDING); + var mapperService = createSemanticMapperService(fieldMapping(b -> { b.field("type", "text"); - b.endObject(); - b.endObject(); - }), indexVersion); - assertSemanticField(mapperService, "field", true, null, semanticIndexOptions); - - mapperService = createSemanticMapperServiceWithIndexVersion(fieldMapping(b -> { - b.field("type", "semantic_text"); - b.field("inference_id", "my_inference_id"); - b.startObject("model_settings"); - b.field("task_type", "sparse_embedding"); - b.endObject(); b.startObject("fields"); - b.startObject("semantic"); - b.field("type", "semantic_text"); - b.field("inference_id", "another_inference_id"); - b.startObject("model_settings"); - b.field("task_type", "sparse_embedding"); - b.endObject(); - b.endObject(); + addSemanticMapping(b, "semantic", "my_inference_id", null, sparseModelSettings, null, null); b.endObject(); - }), indexVersion); - assertSemanticField(mapperService, "field", true, null, semanticIndexOptions); - assertSemanticField(mapperService, "field.semantic", true, null, semanticIndexOptions); + })); + assertSemanticField(mapperService, "field.semantic", true, sparseModelSettings, null, null); - Exception e = expectThrows(MapperParsingException.class, () -> createSemanticMapperService(fieldMapping(b -> { - b.field("type", "semantic_text"); - b.field("inference_id", "my_inference_id"); - b.startObject("fields"); - b.startObject("inference"); - b.field("type", "text"); - b.endObject(); - b.endObject(); - }))); + mapperService = createSemanticMapperService( + semanticMapping("field", "my_inference_id", null, sparseModelSettings, null, null, b -> { + b.startObject("fields"); + b.startObject("text").field("type", "text").endObject(); + b.endObject(); + }) + ); + assertSemanticField(mapperService, "field", true, sparseModelSettings, null, null); + + mapperService = createSemanticMapperService( + semanticMapping("field", "my_inference_id", null, sparseModelSettings, null, null, b -> { + b.startObject("fields"); + addSemanticMapping(b, "semantic", "another_inference_id", null, sparseModelSettings, null, null); + b.endObject(); + }) + ); + assertSemanticField(mapperService, "field", true, sparseModelSettings, null, null); + assertSemanticField(mapperService, "field.semantic", true, sparseModelSettings, null, null); + + Exception e = expectThrows( + MapperParsingException.class, + () -> createSemanticMapperService(semanticMapping("field", "my_inference_id", null, null, null, null, b -> { + b.startObject("fields"); + b.startObject("inference").field("type", "text").endObject(); + b.endObject(); + })) + ); assertThat(e.getMessage(), containsString("is already used by another field")); } } @@ -751,30 +767,19 @@ public void testDynamicUpdate() throws IOException { final String fieldName = "semantic"; final String inferenceId = "test_service"; final String searchInferenceId = "search_test_service"; + final MinimalServiceSettings sparseModelSettings = createRandomModelSettings(TaskType.SPARSE_EMBEDDING); { MapperService mapperService = createSemanticMapperService(semanticMapping(fieldName, inferenceId)); - performDynamicUpdate( - mapperService, - fieldName, - inferenceId, - new MinimalServiceSettings("service", TaskType.SPARSE_EMBEDDING, null, null, null) - ); - var expectedIndexOptions = getDefaultSparseVectorIndexOptionsForMapper(mapperService); - assertSemanticField(mapperService, fieldName, true, null, expectedIndexOptions); + performDynamicUpdate(mapperService, fieldName, inferenceId, sparseModelSettings); + assertSemanticField(mapperService, fieldName, true, sparseModelSettings, null, null); assertInferenceEndpoints(mapperService, fieldName, inferenceId, inferenceId); } { MapperService mapperService = createSemanticMapperService(semanticMapping(fieldName, inferenceId, searchInferenceId)); - performDynamicUpdate( - mapperService, - fieldName, - inferenceId, - new MinimalServiceSettings("service", TaskType.SPARSE_EMBEDDING, null, null, null) - ); - var expectedIndexOptions = getDefaultSparseVectorIndexOptionsForMapper(mapperService); - assertSemanticField(mapperService, fieldName, true, null, expectedIndexOptions); + performDynamicUpdate(mapperService, fieldName, inferenceId, sparseModelSettings); + assertSemanticField(mapperService, fieldName, true, sparseModelSettings, null, null); assertInferenceEndpoints(mapperService, fieldName, inferenceId, searchInferenceId); } } @@ -782,56 +787,33 @@ public void testDynamicUpdate() throws IOException { public void testUpdateModelSettings() throws IOException { for (int depth = 1; depth < 5; depth++) { String fieldName = randomFieldName(depth); + MinimalServiceSettings sparseSettings = createRandomModelSettings(TaskType.SPARSE_EMBEDDING); + MinimalServiceSettings denseSettings = createRandomModelSettings(TaskType.TEXT_EMBEDDING); MapperService mapperService = createSemanticMapperService(semanticMapping(fieldName, "test_model")); - assertSemanticField(mapperService, fieldName, false, null, null); + assertSemanticField(mapperService, fieldName, false, null, null, null); { Exception exc = expectThrows( MapperParsingException.class, - () -> merge( - mapperService, - mapping( - b -> b.startObject(fieldName) - .field("type", "semantic_text") - .field("inference_id", "test_model") - .startObject("model_settings") - .field("inference_id", "test_model") - .endObject() - .endObject() - ) - ) + () -> merge(mapperService, semanticMapping(fieldName, "test_model", null, null, null, null, b -> { + b.startObject("model_settings"); + b.field("inference_id", "test_model"); + b.endObject(); + })) ); assertThat(exc.getMessage(), containsString("Required [task_type]")); } { - merge( - mapperService, - semanticMapping(fieldName, "test_model", new MinimalServiceSettings(null, TaskType.SPARSE_EMBEDDING, null, null, null)) - ); - var expectedIndexOptions = getDefaultSparseVectorIndexOptionsForMapper(mapperService); - assertSemanticField(mapperService, fieldName, true, null, expectedIndexOptions); + merge(mapperService, semanticMapping(fieldName, "test_model", sparseSettings)); + assertSemanticField(mapperService, fieldName, true, sparseSettings, null, null); } { merge(mapperService, semanticMapping(fieldName, "test_model")); - var expectedIndexOptions = getDefaultSparseVectorIndexOptionsForMapper(mapperService); - assertSemanticField(mapperService, fieldName, true, null, expectedIndexOptions); + assertSemanticField(mapperService, fieldName, true, sparseSettings, null, null); } { Exception exc = expectThrows( IllegalArgumentException.class, - () -> merge( - mapperService, - semanticMapping( - fieldName, - "test_model", - new MinimalServiceSettings( - null, - TaskType.TEXT_EMBEDDING, - 10, - SimilarityMeasure.COSINE, - DenseVectorFieldMapper.ElementType.FLOAT - ) - ) - ) + () -> merge(mapperService, semanticMapping(fieldName, "test_model", denseSettings)) ); assertThat(exc.getMessage(), containsString("cannot be changed from type [sparse_vector] to [dense_vector]")); } @@ -844,72 +826,37 @@ public void testDenseVectorIndexOptionValidation() throws IOException { String fieldName = randomFieldName(depth); DenseVectorFieldMapper.DenseVectorIndexOptions indexOptions = DenseVectorFieldTypeTests.randomIndexOptionsAll(); - Exception exc = expectThrows(MapperParsingException.class, () -> createSemanticMapperService(mapping(b -> { - b.startObject(fieldName); - b.field("type", SemanticTextFieldMapper.CONTENT_TYPE); - b.field(INFERENCE_ID_FIELD, inferenceId); - b.startObject(INDEX_OPTIONS_FIELD); - b.startObject("dense_vector"); - b.field("type", indexOptions.getType().name().toLowerCase(Locale.ROOT)); - b.field("unsupported_param", "any_value"); - b.endObject(); - b.endObject(); - b.endObject(); - }))); + Exception exc = expectThrows( + MapperParsingException.class, + () -> createSemanticMapperService(semanticMapping(fieldName, inferenceId, null, null, null, null, b -> { + b.startObject(INDEX_OPTIONS_FIELD); + b.startObject("dense_vector"); + b.field("type", indexOptions.getType().name().toLowerCase(Locale.ROOT)); + b.field("unsupported_param", "any_value"); + b.endObject(); + b.endObject(); + })) + ); assertTrue(exc.getMessage().contains("unsupported parameters")); } } - private void addSparseVectorModelSettingsToBuilder(XContentBuilder b) throws IOException { - b.startObject("model_settings"); - b.field("task_type", TaskType.SPARSE_EMBEDDING); - b.endObject(); - } - public void testSparseVectorIndexOptionsValidationAndMapping() throws IOException { for (int depth = 1; depth < 5; depth++) { String inferenceId = "test_model"; String fieldName = randomFieldName(depth); - IndexVersion indexVersion; - if (useLegacyFormat) { - // Must be in a range that supports both sparse vector index options and legacy format - indexVersion = randomBoolean() - ? IndexVersionUtils.randomVersionBetween( - IndexVersions.SPARSE_VECTOR_PRUNING_INDEX_OPTIONS_SUPPORT, - IndexVersionUtils.getPreviousVersion(IndexVersions.SEMANTIC_TEXT_LEGACY_FORMAT_FORBIDDEN) - ) - : IndexVersionUtils.randomVersionBetween( - IndexVersions.SPARSE_VECTOR_PRUNING_INDEX_OPTIONS_SUPPORT_BACKPORT_8_X, - IndexVersionUtils.getPreviousVersion(IndexVersions.UPGRADE_TO_LUCENE_10_0_0) - ); - } else { - indexVersion = SparseVectorFieldMapperTests.getIndexOptionsCompatibleIndexVersion(); - } + MinimalServiceSettings sparseSettings = createRandomModelSettings(TaskType.SPARSE_EMBEDDING); var sparseVectorIndexOptions = SparseVectorFieldTypeTests.randomSparseVectorIndexOptions(); var expectedIndexOptions = sparseVectorIndexOptions == null ? null : new SemanticIndexOptions(SemanticIndexOptions.SupportedIndexOptions.SPARSE_VECTOR, sparseVectorIndexOptions); // should not throw an exception - MapperService mapper = createSemanticMapperServiceWithIndexVersion(mapping(b -> { - b.startObject(fieldName); - { - b.field("type", SemanticTextFieldMapper.CONTENT_TYPE); - b.field(INFERENCE_ID_FIELD, inferenceId); - addSparseVectorModelSettingsToBuilder(b); - if (sparseVectorIndexOptions != null) { - b.startObject(INDEX_OPTIONS_FIELD); - { - b.field(SparseVectorFieldMapper.CONTENT_TYPE); - sparseVectorIndexOptions.toXContent(b, null); - } - b.endObject(); - } - } - b.endObject(); - }), indexVersion); + MapperService mapper = createSemanticMapperService( + semanticMapping(fieldName, inferenceId, null, sparseSettings, null, expectedIndexOptions) + ); - assertSemanticField(mapper, fieldName, true, null, expectedIndexOptions); + assertSemanticField(mapper, fieldName, true, sparseSettings, null, expectedIndexOptions); } } @@ -921,41 +868,21 @@ public void testSparseVectorMappingUpdate() throws IOException { ); final ChunkingSettings chunkingSettings = generateRandomChunkingSettings(false); - IndexVersion indexVersion = useLegacyFormat - ? IndexVersionUtils.randomVersionBetween( - IndexVersions.SPARSE_VECTOR_PRUNING_INDEX_OPTIONS_SUPPORT, - IndexVersionUtils.getPreviousVersion(IndexVersions.SEMANTIC_TEXT_LEGACY_FORMAT_FORBIDDEN) - ) - : SparseVectorFieldMapperTests.getIndexOptionsCompatibleIndexVersion(); final SemanticIndexOptions indexOptions = randomSemanticIndexOptions(TaskType.SPARSE_EMBEDDING); - String fieldName = "field"; + final String fieldName = "field"; - MapperService mapperService = createSemanticMapperServiceWithIndexVersion( - semanticMapping(fieldName, model.getInferenceEntityId(), null, null, chunkingSettings, indexOptions), - indexVersion + MapperService mapperService = createSemanticMapperService( + semanticMapping(fieldName, model.getInferenceEntityId(), null, null, chunkingSettings, indexOptions) ); - var expectedIndexOptions = (indexOptions == null) - ? new SemanticIndexOptions( - SemanticIndexOptions.SupportedIndexOptions.SPARSE_VECTOR, - SparseVectorFieldMapper.SparseVectorIndexOptions.getDefaultIndexOptions(indexVersion) - ) - : indexOptions; - assertSemanticField(mapperService, fieldName, false, chunkingSettings, expectedIndexOptions); + assertSemanticField(mapperService, fieldName, false, new MinimalServiceSettings(model), chunkingSettings, indexOptions); final SemanticIndexOptions newIndexOptions = randomSemanticIndexOptions(TaskType.SPARSE_EMBEDDING); - expectedIndexOptions = (newIndexOptions == null) - ? new SemanticIndexOptions( - SemanticIndexOptions.SupportedIndexOptions.SPARSE_VECTOR, - SparseVectorFieldMapper.SparseVectorIndexOptions.getDefaultIndexOptions(indexVersion) - ) - : newIndexOptions; - - ChunkingSettings newChunkingSettings = generateRandomChunkingSettingsOtherThan(chunkingSettings); + final ChunkingSettings newChunkingSettings = generateRandomChunkingSettingsOtherThan(chunkingSettings); merge( mapperService, semanticMapping(fieldName, model.getInferenceEntityId(), null, null, newChunkingSettings, newIndexOptions) ); - assertSemanticField(mapperService, fieldName, false, newChunkingSettings, expectedIndexOptions); + assertSemanticField(mapperService, fieldName, false, new MinimalServiceSettings(model), newChunkingSettings, newIndexOptions); } } @@ -964,53 +891,39 @@ public void testUpdateSearchInferenceId() throws IOException { final String searchInferenceId1 = "test_search_inference_id_1"; final String searchInferenceId2 = "test_search_inference_id_2"; - CheckedBiFunction buildMapping = (f, sid) -> mapping(b -> { - b.startObject(f).field("type", "semantic_text").field("inference_id", inferenceId); - if (sid != null) { - b.field("search_inference_id", sid); - } - b.endObject(); - }); - for (int depth = 1; depth < 5; depth++) { String fieldName = randomFieldName(depth); - MapperService mapperService = createSemanticMapperService(buildMapping.apply(fieldName, null)); - assertSemanticField(mapperService, fieldName, false, null, null); + MapperService mapperService = createSemanticMapperService(semanticMapping(fieldName, inferenceId)); + assertSemanticField(mapperService, fieldName, false, null, null, null); assertInferenceEndpoints(mapperService, fieldName, inferenceId, inferenceId); - merge(mapperService, buildMapping.apply(fieldName, searchInferenceId1)); - assertSemanticField(mapperService, fieldName, false, null, null); + merge(mapperService, semanticMapping(fieldName, inferenceId, searchInferenceId1)); + assertSemanticField(mapperService, fieldName, false, null, null, null); assertInferenceEndpoints(mapperService, fieldName, inferenceId, searchInferenceId1); - merge(mapperService, buildMapping.apply(fieldName, searchInferenceId2)); - assertSemanticField(mapperService, fieldName, false, null, null); + merge(mapperService, semanticMapping(fieldName, inferenceId, searchInferenceId2)); + assertSemanticField(mapperService, fieldName, false, null, null, null); assertInferenceEndpoints(mapperService, fieldName, inferenceId, searchInferenceId2); - merge(mapperService, buildMapping.apply(fieldName, null)); - assertSemanticField(mapperService, fieldName, false, null, null); + merge(mapperService, semanticMapping(fieldName, inferenceId)); + assertSemanticField(mapperService, fieldName, false, null, null, null); assertInferenceEndpoints(mapperService, fieldName, inferenceId, inferenceId); - mapperService = createSemanticMapperService( - semanticMapping( - fieldName, - inferenceId, - new MinimalServiceSettings("my-service", TaskType.SPARSE_EMBEDDING, null, null, null) - ) - ); - var expectedIndexOptions = getDefaultSparseVectorIndexOptionsForMapper(mapperService); - assertSemanticField(mapperService, fieldName, true, null, expectedIndexOptions); + MinimalServiceSettings sparseSettings = createRandomModelSettings(TaskType.SPARSE_EMBEDDING); + mapperService = createSemanticMapperService(semanticMapping(fieldName, inferenceId, sparseSettings)); + assertSemanticField(mapperService, fieldName, true, sparseSettings, null, null); assertInferenceEndpoints(mapperService, fieldName, inferenceId, inferenceId); - merge(mapperService, buildMapping.apply(fieldName, searchInferenceId1)); - assertSemanticField(mapperService, fieldName, true, null, expectedIndexOptions); + merge(mapperService, semanticMapping(fieldName, inferenceId, searchInferenceId1)); + assertSemanticField(mapperService, fieldName, true, sparseSettings, null, null); assertInferenceEndpoints(mapperService, fieldName, inferenceId, searchInferenceId1); - merge(mapperService, buildMapping.apply(fieldName, searchInferenceId2)); - assertSemanticField(mapperService, fieldName, true, null, expectedIndexOptions); + merge(mapperService, semanticMapping(fieldName, inferenceId, searchInferenceId2)); + assertSemanticField(mapperService, fieldName, true, sparseSettings, null, null); assertInferenceEndpoints(mapperService, fieldName, inferenceId, searchInferenceId2); - merge(mapperService, buildMapping.apply(fieldName, null)); - assertSemanticField(mapperService, fieldName, true, null, expectedIndexOptions); + merge(mapperService, semanticMapping(fieldName, inferenceId)); + assertSemanticField(mapperService, fieldName, true, sparseSettings, null, null); assertInferenceEndpoints(mapperService, fieldName, inferenceId, inferenceId); } } @@ -1037,6 +950,7 @@ protected Set supportedTaskTypes() { @Override protected IndexVersion getRandomCompatibleIndexVersion() { + // TODO: Bias towards indexVersion.current() return SemanticInferenceMetadataFieldsMapperTests.getRandomCompatibleIndexVersion(useLegacyFormat()); } @@ -1056,28 +970,19 @@ protected void assertChunksTextField(SemanticTextFieldMapper.SemanticTextFieldTy @Override protected void assertEmbeddingsField( MapperService mapperService, - SemanticTextFieldMapper.SemanticTextFieldType fieldType, FieldMapper embeddingsMapper, + MinimalServiceSettings modelSettings, @Nullable SemanticIndexOptions expectedIndexOptions ) { - if (fieldType.getModelSettings().taskType() == TaskType.SPARSE_EMBEDDING) { + if (modelSettings.taskType() == TaskType.SPARSE_EMBEDDING) { assertThat(embeddingsMapper, instanceOf(SparseVectorFieldMapper.class)); SparseVectorFieldMapper sparseVectorFieldMapper = (SparseVectorFieldMapper) embeddingsMapper; - assertEquals(sparseVectorFieldMapper.fieldType().isStored(), fieldType.useLegacyFormat() == false); + assertEquals(sparseVectorFieldMapper.fieldType().isStored(), useLegacyFormat() == false); - SparseVectorFieldMapper.SparseVectorIndexOptions applied = sparseVectorFieldMapper.fieldType().getIndexOptions(); - SparseVectorFieldMapper.SparseVectorIndexOptions expected = expectedIndexOptions == null - ? null - : (SparseVectorFieldMapper.SparseVectorIndexOptions) expectedIndexOptions.indexOptions(); - if (expected == null && applied != null) { - var indexVersionCreated = mapperService.getIndexSettings().getIndexVersionCreated(); - if (SparseVectorFieldMapper.SparseVectorIndexOptions.isDefaultOptions(applied, indexVersionCreated)) { - expected = SparseVectorFieldMapper.SparseVectorIndexOptions.getDefaultIndexOptions(indexVersionCreated); - } - } - assertEquals(expected, applied); + IndexOptions expectedBaseIndexOptions = expectedIndexOptions == null ? null : expectedIndexOptions.indexOptions(); + assertEquals(expectedBaseIndexOptions, sparseVectorFieldMapper.fieldType().getIndexOptions()); } else { - super.assertEmbeddingsField(mapperService, fieldType, embeddingsMapper, expectedIndexOptions); + super.assertEmbeddingsField(mapperService, embeddingsMapper, modelSettings, expectedIndexOptions); } } @@ -1128,22 +1033,15 @@ public void testSuccessfulParse() throws IOException { ); }); - var expectedIndexOptions = (indexOptions == null) - ? new SemanticIndexOptions( - SemanticIndexOptions.SupportedIndexOptions.SPARSE_VECTOR, - SparseVectorFieldMapper.SparseVectorIndexOptions.getDefaultIndexOptions(indexVersion) - ) - : indexOptions; - MapperService mapperService = createSemanticMapperServiceWithIndexVersion(mapping, indexVersion); - assertSemanticField(mapperService, fieldName1, false, null, expectedIndexOptions); + assertSemanticField(mapperService, fieldName1, false, new MinimalServiceSettings(model1), null, indexOptions); assertInferenceEndpoints( mapperService, fieldName1, model1.getInferenceEntityId(), setSearchInferenceId ? searchInferenceId : model1.getInferenceEntityId() ); - assertSemanticField(mapperService, fieldName2, false, null, expectedIndexOptions); + assertSemanticField(mapperService, fieldName2, false, new MinimalServiceSettings(model2), null, indexOptions); assertInferenceEndpoints( mapperService, fieldName2, @@ -1257,10 +1155,7 @@ public void testMissingInferenceId() throws IOException { useLegacyFormat, b -> b.startObject("field") .startObject(INFERENCE_FIELD) - .field( - MODEL_SETTINGS_FIELD, - new MinimalServiceSettings("my-service", TaskType.SPARSE_EMBEDDING, null, null, null) - ) + .field(MODEL_SETTINGS_FIELD, createRandomModelSettings(TaskType.SPARSE_EMBEDDING)) .field(CHUNKS_FIELD, useLegacyFormat ? List.of() : Map.of()) .endObject() .endObject() @@ -1422,11 +1317,11 @@ public void testSettingAndUpdatingChunkingSettings() throws IOException { MapperService mapperService = createSemanticMapperService( semanticMapping(fieldName, model.getInferenceEntityId(), null, null, chunkingSettings, indexOptions) ); - assertSemanticField(mapperService, fieldName, false, chunkingSettings, indexOptions); + assertSemanticField(mapperService, fieldName, false, new MinimalServiceSettings(model), chunkingSettings, indexOptions); ChunkingSettings newChunkingSettings = generateRandomChunkingSettingsOtherThan(chunkingSettings); merge(mapperService, semanticMapping(fieldName, model.getInferenceEntityId(), null, null, newChunkingSettings, indexOptions)); - assertSemanticField(mapperService, fieldName, false, newChunkingSettings, indexOptions); + assertSemanticField(mapperService, fieldName, false, new MinimalServiceSettings(model), newChunkingSettings, indexOptions); } public void testModelSettingsRequiredWithChunks() throws IOException { @@ -1476,6 +1371,7 @@ public void testModelSettingsRequiredWithChunks() throws IOException { public void testPre811IndexSemanticTextDenseVectorRaisesError() throws IOException { assumeTrue("Test only applies to legacy format", useLegacyFormat); Model model = TestModel.createRandomInstance(TaskType.TEXT_EMBEDDING); + MinimalServiceSettings modelSettings = new MinimalServiceSettings(model); String fieldName = randomAlphaOfLength(8); MapperService mapperService = createSemanticMapperService( @@ -1483,10 +1379,10 @@ public void testPre811IndexSemanticTextDenseVectorRaisesError() throws IOExcepti IndexVersions.V_8_0_0, IndexVersionUtils.getPreviousVersion(IndexVersions.NEW_SPARSE_VECTOR) ); - assertSemanticField(mapperService, fieldName, false, null, null); + assertSemanticField(mapperService, fieldName, false, null, null, null); - merge(mapperService, semanticMapping(fieldName, model.getInferenceEntityId(), new MinimalServiceSettings(model))); - assertSemanticField(mapperService, fieldName, true, null, null); + merge(mapperService, semanticMapping(fieldName, model.getInferenceEntityId(), modelSettings)); + assertSemanticField(mapperService, fieldName, true, modelSettings, null, null); DocumentMapper documentMapper = mapperService.documentMapper(); DocumentParsingException e = assertThrows( @@ -1508,6 +1404,7 @@ public void testPre811IndexSemanticTextDenseVectorRaisesError() throws IOExcepti public void testPre811IndexSemanticTextSparseVectorRaisesError() throws IOException { assumeTrue("Test only applies to legacy format", useLegacyFormat); Model model = TestModel.createRandomInstance(TaskType.SPARSE_EMBEDDING); + MinimalServiceSettings modelSettings = new MinimalServiceSettings(model); String fieldName = randomAlphaOfLength(8); MapperService mapperService = createSemanticMapperService( @@ -1515,17 +1412,10 @@ public void testPre811IndexSemanticTextSparseVectorRaisesError() throws IOExcept IndexVersions.V_8_0_0, IndexVersionUtils.getPreviousVersion(IndexVersions.NEW_SPARSE_VECTOR) ); - assertSemanticField(mapperService, fieldName, false, null, null); + assertSemanticField(mapperService, fieldName, false, null, null, null); - merge( - mapperService, - semanticMapping( - fieldName, - model.getInferenceEntityId(), - new MinimalServiceSettings(null, TaskType.SPARSE_EMBEDDING, null, null, null) - ) - ); - assertSemanticField(mapperService, fieldName, true, null, null); + merge(mapperService, semanticMapping(fieldName, model.getInferenceEntityId(), modelSettings)); + assertSemanticField(mapperService, fieldName, true, modelSettings, null, null); DocumentMapper documentMapper = mapperService.documentMapper(); DocumentParsingException e = assertThrows( @@ -1549,7 +1439,7 @@ public void testExistsQuerySparseVector() throws IOException { final String inferenceId = "test_service"; MapperService mapperService = createSemanticMapperService( - semanticMapping(fieldName, inferenceId, new MinimalServiceSettings("my-service", TaskType.SPARSE_EMBEDDING, null, null, null)) + semanticMapping(fieldName, inferenceId, createRandomModelSettings(TaskType.SPARSE_EMBEDDING)) ); SearchExecutionContext searchExecutionContext = createSearchExecutionContext(mapperService); @@ -1562,17 +1452,7 @@ public void testExistsQueryDenseVector() throws IOException { final String inferenceId = "test_service"; MapperService mapperService = createSemanticMapperService( - semanticMapping( - fieldName, - inferenceId, - new MinimalServiceSettings( - "my-service", - TaskType.TEXT_EMBEDDING, - 1024, - SimilarityMeasure.COSINE, - DenseVectorFieldMapper.ElementType.FLOAT - ) - ) + semanticMapping(fieldName, inferenceId, createRandomModelSettings(TaskType.TEXT_EMBEDDING)) ); SearchExecutionContext searchExecutionContext = createSearchExecutionContext(mapperService); @@ -1611,11 +1491,11 @@ public void testMappingWithEndpointMetadata() throws IOException { endpointMetadata ); - MapperService originalMapperService = createSemanticMapperService(fieldMapping(b -> { - b.field("type", "semantic_text"); - b.field("inference_id", "test_service"); - b.field("model_settings", modelSettingsWithMetadata); - })); + MapperService originalMapperService = createSemanticMapperService( + semanticMapping("field", "test_service", null, null, null, null, b -> { + b.field("model_settings", modelSettingsWithMetadata); + }) + ); SemanticTextFieldMapper originalMapper = getSemanticFieldMapper(originalMapperService, "field"); assertThat(originalMapper.fieldType().getModelSettings().endpointMetadata(), equalTo(endpointMetadata)); @@ -1628,26 +1508,6 @@ public void testMappingWithEndpointMetadata() throws IOException { assertThat(parsedMapper.fieldType().getModelSettings().endpointMetadata(), equalTo(EndpointMetadata.EMPTY_INSTANCE)); } - private static SemanticIndexOptions defaultDenseVectorSemanticIndexOptions( - IndexVersion indexVersionCreated, - License.OperationMode operationMode, - Integer dims, - DenseVectorFieldMapper.ElementType elementType, - boolean experimentalFeaturesEnabled - ) { - return new SemanticIndexOptions( - SemanticIndexOptions.SupportedIndexOptions.DENSE_VECTOR, - defaultDenseVectorIndexOptions( - indexVersionCreated, - operationMode == License.OperationMode.ENTERPRISE, - true, - dims, - elementType, - experimentalFeaturesEnabled - ) - ); - } - private static DenseVectorFieldMapper.DenseVectorIndexOptions defaultBbqHnswDenseVectorIndexOptions() { int m = Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN; int efConstruction = Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH; @@ -1655,224 +1515,164 @@ private static DenseVectorFieldMapper.DenseVectorIndexOptions defaultBbqHnswDens return new DenseVectorFieldMapper.BBQHnswIndexOptions(m, efConstruction, false, rescoreVector, -1); } - private static SemanticIndexOptions defaultBbqHnswSemanticIndexOptions() { - return new SemanticIndexOptions(SemanticIndexOptions.SupportedIndexOptions.DENSE_VECTOR, defaultBbqHnswDenseVectorIndexOptions()); - } - - private static SemanticIndexOptions defaultSparseVectorIndexOptions(IndexVersion indexVersion) { - return new SemanticIndexOptions( - SemanticIndexOptions.SupportedIndexOptions.SPARSE_VECTOR, - SparseVectorFieldMapper.SparseVectorIndexOptions.getDefaultIndexOptions(indexVersion) - ); - } - public void testDefaultIndexOptions() throws IOException { for (int i = 0; i < 200; i++) { - final Model model = createRandomSupportedModel(); + final MinimalServiceSettings modelSettings = createRandomModelSettings(); final IndexVersion indexVersion = useLegacyFormat == false && randomBoolean() ? IndexVersion.current() : SemanticInferenceMetadataFieldsMapperTests.getRandomCompatibleIndexVersion(useLegacyFormat); - final TaskType taskType = model.getTaskType(); - final DenseVectorFieldMapper.ElementType elementType = model.getServiceSettings().elementType(); - final Integer dimensions = model.getServiceSettings().dimensions(); - final SimilarityMeasure similarity = model.getServiceSettings().similarity(); - MapperService mapperService = createSemanticMapperServiceWithIndexVersion( - semanticMapping( - "field", - "another_inference_id", - new MinimalServiceSettings(null, taskType, dimensions, similarity, elementType) - ), + semanticMapping("field", "another_inference_id", modelSettings), indexVersion ); - - boolean experimentalFeatures = DENSE_VECTOR_EXPERIMENTAL_FEATURES_SETTING.get(mapperService.getIndexSettings().getSettings()); - assertSemanticField( - mapperService, - "field", - true, - null, - getExpectedDefaultIndexOptions(taskType, elementType, dimensions, indexVersion, experimentalFeatures) - ); + assertSemanticField(mapperService, "field", true, modelSettings, null, null); } } - private SemanticIndexOptions getExpectedDefaultIndexOptions( - TaskType taskType, - DenseVectorFieldMapper.ElementType elementType, - Integer dimensions, - IndexVersion indexVersion, - boolean experimentalFeatures - ) { - return switch (taskType) { - case TEXT_EMBEDDING, EMBEDDING -> { - boolean floatFamilyElementType = elementType == DenseVectorFieldMapper.ElementType.FLOAT - || elementType == DenseVectorFieldMapper.ElementType.BFLOAT16; - if (floatFamilyElementType - && SemanticTextFieldMapper.setExplicitIndexOptionsForSemanticText(indexVersion) - && dimensions >= DenseVectorFieldMapper.BBQ_MIN_DIMS) { - yield defaultBbqHnswSemanticIndexOptions(); - } else if (floatFamilyElementType) { - yield defaultDenseVectorSemanticIndexOptions( - indexVersion, - operationMode, - dimensions, - elementType, - experimentalFeatures - ); - } else { - yield null; - } - } - case SPARSE_EMBEDDING -> defaultSparseVectorIndexOptions(indexVersion); - default -> throw new AssertionError("Unexpected task type [" + taskType + "]"); - }; - } - public void testSpecifiedDenseVectorIndexOptions() throws IOException { + IndexVersion indexVersion = SemanticInferenceMetadataFieldsMapperTests.getRandomCompatibleIndexVersion(useLegacyFormat()); + MinimalServiceSettings denseModelSettings = new MinimalServiceSettings( + randomAlphaOfLength(4), + TaskType.TEXT_EMBEDDING, + 100, + SimilarityMeasure.COSINE, + DenseVectorFieldMapper.ElementType.FLOAT + ); // Specifying index options will override default index option settings - var mapperService = createSemanticMapperService(fieldMapping(b -> { - b.field("type", "semantic_text"); - b.field("inference_id", "another_inference_id"); - b.startObject("model_settings"); - b.field("task_type", "text_embedding"); - b.field("dimensions", 100); - b.field("similarity", "cosine"); - b.field("element_type", "float"); - b.endObject(); - b.startObject("index_options"); - b.startObject("dense_vector"); - b.field("type", "int4_hnsw"); - b.field("m", 20); - b.field("ef_construction", 90); - b.endObject(); - b.endObject(); - }), IndexVersions.INFERENCE_METADATA_FIELDS_BACKPORT); - assertSemanticField( - mapperService, - "field", - true, - null, - new SemanticIndexOptions( - SemanticIndexOptions.SupportedIndexOptions.DENSE_VECTOR, - new DenseVectorFieldMapper.Int4HnswIndexOptions(20, 90, false, null, -1) + SemanticIndexOptions expectedIndexOptions = new SemanticIndexOptions( + SemanticIndexOptions.SupportedIndexOptions.DENSE_VECTOR, + new ExtendedDenseVectorIndexOptions( + new DenseVectorFieldMapper.Int4HnswIndexOptions(20, 90, false, null, -1), + DenseVectorFieldMapper.ElementType.FLOAT // Explicitly override element type to hold it constant ) ); + var mapperService = createSemanticMapperServiceWithIndexVersion( + semanticMapping("field", "another_inference_id", null, denseModelSettings, null, expectedIndexOptions), + indexVersion + ); + assertSemanticField(mapperService, "field", true, denseModelSettings, null, expectedIndexOptions); - // Specifying partial index options will in the remainder index options with defaults - mapperService = createSemanticMapperService(fieldMapping(b -> { - b.field("type", "semantic_text"); - b.field("inference_id", "another_inference_id"); - b.startObject("model_settings"); - b.field("task_type", "text_embedding"); - b.field("dimensions", 100); - b.field("similarity", "cosine"); - b.field("element_type", "float"); - b.endObject(); - b.startObject("index_options"); - b.startObject("dense_vector"); - b.field("type", "int4_hnsw"); - b.endObject(); - b.endObject(); - }), IndexVersions.INFERENCE_METADATA_FIELDS_BACKPORT); + // Specifying partial index options will fill in the remainder index options with defaults + mapperService = createSemanticMapperServiceWithIndexVersion( + semanticMapping("field", "another_inference_id", null, denseModelSettings, null, null, b -> { + b.startObject("index_options"); + b.startObject("dense_vector"); + b.field("element_type", "float"); // Explicitly override element type to hold it constant + b.field("type", "int4_hnsw"); + b.endObject(); + b.endObject(); + }), + indexVersion + ); assertSemanticField( mapperService, "field", true, + denseModelSettings, null, new SemanticIndexOptions( SemanticIndexOptions.SupportedIndexOptions.DENSE_VECTOR, - new DenseVectorFieldMapper.Int4HnswIndexOptions(16, 100, false, null, -1) + new ExtendedDenseVectorIndexOptions( + new DenseVectorFieldMapper.Int4HnswIndexOptions(16, 100, false, null, -1), + DenseVectorFieldMapper.ElementType.FLOAT + ) ) ); // Incompatible index options will fail - Exception e = expectThrows(MapperParsingException.class, () -> createSemanticMapperService(fieldMapping(b -> { - b.field("type", "semantic_text"); - b.field("inference_id", "another_inference_id"); - b.startObject("model_settings"); - b.field("task_type", "sparse_embedding"); - b.endObject(); - b.startObject("index_options"); - b.startObject("dense_vector"); - b.field("type", "int8_hnsw"); - b.endObject(); - b.endObject(); - }), IndexVersions.INFERENCE_METADATA_FIELDS_BACKPORT)); + Exception e = expectThrows( + MapperParsingException.class, + () -> createSemanticMapperServiceWithIndexVersion( + semanticMapping( + "field", + "another_inference_id", + null, + createRandomModelSettings(TaskType.SPARSE_EMBEDDING), + null, + null, + b -> { + b.startObject("index_options"); + b.startObject("dense_vector"); + b.field("type", "int8_hnsw"); + b.endObject(); + b.endObject(); + } + ), + indexVersion + ) + ); assertThat(e.getMessage(), containsString("Invalid task type")); - e = expectThrows(MapperParsingException.class, () -> createSemanticMapperService(fieldMapping(b -> { - b.field("type", "semantic_text"); - b.field("inference_id", "another_inference_id"); - b.startObject("model_settings"); - b.field("task_type", "text_embedding"); - b.field("dimensions", 100); - b.field("similarity", "cosine"); - b.field("element_type", "float"); - b.endObject(); - b.startObject("index_options"); - b.startObject("dense_vector"); - b.field("type", "bbq_flat"); - b.field("ef_construction", 100); - b.endObject(); - b.endObject(); - }), IndexVersions.INFERENCE_METADATA_FIELDS_BACKPORT)); + e = expectThrows( + MapperParsingException.class, + () -> createSemanticMapperServiceWithIndexVersion( + semanticMapping("field", "another_inference_id", null, denseModelSettings, null, null, b -> { + b.startObject("index_options"); + b.startObject("dense_vector"); + b.field("type", "bbq_flat"); + b.field("ef_construction", 100); + b.endObject(); + b.endObject(); + }), + indexVersion + ) + ); assertThat(e.getMessage(), containsString("unsupported parameters: [ef_construction : 100]")); - e = expectThrows(MapperParsingException.class, () -> createSemanticMapperService(fieldMapping(b -> { - b.field("type", "semantic_text"); - b.field("inference_id", "another_inference_id"); - b.startObject("model_settings"); - b.field("task_type", "text_embedding"); - b.field("dimensions", 100); - b.field("similarity", "cosine"); - b.field("element_type", "float"); - b.endObject(); - b.startObject("index_options"); - b.startObject("dense_vector"); - b.field("type", "invalid"); - b.endObject(); - b.endObject(); - }), IndexVersions.INFERENCE_METADATA_FIELDS_BACKPORT)); + e = expectThrows( + MapperParsingException.class, + () -> createSemanticMapperServiceWithIndexVersion( + semanticMapping("field", "another_inference_id", null, denseModelSettings, null, null, b -> { + b.startObject("index_options"); + b.startObject("dense_vector"); + b.field("type", "invalid"); + b.endObject(); + b.endObject(); + }), + indexVersion + ) + ); assertThat(e.getMessage(), containsString("Unsupported index options type invalid")); } public void testSetElementTypeInDenseVectorIndexOptions() throws IOException { - // Specifying the element type prevents defaulting to bfloat16 - IndexVersion indexVersion = SemanticInferenceMetadataFieldsMapperTests.getRandomCompatibleIndexVersion(useLegacyFormat); - var mapperService = createSemanticMapperServiceWithIndexVersion(fieldMapping(b -> { - b.field("type", "semantic_text"); - b.field("inference_id", "another_inference_id"); - b.startObject("model_settings"); - b.field("task_type", "text_embedding"); - b.field("dimensions", 100); - b.field("similarity", "cosine"); - b.field("element_type", "float"); - b.endObject(); - b.startObject("index_options"); - b.startObject("dense_vector"); - b.field("element_type", "float"); - b.endObject(); - b.endObject(); - }), indexVersion); - - SemanticIndexOptions expectedDefaultIndexOptions = getExpectedDefaultIndexOptions( + MinimalServiceSettings denseModelSettings = new MinimalServiceSettings( + randomAlphaOfLength(4), TaskType.TEXT_EMBEDDING, - DenseVectorFieldMapper.ElementType.FLOAT, 100, - indexVersion, - DENSE_VECTOR_EXPERIMENTAL_FEATURES_SETTING.get(mapperService.getIndexSettings().getSettings()) + SimilarityMeasure.COSINE, + DenseVectorFieldMapper.ElementType.FLOAT + ); + + // Specifying the element type prevents defaulting to bfloat16 + IndexVersion indexVersion = SemanticInferenceMetadataFieldsMapperTests.getRandomCompatibleIndexVersion(useLegacyFormat); + var mapperService = createSemanticMapperServiceWithIndexVersion( + semanticMapping( + "field", + "another_inference_id", + null, + denseModelSettings, + null, + new SemanticIndexOptions( + SemanticIndexOptions.SupportedIndexOptions.DENSE_VECTOR, + new ExtendedDenseVectorIndexOptions(null, DenseVectorFieldMapper.ElementType.FLOAT) + ) + ), + indexVersion ); + + SemanticIndexOptions expectedDefaultIndexOptions = getDefaultIndexOptions(denseModelSettings, mapperService); DenseVectorFieldMapper.DenseVectorIndexOptions expectedDenseVectorIndexOptions = expectedDefaultIndexOptions != null - ? (DenseVectorFieldMapper.DenseVectorIndexOptions) expectedDefaultIndexOptions.indexOptions() + ? ((ExtendedDenseVectorIndexOptions) expectedDefaultIndexOptions.indexOptions()).getBaseIndexOptions() : null; assertSemanticField( mapperService, "field", true, + denseModelSettings, null, new SemanticIndexOptions( SemanticIndexOptions.SupportedIndexOptions.DENSE_VECTOR, @@ -1881,44 +1681,25 @@ public void testSetElementTypeInDenseVectorIndexOptions() throws IOException { ); // Can use element type in combination with type - mapperService = createSemanticMapperServiceWithIndexVersion(fieldMapping(b -> { - b.field("type", "semantic_text"); - b.field("inference_id", "another_inference_id"); - b.startObject("model_settings"); - b.field("task_type", "text_embedding"); - b.field("dimensions", 100); - b.field("similarity", "cosine"); - b.field("element_type", "float"); - b.endObject(); - b.startObject("index_options"); - b.startObject("dense_vector"); - b.field("element_type", "float"); - b.field("type", "int4_hnsw"); - b.field("m", 20); - b.field("ef_construction", 90); - b.endObject(); - b.endObject(); - }), indexVersion); - - assertSemanticField( - mapperService, - "field", - true, - null, - new SemanticIndexOptions( - SemanticIndexOptions.SupportedIndexOptions.DENSE_VECTOR, - new ExtendedDenseVectorIndexOptions( - new DenseVectorFieldMapper.Int4HnswIndexOptions(20, 90, false, null, -1), - DenseVectorFieldMapper.ElementType.FLOAT - ) + SemanticIndexOptions int4HnswWithFloatIndexOptions = new SemanticIndexOptions( + SemanticIndexOptions.SupportedIndexOptions.DENSE_VECTOR, + new ExtendedDenseVectorIndexOptions( + new DenseVectorFieldMapper.Int4HnswIndexOptions(20, 90, false, null, -1), + DenseVectorFieldMapper.ElementType.FLOAT ) ); + mapperService = createSemanticMapperServiceWithIndexVersion( + semanticMapping("field", "another_inference_id", null, denseModelSettings, null, int4HnswWithFloatIndexOptions), + indexVersion + ); + + assertSemanticField(mapperService, "field", true, denseModelSettings, null, int4HnswWithFloatIndexOptions); } public void testInvalidElementTypeOverride() { for (int i = 0; i < 10; i++) { - final Model model = TestModel.createRandomInstance(TaskType.TEXT_EMBEDDING); - final DenseVectorFieldMapper.ElementType modelElementType = model.getServiceSettings().elementType(); + final MinimalServiceSettings modelSettings = createRandomModelSettings(TaskType.TEXT_EMBEDDING); + final DenseVectorFieldMapper.ElementType modelElementType = modelSettings.elementType(); final DenseVectorFieldMapper.ElementType overrideElementType; if (modelElementType == DenseVectorFieldMapper.ElementType.FLOAT) { @@ -1927,21 +1708,22 @@ public void testInvalidElementTypeOverride() { overrideElementType = randomValueOtherThan(modelElementType, () -> randomFrom(DenseVectorFieldMapper.ElementType.values())); } - MapperParsingException e = expectThrows(MapperParsingException.class, () -> createSemanticMapperService(fieldMapping(b -> { - b.field("type", "semantic_text"); - b.field("inference_id", "test_inference_id"); - b.startObject("model_settings"); - b.field("task_type", model.getTaskType().toString()); - b.field("dimensions", model.getServiceSettings().dimensions()); - b.field("similarity", model.getServiceSettings().similarity().toString()); - b.field("element_type", modelElementType.toString()); - b.endObject(); - b.startObject("index_options"); - b.startObject("dense_vector"); - b.field("element_type", overrideElementType.toString()); - b.endObject(); - b.endObject(); - }))); + MapperParsingException e = expectThrows( + MapperParsingException.class, + () -> createSemanticMapperService( + semanticMapping( + "field", + "test_inference_id", + null, + modelSettings, + null, + new SemanticIndexOptions( + SemanticIndexOptions.SupportedIndexOptions.DENSE_VECTOR, + new ExtendedDenseVectorIndexOptions(null, overrideElementType) + ) + ) + ) + ); assertThat( e.getMessage(), containsString( @@ -1953,23 +1735,26 @@ public void testInvalidElementTypeOverride() { public void testSpecificSparseVectorIndexOptions() throws IOException { for (int i = 0; i < 10; i++) { + IndexVersion indexVersion = SemanticInferenceMetadataFieldsMapperTests.getRandomCompatibleIndexVersion(useLegacyFormat()); SparseVectorFieldMapper.SparseVectorIndexOptions testIndexOptions = randomSparseVectorIndexOptions(false); - var mapperService = createSemanticMapperService(fieldMapping(b -> { - b.field("type", SemanticTextFieldMapper.CONTENT_TYPE); - b.field(INFERENCE_ID_FIELD, "test_inference_id"); - addSparseVectorModelSettingsToBuilder(b); - b.startObject(INDEX_OPTIONS_FIELD); - { - b.field(SparseVectorFieldMapper.CONTENT_TYPE); - testIndexOptions.toXContent(b, null); - } - b.endObject(); - }), IndexVersions.INFERENCE_METADATA_FIELDS_BACKPORT); + MinimalServiceSettings sparseModelSettings = createRandomModelSettings(TaskType.SPARSE_EMBEDDING); + var mapperService = createSemanticMapperServiceWithIndexVersion( + semanticMapping( + "field", + "test_inference_id", + null, + sparseModelSettings, + null, + new SemanticIndexOptions(SemanticIndexOptions.SupportedIndexOptions.SPARSE_VECTOR, testIndexOptions) + ), + indexVersion + ); assertSemanticField( mapperService, "field", true, + sparseModelSettings, null, new SemanticIndexOptions(SemanticIndexOptions.SupportedIndexOptions.SPARSE_VECTOR, testIndexOptions) ); @@ -1977,88 +1762,78 @@ public void testSpecificSparseVectorIndexOptions() throws IOException { } public void testSparseVectorIndexOptionsValidations() throws IOException { - Exception e = expectThrows(MapperParsingException.class, () -> createSemanticMapperService(fieldMapping(b -> { - b.field("type", SemanticTextFieldMapper.CONTENT_TYPE); - b.field(INFERENCE_ID_FIELD, "test_inference_id"); - b.startObject(INDEX_OPTIONS_FIELD); - { + IndexVersion indexVersion = SemanticInferenceMetadataFieldsMapperTests.getRandomCompatibleIndexVersion(useLegacyFormat()); + Exception e = expectThrows( + MapperParsingException.class, + () -> createSemanticMapperServiceWithIndexVersion(semanticMapping("field", "test_inference_id", null, null, null, null, b -> { + b.startObject(INDEX_OPTIONS_FIELD); b.startObject(SparseVectorFieldMapper.CONTENT_TYPE); - { - b.field("prune", false); - b.startObject("pruning_config"); - { - b.field(TokenPruningConfig.TOKENS_FREQ_RATIO_THRESHOLD.getPreferredName(), 5.0f); - } - b.endObject(); - } + b.field("prune", false); + b.startObject("pruning_config"); + b.field(TokenPruningConfig.TOKENS_FREQ_RATIO_THRESHOLD.getPreferredName(), 5.0f); b.endObject(); - } - b.endObject(); - }), IndexVersions.INFERENCE_METADATA_FIELDS_BACKPORT)); + b.endObject(); + b.endObject(); + }), indexVersion) + ); assertThat(e.getMessage(), containsString("failed to parse field [pruning_config]")); - e = expectThrows(MapperParsingException.class, () -> createSemanticMapperService(fieldMapping(b -> { - b.field("type", SemanticTextFieldMapper.CONTENT_TYPE); - b.field(INFERENCE_ID_FIELD, "test_inference_id"); - b.startObject(INDEX_OPTIONS_FIELD); - { + e = expectThrows( + MapperParsingException.class, + () -> createSemanticMapperServiceWithIndexVersion(semanticMapping("field", "test_inference_id", null, null, null, null, b -> { + b.startObject(INDEX_OPTIONS_FIELD); b.startObject(SparseVectorFieldMapper.CONTENT_TYPE); - { - b.field("prune", true); - b.startObject("pruning_config"); - { - b.field(TokenPruningConfig.TOKENS_FREQ_RATIO_THRESHOLD.getPreferredName(), 1000.0f); - } - b.endObject(); - } + b.field("prune", true); + b.startObject("pruning_config"); + b.field(TokenPruningConfig.TOKENS_FREQ_RATIO_THRESHOLD.getPreferredName(), 1000.0f); b.endObject(); - } - b.endObject(); - }), IndexVersions.INFERENCE_METADATA_FIELDS_BACKPORT)); + b.endObject(); + b.endObject(); + }), indexVersion) + ); var innerClause = e.getCause().getCause().getCause(); assertThat(innerClause.getMessage(), containsString("[tokens_freq_ratio_threshold] must be between [1] and [100], got 1000.0")); } @Override public void testSupportedIndexVersions() throws IOException { - - // Add model settings via extendedSettings to trigger programmatic - // creation of SparseVectorFieldMapper and DenseVectorFieldMapper - Model sparseModel = TestModel.createRandomInstance(TaskType.SPARSE_EMBEDDING); - Map sparseExtensions = Map.of( - "inference_id", - sparseModel.getInferenceEntityId(), - "model_settings", - Map.of("task_type", TaskType.SPARSE_EMBEDDING.toString()) - ); - Model denseModel = TestModel.createRandomInstance(TaskType.TEXT_EMBEDDING); - Map denseExtensions = Map.of( - "inference_id", - denseModel.getInferenceEntityId(), - "model_settings", - Map.of( - "task_type", - TaskType.TEXT_EMBEDDING.toString(), - "dimensions", - denseModel.getServiceSettings().dimensions(), - "similarity", - denseModel.getServiceSettings().similarity(), - "element_type", - denseModel.getServiceSettings().elementType() - ) - ); Set supportedVersions = getSupportedVersions(); for (int i = 0; i < Math.min(supportedVersions.size(), 100); i++) { IndexVersion indexVersion = randomFrom(supportedVersions); - MapperService denseMapperService = createMapperService(indexVersion, fieldMapping(b -> extendedMapping(b, denseExtensions))); - MapperService sparseMapperService = createMapperService(indexVersion, fieldMapping(b -> extendedMapping(b, sparseExtensions))); + createMapperService( + indexVersion, + semanticMapping("field", denseModel.getInferenceEntityId(), new MinimalServiceSettings(denseModel)) + ); + createMapperService( + indexVersion, + semanticMapping("field", sparseModel.getInferenceEntityId(), new MinimalServiceSettings(sparseModel)) + ); } } + @Override + protected MapperService createSemanticMapperService(XContentBuilder mappings, IndexVersion minIndexVersion) throws IOException { + IndexVersion maxIndexVersion = useLegacyFormat() + ? IndexVersionUtils.getPreviousVersion(IndexVersions.SEMANTIC_TEXT_LEGACY_FORMAT_FORBIDDEN) + : IndexVersion.current(); + return createSemanticMapperService(mappings, minIndexVersion, maxIndexVersion); + } + + @Override + protected MapperService createSemanticMapperServiceWithIndexVersion(XContentBuilder mappings, IndexVersion indexVersion) + throws IOException { + validateIndexVersion(indexVersion, useLegacyFormat()); + var settings = Settings.builder() + .put(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), indexVersion) + .put(InferenceMetadataFieldsMapper.USE_LEGACY_SEMANTIC_TEXT_FORMAT.getKey(), useLegacyFormat()) + .build(); + return createMapperService(indexVersion, settings, mappings); + } + public static SemanticIndexOptions randomSemanticIndexOptions() { TaskType taskType = randomFrom(TaskType.SPARSE_EMBEDDING, TaskType.TEXT_EMBEDDING); return randomSemanticIndexOptions(taskType); @@ -2166,4 +1941,16 @@ private static Throwable rootCause(Throwable t) { } return cause; } + + private static void validateIndexVersion(IndexVersion indexVersion, boolean useLegacyFormat) { + if (useLegacyFormat == false + && indexVersion.before(IndexVersions.INFERENCE_METADATA_FIELDS) + && indexVersion.between(IndexVersions.INFERENCE_METADATA_FIELDS_BACKPORT, IndexVersions.UPGRADE_TO_LUCENE_10_0_0) == false) { + throw new AssertionError("Index version " + indexVersion + " does not support new semantic text format"); + } + + if (useLegacyFormat && indexVersion.onOrAfter(IndexVersions.SEMANTIC_TEXT_LEGACY_FORMAT_FORBIDDEN)) { + throw new AssertionError("Index version " + indexVersion + " does not support legacy semantic text format"); + } + } }