From b18ca299725ef3d92079e1a1aac1126348b84fc7 Mon Sep 17 00:00:00 2001 From: Mike Pellegrini Date: Wed, 22 Jul 2026 09:13:19 -0400 Subject: [PATCH 01/24] Move model creation methods --- .../AbstractSemanticMapperTestCase.java | 130 ++++++++++++++++++ .../mapper/SemanticTextFieldMapperTests.java | 126 ----------------- 2 files changed, 130 insertions(+), 126 deletions(-) diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/AbstractSemanticMapperTestCase.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/AbstractSemanticMapperTestCase.java index 8be36896f9db9..cf340d1b87d11 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/AbstractSemanticMapperTestCase.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/AbstractSemanticMapperTestCase.java @@ -38,6 +38,7 @@ import org.elasticsearch.inference.MinimalServiceSettings; import org.elasticsearch.inference.Model; import org.elasticsearch.inference.ServiceSettings; +import org.elasticsearch.inference.SimilarityMeasure; import org.elasticsearch.inference.TaskType; import org.elasticsearch.license.License; import org.elasticsearch.license.XPackLicenseState; @@ -60,6 +61,7 @@ import org.junit.Before; import java.io.IOException; +import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.Map; @@ -69,6 +71,9 @@ import static java.util.Objects.requireNonNull; import static org.elasticsearch.index.IndexVersions.NEW_SPARSE_VECTOR; import static org.elasticsearch.index.IndexVersions.SEMANTIC_TEXT_DEFAULTS_TO_BFLOAT16; +import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.BBQ_MIN_DIMS; +import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapperTestUtils.getSupportedSimilarities; +import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapperTestUtils.randomCompatibleDimensions; import static org.elasticsearch.xpack.inference.mapper.SemanticFieldMapper.INDEX_OPTIONS_FIELD; import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.CHUNKED_EMBEDDINGS_FIELD; import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.CHUNKING_SETTINGS_FIELD; @@ -118,6 +123,15 @@ protected XPackLicenseState getLicenseState() { } } + /** The kinds of incompatibility that can exist between two inference endpoint model settings. */ + protected enum IncompatibilityKind { + TASK_TYPE, + DIMENSIONS, + SIMILARITY, + ELEMENT_TYPE, + DOES_NOT_EXIST + } + protected final License.OperationMode operationMode; protected ModelRegistry globalModelRegistry; private TestThreadPool threadPool; @@ -586,6 +600,122 @@ protected void givenModelSettings(String inferenceId, MinimalServiceSettings mod when(globalModelRegistry.getMinimalServiceSettings(inferenceId)).thenReturn(modelSettings); } + /** + * Creates a {@link TestModel} that is compatible with {@code baseModel} (same task type, dimensions, + * similarity, and element type) but with a distinct service, task settings, and secrets, registered + * under the given inference ID. Compatible models can be substituted via an inference-ID update. + */ + protected TestModel createCompatibleModel(String inferenceId, TestModel baseModel) { + return new TestModel( + inferenceId, + baseModel.getTaskType(), + randomAlphaOfLength(4), + new TestModel.TestServiceSettings( + randomAlphaOfLength(4), + baseModel.getServiceSettings().dimensions(), + baseModel.getServiceSettings().similarity(), + baseModel.getServiceSettings().elementType() + ), + new TestModel.TestTaskSettings(randomInt(3)), + new TestModel.TestSecretSettings(randomAlphaOfLength(4)) + ); + } + + /** + * Creates a {@link TestModel} that is NOT compatible with {@code baseModel}, choosing uniformly among the + * applicable kinds of incompatibility: task type, dimensions, similarity, element type, or the endpoint not + * existing. Setting-based perturbations (dimensions/similarity/element type) only apply to dense base models; + * a sparse base model has null dimensions/similarity/element type, so only task-type and does-not-exist apply. + * + * @return an incompatible model to register under {@code inferenceId}, or {@code null} to indicate the + * caller should NOT register an endpoint (the does-not-exist case). + */ + protected TestModel createIncompatibleModel(String inferenceId, TestModel baseModel) { + final TestModel.TestServiceSettings baseServiceSettings = baseModel.getServiceSettings(); + final DenseVectorFieldMapper.ElementType baseElementType = baseServiceSettings.elementType(); + + List applicable = new ArrayList<>(); + if (supportedTaskTypes().size() > 1) { + applicable.add(IncompatibilityKind.TASK_TYPE); + } + applicable.add(IncompatibilityKind.DOES_NOT_EXIST); + if (baseModel.getTaskType() != TaskType.SPARSE_EMBEDDING) { + applicable.add(IncompatibilityKind.DIMENSIONS); + applicable.add(IncompatibilityKind.ELEMENT_TYPE); + // SIMILARITY only when the element type supports more than one option to perturb to + if (getSupportedSimilarities(baseElementType).size() > 1) { + applicable.add(IncompatibilityKind.SIMILARITY); + } + } + + TaskType taskType = baseModel.getTaskType(); + Integer dimensions = baseServiceSettings.dimensions(); + SimilarityMeasure similarity = baseServiceSettings.similarity(); + DenseVectorFieldMapper.ElementType elementType = baseElementType; + boolean returnNull = false; + + switch (randomFrom(applicable)) { + case DOES_NOT_EXIST -> returnNull = true; + case TASK_TYPE -> { + taskType = randomValueOtherThan(taskType, () -> randomFrom(supportedTaskTypes())); + if (taskType != TaskType.SPARSE_EMBEDDING && baseModel.getTaskType() == TaskType.SPARSE_EMBEDDING) { + // Sparse -> dense: populate required dense settings from a fresh random dense model + TestModel randomDenseModel = TestModel.createRandomInstance(taskType); + dimensions = randomDenseModel.getServiceSettings().dimensions(); + similarity = randomDenseModel.getServiceSettings().similarity(); + elementType = randomDenseModel.getServiceSettings().elementType(); + } else if (taskType == TaskType.SPARSE_EMBEDDING) { + // Dense -> sparse: clear dense-only settings + dimensions = null; + similarity = null; + elementType = null; + } + } + case DIMENSIONS -> dimensions = randomValueOtherThan( + dimensions, + () -> randomCompatibleDimensions(baseElementType, BBQ_MIN_DIMS * 2) + ); + case SIMILARITY -> similarity = randomValueOtherThan(similarity, () -> randomFrom(getSupportedSimilarities(baseElementType))); + case ELEMENT_TYPE -> { + elementType = randomValueOtherThan( + elementType, + () -> randomFrom( + DenseVectorFieldMapper.ElementType.FLOAT, + DenseVectorFieldMapper.ElementType.BYTE, + DenseVectorFieldMapper.ElementType.BIT + ) + ); + // Regenerate dimensions and similarity compatible with the new element type + dimensions = randomCompatibleDimensions(elementType, BBQ_MIN_DIMS * 2); + similarity = randomFrom(getSupportedSimilarities(elementType)); + } + } + + if (returnNull) { + return null; + } + + return buildModel(baseModel, inferenceId, taskType, elementType, dimensions, similarity); + } + + protected static TestModel buildModel( + TestModel baseModel, + String inferenceId, + TaskType taskType, + @Nullable DenseVectorFieldMapper.ElementType elementType, + @Nullable Integer dimensions, + @Nullable SimilarityMeasure similarity + ) { + return new TestModel( + inferenceId, + taskType, + baseModel.getConfigurations().getService(), + new TestModel.TestServiceSettings(baseModel.getServiceSettings().model(), dimensions, similarity, elementType), + baseModel.getTaskSettings(), + baseModel.getSecretSettings() + ); + } + protected static String randomFieldName(int numLevel) { StringBuilder builder = new StringBuilder(); for (int i = 0; i < numLevel; i++) { diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java index 6605ac062b339..12336e5100b18 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java @@ -100,10 +100,7 @@ import java.util.stream.Collectors; import static org.elasticsearch.index.IndexSettings.DENSE_VECTOR_EXPERIMENTAL_FEATURES_SETTING; -import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.BBQ_MIN_DIMS; import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapperTestUtils.defaultDenseVectorIndexOptions; -import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapperTestUtils.getSupportedSimilarities; -import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapperTestUtils.randomCompatibleDimensions; import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldTypeTests.randomIndexOptionsAll; import static org.elasticsearch.index.mapper.vectors.SparseVectorFieldTypeTests.randomSparseVectorIndexOptions; import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.CHUNKS_FIELD; @@ -855,129 +852,6 @@ public void testUpdateInferenceId_GivenModelSettings() throws IOException { } } - /** - * Creates a {@link TestModel} that is compatible with {@code baseModel} (same task type, dimensions, - * similarity, and element type) but with a distinct service, task settings, and secrets, registered - * under the given inference ID. Compatible models can be substituted via an inference-ID update. - */ - private static TestModel createCompatibleModel(String inferenceId, TestModel baseModel) { - return new TestModel( - inferenceId, - baseModel.getTaskType(), - randomAlphaOfLength(4), - new TestModel.TestServiceSettings( - randomAlphaOfLength(4), - baseModel.getServiceSettings().dimensions(), - baseModel.getServiceSettings().similarity(), - baseModel.getServiceSettings().elementType() - ), - new TestModel.TestTaskSettings(randomInt(3)), - new TestModel.TestSecretSettings(randomAlphaOfLength(4)) - ); - } - - /** The kinds of incompatibility that can exist between two inference endpoint model settings. */ - private enum IncompatibilityKind { - TASK_TYPE, - DIMENSIONS, - SIMILARITY, - ELEMENT_TYPE, - DOES_NOT_EXIST - } - - /** - * Creates a {@link TestModel} that is NOT compatible with {@code baseModel}, choosing uniformly among the - * applicable kinds of incompatibility: task type, dimensions, similarity, element type, or the endpoint not - * existing. Setting-based perturbations (dimensions/similarity/element type) only apply to dense base models; - * a sparse base model has null dimensions/similarity/element type, so only task-type and does-not-exist apply. - * - * @return an incompatible model to register under {@code inferenceId}, or {@code null} to indicate the - * caller should NOT register an endpoint (the does-not-exist case). - */ - private static TestModel createIncompatibleModel(String inferenceId, TestModel baseModel) { - final TestModel.TestServiceSettings baseServiceSettings = baseModel.getServiceSettings(); - final DenseVectorFieldMapper.ElementType baseElementType = baseServiceSettings.elementType(); - - List applicable = new ArrayList<>(); - applicable.add(IncompatibilityKind.TASK_TYPE); - applicable.add(IncompatibilityKind.DOES_NOT_EXIST); - if (baseModel.getTaskType() != TaskType.SPARSE_EMBEDDING) { - applicable.add(IncompatibilityKind.DIMENSIONS); - applicable.add(IncompatibilityKind.ELEMENT_TYPE); - // SIMILARITY only when the element type supports more than one option to perturb to - if (getSupportedSimilarities(baseElementType).size() > 1) { - applicable.add(IncompatibilityKind.SIMILARITY); - } - } - - TaskType taskType = baseModel.getTaskType(); - Integer dimensions = baseServiceSettings.dimensions(); - SimilarityMeasure similarity = baseServiceSettings.similarity(); - DenseVectorFieldMapper.ElementType elementType = baseElementType; - boolean returnNull = false; - - switch (randomFrom(applicable)) { - case DOES_NOT_EXIST -> returnNull = true; - case TASK_TYPE -> { - taskType = randomValueOtherThan(taskType, () -> randomFrom(Set.of(TaskType.SPARSE_EMBEDDING, TaskType.TEXT_EMBEDDING))); - if (taskType == TaskType.TEXT_EMBEDDING) { - // Sparse -> dense: populate required dense settings from a fresh random dense model - TestModel randomDenseModel = TestModel.createRandomInstance(TaskType.TEXT_EMBEDDING); - dimensions = randomDenseModel.getServiceSettings().dimensions(); - similarity = randomDenseModel.getServiceSettings().similarity(); - elementType = randomDenseModel.getServiceSettings().elementType(); - } else { - // Dense -> sparse: clear dense-only settings - dimensions = null; - similarity = null; - elementType = null; - } - } - case DIMENSIONS -> dimensions = randomValueOtherThan( - dimensions, - () -> randomCompatibleDimensions(baseElementType, BBQ_MIN_DIMS * 2) - ); - case SIMILARITY -> similarity = randomValueOtherThan(similarity, () -> randomFrom(getSupportedSimilarities(baseElementType))); - case ELEMENT_TYPE -> { - elementType = randomValueOtherThan( - elementType, - () -> randomFrom( - DenseVectorFieldMapper.ElementType.FLOAT, - DenseVectorFieldMapper.ElementType.BYTE, - DenseVectorFieldMapper.ElementType.BIT - ) - ); - // Regenerate dimensions and similarity compatible with the new element type - dimensions = randomCompatibleDimensions(elementType, BBQ_MIN_DIMS * 2); - similarity = randomFrom(getSupportedSimilarities(elementType)); - } - } - - if (returnNull) { - return null; - } - - return buildModel(baseModel, inferenceId, taskType, elementType, dimensions, similarity); - } - - private static TestModel buildModel( - TestModel baseModel, - String inferenceId, - TaskType taskType, - @Nullable DenseVectorFieldMapper.ElementType elementType, - @Nullable Integer dimensions, - @Nullable SimilarityMeasure similarity - ) { - return new TestModel( - inferenceId, - taskType, - baseModel.getConfigurations().getService(), - new TestModel.TestServiceSettings(baseModel.getServiceSettings().model(), dimensions, similarity, elementType), - baseModel.getTaskSettings(), - baseModel.getSecretSettings() - ); - } - public void testDynamicUpdate() throws IOException { final String fieldName = "semantic"; final String inferenceId = "test_service"; From 58b92d1cc618b08fbdc89138fb7c68d9c8d61f9c Mon Sep 17 00:00:00 2001 From: Mike Pellegrini Date: Wed, 22 Jul 2026 09:26:39 -0400 Subject: [PATCH 02/24] Move inference ID update tests to AbstractSemanticMapperTestCase --- .../AbstractSemanticMapperTestCase.java | 106 ++++++++++++++++++ .../mapper/SemanticTextFieldMapperTests.java | 105 ----------------- 2 files changed, 106 insertions(+), 105 deletions(-) diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/AbstractSemanticMapperTestCase.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/AbstractSemanticMapperTestCase.java index cf340d1b87d11..bd29376461d6e 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/AbstractSemanticMapperTestCase.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/AbstractSemanticMapperTestCase.java @@ -14,6 +14,7 @@ import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.CheckedRunnable; import org.elasticsearch.core.Nullable; import org.elasticsearch.features.FeatureService; import org.elasticsearch.index.IndexMode; @@ -83,6 +84,7 @@ import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.TEXT_FIELD; import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.getChunksFieldName; import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.getEmbeddingsFieldName; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; @@ -258,6 +260,110 @@ protected boolean useLegacyFormat() { return false; } + public void testUpdateInferenceId_GivenNoModelSettings() throws IOException { + for (int randomizedRun = 0; randomizedRun < 10; randomizedRun++) { + String fieldName = randomAlphaOfLengthBetween(5, 15); + String oldInferenceId = randomAlphaOfLengthBetween(5, 15); + + TestModel oldModel = null; + if (randomBoolean()) { + oldModel = createRandomSupportedModel(); + givenModelSettings(oldInferenceId, new MinimalServiceSettings(oldModel)); + } + + var mapperService = createSemanticMapperService(semanticMapping(fieldName, oldInferenceId)); + + assertInferenceEndpoints(mapperService, fieldName, oldInferenceId, oldInferenceId); + assertSemanticField(mapperService, fieldName, false, null, null); + if (oldModel != null) { + assertEmbeddingsFieldMapperMatchesModel(mapperService, fieldName, oldModel); + } + + String newInferenceId = randomValueOtherThan(oldInferenceId, () -> randomAlphaOfLengthBetween(5, 15)); + TestModel newModel = null; + if (randomBoolean()) { + newModel = createRandomSupportedModel(); + givenModelSettings(newInferenceId, new MinimalServiceSettings(newModel)); + } + + merge(mapperService, semanticMapping(fieldName, newInferenceId)); + + assertInferenceEndpoints(mapperService, fieldName, newInferenceId, newInferenceId); + assertSemanticField(mapperService, fieldName, false, null, null); + if (newModel != null) { + assertEmbeddingsFieldMapperMatchesModel(mapperService, fieldName, newModel); + } + } + } + + public void testUpdateInferenceId_GivenModelSettings() throws IOException { + for (int randomizedRun = 0; randomizedRun < 20; randomizedRun++) { + final String fieldName = randomAlphaOfLengthBetween(5, 15); + final String oldInferenceId = randomAlphaOfLengthBetween(5, 15); + final String newInferenceId = randomValueOtherThan(oldInferenceId, () -> randomAlphaOfLengthBetween(5, 15)); + + final TestModel oldModel = createRandomSupportedModel(); + final MinimalServiceSettings previousModelSettings = new MinimalServiceSettings(oldModel); + givenModelSettings(oldInferenceId, previousModelSettings); + + final MapperService mapperService = createSemanticMapperService( + semanticMapping(fieldName, oldInferenceId, previousModelSettings) + ); + final SemanticIndexOptions currentIndexOptions = extractCurrentIndexOptions(mapperService, fieldName); + assertInferenceEndpoints(mapperService, fieldName, oldInferenceId, oldInferenceId); + assertSemanticField(mapperService, fieldName, true, null, currentIndexOptions); + assertEmbeddingsFieldMapperMatchesModel(mapperService, fieldName, oldModel); + + final CheckedRunnable mergeRunner = () -> merge(mapperService, semanticMapping(fieldName, newInferenceId)); + + if (randomBoolean()) { + // Compatible: new endpoint has identical task type / dimensions / similarity / element type + TestModel newModel = createCompatibleModel(newInferenceId, oldModel); + MinimalServiceSettings newModelSettings = new MinimalServiceSettings(newModel); + givenModelSettings(newInferenceId, newModelSettings); + + mergeRunner.run(); + assertInferenceEndpoints(mapperService, fieldName, newInferenceId, newInferenceId); + assertSemanticField(mapperService, fieldName, true, null, currentIndexOptions); + assertEmbeddingsFieldMapperMatchesModel(mapperService, fieldName, newModel); + } else { + final TestModel incompatibleModel = createIncompatibleModel(newInferenceId, oldModel); + final String expectedErrorMessage; + if (incompatibleModel == null) { + // Incompatible: new endpoint does not exist + expectedErrorMessage = "Cannot update [" + + contentType() + + "] field [" + + fieldName + + "] because inference endpoint [" + + newInferenceId + + "] does not exist."; + } else { + // Incompatible: new endpoint exists but its model settings are incompatible + MinimalServiceSettings incompatibleModelSettings = new MinimalServiceSettings(incompatibleModel); + givenModelSettings(newInferenceId, incompatibleModelSettings); + + expectedErrorMessage = "Cannot update [" + + contentType() + + "] field [" + + fieldName + + "] because inference endpoint [" + + oldInferenceId + + "] with model settings [" + + previousModelSettings + + "] is not compatible with new inference endpoint [" + + newInferenceId + + "] with model settings [" + + incompatibleModelSettings + + "]"; + } + + IllegalArgumentException exc = assertThrows(IllegalArgumentException.class, mergeRunner::run); + assertThat(exc.getMessage(), containsString(expectedErrorMessage)); + } + } + } + protected XContentBuilder semanticMapping(String fieldName, String inferenceId) throws IOException { return semanticMapping(fieldName, inferenceId, null, null, null, null); } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java index 12336e5100b18..e2154555fb97b 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java @@ -34,7 +34,6 @@ import org.elasticsearch.common.lucene.search.Queries; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.CheckedConsumer; -import org.elasticsearch.core.CheckedRunnable; import org.elasticsearch.core.Nullable; import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexVersion; @@ -748,110 +747,6 @@ public void testMultiFieldsSupport() throws IOException { } } - public void testUpdateInferenceId_GivenNoModelSettings() throws IOException { - for (int randomizedRun = 0; randomizedRun < 10; randomizedRun++) { - String fieldName = randomAlphaOfLengthBetween(5, 15); - String oldInferenceId = randomAlphaOfLengthBetween(5, 15); - - TestModel oldModel = null; - if (randomBoolean()) { - oldModel = createRandomSupportedModel(); - givenModelSettings(oldInferenceId, new MinimalServiceSettings(oldModel)); - } - - var mapperService = createSemanticMapperService(semanticMapping(fieldName, oldInferenceId)); - - assertInferenceEndpoints(mapperService, fieldName, oldInferenceId, oldInferenceId); - assertSemanticField(mapperService, fieldName, false, null, null); - if (oldModel != null) { - assertEmbeddingsFieldMapperMatchesModel(mapperService, fieldName, oldModel); - } - - String newInferenceId = randomValueOtherThan(oldInferenceId, () -> randomAlphaOfLengthBetween(5, 15)); - TestModel newModel = null; - if (randomBoolean()) { - newModel = createRandomSupportedModel(); - givenModelSettings(newInferenceId, new MinimalServiceSettings(newModel)); - } - - merge(mapperService, semanticMapping(fieldName, newInferenceId)); - - assertInferenceEndpoints(mapperService, fieldName, newInferenceId, newInferenceId); - assertSemanticField(mapperService, fieldName, false, null, null); - if (newModel != null) { - assertEmbeddingsFieldMapperMatchesModel(mapperService, fieldName, newModel); - } - } - } - - public void testUpdateInferenceId_GivenModelSettings() throws IOException { - for (int randomizedRun = 0; randomizedRun < 20; randomizedRun++) { - final String fieldName = randomAlphaOfLengthBetween(5, 15); - final String oldInferenceId = randomAlphaOfLengthBetween(5, 15); - final String newInferenceId = randomValueOtherThan(oldInferenceId, () -> randomAlphaOfLengthBetween(5, 15)); - - final TestModel oldModel = createRandomSupportedModel(); - final MinimalServiceSettings previousModelSettings = new MinimalServiceSettings(oldModel); - givenModelSettings(oldInferenceId, previousModelSettings); - - final MapperService mapperService = createSemanticMapperService( - semanticMapping(fieldName, oldInferenceId, previousModelSettings) - ); - final SemanticIndexOptions currentIndexOptions = extractCurrentIndexOptions(mapperService, fieldName); - assertInferenceEndpoints(mapperService, fieldName, oldInferenceId, oldInferenceId); - assertSemanticField(mapperService, fieldName, true, null, currentIndexOptions); - assertEmbeddingsFieldMapperMatchesModel(mapperService, fieldName, oldModel); - - final CheckedRunnable mergeRunner = () -> merge(mapperService, semanticMapping(fieldName, newInferenceId)); - - if (randomBoolean()) { - // Compatible: new endpoint has identical task type / dimensions / similarity / element type - TestModel newModel = createCompatibleModel(newInferenceId, oldModel); - MinimalServiceSettings newModelSettings = new MinimalServiceSettings(newModel); - givenModelSettings(newInferenceId, newModelSettings); - - mergeRunner.run(); - assertInferenceEndpoints(mapperService, fieldName, newInferenceId, newInferenceId); - assertSemanticField(mapperService, fieldName, true, null, currentIndexOptions); - assertEmbeddingsFieldMapperMatchesModel(mapperService, fieldName, newModel); - } else { - final TestModel incompatibleModel = createIncompatibleModel(newInferenceId, oldModel); - final String expectedErrorMessage; - if (incompatibleModel == null) { - // Incompatible: new endpoint does not exist - expectedErrorMessage = "Cannot update [" - + SemanticTextFieldMapper.CONTENT_TYPE - + "] field [" - + fieldName - + "] because inference endpoint [" - + newInferenceId - + "] does not exist."; - } else { - // Incompatible: new endpoint exists but its model settings are incompatible - MinimalServiceSettings incompatibleModelSettings = new MinimalServiceSettings(incompatibleModel); - givenModelSettings(newInferenceId, incompatibleModelSettings); - - expectedErrorMessage = "Cannot update [" - + SemanticTextFieldMapper.CONTENT_TYPE - + "] field [" - + fieldName - + "] because inference endpoint [" - + oldInferenceId - + "] with model settings [" - + previousModelSettings - + "] is not compatible with new inference endpoint [" - + newInferenceId - + "] with model settings [" - + incompatibleModelSettings - + "]"; - } - - IllegalArgumentException exc = assertThrows(IllegalArgumentException.class, mergeRunner::run); - assertThat(exc.getMessage(), containsString(expectedErrorMessage)); - } - } - } - public void testDynamicUpdate() throws IOException { final String fieldName = "semantic"; final String inferenceId = "test_service"; From e18976e5efaa6b2b0ddd700db20a7214f1e3976c Mon Sep 17 00:00:00 2001 From: Mike Pellegrini Date: Wed, 22 Jul 2026 10:48:06 -0400 Subject: [PATCH 03/24] Add getDefaultIndexOptions helper method --- .../AbstractSemanticMapperTestCase.java | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/AbstractSemanticMapperTestCase.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/AbstractSemanticMapperTestCase.java index bd29376461d6e..eaad2acaadaf3 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/AbstractSemanticMapperTestCase.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/AbstractSemanticMapperTestCase.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.inference.mapper; +import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; import org.elasticsearch.cluster.ClusterChangedEvent; @@ -70,9 +71,11 @@ import java.util.function.Supplier; import static java.util.Objects.requireNonNull; +import static org.elasticsearch.index.IndexSettings.DENSE_VECTOR_EXPERIMENTAL_FEATURES_SETTING; import static org.elasticsearch.index.IndexVersions.NEW_SPARSE_VECTOR; import static org.elasticsearch.index.IndexVersions.SEMANTIC_TEXT_DEFAULTS_TO_BFLOAT16; import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.BBQ_MIN_DIMS; +import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapperTestUtils.defaultDenseVectorIndexOptions; import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapperTestUtils.getSupportedSimilarities; import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapperTestUtils.randomCompatibleDimensions; import static org.elasticsearch.xpack.inference.mapper.SemanticFieldMapper.INDEX_OPTIONS_FIELD; @@ -702,6 +705,49 @@ protected SemanticIndexOptions extractCurrentIndexOptions(MapperService mapperSe return currentIndexOptions; } + protected SemanticIndexOptions getDefaultIndexOptions(Model model, MapperService mapperService) { + IndexVersion indexVersion = mapperService.getIndexSettings().getIndexVersionCreated(); + boolean experimentalFeatures = DENSE_VECTOR_EXPERIMENTAL_FEATURES_SETTING.get(mapperService.getIndexSettings().getSettings()); + + TaskType taskType = model.getTaskType(); + DenseVectorFieldMapper.ElementType elementType = model.getServiceSettings().elementType(); + Integer dimensions = model.getServiceSettings().dimensions(); + + return switch (taskType) { + case TEXT_EMBEDDING, EMBEDDING -> { + boolean floatFamilyElementType = elementType == DenseVectorFieldMapper.ElementType.FLOAT + || elementType == DenseVectorFieldMapper.ElementType.BFLOAT16; + if (floatFamilyElementType + && SemanticTextFieldMapper.setExplicitIndexOptionsForSemanticText(indexVersion) + && dimensions >= BBQ_MIN_DIMS) { + yield new SemanticIndexOptions( + SemanticIndexOptions.SupportedIndexOptions.DENSE_VECTOR, + defaultBbqHnswDenseVectorIndexOptions() + ); + } else { + var denseDefaults = defaultDenseVectorIndexOptions( + indexVersion, + operationMode == License.OperationMode.ENTERPRISE, + true, + dimensions, + elementType, + experimentalFeatures + ); + yield denseDefaults == null + ? null + : new SemanticIndexOptions(SemanticIndexOptions.SupportedIndexOptions.DENSE_VECTOR, denseDefaults); + } + } + case SPARSE_EMBEDDING -> { + var sparseDefaults = SparseVectorFieldMapper.SparseVectorIndexOptions.getDefaultIndexOptions(indexVersion); + yield sparseDefaults == null + ? null + : new SemanticIndexOptions(SemanticIndexOptions.SupportedIndexOptions.SPARSE_VECTOR, sparseDefaults); + } + default -> throw new AssertionError("Unexpected task type [" + taskType + "]"); + }; + } + protected void givenModelSettings(String inferenceId, MinimalServiceSettings modelSettings) { when(globalModelRegistry.getMinimalServiceSettings(inferenceId)).thenReturn(modelSettings); } @@ -844,4 +890,13 @@ private static void validateIndexVersion(IndexVersion indexVersion, boolean useL throw new IllegalArgumentException("Index version " + indexVersion + " does not support legacy semantic text format"); } } + + private static DenseVectorFieldMapper.DenseVectorIndexOptions defaultBbqHnswDenseVectorIndexOptions() { + int m = Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN; + int efConstruction = Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH; + DenseVectorFieldMapper.RescoreVector rescoreVector = new DenseVectorFieldMapper.RescoreVector( + SemanticTextFieldMapper.DEFAULT_RESCORE_OVERSAMPLE + ); + return new DenseVectorFieldMapper.BBQHnswIndexOptions(m, efConstruction, false, rescoreVector, -1); + } } From 97b40db5a05f1b16064b305e757639ce97e17b78 Mon Sep 17 00:00:00 2001 From: Mike Pellegrini Date: Wed, 22 Jul 2026 11:14:06 -0400 Subject: [PATCH 04/24] Delegate explicit dense vector index options to a hook --- .../AbstractSemanticMapperTestCase.java | 35 ++++++------------- .../mapper/SemanticTextFieldMapperTests.java | 22 ++++++++++++ 2 files changed, 33 insertions(+), 24 deletions(-) diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/AbstractSemanticMapperTestCase.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/AbstractSemanticMapperTestCase.java index eaad2acaadaf3..0c47713233b03 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/AbstractSemanticMapperTestCase.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/AbstractSemanticMapperTestCase.java @@ -7,7 +7,6 @@ package org.elasticsearch.xpack.inference.mapper; -import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; import org.elasticsearch.cluster.ClusterChangedEvent; @@ -715,17 +714,9 @@ protected SemanticIndexOptions getDefaultIndexOptions(Model model, MapperService return switch (taskType) { case TEXT_EMBEDDING, EMBEDDING -> { - boolean floatFamilyElementType = elementType == DenseVectorFieldMapper.ElementType.FLOAT - || elementType == DenseVectorFieldMapper.ElementType.BFLOAT16; - if (floatFamilyElementType - && SemanticTextFieldMapper.setExplicitIndexOptionsForSemanticText(indexVersion) - && dimensions >= BBQ_MIN_DIMS) { - yield new SemanticIndexOptions( - SemanticIndexOptions.SupportedIndexOptions.DENSE_VECTOR, - defaultBbqHnswDenseVectorIndexOptions() - ); - } else { - var denseDefaults = defaultDenseVectorIndexOptions( + var denseDefaults = getExplicitDenseVectorIndexOptions(model, indexVersion); + if (denseDefaults == null) { + denseDefaults = defaultDenseVectorIndexOptions( indexVersion, operationMode == License.OperationMode.ENTERPRISE, true, @@ -733,10 +724,11 @@ yield new SemanticIndexOptions( elementType, experimentalFeatures ); - yield denseDefaults == null - ? null - : new SemanticIndexOptions(SemanticIndexOptions.SupportedIndexOptions.DENSE_VECTOR, denseDefaults); } + + yield denseDefaults == null + ? null + : new SemanticIndexOptions(SemanticIndexOptions.SupportedIndexOptions.DENSE_VECTOR, denseDefaults); } case SPARSE_EMBEDDING -> { var sparseDefaults = SparseVectorFieldMapper.SparseVectorIndexOptions.getDefaultIndexOptions(indexVersion); @@ -748,6 +740,10 @@ yield new SemanticIndexOptions( }; } + protected DenseVectorFieldMapper.DenseVectorIndexOptions getExplicitDenseVectorIndexOptions(Model model, IndexVersion indexVersion) { + return null; + } + protected void givenModelSettings(String inferenceId, MinimalServiceSettings modelSettings) { when(globalModelRegistry.getMinimalServiceSettings(inferenceId)).thenReturn(modelSettings); } @@ -890,13 +886,4 @@ private static void validateIndexVersion(IndexVersion indexVersion, boolean useL throw new IllegalArgumentException("Index version " + indexVersion + " does not support legacy semantic text format"); } } - - private static DenseVectorFieldMapper.DenseVectorIndexOptions defaultBbqHnswDenseVectorIndexOptions() { - int m = Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN; - int efConstruction = Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH; - DenseVectorFieldMapper.RescoreVector rescoreVector = new DenseVectorFieldMapper.RescoreVector( - SemanticTextFieldMapper.DEFAULT_RESCORE_OVERSAMPLE - ); - return new DenseVectorFieldMapper.BBQHnswIndexOptions(m, efConstruction, false, rescoreVector, -1); - } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java index e2154555fb97b..8b5331fc7b68b 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java @@ -277,6 +277,28 @@ protected void assertSearchable(MappedFieldType fieldType) { assertTrue(fieldType.isSearchable()); } + @Override + protected DenseVectorFieldMapper.DenseVectorIndexOptions getExplicitDenseVectorIndexOptions(Model model, IndexVersion indexVersion) { + TaskType taskType = model.getTaskType(); + DenseVectorFieldMapper.ElementType elementType = model.getServiceSettings().elementType(); + Integer dimensions = model.getServiceSettings().dimensions(); + + if (taskType == TaskType.SPARSE_EMBEDDING) { + return null; + } + + DenseVectorFieldMapper.DenseVectorIndexOptions indexOptions = null; + boolean floatFamilyElementType = elementType == DenseVectorFieldMapper.ElementType.FLOAT + || elementType == DenseVectorFieldMapper.ElementType.BFLOAT16; + if (floatFamilyElementType + && SemanticTextFieldMapper.setExplicitIndexOptionsForSemanticText(indexVersion) + && dimensions >= DenseVectorFieldMapper.BBQ_MIN_DIMS) { + indexOptions = defaultBbqHnswDenseVectorIndexOptions(); + } + + return indexOptions; + } + /** * Randomized sweep over all combinations of endpoint availability and * index version to confirm correct default inference ID. From 584d09083574dd0f15c79c35a2f68c5f996343bd Mon Sep 17 00:00:00 2001 From: Mike Pellegrini Date: Wed, 22 Jul 2026 12:06:41 -0400 Subject: [PATCH 05/24] Update getDefaultIndexOptions to determine the element type override --- .../mapper/AbstractSemanticMapperTestCase.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/AbstractSemanticMapperTestCase.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/AbstractSemanticMapperTestCase.java index 0c47713233b03..b2caf54b83290 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/AbstractSemanticMapperTestCase.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/AbstractSemanticMapperTestCase.java @@ -726,9 +726,15 @@ protected SemanticIndexOptions getDefaultIndexOptions(Model model, MapperService ); } - yield denseDefaults == null + DenseVectorFieldMapper.ElementType elementTypeOverride = indexVersion.onOrAfter(SEMANTIC_TEXT_DEFAULTS_TO_BFLOAT16) + && elementType == DenseVectorFieldMapper.ElementType.FLOAT ? DenseVectorFieldMapper.ElementType.BFLOAT16 : null; + + yield denseDefaults == null && elementTypeOverride == null ? null - : new SemanticIndexOptions(SemanticIndexOptions.SupportedIndexOptions.DENSE_VECTOR, denseDefaults); + : new SemanticIndexOptions( + SemanticIndexOptions.SupportedIndexOptions.DENSE_VECTOR, + new ExtendedDenseVectorIndexOptions(denseDefaults, elementTypeOverride) + ); } case SPARSE_EMBEDDING -> { var sparseDefaults = SparseVectorFieldMapper.SparseVectorIndexOptions.getDefaultIndexOptions(indexVersion); From bf2b1adbef0085c3c37e71d12ee90a910430f6d4 Mon Sep 17 00:00:00 2001 From: Mike Pellegrini Date: Wed, 22 Jul 2026 12:54:26 -0400 Subject: [PATCH 06/24] Don't compute partial default index options in assertSemanticField --- .../mapper/AbstractSemanticMapperTestCase.java | 18 +++++++++--------- .../mapper/SemanticTextFieldMapperTests.java | 14 +++----------- 2 files changed, 12 insertions(+), 20 deletions(-) diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/AbstractSemanticMapperTestCase.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/AbstractSemanticMapperTestCase.java index b2caf54b83290..510b0459397c6 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/AbstractSemanticMapperTestCase.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/AbstractSemanticMapperTestCase.java @@ -602,22 +602,21 @@ protected void assertEmbeddingsField( assertThat(embeddingsMapper, instanceOf(DenseVectorFieldMapper.class)); DenseVectorFieldMapper denseVectorFieldMapper = (DenseVectorFieldMapper) embeddingsMapper; + IndexOptions expectedBaseIndexOptions = null; + DenseVectorFieldMapper.ElementType expectedElementType = modelSettings.elementType(); if (expectedIndexOptions != null) { IndexOptions expectedEmbeddingFieldIndexOptions = expectedIndexOptions.indexOptions(); if (expectedEmbeddingFieldIndexOptions instanceof ExtendedDenseVectorIndexOptions edvio) { - assertEquals(edvio.getBaseIndexOptions(), denseVectorFieldMapper.fieldType().getIndexOptions()); + expectedBaseIndexOptions = edvio.getBaseIndexOptions(); + if (edvio.getElementType() != null) { + expectedElementType = edvio.getElementType(); + } } else { - assertEquals(expectedEmbeddingFieldIndexOptions, denseVectorFieldMapper.fieldType().getIndexOptions()); + expectedBaseIndexOptions = expectedEmbeddingFieldIndexOptions; } - } else { - assertNull(denseVectorFieldMapper.fieldType().getIndexOptions()); } - DenseVectorFieldMapper.ElementType expectedElementType = getExpectedElementType( - indexVersion, - modelSettings.elementType(), - expectedIndexOptions - ); + assertEquals(expectedBaseIndexOptions, denseVectorFieldMapper.fieldType().getIndexOptions()); assertEquals(expectedElementType, denseVectorFieldMapper.fieldType().getElementType()); assertEquals(modelSettings.dimensions().intValue(), denseVectorFieldMapper.fieldType().getVectorDimensions()); if (modelSettings.similarity() != null && indexVersion.onOrAfter(NEW_SPARSE_VECTOR)) { @@ -629,6 +628,7 @@ protected void assertEmbeddingsField( } } + // TODO: Try to remove protected static DenseVectorFieldMapper.ElementType getExpectedElementType( IndexVersion indexVersion, DenseVectorFieldMapper.ElementType modelElementType, diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java index 8b5331fc7b68b..9e1e417f72e2b 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java @@ -57,6 +57,7 @@ import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; import org.elasticsearch.index.mapper.vectors.DenseVectorFieldTypeTests; +import org.elasticsearch.index.mapper.vectors.IndexOptions; import org.elasticsearch.index.mapper.vectors.SparseVectorFieldMapper; import org.elasticsearch.index.mapper.vectors.SparseVectorFieldMapperTests; import org.elasticsearch.index.mapper.vectors.SparseVectorFieldTypeTests; @@ -1087,17 +1088,8 @@ protected void assertEmbeddingsField( SparseVectorFieldMapper sparseVectorFieldMapper = (SparseVectorFieldMapper) embeddingsMapper; assertEquals(sparseVectorFieldMapper.fieldType().isStored(), fieldType.useLegacyFormat() == false); - SparseVectorFieldMapper.SparseVectorIndexOptions applied = sparseVectorFieldMapper.fieldType().getIndexOptions(); - SparseVectorFieldMapper.SparseVectorIndexOptions expected = expectedIndexOptions == null - ? null - : (SparseVectorFieldMapper.SparseVectorIndexOptions) expectedIndexOptions.indexOptions(); - if (expected == null && applied != null) { - var indexVersionCreated = mapperService.getIndexSettings().getIndexVersionCreated(); - if (SparseVectorFieldMapper.SparseVectorIndexOptions.isDefaultOptions(applied, indexVersionCreated)) { - expected = SparseVectorFieldMapper.SparseVectorIndexOptions.getDefaultIndexOptions(indexVersionCreated); - } - } - assertEquals(expected, applied); + IndexOptions expectedBaseIndexOptions = expectedIndexOptions == null ? null : expectedIndexOptions.indexOptions(); + assertEquals(expectedBaseIndexOptions, sparseVectorFieldMapper.fieldType().getIndexOptions()); } else { super.assertEmbeddingsField(mapperService, fieldType, embeddingsMapper, expectedIndexOptions); } From 75fa4a732e9043c05637873b0c20e059c435c19f Mon Sep 17 00:00:00 2001 From: Mike Pellegrini Date: Wed, 22 Jul 2026 15:48:34 -0400 Subject: [PATCH 07/24] Pass model to assertSemanticField --- .../AbstractSemanticMapperTestCase.java | 80 +++++++------ .../mapper/SemanticTextFieldMapperTests.java | 110 ++++++++++++------ 2 files changed, 120 insertions(+), 70 deletions(-) diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/AbstractSemanticMapperTestCase.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/AbstractSemanticMapperTestCase.java index 510b0459397c6..97d4fa00cf713 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/AbstractSemanticMapperTestCase.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/AbstractSemanticMapperTestCase.java @@ -276,7 +276,7 @@ public void testUpdateInferenceId_GivenNoModelSettings() throws IOException { var mapperService = createSemanticMapperService(semanticMapping(fieldName, oldInferenceId)); assertInferenceEndpoints(mapperService, fieldName, oldInferenceId, oldInferenceId); - assertSemanticField(mapperService, fieldName, false, null, null); + assertSemanticField(mapperService, fieldName, false, oldModel, null, null); if (oldModel != null) { assertEmbeddingsFieldMapperMatchesModel(mapperService, fieldName, oldModel); } @@ -291,7 +291,7 @@ public void testUpdateInferenceId_GivenNoModelSettings() throws IOException { merge(mapperService, semanticMapping(fieldName, newInferenceId)); assertInferenceEndpoints(mapperService, fieldName, newInferenceId, newInferenceId); - assertSemanticField(mapperService, fieldName, false, null, null); + assertSemanticField(mapperService, fieldName, false, newModel, null, null); if (newModel != null) { assertEmbeddingsFieldMapperMatchesModel(mapperService, fieldName, newModel); } @@ -311,9 +311,8 @@ public void testUpdateInferenceId_GivenModelSettings() throws IOException { final MapperService mapperService = createSemanticMapperService( semanticMapping(fieldName, oldInferenceId, previousModelSettings) ); - final SemanticIndexOptions currentIndexOptions = extractCurrentIndexOptions(mapperService, fieldName); assertInferenceEndpoints(mapperService, fieldName, oldInferenceId, oldInferenceId); - assertSemanticField(mapperService, fieldName, true, null, currentIndexOptions); + assertSemanticField(mapperService, fieldName, true, oldModel, null, null); assertEmbeddingsFieldMapperMatchesModel(mapperService, fieldName, oldModel); final CheckedRunnable mergeRunner = () -> merge(mapperService, semanticMapping(fieldName, newInferenceId)); @@ -326,7 +325,7 @@ public void testUpdateInferenceId_GivenModelSettings() throws IOException { mergeRunner.run(); assertInferenceEndpoints(mapperService, fieldName, newInferenceId, newInferenceId); - assertSemanticField(mapperService, fieldName, true, null, currentIndexOptions); + assertSemanticField(mapperService, fieldName, true, newModel, null, null); assertEmbeddingsFieldMapperMatchesModel(mapperService, fieldName, newModel); } else { final TestModel incompatibleModel = createIncompatibleModel(newInferenceId, oldModel); @@ -533,8 +532,9 @@ protected void assertSemanticField( MapperService mapperService, String fieldName, boolean expectedModelSettings, - ChunkingSettings expectedChunkingSettings, - SemanticIndexOptions expectedIndexOptions + @Nullable Model model, + @Nullable ChunkingSettings expectedChunkingSettings, + @Nullable SemanticIndexOptions expectedIndexOptions ) { T semanticFieldMapper = getSemanticFieldMapper(mapperService, fieldName); @@ -552,15 +552,28 @@ protected void assertSemanticField( assertThat(chunksMapper.fullPath(), equalTo(getChunksFieldName(fieldName))); assertChunksTextField(semanticFieldType, chunksMapper); - if (expectedModelSettings) { - assertNotNull(semanticFieldType.getModelSettings()); - Mapper embeddingsMapper = chunksMapper.getMapper(CHUNKED_EMBEDDINGS_FIELD); + Mapper embeddingsMapper = chunksMapper.getMapper(CHUNKED_EMBEDDINGS_FIELD); + if (model != null) { + SemanticIndexOptions resolvedExpectedIndexOptions = expectedIndexOptions; + if (resolvedExpectedIndexOptions == null) { + resolvedExpectedIndexOptions = getDefaultIndexOptions(model, mapperService); + } + assertNotNull(embeddingsMapper); assertThat(embeddingsMapper, instanceOf(FieldMapper.class)); FieldMapper embeddingsFieldMapper = (FieldMapper) embeddingsMapper; assertSame(embeddingsFieldMapper.fieldType(), mapperService.mappingLookup().getFieldType(getEmbeddingsFieldName(fieldName))); assertThat(embeddingsMapper.fullPath(), equalTo(getEmbeddingsFieldName(fieldName))); - assertEmbeddingsField(mapperService, semanticFieldType, embeddingsFieldMapper, expectedIndexOptions); + assertEmbeddingsField(mapperService, embeddingsFieldMapper, model, resolvedExpectedIndexOptions); + } else { + assertNull(embeddingsMapper); + } + + if (expectedModelSettings) { + if (model == null) { + throw new IllegalArgumentException("model must be provided to check model settings"); + } + assertModelSettings(semanticFieldType, model); } else { assertNull(semanticFieldType.getModelSettings()); } @@ -585,25 +598,25 @@ protected void assertChunksTextField(U fieldType, NestedObjectMapper chunksMappe } /** - * Asserts the embeddings sub-mapper against the field's model settings and expected index options. The base implementation + * Asserts the embeddings sub-mapper against the referenced {@link Model} and expected index options. The base implementation * covers the dense task types ({@code text_embedding}, {@code embedding}); {@code semantic_text} overrides this to also cover * {@code sparse_embedding}. */ protected void assertEmbeddingsField( MapperService mapperService, - U fieldType, FieldMapper embeddingsMapper, + Model model, @Nullable SemanticIndexOptions expectedIndexOptions ) { IndexVersion indexVersion = mapperService.getIndexSettings().getIndexVersionCreated(); - MinimalServiceSettings modelSettings = fieldType.getModelSettings(); - TaskType taskType = modelSettings.taskType(); + TaskType taskType = model.getTaskType(); + ServiceSettings serviceSettings = model.getServiceSettings(); if (taskType == TaskType.TEXT_EMBEDDING || taskType == TaskType.EMBEDDING) { assertThat(embeddingsMapper, instanceOf(DenseVectorFieldMapper.class)); DenseVectorFieldMapper denseVectorFieldMapper = (DenseVectorFieldMapper) embeddingsMapper; IndexOptions expectedBaseIndexOptions = null; - DenseVectorFieldMapper.ElementType expectedElementType = modelSettings.elementType(); + DenseVectorFieldMapper.ElementType expectedElementType = serviceSettings.elementType(); if (expectedIndexOptions != null) { IndexOptions expectedEmbeddingFieldIndexOptions = expectedIndexOptions.indexOptions(); if (expectedEmbeddingFieldIndexOptions instanceof ExtendedDenseVectorIndexOptions edvio) { @@ -618,16 +631,27 @@ protected void assertEmbeddingsField( assertEquals(expectedBaseIndexOptions, denseVectorFieldMapper.fieldType().getIndexOptions()); assertEquals(expectedElementType, denseVectorFieldMapper.fieldType().getElementType()); - assertEquals(modelSettings.dimensions().intValue(), denseVectorFieldMapper.fieldType().getVectorDimensions()); - if (modelSettings.similarity() != null && indexVersion.onOrAfter(NEW_SPARSE_VECTOR)) { + assertEquals(serviceSettings.dimensions().intValue(), denseVectorFieldMapper.fieldType().getVectorDimensions()); + if (serviceSettings.similarity() != null && indexVersion.onOrAfter(NEW_SPARSE_VECTOR)) { // We don't set similarity on pre 8.11 indices - assertEquals(modelSettings.similarity().vectorSimilarity(), denseVectorFieldMapper.fieldType().getSimilarity()); + assertEquals(serviceSettings.similarity().vectorSimilarity(), denseVectorFieldMapper.fieldType().getSimilarity()); } } else { - throw new AssertionError("Invalid task type [" + modelSettings.taskType() + "]"); + throw new AssertionError("Invalid task type [" + taskType + "]"); } } + protected void assertModelSettings(U fieldType, Model model) { + MinimalServiceSettings actual = fieldType.getModelSettings(); + assertNotNull(actual); + + // TODO: Use canMergeWith + assertEquals(model.getTaskType(), actual.taskType()); + assertEquals(model.getServiceSettings().dimensions(), actual.dimensions()); + assertEquals(model.getServiceSettings().similarity(), actual.similarity()); + assertEquals(model.getServiceSettings().elementType(), actual.elementType()); + } + // TODO: Try to remove protected static DenseVectorFieldMapper.ElementType getExpectedElementType( IndexVersion indexVersion, @@ -651,6 +675,7 @@ protected static DenseVectorFieldMapper.ElementType getExpectedElementType( * Asserts that the generated embeddings sub-mapper (sparse or dense) matches the task type and service settings of the * referenced {@link Model}, including dimensions, element type, and similarity for dense models. */ + // TODO: Remove protected void assertEmbeddingsFieldMapperMatchesModel(MapperService mapperService, String fieldName, Model model) { Mapper embeddingsFieldMapper = mapperService.mappingLookup().getMapper(getEmbeddingsFieldName(fieldName)); switch (model.getTaskType()) { @@ -690,20 +715,6 @@ protected static void assertInferenceEndpoints( assertEquals(expectedSearchInferenceId, semanticFieldType.getSearchInferenceId()); } - protected SemanticIndexOptions extractCurrentIndexOptions(MapperService mapperService, String fieldName) { - SemanticIndexOptions currentIndexOptions = null; - T sfm = getSemanticFieldMapper(mapperService, fieldName); - FieldMapper embeddingsMapper = sfm.fieldType().getEmbeddingsField(); - if (embeddingsMapper instanceof DenseVectorFieldMapper dvm) { - IndexOptions denseIndexOptions = dvm.fieldType().getIndexOptions(); - if (denseIndexOptions != null) { - currentIndexOptions = new SemanticIndexOptions(SemanticIndexOptions.SupportedIndexOptions.DENSE_VECTOR, denseIndexOptions); - } - } - - return currentIndexOptions; - } - protected SemanticIndexOptions getDefaultIndexOptions(Model model, MapperService mapperService) { IndexVersion indexVersion = mapperService.getIndexSettings().getIndexVersionCreated(); boolean experimentalFeatures = DENSE_VECTOR_EXPERIMENTAL_FEATURES_SETTING.get(mapperService.getIndexSettings().getSettings()); @@ -726,6 +737,7 @@ protected SemanticIndexOptions getDefaultIndexOptions(Model model, MapperService ); } + // TODO: Don't duplicate this logic DenseVectorFieldMapper.ElementType elementTypeOverride = indexVersion.onOrAfter(SEMANTIC_TEXT_DEFAULTS_TO_BFLOAT16) && elementType == DenseVectorFieldMapper.ElementType.FLOAT ? DenseVectorFieldMapper.ElementType.BFLOAT16 : null; diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java index 9e1e417f72e2b..a13d5539a7cae 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java @@ -312,7 +312,7 @@ public void testDefaults() throws Exception { MapperService mapperService = createSemanticMapperService(fieldMapping, IndexVersions.SEMANTIC_TEXT_DEFAULTS_TO_JINA_V5); DocumentMapper mapper = mapperService.documentMapper(); assertEquals(Strings.toString(expectedMapping), mapper.mappingSource().toString()); - assertSemanticField(mapperService, fieldName, false, null, null); + assertSemanticField(mapperService, fieldName, false, null, null, null); assertInferenceEndpoints(mapperService, fieldName, DEFAULT_EIS_JINA_V5_INFERENCE_ID, DEFAULT_EIS_JINA_V5_INFERENCE_ID); ParsedDocument doc1 = mapper.parse(source(this::writeField)); @@ -582,7 +582,7 @@ public void testSetInferenceEndpoints() throws IOException { { final XContentBuilder fieldMapping = semanticMapping("field", inferenceId); final MapperService mapperService = createSemanticMapperService(fieldMapping); - assertSemanticField(mapperService, fieldName, false, null, null); + assertSemanticField(mapperService, fieldName, false, null, null, null); assertInferenceEndpoints(mapperService, fieldName, inferenceId, inferenceId); assertSerialization.accept(fieldMapping, mapperService); } @@ -592,14 +592,14 @@ public void testSetInferenceEndpoints() throws IOException { ); final XContentBuilder expectedMapping = semanticMapping("field", DEFAULT_EIS_JINA_V5_INFERENCE_ID, searchInferenceId); final MapperService mapperService = createSemanticMapperService(fieldMapping, IndexVersions.SEMANTIC_TEXT_DEFAULTS_TO_JINA_V5); - assertSemanticField(mapperService, fieldName, false, null, null); + assertSemanticField(mapperService, fieldName, false, null, null, null); assertInferenceEndpoints(mapperService, fieldName, DEFAULT_EIS_JINA_V5_INFERENCE_ID, searchInferenceId); assertSerialization.accept(expectedMapping, mapperService); } { final XContentBuilder fieldMapping = semanticMapping("field", inferenceId, searchInferenceId); MapperService mapperService = createSemanticMapperService(fieldMapping); - assertSemanticField(mapperService, fieldName, false, null, null); + assertSemanticField(mapperService, fieldName, false, null, null, null); assertInferenceEndpoints(mapperService, fieldName, inferenceId, searchInferenceId); assertSerialization.accept(fieldMapping, mapperService); } @@ -710,6 +710,7 @@ public void testMultiFieldsSupport() throws IOException { SemanticIndexOptions semanticIndexOptions = expectedIndexOptions == null ? null : new SemanticIndexOptions(SemanticIndexOptions.SupportedIndexOptions.SPARSE_VECTOR, expectedIndexOptions); + Model sparseModel = TestModel.createRandomInstance(TaskType.SPARSE_EMBEDDING); var mapperService = createSemanticMapperServiceWithIndexVersion(fieldMapping(b -> { b.field("type", "text"); b.startObject("fields"); @@ -722,7 +723,7 @@ public void testMultiFieldsSupport() throws IOException { b.endObject(); b.endObject(); }), indexVersion); - assertSemanticField(mapperService, "field.semantic", true, null, semanticIndexOptions); + assertSemanticField(mapperService, "field.semantic", true, sparseModel, null, semanticIndexOptions); mapperService = createSemanticMapperServiceWithIndexVersion(fieldMapping(b -> { b.field("type", "semantic_text"); @@ -736,7 +737,7 @@ public void testMultiFieldsSupport() throws IOException { b.endObject(); b.endObject(); }), indexVersion); - assertSemanticField(mapperService, "field", true, null, semanticIndexOptions); + assertSemanticField(mapperService, "field", true, sparseModel, null, semanticIndexOptions); mapperService = createSemanticMapperServiceWithIndexVersion(fieldMapping(b -> { b.field("type", "semantic_text"); @@ -754,8 +755,8 @@ public void testMultiFieldsSupport() throws IOException { b.endObject(); b.endObject(); }), indexVersion); - assertSemanticField(mapperService, "field", true, null, semanticIndexOptions); - assertSemanticField(mapperService, "field.semantic", true, null, semanticIndexOptions); + assertSemanticField(mapperService, "field", true, sparseModel, null, semanticIndexOptions); + assertSemanticField(mapperService, "field.semantic", true, sparseModel, null, semanticIndexOptions); Exception e = expectThrows(MapperParsingException.class, () -> createSemanticMapperService(fieldMapping(b -> { b.field("type", "semantic_text"); @@ -776,6 +777,7 @@ public void testDynamicUpdate() throws IOException { final String searchInferenceId = "search_test_service"; { + Model model = TestModel.createRandomInstance(TaskType.SPARSE_EMBEDDING); MapperService mapperService = createSemanticMapperService(semanticMapping(fieldName, inferenceId)); performDynamicUpdate( mapperService, @@ -784,11 +786,12 @@ public void testDynamicUpdate() throws IOException { new MinimalServiceSettings("service", TaskType.SPARSE_EMBEDDING, null, null, null) ); var expectedIndexOptions = getDefaultSparseVectorIndexOptionsForMapper(mapperService); - assertSemanticField(mapperService, fieldName, true, null, expectedIndexOptions); + assertSemanticField(mapperService, fieldName, true, model, null, expectedIndexOptions); assertInferenceEndpoints(mapperService, fieldName, inferenceId, inferenceId); } { + Model model = TestModel.createRandomInstance(TaskType.SPARSE_EMBEDDING); MapperService mapperService = createSemanticMapperService(semanticMapping(fieldName, inferenceId, searchInferenceId)); performDynamicUpdate( mapperService, @@ -797,7 +800,7 @@ public void testDynamicUpdate() throws IOException { new MinimalServiceSettings("service", TaskType.SPARSE_EMBEDDING, null, null, null) ); var expectedIndexOptions = getDefaultSparseVectorIndexOptionsForMapper(mapperService); - assertSemanticField(mapperService, fieldName, true, null, expectedIndexOptions); + assertSemanticField(mapperService, fieldName, true, model, null, expectedIndexOptions); assertInferenceEndpoints(mapperService, fieldName, inferenceId, searchInferenceId); } } @@ -806,7 +809,7 @@ public void testUpdateModelSettings() throws IOException { for (int depth = 1; depth < 5; depth++) { String fieldName = randomFieldName(depth); MapperService mapperService = createSemanticMapperService(semanticMapping(fieldName, "test_model")); - assertSemanticField(mapperService, fieldName, false, null, null); + assertSemanticField(mapperService, fieldName, false, null, null, null); { Exception exc = expectThrows( MapperParsingException.class, @@ -826,17 +829,19 @@ public void testUpdateModelSettings() throws IOException { assertThat(exc.getMessage(), containsString("Required [task_type]")); } { + Model sparseModel = TestModel.createRandomInstance(TaskType.SPARSE_EMBEDDING); merge( mapperService, semanticMapping(fieldName, "test_model", new MinimalServiceSettings(null, TaskType.SPARSE_EMBEDDING, null, null, null)) ); var expectedIndexOptions = getDefaultSparseVectorIndexOptionsForMapper(mapperService); - assertSemanticField(mapperService, fieldName, true, null, expectedIndexOptions); + assertSemanticField(mapperService, fieldName, true, sparseModel, null, expectedIndexOptions); } { + Model sparseModel = TestModel.createRandomInstance(TaskType.SPARSE_EMBEDDING); merge(mapperService, semanticMapping(fieldName, "test_model")); var expectedIndexOptions = getDefaultSparseVectorIndexOptionsForMapper(mapperService); - assertSemanticField(mapperService, fieldName, true, null, expectedIndexOptions); + assertSemanticField(mapperService, fieldName, true, sparseModel, null, expectedIndexOptions); } { Exception exc = expectThrows( @@ -883,6 +888,21 @@ public void testDenseVectorIndexOptionValidation() throws IOException { } } + private static Model denseTextEmbeddingModel( + int dimensions, + SimilarityMeasure similarity, + DenseVectorFieldMapper.ElementType elementType + ) { + return new TestModel( + randomAlphaOfLength(4), + TaskType.TEXT_EMBEDDING, + randomAlphaOfLength(4), + new TestModel.TestServiceSettings(randomAlphaOfLength(4), dimensions, similarity, elementType), + new TestModel.TestTaskSettings(randomInt(3)), + new TestModel.TestSecretSettings(randomAlphaOfLength(4)) + ); + } + private void addSparseVectorModelSettingsToBuilder(XContentBuilder b) throws IOException { b.startObject("model_settings"); b.field("task_type", TaskType.SPARSE_EMBEDDING); @@ -932,7 +952,14 @@ public void testSparseVectorIndexOptionsValidationAndMapping() throws IOExceptio b.endObject(); }), indexVersion); - assertSemanticField(mapper, fieldName, true, null, expectedIndexOptions); + assertSemanticField( + mapper, + fieldName, + true, + TestModel.createRandomInstance(TaskType.SPARSE_EMBEDDING), + null, + expectedIndexOptions + ); } } @@ -963,7 +990,7 @@ public void testSparseVectorMappingUpdate() throws IOException { SparseVectorFieldMapper.SparseVectorIndexOptions.getDefaultIndexOptions(indexVersion) ) : indexOptions; - assertSemanticField(mapperService, fieldName, false, chunkingSettings, expectedIndexOptions); + assertSemanticField(mapperService, fieldName, false, model, chunkingSettings, expectedIndexOptions); final SemanticIndexOptions newIndexOptions = randomSemanticIndexOptions(TaskType.SPARSE_EMBEDDING); expectedIndexOptions = (newIndexOptions == null) @@ -978,7 +1005,7 @@ public void testSparseVectorMappingUpdate() throws IOException { mapperService, semanticMapping(fieldName, model.getInferenceEntityId(), null, null, newChunkingSettings, newIndexOptions) ); - assertSemanticField(mapperService, fieldName, false, newChunkingSettings, expectedIndexOptions); + assertSemanticField(mapperService, fieldName, false, model, newChunkingSettings, expectedIndexOptions); } } @@ -998,21 +1025,22 @@ public void testUpdateSearchInferenceId() throws IOException { for (int depth = 1; depth < 5; depth++) { String fieldName = randomFieldName(depth); MapperService mapperService = createSemanticMapperService(buildMapping.apply(fieldName, null)); - assertSemanticField(mapperService, fieldName, false, null, null); + assertSemanticField(mapperService, fieldName, false, null, null, null); assertInferenceEndpoints(mapperService, fieldName, inferenceId, inferenceId); merge(mapperService, buildMapping.apply(fieldName, searchInferenceId1)); - assertSemanticField(mapperService, fieldName, false, null, null); + assertSemanticField(mapperService, fieldName, false, null, null, null); assertInferenceEndpoints(mapperService, fieldName, inferenceId, searchInferenceId1); merge(mapperService, buildMapping.apply(fieldName, searchInferenceId2)); - assertSemanticField(mapperService, fieldName, false, null, null); + assertSemanticField(mapperService, fieldName, false, null, null, null); assertInferenceEndpoints(mapperService, fieldName, inferenceId, searchInferenceId2); merge(mapperService, buildMapping.apply(fieldName, null)); - assertSemanticField(mapperService, fieldName, false, null, null); + assertSemanticField(mapperService, fieldName, false, null, null, null); assertInferenceEndpoints(mapperService, fieldName, inferenceId, inferenceId); + Model sparseModel = TestModel.createRandomInstance(TaskType.SPARSE_EMBEDDING); mapperService = createSemanticMapperService( semanticMapping( fieldName, @@ -1021,19 +1049,19 @@ public void testUpdateSearchInferenceId() throws IOException { ) ); var expectedIndexOptions = getDefaultSparseVectorIndexOptionsForMapper(mapperService); - assertSemanticField(mapperService, fieldName, true, null, expectedIndexOptions); + assertSemanticField(mapperService, fieldName, true, sparseModel, null, expectedIndexOptions); assertInferenceEndpoints(mapperService, fieldName, inferenceId, inferenceId); merge(mapperService, buildMapping.apply(fieldName, searchInferenceId1)); - assertSemanticField(mapperService, fieldName, true, null, expectedIndexOptions); + assertSemanticField(mapperService, fieldName, true, sparseModel, null, expectedIndexOptions); assertInferenceEndpoints(mapperService, fieldName, inferenceId, searchInferenceId1); merge(mapperService, buildMapping.apply(fieldName, searchInferenceId2)); - assertSemanticField(mapperService, fieldName, true, null, expectedIndexOptions); + assertSemanticField(mapperService, fieldName, true, sparseModel, null, expectedIndexOptions); assertInferenceEndpoints(mapperService, fieldName, inferenceId, searchInferenceId2); merge(mapperService, buildMapping.apply(fieldName, null)); - assertSemanticField(mapperService, fieldName, true, null, expectedIndexOptions); + assertSemanticField(mapperService, fieldName, true, sparseModel, null, expectedIndexOptions); assertInferenceEndpoints(mapperService, fieldName, inferenceId, inferenceId); } } @@ -1079,19 +1107,19 @@ protected void assertChunksTextField(SemanticTextFieldMapper.SemanticTextFieldTy @Override protected void assertEmbeddingsField( MapperService mapperService, - SemanticTextFieldMapper.SemanticTextFieldType fieldType, FieldMapper embeddingsMapper, + Model model, @Nullable SemanticIndexOptions expectedIndexOptions ) { - if (fieldType.getModelSettings().taskType() == TaskType.SPARSE_EMBEDDING) { + if (model.getTaskType() == TaskType.SPARSE_EMBEDDING) { assertThat(embeddingsMapper, instanceOf(SparseVectorFieldMapper.class)); SparseVectorFieldMapper sparseVectorFieldMapper = (SparseVectorFieldMapper) embeddingsMapper; - assertEquals(sparseVectorFieldMapper.fieldType().isStored(), fieldType.useLegacyFormat() == false); + assertEquals(sparseVectorFieldMapper.fieldType().isStored(), useLegacyFormat() == false); IndexOptions expectedBaseIndexOptions = expectedIndexOptions == null ? null : expectedIndexOptions.indexOptions(); assertEquals(expectedBaseIndexOptions, sparseVectorFieldMapper.fieldType().getIndexOptions()); } else { - super.assertEmbeddingsField(mapperService, fieldType, embeddingsMapper, expectedIndexOptions); + super.assertEmbeddingsField(mapperService, embeddingsMapper, model, expectedIndexOptions); } } @@ -1150,14 +1178,14 @@ public void testSuccessfulParse() throws IOException { : indexOptions; MapperService mapperService = createSemanticMapperServiceWithIndexVersion(mapping, indexVersion); - assertSemanticField(mapperService, fieldName1, false, null, expectedIndexOptions); + assertSemanticField(mapperService, fieldName1, false, model1, null, expectedIndexOptions); assertInferenceEndpoints( mapperService, fieldName1, model1.getInferenceEntityId(), setSearchInferenceId ? searchInferenceId : model1.getInferenceEntityId() ); - assertSemanticField(mapperService, fieldName2, false, null, expectedIndexOptions); + assertSemanticField(mapperService, fieldName2, false, model2, null, expectedIndexOptions); assertInferenceEndpoints( mapperService, fieldName2, @@ -1436,11 +1464,11 @@ public void testSettingAndUpdatingChunkingSettings() throws IOException { MapperService mapperService = createSemanticMapperService( semanticMapping(fieldName, model.getInferenceEntityId(), null, null, chunkingSettings, indexOptions) ); - assertSemanticField(mapperService, fieldName, false, chunkingSettings, indexOptions); + assertSemanticField(mapperService, fieldName, false, model, chunkingSettings, indexOptions); ChunkingSettings newChunkingSettings = generateRandomChunkingSettingsOtherThan(chunkingSettings); merge(mapperService, semanticMapping(fieldName, model.getInferenceEntityId(), null, null, newChunkingSettings, indexOptions)); - assertSemanticField(mapperService, fieldName, false, newChunkingSettings, indexOptions); + assertSemanticField(mapperService, fieldName, false, model, newChunkingSettings, indexOptions); } public void testModelSettingsRequiredWithChunks() throws IOException { @@ -1497,10 +1525,10 @@ public void testPre811IndexSemanticTextDenseVectorRaisesError() throws IOExcepti IndexVersions.V_8_0_0, IndexVersionUtils.getPreviousVersion(IndexVersions.NEW_SPARSE_VECTOR) ); - assertSemanticField(mapperService, fieldName, false, null, null); + assertSemanticField(mapperService, fieldName, false, model, null, null); merge(mapperService, semanticMapping(fieldName, model.getInferenceEntityId(), new MinimalServiceSettings(model))); - assertSemanticField(mapperService, fieldName, true, null, null); + assertSemanticField(mapperService, fieldName, true, model, null, null); DocumentMapper documentMapper = mapperService.documentMapper(); DocumentParsingException e = assertThrows( @@ -1529,7 +1557,7 @@ public void testPre811IndexSemanticTextSparseVectorRaisesError() throws IOExcept IndexVersions.V_8_0_0, IndexVersionUtils.getPreviousVersion(IndexVersions.NEW_SPARSE_VECTOR) ); - assertSemanticField(mapperService, fieldName, false, null, null); + assertSemanticField(mapperService, fieldName, false, model, null, null); merge( mapperService, @@ -1539,7 +1567,7 @@ public void testPre811IndexSemanticTextSparseVectorRaisesError() throws IOExcept new MinimalServiceSettings(null, TaskType.SPARSE_EMBEDDING, null, null, null) ) ); - assertSemanticField(mapperService, fieldName, true, null, null); + assertSemanticField(mapperService, fieldName, true, model, null, null); DocumentMapper documentMapper = mapperService.documentMapper(); DocumentParsingException e = assertThrows( @@ -1706,6 +1734,7 @@ public void testDefaultIndexOptions() throws IOException { mapperService, "field", true, + model, null, getExpectedDefaultIndexOptions(taskType, elementType, dimensions, indexVersion, experimentalFeatures) ); @@ -1745,6 +1774,7 @@ yield defaultDenseVectorSemanticIndexOptions( } public void testSpecifiedDenseVectorIndexOptions() throws IOException { + Model denseModel = denseTextEmbeddingModel(100, SimilarityMeasure.COSINE, DenseVectorFieldMapper.ElementType.FLOAT); // Specifying index options will override default index option settings var mapperService = createSemanticMapperService(fieldMapping(b -> { @@ -1768,6 +1798,7 @@ public void testSpecifiedDenseVectorIndexOptions() throws IOException { mapperService, "field", true, + denseModel, null, new SemanticIndexOptions( SemanticIndexOptions.SupportedIndexOptions.DENSE_VECTOR, @@ -1795,6 +1826,7 @@ public void testSpecifiedDenseVectorIndexOptions() throws IOException { mapperService, "field", true, + denseModel, null, new SemanticIndexOptions( SemanticIndexOptions.SupportedIndexOptions.DENSE_VECTOR, @@ -1854,6 +1886,8 @@ public void testSpecifiedDenseVectorIndexOptions() throws IOException { } public void testSetElementTypeInDenseVectorIndexOptions() throws IOException { + Model denseModel = denseTextEmbeddingModel(100, SimilarityMeasure.COSINE, DenseVectorFieldMapper.ElementType.FLOAT); + // Specifying the element type prevents defaulting to bfloat16 IndexVersion indexVersion = SemanticInferenceMetadataFieldsMapperTests.getRandomCompatibleIndexVersion(useLegacyFormat); var mapperService = createSemanticMapperServiceWithIndexVersion(fieldMapping(b -> { @@ -1887,6 +1921,7 @@ public void testSetElementTypeInDenseVectorIndexOptions() throws IOException { mapperService, "field", true, + denseModel, null, new SemanticIndexOptions( SemanticIndexOptions.SupportedIndexOptions.DENSE_VECTOR, @@ -1918,6 +1953,7 @@ public void testSetElementTypeInDenseVectorIndexOptions() throws IOException { mapperService, "field", true, + denseModel, null, new SemanticIndexOptions( SemanticIndexOptions.SupportedIndexOptions.DENSE_VECTOR, @@ -1968,6 +2004,7 @@ public void testInvalidElementTypeOverride() { public void testSpecificSparseVectorIndexOptions() throws IOException { for (int i = 0; i < 10; i++) { SparseVectorFieldMapper.SparseVectorIndexOptions testIndexOptions = randomSparseVectorIndexOptions(false); + Model sparseModel = TestModel.createRandomInstance(TaskType.SPARSE_EMBEDDING); var mapperService = createSemanticMapperService(fieldMapping(b -> { b.field("type", SemanticTextFieldMapper.CONTENT_TYPE); b.field(INFERENCE_ID_FIELD, "test_inference_id"); @@ -1984,6 +2021,7 @@ public void testSpecificSparseVectorIndexOptions() throws IOException { mapperService, "field", true, + sparseModel, null, new SemanticIndexOptions(SemanticIndexOptions.SupportedIndexOptions.SPARSE_VECTOR, testIndexOptions) ); From f06c4d609ec14a6acba5400ea9d29114e6d934a1 Mon Sep 17 00:00:00 2001 From: Mike Pellegrini Date: Thu, 23 Jul 2026 08:27:03 -0400 Subject: [PATCH 08/24] Fix & improve coverage of testDefaults --- .../mapper/SemanticTextFieldMapperTests.java | 144 +++++++++++------- 1 file changed, 93 insertions(+), 51 deletions(-) diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java index a13d5539a7cae..b989c0c909165 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java @@ -86,6 +86,7 @@ import org.elasticsearch.xpack.inference.highlight.SemanticTextHighlighter; import org.elasticsearch.xpack.inference.model.TestModel; import org.elasticsearch.xpack.inference.services.elastic.ElasticInferenceService; +import org.elasticsearch.xpack.inference.services.elasticsearch.ElasticsearchInternalService; import java.io.IOException; import java.util.ArrayList; @@ -133,6 +134,37 @@ public class SemanticTextFieldMapperTests extends AbstractSemanticMapperTestCase SemanticTextFieldMapper, SemanticTextFieldMapper.SemanticTextFieldType> { + private static final TestModel JINA_V5_TEXT_MODEL = new TestModel( + DEFAULT_EIS_JINA_V5_INFERENCE_ID, + TaskType.TEXT_EMBEDDING, + ElasticInferenceService.NAME, + new TestModel.TestServiceSettings("jina", 1024, SimilarityMeasure.COSINE, DenseVectorFieldMapper.ElementType.FLOAT), + new TestModel.TestTaskSettings((Integer) null), + new TestModel.TestSecretSettings("foo") + ); + private static final TestModel DEFAULT_EIS_ELSER_MODEL = new TestModel( + DEFAULT_EIS_ELSER_INFERENCE_ID, + TaskType.SPARSE_EMBEDDING, + ElasticInferenceService.NAME, + new TestModel.TestServiceSettings("elser", null, null, null), + new TestModel.TestTaskSettings((Integer) null), + new TestModel.TestSecretSettings("foo") + ); + private static final TestModel DEFAULT_FALLBACK_ELSER_MODEL = new TestModel( + DEFAULT_FALLBACK_ELSER_INFERENCE_ID, + TaskType.SPARSE_EMBEDDING, + ElasticsearchInternalService.NAME, + new TestModel.TestServiceSettings("elser", null, null, null), + new TestModel.TestTaskSettings((Integer) null), + new TestModel.TestSecretSettings("foo") + ); + + private static final List DEFAULT_MODELS = List.of( + JINA_V5_TEXT_MODEL, + DEFAULT_EIS_ELSER_MODEL, + DEFAULT_FALLBACK_ELSER_MODEL + ); + private final boolean useLegacyFormat; private final IndexVersion testIndexVersion; @@ -148,7 +180,21 @@ public SemanticTextFieldMapperTests(boolean useLegacyFormat, License.OperationMo @Override protected void registerDefaultEndpoints() { - registerDefaultEisEndpoint(); + for (TestModel model : DEFAULT_MODELS) { + globalModelRegistry.putDefaultIdIfAbsent( + new InferenceService.DefaultConfigId( + model.getInferenceEntityId(), + new MinimalServiceSettings(model), + mock(InferenceService.class) + ) + ); + } + } + + private void removeDefaultEndpoints(Set inferenceIds) { + PlainActionFuture removalFuture = new PlainActionFuture<>(); + globalModelRegistry.removeDefaultConfigs(inferenceIds, removalFuture); + safeGet(removalFuture); } @ParametersFactory @@ -168,21 +214,6 @@ protected String minimalIsInvalidRoutingPathErrorMessage(Mapper mapper) { return super.minimalIsInvalidRoutingPathErrorMessage(mapper); } - private void registerDefaultEisEndpoint() { - globalModelRegistry.putDefaultIdIfAbsent( - new InferenceService.DefaultConfigId( - DEFAULT_EIS_JINA_V5_INFERENCE_ID, - MinimalServiceSettings.textEmbedding( - ElasticInferenceService.NAME, - 1024, - SimilarityMeasure.COSINE, - DenseVectorFieldMapper.ElementType.FLOAT - ), - mock(InferenceService.class) - ) - ); - } - @Override protected boolean useLegacyFormat() { return useLegacyFormat; @@ -312,7 +343,7 @@ public void testDefaults() throws Exception { MapperService mapperService = createSemanticMapperService(fieldMapping, IndexVersions.SEMANTIC_TEXT_DEFAULTS_TO_JINA_V5); DocumentMapper mapper = mapperService.documentMapper(); assertEquals(Strings.toString(expectedMapping), mapper.mappingSource().toString()); - assertSemanticField(mapperService, fieldName, false, null, null, null); + assertSemanticField(mapperService, fieldName, false, JINA_V5_TEXT_MODEL, null, null); assertInferenceEndpoints(mapperService, fieldName, DEFAULT_EIS_JINA_V5_INFERENCE_ID, DEFAULT_EIS_JINA_V5_INFERENCE_ID); ParsedDocument doc1 = mapper.parse(source(this::writeField)); @@ -322,32 +353,66 @@ public void testDefaults() throws Exception { assertTrue(fields.isEmpty()); for (int i = 0; i < 20; i++) { + registerDefaultEndpoints(); + boolean jinaAvailable = randomBoolean(); boolean eisElserAvailable = randomBoolean(); boolean postJinaV5 = randomBoolean(); - setDefaultEisEndpoints(jinaAvailable, eisElserAvailable); - - IndexVersion indexVersion = postJinaV5 - ? IndexVersionUtils.randomVersionBetween( + final IndexVersion indexVersion; + if (postJinaV5) { + indexVersion = IndexVersionUtils.randomVersionBetween( IndexVersions.SEMANTIC_TEXT_DEFAULTS_TO_JINA_V5, useLegacyFormat ? IndexVersionUtils.getPreviousVersion(IndexVersions.SEMANTIC_TEXT_LEGACY_FORMAT_FORBIDDEN) : IndexVersion.current() - ) - : IndexVersionUtils.randomPreviousCompatibleWriteVersion(IndexVersions.SEMANTIC_TEXT_DEFAULTS_TO_JINA_V5); + ); + } else if (useLegacyFormat) { + // Legacy format: any write-compatible version below JinaV5 + indexVersion = IndexVersionUtils.randomVersionBetween( + IndexVersions.MINIMUM_COMPATIBLE, + IndexVersionUtils.getPreviousVersion(IndexVersions.SEMANTIC_TEXT_DEFAULTS_TO_JINA_V5) + ); + } else if (randomBoolean()) { + // New format, 9.x, pre-JinaV5 + indexVersion = IndexVersionUtils.randomVersionBetween( + IndexVersions.INFERENCE_METADATA_FIELDS, + IndexVersionUtils.getPreviousVersion(IndexVersions.SEMANTIC_TEXT_DEFAULTS_TO_JINA_V5) + ); + } else { + // New format, 8.x backport window + indexVersion = IndexVersionUtils.randomVersionBetween( + IndexVersions.INFERENCE_METADATA_FIELDS_BACKPORT, + IndexVersionUtils.getPreviousVersion(IndexVersions.UPGRADE_TO_LUCENE_10_0_0) + ); + } + + Set unavailableEndpoints = new HashSet<>(); + if (jinaAvailable == false) { + unavailableEndpoints.add(DEFAULT_EIS_JINA_V5_INFERENCE_ID); + } + if (eisElserAvailable == false) { + unavailableEndpoints.add(DEFAULT_EIS_ELSER_INFERENCE_ID); + } + removeDefaultEndpoints(unavailableEndpoints); - String expectedId; + TestModel expectedModel; if (jinaAvailable && postJinaV5) { - expectedId = DEFAULT_EIS_JINA_V5_INFERENCE_ID; + expectedModel = JINA_V5_TEXT_MODEL; } else if (eisElserAvailable) { - expectedId = DEFAULT_EIS_ELSER_INFERENCE_ID; + expectedModel = DEFAULT_EIS_ELSER_MODEL; } else { - expectedId = DEFAULT_FALLBACK_ELSER_INFERENCE_ID; + expectedModel = DEFAULT_FALLBACK_ELSER_MODEL; } MapperService iterMapperService = createSemanticMapperServiceWithIndexVersion(fieldMapping, indexVersion); - assertInferenceEndpoints(iterMapperService, fieldName, expectedId, expectedId); + assertSemanticField(iterMapperService, fieldName, false, expectedModel, null, null); + assertInferenceEndpoints( + iterMapperService, + fieldName, + expectedModel.getInferenceEntityId(), + expectedModel.getInferenceEntityId() + ); } } @@ -539,29 +604,6 @@ public void testEmptyDefaultInferenceIdSettingThrows() throws Exception { } } - /** - * Resets the model registry to exactly the given endpoint availability for each iteration of the - * randomized sweep in {@link #testDefaults()}. - */ - private void setDefaultEisEndpoints(boolean jinaEnabled, boolean eisElserEnabled) { - PlainActionFuture removalFuture = new PlainActionFuture<>(); - globalModelRegistry.removeDefaultConfigs(Set.of(DEFAULT_EIS_JINA_V5_INFERENCE_ID, DEFAULT_EIS_ELSER_INFERENCE_ID), removalFuture); - removalFuture.actionGet(TEST_REQUEST_TIMEOUT); - - if (jinaEnabled) { - registerDefaultEisEndpoint(); - } - if (eisElserEnabled) { - globalModelRegistry.putDefaultIdIfAbsent( - new InferenceService.DefaultConfigId( - DEFAULT_EIS_ELSER_INFERENCE_ID, - MinimalServiceSettings.sparseEmbedding(ElasticInferenceService.NAME), - mock(InferenceService.class) - ) - ); - } - } - @Override public void testFieldHasValue() { MappedFieldType fieldType = getMappedFieldType(); From 92434cffc9e2951cacf5783069e94b6e1d7b8b83 Mon Sep 17 00:00:00 2001 From: Mike Pellegrini Date: Thu, 23 Jul 2026 09:17:08 -0400 Subject: [PATCH 09/24] Fix testDefaultIndexOptions --- .../mapper/SemanticFieldMapperTests.java | 1 + .../mapper/SemanticTextFieldMapperTests.java | 24 ++++--------------- 2 files changed, 5 insertions(+), 20 deletions(-) diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticFieldMapperTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticFieldMapperTests.java index 0406419cf9cd6..60a1646f28073 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticFieldMapperTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticFieldMapperTests.java @@ -364,6 +364,7 @@ protected Set supportedTaskTypes() { @Override protected IndexVersion getRandomCompatibleIndexVersion() { + // TODO: Bias towards IndexVersion.current() return IndexVersionUtils.randomVersionOnOrAfter(IndexVersions.SEMANTIC_FIELD_TYPE); } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java index b989c0c909165..a9d6539a50ea6 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java @@ -1130,6 +1130,7 @@ protected Set supportedTaskTypes() { @Override protected IndexVersion getRandomCompatibleIndexVersion() { + // TODO: Bias towards indexVersion.current() return SemanticInferenceMetadataFieldsMapperTests.getRandomCompatibleIndexVersion(useLegacyFormat()); } @@ -1757,32 +1758,15 @@ public void testDefaultIndexOptions() throws IOException { ? IndexVersion.current() : SemanticInferenceMetadataFieldsMapperTests.getRandomCompatibleIndexVersion(useLegacyFormat); - final TaskType taskType = model.getTaskType(); - final DenseVectorFieldMapper.ElementType elementType = model.getServiceSettings().elementType(); - final Integer dimensions = model.getServiceSettings().dimensions(); - final SimilarityMeasure similarity = model.getServiceSettings().similarity(); - MapperService mapperService = createSemanticMapperServiceWithIndexVersion( - semanticMapping( - "field", - "another_inference_id", - new MinimalServiceSettings(null, taskType, dimensions, similarity, elementType) - ), + semanticMapping("field", "another_inference_id", new MinimalServiceSettings(model)), indexVersion ); - - boolean experimentalFeatures = DENSE_VECTOR_EXPERIMENTAL_FEATURES_SETTING.get(mapperService.getIndexSettings().getSettings()); - assertSemanticField( - mapperService, - "field", - true, - model, - null, - getExpectedDefaultIndexOptions(taskType, elementType, dimensions, indexVersion, experimentalFeatures) - ); + assertSemanticField(mapperService, "field", true, model, null, null); } } + // TODO: Remove private SemanticIndexOptions getExpectedDefaultIndexOptions( TaskType taskType, DenseVectorFieldMapper.ElementType elementType, From 2924e55d287e5d4b7394d925b88ffdef41c5bc8e Mon Sep 17 00:00:00 2001 From: Mike Pellegrini Date: Thu, 23 Jul 2026 09:25:21 -0400 Subject: [PATCH 10/24] Fix testSetInferenceEndpoints --- .../xpack/inference/mapper/SemanticTextFieldMapperTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java index a9d6539a50ea6..199fa1ffd07cc 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java @@ -634,7 +634,7 @@ public void testSetInferenceEndpoints() throws IOException { ); final XContentBuilder expectedMapping = semanticMapping("field", DEFAULT_EIS_JINA_V5_INFERENCE_ID, searchInferenceId); final MapperService mapperService = createSemanticMapperService(fieldMapping, IndexVersions.SEMANTIC_TEXT_DEFAULTS_TO_JINA_V5); - assertSemanticField(mapperService, fieldName, false, null, null, null); + assertSemanticField(mapperService, fieldName, false, JINA_V5_TEXT_MODEL, null, null); assertInferenceEndpoints(mapperService, fieldName, DEFAULT_EIS_JINA_V5_INFERENCE_ID, searchInferenceId); assertSerialization.accept(expectedMapping, mapperService); } From 59a83296829b2d1dd9a9434b425ce1861d30ba0a Mon Sep 17 00:00:00 2001 From: Mike Pellegrini Date: Thu, 23 Jul 2026 14:02:57 -0400 Subject: [PATCH 11/24] Fix flaky/broken tests in SemanticTextFieldMapperTests --- .../AbstractSemanticMapperTestCase.java | 3 +- .../mapper/SemanticTextFieldMapperTests.java | 62 ++++++++++--------- 2 files changed, 34 insertions(+), 31 deletions(-) diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/AbstractSemanticMapperTestCase.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/AbstractSemanticMapperTestCase.java index 97d4fa00cf713..cf13ddad2c668 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/AbstractSemanticMapperTestCase.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/AbstractSemanticMapperTestCase.java @@ -435,14 +435,13 @@ protected MapperService createSemanticMapperService( IndexVersion minIndexVersion, IndexVersion maxIndexVersion ) throws IOException { - validateIndexVersion(minIndexVersion, useLegacyFormat()); - validateIndexVersion(maxIndexVersion, useLegacyFormat()); IndexVersion indexVersion = IndexVersionUtils.randomVersionBetween(minIndexVersion, maxIndexVersion); return createSemanticMapperServiceWithIndexVersion(mappings, indexVersion); } protected MapperService createSemanticMapperServiceWithIndexVersion(XContentBuilder mappings, IndexVersion indexVersion) throws IOException { + validateIndexVersion(indexVersion, useLegacyFormat()); var settings = Settings.builder() .put(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), indexVersion) .put(InferenceMetadataFieldsMapper.USE_LEGACY_SEMANTIC_TEXT_FORMAT.getKey(), useLegacyFormat()) diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java index 199fa1ffd07cc..83d491225eeb8 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java @@ -278,7 +278,7 @@ protected Object getSampleObjectForDocument() { // legacy value is the {text, inference} object; a minimal one with the resolved inference id and no inference results parses. final String expectedInferenceId = testIndexVersion.onOrAfter(IndexVersions.SEMANTIC_TEXT_DEFAULTS_TO_JINA_V5) ? DEFAULT_EIS_JINA_V5_INFERENCE_ID - : DEFAULT_FALLBACK_ELSER_INFERENCE_ID; + : DEFAULT_EIS_ELSER_INFERENCE_ID; return Map.of( SemanticTextField.TEXT_FIELD, randomAlphaOfLength(10), @@ -1568,7 +1568,7 @@ public void testPre811IndexSemanticTextDenseVectorRaisesError() throws IOExcepti IndexVersions.V_8_0_0, IndexVersionUtils.getPreviousVersion(IndexVersions.NEW_SPARSE_VECTOR) ); - assertSemanticField(mapperService, fieldName, false, model, null, null); + assertSemanticField(mapperService, fieldName, false, null, null, null); merge(mapperService, semanticMapping(fieldName, model.getInferenceEntityId(), new MinimalServiceSettings(model))); assertSemanticField(mapperService, fieldName, true, model, null, null); @@ -1600,16 +1600,9 @@ public void testPre811IndexSemanticTextSparseVectorRaisesError() throws IOExcept IndexVersions.V_8_0_0, IndexVersionUtils.getPreviousVersion(IndexVersions.NEW_SPARSE_VECTOR) ); - assertSemanticField(mapperService, fieldName, false, model, null, null); + assertSemanticField(mapperService, fieldName, false, null, null, null); - merge( - mapperService, - semanticMapping( - fieldName, - model.getInferenceEntityId(), - new MinimalServiceSettings(null, TaskType.SPARSE_EMBEDDING, null, null, null) - ) - ); + merge(mapperService, semanticMapping(fieldName, model.getInferenceEntityId(), new MinimalServiceSettings(model))); assertSemanticField(mapperService, fieldName, true, model, null, null); DocumentMapper documentMapper = mapperService.documentMapper(); @@ -1800,10 +1793,11 @@ yield defaultDenseVectorSemanticIndexOptions( } public void testSpecifiedDenseVectorIndexOptions() throws IOException { + IndexVersion indexVersion = SemanticInferenceMetadataFieldsMapperTests.getRandomCompatibleIndexVersion(useLegacyFormat()); Model denseModel = denseTextEmbeddingModel(100, SimilarityMeasure.COSINE, DenseVectorFieldMapper.ElementType.FLOAT); // Specifying index options will override default index option settings - var mapperService = createSemanticMapperService(fieldMapping(b -> { + var mapperService = createSemanticMapperServiceWithIndexVersion(fieldMapping(b -> { b.field("type", "semantic_text"); b.field("inference_id", "another_inference_id"); b.startObject("model_settings"); @@ -1814,12 +1808,13 @@ public void testSpecifiedDenseVectorIndexOptions() throws IOException { b.endObject(); b.startObject("index_options"); b.startObject("dense_vector"); + b.field("element_type", "float"); // Explicitly override element type to hold it constant b.field("type", "int4_hnsw"); b.field("m", 20); b.field("ef_construction", 90); b.endObject(); b.endObject(); - }), IndexVersions.INFERENCE_METADATA_FIELDS_BACKPORT); + }), indexVersion); assertSemanticField( mapperService, "field", @@ -1828,12 +1823,15 @@ public void testSpecifiedDenseVectorIndexOptions() throws IOException { null, new SemanticIndexOptions( SemanticIndexOptions.SupportedIndexOptions.DENSE_VECTOR, - new DenseVectorFieldMapper.Int4HnswIndexOptions(20, 90, false, null, -1) + new ExtendedDenseVectorIndexOptions( + new DenseVectorFieldMapper.Int4HnswIndexOptions(20, 90, false, null, -1), + DenseVectorFieldMapper.ElementType.FLOAT + ) ) ); // Specifying partial index options will in the remainder index options with defaults - mapperService = createSemanticMapperService(fieldMapping(b -> { + mapperService = createSemanticMapperServiceWithIndexVersion(fieldMapping(b -> { b.field("type", "semantic_text"); b.field("inference_id", "another_inference_id"); b.startObject("model_settings"); @@ -1844,10 +1842,11 @@ public void testSpecifiedDenseVectorIndexOptions() throws IOException { b.endObject(); b.startObject("index_options"); b.startObject("dense_vector"); + b.field("element_type", "float"); // Explicitly override element type to hold it constant b.field("type", "int4_hnsw"); b.endObject(); b.endObject(); - }), IndexVersions.INFERENCE_METADATA_FIELDS_BACKPORT); + }), indexVersion); assertSemanticField( mapperService, "field", @@ -1856,12 +1855,15 @@ public void testSpecifiedDenseVectorIndexOptions() throws IOException { null, new SemanticIndexOptions( SemanticIndexOptions.SupportedIndexOptions.DENSE_VECTOR, - new DenseVectorFieldMapper.Int4HnswIndexOptions(16, 100, false, null, -1) + new ExtendedDenseVectorIndexOptions( + new DenseVectorFieldMapper.Int4HnswIndexOptions(16, 100, false, null, -1), + DenseVectorFieldMapper.ElementType.FLOAT + ) ) ); // Incompatible index options will fail - Exception e = expectThrows(MapperParsingException.class, () -> createSemanticMapperService(fieldMapping(b -> { + Exception e = expectThrows(MapperParsingException.class, () -> createSemanticMapperServiceWithIndexVersion(fieldMapping(b -> { b.field("type", "semantic_text"); b.field("inference_id", "another_inference_id"); b.startObject("model_settings"); @@ -1872,10 +1874,10 @@ public void testSpecifiedDenseVectorIndexOptions() throws IOException { b.field("type", "int8_hnsw"); b.endObject(); b.endObject(); - }), IndexVersions.INFERENCE_METADATA_FIELDS_BACKPORT)); + }), indexVersion)); assertThat(e.getMessage(), containsString("Invalid task type")); - e = expectThrows(MapperParsingException.class, () -> createSemanticMapperService(fieldMapping(b -> { + e = expectThrows(MapperParsingException.class, () -> createSemanticMapperServiceWithIndexVersion(fieldMapping(b -> { b.field("type", "semantic_text"); b.field("inference_id", "another_inference_id"); b.startObject("model_settings"); @@ -1890,10 +1892,10 @@ public void testSpecifiedDenseVectorIndexOptions() throws IOException { b.field("ef_construction", 100); b.endObject(); b.endObject(); - }), IndexVersions.INFERENCE_METADATA_FIELDS_BACKPORT)); + }), indexVersion)); assertThat(e.getMessage(), containsString("unsupported parameters: [ef_construction : 100]")); - e = expectThrows(MapperParsingException.class, () -> createSemanticMapperService(fieldMapping(b -> { + e = expectThrows(MapperParsingException.class, () -> createSemanticMapperServiceWithIndexVersion(fieldMapping(b -> { b.field("type", "semantic_text"); b.field("inference_id", "another_inference_id"); b.startObject("model_settings"); @@ -1907,7 +1909,7 @@ public void testSpecifiedDenseVectorIndexOptions() throws IOException { b.field("type", "invalid"); b.endObject(); b.endObject(); - }), IndexVersions.INFERENCE_METADATA_FIELDS_BACKPORT)); + }), indexVersion)); assertThat(e.getMessage(), containsString("Unsupported index options type invalid")); } @@ -2029,9 +2031,10 @@ public void testInvalidElementTypeOverride() { public void testSpecificSparseVectorIndexOptions() throws IOException { for (int i = 0; i < 10; i++) { + IndexVersion indexVersion = SemanticInferenceMetadataFieldsMapperTests.getRandomCompatibleIndexVersion(useLegacyFormat()); SparseVectorFieldMapper.SparseVectorIndexOptions testIndexOptions = randomSparseVectorIndexOptions(false); Model sparseModel = TestModel.createRandomInstance(TaskType.SPARSE_EMBEDDING); - var mapperService = createSemanticMapperService(fieldMapping(b -> { + var mapperService = createSemanticMapperServiceWithIndexVersion(fieldMapping(b -> { b.field("type", SemanticTextFieldMapper.CONTENT_TYPE); b.field(INFERENCE_ID_FIELD, "test_inference_id"); addSparseVectorModelSettingsToBuilder(b); @@ -2041,7 +2044,7 @@ public void testSpecificSparseVectorIndexOptions() throws IOException { testIndexOptions.toXContent(b, null); } b.endObject(); - }), IndexVersions.INFERENCE_METADATA_FIELDS_BACKPORT); + }), indexVersion); assertSemanticField( mapperService, @@ -2055,7 +2058,8 @@ public void testSpecificSparseVectorIndexOptions() throws IOException { } public void testSparseVectorIndexOptionsValidations() throws IOException { - Exception e = expectThrows(MapperParsingException.class, () -> createSemanticMapperService(fieldMapping(b -> { + IndexVersion indexVersion = SemanticInferenceMetadataFieldsMapperTests.getRandomCompatibleIndexVersion(useLegacyFormat()); + Exception e = expectThrows(MapperParsingException.class, () -> createSemanticMapperServiceWithIndexVersion(fieldMapping(b -> { b.field("type", SemanticTextFieldMapper.CONTENT_TYPE); b.field(INFERENCE_ID_FIELD, "test_inference_id"); b.startObject(INDEX_OPTIONS_FIELD); @@ -2072,10 +2076,10 @@ public void testSparseVectorIndexOptionsValidations() throws IOException { b.endObject(); } b.endObject(); - }), IndexVersions.INFERENCE_METADATA_FIELDS_BACKPORT)); + }), indexVersion)); assertThat(e.getMessage(), containsString("failed to parse field [pruning_config]")); - e = expectThrows(MapperParsingException.class, () -> createSemanticMapperService(fieldMapping(b -> { + e = expectThrows(MapperParsingException.class, () -> createSemanticMapperServiceWithIndexVersion(fieldMapping(b -> { b.field("type", SemanticTextFieldMapper.CONTENT_TYPE); b.field(INFERENCE_ID_FIELD, "test_inference_id"); b.startObject(INDEX_OPTIONS_FIELD); @@ -2092,7 +2096,7 @@ public void testSparseVectorIndexOptionsValidations() throws IOException { b.endObject(); } b.endObject(); - }), IndexVersions.INFERENCE_METADATA_FIELDS_BACKPORT)); + }), indexVersion)); var innerClause = e.getCause().getCause().getCause(); assertThat(innerClause.getMessage(), containsString("[tokens_freq_ratio_threshold] must be between [1] and [100], got 1000.0")); } From ab06cbaa60b29e91720464cd094be27d9847a64a Mon Sep 17 00:00:00 2001 From: Mike Pellegrini Date: Fri, 24 Jul 2026 09:25:41 -0400 Subject: [PATCH 12/24] Apply index version validation only in SemanticTextFieldMapperTests --- .../AbstractSemanticMapperTestCase.java | 24 ++------------ .../mapper/SemanticFieldMapperTests.java | 32 +++++++++++-------- .../mapper/SemanticTextFieldMapperTests.java | 31 ++++++++++++++++++ 3 files changed, 51 insertions(+), 36 deletions(-) diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/AbstractSemanticMapperTestCase.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/AbstractSemanticMapperTestCase.java index cf13ddad2c668..1f9d846484d19 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/AbstractSemanticMapperTestCase.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/AbstractSemanticMapperTestCase.java @@ -20,7 +20,6 @@ import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; -import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.InferenceMetadataFieldsMapper; import org.elasticsearch.index.mapper.LuceneDocument; @@ -424,10 +423,7 @@ protected MapperService createSemanticMapperService(XContentBuilder mappings) th } protected MapperService createSemanticMapperService(XContentBuilder mappings, IndexVersion minIndexVersion) throws IOException { - IndexVersion maxIndexVersion = useLegacyFormat() - ? IndexVersionUtils.getPreviousVersion(IndexVersions.SEMANTIC_TEXT_LEGACY_FORMAT_FORBIDDEN) - : IndexVersion.current(); - return createSemanticMapperService(mappings, minIndexVersion, maxIndexVersion); + return createSemanticMapperService(mappings, minIndexVersion, IndexVersion.current()); } protected MapperService createSemanticMapperService( @@ -441,11 +437,7 @@ protected MapperService createSemanticMapperService( protected MapperService createSemanticMapperServiceWithIndexVersion(XContentBuilder mappings, IndexVersion indexVersion) throws IOException { - validateIndexVersion(indexVersion, useLegacyFormat()); - var settings = Settings.builder() - .put(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), indexVersion) - .put(InferenceMetadataFieldsMapper.USE_LEGACY_SEMANTIC_TEXT_FORMAT.getKey(), useLegacyFormat()) - .build(); + var settings = Settings.builder().put(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), indexVersion).build(); return createMapperService(indexVersion, settings, mappings); } @@ -891,16 +883,4 @@ protected static String randomFieldName(int numLevel) { } return builder.toString(); } - - private static void validateIndexVersion(IndexVersion indexVersion, boolean useLegacyFormat) { - if (useLegacyFormat == false - && indexVersion.before(IndexVersions.INFERENCE_METADATA_FIELDS) - && indexVersion.between(IndexVersions.INFERENCE_METADATA_FIELDS_BACKPORT, IndexVersions.UPGRADE_TO_LUCENE_10_0_0) == false) { - throw new IllegalArgumentException("Index version " + indexVersion + " does not support new semantic text format"); - } - - if (useLegacyFormat && indexVersion.onOrAfter(IndexVersions.SEMANTIC_TEXT_LEGACY_FORMAT_FORBIDDEN)) { - throw new IllegalArgumentException("Index version " + indexVersion + " does not support legacy semantic text format"); - } - } } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticFieldMapperTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticFieldMapperTests.java index 60a1646f28073..73002d654bc9f 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticFieldMapperTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticFieldMapperTests.java @@ -294,25 +294,29 @@ private static String imageObject(String dataUri) throws IOException { return Strings.toString(b); } - public void testSemanticFieldNotSupportedOnOldIndices() throws IOException { - IndexVersion oldVersion = IndexVersionUtils.randomPreviousCompatibleVersion(IndexVersions.SEMANTIC_FIELD_TYPE); + public void testSemanticFieldNotSupportedOnOldIndices() { + for (int i = 0; i < 20; i++) { + IndexVersion oldVersion = IndexVersionUtils.randomPreviousCompatibleVersion(IndexVersions.SEMANTIC_FIELD_TYPE); - var ex = expectThrows( - MapperParsingException.class, - () -> createSemanticMapperServiceWithIndexVersion(semanticMapping("my_field", "test_model"), oldVersion) - ); - assertThat(ex.getMessage(), containsString("[" + SemanticFieldMapper.CONTENT_TYPE + "]")); - assertThat(ex.getMessage(), containsString("is not supported on indices created before version")); - assertThat(ex.getMessage(), containsString(IndexVersions.SEMANTIC_FIELD_TYPE.toString())); + var ex = expectThrows( + MapperParsingException.class, + () -> createSemanticMapperServiceWithIndexVersion(semanticMapping("my_field", "test_model"), oldVersion) + ); + assertThat(ex.getMessage(), containsString("[" + SemanticFieldMapper.CONTENT_TYPE + "]")); + assertThat(ex.getMessage(), containsString("is not supported on indices created before version")); + assertThat(ex.getMessage(), containsString(IndexVersions.SEMANTIC_FIELD_TYPE.toString())); + } } public void testSemanticFieldSupportedOnNewIndices() throws IOException { - IndexVersion newVersion = IndexVersionUtils.randomVersionOnOrAfter(IndexVersions.SEMANTIC_FIELD_TYPE); + for (int i = 0; i < 20; i++) { + IndexVersion newVersion = IndexVersionUtils.randomVersionOnOrAfter(IndexVersions.SEMANTIC_FIELD_TYPE); - // Should not throw; model_settings provided to avoid consulting the model registry - var mapperService = createSemanticMapperServiceWithIndexVersion(mapping(this::minimalMappingWithModelSettings), newVersion); - assertNotNull(mapperService); - assertSemanticFieldMapper(mapperService, "my_field"); + // Should not throw; model_settings provided to avoid consulting the model registry + var mapperService = createSemanticMapperServiceWithIndexVersion(mapping(this::minimalMappingWithModelSettings), newVersion); + assertNotNull(mapperService); + assertSemanticFieldMapper(mapperService, "my_field"); + } } public void testSemanticFieldMappingUpdateNotSupportedOnOldIndices() throws IOException { diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java index 83d491225eeb8..b35fb8cc4c81f 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java @@ -2141,6 +2141,25 @@ public void testSupportedIndexVersions() throws IOException { } } + @Override + protected MapperService createSemanticMapperService(XContentBuilder mappings, IndexVersion minIndexVersion) throws IOException { + IndexVersion maxIndexVersion = useLegacyFormat() + ? IndexVersionUtils.getPreviousVersion(IndexVersions.SEMANTIC_TEXT_LEGACY_FORMAT_FORBIDDEN) + : IndexVersion.current(); + return createSemanticMapperService(mappings, minIndexVersion, maxIndexVersion); + } + + @Override + protected MapperService createSemanticMapperServiceWithIndexVersion(XContentBuilder mappings, IndexVersion indexVersion) + throws IOException { + validateIndexVersion(indexVersion, useLegacyFormat()); + var settings = Settings.builder() + .put(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), indexVersion) + .put(InferenceMetadataFieldsMapper.USE_LEGACY_SEMANTIC_TEXT_FORMAT.getKey(), useLegacyFormat()) + .build(); + return createMapperService(indexVersion, settings, mappings); + } + public static SemanticIndexOptions randomSemanticIndexOptions() { TaskType taskType = randomFrom(TaskType.SPARSE_EMBEDDING, TaskType.TEXT_EMBEDDING); return randomSemanticIndexOptions(taskType); @@ -2248,4 +2267,16 @@ private static Throwable rootCause(Throwable t) { } return cause; } + + private static void validateIndexVersion(IndexVersion indexVersion, boolean useLegacyFormat) { + if (useLegacyFormat == false + && indexVersion.before(IndexVersions.INFERENCE_METADATA_FIELDS) + && indexVersion.between(IndexVersions.INFERENCE_METADATA_FIELDS_BACKPORT, IndexVersions.UPGRADE_TO_LUCENE_10_0_0) == false) { + throw new AssertionError("Index version " + indexVersion + " does not support new semantic text format"); + } + + if (useLegacyFormat && indexVersion.onOrAfter(IndexVersions.SEMANTIC_TEXT_LEGACY_FORMAT_FORBIDDEN)) { + throw new AssertionError("Index version " + indexVersion + " does not support legacy semantic text format"); + } + } } From bbb3ab88d27e6d9fb63919f0e326b5954cb24e55 Mon Sep 17 00:00:00 2001 From: Mike Pellegrini Date: Fri, 24 Jul 2026 10:16:22 -0400 Subject: [PATCH 13/24] Use a single registered test model in SemanticFieldMapperTests --- .../mapper/SemanticFieldMapperTests.java | 66 ++++++++----------- 1 file changed, 27 insertions(+), 39 deletions(-) diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticFieldMapperTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticFieldMapperTests.java index 73002d654bc9f..83a13979ec002 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticFieldMapperTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticFieldMapperTests.java @@ -38,6 +38,7 @@ import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xcontent.json.JsonXContent; import org.elasticsearch.xpack.inference.highlight.SemanticTextHighlighter; +import org.elasticsearch.xpack.inference.model.TestModel; import org.elasticsearch.xpack.inference.services.elastic.ElasticInferenceService; import java.io.IOException; @@ -58,49 +59,35 @@ import static org.mockito.Mockito.mock; public class SemanticFieldMapperTests extends AbstractSemanticMapperTestCase { - private static final String INFERENCE_ID = "inference-id"; + private static final TestModel TEST_MODEL = new TestModel( + "test_inference_id", + EMBEDDING, + ElasticInferenceService.NAME, + new TestModel.TestServiceSettings("test_model", 1024, SimilarityMeasure.COSINE, DenseVectorFieldMapper.ElementType.FLOAT), + new TestModel.TestTaskSettings((Integer) null), + new TestModel.TestSecretSettings("secret") + ); public SemanticFieldMapperTests(License.OperationMode operationMode) { super(operationMode); } - @Override - protected void registerDefaultEndpoints() { - registerMultiModalEisEndpoint(); - } - @ParametersFactory public static Iterable parameters() throws Exception { return List.of(new Object[] { License.OperationMode.BASIC }, new Object[] { License.OperationMode.ENTERPRISE }); } - private void registerMultiModalEisEndpoint() { + @Override + protected void registerDefaultEndpoints() { globalModelRegistry.putDefaultIdIfAbsent( new InferenceService.DefaultConfigId( - INFERENCE_ID, - new MinimalServiceSettings( - ElasticInferenceService.NAME, - EMBEDDING, - 1024, - SimilarityMeasure.COSINE, - DenseVectorFieldMapper.ElementType.FLOAT - ), + TEST_MODEL.getInferenceEntityId(), + new MinimalServiceSettings(TEST_MODEL), mock(InferenceService.class) ) ); } - private void minimalMappingWithModelSettings(XContentBuilder b) throws IOException { - MinimalServiceSettings modelSettings = new MinimalServiceSettings( - "test_service", - TaskType.EMBEDDING, - 128, - SimilarityMeasure.COSINE, - DenseVectorFieldMapper.ElementType.FLOAT - ); - addSemanticMapping(b, "my_field", "test_model", null, modelSettings, null, null); - } - /** * In synthetic-source (and columnar) indices, a {@code semantic} field rebuilds {@code _source} from its internal binary doc * values store. Text round-trips as a string; an image (base64 data URI) round-trips as a {@code {type, format, value}} object, @@ -108,7 +95,7 @@ private void minimalMappingWithModelSettings(XContentBuilder b) throws IOExcepti */ public void testOriginalValueRoundTripFromDocValues() throws IOException { MapperService mapperService = createSemanticMapperServiceWithSourceMode( - mapping(this::minimalMappingWithModelSettings), + semanticMapping("my_field", TEST_MODEL.getInferenceEntityId()), IndexVersion.current(), SourceFieldMapper.Mode.SYNTHETIC ); @@ -144,7 +131,7 @@ public void testSemanticFieldAcceptedInColumnar() throws IOException { for (IndexMode indexMode : List.of(IndexMode.COLUMNAR, IndexMode.LOGSDB_COLUMNAR)) { // Mapping creation succeeding is the assertion: the columnar "every field reconstructable from doc values" check passes. MapperService mapperService = createSemanticMapperServiceWithIndexMode( - mapping(this::minimalMappingWithModelSettings), + semanticMapping("my_field", TEST_MODEL.getInferenceEntityId()), IndexVersion.current(), indexMode ); @@ -168,7 +155,7 @@ public void testSemanticFieldAcceptedInColumnar() throws IOException { */ public void testOriginalValueFetchedFromDocValues() throws IOException { MapperService mapperService = createSemanticMapperServiceWithSourceMode( - mapping(this::minimalMappingWithModelSettings), + semanticMapping("my_field", TEST_MODEL.getInferenceEntityId()), IndexVersion.current(), SourceFieldMapper.Mode.SYNTHETIC ); @@ -237,7 +224,7 @@ private Object expectedFetchedValue(Object input) throws IOException { */ public void testBooleanAndNumericValuesRoundTripAsStrings() throws IOException { MapperService mapperService = createSemanticMapperServiceWithSourceMode( - mapping(this::minimalMappingWithModelSettings), + semanticMapping("my_field", TEST_MODEL.getInferenceEntityId()), IndexVersion.current(), SourceFieldMapper.Mode.SYNTHETIC ); @@ -267,7 +254,7 @@ public void testHighlighterAvoidsSourceWhenValuesInDocValues() throws IOExceptio SearchExecutionContext syntheticContext = createSearchExecutionContext( createSemanticMapperServiceWithSourceMode( - mapping(this::minimalMappingWithModelSettings), + semanticMapping("my_field", TEST_MODEL.getInferenceEntityId()), version, SourceFieldMapper.Mode.SYNTHETIC ) @@ -276,7 +263,7 @@ public void testHighlighterAvoidsSourceWhenValuesInDocValues() throws IOExceptio SearchExecutionContext storedContext = createSearchExecutionContext( createSemanticMapperServiceWithSourceMode( - mapping(this::minimalMappingWithModelSettings), + semanticMapping("my_field", TEST_MODEL.getInferenceEntityId()), version, SourceFieldMapper.Mode.STORED ) @@ -313,7 +300,10 @@ public void testSemanticFieldSupportedOnNewIndices() throws IOException { IndexVersion newVersion = IndexVersionUtils.randomVersionOnOrAfter(IndexVersions.SEMANTIC_FIELD_TYPE); // Should not throw; model_settings provided to avoid consulting the model registry - var mapperService = createSemanticMapperServiceWithIndexVersion(mapping(this::minimalMappingWithModelSettings), newVersion); + var mapperService = createSemanticMapperServiceWithIndexVersion( + semanticMapping("my_field", TEST_MODEL.getInferenceEntityId()), + newVersion + ); assertNotNull(mapperService); assertSemanticFieldMapper(mapperService, "my_field"); } @@ -336,7 +326,7 @@ public void testSemanticFieldMappingUpdateSupportedOnNewIndices() throws IOExcep var mapperService = createSemanticMapperServiceWithIndexVersion(mapping(b -> {}), newVersion); assertNotNull(mapperService); // Should not throw; model_settings provided to avoid consulting the model registry - merge(mapperService, mapping(this::minimalMappingWithModelSettings)); + merge(mapperService, semanticMapping("my_field", TEST_MODEL.getInferenceEntityId())); assertSemanticFieldMapper(mapperService, "my_field"); } @@ -374,8 +364,8 @@ protected IndexVersion getRandomCompatibleIndexVersion() { @Override protected void minimalMapping(XContentBuilder b) throws IOException { - b.field("type", "semantic"); - b.field("inference_id", INFERENCE_ID); + b.field("type", contentType()); + b.field("inference_id", TEST_MODEL.getInferenceEntityId()); } @Override @@ -427,9 +417,7 @@ public void testInvalidInferenceEndpoints() { Exception e = expectThrows( MapperParsingException.class, () -> createMapperService( - fieldMapping( - b -> b.field("type", "semantic").field(INFERENCE_ID_FIELD, INFERENCE_ID).field(SEARCH_INFERENCE_ID_FIELD, "") - ) + fieldMapping(b -> b.field("type", "semantic").field(INFERENCE_ID_FIELD, "foo").field(SEARCH_INFERENCE_ID_FIELD, "")) ) ); assertThat(e.getMessage(), containsString("[search_inference_id] on mapper [field] of type [semantic] must not be empty")); From 1432a8d51b16f64b00366f6e2544032f07b002ae Mon Sep 17 00:00:00 2001 From: Mike Pellegrini Date: Fri, 24 Jul 2026 10:23:45 -0400 Subject: [PATCH 14/24] Remove assertSemanticFieldMapper --- .../inference/mapper/SemanticFieldMapperTests.java | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticFieldMapperTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticFieldMapperTests.java index 83a13979ec002..4c9724ef1a1cb 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticFieldMapperTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticFieldMapperTests.java @@ -15,7 +15,6 @@ import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperParsingException; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.ParsedDocument; @@ -305,7 +304,7 @@ public void testSemanticFieldSupportedOnNewIndices() throws IOException { newVersion ); assertNotNull(mapperService); - assertSemanticFieldMapper(mapperService, "my_field"); + assertSemanticField(mapperService, "my_field", false, TEST_MODEL, null, null); } } @@ -328,12 +327,7 @@ public void testSemanticFieldMappingUpdateSupportedOnNewIndices() throws IOExcep // Should not throw; model_settings provided to avoid consulting the model registry merge(mapperService, semanticMapping("my_field", TEST_MODEL.getInferenceEntityId())); - assertSemanticFieldMapper(mapperService, "my_field"); - } - - private static void assertSemanticFieldMapper(MapperService mapperService, String fieldName) { - Mapper mapper = mapperService.mappingLookup().getMapper(fieldName); - assertThat(mapper, instanceOf(SemanticFieldMapper.class)); + assertSemanticField(mapperService, "my_field", false, TEST_MODEL, null, null); } @Override From 890f86d3857efb9174c24b255686f1e4d784c2d2 Mon Sep 17 00:00:00 2001 From: Mike Pellegrini Date: Fri, 24 Jul 2026 10:51:28 -0400 Subject: [PATCH 15/24] Address some TODOs --- .../AbstractSemanticMapperTestCase.java | 74 ++++--------------- 1 file changed, 14 insertions(+), 60 deletions(-) diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/AbstractSemanticMapperTestCase.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/AbstractSemanticMapperTestCase.java index 1f9d846484d19..9b592462f64f9 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/AbstractSemanticMapperTestCase.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/AbstractSemanticMapperTestCase.java @@ -276,9 +276,6 @@ public void testUpdateInferenceId_GivenNoModelSettings() throws IOException { assertInferenceEndpoints(mapperService, fieldName, oldInferenceId, oldInferenceId); assertSemanticField(mapperService, fieldName, false, oldModel, null, null); - if (oldModel != null) { - assertEmbeddingsFieldMapperMatchesModel(mapperService, fieldName, oldModel); - } String newInferenceId = randomValueOtherThan(oldInferenceId, () -> randomAlphaOfLengthBetween(5, 15)); TestModel newModel = null; @@ -291,9 +288,6 @@ public void testUpdateInferenceId_GivenNoModelSettings() throws IOException { assertInferenceEndpoints(mapperService, fieldName, newInferenceId, newInferenceId); assertSemanticField(mapperService, fieldName, false, newModel, null, null); - if (newModel != null) { - assertEmbeddingsFieldMapperMatchesModel(mapperService, fieldName, newModel); - } } } @@ -312,7 +306,6 @@ public void testUpdateInferenceId_GivenModelSettings() throws IOException { ); assertInferenceEndpoints(mapperService, fieldName, oldInferenceId, oldInferenceId); assertSemanticField(mapperService, fieldName, true, oldModel, null, null); - assertEmbeddingsFieldMapperMatchesModel(mapperService, fieldName, oldModel); final CheckedRunnable mergeRunner = () -> merge(mapperService, semanticMapping(fieldName, newInferenceId)); @@ -325,7 +318,6 @@ public void testUpdateInferenceId_GivenModelSettings() throws IOException { mergeRunner.run(); assertInferenceEndpoints(mapperService, fieldName, newInferenceId, newInferenceId); assertSemanticField(mapperService, fieldName, true, newModel, null, null); - assertEmbeddingsFieldMapperMatchesModel(mapperService, fieldName, newModel); } else { final TestModel incompatibleModel = createIncompatibleModel(newInferenceId, oldModel); final String expectedErrorMessage; @@ -516,8 +508,17 @@ protected void performDynamicUpdate( /** * Asserts the structural invariants of a semantic field mapping: the concrete mapper and field type classes, the nested chunks - * field, the chunks text/offsets sub-fields (see {@link #assertChunksTextField}), the embeddings sub-mapper when model settings - * are expected (see {@link #assertEmbeddingsField}), and the chunking settings. + * field, the chunks text/offsets sub-fields, the embeddings sub-mapper, model settings, chunking settings, and index options. + * + * @param mapperService The mapper service. + * @param fieldName The name of the field to check. + * @param expectedModelSettings Whether model settings are expected. If true, {@code model} is required. + * @param model The model used to validate the embeddings sub-mapper. When null, verifies that the embeddings sub-mapper is not + * configured. + * @param expectedChunkingSettings The expected chunking settings. + * @param expectedIndexOptions The expected index options. When null and {@code model} is provided, the expected default index options + * are automatically determined and validated. When non-null, they must specify the complete index options + * expected, including any element type override. */ protected void assertSemanticField( MapperService mapperService, @@ -562,7 +563,7 @@ protected void assertSemanticField( if (expectedModelSettings) { if (model == null) { - throw new IllegalArgumentException("model must be provided to check model settings"); + throw new AssertionError("model must be provided to check model settings"); } assertModelSettings(semanticFieldType, model); } else { @@ -636,60 +637,13 @@ protected void assertModelSettings(U fieldType, Model model) { MinimalServiceSettings actual = fieldType.getModelSettings(); assertNotNull(actual); - // TODO: Use canMergeWith assertEquals(model.getTaskType(), actual.taskType()); assertEquals(model.getServiceSettings().dimensions(), actual.dimensions()); assertEquals(model.getServiceSettings().similarity(), actual.similarity()); assertEquals(model.getServiceSettings().elementType(), actual.elementType()); - } - - // TODO: Try to remove - protected static DenseVectorFieldMapper.ElementType getExpectedElementType( - IndexVersion indexVersion, - DenseVectorFieldMapper.ElementType modelElementType, - @Nullable SemanticIndexOptions semanticIndexOptions - ) { - if (semanticIndexOptions != null && semanticIndexOptions.indexOptions() instanceof ExtendedDenseVectorIndexOptions edvio) { - if (edvio.getElementType() != null) { - return edvio.getElementType(); - } - } - DenseVectorFieldMapper.ElementType expectedElementType = modelElementType; - if (indexVersion.onOrAfter(SEMANTIC_TEXT_DEFAULTS_TO_BFLOAT16) && expectedElementType == DenseVectorFieldMapper.ElementType.FLOAT) { - expectedElementType = DenseVectorFieldMapper.ElementType.BFLOAT16; - } - return expectedElementType; - } - - /** - * Asserts that the generated embeddings sub-mapper (sparse or dense) matches the task type and service settings of the - * referenced {@link Model}, including dimensions, element type, and similarity for dense models. - */ - // TODO: Remove - protected void assertEmbeddingsFieldMapperMatchesModel(MapperService mapperService, String fieldName, Model model) { - Mapper embeddingsFieldMapper = mapperService.mappingLookup().getMapper(getEmbeddingsFieldName(fieldName)); - switch (model.getTaskType()) { - case SPARSE_EMBEDDING -> assertThat(embeddingsFieldMapper, is(instanceOf(SparseVectorFieldMapper.class))); - case TEXT_EMBEDDING, EMBEDDING -> { - T semanticFieldMapper = getSemanticFieldMapper(mapperService, fieldName); - DenseVectorFieldMapper.ElementType expectedElementType = getExpectedElementType( - mapperService.getIndexSettings().getIndexVersionCreated(), - model.getServiceSettings().elementType(), - semanticFieldMapper.fieldType().getIndexOptions() - ); - assertThat(embeddingsFieldMapper, is(instanceOf(DenseVectorFieldMapper.class))); - DenseVectorFieldMapper denseVectorFieldMapper = (DenseVectorFieldMapper) embeddingsFieldMapper; - ServiceSettings modelServiceSettings = model.getConfigurations().getServiceSettings(); - assertThat(denseVectorFieldMapper.fieldType().getVectorDimensions(), equalTo(modelServiceSettings.dimensions())); - assertThat(denseVectorFieldMapper.fieldType().getElementType(), equalTo(expectedElementType)); - assertThat( - denseVectorFieldMapper.fieldType().getSimilarity(), - equalTo(modelServiceSettings.similarity().vectorSimilarity()) - ); - } - default -> throw new AssertionError("Unexpected task type [" + model.getTaskType() + "]"); - } + MinimalServiceSettings modelServiceSettings = new MinimalServiceSettings(model); + assertTrue(modelServiceSettings.canMergeWith(actual)); } protected static void assertInferenceEndpoints( From e8fe17ffc994c766b3ebe213ea720b7c68429a65 Mon Sep 17 00:00:00 2001 From: Mike Pellegrini Date: Fri, 24 Jul 2026 11:27:43 -0400 Subject: [PATCH 16/24] Remove getExpectedDefaultIndexOptions and associated helpers --- .../AbstractSemanticMapperTestCase.java | 1 - .../mapper/SemanticTextFieldMapperTests.java | 76 +------------------ 2 files changed, 2 insertions(+), 75 deletions(-) diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/AbstractSemanticMapperTestCase.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/AbstractSemanticMapperTestCase.java index 9b592462f64f9..4043db07a6249 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/AbstractSemanticMapperTestCase.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/AbstractSemanticMapperTestCase.java @@ -88,7 +88,6 @@ import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; -import static org.hamcrest.Matchers.is; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java index b35fb8cc4c81f..8bd193893f45b 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java @@ -100,8 +100,6 @@ import java.util.function.BiConsumer; import java.util.stream.Collectors; -import static org.elasticsearch.index.IndexSettings.DENSE_VECTOR_EXPERIMENTAL_FEATURES_SETTING; -import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapperTestUtils.defaultDenseVectorIndexOptions; import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldTypeTests.randomIndexOptionsAll; import static org.elasticsearch.index.mapper.vectors.SparseVectorFieldTypeTests.randomSparseVectorIndexOptions; import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.CHUNKS_FIELD; @@ -1706,26 +1704,6 @@ public void testMappingWithEndpointMetadata() throws IOException { assertThat(parsedMapper.fieldType().getModelSettings().endpointMetadata(), equalTo(EndpointMetadata.EMPTY_INSTANCE)); } - private static SemanticIndexOptions defaultDenseVectorSemanticIndexOptions( - IndexVersion indexVersionCreated, - License.OperationMode operationMode, - Integer dims, - DenseVectorFieldMapper.ElementType elementType, - boolean experimentalFeaturesEnabled - ) { - return new SemanticIndexOptions( - SemanticIndexOptions.SupportedIndexOptions.DENSE_VECTOR, - defaultDenseVectorIndexOptions( - indexVersionCreated, - operationMode == License.OperationMode.ENTERPRISE, - true, - dims, - elementType, - experimentalFeaturesEnabled - ) - ); - } - private static DenseVectorFieldMapper.DenseVectorIndexOptions defaultBbqHnswDenseVectorIndexOptions() { int m = Lucene99HnswVectorsFormat.DEFAULT_MAX_CONN; int efConstruction = Lucene99HnswVectorsFormat.DEFAULT_BEAM_WIDTH; @@ -1733,17 +1711,6 @@ private static DenseVectorFieldMapper.DenseVectorIndexOptions defaultBbqHnswDens return new DenseVectorFieldMapper.BBQHnswIndexOptions(m, efConstruction, false, rescoreVector, -1); } - private static SemanticIndexOptions defaultBbqHnswSemanticIndexOptions() { - return new SemanticIndexOptions(SemanticIndexOptions.SupportedIndexOptions.DENSE_VECTOR, defaultBbqHnswDenseVectorIndexOptions()); - } - - private static SemanticIndexOptions defaultSparseVectorIndexOptions(IndexVersion indexVersion) { - return new SemanticIndexOptions( - SemanticIndexOptions.SupportedIndexOptions.SPARSE_VECTOR, - SparseVectorFieldMapper.SparseVectorIndexOptions.getDefaultIndexOptions(indexVersion) - ); - } - public void testDefaultIndexOptions() throws IOException { for (int i = 0; i < 200; i++) { final Model model = createRandomSupportedModel(); @@ -1759,39 +1726,6 @@ public void testDefaultIndexOptions() throws IOException { } } - // TODO: Remove - private SemanticIndexOptions getExpectedDefaultIndexOptions( - TaskType taskType, - DenseVectorFieldMapper.ElementType elementType, - Integer dimensions, - IndexVersion indexVersion, - boolean experimentalFeatures - ) { - return switch (taskType) { - case TEXT_EMBEDDING, EMBEDDING -> { - boolean floatFamilyElementType = elementType == DenseVectorFieldMapper.ElementType.FLOAT - || elementType == DenseVectorFieldMapper.ElementType.BFLOAT16; - if (floatFamilyElementType - && SemanticTextFieldMapper.setExplicitIndexOptionsForSemanticText(indexVersion) - && dimensions >= DenseVectorFieldMapper.BBQ_MIN_DIMS) { - yield defaultBbqHnswSemanticIndexOptions(); - } else if (floatFamilyElementType) { - yield defaultDenseVectorSemanticIndexOptions( - indexVersion, - operationMode, - dimensions, - elementType, - experimentalFeatures - ); - } else { - yield null; - } - } - case SPARSE_EMBEDDING -> defaultSparseVectorIndexOptions(indexVersion); - default -> throw new AssertionError("Unexpected task type [" + taskType + "]"); - }; - } - public void testSpecifiedDenseVectorIndexOptions() throws IOException { IndexVersion indexVersion = SemanticInferenceMetadataFieldsMapperTests.getRandomCompatibleIndexVersion(useLegacyFormat()); Model denseModel = denseTextEmbeddingModel(100, SimilarityMeasure.COSINE, DenseVectorFieldMapper.ElementType.FLOAT); @@ -1934,15 +1868,9 @@ public void testSetElementTypeInDenseVectorIndexOptions() throws IOException { b.endObject(); }), indexVersion); - SemanticIndexOptions expectedDefaultIndexOptions = getExpectedDefaultIndexOptions( - TaskType.TEXT_EMBEDDING, - DenseVectorFieldMapper.ElementType.FLOAT, - 100, - indexVersion, - DENSE_VECTOR_EXPERIMENTAL_FEATURES_SETTING.get(mapperService.getIndexSettings().getSettings()) - ); + SemanticIndexOptions expectedDefaultIndexOptions = getDefaultIndexOptions(denseModel, mapperService); DenseVectorFieldMapper.DenseVectorIndexOptions expectedDenseVectorIndexOptions = expectedDefaultIndexOptions != null - ? (DenseVectorFieldMapper.DenseVectorIndexOptions) expectedDefaultIndexOptions.indexOptions() + ? ((ExtendedDenseVectorIndexOptions) expectedDefaultIndexOptions.indexOptions()).getBaseIndexOptions() : null; assertSemanticField( From 44100ab8546d922255561e43a1f4a1051ee44bc6 Mon Sep 17 00:00:00 2001 From: Mike Pellegrini Date: Fri, 24 Jul 2026 11:29:37 -0400 Subject: [PATCH 17/24] Remove TODO --- .../xpack/inference/mapper/AbstractSemanticMapperTestCase.java | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/AbstractSemanticMapperTestCase.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/AbstractSemanticMapperTestCase.java index 4043db07a6249..1d9e13fa8e852 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/AbstractSemanticMapperTestCase.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/AbstractSemanticMapperTestCase.java @@ -681,7 +681,6 @@ protected SemanticIndexOptions getDefaultIndexOptions(Model model, MapperService ); } - // TODO: Don't duplicate this logic DenseVectorFieldMapper.ElementType elementTypeOverride = indexVersion.onOrAfter(SEMANTIC_TEXT_DEFAULTS_TO_BFLOAT16) && elementType == DenseVectorFieldMapper.ElementType.FLOAT ? DenseVectorFieldMapper.ElementType.BFLOAT16 : null; From 47eabdff12d06debc2aa437cc2586c9a9dc7e67b Mon Sep 17 00:00:00 2001 From: Mike Pellegrini Date: Fri, 24 Jul 2026 12:00:42 -0400 Subject: [PATCH 18/24] Use semanticMapping instead of XContentBuilders to build mappings --- .../mapper/SemanticTextFieldMapperTests.java | 25 ++++++------------- 1 file changed, 8 insertions(+), 17 deletions(-) diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java index 8bd193893f45b..075d2911f284c 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java @@ -28,7 +28,6 @@ import org.elasticsearch.action.support.PlainActionFuture; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.CheckedBiConsumer; -import org.elasticsearch.common.CheckedBiFunction; import org.elasticsearch.common.Strings; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.lucene.search.Queries; @@ -552,7 +551,7 @@ public void testHighlighterRequiresSourceWhenCopyToSourceFieldNeedsIt() throws I assumeFalse("the legacy format keeps the original value in _source", useLegacyFormat); MapperService mapperService = createSemanticMapperServiceWithSourceMode(mapping(b -> { b.startObject("body").field("type", "text").field("copy_to", "field").endObject(); - b.startObject("field").field("type", "semantic_text").field("inference_id", "test_model").endObject(); + addSemanticMapping(b, "field", "test_model", null, null, null, null); }), IndexVersion.current(), SourceFieldMapper.Mode.SYNTHETIC); SearchExecutionContext context = createSearchExecutionContext(mapperService); assertFalse(new SemanticTextHighlighter().canHighlightWithoutSource(context.getFieldType("field"), context)); @@ -1054,29 +1053,21 @@ public void testUpdateSearchInferenceId() throws IOException { final String searchInferenceId1 = "test_search_inference_id_1"; final String searchInferenceId2 = "test_search_inference_id_2"; - CheckedBiFunction buildMapping = (f, sid) -> mapping(b -> { - b.startObject(f).field("type", "semantic_text").field("inference_id", inferenceId); - if (sid != null) { - b.field("search_inference_id", sid); - } - b.endObject(); - }); - for (int depth = 1; depth < 5; depth++) { String fieldName = randomFieldName(depth); - MapperService mapperService = createSemanticMapperService(buildMapping.apply(fieldName, null)); + MapperService mapperService = createSemanticMapperService(semanticMapping(fieldName, inferenceId)); assertSemanticField(mapperService, fieldName, false, null, null, null); assertInferenceEndpoints(mapperService, fieldName, inferenceId, inferenceId); - merge(mapperService, buildMapping.apply(fieldName, searchInferenceId1)); + merge(mapperService, semanticMapping(fieldName, inferenceId, searchInferenceId1)); assertSemanticField(mapperService, fieldName, false, null, null, null); assertInferenceEndpoints(mapperService, fieldName, inferenceId, searchInferenceId1); - merge(mapperService, buildMapping.apply(fieldName, searchInferenceId2)); + merge(mapperService, semanticMapping(fieldName, inferenceId, searchInferenceId2)); assertSemanticField(mapperService, fieldName, false, null, null, null); assertInferenceEndpoints(mapperService, fieldName, inferenceId, searchInferenceId2); - merge(mapperService, buildMapping.apply(fieldName, null)); + merge(mapperService, semanticMapping(fieldName, inferenceId)); assertSemanticField(mapperService, fieldName, false, null, null, null); assertInferenceEndpoints(mapperService, fieldName, inferenceId, inferenceId); @@ -1092,15 +1083,15 @@ public void testUpdateSearchInferenceId() throws IOException { assertSemanticField(mapperService, fieldName, true, sparseModel, null, expectedIndexOptions); assertInferenceEndpoints(mapperService, fieldName, inferenceId, inferenceId); - merge(mapperService, buildMapping.apply(fieldName, searchInferenceId1)); + merge(mapperService, semanticMapping(fieldName, inferenceId, searchInferenceId1)); assertSemanticField(mapperService, fieldName, true, sparseModel, null, expectedIndexOptions); assertInferenceEndpoints(mapperService, fieldName, inferenceId, searchInferenceId1); - merge(mapperService, buildMapping.apply(fieldName, searchInferenceId2)); + merge(mapperService, semanticMapping(fieldName, inferenceId, searchInferenceId2)); assertSemanticField(mapperService, fieldName, true, sparseModel, null, expectedIndexOptions); assertInferenceEndpoints(mapperService, fieldName, inferenceId, searchInferenceId2); - merge(mapperService, buildMapping.apply(fieldName, null)); + merge(mapperService, semanticMapping(fieldName, inferenceId)); assertSemanticField(mapperService, fieldName, true, sparseModel, null, expectedIndexOptions); assertInferenceEndpoints(mapperService, fieldName, inferenceId, inferenceId); } From 64d1df51a7aded4fdee8a90a2dd59870510c24d3 Mon Sep 17 00:00:00 2001 From: Mike Pellegrini Date: Fri, 24 Jul 2026 14:12:43 -0400 Subject: [PATCH 19/24] Use semanticMapping helpers instead of XContentBuilders to build mappings (part 2) --- .../AbstractSemanticMapperTestCase.java | 60 +- .../mapper/SemanticTextFieldMapperTests.java | 636 +++++++----------- 2 files changed, 301 insertions(+), 395 deletions(-) diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/AbstractSemanticMapperTestCase.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/AbstractSemanticMapperTestCase.java index 1d9e13fa8e852..778f506b8307e 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/AbstractSemanticMapperTestCase.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/AbstractSemanticMapperTestCase.java @@ -14,6 +14,7 @@ import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.core.CheckedRunnable; import org.elasticsearch.core.Nullable; import org.elasticsearch.features.FeatureService; @@ -355,44 +356,84 @@ public void testUpdateInferenceId_GivenModelSettings() throws IOException { } } - protected XContentBuilder semanticMapping(String fieldName, String inferenceId) throws IOException { + protected XContentBuilder semanticMapping(String fieldName, @Nullable String inferenceId) throws IOException { return semanticMapping(fieldName, inferenceId, null, null, null, null); } - protected XContentBuilder semanticMapping(String fieldName, String inferenceId, @Nullable String searchInferenceId) throws IOException { + protected XContentBuilder semanticMapping(String fieldName, @Nullable String inferenceId, @Nullable String searchInferenceId) + throws IOException { return semanticMapping(fieldName, inferenceId, searchInferenceId, null, null, null); } - protected XContentBuilder semanticMapping(String fieldName, String inferenceId, @Nullable MinimalServiceSettings modelSettings) - throws IOException { + protected XContentBuilder semanticMapping( + String fieldName, + @Nullable String inferenceId, + @Nullable MinimalServiceSettings modelSettings + ) throws IOException { return semanticMapping(fieldName, inferenceId, null, modelSettings, null, null); } protected XContentBuilder semanticMapping( String fieldName, - String inferenceId, + @Nullable String inferenceId, @Nullable String searchInferenceId, @Nullable MinimalServiceSettings modelSettings, @Nullable ChunkingSettings chunkingSettings, @Nullable SemanticIndexOptions indexOptions + ) throws IOException { + return semanticMapping(fieldName, inferenceId, searchInferenceId, modelSettings, chunkingSettings, indexOptions, null); + } + + protected XContentBuilder semanticMapping( + String fieldName, + @Nullable String inferenceId, + @Nullable String searchInferenceId, + @Nullable MinimalServiceSettings modelSettings, + @Nullable ChunkingSettings chunkingSettings, + @Nullable SemanticIndexOptions indexOptions, + @Nullable CheckedConsumer additionalFields ) throws IOException { return mapping( - b -> addSemanticMapping(b, fieldName, inferenceId, searchInferenceId, modelSettings, chunkingSettings, indexOptions) + b -> addSemanticMapping( + b, + fieldName, + inferenceId, + searchInferenceId, + modelSettings, + chunkingSettings, + indexOptions, + additionalFields + ) ); } protected void addSemanticMapping( XContentBuilder mappingBuilder, String fieldName, - String inferenceId, + @Nullable String inferenceId, @Nullable String searchInferenceId, @Nullable MinimalServiceSettings modelSettings, @Nullable ChunkingSettings chunkingSettings, @Nullable SemanticIndexOptions indexOptions + ) throws IOException { + addSemanticMapping(mappingBuilder, fieldName, inferenceId, searchInferenceId, modelSettings, chunkingSettings, indexOptions, null); + } + + protected void addSemanticMapping( + XContentBuilder mappingBuilder, + String fieldName, + @Nullable String inferenceId, + @Nullable String searchInferenceId, + @Nullable MinimalServiceSettings modelSettings, + @Nullable ChunkingSettings chunkingSettings, + @Nullable SemanticIndexOptions indexOptions, + @Nullable CheckedConsumer additionalFields ) throws IOException { mappingBuilder.startObject(fieldName); mappingBuilder.field("type", contentType()); - mappingBuilder.field(INFERENCE_ID_FIELD, inferenceId); + if (inferenceId != null) { + mappingBuilder.field(INFERENCE_ID_FIELD, inferenceId); + } if (searchInferenceId != null) { mappingBuilder.field(SEARCH_INFERENCE_ID_FIELD, searchInferenceId); } @@ -405,6 +446,9 @@ protected void addSemanticMapping( if (indexOptions != null) { mappingBuilder.field(INDEX_OPTIONS_FIELD, indexOptions); } + if (additionalFields != null) { + additionalFields.accept(mappingBuilder); + } mappingBuilder.endObject(); } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java index 075d2911f284c..ae12494295493 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java @@ -105,7 +105,6 @@ import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.INFERENCE_FIELD; import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.INFERENCE_ID_FIELD; import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.MODEL_SETTINGS_FIELD; -import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.SEARCH_INFERENCE_ID_FIELD; import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.TEXT_FIELD; import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.getChunksFieldName; import static org.elasticsearch.xpack.inference.mapper.SemanticTextField.getEmbeddingsFieldName; @@ -264,11 +263,6 @@ protected void metaMapping(XContentBuilder b) throws IOException { b.field(INFERENCE_ID_FIELD, DEFAULT_EIS_JINA_V5_INFERENCE_ID); } - protected void extendedMapping(XContentBuilder b, Map extensions) throws IOException { - minimalMapping(b); - b.mapContents(extensions); - } - @Override protected Object getSampleObjectForDocument() { // Only consulted by testSupportsParsingObject, and only for the legacy format (supportsParsingObject() is true only then). A @@ -626,9 +620,7 @@ public void testSetInferenceEndpoints() throws IOException { assertSerialization.accept(fieldMapping, mapperService); } { - final XContentBuilder fieldMapping = fieldMapping( - b -> b.field("type", "semantic_text").field(SEARCH_INFERENCE_ID_FIELD, searchInferenceId) - ); + final XContentBuilder fieldMapping = semanticMapping("field", null, searchInferenceId); final XContentBuilder expectedMapping = semanticMapping("field", DEFAULT_EIS_JINA_V5_INFERENCE_ID, searchInferenceId); final MapperService mapperService = createSemanticMapperService(fieldMapping, IndexVersions.SEMANTIC_TEXT_DEFAULTS_TO_JINA_V5); assertSemanticField(mapperService, fieldName, false, JINA_V5_TEXT_MODEL, null, null); @@ -658,17 +650,11 @@ public void testInvalidInferenceEndpoints() { ); } { - Exception e = expectThrows( - MapperParsingException.class, - () -> createSemanticMapperService(fieldMapping(b -> b.field("type", "semantic_text").field(INFERENCE_ID_FIELD, ""))) - ); + Exception e = expectThrows(MapperParsingException.class, () -> createSemanticMapperService(semanticMapping("field", ""))); assertThat(e.getMessage(), containsString("[inference_id] on mapper [field] of type [semantic_text] must not be empty")); } { - Exception e = expectThrows( - MapperParsingException.class, - () -> createSemanticMapperService(fieldMapping(b -> b.field("type", "semantic_text").field(SEARCH_INFERENCE_ID_FIELD, ""))) - ); + Exception e = expectThrows(MapperParsingException.class, () -> createSemanticMapperService(semanticMapping("field", null, ""))); assertThat(e.getMessage(), containsString("[search_inference_id] on mapper [field] of type [semantic_text] must not be empty")); } } @@ -686,13 +672,14 @@ public void testInvalidTaskTypes() { if (taskType == TaskType.TEXT_EMBEDDING || taskType == TaskType.SPARSE_EMBEDDING || taskType == TaskType.EMBEDDING) { continue; } - Exception e = expectThrows(MapperParsingException.class, () -> createSemanticMapperService(fieldMapping(b -> { - b.field("type", "semantic_text") - .field(INFERENCE_ID_FIELD, "test1") - .startObject("model_settings") - .field("task_type", taskType); - b.endObject(); - }))); + Exception e = expectThrows( + MapperParsingException.class, + () -> createSemanticMapperService(semanticMapping("field", "test1", null, null, null, null, b -> { + b.startObject("model_settings"); + b.field("task_type", taskType); + b.endObject(); + })) + ); assertThat(e.getMessage(), containsString("Wrong [task_type], expected text_embedding, embedding, or sparse_embedding")); } } @@ -705,13 +692,14 @@ public void testMultiFieldNamedInputIsRejected() throws IOException { IndexVersions.SEMANTIC_TEXT_ORIGINAL_VALUES_DOC_VALUES, IndexVersion.current() ); - Exception e = expectThrows(MapperParsingException.class, () -> createSemanticMapperServiceWithIndexVersion(fieldMapping(b -> { - b.field("type", "semantic_text"); - b.field("inference_id", "my_inference_id"); - b.startObject("fields"); - b.startObject("input").field("type", "keyword").endObject(); - b.endObject(); - }), indexVersion)); + Exception e = expectThrows( + MapperParsingException.class, + () -> createSemanticMapperServiceWithIndexVersion(semanticMapping("field", "my_inference_id", null, null, null, null, b -> { + b.startObject("fields"); + b.startObject("input").field("type", "keyword").endObject(); + b.endObject(); + }), indexVersion) + ); assertThat(e.getMessage(), containsString("cannot have a multi-field named [input]")); } @@ -720,13 +708,14 @@ public void testMultiFieldNamedInputIsAccepted() throws IOException { // a multi-field named [input] stays valid. This verifies the gating does not invalidate indices created at an older version. assumeFalse("multi-fields are rejected entirely in the legacy format", useLegacyFormat); IndexVersion indexVersion = IndexVersionUtils.getPreviousVersion(IndexVersions.SEMANTIC_TEXT_ORIGINAL_VALUES_DOC_VALUES); - MapperService mapperService = createSemanticMapperServiceWithIndexVersion(fieldMapping(b -> { - b.field("type", "semantic_text"); - b.field("inference_id", "my_inference_id"); - b.startObject("fields"); - b.startObject("input").field("type", "keyword").endObject(); - b.endObject(); - }), indexVersion); + MapperService mapperService = createSemanticMapperServiceWithIndexVersion( + semanticMapping("field", "my_inference_id", null, null, null, null, b -> { + b.startObject("fields"); + b.startObject("input").field("type", "keyword").endObject(); + b.endObject(); + }), + indexVersion + ); assertNotNull(mapperService.mappingLookup().getMapper("field.input")); } @@ -735,77 +724,49 @@ public void testMultiFieldsSupport() throws IOException { Exception e = expectThrows(MapperParsingException.class, () -> createSemanticMapperService(fieldMapping(b -> { b.field("type", "text"); b.startObject("fields"); - b.startObject("semantic"); - b.field("type", "semantic_text"); - b.field("inference_id", "my_inference_id"); + addSemanticMapping(b, "semantic", "my_inference_id", null, null, null, null); b.endObject(); b.endObject(); }))); assertThat(e.getMessage(), containsString("Field [semantic] of type [semantic_text] can't be used in multifields")); } else { - IndexVersion indexVersion = SparseVectorFieldMapperTests.getIndexOptionsCompatibleIndexVersion(); - SparseVectorFieldMapper.SparseVectorIndexOptions expectedIndexOptions = SparseVectorFieldMapper.SparseVectorIndexOptions - .getDefaultIndexOptions(indexVersion); - SemanticIndexOptions semanticIndexOptions = expectedIndexOptions == null - ? null - : new SemanticIndexOptions(SemanticIndexOptions.SupportedIndexOptions.SPARSE_VECTOR, expectedIndexOptions); Model sparseModel = TestModel.createRandomInstance(TaskType.SPARSE_EMBEDDING); - var mapperService = createSemanticMapperServiceWithIndexVersion(fieldMapping(b -> { - b.field("type", "text"); - b.startObject("fields"); - b.startObject("semantic"); - b.field("type", "semantic_text"); - b.field("inference_id", "my_inference_id"); - b.startObject("model_settings"); - b.field("task_type", "sparse_embedding"); - b.endObject(); - b.endObject(); - b.endObject(); - }), indexVersion); - assertSemanticField(mapperService, "field.semantic", true, sparseModel, null, semanticIndexOptions); - - mapperService = createSemanticMapperServiceWithIndexVersion(fieldMapping(b -> { - b.field("type", "semantic_text"); - b.field("inference_id", "my_inference_id"); - b.startObject("model_settings"); - b.field("task_type", "sparse_embedding"); - b.endObject(); - b.startObject("fields"); - b.startObject("text"); + var mapperService = createSemanticMapperService(fieldMapping(b -> { b.field("type", "text"); - b.endObject(); - b.endObject(); - }), indexVersion); - assertSemanticField(mapperService, "field", true, sparseModel, null, semanticIndexOptions); - - mapperService = createSemanticMapperServiceWithIndexVersion(fieldMapping(b -> { - b.field("type", "semantic_text"); - b.field("inference_id", "my_inference_id"); - b.startObject("model_settings"); - b.field("task_type", "sparse_embedding"); - b.endObject(); b.startObject("fields"); - b.startObject("semantic"); - b.field("type", "semantic_text"); - b.field("inference_id", "another_inference_id"); - b.startObject("model_settings"); - b.field("task_type", "sparse_embedding"); - b.endObject(); + addSemanticMapping(b, "semantic", "my_inference_id", null, new MinimalServiceSettings(sparseModel), null, null); b.endObject(); b.endObject(); - }), indexVersion); - assertSemanticField(mapperService, "field", true, sparseModel, null, semanticIndexOptions); - assertSemanticField(mapperService, "field.semantic", true, sparseModel, null, semanticIndexOptions); + })); + assertSemanticField(mapperService, "field.semantic", true, sparseModel, null, null); - Exception e = expectThrows(MapperParsingException.class, () -> createSemanticMapperService(fieldMapping(b -> { - b.field("type", "semantic_text"); - b.field("inference_id", "my_inference_id"); - b.startObject("fields"); - b.startObject("inference"); - b.field("type", "text"); - b.endObject(); - b.endObject(); - }))); + mapperService = createSemanticMapperService( + semanticMapping("field", "my_inference_id", null, new MinimalServiceSettings(sparseModel), null, null, b -> { + b.startObject("fields"); + b.startObject("text").field("type", "text").endObject(); + b.endObject(); + }) + ); + assertSemanticField(mapperService, "field", true, sparseModel, null, null); + + mapperService = createSemanticMapperService( + semanticMapping("field", "my_inference_id", null, new MinimalServiceSettings(sparseModel), null, null, b -> { + b.startObject("fields"); + addSemanticMapping(b, "semantic", "another_inference_id", null, new MinimalServiceSettings(sparseModel), null, null); + b.endObject(); + }) + ); + assertSemanticField(mapperService, "field", true, sparseModel, null, null); + assertSemanticField(mapperService, "field.semantic", true, sparseModel, null, null); + + Exception e = expectThrows( + MapperParsingException.class, + () -> createSemanticMapperService(semanticMapping("field", "my_inference_id", null, null, null, null, b -> { + b.startObject("fields"); + b.startObject("inference").field("type", "text").endObject(); + b.endObject(); + })) + ); assertThat(e.getMessage(), containsString("is already used by another field")); } } @@ -852,18 +813,11 @@ public void testUpdateModelSettings() throws IOException { { Exception exc = expectThrows( MapperParsingException.class, - () -> merge( - mapperService, - mapping( - b -> b.startObject(fieldName) - .field("type", "semantic_text") - .field("inference_id", "test_model") - .startObject("model_settings") - .field("inference_id", "test_model") - .endObject() - .endObject() - ) - ) + () -> merge(mapperService, semanticMapping(fieldName, "test_model", null, null, null, null, b -> { + b.startObject("model_settings"); + b.field("inference_id", "test_model"); + b.endObject(); + })) ); assertThat(exc.getMessage(), containsString("Required [task_type]")); } @@ -911,18 +865,17 @@ public void testDenseVectorIndexOptionValidation() throws IOException { String fieldName = randomFieldName(depth); DenseVectorFieldMapper.DenseVectorIndexOptions indexOptions = DenseVectorFieldTypeTests.randomIndexOptionsAll(); - Exception exc = expectThrows(MapperParsingException.class, () -> createSemanticMapperService(mapping(b -> { - b.startObject(fieldName); - b.field("type", SemanticTextFieldMapper.CONTENT_TYPE); - b.field(INFERENCE_ID_FIELD, inferenceId); - b.startObject(INDEX_OPTIONS_FIELD); - b.startObject("dense_vector"); - b.field("type", indexOptions.getType().name().toLowerCase(Locale.ROOT)); - b.field("unsupported_param", "any_value"); - b.endObject(); - b.endObject(); - b.endObject(); - }))); + Exception exc = expectThrows( + MapperParsingException.class, + () -> createSemanticMapperService(semanticMapping(fieldName, inferenceId, null, null, null, null, b -> { + b.startObject(INDEX_OPTIONS_FIELD); + b.startObject("dense_vector"); + b.field("type", indexOptions.getType().name().toLowerCase(Locale.ROOT)); + b.field("unsupported_param", "any_value"); + b.endObject(); + b.endObject(); + })) + ); assertTrue(exc.getMessage().contains("unsupported parameters")); } } @@ -942,54 +895,26 @@ private static Model denseTextEmbeddingModel( ); } - private void addSparseVectorModelSettingsToBuilder(XContentBuilder b) throws IOException { - b.startObject("model_settings"); - b.field("task_type", TaskType.SPARSE_EMBEDDING); - b.endObject(); - } - public void testSparseVectorIndexOptionsValidationAndMapping() throws IOException { for (int depth = 1; depth < 5; depth++) { String inferenceId = "test_model"; String fieldName = randomFieldName(depth); - IndexVersion indexVersion; - if (useLegacyFormat) { - // Must be in a range that supports both sparse vector index options and legacy format - indexVersion = randomBoolean() - ? IndexVersionUtils.randomVersionBetween( - IndexVersions.SPARSE_VECTOR_PRUNING_INDEX_OPTIONS_SUPPORT, - IndexVersionUtils.getPreviousVersion(IndexVersions.SEMANTIC_TEXT_LEGACY_FORMAT_FORBIDDEN) - ) - : IndexVersionUtils.randomVersionBetween( - IndexVersions.SPARSE_VECTOR_PRUNING_INDEX_OPTIONS_SUPPORT_BACKPORT_8_X, - IndexVersionUtils.getPreviousVersion(IndexVersions.UPGRADE_TO_LUCENE_10_0_0) - ); - } else { - indexVersion = SparseVectorFieldMapperTests.getIndexOptionsCompatibleIndexVersion(); - } var sparseVectorIndexOptions = SparseVectorFieldTypeTests.randomSparseVectorIndexOptions(); var expectedIndexOptions = sparseVectorIndexOptions == null ? null : new SemanticIndexOptions(SemanticIndexOptions.SupportedIndexOptions.SPARSE_VECTOR, sparseVectorIndexOptions); // should not throw an exception - MapperService mapper = createSemanticMapperServiceWithIndexVersion(mapping(b -> { - b.startObject(fieldName); - { - b.field("type", SemanticTextFieldMapper.CONTENT_TYPE); - b.field(INFERENCE_ID_FIELD, inferenceId); - addSparseVectorModelSettingsToBuilder(b); - if (sparseVectorIndexOptions != null) { - b.startObject(INDEX_OPTIONS_FIELD); - { - b.field(SparseVectorFieldMapper.CONTENT_TYPE); - sparseVectorIndexOptions.toXContent(b, null); - } - b.endObject(); - } - } - b.endObject(); - }), indexVersion); + MapperService mapper = createSemanticMapperService( + semanticMapping( + fieldName, + inferenceId, + null, + new MinimalServiceSettings(null, TaskType.SPARSE_EMBEDDING, null, null, null), + null, + expectedIndexOptions + ) + ); assertSemanticField( mapper, @@ -1678,11 +1603,11 @@ public void testMappingWithEndpointMetadata() throws IOException { endpointMetadata ); - MapperService originalMapperService = createSemanticMapperService(fieldMapping(b -> { - b.field("type", "semantic_text"); - b.field("inference_id", "test_service"); - b.field("model_settings", modelSettingsWithMetadata); - })); + MapperService originalMapperService = createSemanticMapperService( + semanticMapping("field", "test_service", null, null, null, null, b -> { + b.field("model_settings", modelSettingsWithMetadata); + }) + ); SemanticTextFieldMapper originalMapper = getSemanticFieldMapper(originalMapperService, "field"); assertThat(originalMapper.fieldType().getModelSettings().endpointMetadata(), equalTo(endpointMetadata)); @@ -1722,56 +1647,31 @@ public void testSpecifiedDenseVectorIndexOptions() throws IOException { Model denseModel = denseTextEmbeddingModel(100, SimilarityMeasure.COSINE, DenseVectorFieldMapper.ElementType.FLOAT); // Specifying index options will override default index option settings - var mapperService = createSemanticMapperServiceWithIndexVersion(fieldMapping(b -> { - b.field("type", "semantic_text"); - b.field("inference_id", "another_inference_id"); - b.startObject("model_settings"); - b.field("task_type", "text_embedding"); - b.field("dimensions", 100); - b.field("similarity", "cosine"); - b.field("element_type", "float"); - b.endObject(); - b.startObject("index_options"); - b.startObject("dense_vector"); - b.field("element_type", "float"); // Explicitly override element type to hold it constant - b.field("type", "int4_hnsw"); - b.field("m", 20); - b.field("ef_construction", 90); - b.endObject(); - b.endObject(); - }), indexVersion); - assertSemanticField( - mapperService, - "field", - true, - denseModel, - null, - new SemanticIndexOptions( - SemanticIndexOptions.SupportedIndexOptions.DENSE_VECTOR, - new ExtendedDenseVectorIndexOptions( - new DenseVectorFieldMapper.Int4HnswIndexOptions(20, 90, false, null, -1), - DenseVectorFieldMapper.ElementType.FLOAT - ) + SemanticIndexOptions expectedIndexOptions = new SemanticIndexOptions( + SemanticIndexOptions.SupportedIndexOptions.DENSE_VECTOR, + new ExtendedDenseVectorIndexOptions( + new DenseVectorFieldMapper.Int4HnswIndexOptions(20, 90, false, null, -1), + DenseVectorFieldMapper.ElementType.FLOAT // Explicitly override element type to hold it constant ) ); + var mapperService = createSemanticMapperServiceWithIndexVersion( + semanticMapping("field", "another_inference_id", null, new MinimalServiceSettings(denseModel), null, expectedIndexOptions), + indexVersion + ); + assertSemanticField(mapperService, "field", true, denseModel, null, expectedIndexOptions); - // Specifying partial index options will in the remainder index options with defaults - mapperService = createSemanticMapperServiceWithIndexVersion(fieldMapping(b -> { - b.field("type", "semantic_text"); - b.field("inference_id", "another_inference_id"); - b.startObject("model_settings"); - b.field("task_type", "text_embedding"); - b.field("dimensions", 100); - b.field("similarity", "cosine"); - b.field("element_type", "float"); - b.endObject(); - b.startObject("index_options"); - b.startObject("dense_vector"); - b.field("element_type", "float"); // Explicitly override element type to hold it constant - b.field("type", "int4_hnsw"); - b.endObject(); - b.endObject(); - }), indexVersion); + // Specifying partial index options will fill in the remainder index options with defaults + mapperService = createSemanticMapperServiceWithIndexVersion( + semanticMapping("field", "another_inference_id", null, new MinimalServiceSettings(denseModel), null, null, b -> { + b.startObject("index_options"); + b.startObject("dense_vector"); + b.field("element_type", "float"); // Explicitly override element type to hold it constant + b.field("type", "int4_hnsw"); + b.endObject(); + b.endObject(); + }), + indexVersion + ); assertSemanticField( mapperService, "field", @@ -1788,53 +1688,58 @@ public void testSpecifiedDenseVectorIndexOptions() throws IOException { ); // Incompatible index options will fail - Exception e = expectThrows(MapperParsingException.class, () -> createSemanticMapperServiceWithIndexVersion(fieldMapping(b -> { - b.field("type", "semantic_text"); - b.field("inference_id", "another_inference_id"); - b.startObject("model_settings"); - b.field("task_type", "sparse_embedding"); - b.endObject(); - b.startObject("index_options"); - b.startObject("dense_vector"); - b.field("type", "int8_hnsw"); - b.endObject(); - b.endObject(); - }), indexVersion)); + Exception e = expectThrows( + MapperParsingException.class, + () -> createSemanticMapperServiceWithIndexVersion( + semanticMapping( + "field", + "another_inference_id", + null, + new MinimalServiceSettings(null, TaskType.SPARSE_EMBEDDING, null, null, null), + null, + null, + b -> { + b.startObject("index_options"); + b.startObject("dense_vector"); + b.field("type", "int8_hnsw"); + b.endObject(); + b.endObject(); + } + ), + indexVersion + ) + ); assertThat(e.getMessage(), containsString("Invalid task type")); - e = expectThrows(MapperParsingException.class, () -> createSemanticMapperServiceWithIndexVersion(fieldMapping(b -> { - b.field("type", "semantic_text"); - b.field("inference_id", "another_inference_id"); - b.startObject("model_settings"); - b.field("task_type", "text_embedding"); - b.field("dimensions", 100); - b.field("similarity", "cosine"); - b.field("element_type", "float"); - b.endObject(); - b.startObject("index_options"); - b.startObject("dense_vector"); - b.field("type", "bbq_flat"); - b.field("ef_construction", 100); - b.endObject(); - b.endObject(); - }), indexVersion)); + e = expectThrows( + MapperParsingException.class, + () -> createSemanticMapperServiceWithIndexVersion( + semanticMapping("field", "another_inference_id", null, new MinimalServiceSettings(denseModel), null, null, b -> { + b.startObject("index_options"); + b.startObject("dense_vector"); + b.field("type", "bbq_flat"); + b.field("ef_construction", 100); + b.endObject(); + b.endObject(); + }), + indexVersion + ) + ); assertThat(e.getMessage(), containsString("unsupported parameters: [ef_construction : 100]")); - e = expectThrows(MapperParsingException.class, () -> createSemanticMapperServiceWithIndexVersion(fieldMapping(b -> { - b.field("type", "semantic_text"); - b.field("inference_id", "another_inference_id"); - b.startObject("model_settings"); - b.field("task_type", "text_embedding"); - b.field("dimensions", 100); - b.field("similarity", "cosine"); - b.field("element_type", "float"); - b.endObject(); - b.startObject("index_options"); - b.startObject("dense_vector"); - b.field("type", "invalid"); - b.endObject(); - b.endObject(); - }), indexVersion)); + e = expectThrows( + MapperParsingException.class, + () -> createSemanticMapperServiceWithIndexVersion( + semanticMapping("field", "another_inference_id", null, new MinimalServiceSettings(denseModel), null, null, b -> { + b.startObject("index_options"); + b.startObject("dense_vector"); + b.field("type", "invalid"); + b.endObject(); + b.endObject(); + }), + indexVersion + ) + ); assertThat(e.getMessage(), containsString("Unsupported index options type invalid")); } @@ -1843,21 +1748,20 @@ public void testSetElementTypeInDenseVectorIndexOptions() throws IOException { // Specifying the element type prevents defaulting to bfloat16 IndexVersion indexVersion = SemanticInferenceMetadataFieldsMapperTests.getRandomCompatibleIndexVersion(useLegacyFormat); - var mapperService = createSemanticMapperServiceWithIndexVersion(fieldMapping(b -> { - b.field("type", "semantic_text"); - b.field("inference_id", "another_inference_id"); - b.startObject("model_settings"); - b.field("task_type", "text_embedding"); - b.field("dimensions", 100); - b.field("similarity", "cosine"); - b.field("element_type", "float"); - b.endObject(); - b.startObject("index_options"); - b.startObject("dense_vector"); - b.field("element_type", "float"); - b.endObject(); - b.endObject(); - }), indexVersion); + var mapperService = createSemanticMapperServiceWithIndexVersion( + semanticMapping( + "field", + "another_inference_id", + null, + new MinimalServiceSettings(denseModel), + null, + new SemanticIndexOptions( + SemanticIndexOptions.SupportedIndexOptions.DENSE_VECTOR, + new ExtendedDenseVectorIndexOptions(null, DenseVectorFieldMapper.ElementType.FLOAT) + ) + ), + indexVersion + ); SemanticIndexOptions expectedDefaultIndexOptions = getDefaultIndexOptions(denseModel, mapperService); DenseVectorFieldMapper.DenseVectorIndexOptions expectedDenseVectorIndexOptions = expectedDefaultIndexOptions != null @@ -1877,39 +1781,26 @@ public void testSetElementTypeInDenseVectorIndexOptions() throws IOException { ); // Can use element type in combination with type - mapperService = createSemanticMapperServiceWithIndexVersion(fieldMapping(b -> { - b.field("type", "semantic_text"); - b.field("inference_id", "another_inference_id"); - b.startObject("model_settings"); - b.field("task_type", "text_embedding"); - b.field("dimensions", 100); - b.field("similarity", "cosine"); - b.field("element_type", "float"); - b.endObject(); - b.startObject("index_options"); - b.startObject("dense_vector"); - b.field("element_type", "float"); - b.field("type", "int4_hnsw"); - b.field("m", 20); - b.field("ef_construction", 90); - b.endObject(); - b.endObject(); - }), indexVersion); - - assertSemanticField( - mapperService, - "field", - true, - denseModel, - null, - new SemanticIndexOptions( - SemanticIndexOptions.SupportedIndexOptions.DENSE_VECTOR, - new ExtendedDenseVectorIndexOptions( - new DenseVectorFieldMapper.Int4HnswIndexOptions(20, 90, false, null, -1), - DenseVectorFieldMapper.ElementType.FLOAT - ) + SemanticIndexOptions int4HnswWithFloatIndexOptions = new SemanticIndexOptions( + SemanticIndexOptions.SupportedIndexOptions.DENSE_VECTOR, + new ExtendedDenseVectorIndexOptions( + new DenseVectorFieldMapper.Int4HnswIndexOptions(20, 90, false, null, -1), + DenseVectorFieldMapper.ElementType.FLOAT ) ); + mapperService = createSemanticMapperServiceWithIndexVersion( + semanticMapping( + "field", + "another_inference_id", + null, + new MinimalServiceSettings(denseModel), + null, + int4HnswWithFloatIndexOptions + ), + indexVersion + ); + + assertSemanticField(mapperService, "field", true, denseModel, null, int4HnswWithFloatIndexOptions); } public void testInvalidElementTypeOverride() { @@ -1924,21 +1815,22 @@ public void testInvalidElementTypeOverride() { overrideElementType = randomValueOtherThan(modelElementType, () -> randomFrom(DenseVectorFieldMapper.ElementType.values())); } - MapperParsingException e = expectThrows(MapperParsingException.class, () -> createSemanticMapperService(fieldMapping(b -> { - b.field("type", "semantic_text"); - b.field("inference_id", "test_inference_id"); - b.startObject("model_settings"); - b.field("task_type", model.getTaskType().toString()); - b.field("dimensions", model.getServiceSettings().dimensions()); - b.field("similarity", model.getServiceSettings().similarity().toString()); - b.field("element_type", modelElementType.toString()); - b.endObject(); - b.startObject("index_options"); - b.startObject("dense_vector"); - b.field("element_type", overrideElementType.toString()); - b.endObject(); - b.endObject(); - }))); + MapperParsingException e = expectThrows( + MapperParsingException.class, + () -> createSemanticMapperService( + semanticMapping( + "field", + "test_inference_id", + null, + new MinimalServiceSettings(model), + null, + new SemanticIndexOptions( + SemanticIndexOptions.SupportedIndexOptions.DENSE_VECTOR, + new ExtendedDenseVectorIndexOptions(null, overrideElementType) + ) + ) + ) + ); assertThat( e.getMessage(), containsString( @@ -1953,17 +1845,17 @@ public void testSpecificSparseVectorIndexOptions() throws IOException { IndexVersion indexVersion = SemanticInferenceMetadataFieldsMapperTests.getRandomCompatibleIndexVersion(useLegacyFormat()); SparseVectorFieldMapper.SparseVectorIndexOptions testIndexOptions = randomSparseVectorIndexOptions(false); Model sparseModel = TestModel.createRandomInstance(TaskType.SPARSE_EMBEDDING); - var mapperService = createSemanticMapperServiceWithIndexVersion(fieldMapping(b -> { - b.field("type", SemanticTextFieldMapper.CONTENT_TYPE); - b.field(INFERENCE_ID_FIELD, "test_inference_id"); - addSparseVectorModelSettingsToBuilder(b); - b.startObject(INDEX_OPTIONS_FIELD); - { - b.field(SparseVectorFieldMapper.CONTENT_TYPE); - testIndexOptions.toXContent(b, null); - } - b.endObject(); - }), indexVersion); + var mapperService = createSemanticMapperServiceWithIndexVersion( + semanticMapping( + "field", + "test_inference_id", + null, + new MinimalServiceSettings(sparseModel), + null, + new SemanticIndexOptions(SemanticIndexOptions.SupportedIndexOptions.SPARSE_VECTOR, testIndexOptions) + ), + indexVersion + ); assertSemanticField( mapperService, @@ -1978,85 +1870,55 @@ public void testSpecificSparseVectorIndexOptions() throws IOException { public void testSparseVectorIndexOptionsValidations() throws IOException { IndexVersion indexVersion = SemanticInferenceMetadataFieldsMapperTests.getRandomCompatibleIndexVersion(useLegacyFormat()); - Exception e = expectThrows(MapperParsingException.class, () -> createSemanticMapperServiceWithIndexVersion(fieldMapping(b -> { - b.field("type", SemanticTextFieldMapper.CONTENT_TYPE); - b.field(INFERENCE_ID_FIELD, "test_inference_id"); - b.startObject(INDEX_OPTIONS_FIELD); - { + Exception e = expectThrows( + MapperParsingException.class, + () -> createSemanticMapperServiceWithIndexVersion(semanticMapping("field", "test_inference_id", null, null, null, null, b -> { + b.startObject(INDEX_OPTIONS_FIELD); b.startObject(SparseVectorFieldMapper.CONTENT_TYPE); - { - b.field("prune", false); - b.startObject("pruning_config"); - { - b.field(TokenPruningConfig.TOKENS_FREQ_RATIO_THRESHOLD.getPreferredName(), 5.0f); - } - b.endObject(); - } + b.field("prune", false); + b.startObject("pruning_config"); + b.field(TokenPruningConfig.TOKENS_FREQ_RATIO_THRESHOLD.getPreferredName(), 5.0f); b.endObject(); - } - b.endObject(); - }), indexVersion)); + b.endObject(); + b.endObject(); + }), indexVersion) + ); assertThat(e.getMessage(), containsString("failed to parse field [pruning_config]")); - e = expectThrows(MapperParsingException.class, () -> createSemanticMapperServiceWithIndexVersion(fieldMapping(b -> { - b.field("type", SemanticTextFieldMapper.CONTENT_TYPE); - b.field(INFERENCE_ID_FIELD, "test_inference_id"); - b.startObject(INDEX_OPTIONS_FIELD); - { + e = expectThrows( + MapperParsingException.class, + () -> createSemanticMapperServiceWithIndexVersion(semanticMapping("field", "test_inference_id", null, null, null, null, b -> { + b.startObject(INDEX_OPTIONS_FIELD); b.startObject(SparseVectorFieldMapper.CONTENT_TYPE); - { - b.field("prune", true); - b.startObject("pruning_config"); - { - b.field(TokenPruningConfig.TOKENS_FREQ_RATIO_THRESHOLD.getPreferredName(), 1000.0f); - } - b.endObject(); - } + b.field("prune", true); + b.startObject("pruning_config"); + b.field(TokenPruningConfig.TOKENS_FREQ_RATIO_THRESHOLD.getPreferredName(), 1000.0f); b.endObject(); - } - b.endObject(); - }), indexVersion)); + b.endObject(); + b.endObject(); + }), indexVersion) + ); var innerClause = e.getCause().getCause().getCause(); assertThat(innerClause.getMessage(), containsString("[tokens_freq_ratio_threshold] must be between [1] and [100], got 1000.0")); } @Override public void testSupportedIndexVersions() throws IOException { - - // Add model settings via extendedSettings to trigger programmatic - // creation of SparseVectorFieldMapper and DenseVectorFieldMapper - Model sparseModel = TestModel.createRandomInstance(TaskType.SPARSE_EMBEDDING); - Map sparseExtensions = Map.of( - "inference_id", - sparseModel.getInferenceEntityId(), - "model_settings", - Map.of("task_type", TaskType.SPARSE_EMBEDDING.toString()) - ); - Model denseModel = TestModel.createRandomInstance(TaskType.TEXT_EMBEDDING); - Map denseExtensions = Map.of( - "inference_id", - denseModel.getInferenceEntityId(), - "model_settings", - Map.of( - "task_type", - TaskType.TEXT_EMBEDDING.toString(), - "dimensions", - denseModel.getServiceSettings().dimensions(), - "similarity", - denseModel.getServiceSettings().similarity(), - "element_type", - denseModel.getServiceSettings().elementType() - ) - ); Set supportedVersions = getSupportedVersions(); for (int i = 0; i < Math.min(supportedVersions.size(), 100); i++) { IndexVersion indexVersion = randomFrom(supportedVersions); - MapperService denseMapperService = createMapperService(indexVersion, fieldMapping(b -> extendedMapping(b, denseExtensions))); - MapperService sparseMapperService = createMapperService(indexVersion, fieldMapping(b -> extendedMapping(b, sparseExtensions))); + createMapperService( + indexVersion, + semanticMapping("field", denseModel.getInferenceEntityId(), new MinimalServiceSettings(denseModel)) + ); + createMapperService( + indexVersion, + semanticMapping("field", sparseModel.getInferenceEntityId(), new MinimalServiceSettings(sparseModel)) + ); } } From 484fed1396a1ef56bf2561b9f0d322533a3eff2b Mon Sep 17 00:00:00 2001 From: Mike Pellegrini Date: Fri, 24 Jul 2026 14:49:07 -0400 Subject: [PATCH 20/24] Remove manual computation of default index options --- .../mapper/SemanticTextFieldMapperTests.java | 72 +++++-------------- 1 file changed, 16 insertions(+), 56 deletions(-) diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java index ae12494295493..5c944900899d3 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java @@ -659,14 +659,6 @@ public void testInvalidInferenceEndpoints() { } } - private SemanticIndexOptions getDefaultSparseVectorIndexOptionsForMapper(MapperService mapperService) { - var mapperIndexVersion = mapperService.getIndexSettings().getIndexVersionCreated(); - var defaultSparseVectorIndexOptions = SparseVectorFieldMapper.SparseVectorIndexOptions.getDefaultIndexOptions(mapperIndexVersion); - return defaultSparseVectorIndexOptions == null - ? null - : new SemanticIndexOptions(SemanticIndexOptions.SupportedIndexOptions.SPARSE_VECTOR, defaultSparseVectorIndexOptions); - } - public void testInvalidTaskTypes() { for (var taskType : TaskType.values()) { if (taskType == TaskType.TEXT_EMBEDDING || taskType == TaskType.SPARSE_EMBEDDING || taskType == TaskType.EMBEDDING) { @@ -785,8 +777,7 @@ public void testDynamicUpdate() throws IOException { inferenceId, new MinimalServiceSettings("service", TaskType.SPARSE_EMBEDDING, null, null, null) ); - var expectedIndexOptions = getDefaultSparseVectorIndexOptionsForMapper(mapperService); - assertSemanticField(mapperService, fieldName, true, model, null, expectedIndexOptions); + assertSemanticField(mapperService, fieldName, true, model, null, null); assertInferenceEndpoints(mapperService, fieldName, inferenceId, inferenceId); } @@ -799,8 +790,7 @@ public void testDynamicUpdate() throws IOException { inferenceId, new MinimalServiceSettings("service", TaskType.SPARSE_EMBEDDING, null, null, null) ); - var expectedIndexOptions = getDefaultSparseVectorIndexOptionsForMapper(mapperService); - assertSemanticField(mapperService, fieldName, true, model, null, expectedIndexOptions); + assertSemanticField(mapperService, fieldName, true, model, null, null); assertInferenceEndpoints(mapperService, fieldName, inferenceId, searchInferenceId); } } @@ -827,14 +817,12 @@ public void testUpdateModelSettings() throws IOException { mapperService, semanticMapping(fieldName, "test_model", new MinimalServiceSettings(null, TaskType.SPARSE_EMBEDDING, null, null, null)) ); - var expectedIndexOptions = getDefaultSparseVectorIndexOptionsForMapper(mapperService); - assertSemanticField(mapperService, fieldName, true, sparseModel, null, expectedIndexOptions); + assertSemanticField(mapperService, fieldName, true, sparseModel, null, null); } { Model sparseModel = TestModel.createRandomInstance(TaskType.SPARSE_EMBEDDING); merge(mapperService, semanticMapping(fieldName, "test_model")); - var expectedIndexOptions = getDefaultSparseVectorIndexOptionsForMapper(mapperService); - assertSemanticField(mapperService, fieldName, true, sparseModel, null, expectedIndexOptions); + assertSemanticField(mapperService, fieldName, true, sparseModel, null, null); } { Exception exc = expectThrows( @@ -935,41 +923,21 @@ public void testSparseVectorMappingUpdate() throws IOException { ); final ChunkingSettings chunkingSettings = generateRandomChunkingSettings(false); - IndexVersion indexVersion = useLegacyFormat - ? IndexVersionUtils.randomVersionBetween( - IndexVersions.SPARSE_VECTOR_PRUNING_INDEX_OPTIONS_SUPPORT, - IndexVersionUtils.getPreviousVersion(IndexVersions.SEMANTIC_TEXT_LEGACY_FORMAT_FORBIDDEN) - ) - : SparseVectorFieldMapperTests.getIndexOptionsCompatibleIndexVersion(); final SemanticIndexOptions indexOptions = randomSemanticIndexOptions(TaskType.SPARSE_EMBEDDING); - String fieldName = "field"; + final String fieldName = "field"; - MapperService mapperService = createSemanticMapperServiceWithIndexVersion( - semanticMapping(fieldName, model.getInferenceEntityId(), null, null, chunkingSettings, indexOptions), - indexVersion + MapperService mapperService = createSemanticMapperService( + semanticMapping(fieldName, model.getInferenceEntityId(), null, null, chunkingSettings, indexOptions) ); - var expectedIndexOptions = (indexOptions == null) - ? new SemanticIndexOptions( - SemanticIndexOptions.SupportedIndexOptions.SPARSE_VECTOR, - SparseVectorFieldMapper.SparseVectorIndexOptions.getDefaultIndexOptions(indexVersion) - ) - : indexOptions; - assertSemanticField(mapperService, fieldName, false, model, chunkingSettings, expectedIndexOptions); + assertSemanticField(mapperService, fieldName, false, model, chunkingSettings, indexOptions); final SemanticIndexOptions newIndexOptions = randomSemanticIndexOptions(TaskType.SPARSE_EMBEDDING); - expectedIndexOptions = (newIndexOptions == null) - ? new SemanticIndexOptions( - SemanticIndexOptions.SupportedIndexOptions.SPARSE_VECTOR, - SparseVectorFieldMapper.SparseVectorIndexOptions.getDefaultIndexOptions(indexVersion) - ) - : newIndexOptions; - - ChunkingSettings newChunkingSettings = generateRandomChunkingSettingsOtherThan(chunkingSettings); + final ChunkingSettings newChunkingSettings = generateRandomChunkingSettingsOtherThan(chunkingSettings); merge( mapperService, semanticMapping(fieldName, model.getInferenceEntityId(), null, null, newChunkingSettings, newIndexOptions) ); - assertSemanticField(mapperService, fieldName, false, model, newChunkingSettings, expectedIndexOptions); + assertSemanticField(mapperService, fieldName, false, model, newChunkingSettings, newIndexOptions); } } @@ -1004,20 +972,19 @@ public void testUpdateSearchInferenceId() throws IOException { new MinimalServiceSettings("my-service", TaskType.SPARSE_EMBEDDING, null, null, null) ) ); - var expectedIndexOptions = getDefaultSparseVectorIndexOptionsForMapper(mapperService); - assertSemanticField(mapperService, fieldName, true, sparseModel, null, expectedIndexOptions); + assertSemanticField(mapperService, fieldName, true, sparseModel, null, null); assertInferenceEndpoints(mapperService, fieldName, inferenceId, inferenceId); merge(mapperService, semanticMapping(fieldName, inferenceId, searchInferenceId1)); - assertSemanticField(mapperService, fieldName, true, sparseModel, null, expectedIndexOptions); + assertSemanticField(mapperService, fieldName, true, sparseModel, null, null); assertInferenceEndpoints(mapperService, fieldName, inferenceId, searchInferenceId1); merge(mapperService, semanticMapping(fieldName, inferenceId, searchInferenceId2)); - assertSemanticField(mapperService, fieldName, true, sparseModel, null, expectedIndexOptions); + assertSemanticField(mapperService, fieldName, true, sparseModel, null, null); assertInferenceEndpoints(mapperService, fieldName, inferenceId, searchInferenceId2); merge(mapperService, semanticMapping(fieldName, inferenceId)); - assertSemanticField(mapperService, fieldName, true, sparseModel, null, expectedIndexOptions); + assertSemanticField(mapperService, fieldName, true, sparseModel, null, null); assertInferenceEndpoints(mapperService, fieldName, inferenceId, inferenceId); } } @@ -1127,22 +1094,15 @@ public void testSuccessfulParse() throws IOException { ); }); - var expectedIndexOptions = (indexOptions == null) - ? new SemanticIndexOptions( - SemanticIndexOptions.SupportedIndexOptions.SPARSE_VECTOR, - SparseVectorFieldMapper.SparseVectorIndexOptions.getDefaultIndexOptions(indexVersion) - ) - : indexOptions; - MapperService mapperService = createSemanticMapperServiceWithIndexVersion(mapping, indexVersion); - assertSemanticField(mapperService, fieldName1, false, model1, null, expectedIndexOptions); + assertSemanticField(mapperService, fieldName1, false, model1, null, indexOptions); assertInferenceEndpoints( mapperService, fieldName1, model1.getInferenceEntityId(), setSearchInferenceId ? searchInferenceId : model1.getInferenceEntityId() ); - assertSemanticField(mapperService, fieldName2, false, model2, null, expectedIndexOptions); + assertSemanticField(mapperService, fieldName2, false, model2, null, indexOptions); assertInferenceEndpoints( mapperService, fieldName2, From 4103e1eb8dc8430d2bb0ea22472dadeb5f86fc0e Mon Sep 17 00:00:00 2001 From: Mike Pellegrini Date: Fri, 24 Jul 2026 14:58:51 -0400 Subject: [PATCH 21/24] Fix testMultiFieldsSupport --- .../xpack/inference/mapper/SemanticTextFieldMapperTests.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java index 5c944900899d3..92718e5df3be5 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java @@ -718,7 +718,6 @@ public void testMultiFieldsSupport() throws IOException { b.startObject("fields"); addSemanticMapping(b, "semantic", "my_inference_id", null, null, null, null); b.endObject(); - b.endObject(); }))); assertThat(e.getMessage(), containsString("Field [semantic] of type [semantic_text] can't be used in multifields")); } else { @@ -728,7 +727,6 @@ public void testMultiFieldsSupport() throws IOException { b.startObject("fields"); addSemanticMapping(b, "semantic", "my_inference_id", null, new MinimalServiceSettings(sparseModel), null, null); b.endObject(); - b.endObject(); })); assertSemanticField(mapperService, "field.semantic", true, sparseModel, null, null); From 2b603527604ff5c5b270ab5deaedbfd0dca5ff8a Mon Sep 17 00:00:00 2001 From: Mike Pellegrini Date: Fri, 24 Jul 2026 15:07:07 -0400 Subject: [PATCH 22/24] Spotless --- .../xpack/inference/mapper/SemanticTextFieldMapperTests.java | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java index 92718e5df3be5..3fb79f76e62b7 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java @@ -58,7 +58,6 @@ import org.elasticsearch.index.mapper.vectors.DenseVectorFieldTypeTests; import org.elasticsearch.index.mapper.vectors.IndexOptions; import org.elasticsearch.index.mapper.vectors.SparseVectorFieldMapper; -import org.elasticsearch.index.mapper.vectors.SparseVectorFieldMapperTests; import org.elasticsearch.index.mapper.vectors.SparseVectorFieldTypeTests; import org.elasticsearch.index.mapper.vectors.TokenPruningConfig; import org.elasticsearch.index.query.SearchExecutionContext; From 96132b2ab012da06dea71d2a8ea3dadc23ccbc9a Mon Sep 17 00:00:00 2001 From: Mike Pellegrini Date: Fri, 24 Jul 2026 21:34:08 -0400 Subject: [PATCH 23/24] Update assertSemanticField to take MinimalServiceSettings instead of Model --- .../AbstractSemanticMapperTestCase.java | 110 ++++++----- .../mapper/SemanticFieldMapperTests.java | 4 +- .../mapper/SemanticTextFieldMapperTests.java | 175 ++++++------------ 3 files changed, 129 insertions(+), 160 deletions(-) diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/AbstractSemanticMapperTestCase.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/AbstractSemanticMapperTestCase.java index 778f506b8307e..757ab4980fb16 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/AbstractSemanticMapperTestCase.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/AbstractSemanticMapperTestCase.java @@ -37,8 +37,6 @@ import org.elasticsearch.index.mapper.vectors.SparseVectorFieldMapper; import org.elasticsearch.inference.ChunkingSettings; import org.elasticsearch.inference.MinimalServiceSettings; -import org.elasticsearch.inference.Model; -import org.elasticsearch.inference.ServiceSettings; import org.elasticsearch.inference.SimilarityMeasure; import org.elasticsearch.inference.TaskType; import org.elasticsearch.license.License; @@ -275,7 +273,14 @@ public void testUpdateInferenceId_GivenNoModelSettings() throws IOException { var mapperService = createSemanticMapperService(semanticMapping(fieldName, oldInferenceId)); assertInferenceEndpoints(mapperService, fieldName, oldInferenceId, oldInferenceId); - assertSemanticField(mapperService, fieldName, false, oldModel, null, null); + assertSemanticField( + mapperService, + fieldName, + false, + oldModel == null ? null : new MinimalServiceSettings(oldModel), + null, + null + ); String newInferenceId = randomValueOtherThan(oldInferenceId, () -> randomAlphaOfLengthBetween(5, 15)); TestModel newModel = null; @@ -287,7 +292,14 @@ public void testUpdateInferenceId_GivenNoModelSettings() throws IOException { merge(mapperService, semanticMapping(fieldName, newInferenceId)); assertInferenceEndpoints(mapperService, fieldName, newInferenceId, newInferenceId); - assertSemanticField(mapperService, fieldName, false, newModel, null, null); + assertSemanticField( + mapperService, + fieldName, + false, + newModel == null ? null : new MinimalServiceSettings(newModel), + null, + null + ); } } @@ -305,7 +317,7 @@ public void testUpdateInferenceId_GivenModelSettings() throws IOException { semanticMapping(fieldName, oldInferenceId, previousModelSettings) ); assertInferenceEndpoints(mapperService, fieldName, oldInferenceId, oldInferenceId); - assertSemanticField(mapperService, fieldName, true, oldModel, null, null); + assertSemanticField(mapperService, fieldName, true, previousModelSettings, null, null); final CheckedRunnable mergeRunner = () -> merge(mapperService, semanticMapping(fieldName, newInferenceId)); @@ -317,7 +329,7 @@ public void testUpdateInferenceId_GivenModelSettings() throws IOException { mergeRunner.run(); assertInferenceEndpoints(mapperService, fieldName, newInferenceId, newInferenceId); - assertSemanticField(mapperService, fieldName, true, newModel, null, null); + assertSemanticField(mapperService, fieldName, true, newModelSettings, null, null); } else { final TestModel incompatibleModel = createIncompatibleModel(newInferenceId, oldModel); final String expectedErrorMessage; @@ -511,6 +523,14 @@ protected TestModel createRandomSupportedModel() { return TestModel.createRandomInstance(randomFrom(supportedTaskTypes())); } + protected MinimalServiceSettings createRandomModelSettings() { + return new MinimalServiceSettings(createRandomSupportedModel()); + } + + protected MinimalServiceSettings createRandomModelSettings(TaskType taskType) { + return new MinimalServiceSettings(TestModel.createRandomInstance(taskType)); + } + protected T getSemanticFieldMapper(MapperService mapperService, String fieldName) { Mapper mapper = mapperService.mappingLookup().getMapper(fieldName); assertThat(mapper, instanceOf(expectedMapperClass())); @@ -555,19 +575,20 @@ protected void performDynamicUpdate( * * @param mapperService The mapper service. * @param fieldName The name of the field to check. - * @param expectedModelSettings Whether model settings are expected. If true, {@code model} is required. - * @param model The model used to validate the embeddings sub-mapper. When null, verifies that the embeddings sub-mapper is not - * configured. + * @param modelSettingsSetOnFieldType Whether model settings should be set on the field type. If true, {@code modelSettings} is + * required. + * @param modelSettings The model settings used to validate the embeddings sub-mapper. When null, verifies that the embeddings + * sub-mapper is not configured. * @param expectedChunkingSettings The expected chunking settings. - * @param expectedIndexOptions The expected index options. When null and {@code model} is provided, the expected default index options - * are automatically determined and validated. When non-null, they must specify the complete index options - * expected, including any element type override. + * @param expectedIndexOptions The expected index options. When null and {@code modelSettings} is provided, the expected default index + * options are automatically determined and validated. When non-null, they must specify the complete index + * options expected, including any element type override. */ protected void assertSemanticField( MapperService mapperService, String fieldName, - boolean expectedModelSettings, - @Nullable Model model, + boolean modelSettingsSetOnFieldType, + @Nullable MinimalServiceSettings modelSettings, @Nullable ChunkingSettings expectedChunkingSettings, @Nullable SemanticIndexOptions expectedIndexOptions ) { @@ -588,10 +609,10 @@ protected void assertSemanticField( assertChunksTextField(semanticFieldType, chunksMapper); Mapper embeddingsMapper = chunksMapper.getMapper(CHUNKED_EMBEDDINGS_FIELD); - if (model != null) { + if (modelSettings != null) { SemanticIndexOptions resolvedExpectedIndexOptions = expectedIndexOptions; if (resolvedExpectedIndexOptions == null) { - resolvedExpectedIndexOptions = getDefaultIndexOptions(model, mapperService); + resolvedExpectedIndexOptions = getDefaultIndexOptions(modelSettings, mapperService); } assertNotNull(embeddingsMapper); @@ -599,16 +620,16 @@ protected void assertSemanticField( FieldMapper embeddingsFieldMapper = (FieldMapper) embeddingsMapper; assertSame(embeddingsFieldMapper.fieldType(), mapperService.mappingLookup().getFieldType(getEmbeddingsFieldName(fieldName))); assertThat(embeddingsMapper.fullPath(), equalTo(getEmbeddingsFieldName(fieldName))); - assertEmbeddingsField(mapperService, embeddingsFieldMapper, model, resolvedExpectedIndexOptions); + assertEmbeddingsField(mapperService, embeddingsFieldMapper, modelSettings, resolvedExpectedIndexOptions); } else { assertNull(embeddingsMapper); } - if (expectedModelSettings) { - if (model == null) { - throw new AssertionError("model must be provided to check model settings"); + if (modelSettingsSetOnFieldType) { + if (modelSettings == null) { + throw new AssertionError("modelSettings must be provided to check model settings on the field type"); } - assertModelSettings(semanticFieldType, model); + assertModelSettingsOnFieldType(semanticFieldType, modelSettings); } else { assertNull(semanticFieldType.getModelSettings()); } @@ -633,25 +654,24 @@ protected void assertChunksTextField(U fieldType, NestedObjectMapper chunksMappe } /** - * Asserts the embeddings sub-mapper against the referenced {@link Model} and expected index options. The base implementation - * covers the dense task types ({@code text_embedding}, {@code embedding}); {@code semantic_text} overrides this to also cover - * {@code sparse_embedding}. + * Asserts the embeddings sub-mapper against the referenced {@link MinimalServiceSettings} and expected index options. The base + * implementation covers the dense task types ({@code text_embedding}, {@code embedding}); {@code semantic_text} overrides this to + * also cover {@code sparse_embedding}. */ protected void assertEmbeddingsField( MapperService mapperService, FieldMapper embeddingsMapper, - Model model, + MinimalServiceSettings modelSettings, @Nullable SemanticIndexOptions expectedIndexOptions ) { IndexVersion indexVersion = mapperService.getIndexSettings().getIndexVersionCreated(); - TaskType taskType = model.getTaskType(); - ServiceSettings serviceSettings = model.getServiceSettings(); + TaskType taskType = modelSettings.taskType(); if (taskType == TaskType.TEXT_EMBEDDING || taskType == TaskType.EMBEDDING) { assertThat(embeddingsMapper, instanceOf(DenseVectorFieldMapper.class)); DenseVectorFieldMapper denseVectorFieldMapper = (DenseVectorFieldMapper) embeddingsMapper; IndexOptions expectedBaseIndexOptions = null; - DenseVectorFieldMapper.ElementType expectedElementType = serviceSettings.elementType(); + DenseVectorFieldMapper.ElementType expectedElementType = modelSettings.elementType(); if (expectedIndexOptions != null) { IndexOptions expectedEmbeddingFieldIndexOptions = expectedIndexOptions.indexOptions(); if (expectedEmbeddingFieldIndexOptions instanceof ExtendedDenseVectorIndexOptions edvio) { @@ -666,27 +686,26 @@ protected void assertEmbeddingsField( assertEquals(expectedBaseIndexOptions, denseVectorFieldMapper.fieldType().getIndexOptions()); assertEquals(expectedElementType, denseVectorFieldMapper.fieldType().getElementType()); - assertEquals(serviceSettings.dimensions().intValue(), denseVectorFieldMapper.fieldType().getVectorDimensions()); - if (serviceSettings.similarity() != null && indexVersion.onOrAfter(NEW_SPARSE_VECTOR)) { + assertEquals(modelSettings.dimensions().intValue(), denseVectorFieldMapper.fieldType().getVectorDimensions()); + if (modelSettings.similarity() != null && indexVersion.onOrAfter(NEW_SPARSE_VECTOR)) { // We don't set similarity on pre 8.11 indices - assertEquals(serviceSettings.similarity().vectorSimilarity(), denseVectorFieldMapper.fieldType().getSimilarity()); + assertEquals(modelSettings.similarity().vectorSimilarity(), denseVectorFieldMapper.fieldType().getSimilarity()); } } else { throw new AssertionError("Invalid task type [" + taskType + "]"); } } - protected void assertModelSettings(U fieldType, Model model) { + protected void assertModelSettingsOnFieldType(U fieldType, MinimalServiceSettings modelSettings) { MinimalServiceSettings actual = fieldType.getModelSettings(); assertNotNull(actual); - assertEquals(model.getTaskType(), actual.taskType()); - assertEquals(model.getServiceSettings().dimensions(), actual.dimensions()); - assertEquals(model.getServiceSettings().similarity(), actual.similarity()); - assertEquals(model.getServiceSettings().elementType(), actual.elementType()); + assertEquals(modelSettings.taskType(), actual.taskType()); + assertEquals(modelSettings.dimensions(), actual.dimensions()); + assertEquals(modelSettings.similarity(), actual.similarity()); + assertEquals(modelSettings.elementType(), actual.elementType()); - MinimalServiceSettings modelServiceSettings = new MinimalServiceSettings(model); - assertTrue(modelServiceSettings.canMergeWith(actual)); + assertTrue(modelSettings.canMergeWith(actual)); } protected static void assertInferenceEndpoints( @@ -703,17 +722,17 @@ protected static void assertInferenceEndpoints( assertEquals(expectedSearchInferenceId, semanticFieldType.getSearchInferenceId()); } - protected SemanticIndexOptions getDefaultIndexOptions(Model model, MapperService mapperService) { + protected SemanticIndexOptions getDefaultIndexOptions(MinimalServiceSettings modelSettings, MapperService mapperService) { IndexVersion indexVersion = mapperService.getIndexSettings().getIndexVersionCreated(); boolean experimentalFeatures = DENSE_VECTOR_EXPERIMENTAL_FEATURES_SETTING.get(mapperService.getIndexSettings().getSettings()); - TaskType taskType = model.getTaskType(); - DenseVectorFieldMapper.ElementType elementType = model.getServiceSettings().elementType(); - Integer dimensions = model.getServiceSettings().dimensions(); + TaskType taskType = modelSettings.taskType(); + DenseVectorFieldMapper.ElementType elementType = modelSettings.elementType(); + Integer dimensions = modelSettings.dimensions(); return switch (taskType) { case TEXT_EMBEDDING, EMBEDDING -> { - var denseDefaults = getExplicitDenseVectorIndexOptions(model, indexVersion); + var denseDefaults = getExplicitDenseVectorIndexOptions(modelSettings, indexVersion); if (denseDefaults == null) { denseDefaults = defaultDenseVectorIndexOptions( indexVersion, @@ -745,7 +764,10 @@ protected SemanticIndexOptions getDefaultIndexOptions(Model model, MapperService }; } - protected DenseVectorFieldMapper.DenseVectorIndexOptions getExplicitDenseVectorIndexOptions(Model model, IndexVersion indexVersion) { + protected DenseVectorFieldMapper.DenseVectorIndexOptions getExplicitDenseVectorIndexOptions( + MinimalServiceSettings modelSettings, + IndexVersion indexVersion + ) { return null; } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticFieldMapperTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticFieldMapperTests.java index 4c9724ef1a1cb..f6a1f12199efd 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticFieldMapperTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticFieldMapperTests.java @@ -304,7 +304,7 @@ public void testSemanticFieldSupportedOnNewIndices() throws IOException { newVersion ); assertNotNull(mapperService); - assertSemanticField(mapperService, "my_field", false, TEST_MODEL, null, null); + assertSemanticField(mapperService, "my_field", false, new MinimalServiceSettings(TEST_MODEL), null, null); } } @@ -327,7 +327,7 @@ public void testSemanticFieldMappingUpdateSupportedOnNewIndices() throws IOExcep // Should not throw; model_settings provided to avoid consulting the model registry merge(mapperService, semanticMapping("my_field", TEST_MODEL.getInferenceEntityId())); - assertSemanticField(mapperService, "my_field", false, TEST_MODEL, null, null); + assertSemanticField(mapperService, "my_field", false, new MinimalServiceSettings(TEST_MODEL), null, null); } @Override diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java index 3fb79f76e62b7..e3812d0ec143c 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java @@ -300,10 +300,13 @@ protected void assertSearchable(MappedFieldType fieldType) { } @Override - protected DenseVectorFieldMapper.DenseVectorIndexOptions getExplicitDenseVectorIndexOptions(Model model, IndexVersion indexVersion) { - TaskType taskType = model.getTaskType(); - DenseVectorFieldMapper.ElementType elementType = model.getServiceSettings().elementType(); - Integer dimensions = model.getServiceSettings().dimensions(); + protected DenseVectorFieldMapper.DenseVectorIndexOptions getExplicitDenseVectorIndexOptions( + MinimalServiceSettings modelSettings, + IndexVersion indexVersion + ) { + TaskType taskType = modelSettings.taskType(); + DenseVectorFieldMapper.ElementType elementType = modelSettings.elementType(); + Integer dimensions = modelSettings.dimensions(); if (taskType == TaskType.SPARSE_EMBEDDING) { return null; @@ -333,7 +336,7 @@ public void testDefaults() throws Exception { MapperService mapperService = createSemanticMapperService(fieldMapping, IndexVersions.SEMANTIC_TEXT_DEFAULTS_TO_JINA_V5); DocumentMapper mapper = mapperService.documentMapper(); assertEquals(Strings.toString(expectedMapping), mapper.mappingSource().toString()); - assertSemanticField(mapperService, fieldName, false, JINA_V5_TEXT_MODEL, null, null); + assertSemanticField(mapperService, fieldName, false, new MinimalServiceSettings(JINA_V5_TEXT_MODEL), null, null); assertInferenceEndpoints(mapperService, fieldName, DEFAULT_EIS_JINA_V5_INFERENCE_ID, DEFAULT_EIS_JINA_V5_INFERENCE_ID); ParsedDocument doc1 = mapper.parse(source(this::writeField)); @@ -396,7 +399,7 @@ public void testDefaults() throws Exception { } MapperService iterMapperService = createSemanticMapperServiceWithIndexVersion(fieldMapping, indexVersion); - assertSemanticField(iterMapperService, fieldName, false, expectedModel, null, null); + assertSemanticField(iterMapperService, fieldName, false, new MinimalServiceSettings(expectedModel), null, null); assertInferenceEndpoints( iterMapperService, fieldName, @@ -622,7 +625,7 @@ public void testSetInferenceEndpoints() throws IOException { final XContentBuilder fieldMapping = semanticMapping("field", null, searchInferenceId); final XContentBuilder expectedMapping = semanticMapping("field", DEFAULT_EIS_JINA_V5_INFERENCE_ID, searchInferenceId); final MapperService mapperService = createSemanticMapperService(fieldMapping, IndexVersions.SEMANTIC_TEXT_DEFAULTS_TO_JINA_V5); - assertSemanticField(mapperService, fieldName, false, JINA_V5_TEXT_MODEL, null, null); + assertSemanticField(mapperService, fieldName, false, new MinimalServiceSettings(JINA_V5_TEXT_MODEL), null, null); assertInferenceEndpoints(mapperService, fieldName, DEFAULT_EIS_JINA_V5_INFERENCE_ID, searchInferenceId); assertSerialization.accept(expectedMapping, mapperService); } @@ -727,7 +730,7 @@ public void testMultiFieldsSupport() throws IOException { addSemanticMapping(b, "semantic", "my_inference_id", null, new MinimalServiceSettings(sparseModel), null, null); b.endObject(); })); - assertSemanticField(mapperService, "field.semantic", true, sparseModel, null, null); + assertSemanticField(mapperService, "field.semantic", true, new MinimalServiceSettings(sparseModel), null, null); mapperService = createSemanticMapperService( semanticMapping("field", "my_inference_id", null, new MinimalServiceSettings(sparseModel), null, null, b -> { @@ -736,7 +739,7 @@ public void testMultiFieldsSupport() throws IOException { b.endObject(); }) ); - assertSemanticField(mapperService, "field", true, sparseModel, null, null); + assertSemanticField(mapperService, "field", true, new MinimalServiceSettings(sparseModel), null, null); mapperService = createSemanticMapperService( semanticMapping("field", "my_inference_id", null, new MinimalServiceSettings(sparseModel), null, null, b -> { @@ -745,8 +748,8 @@ public void testMultiFieldsSupport() throws IOException { b.endObject(); }) ); - assertSemanticField(mapperService, "field", true, sparseModel, null, null); - assertSemanticField(mapperService, "field.semantic", true, sparseModel, null, null); + assertSemanticField(mapperService, "field", true, new MinimalServiceSettings(sparseModel), null, null); + assertSemanticField(mapperService, "field.semantic", true, new MinimalServiceSettings(sparseModel), null, null); Exception e = expectThrows( MapperParsingException.class, @@ -764,30 +767,19 @@ public void testDynamicUpdate() throws IOException { final String fieldName = "semantic"; final String inferenceId = "test_service"; final String searchInferenceId = "search_test_service"; + final MinimalServiceSettings sparseModelSettings = createRandomModelSettings(TaskType.SPARSE_EMBEDDING); { - Model model = TestModel.createRandomInstance(TaskType.SPARSE_EMBEDDING); MapperService mapperService = createSemanticMapperService(semanticMapping(fieldName, inferenceId)); - performDynamicUpdate( - mapperService, - fieldName, - inferenceId, - new MinimalServiceSettings("service", TaskType.SPARSE_EMBEDDING, null, null, null) - ); - assertSemanticField(mapperService, fieldName, true, model, null, null); + performDynamicUpdate(mapperService, fieldName, inferenceId, sparseModelSettings); + assertSemanticField(mapperService, fieldName, true, sparseModelSettings, null, null); assertInferenceEndpoints(mapperService, fieldName, inferenceId, inferenceId); } { - Model model = TestModel.createRandomInstance(TaskType.SPARSE_EMBEDDING); MapperService mapperService = createSemanticMapperService(semanticMapping(fieldName, inferenceId, searchInferenceId)); - performDynamicUpdate( - mapperService, - fieldName, - inferenceId, - new MinimalServiceSettings("service", TaskType.SPARSE_EMBEDDING, null, null, null) - ); - assertSemanticField(mapperService, fieldName, true, model, null, null); + performDynamicUpdate(mapperService, fieldName, inferenceId, sparseModelSettings); + assertSemanticField(mapperService, fieldName, true, sparseModelSettings, null, null); assertInferenceEndpoints(mapperService, fieldName, inferenceId, searchInferenceId); } } @@ -795,6 +787,8 @@ public void testDynamicUpdate() throws IOException { public void testUpdateModelSettings() throws IOException { for (int depth = 1; depth < 5; depth++) { String fieldName = randomFieldName(depth); + MinimalServiceSettings sparseSettings = createRandomModelSettings(TaskType.SPARSE_EMBEDDING); + MinimalServiceSettings denseSettings = createRandomModelSettings(TaskType.TEXT_EMBEDDING); MapperService mapperService = createSemanticMapperService(semanticMapping(fieldName, "test_model")); assertSemanticField(mapperService, fieldName, false, null, null, null); { @@ -809,35 +803,17 @@ public void testUpdateModelSettings() throws IOException { assertThat(exc.getMessage(), containsString("Required [task_type]")); } { - Model sparseModel = TestModel.createRandomInstance(TaskType.SPARSE_EMBEDDING); - merge( - mapperService, - semanticMapping(fieldName, "test_model", new MinimalServiceSettings(null, TaskType.SPARSE_EMBEDDING, null, null, null)) - ); - assertSemanticField(mapperService, fieldName, true, sparseModel, null, null); + merge(mapperService, semanticMapping(fieldName, "test_model", sparseSettings)); + assertSemanticField(mapperService, fieldName, true, sparseSettings, null, null); } { - Model sparseModel = TestModel.createRandomInstance(TaskType.SPARSE_EMBEDDING); merge(mapperService, semanticMapping(fieldName, "test_model")); - assertSemanticField(mapperService, fieldName, true, sparseModel, null, null); + assertSemanticField(mapperService, fieldName, true, sparseSettings, null, null); } { Exception exc = expectThrows( IllegalArgumentException.class, - () -> merge( - mapperService, - semanticMapping( - fieldName, - "test_model", - new MinimalServiceSettings( - null, - TaskType.TEXT_EMBEDDING, - 10, - SimilarityMeasure.COSINE, - DenseVectorFieldMapper.ElementType.FLOAT - ) - ) - ) + () -> merge(mapperService, semanticMapping(fieldName, "test_model", denseSettings)) ); assertThat(exc.getMessage(), containsString("cannot be changed from type [sparse_vector] to [dense_vector]")); } @@ -884,6 +860,7 @@ public void testSparseVectorIndexOptionsValidationAndMapping() throws IOExceptio for (int depth = 1; depth < 5; depth++) { String inferenceId = "test_model"; String fieldName = randomFieldName(depth); + MinimalServiceSettings sparseSettings = createRandomModelSettings(TaskType.SPARSE_EMBEDDING); var sparseVectorIndexOptions = SparseVectorFieldTypeTests.randomSparseVectorIndexOptions(); var expectedIndexOptions = sparseVectorIndexOptions == null ? null @@ -891,24 +868,10 @@ public void testSparseVectorIndexOptionsValidationAndMapping() throws IOExceptio // should not throw an exception MapperService mapper = createSemanticMapperService( - semanticMapping( - fieldName, - inferenceId, - null, - new MinimalServiceSettings(null, TaskType.SPARSE_EMBEDDING, null, null, null), - null, - expectedIndexOptions - ) + semanticMapping(fieldName, inferenceId, null, sparseSettings, null, expectedIndexOptions) ); - assertSemanticField( - mapper, - fieldName, - true, - TestModel.createRandomInstance(TaskType.SPARSE_EMBEDDING), - null, - expectedIndexOptions - ); + assertSemanticField(mapper, fieldName, true, sparseSettings, null, expectedIndexOptions); } } @@ -926,7 +889,7 @@ public void testSparseVectorMappingUpdate() throws IOException { MapperService mapperService = createSemanticMapperService( semanticMapping(fieldName, model.getInferenceEntityId(), null, null, chunkingSettings, indexOptions) ); - assertSemanticField(mapperService, fieldName, false, model, chunkingSettings, indexOptions); + assertSemanticField(mapperService, fieldName, false, new MinimalServiceSettings(model), chunkingSettings, indexOptions); final SemanticIndexOptions newIndexOptions = randomSemanticIndexOptions(TaskType.SPARSE_EMBEDDING); final ChunkingSettings newChunkingSettings = generateRandomChunkingSettingsOtherThan(chunkingSettings); @@ -934,7 +897,7 @@ public void testSparseVectorMappingUpdate() throws IOException { mapperService, semanticMapping(fieldName, model.getInferenceEntityId(), null, null, newChunkingSettings, newIndexOptions) ); - assertSemanticField(mapperService, fieldName, false, model, newChunkingSettings, newIndexOptions); + assertSemanticField(mapperService, fieldName, false, new MinimalServiceSettings(model), newChunkingSettings, newIndexOptions); } } @@ -961,27 +924,21 @@ public void testUpdateSearchInferenceId() throws IOException { assertSemanticField(mapperService, fieldName, false, null, null, null); assertInferenceEndpoints(mapperService, fieldName, inferenceId, inferenceId); - Model sparseModel = TestModel.createRandomInstance(TaskType.SPARSE_EMBEDDING); - mapperService = createSemanticMapperService( - semanticMapping( - fieldName, - inferenceId, - new MinimalServiceSettings("my-service", TaskType.SPARSE_EMBEDDING, null, null, null) - ) - ); - assertSemanticField(mapperService, fieldName, true, sparseModel, null, null); + MinimalServiceSettings sparseSettings = createRandomModelSettings(TaskType.SPARSE_EMBEDDING); + mapperService = createSemanticMapperService(semanticMapping(fieldName, inferenceId, sparseSettings)); + assertSemanticField(mapperService, fieldName, true, sparseSettings, null, null); assertInferenceEndpoints(mapperService, fieldName, inferenceId, inferenceId); merge(mapperService, semanticMapping(fieldName, inferenceId, searchInferenceId1)); - assertSemanticField(mapperService, fieldName, true, sparseModel, null, null); + assertSemanticField(mapperService, fieldName, true, sparseSettings, null, null); assertInferenceEndpoints(mapperService, fieldName, inferenceId, searchInferenceId1); merge(mapperService, semanticMapping(fieldName, inferenceId, searchInferenceId2)); - assertSemanticField(mapperService, fieldName, true, sparseModel, null, null); + assertSemanticField(mapperService, fieldName, true, sparseSettings, null, null); assertInferenceEndpoints(mapperService, fieldName, inferenceId, searchInferenceId2); merge(mapperService, semanticMapping(fieldName, inferenceId)); - assertSemanticField(mapperService, fieldName, true, sparseModel, null, null); + assertSemanticField(mapperService, fieldName, true, sparseSettings, null, null); assertInferenceEndpoints(mapperService, fieldName, inferenceId, inferenceId); } } @@ -1029,10 +986,10 @@ protected void assertChunksTextField(SemanticTextFieldMapper.SemanticTextFieldTy protected void assertEmbeddingsField( MapperService mapperService, FieldMapper embeddingsMapper, - Model model, + MinimalServiceSettings modelSettings, @Nullable SemanticIndexOptions expectedIndexOptions ) { - if (model.getTaskType() == TaskType.SPARSE_EMBEDDING) { + if (modelSettings.taskType() == TaskType.SPARSE_EMBEDDING) { assertThat(embeddingsMapper, instanceOf(SparseVectorFieldMapper.class)); SparseVectorFieldMapper sparseVectorFieldMapper = (SparseVectorFieldMapper) embeddingsMapper; assertEquals(sparseVectorFieldMapper.fieldType().isStored(), useLegacyFormat() == false); @@ -1040,7 +997,7 @@ protected void assertEmbeddingsField( IndexOptions expectedBaseIndexOptions = expectedIndexOptions == null ? null : expectedIndexOptions.indexOptions(); assertEquals(expectedBaseIndexOptions, sparseVectorFieldMapper.fieldType().getIndexOptions()); } else { - super.assertEmbeddingsField(mapperService, embeddingsMapper, model, expectedIndexOptions); + super.assertEmbeddingsField(mapperService, embeddingsMapper, modelSettings, expectedIndexOptions); } } @@ -1092,14 +1049,14 @@ public void testSuccessfulParse() throws IOException { }); MapperService mapperService = createSemanticMapperServiceWithIndexVersion(mapping, indexVersion); - assertSemanticField(mapperService, fieldName1, false, model1, null, indexOptions); + assertSemanticField(mapperService, fieldName1, false, new MinimalServiceSettings(model1), null, indexOptions); assertInferenceEndpoints( mapperService, fieldName1, model1.getInferenceEntityId(), setSearchInferenceId ? searchInferenceId : model1.getInferenceEntityId() ); - assertSemanticField(mapperService, fieldName2, false, model2, null, indexOptions); + assertSemanticField(mapperService, fieldName2, false, new MinimalServiceSettings(model2), null, indexOptions); assertInferenceEndpoints( mapperService, fieldName2, @@ -1213,10 +1170,7 @@ public void testMissingInferenceId() throws IOException { useLegacyFormat, b -> b.startObject("field") .startObject(INFERENCE_FIELD) - .field( - MODEL_SETTINGS_FIELD, - new MinimalServiceSettings("my-service", TaskType.SPARSE_EMBEDDING, null, null, null) - ) + .field(MODEL_SETTINGS_FIELD, createRandomModelSettings(TaskType.SPARSE_EMBEDDING)) .field(CHUNKS_FIELD, useLegacyFormat ? List.of() : Map.of()) .endObject() .endObject() @@ -1378,11 +1332,11 @@ public void testSettingAndUpdatingChunkingSettings() throws IOException { MapperService mapperService = createSemanticMapperService( semanticMapping(fieldName, model.getInferenceEntityId(), null, null, chunkingSettings, indexOptions) ); - assertSemanticField(mapperService, fieldName, false, model, chunkingSettings, indexOptions); + assertSemanticField(mapperService, fieldName, false, new MinimalServiceSettings(model), chunkingSettings, indexOptions); ChunkingSettings newChunkingSettings = generateRandomChunkingSettingsOtherThan(chunkingSettings); merge(mapperService, semanticMapping(fieldName, model.getInferenceEntityId(), null, null, newChunkingSettings, indexOptions)); - assertSemanticField(mapperService, fieldName, false, model, newChunkingSettings, indexOptions); + assertSemanticField(mapperService, fieldName, false, new MinimalServiceSettings(model), newChunkingSettings, indexOptions); } public void testModelSettingsRequiredWithChunks() throws IOException { @@ -1432,6 +1386,7 @@ public void testModelSettingsRequiredWithChunks() throws IOException { public void testPre811IndexSemanticTextDenseVectorRaisesError() throws IOException { assumeTrue("Test only applies to legacy format", useLegacyFormat); Model model = TestModel.createRandomInstance(TaskType.TEXT_EMBEDDING); + MinimalServiceSettings modelSettings = new MinimalServiceSettings(model); String fieldName = randomAlphaOfLength(8); MapperService mapperService = createSemanticMapperService( @@ -1441,8 +1396,8 @@ public void testPre811IndexSemanticTextDenseVectorRaisesError() throws IOExcepti ); assertSemanticField(mapperService, fieldName, false, null, null, null); - merge(mapperService, semanticMapping(fieldName, model.getInferenceEntityId(), new MinimalServiceSettings(model))); - assertSemanticField(mapperService, fieldName, true, model, null, null); + merge(mapperService, semanticMapping(fieldName, model.getInferenceEntityId(), modelSettings)); + assertSemanticField(mapperService, fieldName, true, modelSettings, null, null); DocumentMapper documentMapper = mapperService.documentMapper(); DocumentParsingException e = assertThrows( @@ -1464,6 +1419,7 @@ public void testPre811IndexSemanticTextDenseVectorRaisesError() throws IOExcepti public void testPre811IndexSemanticTextSparseVectorRaisesError() throws IOException { assumeTrue("Test only applies to legacy format", useLegacyFormat); Model model = TestModel.createRandomInstance(TaskType.SPARSE_EMBEDDING); + MinimalServiceSettings modelSettings = new MinimalServiceSettings(model); String fieldName = randomAlphaOfLength(8); MapperService mapperService = createSemanticMapperService( @@ -1473,8 +1429,8 @@ public void testPre811IndexSemanticTextSparseVectorRaisesError() throws IOExcept ); assertSemanticField(mapperService, fieldName, false, null, null, null); - merge(mapperService, semanticMapping(fieldName, model.getInferenceEntityId(), new MinimalServiceSettings(model))); - assertSemanticField(mapperService, fieldName, true, model, null, null); + merge(mapperService, semanticMapping(fieldName, model.getInferenceEntityId(), modelSettings)); + assertSemanticField(mapperService, fieldName, true, modelSettings, null, null); DocumentMapper documentMapper = mapperService.documentMapper(); DocumentParsingException e = assertThrows( @@ -1498,7 +1454,7 @@ public void testExistsQuerySparseVector() throws IOException { final String inferenceId = "test_service"; MapperService mapperService = createSemanticMapperService( - semanticMapping(fieldName, inferenceId, new MinimalServiceSettings("my-service", TaskType.SPARSE_EMBEDDING, null, null, null)) + semanticMapping(fieldName, inferenceId, createRandomModelSettings(TaskType.SPARSE_EMBEDDING)) ); SearchExecutionContext searchExecutionContext = createSearchExecutionContext(mapperService); @@ -1511,17 +1467,7 @@ public void testExistsQueryDenseVector() throws IOException { final String inferenceId = "test_service"; MapperService mapperService = createSemanticMapperService( - semanticMapping( - fieldName, - inferenceId, - new MinimalServiceSettings( - "my-service", - TaskType.TEXT_EMBEDDING, - 1024, - SimilarityMeasure.COSINE, - DenseVectorFieldMapper.ElementType.FLOAT - ) - ) + semanticMapping(fieldName, inferenceId, createRandomModelSettings(TaskType.TEXT_EMBEDDING)) ); SearchExecutionContext searchExecutionContext = createSearchExecutionContext(mapperService); @@ -1595,7 +1541,7 @@ public void testDefaultIndexOptions() throws IOException { semanticMapping("field", "another_inference_id", new MinimalServiceSettings(model)), indexVersion ); - assertSemanticField(mapperService, "field", true, model, null, null); + assertSemanticField(mapperService, "field", true, new MinimalServiceSettings(model), null, null); } } @@ -1615,7 +1561,7 @@ public void testSpecifiedDenseVectorIndexOptions() throws IOException { semanticMapping("field", "another_inference_id", null, new MinimalServiceSettings(denseModel), null, expectedIndexOptions), indexVersion ); - assertSemanticField(mapperService, "field", true, denseModel, null, expectedIndexOptions); + assertSemanticField(mapperService, "field", true, new MinimalServiceSettings(denseModel), null, expectedIndexOptions); // Specifying partial index options will fill in the remainder index options with defaults mapperService = createSemanticMapperServiceWithIndexVersion( @@ -1633,7 +1579,7 @@ public void testSpecifiedDenseVectorIndexOptions() throws IOException { mapperService, "field", true, - denseModel, + new MinimalServiceSettings(denseModel), null, new SemanticIndexOptions( SemanticIndexOptions.SupportedIndexOptions.DENSE_VECTOR, @@ -1652,7 +1598,7 @@ public void testSpecifiedDenseVectorIndexOptions() throws IOException { "field", "another_inference_id", null, - new MinimalServiceSettings(null, TaskType.SPARSE_EMBEDDING, null, null, null), + createRandomModelSettings(TaskType.SPARSE_EMBEDDING), null, null, b -> { @@ -1720,7 +1666,8 @@ public void testSetElementTypeInDenseVectorIndexOptions() throws IOException { indexVersion ); - SemanticIndexOptions expectedDefaultIndexOptions = getDefaultIndexOptions(denseModel, mapperService); + MinimalServiceSettings denseModelSettings = new MinimalServiceSettings(denseModel); + SemanticIndexOptions expectedDefaultIndexOptions = getDefaultIndexOptions(denseModelSettings, mapperService); DenseVectorFieldMapper.DenseVectorIndexOptions expectedDenseVectorIndexOptions = expectedDefaultIndexOptions != null ? ((ExtendedDenseVectorIndexOptions) expectedDefaultIndexOptions.indexOptions()).getBaseIndexOptions() : null; @@ -1729,7 +1676,7 @@ public void testSetElementTypeInDenseVectorIndexOptions() throws IOException { mapperService, "field", true, - denseModel, + denseModelSettings, null, new SemanticIndexOptions( SemanticIndexOptions.SupportedIndexOptions.DENSE_VECTOR, @@ -1757,7 +1704,7 @@ public void testSetElementTypeInDenseVectorIndexOptions() throws IOException { indexVersion ); - assertSemanticField(mapperService, "field", true, denseModel, null, int4HnswWithFloatIndexOptions); + assertSemanticField(mapperService, "field", true, new MinimalServiceSettings(denseModel), null, int4HnswWithFloatIndexOptions); } public void testInvalidElementTypeOverride() { @@ -1818,7 +1765,7 @@ public void testSpecificSparseVectorIndexOptions() throws IOException { mapperService, "field", true, - sparseModel, + new MinimalServiceSettings(sparseModel), null, new SemanticIndexOptions(SemanticIndexOptions.SupportedIndexOptions.SPARSE_VECTOR, testIndexOptions) ); From 47204e873e577799c360f3e6083e69533cf8ac18 Mon Sep 17 00:00:00 2001 From: Mike Pellegrini Date: Sat, 25 Jul 2026 08:59:56 -0400 Subject: [PATCH 24/24] Use random model settings instead of models --- .../AbstractSemanticMapperTestCase.java | 115 ++++++------------ .../mapper/SemanticTextFieldMapperTests.java | 93 +++++++------- 2 files changed, 76 insertions(+), 132 deletions(-) diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/AbstractSemanticMapperTestCase.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/AbstractSemanticMapperTestCase.java index 757ab4980fb16..8e7f74d35d560 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/AbstractSemanticMapperTestCase.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/AbstractSemanticMapperTestCase.java @@ -264,42 +264,28 @@ public void testUpdateInferenceId_GivenNoModelSettings() throws IOException { String fieldName = randomAlphaOfLengthBetween(5, 15); String oldInferenceId = randomAlphaOfLengthBetween(5, 15); - TestModel oldModel = null; + MinimalServiceSettings oldModelSettings = null; if (randomBoolean()) { - oldModel = createRandomSupportedModel(); - givenModelSettings(oldInferenceId, new MinimalServiceSettings(oldModel)); + oldModelSettings = createRandomModelSettings(); + givenModelSettings(oldInferenceId, oldModelSettings); } var mapperService = createSemanticMapperService(semanticMapping(fieldName, oldInferenceId)); assertInferenceEndpoints(mapperService, fieldName, oldInferenceId, oldInferenceId); - assertSemanticField( - mapperService, - fieldName, - false, - oldModel == null ? null : new MinimalServiceSettings(oldModel), - null, - null - ); + assertSemanticField(mapperService, fieldName, false, oldModelSettings, null, null); String newInferenceId = randomValueOtherThan(oldInferenceId, () -> randomAlphaOfLengthBetween(5, 15)); - TestModel newModel = null; + MinimalServiceSettings newModelSettings = null; if (randomBoolean()) { - newModel = createRandomSupportedModel(); - givenModelSettings(newInferenceId, new MinimalServiceSettings(newModel)); + newModelSettings = createRandomModelSettings(); + givenModelSettings(newInferenceId, newModelSettings); } merge(mapperService, semanticMapping(fieldName, newInferenceId)); assertInferenceEndpoints(mapperService, fieldName, newInferenceId, newInferenceId); - assertSemanticField( - mapperService, - fieldName, - false, - newModel == null ? null : new MinimalServiceSettings(newModel), - null, - null - ); + assertSemanticField(mapperService, fieldName, false, newModelSettings, null, null); } } @@ -309,8 +295,7 @@ public void testUpdateInferenceId_GivenModelSettings() throws IOException { final String oldInferenceId = randomAlphaOfLengthBetween(5, 15); final String newInferenceId = randomValueOtherThan(oldInferenceId, () -> randomAlphaOfLengthBetween(5, 15)); - final TestModel oldModel = createRandomSupportedModel(); - final MinimalServiceSettings previousModelSettings = new MinimalServiceSettings(oldModel); + final MinimalServiceSettings previousModelSettings = createRandomModelSettings(); givenModelSettings(oldInferenceId, previousModelSettings); final MapperService mapperService = createSemanticMapperService( @@ -323,17 +308,16 @@ public void testUpdateInferenceId_GivenModelSettings() throws IOException { if (randomBoolean()) { // Compatible: new endpoint has identical task type / dimensions / similarity / element type - TestModel newModel = createCompatibleModel(newInferenceId, oldModel); - MinimalServiceSettings newModelSettings = new MinimalServiceSettings(newModel); + MinimalServiceSettings newModelSettings = createCompatibleModelSettings(previousModelSettings); givenModelSettings(newInferenceId, newModelSettings); mergeRunner.run(); assertInferenceEndpoints(mapperService, fieldName, newInferenceId, newInferenceId); assertSemanticField(mapperService, fieldName, true, newModelSettings, null, null); } else { - final TestModel incompatibleModel = createIncompatibleModel(newInferenceId, oldModel); + final MinimalServiceSettings incompatibleModelSettings = createIncompatibleModelSettings(previousModelSettings); final String expectedErrorMessage; - if (incompatibleModel == null) { + if (incompatibleModelSettings == null) { // Incompatible: new endpoint does not exist expectedErrorMessage = "Cannot update [" + contentType() @@ -344,7 +328,6 @@ public void testUpdateInferenceId_GivenModelSettings() throws IOException { + "] does not exist."; } else { // Incompatible: new endpoint exists but its model settings are incompatible - MinimalServiceSettings incompatibleModelSettings = new MinimalServiceSettings(incompatibleModel); givenModelSettings(newInferenceId, incompatibleModelSettings); expectedErrorMessage = "Cannot update [" @@ -776,45 +759,38 @@ protected void givenModelSettings(String inferenceId, MinimalServiceSettings mod } /** - * Creates a {@link TestModel} that is compatible with {@code baseModel} (same task type, dimensions, - * similarity, and element type) but with a distinct service, task settings, and secrets, registered - * under the given inference ID. Compatible models can be substituted via an inference-ID update. + * Creates a {@link MinimalServiceSettings} that is compatible with {@code base} (same task type, + * dimensions, similarity, and element type) but with a distinct service name. Compatible settings + * can be substituted via an inference-ID update. */ - protected TestModel createCompatibleModel(String inferenceId, TestModel baseModel) { - return new TestModel( - inferenceId, - baseModel.getTaskType(), + protected MinimalServiceSettings createCompatibleModelSettings(MinimalServiceSettings base) { + return new MinimalServiceSettings( randomAlphaOfLength(4), - new TestModel.TestServiceSettings( - randomAlphaOfLength(4), - baseModel.getServiceSettings().dimensions(), - baseModel.getServiceSettings().similarity(), - baseModel.getServiceSettings().elementType() - ), - new TestModel.TestTaskSettings(randomInt(3)), - new TestModel.TestSecretSettings(randomAlphaOfLength(4)) + base.taskType(), + base.dimensions(), + base.similarity(), + base.elementType() ); } /** - * Creates a {@link TestModel} that is NOT compatible with {@code baseModel}, choosing uniformly among the + * Creates a {@link MinimalServiceSettings} that is NOT compatible with {@code base}, choosing uniformly among the * applicable kinds of incompatibility: task type, dimensions, similarity, element type, or the endpoint not * existing. Setting-based perturbations (dimensions/similarity/element type) only apply to dense base models; * a sparse base model has null dimensions/similarity/element type, so only task-type and does-not-exist apply. * - * @return an incompatible model to register under {@code inferenceId}, or {@code null} to indicate the - * caller should NOT register an endpoint (the does-not-exist case). + * @return new incompatible settings, or {@code null} to indicate the caller should NOT register an endpoint (the does-not-exist case). */ - protected TestModel createIncompatibleModel(String inferenceId, TestModel baseModel) { - final TestModel.TestServiceSettings baseServiceSettings = baseModel.getServiceSettings(); - final DenseVectorFieldMapper.ElementType baseElementType = baseServiceSettings.elementType(); + @Nullable + protected MinimalServiceSettings createIncompatibleModelSettings(MinimalServiceSettings base) { + final DenseVectorFieldMapper.ElementType baseElementType = base.elementType(); List applicable = new ArrayList<>(); if (supportedTaskTypes().size() > 1) { applicable.add(IncompatibilityKind.TASK_TYPE); } applicable.add(IncompatibilityKind.DOES_NOT_EXIST); - if (baseModel.getTaskType() != TaskType.SPARSE_EMBEDDING) { + if (base.taskType() != TaskType.SPARSE_EMBEDDING) { applicable.add(IncompatibilityKind.DIMENSIONS); applicable.add(IncompatibilityKind.ELEMENT_TYPE); // SIMILARITY only when the element type supports more than one option to perturb to @@ -823,17 +799,18 @@ protected TestModel createIncompatibleModel(String inferenceId, TestModel baseMo } } - TaskType taskType = baseModel.getTaskType(); - Integer dimensions = baseServiceSettings.dimensions(); - SimilarityMeasure similarity = baseServiceSettings.similarity(); + TaskType taskType = base.taskType(); + Integer dimensions = base.dimensions(); + SimilarityMeasure similarity = base.similarity(); DenseVectorFieldMapper.ElementType elementType = baseElementType; - boolean returnNull = false; switch (randomFrom(applicable)) { - case DOES_NOT_EXIST -> returnNull = true; + case DOES_NOT_EXIST -> { + return null; + } case TASK_TYPE -> { taskType = randomValueOtherThan(taskType, () -> randomFrom(supportedTaskTypes())); - if (taskType != TaskType.SPARSE_EMBEDDING && baseModel.getTaskType() == TaskType.SPARSE_EMBEDDING) { + if (taskType != TaskType.SPARSE_EMBEDDING && base.taskType() == TaskType.SPARSE_EMBEDDING) { // Sparse -> dense: populate required dense settings from a fresh random dense model TestModel randomDenseModel = TestModel.createRandomInstance(taskType); dimensions = randomDenseModel.getServiceSettings().dimensions(); @@ -866,29 +843,7 @@ protected TestModel createIncompatibleModel(String inferenceId, TestModel baseMo } } - if (returnNull) { - return null; - } - - return buildModel(baseModel, inferenceId, taskType, elementType, dimensions, similarity); - } - - protected static TestModel buildModel( - TestModel baseModel, - String inferenceId, - TaskType taskType, - @Nullable DenseVectorFieldMapper.ElementType elementType, - @Nullable Integer dimensions, - @Nullable SimilarityMeasure similarity - ) { - return new TestModel( - inferenceId, - taskType, - baseModel.getConfigurations().getService(), - new TestModel.TestServiceSettings(baseModel.getServiceSettings().model(), dimensions, similarity, elementType), - baseModel.getTaskSettings(), - baseModel.getSecretSettings() - ); + return new MinimalServiceSettings(base.service(), taskType, dimensions, similarity, elementType); } protected static String randomFieldName(int numLevel) { diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java index e3812d0ec143c..5f05c825221a7 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java @@ -723,33 +723,33 @@ public void testMultiFieldsSupport() throws IOException { }))); assertThat(e.getMessage(), containsString("Field [semantic] of type [semantic_text] can't be used in multifields")); } else { - Model sparseModel = TestModel.createRandomInstance(TaskType.SPARSE_EMBEDDING); + MinimalServiceSettings sparseModelSettings = createRandomModelSettings(TaskType.SPARSE_EMBEDDING); var mapperService = createSemanticMapperService(fieldMapping(b -> { b.field("type", "text"); b.startObject("fields"); - addSemanticMapping(b, "semantic", "my_inference_id", null, new MinimalServiceSettings(sparseModel), null, null); + addSemanticMapping(b, "semantic", "my_inference_id", null, sparseModelSettings, null, null); b.endObject(); })); - assertSemanticField(mapperService, "field.semantic", true, new MinimalServiceSettings(sparseModel), null, null); + assertSemanticField(mapperService, "field.semantic", true, sparseModelSettings, null, null); mapperService = createSemanticMapperService( - semanticMapping("field", "my_inference_id", null, new MinimalServiceSettings(sparseModel), null, null, b -> { + semanticMapping("field", "my_inference_id", null, sparseModelSettings, null, null, b -> { b.startObject("fields"); b.startObject("text").field("type", "text").endObject(); b.endObject(); }) ); - assertSemanticField(mapperService, "field", true, new MinimalServiceSettings(sparseModel), null, null); + assertSemanticField(mapperService, "field", true, sparseModelSettings, null, null); mapperService = createSemanticMapperService( - semanticMapping("field", "my_inference_id", null, new MinimalServiceSettings(sparseModel), null, null, b -> { + semanticMapping("field", "my_inference_id", null, sparseModelSettings, null, null, b -> { b.startObject("fields"); - addSemanticMapping(b, "semantic", "another_inference_id", null, new MinimalServiceSettings(sparseModel), null, null); + addSemanticMapping(b, "semantic", "another_inference_id", null, sparseModelSettings, null, null); b.endObject(); }) ); - assertSemanticField(mapperService, "field", true, new MinimalServiceSettings(sparseModel), null, null); - assertSemanticField(mapperService, "field.semantic", true, new MinimalServiceSettings(sparseModel), null, null); + assertSemanticField(mapperService, "field", true, sparseModelSettings, null, null); + assertSemanticField(mapperService, "field.semantic", true, sparseModelSettings, null, null); Exception e = expectThrows( MapperParsingException.class, @@ -841,21 +841,6 @@ public void testDenseVectorIndexOptionValidation() throws IOException { } } - private static Model denseTextEmbeddingModel( - int dimensions, - SimilarityMeasure similarity, - DenseVectorFieldMapper.ElementType elementType - ) { - return new TestModel( - randomAlphaOfLength(4), - TaskType.TEXT_EMBEDDING, - randomAlphaOfLength(4), - new TestModel.TestServiceSettings(randomAlphaOfLength(4), dimensions, similarity, elementType), - new TestModel.TestTaskSettings(randomInt(3)), - new TestModel.TestSecretSettings(randomAlphaOfLength(4)) - ); - } - public void testSparseVectorIndexOptionsValidationAndMapping() throws IOException { for (int depth = 1; depth < 5; depth++) { String inferenceId = "test_model"; @@ -1532,22 +1517,28 @@ private static DenseVectorFieldMapper.DenseVectorIndexOptions defaultBbqHnswDens public void testDefaultIndexOptions() throws IOException { for (int i = 0; i < 200; i++) { - final Model model = createRandomSupportedModel(); + final MinimalServiceSettings modelSettings = createRandomModelSettings(); final IndexVersion indexVersion = useLegacyFormat == false && randomBoolean() ? IndexVersion.current() : SemanticInferenceMetadataFieldsMapperTests.getRandomCompatibleIndexVersion(useLegacyFormat); MapperService mapperService = createSemanticMapperServiceWithIndexVersion( - semanticMapping("field", "another_inference_id", new MinimalServiceSettings(model)), + semanticMapping("field", "another_inference_id", modelSettings), indexVersion ); - assertSemanticField(mapperService, "field", true, new MinimalServiceSettings(model), null, null); + assertSemanticField(mapperService, "field", true, modelSettings, null, null); } } public void testSpecifiedDenseVectorIndexOptions() throws IOException { IndexVersion indexVersion = SemanticInferenceMetadataFieldsMapperTests.getRandomCompatibleIndexVersion(useLegacyFormat()); - Model denseModel = denseTextEmbeddingModel(100, SimilarityMeasure.COSINE, DenseVectorFieldMapper.ElementType.FLOAT); + MinimalServiceSettings denseModelSettings = new MinimalServiceSettings( + randomAlphaOfLength(4), + TaskType.TEXT_EMBEDDING, + 100, + SimilarityMeasure.COSINE, + DenseVectorFieldMapper.ElementType.FLOAT + ); // Specifying index options will override default index option settings SemanticIndexOptions expectedIndexOptions = new SemanticIndexOptions( @@ -1558,14 +1549,14 @@ public void testSpecifiedDenseVectorIndexOptions() throws IOException { ) ); var mapperService = createSemanticMapperServiceWithIndexVersion( - semanticMapping("field", "another_inference_id", null, new MinimalServiceSettings(denseModel), null, expectedIndexOptions), + semanticMapping("field", "another_inference_id", null, denseModelSettings, null, expectedIndexOptions), indexVersion ); - assertSemanticField(mapperService, "field", true, new MinimalServiceSettings(denseModel), null, expectedIndexOptions); + assertSemanticField(mapperService, "field", true, denseModelSettings, null, expectedIndexOptions); // Specifying partial index options will fill in the remainder index options with defaults mapperService = createSemanticMapperServiceWithIndexVersion( - semanticMapping("field", "another_inference_id", null, new MinimalServiceSettings(denseModel), null, null, b -> { + semanticMapping("field", "another_inference_id", null, denseModelSettings, null, null, b -> { b.startObject("index_options"); b.startObject("dense_vector"); b.field("element_type", "float"); // Explicitly override element type to hold it constant @@ -1579,7 +1570,7 @@ public void testSpecifiedDenseVectorIndexOptions() throws IOException { mapperService, "field", true, - new MinimalServiceSettings(denseModel), + denseModelSettings, null, new SemanticIndexOptions( SemanticIndexOptions.SupportedIndexOptions.DENSE_VECTOR, @@ -1617,7 +1608,7 @@ public void testSpecifiedDenseVectorIndexOptions() throws IOException { e = expectThrows( MapperParsingException.class, () -> createSemanticMapperServiceWithIndexVersion( - semanticMapping("field", "another_inference_id", null, new MinimalServiceSettings(denseModel), null, null, b -> { + semanticMapping("field", "another_inference_id", null, denseModelSettings, null, null, b -> { b.startObject("index_options"); b.startObject("dense_vector"); b.field("type", "bbq_flat"); @@ -1633,7 +1624,7 @@ public void testSpecifiedDenseVectorIndexOptions() throws IOException { e = expectThrows( MapperParsingException.class, () -> createSemanticMapperServiceWithIndexVersion( - semanticMapping("field", "another_inference_id", null, new MinimalServiceSettings(denseModel), null, null, b -> { + semanticMapping("field", "another_inference_id", null, denseModelSettings, null, null, b -> { b.startObject("index_options"); b.startObject("dense_vector"); b.field("type", "invalid"); @@ -1647,7 +1638,13 @@ public void testSpecifiedDenseVectorIndexOptions() throws IOException { } public void testSetElementTypeInDenseVectorIndexOptions() throws IOException { - Model denseModel = denseTextEmbeddingModel(100, SimilarityMeasure.COSINE, DenseVectorFieldMapper.ElementType.FLOAT); + MinimalServiceSettings denseModelSettings = new MinimalServiceSettings( + randomAlphaOfLength(4), + TaskType.TEXT_EMBEDDING, + 100, + SimilarityMeasure.COSINE, + DenseVectorFieldMapper.ElementType.FLOAT + ); // Specifying the element type prevents defaulting to bfloat16 IndexVersion indexVersion = SemanticInferenceMetadataFieldsMapperTests.getRandomCompatibleIndexVersion(useLegacyFormat); @@ -1656,7 +1653,7 @@ public void testSetElementTypeInDenseVectorIndexOptions() throws IOException { "field", "another_inference_id", null, - new MinimalServiceSettings(denseModel), + denseModelSettings, null, new SemanticIndexOptions( SemanticIndexOptions.SupportedIndexOptions.DENSE_VECTOR, @@ -1666,7 +1663,6 @@ public void testSetElementTypeInDenseVectorIndexOptions() throws IOException { indexVersion ); - MinimalServiceSettings denseModelSettings = new MinimalServiceSettings(denseModel); SemanticIndexOptions expectedDefaultIndexOptions = getDefaultIndexOptions(denseModelSettings, mapperService); DenseVectorFieldMapper.DenseVectorIndexOptions expectedDenseVectorIndexOptions = expectedDefaultIndexOptions != null ? ((ExtendedDenseVectorIndexOptions) expectedDefaultIndexOptions.indexOptions()).getBaseIndexOptions() @@ -1693,24 +1689,17 @@ public void testSetElementTypeInDenseVectorIndexOptions() throws IOException { ) ); mapperService = createSemanticMapperServiceWithIndexVersion( - semanticMapping( - "field", - "another_inference_id", - null, - new MinimalServiceSettings(denseModel), - null, - int4HnswWithFloatIndexOptions - ), + semanticMapping("field", "another_inference_id", null, denseModelSettings, null, int4HnswWithFloatIndexOptions), indexVersion ); - assertSemanticField(mapperService, "field", true, new MinimalServiceSettings(denseModel), null, int4HnswWithFloatIndexOptions); + assertSemanticField(mapperService, "field", true, denseModelSettings, null, int4HnswWithFloatIndexOptions); } public void testInvalidElementTypeOverride() { for (int i = 0; i < 10; i++) { - final Model model = TestModel.createRandomInstance(TaskType.TEXT_EMBEDDING); - final DenseVectorFieldMapper.ElementType modelElementType = model.getServiceSettings().elementType(); + final MinimalServiceSettings modelSettings = createRandomModelSettings(TaskType.TEXT_EMBEDDING); + final DenseVectorFieldMapper.ElementType modelElementType = modelSettings.elementType(); final DenseVectorFieldMapper.ElementType overrideElementType; if (modelElementType == DenseVectorFieldMapper.ElementType.FLOAT) { @@ -1726,7 +1715,7 @@ public void testInvalidElementTypeOverride() { "field", "test_inference_id", null, - new MinimalServiceSettings(model), + modelSettings, null, new SemanticIndexOptions( SemanticIndexOptions.SupportedIndexOptions.DENSE_VECTOR, @@ -1748,13 +1737,13 @@ public void testSpecificSparseVectorIndexOptions() throws IOException { for (int i = 0; i < 10; i++) { IndexVersion indexVersion = SemanticInferenceMetadataFieldsMapperTests.getRandomCompatibleIndexVersion(useLegacyFormat()); SparseVectorFieldMapper.SparseVectorIndexOptions testIndexOptions = randomSparseVectorIndexOptions(false); - Model sparseModel = TestModel.createRandomInstance(TaskType.SPARSE_EMBEDDING); + MinimalServiceSettings sparseModelSettings = createRandomModelSettings(TaskType.SPARSE_EMBEDDING); var mapperService = createSemanticMapperServiceWithIndexVersion( semanticMapping( "field", "test_inference_id", null, - new MinimalServiceSettings(sparseModel), + sparseModelSettings, null, new SemanticIndexOptions(SemanticIndexOptions.SupportedIndexOptions.SPARSE_VECTOR, testIndexOptions) ), @@ -1765,7 +1754,7 @@ public void testSpecificSparseVectorIndexOptions() throws IOException { mapperService, "field", true, - new MinimalServiceSettings(sparseModel), + sparseModelSettings, null, new SemanticIndexOptions(SemanticIndexOptions.SupportedIndexOptions.SPARSE_VECTOR, testIndexOptions) );