From 728ed9aad4f59830079251662f495ae8746870c9 Mon Sep 17 00:00:00 2001 From: kishor-kharbas Date: Wed, 15 Jul 2026 11:48:50 -0700 Subject: [PATCH 1/2] new SemanticFieldInferenceFieldsIT test for inference field inclusion/exclusion with semantic field --- .../AbstractInferenceFieldsIT.java | 180 +++++++++++++++ .../integration/IntegrationTestUtils.java | 14 ++ .../SemanticFieldInferenceFieldsIT.java | 172 ++++++++++++++ .../SemanticTextInferenceFieldsIT.java | 209 +++--------------- 4 files changed, 397 insertions(+), 178 deletions(-) create mode 100644 x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/AbstractInferenceFieldsIT.java create mode 100644 x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/SemanticFieldInferenceFieldsIT.java diff --git a/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/AbstractInferenceFieldsIT.java b/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/AbstractInferenceFieldsIT.java new file mode 100644 index 0000000000000..0d381a772a805 --- /dev/null +++ b/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/AbstractInferenceFieldsIT.java @@ -0,0 +1,180 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.integration; + +import org.elasticsearch.action.search.SearchRequest; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.mapper.InferenceMetadataFieldsMapper; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.license.LicenseSettings; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.reindex.ReindexPlugin; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.fetch.subphase.FetchSourceContext; +import org.elasticsearch.search.lookup.SourceFilter; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.xpack.inference.FakeMlPlugin; +import org.elasticsearch.xpack.inference.LocalStateInferencePlugin; +import org.elasticsearch.xpack.inference.mock.TestInferenceServicePlugin; + +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.function.Consumer; + +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse; +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.notNullValue; +import static org.hamcrest.CoreMatchers.nullValue; + +abstract class AbstractInferenceFieldsIT extends ESIntegTestCase { + + @Override + protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { + return Settings.builder().put(LicenseSettings.SELF_GENERATED_LICENSE_TYPE.getKey(), "trial").build(); + } + + @Override + protected Collection> nodePlugins() { + return List.of(LocalStateInferencePlugin.class, TestInferenceServicePlugin.class, ReindexPlugin.class, FakeMlPlugin.class); + } + + @Override + protected boolean forbidPrivateIndexSettings() { + return false; + } + + protected Settings generateIndexSettings(IndexVersion indexVersion) { + int numDataNodes = internalCluster().numDataNodes(); + return Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, indexVersion) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numDataNodes) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .build(); + } + + protected void assertSearchResponse( + String indexName, + QueryBuilder queryBuilder, + Settings indexSettings, + int expectedHits, + @Nullable Consumer searchRequestModifier, + @Nullable Consumer searchResponseValidator + ) throws Exception { + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().query(queryBuilder).size(expectedHits); + SearchRequest searchRequest = new SearchRequest(new String[] { indexName }, searchSourceBuilder); + if (searchRequestModifier != null) { + searchRequestModifier.accept(searchRequest); + } + + ExpectedSource expectedSource = getExpectedSource(indexSettings, searchRequest.source().fetchSource()); + assertResponse(client().search(searchRequest), response -> { + assertThat(response.getSuccessfulShards(), equalTo(response.getTotalShards())); + assertThat(response.getHits().getTotalHits().value(), equalTo((long) expectedHits)); + + for (SearchHit hit : response.getHits()) { + switch (expectedSource) { + case NONE -> assertThat(hit.getSourceAsMap(), nullValue()); + case INFERENCE_FIELDS_EXCLUDED -> { + Map sourceAsMap = hit.getSourceAsMap(); + assertThat(sourceAsMap, notNullValue()); + assertThat(sourceAsMap.containsKey(InferenceMetadataFieldsMapper.NAME), is(false)); + } + case INFERENCE_FIELDS_INCLUDED -> { + Map sourceAsMap = hit.getSourceAsMap(); + assertThat(sourceAsMap, notNullValue()); + assertThat(sourceAsMap.containsKey(InferenceMetadataFieldsMapper.NAME), is(true)); + } + } + } + + if (searchResponseValidator != null) { + searchResponseValidator.accept(response); + } + }); + } + + private static ExpectedSource getExpectedSource(Settings indexSettings, FetchSourceContext fetchSourceContext) { + if (fetchSourceContext != null && fetchSourceContext.fetchSource() == false) { + return ExpectedSource.NONE; + } else if (InferenceMetadataFieldsMapper.isEnabled(indexSettings) == false) { + return ExpectedSource.INFERENCE_FIELDS_EXCLUDED; + } + + if (fetchSourceContext != null) { + SourceFilter filter = fetchSourceContext.filter(); + if (filter != null) { + if (Arrays.asList(filter.getExcludes()).contains(InferenceMetadataFieldsMapper.NAME)) { + return ExpectedSource.INFERENCE_FIELDS_EXCLUDED; + } else if (filter.getIncludes().length > 0) { + return Arrays.asList(filter.getIncludes()).contains(InferenceMetadataFieldsMapper.NAME) + ? ExpectedSource.INFERENCE_FIELDS_INCLUDED + : ExpectedSource.INFERENCE_FIELDS_EXCLUDED; + } + } + + Boolean excludeInferenceFieldsExplicit = fetchSourceContext.excludeInferenceFields(); + if (excludeInferenceFieldsExplicit != null) { + return excludeInferenceFieldsExplicit ? ExpectedSource.INFERENCE_FIELDS_EXCLUDED : ExpectedSource.INFERENCE_FIELDS_INCLUDED; + } + } + + return ExpectedSource.INFERENCE_FIELDS_EXCLUDED; + } + + protected static FetchSourceContext generateRandomFetchSourceContext() { + FetchSourceContext fetchSourceContext = switch (randomIntBetween(0, 4)) { + case 0 -> FetchSourceContext.FETCH_SOURCE; + case 1 -> FetchSourceContext.FETCH_ALL_SOURCE; + case 2 -> FetchSourceContext.FETCH_ALL_SOURCE_EXCLUDE_INFERENCE_FIELDS; + case 3 -> FetchSourceContext.DO_NOT_FETCH_SOURCE; + case 4 -> null; + default -> throw new IllegalStateException("Unhandled randomized case"); + }; + + if (fetchSourceContext != null && fetchSourceContext.fetchSource()) { + String[] includes = null; + String[] excludes = null; + if (randomBoolean()) { + // Randomly include a non-existent field to test explicit inclusion handling + String field = randomBoolean() ? InferenceMetadataFieldsMapper.NAME : randomIdentifier(); + includes = new String[] { field }; + } + if (randomBoolean()) { + // Randomly exclude a non-existent field to test implicit inclusion handling + String field = randomBoolean() ? InferenceMetadataFieldsMapper.NAME : randomIdentifier(); + excludes = new String[] { field }; + } + + if (includes != null || excludes != null) { + fetchSourceContext = FetchSourceContext.of( + fetchSourceContext.fetchSource(), + fetchSourceContext.excludeVectors(), + fetchSourceContext.excludeInferenceFields(), + includes, + excludes + ); + } + } + + return fetchSourceContext; + } + + protected enum ExpectedSource { + NONE, + INFERENCE_FIELDS_EXCLUDED, + INFERENCE_FIELDS_INCLUDED + } +} diff --git a/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/IntegrationTestUtils.java b/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/IntegrationTestUtils.java index e0ecca66b30ba..9d3478f9d3dd3 100644 --- a/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/IntegrationTestUtils.java +++ b/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/IntegrationTestUtils.java @@ -17,6 +17,7 @@ import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.core.inference.action.DeleteInferenceEndpointAction; import org.elasticsearch.xpack.core.inference.action.PutInferenceModelAction; +import org.elasticsearch.xpack.inference.mapper.SemanticFieldMapper; import org.elasticsearch.xpack.inference.mapper.SemanticTextFieldMapper; import org.elasticsearch.xpack.inference.mock.TestDenseInferenceServiceExtension; import org.elasticsearch.xpack.inference.mock.TestSparseInferenceServiceExtension; @@ -101,4 +102,17 @@ public static XContentBuilder generateSemanticTextMapping(Map se return mapping; } + + public static XContentBuilder generateSemanticFieldMapping(Map semanticFields) throws IOException { + XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("properties"); + for (var entry : semanticFields.entrySet()) { + mapping.startObject(entry.getKey()); + mapping.field("type", SemanticFieldMapper.CONTENT_TYPE); + mapping.field("inference_id", entry.getValue()); + mapping.endObject(); + } + mapping.endObject().endObject(); + + return mapping; + } } diff --git a/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/SemanticFieldInferenceFieldsIT.java b/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/SemanticFieldInferenceFieldsIT.java new file mode 100644 index 0000000000000..05780548e3ef4 --- /dev/null +++ b/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/SemanticFieldInferenceFieldsIT.java @@ -0,0 +1,172 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.inference.integration; + +import org.elasticsearch.action.DocWriteResponse; +import org.elasticsearch.common.document.DocumentField; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.inference.TaskType; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.xcontent.XContentBuilder; +import org.junit.After; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.hamcrest.CoreMatchers.is; + +/** + * Integration tests verifying that the {@code _inference_fields} metadata field is correctly + * included or excluded from {@code _source} for the {@code semantic} field type. + * This test covers text and image modalities. + */ +@ESIntegTestCase.ClusterScope(numDataNodes = 3, numClientNodes = 0, supportsDedicatedMasters = false) +public class SemanticFieldInferenceFieldsIT extends AbstractInferenceFieldsIT { + private final String indexName = randomIdentifier(); + private final Map inferenceIds = new HashMap<>(); + + private static final Map EMBEDDING_SERVICE_SETTINGS = Map.of( + "model", + "my_model", + "dimensions", + 256, + "similarity", + "cosine", + "api_key", + "my_api_key" + ); + + // A small JPEG encoded as a base64 data URI for use as a multimodal input. + private static final Map IMAGE_INPUT = Map.of( + "type", + "image", + "value", + "data:image/jpeg;base64,Y2F0IG9uIGEgd2luZG93c2lsbA==" + ); + + @After + public void cleanUp() { + IntegrationTestUtils.deleteIndex(client(), indexName); + for (var entry : inferenceIds.entrySet()) { + IntegrationTestUtils.deleteInferenceEndpoint(client(), entry.getValue(), entry.getKey()); + } + } + + public void testExcludeInferenceFieldsFromSourceWithTextInputs() throws Exception { + final String embeddingInferenceId = randomIdentifier(); + createInferenceEndpoint(TaskType.EMBEDDING, embeddingInferenceId, EMBEDDING_SERVICE_SETTINGS); + final String semanticField = randomIdentifier(); + + for (int i = 0; i < 10; i++) { + final Settings indexSettings = generateIndexSettings(IndexVersion.current()); + XContentBuilder mappings = IntegrationTestUtils.generateSemanticFieldMapping(Map.of(semanticField, embeddingInferenceId)); + assertAcked(prepareCreate(indexName).setSettings(indexSettings).setMapping(mappings)); + + final int docCount = randomIntBetween(10, 50); + indexTextDocuments(semanticField, docCount); + + assertSearchResponse(indexName, matchAllQuery(), indexSettings, docCount, request -> { + request.source().fetchSource(generateRandomFetchSourceContext()).fetchField(semanticField); + }, response -> { + for (SearchHit hit : response.getHits()) { + Map documentFields = hit.getDocumentFields(); + assertThat(documentFields.size(), is(1)); + assertThat(documentFields.containsKey(semanticField), is(true)); + } + }); + + IntegrationTestUtils.deleteIndex(client(), indexName); + } + } + + public void testExcludeInferenceFieldsFromSourceWithImageInputs() throws Exception { + final String embeddingInferenceId = randomIdentifier(); + createInferenceEndpoint(TaskType.EMBEDDING, embeddingInferenceId, EMBEDDING_SERVICE_SETTINGS); + final String semanticField = randomIdentifier(); + + for (int i = 0; i < 10; i++) { + final Settings indexSettings = generateIndexSettings(IndexVersion.current()); + XContentBuilder mappings = IntegrationTestUtils.generateSemanticFieldMapping(Map.of(semanticField, embeddingInferenceId)); + assertAcked(prepareCreate(indexName).setSettings(indexSettings).setMapping(mappings)); + + final int docCount = randomIntBetween(10, 50); + indexImageDocuments(semanticField, docCount); + + assertSearchResponse(indexName, matchAllQuery(), indexSettings, docCount, request -> { + request.source().fetchSource(generateRandomFetchSourceContext()).fetchField(semanticField); + }, response -> { + for (SearchHit hit : response.getHits()) { + Map documentFields = hit.getDocumentFields(); + assertThat(documentFields.size(), is(1)); + assertThat(documentFields.containsKey(semanticField), is(true)); + } + }); + + IntegrationTestUtils.deleteIndex(client(), indexName); + } + } + + public void testExcludeInferenceFieldsFromSourceWithMixedInputs() throws Exception { + final String embeddingInferenceId = randomIdentifier(); + createInferenceEndpoint(TaskType.EMBEDDING, embeddingInferenceId, EMBEDDING_SERVICE_SETTINGS); + final String semanticField = randomIdentifier(); + + for (int i = 0; i < 10; i++) { + final Settings indexSettings = generateIndexSettings(IndexVersion.current()); + XContentBuilder mappings = IntegrationTestUtils.generateSemanticFieldMapping(Map.of(semanticField, embeddingInferenceId)); + assertAcked(prepareCreate(indexName).setSettings(indexSettings).setMapping(mappings)); + + final int textDocCount = randomIntBetween(5, 25); + final int imageDocCount = randomIntBetween(5, 25); + indexTextDocuments(semanticField, textDocCount); + indexImageDocuments(semanticField, imageDocCount); + final int totalDocCount = textDocCount + imageDocCount; + + assertSearchResponse(indexName, matchAllQuery(), indexSettings, totalDocCount, request -> { + request.source().fetchSource(generateRandomFetchSourceContext()).fetchField(semanticField); + }, response -> { + for (SearchHit hit : response.getHits()) { + Map documentFields = hit.getDocumentFields(); + assertThat(documentFields.size(), is(1)); + assertThat(documentFields.containsKey(semanticField), is(true)); + } + }); + + IntegrationTestUtils.deleteIndex(client(), indexName); + } + } + + private void createInferenceEndpoint(TaskType taskType, String inferenceId, Map serviceSettings) throws IOException { + IntegrationTestUtils.createInferenceEndpoint(client(), taskType, inferenceId, serviceSettings); + inferenceIds.put(inferenceId, taskType); + } + + private void indexTextDocuments(String field, int count) { + for (int i = 0; i < count; i++) { + Map source = Map.of(field, randomAlphaOfLength(10)); + DocWriteResponse response = client().prepareIndex(indexName).setSource(source).get(TEST_REQUEST_TIMEOUT); + assertThat(response.getResult(), is(DocWriteResponse.Result.CREATED)); + } + client().admin().indices().prepareRefresh(indexName).get(); + } + + private void indexImageDocuments(String field, int count) { + for (int i = 0; i < count; i++) { + // Unlike text, image inputs are object values {type, value} + Map source = Map.of(field, IMAGE_INPUT); + DocWriteResponse response = client().prepareIndex(indexName).setSource(source).get(TEST_REQUEST_TIMEOUT); + assertThat(response.getResult(), is(DocWriteResponse.Result.CREATED)); + } + client().admin().indices().prepareRefresh(indexName).get(); + } +} diff --git a/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/SemanticTextInferenceFieldsIT.java b/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/SemanticTextInferenceFieldsIT.java index 3fa48af90705a..8a9b52c64a9e7 100644 --- a/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/SemanticTextInferenceFieldsIT.java +++ b/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/SemanticTextInferenceFieldsIT.java @@ -8,50 +8,27 @@ package org.elasticsearch.xpack.inference.integration; import org.elasticsearch.action.DocWriteResponse; -import org.elasticsearch.action.search.SearchRequest; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.document.DocumentField; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.core.Nullable; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; -import org.elasticsearch.index.mapper.InferenceMetadataFieldsMapper; -import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.inference.TaskType; -import org.elasticsearch.license.LicenseSettings; -import org.elasticsearch.plugins.Plugin; -import org.elasticsearch.reindex.ReindexPlugin; import org.elasticsearch.search.SearchHit; -import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.elasticsearch.search.fetch.subphase.FetchSourceContext; -import org.elasticsearch.search.lookup.SourceFilter; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.index.IndexVersionUtils; import org.elasticsearch.xcontent.XContentBuilder; -import org.elasticsearch.xpack.inference.FakeMlPlugin; -import org.elasticsearch.xpack.inference.LocalStateInferencePlugin; -import org.elasticsearch.xpack.inference.mock.TestInferenceServicePlugin; import org.elasticsearch.xpack.inference.queries.SemanticQueryBuilder; import org.junit.After; import java.io.IOException; -import java.util.Arrays; -import java.util.Collection; import java.util.HashMap; -import java.util.List; import java.util.Map; -import java.util.function.Consumer; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse; -import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.CoreMatchers.notNullValue; -import static org.hamcrest.CoreMatchers.nullValue; @ESIntegTestCase.ClusterScope(numDataNodes = 3, numClientNodes = 0, supportsDedicatedMasters = false) -public class SemanticTextInferenceFieldsIT extends ESIntegTestCase { +public class SemanticTextInferenceFieldsIT extends AbstractInferenceFieldsIT { private final String indexName = randomIdentifier(); private final Map inferenceIds = new HashMap<>(); @@ -67,21 +44,6 @@ public class SemanticTextInferenceFieldsIT extends ESIntegTestCase { "my_api_key" ); - @Override - protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) { - return Settings.builder().put(LicenseSettings.SELF_GENERATED_LICENSE_TYPE.getKey(), "trial").build(); - } - - @Override - protected Collection> nodePlugins() { - return List.of(LocalStateInferencePlugin.class, TestInferenceServicePlugin.class, ReindexPlugin.class, FakeMlPlugin.class); - } - - @Override - protected boolean forbidPrivateIndexSettings() { - return false; - } - @After public void cleanUp() { IntegrationTestUtils.deleteIndex(client(), indexName); @@ -124,27 +86,39 @@ private void excludeInferenceFieldsFromSourceTestCase(IndexVersion minIndexVersi indexDocuments(sparseEmbeddingField, docCount); indexDocuments(textEmbeddingField, docCount); - QueryBuilder sparseEmbeddingFieldQuery = new SemanticQueryBuilder(sparseEmbeddingField, randomAlphaOfLength(10)); - assertSearchResponse(sparseEmbeddingFieldQuery, indexSettings, docCount, request -> { - request.source().fetchSource(generateRandomFetchSourceContext()).fetchField(sparseEmbeddingField); - }, response -> { - for (SearchHit hit : response.getHits()) { - Map documentFields = hit.getDocumentFields(); - assertThat(documentFields.size(), is(1)); - assertThat(documentFields.containsKey(sparseEmbeddingField), is(true)); + assertSearchResponse( + indexName, + new SemanticQueryBuilder(sparseEmbeddingField, randomAlphaOfLength(10)), + indexSettings, + docCount, + request -> { + request.source().fetchSource(generateRandomFetchSourceContext()).fetchField(sparseEmbeddingField); + }, + response -> { + for (SearchHit hit : response.getHits()) { + Map documentFields = hit.getDocumentFields(); + assertThat(documentFields.size(), is(1)); + assertThat(documentFields.containsKey(sparseEmbeddingField), is(true)); + } } - }); + ); - QueryBuilder textEmbeddingFieldQuery = new SemanticQueryBuilder(textEmbeddingField, randomAlphaOfLength(10)); - assertSearchResponse(textEmbeddingFieldQuery, indexSettings, docCount, request -> { - request.source().fetchSource(generateRandomFetchSourceContext()).fetchField(textEmbeddingField); - }, response -> { - for (SearchHit hit : response.getHits()) { - Map documentFields = hit.getDocumentFields(); - assertThat(documentFields.size(), is(1)); - assertThat(documentFields.containsKey(textEmbeddingField), is(true)); + assertSearchResponse( + indexName, + new SemanticQueryBuilder(textEmbeddingField, randomAlphaOfLength(10)), + indexSettings, + docCount, + request -> { + request.source().fetchSource(generateRandomFetchSourceContext()).fetchField(textEmbeddingField); + }, + response -> { + for (SearchHit hit : response.getHits()) { + Map documentFields = hit.getDocumentFields(); + assertThat(documentFields.size(), is(1)); + assertThat(documentFields.containsKey(textEmbeddingField), is(true)); + } } - }); + ); IntegrationTestUtils.deleteIndex(client(), indexName); } @@ -155,15 +129,6 @@ private void createInferenceEndpoint(TaskType taskType, String inferenceId, Map< inferenceIds.put(inferenceId, taskType); } - private Settings generateIndexSettings(IndexVersion indexVersion) { - int numDataNodes = internalCluster().numDataNodes(); - return Settings.builder() - .put(IndexMetadata.SETTING_VERSION_CREATED, indexVersion) - .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numDataNodes) - .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) - .build(); - } - private void indexDocuments(String field, int count) { for (int i = 0; i < count; i++) { Map source = Map.of(field, randomAlphaOfLength(10)); @@ -173,116 +138,4 @@ private void indexDocuments(String field, int count) { client().admin().indices().prepareRefresh(indexName).get(); } - - private void assertSearchResponse( - QueryBuilder queryBuilder, - Settings indexSettings, - int expectedHits, - @Nullable Consumer searchRequestModifier, - @Nullable Consumer searchResponseValidator - ) throws Exception { - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().query(queryBuilder).size(expectedHits); - SearchRequest searchRequest = new SearchRequest(new String[] { indexName }, searchSourceBuilder); - if (searchRequestModifier != null) { - searchRequestModifier.accept(searchRequest); - } - - ExpectedSource expectedSource = getExpectedSource(indexSettings, searchRequest.source().fetchSource()); - assertResponse(client().search(searchRequest), response -> { - assertThat(response.getSuccessfulShards(), equalTo(response.getTotalShards())); - assertThat(response.getHits().getTotalHits().value(), equalTo((long) expectedHits)); - - for (SearchHit hit : response.getHits()) { - switch (expectedSource) { - case NONE -> assertThat(hit.getSourceAsMap(), nullValue()); - case INFERENCE_FIELDS_EXCLUDED -> { - Map sourceAsMap = hit.getSourceAsMap(); - assertThat(sourceAsMap, notNullValue()); - assertThat(sourceAsMap.containsKey(InferenceMetadataFieldsMapper.NAME), is(false)); - } - case INFERENCE_FIELDS_INCLUDED -> { - Map sourceAsMap = hit.getSourceAsMap(); - assertThat(sourceAsMap, notNullValue()); - assertThat(sourceAsMap.containsKey(InferenceMetadataFieldsMapper.NAME), is(true)); - } - } - } - - if (searchResponseValidator != null) { - searchResponseValidator.accept(response); - } - }); - } - - private static ExpectedSource getExpectedSource(Settings indexSettings, FetchSourceContext fetchSourceContext) { - if (fetchSourceContext != null && fetchSourceContext.fetchSource() == false) { - return ExpectedSource.NONE; - } else if (InferenceMetadataFieldsMapper.isEnabled(indexSettings) == false) { - return ExpectedSource.INFERENCE_FIELDS_EXCLUDED; - } - - if (fetchSourceContext != null) { - SourceFilter filter = fetchSourceContext.filter(); - if (filter != null) { - if (Arrays.asList(filter.getExcludes()).contains(InferenceMetadataFieldsMapper.NAME)) { - return ExpectedSource.INFERENCE_FIELDS_EXCLUDED; - } else if (filter.getIncludes().length > 0) { - return Arrays.asList(filter.getIncludes()).contains(InferenceMetadataFieldsMapper.NAME) - ? ExpectedSource.INFERENCE_FIELDS_INCLUDED - : ExpectedSource.INFERENCE_FIELDS_EXCLUDED; - } - } - - Boolean excludeInferenceFieldsExplicit = fetchSourceContext.excludeInferenceFields(); - if (excludeInferenceFieldsExplicit != null) { - return excludeInferenceFieldsExplicit ? ExpectedSource.INFERENCE_FIELDS_EXCLUDED : ExpectedSource.INFERENCE_FIELDS_INCLUDED; - } - } - - return ExpectedSource.INFERENCE_FIELDS_EXCLUDED; - } - - private static FetchSourceContext generateRandomFetchSourceContext() { - FetchSourceContext fetchSourceContext = switch (randomIntBetween(0, 4)) { - case 0 -> FetchSourceContext.FETCH_SOURCE; - case 1 -> FetchSourceContext.FETCH_ALL_SOURCE; - case 2 -> FetchSourceContext.FETCH_ALL_SOURCE_EXCLUDE_INFERENCE_FIELDS; - case 3 -> FetchSourceContext.DO_NOT_FETCH_SOURCE; - case 4 -> null; - default -> throw new IllegalStateException("Unhandled randomized case"); - }; - - if (fetchSourceContext != null && fetchSourceContext.fetchSource()) { - String[] includes = null; - String[] excludes = null; - if (randomBoolean()) { - // Randomly include a non-existent field to test explicit inclusion handling - String field = randomBoolean() ? InferenceMetadataFieldsMapper.NAME : randomIdentifier(); - includes = new String[] { field }; - } - if (randomBoolean()) { - // Randomly exclude a non-existent field to test implicit inclusion handling - String field = randomBoolean() ? InferenceMetadataFieldsMapper.NAME : randomIdentifier(); - excludes = new String[] { field }; - } - - if (includes != null || excludes != null) { - fetchSourceContext = FetchSourceContext.of( - fetchSourceContext.fetchSource(), - fetchSourceContext.excludeVectors(), - fetchSourceContext.excludeInferenceFields(), - includes, - excludes - ); - } - } - - return fetchSourceContext; - } - - private enum ExpectedSource { - NONE, - INFERENCE_FIELDS_EXCLUDED, - INFERENCE_FIELDS_INCLUDED - } } From 7a699d2e28d76d940c9228ec177c26a548c42776 Mon Sep 17 00:00:00 2001 From: kishor-kharbas Date: Wed, 22 Jul 2026 15:16:17 -0700 Subject: [PATCH 2/2] review comments - better abstractions, parameterize excludeInferenceFieldsFromSourceTestCase() --- .../AbstractInferenceFieldsIT.java | 52 +++++++ .../SemanticFieldInferenceFieldsIT.java | 133 +++--------------- .../SemanticTextInferenceFieldsIT.java | 101 ++++--------- 3 files changed, 97 insertions(+), 189 deletions(-) diff --git a/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/AbstractInferenceFieldsIT.java b/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/AbstractInferenceFieldsIT.java index 0d381a772a805..dfb7595a7f249 100644 --- a/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/AbstractInferenceFieldsIT.java +++ b/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/AbstractInferenceFieldsIT.java @@ -10,7 +10,9 @@ import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.document.DocumentField; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.CheckedSupplier; import org.elasticsearch.core.Nullable; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.mapper.InferenceMetadataFieldsMapper; @@ -23,16 +25,21 @@ import org.elasticsearch.search.fetch.subphase.FetchSourceContext; import org.elasticsearch.search.lookup.SourceFilter; import org.elasticsearch.test.ESIntegTestCase; +import org.elasticsearch.test.index.IndexVersionUtils; +import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xpack.inference.FakeMlPlugin; import org.elasticsearch.xpack.inference.LocalStateInferencePlugin; import org.elasticsearch.xpack.inference.mock.TestInferenceServicePlugin; +import org.elasticsearch.xpack.inference.queries.SemanticQueryBuilder; +import java.io.IOException; import java.util.Arrays; import java.util.Collection; import java.util.List; import java.util.Map; import java.util.function.Consumer; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse; import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.CoreMatchers.is; @@ -65,6 +72,51 @@ protected Settings generateIndexSettings(IndexVersion indexVersion) { .build(); } + protected void testExcludeInferenceFieldsFromSource( + CheckedSupplier mappingFactory, + Collection fieldsToValidate, + IndexVersion minIndexVersion, + IndexVersion maxIndexVersion, + int iterationCount + ) throws Exception { + final String indexName = randomIdentifier(); + + for (int i = 0; i < iterationCount; i++) { + final IndexVersion indexVersion = IndexVersionUtils.randomVersionBetween(minIndexVersion, maxIndexVersion); + final Settings indexSettings = generateIndexSettings(indexVersion); + XContentBuilder mappings = mappingFactory.get(); + assertAcked(prepareCreate(indexName).setSettings(indexSettings).setMapping(mappings)); + + final int docCount = randomIntBetween(10, 50); + fieldsToValidate.forEach(fieldName -> indexDocuments(indexName, fieldName, docCount)); + + for (String fieldName : fieldsToValidate) { + // For each field defined in the mapping, fetch all documents with that field and verify that + // response only contains the requested fieldName and no inference fields. + assertSearchResponse( + indexName, + new SemanticQueryBuilder(fieldName, randomAlphaOfLength(10)), + indexSettings, + docCount, + request -> { + request.source().fetchSource(generateRandomFetchSourceContext()).fetchField(fieldName); + }, + response -> { + for (SearchHit hit : response.getHits()) { + Map documentFields = hit.getDocumentFields(); + assertThat(documentFields.size(), is(1)); + assertThat(documentFields.containsKey(fieldName), is(true)); + } + } + ); + } + + IntegrationTestUtils.deleteIndex(client(), indexName); + } + } + + protected abstract void indexDocuments(String indexName, String field, int count); + protected void assertSearchResponse( String indexName, QueryBuilder queryBuilder, diff --git a/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/SemanticFieldInferenceFieldsIT.java b/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/SemanticFieldInferenceFieldsIT.java index 05780548e3ef4..2f46621cf6421 100644 --- a/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/SemanticFieldInferenceFieldsIT.java +++ b/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/SemanticFieldInferenceFieldsIT.java @@ -8,21 +8,12 @@ package org.elasticsearch.xpack.inference.integration; import org.elasticsearch.action.DocWriteResponse; -import org.elasticsearch.common.document.DocumentField; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.inference.TaskType; -import org.elasticsearch.search.SearchHit; import org.elasticsearch.test.ESIntegTestCase; -import org.elasticsearch.xcontent.XContentBuilder; -import org.junit.After; -import java.io.IOException; -import java.util.HashMap; import java.util.Map; -import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.CoreMatchers.is; /** @@ -32,9 +23,6 @@ */ @ESIntegTestCase.ClusterScope(numDataNodes = 3, numClientNodes = 0, supportsDedicatedMasters = false) public class SemanticFieldInferenceFieldsIT extends AbstractInferenceFieldsIT { - private final String indexName = randomIdentifier(); - private final Map inferenceIds = new HashMap<>(); - private static final Map EMBEDDING_SERVICE_SETTINGS = Map.of( "model", "my_model", @@ -54,116 +42,27 @@ public class SemanticFieldInferenceFieldsIT extends AbstractInferenceFieldsIT { "data:image/jpeg;base64,Y2F0IG9uIGEgd2luZG93c2lsbA==" ); - @After - public void cleanUp() { - IntegrationTestUtils.deleteIndex(client(), indexName); - for (var entry : inferenceIds.entrySet()) { - IntegrationTestUtils.deleteInferenceEndpoint(client(), entry.getValue(), entry.getKey()); - } - } - - public void testExcludeInferenceFieldsFromSourceWithTextInputs() throws Exception { - final String embeddingInferenceId = randomIdentifier(); - createInferenceEndpoint(TaskType.EMBEDDING, embeddingInferenceId, EMBEDDING_SERVICE_SETTINGS); - final String semanticField = randomIdentifier(); - - for (int i = 0; i < 10; i++) { - final Settings indexSettings = generateIndexSettings(IndexVersion.current()); - XContentBuilder mappings = IntegrationTestUtils.generateSemanticFieldMapping(Map.of(semanticField, embeddingInferenceId)); - assertAcked(prepareCreate(indexName).setSettings(indexSettings).setMapping(mappings)); - - final int docCount = randomIntBetween(10, 50); - indexTextDocuments(semanticField, docCount); - - assertSearchResponse(indexName, matchAllQuery(), indexSettings, docCount, request -> { - request.source().fetchSource(generateRandomFetchSourceContext()).fetchField(semanticField); - }, response -> { - for (SearchHit hit : response.getHits()) { - Map documentFields = hit.getDocumentFields(); - assertThat(documentFields.size(), is(1)); - assertThat(documentFields.containsKey(semanticField), is(true)); - } - }); - - IntegrationTestUtils.deleteIndex(client(), indexName); - } - } - - public void testExcludeInferenceFieldsFromSourceWithImageInputs() throws Exception { - final String embeddingInferenceId = randomIdentifier(); - createInferenceEndpoint(TaskType.EMBEDDING, embeddingInferenceId, EMBEDDING_SERVICE_SETTINGS); - final String semanticField = randomIdentifier(); - - for (int i = 0; i < 10; i++) { - final Settings indexSettings = generateIndexSettings(IndexVersion.current()); - XContentBuilder mappings = IntegrationTestUtils.generateSemanticFieldMapping(Map.of(semanticField, embeddingInferenceId)); - assertAcked(prepareCreate(indexName).setSettings(indexSettings).setMapping(mappings)); - - final int docCount = randomIntBetween(10, 50); - indexImageDocuments(semanticField, docCount); - - assertSearchResponse(indexName, matchAllQuery(), indexSettings, docCount, request -> { - request.source().fetchSource(generateRandomFetchSourceContext()).fetchField(semanticField); - }, response -> { - for (SearchHit hit : response.getHits()) { - Map documentFields = hit.getDocumentFields(); - assertThat(documentFields.size(), is(1)); - assertThat(documentFields.containsKey(semanticField), is(true)); - } - }); - - IntegrationTestUtils.deleteIndex(client(), indexName); - } - } - public void testExcludeInferenceFieldsFromSourceWithMixedInputs() throws Exception { + final String semanticFieldName = randomIdentifier(); final String embeddingInferenceId = randomIdentifier(); - createInferenceEndpoint(TaskType.EMBEDDING, embeddingInferenceId, EMBEDDING_SERVICE_SETTINGS); - final String semanticField = randomIdentifier(); - - for (int i = 0; i < 10; i++) { - final Settings indexSettings = generateIndexSettings(IndexVersion.current()); - XContentBuilder mappings = IntegrationTestUtils.generateSemanticFieldMapping(Map.of(semanticField, embeddingInferenceId)); - assertAcked(prepareCreate(indexName).setSettings(indexSettings).setMapping(mappings)); - - final int textDocCount = randomIntBetween(5, 25); - final int imageDocCount = randomIntBetween(5, 25); - indexTextDocuments(semanticField, textDocCount); - indexImageDocuments(semanticField, imageDocCount); - final int totalDocCount = textDocCount + imageDocCount; - - assertSearchResponse(indexName, matchAllQuery(), indexSettings, totalDocCount, request -> { - request.source().fetchSource(generateRandomFetchSourceContext()).fetchField(semanticField); - }, response -> { - for (SearchHit hit : response.getHits()) { - Map documentFields = hit.getDocumentFields(); - assertThat(documentFields.size(), is(1)); - assertThat(documentFields.containsKey(semanticField), is(true)); - } - }); - - IntegrationTestUtils.deleteIndex(client(), indexName); - } - } - - private void createInferenceEndpoint(TaskType taskType, String inferenceId, Map serviceSettings) throws IOException { - IntegrationTestUtils.createInferenceEndpoint(client(), taskType, inferenceId, serviceSettings); - inferenceIds.put(inferenceId, taskType); - } - - private void indexTextDocuments(String field, int count) { - for (int i = 0; i < count; i++) { - Map source = Map.of(field, randomAlphaOfLength(10)); - DocWriteResponse response = client().prepareIndex(indexName).setSource(source).get(TEST_REQUEST_TIMEOUT); - assertThat(response.getResult(), is(DocWriteResponse.Result.CREATED)); - } - client().admin().indices().prepareRefresh(indexName).get(); + IntegrationTestUtils.createInferenceEndpoint(client(), TaskType.EMBEDDING, embeddingInferenceId, EMBEDDING_SERVICE_SETTINGS); + final var fieldMap = Map.of(semanticFieldName, embeddingInferenceId); + + testExcludeInferenceFieldsFromSource( + () -> IntegrationTestUtils.generateSemanticFieldMapping(fieldMap), + fieldMap.keySet(), + IndexVersion.current(), + IndexVersion.current(), + 20 + ); + + IntegrationTestUtils.deleteInferenceEndpoint(client(), TaskType.EMBEDDING, embeddingInferenceId); } - private void indexImageDocuments(String field, int count) { + @Override + protected void indexDocuments(String indexName, String field, int count) { for (int i = 0; i < count; i++) { - // Unlike text, image inputs are object values {type, value} - Map source = Map.of(field, IMAGE_INPUT); + Map source = Map.of(field, randomBoolean() ? randomAlphaOfLength(10) : IMAGE_INPUT); DocWriteResponse response = client().prepareIndex(indexName).setSource(source).get(TEST_REQUEST_TIMEOUT); assertThat(response.getResult(), is(DocWriteResponse.Result.CREATED)); } diff --git a/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/SemanticTextInferenceFieldsIT.java b/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/SemanticTextInferenceFieldsIT.java index 8a9b52c64a9e7..a6516a599c9ed 100644 --- a/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/SemanticTextInferenceFieldsIT.java +++ b/x-pack/plugin/inference/src/internalClusterTest/java/org/elasticsearch/xpack/inference/integration/SemanticTextInferenceFieldsIT.java @@ -8,28 +8,24 @@ package org.elasticsearch.xpack.inference.integration; import org.elasticsearch.action.DocWriteResponse; -import org.elasticsearch.common.document.DocumentField; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; import org.elasticsearch.inference.TaskType; -import org.elasticsearch.search.SearchHit; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.index.IndexVersionUtils; -import org.elasticsearch.xcontent.XContentBuilder; -import org.elasticsearch.xpack.inference.queries.SemanticQueryBuilder; import org.junit.After; +import org.junit.Before; import java.io.IOException; import java.util.HashMap; import java.util.Map; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.CoreMatchers.is; @ESIntegTestCase.ClusterScope(numDataNodes = 3, numClientNodes = 0, supportsDedicatedMasters = false) public class SemanticTextInferenceFieldsIT extends AbstractInferenceFieldsIT { - private final String indexName = randomIdentifier(); + private final String sparseEmbeddingInferenceId = randomIdentifier(); + private final String textEmbeddingInferenceId = randomIdentifier(); private final Map inferenceIds = new HashMap<>(); private static final Map SPARSE_EMBEDDING_SERVICE_SETTINGS = Map.of("model", "my_model", "api_key", "my_api_key"); @@ -44,14 +40,30 @@ public class SemanticTextInferenceFieldsIT extends AbstractInferenceFieldsIT { "my_api_key" ); + @Before + public void setup() throws IOException { + createInferenceEndpoint(TaskType.SPARSE_EMBEDDING, sparseEmbeddingInferenceId, SPARSE_EMBEDDING_SERVICE_SETTINGS); + createInferenceEndpoint(TaskType.TEXT_EMBEDDING, textEmbeddingInferenceId, TEXT_EMBEDDING_SERVICE_SETTINGS); + } + @After public void cleanUp() { - IntegrationTestUtils.deleteIndex(client(), indexName); for (var entry : inferenceIds.entrySet()) { IntegrationTestUtils.deleteInferenceEndpoint(client(), entry.getValue(), entry.getKey()); } } + @Override + protected void indexDocuments(String indexName, String field, int count) { + for (int i = 0; i < count; i++) { + Map source = Map.of(field, randomAlphaOfLength(10)); + DocWriteResponse response = client().prepareIndex(indexName).setSource(source).get(TEST_REQUEST_TIMEOUT); + assertThat(response.getResult(), is(DocWriteResponse.Result.CREATED)); + } + + client().admin().indices().prepareRefresh(indexName).get(); + } + public void testExcludeInferenceFieldsFromSource() throws Exception { excludeInferenceFieldsFromSourceTestCase(IndexVersion.current(), IndexVersion.current(), 10); } @@ -66,76 +78,21 @@ public void testExcludeInferenceFieldsFromSourceOldIndexVersions() throws Except private void excludeInferenceFieldsFromSourceTestCase(IndexVersion minIndexVersion, IndexVersion maxIndexVersion, int iterations) throws Exception { - final String sparseEmbeddingInferenceId = randomIdentifier(); - final String textEmbeddingInferenceId = randomIdentifier(); - createInferenceEndpoint(TaskType.SPARSE_EMBEDDING, sparseEmbeddingInferenceId, SPARSE_EMBEDDING_SERVICE_SETTINGS); - createInferenceEndpoint(TaskType.TEXT_EMBEDDING, textEmbeddingInferenceId, TEXT_EMBEDDING_SERVICE_SETTINGS); - final String sparseEmbeddingField = randomIdentifier(); final String textEmbeddingField = randomIdentifier(); - - for (int i = 0; i < iterations; i++) { - final IndexVersion indexVersion = IndexVersionUtils.randomVersionBetween(minIndexVersion, maxIndexVersion); - final Settings indexSettings = generateIndexSettings(indexVersion); - XContentBuilder mappings = IntegrationTestUtils.generateSemanticTextMapping( - Map.of(sparseEmbeddingField, sparseEmbeddingInferenceId, textEmbeddingField, textEmbeddingInferenceId) - ); - assertAcked(prepareCreate(indexName).setSettings(indexSettings).setMapping(mappings)); - - final int docCount = randomIntBetween(10, 50); - indexDocuments(sparseEmbeddingField, docCount); - indexDocuments(textEmbeddingField, docCount); - - assertSearchResponse( - indexName, - new SemanticQueryBuilder(sparseEmbeddingField, randomAlphaOfLength(10)), - indexSettings, - docCount, - request -> { - request.source().fetchSource(generateRandomFetchSourceContext()).fetchField(sparseEmbeddingField); - }, - response -> { - for (SearchHit hit : response.getHits()) { - Map documentFields = hit.getDocumentFields(); - assertThat(documentFields.size(), is(1)); - assertThat(documentFields.containsKey(sparseEmbeddingField), is(true)); - } - } - ); - - assertSearchResponse( - indexName, - new SemanticQueryBuilder(textEmbeddingField, randomAlphaOfLength(10)), - indexSettings, - docCount, - request -> { - request.source().fetchSource(generateRandomFetchSourceContext()).fetchField(textEmbeddingField); - }, - response -> { - for (SearchHit hit : response.getHits()) { - Map documentFields = hit.getDocumentFields(); - assertThat(documentFields.size(), is(1)); - assertThat(documentFields.containsKey(textEmbeddingField), is(true)); - } - } - ); - - IntegrationTestUtils.deleteIndex(client(), indexName); - } + final var fieldMap = Map.of(sparseEmbeddingField, sparseEmbeddingInferenceId, textEmbeddingField, textEmbeddingInferenceId); + + testExcludeInferenceFieldsFromSource( + () -> IntegrationTestUtils.generateSemanticTextMapping(fieldMap), + fieldMap.keySet(), + minIndexVersion, + maxIndexVersion, + iterations + ); } private void createInferenceEndpoint(TaskType taskType, String inferenceId, Map serviceSettings) throws IOException { IntegrationTestUtils.createInferenceEndpoint(client(), taskType, inferenceId, serviceSettings); inferenceIds.put(inferenceId, taskType); } - - private void indexDocuments(String field, int count) { - for (int i = 0; i < count; i++) { - Map source = Map.of(field, randomAlphaOfLength(10)); - DocWriteResponse response = client().prepareIndex(indexName).setSource(source).get(TEST_REQUEST_TIMEOUT); - assertThat(response.getResult(), is(DocWriteResponse.Result.CREATED)); - } - - client().admin().indices().prepareRefresh(indexName).get(); - } }