From 625983641ae22473869903560aff0cedb37643f7 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Fri, 24 Jul 2026 04:30:02 -0400 Subject: [PATCH] ESQL: Don't accidentally add flattened subfields (#154508) We were accidentally adding flattened sub fields in queries like: ``` FROM idx_date, idx_flat_date | STATS c = COUNT(metadata.time) ``` In `idx_date` the `metadata.time` field is mapped as `date`. In `idx_flat_date` the `metadata` field is mapped as `flattened`. Without this fix we see the `date` as a real, partially mapped field. So we proceed to process it. On the data node we see the `flattened` sub field and load it like a `keyword` field. Which confuses everything! Because we're expecting a `date` field. Closes #154011 Closes #154484 --- docs/changelog/154508.yaml | 8 + .../index/query/QueryRewriteContext.java | 37 ++++ .../index/query/QueryRewriteContextTests.java | 115 ++++++++++ .../xpack/esql/EsqlSecurityIT.java | 19 ++ .../src/javaRestTest/resources/roles.yml | 16 ++ .../qa/multi_node/FlattenedConflictsIT.java | 200 ++++++++++++++++++ .../planner/EsPhysicalOperationProviders.java | 72 +++++++ .../esql/planner/LocalExecutionPlanner.java | 2 +- .../xpack/esql/stats/SearchContextStats.java | 42 ++-- 9 files changed, 498 insertions(+), 13 deletions(-) create mode 100644 docs/changelog/154508.yaml create mode 100644 x-pack/plugin/esql/qa/server/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/multi_node/FlattenedConflictsIT.java diff --git a/docs/changelog/154508.yaml b/docs/changelog/154508.yaml new file mode 100644 index 0000000000000..d128e28ce4866 --- /dev/null +++ b/docs/changelog/154508.yaml @@ -0,0 +1,8 @@ +area: ES|QL +issues: + - 154743 + - 154484 + - 154011 +pr: 154508 +summary: Don't accidentally add flattened subfields +type: bug diff --git a/server/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java b/server/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java index 16666906a3ff2..3d64440d4536c 100644 --- a/server/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java @@ -27,10 +27,12 @@ import org.elasticsearch.index.SliceIndexing; import org.elasticsearch.index.analysis.IndexAnalyzers; import org.elasticsearch.index.mapper.DateFieldMapper; +import org.elasticsearch.index.mapper.DynamicFieldType; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperBuilderContext; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.mapper.MappingLookup; +import org.elasticsearch.index.mapper.MetadataFieldMapper; import org.elasticsearch.index.mapper.RoutingFieldMapper; import org.elasticsearch.index.mapper.TextFieldMapper; import org.elasticsearch.plugins.internal.rewriter.QueryRewriteInterceptor; @@ -633,6 +635,41 @@ public Iterable> getAllFields() { return () -> Iterators.concat(allEntrySet.iterator(), runtimeEntrySet.iterator()); } + /** + * Returns {@code true} if {@code name} is a concrete mapped field in this index — either an + * explicitly mapped field, a runtime field, or a dynamically-resolved sub-field of a + * {@link MetadataFieldMapper} — and is permitted by the {@link #allowedFields} predicate when set. + *

+ * Unlike {@link SearchExecutionContext#isFieldMapped}, this does not + * include fields that are only dynamically resolved, such as sub-keys of + * {@code flattened} fields. Those sub-keys are not visible to the coordinator + * via field caps and using them in a block loader would cause element-type + * mismatches at runtime. Sub-fields of {@link MetadataFieldMapper} instances + * (e.g. {@code _project._alias}) are allowed because they are registered + * metadata mappers and are reported by field caps. + *

+ *

+ * This is mostly used by ESQL because it exposes flattened + * sub-fields using its own machinery. + *

+ */ + public boolean isMappedField(String name) { + if (allowedFields != null && false == allowedFields.test(name)) { + return false; + } + String fieldName = resolveSliceAlias(name); + if (mappingLookup.getFullNameToFieldType().containsKey(fieldName) || runtimeMappings.containsKey(fieldName)) { + return true; + } + int dotIndex = fieldName.indexOf('.'); + if (dotIndex > 0) { + return mappingLookup.getMapper(fieldName.substring(0, dotIndex)) instanceof MetadataFieldMapper metaMapper + && metaMapper.fieldType() instanceof DynamicFieldType dft + && dft.getChildFieldType(fieldName.substring(dotIndex + 1)) != null; + } + return false; + } + public ResolvedIndices getResolvedIndices() { return resolvedIndices; } diff --git a/server/src/test/java/org/elasticsearch/index/query/QueryRewriteContextTests.java b/server/src/test/java/org/elasticsearch/index/query/QueryRewriteContextTests.java index b27552075f4d2..e252199eb6432 100644 --- a/server/src/test/java/org/elasticsearch/index/query/QueryRewriteContextTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/QueryRewriteContextTests.java @@ -9,17 +9,31 @@ package org.elasticsearch.index.query; +import org.apache.lucene.search.Query; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.routing.allocation.DataTier; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.mapper.DateFieldMapper; +import org.elasticsearch.index.mapper.DocumentParserContext; +import org.elasticsearch.index.mapper.DynamicFieldType; +import org.elasticsearch.index.mapper.IndexType; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.MapperBuilderContext; +import org.elasticsearch.index.mapper.Mapping; import org.elasticsearch.index.mapper.MappingLookup; +import org.elasticsearch.index.mapper.MetadataFieldMapper; +import org.elasticsearch.index.mapper.MockFieldMapper; +import org.elasticsearch.index.mapper.RootObjectMapper; +import org.elasticsearch.index.mapper.ValueFetcher; import org.elasticsearch.indices.DateFieldRangeInfo; import org.elasticsearch.test.ESTestCase; +import java.io.IOException; import java.util.Collections; +import java.util.Map; import static org.elasticsearch.index.query.CoordinatorRewriteContext.TIER_FIELD_TYPE; import static org.hamcrest.Matchers.is; @@ -143,6 +157,107 @@ public void testGetTierPreference() { } } + /** + * Tests that {@link QueryRewriteContext#isMappedField} recognises sub-fields of + * {@link MetadataFieldMapper} instances that implement {@link DynamicFieldType}. + * This mirrors serverless metadata fields such as {@code _project._alias} whose + * root mapper ({@code _project}) is a {@link MetadataFieldMapper} providing + * dynamic child types at query time. + */ + public void testIsMappedFieldMetadataDynamicSubfield() { + DynamicMetadataFieldMapper metaMapper = new DynamicMetadataFieldMapper(); + RootObjectMapper root = new RootObjectMapper.Builder("_doc").build(MapperBuilderContext.root(false, false)); + Mapping mapping = new Mapping(root, new MetadataFieldMapper[] { metaMapper }, Map.of()); + MappingLookup lookup = MappingLookup.fromMapping(mapping, IndexMode.STANDARD); + + IndexMetadata indexMeta = newIndexMeta("test-idx", Settings.builder().build()); + QueryRewriteContext ctx = new QueryRewriteContext( + parserConfig(), + null, + System::currentTimeMillis, + null, + lookup, + Collections.emptyMap(), + new IndexSettings(indexMeta, Settings.EMPTY), + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + false, + false + ); + + // The root metadata field is in fullNameToFieldType, so it maps directly. + assertTrue(ctx.isMappedField(DynamicMetadataFieldMapper.NAME)); + // Sub-field whose key exists in the DynamicFieldType. + assertTrue(ctx.isMappedField(DynamicMetadataFieldMapper.NAME + "." + DynamicMetadataFieldMapper.PRESENT_KEY)); + // Sub-field whose key the DynamicFieldType does not know about. + assertFalse(ctx.isMappedField(DynamicMetadataFieldMapper.NAME + ".absent_key")); + // A completely unmapped field. + assertFalse(ctx.isMappedField("unmapped_field")); + // Sub-field of a completely unmapped parent. + assertFalse(ctx.isMappedField("unmapped_parent.sub_key")); + } + + /** + * A minimal {@link MetadataFieldMapper} whose field type implements {@link DynamicFieldType}, + * used to verify that {@link QueryRewriteContext#isMappedField} correctly recognises + * sub-fields provided by metadata mappers. + */ + private static class DynamicMetadataFieldMapper extends MetadataFieldMapper { + static final String NAME = "_test_meta"; + static final String PRESENT_KEY = "present_key"; + + DynamicMetadataFieldMapper() { + super(new DynamicMetaFieldType()); + } + + @Override + protected String contentType() { + return NAME; + } + + @Override + protected void parseCreateField(DocumentParserContext context) throws IOException {} + + private static class DynamicMetaFieldType extends MappedFieldType implements DynamicFieldType { + DynamicMetaFieldType() { + super(NAME, IndexType.NONE, false, Map.of()); + } + + @Override + public MappedFieldType getChildFieldType(String path) { + if (PRESENT_KEY.equals(path)) { + return new MockFieldMapper.FakeFieldType(NAME + "." + path); + } + return null; + } + + @Override + public String typeName() { + return NAME; + } + + @Override + public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { + throw new UnsupportedOperationException(); + } + + @Override + public Query termQuery(Object value, SearchExecutionContext context) { + throw new UnsupportedOperationException(); + } + } + } + public static IndexMetadata newIndexMeta(String name, Settings indexSettings) { return IndexMetadata.builder(name).settings(indexSettings(IndexVersion.current(), 1, 1).put(indexSettings)).build(); } diff --git a/x-pack/plugin/esql/qa/security/src/javaRestTest/java/org/elasticsearch/xpack/esql/EsqlSecurityIT.java b/x-pack/plugin/esql/qa/security/src/javaRestTest/java/org/elasticsearch/xpack/esql/EsqlSecurityIT.java index d8d92ccf765d8..39a26a3bd5ea4 100644 --- a/x-pack/plugin/esql/qa/security/src/javaRestTest/java/org/elasticsearch/xpack/esql/EsqlSecurityIT.java +++ b/x-pack/plugin/esql/qa/security/src/javaRestTest/java/org/elasticsearch/xpack/esql/EsqlSecurityIT.java @@ -78,6 +78,7 @@ public class EsqlSecurityIT extends ESRestTestCase { .user("ds_repro_broad_reader", "x-pack-test-password", "ds_repro_broad_reader", false) .user("user4", "x-pack-test-password", "user4", false) .user("user5", "x-pack-test-password", "user5", false) + .user("fls_cross_index_user", "x-pack-test-password", "fls_cross_index", false) .user("fls_user", "x-pack-test-password", "fls_user", false) .user("fls_partial_no_source_user", "x-pack-test-password", "fls_partial_no_source", false) .user("fls_per_index_access_user", "x-pack-test-password", "fls_partial_no_source,read_full_mapping", false) @@ -2179,6 +2180,24 @@ public void testDataStream() throws IOException { assertMap(entityAsMap(runESQLCommand("logs_foo_after_2021_alias", "FROM alias-* | STATS COUNT(*)")), oneResult); } + public void testCountAcrossIndicesWithFlsDeniedField() throws Exception { + Response resp = runESQLCommand("fls_cross_index_user", "FROM index,index-user1 | STATS c = COUNT(value)"); + assertOK(resp); + @SuppressWarnings("unchecked") + List> values = (List>) entityAsMap(resp).get("values"); + // index: value allowed — 2 docs (10.0, 20.0); index-user1: value FLS-denied — contributes 0 + assertThat(values.get(0).get(0), equalTo(2)); + } + + public void testSumAcrossIndicesWithFlsDeniedFieldAndCast() throws Exception { + Response resp = runESQLCommand("fls_cross_index_user", "FROM index,index-user1 | STATS s = SUM(value::long)"); + assertOK(resp); + @SuppressWarnings("unchecked") + List> values = (List>) entityAsMap(resp).get("values"); + // index: value allowed — 10.0+20.0 → 30L; index-user1: value FLS-denied — contributes null + assertThat(values.get(0).get(0), equalTo(30)); + } + protected Response runESQLCommand(String user, String command) throws IOException { return runESQLCommand(user, command, null); } diff --git a/x-pack/plugin/esql/qa/security/src/javaRestTest/resources/roles.yml b/x-pack/plugin/esql/qa/security/src/javaRestTest/resources/roles.yml index 82ef86ded417a..83a7b2bf97970 100644 --- a/x-pack/plugin/esql/qa/security/src/javaRestTest/resources/roles.yml +++ b/x-pack/plugin/esql/qa/security/src/javaRestTest/resources/roles.yml @@ -107,6 +107,22 @@ user5: privileges: - read +# Grants value+org on `index` but only `org` on `index-user1` (value is FLS-denied there). +# Exercises COUNT and SUM across indices where a field is visible in one index but +# FLS-denied in another — the FLS analogue of the flattened-field cross-index conflicts +# fixed in issues #154011 and #154484. +fls_cross_index: + cluster: [] + indices: + - names: [ 'index' ] + privileges: [ 'read' ] + field_security: + grant: [ value, org ] + - names: [ 'index-user1' ] + privileges: [ 'read' ] + field_security: + grant: [ org ] + fls_user: cluster: [] indices: diff --git a/x-pack/plugin/esql/qa/server/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/multi_node/FlattenedConflictsIT.java b/x-pack/plugin/esql/qa/server/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/multi_node/FlattenedConflictsIT.java new file mode 100644 index 0000000000000..0bf2f0d15b878 --- /dev/null +++ b/x-pack/plugin/esql/qa/server/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/multi_node/FlattenedConflictsIT.java @@ -0,0 +1,200 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.qa.multi_node; + +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; + +import org.elasticsearch.client.Request; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.WarningsHandler; +import org.elasticsearch.common.Strings; +import org.elasticsearch.test.TestClustersThreadFilter; +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.rest.ESRestTestCase; +import org.junit.Before; +import org.junit.ClassRule; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static org.elasticsearch.test.MapMatcher.assertMap; +import static org.elasticsearch.test.MapMatcher.matchesMap; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; + +/** + * Cross-index type conflicts where a field is a concrete type in one index + * and a sub-key of a {@code flattened} field in another. + *

+ * Each scenario has a {@code SameNode} sibling that pins both indices to the + * same node. + *

+ */ +@ThreadLeakFilters(filters = TestClustersThreadFilter.class) +public class FlattenedConflictsIT extends ESRestTestCase { + + @ClassRule + public static ElasticsearchCluster cluster = Clusters.testCluster(ignored -> {}); + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } + + private List nodeNames; + + @Before + public void discoverNodes() throws Exception { + assumeFalse("Cannot pin shards to specific nodes in serverless mode", isServerless()); + + Request nodesRequest = new Request("GET", "/_nodes"); + nodesRequest.addParameter("filter_path", "nodes.*.name"); + Map nodesResponse = entityAsMap(client().performRequest(nodesRequest)); + @SuppressWarnings("unchecked") + Map nodes = (Map) nodesResponse.get("nodes"); + nodeNames = new ArrayList<>(); + for (Object nodeInfo : nodes.values()) { + @SuppressWarnings("unchecked") + Map info = (Map) nodeInfo; + nodeNames.add((String) info.get("name")); + } + assertThat("Need at least 2 nodes", nodeNames.size(), greaterThanOrEqualTo(2)); + } + + /** + * {@code metadata.time} is {@code date} (element type LONG) in one index + * and a sub-key of a {@code flattened} field in another. + */ + public void testDateVsFlattened() throws Exception { + testDateVsFlattened(nodeNames.get(0), nodeNames.get(1)); + } + + /** Same as {@link #testDateVsFlattened()} but both indices on the same node. */ + public void testDateVsFlattenedSameNode() throws Exception { + testDateVsFlattened(nodeNames.get(0), nodeNames.get(0)); + } + + /** + * {@code features.topic_id} is {@code integer} (element type INT) in one + * index and a sub-key of a {@code flattened} field in another. + * With an explicit {@code ::long} cast the coordinator resolves the union + * but the converter must be applied on both the integer and flattened shards. + */ + public void testIntegerVsFlattenedWithCast() throws Exception { + testIntegerVsFlattenedWithCast(nodeNames.get(0), nodeNames.get(1)); + } + + /** Same as {@link #testIntegerVsFlattenedWithCast()} but both indices on the same node. */ + public void testIntegerVsFlattenedWithCastSameNode() throws Exception { + testIntegerVsFlattenedWithCast(nodeNames.get(0), nodeNames.get(0)); + } + + public void testDateVsFlattened(String node1, String node2) throws Exception { + createIndexPinned("idx_date", """ + { "properties": { "metadata": { "properties": { "time": { "type": "date" } } } } }""", node1); + createIndexPinned("idx_flat_date", """ + { "properties": { "metadata": { "type": "flattened" } } }""", node2); + ensureGreen("idx_date"); + ensureGreen("idx_flat_date"); + for (int i = 0; i < 5; i++) { + indexDoc("idx_date", Integer.toString(i), Strings.format(""" + {"metadata": {"time": "2024-01-0%dT00:00:00.000Z"}}""", i + 1)); + } + for (int i = 0; i < 5; i++) { + indexDoc("idx_flat_date", Integer.toString(i), Strings.format(""" + {"metadata": {"time": "2024-02-0%dT00:00:00.000Z"}}""", i + 1)); + } + refresh("idx_date"); + refresh("idx_flat_date"); + + // The flattened sub-key is not in the real mapping so the data node treats it + // as absent — only the 5 docs from the date index contribute. + List> values = esql(""" + FROM idx_date, idx_flat_date + | STATS c = COUNT(metadata.time) + """); + assertThat(values.size(), equalTo(1)); + assertThat(values.get(0).get(0), equalTo(5)); + } + + public void testIntegerVsFlattenedWithCast(String node1, String node2) throws Exception { + createIndexPinned("idx_obj_int", """ + { "properties": { "features": { "properties": { "topic_id": { "type": "integer" } } } } }""", node1); + createIndexPinned("idx_flat_int", """ + { "properties": { "features": { "type": "flattened" } } }""", node2); + ensureGreen("idx_obj_int"); + ensureGreen("idx_flat_int"); + for (int i = 0; i < 10; i++) { + indexDoc("idx_obj_int", Integer.toString(i), Strings.format(""" + {"features": {"topic_id": %d}}""", i + 1)); + } + for (int i = 0; i < 5; i++) { + indexDoc("idx_flat_int", Integer.toString(i), Strings.format(""" + {"features": {"topic_id": %d}}""", (i + 1) * 10)); + } + refresh("idx_obj_int"); + refresh("idx_flat_int"); + + // int sum: 1+2+...+10 = 55; flattened sub-key treated as absent. + List> values = esql(""" + FROM idx_obj_int, idx_flat_int + | STATS s = SUM(features.topic_id::long) + """); + assertThat(values.size(), equalTo(1)); + assertThat(values.get(0).get(0), equalTo(55)); + } + + private void createIndexPinned(String name, String mapping, String node) throws IOException { + Request request = new Request("PUT", "/" + name); + request.setJsonEntity(Strings.format(""" + { + "settings": { + "index.number_of_shards": 1, + "index.number_of_replicas": 0, + "index.routing.allocation.require._name": "%s" + }, + "mappings": %s + }""", node, mapping)); + assertOK(client().performRequest(request)); + } + + private void indexDoc(String index, String id, String body) throws IOException { + Request request = new Request("PUT", "/" + index + "/_doc/" + id); + request.setJsonEntity(body); + request.addParameter("refresh", "false"); + assertOK(client().performRequest(request)); + } + + private boolean isServerless() throws IOException { + for (Map nodeInfo : getNodesInfo(client()).values()) { + @SuppressWarnings("unchecked") + List> modules = (List>) nodeInfo.get("modules"); + for (Map module : modules) { + if (module.get("name").toString().startsWith("serverless-")) { + return true; + } + } + } + return false; + } + + private List> esql(String query) throws IOException { + Request request = new Request("POST", "/_query"); + request.addParameter("allow_partial_results", "false"); + request.setOptions(RequestOptions.DEFAULT.toBuilder().setWarningsHandler(WarningsHandler.PERMISSIVE).build()); + String escaped = query.replace("\"", "\\\"").replace("\n", "\\n"); + request.setJsonEntity("{\"query\": \"" + escaped + "\"}"); + Map result = entityAsMap(client().performRequest(request)); + assertMap("no partial failures", result, matchesMap().extraOk().entry("is_partial", false)); + @SuppressWarnings("unchecked") + List> values = (List>) result.get("values"); + return values; + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java index 545f573624911..81e5434a4154f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java @@ -45,10 +45,12 @@ import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.AnalysisRegistry; import org.elasticsearch.index.mapper.BlockLoader; +import org.elasticsearch.index.mapper.DynamicFieldType; import org.elasticsearch.index.mapper.IndexType; import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MappingLookup; +import org.elasticsearch.index.mapper.MetadataFieldMapper; import org.elasticsearch.index.mapper.NestedLookup; import org.elasticsearch.index.mapper.SourceFieldMapper; import org.elasticsearch.index.mapper.SourceLoader; @@ -169,6 +171,31 @@ public boolean hasReferences() { * need one in ten documents. */ public abstract double storedFieldsSequentialProportion(); + + /** + * Returns {@code true} if {@code name} is a concrete mapped field in this shard's + * index — including index-level runtime fields and dynamically-resolved sub-fields of + * {@link MetadataFieldMapper} instances, but excluding dynamically-resolved sub-keys of + * {@code flattened} fields. Those sub-keys appear in Lucene's field infos even though they + * are absent from the mapping, and counting them with an EXISTS query would inflate + * aggregation results when field types differ across indices. + *

+ * The default implementation uses the mapping lookup alone. {@link DefaultShardContext} + * overrides this to also respect query-time runtime fields and {@code allowedFields}. + */ + public boolean isMappedField(String name) { + MappingLookup lookup = mappingLookup(); + if (lookup.getFullNameToFieldType().containsKey(name)) { + return true; + } + int dotIndex = name.indexOf('.'); + if (dotIndex > 0) { + return lookup.getMapper(name.substring(0, dotIndex)) instanceof MetadataFieldMapper metaMapper + && metaMapper.fieldType() instanceof DynamicFieldType dft + && dft.getChildFieldType(name.substring(dotIndex + 1)) != null; + } + return false; + } } private final IndexedByShardId shardContexts; @@ -400,6 +427,15 @@ private static class DefaultShardContextForUnmappedField extends DefaultShardCon this.unmappedEsField = unmappedEsField; } + @Override + public boolean isMappedField(String name) { + // For the unmapped field we are loading, bypass the mapped-field gate. + // This allows both truly unmapped fields (which use a source-based loader) and + // dynamic subfields of flattened fields (which use the keyed block loader) to + // produce real values rather than ConstantNull. + return name.equals(fullFieldName) || super.isMappedField(name); + } + @Override public @Nullable MappedFieldType fieldType(String name) { var superResult = super.fieldType(name); @@ -447,6 +483,29 @@ public Function> querySupplierForField( + QueryBuilder builder, + String fieldName + ) { + Function> innerFn = querySupplier(builder); + if ("*".equals(fieldName)) { + return innerFn; + } + return ctx -> { + if (shardContexts.get(ctx.index()).isMappedField(fieldName) == false) { + return List.of(); + } + return innerFn.apply(ctx); + }; + } + @Override public final PhysicalOperation sourcePhysicalOperation(EsQueryExec esQueryExec, LocalExecutionPlannerContext context) { final LuceneOperator.Factory luceneFactory; @@ -742,6 +801,14 @@ public BlockLoader blockLoader( if (asUnsupportedSource) { return ConstantNull.INSTANCE; } + // Don't use fieldType() for the existence check — it resolves sub-keys of + // flattened fields dynamically, but those are not reported by field caps + // and would cause element_type mismatches at runtime. + // Use isMappedField() (virtual) so subclasses such as DefaultShardContextForUnmappedField + // can allow specific unmapped fields to pass through. + if (isMappedField(name) == false) { + return ConstantNull.INSTANCE; + } MappedFieldType fieldType = fieldType(name); if (fieldType == null) { // the field does not exist in this context @@ -780,6 +847,11 @@ public double storedFieldsSequentialProportion() { return EsqlPlugin.STORED_FIELDS_SEQUENTIAL_PROPORTION.get(ctx.getIndexSettings().getSettings()); } + @Override + public boolean isMappedField(String name) { + return ctx.isMappedField(name); + } + @Override public void close() { releasable.close(); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java index 731f3fef2cf3e..c05c545f49dd6 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java @@ -605,7 +605,7 @@ private PhysicalOperation planEsStats(EsStatsQueryExec statsQuery, LocalExecutio EsPhysicalOperationProviders esProvider = (EsPhysicalOperationProviders) physicalOperationProviders; var queryFunction = switch (stat) { - case EsStatsQueryExec.BasicStat basic -> esProvider.querySupplier(basic.filter(statsQuery.query())); + case EsStatsQueryExec.BasicStat basic -> esProvider.querySupplierForField(basic.filter(statsQuery.query()), basic.name()); case EsStatsQueryExec.ByStat byStat -> esProvider.querySupplier(byStat.queryBuilderAndTags()); }; final LuceneOperator.Factory luceneFactory = esProvider.countSource(context, queryFunction, stat.tagTypes(), statsQuery.limit()); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/stats/SearchContextStats.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/stats/SearchContextStats.java index 96f87035d68e0..fc712ea4a3ac8 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/stats/SearchContextStats.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/stats/SearchContextStats.java @@ -109,7 +109,7 @@ private FieldConfig makeFieldConfig(String field) { // even if there are deleted documents, check the existence of a field // since if it's missing, deleted documents won't change that for (SearchExecutionContext context : contexts) { - if (context.isFieldMapped(field)) { + if (context.isMappedField(field)) { MappedFieldType type = context.getFieldType(field); if (fieldType == null) { fieldType = type; @@ -139,7 +139,7 @@ private FieldConfig makeFieldConfig(String field) { private boolean fastNoCacheFieldExists(String field) { for (SearchExecutionContext context : contexts) { - if (context.isFieldMapped(field)) { + if (context.isMappedField(field)) { return true; } } @@ -208,15 +208,33 @@ public long count() { @Override public long count(FieldName field) { var stat = cache.computeIfAbsent(field.string(), this::makeFieldStats); - if (stat.count == null) { - var count = new long[] { 0 }; - boolean completed = doWithContexts(r -> { - count[0] += countEntries(r, field.string()); - return true; - }, false); - stat.count = completed ? count[0] : -1; + if (stat.count != null) { + return stat.count; + } + long count = 0; + for (SearchExecutionContext context : contexts) { + // Skip shards where this field is a dynamic sub-key of a flattened field rather + // than an explicitly mapped field; those shards store the field's terms in Lucene + // even though it is absent from the mapping, so counting without this guard + // inflates the result. + if (context.isMappedField(field.string()) == false) { + continue; + } + for (LeafReaderContext leafContext : context.searcher().getLeafContexts()) { + LeafReader reader = leafContext.reader(); + if (reader.hasDeletions()) { + // Can't use the count + return stat.count = -1L; + } + long c = countEntries(reader, field.string()); + if (c < 0) { + // Can't use the count + return stat.count = -1L; + } + count += c; + } } - return stat.count; + return stat.count = count; } @Override @@ -241,7 +259,7 @@ public Object min(FieldName field) { Long result = null; try { for (final SearchExecutionContext context : contexts) { - if (context.isFieldMapped(field.string()) == false) { + if (context.isMappedField(field.string()) == false) { continue; } final MappedFieldType ctxFieldType = context.getFieldType(field.string()); @@ -272,7 +290,7 @@ public Object max(FieldName field) { Long result = null; try { for (final SearchExecutionContext context : contexts) { - if (context.isFieldMapped(field.string()) == false) { + if (context.isMappedField(field.string()) == false) { continue; } final MappedFieldType ctxFieldType = context.getFieldType(field.string());