ESQL: Don't accidentally add flattened subfields

docs/changelog/154508.yaml

@@ -0,0 +1,8 @@
+area: ES|QL
+issues:
+ - 154743
+ - 154484
+ - 154011
+pr: 154508
+summary: Don't accidentally add flattened subfields
+type: bug

server/src/main/java/org/elasticsearch/index/query/QueryRewriteContext.java

@@ -27,10 +27,12 @@
 import org.elasticsearch.index.SliceIndexing;
 import org.elasticsearch.index.analysis.IndexAnalyzers;
 import org.elasticsearch.index.mapper.DateFieldMapper;
+import org.elasticsearch.index.mapper.DynamicFieldType;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MapperBuilderContext;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.MappingLookup;
+import org.elasticsearch.index.mapper.MetadataFieldMapper;
 import org.elasticsearch.index.mapper.RoutingFieldMapper;
 import org.elasticsearch.index.mapper.TextFieldMapper;
 import org.elasticsearch.plugins.internal.rewriter.QueryRewriteInterceptor;
@@ -633,6 +635,41 @@ public Iterable<Map.Entry<String, MappedFieldType>> getAllFields() {
         return () -> Iterators.concat(allEntrySet.iterator(), runtimeEntrySet.iterator());
     }
 
+    /**
+     * Returns {@code true} if {@code name} is a concrete mapped field in this index — either an
+     * explicitly mapped field, a runtime field, or a dynamically-resolved sub-field of a
+     * {@link MetadataFieldMapper} — and is permitted by the {@link #allowedFields} predicate when set.
+     * <p>
+     *     Unlike {@link SearchExecutionContext#isFieldMapped}, this does <em>not</em>
+     *     include fields that are only dynamically resolved, such as sub-keys of
+     *     {@code flattened} fields. Those sub-keys are not visible to the coordinator
+     *     via field caps and using them in a block loader would cause element-type
+     *     mismatches at runtime. Sub-fields of {@link MetadataFieldMapper} instances
+     *     (e.g. {@code _project._alias}) are allowed because they are registered
+     *     metadata mappers and are reported by field caps.
+     * </p>
+     * <p>
+     *      This is <strong>mostly</strong> used by ESQL because it exposes flattened
+     *      sub-fields using its own machinery.
+     * </p>
+     */
+    public boolean isMappedField(String name) {
+        if (allowedFields != null && false == allowedFields.test(name)) {
+            return false;
+        }
+        String fieldName = resolveSliceAlias(name);
+        if (mappingLookup.getFullNameToFieldType().containsKey(fieldName) || runtimeMappings.containsKey(fieldName)) {
+            return true;
+        }
+        int dotIndex = fieldName.indexOf('.');
+        if (dotIndex > 0) {
+            return mappingLookup.getMapper(fieldName.substring(0, dotIndex)) instanceof MetadataFieldMapper metaMapper
+                && metaMapper.fieldType() instanceof DynamicFieldType dft
+                && dft.getChildFieldType(fieldName.substring(dotIndex + 1)) != null;
+        }
+        return false;
+    }
+
     public ResolvedIndices getResolvedIndices() {
         return resolvedIndices;
     }

server/src/test/java/org/elasticsearch/index/query/QueryRewriteContextTests.java

@@ -9,17 +9,31 @@
 
 package org.elasticsearch.index.query;
 
+import org.apache.lucene.search.Query;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.routing.allocation.DataTier;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.mapper.DateFieldMapper;
+import org.elasticsearch.index.mapper.DocumentParserContext;
+import org.elasticsearch.index.mapper.DynamicFieldType;
+import org.elasticsearch.index.mapper.IndexType;
+import org.elasticsearch.index.mapper.MappedFieldType;
+import org.elasticsearch.index.mapper.MapperBuilderContext;
+import org.elasticsearch.index.mapper.Mapping;
 import org.elasticsearch.index.mapper.MappingLookup;
+import org.elasticsearch.index.mapper.MetadataFieldMapper;
+import org.elasticsearch.index.mapper.MockFieldMapper;
+import org.elasticsearch.index.mapper.RootObjectMapper;
+import org.elasticsearch.index.mapper.ValueFetcher;
 import org.elasticsearch.indices.DateFieldRangeInfo;
 import org.elasticsearch.test.ESTestCase;
 
+import java.io.IOException;
 import java.util.Collections;
+import java.util.Map;
 
 import static org.elasticsearch.index.query.CoordinatorRewriteContext.TIER_FIELD_TYPE;
 import static org.hamcrest.Matchers.is;
@@ -143,6 +157,107 @@ public void testGetTierPreference() {
         }
     }
 
+    /**
+     * Tests that {@link QueryRewriteContext#isMappedField} recognises sub-fields of
+     * {@link MetadataFieldMapper} instances that implement {@link DynamicFieldType}.
+     * This mirrors serverless metadata fields such as {@code _project._alias} whose
+     * root mapper ({@code _project}) is a {@link MetadataFieldMapper} providing
+     * dynamic child types at query time.
+     */
+    public void testIsMappedFieldMetadataDynamicSubfield() {
+        DynamicMetadataFieldMapper metaMapper = new DynamicMetadataFieldMapper();
+        RootObjectMapper root = new RootObjectMapper.Builder("_doc").build(MapperBuilderContext.root(false, false));
+        Mapping mapping = new Mapping(root, new MetadataFieldMapper[] { metaMapper }, Map.of());
+        MappingLookup lookup = MappingLookup.fromMapping(mapping, IndexMode.STANDARD);
+
+        IndexMetadata indexMeta = newIndexMeta("test-idx", Settings.builder().build());
+        QueryRewriteContext ctx = new QueryRewriteContext(
+            parserConfig(),
+            null,
+            System::currentTimeMillis,
+            null,
+            lookup,
+            Collections.emptyMap(),
+            new IndexSettings(indexMeta, Settings.EMPTY),
+            null,
+            null,
+            null,
+            null,
+            null,
+            null,
+            null,
+            null,
+            null,
+            null,
+            null,
+            null,
+            false,
+            false
+        );
+
+        // The root metadata field is in fullNameToFieldType, so it maps directly.
+        assertTrue(ctx.isMappedField(DynamicMetadataFieldMapper.NAME));
+        // Sub-field whose key exists in the DynamicFieldType.
+        assertTrue(ctx.isMappedField(DynamicMetadataFieldMapper.NAME + "." + DynamicMetadataFieldMapper.PRESENT_KEY));
+        // Sub-field whose key the DynamicFieldType does not know about.
+        assertFalse(ctx.isMappedField(DynamicMetadataFieldMapper.NAME + ".absent_key"));
+        // A completely unmapped field.
+        assertFalse(ctx.isMappedField("unmapped_field"));
+        // Sub-field of a completely unmapped parent.
+        assertFalse(ctx.isMappedField("unmapped_parent.sub_key"));
+    }
+
+    /**
+     * A minimal {@link MetadataFieldMapper} whose field type implements {@link DynamicFieldType},
+     * used to verify that {@link QueryRewriteContext#isMappedField} correctly recognises
+     * sub-fields provided by metadata mappers.
+     */
+    private static class DynamicMetadataFieldMapper extends MetadataFieldMapper {
+        static final String NAME = "_test_meta";
+        static final String PRESENT_KEY = "present_key";
+
+        DynamicMetadataFieldMapper() {
+            super(new DynamicMetaFieldType());
+        }
+
+        @Override
+        protected String contentType() {
+            return NAME;
+        }
+
+        @Override
+        protected void parseCreateField(DocumentParserContext context) throws IOException {}
+
+        private static class DynamicMetaFieldType extends MappedFieldType implements DynamicFieldType {
+            DynamicMetaFieldType() {
+                super(NAME, IndexType.NONE, false, Map.of());
+            }
+
+            @Override
+            public MappedFieldType getChildFieldType(String path) {
+                if (PRESENT_KEY.equals(path)) {
+                    return new MockFieldMapper.FakeFieldType(NAME + "." + path);
+                }
+                return null;
+            }
+
+            @Override
+            public String typeName() {
+                return NAME;
+            }
+
+            @Override
+            public ValueFetcher valueFetcher(SearchExecutionContext context, String format) {
+                throw new UnsupportedOperationException();
+            }
+
+            @Override
+            public Query termQuery(Object value, SearchExecutionContext context) {
+                throw new UnsupportedOperationException();
+            }
+        }
+    }
+
     public static IndexMetadata newIndexMeta(String name, Settings indexSettings) {
         return IndexMetadata.builder(name).settings(indexSettings(IndexVersion.current(), 1, 1).put(indexSettings)).build();
     }

x-pack/plugin/esql/qa/security/src/javaRestTest/java/org/elasticsearch/xpack/esql/EsqlSecurityIT.java

@@ -78,6 +78,7 @@ public class EsqlSecurityIT extends ESRestTestCase {
         .user("ds_repro_broad_reader", "x-pack-test-password", "ds_repro_broad_reader", false)
         .user("user4", "x-pack-test-password", "user4", false)
         .user("user5", "x-pack-test-password", "user5", false)
+        .user("fls_cross_index_user", "x-pack-test-password", "fls_cross_index", false)
         .user("fls_user", "x-pack-test-password", "fls_user", false)
         .user("fls_partial_no_source_user", "x-pack-test-password", "fls_partial_no_source", false)
         .user("fls_per_index_access_user", "x-pack-test-password", "fls_partial_no_source,read_full_mapping", false)
@@ -2179,6 +2180,24 @@ public void testDataStream() throws IOException {
         assertMap(entityAsMap(runESQLCommand("logs_foo_after_2021_alias", "FROM alias-* | STATS COUNT(*)")), oneResult);
     }
 
+    public void testCountAcrossIndicesWithFlsDeniedField() throws Exception {
+        Response resp = runESQLCommand("fls_cross_index_user", "FROM index,index-user1 | STATS c = COUNT(value)");
+        assertOK(resp);
+        @SuppressWarnings("unchecked")
+        List<List<Object>> values = (List<List<Object>>) entityAsMap(resp).get("values");
+        // index: value allowed — 2 docs (10.0, 20.0); index-user1: value FLS-denied — contributes 0
+        assertThat(values.get(0).get(0), equalTo(2));
+    }
+
+    public void testSumAcrossIndicesWithFlsDeniedFieldAndCast() throws Exception {
+        Response resp = runESQLCommand("fls_cross_index_user", "FROM index,index-user1 | STATS s = SUM(value::long)");
+        assertOK(resp);
+        @SuppressWarnings("unchecked")
+        List<List<Object>> values = (List<List<Object>>) entityAsMap(resp).get("values");
+        // index: value allowed — 10.0+20.0 → 30L; index-user1: value FLS-denied — contributes null
+        assertThat(values.get(0).get(0), equalTo(30));
+    }
+
     protected Response runESQLCommand(String user, String command) throws IOException {
         return runESQLCommand(user, command, null);
     }

x-pack/plugin/esql/qa/security/src/javaRestTest/resources/roles.yml

@@ -107,6 +107,22 @@ user5:
       privileges:
         - read
 
+# Grants value+org on `index` but only `org` on `index-user1` (value is FLS-denied there).
+# Exercises COUNT and SUM across indices where a field is visible in one index but
+# FLS-denied in another — the FLS analogue of the flattened-field cross-index conflicts
+# fixed in issues #154011 and #154484.
+fls_cross_index:
+  cluster: []
+  indices:
+    - names: [ 'index' ]
+      privileges: [ 'read' ]
+      field_security:
+        grant: [ value, org ]
+    - names: [ 'index-user1' ]
+      privileges: [ 'read' ]
+      field_security:
+        grant: [ org ]
+
 fls_user:
   cluster: []
   indices: