ESQL: gate heterogeneous FROM behind a feature flag

docs/changelog/154987.yaml

@@ -0,0 +1,5 @@
+pr: 154987
+summary: Gate heterogeneous `FROM` (a dataset mixed with an index) behind a feature flag
+area: ES|QL
+type: feature
+issues: []

x-pack/plugin/esql/build.gradle

@@ -196,6 +196,9 @@ tasks.named("internalClusterTest").configure {
   // Same reason: applying the request filter to datasets is off in release builds, so the conformance and
   // heterogeneous request-filter ITs would assert against an unfiltered dataset during release testing.
   systemProperty 'es.esql_request_filter_on_dataset_feature_flag_enabled', 'true'
+  // Same reason: heterogeneous FROM (mixing an index and a dataset) is off in release builds, so the dataset ITs
+  // that exercise the mixed shape would be rejected during release testing rather than asserting the union result.
+  systemProperty 'es.esql_heterogeneous_from_feature_flag_enabled', 'true'
   // Data sources require the project encryption key feature (EsqlPlugin fails fast otherwise); pair the flags.
   systemProperty 'es.project_encryption_key_feature_flag_enabled', 'true'
   systemProperty 'esql.kibana.docs.dir', "${rootDir}/docs/reference/query-languages/esql/kibana/generated"

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlResolveDatasetAction.java

@@ -96,7 +96,12 @@ protected void localClusterStateOperation(Task task, Request request, ProjectSta
             indexNameExpressionResolver
         );
         listener.onResponse(
-            new Response(resolution.resolvedExternalDatasets(), resolution.nonDatasetNames(), resolution.explicitUnauthorized())
+            new Response(
+                resolution.resolvedExternalDatasets(),
+                resolution.nonDatasetNames(),
+                resolution.authorizedNonDatasetNames(),
+                resolution.explicitUnauthorized()
+            )
         );
     }
 
@@ -146,6 +151,14 @@ public IndicesOptions indicesOptions() {
             return DatasetRewriter.RESOLVER_OPTIONS;
         }
 
+        // Include data streams in the security filter's narrowing so an authorized data stream counts as an index-like
+        // in the disabled-heterogeneous-FROM check (a dataset + data-stream wildcard must reject too). The sibling
+        // EsqlResolveViewAction leaves this at the default; datasets differ because that check needs every index-like.
+        @Override
+        public boolean includeDataStreams() {
+            return true;
+        }
+
         @Override
         public void setResolvedIndexExpressions(ResolvedIndexExpressions expressions) {
             this.resolvedIndexExpressions = expressions;
@@ -170,11 +183,18 @@ public String toString() {
     public static class Response extends ActionResponse {
         private final Set<String> datasets;
         private final Set<String> nonDatasetNames;
+        private final Set<String> authorizedNonDatasetNames;
         private final Set<String> explicitUnauthorized;
 
-        public Response(Set<String> datasets, Set<String> nonDatasetNames, Set<String> explicitUnauthorized) {
+        public Response(
+            Set<String> datasets,
+            Set<String> nonDatasetNames,
+            Set<String> authorizedNonDatasetNames,
+            Set<String> explicitUnauthorized
+        ) {
             this.datasets = datasets;
             this.nonDatasetNames = nonDatasetNames;
+            this.authorizedNonDatasetNames = authorizedNonDatasetNames;
             this.explicitUnauthorized = explicitUnauthorized;
         }
 
@@ -184,14 +204,22 @@ public Set<String> datasets() {
         }
 
         /**
-         * Concrete non-dataset names (indices, aliases, data streams) resolved from the same pattern. Non-empty when
-         * the relation targets a mix of datasets and non-datasets; drives heterogeneous-FROM UnionAll building
-         * in {@link DatasetRewriter}.
+         * Concrete non-dataset names (indices, aliases, data streams) resolved from the same pattern. Drives the
+         * heterogeneous-FROM index branch in {@link DatasetRewriter}.
          */
         public Set<String> nonDatasetNames() {
             return nonDatasetNames;
         }
 
+        /**
+         * The subset of {@link #nonDatasetNames()} the caller may read (from the security-narrowed request indices).
+         * Disabled heterogeneous FROM tests this — not the un-narrowed set — so an index the caller can't read is not
+         * counted as a mix.
+         */
+        public Set<String> authorizedNonDatasetNames() {
+            return authorizedNonDatasetNames;
+        }
+
         /** Explicitly-named datasets absent from the authorized set — surfaced as {@code Unknown index} by the rewrite. */
         public Set<String> explicitUnauthorized() {
             return explicitUnauthorized;

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/datasources/DatasetResolver.java

@@ -131,7 +131,12 @@ void replaceDatasets(
                     new ThreadedActionListener<>(executor, l.delegateFailureAndWrap((delegate, response) -> {
                         resolutions.put(
                             relation,
-                            new DatasetResolution(response.datasets(), response.nonDatasetNames(), response.explicitUnauthorized())
+                            new DatasetResolution(
+                                response.datasets(),
+                                response.nonDatasetNames(),
+                                response.authorizedNonDatasetNames(),
+                                response.explicitUnauthorized()
+                            )
                         );
                         delegate.onResponse(null);
                     }))

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/datasources/DatasetRewriter.java

@@ -17,6 +17,7 @@
 import org.elasticsearch.cluster.metadata.ProjectMetadata;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.regex.Regex;
+import org.elasticsearch.common.util.FeatureFlag;
 import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.transport.RemoteClusterAware;
 import org.elasticsearch.xpack.esql.VerificationException;
@@ -38,6 +39,7 @@
 
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.HashMap;
 import java.util.IdentityHashMap;
 import java.util.LinkedHashSet;
@@ -73,6 +75,9 @@ public final class DatasetRewriter {
         .indexAbstractionOptions(IndicesOptions.IndexAbstractionOptions.builder().resolveDatasets(true).resolveViews(false).build())
         .build();
 
+    /** Gates heterogeneous {@code FROM} (a single FROM mixing a dataset and an index): off in release builds. */
+    public static final FeatureFlag HETEROGENEOUS_FROM_FEATURE_FLAG = new FeatureFlag("esql_heterogeneous_from");
+
     private DatasetRewriter() {}
 
     /**
@@ -95,21 +100,13 @@ public static DatasetResolution resolve(
             ? new LinkedHashSet<>()
             : new LinkedHashSet<>(iner.datasets(projectMetadata, RESOLVER_OPTIONS, indicesRequestOf(authorizedIndices)));
 
-        // (b) classify the raw (un-narrowed) patterns into dataset vs non-dataset under an open predicate.
+        // (b) classify the raw (un-narrowed) patterns into dataset vs non-dataset. This non-dataset set drives the index
+        // branch, so it stays un-narrowed — downstream field-caps applies security to it, exactly as before.
         IndexAbstractionResolver resolver = new IndexAbstractionResolver(iner);
         Map<String, IndexAbstraction> indicesLookup = projectMetadata.getIndicesLookup();
-        List<String> localNames = resolver.resolveIndexAbstractions(
-            Arrays.asList(rawPatterns),
-            RESOLVER_OPTIONS,
-            projectMetadata,
-            componentSelector -> indicesLookup.keySet(),
-            (name, selector) -> true,
-            true
-        ).getLocalIndicesList();
-
         Set<String> nonDatasetNames = new LinkedHashSet<>();
         Set<String> rawDatasetNames = new LinkedHashSet<>();
-        for (String name : localNames) {
+        for (String name : classifyLocalNames(resolver, rawPatterns, projectMetadata, indicesLookup)) {
             IndexAbstraction abs = indicesLookup.get(name);
             if (abs == null) {
                 continue; // synthesized name (date math) — neither; skipping is safe for the non-dataset set
@@ -121,6 +118,22 @@ public static DatasetResolution resolve(
             }
         }
 
+        // (c) the non-dataset names the caller is authorized to read — classify the security-narrowed authorizedIndices,
+        // not the open-predicate raw patterns, so the heterogeneity check ignores an index the caller can't see.
+        // Unsecured (authorizedIndices == rawPatterns) reuses (b).
+        Set<String> authorizedNonDatasetNames;
+        if (Arrays.equals(authorizedIndices, rawPatterns)) {
+            authorizedNonDatasetNames = nonDatasetNames;
+        } else {
+            authorizedNonDatasetNames = new LinkedHashSet<>();
+            for (String name : classifyLocalNames(resolver, authorizedIndices, projectMetadata, indicesLookup)) {
+                IndexAbstraction abs = indicesLookup.get(name);
+                if (abs != null && abs.getType() != IndexAbstraction.Type.DATASET) {
+                    authorizedNonDatasetNames.add(name);
+                }
+            }
+        }
+
         // Explicit (non-wildcard) dataset names absent from the authorized set — rewriteOne rejects these as Unknown
         // index rather than silently dropping them from a multi-target FROM.
         Set<String> explicitUnauthorized = new LinkedHashSet<>();
@@ -136,7 +149,24 @@ public static DatasetResolution resolve(
 
         Set<String> result = new LinkedHashSet<>(rawDatasetNames);
         result.retainAll(resolvedExternalDatasets);
-        return new DatasetResolution(result, nonDatasetNames, explicitUnauthorized);
+        return new DatasetResolution(result, nonDatasetNames, authorizedNonDatasetNames, explicitUnauthorized);
+    }
+
+    /** Expands {@code patterns} to their local abstraction names under an open (authorization-blind) predicate. */
+    private static List<String> classifyLocalNames(
+        IndexAbstractionResolver resolver,
+        String[] patterns,
+        ProjectMetadata projectMetadata,
+        Map<String, IndexAbstraction> indicesLookup
+    ) {
+        return resolver.resolveIndexAbstractions(
+            Arrays.asList(patterns),
+            RESOLVER_OPTIONS,
+            projectMetadata,
+            componentSelector -> indicesLookup.keySet(),
+            (name, selector) -> true,
+            true
+        ).getLocalIndicesList();
     }
 
     /** Minimal {@link IndicesRequest} carrier so {@link IndexNameExpressionResolver#datasets} can read the names. */
@@ -161,6 +191,16 @@ public IndicesOptions indicesOptions() {
      * dataset-free project is a no-op.
      */
     public static LogicalPlan rewriteUnsecured(LogicalPlan parsed, ProjectMetadata projectMetadata, IndexNameExpressionResolver iner) {
+        return rewriteUnsecured(parsed, projectMetadata, iner, HETEROGENEOUS_FROM_FEATURE_FLAG.isEnabled());
+    }
+
+    /** Test seam: {@code heterogeneousFromEnabled} overrides the {@link #HETEROGENEOUS_FROM_FEATURE_FLAG} default. */
+    static LogicalPlan rewriteUnsecured(
+        LogicalPlan parsed,
+        ProjectMetadata projectMetadata,
+        IndexNameExpressionResolver iner,
+        boolean heterogeneousFromEnabled
+    ) {
         if (projectMetadata == null) {
             return parsed;
         }
@@ -183,7 +223,7 @@ public static LogicalPlan rewriteUnsecured(LogicalPlan parsed, ProjectMetadata p
             resolutions.put(r, resolve(raw, raw, projectMetadata, iner));
         });
         // Unsecured/test path runs without CPS (single local project): never preserve a wildcard for remote resolution.
-        return rewrite(parsed, projectMetadata, resolutions, false);
+        return rewrite(parsed, projectMetadata, resolutions, false, heterogeneousFromEnabled);
     }
 
     static boolean hasRemotePattern(List<String> patterns) {
@@ -219,6 +259,17 @@ public static LogicalPlan rewrite(
         ProjectMetadata projectMetadata,
         Map<UnresolvedRelation, DatasetResolution> resolutions,
         boolean crossProjectEnabled
+    ) {
+        return rewrite(parsed, projectMetadata, resolutions, crossProjectEnabled, HETEROGENEOUS_FROM_FEATURE_FLAG.isEnabled());
+    }
+
+    /** Test seam: {@code heterogeneousFromEnabled} overrides the {@link #HETEROGENEOUS_FROM_FEATURE_FLAG} default. */
+    static LogicalPlan rewrite(
+        LogicalPlan parsed,
+        ProjectMetadata projectMetadata,
+        Map<UnresolvedRelation, DatasetResolution> resolutions,
+        boolean crossProjectEnabled,
+        boolean heterogeneousFromEnabled
     ) {
         if (projectMetadata == null) {
             return parsed;
@@ -233,7 +284,7 @@ public static LogicalPlan rewrite(
             if (resolution == null) {
                 return r;
             }
-            return rewriteOne(r, datasetMetadata, dataSourceMetadata, resolution, crossProjectEnabled);
+            return rewriteOne(r, datasetMetadata, dataSourceMetadata, resolution, crossProjectEnabled, heterogeneousFromEnabled);
         });
     }
 
@@ -242,7 +293,8 @@ private static LogicalPlan rewriteOne(
         DatasetMetadata datasets,
         DataSourceMetadata dataSources,
         DatasetResolution resolution,
-        boolean crossProjectEnabled
+        boolean crossProjectEnabled,
+        boolean heterogeneousFromEnabled
     ) {
         if (resolution.explicitUnauthorized().isEmpty() == false) {
             // An explicitly-named dataset the caller can't read — same error (and 400) a missing index gives, so an
@@ -258,6 +310,19 @@ private static LogicalPlan rewriteOne(
             // like this and must not be rejected as a "mix".
             return relation;
         }
+        // Heterogeneous FROM disabled: reject iff the caller can read both a dataset and an index-like target in one
+        // FROM. datasetNames is non-empty here, so a non-empty authorized non-dataset set is the mix. Using the
+        // authorized (not raw) set keeps an index the caller can't read out of both the decision and the message. Runs
+        // before any dataset branch is built, so a rejected query never resolves an external schema.
+        if (heterogeneousFromEnabled == false && resolution.authorizedNonDatasetNames().isEmpty() == false) {
+            throw new VerificationException(
+                "Querying both indices and datasets in the same FROM is not supported; dataset(s) requested: "
+                    + truncatedNames(datasetNames)
+                    + ", index(es) requested: "
+                    + truncatedNames(resolution.authorizedNonDatasetNames())
+            );
+        }
+
         if (relation.indexMode() != null && relation.indexMode() != IndexMode.STANDARD) {
             String message = switch (relation.indexMode()) {
                 case TIME_SERIES -> "TS command is not supported for datasets; dataset(s) requested: " + datasetNames;
@@ -270,7 +335,6 @@ private static LogicalPlan rewriteOne(
             };
             throw new VerificationException(message);
         }
-        Set<String> nonDatasetNamesList = resolution.nonDatasetNames();
 
         // One rail for every FROM shape — dataset-only and heterogeneous (index + dataset). The non-remotable-abstraction
         // CPS rule (a remote view/dataset fails; a remote index of the same name reads both) must hold uniformly, so the
@@ -286,7 +350,7 @@ private static LogicalPlan rewriteOne(
         // concrete name and a wildcard (no double read) and the wildcard's remote half reaches field-caps (closing
         // #151977's dropped-remote-wildcard gap). The resolveDatasets rail on this branch also fails a remote
         // dataset/view the wildcard matches. METADATA fields ride along so _index/_id resolve on the index rows.
-        List<String> indexBranch = new ArrayList<>(nonDatasetNamesList);
+        List<String> indexBranch = new ArrayList<>(resolution.nonDatasetNames());
         if (crossProjectEnabled) {
             indexBranch.addAll(crossProjectPatternsToPreserve(patternsOf(relation)));
         }
@@ -359,6 +423,23 @@ static boolean anyPatternCouldMatchDataset(List<String> patterns, Set<String> da
         return false;
     }
 
+    private static final int MAX_NAMES_IN_MESSAGE = 10;
+
+    /** Renders a name list for an error message, capping a wildcard that resolved to many names at {@value #MAX_NAMES_IN_MESSAGE}. */
+    private static String truncatedNames(Collection<String> names) {
+        if (names.size() <= MAX_NAMES_IN_MESSAGE) {
+            return names.toString();
+        }
+        List<String> shown = new ArrayList<>(MAX_NAMES_IN_MESSAGE);
+        for (String name : names) {
+            if (shown.size() == MAX_NAMES_IN_MESSAGE) {
+                break;
+            }
+            shown.add(name);
+        }
+        return shown + " (+" + (names.size() - MAX_NAMES_IN_MESSAGE) + " more)";
+    }
+
     /** Splits a relation's FROM pattern string into its comma-separated parts. */
     static List<String> patternsOf(UnresolvedRelation relation) {
         return Arrays.asList(Strings.splitStringByCommaToArray(relation.indexPattern().indexPattern()));
@@ -474,10 +555,16 @@ private static Map<String, Object> mergeSettings(DataSource parent, Dataset data
     }
 
     /**
-     * Per-relation result of {@link #resolve}: the external dataset names the relation resolved to (security-filtered
-     * to those the caller may read), the concrete non-dataset names resolved from the same pattern (drives
-     * heterogeneous-FROM {@link UnionAll} building), and the explicitly-named datasets absent from the resolved set
-     * (surfaced by {@link #rewriteOne} as {@code Unknown index}).
+     * Per-relation result of {@link #resolve}: {@code resolvedExternalDatasets} (authorized datasets the relation
+     * resolved to); {@code nonDatasetNames} (un-narrowed non-dataset names that build the index branch);
+     * {@code authorizedNonDatasetNames} (the subset the caller may read — what disabled heterogeneous FROM tests, so an
+     * index the caller can't see is neither counted as a mix nor named in the error); and {@code explicitUnauthorized}
+     * (explicitly-named datasets the caller can't read, surfaced by {@link #rewriteOne} as {@code Unknown index}).
      */
-    public record DatasetResolution(Set<String> resolvedExternalDatasets, Set<String> nonDatasetNames, Set<String> explicitUnauthorized) {}
+    public record DatasetResolution(
+        Set<String> resolvedExternalDatasets,
+        Set<String> nonDatasetNames,
+        Set<String> authorizedNonDatasetNames,
+        Set<String> explicitUnauthorized
+    ) {}
 }

x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/datasources/DatasetResolverTests.java

@@ -179,7 +179,7 @@ Response extends org.elasticsearch.action.ActionResponse> void doExecute(
                 ) {
                 assertSame(EsqlResolveDatasetAction.TYPE, action);
                 localCalls.incrementAndGet();
-                listener.onResponse((Response) new EsqlResolveDatasetAction.Response(Set.of(DATASET_NAME), Set.of(), Set.of()));
+                listener.onResponse((Response) new EsqlResolveDatasetAction.Response(Set.of(DATASET_NAME), Set.of(), Set.of(), Set.of()));
             }
         };
     }