From 421c0d9ffd9728601b38c73a5dc540222cde2bb6 Mon Sep 17 00:00:00 2001 From: Oleg Lvovitch Date: Fri, 24 Jul 2026 18:01:43 +0100 Subject: [PATCH] ESQL: gate heterogeneous FROM behind feature flag MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add the esql_heterogeneous_from feature flag (off in release builds). When it is off, a FROM is rejected if, for what the caller is authorized to read, it resolves to both a dataset and an index-like target (index/alias/data stream). Resolving to only datasets works (any expression, wildcards and exclusions included); resolving to only index-likes works. Expression form is never a factor — only what it resolves to. The index-like side is detected from the security-narrowed request indices, so a wildcard that also matched an index the caller cannot read is not a mix: the query reads the dataset, is not rejected, and no error names the unreadable index. Data streams are included in that narrowing. The check runs in DatasetRewriter.rewriteOne, before index field-caps and external-source schema resolution, so a rejected query never resolves a dataset schema. The flag is enabled for the esql test tasks so the on behavior runs in release-tests. The resolve response is local-only, so there is no wire change. --- docs/changelog/154987.yaml | 5 + x-pack/plugin/esql/build.gradle | 3 + .../esql/action/EsqlResolveDatasetAction.java | 38 +++- .../esql/datasources/DatasetResolver.java | 7 +- .../esql/datasources/DatasetRewriter.java | 131 +++++++++-- .../datasources/DatasetResolverTests.java | 2 +- .../datasources/DatasetRewriterTests.java | 204 +++++++++++++++++- 7 files changed, 357 insertions(+), 33 deletions(-) create mode 100644 docs/changelog/154987.yaml diff --git a/docs/changelog/154987.yaml b/docs/changelog/154987.yaml new file mode 100644 index 0000000000000..718e3e8d08090 --- /dev/null +++ b/docs/changelog/154987.yaml @@ -0,0 +1,5 @@ +pr: 154987 +summary: Gate heterogeneous `FROM` (a dataset mixed with an index) behind a feature flag +area: ES|QL +type: feature +issues: [] diff --git a/x-pack/plugin/esql/build.gradle b/x-pack/plugin/esql/build.gradle index 40b276645668e..b7d7ae287afda 100644 --- a/x-pack/plugin/esql/build.gradle +++ b/x-pack/plugin/esql/build.gradle @@ -196,6 +196,9 @@ tasks.named("internalClusterTest").configure { // Same reason: applying the request filter to datasets is off in release builds, so the conformance and // heterogeneous request-filter ITs would assert against an unfiltered dataset during release testing. systemProperty 'es.esql_request_filter_on_dataset_feature_flag_enabled', 'true' + // Same reason: heterogeneous FROM (mixing an index and a dataset) is off in release builds, so the dataset ITs + // that exercise the mixed shape would be rejected during release testing rather than asserting the union result. + systemProperty 'es.esql_heterogeneous_from_feature_flag_enabled', 'true' // Data sources require the project encryption key feature (EsqlPlugin fails fast otherwise); pair the flags. systemProperty 'es.project_encryption_key_feature_flag_enabled', 'true' systemProperty 'esql.kibana.docs.dir', "${rootDir}/docs/reference/query-languages/esql/kibana/generated" diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlResolveDatasetAction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlResolveDatasetAction.java index 14cd4c414eef9..2949110e167e8 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlResolveDatasetAction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlResolveDatasetAction.java @@ -96,7 +96,12 @@ protected void localClusterStateOperation(Task task, Request request, ProjectSta indexNameExpressionResolver ); listener.onResponse( - new Response(resolution.resolvedExternalDatasets(), resolution.nonDatasetNames(), resolution.explicitUnauthorized()) + new Response( + resolution.resolvedExternalDatasets(), + resolution.nonDatasetNames(), + resolution.authorizedNonDatasetNames(), + resolution.explicitUnauthorized() + ) ); } @@ -146,6 +151,14 @@ public IndicesOptions indicesOptions() { return DatasetRewriter.RESOLVER_OPTIONS; } + // Include data streams in the security filter's narrowing so an authorized data stream counts as an index-like + // in the disabled-heterogeneous-FROM check (a dataset + data-stream wildcard must reject too). The sibling + // EsqlResolveViewAction leaves this at the default; datasets differ because that check needs every index-like. + @Override + public boolean includeDataStreams() { + return true; + } + @Override public void setResolvedIndexExpressions(ResolvedIndexExpressions expressions) { this.resolvedIndexExpressions = expressions; @@ -170,11 +183,18 @@ public String toString() { public static class Response extends ActionResponse { private final Set datasets; private final Set nonDatasetNames; + private final Set authorizedNonDatasetNames; private final Set explicitUnauthorized; - public Response(Set datasets, Set nonDatasetNames, Set explicitUnauthorized) { + public Response( + Set datasets, + Set nonDatasetNames, + Set authorizedNonDatasetNames, + Set explicitUnauthorized + ) { this.datasets = datasets; this.nonDatasetNames = nonDatasetNames; + this.authorizedNonDatasetNames = authorizedNonDatasetNames; this.explicitUnauthorized = explicitUnauthorized; } @@ -184,14 +204,22 @@ public Set datasets() { } /** - * Concrete non-dataset names (indices, aliases, data streams) resolved from the same pattern. Non-empty when - * the relation targets a mix of datasets and non-datasets; drives heterogeneous-FROM UnionAll building - * in {@link DatasetRewriter}. + * Concrete non-dataset names (indices, aliases, data streams) resolved from the same pattern. Drives the + * heterogeneous-FROM index branch in {@link DatasetRewriter}. */ public Set nonDatasetNames() { return nonDatasetNames; } + /** + * The subset of {@link #nonDatasetNames()} the caller may read (from the security-narrowed request indices). + * Disabled heterogeneous FROM tests this — not the un-narrowed set — so an index the caller can't read is not + * counted as a mix. + */ + public Set authorizedNonDatasetNames() { + return authorizedNonDatasetNames; + } + /** Explicitly-named datasets absent from the authorized set — surfaced as {@code Unknown index} by the rewrite. */ public Set explicitUnauthorized() { return explicitUnauthorized; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/datasources/DatasetResolver.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/datasources/DatasetResolver.java index c8eefd1ac80e8..80ddd04924de6 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/datasources/DatasetResolver.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/datasources/DatasetResolver.java @@ -131,7 +131,12 @@ void replaceDatasets( new ThreadedActionListener<>(executor, l.delegateFailureAndWrap((delegate, response) -> { resolutions.put( relation, - new DatasetResolution(response.datasets(), response.nonDatasetNames(), response.explicitUnauthorized()) + new DatasetResolution( + response.datasets(), + response.nonDatasetNames(), + response.authorizedNonDatasetNames(), + response.explicitUnauthorized() + ) ); delegate.onResponse(null); })) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/datasources/DatasetRewriter.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/datasources/DatasetRewriter.java index 73f683367f92e..69df6a933655c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/datasources/DatasetRewriter.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/datasources/DatasetRewriter.java @@ -17,6 +17,7 @@ import org.elasticsearch.cluster.metadata.ProjectMetadata; import org.elasticsearch.common.Strings; import org.elasticsearch.common.regex.Regex; +import org.elasticsearch.common.util.FeatureFlag; import org.elasticsearch.index.IndexMode; import org.elasticsearch.transport.RemoteClusterAware; import org.elasticsearch.xpack.esql.VerificationException; @@ -38,6 +39,7 @@ import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.HashMap; import java.util.IdentityHashMap; import java.util.LinkedHashSet; @@ -73,6 +75,9 @@ public final class DatasetRewriter { .indexAbstractionOptions(IndicesOptions.IndexAbstractionOptions.builder().resolveDatasets(true).resolveViews(false).build()) .build(); + /** Gates heterogeneous {@code FROM} (a single FROM mixing a dataset and an index): off in release builds. */ + public static final FeatureFlag HETEROGENEOUS_FROM_FEATURE_FLAG = new FeatureFlag("esql_heterogeneous_from"); + private DatasetRewriter() {} /** @@ -95,21 +100,13 @@ public static DatasetResolution resolve( ? new LinkedHashSet<>() : new LinkedHashSet<>(iner.datasets(projectMetadata, RESOLVER_OPTIONS, indicesRequestOf(authorizedIndices))); - // (b) classify the raw (un-narrowed) patterns into dataset vs non-dataset under an open predicate. + // (b) classify the raw (un-narrowed) patterns into dataset vs non-dataset. This non-dataset set drives the index + // branch, so it stays un-narrowed — downstream field-caps applies security to it, exactly as before. IndexAbstractionResolver resolver = new IndexAbstractionResolver(iner); Map indicesLookup = projectMetadata.getIndicesLookup(); - List localNames = resolver.resolveIndexAbstractions( - Arrays.asList(rawPatterns), - RESOLVER_OPTIONS, - projectMetadata, - componentSelector -> indicesLookup.keySet(), - (name, selector) -> true, - true - ).getLocalIndicesList(); - Set nonDatasetNames = new LinkedHashSet<>(); Set rawDatasetNames = new LinkedHashSet<>(); - for (String name : localNames) { + for (String name : classifyLocalNames(resolver, rawPatterns, projectMetadata, indicesLookup)) { IndexAbstraction abs = indicesLookup.get(name); if (abs == null) { continue; // synthesized name (date math) — neither; skipping is safe for the non-dataset set @@ -121,6 +118,22 @@ public static DatasetResolution resolve( } } + // (c) the non-dataset names the caller is authorized to read — classify the security-narrowed authorizedIndices, + // not the open-predicate raw patterns, so the heterogeneity check ignores an index the caller can't see. + // Unsecured (authorizedIndices == rawPatterns) reuses (b). + Set authorizedNonDatasetNames; + if (Arrays.equals(authorizedIndices, rawPatterns)) { + authorizedNonDatasetNames = nonDatasetNames; + } else { + authorizedNonDatasetNames = new LinkedHashSet<>(); + for (String name : classifyLocalNames(resolver, authorizedIndices, projectMetadata, indicesLookup)) { + IndexAbstraction abs = indicesLookup.get(name); + if (abs != null && abs.getType() != IndexAbstraction.Type.DATASET) { + authorizedNonDatasetNames.add(name); + } + } + } + // Explicit (non-wildcard) dataset names absent from the authorized set — rewriteOne rejects these as Unknown // index rather than silently dropping them from a multi-target FROM. Set explicitUnauthorized = new LinkedHashSet<>(); @@ -136,7 +149,24 @@ public static DatasetResolution resolve( Set result = new LinkedHashSet<>(rawDatasetNames); result.retainAll(resolvedExternalDatasets); - return new DatasetResolution(result, nonDatasetNames, explicitUnauthorized); + return new DatasetResolution(result, nonDatasetNames, authorizedNonDatasetNames, explicitUnauthorized); + } + + /** Expands {@code patterns} to their local abstraction names under an open (authorization-blind) predicate. */ + private static List classifyLocalNames( + IndexAbstractionResolver resolver, + String[] patterns, + ProjectMetadata projectMetadata, + Map indicesLookup + ) { + return resolver.resolveIndexAbstractions( + Arrays.asList(patterns), + RESOLVER_OPTIONS, + projectMetadata, + componentSelector -> indicesLookup.keySet(), + (name, selector) -> true, + true + ).getLocalIndicesList(); } /** Minimal {@link IndicesRequest} carrier so {@link IndexNameExpressionResolver#datasets} can read the names. */ @@ -161,6 +191,16 @@ public IndicesOptions indicesOptions() { * dataset-free project is a no-op. */ public static LogicalPlan rewriteUnsecured(LogicalPlan parsed, ProjectMetadata projectMetadata, IndexNameExpressionResolver iner) { + return rewriteUnsecured(parsed, projectMetadata, iner, HETEROGENEOUS_FROM_FEATURE_FLAG.isEnabled()); + } + + /** Test seam: {@code heterogeneousFromEnabled} overrides the {@link #HETEROGENEOUS_FROM_FEATURE_FLAG} default. */ + static LogicalPlan rewriteUnsecured( + LogicalPlan parsed, + ProjectMetadata projectMetadata, + IndexNameExpressionResolver iner, + boolean heterogeneousFromEnabled + ) { if (projectMetadata == null) { return parsed; } @@ -183,7 +223,7 @@ public static LogicalPlan rewriteUnsecured(LogicalPlan parsed, ProjectMetadata p resolutions.put(r, resolve(raw, raw, projectMetadata, iner)); }); // Unsecured/test path runs without CPS (single local project): never preserve a wildcard for remote resolution. - return rewrite(parsed, projectMetadata, resolutions, false); + return rewrite(parsed, projectMetadata, resolutions, false, heterogeneousFromEnabled); } static boolean hasRemotePattern(List patterns) { @@ -219,6 +259,17 @@ public static LogicalPlan rewrite( ProjectMetadata projectMetadata, Map resolutions, boolean crossProjectEnabled + ) { + return rewrite(parsed, projectMetadata, resolutions, crossProjectEnabled, HETEROGENEOUS_FROM_FEATURE_FLAG.isEnabled()); + } + + /** Test seam: {@code heterogeneousFromEnabled} overrides the {@link #HETEROGENEOUS_FROM_FEATURE_FLAG} default. */ + static LogicalPlan rewrite( + LogicalPlan parsed, + ProjectMetadata projectMetadata, + Map resolutions, + boolean crossProjectEnabled, + boolean heterogeneousFromEnabled ) { if (projectMetadata == null) { return parsed; @@ -233,7 +284,7 @@ public static LogicalPlan rewrite( if (resolution == null) { return r; } - return rewriteOne(r, datasetMetadata, dataSourceMetadata, resolution, crossProjectEnabled); + return rewriteOne(r, datasetMetadata, dataSourceMetadata, resolution, crossProjectEnabled, heterogeneousFromEnabled); }); } @@ -242,7 +293,8 @@ private static LogicalPlan rewriteOne( DatasetMetadata datasets, DataSourceMetadata dataSources, DatasetResolution resolution, - boolean crossProjectEnabled + boolean crossProjectEnabled, + boolean heterogeneousFromEnabled ) { if (resolution.explicitUnauthorized().isEmpty() == false) { // An explicitly-named dataset the caller can't read — same error (and 400) a missing index gives, so an @@ -258,6 +310,19 @@ private static LogicalPlan rewriteOne( // like this and must not be rejected as a "mix". return relation; } + // Heterogeneous FROM disabled: reject iff the caller can read both a dataset and an index-like target in one + // FROM. datasetNames is non-empty here, so a non-empty authorized non-dataset set is the mix. Using the + // authorized (not raw) set keeps an index the caller can't read out of both the decision and the message. Runs + // before any dataset branch is built, so a rejected query never resolves an external schema. + if (heterogeneousFromEnabled == false && resolution.authorizedNonDatasetNames().isEmpty() == false) { + throw new VerificationException( + "Querying both indices and datasets in the same FROM is not supported; dataset(s) requested: " + + truncatedNames(datasetNames) + + ", index(es) requested: " + + truncatedNames(resolution.authorizedNonDatasetNames()) + ); + } + if (relation.indexMode() != null && relation.indexMode() != IndexMode.STANDARD) { String message = switch (relation.indexMode()) { case TIME_SERIES -> "TS command is not supported for datasets; dataset(s) requested: " + datasetNames; @@ -270,7 +335,6 @@ private static LogicalPlan rewriteOne( }; throw new VerificationException(message); } - Set nonDatasetNamesList = resolution.nonDatasetNames(); // One rail for every FROM shape — dataset-only and heterogeneous (index + dataset). The non-remotable-abstraction // CPS rule (a remote view/dataset fails; a remote index of the same name reads both) must hold uniformly, so the @@ -286,7 +350,7 @@ private static LogicalPlan rewriteOne( // concrete name and a wildcard (no double read) and the wildcard's remote half reaches field-caps (closing // #151977's dropped-remote-wildcard gap). The resolveDatasets rail on this branch also fails a remote // dataset/view the wildcard matches. METADATA fields ride along so _index/_id resolve on the index rows. - List indexBranch = new ArrayList<>(nonDatasetNamesList); + List indexBranch = new ArrayList<>(resolution.nonDatasetNames()); if (crossProjectEnabled) { indexBranch.addAll(crossProjectPatternsToPreserve(patternsOf(relation))); } @@ -359,6 +423,23 @@ static boolean anyPatternCouldMatchDataset(List patterns, Set da return false; } + private static final int MAX_NAMES_IN_MESSAGE = 10; + + /** Renders a name list for an error message, capping a wildcard that resolved to many names at {@value #MAX_NAMES_IN_MESSAGE}. */ + private static String truncatedNames(Collection names) { + if (names.size() <= MAX_NAMES_IN_MESSAGE) { + return names.toString(); + } + List shown = new ArrayList<>(MAX_NAMES_IN_MESSAGE); + for (String name : names) { + if (shown.size() == MAX_NAMES_IN_MESSAGE) { + break; + } + shown.add(name); + } + return shown + " (+" + (names.size() - MAX_NAMES_IN_MESSAGE) + " more)"; + } + /** Splits a relation's FROM pattern string into its comma-separated parts. */ static List patternsOf(UnresolvedRelation relation) { return Arrays.asList(Strings.splitStringByCommaToArray(relation.indexPattern().indexPattern())); @@ -474,10 +555,16 @@ private static Map mergeSettings(DataSource parent, Dataset data } /** - * Per-relation result of {@link #resolve}: the external dataset names the relation resolved to (security-filtered - * to those the caller may read), the concrete non-dataset names resolved from the same pattern (drives - * heterogeneous-FROM {@link UnionAll} building), and the explicitly-named datasets absent from the resolved set - * (surfaced by {@link #rewriteOne} as {@code Unknown index}). + * Per-relation result of {@link #resolve}: {@code resolvedExternalDatasets} (authorized datasets the relation + * resolved to); {@code nonDatasetNames} (un-narrowed non-dataset names that build the index branch); + * {@code authorizedNonDatasetNames} (the subset the caller may read — what disabled heterogeneous FROM tests, so an + * index the caller can't see is neither counted as a mix nor named in the error); and {@code explicitUnauthorized} + * (explicitly-named datasets the caller can't read, surfaced by {@link #rewriteOne} as {@code Unknown index}). */ - public record DatasetResolution(Set resolvedExternalDatasets, Set nonDatasetNames, Set explicitUnauthorized) {} + public record DatasetResolution( + Set resolvedExternalDatasets, + Set nonDatasetNames, + Set authorizedNonDatasetNames, + Set explicitUnauthorized + ) {} } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/datasources/DatasetResolverTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/datasources/DatasetResolverTests.java index f408826c07f83..1d7e5ae363617 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/datasources/DatasetResolverTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/datasources/DatasetResolverTests.java @@ -179,7 +179,7 @@ Response extends org.elasticsearch.action.ActionResponse> void doExecute( ) { assertSame(EsqlResolveDatasetAction.TYPE, action); localCalls.incrementAndGet(); - listener.onResponse((Response) new EsqlResolveDatasetAction.Response(Set.of(DATASET_NAME), Set.of(), Set.of())); + listener.onResponse((Response) new EsqlResolveDatasetAction.Response(Set.of(DATASET_NAME), Set.of(), Set.of(), Set.of())); } }; } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/datasources/DatasetRewriterTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/datasources/DatasetRewriterTests.java index e5902d0b4f80a..6d8ec5a7a48c0 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/datasources/DatasetRewriterTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/datasources/DatasetRewriterTests.java @@ -8,7 +8,11 @@ package org.elasticsearch.xpack.esql.datasources; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.Build; +import org.elasticsearch.cluster.metadata.AliasMetadata; import org.elasticsearch.cluster.metadata.DataSourceReference; +import org.elasticsearch.cluster.metadata.DataStream; +import org.elasticsearch.cluster.metadata.DataStreamTestHelper; import org.elasticsearch.cluster.metadata.Dataset; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; @@ -46,6 +50,7 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.not; public class DatasetRewriterTests extends ESTestCase { @@ -153,6 +158,150 @@ public void testMixedMultipleIndicesAndDatasetsProducesUnionAll() { assertThat(union.children().get(2), instanceOf(UnresolvedRelation.class)); } + public void testHeterogeneousFromRejectedWhenFeatureFlagDisabled() { + // With the heterogeneous-FROM gate off (the release-build default), a FROM naming both a concrete index and a + // dataset is rejected instead of producing a UnionAll — the mixed shape is temporarily disabled. + DataSource parent = dataSource("s3_parent", Map.of()); + Dataset dataset = new Dataset("logs", new DataSourceReference("s3_parent"), "s3://logs/", null, Map.of()); + ProjectMetadata project = projectWithIndices(Map.of("s3_parent", parent), Map.of("logs", dataset), Set.of("some_idx")); + + VerificationException ex = expectThrows( + VerificationException.class, + () -> rewriteHeterogeneousDisabled(relationOf("some_idx,logs"), project) + ); + assertThat(ex.getMessage(), containsString("Querying both indices and datasets in the same FROM is not supported")); + } + + public void testWildcardSpanningIndicesAndDatasetsRejectedWhenFeatureFlagDisabled() { + // A wildcard that resolves to both a dataset and an index is a mix and is rejected — the check is on what the + // expression resolves to, not on whether it was a wildcard or an exact name. + DataSource parent = dataSource("s3_parent", Map.of()); + Dataset dataset = new Dataset("logs_ds", new DataSourceReference("s3_parent"), "s3://logs/", null, Map.of()); + ProjectMetadata project = projectWithIndices(Map.of("s3_parent", parent), Map.of("logs_ds", dataset), Set.of("logs_idx")); + + VerificationException ex = expectThrows( + VerificationException.class, + () -> rewriteHeterogeneousDisabled(relationOf("logs_*"), project) + ); + assertThat(ex.getMessage(), containsString("Querying both indices and datasets in the same FROM is not supported")); + } + + public void testWildcardMatchingOnlyDatasetsResolvesWhenFeatureFlagDisabled() { + // A wildcard that resolves to only datasets (no index) is not a mix — it reads the dataset(s), same as an exact + // name. Expression type is never a factor; only what it resolves to is. + DataSource parent = dataSource("s3_parent", Map.of()); + Dataset a = new Dataset("logs_a", new DataSourceReference("s3_parent"), "s3://a/", null, Map.of()); + Dataset b = new Dataset("logs_b", new DataSourceReference("s3_parent"), "s3://b/", null, Map.of()); + ProjectMetadata project = projectWith(Map.of("s3_parent", parent), Map.of("logs_a", a, "logs_b", b)); + + LogicalPlan rewritten = rewriteHeterogeneousDisabled(relationOf("logs_*"), project); + assertThat(rewritten, instanceOf(UnionAll.class)); + assertThat(((UnionAll) rewritten).children(), hasSize(2)); + for (LogicalPlan child : ((UnionAll) rewritten).children()) { + assertThat(child, instanceOf(UnresolvedExternalRelation.class)); + } + } + + public void testSecuredWildcardMatchingDatasetAndUnauthorizedIndexNotRejected() { + // The security case: FROM logs_* matches a dataset the caller can read (logs_ds) and an index it cannot + // (logs_idx). Because the check keys off the caller-authorized non-dataset set, this is NOT a mix — the query is + // not rejected and no error names the unreadable index. The plan proceeds exactly as the flag-on path would: a + // dataset branch plus the (un-narrowed) index branch, which downstream field-caps security-narrows to empty and + // prunes, leaving a dataset-only read. + DataSource parent = dataSource("s3_parent", Map.of()); + Dataset dataset = new Dataset("logs_ds", new DataSourceReference("s3_parent"), "s3://logs/", null, Map.of()); + ProjectMetadata project = projectWithIndices(Map.of("s3_parent", parent), Map.of("logs_ds", dataset), Set.of("logs_idx")); + + // authorized = only the dataset (logs_idx is not readable by this caller): no VerificationException is thrown. + LogicalPlan rewritten = rewriteWithAuthorizedHeterogeneousDisabled(relationOf("logs_*"), project, Set.of("logs_ds")); + assertThat(rewritten, instanceOf(UnionAll.class)); + assertThat(((UnionAll) rewritten).children().get(0), instanceOf(UnresolvedExternalRelation.class)); + } + + public void testSecuredExplicitMixNamesOnlyAuthorizedIndex() { + // When an authorized index is genuinely mixed with a dataset, the reject message names it (it is authorized, so + // no disclosure) — but never a non-dataset the caller can't read. + DataSource parent = dataSource("s3_parent", Map.of()); + Dataset dataset = new Dataset("logs_ds", new DataSourceReference("s3_parent"), "s3://logs/", null, Map.of()); + ProjectMetadata project = projectWithIndices( + Map.of("s3_parent", parent), + Map.of("logs_ds", dataset), + Set.of("ok_idx", "secret_idx") + ); + + VerificationException ex = expectThrows( + VerificationException.class, + () -> rewriteWithAuthorizedHeterogeneousDisabled(relationOf("logs_ds,ok_idx,secret_idx"), project, Set.of("logs_ds", "ok_idx")) + ); + assertThat(ex.getMessage(), containsString("ok_idx")); + assertThat(ex.getMessage(), not(containsString("secret_idx"))); + } + + public void testDatasetOnlyFromUnaffectedWhenHeterogeneousFeatureFlagDisabled() { + // The gate fires only on the mix; a dataset-only FROM still rewrites to an external relation with the flag off. + DataSource parent = dataSource("s3_parent", Map.of()); + Dataset dataset = new Dataset("logs", new DataSourceReference("s3_parent"), "s3://logs/", null, Map.of()); + ProjectMetadata project = projectWith(Map.of("s3_parent", parent), Map.of("logs", dataset)); + + LogicalPlan rewritten = rewriteHeterogeneousDisabled(relationOf("logs"), project); + assertThat(rewritten, instanceOf(UnresolvedExternalRelation.class)); + } + + public void testMultipleDatasetsUnaffectedWhenHeterogeneousFeatureFlagDisabled() { + // A dataset-only multi-target FROM still resolves each dataset with the flag off — no index in the mix. + DataSource parent = dataSource("s3_parent", Map.of()); + Dataset ds1 = new Dataset("ds1", new DataSourceReference("s3_parent"), "s3://a/", null, Map.of()); + Dataset ds2 = new Dataset("ds2", new DataSourceReference("s3_parent"), "s3://b/", null, Map.of()); + ProjectMetadata project = projectWith(Map.of("s3_parent", parent), Map.of("ds1", ds1, "ds2", ds2)); + + LogicalPlan rewritten = rewriteHeterogeneousDisabled(relationOf("ds1,ds2"), project); + assertThat(rewritten, instanceOf(UnionAll.class)); + UnionAll union = (UnionAll) rewritten; + assertThat(union.children(), hasSize(2)); + assertThat(union.children().get(0), instanceOf(UnresolvedExternalRelation.class)); + assertThat(union.children().get(1), instanceOf(UnresolvedExternalRelation.class)); + } + + public void testAliasMixedWithDatasetRejectedWhenFeatureFlagDisabled() { + // An alias is index-like: a dataset mixed with an alias is a mix and is rejected, naming the alias. + ProjectMetadata project = projectWithDatasetAliasAndDataStream(); + + VerificationException ex = expectThrows( + VerificationException.class, + () -> rewriteHeterogeneousDisabled(relationOf("logs_ds,my_alias"), project) + ); + assertThat(ex.getMessage(), containsString("Querying both indices and datasets in the same FROM is not supported")); + assertThat(ex.getMessage(), containsString("my_alias")); + } + + public void testDataStreamMixedWithDatasetRejectedWhenFeatureFlagDisabled() { + // A data stream is index-like too: a dataset mixed with a data stream is a mix and is rejected. This exercises + // the classification the includeDataStreams() request override depends on. + ProjectMetadata project = projectWithDatasetAliasAndDataStream(); + + VerificationException ex = expectThrows( + VerificationException.class, + () -> rewriteHeterogeneousDisabled(relationOf("logs_ds,my_ds"), project) + ); + assertThat(ex.getMessage(), containsString("Querying both indices and datasets in the same FROM is not supported")); + } + + public void testIndexOnlyFromUnaffectedWhenHeterogeneousFeatureFlagDisabled() { + // D = ∅: a FROM that resolves to only an index is untouched with the flag off, even though a dataset is + // registered — the rewriter returns the relation unchanged, exactly as with the flag on. + DataSource parent = dataSource("s3_parent", Map.of()); + Dataset dataset = new Dataset("logs", new DataSourceReference("s3_parent"), "s3://logs/", null, Map.of()); + ProjectMetadata project = projectWithIndices(Map.of("s3_parent", parent), Map.of("logs", dataset), Set.of("some_idx")); + + UnresolvedRelation relation = relationOf("some_idx"); + assertSame(relation, rewriteHeterogeneousDisabled(relation, project)); + } + + public void testHeterogeneousFromFlagDisabledByDefaultInReleaseBuilds() { + // Standard build-gated flag: on in snapshot, off in release — disabled by default in a released build. + assertThat(DatasetRewriter.HETEROGENEOUS_FROM_FEATURE_FLAG.isEnabled(), equalTo(Build.current().isSnapshot())); + } + public void testIndexModeNonStandardRejected() { // Note on coverage: only TIME_SERIES (via TS) and LOOKUP (via LOOKUP JOIN) are user-reachable // through ESQL syntax; LOGSDB has no user-syntax path that constructs an UnresolvedRelation @@ -855,9 +1004,9 @@ public void testResolveClassifiesIndexAndUnauthorizedDatasetPerTarget() { // -- - /** Rewrite with every registered dataset authorized — the unsecured-cluster behavior. */ + /** Rewrite with every registered dataset authorized. Heterogeneous-FROM gate forced on for build-independent results. */ private static LogicalPlan rewrite(LogicalPlan parsed, ProjectMetadata project) { - return DatasetRewriter.rewriteUnsecured(parsed, project, RESOLVER); + return DatasetRewriter.rewriteUnsecured(parsed, project, RESOLVER, true); } /** @@ -867,13 +1016,37 @@ private static LogicalPlan rewrite(LogicalPlan parsed, ProjectMetadata project) */ private static LogicalPlan rewriteWithAuthorized(UnresolvedRelation relation, ProjectMetadata project, Set authorized) { DatasetRewriter.DatasetResolution resolution = resolve(relation.indexPattern().indexPattern(), project, authorized); - return DatasetRewriter.rewrite(relation, project, Map.of(relation, resolution), false); + // Heterogeneous-FROM gate forced on: deterministic across snapshot/release builds (see rewrite() above). + return DatasetRewriter.rewrite(relation, project, Map.of(relation, resolution), false, true); + } + + /** + * Rewrite one relation with every registered dataset authorized (unsecured-cluster behaviour) but the + * heterogeneous-FROM gate forced off, modelling a release build where the feature flag is disabled. + */ + private static LogicalPlan rewriteHeterogeneousDisabled(UnresolvedRelation relation, ProjectMetadata project) { + return DatasetRewriter.rewriteUnsecured(relation, project, RESOLVER, false); + } + + /** + * As {@link #rewriteHeterogeneousDisabled}, but modelling a secured cluster: only {@code authorized} names are + * readable, so the resolution's authorized non-dataset set is narrowed to that subset (the security filter's + * in-flight narrowing of the request indices). + */ + private static LogicalPlan rewriteWithAuthorizedHeterogeneousDisabled( + UnresolvedRelation relation, + ProjectMetadata project, + Set authorized + ) { + DatasetRewriter.DatasetResolution resolution = resolve(relation.indexPattern().indexPattern(), project, authorized); + return DatasetRewriter.rewrite(relation, project, Map.of(relation, resolution), false, false); } /** As {@link #rewriteWithAuthorized}, but with cross-project search (CPS) enabled. */ private static LogicalPlan rewriteWithAuthorizedCps(UnresolvedRelation relation, ProjectMetadata project, Set authorized) { DatasetRewriter.DatasetResolution resolution = resolve(relation.indexPattern().indexPattern(), project, authorized); - return DatasetRewriter.rewrite(relation, project, Map.of(relation, resolution), true); + // Heterogeneous-FROM gate forced on: deterministic across snapshot/release builds (see rewrite() above). + return DatasetRewriter.rewrite(relation, project, Map.of(relation, resolution), true, true); } /** Engine-side resolve of {@code rawPattern} with {@code authorized} as the (filter-narrowed) request indices. */ @@ -923,6 +1096,29 @@ private static ProjectMetadata projectWithIndices( return builder.build(); } + /** Project holding a dataset plus an aliased index and a data stream, to check both classify as index-like. */ + private static ProjectMetadata projectWithDatasetAliasAndDataStream() { + DataSource parent = dataSource("s3_parent", Map.of()); + Dataset dataset = new Dataset("logs_ds", new DataSourceReference("s3_parent"), "s3://logs/", null, Map.of()); + Settings.Builder indexSettings = Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0); + IndexMetadata aliasedIndex = IndexMetadata.builder("aliased_index") + .settings(indexSettings) + .putAlias(AliasMetadata.builder("my_alias").build()) + .build(); + IndexMetadata backing = DataStreamTestHelper.createFirstBackingIndex("my_ds").build(); + DataStream dataStream = DataStreamTestHelper.newInstance("my_ds", List.of(backing.getIndex())); + return ProjectMetadata.builder(ProjectId.DEFAULT) + .putCustom(DataSourceMetadata.TYPE, new DataSourceMetadata(Map.of("s3_parent", parent))) + .datasets(Map.of("logs_ds", dataset)) + .put(aliasedIndex, false) + .put(backing, false) + .put(dataStream) + .build(); + } + private static String tablePathString(UnresolvedExternalRelation relation) { Object value = ((Literal) relation.tablePath()).value(); return value instanceof BytesRef br ? BytesRefs.toString(br) : value.toString();