diff --git a/docs/changelog/154987.yaml b/docs/changelog/154987.yaml new file mode 100644 index 0000000000000..c3a4d8cce27d6 --- /dev/null +++ b/docs/changelog/154987.yaml @@ -0,0 +1,5 @@ +pr: 154987 +summary: Gate wildcard-to-dataset resolution in `FROM` behind a feature flag +area: ES|QL +type: feature +issues: [] diff --git a/x-pack/plugin/esql/build.gradle b/x-pack/plugin/esql/build.gradle index 40b276645668e..7a75f37046b0d 100644 --- a/x-pack/plugin/esql/build.gradle +++ b/x-pack/plugin/esql/build.gradle @@ -165,6 +165,10 @@ tasks.named("test").configure { // inadvertently affected — the per-buffer ledger overhead invalidates timing measurements // (OptimizedReaderPerformanceTests is already excluded from the test task for this reason). systemProperty 'arrow.memory.debug.allocator', 'true' + // Wildcard-dataset resolution is off in release builds, so unit tests that resolve a dataset through a wildcard + // (e.g. DatasetResolverTests) would resolve to indices only during release testing. Enable it so those tests + // exercise the wildcard-dataset path; the exact-name restriction is covered via the boolean test seam. + systemProperty 'es.esql_dataset_wildcards_feature_flag_enabled', 'true' } @@ -196,6 +200,9 @@ tasks.named("internalClusterTest").configure { // Same reason: applying the request filter to datasets is off in release builds, so the conformance and // heterogeneous request-filter ITs would assert against an unfiltered dataset during release testing. systemProperty 'es.esql_request_filter_on_dataset_feature_flag_enabled', 'true' + // Same reason: wildcard-dataset resolution is off in release builds, so the dataset ITs that use FROM to + // match datasets would resolve to indices only during release testing rather than exercising the dataset path. + systemProperty 'es.esql_dataset_wildcards_feature_flag_enabled', 'true' // Data sources require the project encryption key feature (EsqlPlugin fails fast otherwise); pair the flags. systemProperty 'es.project_encryption_key_feature_flag_enabled', 'true' systemProperty 'esql.kibana.docs.dir', "${rootDir}/docs/reference/query-languages/esql/kibana/generated" diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlResolveDatasetAction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlResolveDatasetAction.java index 14cd4c414eef9..e027104700de0 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlResolveDatasetAction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlResolveDatasetAction.java @@ -93,7 +93,8 @@ protected void localClusterStateOperation(Task task, Request request, ProjectSta request.indices(), request.rawPatterns(), project.metadata(), - indexNameExpressionResolver + indexNameExpressionResolver, + DatasetRewriter.DATASET_WILDCARDS_FEATURE_FLAG.isEnabled() ); listener.onResponse( new Response(resolution.resolvedExternalDatasets(), resolution.nonDatasetNames(), resolution.explicitUnauthorized()) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/datasources/DatasetResolver.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/datasources/DatasetResolver.java index c8eefd1ac80e8..cedc83021d988 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/datasources/DatasetResolver.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/datasources/DatasetResolver.java @@ -96,11 +96,12 @@ void replaceDatasets( // Collect the relations worth a round-trip: skip remote-prefixed (datasets are local-only, CCS sees the original // FROM) and skip any relation whose patterns could not match a registered dataset name (ordinary FROM ). + boolean wildcardDatasets = DatasetRewriter.DATASET_WILDCARDS_FEATURE_FLAG.isEnabled(); List relations = new ArrayList<>(); parsed.forEachUp(UnresolvedRelation.class, r -> { List patterns = DatasetRewriter.patternsOf(r); if (DatasetRewriter.hasRemotePattern(patterns) - || DatasetRewriter.anyPatternCouldMatchDataset(patterns, datasetNames) == false) { + || DatasetRewriter.anyPatternCouldMatchDataset(patterns, datasetNames, wildcardDatasets) == false) { return; } relations.add(r); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/datasources/DatasetRewriter.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/datasources/DatasetRewriter.java index 73f683367f92e..ba8993fac3f44 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/datasources/DatasetRewriter.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/datasources/DatasetRewriter.java @@ -17,6 +17,7 @@ import org.elasticsearch.cluster.metadata.ProjectMetadata; import org.elasticsearch.common.Strings; import org.elasticsearch.common.regex.Regex; +import org.elasticsearch.common.util.FeatureFlag; import org.elasticsearch.index.IndexMode; import org.elasticsearch.transport.RemoteClusterAware; import org.elasticsearch.xpack.esql.VerificationException; @@ -57,6 +58,10 @@ * client-side). {@link #resolve} returns, per relation, the authorized concrete dataset names plus whether the * relation also targets non-dataset abstractions. {@link #rewrite}/{@link #rewriteOne} then consume that * {@link DatasetResolution} to build the plan — they no longer resolve, expand, or gate on authorization. + * + *

Whether a wildcard may resolve to a dataset is governed by {@link #DATASET_WILDCARDS_FEATURE_FLAG}: when it is off + * (the release default) a dataset is reachable only by an exact name, so a wildcard resolves to indices exactly as it + * did before datasets existed. Index expressions are otherwise untouched. */ public final class DatasetRewriter { @@ -73,6 +78,14 @@ public final class DatasetRewriter { .indexAbstractionOptions(IndicesOptions.IndexAbstractionOptions.builder().resolveDatasets(true).resolveViews(false).build()) .build(); + /** + * When enabled, a wildcard FROM pattern can resolve to datasets (the full heterogeneous behaviour). When off — the + * default in release builds — datasets are reachable only by an exact name: a wildcard resolves to indices only, + * exactly as it did before datasets existed. Index expressions (wildcards, exclusions, aliases, data streams) are + * never affected; this only governs whether a wildcard may pull in a dataset. + */ + public static final FeatureFlag DATASET_WILDCARDS_FEATURE_FLAG = new FeatureFlag("esql_dataset_wildcards"); + private DatasetRewriter() {} /** @@ -81,12 +94,17 @@ private DatasetRewriter() {} * {@link UnionAll} building), and the explicitly-named-but-unauthorized datasets — which {@link #rewriteOne} * surfaces as {@code Unknown index} (400), the same error a missing index gives, so an unauthorized dataset * can't be told apart from a missing name. + * + * @param wildcardDatasets when {@code false} (the release-build default, see {@link #DATASET_WILDCARDS_FEATURE_FLAG}) + * a dataset is kept only if it was named exactly; a wildcard that also matched it drops it, + * so the wildcard resolves to indices only. */ public static DatasetResolution resolve( String[] authorizedIndices, String[] rawPatterns, ProjectMetadata projectMetadata, - IndexNameExpressionResolver iner + IndexNameExpressionResolver iner, + boolean wildcardDatasets ) { // (a) resolved external datasets: request.indices(), which the security filter already narrowed to the // read-privileged subset on a secured cluster (and equals rawPatterns without security). Empty short-circuits, @@ -121,14 +139,15 @@ public static DatasetResolution resolve( } } - // Explicit (non-wildcard) dataset names absent from the authorized set — rewriteOne rejects these as Unknown - // index rather than silently dropping them from a multi-target FROM. + // The exact (non-wildcard) FROM names, date-math resolved: the only names that reach a dataset when + // wildcard-dataset matching is off, and the set an explicitly-named dataset must come from to be flagged + // unauthorized. Computed once and shared by both uses below. + Set exact = exactNames(Arrays.asList(rawPatterns)); + + // Explicitly-named datasets absent from the authorized set — rewriteOne rejects these as Unknown index rather + // than silently dropping them from a multi-target FROM. Set explicitUnauthorized = new LinkedHashSet<>(); - for (String pattern : rawPatterns) { - if (pattern.isEmpty() || pattern.charAt(0) == '-' || Regex.isSimpleMatchPattern(pattern)) { - continue; - } - String name = IndexNameExpressionResolver.resolveDateMathExpression(pattern); + for (String name : exact) { if (rawDatasetNames.contains(name) && resolvedExternalDatasets.contains(name) == false) { explicitUnauthorized.add(name); } @@ -136,6 +155,11 @@ public static DatasetResolution resolve( Set result = new LinkedHashSet<>(rawDatasetNames); result.retainAll(resolvedExternalDatasets); + if (wildcardDatasets == false) { + // Datasets are reachable only by an exact name: a wildcard that also matched a dataset drops it, leaving the + // wildcard to resolve to indices only (unchanged index behaviour). Explicit names still resolve as datasets. + result.retainAll(exact); + } return new DatasetResolution(result, nonDatasetNames, explicitUnauthorized); } @@ -161,6 +185,16 @@ public IndicesOptions indicesOptions() { * dataset-free project is a no-op. */ public static LogicalPlan rewriteUnsecured(LogicalPlan parsed, ProjectMetadata projectMetadata, IndexNameExpressionResolver iner) { + return rewriteUnsecured(parsed, projectMetadata, iner, DATASET_WILDCARDS_FEATURE_FLAG.isEnabled()); + } + + /** Package-private overload letting tests drive the wildcard-dataset flag without the {@code static final} field. */ + static LogicalPlan rewriteUnsecured( + LogicalPlan parsed, + ProjectMetadata projectMetadata, + IndexNameExpressionResolver iner, + boolean wildcardDatasets + ) { if (projectMetadata == null) { return parsed; } @@ -174,13 +208,13 @@ public static LogicalPlan rewriteUnsecured(LogicalPlan parsed, ProjectMetadata p return; } List patterns = patternsOf(r); - if (hasRemotePattern(patterns) || anyPatternCouldMatchDataset(patterns, datasetNames) == false) { + if (hasRemotePattern(patterns) || anyPatternCouldMatchDataset(patterns, datasetNames, wildcardDatasets) == false) { return; } // Unsecured: the (un-narrowed) raw patterns are the authorized indices — every registered dataset matched // by the pattern is authorized, so resolve() returns it. String[] raw = patterns.toArray(String[]::new); - resolutions.put(r, resolve(raw, raw, projectMetadata, iner)); + resolutions.put(r, resolve(raw, raw, projectMetadata, iner, wildcardDatasets)); }); // Unsecured/test path runs without CPS (single local project): never preserve a wildcard for remote resolution. return rewrite(parsed, projectMetadata, resolutions, false); @@ -338,10 +372,20 @@ private static LogicalPlan rewriteOne( * name. False positives are fine (slow path runs); false negatives would miss datasets, so this * must be at least as permissive as the full resolver. */ - static boolean anyPatternCouldMatchDataset(List patterns, Set datasetNames) { + static boolean anyPatternCouldMatchDataset(List patterns, Set datasetNames, boolean wildcardDatasets) { if (datasetNames.isEmpty()) { return false; } + if (wildcardDatasets == false) { + // Datasets match only exact names — reuse the same exact-name notion resolve() applies (single source of + // truth), so the pre-check and the real resolution can't drift. + for (String name : exactNames(patterns)) { + if (datasetNames.contains(name)) { + return true; + } + } + return false; + } for (String pattern : patterns) { if (pattern.isEmpty() || pattern.charAt(0) == '-') { continue; @@ -359,6 +403,23 @@ static boolean anyPatternCouldMatchDataset(List patterns, Set da return false; } + /** The exact (non-wildcard, non-exclusion) names in {@code patterns}, with date math evaluated. */ + private static Set exactNames(List patterns) { + Set exact = new LinkedHashSet<>(); + for (String pattern : patterns) { + if (isWildcardOrExclusion(pattern)) { + continue; + } + exact.add(IndexNameExpressionResolver.resolveDateMathExpression(pattern)); + } + return exact; + } + + /** A FROM part that contributes no exact name: empty, an exclusion ({@code -x}), or a wildcard. */ + private static boolean isWildcardOrExclusion(String pattern) { + return pattern.isEmpty() || pattern.charAt(0) == '-' || Regex.isSimpleMatchPattern(pattern); + } + /** Splits a relation's FROM pattern string into its comma-separated parts. */ static List patternsOf(UnresolvedRelation relation) { return Arrays.asList(Strings.splitStringByCommaToArray(relation.indexPattern().indexPattern())); @@ -426,7 +487,7 @@ static List crossProjectExactNameShadows(UnresolvedRelati Set seen = new LinkedHashSet<>(); for (int i = 0; i < patterns.size(); i++) { String pattern = patterns.get(i); - if (pattern.isEmpty() || pattern.charAt(0) == '-' || Regex.isSimpleMatchPattern(pattern)) { + if (isWildcardOrExclusion(pattern)) { continue; } // Resolve date-math so a literal-named dataset with a date suffix matches its authorized name. diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/datasources/DatasetRewriterTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/datasources/DatasetRewriterTests.java index e5902d0b4f80a..59f61fe68a09c 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/datasources/DatasetRewriterTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/datasources/DatasetRewriterTests.java @@ -8,7 +8,10 @@ package org.elasticsearch.xpack.esql.datasources; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.cluster.metadata.AliasMetadata; import org.elasticsearch.cluster.metadata.DataSourceReference; +import org.elasticsearch.cluster.metadata.DataStream; +import org.elasticsearch.cluster.metadata.DataStreamTestHelper; import org.elasticsearch.cluster.metadata.Dataset; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; @@ -255,6 +258,190 @@ public void testWildcardMatchesDatasets() { assertThat(union.children().get(1), instanceOf(UnresolvedExternalRelation.class)); } + public void testWildcardMatchingOnlyDatasetsExcludesThemWhenFlagOff() { + // With wildcard-dataset matching off, FROM logs_* does not resolve any dataset: the relation is left untouched + // and flows to normal index resolution (which excludes datasets), so it means "matching indices only". + DataSource parent = dataSource("s3_parent", Map.of()); + Dataset a = new Dataset("logs_a", new DataSourceReference("s3_parent"), "s3://a/", null, Map.of()); + Dataset b = new Dataset("logs_b", new DataSourceReference("s3_parent"), "s3://b/", null, Map.of()); + ProjectMetadata project = projectWith(Map.of("s3_parent", parent), Map.of("logs_a", a, "logs_b", b)); + + UnresolvedRelation relation = relationOf("logs_*"); + assertSame(relation, rewriteWildcardsDisabled(relation, project)); + } + + public void testWildcardSpanningIndexAndDatasetResolvesToIndexOnlyWhenFlagOff() { + // FROM logs_* matching an index and a dataset resolves to the index only (dataset dropped) — not rejected. The + // wildcard is left untouched for normal resolution, which excludes datasets. + DataSource parent = dataSource("s3_parent", Map.of()); + Dataset dataset = new Dataset("logs_ds", new DataSourceReference("s3_parent"), "s3://logs/", null, Map.of()); + ProjectMetadata project = projectWithIndices(Map.of("s3_parent", parent), Map.of("logs_ds", dataset), Set.of("logs_idx")); + + UnresolvedRelation relation = relationOf("logs_*"); + assertSame(relation, rewriteWildcardsDisabled(relation, project)); + } + + public void testExactDatasetNameResolvesWhenFlagOff() { + // A dataset named exactly still resolves with the flag off — only wildcards are restricted. + DataSource parent = dataSource("s3_parent", Map.of()); + Dataset dataset = new Dataset("logs_ds", new DataSourceReference("s3_parent"), "s3://logs/", null, Map.of()); + ProjectMetadata project = projectWith(Map.of("s3_parent", parent), Map.of("logs_ds", dataset)); + + LogicalPlan rewritten = rewriteWildcardsDisabled(relationOf("logs_ds"), project); + assertThat(rewritten, instanceOf(UnresolvedExternalRelation.class)); + } + + public void testPrefixWildcardDoesNotResolveDatasetWhenFlagOff() { + // A prefix wildcard is still a wildcard: it does not reach a dataset with the flag off (datasets need an exact + // name), so the relation is untouched and resolves to indices only. + DataSource parent = dataSource("s3_parent", Map.of()); + Dataset dataset = new Dataset("logs_ds", new DataSourceReference("s3_parent"), "s3://logs/", null, Map.of()); + ProjectMetadata project = projectWith(Map.of("s3_parent", parent), Map.of("logs_ds", dataset)); + + UnresolvedRelation relation = relationOf("logs_d*"); + assertSame(relation, rewriteWildcardsDisabled(relation, project)); + } + + public void testExplicitDatasetAndIndexMixAllowedWhenFlagOff() { + // An explicit dataset named alongside an index still produces the heterogeneous UnionAll with the flag off — + // only accidental wildcard mixing is prevented, not deliberate explicit mixing. + DataSource parent = dataSource("s3_parent", Map.of()); + Dataset dataset = new Dataset("logs_ds", new DataSourceReference("s3_parent"), "s3://logs/", null, Map.of()); + ProjectMetadata project = projectWithIndices(Map.of("s3_parent", parent), Map.of("logs_ds", dataset), Set.of("some_idx")); + + LogicalPlan rewritten = rewriteWildcardsDisabled(relationOf("logs_ds,some_idx"), project); + assertThat(rewritten, instanceOf(UnionAll.class)); + UnionAll union = (UnionAll) rewritten; + assertThat(union.children(), hasSize(2)); + assertThat(union.children().get(0), instanceOf(UnresolvedExternalRelation.class)); + assertThat(union.children().get(1), instanceOf(UnresolvedRelation.class)); + } + + public void testWildcardDoesNotPullDatasetAlongsideExactDatasetWhenFlagOff() { + // FROM logs_a, logs_* with BOTH logs_a and logs_b registered as datasets: the exact name resolves logs_a and + // dispatches, but the wildcard must not pull in logs_b. This is the discriminating case for resolve()'s + // exact-name filter — without it, resolve would return {logs_a, logs_b} and a wildcard would read a dataset. + DataSource parent = dataSource("s3_parent", Map.of()); + Dataset a = new Dataset("logs_a", new DataSourceReference("s3_parent"), "s3://a/", null, Map.of()); + Dataset b = new Dataset("logs_b", new DataSourceReference("s3_parent"), "s3://b/", null, Map.of()); + ProjectMetadata project = projectWith(Map.of("s3_parent", parent), Map.of("logs_a", a, "logs_b", b)); + + DatasetRewriter.DatasetResolution resolution = resolve("logs_a,logs_*", project, Set.of("logs_a", "logs_b"), false); + assertThat(resolution.resolvedExternalDatasets(), containsInAnyOrder("logs_a")); + + // Rewrite-level twin: only logs_a becomes an external relation; logs_b is not pulled in by the wildcard. + LogicalPlan rewritten = rewriteWildcardsDisabled(relationOf("logs_a,logs_*"), project); + assertThat(rewritten, instanceOf(UnresolvedExternalRelation.class)); + assertThat(tablePathString((UnresolvedExternalRelation) rewritten), equalTo("s3://a/")); + } + + public void testWildcardIndexAndExactDatasetUnionWhenFlagOff() { + // FROM foo*, bar — a wildcard over indices plus an explicitly-named dataset — still unions: the wildcard + // resolves to indices (datasets excluded), the exact name resolves the dataset. + DataSource parent = dataSource("s3_parent", Map.of()); + Dataset bar = new Dataset("bar", new DataSourceReference("s3_parent"), "s3://bar/", null, Map.of()); + ProjectMetadata project = projectWithIndices(Map.of("s3_parent", parent), Map.of("bar", bar), Set.of("foo_1", "foo_2")); + + LogicalPlan rewritten = rewriteWildcardsDisabled(relationOf("foo*,bar"), project); + assertThat(rewritten, instanceOf(UnionAll.class)); + UnionAll union = (UnionAll) rewritten; + assertThat(union.children(), hasSize(2)); + // Dataset branch first (buildDatasetBranch loop), then the index branch for the wildcard. + assertThat(union.children().get(0), instanceOf(UnresolvedExternalRelation.class)); + assertThat(union.children().get(1), instanceOf(UnresolvedRelation.class)); + } + + public void testAnyPatternCouldMatchDatasetRespectsFlag() { + // The cheap pre-check that gates dispatch: a wildcard could match a dataset only when wildcard-dataset matching + // is on; an exact name always could; an exclusion never does; no registered datasets short-circuits. + Set datasets = Set.of("logs_ds"); + assertTrue(DatasetRewriter.anyPatternCouldMatchDataset(List.of("logs_*"), datasets, true)); + assertFalse(DatasetRewriter.anyPatternCouldMatchDataset(List.of("logs_*"), datasets, false)); + assertTrue(DatasetRewriter.anyPatternCouldMatchDataset(List.of("logs_ds"), datasets, true)); + assertTrue(DatasetRewriter.anyPatternCouldMatchDataset(List.of("logs_ds"), datasets, false)); + assertFalse(DatasetRewriter.anyPatternCouldMatchDataset(List.of("-logs_ds"), datasets, false)); + assertFalse(DatasetRewriter.anyPatternCouldMatchDataset(List.of("logs_ds"), Set.of(), false)); + } + + public void testBareWildcardExcludesDatasetWhenFlagOff() { + // FROM * with the flag off resolves to indices only — the registered dataset is not swept in. + DataSource parent = dataSource("s3_parent", Map.of()); + Dataset dataset = new Dataset("logs_ds", new DataSourceReference("s3_parent"), "s3://logs/", null, Map.of()); + ProjectMetadata project = projectWithIndices(Map.of("s3_parent", parent), Map.of("logs_ds", dataset), Set.of("some_idx")); + + UnresolvedRelation relation = relationOf("*"); + assertSame(relation, rewriteWildcardsDisabled(relation, project)); + } + + public void testMultipleExactDatasetsResolveWhenFlagOff() { + // Several datasets named exactly still union with the flag off — only wildcards are restricted. + DataSource parent = dataSource("s3_parent", Map.of()); + Dataset ds1 = new Dataset("ds1", new DataSourceReference("s3_parent"), "s3://a/", null, Map.of()); + Dataset ds2 = new Dataset("ds2", new DataSourceReference("s3_parent"), "s3://b/", null, Map.of()); + ProjectMetadata project = projectWith(Map.of("s3_parent", parent), Map.of("ds1", ds1, "ds2", ds2)); + + LogicalPlan rewritten = rewriteWildcardsDisabled(relationOf("ds1,ds2"), project); + assertThat(rewritten, instanceOf(UnionAll.class)); + UnionAll union = (UnionAll) rewritten; + assertThat(union.children(), hasSize(2)); + assertThat(union.children().get(0), instanceOf(UnresolvedExternalRelation.class)); + assertThat(union.children().get(1), instanceOf(UnresolvedExternalRelation.class)); + } + + public void testExclusionOfExactDatasetResolvesToNoDatasetWhenFlagOff() { + // An exact dataset excluded by a trailing -pattern resolves to nothing — the exclusion removes it during + // abstraction expansion, so no dataset survives (flag-independent, but pinned here for the flag-off path). + DataSource parent = dataSource("s3_parent", Map.of()); + Dataset dataset = new Dataset("logs_ds", new DataSourceReference("s3_parent"), "s3://logs/", null, Map.of()); + ProjectMetadata project = projectWith(Map.of("s3_parent", parent), Map.of("logs_ds", dataset)); + + UnresolvedRelation relation = relationOf("logs_ds,-logs_ds"); + assertSame(relation, rewriteWildcardsDisabled(relation, project)); + } + + public void testExplicitUnauthorizedDatasetSurfacedWhenFlagOff() { + // An explicitly-named unauthorized dataset is still flagged (→ Unknown index) with the flag off — the exact + // name reaches resolution and explicitUnauthorized is computed before the wildcard filter. + DataSource parent = dataSource("s3_parent", Map.of()); + Dataset dataset = new Dataset("secret_ds", new DataSourceReference("s3_parent"), "s3://s/", null, Map.of()); + ProjectMetadata project = projectWith(Map.of("s3_parent", parent), Map.of("secret_ds", dataset)); + + DatasetRewriter.DatasetResolution resolution = resolve("secret_ds", project, Set.of(), false); + assertThat(resolution.explicitUnauthorized(), containsInAnyOrder("secret_ds")); + assertThat(resolution.resolvedExternalDatasets(), hasSize(0)); + } + + public void testDatasetWildcardFlagFlipsBehaviourForSameQuery() { + // The same FROM logs_* over a dataset: with the flag ON it reads the dataset, with it OFF it does not (the + // relation is untouched). Proves the flag actually controls wildcard-to-dataset resolution. + DataSource parent = dataSource("s3_parent", Map.of()); + Dataset dataset = new Dataset("logs_ds", new DataSourceReference("s3_parent"), "s3://logs/", null, Map.of()); + ProjectMetadata project = projectWith(Map.of("s3_parent", parent), Map.of("logs_ds", dataset)); + + // Flag on: the wildcard resolves the dataset. + assertThat(rewrite(relationOf("logs_*"), project), instanceOf(UnresolvedExternalRelation.class)); + // Flag off: the wildcard is left untouched (indices only). + UnresolvedRelation relation = relationOf("logs_*"); + assertSame(relation, rewriteWildcardsDisabled(relation, project)); + } + + public void testAliasAlongsideExactDatasetUnionsWhenFlagOff() { + // An alias is index-like: FROM logs_ds, my_alias unions the exact dataset with the alias, flag off. + LogicalPlan rewritten = rewriteWildcardsDisabled(relationOf("logs_ds,my_alias"), projectWithDatasetAliasAndDataStream()); + assertThat(rewritten, instanceOf(UnionAll.class)); + UnionAll union = (UnionAll) rewritten; + assertThat(union.children(), hasSize(2)); + assertThat(union.children().get(0), instanceOf(UnresolvedExternalRelation.class)); + assertThat(union.children().get(1), instanceOf(UnresolvedRelation.class)); + } + + public void testDataStreamAlongsideExactDatasetUnionsWhenFlagOff() { + // A data stream is index-like too: FROM logs_ds, my_ds unions the exact dataset with the data stream, flag off. + LogicalPlan rewritten = rewriteWildcardsDisabled(relationOf("logs_ds,my_ds"), projectWithDatasetAliasAndDataStream()); + assertThat(rewritten, instanceOf(UnionAll.class)); + assertThat(((UnionAll) rewritten).children().get(0), instanceOf(UnresolvedExternalRelation.class)); + } + public void testWildcardMatchingNoDatasetsLeavesPlanUnchanged() { // Pattern matching no datasets and no indices is left for the analyzer to handle (which will // error appropriately based on the request's IndicesOptions). @@ -855,9 +1042,17 @@ public void testResolveClassifiesIndexAndUnauthorizedDatasetPerTarget() { // -- - /** Rewrite with every registered dataset authorized — the unsecured-cluster behavior. */ + /** + * Rewrite with every registered dataset authorized — the unsecured-cluster behavior. Wildcard-dataset matching is + * forced on so the wildcard cases are deterministic regardless of the build's ambient feature flag default. + */ private static LogicalPlan rewrite(LogicalPlan parsed, ProjectMetadata project) { - return DatasetRewriter.rewriteUnsecured(parsed, project, RESOLVER); + return DatasetRewriter.rewriteUnsecured(parsed, project, RESOLVER, true); + } + + /** As {@link #rewrite}, but with wildcard-dataset matching off (release-build behavior: datasets by exact name). */ + private static LogicalPlan rewriteWildcardsDisabled(LogicalPlan parsed, ProjectMetadata project) { + return DatasetRewriter.rewriteUnsecured(parsed, project, RESOLVER, false); } /** @@ -878,8 +1073,17 @@ private static LogicalPlan rewriteWithAuthorizedCps(UnresolvedRelation relation, /** Engine-side resolve of {@code rawPattern} with {@code authorized} as the (filter-narrowed) request indices. */ private static DatasetRewriter.DatasetResolution resolve(String rawPattern, ProjectMetadata project, Set authorized) { + return resolve(rawPattern, project, authorized, true); + } + + private static DatasetRewriter.DatasetResolution resolve( + String rawPattern, + ProjectMetadata project, + Set authorized, + boolean wildcardDatasets + ) { String[] raw = Strings.splitStringByCommaToArray(rawPattern); - return DatasetRewriter.resolve(authorized.toArray(String[]::new), raw, project, RESOLVER); + return DatasetRewriter.resolve(authorized.toArray(String[]::new), raw, project, RESOLVER, wildcardDatasets); } private static UnresolvedRelation relationOf(String pattern) { @@ -923,6 +1127,29 @@ private static ProjectMetadata projectWithIndices( return builder.build(); } + /** Project holding a dataset (logs_ds), an aliased index (my_alias), and a data stream (my_ds). */ + private static ProjectMetadata projectWithDatasetAliasAndDataStream() { + DataSource parent = dataSource("s3_parent", Map.of()); + Dataset dataset = new Dataset("logs_ds", new DataSourceReference("s3_parent"), "s3://logs/", null, Map.of()); + Settings.Builder indexSettings = Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0); + IndexMetadata aliasedIndex = IndexMetadata.builder("aliased_index") + .settings(indexSettings) + .putAlias(AliasMetadata.builder("my_alias").build()) + .build(); + IndexMetadata backing = DataStreamTestHelper.createFirstBackingIndex("my_ds").build(); + DataStream dataStream = DataStreamTestHelper.newInstance("my_ds", List.of(backing.getIndex())); + return ProjectMetadata.builder(ProjectId.DEFAULT) + .putCustom(DataSourceMetadata.TYPE, new DataSourceMetadata(Map.of("s3_parent", parent))) + .datasets(Map.of("logs_ds", dataset)) + .put(aliasedIndex, false) + .put(backing, false) + .put(dataStream) + .build(); + } + private static String tablePathString(UnresolvedExternalRelation relation) { Object value = ((Literal) relation.tablePath()).value(); return value instanceof BytesRef br ? BytesRefs.toString(br) : value.toString();