diff --git a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java index d7c2925d03f88..852a62dc02b99 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java +++ b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java @@ -211,6 +211,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings { IndexSettings.LOGSDB_SORT_ON_MESSAGE_TEMPLATE, IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD, IndexSettings.SLICE_ENABLED, + IndexSettings.FLATTENED_UNMAPPED_FIELDS_ENABLED, IndexSettings.PREFER_ILM_SETTING, DataStreamFailureStoreDefinition.FAILURE_STORE_DEFINITION_VERSION_SETTING, FieldMapper.SYNTHETIC_SOURCE_KEEP_INDEX_SETTING, diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index 13e6df69b852f..e1af676e9ca45 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -33,6 +33,7 @@ import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.SeqNoFieldMapper; import org.elasticsearch.index.mapper.SourceFieldMapper; +import org.elasticsearch.index.mapper.flattened.FlattenedFieldMapper; import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; import org.elasticsearch.index.translog.Translog; import org.elasticsearch.indices.IndicesRequestCache; @@ -855,6 +856,56 @@ public static boolean isValidCodecForSyntheticId(String codecName, IndexVersion return "false"; }, Property.IndexScope, Property.Final); + /** + * Internal, feature-flagged setting that turns on the implicit flattened {@code _unmapped} sink absorbing unmapped fields. + * Only permitted in strict columnar index modes; rejected (as an unknown setting) when the feature flag is off. + * Temporary scaffolding: to be removed once all sink read paths land, when enablement becomes feature flag + strict columnar mode + * with no per-index opt-out. Do not document or expose this setting. + */ + public static final Setting FLATTENED_UNMAPPED_FIELDS_ENABLED = Setting.boolSetting( + "index.mapping.flattened_unmapped_fields.enabled", + false, + new Setting.Validator<>() { + @Override + public void validate(Boolean enabled) { + if (enabled && FlattenedFieldMapper.UNMAPPED_FIELDS_FEATURE_FLAG.isEnabled() == false) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "unknown setting [%s] please check that any required plugins are installed, " + + "or check the breaking changes documentation for removed settings", + FLATTENED_UNMAPPED_FIELDS_ENABLED.getKey() + ) + ); + } + } + + @Override + public void validate(Boolean enabled, Map, Object> settings) { + if (enabled) { + var indexMode = (IndexMode) settings.get(MODE); + if (indexMode.isStrictColumnar() == false) { + throw new IllegalArgumentException( + String.format( + Locale.ROOT, + "The setting [%s] is only permitted in strict columnar index modes. Current mode: [%s].", + FLATTENED_UNMAPPED_FIELDS_ENABLED.getKey(), + indexMode.getName() + ) + ); + } + } + } + + @Override + public Iterator> settings() { + return List.>of(MODE).iterator(); + } + }, + Property.IndexScope, + Property.Final + ); + public static final Setting INDEX_MAPPER_SOURCE_MODE_SETTING = Setting.enumSetting( SourceFieldMapper.Mode.class, settings -> { @@ -1290,6 +1341,7 @@ public Iterator> settings() { private final boolean logsdbSortOnHostName; private final boolean logsdbAddHostNameField; private final boolean sliceEnabled; + private final boolean flattenedUnmappedFieldsEnabled; private volatile long retentionLeaseMillis; /** @@ -1435,6 +1487,10 @@ public boolean isSliceEnabled() { return sliceEnabled; } + public boolean isFlattenedUnmappedFieldsEnabled() { + return flattenedUnmappedFieldsEnabled; + } + /** * Returns true if the index is in logsdb mode and needs a [host.name] keyword field. The default is false */ @@ -1529,6 +1585,7 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti maxRegexLength = scopedSettings.get(MAX_REGEX_LENGTH_SETTING); this.mergePolicyConfig = new MergePolicyConfig(logger, this); sliceEnabled = scopedSettings.get(SLICE_ENABLED); + flattenedUnmappedFieldsEnabled = scopedSettings.get(FLATTENED_UNMAPPED_FIELDS_ENABLED); this.indexSortConfig = new IndexSortConfig(this); searchIdleAfter = scopedSettings.get(INDEX_SEARCH_IDLE_AFTER); defaultPipeline = scopedSettings.get(DEFAULT_PIPELINE); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java index 376f11a0bc132..16a878c97c279 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java @@ -26,6 +26,7 @@ import org.elasticsearch.index.SliceIndexing; import org.elasticsearch.index.fielddata.FieldDataContext; import org.elasticsearch.index.fielddata.IndexFieldDataCache; +import org.elasticsearch.index.mapper.flattened.FlattenedFieldMapper; import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; @@ -945,7 +946,14 @@ private static void parseNullValue(DocumentParserContext context, String lastFie // TODO: passing null to an object seems bogus? parseObjectOrField(context, mapper); } else { - ensureNotStrict(context.resolveDynamic(lastFieldName), context, lastFieldName); + ObjectMapper.Dynamic dynamic = context.resolveDynamic(lastFieldName); + ensureNotStrict(dynamic, context, lastFieldName); + if (dynamic == ObjectMapper.Dynamic.FLATTENED) { + // Absorb the null slot so columnar array order (e.g. [1, null, 3]) is preserved for the unmapped field. + FlattenedFieldMapper sink = (FlattenedFieldMapper) context.mappingLookup() + .getMapper(FlattenedFieldMapper.UNMAPPED_SINK_NAME); + sink.absorbUnmappedNull(context, context.path().pathAsText(lastFieldName)); + } } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java b/server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java index 3d0bd49f66a1b..9e2e2b9e57545 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java @@ -15,6 +15,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.time.DateFormatter; import org.elasticsearch.index.mapper.ObjectMapper.Dynamic; +import org.elasticsearch.index.mapper.flattened.FlattenedFieldMapper; import org.elasticsearch.script.ScriptCompiler; import org.elasticsearch.xcontent.XContentParser; @@ -32,6 +33,7 @@ final class DynamicFieldsBuilder { private static final Concrete CONCRETE = new Concrete(DocumentParser::parseObjectOrField); static final DynamicFieldsBuilder DYNAMIC_TRUE = new DynamicFieldsBuilder(CONCRETE); static final DynamicFieldsBuilder DYNAMIC_RUNTIME = new DynamicFieldsBuilder(new Runtime()); + static final DynamicFieldsBuilder DYNAMIC_FLATTENED = new DynamicFieldsBuilder(new Absorb()); private final Strategy strategy; @@ -143,12 +145,11 @@ boolean createDynamicFieldFromValue(final DocumentParserContext context, String () -> strategy.newDynamicBooleanField(context, name) ); } else if (token == XContentParser.Token.VALUE_EMBEDDED_OBJECT) { - // runtime binary fields are not supported, hence binary objects always get created as concrete fields return createDynamicField( context, name, DynamicTemplate.XContentFieldType.BINARY, - () -> CONCRETE.newDynamicBinaryField(context, name) + () -> strategy.newDynamicBinaryField(context, name) ); } else { return createDynamicStringFieldFromTemplate(context, name); @@ -293,6 +294,8 @@ private interface Strategy { boolean newDynamicBooleanField(DocumentParserContext context, String name) throws IOException; boolean newDynamicDateField(DocumentParserContext context, String name, DateFormatter dateFormatter) throws IOException; + + boolean newDynamicBinaryField(DocumentParserContext context, String name) throws IOException; } /** @@ -382,7 +385,8 @@ public boolean newDynamicDateField(DocumentParserContext context, String name, D ); } - boolean newDynamicBinaryField(DocumentParserContext context, String name) throws IOException { + @Override + public boolean newDynamicBinaryField(DocumentParserContext context, String name) throws IOException { return createDynamicField(new BinaryFieldMapper.Builder(name, SourceFieldMapper.isSynthetic(context.indexSettings())), context); } } @@ -431,5 +435,56 @@ public boolean newDynamicDateField(DocumentParserContext context, String name, D context ); } + + @Override + public boolean newDynamicBinaryField(DocumentParserContext context, String name) throws IOException { + // Runtime binary fields are unsupported, so fall back to the concrete binary field (previous behavior). + return CONCRETE.newDynamicBinaryField(context, name); + } + } + + /** + * Absorbs unmapped leaf values into the implicit flattened {@code _unmapped} sink. Used when the resolved dynamic mode is + * {@link ObjectMapper.Dynamic#FLATTENED}. Dynamic templates still win, since {@link #createDynamicFieldFromValue} tries a matching + * template before falling back to the strategy. + */ + private static final class Absorb implements Strategy { + + private static boolean absorb(DocumentParserContext context, String name) throws IOException { + FlattenedFieldMapper sink = (FlattenedFieldMapper) context.mappingLookup().getMapper(FlattenedFieldMapper.UNMAPPED_SINK_NAME); + sink.absorbUnmappedValue(context, context.path().pathAsText(name)); + return true; + } + + @Override + public boolean newDynamicStringField(DocumentParserContext context, String name) throws IOException { + return absorb(context, name); + } + + @Override + public boolean newDynamicLongField(DocumentParserContext context, String name) throws IOException { + return absorb(context, name); + } + + @Override + public boolean newDynamicDoubleField(DocumentParserContext context, String name) throws IOException { + return absorb(context, name); + } + + @Override + public boolean newDynamicBooleanField(DocumentParserContext context, String name) throws IOException { + return absorb(context, name); + } + + @Override + public boolean newDynamicDateField(DocumentParserContext context, String name, DateFormatter dateFormatter) throws IOException { + return absorb(context, name); + } + + @Override + public boolean newDynamicBinaryField(DocumentParserContext context, String name) { + // Binary (embedded object) values cannot be stringified into flattened; dropped in this phase (feature-flagged, columnar-only). + return true; + } } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappingParser.java b/server/src/main/java/org/elasticsearch/index/mapper/MappingParser.java index 139d3993f77f4..c8b2f8d0b174d 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MappingParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MappingParser.java @@ -15,6 +15,7 @@ import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.core.Nullable; import org.elasticsearch.index.mapper.MapperService.MergeReason; +import org.elasticsearch.index.mapper.flattened.FlattenedFieldMapper; import org.elasticsearch.xcontent.XContentType; import java.util.Collections; @@ -149,6 +150,14 @@ MappingBuilder parseToBuilder(@Nullable String type, MergeReason reason, Map(), mappingParserContext) + ); + } + Map metadataBuilders = metadataBuildersSupplier.get(); Map meta = null; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java index 4bd064c561367..d4fbaa9921e67 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java @@ -24,6 +24,7 @@ import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.mapper.MapperService.MergeReason; +import org.elasticsearch.index.mapper.flattened.FlattenedFieldMapper; import org.elasticsearch.search.lookup.SourceFilter; import org.elasticsearch.xcontent.ToXContent; import org.elasticsearch.xcontent.XContentBuilder; @@ -108,6 +109,12 @@ DynamicFieldsBuilder getDynamicFieldsBuilder() { DynamicFieldsBuilder getDynamicFieldsBuilder() { return DynamicFieldsBuilder.DYNAMIC_RUNTIME; } + }, + FLATTENED { + @Override + DynamicFieldsBuilder getDynamicFieldsBuilder() { + return DynamicFieldsBuilder.DYNAMIC_FLATTENED; + } }; DynamicFieldsBuilder getDynamicFieldsBuilder() { @@ -117,11 +124,19 @@ DynamicFieldsBuilder getDynamicFieldsBuilder() { /** * Get the root-level dynamic setting for a Mapping * - * If no dynamic settings are explicitly configured, we default to {@link #TRUE} + * If no dynamic settings are explicitly configured, we default to {@link #TRUE}, unless the implicit flattened + * {@code _unmapped} sink is present, in which case unmapped fields default to being absorbed ({@link #FLATTENED}). */ static Dynamic getRootDynamic(MappingLookup mappingLookup) { Dynamic rootDynamic = mappingLookup.getMapping().getRoot().dynamic; - return rootDynamic == null ? Defaults.DYNAMIC : rootDynamic; + if (rootDynamic != null) { + return rootDynamic; + } + if (mappingLookup.getMapper(FlattenedFieldMapper.UNMAPPED_SINK_NAME) instanceof FlattenedFieldMapper sink + && sink.isUnmappedSink()) { + return FLATTENED; + } + return Defaults.DYNAMIC; } } @@ -986,12 +1001,14 @@ protected void serializeMappers(XContentBuilder builder, Params params) throws I int count = 0; for (Mapper mapper : sortedMappers) { - if ((mapper instanceof MetadataFieldMapper) == false) { - if (count++ == 0) { - builder.startObject("properties"); - } - mapper.toXContent(builder, params); + // The implicit _unmapped sink is injected on every mapping-source parse, so it must never be serialized back out. + if (mapper instanceof MetadataFieldMapper || (mapper instanceof FlattenedFieldMapper flattened && flattened.isUnmappedSink())) { + continue; + } + if (count++ == 0) { + builder.startObject("properties"); } + mapper.toXContent(builder, params); } if (count > 0) { builder.endObject(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java index 9e211d47c3e70..b212f6f8baa39 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java @@ -701,6 +701,10 @@ private static Dynamic parsePrefixDynamic(String key, Object value) { if (str.equalsIgnoreCase("runtime")) { throw new MapperParsingException("[prefix_properties." + key + ".dynamic] does not support [runtime]"); } + // Dynamic.FLATTENED is an internal resolved value only; it is not user-settable (Dynamic.valueOf would otherwise accept it). + if (str.equalsIgnoreCase("flattened")) { + throw new MapperParsingException("[prefix_properties." + key + ".dynamic] does not support [flattened]"); + } try { return Dynamic.valueOf(str.toUpperCase(Locale.ROOT)); } catch (IllegalArgumentException e) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java index 71ea9100190e6..d4895e1b67d54 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java @@ -44,6 +44,7 @@ import org.elasticsearch.common.lucene.search.AutomatonQueries; import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.common.util.FeatureFlag; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.core.Nullable; import org.elasticsearch.features.NodeFeature; @@ -155,6 +156,16 @@ public final class FlattenedFieldMapper extends FieldMapper implements PassThrou public static final String KEYED_IGNORED_VALUES_FIELD_SUFFIX = "._keyed._ignored"; public static final String TIME_SERIES_DIMENSIONS_ARRAY_PARAM = "time_series_dimensions"; + /** + * Feature flag gating the implicit {@code _unmapped} sink that absorbs unmapped fields on strict columnar indices. + */ + public static final FeatureFlag UNMAPPED_FIELDS_FEATURE_FLAG = new FeatureFlag("flattened_unmapped_fields"); + + /** + * Name of the implicit, non-serialized flattened sink injected under root to absorb unmapped fields as full dotted keys. + */ + public static final String UNMAPPED_SINK_NAME = "_unmapped"; + public static final NodeFeature FLATTENED_MAPPED_SUBFIELDS_FEATURE = new NodeFeature("mapper.flattened.mapped_subfields"); public static final NodeFeature FLATTENED_PASSTHROUGH_FEATURE = new NodeFeature("mapper.flattened.passthrough"); public static final NodeFeature FLATTENED_COLUMNAR_DOCUMENT_ORDER = new NodeFeature("mapper.flattened.columnar_document_order"); @@ -1646,6 +1657,7 @@ public void validateMatchedRoutingPath(final String routingPath) { private final int passthroughPriority; // -1 means passthrough disabled private final boolean passthrough; private final PreserveLeafArrays preserveLeafArrays; + private final boolean unmappedSink; private FlattenedFieldMapper( String leafName, @@ -1680,6 +1692,8 @@ private FlattenedFieldMapper( ((RootFlattenedFieldType) mappedFieldType).usesArrayOrderBinaryDocValues() ); this.preserveLeafArrays = builder.preserveLeafArrays.get(); + // Sink-ness is gated on the setting so a user-declared flattened field named _unmapped on a non-feature index is a normal field. + this.unmappedSink = builder.indexSettings.isFlattenedUnmappedFieldsEnabled() && UNMAPPED_SINK_NAME.equals(mappedFieldType.name()); } public PreserveLeafArrays preserveLeafArrays() { @@ -1760,21 +1774,10 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio return; } - FlattenedFieldArrayContext arrayContext; - // In document-order mode the write path uses KeyedArrayOrderInlineNull; no _offsets sidecar is written. - if (preserveLeafArrays == PreserveLeafArrays.LOSSY || fieldType().usesArrayOrderBinaryDocValues()) { - arrayContext = null; - } else { - arrayContext = (FlattenedFieldArrayContext) context.getOffSetContext( - mappedFieldType.name(), - () -> new FlattenedFieldArrayContext(mappedFieldType.name()) - ); - } - try { // make sure that we don't expand dots in field names while parsing context.path().setWithinLeafObject(true); - fieldParser.parse(context, arrayContext); + fieldParser.parse(context, arrayContext(context)); } finally { context.path().setWithinLeafObject(false); } @@ -1784,6 +1787,38 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio } } + private FlattenedFieldArrayContext arrayContext(DocumentParserContext context) { + // In document-order mode the write path uses KeyedArrayOrderInlineNull; no _offsets sidecar is written. + if (preserveLeafArrays == PreserveLeafArrays.LOSSY || fieldType().usesArrayOrderBinaryDocValues()) { + return null; + } + return (FlattenedFieldArrayContext) context.getOffSetContext( + mappedFieldType.name(), + () -> new FlattenedFieldArrayContext(mappedFieldType.name()) + ); + } + + /** + * Whether this mapper is the implicit {@code _unmapped} sink; other flattened fields (including user-declared ones) return false. + */ + public boolean isUnmappedSink() { + return unmappedSink; + } + + /** + * Absorbs a single unmapped leaf value under its full dotted path from the document root, indexing it as a keyed flattened value. + */ + public void absorbUnmappedValue(DocumentParserContext context, String fullPath) throws IOException { + fieldParser.absorbUnmappedValue(context, arrayContext(context), fullPath); + } + + /** + * Absorbs an unmapped null so columnar array order is preserved; mirrors the flattened parser's own null-slot handling. + */ + public void absorbUnmappedNull(DocumentParserContext context, String fullPath) throws IOException { + fieldParser.absorbUnmappedNull(context, arrayContext(context), fullPath); + } + @Override protected void doXContentBody(XContentBuilder xContentBuilder, Params params) throws IOException { super.doXContentBody(xContentBuilder, params); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldParser.java b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldParser.java index e30a60e8cc45c..e3cba71080a8e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldParser.java @@ -142,27 +142,7 @@ private void parseFieldValue(Context context, XContentParser.Token token, Conten String value = parser.text(); addField(context, path, currentName, value); } else if (token == XContentParser.Token.VALUE_NULL) { - String key = path.pathAsText(currentName); - if (key.contains(SEPARATOR)) { - throw new IllegalArgumentException( - "Keys in [flattened] fields cannot contain the reserved character \\0. Offending key: [" + key + "]." - ); - } - FieldMapper mappedSubField = mappedSubFields.get(key); - if (mappedSubField != null) { - mappedSubField.parse(context.documentParserContext()); - } else if (nullValue != null) { - addField(context, path, currentName, nullValue); - } else if (usesArrayOrderBinaryDocValues) { - // Document-order mode: record a null slot with the key inline so synthetic source can reconstruct it. - MultiValuedBinaryDocValuesField.KeyedArrayOrderInlineNull.recordNull( - context.documentParserContext.doc(), - keyedFieldFullPath, - new BytesRef(key + SEPARATOR) - ); - } else if (preserveLeafArrays == FlattenedFieldMapper.PreserveLeafArrays.EXACT) { - context.arrayContext.recordNull(key); - } + addNull(context, path, currentName); } else { // Note that we throw an exception here just to be safe. We don't actually expect to reach // this case, since XContentParser verifies that the input is well-formed as it parses. @@ -289,6 +269,48 @@ private void addField(Context context, ContentPath path, String currentName, Str } } + private void addNull(Context context, ContentPath path, String currentName) throws IOException { + String key = path.pathAsText(currentName); + if (key.contains(SEPARATOR)) { + throw new IllegalArgumentException( + "Keys in [flattened] fields cannot contain the reserved character \\0. Offending key: [" + key + "]." + ); + } + FieldMapper mappedSubField = mappedSubFields.get(key); + if (mappedSubField != null) { + mappedSubField.parse(context.documentParserContext()); + } else if (nullValue != null) { + addField(context, path, currentName, nullValue); + } else if (usesArrayOrderBinaryDocValues) { + // Document-order mode: record a null slot with the key inline so synthetic source can reconstruct it. + MultiValuedBinaryDocValuesField.KeyedArrayOrderInlineNull.recordNull( + context.documentParserContext.doc(), + keyedFieldFullPath, + new BytesRef(key + SEPARATOR) + ); + } else if (preserveLeafArrays == FlattenedFieldMapper.PreserveLeafArrays.EXACT) { + context.arrayContext.recordNull(key); + } + } + + /** + * Indexes a single unmapped value at {@code fullPath} (already a full dotted key from the document root) reusing {@link #addField}. + */ + void absorbUnmappedValue(DocumentParserContext documentParserContext, FlattenedFieldArrayContext arrayContext, String fullPath) + throws IOException { + Context context = new Context(documentParserContext.parser(), documentParserContext, arrayContext); + addField(context, new ContentPath(), fullPath, documentParserContext.parser().text()); + } + + /** + * Records an unmapped null slot at {@code fullPath} reusing {@link #addNull} so columnar array order is preserved. + */ + void absorbUnmappedNull(DocumentParserContext documentParserContext, FlattenedFieldArrayContext arrayContext, String fullPath) + throws IOException { + Context context = new Context(documentParserContext.parser(), documentParserContext, arrayContext); + addNull(context, new ContentPath(), fullPath); + } + private void validateDepthLimit(ContentPath path) { if (path.length() + 1 > depthLimit) { throw new IllegalArgumentException( diff --git a/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedUnmappedFieldsTests.java b/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedUnmappedFieldsTests.java new file mode 100644 index 0000000000000..e27b1479527ec --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedUnmappedFieldsTests.java @@ -0,0 +1,229 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper.flattened; + +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.settings.IndexScopedSettings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.CheckedConsumer; +import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.mapper.MapperServiceTestCase; +import org.elasticsearch.index.mapper.ParsedDocument; +import org.elasticsearch.xcontent.XContentBuilder; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.not; + +/** + * Write-path coverage for the implicit flattened {@code _unmapped} sink that absorbs unmapped fields on strict columnar indices. + */ +public class FlattenedUnmappedFieldsTests extends MapperServiceTestCase { + + private static final String KEYED = FlattenedFieldMapper.UNMAPPED_SINK_NAME + FlattenedFieldMapper.KEYED_FIELD_SUFFIX; + + private static Settings columnarSettings() { + return Settings.builder() + .put(IndexSettings.MODE.getKey(), IndexMode.COLUMNAR.getName()) + .put(IndexSettings.FLATTENED_UNMAPPED_FIELDS_ENABLED.getKey(), true) + .build(); + } + + private MapperService columnarService(CheckedConsumer mapping) throws IOException { + return createMapperService(columnarSettings(), mapping(mapping)); + } + + private MapperService columnarServiceTop(CheckedConsumer mapping) throws IOException { + return createMapperService(columnarSettings(), topMapping(mapping)); + } + + private static boolean isSink(MapperService mapperService) { + return ((FlattenedFieldMapper) mapperService.mappingLookup().getMapper(FlattenedFieldMapper.UNMAPPED_SINK_NAME)).isUnmappedSink(); + } + + /** + * In columnar mode the sink is doc-values-only: absorbed values land in the keyed {@link + * org.elasticsearch.index.mapper.MultiValuedBinaryDocValuesField.KeyedArrayOrderInlineNull} column, whose blob stores each + * {@code key\0value} slot inline. This checks that byte sequence is present, which is the write-path signal that a value was absorbed. + */ + private static boolean hasKeyedSlot(ParsedDocument doc, String keyedSlot) { + for (IndexableField field : doc.rootDoc().getFields(KEYED)) { + BytesRef blob = field.binaryValue(); + // ISO-8859-1 maps bytes 1:1 to chars, so contains() on the decoded blob is an exact byte-subsequence search for the ASCII slot. + if (blob != null && new String(blob.bytes, blob.offset, blob.length, StandardCharsets.ISO_8859_1).contains(keyedSlot)) { + return true; + } + } + return false; + } + + public void testSettingRejectedOutsideColumnarMode() { + Settings settings = Settings.builder() + .put(IndexSettings.MODE.getKey(), IndexMode.STANDARD.getName()) + .put(IndexSettings.FLATTENED_UNMAPPED_FIELDS_ENABLED.getKey(), true) + .build(); + var scoped = new IndexScopedSettings(settings, IndexScopedSettings.BUILT_IN_INDEX_SETTINGS); + var e = expectThrows(IllegalArgumentException.class, () -> scoped.validate(settings, true)); + assertThat(e.getMessage(), containsString("only permitted in strict columnar index modes")); + } + + public void testSettingAcceptedInColumnarMode() { + Settings settings = Settings.builder() + .put(IndexSettings.MODE.getKey(), randomFrom(IndexMode.COLUMNAR, IndexMode.LOGSDB_COLUMNAR).getName()) + .put(IndexSettings.FLATTENED_UNMAPPED_FIELDS_ENABLED.getKey(), true) + .build(); + var scoped = new IndexScopedSettings(settings, IndexScopedSettings.BUILT_IN_INDEX_SETTINGS); + scoped.validate(settings, true); // no throw + } + + public void testSinkPresentButNotSerialized() throws IOException { + MapperService mapperService = columnarService(b -> {}); + assertTrue(isSink(mapperService)); + assertThat(mapperService.documentMapper().mappingSource().toString(), not(containsString(FlattenedFieldMapper.UNMAPPED_SINK_NAME))); + } + + public void testAbsentWhenSettingOff() throws IOException { + Settings settings = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.COLUMNAR.getName()).build(); + MapperService mapperService = createMapperService(settings, mapping(b -> {})); + assertNull(mapperService.mappingLookup().getMapper(FlattenedFieldMapper.UNMAPPED_SINK_NAME)); + + // With no sink, an unmapped field is dynamically mapped as usual, emitting a mapping update rather than being absorbed. + String field = randomAlphanumericOfLength(8); + ParsedDocument doc = mapperService.documentMapper().parse(source(b -> b.field(field, randomAlphanumericOfLength(6)))); + assertNotNull(doc.dynamicMappingsUpdate()); + assertTrue(doc.rootDoc().getFields(KEYED).isEmpty()); + } + + public void testAbsorbsUnmappedLeafWithFullDottedKey() throws IOException { + DocumentMapper mapper = columnarService(b -> {}).documentMapper(); + String value = randomAlphanumericOfLength(6); + ParsedDocument doc = mapper.parse(source(b -> b.field("foo", value))); + + assertNull("absorbed fields create no mapper, so no dynamic update is emitted", doc.dynamicMappingsUpdate()); + assertTrue(hasKeyedSlot(doc, "foo\0" + value)); + // Nothing is captured into generic _ignored_source in columnar mode. + assertTrue(doc.rootDoc().getFields(IgnoredSourceFieldMapper.NAME).isEmpty()); + } + + public void testAbsorbsNestedUnmappedObjectAsDottedPaths() throws IOException { + DocumentMapper mapper = columnarService(b -> {}).documentMapper(); + String v1 = randomAlphanumericOfLength(5); + String v2 = randomAlphanumericOfLength(5); + ParsedDocument doc = mapper.parse(source(b -> { + b.startObject("outer"); + b.startObject("inner").field("leaf", v1).endObject(); + b.field("other", v2); + b.endObject(); + })); + + assertNull(doc.dynamicMappingsUpdate()); + assertTrue(hasKeyedSlot(doc, "outer.inner.leaf\0" + v1)); + assertTrue(hasKeyedSlot(doc, "outer.other\0" + v2)); + } + + public void testArraysAbsorbPerElementIncludingNull() throws IOException { + DocumentMapper mapper = columnarService(b -> {}).documentMapper(); + ParsedDocument doc = mapper.parse(source(b -> { b.startArray("arr").value(1).nullValue().value(3).endArray(); })); + + assertNull(doc.dynamicMappingsUpdate()); + assertTrue(hasKeyedSlot(doc, "arr\0" + "1")); + assertTrue(hasKeyedSlot(doc, "arr\0" + "3")); + // The null slot is recorded inline as key\0 (no value); exact ordering is verified in the synthetic-source round-trip PR. + assertTrue(hasKeyedSlot(doc, "arr\0")); + assertFalse(doc.rootDoc().getFields(KEYED).isEmpty()); + } + + public void testDynamicTemplateWinsOverAbsorptionWithPathMatch() throws IOException { + DocumentMapper mapper = columnarServiceTop(b -> { + b.startArray("dynamic_templates"); + b.startObject(); + b.startObject("as_long"); + b.field("path_match", "metrics.*"); + b.startObject("mapping").field("type", "long").endObject(); + b.endObject(); + b.endObject(); + b.endArray(); + }).documentMapper(); + + ParsedDocument doc = mapper.parse(source(b -> b.startObject("metrics").field("count", 5).endObject())); + + // The template matched, so a real long mapper is created (mapping update emitted) and the value is NOT absorbed. + assertNotNull(doc.dynamicMappingsUpdate()); + assertFalse(hasKeyedSlot(doc, "metrics.count\0" + "5")); + assertThat(doc.dynamicMappingsUpdate().toString(), containsString("metrics")); + } + + public void testExplicitSubObjectDynamicOverridesAbsorption() throws IOException { + // A sub-object declared dynamic:false becomes a prefix property resolving to FALSE, so unmapped fields under it are + // dropped rather than absorbed; unmapped fields elsewhere still fall back to the FLATTENED sink. + DocumentMapper mapper = columnarService(b -> { + b.startObject("attributes"); + b.field("dynamic", "false"); + b.startObject("properties").startObject("host").field("type", "keyword").endObject().endObject(); + b.endObject(); + }).documentMapper(); + + String top = randomAlphanumericOfLength(5); + ParsedDocument doc = mapper.parse(source(b -> { + b.field("toplevel", top); + b.startObject("attributes").field("unmapped", randomAlphanumericOfLength(5)).endObject(); + })); + + assertTrue("root-level unmapped field is absorbed", hasKeyedSlot(doc, "toplevel\0" + top)); + assertFalse("dynamic:false sub-object drops its unmapped field", hasKeyedSlot(doc, "attributes.unmapped\0")); + } + + public void testSinkSurvivesMerge() throws IOException { + MapperService mapperService = columnarService(b -> {}); + merge(mapperService, mapping(b -> b.startObject("mapped").field("type", "keyword").endObject())); + + assertTrue(isSink(mapperService)); + String serialized = mapperService.documentMapper().mappingSource().toString(); + assertThat(serialized, not(containsString(FlattenedFieldMapper.UNMAPPED_SINK_NAME))); + assertThat(serialized, containsString("mapped")); + + // Absorption still works after the merge. + String value = randomAlphanumericOfLength(6); + ParsedDocument doc = mapperService.documentMapper().parse(source(b -> b.field("afterMerge", value))); + assertTrue(hasKeyedSlot(doc, "afterMerge\0" + value)); + } + + public void testUserUnmappedFieldNotSinkWhenFeatureOff() throws IOException { + // A flattened field literally named _unmapped on an index without the feature is an ordinary field, not the sink. + Settings settings = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.STANDARD.getName()).build(); + MapperService mapperService = createMapperService( + settings, + mapping(b -> b.startObject(FlattenedFieldMapper.UNMAPPED_SINK_NAME).field("type", "flattened").endObject()) + ); + assertFalse(isSink(mapperService)); + // It serializes normally (not skipped), and unmapped fields are dynamically mapped rather than absorbed. + assertThat(mapperService.documentMapper().mappingSource().toString(), containsString(FlattenedFieldMapper.UNMAPPED_SINK_NAME)); + ParsedDocument doc = mapperService.documentMapper().parse(source(b -> b.field("other", randomAlphanumericOfLength(5)))); + assertNotNull(doc.dynamicMappingsUpdate()); + } + + public void testDynamicFlattenedNotUserSettableInPrefixProperties() { + // Dynamic.FLATTENED is an internal resolved value; prefix_properties uses Dynamic.valueOf, so it must be rejected explicitly. + Settings settings = Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.COLUMNAR.getName()).build(); + var e = expectThrows(Exception.class, () -> createMapperService(settings, topMapping(b -> { + b.startObject("prefix_properties"); + b.startObject("attributes").field("dynamic", "flattened").endObject(); + b.endObject(); + }))); + assertThat(e.getMessage(), containsString("does not support [flattened]")); + } +}