elastic · Kubik42 · Jul 24, 2026
diff --git a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java
@@ -211,6 +211,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
                 IndexSettings.LOGSDB_SORT_ON_MESSAGE_TEMPLATE,
                 IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD,
                 IndexSettings.SLICE_ENABLED,
+                IndexSettings.FLATTENED_UNMAPPED_FIELDS_ENABLED,
                 IndexSettings.PREFER_ILM_SETTING,
                 DataStreamFailureStoreDefinition.FAILURE_STORE_DEFINITION_VERSION_SETTING,
                 FieldMapper.SYNTHETIC_SOURCE_KEEP_INDEX_SETTING,

diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java
@@ -33,6 +33,7 @@
 import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.SeqNoFieldMapper;
 import org.elasticsearch.index.mapper.SourceFieldMapper;
+import org.elasticsearch.index.mapper.flattened.FlattenedFieldMapper;
 import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper;
 import org.elasticsearch.index.translog.Translog;
 import org.elasticsearch.indices.IndicesRequestCache;
@@ -855,6 +856,56 @@ public static boolean isValidCodecForSyntheticId(String codecName, IndexVersion
         return "false";
     }, Property.IndexScope, Property.Final);
 
+    /**
+     * Internal, feature-flagged setting that turns on the implicit flattened {@code _unmapped} sink absorbing unmapped fields.
+     * Only permitted in strict columnar index modes; rejected (as an unknown setting) when the feature flag is off.
+     * Temporary scaffolding: to be removed once all sink read paths land, when enablement becomes feature flag + strict columnar mode
+     * with no per-index opt-out. Do not document or expose this setting.
+     */
+    public static final Setting<Boolean> FLATTENED_UNMAPPED_FIELDS_ENABLED = Setting.boolSetting(
+        "index.mapping.flattened_unmapped_fields.enabled",
+        false,
+        new Setting.Validator<>() {
+            @Override
+            public void validate(Boolean enabled) {
+                if (enabled && FlattenedFieldMapper.UNMAPPED_FIELDS_FEATURE_FLAG.isEnabled() == false) {
+                    throw new IllegalArgumentException(
+                        String.format(
+                            Locale.ROOT,
+                            "unknown setting [%s] please check that any required plugins are installed, "
+                                + "or check the breaking changes documentation for removed settings",
+                            FLATTENED_UNMAPPED_FIELDS_ENABLED.getKey()
+                        )
+                    );
+                }
+            }
+
+            @Override
+            public void validate(Boolean enabled, Map<Setting<?>, Object> settings) {
+                if (enabled) {
+                    var indexMode = (IndexMode) settings.get(MODE);
+                    if (indexMode.isStrictColumnar() == false) {
+                        throw new IllegalArgumentException(
+                            String.format(
+                                Locale.ROOT,
+                                "The setting [%s] is only permitted in strict columnar index modes. Current mode: [%s].",
+                                FLATTENED_UNMAPPED_FIELDS_ENABLED.getKey(),
+                                indexMode.getName()
+                            )
+                        );
+                    }
+                }
+            }
+
+            @Override
+            public Iterator<Setting<?>> settings() {
+                return List.<Setting<?>>of(MODE).iterator();
+            }
+        },
+        Property.IndexScope,
+        Property.Final
+    );
+
     public static final Setting<SourceFieldMapper.Mode> INDEX_MAPPER_SOURCE_MODE_SETTING = Setting.enumSetting(
         SourceFieldMapper.Mode.class,
         settings -> {
@@ -1290,6 +1341,7 @@ public Iterator<Setting<?>> settings() {
     private final boolean logsdbSortOnHostName;
     private final boolean logsdbAddHostNameField;
     private final boolean sliceEnabled;
+    private final boolean flattenedUnmappedFieldsEnabled;
     private volatile long retentionLeaseMillis;
 
     /**
@@ -1435,6 +1487,10 @@ public boolean isSliceEnabled() {
         return sliceEnabled;
     }
 
+    public boolean isFlattenedUnmappedFieldsEnabled() {
+        return flattenedUnmappedFieldsEnabled;
+    }
+
     /**
      * Returns <code>true</code> if the index is in logsdb mode and needs a [host.name] keyword field. The default is <code>false</code>
      */
@@ -1529,6 +1585,7 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti
         maxRegexLength = scopedSettings.get(MAX_REGEX_LENGTH_SETTING);
         this.mergePolicyConfig = new MergePolicyConfig(logger, this);
         sliceEnabled = scopedSettings.get(SLICE_ENABLED);
+        flattenedUnmappedFieldsEnabled = scopedSettings.get(FLATTENED_UNMAPPED_FIELDS_ENABLED);
         this.indexSortConfig = new IndexSortConfig(this);
         searchIdleAfter = scopedSettings.get(INDEX_SEARCH_IDLE_AFTER);
         defaultPipeline = scopedSettings.get(DEFAULT_PIPELINE);

diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java
@@ -26,6 +26,7 @@
 import org.elasticsearch.index.SliceIndexing;
 import org.elasticsearch.index.fielddata.FieldDataContext;
 import org.elasticsearch.index.fielddata.IndexFieldDataCache;
+import org.elasticsearch.index.mapper.flattened.FlattenedFieldMapper;
 import org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper;
 import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
@@ -945,7 +946,14 @@ private static void parseNullValue(DocumentParserContext context, String lastFie
             // TODO: passing null to an object seems bogus?
             parseObjectOrField(context, mapper);
         } else {
-            ensureNotStrict(context.resolveDynamic(lastFieldName), context, lastFieldName);
+            ObjectMapper.Dynamic dynamic = context.resolveDynamic(lastFieldName);
+            ensureNotStrict(dynamic, context, lastFieldName);
+            if (dynamic == ObjectMapper.Dynamic.FLATTENED) {
+                // Absorb the null slot so columnar array order (e.g. [1, null, 3]) is preserved for the unmapped field.
+                FlattenedFieldMapper sink = (FlattenedFieldMapper) context.mappingLookup()
+                    .getMapper(FlattenedFieldMapper.UNMAPPED_SINK_NAME);
+                sink.absorbUnmappedNull(context, context.path().pathAsText(lastFieldName));
+            }
         }
     }
 

diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java b/server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.time.DateFormatter;
 import org.elasticsearch.index.mapper.ObjectMapper.Dynamic;
+import org.elasticsearch.index.mapper.flattened.FlattenedFieldMapper;
 import org.elasticsearch.script.ScriptCompiler;
 import org.elasticsearch.xcontent.XContentParser;
 
@@ -32,6 +33,7 @@ final class DynamicFieldsBuilder {
     private static final Concrete CONCRETE = new Concrete(DocumentParser::parseObjectOrField);
     static final DynamicFieldsBuilder DYNAMIC_TRUE = new DynamicFieldsBuilder(CONCRETE);
     static final DynamicFieldsBuilder DYNAMIC_RUNTIME = new DynamicFieldsBuilder(new Runtime());
+    static final DynamicFieldsBuilder DYNAMIC_FLATTENED = new DynamicFieldsBuilder(new Absorb());
 
     private final Strategy strategy;
 
@@ -143,12 +145,11 @@ boolean createDynamicFieldFromValue(final DocumentParserContext context, String
                 () -> strategy.newDynamicBooleanField(context, name)
             );
         } else if (token == XContentParser.Token.VALUE_EMBEDDED_OBJECT) {
-            // runtime binary fields are not supported, hence binary objects always get created as concrete fields
             return createDynamicField(
                 context,
                 name,
                 DynamicTemplate.XContentFieldType.BINARY,
-                () -> CONCRETE.newDynamicBinaryField(context, name)
+                () -> strategy.newDynamicBinaryField(context, name)
             );
         } else {
             return createDynamicStringFieldFromTemplate(context, name);
@@ -293,6 +294,8 @@ private interface Strategy {
         boolean newDynamicBooleanField(DocumentParserContext context, String name) throws IOException;
 
         boolean newDynamicDateField(DocumentParserContext context, String name, DateFormatter dateFormatter) throws IOException;
+
+        boolean newDynamicBinaryField(DocumentParserContext context, String name) throws IOException;
     }
 
     /**
@@ -382,7 +385,8 @@ public boolean newDynamicDateField(DocumentParserContext context, String name, D
             );
         }
 
-        boolean newDynamicBinaryField(DocumentParserContext context, String name) throws IOException {
+        @Override
+        public boolean newDynamicBinaryField(DocumentParserContext context, String name) throws IOException {
             return createDynamicField(new BinaryFieldMapper.Builder(name, SourceFieldMapper.isSynthetic(context.indexSettings())), context);
         }
     }
@@ -431,5 +435,56 @@ public boolean newDynamicDateField(DocumentParserContext context, String name, D
                 context
             );
         }
+
+        @Override
+        public boolean newDynamicBinaryField(DocumentParserContext context, String name) throws IOException {
+            // Runtime binary fields are unsupported, so fall back to the concrete binary field (previous behavior).
+            return CONCRETE.newDynamicBinaryField(context, name);
+        }
+    }
+
+    /**
+     * Absorbs unmapped leaf values into the implicit flattened {@code _unmapped} sink. Used when the resolved dynamic mode is
+     * {@link ObjectMapper.Dynamic#FLATTENED}. Dynamic templates still win, since {@link #createDynamicFieldFromValue} tries a matching
+     * template before falling back to the strategy.
+     */
+    private static final class Absorb implements Strategy {
+
+        private static boolean absorb(DocumentParserContext context, String name) throws IOException {
+            FlattenedFieldMapper sink = (FlattenedFieldMapper) context.mappingLookup().getMapper(FlattenedFieldMapper.UNMAPPED_SINK_NAME);
+            sink.absorbUnmappedValue(context, context.path().pathAsText(name));
+            return true;
+        }
+
+        @Override
+        public boolean newDynamicStringField(DocumentParserContext context, String name) throws IOException {
+            return absorb(context, name);
+        }
+
+        @Override
+        public boolean newDynamicLongField(DocumentParserContext context, String name) throws IOException {
+            return absorb(context, name);
+        }
+
+        @Override
+        public boolean newDynamicDoubleField(DocumentParserContext context, String name) throws IOException {
+            return absorb(context, name);
+        }
+
+        @Override
+        public boolean newDynamicBooleanField(DocumentParserContext context, String name) throws IOException {
+            return absorb(context, name);
+        }
+
+        @Override
+        public boolean newDynamicDateField(DocumentParserContext context, String name, DateFormatter dateFormatter) throws IOException {
+            return absorb(context, name);
+        }
+
+        @Override
+        public boolean newDynamicBinaryField(DocumentParserContext context, String name) {
+            // Binary (embedded object) values cannot be stringified into flattened; dropped in this phase (feature-flagged, columnar-only).
+            return true;
+        }
     }
 }
diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappingParser.java b/server/src/main/java/org/elasticsearch/index/mapper/MappingParser.java
@@ -15,6 +15,7 @@
 import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.index.mapper.MapperService.MergeReason;
+import org.elasticsearch.index.mapper.flattened.FlattenedFieldMapper;
 import org.elasticsearch.xcontent.XContentType;
 
 import java.util.Collections;
@@ -149,6 +150,14 @@ MappingBuilder parseToBuilder(@Nullable String type, MergeReason reason, Map<Str
 
         RootObjectMapper.Builder rootObjectMapper = RootObjectMapper.parse(type, mappingSource, mappingParserContext);
 
+        // The _unmapped sink is derived state, not user mapping: its entire config is a function of index settings, and it carries no
+        // user-settable parameters. So it is never serialized and is instead re-injected here on every mapping-source parse.
+        if (mappingParserContext.getIndexSettings().isFlattenedUnmappedFieldsEnabled()) {
+            rootObjectMapper.add(
+                FlattenedFieldMapper.PARSER.parse(FlattenedFieldMapper.UNMAPPED_SINK_NAME, new HashMap<>(), mappingParserContext)
+            );
+        }
+
         Map<String, MetadataFieldMapper.Builder> metadataBuilders = metadataBuildersSupplier.get();
         Map<String, Object> meta = null;
 

diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java
@@ -24,6 +24,7 @@
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.IndexVersions;
 import org.elasticsearch.index.mapper.MapperService.MergeReason;
+import org.elasticsearch.index.mapper.flattened.FlattenedFieldMapper;
 import org.elasticsearch.search.lookup.SourceFilter;
 import org.elasticsearch.xcontent.ToXContent;
 import org.elasticsearch.xcontent.XContentBuilder;
@@ -108,6 +109,12 @@ DynamicFieldsBuilder getDynamicFieldsBuilder() {
             DynamicFieldsBuilder getDynamicFieldsBuilder() {
                 return DynamicFieldsBuilder.DYNAMIC_RUNTIME;
             }
+        },
+        FLATTENED {
+            @Override
+            DynamicFieldsBuilder getDynamicFieldsBuilder() {
+                return DynamicFieldsBuilder.DYNAMIC_FLATTENED;
+            }
         };
 
         DynamicFieldsBuilder getDynamicFieldsBuilder() {
@@ -117,11 +124,19 @@ DynamicFieldsBuilder getDynamicFieldsBuilder() {
         /**
          * Get the root-level dynamic setting for a Mapping
          *
-         * If no dynamic settings are explicitly configured, we default to {@link #TRUE}
+         * If no dynamic settings are explicitly configured, we default to {@link #TRUE}, unless the implicit flattened
+         * {@code _unmapped} sink is present, in which case unmapped fields default to being absorbed ({@link #FLATTENED}).
          */
         static Dynamic getRootDynamic(MappingLookup mappingLookup) {
             Dynamic rootDynamic = mappingLookup.getMapping().getRoot().dynamic;
-            return rootDynamic == null ? Defaults.DYNAMIC : rootDynamic;
+            if (rootDynamic != null) {
+                return rootDynamic;
+            }
+            if (mappingLookup.getMapper(FlattenedFieldMapper.UNMAPPED_SINK_NAME) instanceof FlattenedFieldMapper sink
+                && sink.isUnmappedSink()) {
+                return FLATTENED;
+            }
+            return Defaults.DYNAMIC;
         }
     }
 
@@ -986,12 +1001,14 @@ protected void serializeMappers(XContentBuilder builder, Params params) throws I
 
         int count = 0;
         for (Mapper mapper : sortedMappers) {
-            if ((mapper instanceof MetadataFieldMapper) == false) {
-                if (count++ == 0) {
-                    builder.startObject("properties");
-                }
-                mapper.toXContent(builder, params);
+            // The implicit _unmapped sink is injected on every mapping-source parse, so it must never be serialized back out.
+            if (mapper instanceof MetadataFieldMapper || (mapper instanceof FlattenedFieldMapper flattened && flattened.isUnmappedSink())) {
+                continue;
+            }
+            if (count++ == 0) {
+                builder.startObject("properties");
             }
+            mapper.toXContent(builder, params);
         }
         if (count > 0) {
             builder.endObject();

diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java
@@ -701,6 +701,10 @@ private static Dynamic parsePrefixDynamic(String key, Object value) {
         if (str.equalsIgnoreCase("runtime")) {
             throw new MapperParsingException("[prefix_properties." + key + ".dynamic] does not support [runtime]");
         }
+        // Dynamic.FLATTENED is an internal resolved value only; it is not user-settable (Dynamic.valueOf would otherwise accept it).
+        if (str.equalsIgnoreCase("flattened")) {
+            throw new MapperParsingException("[prefix_properties." + key + ".dynamic] does not support [flattened]");
+        }
         try {
             return Dynamic.valueOf(str.toUpperCase(Locale.ROOT));
         } catch (IllegalArgumentException e) {