diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java index 057219c341669..a3250dbcf4f17 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java @@ -12,6 +12,7 @@ import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.ActionFuture; +import org.elasticsearch.action.ActionType; import org.elasticsearch.action.DocWriteRequest; import org.elasticsearch.action.DocWriteResponse; import org.elasticsearch.action.RequestBuilder; @@ -41,6 +42,7 @@ import org.elasticsearch.action.admin.indices.stats.ShardStats; import org.elasticsearch.action.admin.indices.template.delete.TransportDeleteComposableIndexTemplateAction; import org.elasticsearch.action.admin.indices.template.get.GetComposableIndexTemplateAction; +import org.elasticsearch.action.admin.indices.template.put.PutComponentTemplateAction; import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest; import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction; import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryRequestBuilder; @@ -52,6 +54,7 @@ import org.elasticsearch.action.datastreams.GetDataStreamAction; import org.elasticsearch.action.datastreams.GetDataStreamAction.Response.DataStreamInfo; import org.elasticsearch.action.datastreams.ModifyDataStreamsAction; +import org.elasticsearch.action.datastreams.UpdateDataStreamMappingsAction; import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.search.MultiSearchRequestBuilder; @@ -62,6 +65,7 @@ import org.elasticsearch.cluster.ClusterStateUpdateTask; import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.cluster.metadata.AliasMetadata; +import org.elasticsearch.cluster.metadata.ComponentTemplate; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.DataStreamAction; @@ -72,6 +76,8 @@ import org.elasticsearch.cluster.metadata.IndexMetadataStats; import org.elasticsearch.cluster.metadata.IndexWriteLoad; import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.metadata.ProjectId; +import org.elasticsearch.cluster.metadata.ProjectMetadata; import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.cluster.routing.IndexRoutingTable; import org.elasticsearch.cluster.routing.ShardRouting; @@ -79,6 +85,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.core.Nullable; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexNotFoundException; @@ -86,6 +93,7 @@ import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.query.TermQueryBuilder; import org.elasticsearch.index.shard.IndexingStats; +import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.InvalidAliasNameException; import org.elasticsearch.indices.InvalidIndexNameException; import org.elasticsearch.plugins.Plugin; @@ -2411,6 +2419,159 @@ public void testShardSizeIsForecastedDuringRollover() throws Exception { assertThat(forecastedShardSizeInBytes.getAsLong(), is(equalTo(expectedTotalSizeInBytes / shardCount))); } + public void testGetEffectiveMappings() throws Exception { + /* + * This test creates a composable template with a mapping and with two component templates with mappings. It then makes sure that + * DataStream.getEffectiveMappings returns a mapping that merges the template's mapping, the component templates' mappings, and the + * mapping override given. It then makes sure we get the same result calling the non-static version of getEffectiveMappings. + */ + ComposableIndexTemplate composableIndexTemplate; + { + ComponentTemplate ct1 = new ComponentTemplate(new Template(null, new CompressedXContent(""" + { + "_doc":{ + "dynamic":"strict", + "properties":{ + "field1":{ + "type":"text" + } + } + } + } + """), null), 3L, null); + ComponentTemplate ct2 = new ComponentTemplate(new Template(null, new CompressedXContent(""" + { + "_doc":{ + "dynamic":"strict", + "properties":{ + "field2":{ + "type":"text" + } + } + } + } + """), null), 3L, null); + client().execute(PutComponentTemplateAction.INSTANCE, new PutComponentTemplateAction.Request("ct1").componentTemplate(ct1)) + .get(); + client().execute(PutComponentTemplateAction.INSTANCE, new PutComponentTemplateAction.Request("ct2").componentTemplate(ct2)) + .get(); + + List componentTemplates = List.of("ct1", "ct2"); + String templateName = "effective-mapping-template"; + TransportPutComposableIndexTemplateAction.Request request = new TransportPutComposableIndexTemplateAction.Request(templateName); + request.indexTemplate( + ComposableIndexTemplate.builder() + .indexPatterns(List.of("effective-*")) + .template(Template.builder().mappings(CompressedXContent.fromJSON(""" + { + "_doc":{ + "dynamic":"strict", + "properties":{ + "field3":{ + "type":"text" + } + } + } + } + """))) + .componentTemplates(componentTemplates) + .dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate()) + .build() + ); + client().execute(TransportPutComposableIndexTemplateAction.TYPE, request).actionGet(); + GetComposableIndexTemplateAction.Response getTemplateResponse = client().execute( + GetComposableIndexTemplateAction.INSTANCE, + new GetComposableIndexTemplateAction.Request(TEST_REQUEST_TIMEOUT, templateName) + ).actionGet(); + composableIndexTemplate = getTemplateResponse.indexTemplates().values().iterator().next(); + } + // The mappingOverrides changes the type of one field, and adds another field: + CompressedXContent mappingOverrides = new CompressedXContent(""" + { + "properties":{ + "field1":{ + "type":"keyword" + }, + "field4":{ + "type":"keyword" + } + } + } + """); + + String dataStreamName = "effective-mappings-test"; + Index writeIndex; + { + CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request( + TEST_REQUEST_TIMEOUT, + TEST_REQUEST_TIMEOUT, + dataStreamName + ); + client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).get(); + writeIndex = getDataStream(dataStreamName).getWriteIndex(); + } + + ProjectMetadata projectMetadata = client().admin() + .cluster() + .state(new ClusterStateRequest(TEST_REQUEST_TIMEOUT)) + .get() + .getState() + .metadata() + .getProject(ProjectId.DEFAULT); + IndicesService indicesService = internalCluster().getInstance(IndicesService.class); + CompressedXContent effectiveMappings = DataStream.getEffectiveMappings( + projectMetadata, + composableIndexTemplate, + mappingOverrides, + writeIndex, + indicesService + ); + assertNotNull(effectiveMappings); + Map effectiveMappingMap = XContentHelper.convertToMap(effectiveMappings.uncompressed(), true, XContentType.JSON) + .v2(); + Map expectedEffectiveMappingMap = Map.of( + "dynamic", + "strict", + "_data_stream_timestamp", + Map.of("enabled", true), + "properties", + Map.of( + "@timestamp", + Map.of("type", "date"), + "field1", + Map.of("type", "keyword"), + "field2", + Map.of("type", "text"), + "field3", + Map.of("type", "text"), + "field4", + Map.of("type", "keyword") + ) + ); + assertThat(effectiveMappingMap, equalTo(expectedEffectiveMappingMap)); + + // Add the same mappingOverrides to the data stream: + client().execute( + new ActionType(UpdateDataStreamMappingsAction.NAME), + new UpdateDataStreamMappingsAction.Request(mappingOverrides, false, TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT).indices( + dataStreamName + ) + ).actionGet(); + assertThat(getDataStream(dataStreamName).getEffectiveMappings(projectMetadata, indicesService), equalTo(effectiveMappings)); + } + + private DataStream getDataStream(String dataStreamName) throws ExecutionException, InterruptedException { + return client().admin() + .cluster() + .state(new ClusterStateRequest(TEST_REQUEST_TIMEOUT)) + .get() + .getState() + .getMetadata() + .getProject(ProjectId.DEFAULT) + .dataStreams() + .get(dataStreamName); + } + private void indexDocsAndEnsureThereIsCapturedWriteLoad(String dataStreamName) throws Exception { assertBusy(() -> { for (int i = 0; i < 10; i++) { diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TransportUpdateDataStreamMappingsActionIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TransportUpdateDataStreamMappingsActionIT.java index 97dce981cfbf3..b4b56d3d43edf 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TransportUpdateDataStreamMappingsActionIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TransportUpdateDataStreamMappingsActionIT.java @@ -57,8 +57,10 @@ public void testGetAndUpdateMappings() throws IOException { Map originalMappings = Map.of( "dynamic", "strict", + "_data_stream_timestamp", + Map.of("enabled", true), "properties", - Map.of("foo1", Map.of("type", "text"), "foo2", Map.of("type", "text")) + Map.of("@timestamp", Map.of("type", "date"), "foo1", Map.of("type", "text"), "foo2", Map.of("type", "text")) ); Map mappingOverrides = Map.of( "properties", @@ -67,8 +69,19 @@ public void testGetAndUpdateMappings() throws IOException { Map expectedEffectiveMappings = Map.of( "dynamic", "strict", + "_data_stream_timestamp", + Map.of("enabled", true), "properties", - Map.of("foo1", Map.of("type", "text"), "foo2", Map.of("type", "keyword"), "foo3", Map.of("type", "text")) + Map.of( + "@timestamp", + Map.of("type", "date"), + "foo1", + Map.of("type", "text"), + "foo2", + Map.of("type", "keyword"), + "foo3", + Map.of("type", "text") + ) ); assertExpectedMappings(dataStreamName, Map.of(), originalMappings); updateMappings(dataStreamName, mappingOverrides, expectedEffectiveMappings, true); diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamMappingsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamMappingsAction.java index c6cf18cdcd758..e22e123d88c51 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamMappingsAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamMappingsAction.java @@ -22,6 +22,7 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.project.ProjectResolver; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.indices.IndicesService; import org.elasticsearch.injection.guice.Inject; import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; @@ -35,6 +36,7 @@ public class TransportGetDataStreamMappingsAction extends TransportLocalProjectM GetDataStreamMappingsAction.Request, GetDataStreamMappingsAction.Response> { private final IndexNameExpressionResolver indexNameExpressionResolver; + private final IndicesService indicesService; @Inject public TransportGetDataStreamMappingsAction( @@ -43,7 +45,8 @@ public TransportGetDataStreamMappingsAction( ThreadPool threadPool, ActionFilters actionFilters, ProjectResolver projectResolver, - IndexNameExpressionResolver indexNameExpressionResolver + IndexNameExpressionResolver indexNameExpressionResolver, + IndicesService indicesService ) { super( GetSettingsAction.NAME, @@ -54,6 +57,7 @@ public TransportGetDataStreamMappingsAction( projectResolver ); this.indexNameExpressionResolver = indexNameExpressionResolver; + this.indicesService = indicesService; } @Override @@ -81,7 +85,7 @@ protected void localClusterStateOperation( new GetDataStreamMappingsAction.DataStreamMappingsResponse( dataStreamName, dataStream.getMappings(), - dataStream.getEffectiveMappings(project.metadata()) + dataStream.getEffectiveMappings(project.metadata(), indicesService) ) ); } diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportUpdateDataStreamMappingsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportUpdateDataStreamMappingsAction.java index beccd0c190542..b4b5c5f23cabb 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportUpdateDataStreamMappingsAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportUpdateDataStreamMappingsAction.java @@ -29,6 +29,7 @@ import org.elasticsearch.common.util.concurrent.EsExecutors; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.mapper.Mapping; +import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.injection.guice.Inject; import org.elasticsearch.tasks.Task; @@ -47,6 +48,7 @@ public class TransportUpdateDataStreamMappingsAction extends TransportMasterNode private final IndexNameExpressionResolver indexNameExpressionResolver; private final SystemIndices systemIndices; private final ProjectResolver projectResolver; + private final IndicesService indicesService; @Inject public TransportUpdateDataStreamMappingsAction( @@ -57,7 +59,8 @@ public TransportUpdateDataStreamMappingsAction( ProjectResolver projectResolver, MetadataDataStreamsService metadataDataStreamsService, IndexNameExpressionResolver indexNameExpressionResolver, - SystemIndices systemIndices + SystemIndices systemIndices, + IndicesService indicesService ) { super( UpdateDataStreamMappingsAction.NAME, @@ -73,6 +76,7 @@ public TransportUpdateDataStreamMappingsAction( this.metadataDataStreamsService = metadataDataStreamsService; this.indexNameExpressionResolver = indexNameExpressionResolver; this.systemIndices = systemIndices; + this.indicesService = indicesService; } @Override @@ -163,7 +167,7 @@ private void updateSingleDataStream( true, null, mappingsOverrides, - dataStream.getEffectiveMappings(clusterService.state().metadata().getProject(projectId)) + dataStream.getEffectiveMappings(clusterService.state().metadata().getProject(projectId), indicesService) ) ); } catch (IOException e) { diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/250_data_stream_mappings.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/250_data_stream_mappings.yml index 98faa3647ff1a..efb5d09cd0dfb 100644 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/250_data_stream_mappings.yml +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/250_data_stream_mappings.yml @@ -37,7 +37,7 @@ setup: name: my-data-stream-1 - match: { data_streams.0.name: my-data-stream-1 } - match: { data_streams.0.mappings: {} } - - length: { data_streams.0.effective_mappings.properties: 1 } + - length: { data_streams.0.effective_mappings.properties: 2 } - do: indices.get_data_stream: @@ -74,7 +74,7 @@ setup: indices.get_data_stream_mappings: name: my-data-stream-1 - match: { data_streams.0.name: my-data-stream-1 } - - length: { data_streams.0.effective_mappings.properties: 2 } + - length: { data_streams.0.effective_mappings.properties: 3 } - match: { data_streams.0.mappings.properties.name.type: "keyword" } - match: { data_streams.0.effective_mappings.properties.name.type: "keyword" } @@ -153,9 +153,23 @@ setup: - match: { data_streams.0.mappings.properties.field2: null } - match: { data_streams.0.mappings.properties.field3.type: "text" } - match: { data_streams.0.effective_mappings.properties.field1.type: "text" } - - match: { data_streams.0.effective_mappings.properties.field2: null } + - match: { data_streams.0.effective_mappings.properties.field2.type: "keyword" } - match: { data_streams.0.effective_mappings.properties.field3.type: "text" } + - do: + cluster.put_component_template: + name: mappings-template + body: + template: + mappings: + properties: + field1: + type: keyword + field2: + type: keyword + field4: + type: keyword + - do: indices.rollover: alias: "my-component-only-data-stream-1" @@ -169,9 +183,10 @@ setup: indices.get_data_stream_mappings: name: my-component-only-data-stream-1 - match: { data_streams.0.name: my-component-only-data-stream-1 } - - length: { data_streams.0.effective_mappings.properties: 2 } + - length: { data_streams.0.effective_mappings.properties: 5 } - match: { data_streams.0.mappings.properties.field1.type: "text" } - match: { data_streams.0.effective_mappings.properties.field3.type: "text" } + - match: { data_streams.0.effective_mappings.properties.field4.type: "keyword" } - do: indices.get_data_stream: @@ -191,6 +206,7 @@ setup: - match: { .$newIndexName.mappings.properties.field1.type: "text" } - match: { .$newIndexName.mappings.properties.field2.type: "keyword" } - match: { .$newIndexName.mappings.properties.field3.type: "text" } + - match: { .$newIndexName.mappings.properties.field4.type: "keyword" } - do: indices.put_data_stream_mappings: @@ -199,4 +215,7 @@ setup: - match: { data_streams.0.name: my-component-only-data-stream-1 } - match: { data_streams.0.applied_to_data_stream: true } - match: { data_streams.0.mappings null } - - match: { data_streams.0.effective_mappings: null } + - match: { data_streams.0.effective_mappings.properties.field1.type: "keyword" } + - match: { data_streams.0.effective_mappings.properties.field2.type: "keyword" } + - match: { data_streams.0.effective_mappings.properties.field3: null } + - match: { data_streams.0.effective_mappings.properties.field4.type: "keyword" } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java index 70ec6678ea165..fa9d075867773 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ComposableIndexTemplate.java @@ -373,7 +373,7 @@ public ComposableIndexTemplate mergeMappings(CompressedXContent mappings) throws } @SuppressWarnings("unchecked") - private CompressedXContent merge(CompressedXContent originalMapping, CompressedXContent mappingAddition) throws IOException { + public static CompressedXContent merge(CompressedXContent originalMapping, CompressedXContent mappingAddition) throws IOException { Map mappingAdditionMap = XContentHelper.convertToMap(mappingAddition.uncompressed(), true, XContentType.JSON).v2(); Map combinedMappingMap = new HashMap<>(); if (originalMapping != null) { @@ -389,7 +389,7 @@ private CompressedXContent merge(CompressedXContent originalMapping, CompressedX return convertMappingMapToXContent(combinedMappingMap); } - private static CompressedXContent convertMappingMapToXContent(Map rawAdditionalMapping) throws IOException { + public static CompressedXContent convertMappingMapToXContent(Map rawAdditionalMapping) throws IOException { CompressedXContent compressedXContent; if (rawAdditionalMapping.isEmpty()) { compressedXContent = EMPTY_MAPPINGS; diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java index ebcb0f4793404..b454aeb5663be 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java @@ -46,8 +46,12 @@ import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.mapper.DateFieldMapper; +import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.xcontent.ConstructingObjectParser; +import org.elasticsearch.xcontent.NamedXContentRegistry; import org.elasticsearch.xcontent.ObjectParser; import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xcontent.ToXContentObject; @@ -73,7 +77,9 @@ import java.util.stream.Collectors; import static org.elasticsearch.cluster.metadata.ComposableIndexTemplate.EMPTY_MAPPINGS; +import static org.elasticsearch.cluster.metadata.ComposableIndexTemplate.convertMappingMapToXContent; import static org.elasticsearch.cluster.metadata.MetadataCreateDataStreamService.lookupTemplateForDataStream; +import static org.elasticsearch.cluster.metadata.MetadataCreateIndexService.collectV2Mappings; import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken; import static org.elasticsearch.index.IndexSettings.LIFECYCLE_ORIGINATION_DATE; import static org.elasticsearch.index.IndexSettings.PREFER_ILM_SETTING; @@ -408,8 +414,72 @@ public Settings getEffectiveSettings(ProjectMetadata projectMetadata) { return templateSettings.merge(settings); } - public CompressedXContent getEffectiveMappings(ProjectMetadata projectMetadata) throws IOException { - return getMatchingIndexTemplate(projectMetadata).mergeMappings(mappings).template().mappings(); + /** + * Returns the mappings that would be used to create the write index if this data stream were rolled over right now. This includes + * the mapping overrides on this data stream, the mapping from the matching composable template, and the mappings from all component + * templates in the matching composable template. + * @param projectMetadata + * @param indicesService Used in the logic that merges all mappings together into one mapping + * @return A JSON CompressedXContent representation of the effective mappings of this data stream + * @throws IOException + */ + public CompressedXContent getEffectiveMappings(ProjectMetadata projectMetadata, IndicesService indicesService) throws IOException { + return getEffectiveMappings(projectMetadata, getMatchingIndexTemplate(projectMetadata), mappings, getWriteIndex(), indicesService); + } + + /** + * Returns the mappings that would be used to create the write index if a data stream with composableTemplate and mappingsOverrides were + * rolled over right now. This includes the mapping overrides, the mapping from the composable template, and the mappings from all + * component templates in the composable template. + * @param projectMetadata + * @param composableTemplate The composable template that matches the data stream's name + * @param mappingsOverrides The mapping overrides to be applied + * @param writeIndex The write index of the data stream whose effective mappings are to be returned. This is used only to determine + * whether data stream mappings ought to be added when calling collectV2Mappings, and to create an IndexService + * object. The mappings that will be used come from templateWithMergedMappings and from the component templates, not + * from this writeIndex. + * @param indicesService + * @return A JSON CompressedXContent representation of the effective mappings for the composableTemplate plus mappingsOverrides + * @throws IOException + */ + @SuppressWarnings("unchecked") + public static CompressedXContent getEffectiveMappings( + ProjectMetadata projectMetadata, + ComposableIndexTemplate composableTemplate, + CompressedXContent mappingsOverrides, + Index writeIndex, + IndicesService indicesService + ) throws IOException { + ComposableIndexTemplate mergedTemplate = composableTemplate.mergeMappings(mappingsOverrides); + final String requestMappings = null; // We are not using any request mappings + final NamedXContentRegistry xContentRegistry = null; // This is only used to parse requestMappings if they are not null + final List mappings = collectV2Mappings( + requestMappings, + projectMetadata, + mergedTemplate, + xContentRegistry, + writeIndex.getName() + ); + return indicesService.withTempIndexService(projectMetadata.index(writeIndex), indexService -> { + MapperService mapperService = indexService.mapperService(); + DocumentMapper documentMapper = mapperService.merge( + MapperService.SINGLE_MAPPING_NAME, + mappings, + MapperService.MergeReason.INDEX_TEMPLATE + ); + Map resultMapping; + Map originalMappingMap = XContentHelper.convertToMap( + documentMapper.mappingSource().uncompressed(), + true, + XContentType.JSON + ).v2(); + if (originalMappingMap.containsKey(MapperService.SINGLE_MAPPING_NAME)) { + resultMapping = (Map) originalMappingMap.get(MapperService.SINGLE_MAPPING_NAME); + } else { + resultMapping = originalMappingMap; + } + return convertMappingMapToXContent(resultMapping); + }); } private ComposableIndexTemplate getMatchingIndexTemplate(ProjectMetadata projectMetadata) { diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java index a131d49fd5269..ca612eb20747a 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java @@ -938,7 +938,7 @@ private static List collectV2Mappings( @Nullable final String requestMappings, final List templateMappings, final NamedXContentRegistry xContentRegistry - ) throws Exception { + ) throws IOException { List result = new ArrayList<>(templateMappings.size() + 1); result.addAll(templateMappings); if (requestMappings != null) { @@ -956,7 +956,7 @@ public static List collectV2Mappings( final ComposableIndexTemplate template, final NamedXContentRegistry xContentRegistry, final String indexName - ) throws Exception { + ) throws IOException { List templateMappings = MetadataIndexTemplateService.collectMappings( template, projectMetadata.componentTemplates(), diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsService.java index 4fdad3c5b1df8..e668bed11ff60 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsService.java @@ -497,11 +497,7 @@ private DataStream createDataStreamForUpdatedDataStreamSettings( final ComposableIndexTemplate template = lookupTemplateForDataStream(dataStreamName, projectMetadata); Settings templateSettings = MetadataIndexTemplateService.resolveSettings(template, projectMetadata.componentTemplates()); Settings mergedEffectiveSettings = templateSettings.merge(mergedDataStreamSettings); - MetadataIndexTemplateService.validateTemplate( - mergedEffectiveSettings, - dataStream.getEffectiveMappings(projectMetadata), - indicesService - ); + MetadataIndexTemplateService.validateTemplate(mergedEffectiveSettings, null, indicesService); return dataStream.copy().setSettings(mergedDataStreamSettings).build(); } @@ -517,12 +513,14 @@ private DataStream createDataStreamForUpdatedDataStreamMappings( DataStream dataStream = dataStreamMap.get(dataStreamName); final ComposableIndexTemplate template = lookupTemplateForDataStream(dataStreamName, projectMetadata); - ComposableIndexTemplate mergedTemplate = template.mergeMappings(mappingsOverrides); - MetadataIndexTemplateService.validateTemplate( - dataStream.getEffectiveSettings(projectMetadata), - mergedTemplate.template().mappings(), + CompressedXContent effectiveMappings = DataStream.getEffectiveMappings( + projectMetadata, + template, + mappingsOverrides, + dataStream.getWriteIndex(), indicesService ); + MetadataIndexTemplateService.validateTemplate(dataStream.getEffectiveSettings(projectMetadata), effectiveMappings, indicesService); return dataStream.copy().setMappings(mappingsOverrides).build(); } diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java index 4af14fdaed0ae..382b3634710fe 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java @@ -29,6 +29,7 @@ import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.mapper.DateFieldMapper; +import org.elasticsearch.indices.IndicesService; import org.elasticsearch.indices.SystemIndices; import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval; import org.elasticsearch.test.AbstractXContentSerializingTestCase; @@ -2688,7 +2689,8 @@ public void testGetEffectiveMappingsNoMatchingTemplate() { // No matching template, so we expect an IllegalArgumentException DataStream dataStream = createTestInstance(); ProjectMetadata.Builder projectMetadataBuilder = ProjectMetadata.builder(randomProjectIdOrDefault()); - assertThrows(IllegalArgumentException.class, () -> dataStream.getEffectiveMappings(projectMetadataBuilder.build())); + IndicesService indicesService = mock(IndicesService.class); + assertThrows(IllegalArgumentException.class, () -> dataStream.getEffectiveMappings(projectMetadataBuilder.build(), indicesService)); } public void testGetEffectiveIndexTemplateDataStreamMappingsOnly() throws IOException {