Remove types from internal monitoring templates and bump to api 7 (#3…

…9888) (#39926)

This commit removes the "doc" type from monitoring internal indexes.
The template still carries the "_doc" type since that is needed for
the internal representation.

This change impacts the following templates:
monitoring-alerts.json
monitoring-beats.json
monitoring-es.json
monitoring-kibana.json
monitoring-logstash.json

As part of the required changes, the system_api_version has been
bumped from "6" to "7" and support for version "2" has been dropped.

A new empty pipeline is now introduced for the version "7", and
the formerly empty "6" pipeline will now remove the type and re-direct
the request to the "7" index.

Additionally, to due to a difference in the internal representation
(which requires the inclusion of "_doc" type) and external representation
(which requires the exclusion of any type) a helper method is introduced
to help convert internal to external representation, and used by the
monitoring HTTP template exporter.

Relates #38637

server/src/main/java/org/elasticsearch/cluster/metadata/IndexTemplateMetaData.java

@@ -345,6 +345,19 @@ public static void toXContentWithTypes(IndexTemplateMetaData indexTemplateMetaDa
             builder.endObject();
         }
 
+        /**
+         * Removes the nested type in the xContent representation of {@link IndexTemplateMetaData}.
+         *
+         * This method is useful to help bridge the gap between an the internal representation which still uses (the legacy format) a
+         * nested type in the mapping, and the external representation which does not use a nested type in the mapping.
+         */
+        public static void removeType(IndexTemplateMetaData indexTemplateMetaData, XContentBuilder builder) throws IOException {
+            builder.startObject();
+            toInnerXContent(indexTemplateMetaData, builder,
+                new ToXContent.MapParams(Collections.singletonMap("reduce_mappings", "true")), false);
+            builder.endObject();
+        }
+
         /**
          * Serializes the template to xContent, making sure not to nest mappings under the
          * type name.
@@ -361,6 +374,7 @@ public static void toXContent(IndexTemplateMetaData indexTemplateMetaData,
             builder.endObject();
         }
 
+
         static void toInnerXContentWithTypes(IndexTemplateMetaData indexTemplateMetaData,
                                              XContentBuilder builder,
                                              ToXContent.Params params) throws IOException {

x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/monitoring/collector/ccr/AutoFollowStatsMonitoringDocTests.java

@@ -148,7 +148,7 @@ public void testShardFollowNodeTaskStatusFieldsMapped() throws IOException {
         Map<String, Object> template =
             XContentHelper.convertToMap(XContentType.JSON.xContent(), MonitoringTemplateUtils.loadTemplate("es"), false);
         Map<?, ?> autoFollowStatsMapping =
-            (Map<?, ?>) XContentMapValues.extractValue("mappings.doc.properties.ccr_auto_follow_stats.properties", template);
+            (Map<?, ?>) XContentMapValues.extractValue("mappings._doc.properties.ccr_auto_follow_stats.properties", template);
 
         assertThat(serializedStatus.size(), equalTo(autoFollowStatsMapping.size()));
         for (Map.Entry<String, Object> entry : serializedStatus.entrySet()) {

x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/monitoring/collector/ccr/FollowStatsMonitoringDocTests.java

@@ -237,7 +237,8 @@ public void testShardFollowNodeTaskStatusFieldsMapped() throws IOException {
 
         Map<String, Object> template =
             XContentHelper.convertToMap(XContentType.JSON.xContent(), MonitoringTemplateUtils.loadTemplate("es"), false);
-        Map<?, ?> followStatsMapping = (Map<?, ?>) XContentMapValues.extractValue("mappings.doc.properties.ccr_stats.properties", template);
+        Map<?, ?> followStatsMapping = (Map<?, ?>) XContentMapValues
+            .extractValue("mappings._doc.properties.ccr_stats.properties", template);
         assertThat(serializedStatus.size(), equalTo(followStatsMapping.size()));
         for (Map.Entry<String, Object> entry : serializedStatus.entrySet()) {
             String fieldName = entry.getKey();

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/monitoring/action/MonitoringBulkRequest.java

@@ -66,14 +66,13 @@ public MonitoringBulkRequest add(MonitoringBulkDoc doc) {
      * Parses a monitoring bulk request and builds the list of documents to be indexed.
      */
     public MonitoringBulkRequest add(final MonitoredSystem system,
-                                     final String defaultType,
                                      final BytesReference content,
                                      final XContentType xContentType,
                                      final long timestamp,
                                      final long intervalMillis) throws IOException {
 
         // MonitoringBulkRequest accepts a body request that has the same format as the BulkRequest
-        new BulkRequestParser(false).parse(content, null, defaultType, null, null, null, true, xContentType,
+        new BulkRequestParser(false).parse(content, null, null, null, null, true, xContentType,
                 indexRequest -> {
                     // we no longer accept non-timestamped indexes from Kibana, LS, or Beats because we do not use the data
                     // and it was duplicated anyway; by simply dropping it, we allow BWC for older clients that still send it

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/monitoring/action/MonitoringBulkRequestBuilder.java

@@ -26,12 +26,11 @@ public MonitoringBulkRequestBuilder add(MonitoringBulkDoc doc) {
     }
 
     public MonitoringBulkRequestBuilder add(final MonitoredSystem system,
-                                            final String type,
                                             final BytesReference content,
                                             final XContentType xContentType,
                                             final long timestamp,
                                             final long intervalMillis) throws IOException {
-        request.add(system, type, content, xContentType, timestamp, intervalMillis);
+        request.add(system, content, xContentType, timestamp, intervalMillis);
         return this;
     }
 

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/monitoring/exporter/MonitoringTemplateUtils.java

@@ -33,18 +33,19 @@ public final class MonitoringTemplateUtils {
 
     /**
      * Current version of templates used in their name to differentiate from breaking changes (separate from product version).
+     * Version 7 has the same structure as version 6, but uses the `_doc` type.
      */
-    public static final String TEMPLATE_VERSION = "6";
+    public static final String TEMPLATE_VERSION = "7";
     /**
      * The previous version of templates, which we still support via the REST /_monitoring/bulk endpoint because
      * nothing changed for those documents.
      */
-    public static final String OLD_TEMPLATE_VERSION = "2";
+    public static final String OLD_TEMPLATE_VERSION = "6";
 
     /**
      * IDs of templates that can be used with {@linkplain #loadTemplate(String) loadTemplate}.
      */
-    public static final String[] TEMPLATE_IDS = { "alerts", "es", "kibana", "logstash", "beats" };
+    public static final String[] TEMPLATE_IDS = { "alerts-7", "es", "kibana", "logstash", "beats" };
 
     /**
      * IDs of templates that can be used with {@linkplain #createEmptyTemplate(String) createEmptyTemplate} that are not managed by a
@@ -54,7 +55,7 @@ public final class MonitoringTemplateUtils {
      * instances will attempt to create a named template based on the templates that they expect (e.g., ".monitoring-es-2") and not the
      * ones that we are creating.
      */
-    public static final String[] OLD_TEMPLATE_IDS = { "data", "es", "kibana", "logstash", "alerts" };
+    public static final String[] OLD_TEMPLATE_IDS = { "data", "es", "kibana", "logstash" }; //excluding alerts since 6.x watches use it
 
     /**
      * IDs of pipelines that can be used with
@@ -99,7 +100,7 @@ public static String loadTemplate(final String id) {
      * @see #OLD_TEMPLATE_VERSION
      */
     public static String createEmptyTemplate(final String id) {
-        // e.g., { "index_patterns": [ ".monitoring-data-2*" ], "version": 6000002 }
+        // e.g., { "index_patterns": [ ".monitoring-data-6*" ], "version": 6000002 }
         return "{\"index_patterns\":[\".monitoring-" + id + "-" + OLD_TEMPLATE_VERSION + "*\"],\"version\":" + LAST_UPDATED_VERSION + "}";
     }
 
@@ -120,7 +121,7 @@ public static String pipelineName(String id) {
      * The expectation is that you will call either {@link Strings#toString(XContentBuilder)} or
      * {@link BytesReference#bytes(XContentBuilder)}}.
      *
-     * @param id The API version (e.g., "2") to use
+     * @param id The API version (e.g., "6") to use
      * @param type The type of data you want to format for the request
      * @return Never {@code null}. Always an ended-object.
      * @throws IllegalArgumentException if {@code apiVersion} is unrecognized
@@ -131,103 +132,54 @@ public static XContentBuilder loadPipeline(final String id, final XContentType t
             case TEMPLATE_VERSION:
                 return emptyPipeline(type);
             case OLD_TEMPLATE_VERSION:
-                return pipelineForApiVersion2(type);
+                return pipelineForApiVersion6(type);
         }
 
         throw new IllegalArgumentException("unrecognized pipeline API version [" + id + "]");
     }
 
     /**
      * Create a pipeline to upgrade documents from {@link MonitoringTemplateUtils#OLD_TEMPLATE_VERSION}
-     * <pre><code>
-     * {
-     *   "description" : "This pipeline upgrades documents ...",
-     *   "version": 6000001,
-     *   "processors": [ ]
-     * }
-     * </code></pre>
      * The expectation is that you will call either {@link Strings#toString(XContentBuilder)} or
      * {@link BytesReference#bytes(XContentBuilder)}}.
      *
      * @param type The type of data you want to format for the request
      * @return Never {@code null}. Always an ended-object.
      * @see #LAST_UPDATED_VERSION
      */
-    static XContentBuilder pipelineForApiVersion2(final XContentType type) {
+    static XContentBuilder pipelineForApiVersion6(final XContentType type) {
         try {
-            // For now: We prepend the API version to the string so that it's easy to parse in the future; if we ever add metadata
-            //  to pipelines, then it would better serve this use case
             return XContentBuilder.builder(type.xContent()).startObject()
                     .field("description", "This pipeline upgrades documents from the older version of the Monitoring API to " +
-                                                "the newer version (" + TEMPLATE_VERSION + ") by fixing breaking " +
-                                                "changes in those older documents before they are indexed from the older version (" +
-                                                OLD_TEMPLATE_VERSION + ").")
+                        "the newer version (" + TEMPLATE_VERSION + ") by fixing breaking " +
+                        "changes in those older documents before they are indexed from the older version (" +
+                        OLD_TEMPLATE_VERSION + ").")
                     .field("version", LAST_UPDATED_VERSION)
                     .startArray("processors")
                         .startObject()
-                            // Drop the .monitoring-data-2 index and effectively drop unnecessary data (duplicate or simply unused)
+                            // remove the type
                             .startObject("script")
-                                .field("source",
-                                       "boolean legacyIndex = ctx._index == '.monitoring-data-2';" +
-                                       "if (legacyIndex || ctx._index.startsWith('.monitoring-es-2')) {" +
-                                         "if (ctx._type == 'cluster_info') {" +
-                                           "ctx._type = 'cluster_stats';" +
-                                           "ctx._id = null;" +
-                                         "} else if (legacyIndex || ctx._type == 'cluster_stats' || ctx._type == 'node') {" +
-                                           "String index = ctx._index;" +
-                                           "Object clusterUuid = ctx.cluster_uuid;" +
-                                           "Object timestamp = ctx.timestamp;" +
-
-                                           "ctx.clear();" +
-
-                                           "ctx._id = 'xpack_monitoring_2_drop_bucket';" +
-                                           "ctx._index = index;" +
-                                           "ctx._type = 'legacy_data';" +
-                                           "ctx.timestamp = timestamp;" +
-                                           "ctx.cluster_uuid = clusterUuid;" +
-                                         "}" +
-                                         "if (legacyIndex) {" +
-                                           "ctx._index = '<.monitoring-es-" + TEMPLATE_VERSION + "-{now}>';" +
-                                         "}" +
-                                       "}")
-                            .endObject()
-                        .endObject()
-                        .startObject()
-                            .startObject("rename")
-                                .field("field", "_type")
-                                .field("target_field", "type")
-                            .endObject()
-                        .endObject()
-                        .startObject()
-                            .startObject("set")
-                                .field("field", "_type")
-                                .field("value", "doc")
+                                .field("source","ctx._type = null" )
                             .endObject()
                         .endObject()
                         .startObject()
+                            // ensure the data lands in the correct index
                             .startObject("gsub")
                                 .field("field", "_index")
-                                .field("pattern", "(.monitoring-\\w+-)2(-.+)")
+                                .field("pattern", "(.monitoring-\\w+-)6(-.+)")
                                 .field("replacement", "$1" + TEMPLATE_VERSION + "$2")
                             .endObject()
                         .endObject()
                     .endArray()
-                    .endObject();
+                .endObject();
         } catch (final IOException e) {
             throw new RuntimeException("Failed to create pipeline to upgrade from older version [" + OLD_TEMPLATE_VERSION +
-                                       "] to the newer version [" + TEMPLATE_VERSION + "].", e);
+                "] to the newer version [" + TEMPLATE_VERSION + "].", e);
         }
     }
 
     /**
      * Create an empty pipeline.
-     * <pre><code>
-     * {
-     *   "description" : "This is a placeholder pipeline ...",
-     *   "version": 6000001,
-     *   "processors": [ ]
-     * }
-     * </code></pre>
      * The expectation is that you will call either {@link Strings#toString(XContentBuilder)} or
      * {@link BytesReference#bytes(XContentBuilder)}}.
      *

x-pack/plugin/core/src/main/resources/monitoring-alerts.json → x-pack/plugin/core/src/main/resources/monitoring-alerts-7.json

@@ -6,12 +6,12 @@
       "number_of_shards": 1,
       "number_of_replicas": 0,
       "auto_expand_replicas": "0-1",
-      "format": 6,
+      "format": 7,
       "codec": "best_compression"
     }
   },
   "mappings": {
-    "doc": {
+    "_doc": {
       "dynamic": false,
       "properties": {
         "timestamp": {

x-pack/plugin/core/src/main/resources/monitoring-beats.json

@@ -5,13 +5,13 @@
   "settings": {
     "index.auto_expand_replicas": "0-1",
     "index.codec": "best_compression",
-    "index.format": 6,
+    "index.format": 7,
     "index.number_of_replicas": 0,
     "index.number_of_shards": 1
   },
   "version": 7000099,
   "mappings": {
-    "doc": {
+    "_doc": {
       "dynamic": false,
       "properties": {
         "beats_state": {

x-pack/plugin/core/src/main/resources/monitoring-es.json

@@ -5,11 +5,11 @@
     "index.number_of_shards": 1,
     "index.number_of_replicas": 0,
     "index.auto_expand_replicas": "0-1",
-    "index.format": 6,
+    "index.format": 7,
     "index.codec": "best_compression"
   },
   "mappings": {
-    "doc": {
+    "_doc": {
       "date_detection": false,
       "dynamic": false,
       "properties": {

x-pack/plugin/core/src/main/resources/monitoring-kibana.json

@@ -5,11 +5,11 @@
     "index.number_of_shards": 1,
     "index.number_of_replicas": 0,
     "index.auto_expand_replicas": "0-1",
-    "index.format": 6,
+    "index.format": 7,
     "index.codec": "best_compression"
   },
   "mappings": {
-    "doc": {
+    "_doc": {
       "dynamic": false,
       "properties": {
         "cluster_uuid": {

x-pack/plugin/core/src/main/resources/monitoring-logstash.json

@@ -5,11 +5,11 @@
     "index.number_of_shards": 1,
     "index.number_of_replicas": 0,
     "index.auto_expand_replicas": "0-1",
-    "index.format": 6,
+    "index.format": 7,
     "index.codec": "best_compression"
   },
   "mappings": {
-    "doc": {
+    "_doc": {
       "dynamic": false,
       "properties": {
         "cluster_uuid": {

x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/HttpExporter.java

@@ -592,7 +592,8 @@ private static void configureTemplateResources(final Config config,
             resources.add(new TemplateHttpResource(resourceOwnerName, templateTimeout, templateName, templateLoader));
         }
 
-        // add old templates, like ".monitoring-data-2" and ".monitoring-es-2" so that other versions can continue to work
+        // Add dummy templates (e.g. ".monitoring-es-6") to enable the ability to check which version of the actual
+        // index template (e.g. ".monitoring-es") should be applied.
         boolean createLegacyTemplates =
                 TEMPLATE_CREATE_LEGACY_VERSIONS_SETTING.getConcreteSettingForNamespace(config.name()).get(config.settings());
         if (createLegacyTemplates) {

x-pack/plugin/monitoring/src/main/java/org/elasticsearch/xpack/monitoring/exporter/http/TemplateHttpResource.java

@@ -12,19 +12,26 @@
 import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.client.RestClient;
+import org.elasticsearch.cluster.metadata.IndexTemplateMetaData;
 import org.elasticsearch.common.Nullable;
+import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.xcontent.LoggingDeprecationHandler;
+import org.elasticsearch.common.xcontent.NamedXContentRegistry;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.xpack.core.monitoring.exporter.MonitoringTemplateUtils;
 
+import java.io.IOException;
 import java.util.Collections;
 import java.util.Map;
 import java.util.Objects;
 import java.util.TreeMap;
 import java.util.function.Supplier;
 
-import static org.elasticsearch.rest.BaseRestHandler.INCLUDE_TYPE_NAME_PARAMETER;
-
 /**
  * {@code TemplateHttpResource}s allow the checking and uploading of templates to a remote cluster.
  * <p>
@@ -87,9 +94,8 @@ protected void doCheck(final RestClient client, final ActionListener<Boolean> li
      */
     @Override
     protected void doPublish(final RestClient client, final ActionListener<Boolean> listener) {
-        Map<String, String> parameters = Collections.singletonMap(INCLUDE_TYPE_NAME_PARAMETER, "true");
         putResource(client, listener, logger,
-                    "/_template", templateName, parameters, this::templateToHttpEntity, "monitoring template",
+                    "/_template", templateName, Collections.emptyMap(), this::templateToHttpEntity, "monitoring template",
                     resourceOwnerName, "monitoring cluster");
     }
 
@@ -98,8 +104,17 @@ protected void doPublish(final RestClient client, final ActionListener<Boolean>
      *
      * @return Never {@code null}.
      */
-    HttpEntity templateToHttpEntity() {
-        return new StringEntity(template.get(), ContentType.APPLICATION_JSON);
+     HttpEntity templateToHttpEntity() {
+        // the internal representation of a template has type nested under mappings.
+        // this uses xContent to help remove the type before sending to the remote cluster
+        try (XContentParser parser = XContentFactory.xContent(XContentType.JSON)
+            .createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, template.get())) {
+            XContentBuilder builder = JsonXContent.contentBuilder();
+            IndexTemplateMetaData.Builder.removeType(IndexTemplateMetaData.Builder.fromXContent(parser, templateName), builder);
+            return new StringEntity(BytesReference.bytes(builder).utf8ToString(), ContentType.APPLICATION_JSON);
+        } catch (IOException ex) {
+            throw new IllegalStateException("Cannot serialize template [" + templateName + "] for monitoring export", ex);
+        }
     }
 
 }