diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index cbe2d8ba4a82d..10ca81c9ada3c 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -22,3 +22,6 @@ x-pack/plugin/core/src/main/resources/fleet-* @elastic/fleet # Kibana Security x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/KibanaOwnedReservedRoleDescriptors.java @elastic/kibana-security + +# APM +x-pack/plugin/apm-data @elastic/apm-server diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java index f157ef5ced63f..26d55885fe7a3 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java @@ -1882,6 +1882,13 @@ protected static boolean isXPackTemplate(String name) { // We have a naming convention that internal component templates contain `@`. See also index-templates.asciidoc. return true; } + if (name.startsWith("apm@") + || name.startsWith("apm-") + || name.startsWith("traces-apm") + || name.startsWith("metrics-apm") + || name.startsWith("logs-apm")) { + return true; + } switch (name) { case ".watches": case "security_audit_log": diff --git a/x-pack/plugin/apm-data/README.md b/x-pack/plugin/apm-data/README.md new file mode 100644 index 0000000000000..c4a0d97cb09c0 --- /dev/null +++ b/x-pack/plugin/apm-data/README.md @@ -0,0 +1,33 @@ +## APM Data plugin + +The APM data plugin installs index templates, component templates, and ingest pipelines for Elastic APM. + +All resources are defined as YAML under [src/main/resources](src/main/resources). + +The APM index templates rely on mappings from `x-pack-core`. +See [x-pack/plugin/core/src/main/resources](../core/src/main/resources). + +This plugin is intended to work with data produced by https://github.com/elastic/apm-data. + +## Testing + +## Unit testing + +Java unit tests cover basic, low-level details of the plugin, such as the parsing and loading of resources. +These can be run with: + +``` +./gradlew x-pack:plugin:apm-data:test +``` + +## Integration testing + +The index templates and ingest pipeline functionality is tested using YAML REST tests. +These can be run with: + +``` +./gradlew x-pack:plugin:apm-data:yamlRestTest +``` + +Refer to the [rest-api-spec documentation](../../../rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/README.asciidoc) +for information about writing YAML REST tests. diff --git a/x-pack/plugin/apm-data/build.gradle b/x-pack/plugin/apm-data/build.gradle new file mode 100644 index 0000000000000..d70eec8268cee --- /dev/null +++ b/x-pack/plugin/apm-data/build.gradle @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +apply plugin: 'elasticsearch.internal-es-plugin' +apply plugin: 'elasticsearch.internal-yaml-rest-test' +apply plugin: 'elasticsearch.internal-cluster-test' + +esplugin { + name 'x-pack-apm-data' + description 'The APM plugin defines APM data streams and ingest pipelines.' + classname 'org.elasticsearch.xpack.apmdata.APMPlugin' + extendedPlugins = ['x-pack-core'] +} + +dependencies { + compileOnly project(path: xpackModule('core')) + testImplementation project(path: ':x-pack:plugin:stack') + testImplementation(testArtifact(project(xpackModule('core')))) + clusterModules project(':modules:data-streams') + clusterModules project(':modules:ingest-common') + clusterModules project(':modules:ingest-geoip') + clusterModules project(':modules:ingest-user-agent') + clusterModules project(':modules:mapper-extras') + clusterModules project(xpackModule('analytics')) + clusterModules project(xpackModule('ilm')) + clusterModules project(xpackModule('mapper-aggregate-metric')) + clusterModules project(xpackModule('mapper-constant-keyword')) + clusterModules project(xpackModule('stack')) + clusterModules project(xpackModule('wildcard')) +} diff --git a/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistry.java b/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistry.java new file mode 100644 index 0000000000000..665ecc16a1e14 --- /dev/null +++ b/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistry.java @@ -0,0 +1,169 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.apmdata; + +import org.elasticsearch.client.internal.Client; +import org.elasticsearch.cluster.metadata.ComponentTemplate; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xcontent.NamedXContentRegistry; +import org.elasticsearch.xcontent.XContentParserConfiguration; +import org.elasticsearch.xcontent.yaml.YamlXContent; +import org.elasticsearch.xpack.core.ClientHelper; +import org.elasticsearch.xpack.core.XPackSettings; +import org.elasticsearch.xpack.core.template.IndexTemplateRegistry; +import org.elasticsearch.xpack.core.template.IngestPipelineConfig; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import static org.elasticsearch.xpack.apmdata.ResourceUtils.APM_TEMPLATE_VERSION_VARIABLE; +import static org.elasticsearch.xpack.apmdata.ResourceUtils.loadResource; +import static org.elasticsearch.xpack.apmdata.ResourceUtils.loadVersionedResourceUTF8; + +/** + * Creates all index templates and ingest pipelines that are required for using Elastic APM. + */ +public class APMIndexTemplateRegistry extends IndexTemplateRegistry { + private final int version; + + private final Map componentTemplates; + private final Map composableIndexTemplates; + private final List ingestPipelines; + private final boolean enabled; + + @SuppressWarnings("unchecked") + public APMIndexTemplateRegistry( + Settings nodeSettings, + ClusterService clusterService, + ThreadPool threadPool, + Client client, + NamedXContentRegistry xContentRegistry + ) { + super(nodeSettings, clusterService, threadPool, client, xContentRegistry); + + try { + final Map apmResources = XContentHelper.convertToMap( + YamlXContent.yamlXContent, + loadResource("/resources.yaml"), + false + ); + version = (((Number) apmResources.get("version")).intValue()); + final List componentTemplateNames = (List) apmResources.get("component-templates"); + final List indexTemplateNames = (List) apmResources.get("index-templates"); + final List ingestPipelineConfigs = (List) apmResources.get("ingest-pipelines"); + + componentTemplates = componentTemplateNames.stream() + .map(o -> (String) o) + .collect(Collectors.toMap(name -> name, name -> loadComponentTemplate(name, version))); + composableIndexTemplates = indexTemplateNames.stream() + .map(o -> (String) o) + .collect(Collectors.toMap(name -> name, name -> loadIndexTemplate(name, version))); + ingestPipelines = ingestPipelineConfigs.stream().map(o -> (Map>) o).map(map -> { + Map.Entry> pipelineConfig = map.entrySet().iterator().next(); + return loadIngestPipeline(pipelineConfig.getKey(), version, (List) pipelineConfig.getValue().get("dependencies")); + }).collect(Collectors.toList()); + + enabled = XPackSettings.APM_DATA_ENABLED.get(nodeSettings); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public int getVersion() { + return version; + } + + public boolean isEnabled() { + return enabled; + } + + public void close() { + clusterService.removeListener(this); + } + + @Override + protected String getOrigin() { + return ClientHelper.APM_ORIGIN; + } + + @Override + protected boolean requiresMasterNode() { + return true; + } + + @Override + protected Map getComponentTemplateConfigs() { + if (enabled) { + return componentTemplates; + } else { + return Map.of(); + } + } + + @Override + protected Map getComposableTemplateConfigs() { + if (enabled) { + return composableIndexTemplates; + } else { + return Map.of(); + } + } + + @Override + protected List getIngestPipelines() { + if (enabled) { + return ingestPipelines; + } else { + return Collections.emptyList(); + } + } + + private static ComponentTemplate loadComponentTemplate(String name, int version) { + try { + final byte[] content = loadVersionedResourceUTF8("/component-templates/" + name + ".yaml", version); + return ComponentTemplate.parse(YamlXContent.yamlXContent.createParser(XContentParserConfiguration.EMPTY, content)); + } catch (Exception e) { + throw new RuntimeException("failed to load APM component template: " + name, e); + } + } + + private static ComposableIndexTemplate loadIndexTemplate(String name, int version) { + try { + final byte[] content = loadVersionedResourceUTF8("/index-templates/" + name + ".yaml", version); + return ComposableIndexTemplate.parse(YamlXContent.yamlXContent.createParser(XContentParserConfiguration.EMPTY, content)); + } catch (Exception e) { + throw new RuntimeException("failed to load APM index template: " + name, e); + } + } + + private static IngestPipelineConfig loadIngestPipeline(String name, int version, @Nullable List dependencies) { + if (dependencies == null) { + dependencies = Collections.emptyList(); + } + return new YamlIngestPipelineConfig( + name, + "/ingest-pipelines/" + name + ".yaml", + version, + APM_TEMPLATE_VERSION_VARIABLE, + dependencies + ); + } + + @Override + protected boolean applyRolloverAfterTemplateV2Upgrade() { + return true; + } +} diff --git a/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/APMPlugin.java b/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/APMPlugin.java new file mode 100644 index 0000000000000..3d024078e1783 --- /dev/null +++ b/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/APMPlugin.java @@ -0,0 +1,73 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.apmdata; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.lucene.util.SetOnce; +import org.elasticsearch.client.internal.Client; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.routing.allocation.AllocationService; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.env.NodeEnvironment; +import org.elasticsearch.indices.IndicesService; +import org.elasticsearch.plugins.ActionPlugin; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.repositories.RepositoriesService; +import org.elasticsearch.script.ScriptService; +import org.elasticsearch.telemetry.TelemetryProvider; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.watcher.ResourceWatcherService; +import org.elasticsearch.xcontent.NamedXContentRegistry; + +import java.util.Collection; +import java.util.List; +import java.util.function.Supplier; + +public class APMPlugin extends Plugin implements ActionPlugin { + private static final Logger logger = LogManager.getLogger(APMPlugin.class); + private final Settings settings; + + private final SetOnce registry = new SetOnce<>(); + + public APMPlugin(Settings settings) { + this.settings = settings; + } + + @Override + public Collection createComponents( + Client client, + ClusterService clusterService, + ThreadPool threadPool, + ResourceWatcherService resourceWatcherService, + ScriptService scriptService, + NamedXContentRegistry xContentRegistry, + Environment environment, + NodeEnvironment nodeEnvironment, + NamedWriteableRegistry namedWriteableRegistry, + IndexNameExpressionResolver indexNameExpressionResolver, + Supplier repositoriesServiceSupplier, + TelemetryProvider telemetryProvider, + AllocationService allocationService, + IndicesService indicesService + ) { + registry.set(new APMIndexTemplateRegistry(settings, clusterService, threadPool, client, xContentRegistry)); + APMIndexTemplateRegistry registryInstance = registry.get(); + logger.info("APM is {}", registryInstance.isEnabled() ? "enabled" : "disabled"); + registryInstance.initialize(); + return List.of(registryInstance); + } + + @Override + public void close() { + registry.get().close(); + } +} diff --git a/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/ResourceUtils.java b/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/ResourceUtils.java new file mode 100644 index 0000000000000..b9a6edfb958f3 --- /dev/null +++ b/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/ResourceUtils.java @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.apmdata; + +import org.elasticsearch.common.io.Streams; +import org.elasticsearch.xpack.core.template.TemplateUtils; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; + +public class ResourceUtils { + + public static final String APM_TEMPLATE_VERSION_VARIABLE = "xpack.apmdata.template.version"; + + static byte[] loadVersionedResourceUTF8(String name, int version) { + try { + String content = loadResource(name); + content = TemplateUtils.replaceVariable(content, APM_TEMPLATE_VERSION_VARIABLE, String.valueOf(version)); + return content.getBytes(StandardCharsets.UTF_8); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + static String loadResource(String name) throws IOException { + InputStream is = APMIndexTemplateRegistry.class.getResourceAsStream(name); + if (is == null) { + throw new IOException("Resource [" + name + "] not found in classpath."); + } + return Streams.readFully(is).utf8ToString(); + } +} diff --git a/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/YamlIngestPipelineConfig.java b/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/YamlIngestPipelineConfig.java new file mode 100644 index 0000000000000..938fd69f80abe --- /dev/null +++ b/x-pack/plugin/apm-data/src/main/java/org/elasticsearch/xpack/apmdata/YamlIngestPipelineConfig.java @@ -0,0 +1,36 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.apmdata; + +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.core.template.IngestPipelineConfig; + +import java.util.List; + +import static org.elasticsearch.xpack.apmdata.ResourceUtils.loadVersionedResourceUTF8; + +/** + * An APM-plugin-specific implementation that loads ingest pipelines in yaml format from a local resources repository + */ +public class YamlIngestPipelineConfig extends IngestPipelineConfig { + public YamlIngestPipelineConfig(String id, String resource, int version, String versionProperty, List dependencies) { + super(id, resource, version, versionProperty, dependencies); + } + + @Override + public XContentType getXContentType() { + return XContentType.YAML; + } + + @Override + public BytesReference loadConfig() { + return new BytesArray(loadVersionedResourceUTF8("/ingest-pipelines/" + id + ".yaml", version)); + } +} diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/apm-10d@lifecycle.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/apm-10d@lifecycle.yaml new file mode 100644 index 0000000000000..ba0e06c95ddc1 --- /dev/null +++ b/x-pack/plugin/apm-data/src/main/resources/component-templates/apm-10d@lifecycle.yaml @@ -0,0 +1,8 @@ +--- +version: ${xpack.apmdata.template.version} +_meta: + description: Data stream lifecycle for 10 days of retention + managed: true +template: + lifecycle: + data_retention: "10d" diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/apm-180d@lifecycle.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/apm-180d@lifecycle.yaml new file mode 100644 index 0000000000000..00e97d5371bca --- /dev/null +++ b/x-pack/plugin/apm-data/src/main/resources/component-templates/apm-180d@lifecycle.yaml @@ -0,0 +1,8 @@ +--- +version: ${xpack.apmdata.template.version} +_meta: + description: Data stream lifecycle for 180 days of retention + managed: true +template: + lifecycle: + data_retention: "180d" diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/apm-390d@lifecycle.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/apm-390d@lifecycle.yaml new file mode 100644 index 0000000000000..b2e95a4da525f --- /dev/null +++ b/x-pack/plugin/apm-data/src/main/resources/component-templates/apm-390d@lifecycle.yaml @@ -0,0 +1,8 @@ +--- +version: ${xpack.apmdata.template.version} +_meta: + description: Data stream lifecycle for 390 days of retention + managed: true +template: + lifecycle: + data_retention: "390d" diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/apm-90d@lifecycle.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/apm-90d@lifecycle.yaml new file mode 100644 index 0000000000000..4b6ebe236eeac --- /dev/null +++ b/x-pack/plugin/apm-data/src/main/resources/component-templates/apm-90d@lifecycle.yaml @@ -0,0 +1,8 @@ +--- +version: ${xpack.apmdata.template.version} +_meta: + description: Data stream lifecycle for 90 days of retention + managed: true +template: + lifecycle: + data_retention: "90d" diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/apm@mappings.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/apm@mappings.yaml new file mode 100644 index 0000000000000..ac6462c86676c --- /dev/null +++ b/x-pack/plugin/apm-data/src/main/resources/component-templates/apm@mappings.yaml @@ -0,0 +1,18 @@ +version: ${xpack.apmdata.template.version} +_meta: + description: Default mappings for all APM data streams + managed: true +template: + mappings: + dynamic: true + dynamic_templates: + - numeric_labels: + path_match: numeric_labels.* + mapping: + type: scaled_float + scaling_factor: 1000000 + properties: + data_stream.dataset: + type: constant_keyword + data_stream.namespace: + type: constant_keyword diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/apm@settings.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/apm@settings.yaml new file mode 100644 index 0000000000000..3ca15224dafc4 --- /dev/null +++ b/x-pack/plugin/apm-data/src/main/resources/component-templates/apm@settings.yaml @@ -0,0 +1,10 @@ +version: ${xpack.apmdata.template.version} +_meta: + description: Default settings for all APM data streams + managed: true +template: + settings: + index: + sort: + field: "@timestamp" + order: desc diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/logs-apm.error@mappings.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/logs-apm.error@mappings.yaml new file mode 100644 index 0000000000000..c946403c795dd --- /dev/null +++ b/x-pack/plugin/apm-data/src/main/resources/component-templates/logs-apm.error@mappings.yaml @@ -0,0 +1,26 @@ +--- +version: ${xpack.apmdata.template.version} +_meta: + description: Default mappings for logs-apm.error-* data streams + managed: true +template: + mappings: + properties: + error.custom: + type: flattened + error.exception.attributes: + type: flattened + error.exception.stacktrace: + type: flattened + error.grouping_name: + type: keyword + script: | + def logMessage = params['_source'].error?.log?.message; + if (logMessage != null) { + emit(logMessage); + return; + } + def exception = params['_source'].error?.exception; + if (exception != null && exception.length > 0) { + emit(exception[0].message); + } diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_destination@mappings.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_destination@mappings.yaml new file mode 100644 index 0000000000000..2997dc923b9c4 --- /dev/null +++ b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_destination@mappings.yaml @@ -0,0 +1,23 @@ +--- +version: ${xpack.apmdata.template.version} +_meta: + description: Default mappings for metrics-apm.service_destination.* data streams + managed: true +template: + mappings: + properties: + metricset.name: + type: constant_keyword + value: service_destination + metricset.interval: + type: constant_keyword + transaction.duration.histogram: + type: histogram + transaction.duration.summary: + type: aggregate_metric_double + metrics: [sum, value_count] + default_metric: sum + event.success_count: + type: aggregate_metric_double + metrics: [sum, value_count] + default_metric: sum diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_summary@mappings.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_summary@mappings.yaml new file mode 100644 index 0000000000000..011dfaf6a94b1 --- /dev/null +++ b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_summary@mappings.yaml @@ -0,0 +1,23 @@ +--- +version: ${xpack.apmdata.template.version} +_meta: + description: Default mappings for metrics-apm.service_summary.* data streams + managed: true +template: + mappings: + properties: + metricset.name: + type: constant_keyword + value: service_summary + metricset.interval: + type: constant_keyword + transaction.duration.histogram: + type: histogram + transaction.duration.summary: + type: aggregate_metric_double + metrics: [sum, value_count] + default_metric: sum + event.success_count: + type: aggregate_metric_double + metrics: [sum, value_count] + default_metric: sum diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_transaction@mappings.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_transaction@mappings.yaml new file mode 100644 index 0000000000000..5f8f5e8dc8128 --- /dev/null +++ b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.service_transaction@mappings.yaml @@ -0,0 +1,22 @@ +version: ${xpack.apmdata.template.version} +_meta: + description: Default mappings for metrics-apm.service_transaction.* data streams + managed: true +template: + mappings: + properties: + metricset.name: + type: constant_keyword + value: service_transaction + metricset.interval: + type: constant_keyword + transaction.duration.histogram: + type: histogram + transaction.duration.summary: + type: aggregate_metric_double + metrics: [sum, value_count] + default_metric: sum + event.success_count: + type: aggregate_metric_double + metrics: [sum, value_count] + default_metric: sum diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.transaction@mappings.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.transaction@mappings.yaml new file mode 100644 index 0000000000000..81f5d6c70b229 --- /dev/null +++ b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm.transaction@mappings.yaml @@ -0,0 +1,22 @@ +version: ${xpack.apmdata.template.version} +_meta: + description: Default mappings for metrics-apm.transaction.* data streams + managed: true +template: + mappings: + properties: + metricset.name: + type: constant_keyword + value: transaction + metricset.interval: + type: constant_keyword + transaction.duration.histogram: + type: histogram + transaction.duration.summary: + type: aggregate_metric_double + metrics: [sum, value_count] + default_metric: sum + event.success_count: + type: aggregate_metric_double + metrics: [sum, value_count] + default_metric: sum diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm@mappings.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm@mappings.yaml new file mode 100644 index 0000000000000..af28cbb7415a0 --- /dev/null +++ b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm@mappings.yaml @@ -0,0 +1,13 @@ +--- +version: ${xpack.apmdata.template.version} +_meta: + description: Default mappings for metrics-apm.* data streams + managed: true +template: + mappings: + _source: + mode: synthetic + properties: + processor.event: + type: constant_keyword + value: metric diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm@settings.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm@settings.yaml new file mode 100644 index 0000000000000..e6c84b6ed06f9 --- /dev/null +++ b/x-pack/plugin/apm-data/src/main/resources/component-templates/metrics-apm@settings.yaml @@ -0,0 +1,8 @@ +--- +version: ${xpack.apmdata.template.version} +_meta: + description: Default settings for metrics-apm.* data streams + managed: true +template: + settings: + codec: best_compression diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/traces-apm.rum@mappings.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/traces-apm.rum@mappings.yaml new file mode 100644 index 0000000000000..833814d3fc4cc --- /dev/null +++ b/x-pack/plugin/apm-data/src/main/resources/component-templates/traces-apm.rum@mappings.yaml @@ -0,0 +1,26 @@ +--- +version: ${xpack.apmdata.template.version} +_meta: + description: Default mappings for traces-apm.rum-* data streams + managed: true +template: + mappings: + properties: + transaction.experience.cls: + type: scaled_float + scaling_factor: 1000 + transaction.experience.fid: + type: scaled_float + scaling_factor: 1000 + transaction.experience.tbt: + type: scaled_float + scaling_factor: 1000 + transaction.experience.longtask.count: + type: scaled_float + scaling_factor: 1000 + transaction.experience.longtask.max: + type: scaled_float + scaling_factor: 1000 + transaction.experience.longtask.sum: + type: scaled_float + scaling_factor: 1000 diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/traces-apm@mappings.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/traces-apm@mappings.yaml new file mode 100644 index 0000000000000..558a5da81e4f7 --- /dev/null +++ b/x-pack/plugin/apm-data/src/main/resources/component-templates/traces-apm@mappings.yaml @@ -0,0 +1,34 @@ +--- +version: ${xpack.apmdata.template.version} +_meta: + description: Default mappings for traces-apm* data streams + managed: true +template: + mappings: + properties: + processor.event: + type: keyword + event.success_count: + type: byte + index: false + span.duration.us: + type: long + transaction.duration.us: + type: long + http.response.transfer_size: + type: long + index: false + http.response.encoded_body_size: + type: long + index: false + http.response.decoded_body_size: + type: long + index: false + span.representative_count: + type: scaled_float + scaling_factor: 1000 + index: false + transaction.representative_count: + type: scaled_float + scaling_factor: 1000 + index: false diff --git a/x-pack/plugin/apm-data/src/main/resources/component-templates/traces@mappings.yaml b/x-pack/plugin/apm-data/src/main/resources/component-templates/traces@mappings.yaml new file mode 100644 index 0000000000000..51c987df4df60 --- /dev/null +++ b/x-pack/plugin/apm-data/src/main/resources/component-templates/traces@mappings.yaml @@ -0,0 +1,11 @@ +--- +version: ${xpack.apmdata.template.version} +_meta: + description: Default mappings for traces data streams + managed: true +template: + mappings: + properties: + data_stream.type: + type: constant_keyword + value: traces diff --git a/x-pack/plugin/apm-data/src/main/resources/index-templates/README.md b/x-pack/plugin/apm-data/src/main/resources/index-templates/README.md new file mode 100644 index 0000000000000..99f4f4218ca90 --- /dev/null +++ b/x-pack/plugin/apm-data/src/main/resources/index-templates/README.md @@ -0,0 +1,7 @@ +Mappings and settings that must not be overridden are to be defined +in the index templates, as these will take precedence over component +template mappings and settings. This includes: + + - `index.default_pipeline` + - `index.final_pipeline` + - values for `data_stream.*` diff --git a/x-pack/plugin/apm-data/src/main/resources/index-templates/logs-apm.app@template.yaml b/x-pack/plugin/apm-data/src/main/resources/index-templates/logs-apm.app@template.yaml new file mode 100644 index 0000000000000..0ebbb99a1e379 --- /dev/null +++ b/x-pack/plugin/apm-data/src/main/resources/index-templates/logs-apm.app@template.yaml @@ -0,0 +1,24 @@ +version: ${xpack.apmdata.template.version} +index_patterns: ["logs-apm.app.*-*"] +priority: 140 +data_stream: {} +allow_auto_create: true +_meta: + description: Index template for logs-apm.app.*-* + managed: true +composed_of: +- logs@mappings +- apm@mappings +- apm@settings +- apm-10d@lifecycle +- apm@custom +- logs-apm.app@custom +- ecs@mappings +ignore_missing_component_templates: +- logs-apm.app@custom +- apm@custom +template: + settings: + index: + default_pipeline: apm@default-pipeline + final_pipeline: apm@pipeline diff --git a/x-pack/plugin/apm-data/src/main/resources/index-templates/logs-apm.error@template.yaml b/x-pack/plugin/apm-data/src/main/resources/index-templates/logs-apm.error@template.yaml new file mode 100644 index 0000000000000..831f7cc404415 --- /dev/null +++ b/x-pack/plugin/apm-data/src/main/resources/index-templates/logs-apm.error@template.yaml @@ -0,0 +1,31 @@ +--- +version: ${xpack.apmdata.template.version} +index_patterns: ["logs-apm.error-*"] +priority: 140 +data_stream: {} +allow_auto_create: true +_meta: + description: Index template for logs-apm.error-* + managed: true +composed_of: +- logs@mappings +- apm@mappings +- apm@settings +- apm-10d@lifecycle +- logs-apm.error@mappings +- apm@custom +- logs-apm.error@custom +- ecs@mappings +ignore_missing_component_templates: +- logs-apm.error@custom +- apm@custom +template: + mappings: + properties: + processor.event: + type: constant_keyword + value: error + settings: + index: + default_pipeline: apm@default-pipeline + final_pipeline: apm@pipeline diff --git a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.app@template.yaml b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.app@template.yaml new file mode 100644 index 0000000000000..bdd1fa363bcf4 --- /dev/null +++ b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.app@template.yaml @@ -0,0 +1,26 @@ +version: ${xpack.apmdata.template.version} +index_patterns: ["metrics-apm.app.*-*"] +priority: 140 +data_stream: {} +allow_auto_create: true +_meta: + description: "Index template for metrics-apm.app.*-*" + managed: true +composed_of: +- metrics@mappings +- apm@mappings +- apm@settings +- apm-90d@lifecycle +- metrics-apm@mappings +- metrics-apm@settings +- apm@custom +- metrics-apm.app@custom +- ecs@mappings +ignore_missing_component_templates: +- apm@custom +- metrics-apm.app@custom +template: + settings: + index: + default_pipeline: apm@default-pipeline + final_pipeline: metrics-apm@pipeline diff --git a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.internal@template.yaml b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.internal@template.yaml new file mode 100644 index 0000000000000..205784e22e685 --- /dev/null +++ b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.internal@template.yaml @@ -0,0 +1,32 @@ +--- +version: ${xpack.apmdata.template.version} +index_patterns: ["metrics-apm.internal-*"] +priority: 140 +data_stream: {} +allow_auto_create: true +_meta: + description: "Index template for metrics-apm.internal-*" + managed: true +composed_of: +- metrics@mappings +- apm@mappings +- apm@settings +- apm-90d@lifecycle +- metrics-apm@mappings +- metrics-apm@settings +- apm@custom +- metrics-apm.internal@custom +- ecs@mappings +ignore_missing_component_templates: +- apm@custom +- metrics-apm.internal@custom +template: + settings: + index: + default_pipeline: apm@default-pipeline + final_pipeline: metrics-apm@pipeline + mappings: + properties: + data_stream.dataset: + type: constant_keyword + value: apm.internal diff --git a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_destination.10m@template.yaml b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_destination.10m@template.yaml new file mode 100644 index 0000000000000..6279e044fbfcf --- /dev/null +++ b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_destination.10m@template.yaml @@ -0,0 +1,29 @@ +--- +version: ${xpack.apmdata.template.version} +index_patterns: [metrics-apm.service_destination.10m-*] +priority: 140 +data_stream: + hidden: true +allow_auto_create: true +_meta: + description: Index template for metrics-apm.service_destination.10m-* + managed: true +composed_of: +- metrics@mappings +- apm@mappings +- apm@settings +- apm-180d@lifecycle +- metrics-apm@mappings +- metrics-apm@settings +- metrics-apm.service_destination@mappings +- apm@custom +- metrics-apm.service_destination@custom +- ecs@mappings +ignore_missing_component_templates: +- apm@custom +- metrics-apm.service_destination@custom +template: + settings: + index: + default_pipeline: apm@default-pipeline + final_pipeline: metrics-apm@pipeline diff --git a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_destination.1m@template.yaml b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_destination.1m@template.yaml new file mode 100644 index 0000000000000..10e4ca5b39a52 --- /dev/null +++ b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_destination.1m@template.yaml @@ -0,0 +1,28 @@ +--- +version: ${xpack.apmdata.template.version} +index_patterns: [metrics-apm.service_destination.1m-*] +priority: 140 +data_stream: {} +allow_auto_create: true +_meta: + description: Index template for metrics-apm.service_destination.1m-* + managed: true +composed_of: +- metrics@mappings +- apm@mappings +- apm@settings +- apm-90d@lifecycle +- metrics-apm@mappings +- metrics-apm@settings +- metrics-apm.service_destination@mappings +- apm@custom +- metrics-apm.service_destination@custom +- ecs@mappings +ignore_missing_component_templates: +- apm@custom +- metrics-apm.service_destination@custom +template: + settings: + index: + default_pipeline: apm@default-pipeline + final_pipeline: metrics-apm@pipeline diff --git a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_destination.60m@template.yaml b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_destination.60m@template.yaml new file mode 100644 index 0000000000000..dbac0d0d17d89 --- /dev/null +++ b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_destination.60m@template.yaml @@ -0,0 +1,29 @@ +--- +version: ${xpack.apmdata.template.version} +index_patterns: [metrics-apm.service_destination.60m-*] +priority: 140 +data_stream: + hidden: true +allow_auto_create: true +_meta: + description: Index template for metrics-apm.service_destination.60m-* + managed: true +composed_of: +- metrics@mappings +- apm@mappings +- apm@settings +- apm-390d@lifecycle +- metrics-apm@mappings +- metrics-apm@settings +- metrics-apm.service_destination@mappings +- apm@custom +- metrics-apm.service_destination@custom +- ecs@mappings +ignore_missing_component_templates: +- apm@custom +- metrics-apm.service_destination@custom +template: + settings: + index: + default_pipeline: apm@default-pipeline + final_pipeline: metrics-apm@pipeline diff --git a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_summary.10m@template.yaml b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_summary.10m@template.yaml new file mode 100644 index 0000000000000..af99e419d4a56 --- /dev/null +++ b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_summary.10m@template.yaml @@ -0,0 +1,29 @@ +--- +version: ${xpack.apmdata.template.version} +index_patterns: [metrics-apm.service_summary.10m-*] +priority: 140 +data_stream: + hidden: true +allow_auto_create: true +_meta: + description: Index template for metrics-apm.service_summary.10m-* + managed: true +composed_of: +- metrics@mappings +- apm@mappings +- apm@settings +- apm-180d@lifecycle +- metrics-apm@mappings +- metrics-apm@settings +- metrics-apm.service_summary@mappings +- apm@custom +- metrics-apm.service_summary@custom +- ecs@mappings +ignore_missing_component_templates: +- apm@custom +- metrics-apm.service_summary@custom +template: + settings: + index: + default_pipeline: apm@default-pipeline + final_pipeline: metrics-apm@pipeline diff --git a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_summary.1m@template.yaml b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_summary.1m@template.yaml new file mode 100644 index 0000000000000..29c28953d6b40 --- /dev/null +++ b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_summary.1m@template.yaml @@ -0,0 +1,28 @@ +--- +version: ${xpack.apmdata.template.version} +index_patterns: [metrics-apm.service_summary.1m-*] +priority: 140 +data_stream: {} +allow_auto_create: true +_meta: + description: Index template for metrics-apm.service_summary.1m-* + managed: true +composed_of: +- metrics@mappings +- apm@mappings +- apm@settings +- apm-90d@lifecycle +- metrics-apm@mappings +- metrics-apm@settings +- metrics-apm.service_summary@mappings +- apm@custom +- metrics-apm.service_summary@custom +- ecs@mappings +ignore_missing_component_templates: +- apm@custom +- metrics-apm.service_summary@custom +template: + settings: + index: + default_pipeline: apm@default-pipeline + final_pipeline: metrics-apm@pipeline diff --git a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_summary.60m@template.yaml b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_summary.60m@template.yaml new file mode 100644 index 0000000000000..bdbd4900df3bb --- /dev/null +++ b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_summary.60m@template.yaml @@ -0,0 +1,29 @@ +--- +version: ${xpack.apmdata.template.version} +index_patterns: [metrics-apm.service_summary.60m-*] +priority: 140 +data_stream: + hidden: true +allow_auto_create: true +_meta: + description: Index template for metrics-apm.service_summary.60m-* + managed: true +composed_of: +- metrics@mappings +- apm@mappings +- apm@settings +- apm-390d@lifecycle +- metrics-apm@mappings +- metrics-apm@settings +- metrics-apm.service_summary@mappings +- apm@custom +- metrics-apm.service_summary@custom +- ecs@mappings +ignore_missing_component_templates: +- apm@custom +- metrics-apm.service_summary@custom +template: + settings: + index: + default_pipeline: apm@default-pipeline + final_pipeline: metrics-apm@pipeline diff --git a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_transaction.10m@template.yaml b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_transaction.10m@template.yaml new file mode 100644 index 0000000000000..8b4e88391a475 --- /dev/null +++ b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_transaction.10m@template.yaml @@ -0,0 +1,29 @@ +--- +version: ${xpack.apmdata.template.version} +index_patterns: [metrics-apm.service_transaction.10m-*] +priority: 140 +data_stream: + hidden: true +allow_auto_create: true +_meta: + description: Index template for metrics-apm.service_transaction.10m-* + managed: true +composed_of: +- metrics@mappings +- apm@mappings +- apm@settings +- apm-180d@lifecycle +- metrics-apm@mappings +- metrics-apm@settings +- metrics-apm.service_transaction@mappings +- apm@custom +- metrics-apm.service_transaction@custom +- ecs@mappings +ignore_missing_component_templates: +- apm@custom +- metrics-apm.service_transaction@custom +template: + settings: + index: + default_pipeline: apm@default-pipeline + final_pipeline: metrics-apm@pipeline diff --git a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_transaction.1m@template.yaml b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_transaction.1m@template.yaml new file mode 100644 index 0000000000000..811067f8e6f30 --- /dev/null +++ b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_transaction.1m@template.yaml @@ -0,0 +1,28 @@ +--- +version: ${xpack.apmdata.template.version} +index_patterns: [metrics-apm.service_transaction.1m-*] +priority: 140 +data_stream: {} +allow_auto_create: true +_meta: + description: Index template for metrics-apm.service_transaction.1m-* + managed: true +composed_of: +- metrics@mappings +- apm@mappings +- apm@settings +- apm-90d@lifecycle +- metrics-apm@mappings +- metrics-apm@settings +- metrics-apm.service_transaction@mappings +- apm@custom +- metrics-apm.service_transaction@custom +- ecs@mappings +ignore_missing_component_templates: +- apm@custom +- metrics-apm.service_transaction@custom +template: + settings: + index: + default_pipeline: apm@default-pipeline + final_pipeline: metrics-apm@pipeline diff --git a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_transaction.60m@template.yaml b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_transaction.60m@template.yaml new file mode 100644 index 0000000000000..db28b7c56aaab --- /dev/null +++ b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.service_transaction.60m@template.yaml @@ -0,0 +1,29 @@ +--- +version: ${xpack.apmdata.template.version} +index_patterns: [metrics-apm.service_transaction.60m-*] +priority: 140 +data_stream: + hidden: true +allow_auto_create: true +_meta: + description: Index template for metrics-apm.service_transaction.60m-* + managed: true +composed_of: +- metrics@mappings +- apm@mappings +- apm@settings +- apm-390d@lifecycle +- metrics-apm@mappings +- metrics-apm@settings +- metrics-apm.service_transaction@mappings +- apm@custom +- metrics-apm.service_transaction@custom +- ecs@mappings +ignore_missing_component_templates: +- apm@custom +- metrics-apm.service_transaction@custom +template: + settings: + index: + default_pipeline: apm@default-pipeline + final_pipeline: metrics-apm@pipeline diff --git a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.transaction.10m@template.yaml b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.transaction.10m@template.yaml new file mode 100644 index 0000000000000..548f73656fda4 --- /dev/null +++ b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.transaction.10m@template.yaml @@ -0,0 +1,29 @@ +--- +version: ${xpack.apmdata.template.version} +index_patterns: [metrics-apm.transaction.10m-*] +priority: 140 +data_stream: + hidden: true +allow_auto_create: true +_meta: + description: Index template for metrics-apm.transaction.10m-* + managed: true +composed_of: +- metrics@mappings +- apm@mappings +- apm@settings +- apm-180d@lifecycle +- metrics-apm@mappings +- metrics-apm@settings +- metrics-apm.transaction@mappings +- apm@custom +- metrics-apm.transaction@custom +- ecs@mappings +ignore_missing_component_templates: +- apm@custom +- metrics-apm.transaction@custom +template: + settings: + index: + default_pipeline: apm@default-pipeline + final_pipeline: metrics-apm@pipeline diff --git a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.transaction.1m@template.yaml b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.transaction.1m@template.yaml new file mode 100644 index 0000000000000..6206e7c126c48 --- /dev/null +++ b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.transaction.1m@template.yaml @@ -0,0 +1,28 @@ +--- +version: ${xpack.apmdata.template.version} +index_patterns: [metrics-apm.transaction.1m-*] +priority: 140 +data_stream: {} +allow_auto_create: true +_meta: + description: Index template for metrics-apm.transaction.1m-* + managed: true +composed_of: +- metrics@mappings +- apm@mappings +- apm@settings +- apm-90d@lifecycle +- metrics-apm@mappings +- metrics-apm@settings +- metrics-apm.transaction@mappings +- apm@custom +- metrics-apm.transaction@custom +- ecs@mappings +ignore_missing_component_templates: +- apm@custom +- metrics-apm.transaction@custom +template: + settings: + index: + default_pipeline: apm@default-pipeline + final_pipeline: metrics-apm@pipeline diff --git a/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.transaction.60m@template.yaml b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.transaction.60m@template.yaml new file mode 100644 index 0000000000000..4ad00aecf23a5 --- /dev/null +++ b/x-pack/plugin/apm-data/src/main/resources/index-templates/metrics-apm.transaction.60m@template.yaml @@ -0,0 +1,29 @@ +--- +version: ${xpack.apmdata.template.version} +index_patterns: [metrics-apm.transaction.60m-*] +priority: 140 +data_stream: + hidden: true +allow_auto_create: true +_meta: + description: Index template for metrics-apm.transaction.60m-* + managed: true +composed_of: +- metrics@mappings +- apm@mappings +- apm@settings +- apm-390d@lifecycle +- metrics-apm@mappings +- metrics-apm@settings +- metrics-apm.transaction@mappings +- apm@custom +- metrics-apm.transaction@custom +- ecs@mappings +ignore_missing_component_templates: +- apm@custom +- metrics-apm.transaction@custom +template: + settings: + index: + default_pipeline: apm@default-pipeline + final_pipeline: metrics-apm@pipeline diff --git a/x-pack/plugin/apm-data/src/main/resources/index-templates/traces-apm.rum@template.yaml b/x-pack/plugin/apm-data/src/main/resources/index-templates/traces-apm.rum@template.yaml new file mode 100644 index 0000000000000..174faf432eb6e --- /dev/null +++ b/x-pack/plugin/apm-data/src/main/resources/index-templates/traces-apm.rum@template.yaml @@ -0,0 +1,37 @@ +--- +version: ${xpack.apmdata.template.version} +index_patterns: ["traces-apm.rum-*"] +priority: 140 +data_stream: {} +allow_auto_create: true +_meta: + description: Index template for traces-apm.rum-* + managed: true +composed_of: +- traces@mappings +- apm@mappings +- apm@settings +- apm-90d@lifecycle +- traces-apm@mappings +- traces-apm.rum@mappings +- apm@custom +- traces-apm@custom +- traces-apm.rum@custom +- ecs@mappings +ignore_missing_component_templates: +- traces-apm.rum@custom +- traces-apm@custom +- apm@custom +template: + settings: + index: + default_pipeline: apm@default-pipeline + final_pipeline: traces-apm@pipeline + mappings: + properties: + data_stream.type: + type: constant_keyword + value: traces + data_stream.dataset: + type: constant_keyword + value: apm.rum diff --git a/x-pack/plugin/apm-data/src/main/resources/index-templates/traces-apm.sampled@template.yaml b/x-pack/plugin/apm-data/src/main/resources/index-templates/traces-apm.sampled@template.yaml new file mode 100644 index 0000000000000..8c65c69bc3afa --- /dev/null +++ b/x-pack/plugin/apm-data/src/main/resources/index-templates/traces-apm.sampled@template.yaml @@ -0,0 +1,32 @@ +--- +version: ${xpack.apmdata.template.version} +index_patterns: ["traces-apm.sampled-*"] +priority: 140 +data_stream: {} +allow_auto_create: true +_meta: + description: Index template for traces-apm.sampled-* + managed: true +composed_of: +- traces@mappings +- apm@mappings +- apm@settings +- apm@custom +- ecs@mappings +ignore_missing_component_templates: +- apm@custom +template: + lifecycle: + data_retention: 1h + mappings: + properties: + data_stream.type: + type: constant_keyword + value: traces + data_stream.dataset: + type: constant_keyword + value: apm.sampled + event.ingested: + type: date + format: date_time_no_millis + script: "emit(System.currentTimeMillis())" diff --git a/x-pack/plugin/apm-data/src/main/resources/index-templates/traces-apm@template.yaml b/x-pack/plugin/apm-data/src/main/resources/index-templates/traces-apm@template.yaml new file mode 100644 index 0000000000000..fb6670a7f7143 --- /dev/null +++ b/x-pack/plugin/apm-data/src/main/resources/index-templates/traces-apm@template.yaml @@ -0,0 +1,34 @@ +--- +version: ${xpack.apmdata.template.version} +index_patterns: ["traces-apm-*"] +priority: 140 +data_stream: {} +allow_auto_create: true +_meta: + description: Index template for traces-apm-* + managed: true +composed_of: +- traces@mappings +- apm@mappings +- apm@settings +- apm-10d@lifecycle +- traces-apm@mappings +- apm@custom +- traces-apm@custom +- ecs@mappings +ignore_missing_component_templates: +- traces-apm@custom +- apm@custom +template: + settings: + index: + default_pipeline: apm@default-pipeline + final_pipeline: traces-apm@pipeline + mappings: + properties: + data_stream.type: + type: constant_keyword + value: traces + data_stream.dataset: + type: constant_keyword + value: apm diff --git a/x-pack/plugin/apm-data/src/main/resources/ingest-pipelines/apm@default-pipeline.yaml b/x-pack/plugin/apm-data/src/main/resources/ingest-pipelines/apm@default-pipeline.yaml new file mode 100644 index 0000000000000..e1290215a5a59 --- /dev/null +++ b/x-pack/plugin/apm-data/src/main/resources/ingest-pipelines/apm@default-pipeline.yaml @@ -0,0 +1,45 @@ +--- +version: ${xpack.apmdata.template.version} +_meta: + managed: true +description: | + Built-in default ingest pipeline for all APM data streams. + + This pipeline exists purely for routing, which cannot be + performed in a final pipeline, and for invoking user-defined + custom pipelines. All built-in processing occurs in the final + pipelines. +processors: + # Older versions of apm-server write various metrics to the + # metrics-apm.internal data stream, which newer versions break + # into separate datasets. We reroute these metrics coming from + # older versions of apm-server based on 'metricset.name'. +- set: + if: | + (ctx.data_stream?.dataset == 'apm.internal' || ctx['data_stream.dataset'] == 'apm.internal') && + (ctx.metricset?.name == 'transaction' || ctx.metricset?.name == 'service_destination') + field: metricset.interval + value: 1m + override: false +- reroute: + if: | + (ctx.data_stream?.dataset == 'apm.internal' || ctx['data_stream.dataset'] == 'apm.internal') && + (ctx.metricset?.name == 'transaction') + dataset: apm.transaction.1m +- reroute: + if: | + (ctx.data_stream?.dataset == 'apm.internal' || ctx['data_stream.dataset'] == 'apm.internal') && + (ctx.metricset?.name == 'service_destination') + dataset: apm.service_destination.1m + +# Invoke user-defined custom pipelines, in ascending order +# of specificity. +- pipeline: + name: apm@custom + ignore_missing_pipeline: true +- pipeline: + name: "{{{data_stream.type}}}-apm@custom" + ignore_missing_pipeline: true +- pipeline: + name: "{{{data_stream.type}}}-{{{data_stream.dataset}}}@custom" + ignore_missing_pipeline: true diff --git a/x-pack/plugin/apm-data/src/main/resources/ingest-pipelines/apm@pipeline.yaml b/x-pack/plugin/apm-data/src/main/resources/ingest-pipelines/apm@pipeline.yaml new file mode 100644 index 0000000000000..40161e6ddcbf4 --- /dev/null +++ b/x-pack/plugin/apm-data/src/main/resources/ingest-pipelines/apm@pipeline.yaml @@ -0,0 +1,26 @@ +--- +version: ${xpack.apmdata.template.version} +_meta: + managed: true +description: | + Built-in ingest pipeline for all APM data streams. +processors: +- rename: + field: process.ppid + target_field: process.parent.pid + ignore_failure: true + ignore_missing: true +- geoip: + database_file: GeoLite2-City.mmdb + field: client.ip + target_field: client.geo + download_database_on_pipeline_creation: false + ignore_missing: true + on_failure: + - remove: + field: client.ip +- user_agent: + field: user_agent.original + target_field: user_agent + ignore_failure: true + ignore_missing: true diff --git a/x-pack/plugin/apm-data/src/main/resources/ingest-pipelines/metrics-apm@pipeline.yaml b/x-pack/plugin/apm-data/src/main/resources/ingest-pipelines/metrics-apm@pipeline.yaml new file mode 100644 index 0000000000000..0e7cf1ce5521a --- /dev/null +++ b/x-pack/plugin/apm-data/src/main/resources/ingest-pipelines/metrics-apm@pipeline.yaml @@ -0,0 +1,50 @@ +--- +version: ${xpack.apmdata.template.version} +_meta: + managed: true +description: Built-in ingest pipeline for metrics-apm.*-* data streams +processors: + # Older versions of apm-server write a '_metric_descriptions' field, + # which describes metrics written at the top level of the source. + # Newer versions write metrics as 'metricset.samples', which carry + # both the metric description and value(s). +- script: + if: ctx._metric_descriptions != null + source: | + Map dynamic_templates = new HashMap(); + for (entry in ctx._metric_descriptions.entrySet()) { + String name = entry.getKey(); + Map description = entry.getValue(); + String metric_type = description.type; + if (metric_type == "histogram") { + dynamic_templates[name] = "histogram_metrics"; + } else if (metric_type == "summary") { + dynamic_templates[name] = "summary_metrics"; + } else { + dynamic_templates[name] = "double_metrics"; + } + } + ctx._dynamic_templates = dynamic_templates; + ctx.remove("_metric_descriptions"); +- script: + if: ctx.metricset?.samples != null + source: | + Map dynamic_templates = new HashMap(); + for (sample in ctx.metricset.samples) { + String name = sample.name; + String metric_type = sample.type; + if (metric_type == "histogram") { + dynamic_templates[name] = "histogram_metrics"; + ctx.put(name, ["values": sample.values, "counts": sample.counts]); + } else if (metric_type == "summary") { + dynamic_templates[name] = "summary_metrics"; + ctx.put(name, ["value_count": sample.value_count, "sum": sample.sum]); + } else { + dynamic_templates[name] = "double_metrics"; + ctx.put(name, sample.value); + } + } + ctx._dynamic_templates = dynamic_templates; + ctx.metricset.remove("samples"); +- pipeline: + name: apm@pipeline diff --git a/x-pack/plugin/apm-data/src/main/resources/ingest-pipelines/traces-apm@pipeline.yaml b/x-pack/plugin/apm-data/src/main/resources/ingest-pipelines/traces-apm@pipeline.yaml new file mode 100644 index 0000000000000..f79c07ff40ac0 --- /dev/null +++ b/x-pack/plugin/apm-data/src/main/resources/ingest-pipelines/traces-apm@pipeline.yaml @@ -0,0 +1,42 @@ +--- +version: ${xpack.apmdata.template.version} +_meta: + managed: true +description: Built-in ingest pipeline for traces-apm-* data streams +processors: +- set: + if: ctx.transaction?.type != null + field: processor.event + value: transaction +- set: + field: processor.event + value: span + override: false +- script: + # Older versions of apm-server write 'transaction.duration.us' or + # 'span.duration.us'. Newer versions write 'event.duration', which + # should be rewritten. + if: 'ctx[ctx.processor.event]?.duration == null' + source: | + def eventDuration = ctx.event?.duration ?: 0; + def rootObjectName = ctx.processor?.event; + def rootObject = ctx[rootObjectName]; + if (rootObject == null) { + rootObject = [:]; + ctx[rootObjectName] = rootObject; + } + rootObject.duration = ["us": (long)(eventDuration/1000)]; +- remove: + field: ["event.duration"] + ignore_failure: true + ignore_missing: true +- set: + if: ctx.event?.outcome == 'success' + field: event.success_count + value: 1 +- set: + if: ctx.event?.outcome == 'failure' + field: event.success_count + value: 0 +- pipeline: + name: apm@pipeline diff --git a/x-pack/plugin/apm-data/src/main/resources/resources.yaml b/x-pack/plugin/apm-data/src/main/resources/resources.yaml new file mode 100644 index 0000000000000..71b54ae6297db --- /dev/null +++ b/x-pack/plugin/apm-data/src/main/resources/resources.yaml @@ -0,0 +1,59 @@ +# "version" holds the version of the templates and ingest pipelines installed +# by xpack-plugin apm-data. This must be increased whenever an existing template or +# pipeline is changed, in order for it to be updated on Elasticsearch upgrade. +version: 1 + +component-templates: + # Data lifecycle. + - apm-10d@lifecycle + - apm-90d@lifecycle + - apm-180d@lifecycle + - apm-390d@lifecycle + # Common mappings and settings. + # - *-apm* data streams compose apm@* + # - metrics-apm* data streams additionally compose metrics-apm@* + - apm@mappings + - apm@settings + - metrics-apm@mappings + - metrics-apm@settings + # Data stream-specific mappings. + - logs-apm.error@mappings + - metrics-apm.service_destination@mappings + - metrics-apm.service_summary@mappings + - metrics-apm.service_transaction@mappings + - metrics-apm.transaction@mappings + - traces@mappings + - traces-apm@mappings + - traces-apm.rum@mappings + +index-templates: + - logs-apm.app@template + - logs-apm.error@template + - metrics-apm.app@template + - metrics-apm.internal@template + - metrics-apm.service_destination.1m@template + - metrics-apm.service_destination.10m@template + - metrics-apm.service_destination.60m@template + - metrics-apm.service_summary.1m@template + - metrics-apm.service_summary.10m@template + - metrics-apm.service_summary.60m@template + - metrics-apm.service_transaction.1m@template + - metrics-apm.service_transaction.10m@template + - metrics-apm.service_transaction.60m@template + - metrics-apm.transaction.1m@template + - metrics-apm.transaction.10m@template + - metrics-apm.transaction.60m@template + - traces-apm@template + - traces-apm.rum@template + - traces-apm.sampled@template + +# Ingest pipeline configuration requires to manually specify pipeline dependencies +ingest-pipelines: + - apm@default-pipeline: {} + - apm@pipeline: {} + - traces-apm@pipeline: + dependencies: + - apm@pipeline + - metrics-apm@pipeline: + dependencies: + - apm@pipeline diff --git a/x-pack/plugin/apm-data/src/test/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistryTests.java b/x-pack/plugin/apm-data/src/test/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistryTests.java new file mode 100644 index 0000000000000..6b16de4ac6458 --- /dev/null +++ b/x-pack/plugin/apm-data/src/test/java/org/elasticsearch/xpack/apmdata/APMIndexTemplateRegistryTests.java @@ -0,0 +1,387 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.apmdata; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.admin.indices.template.put.PutComponentTemplateAction; +import org.elasticsearch.action.admin.indices.template.put.PutComposableIndexTemplateAction; +import org.elasticsearch.action.ingest.PutPipelineAction; +import org.elasticsearch.action.ingest.PutPipelineRequest; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.cluster.ClusterChangedEvent; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlocks; +import org.elasticsearch.cluster.metadata.ComponentTemplate; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; +import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodeUtils; +import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.ingest.IngestMetadata; +import org.elasticsearch.ingest.PipelineConfiguration; +import org.elasticsearch.test.ClusterServiceUtils; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.threadpool.TestThreadPool; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xcontent.NamedXContentRegistry; +import org.elasticsearch.xcontent.XContentType; +import org.elasticsearch.xpack.core.ilm.IndexLifecycleMetadata; +import org.elasticsearch.xpack.core.ilm.LifecyclePolicy; +import org.elasticsearch.xpack.core.ilm.LifecyclePolicyMetadata; +import org.elasticsearch.xpack.core.ilm.OperationMode; +import org.elasticsearch.xpack.core.template.IngestPipelineConfig; +import org.elasticsearch.xpack.stack.StackTemplateRegistry; +import org.elasticsearch.xpack.stack.StackTemplateRegistryAccessor; +import org.junit.After; +import org.junit.Before; + +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; + +import static org.elasticsearch.xpack.core.XPackSettings.APM_DATA_ENABLED; +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.not; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; + +public class APMIndexTemplateRegistryTests extends ESTestCase { + private APMIndexTemplateRegistry apmIndexTemplateRegistry; + private StackTemplateRegistryAccessor stackTemplateRegistryAccessor; + private ClusterService clusterService; + private ThreadPool threadPool; + private VerifyingClient client; + + @Before + public void createRegistryAndClient() { + threadPool = new TestThreadPool(this.getClass().getName()); + client = new VerifyingClient(threadPool); + clusterService = ClusterServiceUtils.createClusterService(threadPool); + stackTemplateRegistryAccessor = new StackTemplateRegistryAccessor( + new StackTemplateRegistry(Settings.EMPTY, clusterService, threadPool, client, NamedXContentRegistry.EMPTY) + ); + apmIndexTemplateRegistry = new APMIndexTemplateRegistry( + Settings.builder().put(APM_DATA_ENABLED.getKey(), true).build(), + clusterService, + threadPool, + client, + NamedXContentRegistry.EMPTY + ); + } + + @After + @Override + public void tearDown() throws Exception { + super.tearDown(); + threadPool.shutdownNow(); + } + + public void testThatMissingMasterNodeDoesNothing() { + DiscoveryNode localNode = DiscoveryNodeUtils.create("node"); + DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").add(localNode).build(); + + client.setVerifier((a, r, l) -> { + fail("if the master is missing nothing should happen"); + return null; + }); + + ClusterChangedEvent event = createClusterChangedEvent(Map.of(), Map.of(), nodes); + apmIndexTemplateRegistry.clusterChanged(event); + } + + public void testThatIndependentTemplatesAreAddedImmediatelyIfMissing() throws Exception { + DiscoveryNode node = DiscoveryNodeUtils.create("node"); + DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build(); + + AtomicInteger actualInstalledIndexTemplates = new AtomicInteger(0); + AtomicInteger actualInstalledComponentTemplates = new AtomicInteger(0); + AtomicInteger actualInstalledIngestPipelines = new AtomicInteger(0); + + client.setVerifier( + (action, request, listener) -> verifyActions( + actualInstalledIndexTemplates, + actualInstalledComponentTemplates, + actualInstalledIngestPipelines, + action, + request, + listener + ) + ); + apmIndexTemplateRegistry.clusterChanged(createClusterChangedEvent(Map.of(), Map.of(), nodes)); + + assertBusy(() -> assertThat(actualInstalledIngestPipelines.get(), equalTo(getIndependentPipelineConfigs().size()))); + assertBusy(() -> assertThat(actualInstalledComponentTemplates.get(), equalTo(getIndependentComponentTemplateConfigs().size()))); + + // index templates should not be installed as they are dependent in component templates and ingest pipelines + assertThat(actualInstalledIndexTemplates.get(), equalTo(0)); + } + + public void testIngestPipelines() { + DiscoveryNode node = DiscoveryNodeUtils.create("node"); + DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build(); + + final List pipelineConfigs = apmIndexTemplateRegistry.getIngestPipelines(); + assertThat(pipelineConfigs, is(not(empty()))); + + pipelineConfigs.forEach(ingestPipelineConfig -> { + AtomicInteger putPipelineRequestsLocal = new AtomicInteger(0); + client.setVerifier((a, r, l) -> { + if (r instanceof PutPipelineRequest && ingestPipelineConfig.getId().equals(((PutPipelineRequest) r).getId())) { + putPipelineRequestsLocal.incrementAndGet(); + } + return AcknowledgedResponse.TRUE; + }); + + apmIndexTemplateRegistry.clusterChanged( + createClusterChangedEvent(Map.of(), Map.of(), ingestPipelineConfig.getPipelineDependencies(), nodes) + ); + try { + assertBusy(() -> assertThat(putPipelineRequestsLocal.get(), greaterThanOrEqualTo(1))); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + } + + public void testComponentTemplates() throws Exception { + DiscoveryNode node = DiscoveryNodeUtils.create("node"); + DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build(); + + AtomicInteger actualInstalledIndexTemplates = new AtomicInteger(0); + AtomicInteger actualInstalledComponentTemplates = new AtomicInteger(0); + AtomicInteger actualInstalledIngestPipelines = new AtomicInteger(0); + + client.setVerifier( + (action, request, listener) -> verifyActions( + actualInstalledIndexTemplates, + actualInstalledComponentTemplates, + actualInstalledIngestPipelines, + action, + request, + listener + ) + ); + apmIndexTemplateRegistry.clusterChanged( + createClusterChangedEvent( + Map.of(), + Map.of(), + apmIndexTemplateRegistry.getIngestPipelines().stream().map(IngestPipelineConfig::getId).collect(Collectors.toList()), + nodes + ) + ); + + assertBusy( + () -> assertThat( + actualInstalledComponentTemplates.get(), + equalTo(apmIndexTemplateRegistry.getComponentTemplateConfigs().size()) + ) + ); + + // ingest pipelines should not have been installed as we used a cluster state that includes them already + assertThat(actualInstalledIngestPipelines.get(), equalTo(0)); + // index templates should not be installed as they are dependent in component templates and ingest pipelines + assertThat(actualInstalledIndexTemplates.get(), equalTo(0)); + } + + public void testIndexTemplates() throws Exception { + DiscoveryNode node = DiscoveryNodeUtils.create("node"); + DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build(); + + AtomicInteger actualInstalledIndexTemplates = new AtomicInteger(0); + AtomicInteger actualInstalledComponentTemplates = new AtomicInteger(0); + AtomicInteger actualInstalledIngestPipelines = new AtomicInteger(0); + + client.setVerifier( + (action, request, listener) -> verifyActions( + actualInstalledIndexTemplates, + actualInstalledComponentTemplates, + actualInstalledIngestPipelines, + action, + request, + listener + ) + ); + + // we need to add all component templates of the APM registry as well as the stack registry to ensure that all dependencies of + // the APM-registry index templates exist + Map componentTemplates = new HashMap<>(); + apmIndexTemplateRegistry.getComponentTemplateConfigs() + .forEach((key, value) -> componentTemplates.put(key, apmIndexTemplateRegistry.getVersion())); + stackTemplateRegistryAccessor.getComponentTemplateConfigs() + .forEach((key, value) -> componentTemplates.put(key, apmIndexTemplateRegistry.getVersion())); + apmIndexTemplateRegistry.clusterChanged( + createClusterChangedEvent( + componentTemplates, + Map.of(), + apmIndexTemplateRegistry.getIngestPipelines().stream().map(IngestPipelineConfig::getId).collect(Collectors.toList()), + nodes + ) + ); + + assertBusy( + () -> assertThat(actualInstalledIndexTemplates.get(), equalTo(apmIndexTemplateRegistry.getComposableTemplateConfigs().size())) + ); + + // ingest pipelines and component templates should not have been installed as we used a cluster state that includes them already + assertThat(actualInstalledComponentTemplates.get(), equalTo(0)); + assertThat(actualInstalledIngestPipelines.get(), equalTo(0)); + } + + private Map getIndependentComponentTemplateConfigs() { + return apmIndexTemplateRegistry.getComponentTemplateConfigs().entrySet().stream().filter(template -> { + Settings settings = template.getValue().template().settings(); + return settings == null || (settings.get("index.default_pipeline") == null && settings.get("index.final_pipeline") == null); + }).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + } + + private List getIndependentPipelineConfigs() { + return apmIndexTemplateRegistry.getIngestPipelines() + .stream() + .filter(pipelineConfig -> pipelineConfig.getPipelineDependencies().isEmpty()) + .collect(Collectors.toList()); + } + + private ActionResponse verifyActions( + AtomicInteger indexTemplatesCounter, + AtomicInteger componentTemplatesCounter, + AtomicInteger ingestPipelinesCounter, + ActionType action, + ActionRequest request, + ActionListener listener + ) { + if (action instanceof PutComponentTemplateAction) { + componentTemplatesCounter.incrementAndGet(); + return AcknowledgedResponse.TRUE; + } else if (action instanceof PutComposableIndexTemplateAction) { + indexTemplatesCounter.incrementAndGet(); + assertThat(request, instanceOf(PutComposableIndexTemplateAction.Request.class)); + final PutComposableIndexTemplateAction.Request putRequest = ((PutComposableIndexTemplateAction.Request) request); + assertThat(putRequest.indexTemplate().version(), equalTo((long) apmIndexTemplateRegistry.getVersion())); + assertNotNull(listener); + return AcknowledgedResponse.TRUE; + } else if (action instanceof PutPipelineAction) { + ingestPipelinesCounter.incrementAndGet(); + return AcknowledgedResponse.TRUE; + } else { + fail("client called with unexpected request:" + request.toString()); + return null; + } + } + + private ClusterChangedEvent createClusterChangedEvent( + Map existingComponentTemplates, + Map existingComposableTemplates, + DiscoveryNodes nodes + ) { + return createClusterChangedEvent(existingComponentTemplates, existingComposableTemplates, Collections.emptyList(), nodes); + } + + private ClusterChangedEvent createClusterChangedEvent( + Map existingComponentTemplates, + Map existingComposableTemplates, + List ingestPipelines, + DiscoveryNodes nodes + ) { + return createClusterChangedEvent(existingComponentTemplates, existingComposableTemplates, ingestPipelines, Map.of(), nodes); + } + + private ClusterChangedEvent createClusterChangedEvent( + Map existingComponentTemplates, + Map existingComposableTemplates, + List ingestPipelines, + Map existingPolicies, + DiscoveryNodes nodes + ) { + ClusterState cs = createClusterState( + Settings.EMPTY, + existingComponentTemplates, + existingComposableTemplates, + ingestPipelines, + existingPolicies, + nodes + ); + ClusterChangedEvent realEvent = new ClusterChangedEvent( + "created-from-test", + cs, + ClusterState.builder(new ClusterName("test")).build() + ); + ClusterChangedEvent event = spy(realEvent); + when(event.localNodeMaster()).thenReturn(nodes.isLocalNodeElectedMaster()); + + return event; + } + + private ClusterState createClusterState( + Settings nodeSettings, + Map existingComponentTemplates, + Map existingComposableTemplates, + List ingestPipelines, + Map existingPolicies, + DiscoveryNodes nodes + ) { + Map componentTemplates = new HashMap<>(); + for (Map.Entry template : existingComponentTemplates.entrySet()) { + ComponentTemplate mockTemplate = mock(ComponentTemplate.class); + when(mockTemplate.version()).thenReturn(template.getValue() == null ? null : (long) template.getValue()); + componentTemplates.put(template.getKey(), mockTemplate); + } + Map composableTemplates = new HashMap<>(); + for (Map.Entry template : existingComposableTemplates.entrySet()) { + ComposableIndexTemplate mockTemplate = mock(ComposableIndexTemplate.class); + when(mockTemplate.version()).thenReturn(template.getValue() == null ? null : (long) template.getValue()); + composableTemplates.put(template.getKey(), mockTemplate); + } + + Map existingILMMeta = existingPolicies.entrySet() + .stream() + .collect(Collectors.toMap(Map.Entry::getKey, e -> new LifecyclePolicyMetadata(e.getValue(), Map.of(), 1, 1))); + IndexLifecycleMetadata ilmMeta = new IndexLifecycleMetadata(existingILMMeta, OperationMode.RUNNING); + + Map ingestPipelineConfigurations = new HashMap<>(); + if (ingestPipelines.isEmpty() == false) { + for (IngestPipelineConfig ingestPipelineConfig : apmIndexTemplateRegistry.getIngestPipelines()) { + if (ingestPipelines.contains(ingestPipelineConfig.getId())) { + // we cannot mock PipelineConfiguration as it is a final class + ingestPipelineConfigurations.put( + ingestPipelineConfig.getId(), + new PipelineConfiguration(ingestPipelineConfig.getId(), ingestPipelineConfig.loadConfig(), XContentType.YAML) + ); + } + } + } + IngestMetadata ingestMetadata = new IngestMetadata(ingestPipelineConfigurations); + + return ClusterState.builder(new ClusterName("test")) + .metadata( + Metadata.builder() + .componentTemplates(componentTemplates) + .indexTemplates(composableTemplates) + .transientSettings(nodeSettings) + .putCustom(IndexLifecycleMetadata.TYPE, ilmMeta) + .putCustom(IngestMetadata.TYPE, ingestMetadata) + .build() + ) + .blocks(new ClusterBlocks.Builder().build()) + .nodes(nodes) + .build(); + } +} diff --git a/x-pack/plugin/apm-data/src/test/java/org/elasticsearch/xpack/apmdata/VerifyingClient.java b/x-pack/plugin/apm-data/src/test/java/org/elasticsearch/xpack/apmdata/VerifyingClient.java new file mode 100644 index 0000000000000..450a275f6a1c1 --- /dev/null +++ b/x-pack/plugin/apm-data/src/test/java/org/elasticsearch/xpack/apmdata/VerifyingClient.java @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.apmdata; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.ActionType; +import org.elasticsearch.common.TriFunction; +import org.elasticsearch.test.client.NoOpClient; +import org.elasticsearch.threadpool.ThreadPool; +import org.junit.Assert; + +/** + * A client that delegates to a verifying function for action/request/listener + */ +public class VerifyingClient extends NoOpClient { + + private TriFunction, ActionRequest, ActionListener, ActionResponse> verifier = (a, r, l) -> { + Assert.fail("verifier not set"); + return null; + }; + + VerifyingClient(ThreadPool threadPool) { + super(threadPool); + } + + @Override + @SuppressWarnings("unchecked") + protected void doExecute( + ActionType action, + Request request, + ActionListener listener + ) { + try { + listener.onResponse((Response) verifier.apply(action, request, listener)); + } catch (Exception e) { + listener.onFailure(e); + } + } + + public VerifyingClient setVerifier(TriFunction, ActionRequest, ActionListener, ActionResponse> verifier) { + this.verifier = verifier; + return this; + } +} diff --git a/x-pack/plugin/apm-data/src/test/java/org/elasticsearch/xpack/stack/StackTemplateRegistryAccessor.java b/x-pack/plugin/apm-data/src/test/java/org/elasticsearch/xpack/stack/StackTemplateRegistryAccessor.java new file mode 100644 index 0000000000000..849518299fec2 --- /dev/null +++ b/x-pack/plugin/apm-data/src/test/java/org/elasticsearch/xpack/stack/StackTemplateRegistryAccessor.java @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.stack; + +import org.elasticsearch.cluster.metadata.ComponentTemplate; + +import java.util.Map; + +/** + * An accessor class for package-private functionalities of {@link StackTemplateRegistry} + */ +public class StackTemplateRegistryAccessor { + + final StackTemplateRegistry registry; + + public StackTemplateRegistryAccessor(StackTemplateRegistry registry) { + this.registry = registry; + } + + public Map getComponentTemplateConfigs() { + return registry.getComponentTemplateConfigs(); + } +} diff --git a/x-pack/plugin/apm-data/src/yamlRestTest/java/org/elasticsearch/xpack/apmdata/APMYamlTestSuiteIT.java b/x-pack/plugin/apm-data/src/yamlRestTest/java/org/elasticsearch/xpack/apmdata/APMYamlTestSuiteIT.java new file mode 100644 index 0000000000000..5835a41479a68 --- /dev/null +++ b/x-pack/plugin/apm-data/src/yamlRestTest/java/org/elasticsearch/xpack/apmdata/APMYamlTestSuiteIT.java @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.apmdata; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.elasticsearch.test.cluster.ElasticsearchCluster; +import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; +import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; +import org.junit.ClassRule; + +public class APMYamlTestSuiteIT extends ESClientYamlSuiteTestCase { + + @ClassRule + public static ElasticsearchCluster cluster = ElasticsearchCluster.local() + .module("constant-keyword") + .module("data-streams") + .module("ingest-common") + .module("ingest-geoip") + .module("ingest-user-agent") + .module("mapper-extras") + .module("wildcard") + .module("x-pack-analytics") + .module("x-pack-apm-data") + .module("x-pack-aggregate-metric") + .module("x-pack-ilm") + .module("x-pack-stack") + .setting("ingest.geoip.downloader.enabled", "false") + .setting("xpack.apm_data.enabled", "true") + .build(); + + public APMYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { + super(testCandidate); + } + + @ParametersFactory + public static Iterable parameters() throws Exception { + return ESClientYamlSuiteTestCase.createParameters(); + } + + @Override + protected String getTestRestCluster() { + return cluster.getHttpAddresses(); + } +} diff --git a/x-pack/plugin/apm-data/src/yamlRestTest/resources/rest-api-spec/test/10_apm.yml b/x-pack/plugin/apm-data/src/yamlRestTest/resources/rest-api-spec/test/10_apm.yml new file mode 100644 index 0000000000000..0c538c345ebaa --- /dev/null +++ b/x-pack/plugin/apm-data/src/yamlRestTest/resources/rest-api-spec/test/10_apm.yml @@ -0,0 +1,182 @@ +--- +setup: + - do: + cluster.health: + wait_for_events: languid + +--- +"Test traces-apm* template installation": + - skip: + reason: contains is a newly added assertion + features: contains + - do: + indices.get_index_template: + name: traces-apm* + - length: {index_templates: 3} + - contains: {index_templates: {name: traces-apm@template}} + - contains: {index_templates: {name: traces-apm.rum@template}} + - contains: {index_templates: {name: traces-apm.sampled@template}} + +--- +"Test metrics-apm* template installation": + - skip: + reason: contains is a newly added assertion + features: contains + - do: + indices.get_index_template: + name: metrics-apm* + - length: {index_templates: 14} + - contains: {index_templates: {name: metrics-apm.app@template}} + - contains: {index_templates: {name: metrics-apm.internal@template}} + - contains: {index_templates: {name: metrics-apm.service_destination.1m@template}} + - contains: {index_templates: {name: metrics-apm.service_destination.10m@template}} + - contains: {index_templates: {name: metrics-apm.service_destination.60m@template}} + - contains: {index_templates: {name: metrics-apm.service_summary.1m@template}} + - contains: {index_templates: {name: metrics-apm.service_summary.10m@template}} + - contains: {index_templates: {name: metrics-apm.service_summary.60m@template}} + - contains: {index_templates: {name: metrics-apm.service_transaction.1m@template}} + - contains: {index_templates: {name: metrics-apm.service_transaction.10m@template}} + - contains: {index_templates: {name: metrics-apm.service_transaction.60m@template}} + - contains: {index_templates: {name: metrics-apm.transaction.1m@template}} + - contains: {index_templates: {name: metrics-apm.transaction.10m@template}} + - contains: {index_templates: {name: metrics-apm.transaction.60m@template}} + +--- +"Test logs-apm* template installation": + - skip: + reason: contains is a newly added assertion + features: contains + - do: + indices.get_index_template: + name: logs-apm* + - length: {index_templates: 2} + - contains: {index_templates: {name: logs-apm.app@template}} + - contains: {index_templates: {name: logs-apm.error@template}} + +--- +"Test traces-apm-* data stream indexing": + - do: + index: + index: traces-apm-testing + refresh: true + body: + "@timestamp": "2017-06-22" + data_stream.type: traces + data_stream.dataset: apm + data_stream.namespace: testing + event: + outcome: success + duration: 123000 + - do: + indices.get_data_stream: + name: traces-apm-testing + - do: + search: + index: traces-apm-testing + body: + fields: ["*"] + - length: {hits.hits: 1} + - match: {hits.hits.0.fields.event\.success_count: [1]} + - match: {hits.hits.0.fields.span\.duration\.us: [123]} + +--- +"Test metrics-apm.internal-* data stream rerouting": + - do: + bulk: + index: metrics-apm.internal-testing + refresh: true + body: + - create: {} + - "@timestamp": "2017-06-22" + data_stream.type: metrics + data_stream.dataset: apm.internal + data_stream.namespace: testing + metricset: + name: transaction + - create: {} + - "@timestamp": "2017-06-22" + data_stream.type: metrics + data_stream.dataset: apm.internal + data_stream.namespace: testing + metricset: + name: service_destination + - create: {} + - "@timestamp": "2017-06-22" + data_stream.type: metrics + data_stream.dataset: apm.internal + data_stream.namespace: testing + metricset: + name: app_config # should not be rerouted + - do: + indices.get_data_stream: + name: metrics-apm.transaction.1m-testing + - do: + indices.get_data_stream: + name: metrics-apm.service_destination.1m-testing + - do: + indices.get_data_stream: + name: metrics-apm.internal-testing + - do: + search: + index: metrics-apm* + - length: {hits.hits: 3} + - match: {hits.hits.0._source.data_stream.dataset: "apm.internal"} + - match: {hits.hits.1._source.data_stream.dataset: "apm.service_destination.1m"} + - match: {hits.hits.1._source.metricset.interval: "1m"} + - match: {hits.hits.2._source.data_stream.dataset: "apm.transaction.1m"} + - match: {hits.hits.2._source.metricset.interval: "1m"} + +--- +"Test metrics-apm.app-* dynamic mapping": + - do: + bulk: + index: metrics-apm.app.svc1-testing + refresh: true + body: + - create: {} + - "@timestamp": "2017-06-22" + data_stream.type: metrics + data_stream.dataset: apm.app.svc1 + data_stream.namespace: testing + metricset: + name: app + samples: + - name: double_metric + type: gauge + value: 123 + - name: summary_metric + type: summary + value_count: 123 + sum: 456.789 + - name: histogram_metric + type: histogram + counts: [1, 2, 3] + values: [1.5, 2.5, 3.5] + - set: + items.0.create._index: index + - do: + indices.get_field_mapping: + index: metrics-apm.app.svc1-testing + fields: [double_metric, summary_metric, histogram_metric] + - match: + $body: + $index: + mappings: + double_metric: + full_name: double_metric + mapping: + double_metric: + type: float + index: false + summary_metric: + full_name: summary_metric + mapping: + summary_metric: + type: aggregate_metric_double + metrics : [sum, value_count] + default_metric: value_count + histogram_metric: + full_name: histogram_metric + mapping: + histogram_metric: + type: histogram diff --git a/x-pack/plugin/core/build.gradle b/x-pack/plugin/core/build.gradle index 6f9c6b93ac373..c7d44d4557801 100644 --- a/x-pack/plugin/core/build.gradle +++ b/x-pack/plugin/core/build.gradle @@ -44,6 +44,7 @@ dependencies { api "commons-logging:commons-logging:${versions.commonslogging}" api "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}" api "commons-codec:commons-codec:${versions.commonscodec}" + testImplementation project(path: ':modules:data-streams') // security deps api 'com.unboundid:unboundid-ldapsdk:6.0.3' diff --git a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/template/IndexTemplateRegistryRolloverIT.java b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/template/IndexTemplateRegistryRolloverIT.java new file mode 100644 index 0000000000000..12f9ce685a8b6 --- /dev/null +++ b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/template/IndexTemplateRegistryRolloverIT.java @@ -0,0 +1,82 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.template; + +import org.elasticsearch.action.datastreams.CreateDataStreamAction; +import org.elasticsearch.action.datastreams.GetDataStreamAction; +import org.elasticsearch.action.support.master.AcknowledgedResponse; +import org.elasticsearch.client.internal.Client; +import org.elasticsearch.cluster.ClusterChangedEvent; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.DataStream; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.datastreams.DataStreamsPlugin; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.ESIntegTestCase; +import org.junit.Before; + +import java.util.Collection; +import java.util.List; + +import static org.elasticsearch.xpack.core.template.RolloverEnabledTestTemplateRegistry.TEST_INDEX_PATTERN; +import static org.elasticsearch.xpack.core.template.RolloverEnabledTestTemplateRegistry.TEST_INDEX_TEMPLATE_ID; +import static org.hamcrest.Matchers.endsWith; +import static org.hamcrest.Matchers.hasSize; + +@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, numClientNodes = 0) +public class IndexTemplateRegistryRolloverIT extends ESIntegTestCase { + + private ClusterService clusterService; + private Client client; + private RolloverEnabledTestTemplateRegistry registry; + + @Override + protected Collection> nodePlugins() { + return List.of(DataStreamsPlugin.class); + } + + @Before + public void setup() { + clusterService = internalCluster().clusterService(internalCluster().getMasterName()); + client = client(); + registry = new RolloverEnabledTestTemplateRegistry( + clusterService.getSettings(), + clusterService, + clusterService.threadPool(), + client, + xContentRegistry(), + 3L + ); + registry.initialize(); + ensureGreen(); + } + + public void testRollover() throws Exception { + ClusterState state = clusterService.state(); + registry.clusterChanged(new ClusterChangedEvent(IndexTemplateRegistryRolloverIT.class.getName(), state, state)); + assertBusy(() -> { assertTrue(clusterService.state().metadata().templatesV2().containsKey(TEST_INDEX_TEMPLATE_ID)); }); + String dsName = TEST_INDEX_PATTERN.replace('*', '1'); + CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(dsName); + AcknowledgedResponse acknowledgedResponse = client.execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).get(); + assertTrue(acknowledgedResponse.isAcknowledged()); + assertNumberOfBackingIndices(1); + registry.incrementVersion(); + registry.clusterChanged(new ClusterChangedEvent(IndexTemplateRegistryRolloverIT.class.getName(), clusterService.state(), state)); + assertBusy(() -> assertNumberOfBackingIndices(2)); + } + + private void assertNumberOfBackingIndices(final int expected) { + GetDataStreamAction.Request getDataStreamRequest = new GetDataStreamAction.Request(new String[] { TEST_INDEX_PATTERN }); + GetDataStreamAction.Response getDataStreamResponse = client.execute(GetDataStreamAction.INSTANCE, getDataStreamRequest).actionGet(); + List dataStreams = getDataStreamResponse.getDataStreams(); + assertThat(dataStreams, hasSize(1)); + DataStream dataStream = dataStreams.get(0).getDataStream(); + assertThat(dataStream.getIndices(), hasSize(expected)); + assertThat(dataStream.getWriteIndex().getName(), endsWith(String.valueOf(expected))); + } +} diff --git a/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/template/RolloverEnabledTestTemplateRegistry.java b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/template/RolloverEnabledTestTemplateRegistry.java new file mode 100644 index 0000000000000..3f2856472a182 --- /dev/null +++ b/x-pack/plugin/core/src/internalClusterTest/java/org/elasticsearch/xpack/core/template/RolloverEnabledTestTemplateRegistry.java @@ -0,0 +1,67 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.template; + +import org.elasticsearch.client.internal.Client; +import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.xcontent.NamedXContentRegistry; + +import java.util.List; +import java.util.Map; + +public class RolloverEnabledTestTemplateRegistry extends IndexTemplateRegistry { + public static final String TEST_INDEX_TEMPLATE_ID = "test-index-template"; + public static final String TEST_INDEX_PATTERN = "test-rollover-*"; + private volatile long version; + + public RolloverEnabledTestTemplateRegistry( + Settings nodeSettings, + ClusterService clusterService, + ThreadPool threadPool, + Client client, + NamedXContentRegistry xContentRegistry, + long version + ) { + super(nodeSettings, clusterService, threadPool, client, xContentRegistry); + this.version = version; + } + + @Override + protected String getOrigin() { + return "test-with-rollover-enabled"; + } + + @Override + protected Map getComposableTemplateConfigs() { + return Map.of( + TEST_INDEX_TEMPLATE_ID, + new ComposableIndexTemplate( + List.of(TEST_INDEX_PATTERN), + null, + null, + 100L, + version, + null, + new ComposableIndexTemplate.DataStreamTemplate() + ) + ); + } + + @Override + protected boolean applyRolloverAfterTemplateV2Upgrade() { + return true; + } + + @SuppressWarnings("NonAtomicOperationOnVolatileField") + public void incrementVersion() { + version++; + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ClientHelper.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ClientHelper.java index fc81fc2f1eb75..4a2f6fbd8289f 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ClientHelper.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ClientHelper.java @@ -193,6 +193,7 @@ private static String maybeRewriteSingleAuthenticationHeaderForVersion( public static final String FLEET_ORIGIN = "fleet"; public static final String ENT_SEARCH_ORIGIN = "enterprise_search"; public static final String INFERENCE_ORIGIN = "inference"; + public static final String APM_ORIGIN = "apm"; private ClientHelper() {} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java index b37f6acfe7011..9a53653c454e3 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java @@ -92,6 +92,13 @@ public Iterator> settings() { Setting.Property.NodeScope ); + /** Setting for enabling or disabling APM Data. Defaults to false. */ + public static final Setting APM_DATA_ENABLED = Setting.boolSetting( + "xpack.apm_data.enabled", + false, + Setting.Property.NodeScope + ); + /** Setting for enabling or disabling enterprise search. Defaults to true. */ public static final Setting ENTERPRISE_SEARCH_ENABLED = Setting.boolSetting( "xpack.ent_search.enabled", @@ -314,6 +321,7 @@ public static List> getAllSettings() { settings.add(GRAPH_ENABLED); settings.add(MACHINE_LEARNING_ENABLED); settings.add(PROFILING_ENABLED); + settings.add(APM_DATA_ENABLED); settings.add(ENTERPRISE_SEARCH_ENABLED); settings.add(AUDIT_ENABLED); settings.add(WATCHER_ENABLED); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/IndexTemplateRegistry.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/IndexTemplateRegistry.java index 0ca489b32eb6d..18a2be0e9b358 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/IndexTemplateRegistry.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/IndexTemplateRegistry.java @@ -10,11 +10,15 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.admin.indices.rollover.RolloverAction; +import org.elasticsearch.action.admin.indices.rollover.RolloverRequest; +import org.elasticsearch.action.admin.indices.rollover.RolloverResponse; import org.elasticsearch.action.admin.indices.template.put.PutComponentTemplateAction; import org.elasticsearch.action.admin.indices.template.put.PutComposableIndexTemplateAction; import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest; import org.elasticsearch.action.ingest.PutPipelineAction; import org.elasticsearch.action.ingest.PutPipelineRequest; +import org.elasticsearch.action.support.GroupedActionListener; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.ClusterChangedEvent; @@ -22,10 +26,14 @@ import org.elasticsearch.cluster.ClusterStateListener; import org.elasticsearch.cluster.metadata.ComponentTemplate; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; +import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.IndexTemplateMetadata; +import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.cluster.metadata.MetadataIndexTemplateService; import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.regex.Regex; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.Nullable; import org.elasticsearch.core.TimeValue; @@ -43,8 +51,10 @@ import java.io.IOException; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.Optional; @@ -385,17 +395,20 @@ private void addComposableTemplatesIfMissing(ClusterState state) { final AtomicBoolean creationCheck = templateCreationsInProgress.computeIfAbsent(templateName, key -> new AtomicBoolean(false)); if (creationCheck.compareAndSet(false, true)) { ComposableIndexTemplate currentTemplate = state.metadata().templatesV2().get(templateName); - boolean componentTemplatesAvailable = componentTemplatesExist(state, newTemplate.getValue()); + boolean componentTemplatesAvailable = componentTemplatesInstalled(state, newTemplate.getValue()); if (componentTemplatesAvailable == false) { creationCheck.set(false); - logger.trace( - "not adding composable template [{}] for [{}] because its required component templates do not exist", - templateName, - getOrigin() - ); + if (logger.isTraceEnabled()) { + logger.trace( + "not adding composable template [{}] for [{}] because its required component templates do not exist or do not " + + "have the right version", + templateName, + getOrigin() + ); + } } else if (Objects.isNull(currentTemplate)) { logger.debug("adding composable template [{}] for [{}], because it doesn't exist", templateName, getOrigin()); - putComposableTemplate(templateName, newTemplate.getValue(), creationCheck); + putComposableTemplate(state, templateName, newTemplate.getValue(), creationCheck, false); } else if (Objects.isNull(currentTemplate.version()) || newTemplate.getValue().version() > currentTemplate.version()) { // IndexTemplateConfig now enforces templates contain a `version` property, so if the template doesn't have one we can // safely assume it's an old version of the template. @@ -406,7 +419,7 @@ private void addComposableTemplatesIfMissing(ClusterState state) { currentTemplate.version(), newTemplate.getValue().version() ); - putComposableTemplate(templateName, newTemplate.getValue(), creationCheck); + putComposableTemplate(state, templateName, newTemplate.getValue(), creationCheck, true); } else { creationCheck.set(false); logger.trace( @@ -427,10 +440,33 @@ private void addComposableTemplatesIfMissing(ClusterState state) { } /** - * Returns true if the cluster state contains all of the component templates needed by the composable template + * Returns true if the cluster state contains all of the component templates needed by the composable template. If this registry + * requires automatic rollover after index template upgrades (see {@link #applyRolloverAfterTemplateV2Upgrade()}), this method also + * verifies that the installed components templates are of the right version. */ - private static boolean componentTemplatesExist(ClusterState state, ComposableIndexTemplate indexTemplate) { - return state.metadata().componentTemplates().keySet().containsAll(indexTemplate.getRequiredComponentTemplates()); + private boolean componentTemplatesInstalled(ClusterState state, ComposableIndexTemplate indexTemplate) { + if (applyRolloverAfterTemplateV2Upgrade() == false) { + // component templates and index templates can be updated independently, we only need to know that the required component + // templates are available + return state.metadata().componentTemplates().keySet().containsAll(indexTemplate.getRequiredComponentTemplates()); + } + Map componentTemplateConfigs = getComponentTemplateConfigs(); + Map installedTemplates = state.metadata().componentTemplates(); + for (String templateName : indexTemplate.getRequiredComponentTemplates()) { + ComponentTemplate installedTemplate = installedTemplates.get(templateName); + // if a required component templates is not installed - the current cluster state cannot allow this index template yet + if (installedTemplate == null) { + return false; + } + ComponentTemplate templateConfig = componentTemplateConfigs.get(templateName); + // note: currently we only take care of component templates that are installed by the same registry as the index template. We + // don't enforce proper version for component templates that come from outside of this registry. + // See https://github.com/elastic/elasticsearch/issues/99647 + if (templateConfig != null && templateConfig.version().equals(installedTemplate.version()) == false) { + return false; + } + } + return true; } private void putLegacyTemplate(final IndexTemplateConfig config, final AtomicBoolean creationCheck) { @@ -502,9 +538,11 @@ public void onFailure(Exception e) { } private void putComposableTemplate( + ClusterState state, final String templateName, final ComposableIndexTemplate indexTemplate, - final AtomicBoolean creationCheck + final AtomicBoolean creationCheck, + final boolean isUpgrade ) { final Executor executor = threadPool.generic(); executor.execute(() -> { @@ -519,8 +557,14 @@ private void putComposableTemplate( new ActionListener() { @Override public void onResponse(AcknowledgedResponse response) { - creationCheck.set(false); - if (response.isAcknowledged() == false) { + if (response.isAcknowledged()) { + if (isUpgrade && applyRolloverAfterTemplateV2Upgrade()) { + invokeRollover(state, templateName, indexTemplate, creationCheck); + } else { + creationCheck.set(false); + } + } else { + creationCheck.set(false); logger.error( "error adding composable template [{}] for [{}], request was not acknowledged", templateName, @@ -691,7 +735,11 @@ private static PipelineConfiguration findInstalledPipeline(ClusterState state, S private void putIngestPipeline(final IngestPipelineConfig pipelineConfig, final AtomicBoolean creationCheck) { final Executor executor = threadPool.generic(); executor.execute(() -> { - PutPipelineRequest request = new PutPipelineRequest(pipelineConfig.getId(), pipelineConfig.loadConfig(), XContentType.JSON); + PutPipelineRequest request = new PutPipelineRequest( + pipelineConfig.getId(), + pipelineConfig.loadConfig(), + pipelineConfig.getXContentType() + ); request.masterNodeTimeout(TimeValue.timeValueMinutes(1)); executeAsyncWithOrigin( @@ -724,6 +772,17 @@ public void onFailure(Exception e) { }); } + /** + * Allows registries to opt-in for automatic rollover of "relevant" data streams immediately after a composable index template gets + * upgraded. If set to {@code true}, then every time a composable index template is being upgraded, all data streams of which name + * matches this template's index patterns AND of all matching templates the upgraded one has the highest priority, will be rolled over. + * + * @return {@code true} if this registry wants to apply automatic rollovers after template V2 upgrades + */ + protected boolean applyRolloverAfterTemplateV2Upgrade() { + return false; + } + /** * Called when creation of an ingest pipeline fails. * @@ -733,4 +792,91 @@ public void onFailure(Exception e) { protected void onPutPipelineFailure(String pipelineId, Exception e) { logger.error(() -> format("error adding ingest pipeline template [%s] for [%s]", pipelineId, getOrigin()), e); } + + private void invokeRollover( + final ClusterState state, + final String templateName, + final ComposableIndexTemplate indexTemplate, + final AtomicBoolean creationCheck + ) { + final Executor executor = threadPool.generic(); + executor.execute(() -> { + List rolloverTargets = findRolloverTargetDataStreams(state, templateName, indexTemplate); + if (rolloverTargets.isEmpty() == false) { + GroupedActionListener groupedActionListener = new GroupedActionListener<>( + rolloverTargets.size(), + new ActionListener<>() { + @Override + public void onResponse(Collection rolloverResponses) { + creationCheck.set(false); + onRolloversBulkResponse(rolloverResponses); + } + + @Override + public void onFailure(Exception e) { + creationCheck.set(false); + onRolloverFailure(e); + } + } + ); + for (String rolloverTarget : rolloverTargets) { + logger.info( + "rolling over data stream [{}] as a followup to the upgrade of the [{}] index template [{}]", + rolloverTarget, + getOrigin(), + templateName + ); + RolloverRequest request = new RolloverRequest(rolloverTarget, null); + request.masterNodeTimeout(TimeValue.timeValueMinutes(1)); + executeAsyncWithOrigin( + client.threadPool().getThreadContext(), + getOrigin(), + request, + groupedActionListener, + (req, listener) -> client.execute(RolloverAction.INSTANCE, req, listener) + ); + } + } + }); + } + + void onRolloversBulkResponse(Collection rolloverResponses) { + for (RolloverResponse rolloverResponse : rolloverResponses) { + if (rolloverResponse.isRolledOver() == false) { + logger.warn("rollover of the [{}] index [{}] failed", getOrigin(), rolloverResponse.getOldIndex()); + } + } + } + + void onRolloverFailure(Exception e) { + logger.error(String.format(Locale.ROOT, "[%s] related rollover failed", getOrigin()), e); + for (Throwable throwable : e.getSuppressed()) { + logger.error(String.format(Locale.ROOT, "[%s] related rollover failed", getOrigin()), throwable); + } + } + + /** + * Finds rollover target data streams based on the provided index template. A matching rollover target data stream is such that: + *
    + *
  1. matches the provided index template's index patterns + *
  2. of all index templates that have index patterns matching its name, the one with the highest priority is the one provided + * as argument + *
+ * + * @param state the cluster state + * @param templateName the ID by which the provided index template is being registered + * @param indexTemplate the index template for which a data stream is looked up as rollover target + * @return the list of rollover targets matching the provided index template + */ + static List findRolloverTargetDataStreams(ClusterState state, String templateName, ComposableIndexTemplate indexTemplate) { + final Metadata metadata = state.metadata(); + return metadata.dataStreams() + .values() + .stream() + // Limit to checking data streams that match any of the index template's index patterns + .filter(ds -> indexTemplate.indexPatterns().stream().anyMatch(pattern -> Regex.simpleMatch(pattern, ds.getName()))) + .filter(ds -> templateName.equals(MetadataIndexTemplateService.findV2Template(metadata, ds.getName(), ds.isHidden()))) + .map(DataStream::getName) + .collect(Collectors.toList()); + } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/IngestPipelineConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/IngestPipelineConfig.java index d0b748108c948..a216030f1c2e0 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/IngestPipelineConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/IngestPipelineConfig.java @@ -7,8 +7,8 @@ package org.elasticsearch.xpack.core.template; -import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.xcontent.XContentType; import java.util.Collections; import java.util.List; @@ -17,14 +17,14 @@ /** * Describes an ingest pipeline to be loaded from a resource file for use with an {@link IndexTemplateRegistry}. */ -public class IngestPipelineConfig { - private final String id; - private final String resource; - private final int version; - private final String versionProperty; +public abstract class IngestPipelineConfig { + protected final String id; + protected final String resource; + protected final int version; + protected final String versionProperty; /** - * A list of this pipeline's dependencies, for example- such referred to through a pipeline processor. + * A list of this pipeline's dependencies, for example - such referred to through a pipeline processor. * This list is used to enforce proper ordering of pipeline installation, so that a pipeline gets installed only if all its * dependencies are already installed. */ @@ -50,15 +50,11 @@ public int getVersion() { return version; } - public String getVersionProperty() { - return versionProperty; - } - public List getPipelineDependencies() { return dependencies; } - public BytesReference loadConfig() { - return new BytesArray(TemplateUtils.loadTemplate(resource, String.valueOf(version), versionProperty)); - } + public abstract XContentType getXContentType(); + + public abstract BytesReference loadConfig(); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/JsonIngestPipelineConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/JsonIngestPipelineConfig.java new file mode 100644 index 0000000000000..fc2ca7cbce186 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/template/JsonIngestPipelineConfig.java @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.core.template; + +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.xcontent.XContentType; + +import java.util.List; + +public class JsonIngestPipelineConfig extends IngestPipelineConfig { + public JsonIngestPipelineConfig(String id, String resource, int version, String versionProperty) { + super(id, resource, version, versionProperty); + } + + public JsonIngestPipelineConfig(String id, String resource, int version, String versionProperty, List dependencies) { + super(id, resource, version, versionProperty, dependencies); + } + + @Override + public XContentType getXContentType() { + return XContentType.JSON; + } + + @Override + public BytesReference loadConfig() { + return new BytesArray(TemplateUtils.loadTemplate(resource, String.valueOf(version), versionProperty)); + } +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicensesManagerServiceTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicensesManagerServiceTests.java index 261b9d10a47dd..7207a823c543b 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicensesManagerServiceTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/license/LicensesManagerServiceTests.java @@ -42,6 +42,7 @@ protected Settings nodeSettings() { .put(XPackSettings.GRAPH_ENABLED.getKey(), false) .put(XPackSettings.MACHINE_LEARNING_ENABLED.getKey(), false) .put(XPackSettings.PROFILING_ENABLED.getKey(), false) + .put(XPackSettings.APM_DATA_ENABLED.getKey(), false) .build(); } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/template/IndexTemplateRegistryTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/template/IndexTemplateRegistryTests.java index ec634418e46e2..ff0e99e4a1e17 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/template/IndexTemplateRegistryTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/template/IndexTemplateRegistryTests.java @@ -11,6 +11,9 @@ import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.admin.indices.rollover.RolloverAction; +import org.elasticsearch.action.admin.indices.rollover.RolloverRequest; +import org.elasticsearch.action.admin.indices.rollover.RolloverResponse; import org.elasticsearch.action.admin.indices.template.put.PutComponentTemplateAction; import org.elasticsearch.action.admin.indices.template.put.PutComposableIndexTemplateAction; import org.elasticsearch.action.ingest.PutPipelineAction; @@ -22,6 +25,7 @@ import org.elasticsearch.cluster.block.ClusterBlocks; import org.elasticsearch.cluster.metadata.ComponentTemplate; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; +import org.elasticsearch.cluster.metadata.DataStreamTestHelper; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.node.DiscoveryNode; import org.elasticsearch.cluster.node.DiscoveryNodeUtils; @@ -31,6 +35,7 @@ import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.Strings; +import org.elasticsearch.index.Index; import org.elasticsearch.ingest.IngestMetadata; import org.elasticsearch.ingest.PipelineConfiguration; import org.elasticsearch.test.ClusterServiceUtils; @@ -54,17 +59,24 @@ import org.junit.Before; import java.io.IOException; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.oneOf; +import static org.hamcrest.Matchers.startsWith; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; @@ -230,6 +242,49 @@ public void testThatComposableTemplateIsAddedIfDependenciesExist() throws Except assertBusy(() -> assertThat(calledTimes.get(), equalTo(1))); } + public void testThatComposableTemplateIsAddedIfDependenciesHaveRightVersion() throws Exception { + DiscoveryNode node = DiscoveryNodeUtils.create("node"); + DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build(); + + AtomicInteger calledTimes = new AtomicInteger(0); + client.setVerifier((action, request, listener) -> { + if (action instanceof PutComposableIndexTemplateAction) { + assertPutComposableIndexTemplateAction(calledTimes, action, request, listener); + return AcknowledgedResponse.TRUE; + } else if (action instanceof PutComponentTemplateAction) { + // ignore the component template upgrade + return AcknowledgedResponse.TRUE; + } else if (action instanceof PutLifecycleAction) { + // ignore lifecycle policies in this case + return AcknowledgedResponse.TRUE; + } else if (action instanceof PutPipelineAction) { + // ignore pipelines in this case + return AcknowledgedResponse.TRUE; + } else { + // other components should be added as they already exist with the right version already + fail("client called with unexpected request: " + request.toString()); + return null; + } + }); + + // unless the registry requires rollovers after index template updates, the dependencies only need to be available, without regard + // to their version + ClusterChangedEvent event = createClusterChangedEvent(Collections.singletonMap("custom-plugin-settings", 2), nodes); + registry.clusterChanged(event); + assertBusy(() -> assertThat(calledTimes.get(), equalTo(1))); + + // when a registry requires rollovers after index template updates, the upgrade should occur only if the dependencies are have + // the required version + registry.setApplyRollover(true); + calledTimes.set(0); + registry.clusterChanged(event); + Thread.sleep(100L); + assertThat(calledTimes.get(), equalTo(0)); + event = createClusterChangedEvent(Collections.singletonMap("custom-plugin-settings", 3), nodes); + registry.clusterChanged(event); + assertBusy(() -> assertThat(calledTimes.get(), equalTo(1))); + } + public void testThatTemplatesAreUpgradedWhenNeeded() throws Exception { DiscoveryNode node = DiscoveryNodeUtils.create("node"); DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build(); @@ -271,6 +326,155 @@ public void testThatTemplatesAreUpgradedWhenNeeded() throws Exception { assertBusy(() -> assertThat(calledTimes.get(), equalTo(4))); } + public void testAutomaticRollover() throws Exception { + DiscoveryNode node = DiscoveryNodeUtils.create("node"); + DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build(); + + ClusterState state = createClusterState( + Map.of("custom-plugin-settings", 3, "custom-plugin-template", 3), + Collections.emptyMap(), + Map.of("custom-plugin-default_pipeline", 3, "custom-plugin-final_pipeline", 3), + nodes + ); + Map composableTemplateConfigs = registry.getComposableTemplateConfigs(); + for (Map.Entry entry : composableTemplateConfigs.entrySet()) { + ComposableIndexTemplate template = entry.getValue(); + state = ClusterState.builder(state) + .metadata( + Metadata.builder(Objects.requireNonNull(state).metadata()) + .put( + entry.getKey(), + new ComposableIndexTemplate( + template.indexPatterns(), + template.template(), + template.composedOf(), + template.priority(), + 2L, + template.metadata(), + template.getDataStreamTemplate() + ) + ) + ) + .build(); + } + state = ClusterState.builder(state) + .metadata( + Metadata.builder(Objects.requireNonNull(state).metadata()) + .put(DataStreamTestHelper.newInstance("logs-my_app-1", Collections.singletonList(new Index(".ds-ds1-000001", "ds1i")))) + .put(DataStreamTestHelper.newInstance("logs-my_app-2", Collections.singletonList(new Index(".ds-ds2-000001", "ds2i")))) + .put( + DataStreamTestHelper.newInstance("traces-my_app-1", Collections.singletonList(new Index(".ds-ds3-000001", "ds3i"))) + ) + ) + .build(); + ClusterChangedEvent event = createClusterChangedEvent(nodes, state); + + AtomicInteger rolloverCounter = new AtomicInteger(0); + AtomicInteger putIndexTemplateCounter = new AtomicInteger(0); + client.setVerifier((action, request, listener) -> { + if (action instanceof RolloverAction) { + rolloverCounter.incrementAndGet(); + RolloverRequest rolloverRequest = ((RolloverRequest) request); + assertThat(rolloverRequest.getRolloverTarget(), startsWith("logs-my_app-")); + } else if (action instanceof PutComposableIndexTemplateAction) { + putIndexTemplateCounter.incrementAndGet(); + } + return AcknowledgedResponse.TRUE; + }); + + registry.clusterChanged(event); + assertBusy(() -> assertThat(putIndexTemplateCounter.get(), equalTo(1))); + // no rollover on upgrade because the test registry doesn't support automatic rollover by default + Thread.sleep(100L); + assertThat(rolloverCounter.get(), equalTo(0)); + + // test successful rollovers + registry.setApplyRollover(true); + putIndexTemplateCounter.set(0); + registry.clusterChanged(event); + assertBusy(() -> assertThat(putIndexTemplateCounter.get(), equalTo(1))); + assertBusy(() -> assertThat(rolloverCounter.get(), equalTo(2))); + AtomicReference> rolloverResponsesRef = registry.getRolloverResponses(); + assertBusy(() -> assertNotNull(rolloverResponsesRef.get())); + Collection rolloverResponses = rolloverResponsesRef.get(); + assertThat(rolloverResponses, hasSize(2)); + + // test again, to verify that the per-index-template creation lock gets released for reuse + putIndexTemplateCounter.set(0); + rolloverCounter.set(0); + registry.clusterChanged(event); + assertBusy(() -> assertThat(putIndexTemplateCounter.get(), equalTo(1))); + assertBusy(() -> assertThat(rolloverCounter.get(), equalTo(2))); + + // test rollover failures + putIndexTemplateCounter.set(0); + rolloverCounter.set(0); + client.setVerifier((action, request, listener) -> { + if (action instanceof RolloverAction) { + rolloverCounter.incrementAndGet(); + RolloverRequest rolloverRequest = ((RolloverRequest) request); + assertThat(rolloverRequest.getRolloverTarget(), startsWith("logs-my_app-")); + throw new RuntimeException("Failed to rollover " + rolloverRequest.getRolloverTarget()); + } else if (action instanceof PutComposableIndexTemplateAction) { + putIndexTemplateCounter.incrementAndGet(); + } + return AcknowledgedResponse.TRUE; + }); + registry.clusterChanged(event); + assertBusy(() -> assertThat(putIndexTemplateCounter.get(), equalTo(1))); + assertBusy(() -> assertThat(rolloverCounter.get(), equalTo(2))); + AtomicReference rolloverFailureRef = registry.getRolloverFailure(); + assertBusy(() -> assertNotNull(rolloverFailureRef.get())); + Exception rolloverFailure = rolloverFailureRef.get(); + assertThat(rolloverFailure.getMessage(), startsWith("Failed to rollover logs-my_app-")); + Throwable[] suppressed = rolloverFailure.getSuppressed(); + assertThat(suppressed.length, equalTo(1)); + assertThat(suppressed[0].getMessage(), startsWith("Failed to rollover logs-my_app-")); + } + + public void testNoRolloverForFreshInstalledIndexTemplate() throws Exception { + DiscoveryNode node = DiscoveryNodeUtils.create("node"); + DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build(); + + ClusterState state = createClusterState( + Map.of("custom-plugin-settings", 3, "custom-plugin-template", 3), + Collections.emptyMap(), + Map.of("custom-plugin-default_pipeline", 3, "custom-plugin-final_pipeline", 3), + nodes + ); + state = ClusterState.builder(state) + .metadata( + Metadata.builder(Objects.requireNonNull(state).metadata()) + .put(DataStreamTestHelper.newInstance("logs-my_app-1", Collections.singletonList(new Index(".ds-ds1-000001", "ds1i")))) + .put(DataStreamTestHelper.newInstance("logs-my_app-2", Collections.singletonList(new Index(".ds-ds2-000001", "ds2i")))) + .put( + DataStreamTestHelper.newInstance("traces-my_app-1", Collections.singletonList(new Index(".ds-ds3-000001", "ds3i"))) + ) + ) + .build(); + ClusterChangedEvent event = createClusterChangedEvent(nodes, state); + + AtomicInteger rolloverCounter = new AtomicInteger(0); + AtomicInteger putIndexTemplateCounter = new AtomicInteger(0); + client.setVerifier((action, request, listener) -> { + if (action instanceof RolloverAction) { + rolloverCounter.incrementAndGet(); + RolloverRequest rolloverRequest = ((RolloverRequest) request); + assertThat(rolloverRequest.getRolloverTarget(), startsWith("logs-my_app-")); + } else if (action instanceof PutComposableIndexTemplateAction) { + putIndexTemplateCounter.incrementAndGet(); + } + return AcknowledgedResponse.TRUE; + }); + + registry.setApplyRollover(true); + registry.clusterChanged(event); + assertBusy(() -> assertThat(putIndexTemplateCounter.get(), equalTo(1))); + // the index component is first installed, not upgraded, therefore rollover should not be triggered + Thread.sleep(100L); + assertThat(rolloverCounter.get(), equalTo(0)); + } + public void testThatTemplatesAreNotUpgradedWhenNotNeeded() throws Exception { DiscoveryNode node = DiscoveryNodeUtils.create("node"); DiscoveryNodes nodes = DiscoveryNodes.builder().localNodeId("node").masterNodeId("node").add(node).build(); @@ -551,10 +755,14 @@ private ClusterChangedEvent createClusterChangedEvent( Map existingIngestPipelines, DiscoveryNodes nodes ) { - ClusterState cs = createClusterState(Settings.EMPTY, existingTemplates, existingPolicies, existingIngestPipelines, nodes); + ClusterState clusterState = createClusterState(existingTemplates, existingPolicies, existingIngestPipelines, nodes); + return createClusterChangedEvent(nodes, clusterState); + } + + private ClusterChangedEvent createClusterChangedEvent(DiscoveryNodes nodes, ClusterState state) { ClusterChangedEvent realEvent = new ClusterChangedEvent( "created-from-test", - cs, + state, ClusterState.builder(new ClusterName("test")).build() ); ClusterChangedEvent event = spy(realEvent); @@ -564,7 +772,6 @@ private ClusterChangedEvent createClusterChangedEvent( } private ClusterState createClusterState( - Settings nodeSettings, Map existingComponentTemplates, Map existingPolicies, Map existingIngestPipelines, @@ -600,7 +807,7 @@ private ClusterState createClusterState( .metadata( Metadata.builder() .componentTemplates(componentTemplates) - .transientSettings(nodeSettings) + .transientSettings(Settings.EMPTY) .putCustom(IndexLifecycleMetadata.TYPE, ilmMeta) .putCustom(IngestMetadata.TYPE, ingestMetadata) .build() @@ -610,6 +817,62 @@ private ClusterState createClusterState( .build(); } + // ------------- functionality unit test -------- + + public void testFindRolloverTargetDataStreams() { + ClusterState state = ClusterState.EMPTY_STATE; + state = ClusterState.builder(state) + .metadata( + Metadata.builder(state.metadata()) + .put(DataStreamTestHelper.newInstance("ds1", Collections.singletonList(new Index(".ds-ds1-000001", "ds1i")))) + .put(DataStreamTestHelper.newInstance("ds2", Collections.singletonList(new Index(".ds-ds2-000001", "ds2i")))) + .put(DataStreamTestHelper.newInstance("ds3", Collections.singletonList(new Index(".ds-ds3-000001", "ds3i")))) + .put(DataStreamTestHelper.newInstance("ds4", Collections.singletonList(new Index(".ds-ds4-000001", "ds4i")))) + ) + .build(); + + ComposableIndexTemplate it1 = new ComposableIndexTemplate( + List.of("ds1*", "ds2*", "ds3*"), + null, + null, + 100L, + null, + null, + new ComposableIndexTemplate.DataStreamTemplate(), + null + ); + + ComposableIndexTemplate it2 = new ComposableIndexTemplate( + List.of("ds2*"), + null, + null, + 200L, + null, + null, + new ComposableIndexTemplate.DataStreamTemplate(), + null + ); + + ComposableIndexTemplate it5 = new ComposableIndexTemplate( + List.of("ds5*"), + null, + null, + 200L, + null, + null, + new ComposableIndexTemplate.DataStreamTemplate(), + null + ); + + state = ClusterState.builder(state) + .metadata(Metadata.builder(state.metadata()).put("it1", it1).put("it2", it2).put("it5", it5)) + .build(); + + assertThat(IndexTemplateRegistry.findRolloverTargetDataStreams(state, "it1", it1), containsInAnyOrder("ds1", "ds3")); + assertThat(IndexTemplateRegistry.findRolloverTargetDataStreams(state, "it2", it2), contains("ds2")); + assertThat(IndexTemplateRegistry.findRolloverTargetDataStreams(state, "it5", it5), empty()); + } + // ------------- /** diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/template/TestRegistryWithCustomPlugin.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/template/TestRegistryWithCustomPlugin.java index d63e3f33d28af..349fdfe1259c9 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/template/TestRegistryWithCustomPlugin.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/template/TestRegistryWithCustomPlugin.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.core.template; +import org.elasticsearch.action.admin.indices.rollover.RolloverResponse; import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.metadata.ComponentTemplate; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; @@ -19,16 +20,23 @@ import org.elasticsearch.xpack.core.ilm.LifecyclePolicy; import java.io.IOException; +import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; class TestRegistryWithCustomPlugin extends IndexTemplateRegistry { public static final int REGISTRY_VERSION = 3; public static final String TEMPLATE_VERSION_VARIABLE = "xpack.custom_plugin.template.version"; - private boolean policyUpgradeRequired = false; + private final AtomicBoolean policyUpgradeRequired = new AtomicBoolean(false); + private final AtomicBoolean applyRollover = new AtomicBoolean(false); + + private final AtomicReference> rolloverResponses = new AtomicReference<>(); + private final AtomicReference rolloverFailure = new AtomicReference<>(); TestRegistryWithCustomPlugin( Settings nodeSettings, @@ -75,14 +83,14 @@ protected Map getComposableTemplateConfigs() { @Override protected List getIngestPipelines() { return List.of( - new IngestPipelineConfig( + new JsonIngestPipelineConfig( "custom-plugin-default_pipeline", "/org/elasticsearch/xpack/core/template/custom-plugin-default_pipeline.json", REGISTRY_VERSION, TEMPLATE_VERSION_VARIABLE, Collections.singletonList("custom-plugin-final_pipeline") ), - new IngestPipelineConfig( + new JsonIngestPipelineConfig( "custom-plugin-final_pipeline", "/org/elasticsearch/xpack/core/template/custom-plugin-final_pipeline.json", REGISTRY_VERSION, @@ -102,11 +110,38 @@ protected List getLifecyclePolicies() { @Override protected boolean isUpgradeRequired(LifecyclePolicy currentPolicy, LifecyclePolicy newPolicy) { - return policyUpgradeRequired; + return policyUpgradeRequired.get(); } public void setPolicyUpgradeRequired(boolean policyUpgradeRequired) { - this.policyUpgradeRequired = policyUpgradeRequired; + this.policyUpgradeRequired.set(policyUpgradeRequired); + } + + @Override + protected boolean applyRolloverAfterTemplateV2Upgrade() { + return applyRollover.get(); + } + + public void setApplyRollover(boolean shouldApplyRollover) { + applyRollover.set(shouldApplyRollover); + } + + @Override + void onRolloversBulkResponse(Collection rolloverResponses) { + this.rolloverResponses.set(rolloverResponses); + } + + public AtomicReference> getRolloverResponses() { + return rolloverResponses; + } + + @Override + void onRolloverFailure(Exception e) { + rolloverFailure.set(e); + } + + public AtomicReference getRolloverFailure() { + return rolloverFailure; } @Override diff --git a/x-pack/plugin/core/template-resources/src/main/resources/metrics@mappings.json b/x-pack/plugin/core/template-resources/src/main/resources/metrics@mappings.json index 743c98d196a16..5741b441256f9 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/metrics@mappings.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/metrics@mappings.json @@ -19,6 +19,22 @@ "index": false } } + }, + { + "histogram_metrics": { + "mapping": { + "type": "histogram" + } + } + }, + { + "summary_metrics": { + "mapping": { + "type": "aggregate_metric_double", + "metrics": ["sum", "value_count"], + "default_metric": "value_count" + } + } } ], "properties": { diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/AnalyticsTemplateRegistry.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/AnalyticsTemplateRegistry.java index 0ecdca59c0f33..7472063e92e11 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/AnalyticsTemplateRegistry.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/analytics/AnalyticsTemplateRegistry.java @@ -20,6 +20,7 @@ import org.elasticsearch.xpack.core.template.IndexTemplateConfig; import org.elasticsearch.xpack.core.template.IndexTemplateRegistry; import org.elasticsearch.xpack.core.template.IngestPipelineConfig; +import org.elasticsearch.xpack.core.template.JsonIngestPipelineConfig; import java.io.IOException; import java.util.HashMap; @@ -79,7 +80,7 @@ public class AnalyticsTemplateRegistry extends IndexTemplateRegistry { @Override protected List getIngestPipelines() { return List.of( - new IngestPipelineConfig( + new JsonIngestPipelineConfig( EVENT_DATA_STREAM_INGEST_PIPELINE_NAME, ROOT_RESOURCE_PATH + EVENT_DATA_STREAM_INGEST_PIPELINE_NAME + ".json", REGISTRY_VERSION, diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorTemplateRegistry.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorTemplateRegistry.java index 514f4b1edd812..e497088f789f7 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorTemplateRegistry.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/connector/ConnectorTemplateRegistry.java @@ -21,6 +21,7 @@ import org.elasticsearch.xpack.core.template.IndexTemplateConfig; import org.elasticsearch.xpack.core.template.IndexTemplateRegistry; import org.elasticsearch.xpack.core.template.IngestPipelineConfig; +import org.elasticsearch.xpack.core.template.JsonIngestPipelineConfig; import java.io.IOException; import java.util.HashMap; @@ -111,7 +112,7 @@ public class ConnectorTemplateRegistry extends IndexTemplateRegistry { @Override protected List getIngestPipelines() { return List.of( - new IngestPipelineConfig( + new JsonIngestPipelineConfig( ENT_SEARCH_GENERIC_PIPELINE_NAME, ROOT_RESOURCE_PATH + ENT_SEARCH_GENERIC_PIPELINE_FILE + ".json", REGISTRY_VERSION, diff --git a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterIntegTestCase.java b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterIntegTestCase.java index 1fd62d0a3948f..a52cf9b0505dc 100644 --- a/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterIntegTestCase.java +++ b/x-pack/plugin/monitoring/src/test/java/org/elasticsearch/xpack/monitoring/exporter/local/LocalExporterIntegTestCase.java @@ -49,6 +49,7 @@ protected Settings localExporterSettings() { .put("xpack.monitoring.exporters." + exporterName + ".cluster_alerts.management.enabled", false) .put(XPackSettings.MACHINE_LEARNING_ENABLED.getKey(), false) .put(XPackSettings.PROFILING_ENABLED.getKey(), false) + .put(XPackSettings.APM_DATA_ENABLED.getKey(), false) .build(); } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationUtils.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationUtils.java index d93ee6ad36c67..475102e1a2152 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationUtils.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationUtils.java @@ -25,6 +25,7 @@ import static org.elasticsearch.ingest.IngestService.INGEST_ORIGIN; import static org.elasticsearch.persistent.PersistentTasksService.PERSISTENT_TASK_ORIGIN; import static org.elasticsearch.synonyms.SynonymsManagementAPIService.SYNONYMS_ORIGIN; +import static org.elasticsearch.xpack.core.ClientHelper.APM_ORIGIN; import static org.elasticsearch.xpack.core.ClientHelper.ASYNC_SEARCH_ORIGIN; import static org.elasticsearch.xpack.core.ClientHelper.DEPRECATION_ORIGIN; import static org.elasticsearch.xpack.core.ClientHelper.ENRICH_ORIGIN; @@ -144,6 +145,7 @@ public static void switchUserBasedOnActionOriginAndExecute( case IDP_ORIGIN: case INGEST_ORIGIN: case PROFILING_ORIGIN: + case APM_ORIGIN: case STACK_ORIGIN: case SEARCHABLE_SNAPSHOTS_ORIGIN: case LOGSTASH_MANAGEMENT_ORIGIN: diff --git a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/LegacyStackTemplateRegistry.java b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/LegacyStackTemplateRegistry.java index 26bcfb66bf818..9fb33db74964a 100644 --- a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/LegacyStackTemplateRegistry.java +++ b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/LegacyStackTemplateRegistry.java @@ -25,6 +25,7 @@ import org.elasticsearch.xpack.core.template.IndexTemplateConfig; import org.elasticsearch.xpack.core.template.IndexTemplateRegistry; import org.elasticsearch.xpack.core.template.IngestPipelineConfig; +import org.elasticsearch.xpack.core.template.JsonIngestPipelineConfig; import org.elasticsearch.xpack.core.template.LifecyclePolicyConfig; import java.io.IOException; @@ -231,8 +232,8 @@ protected Map getComposableTemplateConfigs() { } private static final List INGEST_PIPELINE_CONFIGS = List.of( - new IngestPipelineConfig("logs@json-message", "/logs@json-pipeline.json", REGISTRY_VERSION, TEMPLATE_VERSION_VARIABLE), - new IngestPipelineConfig("logs-default-pipeline", "/logs@default-pipeline.json", REGISTRY_VERSION, TEMPLATE_VERSION_VARIABLE) + new JsonIngestPipelineConfig("logs@json-message", "/logs@json-pipeline.json", REGISTRY_VERSION, TEMPLATE_VERSION_VARIABLE), + new JsonIngestPipelineConfig("logs-default-pipeline", "/logs@default-pipeline.json", REGISTRY_VERSION, TEMPLATE_VERSION_VARIABLE) ); @Override diff --git a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java index f81697982c803..3471d312d9df8 100644 --- a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java +++ b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java @@ -26,6 +26,7 @@ import org.elasticsearch.xpack.core.template.IndexTemplateConfig; import org.elasticsearch.xpack.core.template.IndexTemplateRegistry; import org.elasticsearch.xpack.core.template.IngestPipelineConfig; +import org.elasticsearch.xpack.core.template.JsonIngestPipelineConfig; import org.elasticsearch.xpack.core.template.LifecyclePolicyConfig; import java.io.IOException; @@ -250,8 +251,8 @@ protected Map getComposableTemplateConfigs() { } private static final List INGEST_PIPELINE_CONFIGS = List.of( - new IngestPipelineConfig("logs@json-pipeline", "/logs@json-pipeline.json", REGISTRY_VERSION, TEMPLATE_VERSION_VARIABLE), - new IngestPipelineConfig("logs@default-pipeline", "/logs@default-pipeline.json", REGISTRY_VERSION, TEMPLATE_VERSION_VARIABLE) + new JsonIngestPipelineConfig("logs@json-pipeline", "/logs@json-pipeline.json", REGISTRY_VERSION, TEMPLATE_VERSION_VARIABLE), + new JsonIngestPipelineConfig("logs@default-pipeline", "/logs@default-pipeline.json", REGISTRY_VERSION, TEMPLATE_VERSION_VARIABLE) ); @Override