From a33ee29d9ce4f4783a54cc2c7d9e6d70888c4fdb Mon Sep 17 00:00:00 2001 From: Nicolas Ruflin Date: Tue, 22 Jan 2019 21:09:23 +0100 Subject: [PATCH] Add docs for ILM feature (#9263) * Add docs for ILM feature This is a follow up for https://github.com/elastic/beats/pull/7963 * Add edits * Minor fixes * Add changes from the review * Make cross references consistent * Fix rebase error --- auditbeat/docs/configuring-howto.asciidoc | 3 + filebeat/docs/configuring-howto.asciidoc | 3 + heartbeat/docs/configuring-howto.asciidoc | 3 + journalbeat/docs/configuring-howto.asciidoc | 3 + libbeat/docs/outputconfig.asciidoc | 15 +- libbeat/docs/shared-ilm.asciidoc | 143 ++++++++++++++++++ metricbeat/docs/configuring-howto.asciidoc | 3 + packetbeat/docs/configuring-howto.asciidoc | 3 + winlogbeat/docs/configuring-howto.asciidoc | 3 + .../docs/configuring-howto.asciidoc | 4 + 10 files changed, 182 insertions(+), 1 deletion(-) create mode 100644 libbeat/docs/shared-ilm.asciidoc diff --git a/auditbeat/docs/configuring-howto.asciidoc b/auditbeat/docs/configuring-howto.asciidoc index 68f049a7c501..13e5ebda5e46 100644 --- a/auditbeat/docs/configuring-howto.asciidoc +++ b/auditbeat/docs/configuring-howto.asciidoc @@ -24,6 +24,7 @@ The following topics describe how to configure {beatname_uc}: * <<{beatname_lc}-configuration-reloading>> * <> * <> +* <> * <> * <> * <> @@ -53,6 +54,8 @@ include::{libbeat-dir}/docs/queueconfig.asciidoc[] include::{libbeat-dir}/docs/outputconfig.asciidoc[] +include::{libbeat-dir}/docs/shared-ilm.asciidoc[] + include::{libbeat-dir}/docs/shared-ssl-config.asciidoc[] include::./auditbeat-filtering.asciidoc[] diff --git a/filebeat/docs/configuring-howto.asciidoc b/filebeat/docs/configuring-howto.asciidoc index a08fe2ed131d..ac6a70b3b949 100644 --- a/filebeat/docs/configuring-howto.asciidoc +++ b/filebeat/docs/configuring-howto.asciidoc @@ -28,6 +28,7 @@ The following topics describe how to configure Filebeat: * <> * <> * <> +* <> * <> * <> * <> @@ -61,6 +62,8 @@ include::{libbeat-dir}/docs/queueconfig.asciidoc[] include::{libbeat-dir}/docs/outputconfig.asciidoc[] +include::../../libbeat/docs/shared-ilm.asciidoc[] + include::./load-balancing.asciidoc[] include::{libbeat-dir}/docs/shared-ssl-config.asciidoc[] diff --git a/heartbeat/docs/configuring-howto.asciidoc b/heartbeat/docs/configuring-howto.asciidoc index 270caac4d935..d9982e04eb48 100644 --- a/heartbeat/docs/configuring-howto.asciidoc +++ b/heartbeat/docs/configuring-howto.asciidoc @@ -25,6 +25,7 @@ The following topics describe how to configure Heartbeat: * <> * <> * <> +* <> * <> * <> * <> @@ -50,6 +51,8 @@ include::{libbeat-dir}/docs/queueconfig.asciidoc[] include::{libbeat-dir}/docs/outputconfig.asciidoc[] +include::{libbeat-dir}/docs/shared-ilm.asciidoc[] + include::{libbeat-dir}/docs/shared-ssl-config.asciidoc[] include::./heartbeat-filtering.asciidoc[] diff --git a/journalbeat/docs/configuring-howto.asciidoc b/journalbeat/docs/configuring-howto.asciidoc index 45b1f98058a5..7e475b8a113d 100644 --- a/journalbeat/docs/configuring-howto.asciidoc +++ b/journalbeat/docs/configuring-howto.asciidoc @@ -16,6 +16,7 @@ The following topics describe how to configure {beatname_uc}: * <> * <> * <> +* <> * <> * <> * <> @@ -39,6 +40,8 @@ include::{libbeat-dir}/docs/queueconfig.asciidoc[] include::{libbeat-dir}/docs/outputconfig.asciidoc[] +include::{libbeat-dir}/docs/shared-ilm.asciidoc[] + include::{libbeat-dir}/docs/shared-ssl-config.asciidoc[] include::./filtering.asciidoc[] diff --git a/libbeat/docs/outputconfig.asciidoc b/libbeat/docs/outputconfig.asciidoc index af888bae2903..0721236c4f57 100644 --- a/libbeat/docs/outputconfig.asciidoc +++ b/libbeat/docs/outputconfig.asciidoc @@ -86,13 +86,17 @@ output.elasticsearch: protocol: "https" username: "{beatname_lc}_internal" password: "{pwd}" - ------------------------------------------------------------------------------ For more information about securing {beatname_uc}, see <>. +If you are indexing large amounts of time-series data, you might also want to +configure {beatname_uc} to use index lifecycle management. For more information +about configuring and using index lifecycle management with {beatname_uc}, see +<>. + ==== Compatibility This output works with all compatible versions of Elasticsearch. See the @@ -320,6 +324,15 @@ This configuration results in indices named `sev1`, `sev2`, and `sev3`. The `mappings` setting simplifies the configuration, but is limited to string values. You cannot specify format strings within the mapping pairs. +//TODO: MOVE ILM OPTIONS TO APPEAR LOGICALLY BASED ON LOCATION IN THE YAML FILE. + +[[ilm-es]] +===== `ilm` + +Configuration options for index lifecycle management. + +See <> for more information. + ifndef::no-pipeline[] [[pipeline-option-es]] ===== `pipeline` diff --git a/libbeat/docs/shared-ilm.asciidoc b/libbeat/docs/shared-ilm.asciidoc new file mode 100644 index 000000000000..0019698d9641 --- /dev/null +++ b/libbeat/docs/shared-ilm.asciidoc @@ -0,0 +1,143 @@ +[[ilm]] +[role="xpack"] +== Set up index lifecycle management + +beta[] + +You can use the {ref}/getting-started-index-lifecycle-management.html[index +lifecycle management] feature in {es} to manage your {beatname_uc} indices as +they age. For example, instead of having {beatname_uc} create daily indices +where index size can vary based on the number of Beats and number of events +sent, you can use an index lifecycle policy that automates a rollover to a new +index when the existing index reaches a specified size or age. + +{beatname_uc} provides a default policy that you can load when you set up +{beatname_uc}. The default policy is applied to any new indices created by +{beatname_uc}. You can edit the policy to modify the lifecycle of both new and +existing indices. + +To use index lifecycle management on {beatname_uc} indices: + +. Enable index lifecycle management by setting `ilm.enabled: true` in the {es} +output configuration. For example: ++ +-- +[source,yaml] +------------------------------------------------------------------------------ +output.elasticsearch: + hosts: ["localhost:9200"] + ilm.enabled: true +------------------------------------------------------------------------------ + +This configuration overwrites your index settings and adjusts the {beatname_uc} +template to use the correct settings for index lifecycle management. + +NOTE: If you've previously loaded the index template for this version into {es}, +you must overwrite the template by setting `setup.template.overwrite: true`. + +The rollover alias is set to +{beatname_lc}-\{beat.version\}+ by default. You +can change the prefix used in the alias by setting `ilm.rollover_alias`, but you +can't remove `{beat.version}` from the rollover alias name. The default pattern +used for the rollover index is `%{now/d}-000001`. You can change the +pattern by setting `ilm.pattern`. For example: + +["source","yaml",subs="attributes"] +---- +output.elasticsearch: + hosts: ["localhost"] + ilm.enabled: true + ilm.rollover_alias: "{beatname_lc}" + ilm.pattern: "{now/d}-000001" <1> +---- +<1> Date math is supported here. For more information, see +{ref}/indices-rollover-index.html#_using_date_math_with_the_rollover_api[Using +date math with the rollover API]. + +NOTE: If you modify the `rollover_alias` or `pattern` settings after loading the +index template, you must overwrite the template to apply the changes. +-- + +. Load the default policy into {es}. You can either use the `setup` command to +load the policy without modifying it, or modify the policy and load it manually. ++ +-- +To use the setup command, run: + +["source","shell",subs="attributes"] +---- +{beatname_lc} setup --ilm-policy +---- + +After loading the default policy, you can edit it in the *Index lifecycle policies* +UI in {kib}. For more information about working with the UI, see +{kibana-ref}/index-lifecycle-policies.html[Index lifecyle policies]. + +To modify the default policy before loading it, run +{beatname_lc} export +ilm-policy+ to print the policy to stdout. Modify the policy then use the +{ref}/ilm-put-lifecycle.html[Create lifecycle policy API] to load it into {es}. + +-- + +[float] +=== Advanced ILM settings + +WARNING: We recommend that you avoid modifying these settings unless you know +what you're doing. + +The default index lifecycle management settings work best for common use cases +that work with the automated alias setup described earlier. It is possible to +use a multiple write alias with dynamic index patterns, but this requires manual +set up. This section describes the configuration options you need to change. + +Let's assume you have the index pattern `customname-%{event.module}` where +`event.module` can have the values `system` and `apache`. First you must set up +a rollover index for `customname-system` and `customname-apache`. For details on +how to do this, see +{ref}/indices-rollover-index.html#_using_date_math_with_the_rollover_api[Rollover +Index]. + +Next, set the index pattern in the {es} output. For example: + +["source","yaml",subs="attributes"] +---- +output.elasticesarch.index: customname-%{event.module} <1> +---- +<1> For this example to work, every event must contain `event.module`. + +If you change the index name, you must also set the template name, template +pattern, rollover alias, and lifecycle name. The best way to set these is +through an {es} template. It's possible to disable the template loading in +{beatname_uc} and specify these settings in your own template. Or you can use +the following config options in {beatname_uc}: + +[source,yaml] +---- +setup.template.name: "customname" +setup.template.pattern: "customname-*" +setup.template.settings.index.lifecycle.rollover_alias: "customname" +setup.template.settings.index.lifecycle.name: "beats-default-policy" +---- + +IMPORTANT: If you set the options manually as shown in this example, do *not* +set `ilm.enabled`, or the settings specified in the configuration file will be +overwritten. + +This configuration results in a managed index named something like ++customname-{localdate}-000001+ and the following index settings: + +["source","shell"] +---- +"aliases" : { + "customname" : { + "is_write_index" : true + } +}, +... + "index" : { + "lifecycle" : { + "name" : "beats-default-policy", + "rollover_alias" : "customname" + }, +---- + + diff --git a/metricbeat/docs/configuring-howto.asciidoc b/metricbeat/docs/configuring-howto.asciidoc index 69977be1f2c6..fef5f4ca8d3c 100644 --- a/metricbeat/docs/configuring-howto.asciidoc +++ b/metricbeat/docs/configuring-howto.asciidoc @@ -26,6 +26,7 @@ The following topics describe how to configure {beatname_uc}: * <> * <> * <> +* <> * <> * <> * <> @@ -53,6 +54,8 @@ include::{libbeat-dir}/docs/queueconfig.asciidoc[] include::{libbeat-dir}/docs/outputconfig.asciidoc[] +include::{libbeat-dir}/docs/shared-ilm.asciidoc[] + include::{libbeat-dir}/docs/shared-ssl-config.asciidoc[] include::./metricbeat-filtering.asciidoc[] diff --git a/packetbeat/docs/configuring-howto.asciidoc b/packetbeat/docs/configuring-howto.asciidoc index a3b7e46c90ab..c64dc68dfe5f 100644 --- a/packetbeat/docs/configuring-howto.asciidoc +++ b/packetbeat/docs/configuring-howto.asciidoc @@ -26,6 +26,7 @@ The following topics describe how to configure Packetbeat: * <> * <> * <> +* <> * <> * <> * <> @@ -50,6 +51,8 @@ include::{libbeat-dir}/docs/queueconfig.asciidoc[] include::{libbeat-dir}/docs/outputconfig.asciidoc[] +include::{libbeat-dir}/docs/shared-ilm.asciidoc[] + include::{libbeat-dir}/docs/shared-ssl-config.asciidoc[] include::./packetbeat-filtering.asciidoc[] diff --git a/winlogbeat/docs/configuring-howto.asciidoc b/winlogbeat/docs/configuring-howto.asciidoc index 664ea25259a8..12b0e9fa4671 100644 --- a/winlogbeat/docs/configuring-howto.asciidoc +++ b/winlogbeat/docs/configuring-howto.asciidoc @@ -21,6 +21,7 @@ The following topics describe how to configure Winlogbeat: * <> * <> * <> +* <> * <> * <> * <> @@ -44,6 +45,8 @@ include::{libbeat-dir}/docs/queueconfig.asciidoc[] include::{libbeat-dir}/docs/outputconfig.asciidoc[] +include::{libbeat-dir}/docs/shared-ilm.asciidoc[] + include::{libbeat-dir}/docs/shared-ssl-config.asciidoc[] include::./winlogbeat-filtering.asciidoc[] diff --git a/x-pack/functionbeat/docs/configuring-howto.asciidoc b/x-pack/functionbeat/docs/configuring-howto.asciidoc index 239c107a6374..20eff8bb4422 100644 --- a/x-pack/functionbeat/docs/configuring-howto.asciidoc +++ b/x-pack/functionbeat/docs/configuring-howto.asciidoc @@ -16,6 +16,7 @@ The following topics describe how to configure {beatname_uc}: * <> * <> * <> +* <> * <> * <> * <> @@ -41,6 +42,9 @@ include::{libbeat-dir}/docs/queueconfig.asciidoc[] [role="xpack"] include::{libbeat-dir}/docs/outputconfig.asciidoc[] +[role="xpack"] +include::{libbeat-dir}/docs/shared-ilm.asciidoc[] + [role="xpack"] include::{libbeat-dir}/docs/shared-ssl-config.asciidoc[]