diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index ddc1d5a6d001..714b62fb0841 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -72,6 +72,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha1...master[Check the HEAD d - Unify dashboard exporter tools. {pull}9097[9097] - Use _doc as document type of the Elasticsearch major version is 7. {pull}9056[9056] - Add cache.ttl to add_host_metadata. {pull}9359[9359] +- Add support for index lifecycle management (beta). {pull}7963[7963] *Auditbeat* diff --git a/auditbeat/auditbeat.reference.yml b/auditbeat/auditbeat.reference.yml index f8c69a5000b2..f69d819e615d 100644 --- a/auditbeat/auditbeat.reference.yml +++ b/auditbeat/auditbeat.reference.yml @@ -357,6 +357,11 @@ output.elasticsearch: # IPv6 addresses should always be defined as: https://[2001:db8::1]:9200 hosts: ["localhost:9200"] + # Enabled ilm (beta) to use index lifecycle management instead daily indices. + #ilm.enabled: false + #ilm.rollover_alias: "auditbeat" + #ilm.pattern: "{now/d}-000001" + # Set gzip compression level. #compression_level: 0 diff --git a/auditbeat/auditbeat.yml b/auditbeat/auditbeat.yml index 3e0e41e4c037..79a1b1cb9028 100644 --- a/auditbeat/auditbeat.yml +++ b/auditbeat/auditbeat.yml @@ -122,6 +122,9 @@ output.elasticsearch: # Array of hosts to connect to. hosts: ["localhost:9200"] + # Enabled ilm (beta) to use index lifecycle management instead daily indices. + #ilm.enabled: false + # Optional protocol and basic auth credentials. #protocol: "https" #username: "elastic" diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml index 38677dd6068c..ce4b822bbb48 100644 --- a/filebeat/filebeat.reference.yml +++ b/filebeat/filebeat.reference.yml @@ -1031,6 +1031,11 @@ output.elasticsearch: # IPv6 addresses should always be defined as: https://[2001:db8::1]:9200 hosts: ["localhost:9200"] + # Enabled ilm (beta) to use index lifecycle management instead daily indices. + #ilm.enabled: false + #ilm.rollover_alias: "filebeat" + #ilm.pattern: "{now/d}-000001" + # Set gzip compression level. #compression_level: 0 diff --git a/filebeat/filebeat.yml b/filebeat/filebeat.yml index 5de1a4d1e55c..78467764cc94 100644 --- a/filebeat/filebeat.yml +++ b/filebeat/filebeat.yml @@ -149,6 +149,9 @@ output.elasticsearch: # Array of hosts to connect to. hosts: ["localhost:9200"] + # Enabled ilm (beta) to use index lifecycle management instead daily indices. + #ilm.enabled: false + # Optional protocol and basic auth credentials. #protocol: "https" #username: "elastic" diff --git a/heartbeat/heartbeat.reference.yml b/heartbeat/heartbeat.reference.yml index e836fd6fcbd0..a5571143e09b 100644 --- a/heartbeat/heartbeat.reference.yml +++ b/heartbeat/heartbeat.reference.yml @@ -490,6 +490,11 @@ output.elasticsearch: # IPv6 addresses should always be defined as: https://[2001:db8::1]:9200 hosts: ["localhost:9200"] + # Enabled ilm (beta) to use index lifecycle management instead daily indices. + #ilm.enabled: false + #ilm.rollover_alias: "heartbeat" + #ilm.pattern: "{now/d}-000001" + # Set gzip compression level. #compression_level: 0 diff --git a/heartbeat/heartbeat.yml b/heartbeat/heartbeat.yml index da3f3d9aa0c1..1e20e85e6b3b 100644 --- a/heartbeat/heartbeat.yml +++ b/heartbeat/heartbeat.yml @@ -96,6 +96,9 @@ output.elasticsearch: # Array of hosts to connect to. hosts: ["localhost:9200"] + # Enabled ilm (beta) to use index lifecycle management instead daily indices. + #ilm.enabled: false + # Optional protocol and basic auth credentials. #protocol: "https" #username: "elastic" diff --git a/journalbeat/journalbeat.reference.yml b/journalbeat/journalbeat.reference.yml index b5b03a133dc8..2e936bc1cd6f 100644 --- a/journalbeat/journalbeat.reference.yml +++ b/journalbeat/journalbeat.reference.yml @@ -291,6 +291,11 @@ output.elasticsearch: # IPv6 addresses should always be defined as: https://[2001:db8::1]:9200 hosts: ["localhost:9200"] + # Enabled ilm (beta) to use index lifecycle management instead daily indices. + #ilm.enabled: false + #ilm.rollover_alias: "journalbeat" + #ilm.pattern: "{now/d}-000001" + # Set gzip compression level. #compression_level: 0 diff --git a/journalbeat/journalbeat.yml b/journalbeat/journalbeat.yml index 414e249382aa..a12f6c28887f 100644 --- a/journalbeat/journalbeat.yml +++ b/journalbeat/journalbeat.yml @@ -112,6 +112,9 @@ output.elasticsearch: # Array of hosts to connect to. hosts: ["localhost:9200"] + # Enabled ilm (beta) to use index lifecycle management instead daily indices. + #ilm.enabled: false + # Optional protocol and basic auth credentials. #protocol: "https" #username: "elastic" diff --git a/libbeat/_meta/config.reference.yml b/libbeat/_meta/config.reference.yml index 6d987f7e1488..3321ed51e9c7 100644 --- a/libbeat/_meta/config.reference.yml +++ b/libbeat/_meta/config.reference.yml @@ -245,6 +245,11 @@ output.elasticsearch: # IPv6 addresses should always be defined as: https://[2001:db8::1]:9200 hosts: ["localhost:9200"] + # Enabled ilm (beta) to use index lifecycle management instead daily indices. + #ilm.enabled: false + #ilm.rollover_alias: "beat-index-prefix" + #ilm.pattern: "{now/d}-000001" + # Set gzip compression level. #compression_level: 0 diff --git a/libbeat/_meta/config.yml b/libbeat/_meta/config.yml index 3559eb140321..0847a93afd14 100644 --- a/libbeat/_meta/config.yml +++ b/libbeat/_meta/config.yml @@ -66,6 +66,9 @@ output.elasticsearch: # Array of hosts to connect to. hosts: ["localhost:9200"] + # Enabled ilm (beta) to use index lifecycle management instead daily indices. + #ilm.enabled: false + # Optional protocol and basic auth credentials. #protocol: "https" #username: "elastic" diff --git a/libbeat/cmd/export.go b/libbeat/cmd/export.go index e9d6015992da..07f9cd16da0a 100644 --- a/libbeat/cmd/export.go +++ b/libbeat/cmd/export.go @@ -33,6 +33,7 @@ func genExportCmd(settings instance.Settings, name, idxPrefix, beatVersion strin exportCmd.AddCommand(export.GenExportConfigCmd(settings, name, idxPrefix, beatVersion)) exportCmd.AddCommand(export.GenTemplateConfigCmd(settings, name, idxPrefix, beatVersion)) exportCmd.AddCommand(export.GenDashboardCmd(name, idxPrefix, beatVersion)) + exportCmd.AddCommand(export.GenGetILMPolicyCmd()) return exportCmd } diff --git a/libbeat/cmd/export/ilm_policy.go b/libbeat/cmd/export/ilm_policy.go new file mode 100644 index 000000000000..a9e4542fc29e --- /dev/null +++ b/libbeat/cmd/export/ilm_policy.go @@ -0,0 +1,39 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package export + +import ( + "fmt" + + "github.com/spf13/cobra" + + "github.com/elastic/beats/libbeat/cmd/instance" +) + +// GenGetILMPolicyCmd is the command used to export the ilm policy. +func GenGetILMPolicyCmd() *cobra.Command { + genTemplateConfigCmd := &cobra.Command{ + Use: "ilm-policy", + Short: "Export ILM policy", + Run: func(cmd *cobra.Command, args []string) { + fmt.Println(instance.ILMPolicy.StringToPrint()) + }, + } + + return genTemplateConfigCmd +} diff --git a/libbeat/cmd/instance/beat.go b/libbeat/cmd/instance/beat.go index 9e9c23d756e9..9a53bd400a5c 100644 --- a/libbeat/cmd/instance/beat.go +++ b/libbeat/cmd/instance/beat.go @@ -105,6 +105,9 @@ type beatConfig struct { Dashboards *common.Config `config:"setup.dashboards"` Template *common.Config `config:"setup.template"` Kibana *common.Config `config:"setup.kibana"` + + // ILM Config options + ILM *common.Config `config:"output.elasticsearch.ilm"` } var ( @@ -430,7 +433,7 @@ func (b *Beat) TestConfig(bt beat.Creator) error { } // Setup registers ES index template, kibana dashboards, ml jobs and pipelines. -func (b *Beat) Setup(bt beat.Creator, template, setupDashboards, machineLearning, pipelines bool) error { +func (b *Beat) Setup(bt beat.Creator, template, setupDashboards, machineLearning, pipelines, policy bool) error { return handleError(func() error { err := b.Init() if err != nil { @@ -509,6 +512,13 @@ func (b *Beat) Setup(bt beat.Creator, template, setupDashboards, machineLearning fmt.Println("Loaded Ingest pipelines") } + if policy { + if err := b.loadILMPolicy(); err != nil { + return err + } + fmt.Println("Loaded Index Lifecycle Management (ILM) policy") + } + return nil }()) } @@ -719,11 +729,11 @@ func (b *Beat) loadDashboards(ctx context.Context, force bool) error { // the elasticsearch output. It is important the the registration happens before // the publisher is created. func (b *Beat) registerTemplateLoading() error { - var cfg template.TemplateConfig + var templateCfg template.TemplateConfig // Check if outputting to file is enabled, and output to file if it is if b.Config.Template.Enabled() { - err := b.Config.Template.Unpack(&cfg) + err := b.Config.Template.Unpack(&templateCfg) if err != nil { return fmt.Errorf("unpacking template config fails: %v", err) } @@ -741,8 +751,82 @@ func (b *Beat) registerTemplateLoading() error { return err } - if esCfg.Index != "" && (cfg.Name == "" || cfg.Pattern == "") && (b.Config.Template == nil || b.Config.Template.Enabled()) { - return fmt.Errorf("setup.template.name and setup.template.pattern have to be set if index name is modified.") + if esCfg.Index != "" && + (templateCfg.Name == "" || templateCfg.Pattern == "") && + (b.Config.Template == nil || b.Config.Template.Enabled()) { + return errors.New("setup.template.name and setup.template.pattern have to be set if index name is modified") + } + + if b.Config.ILM.Enabled() { + cfgwarn.Beta("Index lifecycle management is enabled which is in beta.") + + ilmCfg, err := getILMConfig(b) + if err != nil { + return err + } + + // In case no template settings are set, config must be created + if b.Config.Template == nil { + b.Config.Template = common.NewConfig() + } + // Template name and pattern can't be configure when using ILM + logp.Info("Set setup.template.name to '%s' as ILM is enabled.", ilmCfg.RolloverAlias) + err = b.Config.Template.SetString("name", -1, ilmCfg.RolloverAlias) + if err != nil { + return errw.Wrap(err, "error setting setup.template.name") + } + pattern := fmt.Sprintf("%s-*", ilmCfg.RolloverAlias) + logp.Info("Set setup.template.pattern to '%s' as ILM is enabled.", pattern) + err = b.Config.Template.SetString("pattern", -1, pattern) + if err != nil { + return errw.Wrap(err, "error setting setup.template.pattern") + } + + // rollover_alias and lifecycle.name can't be configured and will be overwritten + logp.Info("Set settings.index.lifecycle.rollover_alias in template to %s as ILM is enabled.", ilmCfg.RolloverAlias) + err = b.Config.Template.SetString("settings.index.lifecycle.rollover_alias", -1, ilmCfg.RolloverAlias) + if err != nil { + return errw.Wrap(err, "error setting settings.index.lifecycle.rollover_alias") + } + logp.Info("Set settings.index.lifecycle.name in template to %s as ILM is enabled.", ILMPolicyName) + err = b.Config.Template.SetString("settings.index.lifecycle.name", -1, ILMPolicyName) + if err != nil { + return errw.Wrap(err, "error setting settings.index.lifecycle.name") + } + + // Set the ingestion index to the rollover alias + logp.Info("Set output.elasticsearch.index to '%s' as ILM is enabled.", ilmCfg.RolloverAlias) + esCfg.Index = ilmCfg.RolloverAlias + err = b.Config.Output.Config().SetString("index", -1, ilmCfg.RolloverAlias) + if err != nil { + return errw.Wrap(err, "error setting output.elasticsearch.index") + } + + writeAliasCallback, err := b.writeAliasLoadingCallback() + if err != nil { + return err + } + + // Load write alias already on + esConfig := b.Config.Output.Config() + + // Check that ILM is enabled and the right elasticsearch version exists + esClient, err := elasticsearch.NewConnectedClient(esConfig) + if err != nil { + return err + } + + err = checkElasticsearchVersionIlm(esClient) + if err != nil { + return err + } + + err = checkILMFeatureEnabled(esClient) + if err != nil { + return err + } + + elasticsearch.RegisterConnectCallback(writeAliasCallback) } if b.Config.Template == nil || (b.Config.Template != nil && b.Config.Template.Enabled()) { @@ -754,6 +838,8 @@ func (b *Beat) registerTemplateLoading() error { return err } elasticsearch.RegisterConnectCallback(callback) + } else if b.Config.ILM.Enabled() { + return errors.New("templates cannot be disable when using ILM") } } diff --git a/libbeat/cmd/instance/ilm.go b/libbeat/cmd/instance/ilm.go new file mode 100644 index 000000000000..addaa24258e5 --- /dev/null +++ b/libbeat/cmd/instance/ilm.go @@ -0,0 +1,206 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package instance + +import ( + "encoding/json" + "fmt" + "net/url" + + "github.com/pkg/errors" + + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/logp" + "github.com/elastic/beats/libbeat/outputs/elasticsearch" +) + +type ilmConfig struct { + RolloverAlias string `config:"ilm.rollover_alias" ` + Pattern string `config:"ilm.pattern"` +} + +// ILMPolicy contains the default policy +var ILMPolicy = common.MapStr{ + "policy": common.MapStr{ + "phases": common.MapStr{ + "hot": common.MapStr{ + "actions": common.MapStr{ + "rollover": common.MapStr{ + "max_size": "50gb", + "max_age": "30d", + }, + }, + }, + }, + }, +} + +const ( + // ILMPolicyName is the default policy name + ILMPolicyName = "beats-default-policy" + // ILMDefaultPattern is the default pattern + ILMDefaultPattern = "{now/d}-000001" +) + +// Build and return a callback to load ILM write alias +func (b *Beat) writeAliasLoadingCallback() (func(esClient *elasticsearch.Client) error, error) { + callback := func(esClient *elasticsearch.Client) error { + if b.Config.ILM == nil { + b.Config.ILM = common.NewConfig() + } + + config, err := getILMConfig(b) + if err != nil { + return err + } + + // Escaping because of date pattern + pattern := url.PathEscape(config.Pattern) + // This always assume it's a date pattern by sourrounding it by <...> + firstIndex := fmt.Sprintf("%%3C%s-%s%%3E", config.RolloverAlias, pattern) + + // Check if alias already exists + status, b, err := esClient.Request("HEAD", "/_alias/"+config.RolloverAlias, "", nil, nil) + if err != nil && status != 404 { + logp.Err("Failed to check for alias: %s: %+v", err, string(b)) + return errors.Wrap(err, "failed to check for alias") + } + if status == 200 { + logp.Info("Write alias already exists") + return nil + } + + body := common.MapStr{ + "aliases": common.MapStr{ + config.RolloverAlias: common.MapStr{ + "is_write_index": true, + }, + }, + } + + // Create alias with write index + code, res, err := esClient.Request("PUT", "/"+firstIndex, "", nil, body) + if code == 400 { + logp.Err("Error creating alias with write index. As return code is 400, assuming already exists: %s, %s", err, string(res)) + return nil + + } else if err != nil { + logp.Err("Error creating alias with write index: %s, %s", err, string(res)) + return errors.Wrap(err, "failed to create write alias: "+string(res)) + } + + logp.Info("Alias with write index created: %s", firstIndex) + + return nil + } + + return callback, nil +} + +func (b *Beat) loadILMPolicy() error { + esClient, err := getElasticsearchClient(b) + if err != nil { + return err + } + + _, _, err = esClient.Request("PUT", "/_ilm/policy/"+ILMPolicyName, "", nil, ILMPolicy) + return err +} + +func getElasticsearchClient(b *Beat) (*elasticsearch.Client, error) { + outCfg := b.Config.Output + if outCfg.Name() != "elasticsearch" { + return nil, fmt.Errorf("Policy loading requested but the Elasticsearch output is not configured/enabled") + } + + esConfig := outCfg.Config() + + return elasticsearch.NewConnectedClient(esConfig) +} + +func loadConfigWithDefaults(config *ilmConfig, b *Beat) { + if config.RolloverAlias == "" { + config.RolloverAlias = fmt.Sprintf("%s-%s", b.Info.Beat, b.Info.Version) + } + + if config.Pattern == "" { + config.Pattern = ILMDefaultPattern + } +} + +func checkElasticsearchVersionIlm(client *elasticsearch.Client) error { + esV := client.GetVersion() + requiredVersion, err := common.NewVersion("6.6.0") + if err != nil { + return err + } + + if esV.LessThan(requiredVersion) { + return fmt.Errorf("ILM requires at least Elasticsearch 6.6.0. Used version: %s", esV.String()) + } + + return nil +} + +func checkILMFeatureEnabled(client *elasticsearch.Client) error { + code, body, err := client.Request("GET", "/_xpack", "", nil, nil) + + // If we get a 400, it's assumed to be the OSS version of Elasticsearch + if code == 400 { + return fmt.Errorf("ILM feature is not available in this Elasticsearch version") + } + if err != nil { + return err + } + + var response struct { + Features struct { + ILM struct { + Available bool `json:"available"` + Enabled bool `json:"enabled"` + } `json:"ilm"` + } `json:"features"` + } + + err = json.Unmarshal(body, &response) + if err != nil { + return fmt.Errorf("failed to parse JSON response: %v", err) + } + + if !response.Features.ILM.Available { + return fmt.Errorf("ILM feature is not available in Elasticsearch") + } + + if !response.Features.ILM.Enabled { + return fmt.Errorf("ILM feature is not enabled in Elasticsearch") + } + + return nil +} + +func getILMConfig(b *Beat) (*ilmConfig, error) { + config := &ilmConfig{} + err := b.Config.Output.Config().Unpack(config) + if err != nil { + return nil, errors.Wrap(err, "problem unpacking ilm configs") + } + + loadConfigWithDefaults(config, b) + + return config, nil +} diff --git a/libbeat/cmd/setup.go b/libbeat/cmd/setup.go index 2b4f5cd326be..fd096a343707 100644 --- a/libbeat/cmd/setup.go +++ b/libbeat/cmd/setup.go @@ -37,6 +37,7 @@ func genSetupCmd(name, idxPrefix, version string, beatCreator beat.Creator) *cob * Kibana dashboards (where available). * ML jobs (where available). * Ingest pipelines (where available). + * ILM policy (for Elasticsearch 6.5 and newer). `, Run: func(cmd *cobra.Command, args []string) { beat, err := instance.NewBeat(name, idxPrefix, version) @@ -49,16 +50,18 @@ func genSetupCmd(name, idxPrefix, version string, beatCreator beat.Creator) *cob dashboards, _ := cmd.Flags().GetBool("dashboards") machineLearning, _ := cmd.Flags().GetBool("machine-learning") pipelines, _ := cmd.Flags().GetBool("pipelines") + policy, _ := cmd.Flags().GetBool("ilm-policy") // No flags: setup all - if !template && !dashboards && !machineLearning && !pipelines { + if !template && !dashboards && !machineLearning && !pipelines && !policy { template = true dashboards = true machineLearning = true pipelines = true + policy = true } - if err = beat.Setup(beatCreator, template, dashboards, machineLearning, pipelines); err != nil { + if err = beat.Setup(beatCreator, template, dashboards, machineLearning, pipelines, policy); err != nil { os.Exit(1) } }, @@ -68,6 +71,7 @@ func genSetupCmd(name, idxPrefix, version string, beatCreator beat.Creator) *cob setup.Flags().Bool("dashboards", false, "Setup dashboards") setup.Flags().Bool("machine-learning", false, "Setup machine learning job configurations") setup.Flags().Bool("pipelines", false, "Setup Ingest pipelines") + setup.Flags().Bool("ilm-policy", false, "Setup ILM policy") return &setup } diff --git a/libbeat/docs/command-reference.asciidoc b/libbeat/docs/command-reference.asciidoc index e30cabe3b5d6..577a7ab58300 100644 --- a/libbeat/docs/command-reference.asciidoc +++ b/libbeat/docs/command-reference.asciidoc @@ -184,6 +184,10 @@ If {kib} is not running on `localhost:5061`, you must also adjust the Exports the index template to stdout. You can specify the `--es.version` and `--index` flags to further define what gets exported. +[[ilm-policy-subcommand]] +*`ilm-policy`*:: +Exports ILM policy to stdout. + *FLAGS* *`--es.version VERSION`*:: diff --git a/libbeat/tests/system/config/libbeat.yml.j2 b/libbeat/tests/system/config/libbeat.yml.j2 index a8e494e63eb9..839836be2875 100644 --- a/libbeat/tests/system/config/libbeat.yml.j2 +++ b/libbeat/tests/system/config/libbeat.yml.j2 @@ -66,6 +66,9 @@ output: {% if elasticsearch.index %} index: {{elasticsearch.index}} {% endif %} + {% if elasticsearch.ilm %} + ilm.enabled: {{elasticsearch.ilm}} + {% endif %} {%- endif %} {% if logstash %} diff --git a/libbeat/tests/system/test_ilm.py b/libbeat/tests/system/test_ilm.py new file mode 100644 index 000000000000..305f6f4e2fe7 --- /dev/null +++ b/libbeat/tests/system/test_ilm.py @@ -0,0 +1,253 @@ +from base import BaseTest +import os +from elasticsearch import Elasticsearch, TransportError +from nose.plugins.attrib import attr +import unittest +import shutil +import logging +import datetime + +INTEGRATION_TESTS = os.environ.get('INTEGRATION_TESTS', False) + + +class Test(BaseTest): + + def setUp(self): + super(BaseTest, self).setUp() + + self.elasticsearch_url = self.get_elasticsearch_url() + print("Using elasticsearch: {}".format(self.elasticsearch_url)) + self.es = Elasticsearch([self.elasticsearch_url]) + self.alias_name = "mockbeat-9.9.9" + self.policy_name = "beats-default-policy" + logging.getLogger("urllib3").setLevel(logging.WARNING) + logging.getLogger("elasticsearch").setLevel(logging.ERROR) + + @unittest.skipUnless(INTEGRATION_TESTS, "integration test") + @attr('integration') + def test_enabled(self): + """ + Test ilm enabled + """ + + self.render_config_template( + elasticsearch={ + "hosts": self.get_elasticsearch_url(), + "ilm.enabled": True, + }, + ) + + self.clean() + + proc = self.start_beat() + self.wait_until(lambda: self.log_contains("mockbeat start running.")) + self.wait_until(lambda: self.log_contains("Set setup.template.name")) + self.wait_until(lambda: self.log_contains("PublishEvents: 1 events have been published")) + proc.check_kill_and_wait() + + # Check if template is loaded with settings + template = self.es.transport.perform_request('GET', '/_template/' + self.alias_name) + + assert template[self.alias_name]["settings"]["index"]["lifecycle"]["name"] == "beats-default-policy" + assert template[self.alias_name]["settings"]["index"]["lifecycle"]["rollover_alias"] == self.alias_name + + # Make sure the correct index + alias was created + alias = self.es.transport.perform_request('GET', '/_alias/' + self.alias_name) + d = datetime.datetime.now() + now = d.strftime("%Y.%m.%d") + index_name = self.alias_name + "-" + now + "-000001" + assert index_name in alias + assert alias[index_name]["aliases"][self.alias_name]["is_write_index"] == True + + # Asserts that data is actually written to the ILM indices + self.wait_until(lambda: self.es.transport.perform_request( + 'GET', '/' + index_name + '/_search')["hits"]["total"] > 0) + + data = self.es.transport.perform_request('GET', '/' + index_name + '/_search') + assert data["hits"]["total"] > 0 + + @unittest.skipUnless(INTEGRATION_TESTS, "integration test") + @attr('integration') + def test_rollover_alias(self): + """ + Test ilm rollover alias setting + """ + + alias_name = "foo" + self.render_config_template( + elasticsearch={ + "hosts": self.get_elasticsearch_url(), + "ilm.enabled": True, + "ilm.pattern": "1", + "ilm.rollover_alias": alias_name + }, + ) + + self.clean(alias_name=alias_name) + + proc = self.start_beat() + self.wait_until(lambda: self.log_contains("mockbeat start running.")) + self.wait_until(lambda: self.log_contains("Set setup.template.name")) + self.wait_until(lambda: self.log_contains("PublishEvents: 1 events have been published")) + proc.check_kill_and_wait() + + # Make sure the correct index + alias was created + print '/_alias/' + alias_name + logfile = self.beat_name + ".log" + with open(os.path.join(self.working_dir, logfile), "r") as f: + print f.read() + + alias = self.es.transport.perform_request('GET', '/_alias/' + alias_name) + index_name = alias_name + "-1" + assert index_name in alias + + @unittest.skipUnless(INTEGRATION_TESTS, "integration test") + @attr('integration') + def test_pattern(self): + """ + Test ilm pattern setting + """ + + self.render_config_template( + elasticsearch={ + "hosts": self.get_elasticsearch_url(), + "ilm.enabled": True, + "ilm.pattern": "1" + }, + ) + + self.clean() + + proc = self.start_beat() + self.wait_until(lambda: self.log_contains("mockbeat start running.")) + self.wait_until(lambda: self.log_contains("Set setup.template.name")) + self.wait_until(lambda: self.log_contains("PublishEvents: 1 events have been published")) + proc.check_kill_and_wait() + + # Make sure the correct index + alias was created + print '/_alias/' + self.alias_name + logfile = self.beat_name + ".log" + with open(os.path.join(self.working_dir, logfile), "r") as f: + print f.read() + + alias = self.es.transport.perform_request('GET', '/_alias/' + self.alias_name) + index_name = self.alias_name + "-1" + assert index_name in alias + + @unittest.skipUnless(INTEGRATION_TESTS, "integration test") + @attr('integration') + def test_pattern_date(self): + """ + Test ilm pattern with date inside + """ + + self.render_config_template( + elasticsearch={ + "hosts": self.get_elasticsearch_url(), + "ilm.enabled": True, + "ilm.pattern": "'{now/d}'" + }, + ) + + self.clean() + + proc = self.start_beat() + self.wait_until(lambda: self.log_contains("mockbeat start running.")) + self.wait_until(lambda: self.log_contains("Set setup.template.name")) + self.wait_until(lambda: self.log_contains("PublishEvents: 1 events have been published")) + proc.check_kill_and_wait() + + # Make sure the correct index + alias was created + print '/_alias/' + self.alias_name + logfile = self.beat_name + ".log" + with open(os.path.join(self.working_dir, logfile), "r") as f: + print f.read() + + # Make sure the correct index + alias was created + alias = self.es.transport.perform_request('GET', '/_alias/' + self.alias_name) + d = datetime.datetime.now() + now = d.strftime("%Y.%m.%d") + index_name = self.alias_name + "-" + now + assert index_name in alias + assert alias[index_name]["aliases"][self.alias_name]["is_write_index"] == True + + @unittest.skipUnless(INTEGRATION_TESTS, "integration test") + @attr('integration') + def test_setup_ilm_policy(self): + """ + Test ilm policy setup + """ + + self.clean() + + shutil.copy(self.beat_path + "/_meta/config.yml", + os.path.join(self.working_dir, "libbeat.yml")) + shutil.copy(self.beat_path + "/fields.yml", + os.path.join(self.working_dir, "fields.yml")) + + exit_code = self.run_beat( + logging_args=["-v", "-d", "*"], + extra_args=["setup", + "--ilm-policy", + "-path.config", self.working_dir, + "-E", "output.elasticsearch.hosts=['" + self.get_elasticsearch_url() + "']"], + config="libbeat.yml") + + assert exit_code == 0 + + policy = self.es.transport.perform_request('GET', "/_ilm/policy/" + self.policy_name) + assert self.policy_name in policy + + @attr('integration') + def test_export_ilm_policy(self): + """ + Test ilm policy export + """ + + self.clean() + + shutil.copy(self.beat_path + "/_meta/config.yml", + os.path.join(self.working_dir, "libbeat.yml")) + shutil.copy(self.beat_path + "/fields.yml", + os.path.join(self.working_dir, "fields.yml")) + + exit_code = self.run_beat( + logging_args=["-v", "-d", "*"], + extra_args=["export", + "ilm-policy", + ], + config="libbeat.yml") + + assert exit_code == 0 + + assert self.log_contains('"max_age": "30d"') + assert self.log_contains('"max_size": "50gb"') + + def clean(self, alias_name=""): + + if alias_name == "": + alias_name = self.alias_name + + # Delete existing indices and aliases with it policy + try: + self.es.transport.perform_request('DELETE', "/" + alias_name + "*") + except: + pass + + # Delete any existing policy + try: + self.es.transport.perform_request('DELETE', "/_ilm/policy/" + self.policy_name) + except: + pass + + # Delete templates + try: + self.es.transport.perform_request('DELETE', "/_template/mockbeat*") + except: + pass + + # Delete indices + try: + self.es.transport.perform_request('DELETE', "/foo*,mockbeat*") + except: + pass diff --git a/libbeat/tests/system/test_template.py b/libbeat/tests/system/test_template.py index f91d774d8418..66a67f4c0a0f 100644 --- a/libbeat/tests/system/test_template.py +++ b/libbeat/tests/system/test_template.py @@ -21,7 +21,7 @@ def test_index_modified(self): assert exit_code == 1 assert self.log_contains( - "setup.template.name and setup.template.pattern have to be set if index name is modified.") is True + "setup.template.name and setup.template.pattern have to be set if index name is modified") is True def test_index_not_modified(self): """ @@ -48,7 +48,7 @@ def test_index_modified_no_pattern(self): assert exit_code == 1 assert self.log_contains( - "setup.template.name and setup.template.pattern have to be set if index name is modified.") is True + "setup.template.name and setup.template.pattern have to be set if index name is modified") is True def test_index_modified_no_name(self): """ @@ -63,7 +63,7 @@ def test_index_modified_no_name(self): assert exit_code == 1 assert self.log_contains( - "setup.template.name and setup.template.pattern have to be set if index name is modified.") is True + "setup.template.name and setup.template.pattern have to be set if index name is modified") is True def test_index_with_pattern_name(self): """ diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 24449a776bf5..bc8d641356e5 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -930,6 +930,11 @@ output.elasticsearch: # IPv6 addresses should always be defined as: https://[2001:db8::1]:9200 hosts: ["localhost:9200"] + # Enabled ilm (beta) to use index lifecycle management instead daily indices. + #ilm.enabled: false + #ilm.rollover_alias: "metricbeat" + #ilm.pattern: "{now/d}-000001" + # Set gzip compression level. #compression_level: 0 diff --git a/metricbeat/metricbeat.yml b/metricbeat/metricbeat.yml index 2ff8c47f6e42..aa4765e82205 100644 --- a/metricbeat/metricbeat.yml +++ b/metricbeat/metricbeat.yml @@ -93,6 +93,9 @@ output.elasticsearch: # Array of hosts to connect to. hosts: ["localhost:9200"] + # Enabled ilm (beta) to use index lifecycle management instead daily indices. + #ilm.enabled: false + # Optional protocol and basic auth credentials. #protocol: "https" #username: "elastic" diff --git a/packetbeat/packetbeat.reference.yml b/packetbeat/packetbeat.reference.yml index 8ce05f35094c..f67837cabb09 100644 --- a/packetbeat/packetbeat.reference.yml +++ b/packetbeat/packetbeat.reference.yml @@ -734,6 +734,11 @@ output.elasticsearch: # IPv6 addresses should always be defined as: https://[2001:db8::1]:9200 hosts: ["localhost:9200"] + # Enabled ilm (beta) to use index lifecycle management instead daily indices. + #ilm.enabled: false + #ilm.rollover_alias: "packetbeat" + #ilm.pattern: "{now/d}-000001" + # Set gzip compression level. #compression_level: 0 diff --git a/packetbeat/packetbeat.yml b/packetbeat/packetbeat.yml index 284af2be4a50..36b2d1142955 100644 --- a/packetbeat/packetbeat.yml +++ b/packetbeat/packetbeat.yml @@ -176,6 +176,9 @@ output.elasticsearch: # Array of hosts to connect to. hosts: ["localhost:9200"] + # Enabled ilm (beta) to use index lifecycle management instead daily indices. + #ilm.enabled: false + # Optional protocol and basic auth credentials. #protocol: "https" #username: "elastic" diff --git a/winlogbeat/winlogbeat.reference.yml b/winlogbeat/winlogbeat.reference.yml index e4ad21717944..449d2bed9267 100644 --- a/winlogbeat/winlogbeat.reference.yml +++ b/winlogbeat/winlogbeat.reference.yml @@ -274,6 +274,11 @@ output.elasticsearch: # IPv6 addresses should always be defined as: https://[2001:db8::1]:9200 hosts: ["localhost:9200"] + # Enabled ilm (beta) to use index lifecycle management instead daily indices. + #ilm.enabled: false + #ilm.rollover_alias: "winlogbeat" + #ilm.pattern: "{now/d}-000001" + # Set gzip compression level. #compression_level: 0 diff --git a/winlogbeat/winlogbeat.yml b/winlogbeat/winlogbeat.yml index 3b26bf8d3a93..7661b770cda0 100644 --- a/winlogbeat/winlogbeat.yml +++ b/winlogbeat/winlogbeat.yml @@ -97,6 +97,9 @@ output.elasticsearch: # Array of hosts to connect to. hosts: ["localhost:9200"] + # Enabled ilm (beta) to use index lifecycle management instead daily indices. + #ilm.enabled: false + # Optional protocol and basic auth credentials. #protocol: "https" #username: "elastic" diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index eb361dfdf18e..294c9b42f1a6 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -1041,6 +1041,11 @@ output.elasticsearch: # IPv6 addresses should always be defined as: https://[2001:db8::1]:9200 hosts: ["localhost:9200"] + # Enabled ilm (beta) to use index lifecycle management instead daily indices. + #ilm.enabled: false + #ilm.rollover_alias: "filebeat" + #ilm.pattern: "{now/d}-000001" + # Set gzip compression level. #compression_level: 0 diff --git a/x-pack/filebeat/filebeat.yml b/x-pack/filebeat/filebeat.yml index 5de1a4d1e55c..78467764cc94 100644 --- a/x-pack/filebeat/filebeat.yml +++ b/x-pack/filebeat/filebeat.yml @@ -149,6 +149,9 @@ output.elasticsearch: # Array of hosts to connect to. hosts: ["localhost:9200"] + # Enabled ilm (beta) to use index lifecycle management instead daily indices. + #ilm.enabled: false + # Optional protocol and basic auth credentials. #protocol: "https" #username: "elastic" diff --git a/x-pack/functionbeat/functionbeat.reference.yml b/x-pack/functionbeat/functionbeat.reference.yml index 86a96abdc52f..6ac195923d6a 100644 --- a/x-pack/functionbeat/functionbeat.reference.yml +++ b/x-pack/functionbeat/functionbeat.reference.yml @@ -339,6 +339,11 @@ output.elasticsearch: # IPv6 addresses should always be defined as: https://[2001:db8::1]:9200 hosts: ["localhost:9200"] + # Enabled ilm (beta) to use index lifecycle management instead daily indices. + #ilm.enabled: false + #ilm.rollover_alias: "functionbeat" + #ilm.pattern: "{now/d}-000001" + # Set gzip compression level. #compression_level: 0 diff --git a/x-pack/functionbeat/functionbeat.yml b/x-pack/functionbeat/functionbeat.yml index 57e02e52010f..d595636e2252 100644 --- a/x-pack/functionbeat/functionbeat.yml +++ b/x-pack/functionbeat/functionbeat.yml @@ -157,6 +157,9 @@ output.elasticsearch: # Array of hosts to connect to. hosts: ["localhost:9200"] + # Enabled ilm (beta) to use index lifecycle management instead daily indices. + #ilm.enabled: false + # Optional protocol and basic auth credentials. #protocol: "https" #username: "elastic" diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index dc7162238060..f6b7ce9cf927 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -938,6 +938,11 @@ output.elasticsearch: # IPv6 addresses should always be defined as: https://[2001:db8::1]:9200 hosts: ["localhost:9200"] + # Enabled ilm (beta) to use index lifecycle management instead daily indices. + #ilm.enabled: false + #ilm.rollover_alias: "metricbeat" + #ilm.pattern: "{now/d}-000001" + # Set gzip compression level. #compression_level: 0 diff --git a/x-pack/metricbeat/metricbeat.yml b/x-pack/metricbeat/metricbeat.yml index 2ff8c47f6e42..aa4765e82205 100644 --- a/x-pack/metricbeat/metricbeat.yml +++ b/x-pack/metricbeat/metricbeat.yml @@ -93,6 +93,9 @@ output.elasticsearch: # Array of hosts to connect to. hosts: ["localhost:9200"] + # Enabled ilm (beta) to use index lifecycle management instead daily indices. + #ilm.enabled: false + # Optional protocol and basic auth credentials. #protocol: "https" #username: "elastic"