Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Cherry-pick #7963 to 6.x: Add ILM support to Beats #9418

Merged
merged 3 commits into from
Dec 7, 2018
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.asciidoc
Original file line number Diff line number Diff line change
@@ -84,6 +84,7 @@ https://github.com/elastic/beats/compare/v6.5.0...6.x[Check the HEAD diff]
- Dissect will now flag event on parsing error. {pull}8751[8751]
- Added the `redirect_stderr` option that allows panics to be logged to log files. {pull}8430[8430]
- Add cache.ttl to add_host_metadata. {pull}9359[9359]
- Add support for index lifecycle management (beta). {pull}7963[7963]

*Auditbeat*

5 changes: 5 additions & 0 deletions auditbeat/auditbeat.reference.yml
Original file line number Diff line number Diff line change
@@ -352,6 +352,11 @@ output.elasticsearch:
# IPv6 addresses should always be defined as: https://[2001:db8::1]:9200
hosts: ["localhost:9200"]

# Enabled ilm (beta) to use index lifecycle management instead daily indices.
#ilm.enabled: false
#ilm.rollover_alias: "auditbeat"
#ilm.pattern: "{now/d}-000001"

# Set gzip compression level.
#compression_level: 0

3 changes: 3 additions & 0 deletions auditbeat/auditbeat.yml
Original file line number Diff line number Diff line change
@@ -122,6 +122,9 @@ output.elasticsearch:
# Array of hosts to connect to.
hosts: ["localhost:9200"]

# Enabled ilm (beta) to use index lifecycle management instead daily indices.
#ilm.enabled: false

# Optional protocol and basic auth credentials.
#protocol: "https"
#username: "elastic"
5 changes: 5 additions & 0 deletions filebeat/filebeat.reference.yml
Original file line number Diff line number Diff line change
@@ -1031,6 +1031,11 @@ output.elasticsearch:
# IPv6 addresses should always be defined as: https://[2001:db8::1]:9200
hosts: ["localhost:9200"]

# Enabled ilm (beta) to use index lifecycle management instead daily indices.
#ilm.enabled: false
#ilm.rollover_alias: "filebeat"
#ilm.pattern: "{now/d}-000001"

# Set gzip compression level.
#compression_level: 0

3 changes: 3 additions & 0 deletions filebeat/filebeat.yml
Original file line number Diff line number Diff line change
@@ -149,6 +149,9 @@ output.elasticsearch:
# Array of hosts to connect to.
hosts: ["localhost:9200"]

# Enabled ilm (beta) to use index lifecycle management instead daily indices.
#ilm.enabled: false

# Optional protocol and basic auth credentials.
#protocol: "https"
#username: "elastic"
5 changes: 5 additions & 0 deletions heartbeat/heartbeat.reference.yml
Original file line number Diff line number Diff line change
@@ -490,6 +490,11 @@ output.elasticsearch:
# IPv6 addresses should always be defined as: https://[2001:db8::1]:9200
hosts: ["localhost:9200"]

# Enabled ilm (beta) to use index lifecycle management instead daily indices.
#ilm.enabled: false
#ilm.rollover_alias: "heartbeat"
#ilm.pattern: "{now/d}-000001"

# Set gzip compression level.
#compression_level: 0

3 changes: 3 additions & 0 deletions heartbeat/heartbeat.yml
Original file line number Diff line number Diff line change
@@ -96,6 +96,9 @@ output.elasticsearch:
# Array of hosts to connect to.
hosts: ["localhost:9200"]

# Enabled ilm (beta) to use index lifecycle management instead daily indices.
#ilm.enabled: false

# Optional protocol and basic auth credentials.
#protocol: "https"
#username: "elastic"
5 changes: 5 additions & 0 deletions journalbeat/journalbeat.reference.yml
Original file line number Diff line number Diff line change
@@ -303,6 +303,11 @@ output.elasticsearch:
# IPv6 addresses should always be defined as: https://[2001:db8::1]:9200
hosts: ["localhost:9200"]

# Enabled ilm (beta) to use index lifecycle management instead daily indices.
#ilm.enabled: false
#ilm.rollover_alias: "journalbeat"
#ilm.pattern: "{now/d}-000001"

# Set gzip compression level.
#compression_level: 0

3 changes: 3 additions & 0 deletions journalbeat/journalbeat.yml
Original file line number Diff line number Diff line change
@@ -124,6 +124,9 @@ output.elasticsearch:
# Array of hosts to connect to.
hosts: ["localhost:9200"]

# Enabled ilm (beta) to use index lifecycle management instead daily indices.
#ilm.enabled: false

# Optional protocol and basic auth credentials.
#protocol: "https"
#username: "elastic"
5 changes: 5 additions & 0 deletions libbeat/_meta/config.reference.yml
Original file line number Diff line number Diff line change
@@ -245,6 +245,11 @@ output.elasticsearch:
# IPv6 addresses should always be defined as: https://[2001:db8::1]:9200
hosts: ["localhost:9200"]

# Enabled ilm (beta) to use index lifecycle management instead daily indices.
#ilm.enabled: false
#ilm.rollover_alias: "beat-index-prefix"
#ilm.pattern: "{now/d}-000001"

# Set gzip compression level.
#compression_level: 0

3 changes: 3 additions & 0 deletions libbeat/_meta/config.yml
Original file line number Diff line number Diff line change
@@ -66,6 +66,9 @@ output.elasticsearch:
# Array of hosts to connect to.
hosts: ["localhost:9200"]

# Enabled ilm (beta) to use index lifecycle management instead daily indices.
#ilm.enabled: false

# Optional protocol and basic auth credentials.
#protocol: "https"
#username: "elastic"
1 change: 1 addition & 0 deletions libbeat/cmd/export.go
Original file line number Diff line number Diff line change
@@ -33,6 +33,7 @@ func genExportCmd(settings instance.Settings, name, idxPrefix, beatVersion strin
exportCmd.AddCommand(export.GenExportConfigCmd(settings, name, idxPrefix, beatVersion))
exportCmd.AddCommand(export.GenTemplateConfigCmd(settings, name, idxPrefix, beatVersion))
exportCmd.AddCommand(export.GenDashboardCmd(name, idxPrefix, beatVersion))
exportCmd.AddCommand(export.GenGetILMPolicyCmd())

return exportCmd
}
39 changes: 39 additions & 0 deletions libbeat/cmd/export/ilm_policy.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
// Licensed to Elasticsearch B.V. under one or more contributor
// license agreements. See the NOTICE file distributed with
// this work for additional information regarding copyright
// ownership. Elasticsearch B.V. licenses this file to you under
// the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

package export

import (
"fmt"

"github.com/spf13/cobra"

"github.com/elastic/beats/libbeat/cmd/instance"
)

// GenGetILMPolicyCmd is the command used to export the ilm policy.
func GenGetILMPolicyCmd() *cobra.Command {
genTemplateConfigCmd := &cobra.Command{
Use: "ilm-policy",
Short: "Export ILM policy",
Run: func(cmd *cobra.Command, args []string) {
fmt.Println(instance.ILMPolicy.StringToPrint())
},
}

return genTemplateConfigCmd
}
96 changes: 91 additions & 5 deletions libbeat/cmd/instance/beat.go
Original file line number Diff line number Diff line change
@@ -105,6 +105,9 @@ type beatConfig struct {
Dashboards *common.Config `config:"setup.dashboards"`
Template *common.Config `config:"setup.template"`
Kibana *common.Config `config:"setup.kibana"`

// ILM Config options
ILM *common.Config `config:"output.elasticsearch.ilm"`
}

var (
@@ -430,7 +433,7 @@ func (b *Beat) TestConfig(bt beat.Creator) error {
}

// Setup registers ES index template, kibana dashboards, ml jobs and pipelines.
func (b *Beat) Setup(bt beat.Creator, template, setupDashboards, machineLearning, pipelines bool) error {
func (b *Beat) Setup(bt beat.Creator, template, setupDashboards, machineLearning, pipelines, policy bool) error {
return handleError(func() error {
err := b.Init()
if err != nil {
@@ -509,6 +512,13 @@ func (b *Beat) Setup(bt beat.Creator, template, setupDashboards, machineLearning
fmt.Println("Loaded Ingest pipelines")
}

if policy {
if err := b.loadILMPolicy(); err != nil {
return err
}
fmt.Println("Loaded Index Lifecycle Management (ILM) policy")
}

return nil
}())
}
@@ -724,11 +734,11 @@ func (b *Beat) loadDashboards(ctx context.Context, force bool) error {
// the elasticsearch output. It is important the the registration happens before
// the publisher is created.
func (b *Beat) registerTemplateLoading() error {
var cfg template.TemplateConfig
var templateCfg template.TemplateConfig

// Check if outputting to file is enabled, and output to file if it is
if b.Config.Template.Enabled() {
err := b.Config.Template.Unpack(&cfg)
err := b.Config.Template.Unpack(&templateCfg)
if err != nil {
return fmt.Errorf("unpacking template config fails: %v", err)
}
@@ -746,8 +756,82 @@ func (b *Beat) registerTemplateLoading() error {
return err
}

if esCfg.Index != "" && (cfg.Name == "" || cfg.Pattern == "") && (b.Config.Template == nil || b.Config.Template.Enabled()) {
return fmt.Errorf("setup.template.name and setup.template.pattern have to be set if index name is modified.")
if esCfg.Index != "" &&
(templateCfg.Name == "" || templateCfg.Pattern == "") &&
(b.Config.Template == nil || b.Config.Template.Enabled()) {
return errors.New("setup.template.name and setup.template.pattern have to be set if index name is modified")
}

if b.Config.ILM.Enabled() {
cfgwarn.Beta("Index lifecycle management is enabled which is in beta.")

ilmCfg, err := getILMConfig(b)
if err != nil {
return err
}

// In case no template settings are set, config must be created
if b.Config.Template == nil {
b.Config.Template = common.NewConfig()
}
// Template name and pattern can't be configure when using ILM
logp.Info("Set setup.template.name to '%s' as ILM is enabled.", ilmCfg.RolloverAlias)
err = b.Config.Template.SetString("name", -1, ilmCfg.RolloverAlias)
if err != nil {
return errw.Wrap(err, "error setting setup.template.name")
}
pattern := fmt.Sprintf("%s-*", ilmCfg.RolloverAlias)
logp.Info("Set setup.template.pattern to '%s' as ILM is enabled.", pattern)
err = b.Config.Template.SetString("pattern", -1, pattern)
if err != nil {
return errw.Wrap(err, "error setting setup.template.pattern")
}

// rollover_alias and lifecycle.name can't be configured and will be overwritten
logp.Info("Set settings.index.lifecycle.rollover_alias in template to %s as ILM is enabled.", ilmCfg.RolloverAlias)
err = b.Config.Template.SetString("settings.index.lifecycle.rollover_alias", -1, ilmCfg.RolloverAlias)
if err != nil {
return errw.Wrap(err, "error setting settings.index.lifecycle.rollover_alias")
}
logp.Info("Set settings.index.lifecycle.name in template to %s as ILM is enabled.", ILMPolicyName)
err = b.Config.Template.SetString("settings.index.lifecycle.name", -1, ILMPolicyName)
if err != nil {
return errw.Wrap(err, "error setting settings.index.lifecycle.name")
}

// Set the ingestion index to the rollover alias
logp.Info("Set output.elasticsearch.index to '%s' as ILM is enabled.", ilmCfg.RolloverAlias)
esCfg.Index = ilmCfg.RolloverAlias
err = b.Config.Output.Config().SetString("index", -1, ilmCfg.RolloverAlias)
if err != nil {
return errw.Wrap(err, "error setting output.elasticsearch.index")
}

writeAliasCallback, err := b.writeAliasLoadingCallback()
if err != nil {
return err
}

// Load write alias already on
esConfig := b.Config.Output.Config()

// Check that ILM is enabled and the right elasticsearch version exists
esClient, err := elasticsearch.NewConnectedClient(esConfig)
if err != nil {
return err
}

err = checkElasticsearchVersionIlm(esClient)
if err != nil {
return err
}

err = checkILMFeatureEnabled(esClient)
if err != nil {
return err
}

elasticsearch.RegisterConnectCallback(writeAliasCallback)
}

if b.Config.Template == nil || (b.Config.Template != nil && b.Config.Template.Enabled()) {
@@ -759,6 +843,8 @@ func (b *Beat) registerTemplateLoading() error {
return err
}
elasticsearch.RegisterConnectCallback(callback)
} else if b.Config.ILM.Enabled() {
return errors.New("templates cannot be disable when using ILM")
}
}

Loading