diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 24779b6bb0b..6857f7a1056 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -226,7 +226,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...v7.0.0-beta1[Check the - The `elasticsearch/deprecation` fileset now indexes the `component` field under `elasticsearch` instead of `elasticsearch.server`. {pull}10445[10445] - Remove field `kafka.log.trace.full` from kafka.log fielset. {pull}10398[10398] - Change field `kafka.log.class` for kafka.log fileset from text to keyword. {pull}10398[10398] -- Address add_kubernetes_metadata processor issue where old source field is +- Address add_kubernetes_metadata processor issue where old source field is still used for matcher. {issue}10505[10505] {pull}10506[10506] - Change type of haproxy.source from text to keyword. {pull}10506[10506] - Rename `event.type` to `suricata.eve.event_type` in Suricata module because event.type is reserved for future use by ECS. {pull}10575[10575] diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 9158760ab50..6d7ac66affa 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -60,7 +60,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Improve detection of file deletion on Windows. {pull}10747[10747] - Fix goroutine leak happening when harvesters are dynamically stopped. {pull}11263[11263] - Fix `add_docker_metadata` source matching, using `log.file.path` field now. {pull}11577[11577] -- Add missing Kubernetes metadata fields to Filebeat CoreDNS module, and fix a documentation error. {pull}11591[11591] +- Add missing Kubernetes metadata fields to Filebeat CoreDNS module, and fix a documentation error. {pull}11591[11591] *Heartbeat* @@ -120,6 +120,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add `coredns` metricbeat module. {pull}10585[10585] - Add SSL support for Metricbeat HTTP server. {pull}11482[11482] {issue}11457[11457] - The `elasticsearch.index` metricset (with `xpack.enabled: true`) now collects `refresh.external_total_time_in_millis` fields from Elasticsearch. {pull}11616[11616] +- Allow module configurations to have variants {pull}9118[9118] *Packetbeat* diff --git a/libbeat/cfgfile/glob_manager.go b/libbeat/cfgfile/glob_manager.go index 0a90357b204..f99790640a2 100644 --- a/libbeat/cfgfile/glob_manager.go +++ b/libbeat/cfgfile/glob_manager.go @@ -20,6 +20,7 @@ package cfgfile import ( "os" "path/filepath" + "sort" "strings" "github.com/pkg/errors" @@ -110,6 +111,7 @@ func (g *GlobManager) ListEnabled() []*CfgFile { } } + sort.Sort(byCfgFileDisplayNames(enabled)) return enabled } @@ -122,6 +124,7 @@ func (g *GlobManager) ListDisabled() []*CfgFile { } } + sort.Sort(byCfgFileDisplayNames(disabled)) return disabled } @@ -182,3 +185,31 @@ func (g *GlobManager) Disable(name string) error { return errors.Errorf("module %s not found", name) } + +// For sorting config files in the desired order, so variants will +// show up after the default, e.g. elasticsearch-xpack will show up +// after elasticsearch. +type byCfgFileDisplayNames []*CfgFile + +func (s byCfgFileDisplayNames) Len() int { + return len(s) +} + +func (s byCfgFileDisplayNames) Swap(i, j int) { + s[i], s[j] = s[j], s[i] +} + +func (s byCfgFileDisplayNames) Less(i, j int) bool { + namei := s[i].Name + namej := s[j].Name + + if strings.HasPrefix(namei, namej) { + // namei starts with namej, so namei is longer and we want it to come after namej + return false + } else if strings.HasPrefix(namej, namei) { + // namej starts with namei, so namej is longer and we want it to come after namei + return true + } else { + return namei < namej + } +} diff --git a/libbeat/cfgfile/glob_manager_test.go b/libbeat/cfgfile/glob_manager_test.go index c2816b5e01d..808f1bc3394 100644 --- a/libbeat/cfgfile/glob_manager_test.go +++ b/libbeat/cfgfile/glob_manager_test.go @@ -47,6 +47,8 @@ func TestGlobManager(t *testing.T) { assert.NoError(t, err) err = ioutil.WriteFile(dir+"/config2.yml", content, 0644) assert.NoError(t, err) + err = ioutil.WriteFile(dir+"/config2-alt.yml.disabled", content, 0644) + assert.NoError(t, err) err = ioutil.WriteFile(dir+"/config3.yml.disabled", content, 0644) assert.NoError(t, err) @@ -59,15 +61,17 @@ func TestGlobManager(t *testing.T) { assert.True(t, manager.Exists("config1")) assert.True(t, manager.Exists("config2")) + assert.True(t, manager.Exists("config2-alt")) assert.True(t, manager.Exists("config3")) assert.False(t, manager.Exists("config4")) assert.True(t, manager.Enabled("config1")) assert.True(t, manager.Enabled("config2")) + assert.False(t, manager.Enabled("config2-alt")) assert.False(t, manager.Enabled("config3")) assert.Equal(t, len(manager.ListEnabled()), 2) - assert.Equal(t, len(manager.ListDisabled()), 1) + assert.Equal(t, len(manager.ListDisabled()), 2) // Test disable if err = manager.Disable("config2"); err != nil { @@ -75,7 +79,7 @@ func TestGlobManager(t *testing.T) { } assert.Equal(t, len(manager.ListEnabled()), 1) - assert.Equal(t, len(manager.ListDisabled()), 2) + assert.Equal(t, len(manager.ListDisabled()), 3) enabled := manager.ListEnabled() assert.Equal(t, enabled[0].Name, "config1") @@ -87,11 +91,13 @@ func TestGlobManager(t *testing.T) { } assert.Equal(t, len(manager.ListEnabled()), 2) - assert.Equal(t, len(manager.ListDisabled()), 1) + assert.Equal(t, len(manager.ListDisabled()), 2) disabled := manager.ListDisabled() assert.Equal(t, disabled[0].Name, "config2") assert.Equal(t, disabled[0].Enabled, false) + assert.Equal(t, disabled[1].Name, "config2-alt") + assert.Equal(t, disabled[1].Enabled, false) // Check correct files layout: files, err := filepath.Glob(dir + "/*") @@ -101,7 +107,32 @@ func TestGlobManager(t *testing.T) { assert.Equal(t, files, []string{ filepath.Join(dir, "config1.yml"), + filepath.Join(dir, "config2-alt.yml.disabled"), filepath.Join(dir, "config2.yml.disabled"), filepath.Join(dir, "config3.yml"), }) } + +func TestCfgFileSorting(t *testing.T) { + cfgFiles := byCfgFileDisplayNames{ + &CfgFile{ + "foo", + "modules.d/foo.yml", + false, + }, + &CfgFile{ + "foo-variant", + "modules.d/foo-variant.yml", + false, + }, + &CfgFile{ + "fox", + "modules.d/fox.yml", + false, + }, + } + + assert.True(t, cfgFiles.Less(0, 1)) + assert.False(t, cfgFiles.Less(1, 0)) + assert.True(t, cfgFiles.Less(0, 2)) +} diff --git a/metricbeat/docs/metricbeat-options.asciidoc b/metricbeat/docs/metricbeat-options.asciidoc index 6b0d39fbc74..6ab2a311201 100644 --- a/metricbeat/docs/metricbeat-options.asciidoc +++ b/metricbeat/docs/metricbeat-options.asciidoc @@ -113,6 +113,20 @@ metricbeat.modules: period: 30s ---- +[float] +[[config-variants]] +== Configuration variants + +Every module come with a default configuration file. Some modules also come with +one or more variant configuration files containing common alternative configurations +for that module. + +When you see the list of enabled and disabled modules, those modules with configuration +variants will be shown as `-`. You can enable or disable +specific configuration variants of a module by specifying `metricbeat modules enable +-` and `metricbeat modules disable -` +respectively. + [float] [[config-combos]] == Configuration combinations diff --git a/metricbeat/module/elasticsearch/_meta/config-xpack.yml b/metricbeat/module/elasticsearch/_meta/config-xpack.yml new file mode 100644 index 00000000000..982fbf3bf1f --- /dev/null +++ b/metricbeat/module/elasticsearch/_meta/config-xpack.yml @@ -0,0 +1,16 @@ +- module: elasticsearch + metricsets: + - ccr + - cluster_stats + - index + - index_recovery + - index_summary + - ml_job + - node_stats + - shard + period: 10s + hosts: ["http://localhost:9200"] + #username: "user" + #password: "secret" + xpack.enabled: true + diff --git a/metricbeat/module/kibana/_meta/config-xpack.yml b/metricbeat/module/kibana/_meta/config-xpack.yml new file mode 100644 index 00000000000..ac67948c7ab --- /dev/null +++ b/metricbeat/module/kibana/_meta/config-xpack.yml @@ -0,0 +1,9 @@ +- module: kibana + metricsets: + - stats + period: 10s + hosts: ["localhost:5601"] + #basepath: "" + #username: "user" + #password: "secret" + xpack.enabled: true diff --git a/metricbeat/modules.d/elasticsearch-xpack.yml.disabled b/metricbeat/modules.d/elasticsearch-xpack.yml.disabled new file mode 100644 index 00000000000..b542e7f1247 --- /dev/null +++ b/metricbeat/modules.d/elasticsearch-xpack.yml.disabled @@ -0,0 +1,19 @@ +# Module: elasticsearch +# Docs: https://www.elastic.co/guide/en/beats/metricbeat/master/metricbeat-module-elasticsearch.html + +- module: elasticsearch + metricsets: + - ccr + - cluster_stats + - index + - index_recovery + - index_summary + - ml_job + - node_stats + - shard + period: 10s + hosts: ["http://localhost:9200"] + #username: "user" + #password: "secret" + xpack.enabled: true + diff --git a/metricbeat/modules.d/kibana-xpack.yml.disabled b/metricbeat/modules.d/kibana-xpack.yml.disabled new file mode 100644 index 00000000000..84675eeb340 --- /dev/null +++ b/metricbeat/modules.d/kibana-xpack.yml.disabled @@ -0,0 +1,12 @@ +# Module: kibana +# Docs: https://www.elastic.co/guide/en/beats/metricbeat/master/metricbeat-module-kibana.html + +- module: kibana + metricsets: + - stats + period: 10s + hosts: ["localhost:5601"] + #basepath: "" + #username: "user" + #password: "secret" + xpack.enabled: true