From 267c76e1ba0c3c351eef50d79089dd154c9f6120 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20P=C3=A9rez-Aradros=20Herce?= Date: Wed, 12 Dec 2018 02:26:38 +0100 Subject: [PATCH] Add Central Management blacklisting (#9099) * Add Central Management blacklisting This change allows to define local blacklist for configurations coming from Central Management. If a configuration block matches the given regular expession, it will be ignored: For example: ``` management: blacklist: output: console metricbeat.modules.module: k.{18}s ``` --- x-pack/libbeat/management/blacklist.go | 166 +++++++++++ x-pack/libbeat/management/blacklist_test.go | 280 ++++++++++++++++++ x-pack/libbeat/management/config.go | 7 + x-pack/libbeat/management/manager.go | 50 +++- .../libbeat/tests/system/test_management.py | 49 ++- 5 files changed, 534 insertions(+), 18 deletions(-) create mode 100644 x-pack/libbeat/management/blacklist.go create mode 100644 x-pack/libbeat/management/blacklist_test.go diff --git a/x-pack/libbeat/management/blacklist.go b/x-pack/libbeat/management/blacklist.go new file mode 100644 index 00000000000..677faf9ac48 --- /dev/null +++ b/x-pack/libbeat/management/blacklist.go @@ -0,0 +1,166 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package management + +import ( + "fmt" + "regexp" + "strings" + + "github.com/joeshaw/multierror" + "github.com/pkg/errors" + + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/x-pack/libbeat/management/api" +) + +// ConfigBlacklist takes a ConfigBlocks object and filter it based on the given +// blacklist settings +type ConfigBlacklist struct { + patterns map[string]*regexp.Regexp +} + +// ConfigBlacklistSettings holds a list of fields and regular expressions to blacklist +type ConfigBlacklistSettings struct { + Patterns map[string]string `yaml:",inline"` +} + +// Unpack unpacks nested fields set with dot notation like foo.bar into the proper nesting +// in a nested map/slice structure. +func (f *ConfigBlacklistSettings) Unpack(from interface{}) error { + m, ok := from.(map[string]interface{}) + if !ok { + return fmt.Errorf("wrong type, map is expected") + } + + f.Patterns = map[string]string{} + for k, v := range common.MapStr(m).Flatten() { + f.Patterns[k] = fmt.Sprintf("%s", v) + } + + return nil +} + +// NewConfigBlacklist filters configs from CM according to a given blacklist +func NewConfigBlacklist(cfg ConfigBlacklistSettings) (*ConfigBlacklist, error) { + list := ConfigBlacklist{ + patterns: map[string]*regexp.Regexp{}, + } + + for field, pattern := range cfg.Patterns { + exp, err := regexp.Compile(pattern) + if err != nil { + return nil, errors.Wrap(err, fmt.Sprintf("Given expression is not a valid regexp: %s", pattern)) + } + + list.patterns[field] = exp + } + + return &list, nil +} + +// Filter an error if any of the given config blocks is blacklisted +func (c *ConfigBlacklist) Filter(configBlocks api.ConfigBlocks) error { + var errs multierror.Errors + + for _, configs := range configBlocks { + for _, block := range configs.Blocks { + if c.isBlacklisted(configs.Type, block) { + errs = append(errs, fmt.Errorf("Config for '%s' is blacklisted", configs.Type)) + } + } + } + + return errs.Err() +} + +func (c *ConfigBlacklist) isBlacklisted(blockType string, block *api.ConfigBlock) bool { + cfg, err := block.ConfigWithMeta() + if err != nil { + return false + } + + for field, pattern := range c.patterns { + prefix := blockType + if strings.Contains(field, ".") { + prefix += "." + } + + if strings.HasPrefix(field, prefix) { + // This pattern affects a field on this block type + field = field[len(prefix):] + var segments []string + if len(field) > 0 { + segments = strings.Split(field, ".") + } + if c.isBlacklistedBlock(pattern, segments, cfg.Config) { + return true + } + } + } + + return false +} + +func (c *ConfigBlacklist) isBlacklistedBlock(pattern *regexp.Regexp, segments []string, current *common.Config) bool { + if current.IsDict() { + switch len(segments) { + case 0: + for _, field := range current.GetFields() { + if pattern.MatchString(field) { + return true + } + } + + case 1: + // Check field in the dict + val, err := current.String(segments[0], -1) + if err == nil { + return pattern.MatchString(val) + } + // not a string, traverse + child, _ := current.Child(segments[0], -1) + return child != nil && c.isBlacklistedBlock(pattern, segments[1:], child) + + default: + // traverse the tree + child, _ := current.Child(segments[0], -1) + return child != nil && c.isBlacklistedBlock(pattern, segments[1:], child) + + } + } + + if current.IsArray() { + switch len(segments) { + case 0: + // List of elements, match strings + for count, _ := current.CountField(""); count > 0; count-- { + val, err := current.String("", count-1) + if err == nil && pattern.MatchString(val) { + return true + } + + // not a string, traverse + child, _ := current.Child("", count-1) + if child != nil { + if c.isBlacklistedBlock(pattern, segments, child) { + return true + } + } + } + + default: + // List of elements, explode traversal to all of them + for count, _ := current.CountField(""); count > 0; count-- { + child, _ := current.Child("", count-1) + if child != nil && c.isBlacklistedBlock(pattern, segments, child) { + return true + } + } + } + } + + return false +} diff --git a/x-pack/libbeat/management/blacklist_test.go b/x-pack/libbeat/management/blacklist_test.go new file mode 100644 index 00000000000..b6fb8cd70b3 --- /dev/null +++ b/x-pack/libbeat/management/blacklist_test.go @@ -0,0 +1,280 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package management + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/x-pack/libbeat/management/api" +) + +func TestConfigBlacklistSettingsUnpack(t *testing.T) { + tests := []struct { + name string + config *common.Config + error bool + expected ConfigBlacklistSettings + }{ + { + name: "Simple config", + config: common.MustNewConfigFrom(map[string]interface{}{ + "foo": "bar", + }), + expected: ConfigBlacklistSettings{ + Patterns: map[string]string{ + "foo": "bar", + }, + }, + }, + { + name: "Wrong config", + config: common.MustNewConfigFrom([]string{"a", "b"}), + error: true, + }, + { + name: "Tree config", + config: common.MustNewConfigFrom(map[string]interface{}{ + "foo": map[string]interface{}{ + "bar": "baz", + }, + }), + expected: ConfigBlacklistSettings{ + Patterns: map[string]string{ + "foo.bar": "baz", + }, + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + var result ConfigBlacklistSettings + err := test.config.Unpack(&result) + if test.error { + assert.Error(t, err) + } + assert.Equal(t, test.expected, result) + }) + } +} + +func TestConfigBlacklist(t *testing.T) { + tests := []struct { + name string + patterns map[string]string + blocks api.ConfigBlocks + blacklisted bool + }{ + { + name: "No patterns", + blacklisted: false, + blocks: api.ConfigBlocks{ + api.ConfigBlocksWithType{ + Type: "output", + Blocks: []*api.ConfigBlock{ + { + Raw: map[string]interface{}{ + "output": "console", + }, + }, + }, + }, + }, + }, + { + name: "Blacklisted dict key", + blacklisted: true, + patterns: map[string]string{ + "output": "^console$", + }, + blocks: api.ConfigBlocks{ + api.ConfigBlocksWithType{ + Type: "output", + Blocks: []*api.ConfigBlock{ + { + Raw: map[string]interface{}{ + "console": map[string]interface{}{ + "pretty": "true", + }, + }, + }, + { + Raw: map[string]interface{}{ + "elasticsearch": map[string]interface{}{ + "host": "localhost", + }, + }, + }, + }, + }, + }, + }, + { + name: "Blacklisted value key", + blacklisted: true, + patterns: map[string]string{ + "metricbeat.modules.module": "k.{8}s", + }, + blocks: api.ConfigBlocks{ + api.ConfigBlocksWithType{ + Type: "metricbeat.modules", + Blocks: []*api.ConfigBlock{ + { + Raw: map[string]interface{}{ + "module": "kubernetes", + "hosts": "localhost:10255", + }, + }, + }, + }, + }, + }, + { + name: "Blacklisted value in a list", + blacklisted: true, + patterns: map[string]string{ + "metricbeat.modules.metricsets": "event", + }, + blocks: api.ConfigBlocks{ + api.ConfigBlocksWithType{ + Type: "metricbeat.modules", + Blocks: []*api.ConfigBlock{ + { + Raw: map[string]interface{}{ + "module": "kubernetes", + "metricsets": []string{ + "event", + "default", + }, + }, + }, + { + Raw: map[string]interface{}{ + "module": "kubernetes", + "metricsets": []string{ + "default", + }, + }, + }, + }, + }, + }, + }, + { + name: "Blacklisted value in a deep list", + blacklisted: true, + patterns: map[string]string{ + "filebeat.inputs.containers.ids": "1ffeb0dbd13", + }, + blocks: api.ConfigBlocks{ + api.ConfigBlocksWithType{ + Type: "metricbeat.modules", + Blocks: []*api.ConfigBlock{ + { + Raw: map[string]interface{}{ + "module": "kubernetes", + "metricsets": []string{ + "event", + "default", + }, + }, + }, + }, + }, + api.ConfigBlocksWithType{ + Type: "filebeat.inputs", + Blocks: []*api.ConfigBlock{ + { + Raw: map[string]interface{}{ + "type": "docker", + "containers": map[string]interface{}{ + "ids": []string{ + "1ffeb0dbd13", + }, + }, + }, + }, + { + Raw: map[string]interface{}{ + "type": "docker", + "containers": map[string]interface{}{ + "ids": []string{ + "256425931c2", + }, + }, + }, + }, + }, + }, + }, + }, + { + name: "Blacklisted dict key in a list", + blacklisted: true, + patterns: map[string]string{ + "list.of.elements": "forbidden", + "list.of.elements.disallowed": "yes", + }, + blocks: api.ConfigBlocks{ + api.ConfigBlocksWithType{ + Type: "list", + Blocks: []*api.ConfigBlock{ + { + Raw: map[string]interface{}{ + "of": map[string]interface{}{ + "elements": []interface{}{ + map[string]interface{}{ + "forbidden": "yes", + }, + }, + }, + }, + }, + { + Raw: map[string]interface{}{ + "of": map[string]interface{}{ + "elements": []interface{}{ + map[string]interface{}{ + "allowed": "yes", + }, + }, + }, + }, + }, + { + Raw: map[string]interface{}{ + "of": map[string]interface{}{ + "elements": []interface{}{ + map[string]interface{}{ + "disallowed": "yes", + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + cfg := ConfigBlacklistSettings{ + Patterns: test.patterns, + } + bl, err := NewConfigBlacklist(cfg) + if err != nil { + t.Fatal(err) + } + + err = bl.Filter(test.blocks) + assert.Equal(t, test.blacklisted, err != nil) + }) + } +} diff --git a/x-pack/libbeat/management/config.go b/x-pack/libbeat/management/config.go index 552525cf6b3..3781938f59c 100644 --- a/x-pack/libbeat/management/config.go +++ b/x-pack/libbeat/management/config.go @@ -77,11 +77,18 @@ type Config struct { AccessToken string `config:"access_token" yaml:"access_token"` Kibana *kibana.ClientConfig `config:"kibana" yaml:"kibana"` + + Blacklist ConfigBlacklistSettings `config:"blacklist" yaml:"blacklist"` } func defaultConfig() *Config { return &Config{ Period: 60 * time.Second, + Blacklist: ConfigBlacklistSettings{ + Patterns: map[string]string{ + "output": "console|file", + }, + }, } } diff --git a/x-pack/libbeat/management/manager.go b/x-pack/libbeat/management/manager.go index 9c494fd8996..81aff95bc5d 100644 --- a/x-pack/libbeat/management/manager.go +++ b/x-pack/libbeat/management/manager.go @@ -31,14 +31,15 @@ func init() { // ConfigManager handles internal config updates. By retrieving // new configs from Kibana and applying them to the Beat type ConfigManager struct { - config *Config - cache *Cache - logger *logp.Logger - client *api.Client - beatUUID uuid.UUID - done chan struct{} - registry *reload.Registry - wg sync.WaitGroup + config *Config + cache *Cache + logger *logp.Logger + client *api.Client + beatUUID uuid.UUID + done chan struct{} + registry *reload.Registry + wg sync.WaitGroup + blacklist *ConfigBlacklist } // NewConfigManager returns a X-Pack Beats Central Management manager @@ -56,9 +57,17 @@ func NewConfigManager(config *common.Config, registry *reload.Registry, beatUUID func NewConfigManagerWithConfig(c *Config, registry *reload.Registry, beatUUID uuid.UUID) (management.ConfigManager, error) { var client *api.Client var cache *Cache + var blacklist *ConfigBlacklist + if c.Enabled { var err error + // Initialize configs blacklist + blacklist, err = NewConfigBlacklist(c.Blacklist) + if err != nil { + return nil, errors.Wrap(err, "wrong settings for configurations blacklist") + } + // Initialize central management settings cache cache = &Cache{ ConfigOK: true, @@ -74,16 +83,18 @@ func NewConfigManagerWithConfig(c *Config, registry *reload.Registry, beatUUID u if err != nil { return nil, errors.Wrap(err, "initializing kibana client") } + } return &ConfigManager{ - config: c, - cache: cache, - logger: logp.NewLogger(management.DebugK), - client: client, - done: make(chan struct{}), - beatUUID: beatUUID, - registry: registry, + config: c, + cache: cache, + blacklist: blacklist, + logger: logp.NewLogger(management.DebugK), + client: client, + done: make(chan struct{}), + beatUUID: beatUUID, + registry: registry, }, nil } @@ -187,6 +198,13 @@ func (cm *ConfigManager) apply() { missing[name] = true } + // Filter unwanted configs from the list + errors := cm.blacklist.Filter(cm.cache.Configs) + if errors != nil { + cm.logger.Error(errors) + return + } + // Reload configs for _, b := range cm.cache.Configs { err := cm.reload(b.Type, b.Blocks) @@ -202,7 +220,7 @@ func (cm *ConfigManager) apply() { } if !configOK { - logp.Info("Failed to apply settings, reporting error on next fetch") + cm.logger.Info("Failed to apply settings, reporting error on next fetch") } // Update configOK flag with the result of this apply diff --git a/x-pack/libbeat/tests/system/test_management.py b/x-pack/libbeat/tests/system/test_management.py index dee3b133dea..0345d1ce8e4 100644 --- a/x-pack/libbeat/tests/system/test_management.py +++ b/x-pack/libbeat/tests/system/test_management.py @@ -93,7 +93,11 @@ def test_fetch_configs(self): ]) # Start beat - proc = self.start_beat(extra_args=["-E", "management.period=1s"]) + proc = self.start_beat(extra_args=[ + "-E", "management.period=1s", + # do not blacklist file/elasticsearch outputs + "-E", "management.blacklist.output='foo'", + ]) # Wait for beat to apply new conf self.wait_log_contains("Applying settings for output") @@ -151,7 +155,10 @@ def test_configs_cache(self): output_file = os.path.join("output", "mockbeat_managed") # Start beat - proc = self.start_beat() + proc = self.start_beat(extra_args=[ + # do not blacklist file output + "-E", "management.blacklist.output='elasticsearch'", + ]) self.wait_until(cond=lambda: self.output_has( 1, output_file=output_file)) proc.check_kill_and_wait() @@ -163,11 +170,49 @@ def test_configs_cache(self): proc = self.start_beat(extra_args=[ "-E", "management.kibana.host=wronghost", "-E", "management.kibana.timeout=0.5s", + # do not blacklist file output + "-E", "management.blacklist.output='elasticsearch'", ]) self.wait_until(cond=lambda: self.output_has( 1, output_file=output_file)) proc.check_kill_and_wait() + def test_blacklist(self): + """ + Blacklist blocks bad configs + """ + # Enroll the beat + config_path = os.path.join(self.working_dir, "mockbeat.yml") + self.render_config_template("mockbeat", config_path) + exit_code = self.enroll(KIBANA_PASSWORD) + assert exit_code == 0 + + # Update output configuration + self.create_and_assing_tag([ + { + "type": "output", + "configs": [ + { + "output": "file", + "file": { + "path": os.path.join(self.working_dir, "output"), + "filename": "mockbeat_managed", + } + } + ] + } + ]) + + output_file = os.path.join("output", "mockbeat_managed") + + # Start beat + proc = self.start_beat() + + self.wait_until( + cond=lambda: self.log_contains("Config for 'output' is blacklisted")) + proc.check_kill_and_wait() + assert not os.path.isfile(os.path.join(self.working_dir, output_file)) + def enroll(self, password): return self.run_beat( extra_args=["enroll", self.get_kibana_url(),