diff --git a/filebeat/autodiscover/builder/hints/config.go b/filebeat/autodiscover/builder/hints/config.go index 007ae2afdd7f..adbe922262bc 100644 --- a/filebeat/autodiscover/builder/hints/config.go +++ b/filebeat/autodiscover/builder/hints/config.go @@ -3,8 +3,8 @@ package hints import "github.com/elastic/beats/libbeat/common" type config struct { - Key string `config:"key"` - Config []*common.Config `config:"config"` + Key string `config:"key"` + Config *common.Config `config:"config"` } func defaultConfig() config { @@ -12,13 +12,13 @@ func defaultConfig() config { "type": "docker", "containers": map[string]interface{}{ "ids": []string{ - "${data.docker.container.id}", + "${data.container.id}", }, }, } cfg, _ := common.NewConfigFrom(rawCfg) return config{ Key: "logs", - Config: []*common.Config{cfg}, + Config: cfg, } } diff --git a/filebeat/autodiscover/builder/hints/logs.go b/filebeat/autodiscover/builder/hints/logs.go index b9ae5af5a0e6..20c1a678be80 100644 --- a/filebeat/autodiscover/builder/hints/logs.go +++ b/filebeat/autodiscover/builder/hints/logs.go @@ -2,7 +2,9 @@ package hints import ( "fmt" + "regexp" + "github.com/elastic/beats/filebeat/fileset" "github.com/elastic/beats/libbeat/autodiscover" "github.com/elastic/beats/libbeat/autodiscover/builder" "github.com/elastic/beats/libbeat/autodiscover/template" @@ -22,9 +24,13 @@ const ( excludeLines = "exclude_lines" ) +// validModuleNames to sanitize user input +var validModuleNames = regexp.MustCompile("[^a-zA-Z0-9]+") + type logHints struct { - Key string - Config []*common.Config + Key string + Config *common.Config + Registry *fileset.ModuleRegistry } // NewLogHints builds a log hints builder @@ -37,16 +43,21 @@ func NewLogHints(cfg *common.Config) (autodiscover.Builder, error) { return nil, fmt.Errorf("unable to unpack hints config due to error: %v", err) } - return &logHints{config.Key, config.Config}, nil + moduleRegistry, err := fileset.NewModuleRegistry([]*common.Config{}, "", false) + if err != nil { + return nil, err + } + + return &logHints{config.Key, config.Config, moduleRegistry}, nil } // Create config based on input hints in the bus event func (l *logHints) CreateConfig(event bus.Event) []*common.Config { - var config []*common.Config - + // Clone original config + config, _ := common.NewConfigFrom(l.Config) host, _ := event["host"].(string) if host == "" { - return config + return []*common.Config{} } var hints common.MapStr @@ -56,11 +67,9 @@ func (l *logHints) CreateConfig(event bus.Event) []*common.Config { } if builder.IsNoOp(hints, l.Key) == true { - return config + return []*common.Config{config} } - //TODO: Add module support - tempCfg := common.MapStr{} mline := l.getMultiline(hints) if len(mline) != 0 { @@ -74,18 +83,34 @@ func (l *logHints) CreateConfig(event bus.Event) []*common.Config { } // Merge config template with the configs from the annotations - for _, c := range l.Config { - if err := c.Merge(tempCfg); err != nil { - logp.Debug("hints.builder", "config merge failed with error: %v", err) - } else { - logp.Debug("hints.builder", "generated config %v", *c) - config = append(config, c) + if err := config.Merge(tempCfg); err != nil { + logp.Debug("hints.builder", "config merge failed with error: %v", err) + return []*common.Config{config} + } + + module := l.getModule(hints) + if module != "" { + moduleConf := map[string]interface{}{ + "module": module, } + + filesets := l.getFilesets(hints, module) + for fileset, conf := range filesets { + filesetConf, _ := common.NewConfigFrom(config) + filesetConf.SetString("containers.stream", -1, conf.Stream) + + moduleConf[fileset+".enabled"] = conf.Enabled + moduleConf[fileset+".input"] = filesetConf + + logp.Debug("hints.builder", "generated config %+v", moduleConf) + } + config, _ = common.NewConfigFrom(moduleConf) } + logp.Debug("hints.builder", "generated config %+v", config) + // Apply information in event to the template to generate the final config - config = template.ApplyConfigTemplate(event, config) - return config + return template.ApplyConfigTemplate(event, []*common.Config{config}) } func (l *logHints) getMultiline(hints common.MapStr) common.MapStr { @@ -99,3 +124,60 @@ func (l *logHints) getIncludeLines(hints common.MapStr) []string { func (l *logHints) getExcludeLines(hints common.MapStr) []string { return builder.GetHintAsList(hints, l.Key, excludeLines) } + +func (l *logHints) getModule(hints common.MapStr) string { + module := builder.GetHintString(hints, l.Key, "module") + // for security, strip module name + return validModuleNames.ReplaceAllString(module, "") +} + +type filesetConfig struct { + Enabled bool + Stream string +} + +// Return a map containing filesets -> enabled & stream (stdout, stderr, all) +func (l *logHints) getFilesets(hints common.MapStr, module string) map[string]*filesetConfig { + var configured bool + filesets := make(map[string]*filesetConfig) + + moduleFilesets, err := l.Registry.ModuleFilesets(module) + if err != nil { + logp.Err("Error retrieving module filesets", err) + return nil + } + + for _, fileset := range moduleFilesets { + filesets[fileset] = &filesetConfig{Enabled: false, Stream: "all"} + } + + // If a single fileset is given, pass all streams to it + fileset := builder.GetHintString(hints, l.Key, "fileset") + if fileset != "" { + if conf, ok := filesets[fileset]; ok { + conf.Enabled = true + configured = true + } + } + + // If fileset is defined per stream, return all of them + for _, stream := range []string{"all", "stdout", "stderr"} { + fileset := builder.GetHintString(hints, l.Key, "fileset."+stream) + if fileset != "" { + if conf, ok := filesets[fileset]; ok { + conf.Enabled = true + conf.Stream = stream + configured = true + } + } + } + + // No fileseat defined, return defaults for the module, all streams to all filesets + if !configured { + for _, conf := range filesets { + conf.Enabled = true + } + } + + return filesets +} diff --git a/filebeat/autodiscover/builder/hints/logs_test.go b/filebeat/autodiscover/builder/hints/logs_test.go index 38c98eeeaceb..6b1de4da246e 100644 --- a/filebeat/autodiscover/builder/hints/logs_test.go +++ b/filebeat/autodiscover/builder/hints/logs_test.go @@ -1,22 +1,25 @@ package hints import ( + "path/filepath" "testing" "github.com/stretchr/testify/assert" "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/common/bus" + "github.com/elastic/beats/libbeat/paths" ) func TestGenerateHints(t *testing.T) { tests := []struct { + msg string event bus.Event len int result common.MapStr }{ - // Hints without host should return nothing { + msg: "Hints without host should return nothing", event: bus.Event{ "hints": common.MapStr{ "metrics": common.MapStr{ @@ -27,8 +30,8 @@ func TestGenerateHints(t *testing.T) { len: 0, result: common.MapStr{}, }, - // Empty event hints should return default config { + msg: "Empty event hints should return default config", event: bus.Event{ "host": "1.2.3.4", "kubernetes": common.MapStr{ @@ -37,11 +40,9 @@ func TestGenerateHints(t *testing.T) { "id": "abc", }, }, - "docker": common.MapStr{ - "container": common.MapStr{ - "name": "foobar", - "id": "abc", - }, + "container": common.MapStr{ + "name": "foobar", + "id": "abc", }, }, len: 1, @@ -52,8 +53,8 @@ func TestGenerateHints(t *testing.T) { }, }, }, - // Hint with include|exclude_lines must be part of the input config { + msg: "Hint with include|exclude_lines must be part of the input config", event: bus.Event{ "host": "1.2.3.4", "kubernetes": common.MapStr{ @@ -62,11 +63,9 @@ func TestGenerateHints(t *testing.T) { "id": "abc", }, }, - "docker": common.MapStr{ - "container": common.MapStr{ - "name": "foobar", - "id": "abc", - }, + "container": common.MapStr{ + "name": "foobar", + "id": "abc", }, "hints": common.MapStr{ "logs": common.MapStr{ @@ -85,8 +84,8 @@ func TestGenerateHints(t *testing.T) { "exclude_lines": []interface{}{"^test2", "^test3"}, }, }, - // Hint with multiline config must have a multiline in the input config { + msg: "Hint with multiline config must have a multiline in the input config", event: bus.Event{ "host": "1.2.3.4", "kubernetes": common.MapStr{ @@ -95,11 +94,9 @@ func TestGenerateHints(t *testing.T) { "id": "abc", }, }, - "docker": common.MapStr{ - "container": common.MapStr{ - "name": "foobar", - "id": "abc", - }, + "container": common.MapStr{ + "name": "foobar", + "id": "abc", }, "hints": common.MapStr{ "logs": common.MapStr{ @@ -122,23 +119,176 @@ func TestGenerateHints(t *testing.T) { }, }, }, + { + msg: "Hint with module should attach input to its filesets", + event: bus.Event{ + "host": "1.2.3.4", + "kubernetes": common.MapStr{ + "container": common.MapStr{ + "name": "foobar", + "id": "abc", + }, + }, + "container": common.MapStr{ + "name": "foobar", + "id": "abc", + }, + "hints": common.MapStr{ + "logs": common.MapStr{ + "module": "apache2", + }, + }, + }, + len: 1, + result: common.MapStr{ + "module": "apache2", + "error": map[string]interface{}{ + "enabled": true, + "input": map[string]interface{}{ + "type": "docker", + "containers": map[string]interface{}{ + "stream": "all", + "ids": []interface{}{"abc"}, + }, + }, + }, + "access": map[string]interface{}{ + "enabled": true, + "input": map[string]interface{}{ + "type": "docker", + "containers": map[string]interface{}{ + "stream": "all", + "ids": []interface{}{"abc"}, + }, + }, + }, + }, + }, + { + msg: "Hint with module should honor defined filesets", + event: bus.Event{ + "host": "1.2.3.4", + "kubernetes": common.MapStr{ + "container": common.MapStr{ + "name": "foobar", + "id": "abc", + }, + }, + "container": common.MapStr{ + "name": "foobar", + "id": "abc", + }, + "hints": common.MapStr{ + "logs": common.MapStr{ + "module": "apache2", + "fileset": "access", + }, + }, + }, + len: 1, + result: common.MapStr{ + "module": "apache2", + "access": map[string]interface{}{ + "enabled": true, + "input": map[string]interface{}{ + "type": "docker", + "containers": map[string]interface{}{ + "stream": "all", + "ids": []interface{}{"abc"}, + }, + }, + }, + "error": map[string]interface{}{ + "enabled": false, + "input": map[string]interface{}{ + "type": "docker", + "containers": map[string]interface{}{ + "stream": "all", + "ids": []interface{}{"abc"}, + }, + }, + }, + }, + }, + { + msg: "Hint with module should honor defined filesets with streams", + event: bus.Event{ + "host": "1.2.3.4", + "kubernetes": common.MapStr{ + "container": common.MapStr{ + "name": "foobar", + "id": "abc", + }, + }, + "container": common.MapStr{ + "name": "foobar", + "id": "abc", + }, + "hints": common.MapStr{ + "logs": common.MapStr{ + "module": "apache2", + "fileset.stdout": "access", + "fileset.stderr": "error", + }, + }, + }, + len: 1, + result: common.MapStr{ + "module": "apache2", + "access": map[string]interface{}{ + "enabled": true, + "input": map[string]interface{}{ + "type": "docker", + "containers": map[string]interface{}{ + "stream": "stdout", + "ids": []interface{}{"abc"}, + }, + }, + }, + "error": map[string]interface{}{ + "enabled": true, + "input": map[string]interface{}{ + "type": "docker", + "containers": map[string]interface{}{ + "stream": "stderr", + "ids": []interface{}{"abc"}, + }, + }, + }, + }, + }, } for _, test := range tests { - cfg := defaultConfig() - l := logHints{ - Key: cfg.Key, - Config: cfg.Config, + cfg, _ := common.NewConfigFrom(map[string]interface{}{ + "type": "docker", + "containers": map[string]interface{}{ + "ids": []string{ + "${data.container.id}", + }, + }, + }) + + // Configure path for modules access + abs, _ := filepath.Abs("../../..") + err := paths.InitPaths(&paths.Path{ + Home: abs, + }) + + l, err := NewLogHints(cfg) + if err != nil { + t.Fatal(err) } + cfgs := l.CreateConfig(test.event) - assert.Equal(t, len(cfgs), test.len) + assert.Equal(t, len(cfgs), test.len, test.msg) if test.len != 0 { config := common.MapStr{} err := cfgs[0].Unpack(&config) - assert.Nil(t, err) + assert.Nil(t, err, test.msg) - assert.Equal(t, config, test.result) + assert.Equal(t, test.result, config, test.msg) } } diff --git a/filebeat/fileset/modules.go b/filebeat/fileset/modules.go index 6c2cc897c22e..35fcc2251078 100644 --- a/filebeat/fileset/modules.go +++ b/filebeat/fileset/modules.go @@ -409,3 +409,10 @@ func (reg *ModuleRegistry) ModuleNames() []string { } return modules } + +// ModuleFilesets return the list of available filesets for the given module +// it returns an empty list if the module doesn't exist +func (reg *ModuleRegistry) ModuleFilesets(module string) ([]string, error) { + modulesPath := paths.Resolve(paths.Home, "module") + return getModuleFilesets(modulesPath, module) +} diff --git a/libbeat/autodiscover/providers/docker/docker.go b/libbeat/autodiscover/providers/docker/docker.go index 1e7ff9353494..af83f3a9fbf2 100644 --- a/libbeat/autodiscover/providers/docker/docker.go +++ b/libbeat/autodiscover/providers/docker/docker.go @@ -176,9 +176,9 @@ func (d *Provider) generateHints(event bus.Event) bus.Event { e := bus.Event{} var dockerMeta common.MapStr - if rawDocker, ok := event["docker"]; ok { + if rawDocker, err := common.MapStr(event).GetValue("docker.container"); err == nil { dockerMeta = rawDocker.(common.MapStr) - e["docker"] = dockerMeta + e["container"] = dockerMeta } if host, ok := event["host"]; ok { @@ -187,7 +187,7 @@ func (d *Provider) generateHints(event bus.Event) bus.Event { if port, ok := event["port"]; ok { e["port"] = port } - if labels, err := dockerMeta.GetValue("container.labels"); err == nil { + if labels, err := dockerMeta.GetValue("labels"); err == nil { hints := builder.GenerateHints(labels.(common.MapStr), "", d.config.Prefix) e["hints"] = hints } diff --git a/libbeat/autodiscover/providers/docker/docker_test.go b/libbeat/autodiscover/providers/docker/docker_test.go index 63cc9e614bf3..91c8cc04f0fe 100644 --- a/libbeat/autodiscover/providers/docker/docker_test.go +++ b/libbeat/autodiscover/providers/docker/docker_test.go @@ -30,11 +30,9 @@ func TestGenerateHints(t *testing.T) { }, }, result: bus.Event{ - "docker": common.MapStr{ - "container": common.MapStr{ - "id": "abc", - "name": "foobar", - }, + "container": common.MapStr{ + "id": "abc", + "name": "foobar", }, }, }, @@ -55,15 +53,13 @@ func TestGenerateHints(t *testing.T) { }, }, result: bus.Event{ - "docker": common.MapStr{ - "container": common.MapStr{ - "id": "abc", - "name": "foobar", - "labels": getNestedAnnotations(common.MapStr{ - "do.not.include": "true", - "co.elastic.logs/disable": "true", - }), - }, + "container": common.MapStr{ + "id": "abc", + "name": "foobar", + "labels": getNestedAnnotations(common.MapStr{ + "do.not.include": "true", + "co.elastic.logs/disable": "true", + }), }, "hints": common.MapStr{ "logs": common.MapStr{