diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 0c551133719..a3778a2ffd4 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -58,6 +58,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix index names for indexing not always guaranteed to be lower case. {pull}16081[16081] - Add `ssl.ca_sha256` option to the supported TLS option, this allow to check that a specific certificate is used as part of the verified chain. {issue}15717[15717] - Fix loading processors from annotation hints. {pull}16348[16348] +- Upgrade go-ucfg to latest v0.8.3. {pull}16450{16450} *Auditbeat* @@ -73,6 +74,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Prevent Elasticsearch from spewing log warnings about redundant wildcards when setting up ingest pipelines for the `elasticsearch` module. {issue}15840[15840] {pull}15900[15900] - Fix mapping error for cloudtrail additionalEventData field {pull}16088[16088] - Fix a connection error in httpjson input. {pull}16123[16123] +- Fix merging of fileset inputs to replace paths and append processors. {pull}16450{16450} *Heartbeat* diff --git a/NOTICE.txt b/NOTICE.txt index cae8fed4047..801a192509d 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -1484,8 +1484,8 @@ Apache License 2.0 -------------------------------------------------------------------- Dependency: github.com/elastic/go-ucfg -Version: v0.8.2 -Revision: ab69586e10820006b250d04c98cc3b848e227808 +Version: v0.8.3 +Revision: f3ae3b220df32cfcae011ec6bbf87a711e6bf06b License type (autodetected): Apache-2.0 ./vendor/github.com/elastic/go-ucfg/LICENSE: -------------------------------------------------------------------- diff --git a/filebeat/fileset/fileset.go b/filebeat/fileset/fileset.go index e1419ee5a93..eccd019fcb5 100644 --- a/filebeat/fileset/fileset.go +++ b/filebeat/fileset/fileset.go @@ -34,6 +34,8 @@ import ( "strings" "text/template" + "github.com/elastic/go-ucfg" + errw "github.com/pkg/errors" "gopkg.in/yaml.v2" @@ -355,7 +357,7 @@ func (fs *Fileset) getInputConfig() (*common.Config, error) { if err != nil { return nil, fmt.Errorf("Error creating config from input overrides: %v", err) } - cfg, err = common.MergeConfigs(cfg, overrides) + cfg, err = common.MergeConfigsWithOptions([]*common.Config{cfg, overrides}, ucfg.FieldReplaceValues("**.paths"), ucfg.FieldAppendValues("**.processors")) if err != nil { return nil, fmt.Errorf("Error applying config overrides: %v", err) } diff --git a/libbeat/common/config.go b/libbeat/common/config.go index 2f779d996bb..448b2476156 100644 --- a/libbeat/common/config.go +++ b/libbeat/common/config.go @@ -116,6 +116,16 @@ func MergeConfigs(cfgs ...*Config) (*Config, error) { return config, nil } +func MergeConfigsWithOptions(cfgs []*Config, options ...ucfg.Option) (*Config, error) { + config := NewConfig() + for _, c := range cfgs { + if err := config.MergeWithOpts(c, options...); err != nil { + return nil, err + } + } + return config, nil +} + func NewConfigWithYAML(in []byte, source string) (*Config, error) { opts := append( []ucfg.Option{ @@ -164,6 +174,14 @@ func (c *Config) Merge(from interface{}) error { return c.access().Merge(from, configOpts...) } +func (c *Config) MergeWithOpts(from interface{}, opts ...ucfg.Option) error { + o := configOpts + if opts != nil { + o = append(o, opts...) + } + return c.access().Merge(from, o...) +} + func (c *Config) Unpack(to interface{}) error { return c.access().Unpack(to, configOpts...) } diff --git a/vendor/github.com/elastic/go-ucfg/CHANGELOG.md b/vendor/github.com/elastic/go-ucfg/CHANGELOG.md index 01a9ea1d295..f0326ac78ef 100644 --- a/vendor/github.com/elastic/go-ucfg/CHANGELOG.md +++ b/vendor/github.com/elastic/go-ucfg/CHANGELOG.md @@ -14,6 +14,11 @@ This project adheres to [Semantic Versioning](http://semver.org/). ### Fixed +## [0.8.3] + +### Added +- Added ability to adjust merging behavior based on field names in configuration. Using `ucfg.FieldMergeValues`, `ucfg.FieldReplaceValues`, `ucfg.FieldAppendValues`, and `ucfg.FieldPrependValues`. #151 + ## [0.8.2] ### Fixed @@ -263,7 +268,8 @@ This project adheres to [Semantic Versioning](http://semver.org/). - Introduced CHANGELOG.md for documenting changes to ucfg. -[Unreleased]: https://github.com/elastic/go-ucfg/compare/v0.8.2...HEAD +[Unreleased]: https://github.com/elastic/go-ucfg/compare/v0.8.3...HEAD +[0.8.3]: https://github.com/elastic/go-ucfg/compare/v0.8.2...v0.8.3 [0.8.2]: https://github.com/elastic/go-ucfg/compare/v0.8.1...v0.8.2 [0.8.1]: https://github.com/elastic/go-ucfg/compare/v0.8.0...v0.8.1 [0.8.0]: https://github.com/elastic/go-ucfg/compare/v0.7.0...v0.8.0 diff --git a/vendor/github.com/elastic/go-ucfg/merge.go b/vendor/github.com/elastic/go-ucfg/merge.go index fa2dd074566..ff07a97c8f7 100644 --- a/vendor/github.com/elastic/go-ucfg/merge.go +++ b/vendor/github.com/elastic/go-ucfg/merge.go @@ -115,6 +115,10 @@ func mergeConfigDict(opts *options, to, from *Config) Error { } old, _ := to.fields.get(k) + opts, err := fieldOptsOverride(opts, k, -1) + if err != nil { + return err + } merged, err := mergeValues(opts, old, v) if err != nil { return err @@ -128,7 +132,12 @@ func mergeConfigDict(opts *options, to, from *Config) Error { } func mergeConfigArr(opts *options, to, from *Config) Error { - switch opts.configValueHandling { + currHandling := opts.configValueHandling + opts, err := fieldOptsOverride(opts, "*", -1) + if err != nil { + return err + } + switch currHandling { case cfgReplaceValue: return mergeConfigReplaceArr(opts, to, from) @@ -177,8 +186,13 @@ func mergeConfigMergeArr(opts *options, to, from *Config) Error { field: fmt.Sprintf("%v", i), } + // possible for individual index to be replaced + idxOpts, err := fieldOptsOverride(opts, "", i) + if err != nil { + return err + } old := to.fields.array()[i] - merged, err := mergeValues(opts, old, arr[i]) + merged, err := mergeValues(idxOpts, old, arr[i]) if err != nil { return err } @@ -515,3 +529,57 @@ func normalizeString(ctx context, opts *options, str string) (value, Error) { return newSplice(ctx, opts.meta, varexp), nil } + +func fieldOptsOverride(opts *options, fieldName string, idx int) (*options, Error) { + if opts.fieldHandlingTree == nil { + return opts, nil + } + cfgHandling, child, ok := opts.fieldHandlingTree.fieldHandling(fieldName, idx) + child, err := includeWildcard(child, opts.fieldHandlingTree) + if err != nil { + return nil, err + } + if !ok { + // Only return a new `options` when arriving at new nested child. This + // combined with optimizations in `includeWildcard` will ensure that only + // a new opts will be created and returned when absolutely required. + if child != nil && opts.fieldHandlingTree != child { + newOpts := *opts + newOpts.fieldHandlingTree = child + opts = &newOpts + } + return opts, nil + } + // Only return a new `options` if absolutely required. + if opts.configValueHandling != cfgHandling || opts.fieldHandlingTree != child { + newOpts := *opts + newOpts.configValueHandling = cfgHandling + newOpts.fieldHandlingTree = child + opts = &newOpts + } + return opts, nil +} + +func includeWildcard(child *fieldHandlingTree, parent *fieldHandlingTree) (*fieldHandlingTree, Error) { + if parent == nil { + return child, nil + } + wildcard, err := parent.wildcard() + if err != nil { + return child, nil + } + if child == nil && len(parent.fields.dict()) == 1 { + // parent is already config with just wildcard + return parent, nil + } + sub := newFieldHandlingTree() + if child != nil { + if err := sub.merge(child); err != nil { + return nil, err.(Error) + } + } + if err := sub.setWildcard(wildcard); err != nil { + return nil, err.(Error) + } + return sub, nil +} diff --git a/vendor/github.com/elastic/go-ucfg/opts.go b/vendor/github.com/elastic/go-ucfg/opts.go index ba954242829..68c23e562c9 100644 --- a/vendor/github.com/elastic/go-ucfg/opts.go +++ b/vendor/github.com/elastic/go-ucfg/opts.go @@ -18,7 +18,9 @@ package ucfg import ( + "fmt" "os" + "strings" "github.com/elastic/go-ucfg/parse" ) @@ -38,6 +40,7 @@ type options struct { noParse bool configValueHandling configHandling + fieldHandlingTree *fieldHandlingTree // temporary cache of parsed splice values for lifetime of call to // Unpack/Pack/Get/... @@ -48,6 +51,9 @@ type options struct { type valueCache map[string]spliceValue +// specific API on top of Config to handle adjusting merging behavior per fields +type fieldHandlingTree Config + // id used to store intermediate parse results in current execution context. // As parsing results might differ between multiple calls due to: // splice being shared between multiple configurations, or environment @@ -162,6 +168,57 @@ func makeOptValueHandling(h configHandling) Option { } } +var ( + // FieldMergeValues option configures all merging and unpacking operations to use + // the default merging behavior for the specified field. This overrides the any struct + // tags during unpack for the field. Nested field names can be defined using dot + // notation. + FieldMergeValues = makeFieldOptValueHandling(cfgMergeValues) + + // FieldReplaceValues option configures all merging and unpacking operations to + // replace old dictionaries and arrays while merging for the specified field. This + // overrides the any struct tags during unpack for the field. Nested field names + // can be defined using dot notation. + FieldReplaceValues = makeFieldOptValueHandling(cfgReplaceValue) + + // FieldAppendValues option configures all merging and unpacking operations to + // merge dictionaries and append arrays to existing arrays while merging for the + // specified field. This overrides the any struct tags during unpack for the field. + // Nested field names can be defined using dot notation. + FieldAppendValues = makeFieldOptValueHandling(cfgArrAppend) + + // FieldPrependValues option configures all merging and unpacking operations to + // merge dictionaries and prepend arrays to existing arrays while merging for the + // specified field. This overrides the any struct tags during unpack for the field. + // Nested field names can be defined using dot notation. + FieldPrependValues = makeFieldOptValueHandling(cfgArrPrepend) +) + +func makeFieldOptValueHandling(h configHandling) func(...string) Option { + return func(fieldName ...string) Option { + if len(fieldName) == 0 { + return func(_ *options) {} + } + + table := make(map[string]configHandling) + for _, name := range fieldName { + // field value config options are rendered into a Config; the '*' represents the handling method + // for everything nested under this field. + if !strings.HasSuffix(name, ".*") { + name = fmt.Sprintf("%s.*", name) + } + table[name] = h + } + + return func(o *options) { + if o.fieldHandlingTree == nil { + o.fieldHandlingTree = newFieldHandlingTree() + } + o.fieldHandlingTree.merge(table, PathSep(o.pathSep)) + } + } +} + // VarExp option enables support for variable expansion. Resolve and Env options will only be effective if VarExp is set. var VarExp Option = doVarExp @@ -200,3 +257,59 @@ func (cache valueCache) cachedValue( } return v, err } + +func newFieldHandlingTree() *fieldHandlingTree { + return (*fieldHandlingTree)(New()) +} + +func (t *fieldHandlingTree) merge(other interface{}, opts ...Option) error { + cfg := (*Config)(t) + return cfg.Merge(other, opts...) +} + +func (t *fieldHandlingTree) child(fieldName string, idx int) (*fieldHandlingTree, error) { + cfg := (*Config)(t) + child, err := cfg.Child(fieldName, idx) + if err != nil { + return nil, err + } + return (*fieldHandlingTree)(child), nil +} + +func (t *fieldHandlingTree) configHandling(fieldName string, idx int) (configHandling, error) { + cfg := (*Config)(t) + handling, err := cfg.Uint(fieldName, idx) + if err != nil { + return cfgDefaultHandling, err + } + return configHandling(handling), nil +} + +func (t *fieldHandlingTree) wildcard() (*fieldHandlingTree, error) { + return t.child("**", -1) +} + +func (t *fieldHandlingTree) setWildcard(wildcard *fieldHandlingTree) error { + cfg := (*Config)(t) + return cfg.SetChild("**", -1, (*Config)(wildcard)) +} + +func (t *fieldHandlingTree) fieldHandling(fieldName string, idx int) (configHandling, *fieldHandlingTree, bool) { + child, err := t.child(fieldName, idx) + if err == nil { + cfgHandling, err := child.configHandling("*", -1) + if err == nil { + return cfgHandling, child, true + } + } + // try wildcard match + wildcard, err := t.wildcard() + if err != nil { + return cfgDefaultHandling, child, false + } + cfgHandling, cfg, ok := wildcard.fieldHandling(fieldName, idx) + if ok { + return cfgHandling, cfg, ok + } + return cfgDefaultHandling, child, ok +} diff --git a/vendor/vendor.json b/vendor/vendor.json index fdd29cff5ea..a729034172a 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -3280,76 +3280,76 @@ "versionExact": "v0.0.7" }, { - "checksumSHA1": "VPcK6uPDligSrbdi1yVeGivPvno=", + "checksumSHA1": "6EwlkLbm/yUYgLddYEiQRZioMNM=", "path": "github.com/elastic/go-ucfg", - "revision": "ab69586e10820006b250d04c98cc3b848e227808", - "revisionTime": "2020-02-07T21:56:35Z", - "version": "v0.8.2", - "versionExact": "v0.8.2" + "revision": "f3ae3b220df32cfcae011ec6bbf87a711e6bf06b", + "revisionTime": "2020-02-20T11:45:39Z", + "version": "v0.8.3", + "versionExact": "v0.8.3" }, { "checksumSHA1": "X+R/CD8SokJrmlxFTx2nSevRDhQ=", "path": "github.com/elastic/go-ucfg/cfgutil", - "revision": "ab69586e10820006b250d04c98cc3b848e227808", - "revisionTime": "2020-02-07T21:56:35Z", - "version": "v0.8.2", - "versionExact": "v0.8.2" + "revision": "f3ae3b220df32cfcae011ec6bbf87a711e6bf06b", + "revisionTime": "2020-02-20T11:45:39Z", + "version": "v0.8.3", + "versionExact": "v0.8.3" }, { "checksumSHA1": "B58nBBiUIstpkXfr9yaYN9GzdLE=", "path": "github.com/elastic/go-ucfg/diff", - "revision": "ab69586e10820006b250d04c98cc3b848e227808", - "revisionTime": "2020-02-07T21:56:35Z", - "version": "v0.8.2", - "versionExact": "v0.8.2" + "revision": "f3ae3b220df32cfcae011ec6bbf87a711e6bf06b", + "revisionTime": "2020-02-20T11:45:39Z", + "version": "v0.8.3", + "versionExact": "v0.8.3" }, { "checksumSHA1": "E6k6DWkpI+LOKDIFRqRmNH9GORc=", "path": "github.com/elastic/go-ucfg/flag", - "revision": "ab69586e10820006b250d04c98cc3b848e227808", - "revisionTime": "2020-02-07T21:56:35Z", - "version": "v0.8.2", - "versionExact": "v0.8.2" + "revision": "f3ae3b220df32cfcae011ec6bbf87a711e6bf06b", + "revisionTime": "2020-02-20T11:45:39Z", + "version": "v0.8.3", + "versionExact": "v0.8.3" }, { "checksumSHA1": "NhiQQjYMs/ViCbmEq9tll2uCaYo=", "path": "github.com/elastic/go-ucfg/hjson", - "revision": "ab69586e10820006b250d04c98cc3b848e227808", - "revisionTime": "2020-02-07T21:56:35Z", - "version": "v0.8.2", - "versionExact": "v0.8.2" + "revision": "f3ae3b220df32cfcae011ec6bbf87a711e6bf06b", + "revisionTime": "2020-02-20T11:45:39Z", + "version": "v0.8.3", + "versionExact": "v0.8.3" }, { "checksumSHA1": "esXpiQlEvTOUwsE0nNesso8albo=", "path": "github.com/elastic/go-ucfg/internal/parse", "revision": "0539807037ce820e147797f051ff32b05f4f9288", "revisionTime": "2019-01-28T11:18:48Z", - "version": "v0.8.2", - "versionExact": "v0.8.2" + "version": "v0.8.3", + "versionExact": "v0.8.3" }, { "checksumSHA1": "cfMNsyQm0gZOV0hRJrBSdKDQSBo=", "path": "github.com/elastic/go-ucfg/json", - "revision": "ab69586e10820006b250d04c98cc3b848e227808", - "revisionTime": "2020-02-07T21:56:35Z", - "version": "v0.8.2", - "versionExact": "v0.8.2" + "revision": "f3ae3b220df32cfcae011ec6bbf87a711e6bf06b", + "revisionTime": "2020-02-20T11:45:39Z", + "version": "v0.8.3", + "versionExact": "v0.8.3" }, { "checksumSHA1": "ZISq+zzSb0OLzvwLlf1ObdgnFmM=", "path": "github.com/elastic/go-ucfg/parse", - "revision": "ab69586e10820006b250d04c98cc3b848e227808", - "revisionTime": "2020-02-07T21:56:35Z", - "version": "v0.8.2", - "versionExact": "v0.8.2" + "revision": "f3ae3b220df32cfcae011ec6bbf87a711e6bf06b", + "revisionTime": "2020-02-20T11:45:39Z", + "version": "v0.8.3", + "versionExact": "v0.8.3" }, { "checksumSHA1": "cnJVnptTvXNLzxVhd266k19/pQg=", "path": "github.com/elastic/go-ucfg/yaml", - "revision": "ab69586e10820006b250d04c98cc3b848e227808", - "revisionTime": "2020-02-07T21:56:35Z", - "version": "v0.8.2", - "versionExact": "v0.8.2" + "revision": "f3ae3b220df32cfcae011ec6bbf87a711e6bf06b", + "revisionTime": "2020-02-20T11:45:39Z", + "version": "v0.8.3", + "versionExact": "v0.8.3" }, { "checksumSHA1": "iI1JCWsrAPpoKcEo/i6G3lRLIVs=",