From 7dbb4f1e380c6be14eb775057a94ea4e02d49a4d Mon Sep 17 00:00:00 2001 From: Mario Castro Date: Thu, 27 Sep 2018 10:39:50 +0200 Subject: [PATCH] Added 'query' parameter to metricbeats global configuration (#8292) Added 'query' parameter to metricbeats global configuration --- CHANGELOG.asciidoc | 1 + metricbeat/docs/metricbeat-options.asciidoc | 17 +++++++++++ metricbeat/mb/mb.go | 34 +++++++++++++++++++-- metricbeat/mb/mb_test.go | 25 ++++++++++++++- metricbeat/mb/parse/url.go | 10 ++++++ metricbeat/mb/parse/url_test.go | 3 ++ 6 files changed, 87 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index eea6a4717a1b..09a51f4c3a68 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -136,6 +136,7 @@ https://github.com/elastic/beats/compare/v6.4.0...master[Check the HEAD diff] - Allow TCP helper to support delimiters and graphite module to accept multiple metrics in a single payload. {pull}8278[8278] - Added 'died' PID state to process_system metricset on system module{pull}8275[8275] - Added `ccr` metricset to Elasticsearch module. {pull}8335[8335] +- Added support for query params in configuration {issue}8286[8286] {pull}8292[8292] - Support for Kafka 2.0.0 {pull}8399[8399] *Packetbeat* diff --git a/metricbeat/docs/metricbeat-options.asciidoc b/metricbeat/docs/metricbeat-options.asciidoc index 09f62cd1cfd7..94afdf33fdac 100644 --- a/metricbeat/docs/metricbeat-options.asciidoc +++ b/metricbeat/docs/metricbeat-options.asciidoc @@ -248,3 +248,20 @@ and then use the value in an HTTP Authorization header. An optional base path to be used in HTTP URIs. If defined, Metricbeat will insert this value as the first segment in the HTTP URI path. + +[float] +==== `query` + +An optional value to pass common query params in YAML. Instead of setting the query params +within hosts values using the syntax `?key=value&key2&value2`, you can set it here like this: + +[source,yaml] +---- +query: + key: value + key2: value2 + list: + - 1.1 + - 2.95 + - -15 +---- diff --git a/metricbeat/mb/mb.go b/metricbeat/mb/mb.go index e19b3fb108c0..b9617274d222 100644 --- a/metricbeat/mb/mb.go +++ b/metricbeat/mb/mb.go @@ -23,6 +23,7 @@ package mb import ( "fmt" + "net/url" "time" "github.com/elastic/beats/libbeat/common" @@ -306,17 +307,46 @@ type ModuleConfig struct { MetricSets []string `config:"metricsets"` Enabled bool `config:"enabled"` Raw bool `config:"raw"` + Query QueryParams `config:"query"` } func (c ModuleConfig) String() string { return fmt.Sprintf(`{Module:"%v", MetricSets:%v, Enabled:%v, `+ - `Hosts:[%v hosts], Period:"%v", Timeout:"%v", Raw:%v}`, + `Hosts:[%v hosts], Period:"%v", Timeout:"%v", Raw:%v, Query:%v}`, c.Module, c.MetricSets, c.Enabled, len(c.Hosts), c.Period, c.Timeout, - c.Raw) + c.Raw, c.Query) } func (c ModuleConfig) GoString() string { return c.String() } +// QueryParams is a convenient map[string]interface{} wrapper to implement the String interface which returns the +// values in common query params format (key=value&key2=value2) which is the way that the url package expects this +// params (without the initial '?') +type QueryParams map[string]interface{} + +// String returns the values in common query params format (key=value&key2=value2) which is the way that the url +// package expects this params (without the initial '?') +func (q QueryParams) String() (s string) { + u := url.Values{} + + for k, v := range q { + if values, ok := v.([]interface{}); ok { + for _, innerValue := range values { + u.Add(k, fmt.Sprintf("%v", innerValue)) + } + } else { + //nil values in YAML shouldn't be stringified anyhow + if v == nil { + u.Add(k, "") + } else { + u.Add(k, fmt.Sprintf("%v", v)) + } + } + } + + return u.Encode() +} + // defaultModuleConfig contains the default values for ModuleConfig instances. var defaultModuleConfig = ModuleConfig{ Enabled: true, diff --git a/metricbeat/mb/mb_test.go b/metricbeat/mb/mb_test.go index c2b7ccfd36be..3e3b21ea30a1 100644 --- a/metricbeat/mb/mb_test.go +++ b/metricbeat/mb/mb_test.go @@ -206,7 +206,7 @@ func TestNewModulesHostParser(t *testing.T) { r := newTestRegistry(t) factory := func(base BaseMetricSet) (MetricSet, error) { - return &testMetricSet{base}, nil + return &testMetricSet{BaseMetricSet: base}, nil } hostParser := func(m Module, rawHost string) (HostData, error) { @@ -375,3 +375,26 @@ func newConfig(t testing.TB, moduleConfig interface{}) *common.Config { } return config } + +func TestModuleConfigQueryParams(t *testing.T) { + qp := QueryParams{ + "stringKey": "value", + "intKey": 10, + "floatKey": 11.5, + "boolKey": true, + "nullKey": nil, + "arKey": []interface{}{1, 2}, + } + + res := qp.String() + + expectedValues := []string{"stringKey=value", "intKey=10", "floatKey=11.5", "boolKey=true", "nullKey=", "arKey=1", "arKey=2"} + for _, expected := range expectedValues { + assert.Contains(t, res, expected) + } + + assert.NotContains(t, res, "?") + assert.NotContains(t, res, "%") + assert.NotEqual(t, "&", res[0]) + assert.NotEqual(t, "&", res[len(res)-1]) +} diff --git a/metricbeat/mb/parse/url.go b/metricbeat/mb/parse/url.go index e8945d9df1ff..66c970cbd6bb 100644 --- a/metricbeat/mb/parse/url.go +++ b/metricbeat/mb/parse/url.go @@ -50,6 +50,16 @@ func (b URLHostParserBuilder) Build() mb.HostParser { return mb.HostData{}, err } + query, ok := conf["query"] + if ok { + queryMap, ok := query.(map[string]interface{}) + if !ok { + return mb.HostData{}, errors.Errorf("'query' config for module %v is not a map", module.Name()) + } + + b.QueryParams = mb.QueryParams(queryMap).String() + } + var user, pass, path, basePath string t, ok := conf["username"] if ok { diff --git a/metricbeat/mb/parse/url_test.go b/metricbeat/mb/parse/url_test.go index 571a5c0d13a4..8465f4f16794 100644 --- a/metricbeat/mb/parse/url_test.go +++ b/metricbeat/mb/parse/url_test.go @@ -20,6 +20,8 @@ package parse import ( "testing" + "github.com/elastic/beats/metricbeat/mb" + mbtest "github.com/elastic/beats/metricbeat/mb/testing" "github.com/stretchr/testify/assert" @@ -118,6 +120,7 @@ func TestURLHostParserBuilder(t *testing.T) { {map[string]interface{}{"basepath": "foo/"}, URLHostParserBuilder{DefaultPath: "/default"}, "http://example.com/foo/default"}, {map[string]interface{}{"basepath": "/foo/"}, URLHostParserBuilder{DefaultPath: "/default"}, "http://example.com/foo/default"}, {map[string]interface{}{"basepath": "foo"}, URLHostParserBuilder{DefaultPath: "/default"}, "http://example.com/foo/default"}, + {map[string]interface{}{"basepath": "foo"}, URLHostParserBuilder{DefaultPath: "/queryParams", QueryParams: mb.QueryParams{"key": "value"}.String()}, "http://example.com/foo/queryParams?key=value"}, } for _, test := range cases {