Cherry-pick #8292 to 6.x: Added 'query' parameter to metricbeats global configuration

CHANGELOG.asciidoc

@@ -113,6 +113,8 @@ https://github.com/elastic/beats/compare/v6.4.0...6.x[Check the HEAD diff]
 - Added 'died' PID state to process_system metricset on system module {pull}8275[8275]
 - Add `metrics` metricset to MongoDB module. {pull}7611[7611]
 - Added `ccr` metricset to Elasticsearch module. {pull}8335[8335]
+- Added support for query params in configuration {issue}8286[8286] {pull}8292[8292]
+- Support for Kafka 2.0.0 {pull}8399[8399]
 
 *Packetbeat*
 

metricbeat/docs/metricbeat-options.asciidoc

@@ -240,3 +240,20 @@ and then use the value in an HTTP Authorization header.
 
 An optional base path to be used in HTTP URIs. If defined, Metricbeat will insert this value
 as the first segment in the HTTP URI path.
+
+[float]
+==== `query`
+
+An optional value to pass common query params in YAML. Instead of setting the query params 
+within hosts values using the syntax `?key=value&key2&value2`, you can set it here like this:
+
+[source,yaml]
+----
+query:
+  key: value
+  key2: value2
+  list:
+  - 1.1
+  - 2.95
+  - -15
+----

metricbeat/mb/mb.go

@@ -23,6 +23,7 @@ package mb
 
 import (
 	"fmt"
+	"net/url"
 	"time"
 
 	"github.com/elastic/beats/libbeat/common"
@@ -306,17 +307,46 @@ type ModuleConfig struct {
 	MetricSets []string      `config:"metricsets"`
 	Enabled    bool          `config:"enabled"`
 	Raw        bool          `config:"raw"`
+	Query      QueryParams   `config:"query"`
 }
 
 func (c ModuleConfig) String() string {
 	return fmt.Sprintf(`{Module:"%v", MetricSets:%v, Enabled:%v, `+
-		`Hosts:[%v hosts], Period:"%v", Timeout:"%v", Raw:%v}`,
+		`Hosts:[%v hosts], Period:"%v", Timeout:"%v", Raw:%v, Query:%v}`,
 		c.Module, c.MetricSets, c.Enabled, len(c.Hosts), c.Period, c.Timeout,
-		c.Raw)
+		c.Raw, c.Query)
 }
 
 func (c ModuleConfig) GoString() string { return c.String() }
 
+// QueryParams is a convenient map[string]interface{} wrapper to implement the String interface which returns the
+// values in common query params format (key=value&key2=value2) which is the way that the url package expects this
+// params (without the initial '?')
+type QueryParams map[string]interface{}
+
+// String returns the values in common query params format (key=value&key2=value2) which is the way that the url
+// package expects this params (without the initial '?')
+func (q QueryParams) String() (s string) {
+	u := url.Values{}
+
+	for k, v := range q {
+		if values, ok := v.([]interface{}); ok {
+			for _, innerValue := range values {
+				u.Add(k, fmt.Sprintf("%v", innerValue))
+			}
+		} else {
+			//nil values in YAML shouldn't be stringified anyhow
+			if v == nil {
+				u.Add(k, "")
+			} else {
+				u.Add(k, fmt.Sprintf("%v", v))
+			}
+		}
+	}
+
+	return u.Encode()
+}
+
 // defaultModuleConfig contains the default values for ModuleConfig instances.
 var defaultModuleConfig = ModuleConfig{
 	Enabled: true,

metricbeat/mb/mb_test.go

@@ -206,7 +206,7 @@ func TestNewModulesHostParser(t *testing.T) {
 	r := newTestRegistry(t)
 
 	factory := func(base BaseMetricSet) (MetricSet, error) {
-		return &testMetricSet{base}, nil
+		return &testMetricSet{BaseMetricSet: base}, nil
 	}
 
 	hostParser := func(m Module, rawHost string) (HostData, error) {
@@ -375,3 +375,26 @@ func newConfig(t testing.TB, moduleConfig interface{}) *common.Config {
 	}
 	return config
 }
+
+func TestModuleConfigQueryParams(t *testing.T) {
+	qp := QueryParams{
+		"stringKey": "value",
+		"intKey":    10,
+		"floatKey":  11.5,
+		"boolKey":   true,
+		"nullKey":   nil,
+		"arKey":     []interface{}{1, 2},
+	}
+
+	res := qp.String()
+
+	expectedValues := []string{"stringKey=value", "intKey=10", "floatKey=11.5", "boolKey=true", "nullKey=", "arKey=1", "arKey=2"}
+	for _, expected := range expectedValues {
+		assert.Contains(t, res, expected)
+	}
+
+	assert.NotContains(t, res, "?")
+	assert.NotContains(t, res, "%")
+	assert.NotEqual(t, "&", res[0])
+	assert.NotEqual(t, "&", res[len(res)-1])
+}

metricbeat/mb/parse/url.go

@@ -50,6 +50,16 @@ func (b URLHostParserBuilder) Build() mb.HostParser {
 			return mb.HostData{}, err
 		}
 
+		query, ok := conf["query"]
+		if ok {
+			queryMap, ok := query.(map[string]interface{})
+			if !ok {
+				return mb.HostData{}, errors.Errorf("'query' config for module %v is not a map", module.Name())
+			}
+
+			b.QueryParams = mb.QueryParams(queryMap).String()
+		}
+
 		var user, pass, path, basePath string
 		t, ok := conf["username"]
 		if ok {

metricbeat/mb/parse/url_test.go

@@ -20,6 +20,8 @@ package parse
 import (
 	"testing"
 
+	"github.com/elastic/beats/metricbeat/mb"
+
 	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
 
 	"github.com/stretchr/testify/assert"
@@ -118,6 +120,7 @@ func TestURLHostParserBuilder(t *testing.T) {
 		{map[string]interface{}{"basepath": "foo/"}, URLHostParserBuilder{DefaultPath: "/default"}, "http://example.com/foo/default"},
 		{map[string]interface{}{"basepath": "/foo/"}, URLHostParserBuilder{DefaultPath: "/default"}, "http://example.com/foo/default"},
 		{map[string]interface{}{"basepath": "foo"}, URLHostParserBuilder{DefaultPath: "/default"}, "http://example.com/foo/default"},
+		{map[string]interface{}{"basepath": "foo"}, URLHostParserBuilder{DefaultPath: "/queryParams", QueryParams: mb.QueryParams{"key": "value"}.String()}, "http://example.com/foo/queryParams?key=value"},
 	}
 
 	for _, test := range cases {