diff --git a/metricbeat/module/jolokia/jmx/_meta/docs.asciidoc b/metricbeat/module/jolokia/jmx/_meta/docs.asciidoc index f2f07f9a60c9..e61463653450 100644 --- a/metricbeat/module/jolokia/jmx/_meta/docs.asciidoc +++ b/metricbeat/module/jolokia/jmx/_meta/docs.asciidoc @@ -20,12 +20,13 @@ configure the following mapping: metricsets: ["jmx"] hosts: ["localhost:8778"] namespace: "testnamespace" <1> + http_method: "POST" <2> jmx.mappings: - mbean: 'java.lang:type=Runtime' attributes: - attr: Uptime - field: uptime <2> - event: uptime <3> + field: uptime <3> + event: uptime <4> target: url: "service:jmx:rmi:///jndi/rmi://targethost:9999/jmxrmi" user: "jolokia" @@ -33,9 +34,11 @@ configure the following mapping: ---- <1> The `namespace` setting is required. This setting is used along with the module name to qualify field names in the output event. -<2> The field where the returned value will be saved. This field will be called +<2> The `http_method` setting is optional. By default all requests to Jolokia +are performed using `POST` HTTP method. This setting allows only two values: `POST` or `GET`. +<3> The field where the returned value will be saved. This field will be called `jolokia.testnamespace.uptime` in the output event. -<3> The `event` setting is optional. Use this setting to group all attributes +<4> The `event` setting is optional. Use this setting to group all attributes with the same `event` value into the same event when sending data to Elastic. If the underlying attribute is an object (such as the `HeapMemoryUsage` @@ -54,12 +57,55 @@ configure multiple modules. When wildcards are used, an event is sent to Elastic for each matching MBean, and an `mbean` field is added to the event. +[float] +=== Accessing Jolokia via POST or GET method + +All requests to Jolokia are made by default using HTTP POST method. However, there are specific circumstances +on the environment where Jolokia agent is deployed, in which POST method can be unavailable. In this case you can use +HTTP GET method, by defining `http_method` attribute. In general you can use either POST or GET, but GET has the following +drawbacks: + +1. https://jolokia.org/reference/html/protocol.html#protocol-proxy[Proxy requests] +are not allowed. +2. If more than one `jmx.mappings` are defined, then Metricbeat will perform as many GET requests as the mappings defined. + For example the following configuration with 3 mappings will create 3 GET requests, one for every MBean. On the contrary, if you use HTTP POST, Metricbeat will create only 1 request to Jolokia. + +[source,yaml] +---- +- module: jolokia + metricsets: ["jmx"] + enabled: true + period: 10s + hosts: ["localhost:8080"] + namespace: "jolokia_metrics" + path: "/jolokia" + http_method: 'GET' + jmx.mappings: + - mbean: 'java.lang:type=Memory' + attributes: + - attr: HeapMemoryUsage + field: memory.heap_usage + - attr: NonHeapMemoryUsage + field: memory.non_heap_usage + - mbean: 'Catalina:name=*,type=ThreadPool' + attributes: + - attr: port + field: catalina.port + - attr: maxConnections + field: catalina.maxConnections + - mbean: 'java.lang:type=Runtime' + attributes: + - attr: Uptime + field: uptime +---- + [float] === Limitations -All Jolokia requests have `canonicalNaming` set to `false`. See the +1. All Jolokia requests have `canonicalNaming` set to `false`. See the https://jolokia.org/reference/html/protocol.html[Jolokia Protocol] documentation for more detail about this parameter. - +2. If `http_method` is set to `GET`, then https://jolokia.org/reference/html/protocol.html#protocol-proxy[Proxy requests] +are not allowed. Thus, setting a value to `target` section is going to fail with an error. [float] === Exposed fields, dashboards, indexes, etc. diff --git a/metricbeat/module/jolokia/jmx/_meta/test/jolokia_get_response.json b/metricbeat/module/jolokia/jmx/_meta/test/jolokia_get_response.json new file mode 100644 index 000000000000..1ab7462ddd13 --- /dev/null +++ b/metricbeat/module/jolokia/jmx/_meta/test/jolokia_get_response.json @@ -0,0 +1,26 @@ +{ + "request": { + "mbean": "java.lang:type=Memory", + "attribute": [ + "HeapMemoryUsage", + "NonHeapMemoryUsage" + ], + "type": "read" + }, + "value": { + "HeapMemoryUsage": { + "init": 1073741824, + "committed": 1037959168, + "max": 1037959168, + "used": 227420472 + }, + "NonHeapMemoryUsage": { + "init": 2555904, + "committed": 53477376, + "max": -1, + "used": 50519768 + } + }, + "timestamp": 1472298687, + "status": 200 +} \ No newline at end of file diff --git a/metricbeat/module/jolokia/jmx/config.go b/metricbeat/module/jolokia/jmx/config.go index 0ad6a734434c..ad6be581e94e 100644 --- a/metricbeat/module/jolokia/jmx/config.go +++ b/metricbeat/module/jolokia/jmx/config.go @@ -19,10 +19,17 @@ package jmx import ( "encoding/json" + "fmt" "regexp" "sort" "strings" + + "github.com/pkg/errors" + + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/logp" + "github.com/elastic/beats/metricbeat/helper" ) type JMXMapping struct { @@ -103,41 +110,326 @@ func (m AttributeMapping) Get(mbean, attr string) (Attribute, bool) { return a, found } -// Parse strings with properties with the format key=value, being: -// - key a nonempty string of characters which may not contain any of the characters, -// comma (,), equals (=), colon, asterisk, or question mark. -// - value a string that can be quoted or unquoted, if unquoted it cannot be empty and -// cannot contain any of the characters comma, equals, colon, or quote. -var propertyRegexp = regexp.MustCompile("[^,=:*?]+=([^,=:\"]+|\".*\")") +// MBeanName is an internal struct used to store +// the information by the parsed `mbean` (bean name) configuration +// field in `jmx.mappings`. +type MBeanName struct { + Domain string + Properties map[string]string +} -func canonicalizeMBeanName(name string) (string, error) { - // From https://docs.oracle.com/javase/8/docs/api/javax/management/ObjectName.html#getCanonicalName-- - // - // Returns the canonical form of the name; that is, a string representation where the - // properties are sorted in lexical order. - // The canonical form of the name is a String consisting of the domain part, - // a colon (:), the canonical key property list, and a pattern indication. - // - parts := strings.SplitN(name, ":", 2) +var mbeanRegexp = regexp.MustCompile("([^,=:*?]+)=([^,=:\"]+|\".*\")") + +// This replacer is responsible for adding a "!" before special characters in GET request URIs +// For more information refer: https://jolokia.org/reference/html/protocol.html +var mbeanGetEscapeReplacer = strings.NewReplacer("\"", "!\"", ".", "!.", "!", "!!", "/", "!/") + +// Canonicalize Returns the canonical form of the name; that is, a string representation where the +// properties are sorted in lexical order. +// The canonical form of the name is a String consisting of the domain part, +// a colon (:), the canonical key property list, and a pattern indication. +// +// For more information refer to Java 8 [getCanonicalName()](https://docs.oracle.com/javase/8/docs/api/javax/management/ObjectName.html#getCanonicalName--) +// method. +// +// Set "escape" parameter to true if you want to use the canonicalized name for a Jolokia HTTP GET request, false otherwise. +func (m *MBeanName) Canonicalize(escape bool) string { + + var propertySlice []string + + for key, value := range m.Properties { + + tmpVal := value + if escape { + tmpVal = mbeanGetEscapeReplacer.Replace(value) + } + + propertySlice = append(propertySlice, key+"="+tmpVal) + } + + sort.Strings(propertySlice) + + return m.Domain + ":" + strings.Join(propertySlice, ",") +} + +// ParseMBeanName is a factory function which parses a Managed Bean name string +// identified by mBeanName and returns a new MBean object which +// contains all the information, i.e. domain and properties of the MBean. +// +// The Mbean string has to abide by the rules which are imposed by Java. +// For more info: https://docs.oracle.com/javase/8/docs/api/javax/management/ObjectName.html#getCanonicalName-- +func ParseMBeanName(mBeanName string) (*MBeanName, error) { + + // Split mbean string in two parts: the bean domain and the properties + parts := strings.SplitN(mBeanName, ":", 2) if len(parts) != 2 || parts[0] == "" || parts[1] == "" { - return name, fmt.Errorf("domain and properties needed in mbean name: %s", name) + return nil, fmt.Errorf("domain and properties needed in mbean name: %s", mBeanName) } - domain := parts[0] - // Using this regexp instead of just splitting by commas because values can be quoted - // and contain commas, what complicates the parsing. - properties := propertyRegexp.FindAllString(parts[1], -1) - propertyList := strings.Join(properties, ",") + // Create a new MBean object + mybean := &MBeanName{ + Domain: parts[0], + } + + // First of all verify that all bean properties are + // in the form key=value + tmpProps := propertyRegexp.FindAllString(parts[1], -1) + propertyList := strings.Join(tmpProps, ",") if len(propertyList) != len(parts[1]) { // Some property didn't match - return name, fmt.Errorf("mbean properties must be in the form key=value: %s", name) + return nil, fmt.Errorf("mbean properties must be in the form key=value: %s", mBeanName) + } + + // Using this regexp we will split the properties in a 2 dimensional array + // instead of just splitting by commas because values can be quoted + // and contain commas, what complicates the parsing. + // For example this MBean property string: + // + // name=HttpRequest1,type=RequestProcessor,worker="http-nio-8080" + // + // will become: + // + // [][]string{ + // []string{"name=HttpRequest1", "name", "HttpRequest1"}, + // []string{"type=RequestProcessor", "type", "RequestProcessor"}, + // []string{"worker=\"http-nio-8080\"", "worker", "\"http-nio-8080\""} + // } + properties := mbeanRegexp.FindAllStringSubmatch(parts[1], -1) + + // If we could not parse MBean properties + if properties == nil { + return nil, fmt.Errorf("mbean properties must be in the form key=value: %s", mBeanName) + } + + // Initialise properties map + mybean.Properties = make(map[string]string) + + for _, prop := range properties { + + // If every row does not have 3 columns, then + // parsing must have failed. + if (prop == nil) || (len(prop) < 3) { + // Some property didn't match + return nil, fmt.Errorf("mbean properties must be in the form key=value: %s", mBeanName) + } + + mybean.Properties[prop[1]] = prop[2] } - sort.Strings(properties) - return domain + ":" + strings.Join(properties, ","), nil + return mybean, nil } -func buildRequestBodyAndMapping(mappings []JMXMapping) ([]byte, AttributeMapping, error) { +// JolokiaHTTPRequest is a small struct which contains all request information +// needed to construct a reqest helper.HTTP object which will be sent to Jolokia. +// It is just an intermediary structure which can be easily tested as helper.HTTP +// fields are all private. +type JolokiaHTTPRequest struct { + // HttpMethod can be either "GET" or "POST" + HTTPMethod string + // URI which will be used to query Jolokia + URI string + // Request body which is only filled if the http method is "POST" + Body []byte +} + +// JolokiaHTTPRequestFetcher is an interface which describes +// the behaviour of the builder which generates,fetches the HTTP request, +// which is sent to Jolokia and then parses and maps the response to +// Metricbeat events. +type JolokiaHTTPRequestFetcher interface { + // BuildRequestsAndMappings builds the request information and mappings needed to fetch information from Jolokia server + BuildRequestsAndMappings(configMappings []JMXMapping) ([]*JolokiaHTTPRequest, AttributeMapping, error) + // Fetches the information from Jolokia server regarding MBeans + Fetch(m *MetricSet) ([]common.MapStr, error) + EventMapping(content []byte, mapping AttributeMapping) ([]common.MapStr, error) +} + +// JolokiaHTTPGetFetcher constructs and executes an HTTP GET request +// which will read MBean information from Jolokia +type JolokiaHTTPGetFetcher struct { +} + +// BuildRequestsAndMappings generates HTTP GET request +// such as URI,Body. +func (pc *JolokiaHTTPGetFetcher) BuildRequestsAndMappings(configMappings []JMXMapping) ([]*JolokiaHTTPRequest, AttributeMapping, error) { + + // Create Jolokia URLs + uris, responseMapping, err := pc.buildGetRequestURIs(configMappings) + if err != nil { + return nil, nil, err + } + + // Create one or more HTTP GET requests + var httpRequests []*JolokiaHTTPRequest + for _, i := range uris { + http := &JolokiaHTTPRequest{ + HTTPMethod: "GET", + URI: i, + } + + httpRequests = append(httpRequests, http) + } + + return httpRequests, responseMapping, err +} + +// Builds a GET URI which will have the following format: +// +// /read///[path]?ignoreErrors=true&canonicalNaming=false +func (pc *JolokiaHTTPGetFetcher) buildJolokiaGETUri(mbean string, attr []Attribute) string { + initialURI := "/read/%s?ignoreErrors=true&canonicalNaming=false" + + var attrList []string + for _, attribute := range attr { + attrList = append(attrList, attribute.Attr) + } + + tmpURL := mbean + "/" + strings.Join(attrList, ",") + + tmpURL = fmt.Sprintf(initialURI, tmpURL) + + return tmpURL +} + +func (pc *JolokiaHTTPGetFetcher) mBeanAttributeHasField(attr *Attribute) bool { + + if attr.Field != "" && (strings.Trim(attr.Field, " ") != "") { + return true + } + + return false +} + +func (pc *JolokiaHTTPGetFetcher) buildGetRequestURIs(mappings []JMXMapping) ([]string, AttributeMapping, error) { + + responseMapping := make(AttributeMapping) + var urls []string + + // At least Jolokia 1.5 responses with canonicalized MBean names when using + // wildcards, even when canonicalNaming is set to false, this makes mappings to fail. + // So use canonicalized names everywhere. + // If Jolokia returns non-canonicalized MBean names, then we'll need to canonicalize + // them or change our approach to mappings. + + for _, mapping := range mappings { + mbean, err := ParseMBeanName(mapping.MBean) + if err != nil { + return urls, nil, err + } + + if len(mapping.Target.URL) != 0 { + err := errors.New("Proxy requests are only valid when using POST method") + return urls, nil, err + } + + // For every attribute we will build a response mapping + for _, attribute := range mapping.Attributes { + responseMapping[attributeMappingKey{mbean.Canonicalize(true), attribute.Attr}] = attribute + } + + // Build a new URI for all attributes + urls = append(urls, pc.buildJolokiaGETUri(mbean.Canonicalize(true), mapping.Attributes)) + + } + + return urls, responseMapping, nil +} + +// Fetch perfrorms one or more GET requests to Jolokia server and gets information about MBeans. +func (pc *JolokiaHTTPGetFetcher) Fetch(m *MetricSet) ([]common.MapStr, error) { + + var allEvents []common.MapStr + + // Prepare Http request objects and attribute mappings according to selected Http method + httpReqs, mapping, err := pc.BuildRequestsAndMappings(m.mapping) + if err != nil { + return nil, err + } + + // Log request information + if logp.IsDebug(metricsetName) { + for _, r := range httpReqs { + m.log.Debugw("Jolokia request URI and body", + "httpMethod", r.HTTPMethod, "URI", r.URI, "body", string(r.Body), "type", "request") + } + } + + for _, r := range httpReqs { + + http, err := helper.NewHTTP(m.BaseMetricSet) + + http.SetMethod(r.HTTPMethod) + http.SetURI(m.BaseMetricSet.HostData().SanitizedURI + r.URI) + + resBody, err := http.FetchContent() + if err != nil { + return nil, err + } + + if logp.IsDebug(metricsetName) { + m.log.Debugw("Jolokia response body", + "host", m.HostData().Host, "uri", http.GetURI(), "body", string(resBody), "type", "response") + } + + // Map response to Metricbeat events + events, err := pc.EventMapping(resBody, mapping) + if err != nil { + return nil, err + } + + allEvents = append(allEvents, events...) + } + + return allEvents, nil + +} + +// EventMapping maps a Jolokia response from a GET request is to one or more Metricbeat events +func (pc *JolokiaHTTPGetFetcher) EventMapping(content []byte, mapping AttributeMapping) ([]common.MapStr, error) { + + var singleEntry Entry + + // When we use GET, the response is a single Entry + if err := json.Unmarshal(content, &singleEntry); err != nil { + return nil, errors.Wrapf(err, "failed to unmarshal jolokia JSON response '%v'", string(content)) + } + + return eventMapping([]Entry{singleEntry}, mapping) +} + +// JolokiaHTTPPostFetcher constructs and executes an HTTP GET request +// which will read MBean information from Jolokia +type JolokiaHTTPPostFetcher struct { +} + +// BuildRequestsAndMappings generates HTTP POST request +// such as URI,Body. +func (pc *JolokiaHTTPPostFetcher) BuildRequestsAndMappings(configMappings []JMXMapping) ([]*JolokiaHTTPRequest, AttributeMapping, error) { + + body, mapping, err := pc.buildRequestBodyAndMapping(configMappings) + if err != nil { + return nil, nil, err + } + + http := &JolokiaHTTPRequest{ + HTTPMethod: "POST", + Body: body, + } + + // Create an array with only one HTTP POST request + httpRequests := []*JolokiaHTTPRequest{http} + + return httpRequests, mapping, nil +} + +// Parse strings with properties with the format key=value, being: +// - key a nonempty string of characters which may not contain any of the characters, +// comma (,), equals (=), colon, asterisk, or question mark. +// - value a string that can be quoted or unquoted, if unquoted it cannot be empty and +// cannot contain any of the characters comma, equals, colon, or quote. +var propertyRegexp = regexp.MustCompile("[^,=:*?]+=([^,=:\"]+|\".*\")") + +func (pc *JolokiaHTTPPostFetcher) buildRequestBodyAndMapping(mappings []JMXMapping) ([]byte, AttributeMapping, error) { responseMapping := make(AttributeMapping) var blocks []RequestBlock @@ -151,10 +443,13 @@ func buildRequestBodyAndMapping(mappings []JMXMapping) ([]byte, AttributeMapping "canonicalNaming": true, } for _, mapping := range mappings { - mbean, err := canonicalizeMBeanName(mapping.MBean) + mbeanObj, err := ParseMBeanName(mapping.MBean) if err != nil { return nil, nil, err } + + mbean := mbeanObj.Canonicalize(false) + rb := RequestBlock{ Type: "read", MBean: mbean, @@ -178,3 +473,70 @@ func buildRequestBodyAndMapping(mappings []JMXMapping) ([]byte, AttributeMapping content, err := json.Marshal(blocks) return content, responseMapping, err } + +// Fetch perfrorms a POST request to Jolokia server and gets information about MBeans. +func (pc *JolokiaHTTPPostFetcher) Fetch(m *MetricSet) ([]common.MapStr, error) { + + // Prepare Http POST request object and attribute mappings according to selected Http method + httpReqs, mapping, err := pc.BuildRequestsAndMappings(m.mapping) + if err != nil { + return nil, err + } + + // Log request information + if logp.IsDebug(metricsetName) { + for _, r := range httpReqs { + m.log.Debugw("Jolokia request URI and body", + "httpMethod", r.HTTPMethod, "URI", r.URI, "body", string(r.Body), "type", "request") + } + } + + http, err := helper.NewHTTP(m.BaseMetricSet) + + http.SetMethod(httpReqs[0].HTTPMethod) + http.SetBody(httpReqs[0].Body) + + resBody, err := http.FetchContent() + if err != nil { + return nil, err + } + + if logp.IsDebug(metricsetName) { + m.log.Debugw("Jolokia response body", + "host", m.HostData().Host, "uri", http.GetURI(), "body", string(resBody), "type", "response") + } + + // Map response to Metricbeat events + events, err := pc.EventMapping(resBody, mapping) + if err != nil { + return nil, err + } + + return events, nil +} + +// EventMapping maps a Jolokia response from a POST request is to one or more Metricbeat events +func (pc *JolokiaHTTPPostFetcher) EventMapping(content []byte, mapping AttributeMapping) ([]common.MapStr, error) { + + var entries []Entry + + // When we use POST, the response is an array of Entry objects + if err := json.Unmarshal(content, &entries); err != nil { + + return nil, errors.Wrapf(err, "failed to unmarshal jolokia JSON response '%v'", string(content)) + } + + return eventMapping(entries, mapping) +} + +// NewJolokiaHTTPRequestFetcher is a factory method which creates and returns an implementation +// class of JolokiaHTTPRequestFetcher interface. HTTP GET and POST are currently supported. +func NewJolokiaHTTPRequestFetcher(httpMethod string) JolokiaHTTPRequestFetcher { + + if httpMethod == "GET" { + return &JolokiaHTTPGetFetcher{} + } + + return &JolokiaHTTPPostFetcher{} + +} diff --git a/metricbeat/module/jolokia/jmx/config_test.go b/metricbeat/module/jolokia/jmx/config_test.go index 34497e7450f1..b6ccc1b7ccb5 100644 --- a/metricbeat/module/jolokia/jmx/config_test.go +++ b/metricbeat/module/jolokia/jmx/config_test.go @@ -23,10 +23,61 @@ import ( "github.com/stretchr/testify/assert" ) -func TestCanonicalMBeanName(t *testing.T) { +func TestBuildJolokiaGETUri(t *testing.T) { + cases := []struct { + mbean string + attributes []Attribute + expected string + }{ + { + mbean: `java.lang:type=Memory`, + attributes: []Attribute{ + Attribute{ + Attr: `HeapMemoryUsage`, + Field: `heapMemoryUsage`, + }, + }, + expected: `/read/java.lang:type=Memory/HeapMemoryUsage?ignoreErrors=true&canonicalNaming=false`, + }, + { + mbean: `java.lang:type=Memory`, + attributes: []Attribute{ + Attribute{ + Attr: `HeapMemoryUsage`, + Field: `heapMemoryUsage`, + }, + Attribute{ + Attr: `NonHeapMemoryUsage`, + Field: `nonHeapMemoryUsage`, + }, + }, + expected: `/read/java.lang:type=Memory/HeapMemoryUsage,NonHeapMemoryUsage?ignoreErrors=true&canonicalNaming=false`, + }, + { + mbean: `Catalina:name=HttpRequest1,type=RequestProcessor,worker=!"http-nio-8080!"`, + attributes: []Attribute{ + Attribute{ + Attr: `globalProcessor`, + Field: `maxTime`, + }}, + expected: `/read/Catalina:name=HttpRequest1,type=RequestProcessor,worker=!"http-nio-8080!"/globalProcessor?ignoreErrors=true&canonicalNaming=false`, + }, + } + + for _, c := range cases { + jolokiaGETFetcher := &JolokiaHTTPGetFetcher{} + getURI := jolokiaGETFetcher.buildJolokiaGETUri(c.mbean, c.attributes) + + assert.Equal(t, c.expected, getURI, "mbean: "+c.mbean) + + } +} + +func TestParseMBean(t *testing.T) { + cases := []struct { mbean string - expected string + expected *MBeanName ok bool }{ { @@ -50,49 +101,580 @@ func TestCanonicalMBeanName(t *testing.T) { ok: false, }, { - mbean: `java.lang:type=Runtime`, + mbean: `java.lang:type=Runtime`, + expected: &MBeanName{ + Domain: `java.lang`, + Properties: map[string]string{ + "type": "Runtime", + }, + }, + ok: true, + }, + { + mbean: `java.lang:name=Foo,type=Runtime`, + expected: &MBeanName{ + Domain: `java.lang`, + Properties: map[string]string{ + "name": "Foo", + "type": "Runtime", + }, + }, + ok: true, + }, + { + mbean: `java.lang:name=Foo,type=Runtime`, + expected: &MBeanName{ + Domain: `java.lang`, + Properties: map[string]string{ + "name": "Foo", + "type": "Runtime", + }, + }, + ok: true, + }, + { + mbean: `java.lang:type=Runtime,name=Foo*`, + expected: &MBeanName{ + Domain: `java.lang`, + Properties: map[string]string{ + "name": "Foo*", + "type": "Runtime", + }, + }, + ok: true, + }, + { + mbean: `java.lang:type=Runtime,name=*`, + expected: &MBeanName{ + Domain: `java.lang`, + Properties: map[string]string{ + "name": "*", + "type": "Runtime", + }, + }, + ok: true, + }, + { + mbean: `java.lang:name="foo,bar",type=Runtime`, + expected: &MBeanName{ + Domain: `java.lang`, + Properties: map[string]string{ + "name": `"foo,bar"`, + "type": "Runtime", + }, + }, + ok: true, + }, + { + mbean: `java.lang:type=Memory`, + expected: &MBeanName{ + Domain: `java.lang`, + Properties: map[string]string{ + "type": "Memory", + }, + }, + ok: true, + }, + { + mbean: `Catalina:name=HttpRequest1,type=RequestProcessor,worker="http-nio-8080"`, + expected: &MBeanName{ + Domain: `Catalina`, + Properties: map[string]string{ + "name": "HttpRequest1", + "type": "RequestProcessor", + "worker": `"http-nio-8080"`, + }, + }, + ok: true, + }, + } + + for _, c := range cases { + beanObj, err := ParseMBeanName(c.mbean) + + if c.ok { + assert.NoError(t, err, "failed parsing for: "+c.mbean) + assert.Equal(t, c.expected, beanObj, "mbean: "+c.mbean) + } else { + assert.Error(t, err, "should have failed for: "+c.mbean) + } + } + +} + +func TestCanonicalizeMbeanName(t *testing.T) { + + cases := []struct { + mbean *MBeanName + expected string + escape bool + }{ + + { + mbean: &MBeanName{ + Domain: `java.lang`, + Properties: map[string]string{ + "type": "Runtime", + }, + }, + escape: true, expected: `java.lang:type=Runtime`, - ok: true, }, { - mbean: `java.lang:name=Foo,type=Runtime`, + mbean: &MBeanName{ + Domain: `java.lang`, + Properties: map[string]string{ + "type": "Runtime", + }, + }, + escape: false, + expected: `java.lang:type=Runtime`, + }, + { + mbean: &MBeanName{ + Domain: `java.lang`, + Properties: map[string]string{ + "name": "Foo", + "type": "Runtime", + }, + }, + escape: true, expected: `java.lang:name=Foo,type=Runtime`, - ok: true, }, { - mbean: `java.lang:type=Runtime,name=Foo`, + mbean: &MBeanName{ + Domain: `java.lang`, + Properties: map[string]string{ + "name": "Foo", + "type": "Runtime", + }, + }, + escape: false, expected: `java.lang:name=Foo,type=Runtime`, - ok: true, }, { - mbean: `java.lang:type=Runtime,name=Foo*`, + mbean: &MBeanName{ + Domain: `java.lang`, + Properties: map[string]string{ + "name": "Foo", + "type": "Runtime", + }, + }, + escape: true, + expected: `java.lang:name=Foo,type=Runtime`, + }, + { + mbean: &MBeanName{ + Domain: `java.lang`, + Properties: map[string]string{ + "name": "Foo*", + "type": "Runtime", + }, + }, + escape: true, expected: `java.lang:name=Foo*,type=Runtime`, - ok: true, }, { - mbean: `java.lang:type=Runtime,name=*`, + mbean: &MBeanName{ + Domain: `java.lang`, + Properties: map[string]string{ + "name": "*", + "type": "Runtime", + }, + }, + escape: true, expected: `java.lang:name=*,type=Runtime`, - ok: true, }, { - mbean: `java.lang:type=Runtime,name="foo,bar"`, - expected: `java.lang:name="foo,bar",type=Runtime`, - ok: true, + mbean: &MBeanName{ + Domain: `java.lang`, + Properties: map[string]string{ + "name": `"foo,bar"`, + "type": "Runtime", + }, + }, + escape: true, + expected: `java.lang:name=!"foo,bar!",type=Runtime`, + }, + { + expected: `java.lang:type=Memory`, + mbean: &MBeanName{ + Domain: `java.lang`, + Properties: map[string]string{ + "type": "Memory", + }, + }, + escape: true, + }, + { + expected: `jboss.jmx:alias=jmx!/rmi!/RMIAdaptor!/State`, + mbean: &MBeanName{ + Domain: `jboss.jmx`, + Properties: map[string]string{ + "alias": "jmx/rmi/RMIAdaptor/State", + }, + }, + escape: true, }, { - mbean: `Catalina:type=RequestProcessor,worker="http-nio-8080",name=HttpRequest1`, - expected: `Catalina:name=HttpRequest1,type=RequestProcessor,worker="http-nio-8080"`, - ok: true, + mbean: &MBeanName{ + Domain: `Catalina`, + Properties: map[string]string{ + "name": "HttpRequest1", + "type": "RequestProcessor", + "worker": `"http-nio-8080"`, + }, + }, + escape: true, + expected: `Catalina:name=HttpRequest1,type=RequestProcessor,worker=!"http-nio-8080!"`, }, } for _, c := range cases { - canonical, err := canonicalizeMBeanName(c.mbean) - if c.ok { - assert.NoError(t, err, "failed parsing for: "+c.mbean) - assert.Equal(t, c.expected, canonical, "mbean: "+c.mbean) - } else { - assert.Error(t, err, "should have failed for: "+c.mbean) + canonicalString := c.mbean.Canonicalize(c.escape) + + assert.Equal(t, c.expected, canonicalString) + } + +} + +func TestMBeanAttributeHasField(t *testing.T) { + + cases := []struct { + attribute *Attribute + expected bool + }{ + + { + attribute: &Attribute{ + Attr: "CollectionTime", + Field: "", + }, + expected: false, + }, + { + attribute: &Attribute{ + Attr: "CollectionTime", + Field: " ", + }, + + expected: false, + }, + { + attribute: &Attribute{ + Attr: "CollectionTime", + Field: "gc.cms_collection_time", + }, + expected: true, + }, + } + + for _, c := range cases { + jolokiaGETFetcher := &JolokiaHTTPGetFetcher{} + hasField := jolokiaGETFetcher.mBeanAttributeHasField(c.attribute) + + assert.Equal(t, c.expected, hasField, "mbean attribute: "+c.attribute.Attr, "mbean attribute field: "+c.attribute.Field) + } +} + +func TestBuildGETRequestsAndMappings(t *testing.T) { + + cases := []struct { + mappings []JMXMapping + httpMethod string + uris []string + attributeMappings AttributeMapping + ok bool + }{ + { + mappings: []JMXMapping{ + { + + MBean: "java.lang:type=Runtime", + Attributes: []Attribute{ + { + Attr: "Uptime", + Field: "uptime", + }, + }, + Target: Target{ + URL: `service:jmx:rmi:///jndi/rmi://targethost:9999/jmxrmi`, + User: "jolokia", + Password: "password", + }, + }, + { + MBean: "java.lang:type=GarbageCollector,name=ConcurrentMarkSweep", + Attributes: []Attribute{ + { + Attr: "CollectionTime", + Field: "gc.cms_collection_time", + }, + { + Attr: "CollectionCount", + Field: "gc.cms_collection_count", + }, + }, + Target: Target{ + URL: `service:jmx:rmi:///jndi/rmi://targethost:9999/jmxrmi`, + User: "jolokia", + Password: "password", + }, + }, + { + MBean: "java.lang:type=Memory", + Attributes: []Attribute{ + { + Attr: "HeapMemoryUsage", + Field: "memory.heap_usage", + }, + { + Attr: "NonHeapMemoryUsage", + Field: "memory.non_heap_usage", + }, + }, + Target: Target{ + URL: `service:jmx:rmi:///jndi/rmi://targethost:9999/jmxrmi`, + User: "jolokia", + Password: "password", + }, + }, + }, + ok: false, + }, + { + mappings: []JMXMapping{ + { + + MBean: "java.lang:type=Runtime", + Attributes: []Attribute{ + { + Attr: "Uptime", + Field: "uptime", + }, + }, + }, + { + MBean: "java.lang:type=GarbageCollector,name=ConcurrentMarkSweep", + Attributes: []Attribute{ + { + Attr: "CollectionTime", + Field: "gc.cms_collection_time", + }, + { + Attr: "CollectionCount", + Field: "gc.cms_collection_count", + }, + }, + }, + { + MBean: "java.lang:type=Memory", + Attributes: []Attribute{ + { + Attr: "HeapMemoryUsage", + Field: "memory.heap_usage", + }, + { + Attr: "NonHeapMemoryUsage", + Field: "memory.non_heap_usage", + }, + }, + }, + }, + httpMethod: "GET", + uris: []string{ + "/read/java.lang:type=Runtime/Uptime?ignoreErrors=true&canonicalNaming=false", + "/read/java.lang:name=ConcurrentMarkSweep,type=GarbageCollector/CollectionTime,CollectionCount?ignoreErrors=true&canonicalNaming=false", + "/read/java.lang:type=Memory/HeapMemoryUsage,NonHeapMemoryUsage?ignoreErrors=true&canonicalNaming=false", + }, + attributeMappings: map[attributeMappingKey]Attribute{ + attributeMappingKey{"java.lang:type=Runtime", "Uptime"}: Attribute{ + Attr: "Uptime", + Field: "uptime", + }, + attributeMappingKey{"java.lang:name=ConcurrentMarkSweep,type=GarbageCollector", "CollectionTime"}: Attribute{ + Attr: "CollectionTime", + Field: "gc.cms_collection_time", + }, + attributeMappingKey{"java.lang:name=ConcurrentMarkSweep,type=GarbageCollector", "CollectionCount"}: Attribute{ + Attr: "CollectionCount", + Field: "gc.cms_collection_count", + }, + attributeMappingKey{"java.lang:type=Memory", "HeapMemoryUsage"}: Attribute{ + Attr: "HeapMemoryUsage", + Field: "memory.heap_usage", + }, + attributeMappingKey{"java.lang:type=Memory", "NonHeapMemoryUsage"}: Attribute{ + Attr: "NonHeapMemoryUsage", + Field: "memory.non_heap_usage", + }, + }, + ok: true, + }, + } + + for _, c := range cases { + + jolokiaGETFetcher := &JolokiaHTTPGetFetcher{} + + httpReqs, attrMaps, myerr := jolokiaGETFetcher.BuildRequestsAndMappings(c.mappings) + + if c.ok == false { + assert.Error(t, myerr, "should have failed for httpMethod: "+c.httpMethod) + continue } + + assert.Nil(t, myerr) + assert.NotNil(t, attrMaps) + + // Test returned URIs + for i, r := range httpReqs { + assert.Equal(t, c.uris[i], r.URI, "request uri: ", r.URI) + } + + assert.Equal(t, c.attributeMappings, attrMaps) + + } + +} +func TestBuildPOSTRequestsAndMappings(t *testing.T) { + + cases := []struct { + mappings []JMXMapping + httpMethod string + body string + attributeMappings AttributeMapping + }{ + + { + mappings: []JMXMapping{ + { + + MBean: "java.lang:type=Runtime", + Attributes: []Attribute{ + { + Attr: "Uptime", + Field: "uptime", + }, + }, + Target: Target{ + URL: `service:jmx:rmi:///jndi/rmi://targethost:9999/jmxrmi`, + User: "jolokia", + Password: "password", + }, + }, + { + + MBean: "java.lang:type=Runtime", + Attributes: []Attribute{ + { + Attr: "Uptime", + Field: "uptime", + }, + }, + }, { + MBean: "java.lang:type=GarbageCollector,name=ConcurrentMarkSweep", + Attributes: []Attribute{ + { + Attr: "CollectionTime", + Field: "gc.cms_collection_time", + }, + { + Attr: "CollectionCount", + Field: "gc.cms_collection_count", + }, + }, + }, + { + MBean: "java.lang:type=Memory", + Attributes: []Attribute{ + { + Attr: "HeapMemoryUsage", + Field: "memory.heap_usage", + }, + { + Attr: "NonHeapMemoryUsage", + Field: "memory.non_heap_usage", + }, + }, + }, + }, + httpMethod: "POST", + body: `[{"type":"read","mbean":"java.lang:type=Runtime","attribute":["Uptime"],"config":{"canonicalNaming":true,"ignoreErrors":true},"target":{"url":"service:jmx:rmi:///jndi/rmi://targethost:9999/jmxrmi","user":"jolokia","password":"password"}},{"type":"read","mbean":"java.lang:type=Runtime","attribute":["Uptime"],"config":{"canonicalNaming":true,"ignoreErrors":true}},{"type":"read","mbean":"java.lang:name=ConcurrentMarkSweep,type=GarbageCollector","attribute":["CollectionTime","CollectionCount"],"config":{"canonicalNaming":true,"ignoreErrors":true}},{"type":"read","mbean":"java.lang:type=Memory","attribute":["HeapMemoryUsage","NonHeapMemoryUsage"],"config":{"canonicalNaming":true,"ignoreErrors":true}}]`, + attributeMappings: map[attributeMappingKey]Attribute{ + attributeMappingKey{"java.lang:type=Runtime", "Uptime"}: Attribute{ + Attr: "Uptime", + Field: "uptime", + }, + attributeMappingKey{"java.lang:name=ConcurrentMarkSweep,type=GarbageCollector", "CollectionTime"}: Attribute{ + Attr: "CollectionTime", + Field: "gc.cms_collection_time", + }, + attributeMappingKey{"java.lang:name=ConcurrentMarkSweep,type=GarbageCollector", "CollectionCount"}: Attribute{ + Attr: "CollectionCount", + Field: "gc.cms_collection_count", + }, + attributeMappingKey{"java.lang:type=Memory", "HeapMemoryUsage"}: Attribute{ + Attr: "HeapMemoryUsage", + Field: "memory.heap_usage", + }, + attributeMappingKey{"java.lang:type=Memory", "NonHeapMemoryUsage"}: Attribute{ + Attr: "NonHeapMemoryUsage", + Field: "memory.non_heap_usage", + }, + }, + }, + } + + for _, c := range cases { + + jolokiaPOSTBuilder := &JolokiaHTTPPostFetcher{} + + httpReqs, attrMaps, myerr := jolokiaPOSTBuilder.BuildRequestsAndMappings(c.mappings) + + assert.Nil(t, myerr) + assert.NotNil(t, attrMaps) + + // Test returned URIs + for _, r := range httpReqs { + // assert.Equal(t, c.uris[i], r.Uri, "request uri: ", r.Uri) + assert.Equal(t, c.body, string(r.Body), "body", r.Body) + } + + assert.Equal(t, c.attributeMappings, attrMaps) + + } + +} + +func TestNewJolokiaHTTPClient(t *testing.T) { + + cases := []struct { + httpMethod string + expected JolokiaHTTPRequestFetcher + }{ + + { + httpMethod: "GET", + expected: &JolokiaHTTPGetFetcher{}, + }, + { + httpMethod: "", + expected: &JolokiaHTTPPostFetcher{}, + }, + { + httpMethod: "GET", + expected: &JolokiaHTTPGetFetcher{}, + }, + { + httpMethod: "POST", + expected: &JolokiaHTTPPostFetcher{}, + }, + } + + for _, c := range cases { + jolokiaGETClient := NewJolokiaHTTPRequestFetcher(c.httpMethod) + + assert.Equal(t, c.expected, jolokiaGETClient, "httpMethod: "+c.httpMethod) } } diff --git a/metricbeat/module/jolokia/jmx/data.go b/metricbeat/module/jolokia/jmx/data.go index 74a91a0d887e..10e4b477a0ee 100644 --- a/metricbeat/module/jolokia/jmx/data.go +++ b/metricbeat/module/jolokia/jmx/data.go @@ -18,7 +18,6 @@ package jmx import ( - "encoding/json" "strings" "github.com/joeshaw/multierror" @@ -96,11 +95,7 @@ type eventKey struct { mbean, event string } -func eventMapping(content []byte, mapping AttributeMapping) ([]common.MapStr, error) { - var entries []Entry - if err := json.Unmarshal(content, &entries); err != nil { - return nil, errors.Wrapf(err, "failed to unmarshal jolokia JSON response '%v'", string(content)) - } +func eventMapping(entries []Entry, mapping AttributeMapping) ([]common.MapStr, error) { // Generate a different event for each wildcard mbean, and and additional one // for non-wildcard requested mbeans, group them by event name if defined diff --git a/metricbeat/module/jolokia/jmx/data_test.go b/metricbeat/module/jolokia/jmx/data_test.go index 4336bb6a97f7..bbbdb3d765a8 100644 --- a/metricbeat/module/jolokia/jmx/data_test.go +++ b/metricbeat/module/jolokia/jmx/data_test.go @@ -54,7 +54,12 @@ func TestEventMapper(t *testing.T) { Attr: "serverInfo", Field: "server_info"}, } - events, err := eventMapping(jolokiaResponse, mapping) + // Construct a new POST response event mapper + eventMapper := NewJolokiaHTTPRequestFetcher("POST") + + // Map response to Metricbeat events + events, err := eventMapper.EventMapping(jolokiaResponse, mapping) + assert.Nil(t, err) expected := []common.MapStr{ @@ -91,6 +96,8 @@ func TestEventMapper(t *testing.T) { assert.ElementsMatch(t, expected, events) } +// TestEventGroupingMapper tests responses which are returned +// from a Jolokia POST request. func TestEventGroupingMapper(t *testing.T) { absPath, err := filepath.Abs("./_meta/test") @@ -118,7 +125,12 @@ func TestEventGroupingMapper(t *testing.T) { Attr: "serverInfo", Field: "server_info"}, } - events, err := eventMapping(jolokiaResponse, mapping) + // Construct a new POST response event mapper + eventMapper := NewJolokiaHTTPRequestFetcher("POST") + + // Map response to Metricbeat events + events, err := eventMapper.EventMapping(jolokiaResponse, mapping) + assert.Nil(t, err) expected := []common.MapStr{ @@ -159,6 +171,57 @@ func TestEventGroupingMapper(t *testing.T) { assert.ElementsMatch(t, expected, events) } +// TestEventGroupingMapperGetRequest tests responses which are returned +// from a Jolokia GET request. The difference from POST responses is that +// GET method returns a single Entry, whereas POST method returns an array +// of Entry objects +func TestEventGroupingMapperGetRequest(t *testing.T) { + absPath, err := filepath.Abs("./_meta/test") + + assert.NotNil(t, absPath) + assert.Nil(t, err) + + jolokiaResponse, err := ioutil.ReadFile(absPath + "/jolokia_get_response.json") + + assert.Nil(t, err) + + var mapping = AttributeMapping{ + attributeMappingKey{"java.lang:type=Memory", "HeapMemoryUsage"}: Attribute{ + Attr: "HeapMemoryUsage", Field: "memory.heap_usage", Event: "memory"}, + attributeMappingKey{"java.lang:type=Memory", "NonHeapMemoryUsage"}: Attribute{ + Attr: "NonHEapMemoryUsage", Field: "memory.non_heap_usage", Event: "memory"}, + } + + // Construct a new GET response event mapper + eventMapper := NewJolokiaHTTPRequestFetcher("GET") + + // Map response to Metricbeat events + events, err := eventMapper.EventMapping(jolokiaResponse, mapping) + + assert.Nil(t, err) + + expected := []common.MapStr{ + { + "memory": common.MapStr{ + "heap_usage": map[string]interface{}{ + "init": float64(1073741824), + "committed": float64(1037959168), + "max": float64(1037959168), + "used": float64(227420472), + }, + "non_heap_usage": map[string]interface{}{ + "init": float64(2555904), + "committed": float64(53477376), + "max": float64(-1), + "used": float64(50519768), + }, + }, + }, + } + + assert.ElementsMatch(t, expected, events) +} + func TestEventMapperWithWildcard(t *testing.T) { absPath, err := filepath.Abs("./_meta/test") @@ -176,7 +239,11 @@ func TestEventMapperWithWildcard(t *testing.T) { Attr: "maxConnections", Field: "max_connections"}, } - events, err := eventMapping(jolokiaResponse, mapping) + // Construct a new POST response event mapper + eventMapper := NewJolokiaHTTPRequestFetcher("POST") + + // Map response to Metricbeat events + events, err := eventMapper.EventMapping(jolokiaResponse, mapping) assert.Nil(t, err) assert.Equal(t, 2, len(events)) @@ -213,7 +280,11 @@ func TestEventGroupingMapperWithWildcard(t *testing.T) { Attr: "maxConnections", Field: "max_connections", Event: "network"}, } - events, err := eventMapping(jolokiaResponse, mapping) + // Construct a new POST response event mapper + eventMapper := NewJolokiaHTTPRequestFetcher("POST") + + // Map response to Metricbeat events + events, err := eventMapper.EventMapping(jolokiaResponse, mapping) assert.Nil(t, err) assert.Equal(t, 4, len(events)) diff --git a/metricbeat/module/jolokia/jmx/jmx.go b/metricbeat/module/jolokia/jmx/jmx.go index c6f3208941a1..7ad38bfe1dfa 100644 --- a/metricbeat/module/jolokia/jmx/jmx.go +++ b/metricbeat/module/jolokia/jmx/jmx.go @@ -22,7 +22,6 @@ import ( "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/logp" - "github.com/elastic/beats/metricbeat/helper" "github.com/elastic/beats/metricbeat/mb" "github.com/elastic/beats/metricbeat/mb/parse" ) @@ -54,76 +53,55 @@ var ( // MetricSet type defines all fields of the MetricSet type MetricSet struct { mb.BaseMetricSet - mapping AttributeMapping + mapping []JMXMapping namespace string - http *helper.HTTP + http JolokiaHTTPRequestFetcher log *logp.Logger } // New create a new instance of the MetricSet func New(base mb.BaseMetricSet) (mb.MetricSet, error) { config := struct { - Namespace string `config:"namespace" validate:"required"` - Mappings []JMXMapping `config:"jmx.mappings" validate:"required"` + Namespace string `config:"namespace" validate:"required"` + HTTPMethod string `config:"http_method"` + Mappings []JMXMapping `config:"jmx.mappings" validate:"required"` }{} if err := base.Module().UnpackConfig(&config); err != nil { return nil, err } - body, mapping, err := buildRequestBodyAndMapping(config.Mappings) - if err != nil { - return nil, err - } - - http, err := helper.NewHTTP(base) - if err != nil { - return nil, err - } - http.SetMethod("POST") - http.SetBody(body) + jolokiaHTTPBuild := NewJolokiaHTTPRequestFetcher(config.HTTPMethod) log := logp.NewLogger(metricsetName).With("host", base.HostData().Host) - if logp.IsDebug(metricsetName) { - log.Debugw("Jolokia request body", - "body", string(body), "type", "request") - } - return &MetricSet{ BaseMetricSet: base, - mapping: mapping, + mapping: config.Mappings, namespace: config.Namespace, - http: http, + http: jolokiaHTTPBuild, log: log, }, nil } // Fetch methods implements the data gathering and data conversion to the right format func (m *MetricSet) Fetch() ([]common.MapStr, error) { - body, err := m.http.FetchContent() - if err != nil { - return nil, err - } - if logp.IsDebug(metricsetName) { - m.log.Debugw("Jolokia response body", - "host", m.HostData().Host, "body", string(body), "type", "response") - } + var allEvents []common.MapStr - events, err := eventMapping(body, m.mapping) + allEvents, err := m.http.Fetch(m) if err != nil { return nil, err } // Set dynamic namespace. var errs multierror.Errors - for _, event := range events { - _, err = event.Put(mb.NamespaceKey, m.namespace) + for _, event := range allEvents { + _, err := event.Put(mb.NamespaceKey, m.namespace) if err != nil { errs = append(errs, err) } } - return events, errs.Err() + return allEvents, errs.Err() } diff --git a/metricbeat/module/jolokia/jmx/jmx_integration_test.go b/metricbeat/module/jolokia/jmx/jmx_integration_test.go index e66cd516bda3..e8cff3b62cfd 100644 --- a/metricbeat/module/jolokia/jmx/jmx_integration_test.go +++ b/metricbeat/module/jolokia/jmx/jmx_integration_test.go @@ -151,6 +151,41 @@ func getConfigs() []map[string]interface{} { }, }, }, + { + "module": "jolokia", + "metricsets": []string{"jmx"}, + "hosts": []string{getEnvHost() + ":" + getEnvPort()}, + "namespace": "testnamespace", + "http_method": "GET", + "jmx.mappings": []map[string]interface{}{ + { + "mbean": "java.lang:type=GarbageCollector,name=ConcurrentMarkSweep", + "attributes": []map[string]string{ + { + "attr": "CollectionTime", + "field": "gc.cms_collection_time", + }, + { + "attr": "CollectionCount", + "field": "gc.cms_collection_count", + }, + }, + }, + { + "mbean": "java.lang:type=Memory", + "attributes": []map[string]string{ + { + "attr": "HeapMemoryUsage", + "field": "memory.heap_usage", + }, + { + "attr": "NonHeapMemoryUsage", + "field": "memory.non_heap_usage", + }, + }, + }, + }, + }, } }