diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index bfe883a9a9c0..8d169cee2758 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -71,6 +71,7 @@ https://github.com/elastic/beats/compare/v7.0.0-beta1...master[Check the HEAD di - Migrate docker autodiscover to ECS. {issue}10757[10757] {pull}10862[10862] - Fix issue in kubernetes module preventing usage percentages to be properly calculated. {pull}10946[10946] +- Fix parsing error using GET in Jolokia module. {pull}11075[11075] {issue}11071[11071] *Packetbeat* diff --git a/metricbeat/module/jolokia/jmx/_meta/data.json b/metricbeat/module/jolokia/jmx/_meta/data.json index 6716f360e0aa..3fe106a33229 100644 --- a/metricbeat/module/jolokia/jmx/_meta/data.json +++ b/metricbeat/module/jolokia/jmx/_meta/data.json @@ -1,33 +1,38 @@ { "@timestamp": "2017-10-12T08:05:34.853Z", - "beat": { + "agent": { "hostname": "host.example.com", "name": "host.example.com" }, + "event": { + "dataset": "jolokia.testnamespace", + "duration": 115000, + "module": "jolokia" + }, "jolokia": { "testnamespace": { "memory": { "heap_usage": { - "committed": 112721920, - "init": 64673792, - "max": 921174016, - "used": 81023984 + "committed": 514850816, + "init": 536870912, + "max": 7635730432, + "used": 85208288 }, "non_heap_usage": { - "committed": 24576000, - "init": 24576000, - "max": 224395264, - "used": 17684240 + "committed": 30277632, + "init": 2555904, + "max": -1, + "used": 29298840 } }, - "uptime": 580487 + "uptime": 6749672 } }, "metricset": { - "host": "jolokia:8778", - "module": "jolokia", - "name": "jmx", - "namespace": "testnamespace", - "rtt": 115 + "name": "jmx" + }, + "service": { + "address": "127.0.0.1:8778", + "type": "jolokia" } } \ No newline at end of file diff --git a/metricbeat/module/jolokia/jmx/_meta/test/jolokia_get_response_uptime.json b/metricbeat/module/jolokia/jmx/_meta/test/jolokia_get_response_uptime.json new file mode 100644 index 000000000000..25ebf90f0c1a --- /dev/null +++ b/metricbeat/module/jolokia/jmx/_meta/test/jolokia_get_response_uptime.json @@ -0,0 +1,10 @@ +{ + "request":{ + "mbean":"java.lang:type=Runtime", + "attribute":"Uptime", + "type":"read" + }, + "value":88622, + "timestamp":1551739190, + "status":200 +} diff --git a/metricbeat/module/jolokia/jmx/data.go b/metricbeat/module/jolokia/jmx/data.go index 10e4b477a0ee..2ce2c73a8627 100644 --- a/metricbeat/module/jolokia/jmx/data.go +++ b/metricbeat/module/jolokia/jmx/data.go @@ -33,9 +33,10 @@ const ( type Entry struct { Request struct { - Mbean string `json:"mbean"` + Mbean string `json:"mbean"` + Attribute interface{} `json:"attribute"` } - Value map[string]interface{} + Value interface{} } // Map responseBody to common.MapStr @@ -90,7 +91,22 @@ type Entry struct { // "timestamp": 1519409583 // "status": 200, // } -// } +// ] +// +// A response with single value +// +// [ +// { +// "request": { +// "mbean":"java.lang:type=Runtime", +// "attribute":"Uptime", +// "type":"read" +// }, +// "value":88622, +// "timestamp":1551739190, +// "status":200 +// } +// ] type eventKey struct { mbean, event string } @@ -103,32 +119,24 @@ func eventMapping(entries []Entry, mapping AttributeMapping) ([]common.MapStr, e var errs multierror.Errors for _, v := range entries { - hasWildcard := strings.Contains(v.Request.Mbean, "*") - for attribute, value := range v.Value { - if !hasWildcard { - err := parseResponseEntry(v.Request.Mbean, v.Request.Mbean, attribute, value, mbeanEvents, mapping) - if err != nil { - errs = append(errs, err) - } - continue - } - - // If there was a wildcard, we are going to have an additional - // nesting level in response values, and attribute here is going - // to be actually the matching mbean name - values, ok := value.(map[string]interface{}) - if !ok { - errs = append(errs, errors.Errorf("expected map of values for %s", v.Request.Mbean)) - continue - } + if v.Value == nil || v.Request.Attribute == nil { + continue + } - responseMbean := attribute - for attribute, value := range values { - err := parseResponseEntry(v.Request.Mbean, responseMbean, attribute, value, mbeanEvents, mapping) + switch attribute := v.Request.Attribute.(type) { + case string: + switch entryValues := v.Value.(type) { + case float64: + err := parseResponseEntry(v.Request.Mbean, v.Request.Mbean, attribute, entryValues, mbeanEvents, mapping) if err != nil { errs = append(errs, err) } + case map[string]interface{}: + constructEvents(entryValues, v, mbeanEvents, mapping, errs) } + case []interface{}: + entryValues := v.Value.(map[string]interface{}) + constructEvents(entryValues, v, mbeanEvents, mapping, errs) } } @@ -140,6 +148,36 @@ func eventMapping(entries []Entry, mapping AttributeMapping) ([]common.MapStr, e return events, errs.Err() } +func constructEvents(entryValues map[string]interface{}, v Entry, mbeanEvents map[eventKey]common.MapStr, mapping AttributeMapping, errs multierror.Errors) { + hasWildcard := strings.Contains(v.Request.Mbean, "*") + for attribute, value := range entryValues { + if !hasWildcard { + err := parseResponseEntry(v.Request.Mbean, v.Request.Mbean, attribute, value, mbeanEvents, mapping) + if err != nil { + errs = append(errs, err) + } + continue + } + + // If there was a wildcard, we are going to have an additional + // nesting level in response values, and attribute here is going + // to be actually the matching mbean name + values, ok := value.(map[string]interface{}) + if !ok { + errs = append(errs, errors.Errorf("expected map of values for %s", v.Request.Mbean)) + continue + } + + responseMbean := attribute + for attribute, value := range values { + err := parseResponseEntry(v.Request.Mbean, responseMbean, attribute, value, mbeanEvents, mapping) + if err != nil { + errs = append(errs, err) + } + } + } +} + func selectEvent(events map[eventKey]common.MapStr, key eventKey) common.MapStr { event, found := events[key] if !found { @@ -177,13 +215,15 @@ func parseResponseEntry( // In case the attributeValue is a map the keys are dedotted data := attributeValue - c, ok := data.(map[string]interface{}) - if ok { + switch aValue := attributeValue.(type) { + case map[string]interface{}: newData := map[string]interface{}{} - for k, v := range c { + for k, v := range aValue { newData[common.DeDot(k)] = v } data = newData + case float64: + data = aValue } _, err := event.Put(field.Field, data) return err diff --git a/metricbeat/module/jolokia/jmx/data_test.go b/metricbeat/module/jolokia/jmx/data_test.go index bbbdb3d765a8..2fa8dd1429e9 100644 --- a/metricbeat/module/jolokia/jmx/data_test.go +++ b/metricbeat/module/jolokia/jmx/data_test.go @@ -22,7 +22,7 @@ import ( "path/filepath" "testing" - "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/elastic/beats/libbeat/common" ) @@ -30,12 +30,12 @@ import ( func TestEventMapper(t *testing.T) { absPath, err := filepath.Abs("./_meta/test") - assert.NotNil(t, absPath) - assert.Nil(t, err) + require.NotNil(t, absPath) + require.NoError(t, err) jolokiaResponse, err := ioutil.ReadFile(absPath + "/jolokia_response.json") - assert.Nil(t, err) + require.NoError(t, err) var mapping = AttributeMapping{ attributeMappingKey{"java.lang:type=Runtime", "Uptime"}: Attribute{ @@ -60,7 +60,7 @@ func TestEventMapper(t *testing.T) { // Map response to Metricbeat events events, err := eventMapper.EventMapping(jolokiaResponse, mapping) - assert.Nil(t, err) + require.NoError(t, err) expected := []common.MapStr{ { @@ -93,7 +93,7 @@ func TestEventMapper(t *testing.T) { }, } - assert.ElementsMatch(t, expected, events) + require.ElementsMatch(t, expected, events) } // TestEventGroupingMapper tests responses which are returned @@ -101,12 +101,12 @@ func TestEventMapper(t *testing.T) { func TestEventGroupingMapper(t *testing.T) { absPath, err := filepath.Abs("./_meta/test") - assert.NotNil(t, absPath) - assert.Nil(t, err) + require.NotNil(t, absPath) + require.NoError(t, err) jolokiaResponse, err := ioutil.ReadFile(absPath + "/jolokia_response.json") - assert.Nil(t, err) + require.NoError(t, err) var mapping = AttributeMapping{ attributeMappingKey{"java.lang:type=Runtime", "Uptime"}: Attribute{ @@ -131,7 +131,7 @@ func TestEventGroupingMapper(t *testing.T) { // Map response to Metricbeat events events, err := eventMapper.EventMapping(jolokiaResponse, mapping) - assert.Nil(t, err) + require.NoError(t, err) expected := []common.MapStr{ { @@ -168,7 +168,7 @@ func TestEventGroupingMapper(t *testing.T) { }, } - assert.ElementsMatch(t, expected, events) + require.ElementsMatch(t, expected, events) } // TestEventGroupingMapperGetRequest tests responses which are returned @@ -178,12 +178,12 @@ func TestEventGroupingMapper(t *testing.T) { func TestEventGroupingMapperGetRequest(t *testing.T) { absPath, err := filepath.Abs("./_meta/test") - assert.NotNil(t, absPath) - assert.Nil(t, err) + require.NotNil(t, absPath) + require.NoError(t, err) jolokiaResponse, err := ioutil.ReadFile(absPath + "/jolokia_get_response.json") - assert.Nil(t, err) + require.NoError(t, err) var mapping = AttributeMapping{ attributeMappingKey{"java.lang:type=Memory", "HeapMemoryUsage"}: Attribute{ @@ -198,7 +198,7 @@ func TestEventGroupingMapperGetRequest(t *testing.T) { // Map response to Metricbeat events events, err := eventMapper.EventMapping(jolokiaResponse, mapping) - assert.Nil(t, err) + require.NoError(t, err) expected := []common.MapStr{ { @@ -219,18 +219,54 @@ func TestEventGroupingMapperGetRequest(t *testing.T) { }, } - assert.ElementsMatch(t, expected, events) + require.ElementsMatch(t, expected, events) +} + +// TestEventGroupingMapperGetRequestUptime tests responses which are returned +// from a Jolokia GET request and only has one uptime runtime value. +func TestEventGroupingMapperGetRequestUptime(t *testing.T) { + absPath, err := filepath.Abs("./_meta/test") + + require.NotNil(t, absPath) + require.NoError(t, err) + + jolokiaResponse, err := ioutil.ReadFile(absPath + "/jolokia_get_response_uptime.json") + + require.NoError(t, err) + + var mapping = AttributeMapping{ + attributeMappingKey{"java.lang:type=Runtime", "Uptime"}: Attribute{ + Field: "runtime.uptime", Event: "runtime"}, + } + + // Construct a new GET response event mapper + eventMapper := NewJolokiaHTTPRequestFetcher("GET") + + // Map response to Metricbeat events + events, err := eventMapper.EventMapping(jolokiaResponse, mapping) + + require.NoError(t, err) + + expected := []common.MapStr{ + { + "runtime": common.MapStr{ + "uptime": float64(88622), + }, + }, + } + + require.ElementsMatch(t, expected, events) } func TestEventMapperWithWildcard(t *testing.T) { absPath, err := filepath.Abs("./_meta/test") - assert.NotNil(t, absPath) - assert.Nil(t, err) + require.NotNil(t, absPath) + require.NoError(t, err) jolokiaResponse, err := ioutil.ReadFile(absPath + "/jolokia_response_wildcard.json") - assert.Nil(t, err) + require.NoError(t, err) var mapping = AttributeMapping{ attributeMappingKey{"Catalina:name=*,type=ThreadPool", "port"}: Attribute{ @@ -244,8 +280,8 @@ func TestEventMapperWithWildcard(t *testing.T) { // Map response to Metricbeat events events, err := eventMapper.EventMapping(jolokiaResponse, mapping) - assert.Nil(t, err) - assert.Equal(t, 2, len(events)) + require.NoError(t, err) + require.Equal(t, 2, len(events)) expected := []common.MapStr{ { @@ -260,18 +296,18 @@ func TestEventMapperWithWildcard(t *testing.T) { }, } - assert.ElementsMatch(t, expected, events) + require.ElementsMatch(t, expected, events) } func TestEventGroupingMapperWithWildcard(t *testing.T) { absPath, err := filepath.Abs("./_meta/test") - assert.NotNil(t, absPath) - assert.Nil(t, err) + require.NotNil(t, absPath) + require.NoError(t, err) jolokiaResponse, err := ioutil.ReadFile(absPath + "/jolokia_response_wildcard.json") - assert.Nil(t, err) + require.NoError(t, err) var mapping = AttributeMapping{ attributeMappingKey{"Catalina:name=*,type=ThreadPool", "port"}: Attribute{ @@ -285,8 +321,8 @@ func TestEventGroupingMapperWithWildcard(t *testing.T) { // Map response to Metricbeat events events, err := eventMapper.EventMapping(jolokiaResponse, mapping) - assert.Nil(t, err) - assert.Equal(t, 4, len(events)) + require.NoError(t, err) + require.Equal(t, 4, len(events)) expected := []common.MapStr{ { @@ -307,5 +343,5 @@ func TestEventGroupingMapperWithWildcard(t *testing.T) { }, } - assert.ElementsMatch(t, expected, events) + require.ElementsMatch(t, expected, events) } diff --git a/metricbeat/module/jolokia/jmx/jmx_integration_test.go b/metricbeat/module/jolokia/jmx/jmx_integration_test.go index e8cff3b62cfd..1677395e4bb0 100644 --- a/metricbeat/module/jolokia/jmx/jmx_integration_test.go +++ b/metricbeat/module/jolokia/jmx/jmx_integration_test.go @@ -184,6 +184,15 @@ func getConfigs() []map[string]interface{} { }, }, }, + { + "mbean": "java.lang:type=Runtime", + "attributes": []map[string]string{ + { + "attr": "Uptime", + "field": "uptime", + }, + }, + }, }, }, }