From 7ac79aeac021ec0c851eb01e5756914fe486235e Mon Sep 17 00:00:00 2001 From: vas78 Date: Tue, 22 Nov 2016 21:18:56 +0100 Subject: [PATCH] Jolokia Module with dynamic JMX Metricset This is the implementation of a module for Jolokia which contains a dynamic jmx metricset. An example configuration looks as following: ``` - module: jolokia metricsets: ["jmx"] enabled: true period: 1s hosts: ["localhost:8778"] namespace: "metrics" jmx.mappings: - mbean: 'java.lang:type=Runtime' attributes: - attr: Uptime field: uptime - mbean: 'java.lang:type=GarbageCollector,name=ConcurrentMarkSweep' attributes: - attr: CollectionTime field: gc.cms_collection_time - attr: CollectionCount field: gc.cms_collection_count - mbean: 'java.lang:type=Memory' attributes: - attr: HeapMemoryUsage field: memory.heap_usage - attr: NonHeapMemoryUsage field: memory.non_heap_usage ``` For each mbeat the attributes which should be fetched can be defined. The field defines under which field name the event will be put. The namespace defines the metricset namespace. This PR replaces https://github.com/elastic/beats/pull/3051 Further changes: * Added support for method and body to http helper * Handle empty fields in generators. This happens for a module which only contains dynamic metricsets which is currently the case for jolokia. TODO: * [x] Add system tests * [x] Check documentation * [x] Add integration test * [ ] Open issue for metricset which contains basic memory info --- CHANGELOG.asciidoc | 1 + libbeat/scripts/generate_index_pattern.py | 4 + libbeat/scripts/generate_template.py | 4 + libbeat/tests/system/beat/beat.py | 4 + metricbeat/docker-compose.yml | 5 + metricbeat/docs/fields.asciidoc | 15 +++ metricbeat/docs/modules/jolokia.asciidoc | 43 +++++++++ metricbeat/docs/modules/jolokia/jmx.asciidoc | 19 ++++ metricbeat/docs/modules_list.asciidoc | 2 + metricbeat/helper/http.go | 22 ++++- metricbeat/include/list.go | 2 + metricbeat/metricbeat.full.yml | 12 +++ metricbeat/module/jolokia/_meta/Dockerfile | 10 ++ metricbeat/module/jolokia/_meta/config.yml | 10 ++ metricbeat/module/jolokia/_meta/docs.asciidoc | 6 ++ metricbeat/module/jolokia/_meta/env | 2 + metricbeat/module/jolokia/_meta/fields.yml | 13 +++ metricbeat/module/jolokia/doc.go | 4 + metricbeat/module/jolokia/jmx/_meta/data.json | 34 +++++++ .../module/jolokia/jmx/_meta/docs.asciidoc | 49 ++++++++++ .../module/jolokia/jmx/_meta/fields.yml | 0 .../module/jolokia/jmx/_meta/test/config.yml | 68 +++++++++++++ .../jmx/_meta/test/jolokia_response.json | 56 +++++++++++ metricbeat/module/jolokia/jmx/config.go | 61 ++++++++++++ metricbeat/module/jolokia/jmx/data.go | 87 +++++++++++++++++ metricbeat/module/jolokia/jmx/data_test.go | 44 +++++++++ metricbeat/module/jolokia/jmx/jmx.go | 96 +++++++++++++++++++ .../jolokia/jmx/jmx_integration_test.go | 93 ++++++++++++++++++ .../tests/system/config/metricbeat.yml.j2 | 7 ++ metricbeat/tests/system/test_jolokia.py | 44 +++++++++ 30 files changed, 816 insertions(+), 1 deletion(-) create mode 100644 metricbeat/docs/modules/jolokia.asciidoc create mode 100644 metricbeat/docs/modules/jolokia/jmx.asciidoc create mode 100644 metricbeat/module/jolokia/_meta/Dockerfile create mode 100644 metricbeat/module/jolokia/_meta/config.yml create mode 100644 metricbeat/module/jolokia/_meta/docs.asciidoc create mode 100644 metricbeat/module/jolokia/_meta/env create mode 100644 metricbeat/module/jolokia/_meta/fields.yml create mode 100644 metricbeat/module/jolokia/doc.go create mode 100644 metricbeat/module/jolokia/jmx/_meta/data.json create mode 100644 metricbeat/module/jolokia/jmx/_meta/docs.asciidoc create mode 100644 metricbeat/module/jolokia/jmx/_meta/fields.yml create mode 100644 metricbeat/module/jolokia/jmx/_meta/test/config.yml create mode 100644 metricbeat/module/jolokia/jmx/_meta/test/jolokia_response.json create mode 100644 metricbeat/module/jolokia/jmx/config.go create mode 100644 metricbeat/module/jolokia/jmx/data.go create mode 100644 metricbeat/module/jolokia/jmx/data_test.go create mode 100644 metricbeat/module/jolokia/jmx/jmx.go create mode 100644 metricbeat/module/jolokia/jmx/jmx_integration_test.go create mode 100644 metricbeat/tests/system/test_jolokia.py diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 036f7be965b..0c2e5803c7d 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -168,6 +168,7 @@ https://github.com/elastic/beats/compare/v5.1.2...v5.2.0[View commits] - Experimental Prometheus module. {pull}3202[3202] - Add system socket module that reports all TCP sockets. {pull}3246[3246] - Kafka consumer groups metricset. {pull}3240[3240] +- Add jolokia module with dynamic jmx metricset. {pull}3570[3570] *Winlogbeat* diff --git a/libbeat/scripts/generate_index_pattern.py b/libbeat/scripts/generate_index_pattern.py index 94c40a673ed..2e748f35cb1 100644 --- a/libbeat/scripts/generate_index_pattern.py +++ b/libbeat/scripts/generate_index_pattern.py @@ -19,6 +19,10 @@ def fields_to_json(section, path, output): + # Need in case there are no fields + if section["fields"] is None: + section["fields"] = {} + for field in section["fields"]: if path == "": newpath = field["name"] diff --git a/libbeat/scripts/generate_template.py b/libbeat/scripts/generate_template.py index 8abf5d5190c..a27543a327a 100644 --- a/libbeat/scripts/generate_template.py +++ b/libbeat/scripts/generate_template.py @@ -135,6 +135,10 @@ def dedot(group): fields = [] dedotted = {} + # Need in case there are no fields + if group["fields"] is None: + group["fields"] = {} + for field in group["fields"]: if "." in field["name"]: # dedot diff --git a/libbeat/tests/system/beat/beat.py b/libbeat/tests/system/beat/beat.py index e1eb9f24ab6..d3aba834c74 100644 --- a/libbeat/tests/system/beat/beat.py +++ b/libbeat/tests/system/beat/beat.py @@ -390,6 +390,10 @@ def load_fields(self, fields_doc="../../_meta/fields.generated.yml"): def extract_fields(doc_list, name): fields = [] dictfields = [] + + if doc_list is None: + return fields, dictfields + for field in doc_list: # Chain together names diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml index ff41f65678e..626682417f5 100644 --- a/metricbeat/docker-compose.yml +++ b/metricbeat/docker-compose.yml @@ -18,6 +18,7 @@ services: - ceph - couchbase - haproxy + - jolokia - kafka - mongodb - mysql @@ -33,6 +34,7 @@ services: - ${PWD}/module/ceph/_meta/env - ${PWD}/module/couchbase/_meta/env - ${PWD}/module/haproxy/_meta/env + - ${PWD}/module/jolokia/_meta/env - ${PWD}/module/kafka/_meta/env - ${PWD}/module/mongodb/_meta/env - ${PWD}/module/mysql/_meta/env @@ -56,6 +58,9 @@ services: haproxy: build: ${PWD}/module/haproxy/_meta + jolokia: + build: ${PWD}/module/jolokia/_meta + kafka: build: ${PWD}/module/kafka/_meta diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index b228d9cb821..f74dc258851 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -20,6 +20,7 @@ grouped in the following categories: * <> * <> * <> +* <> * <> * <> * <> @@ -2804,6 +2805,20 @@ type: integer The average queue time in ms over the last 1024 requests. +[[exported-fields-jolokia]] +== Jolokia Fields + +[]beta +Jolokia Module + + + +[float] +== jolokia Fields + +jolokia contains metrics exposed via jolokia agent + + [[exported-fields-kafka]] == kafka Fields diff --git a/metricbeat/docs/modules/jolokia.asciidoc b/metricbeat/docs/modules/jolokia.asciidoc new file mode 100644 index 00000000000..ea006397f8c --- /dev/null +++ b/metricbeat/docs/modules/jolokia.asciidoc @@ -0,0 +1,43 @@ +//// +This file is generated! See scripts/docs_collector.py +//// + +[[metricbeat-module-jolokia]] +== Jolokia Module + +beta[] + +This is the Jolokia Module. + + + +[float] +=== Example Configuration + +The Jolokia module supports the standard configuration options that are described +in <>. Here is an example configuration: + +[source,yaml] +---- +metricbeat.modules: +#- module: jolokia +# metricsets: ["jmx"] +# enabled: true +# period: 10s +# hosts: ["localhost"] +# namespace: "metrics" +# path: "/jolokia/?ignoreErrors=true&canonicalNaming=false" +# jmx.mapping: +# jmx.application: +# jmx.instance: +---- + +[float] +=== Metricsets + +The following metricsets are available: + +* <> + +include::jolokia/jmx.asciidoc[] + diff --git a/metricbeat/docs/modules/jolokia/jmx.asciidoc b/metricbeat/docs/modules/jolokia/jmx.asciidoc new file mode 100644 index 00000000000..99290cdf069 --- /dev/null +++ b/metricbeat/docs/modules/jolokia/jmx.asciidoc @@ -0,0 +1,19 @@ +//// +This file is generated! See scripts/docs_collector.py +//// + +[[metricbeat-metricset-jolokia-jmx]] +include::../../../module/jolokia/jmx/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/jolokia/jmx/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index ceecfd51fc5..f56c3d6c1ee 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -7,6 +7,7 @@ This file is generated! See scripts/docs_collector.py * <> * <> * <> + * <> * <> * <> * <> @@ -26,6 +27,7 @@ include::modules/ceph.asciidoc[] include::modules/couchbase.asciidoc[] include::modules/docker.asciidoc[] include::modules/haproxy.asciidoc[] +include::modules/jolokia.asciidoc[] include::modules/kafka.asciidoc[] include::modules/mongodb.asciidoc[] include::modules/mysql.asciidoc[] diff --git a/metricbeat/helper/http.go b/metricbeat/helper/http.go index f3eb49d5d37..d103cad5d79 100644 --- a/metricbeat/helper/http.go +++ b/metricbeat/helper/http.go @@ -5,6 +5,7 @@ import ( "bytes" "encoding/json" "fmt" + "io" "io/ioutil" "net/http" @@ -15,6 +16,8 @@ type HTTP struct { base mb.BaseMetricSet client *http.Client // HTTP client that is reused across requests. headers map[string]string + method string + body []byte } // NewHTTP creates new http helper @@ -23,6 +26,8 @@ func NewHTTP(base mb.BaseMetricSet) *HTTP { base: base, client: &http.Client{Timeout: base.Module().Config().Timeout}, headers: map[string]string{}, + method: "GET", + body: nil, } } @@ -30,7 +35,14 @@ func NewHTTP(base mb.BaseMetricSet) *HTTP { // It's important that resp.Body has to be closed if this method is used. Before using this method // check if one of the other Fetch* methods could be used as they ensure that the Body is properly closed. func (h *HTTP) FetchResponse() (*http.Response, error) { - req, err := http.NewRequest("GET", h.base.HostData().SanitizedURI, nil) + + // Create a fresh reader every time + var reader io.Reader + if h.body != nil { + reader = bytes.NewReader(h.body) + } + + req, err := http.NewRequest(h.method, h.base.HostData().SanitizedURI, reader) if h.base.HostData().User != "" || h.base.HostData().Password != "" { req.SetBasicAuth(h.base.HostData().User, h.base.HostData().Password) } @@ -51,6 +63,14 @@ func (h *HTTP) SetHeader(key, value string) { h.headers[key] = value } +func (h *HTTP) SetMethod(method string) { + h.method = method +} + +func (h *HTTP) SetBody(body []byte) { + h.body = body +} + // FetchContent makes an HTTP request to the configured url and returns the body content. func (h *HTTP) FetchContent() ([]byte, error) { resp, err := h.FetchResponse() diff --git a/metricbeat/include/list.go b/metricbeat/include/list.go index 68f6d060d4b..b1deebf725b 100644 --- a/metricbeat/include/list.go +++ b/metricbeat/include/list.go @@ -31,6 +31,8 @@ import ( _ "github.com/elastic/beats/metricbeat/module/haproxy" _ "github.com/elastic/beats/metricbeat/module/haproxy/info" _ "github.com/elastic/beats/metricbeat/module/haproxy/stat" + _ "github.com/elastic/beats/metricbeat/module/jolokia" + _ "github.com/elastic/beats/metricbeat/module/jolokia/jmx" _ "github.com/elastic/beats/metricbeat/module/kafka" _ "github.com/elastic/beats/metricbeat/module/kafka/consumergroup" _ "github.com/elastic/beats/metricbeat/module/kafka/partition" diff --git a/metricbeat/metricbeat.full.yml b/metricbeat/metricbeat.full.yml index 04649c091d6..7f539dbd9c5 100644 --- a/metricbeat/metricbeat.full.yml +++ b/metricbeat/metricbeat.full.yml @@ -128,6 +128,18 @@ metricbeat.modules: #period: 10s #hosts: ["tcp://127.0.0.1:14567"] +#------------------------------- Jolokia Module ------------------------------ +#- module: jolokia +# metricsets: ["jmx"] +# enabled: true +# period: 10s +# hosts: ["localhost"] +# namespace: "metrics" +# path: "/jolokia/?ignoreErrors=true&canonicalNaming=false" +# jmx.mapping: +# jmx.application: +# jmx.instance: + #-------------------------------- kafka Module ------------------------------- #- module: kafka #metricsets: ["partition"] diff --git a/metricbeat/module/jolokia/_meta/Dockerfile b/metricbeat/module/jolokia/_meta/Dockerfile new file mode 100644 index 00000000000..fdd0e30b806 --- /dev/null +++ b/metricbeat/module/jolokia/_meta/Dockerfile @@ -0,0 +1,10 @@ +# Tomcat is started to fetch Jolokia metrics from it +FROM jolokia/java-jolokia:7 +ENV TOMCAT_VERSION 7.0.55 +ENV TC apache-tomcat-${TOMCAT_VERSION} + +EXPOSE 8778 +RUN wget http://archive.apache.org/dist/tomcat/tomcat-7/v${TOMCAT_VERSION}/bin/${TC}.tar.gz +RUN tar xzf ${TC}.tar.gz -C /opt + +CMD env CATALINA_OPTS=$(jolokia_opts) /opt/${TC}/bin/catalina.sh run diff --git a/metricbeat/module/jolokia/_meta/config.yml b/metricbeat/module/jolokia/_meta/config.yml new file mode 100644 index 00000000000..8658d1f282b --- /dev/null +++ b/metricbeat/module/jolokia/_meta/config.yml @@ -0,0 +1,10 @@ +#- module: jolokia +# metricsets: ["jmx"] +# enabled: true +# period: 10s +# hosts: ["localhost"] +# namespace: "metrics" +# path: "/jolokia/?ignoreErrors=true&canonicalNaming=false" +# jmx.mapping: +# jmx.application: +# jmx.instance: diff --git a/metricbeat/module/jolokia/_meta/docs.asciidoc b/metricbeat/module/jolokia/_meta/docs.asciidoc new file mode 100644 index 00000000000..af59ed28f1c --- /dev/null +++ b/metricbeat/module/jolokia/_meta/docs.asciidoc @@ -0,0 +1,6 @@ +== Jolokia Module + +beta[] + +This is the Jolokia Module. + diff --git a/metricbeat/module/jolokia/_meta/env b/metricbeat/module/jolokia/_meta/env new file mode 100644 index 00000000000..9c0340b6f3c --- /dev/null +++ b/metricbeat/module/jolokia/_meta/env @@ -0,0 +1,2 @@ +JOLOKIA_HOST=jolokia +JOLOKIA_PORT=8778 diff --git a/metricbeat/module/jolokia/_meta/fields.yml b/metricbeat/module/jolokia/_meta/fields.yml new file mode 100644 index 00000000000..dde458c4980 --- /dev/null +++ b/metricbeat/module/jolokia/_meta/fields.yml @@ -0,0 +1,13 @@ +- key: jolokia + title: "Jolokia" + description: > + []beta + + Jolokia Module + short_config: false + fields: + - name: jolokia + type: group + description: > + jolokia contains metrics exposed via jolokia agent + fields: diff --git a/metricbeat/module/jolokia/doc.go b/metricbeat/module/jolokia/doc.go new file mode 100644 index 00000000000..149effe70b0 --- /dev/null +++ b/metricbeat/module/jolokia/doc.go @@ -0,0 +1,4 @@ +/* +Package jolokia is a Metricbeat module that contains MetricSets. +*/ +package jolokia diff --git a/metricbeat/module/jolokia/jmx/_meta/data.json b/metricbeat/module/jolokia/jmx/_meta/data.json new file mode 100644 index 00000000000..2405a1c4da8 --- /dev/null +++ b/metricbeat/module/jolokia/jmx/_meta/data.json @@ -0,0 +1,34 @@ +{ + "@timestamp": "2016-05-23T08:05:34.853Z", + "beat": { + "hostname": "host.example.com", + "name": "host.example.com" + }, + "jolokia": { + "testnamespace": { + "memory": { + "heap_usage": { + "committed": 1.09051904e+08, + "init": 3.2753408e+07, + "max": 6.20756992e+08, + "used": 5.8796168e+07 + }, + "non_heap_usage": { + "committed": 3.244032e+07, + "init": 2.4576e+07, + "max": 2.24395264e+08, + "used": 1.7975176e+07 + } + }, + "uptime": 6.1802139e+07 + } + }, + "metricset": { + "host": "127.0.0.1:8778", + "module": "jolokia", + "name": "jmx", + "namespace": "testnamespace", + "rtt": 115 + }, + "type": "metricsets" +} \ No newline at end of file diff --git a/metricbeat/module/jolokia/jmx/_meta/docs.asciidoc b/metricbeat/module/jolokia/jmx/_meta/docs.asciidoc new file mode 100644 index 00000000000..901bb62a5c0 --- /dev/null +++ b/metricbeat/module/jolokia/jmx/_meta/docs.asciidoc @@ -0,0 +1,49 @@ +=== jolokia jmx MetricSet + +This is the jmx metricset of the module jolokia. + +[float] +=== Features and configuration +Tested with Jolokia 1.3.4. + +Metrics to be collected from each Jolokia instance are defined in the mapping section with an MBean ObjectName and +an array of Attributes to be requested with Elastic field names under which the return values should be saved. + +For example: to get the "Uptime" attribute from the "java.lang:type=Runtime" MBean and map it to something like +"uptime" (actually "jolokia.jmx.uptime", the prexif is added by beats framework) you have to configure following +mapping: + +``` +- module: jolokia + metricsets: ["jmx"] + hosts: ["localhost:8778"] + namespace: "metrics" + jmx.mappings: + - mbean: 'java.lang:type=Runtime' + attributes: + - attr: Uptime + field: uptime +``` + +In case the underlying attribute is an object (e.g. see HeapMemoryUsage attribute in java.lang:type=Memory) it`s +structure will be published to Elastic "as is". + +It is possible to configure nested metric aliases by using dots in the mapping name (e.g. gc.cms_collection_time). For examples please refer to the +/jolokia/jmx/test/config.yml. + +All metrics from a single mapping will be POSTed to the defined host/port and sent to Elastic as a single event. +To make it possible to differentiate between metrics from multiple similar applications running on the same host, +please configure multiple modules. + +It is required to set a namespace in the general module config section. + +[float] +=== Limitations +No authentication against Jolokia is supported yet. No wildcards in Jolokia requests supported yet. +All Jolokia requests have canonicalNaming set to false (details see here: https://jolokia.org/reference/html/protocol.html). + + +[float] +=== Exposed fields, Dashboards, Indexes, etc. +Since this is a very general module that can be tailored for any application that exposes it's metrics over Jolokia, it +comes with no exposed fields description, dashboards or index patterns. diff --git a/metricbeat/module/jolokia/jmx/_meta/fields.yml b/metricbeat/module/jolokia/jmx/_meta/fields.yml new file mode 100644 index 00000000000..e69de29bb2d diff --git a/metricbeat/module/jolokia/jmx/_meta/test/config.yml b/metricbeat/module/jolokia/jmx/_meta/test/config.yml new file mode 100644 index 00000000000..97ccd8f683e --- /dev/null +++ b/metricbeat/module/jolokia/jmx/_meta/test/config.yml @@ -0,0 +1,68 @@ +###################### Metricbeat Configuration Example ####################### + +#========================== Modules configuration ============================ +metricbeat.modules: + +#------------------------------ jolokia Module ----------------------------- +- module: jolokia + metricsets: ["jmx"] + enabled: true + period: 10s + namespace: "jolokia_metrics" + hosts: ["localhost:4008"] + jmx.mappings: + - mbean: 'java.lang:type=Runtime' + attributes: + - attr: Uptime + field: uptime + - mbean: 'java.lang:type=GarbageCollector,name=ConcurrentMarkSweep' + attributes: + - attr: CollectionTime + field: gc.cms_collection_time + - attr: CollectionCount + field: gc.cms_collection_count + - mbean: 'java.lang:type=Memory' + attributes: + - attr: HeapMemoryUsage + field: memory.heap_usage + - attr: NonHeapMemoryUsage + field: memory.non_heap_usage + +- module: jolokia + metricsets: ["jmx"] + enabled: true + period: 10s + namespace: "jolokia_metrics" + hosts: ["localhost:4002"] + jmx.mappings: + - mbean: 'org.apache.cassandra.metrics:type=ClientRequest,scope=Read,name=Latency' + attributes: + - attr: OneMinuteRate + field: client_request.read_latency_one_min_rate + - attr: Count + field: client_request.read_latency + - mbean: 'org.apache.cassandra.metrics:type=ClientRequest,scope=Write,name=Latency' + attributes: + - attr: OneMinuteRate + field: client_request.write_latency_one_min_rate + - attr: Count + field: client_request.write_latency + - mbean: 'org.apache.cassandra.metrics:type=Compaction,name=CompletedTasks' + attributes: + - attr: Value + field: compaction.completed_tasks + - mbean: 'org.apache.cassandra.metrics:type=Compaction,name=PendingTasks' + attributes: + - attr: Value + field: compaction.pending_tasks +#================================ Outputs ===================================== + +#-------------------------- Elasticsearch output ------------------------------ +output.elasticsearch: + # Array of hosts to connect to. + hosts: ["localhost:9200"] + + # Optional protocol and basic auth credentials. + #protocol: "https" + #username: "elastic" + #password: "changeme" diff --git a/metricbeat/module/jolokia/jmx/_meta/test/jolokia_response.json b/metricbeat/module/jolokia/jmx/_meta/test/jolokia_response.json new file mode 100644 index 00000000000..effa8f9fc51 --- /dev/null +++ b/metricbeat/module/jolokia/jmx/_meta/test/jolokia_response.json @@ -0,0 +1,56 @@ +[ + { + "request": { + "mbean": "java.lang:type=Runtime", + "attribute": "Uptime", + "type": "read" + }, + "value": { + "Uptime": 47283 + }, + "timestamp": 1472298687, + "status": 200 + }, + { + "request": { + "mbean": "java.lang:type=GarbageCollector,name=ConcurrentMarkSweep", + "attribute": [ + "CollectionTime", + "CollectionCount" + ], + "type": "read" + }, + "value": { + "CollectionTime": 53, + "CollectionCount": 1 + }, + "timestamp": 1472298687, + "status": 200 + }, + { + "request": { + "mbean": "java.lang:type=Memory", + "attribute": [ + "HeapMemoryUsage", + "NonHeapMemoryUsage" + ], + "type": "read" + }, + "value": { + "HeapMemoryUsage": { + "init": 1073741824, + "committed": 1037959168, + "max": 1037959168, + "used": 227420472 + }, + "NonHeapMemoryUsage": { + "init": 2555904, + "committed": 53477376, + "max": -1, + "used": 50519768 + } + }, + "timestamp": 1472298687, + "status": 200 + } +] diff --git a/metricbeat/module/jolokia/jmx/config.go b/metricbeat/module/jolokia/jmx/config.go new file mode 100644 index 00000000000..26a263e6e97 --- /dev/null +++ b/metricbeat/module/jolokia/jmx/config.go @@ -0,0 +1,61 @@ +package jmx + +import "encoding/json" + +type JMXMapping struct { + MBean string + Attributes []Attribute +} + +type Attribute struct { + Attr string + Field string +} + +// RequestBlock is used to build the request blocks of the following format: +// +// [ +// { +// "type":"read", +// "mbean":"java.lang:type=Runtime", +// "attribute":[ +// "Uptime" +// ] +// }, +// { +// "type":"read", +// "mbean":"java.lang:type=GarbageCollector,name=ConcurrentMarkSweep", +// "attribute":[ +// "CollectionTime", +// "CollectionCount" +// ] +// } +// ] +type RequestBlock struct { + Type string `json:"type"` + MBean string `json:"mbean"` + Attribute []string `json:"attribute"` +} + +func buildRequestBodyAndMapping(mappings []JMXMapping) ([]byte, map[string]string, error) { + + responseMapping := map[string]string{} + blocks := []RequestBlock{} + + for _, mapping := range mappings { + + rb := RequestBlock{ + Type: "read", + MBean: mapping.MBean, + } + + for _, attribute := range mapping.Attributes { + rb.Attribute = append(rb.Attribute, attribute.Attr) + responseMapping[mapping.MBean+"_"+attribute.Attr] = attribute.Field + } + blocks = append(blocks, rb) + } + + content, err := json.Marshal(blocks) + return content, responseMapping, err +} diff --git a/metricbeat/module/jolokia/jmx/data.go b/metricbeat/module/jolokia/jmx/data.go new file mode 100644 index 00000000000..37ca5b5d879 --- /dev/null +++ b/metricbeat/module/jolokia/jmx/data.go @@ -0,0 +1,87 @@ +package jmx + +import ( + "encoding/json" + "fmt" + + "github.com/elastic/beats/libbeat/common" + "github.com/joeshaw/multierror" +) + +type Entry struct { + Request struct { + Mbean string `json:"mbean"` + } + Value map[string]interface{} +} + +// Map responseBody to common.MapStr +// +// A response has the following structure +// [ +// { +// "request": { +// "mbean": "java.lang:type=Memory", +// "attribute": [ +// "HeapMemoryUsage", +// "NonHeapMemoryUsage" +// ], +// "type": "read" +// }, +// "value": { +// "HeapMemoryUsage": { +// "init": 1073741824, +// "committed": 1037959168, +// "max": 1037959168, +// "used": 227420472 +// }, +// "NonHeapMemoryUsage": { +// "init": 2555904, +// "committed": 53477376, +// "max": -1, +// "used": 50519768 +// } +// }, +// "timestamp": 1472298687, +// "status": 200 +// } +// ] +func eventMapping(content []byte, mapping map[string]string) (common.MapStr, error) { + + var entries []Entry + err := json.Unmarshal(content, &entries) + if err != nil { + return nil, fmt.Errorf("Cannot unmarshal json response: %s", err) + } + + event := common.MapStr{} + var errs multierror.Errors + + for _, v := range entries { + for attribute, value := range v.Value { + // Extend existing event + err := parseResponseEntry(v.Request.Mbean, attribute, value, event, mapping) + if err != nil { + errs = append(errs, err) + } + } + } + + return event, errs.Err() + +} + +func parseResponseEntry(mbeanName string, attributeName string, attibuteValue interface{}, + event common.MapStr, mapping map[string]string) error { + + //create metric name by merging mbean and attribute fields + var metricName = mbeanName + "_" + attributeName + + key, exists := mapping[metricName] + if !exists { + return fmt.Errorf("No key found for metric: '%s', skipping...", metricName) + } + + _, err := event.Put(key, attibuteValue) + return err +} diff --git a/metricbeat/module/jolokia/jmx/data_test.go b/metricbeat/module/jolokia/jmx/data_test.go new file mode 100644 index 00000000000..73f00660b19 --- /dev/null +++ b/metricbeat/module/jolokia/jmx/data_test.go @@ -0,0 +1,44 @@ +package jmx + +import ( + "io/ioutil" + "path/filepath" + "testing" + + "github.com/elastic/beats/libbeat/common" + "github.com/stretchr/testify/assert" +) + +func TestEventMapper(t *testing.T) { + absPath, err := filepath.Abs("./_meta/test") + + assert.NotNil(t, absPath) + assert.Nil(t, err) + + jolokiaResponse, err := ioutil.ReadFile(absPath + "/jolokia_response.json") + + assert.Nil(t, err) + + var mapping = map[string]string{ + "java.lang:type=Runtime_Uptime": "uptime", + "java.lang:type=GarbageCollector,name=ConcurrentMarkSweep_CollectionTime": "gc.cms_collection_time", + "java.lang:type=GarbageCollector,name=ConcurrentMarkSweep_CollectionCount": "gc.cms_collection_count", + "java.lang:type=Memory_HeapMemoryUsage": "memory.heap_usage", + "java.lang:type=Memory_NonHeapMemoryUsage": "memory.non_heap_usage", + } + + event, err := eventMapping(jolokiaResponse, mapping) + + assert.Nil(t, err) + assert.EqualValues(t, 47283, event["uptime"]) + assert.EqualValues(t, 53, event["gc"].(common.MapStr)["cms_collection_time"]) + assert.EqualValues(t, 1, event["gc"].(common.MapStr)["cms_collection_count"]) + assert.EqualValues(t, 1073741824, event["memory"].(common.MapStr)["heap_usage"].(map[string]interface{})["init"]) + assert.EqualValues(t, 1037959168, event["memory"].(common.MapStr)["heap_usage"].(map[string]interface{})["committed"]) + assert.EqualValues(t, 1037959168, event["memory"].(common.MapStr)["heap_usage"].(map[string]interface{})["max"]) + assert.EqualValues(t, 227420472, event["memory"].(common.MapStr)["heap_usage"].(map[string]interface{})["used"]) + assert.EqualValues(t, 2555904, event["memory"].(common.MapStr)["non_heap_usage"].(map[string]interface{})["init"]) + assert.EqualValues(t, 53477376, event["memory"].(common.MapStr)["non_heap_usage"].(map[string]interface{})["committed"]) + assert.EqualValues(t, -1, event["memory"].(common.MapStr)["non_heap_usage"].(map[string]interface{})["max"]) + assert.EqualValues(t, 50519768, event["memory"].(common.MapStr)["non_heap_usage"].(map[string]interface{})["used"]) +} diff --git a/metricbeat/module/jolokia/jmx/jmx.go b/metricbeat/module/jolokia/jmx/jmx.go new file mode 100644 index 00000000000..b2b1de38bad --- /dev/null +++ b/metricbeat/module/jolokia/jmx/jmx.go @@ -0,0 +1,96 @@ +package jmx + +import ( + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/logp" + "github.com/elastic/beats/metricbeat/helper" + "github.com/elastic/beats/metricbeat/mb" + "github.com/elastic/beats/metricbeat/mb/parse" +) + +var ( + debugf = logp.MakeDebug("jolokia-jmx") +) + +// init registers the MetricSet with the central registry. +func init() { + if err := mb.Registry.AddMetricSet("jolokia", "jmx", New, hostParser); err != nil { + panic(err) + } +} + +const ( + // defaultScheme is the default scheme to use when it is not specified in + // the host config. + defaultScheme = "http" + + // defaultPath is the default path to the ngx_http_stub_status_module endpoint on Nginx. + defaultPath = "/jolokia/?ignoreErrors=true&canonicalNaming=false" +) + +var ( + hostParser = parse.URLHostParserBuilder{ + DefaultScheme: defaultScheme, + PathConfigKey: "path", + DefaultPath: defaultPath, + }.Build() +) + +// MetricSet type defines all fields of the MetricSet +type MetricSet struct { + mb.BaseMetricSet + mapping map[string]string + namespace string + http *helper.HTTP +} + +// New create a new instance of the MetricSet +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + logp.Warn("BETA: The jolokia jmx metricset is beta") + + // Additional configuration options + config := struct { + Namespace string `config:"namespace" validate:"required"` + Mappings []JMXMapping `config:"jmx.mappings" validate:"required"` + }{} + + if err := base.Module().UnpackConfig(&config); err != nil { + return nil, err + } + + body, mapping, err := buildRequestBodyAndMapping(config.Mappings) + if err != nil { + return nil, err + } + + http := helper.NewHTTP(base) + http.SetMethod("POST") + http.SetBody(body) + + return &MetricSet{ + BaseMetricSet: base, + mapping: mapping, + namespace: config.Namespace, + http: http, + }, nil + +} + +// Fetch methods implements the data gathering and data conversion to the right format +func (m *MetricSet) Fetch() (common.MapStr, error) { + + body, err := m.http.FetchContent() + if err != nil { + return nil, err + } + + event, err := eventMapping(body, m.mapping) + if err != nil { + return nil, err + } + + // Set dynamic namespace + event["_namespace"] = m.namespace + + return event, nil +} diff --git a/metricbeat/module/jolokia/jmx/jmx_integration_test.go b/metricbeat/module/jolokia/jmx/jmx_integration_test.go new file mode 100644 index 00000000000..f3df7327273 --- /dev/null +++ b/metricbeat/module/jolokia/jmx/jmx_integration_test.go @@ -0,0 +1,93 @@ +// +build integration + +package jmx + +import ( + "os" + "testing" + + mbtest "github.com/elastic/beats/metricbeat/mb/testing" + "github.com/stretchr/testify/assert" +) + +func TestFetch(t *testing.T) { + f := mbtest.NewEventFetcher(t, getConfig()) + event, err := f.Fetch() + if !assert.NoError(t, err) { + t.FailNow() + } + + t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event) +} + +func TestData(t *testing.T) { + f := mbtest.NewEventFetcher(t, getConfig()) + err := mbtest.WriteEvent(f, t) + if err != nil { + t.Fatal("write", err) + } +} + +func getConfig() map[string]interface{} { + return map[string]interface{}{ + "module": "jolokia", + "metricsets": []string{"jmx"}, + "hosts": []string{getEnvHost() + ":" + getEnvPort()}, + "namespace": "testnamespace", + "jmx.mappings": []map[string]interface{}{ + { + "mbean": "java.lang:type=Runtime", + "attributes": []map[string]string{ + { + "attr": "Uptime", + "field": "uptime", + }, + }, + }, + { + "mbean": "java.lang:type=GarbageCollector,name=ConcurrentMarkSweep", + "attributes": []map[string]string{ + { + "attr": "CollectionTime", + "field": "gc.cms_collection_time", + }, + { + "attr": "CollectionCount", + "field": "gc.cms_collection_count", + }, + }, + }, + { + "mbean": "java.lang:type=Memory", + "attributes": []map[string]string{ + { + "attr": "HeapMemoryUsage", + "field": "memory.heap_usage", + }, + { + "attr": "NonHeapMemoryUsage", + "field": "memory.non_heap_usage", + }, + }, + }, + }, + } +} + +func getEnvHost() string { + host := os.Getenv("JOLOKIA_HOST") + + if len(host) == 0 { + host = "127.0.0.1" + } + return host +} + +func getEnvPort() string { + port := os.Getenv("JOLOKIA_PORT") + + if len(port) == 0 { + port = "8778" + } + return port +} diff --git a/metricbeat/tests/system/config/metricbeat.yml.j2 b/metricbeat/tests/system/config/metricbeat.yml.j2 index da5a6b15609..4a0fab297e2 100644 --- a/metricbeat/tests/system/config/metricbeat.yml.j2 +++ b/metricbeat/tests/system/config/metricbeat.yml.j2 @@ -41,6 +41,10 @@ metricbeat.modules: timeout: {{ m.timeout }} {% endif -%} + {% if m.namespace -%} + namespace: {{ m.namespace }} + {% endif -%} + {% if m.processes -%} processes: {{ m.processes }} {% endif -%} @@ -78,6 +82,9 @@ metricbeat.modules: {{ k }}: {{ v }} {% endfor %} {% endif -%} + {% if m.additional_content -%} + {{ m.additional_content }} + {% endif -%} {%- endfor %} {% if reload -%} diff --git a/metricbeat/tests/system/test_jolokia.py b/metricbeat/tests/system/test_jolokia.py new file mode 100644 index 00000000000..6c37b46f25a --- /dev/null +++ b/metricbeat/tests/system/test_jolokia.py @@ -0,0 +1,44 @@ +import os +import metricbeat +import unittest +from nose.plugins.attrib import attr + + +class Test(metricbeat.BaseTest): + + @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") + def test_jmx(self): + """ + jolokia jmx metricset test + """ + + additional_content = """ + jmx.mappings: + - mbean: 'java.lang:type=Runtime' + attributes: + - attr: Uptime + field: uptime +""" + + self.render_config_template(modules=[{ + "name": "jolokia", + "metricsets": ["jmx"], + "hosts": self.get_hosts(), + "period": "1s", + "namespace": "test", + "additional_content": additional_content, + }]) + proc = self.start_beat() + self.wait_until(lambda: self.output_lines() > 0, max_timeout=20) + proc.check_kill_and_wait() + + output = self.read_output_json() + self.assertTrue(len(output) >= 1) + evt = output[0] + print(evt) + + assert evt["jolokia"]["test"]["uptime"] > 0 + + def get_hosts(self): + return [os.getenv('JOLOKIA_HOST', 'localhost') + ':' + + os.getenv('JOLOKIA_PORT', '8778')]