Metricbeat: Support Jolokia proxy mode (#6475)

CHANGELOG.asciidoc

@@ -228,6 +228,7 @@ https://github.com/elastic/beats/compare/v6.2.3...master[Check the HEAD diff]
 - Release config reloading feature as GA. {pull}6891[6891]
 - Add experimental Elasticsearch index metricset. {pull}6881[6881]
 - Add dashboards and visualizations for haproxy metrics. {pull}6934[6934]
+- Add Jolokia agent in proxy mode. {pull}6475[6475]
 - Add message rates to the RabbitMQ queue metricset {issue}6442[6442] {pull}6606[6606]
 - Add exchanges metricset to the RabbitMQ module {issue}6442[6442] {pull}6607[6607]
 - Add Elasticsearch index_summary metricset. {pull}6918[6918]

metricbeat/module/jolokia/_meta/Dockerfile

@@ -1,11 +1,31 @@
 # Tomcat is started to fetch Jolokia metrics from it
-FROM jolokia/java-jolokia:7
-ENV TOMCAT_VERSION 7.0.55
+FROM java:8-jdk-alpine
+
+ENV TOMCAT_VERSION 7.0.86
 ENV TC apache-tomcat-${TOMCAT_VERSION}
+ENV JOLOKIA_VERSION 1.5.0
+
+RUN apk update;\
+    apk add curl
 
-HEALTHCHECK --interval=1s --retries=90 CMD curl -f curl localhost:8778/jolokia/
+HEALTHCHECK --interval=1s --retries=90 CMD curl -f localhost:8778/jolokia/
 EXPOSE 8778
-RUN wget -q http://archive.apache.org/dist/tomcat/tomcat-7/v${TOMCAT_VERSION}/bin/${TC}.tar.gz
-RUN tar xzf ${TC}.tar.gz -C /opt
 
-CMD env CATALINA_OPTS=$(jolokia_opts) /opt/${TC}/bin/catalina.sh run
+# Prepare a server where jolokia runs in proxy mode
+RUN wget http://archive.apache.org/dist/tomcat/tomcat-7/v${TOMCAT_VERSION}/bin/${TC}.tar.gz;\
+    tar xzf ${TC}.tar.gz -C /usr;\
+    rm ${TC}.tar.gz;\
+    sed -i -e 's/Connector port="8080"/Connector port="8778"/g' /usr/${TC}/conf/server.xml;\
+    wget http://central.maven.org/maven2/org/jolokia/jolokia-war/${JOLOKIA_VERSION}/jolokia-war-${JOLOKIA_VERSION}.war -O /usr/${TC}/webapps/jolokia.war
+
+# JMX setting to request authentication with remote connection
+RUN echo "monitorRole QED" >> /usr/lib/jvm/java-1.8-openjdk/jre/lib/management/jmxremote.password;\
+    echo "controlRole R&D" >> /usr/lib/jvm/java-1.8-openjdk/jre/lib/management/jmxremote.password;\
+    chmod 600 /usr/lib/jvm/java-1.8-openjdk/jre/lib/management/jmxremote.password
+
+# Start tomcat to accept JMX connection and enable jolokia proxy mode
+CMD env CATALINA_OPTS="$(jolokia_opts)\
+    -Dcom.sun.management.jmxremote.port=7091\
+    -Dcom.sun.management.jmxremote.ssl=false\
+    -Dcom.sun.management.jmxremote.authenticate=true\
+    -Dorg.jolokia.jsr160ProxyEnabled" /usr/${TC}/bin/catalina.sh run

metricbeat/module/jolokia/jmx/_meta/docs.asciidoc

@@ -23,6 +23,10 @@ mapping:
         - attr: Uptime
           field: uptime
           event: uptime
+      target:
+          url: "service:jmx:rmi:///jndi/rmi://targethost:9999/jmxrmi"
+          user: "jolokia"
+          password: "s!cr!t"
 ---
 
 In case the underlying attribute is an object (e.g. see HeapMemoryUsage attribute in java.lang:type=Memory) its

metricbeat/module/jolokia/jmx/_meta/test/config.yml

@@ -55,6 +55,23 @@ metricbeat.modules:
       attributes:
         - attr: Value
           field: compaction.pending_tasks
+
+- module: jolokia
+  metricsets: ["jmx"]
+  enabled: true
+  period: 10s
+  namespace: "jolokia_metrics"
+  hosts: ["localhost:4004"]
+  jmx.mappings:
+    - mbean: 'Catalina:type=Server'
+      attributes:
+        - attr: serverInfo
+          field: server_info
+      target:
+        url: 'service:jmx:rmi:///jndi/rmi://jolokia:7091/jmxrmi'
+        user: 'monitorRole'
+        password: 'QED'
+
 #================================ Outputs =====================================
 
 #-------------------------- Elasticsearch output ------------------------------

metricbeat/module/jolokia/jmx/_meta/test/jolokia_response.json

@@ -71,5 +71,24 @@
         },
         "timestamp": 1472298687,
         "status": 200
+    },
+    {
+        "request": {
+            "mbean": "Catalina:type=Server",
+            "attribute": "serverInfo",
+            "type": "read",
+            "target": {
+                "env": {
+                    "password": "QED",
+                    "user": "monitorRole"
+                },
+                "url": "service:jmx:rmi:///jndi/rmi://localhost:7091/jmxrmi"
+            }
+        },
+        "value": {
+            "serverInfo": "Apache Tomcat/9.0.7"
+        },
+        "timestamp": 1523785646,
+        "status": 200
     }
 ]

metricbeat/module/jolokia/jmx/config.go

@@ -5,6 +5,7 @@ import "encoding/json"
 type JMXMapping struct {
 	MBean      string
 	Attributes []Attribute
+	Target     Target
 }
 
 type Attribute struct {
@@ -13,6 +14,13 @@ type Attribute struct {
 	Event string
 }
 
+// Target inputs the value you want to set for jolokia target block
+type Target struct {
+	URL      string
+	User     string
+	Password string
+}
+
 // RequestBlock is used to build the request blocks of the following format:
 //
 // [
@@ -29,14 +37,33 @@ type Attribute struct {
 //       "attribute":[
 //          "CollectionTime",
 //          "CollectionCount"
-//       ]
+//       ],
+//       "target":{
+//          "url":"service:jmx:rmi:///jndi/rmi://targethost:9999/jmxrmi",
+//          "user":"jolokia",
+//          "password":"s!cr!t"
+//       }
 //    }
 // ]
 type RequestBlock struct {
 	Type      string                 `json:"type"`
 	MBean     string                 `json:"mbean"`
 	Attribute []string               `json:"attribute"`
 	Config    map[string]interface{} `json:"config"`
+	Target    *TargetBlock           `json:"target,omitempty"`
+}
+
+// TargetBlock is used to build the target blocks of the following format into RequestBlock.
+//
+// "target":{
+//    "url":"service:jmx:rmi:///jndi/rmi://targethost:9999/jmxrmi",
+//    "user":"jolokia",
+//    "password":"s!cr!t"
+// }
+type TargetBlock struct {
+	URL      string `json:"url"`
+	User     string `json:"user,omitempty"`
+	Password string `json:"password,omitempty"`
 }
 
 type attributeMappingKey struct {
@@ -68,6 +95,13 @@ func buildRequestBodyAndMapping(mappings []JMXMapping) ([]byte, AttributeMapping
 			Config: config,
 		}
 
+		if len(mapping.Target.URL) != 0 {
+			rb.Target = new(TargetBlock)
+			rb.Target.URL = mapping.Target.URL
+			rb.Target.User = mapping.Target.User
+			rb.Target.Password = mapping.Target.Password
+		}
+
 		for _, attribute := range mapping.Attributes {
 			rb.Attribute = append(rb.Attribute, attribute.Attr)
 			responseMapping[attributeMappingKey{mapping.MBean, attribute.Attr}] = attribute

metricbeat/module/jolokia/jmx/data_test.go

@@ -33,6 +33,8 @@ func TestEventMapper(t *testing.T) {
 			Attr: "NonHEapMemoryUsage", Field: "memory.non_heap_usage"},
 		attributeMappingKey{"org.springframework.boot:type=Endpoint,name=metricsEndpoint", "Metrics"}: Attribute{
 			Attr: "Metrics", Field: "metrics"},
+		attributeMappingKey{"Catalina:type=Server", "serverInfo"}: Attribute{
+			Attr: "serverInfo", Field: "server_info"},
 	}
 
 	events, err := eventMapping(jolokiaResponse, mapping)
@@ -65,6 +67,7 @@ func TestEventMapper(t *testing.T) {
 				"classes_loaded":          float64(19127),
 				"classes_unloaded":        float64(270),
 			},
+			"server_info": "Apache Tomcat/9.0.7",
 		},
 	}
 
@@ -94,6 +97,8 @@ func TestEventGroupingMapper(t *testing.T) {
 			Attr: "NonHEapMemoryUsage", Field: "memory.non_heap_usage", Event: "memory"},
 		attributeMappingKey{"org.springframework.boot:type=Endpoint,name=metricsEndpoint", "Metrics"}: Attribute{
 			Attr: "Metrics", Field: "metrics"},
+		attributeMappingKey{"Catalina:type=Server", "serverInfo"}: Attribute{
+			Attr: "serverInfo", Field: "server_info"},
 	}
 
 	events, err := eventMapping(jolokiaResponse, mapping)
@@ -108,6 +113,7 @@ func TestEventGroupingMapper(t *testing.T) {
 				"classes_loaded":          float64(19127),
 				"classes_unloaded":        float64(270),
 			},
+			"server_info": "Apache Tomcat/9.0.7",
 		},
 		{
 			"gc": common.MapStr{

metricbeat/module/jolokia/jmx/jmx_integration_test.go

@@ -102,6 +102,34 @@ func getConfigs() []map[string]interface{} {
 						},
 					},
 				},
+				{
+					"mbean": "Catalina:type=Server",
+					"attributes": []map[string]string{
+						{
+							"attr":  "serverNumber",
+							"field": "server_number_dosntconnect",
+						},
+					},
+					"target": &TargetBlock{
+						URL:      "service:jmx:rmi:///jndi/rmi://localhost:7091/jmxrmi",
+						User:     "monitorRole",
+						Password: "IGNORE",
+					},
+				},
+				{
+					"mbean": "Catalina:type=Server",
+					"attributes": []map[string]string{
+						{
+							"attr":  "serverInfo",
+							"field": "server_info_proxy",
+						},
+					},
+					"target": &TargetBlock{
+						URL:      "service:jmx:rmi:///jndi/rmi://localhost:7091/jmxrmi",
+						User:     "monitorRole",
+						Password: "QED",
+					},
+				},
 			},
 		},
 	}