[Metricbeat] Add new module for Redis Enterprise (elastic#16482)

Add metricsets: node, proxy

(cherry picked from commit 96bebe4)

CHANGELOG.next.asciidoc

@@ -272,6 +272,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add metricsets based on Ceph Manager Daemon to the `ceph` module. {issue}7723[7723] {pull}16254[16254]
 - Add Load Balancing metricset to GCP {pull}15559[15559]
 - Release `statsd` module as GA. {pull}16447[16447] {issue}14280[14280]
+- Add `redisenterprise` module. {pull}16482[16482] {issue}15269[15269]
 
 *Packetbeat*
 

libbeat/tests/compose/compose.go

@@ -81,6 +81,13 @@ func EnsureUp(t testing.TB, service string, options ...UpOption) HostInfo {
 	// Wait for health
 	err = compose.Wait(upOptions.Timeout, service)
 	if err != nil {
+		inspected, inspectErr := compose.Inspect(service)
+		if inspectErr != nil {
+			t.Logf("inspection error: %v", err)
+		} else {
+			t.Logf("Container state (service: '%s'): %s", service, inspected)
+		}
+
 		t.Fatal(err)
 	}
 

libbeat/tests/compose/project.go

@@ -92,6 +92,7 @@ type Driver interface {
 	Kill(ctx context.Context, signal string, service string) error
 	KillOld(ctx context.Context, except []string) error
 	Ps(ctx context.Context, filter ...string) ([]ContainerStatus, error)
+	Inspect(ctx context.Context, serviceName string) (string, error)
 
 	LockFile() string
 
@@ -227,6 +228,14 @@ func (c *Project) KillOld(except []string) error {
 	return c.Driver.KillOld(context.TODO(), except)
 }
 
+// Inspect a container
+func (c *Project) Inspect(service string) (string, error) {
+	c.Lock()
+	defer c.Unlock()
+
+	return c.Driver.Inspect(context.Background(), service)
+}
+
 // Lock acquires the lock (300s) timeout
 // Normally it should only be seconds that the lock is used, but in some cases it can take longer.
 // Pid is written to the lock file, and it is used to check if process holding the process is still

libbeat/tests/compose/wrapper.go

@@ -21,6 +21,7 @@ import (
 	"archive/tar"
 	"bytes"
 	"context"
+	"encoding/json"
 	"fmt"
 	"net"
 	"os"
@@ -371,6 +372,43 @@ func (d *wrapperDriver) serviceNames(ctx context.Context) ([]string, error) {
 	return strings.Fields(stdout.String()), nil
 }
 
+// Inspect a container.
+func (d *wrapperDriver) Inspect(ctx context.Context, serviceName string) (string, error) {
+	list, err := d.client.ContainerList(ctx, types.ContainerListOptions{All: true})
+	if err != nil {
+		return "", errors.Wrap(err, "listing containers to be inspected")
+	}
+
+	var found bool
+	var c types.Container
+	for _, container := range list {
+		aServiceName, ok := container.Labels[labelComposeService]
+		if ok && serviceName == aServiceName {
+			c = container
+			found = true
+			break
+		}
+	}
+
+	if !found {
+		return "", errors.Errorf("container not found for service '%s'", serviceName)
+	}
+
+	inspect, err := d.client.ContainerInspect(ctx, c.ID)
+	if err != nil {
+		return "", errors.Wrap(err, "container failed inspection")
+	} else if inspect.State == nil {
+		return "empty container state", nil
+	}
+
+	state, err := json.Marshal(inspect.State)
+	if err != nil {
+		return "", errors.Wrap(err, "container inspection failed")
+	}
+
+	return string(state), nil
+}
+
 func makeFilter(project, service string, projectFilter Filter) filters.Args {
 	f := filters.NewArgs()
 	f.Add("label", fmt.Sprintf("%s=%s", labelComposeProject, project))

metricbeat/docs/fields.asciidoc

@@ -66,6 +66,7 @@ grouped in the following categories:
 * <<exported-fields-prometheus>>
 * <<exported-fields-rabbitmq>>
 * <<exported-fields-redis>>
+* <<exported-fields-redisenterprise>>
 * <<exported-fields-sql>>
 * <<exported-fields-stan>>
 * <<exported-fields-statsd>>
@@ -32546,6 +32547,20 @@ type: long
 
 --
 
+[[exported-fields-redisenterprise]]
+== Redis Enterprise fields
+
+Redis metrics collected from Redis Enterprise Server.
+
+
+
+[float]
+=== redisenterprise
+
+`redisenterprise` contains the information and statistics from Redis Enterprise Server.
+
+
+
 [[exported-fields-sql]]
 == sql fields
 

metricbeat/docs/modules/redis.asciidoc

@@ -33,8 +33,8 @@ redis://HOST[:PORT][?password=PASSWORD[&db=DATABASE]]
 [float]
 === Compatibility
 
-The Redis metricsets were tested with Redis 3.2.12, 4.0.11 and 5.0-rc4,  and are expected
-to work with all versions >= 3.0.
+The redis metricsets `info`, `key` and `keyspace` are compatible with all distributions of Redis (OSS and enterprise).
+They were tested with Redis 3.2.12, 4.0.11 and 5.0-rc4, and are expected to work with all versions >= 3.0.
 
 
 [float]

metricbeat/docs/modules/redisenterprise.asciidoc

@@ -0,0 +1,62 @@
+////
+This file is generated! See scripts/mage/docs_collector.go
+////
+
+[[metricbeat-module-redisenterprise]]
+[role="xpack"]
+== Redis Enterprise module
+
+beta[]
+
+This module periodically fetches metrics from https://redislabs.com/redis-enterprise/[Redis Enterprise Software].
+
+The defaut metricsets are `node` and `proxy`.
+
+[float]
+=== Module-specific configuration notes
+
+The Redis module has these additional config options:
+
+*`hosts`*:: URLs that are used to connect to Redis.
+URL format:
+https://HOST[:PORT]
+
+[float]
+=== Compatibility
+
+The metricsets `node` and `proxy` are compatible with Redis Enterprise Software (RES). There were tested with RES
+5.4.10-22 and are expected to work with all versions >= 5.0.2.
+
+
+[float]
+=== Example configuration
+
+The Redis Enterprise module supports the standard configuration options that are described
+in <<configuration-metricbeat>>. Here is an example configuration:
+
+[source,yaml]
+----
+metricbeat.modules:
+- module: redisenterprise
+  metricsets:
+    - node
+    - proxy
+  period: 1m
+
+  # Metrics endpoint
+  hosts: ["https://127.0.0.1:8070/"]
+----
+
+[float]
+=== Metricsets
+
+The following metricsets are available:
+
+* <<metricbeat-metricset-redisenterprise-node,node>>
+
+* <<metricbeat-metricset-redisenterprise-proxy,proxy>>
+
+include::redisenterprise/node.asciidoc[]
+
+include::redisenterprise/proxy.asciidoc[]
+

metricbeat/docs/modules/redisenterprise/node.asciidoc

@@ -0,0 +1,24 @@
+////
+This file is generated! See scripts/mage/docs_collector.go
+////
+
+[[metricbeat-metricset-redisenterprise-node]]
+=== Redis Enterprise node metricset
+
+beta[]
+
+include::../../../../x-pack/metricbeat/module/redisenterprise/node/_meta/docs.asciidoc[]
+
+This is a default metricset. If the host module is unconfigured, this metricset is enabled by default.
+
+==== Fields
+
+For a description of each field in the metricset, see the
+<<exported-fields-redisenterprise,exported fields>> section.
+
+Here is an example document generated by this metricset:
+
+[source,json]
+----
+include::../../../../x-pack/metricbeat/module/redisenterprise/node/_meta/data.json[]
+----

metricbeat/docs/modules/redisenterprise/proxy.asciidoc

@@ -0,0 +1,24 @@
+////
+This file is generated! See scripts/mage/docs_collector.go
+////
+
+[[metricbeat-metricset-redisenterprise-proxy]]
+=== Redis Enterprise proxy metricset
+
+beta[]
+
+include::../../../../x-pack/metricbeat/module/redisenterprise/proxy/_meta/docs.asciidoc[]
+
+This is a default metricset. If the host module is unconfigured, this metricset is enabled by default.
+
+==== Fields
+
+For a description of each field in the metricset, see the
+<<exported-fields-redisenterprise,exported fields>> section.
+
+Here is an example document generated by this metricset:
+
+[source,json]
+----
+include::../../../../x-pack/metricbeat/module/redisenterprise/proxy/_meta/data.json[]
+----

metricbeat/docs/modules_list.asciidoc

@@ -205,6 +205,9 @@ This file is generated! See scripts/mage/docs_collector.go
 .3+| .3+|  |<<metricbeat-metricset-redis-info,info>>   
 |<<metricbeat-metricset-redis-key,key>>   
 |<<metricbeat-metricset-redis-keyspace,keyspace>>   
+|<<metricbeat-module-redisenterprise,Redis Enterprise>>  beta[]   |image:./images/icon-no.png[No prebuilt dashboards]    |  
+.2+| .2+|  |<<metricbeat-metricset-redisenterprise-node,node>> beta[]  
+|<<metricbeat-metricset-redisenterprise-proxy,proxy>> beta[]  
 |<<metricbeat-module-sql,sql>>  beta[]   |image:./images/icon-no.png[No prebuilt dashboards]    |  
 .1+| .1+|  |<<metricbeat-metricset-sql-query,query>> beta[]  
 |<<metricbeat-module-stan,Stan>>  beta[]   |image:./images/icon-yes.png[Prebuilt dashboards are available]    |  
@@ -300,6 +303,7 @@ include::modules/postgresql.asciidoc[]
 include::modules/prometheus.asciidoc[]
 include::modules/rabbitmq.asciidoc[]
 include::modules/redis.asciidoc[]
+include::modules/redisenterprise.asciidoc[]
 include::modules/sql.asciidoc[]
 include::modules/stan.asciidoc[]
 include::modules/statsd.asciidoc[]

metricbeat/module/redis/_meta/docs.asciidoc

@@ -26,5 +26,5 @@ redis://HOST[:PORT][?password=PASSWORD[&db=DATABASE]]
 [float]
 === Compatibility
 
-The Redis metricsets were tested with Redis 3.2.12, 4.0.11 and 5.0-rc4,  and are expected
-to work with all versions >= 3.0.
+The redis metricsets `info`, `key` and `keyspace` are compatible with all distributions of Redis (OSS and enterprise).
+They were tested with Redis 3.2.12, 4.0.11 and 5.0-rc4, and are expected to work with all versions >= 3.0.

metricbeat/module/redis/docker-compose.yml

@@ -7,5 +7,6 @@ services:
       context: ./_meta
       args:
         REDIS_VERSION: ${REDIS_VERSION:-3.2.12}
+    privileged: true
     ports:
       - 6379

metricbeat/module/redis/module.yml

@@ -1,3 +1,4 @@
+name: redis
 dashboards:
-    - id: AV4YjZ5pux-M-tCAunxK
-      file: Metricbeat-redis-overview.json
+  - id: AV4YjZ5pux-M-tCAunxK
+    file: Metricbeat-redis-overview.json

metricbeat/module/redis/test_redis.py

@@ -34,6 +34,7 @@ def test_info(self):
         """
         Test redis info metricset
         """
+
         self.render_config_template(modules=[{
             "name": "redis",
             "metricsets": ["info"],
@@ -137,6 +138,7 @@ def test_module_processors(self):
         """
         Test local processors for Redis info event.
         """
+
         fields = ["clients", "cpu"]
         eventFields = ['beat', 'metricset', 'service', 'event']
         eventFields += ['redis.info.' + f for f in fields]

x-pack/metricbeat/metricbeat.reference.yml

@@ -978,6 +978,16 @@ metricbeat.modules:
   # Redis AUTH password. Empty by default.
   #password: foobared
 
+#--------------------------- Redis Enterprise Module ---------------------------
+- module: redisenterprise
+  metricsets:
+    - node
+    - proxy
+  period: 1m
+
+  # Metrics endpoint
+  hosts: ["https://127.0.0.1:8070/"]
+
 #--------------------------------- Sql Module ---------------------------------
 - module: sql
   metricsets:

x-pack/metricbeat/module/redisenterprise/_meta/Dockerfile

@@ -0,0 +1,6 @@
+ARG REDISENTERPRISE_VERSION
+FROM redislabs/redis:${REDISENTERPRISE_VERSION}
+
+# Wait for the health endpoint to have monitors information
+ADD healthcheck.sh /
+HEALTHCHECK --interval=1s --retries=300 CMD /healthcheck.sh

x-pack/metricbeat/module/redisenterprise/_meta/config.reference.yml

@@ -0,0 +1,8 @@
+- module: redisenterprise
+  metricsets:
+    - node
+    - proxy
+  period: 1m
+
+  # Metrics endpoint
+  hosts: ["https://127.0.0.1:8070/"]

x-pack/metricbeat/module/redisenterprise/_meta/config.yml

@@ -0,0 +1,8 @@
+- module: redisenterprise
+  metricsets:
+    - node
+    - proxy
+  period: 1m
+
+  # Metrics endpoint
+  hosts: ["https://127.0.0.1:8070/"]

x-pack/metricbeat/module/redisenterprise/_meta/docs.asciidoc

@@ -0,0 +1,18 @@
+This module periodically fetches metrics from https://redislabs.com/redis-enterprise/[Redis Enterprise Software].
+
+The defaut metricsets are `node` and `proxy`.
+
+[float]
+=== Module-specific configuration notes
+
+The Redis module has these additional config options:
+
+*`hosts`*:: URLs that are used to connect to Redis.
+URL format:
+https://HOST[:PORT]
+
+[float]
+=== Compatibility
+
+The metricsets `node` and `proxy` are compatible with Redis Enterprise Software (RES). There were tested with RES
+5.4.10-22 and are expected to work with all versions >= 5.0.2.

x-pack/metricbeat/module/redisenterprise/_meta/fields.yml

@@ -0,0 +1,11 @@
+- key: redisenterprise
+  title: "Redis Enterprise"
+  description: >
+    Redis metrics collected from Redis Enterprise Server.
+  release: beta
+  fields:
+    - name: redisenterprise
+      type: group
+      description: >
+        `redisenterprise` contains the information and statistics from Redis Enterprise Server.
+      fields: