Add basic shard metricset to Elasticsearch

CHANGELOG.asciidoc

@@ -232,6 +232,7 @@ https://github.com/elastic/beats/compare/v6.2.3...master[Check the HEAD diff]
 - Add exchanges metricset to the RabbitMQ module {issue}6442[6442] {pull}6607[6607]
 - Add Elasticsearch index_summary metricset. {pull}6918[6918]
 - Add config option `management_path_prefix` for RabbitMQ module to configure management plugin path prefix {issue}6875[6875] {pull}7074[7074]
+- Add shard metricset to Elasticsearch module. {pull}7006[7006]
 
 *Packetbeat*
 

metricbeat/docs/fields.asciidoc

@@ -3337,6 +3337,16 @@ type: keyword
 Elasticsearch cluster id.
 
 
+--
+
+*`elasticsearch.cluster.state.id`*::
++
+--
+type: keyword
+
+Elasticsearch state id.
+
+
 --
 
 [float]
@@ -3951,6 +3961,43 @@ format: bytes
 
 
 
+--
+
+[float]
+== shard fields
+
+shard fields
+
+
+
+*`elasticsearch.shard.primary`*::
++
+--
+type: boolean
+
+True if this is the primary shard.
+
+
+--
+
+*`elasticsearch.shard.number`*::
++
+--
+type: long
+
+The number of this shard.
+
+
+--
+
+*`elasticsearch.shard.state`*::
++
+--
+type: keyword
+
+The state of this shard.
+
+
 --
 
 [[exported-fields-etcd]]

metricbeat/docs/modules/elasticsearch.asciidoc

@@ -23,10 +23,11 @@ in <<configuration-metricbeat>>. Here is an example configuration:
 metricbeat.modules:
 - module: elasticsearch
   metricsets:
-   #- index
-   #- index_summary
-   - node
-   - node_stats
+    - node
+    - node_stats
+    #- index
+    #- index_summary
+    #- shard
   period: 10s
   hosts: ["localhost:9200"]
 ----
@@ -46,6 +47,8 @@ The following metricsets are available:
 
 * <<metricbeat-metricset-elasticsearch-node_stats,node_stats>>
 
+* <<metricbeat-metricset-elasticsearch-shard,shard>>
+
 include::elasticsearch/index.asciidoc[]
 
 include::elasticsearch/index_summary.asciidoc[]
@@ -54,3 +57,5 @@ include::elasticsearch/node.asciidoc[]
 
 include::elasticsearch/node_stats.asciidoc[]
 
+include::elasticsearch/shard.asciidoc[]
+

metricbeat/docs/modules/elasticsearch/shard.asciidoc

@@ -0,0 +1,23 @@
+////
+This file is generated! See scripts/docs_collector.py
+////
+
+[[metricbeat-metricset-elasticsearch-shard]]
+=== Elasticsearch shard metricset
+
+experimental[]
+
+include::../../../module/elasticsearch/shard/_meta/docs.asciidoc[]
+
+
+==== Fields
+
+For a description of each field in the metricset, see the
+<<exported-fields-elasticsearch,exported fields>> section.
+
+Here is an example document generated by this metricset:
+
+[source,json]
+----
+include::../../../module/elasticsearch/shard/_meta/data.json[]
+----

metricbeat/docs/modules_list.asciidoc

@@ -33,10 +33,11 @@ This file is generated! See scripts/docs_collector.py
 |<<metricbeat-module-dropwizard,Dropwizard>>  beta[]   |image:./images/icon-no.png[No prebuilt dashboards]    |  
 .1+| .1+|  |<<metricbeat-metricset-dropwizard-collector,collector>> beta[]  
 |<<metricbeat-module-elasticsearch,Elasticsearch>>  beta[]   |image:./images/icon-no.png[No prebuilt dashboards]    |  
-.4+| .4+|  |<<metricbeat-metricset-elasticsearch-index,index>> experimental[]  
+.5+| .5+|  |<<metricbeat-metricset-elasticsearch-index,index>> experimental[]  
 |<<metricbeat-metricset-elasticsearch-index_summary,index_summary>> experimental[]  
 |<<metricbeat-metricset-elasticsearch-node,node>> beta[]  
 |<<metricbeat-metricset-elasticsearch-node_stats,node_stats>> beta[]  
+|<<metricbeat-metricset-elasticsearch-shard,shard>> experimental[]  
 |<<metricbeat-module-etcd,Etcd>>  beta[]   |image:./images/icon-no.png[No prebuilt dashboards]    |  
 .3+| .3+|  |<<metricbeat-metricset-etcd-leader,leader>> beta[]  
 |<<metricbeat-metricset-etcd-self,self>> beta[]  

metricbeat/include/list.go

@@ -40,6 +40,7 @@ import (
 	_ "github.com/elastic/beats/metricbeat/module/elasticsearch/index_summary"
 	_ "github.com/elastic/beats/metricbeat/module/elasticsearch/node"
 	_ "github.com/elastic/beats/metricbeat/module/elasticsearch/node_stats"
+	_ "github.com/elastic/beats/metricbeat/module/elasticsearch/shard"
 	_ "github.com/elastic/beats/metricbeat/module/etcd"
 	_ "github.com/elastic/beats/metricbeat/module/etcd/leader"
 	_ "github.com/elastic/beats/metricbeat/module/etcd/self"

metricbeat/metricbeat.reference.yml

@@ -196,10 +196,11 @@ metricbeat.modules:
 #---------------------------- Elasticsearch Module ---------------------------
 - module: elasticsearch
   metricsets:
-   #- index
-   #- index_summary
-   - node
-   - node_stats
+    - node
+    - node_stats
+    #- index
+    #- index_summary
+    #- shard
   period: 10s
   hosts: ["localhost:9200"]
 

metricbeat/module/elasticsearch/_meta/config.reference.yml

@@ -1,8 +1,9 @@
 - module: elasticsearch
   metricsets:
-   #- index
-   #- index_summary
-   - node
-   - node_stats
+    - node
+    - node_stats
+    #- index
+    #- index_summary
+    #- shard
   period: 10s
   hosts: ["localhost:9200"]

metricbeat/module/elasticsearch/_meta/fields.yml

@@ -19,3 +19,8 @@
           type: keyword
           description: >
             Elasticsearch cluster id.
+
+        - name: cluster.state.id
+          type: keyword
+          description: >
+            Elasticsearch state id.

metricbeat/module/elasticsearch/elasticsearch_integration_test.go

@@ -17,13 +17,15 @@ import (
 	_ "github.com/elastic/beats/metricbeat/module/elasticsearch/index_summary"
 	_ "github.com/elastic/beats/metricbeat/module/elasticsearch/node"
 	_ "github.com/elastic/beats/metricbeat/module/elasticsearch/node_stats"
+	_ "github.com/elastic/beats/metricbeat/module/elasticsearch/shard"
 )
 
 var metricSets = []string{
 	"index",
 	"index_summary",
 	"node",
 	"node_stats",
+	"shard",
 }
 
 func TestFetch(t *testing.T) {

metricbeat/module/elasticsearch/shard/_meta/data-xpack.json

@@ -0,0 +1,29 @@
+{
+  "_index": ".monitoring-es-6-2018.04.29",
+  "_type": "doc",
+  "_id": "zCHJWMeqT1StL1M28ml_Vw:r4XD9O8eTrCHyN_GJswZ5A:heartbeat-6.0.0-rc1-2018.04.14:0:p",
+  "_score": 2.0136,
+  "_source": {
+    "cluster_uuid": "3zVAmPiRRNK6TYXeqCVbqg",
+    "timestamp": "2018-04-29T00:00:30.108Z",
+    "interval_ms": 10000,
+    "type": "shards",
+    "source_node": {
+      "uuid": "r4XD9O8eTrCHyN_GJswZ5A",
+      "host": "172.25.133.112",
+      "transport_address": "172.25.133.112:19608",
+      "ip": "172.25.133.112",
+      "name": "instance-0000000016",
+      "timestamp": "2018-04-29T00:00:30.073Z"
+    },
+    "state_uuid": "zCHJWMeqT1StL1M28ml_Vw",
+    "shard": {
+      "state": "STARTED",
+      "primary": true,
+      "node": "r4XD9O8eTrCHyN_GJswZ5A",
+      "relocating_node": null,
+      "shard": 0,
+      "index": "heartbeat-6.0.0-rc1-2018.04.14"
+    }
+  }
+}

metricbeat/module/elasticsearch/shard/_meta/data.json

@@ -0,0 +1,35 @@
+{
+    "@timestamp": "2017-10-12T08:05:34.853Z",
+    "beat": {
+        "hostname": "host.example.com",
+        "name": "host.example.com"
+    },
+    "elasticsearch": {
+        "cluster": {
+            "name": "elasticsearch",
+            "state": {
+                "id": "dJaHX6fxSqSVMOsL4QZwwQ"
+            }
+        },
+        "index": {
+            "name": "filebeat-7.0.0-alpha1-2018.05.09"
+        },
+        "node": {
+            "name": "523zXyT6TRWiqXcQItnkyQ"
+        },
+        "shard": {
+            "primary": true,
+            "state": "STARTED"
+        }
+    },
+    "metricset": {
+        "host": "127.0.0.1:9200",
+        "module": "elasticsearch",
+        "name": "shard",
+        "namespace": "elasticsearch.shard",
+        "rtt": 115
+    },
+    "service": {
+        "name": "elasticsearch"
+    }
+}

metricbeat/module/elasticsearch/shard/_meta/docs.asciidoc

@@ -0,0 +1,2 @@
+The `shard` metricset interrogates the
+https://www.elastic.co/guide/en/elasticsearch/reference/6.2/cluster-state.html[Cluster State API endpoint] to fetch information about all shards.

metricbeat/module/elasticsearch/shard/_meta/fields.yml

@@ -0,0 +1,18 @@
+- name: shard
+  type: group
+  description: >
+    shard fields
+  fields:
+    - name: primary
+      type: boolean
+      description: >
+        True if this is the primary shard.
+    - name: number
+      type: long
+      description: >
+        The number of this shard.
+
+    - name: state
+      type: keyword
+      description: >
+        The state of this shard.