Enhance node_stats metricset with index stats (#4077)

Most of the index stats were added to the node_stats metricset. This allows to fetch the metric stats from each node instead of globally. The difference between these stats is that these are node specific. That means if there are multiple shards / replicas the sum of all doc counts from all nodes is not the same as the doc count in all indices. Remove stats metricset as not needed anymore.
elastic · Apr 21, 2017 · 5b71497 · 5b71497
1 parent 796fe56
commit 5b71497
Show file tree

Hide file tree

Showing 24 changed files with 140 additions and 2,425 deletions.
diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc
@@ -1967,6 +1967,55 @@ type: keyword
 Node name.
 
 
+[float]
+== indices Fields
+
+Node indices stats
+
+
+
+[float]
+=== elasticsearch.node_stats.indices.docs.count
+
+type: long
+
+Total number of existing documents.
+
+
+[float]
+=== elasticsearch.node_stats.indices.docs.deleted
+
+type: long
+
+Total number of deleted documents.
+
+
+[float]
+=== elasticsearch.node_stats.indices.segments.count
+
+type: long
+
+Total number of segments.
+
+
+[float]
+=== elasticsearch.node_stats.indices.segments.memory.bytes
+
+type: long
+
+format: bytes
+
+Total size of segments in bytes.
+
+
+[float]
+=== elasticsearch.node_stats.indices.store.size.bytes
+
+type: long
+
+Total size of the store in bytes.
+
+
 [float]
 == jvm.mem.pools Fields
 
@@ -2104,7 +2153,7 @@ format: bytes
 Used bytes.
 
 [float]
-== gc.collectors Fields
+== jvm.gc.collectors Fields
 
 GC collector stats.
 
@@ -2113,19 +2162,19 @@ GC collector stats.
 [float]
 == old.collection Fields
 
-Old colletion gc.
+Old collection gc.
 
 
 
 [float]
-=== elasticsearch.node_stats.gc.collectors.old.collection.count
+=== elasticsearch.node_stats.jvm.gc.collectors.old.collection.count
 
 type: long
 
 
 
 [float]
-=== elasticsearch.node_stats.gc.collectors.old.collection.ms
+=== elasticsearch.node_stats.jvm.gc.collectors.old.collection.ms
 
 type: long
 
@@ -2139,90 +2188,17 @@ Young collection gc.
 
 
 [float]
-=== elasticsearch.node_stats.gc.collectors.young.collection.count
-
-type: long
-
-
-
-[float]
-=== elasticsearch.node_stats.gc.collectors.young.collection.ms
-
-type: long
-
-
-
-[float]
-== stats Fields
-
-stats
-
-
-
-[float]
-=== elasticsearch.stats.docs.count
-
-type: long
-
-Total number of existing documents.
-
-
-[float]
-=== elasticsearch.stats.docs.deleted
-
-type: long
-
-Total number of deleted documents.
-
-
-[float]
-=== elasticsearch.stats.segments.count
-
-type: long
-
-Total number of segments.
-
-
-[float]
-=== elasticsearch.stats.segments.memory.bytes
-
-type: long
-
-format: bytes
-
-Total size of segments in bytes.
-
-
-[float]
-=== elasticsearch.stats.shards.failed
+=== elasticsearch.node_stats.jvm.gc.collectors.young.collection.count
 
 type: long
 
-Number of failed shards.
 
 
 [float]
-=== elasticsearch.stats.shards.successful
+=== elasticsearch.node_stats.jvm.gc.collectors.young.collection.ms
 
 type: long
 
-Number of successful shards.
-
-
-[float]
-=== elasticsearch.stats.shards.total
-
-type: long
-
-Total number of shards.
-
-
-[float]
-=== elasticsearch.stats.store.size.bytes
-
-type: long
-
-Total size of the store in bytes.
 
 
 [[exported-fields-golang]]

diff --git a/metricbeat/docs/modules/elasticsearch.asciidoc b/metricbeat/docs/modules/elasticsearch.asciidoc
@@ -18,11 +18,12 @@ in <<configuration-metricbeat>>. Here is an example configuration:
 ----
 metricbeat.modules:
 - module: elasticsearch
-  metricsets: ["node", "node_stats", "stats"]
+  metricsets: ["node", "node_stats"]
   enabled: false
   period: 10s
   hosts: ["localhost:9200"]
 
+
 ----
 
 [float]
@@ -34,11 +35,7 @@ The following metricsets are available:
 
 * <<metricbeat-metricset-elasticsearch-node_stats,node_stats>>
 
-* <<metricbeat-metricset-elasticsearch-stats,stats>>
-
 include::elasticsearch/node.asciidoc[]
 
 include::elasticsearch/node_stats.asciidoc[]
 
-include::elasticsearch/stats.asciidoc[]
-
diff --git a/metricbeat/docs/modules/elasticsearch/stats.asciidoc b/metricbeat/docs/modules/elasticsearch/stats.asciidoc
diff --git a/metricbeat/include/list.go b/metricbeat/include/list.go
@@ -33,7 +33,6 @@ import (
 	_ "github.com/elastic/beats/metricbeat/module/elasticsearch"
 	_ "github.com/elastic/beats/metricbeat/module/elasticsearch/node"
 	_ "github.com/elastic/beats/metricbeat/module/elasticsearch/node_stats"
-	_ "github.com/elastic/beats/metricbeat/module/elasticsearch/stats"
 	_ "github.com/elastic/beats/metricbeat/module/golang"
 	_ "github.com/elastic/beats/metricbeat/module/golang/expvar"
 	_ "github.com/elastic/beats/metricbeat/module/golang/heap"

diff --git a/metricbeat/metricbeat.full.yml b/metricbeat/metricbeat.full.yml
@@ -135,12 +135,13 @@ metricbeat.modules:
 
 #---------------------------- elasticsearch Module ---------------------------
 - module: elasticsearch
-  metricsets: ["node", "node_stats", "stats"]
+  metricsets: ["node", "node_stats"]
   enabled: false
   period: 10s
   hosts: ["localhost:9200"]
 
 
+
 #------------------------------- golang Module -------------------------------
 - module: golang
   metricsets: ["expvar","heap"]

diff --git a/metricbeat/module/elasticsearch/_meta/config.yml b/metricbeat/module/elasticsearch/_meta/config.yml
@@ -1,6 +1,7 @@
 - module: elasticsearch
-  metricsets: ["node", "node_stats", "stats"]
+  metricsets: ["node", "node_stats"]
   enabled: false
   period: 10s
   hosts: ["localhost:9200"]
 
+
diff --git a/metricbeat/module/elasticsearch/node_stats/_meta/data.json b/metricbeat/module/elasticsearch/node_stats/_meta/data.json
@@ -9,22 +9,39 @@
             "name": "elasticsearch"
         },
         "node": {
-            "name": "hU3Tsxk0TXCAs8vIfBNJ2g"
+            "name": "OIBwp_5vSAG16PyX6B4eUA"
         },
         "node_stats": {
+            "indices": {
+                "docs": {
+                    "count": 2,
+                    "deleted": 0
+                },
+                "segments": {
+                    "count": 2,
+                    "memory": {
+                        "bytes": 3476
+                    }
+                },
+                "store": {
+                    "size": {
+                        "bytes": 8078
+                    }
+                }
+            },
             "jvm": {
                 "gc": {
                     "collectors": {
                         "old": {
                             "collection": {
                                 "count": 1,
-                                "ms": 187
+                                "ms": 71
                             }
                         },
                         "young": {
                             "collection": {
                                 "count": 4,
-                                "ms": 2125
+                                "ms": 193
                             }
                         }
                     }
@@ -36,27 +53,27 @@
                                 "bytes": 362414080
                             },
                             "peak": {
-                                "bytes": 13945280
+                                "bytes": 16151432
                             },
                             "peak_max": {
                                 "bytes": 362414080
                             },
                             "used": {
-                                "bytes": 13945280
+                                "bytes": 16151432
                             }
                         },
                         "survivor": {
                             "max": {
                                 "bytes": 17432576
                             },
                             "peak": {
-                                "bytes": 17432560
+                                "bytes": 17432568
                             },
                             "peak_max": {
                                 "bytes": 17432576
                             },
                             "used": {
-                                "bytes": 17432560
+                                "bytes": 17432568
                             }
                         },
                         "young": {
@@ -70,7 +87,7 @@
                                 "bytes": 139591680
                             },
                             "used": {
-                                "bytes": 137025152
+                                "bytes": 136064336
                             }
                         }
                     }
@@ -79,7 +96,7 @@
         }
     },
     "metricset": {
-        "host": "elasticsearch:9200",
+        "host": "127.0.0.1:9200",
         "module": "elasticsearch",
         "name": "node_stats",
         "rtt": 115

diff --git a/metricbeat/module/elasticsearch/node_stats/_meta/docs.asciidoc b/metricbeat/module/elasticsearch/node_stats/_meta/docs.asciidoc
@@ -1,3 +1,5 @@
 === elasticsearch node_stats MetricSet
 
 This is the node_stats metricset of the module elasticsearch. It fetches data from `/_nodes/_local/stats`. The data received is only for the local node so this metricset has to be run on each elasticsearch node.
+
+Note: The indices stats are node specific. That means for example the total number of docs reported by all nodes together is not the total number of documents in all indices as there can also be replicas.