From e788d184d0e033df1a7911839d0c94b70cac675b Mon Sep 17 00:00:00 2001 From: ruflin Date: Mon, 23 Apr 2018 13:46:40 +0200 Subject: [PATCH] Add basic shard metricset to Elasticsearch --- CHANGELOG.asciidoc | 1 + metricbeat/docs/fields.asciidoc | 47 ++ .../docs/modules/elasticsearch.asciidoc | 13 +- .../docs/modules/elasticsearch/shard.asciidoc | 23 + metricbeat/docs/modules_list.asciidoc | 3 +- metricbeat/include/list.go | 1 + metricbeat/metricbeat.reference.yml | 9 +- .../elasticsearch/_meta/config.reference.yml | 9 +- .../module/elasticsearch/_meta/fields.yml | 5 + .../elasticsearch_integration_test.go | 2 + .../elasticsearch/shard/_meta/data-xpack.json | 29 + .../elasticsearch/shard/_meta/data.json | 35 + .../elasticsearch/shard/_meta/docs.asciidoc | 2 + .../elasticsearch/shard/_meta/fields.yml | 18 + .../shard/_meta/test/routing_table.623.json | 689 ++++++++++++++++++ metricbeat/module/elasticsearch/shard/data.go | 63 ++ .../module/elasticsearch/shard/data_test.go | 27 + .../module/elasticsearch/shard/shard.go | 56 ++ .../elasticsearch/test_elasticsearch.py | 1 + 19 files changed, 1020 insertions(+), 13 deletions(-) create mode 100644 metricbeat/docs/modules/elasticsearch/shard.asciidoc create mode 100644 metricbeat/module/elasticsearch/shard/_meta/data-xpack.json create mode 100644 metricbeat/module/elasticsearch/shard/_meta/data.json create mode 100644 metricbeat/module/elasticsearch/shard/_meta/docs.asciidoc create mode 100644 metricbeat/module/elasticsearch/shard/_meta/fields.yml create mode 100644 metricbeat/module/elasticsearch/shard/_meta/test/routing_table.623.json create mode 100644 metricbeat/module/elasticsearch/shard/data.go create mode 100644 metricbeat/module/elasticsearch/shard/data_test.go create mode 100644 metricbeat/module/elasticsearch/shard/shard.go diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 1da7080e1a1..9c65179a4cb 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -232,6 +232,7 @@ https://github.com/elastic/beats/compare/v6.2.3...master[Check the HEAD diff] - Add exchanges metricset to the RabbitMQ module {issue}6442[6442] {pull}6607[6607] - Add Elasticsearch index_summary metricset. {pull}6918[6918] - Add config option `management_path_prefix` for RabbitMQ module to configure management plugin path prefix {issue}6875[6875] {pull}7074[7074] +- Add shard metricset to Elasticsearch module. {pull}7006[7006] *Packetbeat* diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 211fc09d724..314eef061fd 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -3337,6 +3337,16 @@ type: keyword Elasticsearch cluster id. +-- + +*`elasticsearch.cluster.state.id`*:: ++ +-- +type: keyword + +Elasticsearch state id. + + -- [float] @@ -3951,6 +3961,43 @@ format: bytes +-- + +[float] +== shard fields + +shard fields + + + +*`elasticsearch.shard.primary`*:: ++ +-- +type: boolean + +True if this is the primary shard. + + +-- + +*`elasticsearch.shard.number`*:: ++ +-- +type: long + +The number of this shard. + + +-- + +*`elasticsearch.shard.state`*:: ++ +-- +type: keyword + +The state of this shard. + + -- [[exported-fields-etcd]] diff --git a/metricbeat/docs/modules/elasticsearch.asciidoc b/metricbeat/docs/modules/elasticsearch.asciidoc index ca64afc030e..8dffe6ed93e 100644 --- a/metricbeat/docs/modules/elasticsearch.asciidoc +++ b/metricbeat/docs/modules/elasticsearch.asciidoc @@ -23,10 +23,11 @@ in <>. Here is an example configuration: metricbeat.modules: - module: elasticsearch metricsets: - #- index - #- index_summary - - node - - node_stats + - node + - node_stats + #- index + #- index_summary + #- shard period: 10s hosts: ["localhost:9200"] ---- @@ -46,6 +47,8 @@ The following metricsets are available: * <> +* <> + include::elasticsearch/index.asciidoc[] include::elasticsearch/index_summary.asciidoc[] @@ -54,3 +57,5 @@ include::elasticsearch/node.asciidoc[] include::elasticsearch/node_stats.asciidoc[] +include::elasticsearch/shard.asciidoc[] + diff --git a/metricbeat/docs/modules/elasticsearch/shard.asciidoc b/metricbeat/docs/modules/elasticsearch/shard.asciidoc new file mode 100644 index 00000000000..c463610725a --- /dev/null +++ b/metricbeat/docs/modules/elasticsearch/shard.asciidoc @@ -0,0 +1,23 @@ +//// +This file is generated! See scripts/docs_collector.py +//// + +[[metricbeat-metricset-elasticsearch-shard]] +=== Elasticsearch shard metricset + +experimental[] + +include::../../../module/elasticsearch/shard/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/elasticsearch/shard/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index 9fdebb25334..4b454663760 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -33,10 +33,11 @@ This file is generated! See scripts/docs_collector.py |<> beta[] |image:./images/icon-no.png[No prebuilt dashboards] | .1+| .1+| |<> beta[] |<> beta[] |image:./images/icon-no.png[No prebuilt dashboards] | -.4+| .4+| |<> experimental[] +.5+| .5+| |<> experimental[] |<> experimental[] |<> beta[] |<> beta[] +|<> experimental[] |<> beta[] |image:./images/icon-no.png[No prebuilt dashboards] | .3+| .3+| |<> beta[] |<> beta[] diff --git a/metricbeat/include/list.go b/metricbeat/include/list.go index f8d7535572f..cd2892f7749 100644 --- a/metricbeat/include/list.go +++ b/metricbeat/include/list.go @@ -40,6 +40,7 @@ import ( _ "github.com/elastic/beats/metricbeat/module/elasticsearch/index_summary" _ "github.com/elastic/beats/metricbeat/module/elasticsearch/node" _ "github.com/elastic/beats/metricbeat/module/elasticsearch/node_stats" + _ "github.com/elastic/beats/metricbeat/module/elasticsearch/shard" _ "github.com/elastic/beats/metricbeat/module/etcd" _ "github.com/elastic/beats/metricbeat/module/etcd/leader" _ "github.com/elastic/beats/metricbeat/module/etcd/self" diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 0bb8f8bec92..01995bfa5de 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -196,10 +196,11 @@ metricbeat.modules: #---------------------------- Elasticsearch Module --------------------------- - module: elasticsearch metricsets: - #- index - #- index_summary - - node - - node_stats + - node + - node_stats + #- index + #- index_summary + #- shard period: 10s hosts: ["localhost:9200"] diff --git a/metricbeat/module/elasticsearch/_meta/config.reference.yml b/metricbeat/module/elasticsearch/_meta/config.reference.yml index cb730d3493f..3a322d4d682 100644 --- a/metricbeat/module/elasticsearch/_meta/config.reference.yml +++ b/metricbeat/module/elasticsearch/_meta/config.reference.yml @@ -1,8 +1,9 @@ - module: elasticsearch metricsets: - #- index - #- index_summary - - node - - node_stats + - node + - node_stats + #- index + #- index_summary + #- shard period: 10s hosts: ["localhost:9200"] diff --git a/metricbeat/module/elasticsearch/_meta/fields.yml b/metricbeat/module/elasticsearch/_meta/fields.yml index 55a55bf8a9f..377b39d0843 100644 --- a/metricbeat/module/elasticsearch/_meta/fields.yml +++ b/metricbeat/module/elasticsearch/_meta/fields.yml @@ -19,3 +19,8 @@ type: keyword description: > Elasticsearch cluster id. + + - name: cluster.state.id + type: keyword + description: > + Elasticsearch state id. diff --git a/metricbeat/module/elasticsearch/elasticsearch_integration_test.go b/metricbeat/module/elasticsearch/elasticsearch_integration_test.go index dcdd93a5a63..9b15da53433 100644 --- a/metricbeat/module/elasticsearch/elasticsearch_integration_test.go +++ b/metricbeat/module/elasticsearch/elasticsearch_integration_test.go @@ -17,6 +17,7 @@ import ( _ "github.com/elastic/beats/metricbeat/module/elasticsearch/index_summary" _ "github.com/elastic/beats/metricbeat/module/elasticsearch/node" _ "github.com/elastic/beats/metricbeat/module/elasticsearch/node_stats" + _ "github.com/elastic/beats/metricbeat/module/elasticsearch/shard" ) var metricSets = []string{ @@ -24,6 +25,7 @@ var metricSets = []string{ "index_summary", "node", "node_stats", + "shard", } func TestFetch(t *testing.T) { diff --git a/metricbeat/module/elasticsearch/shard/_meta/data-xpack.json b/metricbeat/module/elasticsearch/shard/_meta/data-xpack.json new file mode 100644 index 00000000000..e7c4eb03c09 --- /dev/null +++ b/metricbeat/module/elasticsearch/shard/_meta/data-xpack.json @@ -0,0 +1,29 @@ +{ + "_index": ".monitoring-es-6-2018.04.29", + "_type": "doc", + "_id": "zCHJWMeqT1StL1M28ml_Vw:r4XD9O8eTrCHyN_GJswZ5A:heartbeat-6.0.0-rc1-2018.04.14:0:p", + "_score": 2.0136, + "_source": { + "cluster_uuid": "3zVAmPiRRNK6TYXeqCVbqg", + "timestamp": "2018-04-29T00:00:30.108Z", + "interval_ms": 10000, + "type": "shards", + "source_node": { + "uuid": "r4XD9O8eTrCHyN_GJswZ5A", + "host": "172.25.133.112", + "transport_address": "172.25.133.112:19608", + "ip": "172.25.133.112", + "name": "instance-0000000016", + "timestamp": "2018-04-29T00:00:30.073Z" + }, + "state_uuid": "zCHJWMeqT1StL1M28ml_Vw", + "shard": { + "state": "STARTED", + "primary": true, + "node": "r4XD9O8eTrCHyN_GJswZ5A", + "relocating_node": null, + "shard": 0, + "index": "heartbeat-6.0.0-rc1-2018.04.14" + } + } +} diff --git a/metricbeat/module/elasticsearch/shard/_meta/data.json b/metricbeat/module/elasticsearch/shard/_meta/data.json new file mode 100644 index 00000000000..23f198b6569 --- /dev/null +++ b/metricbeat/module/elasticsearch/shard/_meta/data.json @@ -0,0 +1,35 @@ +{ + "@timestamp": "2017-10-12T08:05:34.853Z", + "beat": { + "hostname": "host.example.com", + "name": "host.example.com" + }, + "elasticsearch": { + "cluster": { + "name": "elasticsearch", + "state": { + "id": "dJaHX6fxSqSVMOsL4QZwwQ" + } + }, + "index": { + "name": "filebeat-7.0.0-alpha1-2018.05.09" + }, + "node": { + "name": "523zXyT6TRWiqXcQItnkyQ" + }, + "shard": { + "primary": true, + "state": "STARTED" + } + }, + "metricset": { + "host": "127.0.0.1:9200", + "module": "elasticsearch", + "name": "shard", + "namespace": "elasticsearch.shard", + "rtt": 115 + }, + "service": { + "name": "elasticsearch" + } +} \ No newline at end of file diff --git a/metricbeat/module/elasticsearch/shard/_meta/docs.asciidoc b/metricbeat/module/elasticsearch/shard/_meta/docs.asciidoc new file mode 100644 index 00000000000..36ff6f31222 --- /dev/null +++ b/metricbeat/module/elasticsearch/shard/_meta/docs.asciidoc @@ -0,0 +1,2 @@ +The `shard` metricset interrogates the +https://www.elastic.co/guide/en/elasticsearch/reference/6.2/cluster-state.html[Cluster State API endpoint] to fetch information about all shards. diff --git a/metricbeat/module/elasticsearch/shard/_meta/fields.yml b/metricbeat/module/elasticsearch/shard/_meta/fields.yml new file mode 100644 index 00000000000..ee4c7f281fc --- /dev/null +++ b/metricbeat/module/elasticsearch/shard/_meta/fields.yml @@ -0,0 +1,18 @@ +- name: shard + type: group + description: > + shard fields + fields: + - name: primary + type: boolean + description: > + True if this is the primary shard. + - name: number + type: long + description: > + The number of this shard. + + - name: state + type: keyword + description: > + The state of this shard. diff --git a/metricbeat/module/elasticsearch/shard/_meta/test/routing_table.623.json b/metricbeat/module/elasticsearch/shard/_meta/test/routing_table.623.json new file mode 100644 index 00000000000..d9754c399a0 --- /dev/null +++ b/metricbeat/module/elasticsearch/shard/_meta/test/routing_table.623.json @@ -0,0 +1,689 @@ +{ + "cluster_name": "docker-cluster", + "compressed_size_in_bytes": 49692, + "routing_table": { + "indices": { + ".watcher-history-7-2018.04.17": { + "shards": { + "0": [ + { + "state": "STARTED", + "primary": true, + "node": "VnS118cKS7-8mz8l2jZZKQ", + "relocating_node": null, + "shard": 0, + "index": ".watcher-history-7-2018.04.17", + "allocation_id": { + "id": "ySc29b2lREq85LpZZ6tQqQ" + } + } + ] + } + }, + ".watcher-history-7-2018.04.20": { + "shards": { + "0": [ + { + "state": "STARTED", + "primary": true, + "node": "VnS118cKS7-8mz8l2jZZKQ", + "relocating_node": null, + "shard": 0, + "index": ".watcher-history-7-2018.04.20", + "allocation_id": { + "id": "2Mb2YYsEQkKHSYx_BWmziQ" + } + } + ] + } + }, + ".monitoring-es-6-2018.04.17": { + "shards": { + "0": [ + { + "state": "STARTED", + "primary": true, + "node": "VnS118cKS7-8mz8l2jZZKQ", + "relocating_node": null, + "shard": 0, + "index": ".monitoring-es-6-2018.04.17", + "allocation_id": { + "id": "d9nN0jCSQK6o6eVd6asI2A" + } + } + ] + } + }, + ".monitoring-es-6-2018.04.21": { + "shards": { + "0": [ + { + "state": "STARTED", + "primary": true, + "node": "VnS118cKS7-8mz8l2jZZKQ", + "relocating_node": null, + "shard": 0, + "index": ".monitoring-es-6-2018.04.21", + "allocation_id": { + "id": "-NKgVp_iTQqrbsBGl4B5sQ" + } + } + ] + } + }, + ".triggered_watches": { + "shards": { + "0": [ + { + "state": "STARTED", + "primary": true, + "node": "VnS118cKS7-8mz8l2jZZKQ", + "relocating_node": null, + "shard": 0, + "index": ".triggered_watches", + "allocation_id": { + "id": "hpGM4kcnRw2XSO9kx0PzzA" + } + } + ] + } + }, + ".monitoring-es-6-2018.04.23": { + "shards": { + "0": [ + { + "state": "STARTED", + "primary": true, + "node": "VnS118cKS7-8mz8l2jZZKQ", + "relocating_node": null, + "shard": 0, + "index": ".monitoring-es-6-2018.04.23", + "allocation_id": { + "id": "5rCyAFEPT8u6Rt4CACrhUA" + } + } + ] + } + }, + "testindex": { + "shards": { + "0": [ + { + "state": "STARTED", + "primary": true, + "node": "VnS118cKS7-8mz8l2jZZKQ", + "relocating_node": null, + "shard": 0, + "index": "testindex", + "allocation_id": { + "id": "pEwU3PKtTE2nWFOJ1UnzMA" + } + }, + { + "state": "UNASSIGNED", + "primary": false, + "node": null, + "relocating_node": null, + "shard": 0, + "index": "testindex", + "recovery_source": { + "type": "PEER" + }, + "unassigned_info": { + "reason": "INDEX_CREATED", + "at": "2018-04-17T09:06:31.523Z", + "delayed": false, + "allocation_status": "no_attempt" + } + } + ], + "1": [ + { + "state": "STARTED", + "primary": true, + "node": "VnS118cKS7-8mz8l2jZZKQ", + "relocating_node": null, + "shard": 1, + "index": "testindex", + "allocation_id": { + "id": "OQCrOwsTQb68kfI6wvfvfA" + } + }, + { + "state": "UNASSIGNED", + "primary": false, + "node": null, + "relocating_node": null, + "shard": 1, + "index": "testindex", + "recovery_source": { + "type": "PEER" + }, + "unassigned_info": { + "reason": "INDEX_CREATED", + "at": "2018-04-17T09:06:31.523Z", + "delayed": false, + "allocation_status": "no_attempt" + } + } + ], + "2": [ + { + "state": "STARTED", + "primary": true, + "node": "VnS118cKS7-8mz8l2jZZKQ", + "relocating_node": null, + "shard": 2, + "index": "testindex", + "allocation_id": { + "id": "KU2vy0mHT3yP5cTVFAxc9A" + } + }, + { + "state": "UNASSIGNED", + "primary": false, + "node": null, + "relocating_node": null, + "shard": 2, + "index": "testindex", + "recovery_source": { + "type": "PEER" + }, + "unassigned_info": { + "reason": "INDEX_CREATED", + "at": "2018-04-17T09:06:31.523Z", + "delayed": false, + "allocation_status": "no_attempt" + } + } + ], + "3": [ + { + "state": "STARTED", + "primary": true, + "node": "VnS118cKS7-8mz8l2jZZKQ", + "relocating_node": null, + "shard": 3, + "index": "testindex", + "allocation_id": { + "id": "f6jJdqh9SQe2SRybz0mN2A" + } + }, + { + "state": "UNASSIGNED", + "primary": false, + "node": null, + "relocating_node": null, + "shard": 3, + "index": "testindex", + "recovery_source": { + "type": "PEER" + }, + "unassigned_info": { + "reason": "INDEX_CREATED", + "at": "2018-04-17T09:06:31.523Z", + "delayed": false, + "allocation_status": "no_attempt" + } + } + ], + "4": [ + { + "state": "STARTED", + "primary": true, + "node": "VnS118cKS7-8mz8l2jZZKQ", + "relocating_node": null, + "shard": 4, + "index": "testindex", + "allocation_id": { + "id": "k36nAlsmTC2N1cJF4NAd7A" + } + }, + { + "state": "UNASSIGNED", + "primary": false, + "node": null, + "relocating_node": null, + "shard": 4, + "index": "testindex", + "recovery_source": { + "type": "PEER" + }, + "unassigned_info": { + "reason": "INDEX_CREATED", + "at": "2018-04-17T09:06:31.523Z", + "delayed": false, + "allocation_status": "no_attempt" + } + } + ] + } + }, + ".monitoring-es-6-2018.04.20": { + "shards": { + "0": [ + { + "state": "STARTED", + "primary": true, + "node": "VnS118cKS7-8mz8l2jZZKQ", + "relocating_node": null, + "shard": 0, + "index": ".monitoring-es-6-2018.04.20", + "allocation_id": { + "id": "k4E-0Tb4Tu26OJVFapPqYA" + } + } + ] + } + }, + ".monitoring-es-6-2018.04.18": { + "shards": { + "0": [ + { + "state": "STARTED", + "primary": true, + "node": "VnS118cKS7-8mz8l2jZZKQ", + "relocating_node": null, + "shard": 0, + "index": ".monitoring-es-6-2018.04.18", + "allocation_id": { + "id": "zuzpxvQ6SXaJC6P_KdS5Hw" + } + } + ] + } + }, + ".watcher-history-7-2018.04.19": { + "shards": { + "0": [ + { + "state": "STARTED", + "primary": true, + "node": "VnS118cKS7-8mz8l2jZZKQ", + "relocating_node": null, + "shard": 0, + "index": ".watcher-history-7-2018.04.19", + "allocation_id": { + "id": "5LMJgFYESsSGqe8znMRvHw" + } + } + ] + } + }, + ".monitoring-logstash-6-2018.04.23": { + "shards": { + "0": [ + { + "state": "STARTED", + "primary": true, + "node": "VnS118cKS7-8mz8l2jZZKQ", + "relocating_node": null, + "shard": 0, + "index": ".monitoring-logstash-6-2018.04.23", + "allocation_id": { + "id": "ruhKHCXnSeCs3XF36V2Cfw" + } + } + ] + } + }, + ".watcher-history-7-2018.04.21": { + "shards": { + "0": [ + { + "state": "STARTED", + "primary": true, + "node": "VnS118cKS7-8mz8l2jZZKQ", + "relocating_node": null, + "shard": 0, + "index": ".watcher-history-7-2018.04.21", + "allocation_id": { + "id": "N4vZGytQRy2pz8u9V1CBwA" + } + } + ] + } + }, + "metricbeat-7.0.0-alpha1-2018.04.23": { + "shards": { + "0": [ + { + "state": "STARTED", + "primary": true, + "node": "VnS118cKS7-8mz8l2jZZKQ", + "relocating_node": null, + "shard": 0, + "index": "metricbeat-7.0.0-alpha1-2018.04.23", + "allocation_id": { + "id": "Vi5fvZAvQe-W8zu0cv3g2A" + } + }, + { + "state": "UNASSIGNED", + "primary": false, + "node": null, + "relocating_node": null, + "shard": 0, + "index": "metricbeat-7.0.0-alpha1-2018.04.23", + "recovery_source": { + "type": "PEER" + }, + "unassigned_info": { + "reason": "INDEX_CREATED", + "at": "2018-04-23T06:45:57.545Z", + "delayed": false, + "allocation_status": "no_attempt" + } + } + ] + } + }, + ".watcher-history-7-2018.04.23": { + "shards": { + "0": [ + { + "state": "STARTED", + "primary": true, + "node": "VnS118cKS7-8mz8l2jZZKQ", + "relocating_node": null, + "shard": 0, + "index": ".watcher-history-7-2018.04.23", + "allocation_id": { + "id": "4crt-Ub4SVSoxgok3H-bpw" + } + } + ] + } + }, + ".watcher-history-7-2018.04.18": { + "shards": { + "0": [ + { + "state": "STARTED", + "primary": true, + "node": "VnS118cKS7-8mz8l2jZZKQ", + "relocating_node": null, + "shard": 0, + "index": ".watcher-history-7-2018.04.18", + "allocation_id": { + "id": "UYLupaFoS2Gby94JdNYnsQ" + } + } + ] + } + }, + ".monitoring-kibana-6-2018.04.19": { + "shards": { + "0": [ + { + "state": "STARTED", + "primary": true, + "node": "VnS118cKS7-8mz8l2jZZKQ", + "relocating_node": null, + "shard": 0, + "index": ".monitoring-kibana-6-2018.04.19", + "allocation_id": { + "id": "euMN_YV_QcWuKCZS1iyz9Q" + } + } + ] + } + }, + ".monitoring-es-6-2018.04.19": { + "shards": { + "0": [ + { + "state": "STARTED", + "primary": true, + "node": "VnS118cKS7-8mz8l2jZZKQ", + "relocating_node": null, + "shard": 0, + "index": ".monitoring-es-6-2018.04.19", + "allocation_id": { + "id": "rgBeiNWXRwKiO1h0v1NLZQ" + } + } + ] + } + }, + "metricbeat-7.0.0-alpha1-2018.04.17": { + "shards": { + "0": [ + { + "state": "STARTED", + "primary": true, + "node": "VnS118cKS7-8mz8l2jZZKQ", + "relocating_node": null, + "shard": 0, + "index": "metricbeat-7.0.0-alpha1-2018.04.17", + "allocation_id": { + "id": "rV1DH2rdRiyhNhEXgefwzg" + } + }, + { + "state": "UNASSIGNED", + "primary": false, + "node": null, + "relocating_node": null, + "shard": 0, + "index": "metricbeat-7.0.0-alpha1-2018.04.17", + "recovery_source": { + "type": "PEER" + }, + "unassigned_info": { + "reason": "INDEX_CREATED", + "at": "2018-04-17T12:12:57.093Z", + "delayed": false, + "allocation_status": "no_attempt" + } + } + ] + } + }, + ".monitoring-kibana-6-2018.04.22": { + "shards": { + "0": [ + { + "state": "STARTED", + "primary": true, + "node": "VnS118cKS7-8mz8l2jZZKQ", + "relocating_node": null, + "shard": 0, + "index": ".monitoring-kibana-6-2018.04.22", + "allocation_id": { + "id": "KJbmsWimTVyWn-znN-gi0Q" + } + } + ] + } + }, + ".monitoring-alerts-6": { + "shards": { + "0": [ + { + "state": "STARTED", + "primary": true, + "node": "VnS118cKS7-8mz8l2jZZKQ", + "relocating_node": null, + "shard": 0, + "index": ".monitoring-alerts-6", + "allocation_id": { + "id": "FS2e938ERji7jRuFQ_QFVQ" + } + } + ] + } + }, + ".monitoring-es-6-2018.04.22": { + "shards": { + "0": [ + { + "state": "STARTED", + "primary": true, + "node": "VnS118cKS7-8mz8l2jZZKQ", + "relocating_node": null, + "shard": 0, + "index": ".monitoring-es-6-2018.04.22", + "allocation_id": { + "id": "YfSdMq9rRaGa4RTf1Bar3w" + } + } + ] + } + }, + ".monitoring-kibana-6-2018.04.20": { + "shards": { + "0": [ + { + "state": "STARTED", + "primary": true, + "node": "VnS118cKS7-8mz8l2jZZKQ", + "relocating_node": null, + "shard": 0, + "index": ".monitoring-kibana-6-2018.04.20", + "allocation_id": { + "id": "Wk3DGqB0TnqnXtbdu5qzfQ" + } + } + ] + } + }, + ".watches": { + "shards": { + "0": [ + { + "state": "STARTED", + "primary": true, + "node": "VnS118cKS7-8mz8l2jZZKQ", + "relocating_node": null, + "shard": 0, + "index": ".watches", + "allocation_id": { + "id": "L6v93p4qQa-qFENtJUuprQ" + } + } + ] + } + }, + ".monitoring-kibana-6-2018.04.21": { + "shards": { + "0": [ + { + "state": "STARTED", + "primary": true, + "node": "VnS118cKS7-8mz8l2jZZKQ", + "relocating_node": null, + "shard": 0, + "index": ".monitoring-kibana-6-2018.04.21", + "allocation_id": { + "id": "b9DbSr7tRB69YEbyDVVa0Q" + } + } + ] + } + }, + ".monitoring-kibana-6-2018.04.17": { + "shards": { + "0": [ + { + "state": "STARTED", + "primary": true, + "node": "VnS118cKS7-8mz8l2jZZKQ", + "relocating_node": null, + "shard": 0, + "index": ".monitoring-kibana-6-2018.04.17", + "allocation_id": { + "id": "98-Z-UcJR2uKs8P9Tf7t7w" + } + } + ] + } + }, + ".monitoring-kibana-6-2018.04.18": { + "shards": { + "0": [ + { + "state": "STARTED", + "primary": true, + "node": "VnS118cKS7-8mz8l2jZZKQ", + "relocating_node": null, + "shard": 0, + "index": ".monitoring-kibana-6-2018.04.18", + "allocation_id": { + "id": "fNDxGJ51QseQur63ikfw7w" + } + } + ] + } + }, + ".monitoring-kibana-6-2018.04.23": { + "shards": { + "0": [ + { + "state": "STARTED", + "primary": true, + "node": "VnS118cKS7-8mz8l2jZZKQ", + "relocating_node": null, + "shard": 0, + "index": ".monitoring-kibana-6-2018.04.23", + "allocation_id": { + "id": "lwqG9MdkQLykIL0oFTlh9w" + } + } + ] + } + }, + "metricbeat-7.0.0-alpha1-2018.04.20": { + "shards": { + "0": [ + { + "state": "STARTED", + "primary": true, + "node": "VnS118cKS7-8mz8l2jZZKQ", + "relocating_node": null, + "shard": 0, + "index": "metricbeat-7.0.0-alpha1-2018.04.20", + "allocation_id": { + "id": "i7tSPiCyS62l8V0pl7KosQ" + } + }, + { + "state": "UNASSIGNED", + "primary": false, + "node": null, + "relocating_node": null, + "shard": 0, + "index": "metricbeat-7.0.0-alpha1-2018.04.20", + "recovery_source": { + "type": "PEER" + }, + "unassigned_info": { + "reason": "INDEX_CREATED", + "at": "2018-04-20T10:54:33.134Z", + "delayed": false, + "allocation_status": "no_attempt" + } + } + ] + } + }, + ".watcher-history-7-2018.04.22": { + "shards": { + "0": [ + { + "state": "STARTED", + "primary": true, + "node": "VnS118cKS7-8mz8l2jZZKQ", + "relocating_node": null, + "shard": 0, + "index": ".watcher-history-7-2018.04.22", + "allocation_id": { + "id": "inTIuRJzQaGOELq5Wo-33g" + } + } + ] + } + } + } + } +} diff --git a/metricbeat/module/elasticsearch/shard/data.go b/metricbeat/module/elasticsearch/shard/data.go new file mode 100644 index 00000000000..2c0d7be5b00 --- /dev/null +++ b/metricbeat/module/elasticsearch/shard/data.go @@ -0,0 +1,63 @@ +package shard + +import ( + "encoding/json" + + "github.com/elastic/beats/libbeat/common" + s "github.com/elastic/beats/libbeat/common/schema" + c "github.com/elastic/beats/libbeat/common/schema/mapstriface" + "github.com/elastic/beats/metricbeat/mb" +) + +var ( + schema = s.Schema{ + "state": c.Str("state"), + "primary": c.Bool("primary"), + "node": c.Str("node"), + "index": c.Str("index"), + "shard": c.Int("number"), + } +) + +type stateStruct struct { + ClusterName string `json:"cluster_name"` + StateID string `json:"state_uuid"` + RoutingTable struct { + Indices map[string]struct { + Shards map[string][]map[string]interface{} `json:"shards"` + } `json:"indices"` + } `json:"routing_table"` +} + +func eventsMapping(r mb.ReporterV2, content []byte) { + stateData := &stateStruct{} + err := json.Unmarshal(content, stateData) + if err != nil { + r.Error(err) + return + } + + for _, index := range stateData.RoutingTable.Indices { + for _, shards := range index.Shards { + for _, shard := range shards { + event := mb.Event{} + + fields, _ := schema.Apply(shard) + + event.ModuleFields = common.MapStr{} + event.ModuleFields.Put("node.name", fields["node"]) + delete(fields, "node") + event.ModuleFields.Put("index.name", fields["index"]) + delete(fields, "index") + event.MetricSetFields = fields + event.ModuleFields.Put("cluster.state.id", stateData.StateID) + event.ModuleFields.Put("cluster.name", stateData.ClusterName) + + event.RootFields = common.MapStr{} + event.RootFields.Put("service.name", "elasticsearch") + + r.Event(event) + } + } + } +} diff --git a/metricbeat/module/elasticsearch/shard/data_test.go b/metricbeat/module/elasticsearch/shard/data_test.go new file mode 100644 index 00000000000..321078a6af4 --- /dev/null +++ b/metricbeat/module/elasticsearch/shard/data_test.go @@ -0,0 +1,27 @@ +package shard + +import ( + "io/ioutil" + "path/filepath" + "testing" + + mbtest "github.com/elastic/beats/metricbeat/mb/testing" + + "github.com/stretchr/testify/assert" +) + +func TestStats(t *testing.T) { + files, err := filepath.Glob("./_meta/test/routing_table.*.json") + assert.NoError(t, err) + + for _, f := range files { + input, err := ioutil.ReadFile(f) + assert.NoError(t, err) + + reporter := &mbtest.CapturingReporterV2{} + eventsMapping(reporter, input) + + assert.True(t, len(reporter.GetEvents()) >= 1) + assert.Equal(t, 0, len(reporter.GetErrors())) + } +} diff --git a/metricbeat/module/elasticsearch/shard/shard.go b/metricbeat/module/elasticsearch/shard/shard.go new file mode 100644 index 00000000000..88704dadaab --- /dev/null +++ b/metricbeat/module/elasticsearch/shard/shard.go @@ -0,0 +1,56 @@ +package shard + +import ( + "github.com/elastic/beats/libbeat/common/cfgwarn" + "github.com/elastic/beats/metricbeat/helper" + "github.com/elastic/beats/metricbeat/mb" + "github.com/elastic/beats/metricbeat/mb/parse" +) + +func init() { + mb.Registry.MustAddMetricSet("elasticsearch", "shard", New, + mb.WithHostParser(hostParser), + mb.DefaultMetricSet(), + mb.WithNamespace("elasticsearch.shard"), + ) +} + +var ( + hostParser = parse.URLHostParserBuilder{ + DefaultScheme: "http", + PathConfigKey: "path", + // Get the stats from the local node + DefaultPath: "_cluster/state/version,master_node,routing_table", + }.Build() +) + +// MetricSet type defines all fields of the MetricSet +type MetricSet struct { + mb.BaseMetricSet + http *helper.HTTP +} + +// New create a new instance of the MetricSet +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + cfgwarn.Beta("The elasticsearch shard metricset is beta") + + http, err := helper.NewHTTP(base) + if err != nil { + return nil, err + } + return &MetricSet{ + BaseMetricSet: base, + http: http, + }, nil +} + +// Fetch methods implements the data gathering and data conversion to the right format +func (m *MetricSet) Fetch(r mb.ReporterV2) { + content, err := m.http.FetchContent() + if err != nil { + r.Error(err) + return + } + + eventsMapping(r, content) +} diff --git a/metricbeat/module/elasticsearch/test_elasticsearch.py b/metricbeat/module/elasticsearch/test_elasticsearch.py index 1940a516867..a8d113bf6ad 100644 --- a/metricbeat/module/elasticsearch/test_elasticsearch.py +++ b/metricbeat/module/elasticsearch/test_elasticsearch.py @@ -20,6 +20,7 @@ class Test(metricbeat.BaseTest): "index_summary", "node_stats", "node", + "shard" ]) @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") def test_metricsets(self, metricset):