From 2fdbdb3a9f86e47e279de9b59aa30b2a759b30e6 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Tue, 16 Oct 2018 20:09:06 +0100 Subject: [PATCH] Get cluster UUID from Info API call (#8592) (#8610) Starting from 6.5.0, Elasticsearch [will return](https://github.com/elastic/elasticsearch/pull/32206) the `cluster_uuid` as part of the `GET _cluster/stats` API response. However, the Elasticsearch metricbeat module has been in existence since 6.3.0 so it must fetch the cluster UUID from the `GET /` Elasticsearch API. This PR makes it so the `GET /` API is always called and the cluster UUID from it is used. This is obviously not ideal in terms of API calls, but it's the simplest fix (for now, until we can figure out a nicer way to deal with version differences). (cherry picked from commit ccfc8d77cf62ec2a62b22d99ebe7101916da28a5) --- .../_meta/test/cluster_stats.700.json | 271 +++++++++--------- .../cluster_stats/cluster_stats.go | 10 +- .../elasticsearch/cluster_stats/data.go | 28 +- .../elasticsearch/cluster_stats/data_test.go | 2 +- .../elasticsearch/cluster_stats/data_xpack.go | 7 +- 5 files changed, 153 insertions(+), 165 deletions(-) diff --git a/metricbeat/module/elasticsearch/cluster_stats/_meta/test/cluster_stats.700.json b/metricbeat/module/elasticsearch/cluster_stats/_meta/test/cluster_stats.700.json index 50180bfdc37c..68ec01695b74 100644 --- a/metricbeat/module/elasticsearch/cluster_stats/_meta/test/cluster_stats.700.json +++ b/metricbeat/module/elasticsearch/cluster_stats/_meta/test/cluster_stats.700.json @@ -1,147 +1,144 @@ { - "_nodes":{ - "total":2, - "successful":2, - "failed":0 + "_nodes": { + "total": 2, + "successful": 2, + "failed": 0 }, - "cluster_name":"elasticsearch", - "timestamp":1532433421874, - "status":"yellow", - "indices":{ - "count":4, - "shards":{ - "total":16, - "primaries":8, - "replication":1, - "index":{ - "shards":{ - "min":2, - "max":10, - "avg":4 - }, - "primaries":{ - "min":1, - "max":5, - "avg":2 - }, - "replication":{ - "min":1, - "max":1, - "avg":1 - } - } - }, - "docs":{ - "count":145, - "deleted":0 - }, - "store":{ - "size_in_bytes":1030085 - }, - "fielddata":{ - "memory_size_in_bytes":0, - "evictions":0 - }, - "query_cache":{ - "memory_size_in_bytes":0, - "total_count":0, - "hit_count":0, - "miss_count":0, - "cache_size":0, - "cache_count":0, - "evictions":0 - }, - "completion":{ - "size_in_bytes":0 - }, - "segments":{ - "count":26, - "memory_in_bytes":108394, - "terms_memory_in_bytes":83123, - "stored_fields_memory_in_bytes":8112, - "term_vectors_memory_in_bytes":0, - "norms_memory_in_bytes":0, - "points_memory_in_bytes":3047, - "doc_values_memory_in_bytes":14112, - "index_writer_memory_in_bytes":0, - "version_map_memory_in_bytes":0, - "fixed_bit_set_memory_in_bytes":0, - "max_unsafe_auto_id_timestamp":1532433381676, - "file_sizes":{ - + "cluster_name": "elasticsearch", + "cluster_uuid": "zLXBfdzPSrO12OBIz3ybyg", + "timestamp": 1532433421874, + "status": "yellow", + "indices": { + "count": 4, + "shards": { + "total": 16, + "primaries": 8, + "replication": 1, + "index": { + "shards": { + "min": 2, + "max": 10, + "avg": 4 + }, + "primaries": { + "min": 1, + "max": 5, + "avg": 2 + }, + "replication": { + "min": 1, + "max": 1, + "avg": 1 + } + } + }, + "docs": { + "count": 145, + "deleted": 0 + }, + "store": { + "size_in_bytes": 1030085 + }, + "fielddata": { + "memory_size_in_bytes": 0, + "evictions": 0 + }, + "query_cache": { + "memory_size_in_bytes": 0, + "total_count": 0, + "hit_count": 0, + "miss_count": 0, + "cache_size": 0, + "cache_count": 0, + "evictions": 0 + }, + "completion": { + "size_in_bytes": 0 + }, + "segments": { + "count": 26, + "memory_in_bytes": 108394, + "terms_memory_in_bytes": 83123, + "stored_fields_memory_in_bytes": 8112, + "term_vectors_memory_in_bytes": 0, + "norms_memory_in_bytes": 0, + "points_memory_in_bytes": 3047, + "doc_values_memory_in_bytes": 14112, + "index_writer_memory_in_bytes": 0, + "version_map_memory_in_bytes": 0, + "fixed_bit_set_memory_in_bytes": 0, + "max_unsafe_auto_id_timestamp": 1532433381676, + "file_sizes": {} } - } }, - "nodes":{ - "count":{ - "total":2, - "data":2, - "coordinating_only":0, - "master":2, - "ingest":2 - }, - "versions":[ - "7.0.0-alpha1" - ], - "os":{ - "available_processors":16, - "allocated_processors":16, - "names":[ - { - "name":"Mac OS X", - "count":2 - } - ], - "mem":{ - "total_in_bytes":34359738368, - "free_in_bytes":735543296, - "used_in_bytes":33624195072, - "free_percent":2, - "used_percent":98 - } - }, - "process":{ - "cpu":{ - "percent":0 + "nodes": { + "count": { + "total": 2, + "data": 2, + "coordinating_only": 0, + "master": 2, + "ingest": 2 }, - "open_file_descriptors":{ - "min":364, - "max":371, - "avg":367 - } - }, - "jvm":{ - "max_uptime_in_millis":228721, - "versions":[ - { - "version":"10.0.1", - "vm_name":"Java HotSpot(TM) 64-Bit Server VM", - "vm_version":"10.0.1+10", - "vm_vendor":"Oracle Corporation", - "count":2 - } + "versions": [ + "7.0.0-alpha1" ], - "mem":{ - "heap_used_in_bytes":420771816, - "heap_max_in_bytes":2075918336 + "os": { + "available_processors": 16, + "allocated_processors": 16, + "names": [ + { + "name": "Mac OS X", + "count": 2 + } + ], + "mem": { + "total_in_bytes": 34359738368, + "free_in_bytes": 735543296, + "used_in_bytes": 33624195072, + "free_percent": 2, + "used_percent": 98 + } + }, + "process": { + "cpu": { + "percent": 0 + }, + "open_file_descriptors": { + "min": 364, + "max": 371, + "avg": 367 + } + }, + "jvm": { + "max_uptime_in_millis": 228721, + "versions": [ + { + "version": "10.0.1", + "vm_name": "Java HotSpot(TM) 64-Bit Server VM", + "vm_version": "10.0.1+10", + "vm_vendor": "Oracle Corporation", + "count": 2 + } + ], + "mem": { + "heap_used_in_bytes": 420771816, + "heap_max_in_bytes": 2075918336 + }, + "threads": 164 }, - "threads":164 - }, - "fs":{ - "total_in_bytes":499963170816, - "free_in_bytes":410995167232, - "available_in_bytes":408143978496 - }, - "plugins":[ - - ], - "network_types":{ - "transport_types":{ - "security4":2 + "fs": { + "total_in_bytes": 499963170816, + "free_in_bytes": 410995167232, + "available_in_bytes": 408143978496 }, - "http_types":{ - "security4":2 + "plugins": [], + "network_types": { + "transport_types": { + "security4": 2 + }, + "http_types": { + "security4": 2 + } } - } } - } +} diff --git a/metricbeat/module/elasticsearch/cluster_stats/cluster_stats.go b/metricbeat/module/elasticsearch/cluster_stats/cluster_stats.go index a7c043c8a6c0..ae6d289b8212 100644 --- a/metricbeat/module/elasticsearch/cluster_stats/cluster_stats.go +++ b/metricbeat/module/elasticsearch/cluster_stats/cluster_stats.go @@ -73,9 +73,15 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) { return } + info, err := elasticsearch.GetInfo(m.HTTP, m.HostData().SanitizedURI+clusterStatsPath) + if err != nil { + r.Error(errors.Wrap(err, "failed to get info from Elasticsearch")) + return + } + if m.MetricSet.XPack { - eventMappingXPack(r, m, content) + eventMappingXPack(r, m, *info, content) } else { - eventMapping(r, content) + eventMapping(r, *info, content) } } diff --git a/metricbeat/module/elasticsearch/cluster_stats/data.go b/metricbeat/module/elasticsearch/cluster_stats/data.go index e2cd56db1e2c..d0d680562afe 100644 --- a/metricbeat/module/elasticsearch/cluster_stats/data.go +++ b/metricbeat/module/elasticsearch/cluster_stats/data.go @@ -25,7 +25,6 @@ import ( "github.com/elastic/beats/libbeat/common" s "github.com/elastic/beats/libbeat/common/schema" c "github.com/elastic/beats/libbeat/common/schema/mapstriface" - "github.com/elastic/beats/metricbeat/helper/elastic" "github.com/elastic/beats/metricbeat/mb" "github.com/elastic/beats/metricbeat/module/elasticsearch" ) @@ -53,13 +52,20 @@ var ( } ) -func eventMapping(r mb.ReporterV2, content []byte) error { +func eventMapping(r mb.ReporterV2, info elasticsearch.Info, content []byte) error { + var event mb.Event + event.RootFields = common.MapStr{} + event.RootFields.Put("service.name", elasticsearch.ModuleName) + + event.ModuleFields = common.MapStr{} + event.ModuleFields.Put("cluster.name", info.ClusterName) + event.ModuleFields.Put("cluster.id", info.ClusterID) + var data map[string]interface{} err := json.Unmarshal(content, &data) if err != nil { err = errors.Wrap(err, "failure parsing Elasticsearch Cluster Stats API response") r.Error(err) - return err } metricSetFields, err := schema.Apply(data) @@ -69,22 +75,6 @@ func eventMapping(r mb.ReporterV2, content []byte) error { return err } - clusterName, ok := data["cluster_name"] - if !ok { - return elastic.ReportErrorForMissingField("cluster_name", elastic.Elasticsearch, r) - } - - var event mb.Event - event.RootFields = common.MapStr{} - event.RootFields.Put("service.name", elasticsearch.ModuleName) - - event.ModuleFields = common.MapStr{} - event.ModuleFields.Put("cluster.name", clusterName) - clusterUUID, ok := data["cluster_uuid"] - if ok { - event.ModuleFields.Put("cluster.id", clusterUUID) - } - event.MetricSetFields = metricSetFields r.Event(event) diff --git a/metricbeat/module/elasticsearch/cluster_stats/data_test.go b/metricbeat/module/elasticsearch/cluster_stats/data_test.go index 3fe38b8d0c96..6a4c35ebad06 100644 --- a/metricbeat/module/elasticsearch/cluster_stats/data_test.go +++ b/metricbeat/module/elasticsearch/cluster_stats/data_test.go @@ -26,5 +26,5 @@ import ( ) func TestMapper(t *testing.T) { - elasticsearch.TestMapper(t, "./_meta/test/cluster_stats.*.json", eventMapping) + elasticsearch.TestMapperWithInfo(t, "./_meta/test/cluster_stats.*.json", eventMapping) } diff --git a/metricbeat/module/elasticsearch/cluster_stats/data_xpack.go b/metricbeat/module/elasticsearch/cluster_stats/data_xpack.go index 298e5d263ef8..c3fd7fb4ddb1 100644 --- a/metricbeat/module/elasticsearch/cluster_stats/data_xpack.go +++ b/metricbeat/module/elasticsearch/cluster_stats/data_xpack.go @@ -142,7 +142,7 @@ func apmIndicesExist(clusterState common.MapStr) (bool, error) { return false, nil } -func eventMappingXPack(r mb.ReporterV2, m *MetricSet, content []byte) error { +func eventMappingXPack(r mb.ReporterV2, m *MetricSet, info elasticsearch.Info, content []byte) error { var data map[string]interface{} err := json.Unmarshal(content, &data) if err != nil { @@ -160,11 +160,6 @@ func eventMappingXPack(r mb.ReporterV2, m *MetricSet, content []byte) error { return fmt.Errorf("cluster name is not a string") } - info, err := elasticsearch.GetInfo(m.HTTP, m.HTTP.GetURI()) - if err != nil { - return errors.Wrap(err, "failed to get info from Elasticsearch") - } - license, err := elasticsearch.GetLicense(m.HTTP, m.HTTP.GetURI()) if err != nil { return errors.Wrap(err, "failed to get license from Elasticsearch")