diff --git a/metricbeat/module/elasticsearch/cluster_stats/_meta/test/cluster_stats.700.json b/metricbeat/module/elasticsearch/cluster_stats/_meta/test/cluster_stats.700.json index 50180bfdc37..68ec01695b7 100644 --- a/metricbeat/module/elasticsearch/cluster_stats/_meta/test/cluster_stats.700.json +++ b/metricbeat/module/elasticsearch/cluster_stats/_meta/test/cluster_stats.700.json @@ -1,147 +1,144 @@ { - "_nodes":{ - "total":2, - "successful":2, - "failed":0 + "_nodes": { + "total": 2, + "successful": 2, + "failed": 0 }, - "cluster_name":"elasticsearch", - "timestamp":1532433421874, - "status":"yellow", - "indices":{ - "count":4, - "shards":{ - "total":16, - "primaries":8, - "replication":1, - "index":{ - "shards":{ - "min":2, - "max":10, - "avg":4 - }, - "primaries":{ - "min":1, - "max":5, - "avg":2 - }, - "replication":{ - "min":1, - "max":1, - "avg":1 - } - } - }, - "docs":{ - "count":145, - "deleted":0 - }, - "store":{ - "size_in_bytes":1030085 - }, - "fielddata":{ - "memory_size_in_bytes":0, - "evictions":0 - }, - "query_cache":{ - "memory_size_in_bytes":0, - "total_count":0, - "hit_count":0, - "miss_count":0, - "cache_size":0, - "cache_count":0, - "evictions":0 - }, - "completion":{ - "size_in_bytes":0 - }, - "segments":{ - "count":26, - "memory_in_bytes":108394, - "terms_memory_in_bytes":83123, - "stored_fields_memory_in_bytes":8112, - "term_vectors_memory_in_bytes":0, - "norms_memory_in_bytes":0, - "points_memory_in_bytes":3047, - "doc_values_memory_in_bytes":14112, - "index_writer_memory_in_bytes":0, - "version_map_memory_in_bytes":0, - "fixed_bit_set_memory_in_bytes":0, - "max_unsafe_auto_id_timestamp":1532433381676, - "file_sizes":{ - + "cluster_name": "elasticsearch", + "cluster_uuid": "zLXBfdzPSrO12OBIz3ybyg", + "timestamp": 1532433421874, + "status": "yellow", + "indices": { + "count": 4, + "shards": { + "total": 16, + "primaries": 8, + "replication": 1, + "index": { + "shards": { + "min": 2, + "max": 10, + "avg": 4 + }, + "primaries": { + "min": 1, + "max": 5, + "avg": 2 + }, + "replication": { + "min": 1, + "max": 1, + "avg": 1 + } + } + }, + "docs": { + "count": 145, + "deleted": 0 + }, + "store": { + "size_in_bytes": 1030085 + }, + "fielddata": { + "memory_size_in_bytes": 0, + "evictions": 0 + }, + "query_cache": { + "memory_size_in_bytes": 0, + "total_count": 0, + "hit_count": 0, + "miss_count": 0, + "cache_size": 0, + "cache_count": 0, + "evictions": 0 + }, + "completion": { + "size_in_bytes": 0 + }, + "segments": { + "count": 26, + "memory_in_bytes": 108394, + "terms_memory_in_bytes": 83123, + "stored_fields_memory_in_bytes": 8112, + "term_vectors_memory_in_bytes": 0, + "norms_memory_in_bytes": 0, + "points_memory_in_bytes": 3047, + "doc_values_memory_in_bytes": 14112, + "index_writer_memory_in_bytes": 0, + "version_map_memory_in_bytes": 0, + "fixed_bit_set_memory_in_bytes": 0, + "max_unsafe_auto_id_timestamp": 1532433381676, + "file_sizes": {} } - } }, - "nodes":{ - "count":{ - "total":2, - "data":2, - "coordinating_only":0, - "master":2, - "ingest":2 - }, - "versions":[ - "7.0.0-alpha1" - ], - "os":{ - "available_processors":16, - "allocated_processors":16, - "names":[ - { - "name":"Mac OS X", - "count":2 - } - ], - "mem":{ - "total_in_bytes":34359738368, - "free_in_bytes":735543296, - "used_in_bytes":33624195072, - "free_percent":2, - "used_percent":98 - } - }, - "process":{ - "cpu":{ - "percent":0 + "nodes": { + "count": { + "total": 2, + "data": 2, + "coordinating_only": 0, + "master": 2, + "ingest": 2 }, - "open_file_descriptors":{ - "min":364, - "max":371, - "avg":367 - } - }, - "jvm":{ - "max_uptime_in_millis":228721, - "versions":[ - { - "version":"10.0.1", - "vm_name":"Java HotSpot(TM) 64-Bit Server VM", - "vm_version":"10.0.1+10", - "vm_vendor":"Oracle Corporation", - "count":2 - } + "versions": [ + "7.0.0-alpha1" ], - "mem":{ - "heap_used_in_bytes":420771816, - "heap_max_in_bytes":2075918336 + "os": { + "available_processors": 16, + "allocated_processors": 16, + "names": [ + { + "name": "Mac OS X", + "count": 2 + } + ], + "mem": { + "total_in_bytes": 34359738368, + "free_in_bytes": 735543296, + "used_in_bytes": 33624195072, + "free_percent": 2, + "used_percent": 98 + } + }, + "process": { + "cpu": { + "percent": 0 + }, + "open_file_descriptors": { + "min": 364, + "max": 371, + "avg": 367 + } + }, + "jvm": { + "max_uptime_in_millis": 228721, + "versions": [ + { + "version": "10.0.1", + "vm_name": "Java HotSpot(TM) 64-Bit Server VM", + "vm_version": "10.0.1+10", + "vm_vendor": "Oracle Corporation", + "count": 2 + } + ], + "mem": { + "heap_used_in_bytes": 420771816, + "heap_max_in_bytes": 2075918336 + }, + "threads": 164 }, - "threads":164 - }, - "fs":{ - "total_in_bytes":499963170816, - "free_in_bytes":410995167232, - "available_in_bytes":408143978496 - }, - "plugins":[ - - ], - "network_types":{ - "transport_types":{ - "security4":2 + "fs": { + "total_in_bytes": 499963170816, + "free_in_bytes": 410995167232, + "available_in_bytes": 408143978496 }, - "http_types":{ - "security4":2 + "plugins": [], + "network_types": { + "transport_types": { + "security4": 2 + }, + "http_types": { + "security4": 2 + } } - } } - } +} diff --git a/metricbeat/module/elasticsearch/cluster_stats/cluster_stats.go b/metricbeat/module/elasticsearch/cluster_stats/cluster_stats.go index a7c043c8a6c..ae6d289b821 100644 --- a/metricbeat/module/elasticsearch/cluster_stats/cluster_stats.go +++ b/metricbeat/module/elasticsearch/cluster_stats/cluster_stats.go @@ -73,9 +73,15 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) { return } + info, err := elasticsearch.GetInfo(m.HTTP, m.HostData().SanitizedURI+clusterStatsPath) + if err != nil { + r.Error(errors.Wrap(err, "failed to get info from Elasticsearch")) + return + } + if m.MetricSet.XPack { - eventMappingXPack(r, m, content) + eventMappingXPack(r, m, *info, content) } else { - eventMapping(r, content) + eventMapping(r, *info, content) } } diff --git a/metricbeat/module/elasticsearch/cluster_stats/data.go b/metricbeat/module/elasticsearch/cluster_stats/data.go index e2cd56db1e2..d0d680562af 100644 --- a/metricbeat/module/elasticsearch/cluster_stats/data.go +++ b/metricbeat/module/elasticsearch/cluster_stats/data.go @@ -25,7 +25,6 @@ import ( "github.com/elastic/beats/libbeat/common" s "github.com/elastic/beats/libbeat/common/schema" c "github.com/elastic/beats/libbeat/common/schema/mapstriface" - "github.com/elastic/beats/metricbeat/helper/elastic" "github.com/elastic/beats/metricbeat/mb" "github.com/elastic/beats/metricbeat/module/elasticsearch" ) @@ -53,13 +52,20 @@ var ( } ) -func eventMapping(r mb.ReporterV2, content []byte) error { +func eventMapping(r mb.ReporterV2, info elasticsearch.Info, content []byte) error { + var event mb.Event + event.RootFields = common.MapStr{} + event.RootFields.Put("service.name", elasticsearch.ModuleName) + + event.ModuleFields = common.MapStr{} + event.ModuleFields.Put("cluster.name", info.ClusterName) + event.ModuleFields.Put("cluster.id", info.ClusterID) + var data map[string]interface{} err := json.Unmarshal(content, &data) if err != nil { err = errors.Wrap(err, "failure parsing Elasticsearch Cluster Stats API response") r.Error(err) - return err } metricSetFields, err := schema.Apply(data) @@ -69,22 +75,6 @@ func eventMapping(r mb.ReporterV2, content []byte) error { return err } - clusterName, ok := data["cluster_name"] - if !ok { - return elastic.ReportErrorForMissingField("cluster_name", elastic.Elasticsearch, r) - } - - var event mb.Event - event.RootFields = common.MapStr{} - event.RootFields.Put("service.name", elasticsearch.ModuleName) - - event.ModuleFields = common.MapStr{} - event.ModuleFields.Put("cluster.name", clusterName) - clusterUUID, ok := data["cluster_uuid"] - if ok { - event.ModuleFields.Put("cluster.id", clusterUUID) - } - event.MetricSetFields = metricSetFields r.Event(event) diff --git a/metricbeat/module/elasticsearch/cluster_stats/data_test.go b/metricbeat/module/elasticsearch/cluster_stats/data_test.go index 3fe38b8d0c9..6a4c35ebad0 100644 --- a/metricbeat/module/elasticsearch/cluster_stats/data_test.go +++ b/metricbeat/module/elasticsearch/cluster_stats/data_test.go @@ -26,5 +26,5 @@ import ( ) func TestMapper(t *testing.T) { - elasticsearch.TestMapper(t, "./_meta/test/cluster_stats.*.json", eventMapping) + elasticsearch.TestMapperWithInfo(t, "./_meta/test/cluster_stats.*.json", eventMapping) } diff --git a/metricbeat/module/elasticsearch/cluster_stats/data_xpack.go b/metricbeat/module/elasticsearch/cluster_stats/data_xpack.go index 298e5d263ef..c3fd7fb4ddb 100644 --- a/metricbeat/module/elasticsearch/cluster_stats/data_xpack.go +++ b/metricbeat/module/elasticsearch/cluster_stats/data_xpack.go @@ -142,7 +142,7 @@ func apmIndicesExist(clusterState common.MapStr) (bool, error) { return false, nil } -func eventMappingXPack(r mb.ReporterV2, m *MetricSet, content []byte) error { +func eventMappingXPack(r mb.ReporterV2, m *MetricSet, info elasticsearch.Info, content []byte) error { var data map[string]interface{} err := json.Unmarshal(content, &data) if err != nil { @@ -160,11 +160,6 @@ func eventMappingXPack(r mb.ReporterV2, m *MetricSet, content []byte) error { return fmt.Errorf("cluster name is not a string") } - info, err := elasticsearch.GetInfo(m.HTTP, m.HTTP.GetURI()) - if err != nil { - return errors.Wrap(err, "failed to get info from Elasticsearch") - } - license, err := elasticsearch.GetLicense(m.HTTP, m.HTTP.GetURI()) if err != nil { return errors.Wrap(err, "failed to get license from Elasticsearch")