Skip to content

Commit

Permalink
Get cluster UUID from Info API call (#8592)
Browse files Browse the repository at this point in the history 
Starting from 6.5.0, Elasticsearch [will return](elastic/elasticsearch#32206) the `cluster_uuid` as part of the `GET _cluster/stats` API response. However, the Elasticsearch metricbeat module has been in existence since 6.3.0 so it must fetch the cluster UUID from the `GET /` Elasticsearch API.

This PR makes it so the `GET /` API is always called and the cluster UUID from it is used. This is obviously not ideal in terms of API calls, but it's the simplest fix (for now, until we can figure out a nicer way to deal with version differences).

(cherry picked from commit ccfc8d7)
  • Loading branch information
@ycombinator
ycombinator committed Oct 16, 2018
1 parent c7935f7 commit 6753198
Show file tree
Hide file tree
Showing 5 changed files with 153 additions and 165 deletions.
271 changes: 134 additions & 137 deletions metricbeat/module/elasticsearch/cluster_stats/_meta/test/cluster_stats.700.json
View file
Original file line number Diff line number Diff line change
@@ -1,147 +1,144 @@
{
"_nodes":{
"total":2,
"successful":2,
"failed":0
"_nodes": {
"total": 2,
"successful": 2,
"failed": 0
},
"cluster_name":"elasticsearch",
"timestamp":1532433421874,
"status":"yellow",
"indices":{
"count":4,
"shards":{
"total":16,
"primaries":8,
"replication":1,
"index":{
"shards":{
"min":2,
"max":10,
"avg":4
},
"primaries":{
"min":1,
"max":5,
"avg":2
},
"replication":{
"min":1,
"max":1,
"avg":1
}
}
},
"docs":{
"count":145,
"deleted":0
},
"store":{
"size_in_bytes":1030085
},
"fielddata":{
"memory_size_in_bytes":0,
"evictions":0
},
"query_cache":{
"memory_size_in_bytes":0,
"total_count":0,
"hit_count":0,
"miss_count":0,
"cache_size":0,
"cache_count":0,
"evictions":0
},
"completion":{
"size_in_bytes":0
},
"segments":{
"count":26,
"memory_in_bytes":108394,
"terms_memory_in_bytes":83123,
"stored_fields_memory_in_bytes":8112,
"term_vectors_memory_in_bytes":0,
"norms_memory_in_bytes":0,
"points_memory_in_bytes":3047,
"doc_values_memory_in_bytes":14112,
"index_writer_memory_in_bytes":0,
"version_map_memory_in_bytes":0,
"fixed_bit_set_memory_in_bytes":0,
"max_unsafe_auto_id_timestamp":1532433381676,
"file_sizes":{

"cluster_name": "elasticsearch",
"cluster_uuid": "zLXBfdzPSrO12OBIz3ybyg",
"timestamp": 1532433421874,
"status": "yellow",
"indices": {
"count": 4,
"shards": {
"total": 16,
"primaries": 8,
"replication": 1,
"index": {
"shards": {
"min": 2,
"max": 10,
"avg": 4
},
"primaries": {
"min": 1,
"max": 5,
"avg": 2
},
"replication": {
"min": 1,
"max": 1,
"avg": 1
}
}
},
"docs": {
"count": 145,
"deleted": 0
},
"store": {
"size_in_bytes": 1030085
},
"fielddata": {
"memory_size_in_bytes": 0,
"evictions": 0
},
"query_cache": {
"memory_size_in_bytes": 0,
"total_count": 0,
"hit_count": 0,
"miss_count": 0,
"cache_size": 0,
"cache_count": 0,
"evictions": 0
},
"completion": {
"size_in_bytes": 0
},
"segments": {
"count": 26,
"memory_in_bytes": 108394,
"terms_memory_in_bytes": 83123,
"stored_fields_memory_in_bytes": 8112,
"term_vectors_memory_in_bytes": 0,
"norms_memory_in_bytes": 0,
"points_memory_in_bytes": 3047,
"doc_values_memory_in_bytes": 14112,
"index_writer_memory_in_bytes": 0,
"version_map_memory_in_bytes": 0,
"fixed_bit_set_memory_in_bytes": 0,
"max_unsafe_auto_id_timestamp": 1532433381676,
"file_sizes": {}
}
}
},
"nodes":{
"count":{
"total":2,
"data":2,
"coordinating_only":0,
"master":2,
"ingest":2
},
"versions":[
"7.0.0-alpha1"
],
"os":{
"available_processors":16,
"allocated_processors":16,
"names":[
{
"name":"Mac OS X",
"count":2
}
],
"mem":{
"total_in_bytes":34359738368,
"free_in_bytes":735543296,
"used_in_bytes":33624195072,
"free_percent":2,
"used_percent":98
}
},
"process":{
"cpu":{
"percent":0
"nodes": {
"count": {
"total": 2,
"data": 2,
"coordinating_only": 0,
"master": 2,
"ingest": 2
},
"open_file_descriptors":{
"min":364,
"max":371,
"avg":367
}
},
"jvm":{
"max_uptime_in_millis":228721,
"versions":[
{
"version":"10.0.1",
"vm_name":"Java HotSpot(TM) 64-Bit Server VM",
"vm_version":"10.0.1+10",
"vm_vendor":"Oracle Corporation",
"count":2
}
"versions": [
"7.0.0-alpha1"
],
"mem":{
"heap_used_in_bytes":420771816,
"heap_max_in_bytes":2075918336
"os": {
"available_processors": 16,
"allocated_processors": 16,
"names": [
{
"name": "Mac OS X",
"count": 2
}
],
"mem": {
"total_in_bytes": 34359738368,
"free_in_bytes": 735543296,
"used_in_bytes": 33624195072,
"free_percent": 2,
"used_percent": 98
}
},
"process": {
"cpu": {
"percent": 0
},
"open_file_descriptors": {
"min": 364,
"max": 371,
"avg": 367
}
},
"jvm": {
"max_uptime_in_millis": 228721,
"versions": [
{
"version": "10.0.1",
"vm_name": "Java HotSpot(TM) 64-Bit Server VM",
"vm_version": "10.0.1+10",
"vm_vendor": "Oracle Corporation",
"count": 2
}
],
"mem": {
"heap_used_in_bytes": 420771816,
"heap_max_in_bytes": 2075918336
},
"threads": 164
},
"threads":164
},
"fs":{
"total_in_bytes":499963170816,
"free_in_bytes":410995167232,
"available_in_bytes":408143978496
},
"plugins":[

],
"network_types":{
"transport_types":{
"security4":2
"fs": {
"total_in_bytes": 499963170816,
"free_in_bytes": 410995167232,
"available_in_bytes": 408143978496
},
"http_types":{
"security4":2
"plugins": [],
"network_types": {
"transport_types": {
"security4": 2
},
"http_types": {
"security4": 2
}
}
}
}
}
}
10 changes: 8 additions & 2 deletions metricbeat/module/elasticsearch/cluster_stats/cluster_stats.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,9 +73,15 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) {
return
}

info, err := elasticsearch.GetInfo(m.HTTP, m.HostData().SanitizedURI+clusterStatsPath)
if err != nil {
r.Error(errors.Wrap(err, "failed to get info from Elasticsearch"))
return
}

if m.MetricSet.XPack {
eventMappingXPack(r, m, content)
eventMappingXPack(r, m, *info, content)
} else {
eventMapping(r, content)
eventMapping(r, *info, content)
}
}
28 changes: 9 additions & 19 deletions metricbeat/module/elasticsearch/cluster_stats/data.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@ import (
"github.com/elastic/beats/libbeat/common"
s "github.com/elastic/beats/libbeat/common/schema"
c "github.com/elastic/beats/libbeat/common/schema/mapstriface"
"github.com/elastic/beats/metricbeat/helper/elastic"
"github.com/elastic/beats/metricbeat/mb"
"github.com/elastic/beats/metricbeat/module/elasticsearch"
)
Expand Down Expand Up @@ -53,13 +52,20 @@ var (
}
)

func eventMapping(r mb.ReporterV2, content []byte) error {
func eventMapping(r mb.ReporterV2, info elasticsearch.Info, content []byte) error {
var event mb.Event
event.RootFields = common.MapStr{}
event.RootFields.Put("service.name", elasticsearch.ModuleName)

event.ModuleFields = common.MapStr{}
event.ModuleFields.Put("cluster.name", info.ClusterName)
event.ModuleFields.Put("cluster.id", info.ClusterID)

var data map[string]interface{}
err := json.Unmarshal(content, &data)
if err != nil {
err = errors.Wrap(err, "failure parsing Elasticsearch Cluster Stats API response")
r.Error(err)
return err
}

metricSetFields, err := schema.Apply(data)
Expand All @@ -69,22 +75,6 @@ func eventMapping(r mb.ReporterV2, content []byte) error {
return err
}

clusterName, ok := data["cluster_name"]
if !ok {
return elastic.ReportErrorForMissingField("cluster_name", elastic.Elasticsearch, r)
}

var event mb.Event
event.RootFields = common.MapStr{}
event.RootFields.Put("service.name", elasticsearch.ModuleName)

event.ModuleFields = common.MapStr{}
event.ModuleFields.Put("cluster.name", clusterName)
clusterUUID, ok := data["cluster_uuid"]
if ok {
event.ModuleFields.Put("cluster.id", clusterUUID)
}

event.MetricSetFields = metricSetFields

r.Event(event)
Expand Down
2 changes: 1 addition & 1 deletion metricbeat/module/elasticsearch/cluster_stats/data_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,5 +26,5 @@ import (
)

func TestMapper(t *testing.T) {
elasticsearch.TestMapper(t, "./_meta/test/cluster_stats.*.json", eventMapping)
elasticsearch.TestMapperWithInfo(t, "./_meta/test/cluster_stats.*.json", eventMapping)
}
7 changes: 1 addition & 6 deletions metricbeat/module/elasticsearch/cluster_stats/data_xpack.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -142,7 +142,7 @@ func apmIndicesExist(clusterState common.MapStr) (bool, error) {
return false, nil
}

func eventMappingXPack(r mb.ReporterV2, m *MetricSet, content []byte) error {
func eventMappingXPack(r mb.ReporterV2, m *MetricSet, info elasticsearch.Info, content []byte) error {
var data map[string]interface{}
err := json.Unmarshal(content, &data)
if err != nil {
Expand All @@ -160,11 +160,6 @@ func eventMappingXPack(r mb.ReporterV2, m *MetricSet, content []byte) error {
return fmt.Errorf("cluster name is not a string")
}

info, err := elasticsearch.GetInfo(m.HTTP, m.HTTP.GetURI())
if err != nil {
return errors.Wrap(err, "failed to get info from Elasticsearch")
}

license, err := elasticsearch.GetLicense(m.HTTP, m.HTTP.GetURI())
if err != nil {
return errors.Wrap(err, "failed to get license from Elasticsearch")
Expand Down

0 comments on commit 6753198

Please sign in to comment.