From 27e56d12d06deefc78859d33d53cd2066107be8b Mon Sep 17 00:00:00 2001 From: Manabu Matsuzaki Date: Wed, 16 Aug 2017 21:33:05 +0900 Subject: [PATCH 01/12] add index metrics --- collector/indices.go | 172 ++++++++++++++++++++++++++++++++++ collector/indices_response.go | 169 +++++++++++++++++++++++++++++++++ collector/indices_test.go | 54 +++++++++++ main.go | 1 + 4 files changed, 396 insertions(+) create mode 100644 collector/indices.go create mode 100644 collector/indices_response.go create mode 100644 collector/indices_test.go diff --git a/collector/indices.go b/collector/indices.go new file mode 100644 index 00000000..329078dc --- /dev/null +++ b/collector/indices.go @@ -0,0 +1,172 @@ +package collector + +import ( + "fmt" + "encoding/json" + "net/http" + "net/url" + + "github.com/go-kit/kit/log" + "github.com/go-kit/kit/log/level" + "github.com/prometheus/client_golang/prometheus" +) + +var ( + defaultIndexLabels = []string{"cluster", "index"} + defaultIndexLabelValues = func(clusterName string, indexName string) []string { + return []string{clusterName, indexName} + } +) + +type indexMetric struct { + Type prometheus.ValueType + Desc *prometheus.Desc + Value func(indexStats IndexStatsIndexResponse) float64 + Labels func(clusterName string, indexName string) []string +} + +type Indices struct { + logger log.Logger + client *http.Client + url *url.URL + + up prometheus.Gauge + totalScrapes prometheus.Counter + jsonParseFailures prometheus.Counter + + indexMetrics []*indexMetric +} + +func NewIndices(logger log.Logger, client *http.Client, url *url.URL) *Indices { + return &Indices{ + logger: logger, + client: client, + url: url, + + up: prometheus.NewGauge(prometheus.GaugeOpts{ + Name: prometheus.BuildFQName(namespace, "index_stats", "up"), + Help: "Was the last scrape of the ElasticSearch index endpoint successful.", + }), + totalScrapes: prometheus.NewCounter(prometheus.CounterOpts{ + Name: prometheus.BuildFQName(namespace, "index_stats", "total_scrapes"), + Help: "Current total ElasticSearch index scrapes.", + }), + jsonParseFailures: prometheus.NewCounter(prometheus.CounterOpts{ + Name: prometheus.BuildFQName(namespace, "index_stats", "json_parse_failures"), + Help: "Number of errors while parsing JSON.", + }), + + indexMetrics: []*indexMetric{ + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "docs_primary"), + "Count of documents which only primary shards", + defaultIndexLabels, nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.Docs.Count) + }, + Labels: defaultIndexLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "store_size_bytes_primary"), + "Current total size of stored index data in bytes which only primary shards on all nodes", + defaultIndexLabels, nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.Store.SizeInBytes) + }, + Labels: defaultIndexLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "store_size_bytes_total"), + "Current total size of stored index data in bytes which all shards on all nodes", + defaultIndexLabels, nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Total.Store.SizeInBytes) + }, + Labels: defaultIndexLabelValues, + }, + }, + } +} + +func (i *Indices) Describe(ch chan<- *prometheus.Desc) { + for _, metric := range i.indexMetrics { + ch <- metric.Desc + } + ch <- i.up.Desc() + ch <- i.totalScrapes.Desc() + ch <- i.jsonParseFailures.Desc() +} + +func (c *Indices) fetchAndDecodeStats() (indexStatsResponse, error) { + var isr indexStatsResponse + + u := *c.url + u.Path = "/_all/_stats" + + res, err := c.client.Get(u.String()) + if err != nil { + return isr, fmt.Errorf("failed to get index stats from %s://%s:%s/%s: %s", + u.Scheme, u.Hostname(), u.Port(), u.Path, err) + } + defer res.Body.Close() + if res.StatusCode != http.StatusOK { + return isr, fmt.Errorf("HTTP Request failed with code %d", res.StatusCode) + } + + if err := json.NewDecoder(res.Body).Decode(&isr); err != nil { + c.jsonParseFailures.Inc() + return isr, err + } + return isr, nil +} + +func (i *Indices) Collect(ch chan<- prometheus.Metric) { + i.totalScrapes.Inc() + defer func() { + ch <- i.up + ch <- i.totalScrapes + ch <- i.jsonParseFailures + }() + + clusterHealth := NewClusterHealth(i.logger, i.client, i.url) + clusterHealthResponse, err := clusterHealth.fetchAndDecodeClusterHealth() + if err != nil { + i.up.Set(0) + level.Warn(i.logger).Log( + "msg", "failed to fetch and decode cluster health", + "err", err, + ) + return + } + + indexStatsResponse, err := i.fetchAndDecodeStats() + if err != nil { + i.up.Set(0) + level.Warn(i.logger).Log( + "msg", "failed to fetch and decode index stats", + "err", err, + ) + return + } + i.up.Set(1) + + for indexName, indexStats := range indexStatsResponse.Indices { + for _, metric := range i.indexMetrics { + ch <- prometheus.MustNewConstMetric( + metric.Desc, + metric.Type, + metric.Value(indexStats), + metric.Labels(clusterHealthResponse.ClusterName, indexName)..., + ) + } + } +} diff --git a/collector/indices_response.go b/collector/indices_response.go new file mode 100644 index 00000000..f65df501 --- /dev/null +++ b/collector/indices_response.go @@ -0,0 +1,169 @@ +package collector + +// indexStatsResponse is a representation of a Elasticsearch Index Stats +type indexStatsResponse struct { + Shards IndexStatsShardsResponse `json:"_shards"` + All IndexStatsIndexResponse `json:"_all"` + Indices map[string]IndexStatsIndexResponse `json:"indices"` +} + +type IndexStatsShardsResponse struct { + Total int64 `json:"total"` + Successful int64 `json:"successful"` + Failed int64 `json:"failed"` +} + +type IndexStatsIndexResponse struct { + Primaries IndexStatsIndexDetailResponse `json:"primaries"` + Total IndexStatsIndexDetailResponse `json:"total"` +} + +type IndexStatsIndexDetailResponse struct { + Docs IndexStatsIndexDocsResponse `json:"docs"` + Store IndexStatsIndexStoreResponse `json:"store"` + Indexing IndexStatsIndexIndexingResponse `json:"indexing"` + Get IndexStatsIndexGetResponse `json:"get"` + Search IndexStatsIndexSearchResponse `json:"search"` + Merges IndexStatsIndexMergesResponse `json:"merges"` + Refresh IndexStatsIndexRefreshResponse `json:"refresh"` + Flush IndexStatsIndexFlushResponse `json:"flush"` + Warmer IndexStatsIndexWarmerResponse `json:"warmer"` + QueryCache IndexStatsIndexQueryCacheResponse `json:"query_cache"` + Fielddata IndexStatsIndexFielddataResponse `json:"fielddata"` + Completion IndexStatsIndexCompletionResponse `json:"completion"` + Segments IndexStatsIndexSegmentsResponse `json:"segments"` + Translog IndexStatsIndexTranslogResponse `json:"translog"` + RequestCache IndexStatsIndexRequestCacheResponse `json:"request_cache"` + Recovery IndexStatsIndexRecoveryResponse `json:"recovery"` +} + +type IndexStatsIndexDocsResponse struct { + Count int64 `json:"count"` + Deleted int64 `json:"deleted"` +} + +type IndexStatsIndexStoreResponse struct { + SizeInBytes int64 `json:"size_in_bytes"` + ThrottleTimeInMillis int64 `json:"throttle_time_in_millis"` +} + +type IndexStatsIndexIndexingResponse struct { + IndexTotal int64 `json:"index_total"` + IndexTimeInMillis int64 `json:"index_time_in_millis"` + IndexCurrent int64 `json:"index_current"` + IndexFailed int64 `json:"index_failed"` + DeleteTotal int64 `json:"delete_total"` + DeleteTimeInMillis int64 `json:"delete_time_in_millis"` + DeleteCurrent int64 `json:"delete_current"` + NoopUpdateTotal int64 `json:"noop_update_total"` + IsThrottled bool `json:"is_throttled"` + ThrottleTimeInMillis int64 `json:"throttle_time_in_millis"` +} + +type IndexStatsIndexGetResponse struct { + Total int64 `json:"total"` + TimeInMillis int64 `json:"time_in_millis"` + ExistsTotal int64 `json:"exists_total"` + ExistsTimeInMillis int64 `json:"exists_time_in_millis"` + MissingTotal int64 `json:"missing_total"` + MissingTimeInMillis int64 `json:"missing_time_in_millis"` + Current int64 `json:"current"` +} + +type IndexStatsIndexSearchResponse struct { + OpenContexts int64 `json:"open_contexts"` + QueryTotal int64 `json:"query_total"` + QueryTimeInMillis int64 `json:"query_time_in_millis"` + QueryCurrent int64 `json:"query_current"` + FetchTotal int64 `json:"fetch_total"` + FetchTimeInMillis int64 `json:"fetch_time_in_millis"` + FetchCurrent int64 `json:"fetch_current"` + ScrollTotal int64 `json:"scroll_total"` + ScrollTimeInMillis int64 `json:"scroll_time_in_millis"` + ScrollCurrent int64 `json:"scroll_current"` + SuggestTotal int64 `json:"suggest_total"` + SuggestTimeInMillis int64 `json:"suggest_time_in_millis"` + SuggestCurrent int64 `json:"suggest_current"` +} + +type IndexStatsIndexMergesResponse struct { + Current int64 `json:"current"` + CurrentDocs int64 `json:"current_docs"` + CurrentSizeInBytes int64 `json:"current_size_in_bytes"` + Total int64 `json:"total"` + TotalTimeInMillis int64 `json:"total_time_in_millis"` + TotalDocs int64 `json:"total_docs"` + TotalSizeInBytes int64 `json:"total_size_in_bytes"` + TotalStoppedTimeInMillis int64 `json:"total_stopped_time_in_millis"` + TotalThrottledTimeInMillis int64 `json:"total_throttled_time_in_millis"` + TotalAutoThrottleInBytes int64 `json:"total_auto_throttle_in_bytes"` +} + +type IndexStatsIndexRefreshResponse struct { + Total int64 `json:"total"` + TotalTimeInMillis int64 `json:"total_time_in_millis"` + Listeners int64 `json:"listeners"` +} + +type IndexStatsIndexFlushResponse struct { + Total int64 `json:"total"` + TotalTimeInMillis int64 `json:"total_time_in_millis"` +} + +type IndexStatsIndexWarmerResponse struct { + Current int64 `json:"current"` + Total int64 `json:"total"` + TotalTimeInMillis int64 `json:"total_time_in_millis"` +} + +type IndexStatsIndexQueryCacheResponse struct { + MemorySizeInBytes int64 `json:"memory_size_in_bytes"` + TotalCount int64 `json:"total_count"` + HitCount int64 `json:"hit_count"` + MissCount int64 `json:"miss_count"` + CacheSize int64 `json:"cache_size"` + CacheCount int64 `json:"cache_count"` + Evictions int64 `json:"evictions"` +} + +type IndexStatsIndexFielddataResponse struct { + MemorySizeInBytes int64 `json:"memory_size_in_bytes"` + Evictions int64 `json:"evictions"` +} + +type IndexStatsIndexCompletionResponse struct { + SizeInBytes int64 `json:"size_in_bytes"` +} + +type IndexStatsIndexSegmentsResponse struct { + Count int64 `json:"count"` + MemoryInBytes int64 `json:"memory_in_bytes"` + TermsMemoryInBytes int64 `json:"terms_memory_in_bytes"` + StoredFieldsMemoryInBytes int64 `json:"stored_fields_memory_in_bytes"` + TermVectorsMemoryInBytes int64 `json:"term_vectors_memory_in_bytes"` + NormsMemoryInBytes int64 `json:"norms_memory_in_bytes"` + PointsMemoryInBytes int64 `json:"points_memory_in_bytes"` + DocValuesMemoryInBytes int64 `json:"doc_values_memory_in_bytes"` + IndexWriterMemoryInBytes int64 `json:"index_writer_memory_in_bytes"` + VersionMapMemoryInBytes int64 `json:"version_map_memory_in_bytes"` + FixedBitSetMemoryInBytes int64 `json:"fixed_bit_set_memory_in_bytes"` + MaxUnsafeAutoIdTimestamp int64 `json:"max_unsafe_auto_id_timestamp"` +} + +type IndexStatsIndexTranslogResponse struct { + Operations int64 `json:"operations"` + SizeInBytes int64 `json:"size_in_bytes"` +} + +type IndexStatsIndexRequestCacheResponse struct { + MemorySizeInBytes int64 `json:"memory_size_in_bytes"` + Evictions int64 `json:"evictions"` + HitCount int64 `json:"hit_count"` + MissCount int64 `json:"miss_count"` +} + +type IndexStatsIndexRecoveryResponse struct { + CurrentAsSource int64 `json:"current_as_source"` + CurrentAsTarget int64 `json:"current_as_target"` + ThrottleTimeInMillis int64 `json:"throttle_time_in_millis"` +} diff --git a/collector/indices_test.go b/collector/indices_test.go new file mode 100644 index 00000000..b93e495a --- /dev/null +++ b/collector/indices_test.go @@ -0,0 +1,54 @@ +package collector + +import ( + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "testing" + + "github.com/go-kit/kit/log" +) + +func TestIndices(t *testing.T) { + // Testcases created using: + // docker run -d -p 9200:9200 elasticsearch:VERSION-alpine + // curl -XPUT http://localhost:9200/foo_1/type1/1 -d '{"title":"abc","content":"hello"}' + // curl -XPUT http://localhost:9200/foo_1/type1/2 -d '{"title":"def","content":"world"}' + // curl -XPUT http://localhost:9200/foo_2/type1/1 -d '{"title":"abc001","content":"hello001"}' + // curl -XPUT http://localhost:9200/foo_2/type1/2 -d '{"title":"def002","content":"world002"}' + // curl -XPUT http://localhost:9200/foo_2/type1/3 -d '{"title":"def003","content":"world003"}' + // curl http://localhost:9200/_all/_stats + ti := map[string]string{ + "5.5.1": `{"_shards":{"total":34,"successful":34,"failed":0},"_all":{"primaries":{"docs":{"count":256658,"deleted":148},"store":{"size_in_bytes":175959144,"throttle_time_in_millis":0},"indexing":{"index_total":413652,"index_time_in_millis":316231,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":88438,"time_in_millis":2799,"exists_total":88438,"exists_time_in_millis":2799,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":88458,"query_time_in_millis":2513,"query_current":0,"fetch_total":88446,"fetch_time_in_millis":2685,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":10275,"total_time_in_millis":5895937,"total_docs":80376844,"total_size_in_bytes":49005089428,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":274,"total_auto_throttle_in_bytes":354609338},"refresh":{"total":100264,"total_time_in_millis":818781,"listeners":0},"flush":{"total":5,"total_time_in_millis":124},"warmer":{"current":0,"total":100286,"total_time_in_millis":23851},"query_cache":{"memory_size_in_bytes":0,"total_count":22,"hit_count":0,"miss_count":22,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":38,"memory_in_bytes":720166,"terms_memory_in_bytes":481188,"stored_fields_memory_in_bytes":31920,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":3072,"points_memory_in_bytes":61858,"doc_values_memory_in_bytes":142128,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":137331,"size_in_bytes":167642357},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":1},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":513316,"deleted":324},"store":{"size_in_bytes":352143858,"throttle_time_in_millis":0},"indexing":{"index_total":827302,"index_time_in_millis":625030,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":176883,"time_in_millis":6265,"exists_total":176883,"exists_time_in_millis":6265,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":176913,"query_time_in_millis":5115,"query_current":0,"fetch_total":176891,"fetch_time_in_millis":5862,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":20547,"total_time_in_millis":11765629,"total_docs":160766745,"total_size_in_bytes":98024993696,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":522,"total_auto_throttle_in_bytes":709218676},"refresh":{"total":200378,"total_time_in_millis":1620051,"listeners":0},"flush":{"total":10,"total_time_in_millis":264},"warmer":{"current":0,"total":200427,"total_time_in_millis":46689},"query_cache":{"memory_size_in_bytes":0,"total_count":54,"hit_count":0,"miss_count":54,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":776,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":86,"memory_in_bytes":1496932,"terms_memory_in_bytes":1008932,"stored_fields_memory_in_bytes":66896,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":6784,"points_memory_in_bytes":124368,"doc_values_memory_in_bytes":289952,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":1502755202429,"file_sizes":{}},"translog":{"operations":274662,"size_in_bytes":335284714},"request_cache":{"memory_size_in_bytes":1794,"evictions":0,"hit_count":0,"miss_count":3},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}},"indices":{".monitoring-kibana-6-2017.08.16":{"primaries":{"docs":{"count":12459,"deleted":0},"store":{"size_in_bytes":4880578,"throttle_time_in_millis":0},"indexing":{"index_total":12459,"index_time_in_millis":20073,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":1372,"total_time_in_millis":275560,"total_docs":8619963,"total_size_in_bytes":3518738895,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":20971520},"refresh":{"total":12724,"total_time_in_millis":79600,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":12725,"total_time_in_millis":2653},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":2,"memory_in_bytes":34337,"terms_memory_in_bytes":30104,"stored_fields_memory_in_bytes":968,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":128,"points_memory_in_bytes":1393,"doc_values_memory_in_bytes":1744,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":12459,"size_in_bytes":12175093},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":24918,"deleted":0},"store":{"size_in_bytes":9682148,"throttle_time_in_millis":0},"indexing":{"index_total":24918,"index_time_in_millis":37852,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":1,"query_time_in_millis":10,"query_current":0,"fetch_total":1,"fetch_time_in_millis":1,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":2743,"total_time_in_millis":549689,"total_docs":17234974,"total_size_in_bytes":7034624631,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":41943040},"refresh":{"total":25390,"total_time_in_millis":155546,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":25392,"total_time_in_millis":5196},"query_cache":{"memory_size_in_bytes":0,"total_count":3,"hit_count":0,"miss_count":3,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":7,"memory_in_bytes":83565,"terms_memory_in_bytes":73488,"stored_fields_memory_in_bytes":2968,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":448,"points_memory_in_bytes":2897,"doc_values_memory_in_bytes":3764,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":24918,"size_in_bytes":24350186},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}},".monitoring-kibana-6-2017.08.15":{"primaries":{"docs":{"count":25917,"deleted":0},"store":{"size_in_bytes":9617967,"throttle_time_in_millis":0},"indexing":{"index_total":25917,"index_time_in_millis":36304,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":1,"query_time_in_millis":10,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":2055,"total_time_in_millis":746516,"total_docs":27797269,"total_size_in_bytes":10672726928,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":20971520},"refresh":{"total":19569,"total_time_in_millis":123083,"listeners":0},"flush":{"total":1,"total_time_in_millis":18},"warmer":{"current":0,"total":19571,"total_time_in_millis":3995},"query_cache":{"memory_size_in_bytes":0,"total_count":3,"hit_count":0,"miss_count":3,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":3,"memory_in_bytes":58226,"terms_memory_in_bytes":52997,"stored_fields_memory_in_bytes":1912,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":192,"points_memory_in_bytes":2721,"doc_values_memory_in_bytes":404,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":0,"size_in_bytes":43},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":51834,"deleted":0},"store":{"size_in_bytes":19309686,"throttle_time_in_millis":0},"indexing":{"index_total":51834,"index_time_in_millis":68258,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":1,"query_time_in_millis":10,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":4109,"total_time_in_millis":1477106,"total_docs":55598675,"total_size_in_bytes":21347372840,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":41943040},"refresh":{"total":39103,"total_time_in_millis":238886,"listeners":0},"flush":{"total":2,"total_time_in_millis":44},"warmer":{"current":0,"total":39108,"total_time_in_millis":7775},"query_cache":{"memory_size_in_bytes":0,"total_count":3,"hit_count":0,"miss_count":3,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":11,"memory_in_bytes":141204,"terms_memory_in_bytes":128132,"stored_fields_memory_in_bytes":5472,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":704,"points_memory_in_bytes":5628,"doc_values_memory_in_bytes":1268,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":1502755202429,"file_sizes":{}},"translog":{"operations":0,"size_in_bytes":86},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}},".monitoring-kibana-6-2017.08.14":{"primaries":{"docs":{"count":6147,"deleted":0},"store":{"size_in_bytes":2467088,"throttle_time_in_millis":0},"indexing":{"index_total":6147,"index_time_in_millis":9228,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":1,"query_time_in_millis":32,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":397,"total_time_in_millis":48142,"total_docs":1231276,"total_size_in_bytes":559560922,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":20971520},"refresh":{"total":3733,"total_time_in_millis":28841,"listeners":0},"flush":{"total":1,"total_time_in_millis":7},"warmer":{"current":0,"total":3735,"total_time_in_millis":1168},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":3,"memory_in_bytes":29099,"terms_memory_in_bytes":25724,"stored_fields_memory_in_bytes":1104,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":192,"points_memory_in_bytes":715,"doc_values_memory_in_bytes":1364,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":0,"size_in_bytes":43},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":12294,"deleted":0},"store":{"size_in_bytes":5029190,"throttle_time_in_millis":0},"indexing":{"index_total":12294,"index_time_in_millis":16557,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":1,"query_time_in_millis":32,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":792,"total_time_in_millis":96523,"total_docs":2467549,"total_size_in_bytes":1121117660,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":41943040},"refresh":{"total":7449,"total_time_in_millis":51544,"listeners":0},"flush":{"total":2,"total_time_in_millis":16},"warmer":{"current":0,"total":7453,"total_time_in_millis":1872},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":11,"memory_in_bytes":83924,"terms_memory_in_bytes":73590,"stored_fields_memory_in_bytes":3768,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":704,"points_memory_in_bytes":1618,"doc_values_memory_in_bytes":4244,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":0,"size_in_bytes":86},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}},".kibana":{"primaries":{"docs":{"count":1,"deleted":0},"store":{"size_in_bytes":3967,"throttle_time_in_millis":0},"indexing":{"index_total":1,"index_time_in_millis":2,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":88438,"time_in_millis":2799,"exists_total":88438,"exists_time_in_millis":2799,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":88441,"query_time_in_millis":2287,"query_current":0,"fetch_total":88441,"fetch_time_in_millis":2685,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":20971520},"refresh":{"total":2,"total_time_in_millis":19,"listeners":0},"flush":{"total":1,"total_time_in_millis":7},"warmer":{"current":0,"total":5,"total_time_in_millis":12},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":1,"memory_in_bytes":2219,"terms_memory_in_bytes":1751,"stored_fields_memory_in_bytes":312,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":64,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":92,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":0,"size_in_bytes":43},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":2,"deleted":0},"store":{"size_in_bytes":7934,"throttle_time_in_millis":0},"indexing":{"index_total":2,"index_time_in_millis":4,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":176883,"time_in_millis":6265,"exists_total":176883,"exists_time_in_millis":6265,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":176879,"query_time_in_millis":4703,"query_current":0,"fetch_total":176879,"fetch_time_in_millis":5860,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":41943040},"refresh":{"total":4,"total_time_in_millis":33,"listeners":0},"flush":{"total":2,"total_time_in_millis":14},"warmer":{"current":0,"total":10,"total_time_in_millis":26},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":2,"memory_in_bytes":4438,"terms_memory_in_bytes":3502,"stored_fields_memory_in_bytes":624,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":128,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":184,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":0,"size_in_bytes":86},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}},".monitoring-es-6-2017.08.15":{"primaries":{"docs":{"count":120970,"deleted":60},"store":{"size_in_bytes":90471073,"throttle_time_in_millis":0},"indexing":{"index_total":207360,"index_time_in_millis":142584,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":1,"query_time_in_millis":85,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":3781,"total_time_in_millis":2188959,"total_docs":19326607,"total_size_in_bytes":15981823826,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":274,"total_auto_throttle_in_bytes":19065018},"refresh":{"total":37364,"total_time_in_millis":337011,"listeners":0},"flush":{"total":1,"total_time_in_millis":67},"warmer":{"current":0,"total":37365,"total_time_in_millis":8943},"query_cache":{"memory_size_in_bytes":0,"total_count":9,"hit_count":0,"miss_count":9,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":9,"memory_in_bytes":297633,"terms_memory_in_bytes":184606,"stored_fields_memory_in_bytes":13736,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":576,"points_memory_in_bytes":31279,"doc_values_memory_in_bytes":67436,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":0,"size_in_bytes":43},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":241940,"deleted":120},"store":{"size_in_bytes":180988889,"throttle_time_in_millis":0},"indexing":{"index_total":414720,"index_time_in_millis":286230,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":2,"query_time_in_millis":135,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":7561,"total_time_in_millis":4425176,"total_docs":38722769,"total_size_in_bytes":32018268991,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":522,"total_auto_throttle_in_bytes":38130036},"refresh":{"total":74706,"total_time_in_millis":675900,"listeners":0},"flush":{"total":2,"total_time_in_millis":148},"warmer":{"current":0,"total":74710,"total_time_in_millis":18069},"query_cache":{"memory_size_in_bytes":0,"total_count":24,"hit_count":0,"miss_count":24,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":424,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":16,"memory_in_bytes":590531,"terms_memory_in_bytes":362031,"stored_fields_memory_in_bytes":26728,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":1024,"points_memory_in_bytes":62748,"doc_values_memory_in_bytes":138000,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":1502755201903,"file_sizes":{}},"translog":{"operations":0,"size_in_bytes":86},"request_cache":{"memory_size_in_bytes":897,"evictions":0,"hit_count":0,"miss_count":1},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}},".monitoring-es-6-2017.08.14":{"primaries":{"docs":{"count":24614,"deleted":12},"store":{"size_in_bytes":17680221,"throttle_time_in_millis":0},"indexing":{"index_total":36896,"index_time_in_millis":35441,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":1,"query_time_in_millis":24,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":927,"total_time_in_millis":1245791,"total_docs":11392506,"total_size_in_bytes":8435013687,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":20971520},"refresh":{"total":9432,"total_time_in_millis":90529,"listeners":0},"flush":{"total":1,"total_time_in_millis":25},"warmer":{"current":0,"total":9434,"total_time_in_millis":3098},"query_cache":{"memory_size_in_bytes":0,"total_count":2,"hit_count":0,"miss_count":2,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":7,"memory_in_bytes":83467,"terms_memory_in_bytes":59601,"stored_fields_memory_in_bytes":4040,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":448,"points_memory_in_bytes":7054,"doc_values_memory_in_bytes":12324,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":0,"size_in_bytes":43},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":49228,"deleted":18},"store":{"size_in_bytes":35172184,"throttle_time_in_millis":0},"indexing":{"index_total":73792,"index_time_in_millis":69939,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":2,"query_time_in_millis":61,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":1853,"total_time_in_millis":2468444,"total_docs":22769560,"total_size_in_bytes":16842830754,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":41943040},"refresh":{"total":18843,"total_time_in_millis":176757,"listeners":0},"flush":{"total":2,"total_time_in_millis":42},"warmer":{"current":0,"total":18847,"total_time_in_millis":5819},"query_cache":{"memory_size_in_bytes":0,"total_count":7,"hit_count":0,"miss_count":7,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":352,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":9,"memory_in_bytes":137869,"terms_memory_in_bytes":97131,"stored_fields_memory_in_bytes":6680,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":576,"points_memory_in_bytes":13662,"doc_values_memory_in_bytes":19820,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":0,"size_in_bytes":86},"request_cache":{"memory_size_in_bytes":897,"evictions":0,"hit_count":0,"miss_count":1},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}},"foo_2":{"primaries":{"docs":{"count":3,"deleted":0},"store":{"size_in_bytes":12138,"throttle_time_in_millis":0},"indexing":{"index_total":3,"index_time_in_millis":4,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":5,"query_time_in_millis":0,"query_current":0,"fetch_total":2,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":104857600},"refresh":{"total":3,"total_time_in_millis":12,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":8,"total_time_in_millis":0},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":3,"memory_in_bytes":7764,"terms_memory_in_bytes":5976,"stored_fields_memory_in_bytes":936,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":576,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":276,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":3,"size_in_bytes":524},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":6,"deleted":0},"store":{"size_in_bytes":24276,"throttle_time_in_millis":0},"indexing":{"index_total":5,"index_time_in_millis":6,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":10,"query_time_in_millis":0,"query_current":0,"fetch_total":5,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":209715200},"refresh":{"total":5,"total_time_in_millis":20,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":16,"total_time_in_millis":0},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":6,"memory_in_bytes":15528,"terms_memory_in_bytes":11952,"stored_fields_memory_in_bytes":1872,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":1152,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":552,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":6,"size_in_bytes":1048},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}},"foo_1":{"primaries":{"docs":{"count":2,"deleted":0},"store":{"size_in_bytes":8246,"throttle_time_in_millis":0},"indexing":{"index_total":4,"index_time_in_millis":7,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":7,"query_time_in_millis":8,"query_current":0,"fetch_total":3,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":104857600},"refresh":{"total":4,"total_time_in_millis":18,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":9,"total_time_in_millis":0},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":2,"memory_in_bytes":5176,"terms_memory_in_bytes":3984,"stored_fields_memory_in_bytes":624,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":384,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":184,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":4,"size_in_bytes":609},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":4,"deleted":0},"store":{"size_in_bytes":16492,"throttle_time_in_millis":0},"indexing":{"index_total":7,"index_time_in_millis":13,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":15,"query_time_in_millis":21,"query_current":0,"fetch_total":5,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":209715200},"refresh":{"total":7,"total_time_in_millis":32,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":18,"total_time_in_millis":0},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":4,"memory_in_bytes":10352,"terms_memory_in_bytes":7968,"stored_fields_memory_in_bytes":1248,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":768,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":368,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":8,"size_in_bytes":1218},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}},".monitoring-es-6-2017.08.16":{"primaries":{"docs":{"count":66545,"deleted":76},"store":{"size_in_bytes":50817866,"throttle_time_in_millis":0},"indexing":{"index_total":124865,"index_time_in_millis":72588,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":1,"query_time_in_millis":67,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":1743,"total_time_in_millis":1390969,"total_docs":12009223,"total_size_in_bytes":9837225170,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":20971520},"refresh":{"total":17433,"total_time_in_millis":159668,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":17434,"total_time_in_millis":3982},"query_cache":{"memory_size_in_bytes":0,"total_count":8,"hit_count":0,"miss_count":8,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":8,"memory_in_bytes":202245,"terms_memory_in_bytes":116445,"stored_fields_memory_in_bytes":8288,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":512,"points_memory_in_bytes":18696,"doc_values_memory_in_bytes":58304,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":124865,"size_in_bytes":155465916},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":1},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":133090,"deleted":186},"store":{"size_in_bytes":101913059,"throttle_time_in_millis":0},"indexing":{"index_total":249730,"index_time_in_millis":146171,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":2,"query_time_in_millis":143,"query_current":0,"fetch_total":1,"fetch_time_in_millis":1,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":3489,"total_time_in_millis":2748691,"total_docs":23973218,"total_size_in_bytes":19660778820,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":41943040},"refresh":{"total":34871,"total_time_in_millis":321333,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":34873,"total_time_in_millis":7932},"query_cache":{"memory_size_in_bytes":0,"total_count":17,"hit_count":0,"miss_count":17,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":20,"memory_in_bytes":429521,"terms_memory_in_bytes":251138,"stored_fields_memory_in_bytes":17536,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":1280,"points_memory_in_bytes":37815,"doc_values_memory_in_bytes":121752,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":249730,"size_in_bytes":310931832},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":1},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}}}}`, + } + for ver, out := range ti { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, out) + })) + defer ts.Close() + + u, err := url.Parse(ts.URL) + if err != nil { + t.Fatalf("Failed to parse URL: %s", err) + } + i := NewIndices(log.NewNopLogger(), http.DefaultClient, u) + stats, err := i.fetchAndDecodeStats() + if err != nil { + t.Fatalf("Failed to fetch or decode indices stats: %s", err) + } + t.Logf("[%s] Index Response: %+v", ver, stats) + if len(stats.Indices) != 9 { + t.Errorf("Wrong number of indices") + } + if stats.Indices["foo_1"].Primaries.Docs.Count != 2 { + t.Errorf("Wrong number of primary docs") + } + if stats.Indices["foo_1"].Primaries.Store.SizeInBytes != 8246 { + t.Errorf("Wrong number of primary store size in bytes") + } + if stats.Indices["foo_1"].Total.Store.SizeInBytes != 16492 { + t.Errorf("Wrong number of total store size in bytes") + } + } +} diff --git a/main.go b/main.go index a4028a2a..d8875f99 100644 --- a/main.go +++ b/main.go @@ -55,6 +55,7 @@ func main() { prometheus.MustRegister(collector.NewClusterHealth(logger, httpClient, esURL)) prometheus.MustRegister(collector.NewNodes(logger, httpClient, esURL, *esAllNodes)) + prometheus.MustRegister(collector.NewIndices(logger, httpClient, esURL)) http.Handle(*metricsPath, prometheus.Handler()) http.HandleFunc("/", IndexHandler(*metricsPath)) From 0bf3dcc7582e0e8222dd34b280d6f61204d6b633 Mon Sep 17 00:00:00 2001 From: Manabu Matsuzaki Date: Wed, 16 Aug 2017 21:38:13 +0900 Subject: [PATCH 02/12] re-format --- collector/indices.go | 2 +- collector/indices_response.go | 208 +++++++++++++++++----------------- 2 files changed, 105 insertions(+), 105 deletions(-) diff --git a/collector/indices.go b/collector/indices.go index 329078dc..6509985d 100644 --- a/collector/indices.go +++ b/collector/indices.go @@ -1,8 +1,8 @@ package collector import ( - "fmt" "encoding/json" + "fmt" "net/http" "net/url" diff --git a/collector/indices_response.go b/collector/indices_response.go index f65df501..0501c817 100644 --- a/collector/indices_response.go +++ b/collector/indices_response.go @@ -2,15 +2,15 @@ package collector // indexStatsResponse is a representation of a Elasticsearch Index Stats type indexStatsResponse struct { - Shards IndexStatsShardsResponse `json:"_shards"` - All IndexStatsIndexResponse `json:"_all"` + Shards IndexStatsShardsResponse `json:"_shards"` + All IndexStatsIndexResponse `json:"_all"` Indices map[string]IndexStatsIndexResponse `json:"indices"` } type IndexStatsShardsResponse struct { - Total int64 `json:"total"` - Successful int64 `json:"successful"` - Failed int64 `json:"failed"` + Total int64 `json:"total"` + Successful int64 `json:"successful"` + Failed int64 `json:"failed"` } type IndexStatsIndexResponse struct { @@ -19,151 +19,151 @@ type IndexStatsIndexResponse struct { } type IndexStatsIndexDetailResponse struct { - Docs IndexStatsIndexDocsResponse `json:"docs"` - Store IndexStatsIndexStoreResponse `json:"store"` - Indexing IndexStatsIndexIndexingResponse `json:"indexing"` - Get IndexStatsIndexGetResponse `json:"get"` - Search IndexStatsIndexSearchResponse `json:"search"` - Merges IndexStatsIndexMergesResponse `json:"merges"` - Refresh IndexStatsIndexRefreshResponse `json:"refresh"` - Flush IndexStatsIndexFlushResponse `json:"flush"` - Warmer IndexStatsIndexWarmerResponse `json:"warmer"` - QueryCache IndexStatsIndexQueryCacheResponse `json:"query_cache"` - Fielddata IndexStatsIndexFielddataResponse `json:"fielddata"` - Completion IndexStatsIndexCompletionResponse `json:"completion"` - Segments IndexStatsIndexSegmentsResponse `json:"segments"` - Translog IndexStatsIndexTranslogResponse `json:"translog"` - RequestCache IndexStatsIndexRequestCacheResponse `json:"request_cache"` - Recovery IndexStatsIndexRecoveryResponse `json:"recovery"` + Docs IndexStatsIndexDocsResponse `json:"docs"` + Store IndexStatsIndexStoreResponse `json:"store"` + Indexing IndexStatsIndexIndexingResponse `json:"indexing"` + Get IndexStatsIndexGetResponse `json:"get"` + Search IndexStatsIndexSearchResponse `json:"search"` + Merges IndexStatsIndexMergesResponse `json:"merges"` + Refresh IndexStatsIndexRefreshResponse `json:"refresh"` + Flush IndexStatsIndexFlushResponse `json:"flush"` + Warmer IndexStatsIndexWarmerResponse `json:"warmer"` + QueryCache IndexStatsIndexQueryCacheResponse `json:"query_cache"` + Fielddata IndexStatsIndexFielddataResponse `json:"fielddata"` + Completion IndexStatsIndexCompletionResponse `json:"completion"` + Segments IndexStatsIndexSegmentsResponse `json:"segments"` + Translog IndexStatsIndexTranslogResponse `json:"translog"` + RequestCache IndexStatsIndexRequestCacheResponse `json:"request_cache"` + Recovery IndexStatsIndexRecoveryResponse `json:"recovery"` } type IndexStatsIndexDocsResponse struct { - Count int64 `json:"count"` - Deleted int64 `json:"deleted"` + Count int64 `json:"count"` + Deleted int64 `json:"deleted"` } type IndexStatsIndexStoreResponse struct { - SizeInBytes int64 `json:"size_in_bytes"` - ThrottleTimeInMillis int64 `json:"throttle_time_in_millis"` + SizeInBytes int64 `json:"size_in_bytes"` + ThrottleTimeInMillis int64 `json:"throttle_time_in_millis"` } type IndexStatsIndexIndexingResponse struct { - IndexTotal int64 `json:"index_total"` - IndexTimeInMillis int64 `json:"index_time_in_millis"` - IndexCurrent int64 `json:"index_current"` - IndexFailed int64 `json:"index_failed"` - DeleteTotal int64 `json:"delete_total"` - DeleteTimeInMillis int64 `json:"delete_time_in_millis"` - DeleteCurrent int64 `json:"delete_current"` - NoopUpdateTotal int64 `json:"noop_update_total"` - IsThrottled bool `json:"is_throttled"` - ThrottleTimeInMillis int64 `json:"throttle_time_in_millis"` + IndexTotal int64 `json:"index_total"` + IndexTimeInMillis int64 `json:"index_time_in_millis"` + IndexCurrent int64 `json:"index_current"` + IndexFailed int64 `json:"index_failed"` + DeleteTotal int64 `json:"delete_total"` + DeleteTimeInMillis int64 `json:"delete_time_in_millis"` + DeleteCurrent int64 `json:"delete_current"` + NoopUpdateTotal int64 `json:"noop_update_total"` + IsThrottled bool `json:"is_throttled"` + ThrottleTimeInMillis int64 `json:"throttle_time_in_millis"` } type IndexStatsIndexGetResponse struct { - Total int64 `json:"total"` - TimeInMillis int64 `json:"time_in_millis"` - ExistsTotal int64 `json:"exists_total"` - ExistsTimeInMillis int64 `json:"exists_time_in_millis"` - MissingTotal int64 `json:"missing_total"` - MissingTimeInMillis int64 `json:"missing_time_in_millis"` - Current int64 `json:"current"` + Total int64 `json:"total"` + TimeInMillis int64 `json:"time_in_millis"` + ExistsTotal int64 `json:"exists_total"` + ExistsTimeInMillis int64 `json:"exists_time_in_millis"` + MissingTotal int64 `json:"missing_total"` + MissingTimeInMillis int64 `json:"missing_time_in_millis"` + Current int64 `json:"current"` } type IndexStatsIndexSearchResponse struct { - OpenContexts int64 `json:"open_contexts"` - QueryTotal int64 `json:"query_total"` - QueryTimeInMillis int64 `json:"query_time_in_millis"` - QueryCurrent int64 `json:"query_current"` - FetchTotal int64 `json:"fetch_total"` - FetchTimeInMillis int64 `json:"fetch_time_in_millis"` - FetchCurrent int64 `json:"fetch_current"` - ScrollTotal int64 `json:"scroll_total"` - ScrollTimeInMillis int64 `json:"scroll_time_in_millis"` - ScrollCurrent int64 `json:"scroll_current"` - SuggestTotal int64 `json:"suggest_total"` - SuggestTimeInMillis int64 `json:"suggest_time_in_millis"` - SuggestCurrent int64 `json:"suggest_current"` + OpenContexts int64 `json:"open_contexts"` + QueryTotal int64 `json:"query_total"` + QueryTimeInMillis int64 `json:"query_time_in_millis"` + QueryCurrent int64 `json:"query_current"` + FetchTotal int64 `json:"fetch_total"` + FetchTimeInMillis int64 `json:"fetch_time_in_millis"` + FetchCurrent int64 `json:"fetch_current"` + ScrollTotal int64 `json:"scroll_total"` + ScrollTimeInMillis int64 `json:"scroll_time_in_millis"` + ScrollCurrent int64 `json:"scroll_current"` + SuggestTotal int64 `json:"suggest_total"` + SuggestTimeInMillis int64 `json:"suggest_time_in_millis"` + SuggestCurrent int64 `json:"suggest_current"` } type IndexStatsIndexMergesResponse struct { - Current int64 `json:"current"` - CurrentDocs int64 `json:"current_docs"` - CurrentSizeInBytes int64 `json:"current_size_in_bytes"` - Total int64 `json:"total"` - TotalTimeInMillis int64 `json:"total_time_in_millis"` - TotalDocs int64 `json:"total_docs"` - TotalSizeInBytes int64 `json:"total_size_in_bytes"` - TotalStoppedTimeInMillis int64 `json:"total_stopped_time_in_millis"` - TotalThrottledTimeInMillis int64 `json:"total_throttled_time_in_millis"` - TotalAutoThrottleInBytes int64 `json:"total_auto_throttle_in_bytes"` + Current int64 `json:"current"` + CurrentDocs int64 `json:"current_docs"` + CurrentSizeInBytes int64 `json:"current_size_in_bytes"` + Total int64 `json:"total"` + TotalTimeInMillis int64 `json:"total_time_in_millis"` + TotalDocs int64 `json:"total_docs"` + TotalSizeInBytes int64 `json:"total_size_in_bytes"` + TotalStoppedTimeInMillis int64 `json:"total_stopped_time_in_millis"` + TotalThrottledTimeInMillis int64 `json:"total_throttled_time_in_millis"` + TotalAutoThrottleInBytes int64 `json:"total_auto_throttle_in_bytes"` } type IndexStatsIndexRefreshResponse struct { - Total int64 `json:"total"` - TotalTimeInMillis int64 `json:"total_time_in_millis"` - Listeners int64 `json:"listeners"` + Total int64 `json:"total"` + TotalTimeInMillis int64 `json:"total_time_in_millis"` + Listeners int64 `json:"listeners"` } type IndexStatsIndexFlushResponse struct { - Total int64 `json:"total"` - TotalTimeInMillis int64 `json:"total_time_in_millis"` + Total int64 `json:"total"` + TotalTimeInMillis int64 `json:"total_time_in_millis"` } type IndexStatsIndexWarmerResponse struct { - Current int64 `json:"current"` - Total int64 `json:"total"` - TotalTimeInMillis int64 `json:"total_time_in_millis"` + Current int64 `json:"current"` + Total int64 `json:"total"` + TotalTimeInMillis int64 `json:"total_time_in_millis"` } type IndexStatsIndexQueryCacheResponse struct { - MemorySizeInBytes int64 `json:"memory_size_in_bytes"` - TotalCount int64 `json:"total_count"` - HitCount int64 `json:"hit_count"` - MissCount int64 `json:"miss_count"` - CacheSize int64 `json:"cache_size"` - CacheCount int64 `json:"cache_count"` - Evictions int64 `json:"evictions"` + MemorySizeInBytes int64 `json:"memory_size_in_bytes"` + TotalCount int64 `json:"total_count"` + HitCount int64 `json:"hit_count"` + MissCount int64 `json:"miss_count"` + CacheSize int64 `json:"cache_size"` + CacheCount int64 `json:"cache_count"` + Evictions int64 `json:"evictions"` } type IndexStatsIndexFielddataResponse struct { - MemorySizeInBytes int64 `json:"memory_size_in_bytes"` - Evictions int64 `json:"evictions"` + MemorySizeInBytes int64 `json:"memory_size_in_bytes"` + Evictions int64 `json:"evictions"` } type IndexStatsIndexCompletionResponse struct { - SizeInBytes int64 `json:"size_in_bytes"` + SizeInBytes int64 `json:"size_in_bytes"` } type IndexStatsIndexSegmentsResponse struct { - Count int64 `json:"count"` - MemoryInBytes int64 `json:"memory_in_bytes"` - TermsMemoryInBytes int64 `json:"terms_memory_in_bytes"` - StoredFieldsMemoryInBytes int64 `json:"stored_fields_memory_in_bytes"` - TermVectorsMemoryInBytes int64 `json:"term_vectors_memory_in_bytes"` - NormsMemoryInBytes int64 `json:"norms_memory_in_bytes"` - PointsMemoryInBytes int64 `json:"points_memory_in_bytes"` - DocValuesMemoryInBytes int64 `json:"doc_values_memory_in_bytes"` - IndexWriterMemoryInBytes int64 `json:"index_writer_memory_in_bytes"` - VersionMapMemoryInBytes int64 `json:"version_map_memory_in_bytes"` - FixedBitSetMemoryInBytes int64 `json:"fixed_bit_set_memory_in_bytes"` - MaxUnsafeAutoIdTimestamp int64 `json:"max_unsafe_auto_id_timestamp"` + Count int64 `json:"count"` + MemoryInBytes int64 `json:"memory_in_bytes"` + TermsMemoryInBytes int64 `json:"terms_memory_in_bytes"` + StoredFieldsMemoryInBytes int64 `json:"stored_fields_memory_in_bytes"` + TermVectorsMemoryInBytes int64 `json:"term_vectors_memory_in_bytes"` + NormsMemoryInBytes int64 `json:"norms_memory_in_bytes"` + PointsMemoryInBytes int64 `json:"points_memory_in_bytes"` + DocValuesMemoryInBytes int64 `json:"doc_values_memory_in_bytes"` + IndexWriterMemoryInBytes int64 `json:"index_writer_memory_in_bytes"` + VersionMapMemoryInBytes int64 `json:"version_map_memory_in_bytes"` + FixedBitSetMemoryInBytes int64 `json:"fixed_bit_set_memory_in_bytes"` + MaxUnsafeAutoIdTimestamp int64 `json:"max_unsafe_auto_id_timestamp"` } type IndexStatsIndexTranslogResponse struct { - Operations int64 `json:"operations"` - SizeInBytes int64 `json:"size_in_bytes"` + Operations int64 `json:"operations"` + SizeInBytes int64 `json:"size_in_bytes"` } type IndexStatsIndexRequestCacheResponse struct { - MemorySizeInBytes int64 `json:"memory_size_in_bytes"` - Evictions int64 `json:"evictions"` - HitCount int64 `json:"hit_count"` - MissCount int64 `json:"miss_count"` + MemorySizeInBytes int64 `json:"memory_size_in_bytes"` + Evictions int64 `json:"evictions"` + HitCount int64 `json:"hit_count"` + MissCount int64 `json:"miss_count"` } type IndexStatsIndexRecoveryResponse struct { - CurrentAsSource int64 `json:"current_as_source"` - CurrentAsTarget int64 `json:"current_as_target"` - ThrottleTimeInMillis int64 `json:"throttle_time_in_millis"` + CurrentAsSource int64 `json:"current_as_source"` + CurrentAsTarget int64 `json:"current_as_target"` + ThrottleTimeInMillis int64 `json:"throttle_time_in_millis"` } From e021ec26d8e8f7a5e952db459c2824b2ae6b4f02 Mon Sep 17 00:00:00 2001 From: Manabu Matsuzaki Date: Tue, 22 Aug 2017 01:33:57 +0900 Subject: [PATCH 03/12] moved all indices metric from `collector/nodes.go` to `collector/indices.go` --- collector/indices.go | 494 +++++++++++++++++++++++++++++++++++++- collector/indices_test.go | 4 +- collector/nodes.go | 456 ----------------------------------- main.go | 2 +- 4 files changed, 494 insertions(+), 462 deletions(-) diff --git a/collector/indices.go b/collector/indices.go index 6509985d..9ec034ca 100644 --- a/collector/indices.go +++ b/collector/indices.go @@ -29,19 +29,22 @@ type Indices struct { logger log.Logger client *http.Client url *url.URL + all bool up prometheus.Gauge totalScrapes prometheus.Counter jsonParseFailures prometheus.Counter + nodeMetrics []*nodeMetric indexMetrics []*indexMetric } -func NewIndices(logger log.Logger, client *http.Client, url *url.URL) *Indices { +func NewIndices(logger log.Logger, client *http.Client, url *url.URL, all bool) *Indices { return &Indices{ logger: logger, client: client, url: url, + all: all, up: prometheus.NewGauge(prometheus.GaugeOpts{ Name: prometheus.BuildFQName(namespace, "index_stats", "up"), @@ -56,6 +59,464 @@ func NewIndices(logger log.Logger, client *http.Client, url *url.URL) *Indices { Help: "Number of errors while parsing JSON.", }), + nodeMetrics: []*nodeMetric{ + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "fielddata_memory_size_bytes"), + "Field data cache memory usage in bytes", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.FieldData.MemorySize) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "fielddata_evictions"), + "Evictions from field data", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.FieldData.Evictions) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "filter_cache_memory_size_bytes"), + "Filter cache memory usage in bytes", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.FilterCache.MemorySize) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "filter_cache_evictions"), + "Evictions from filter cache", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.FilterCache.Evictions) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "query_cache_memory_size_bytes"), + "Query cache memory usage in bytes", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.QueryCache.MemorySize) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "query_cache_evictions"), + "Evictions from query cache", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.QueryCache.Evictions) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "request_cache_memory_size_bytes"), + "Request cache memory usage in bytes", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.RequestCache.MemorySize) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "request_cache_evictions"), + "Evictions from request cache", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.RequestCache.Evictions) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "translog_operations"), + "Total translog operations", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Translog.Operations) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "translog_size_in_bytes"), + "Total translog size in bytes", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Translog.Size) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "get_time_seconds"), + "Total get time in seconds", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Get.Time / 1000) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "get_total"), + "Total get", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Get.Total) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "get_missing_time_seconds"), + "Total time of get missing in seconds", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Get.MissingTime / 1000) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "get_missing_total"), + "Total get missing", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Get.MissingTotal) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "get_exists_time_seconds"), + "Total time get exists in seconds", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Get.ExistsTime / 1000) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "get_exists_total"), + "Total get exists operations", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Get.ExistsTotal) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_refresh", "time_seconds_total"), + "Total refreshes", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Refresh.TotalTime / 1000) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_refresh", "total"), + "Total time spent refreshing in seconds", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Refresh.Total) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "search_query_time_seconds"), + "Total search query time in seconds", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Search.QueryTime / 1000) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "search_query_total"), + "Total number of queries", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Search.QueryTotal) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "search_fetch_time_seconds"), + "Total search fetch time in seconds", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Search.FetchTime / 1000) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "search_fetch_total"), + "Total number of fetches", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Search.FetchTotal) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "docs"), + "Count of documents on this node", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Docs.Count) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "docs_deleted"), + "Count of deleted documents on this node", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Docs.Deleted) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "store_size_bytes"), + "Current size of stored index data in bytes", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Store.Size) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "store_throttle_time_seconds_total"), + "Throttle time for index store in seconds", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Store.ThrottleTime / 1000) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "segments_memory_bytes"), + "Current memory size of segments in bytes", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Segments.Memory) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "segments_count"), + "Count of index segments on this node", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Segments.Count) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "flush_total"), + "Total flushes", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Flush.Total) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "flush_time_seconds"), + "Cumulative flush time in seconds", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Flush.Time / 1000) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_indexing", "index_time_seconds_total"), + "Cumulative index time in seconds", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Indexing.IndexTime / 1000) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_indexing", "index_total"), + "Total index calls", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Indexing.IndexTotal) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_indexing", "delete_time_seconds_total"), + "Total time indexing delete in seconds", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Indexing.DeleteTime / 1000) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_indexing", "delete_total"), + "Total indexing deletes", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Indexing.DeleteTotal) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_merges", "total"), + "Total merges", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Merges.Total) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_merges", "docs_total"), + "Cumulative docs merged", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Merges.TotalDocs) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_merges", "total_size_bytes_total"), + "Total merge size in bytes", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Merges.TotalSize) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_merges", "total_time_seconds_total"), + "Total time spent merging in seconds", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Merges.TotalTime / 1000) + }, + Labels: defaultNodeLabelValues, + }, + }, indexMetrics: []*indexMetric{ { Type: prometheus.GaugeValue, @@ -106,7 +567,7 @@ func (i *Indices) Describe(ch chan<- *prometheus.Desc) { ch <- i.jsonParseFailures.Desc() } -func (c *Indices) fetchAndDecodeStats() (indexStatsResponse, error) { +func (c *Indices) fetchAndDecodeIndexStats() (indexStatsResponse, error) { var isr indexStatsResponse u := *c.url @@ -137,6 +598,7 @@ func (i *Indices) Collect(ch chan<- prometheus.Metric) { ch <- i.jsonParseFailures }() + // clusterHealth clusterHealth := NewClusterHealth(i.logger, i.client, i.url) clusterHealthResponse, err := clusterHealth.fetchAndDecodeClusterHealth() if err != nil { @@ -148,7 +610,20 @@ func (i *Indices) Collect(ch chan<- prometheus.Metric) { return } - indexStatsResponse, err := i.fetchAndDecodeStats() + // nodes + nodes := NewNodes(i.logger, i.client, i.url, i.all) + nodeStatsResponse, err := nodes.fetchAndDecodeNodeStats() + if err != nil { + i.up.Set(0) + level.Warn(i.logger).Log( + "msg", "failed to fetch and decode node stats", + "err", err, + ) + return + } + + // indices + indexStatsResponse, err := i.fetchAndDecodeIndexStats() if err != nil { i.up.Set(0) level.Warn(i.logger).Log( @@ -159,6 +634,19 @@ func (i *Indices) Collect(ch chan<- prometheus.Metric) { } i.up.Set(1) + // Node stats + for _, node := range nodeStatsResponse.Nodes { + for _, metric := range i.nodeMetrics { + ch <- prometheus.MustNewConstMetric( + metric.Desc, + metric.Type, + metric.Value(node), + metric.Labels(nodeStatsResponse.ClusterName, node)..., + ) + } + } + + // Index stats for indexName, indexStats := range indexStatsResponse.Indices { for _, metric := range i.indexMetrics { ch <- prometheus.MustNewConstMetric( diff --git a/collector/indices_test.go b/collector/indices_test.go index b93e495a..6907f99e 100644 --- a/collector/indices_test.go +++ b/collector/indices_test.go @@ -32,8 +32,8 @@ func TestIndices(t *testing.T) { if err != nil { t.Fatalf("Failed to parse URL: %s", err) } - i := NewIndices(log.NewNopLogger(), http.DefaultClient, u) - stats, err := i.fetchAndDecodeStats() + i := NewIndices(log.NewNopLogger(), http.DefaultClient, u, true) + stats, err := i.fetchAndDecodeIndexStats() if err != nil { t.Fatalf("Failed to fetch or decode indices stats: %s", err) } diff --git a/collector/nodes.go b/collector/nodes.go index 92f3d6e7..2bdd8372 100644 --- a/collector/nodes.go +++ b/collector/nodes.go @@ -100,462 +100,6 @@ func NewNodes(logger log.Logger, client *http.Client, url *url.URL, all bool) *N }), nodeMetrics: []*nodeMetric{ - { - Type: prometheus.GaugeValue, - Desc: prometheus.NewDesc( - prometheus.BuildFQName(namespace, "indices", "fielddata_memory_size_bytes"), - "Field data cache memory usage in bytes", - defaultNodeLabels, nil, - ), - Value: func(node NodeStatsNodeResponse) float64 { - return float64(node.Indices.FieldData.MemorySize) - }, - Labels: defaultNodeLabelValues, - }, - { - Type: prometheus.CounterValue, - Desc: prometheus.NewDesc( - prometheus.BuildFQName(namespace, "indices", "fielddata_evictions"), - "Evictions from field data", - defaultNodeLabels, nil, - ), - Value: func(node NodeStatsNodeResponse) float64 { - return float64(node.Indices.FieldData.Evictions) - }, - Labels: defaultNodeLabelValues, - }, - { - Type: prometheus.GaugeValue, - Desc: prometheus.NewDesc( - prometheus.BuildFQName(namespace, "indices", "filter_cache_memory_size_bytes"), - "Filter cache memory usage in bytes", - defaultNodeLabels, nil, - ), - Value: func(node NodeStatsNodeResponse) float64 { - return float64(node.Indices.FilterCache.MemorySize) - }, - Labels: defaultNodeLabelValues, - }, - { - Type: prometheus.CounterValue, - Desc: prometheus.NewDesc( - prometheus.BuildFQName(namespace, "indices", "filter_cache_evictions"), - "Evictions from filter cache", - defaultNodeLabels, nil, - ), - Value: func(node NodeStatsNodeResponse) float64 { - return float64(node.Indices.FilterCache.Evictions) - }, - Labels: defaultNodeLabelValues, - }, - { - Type: prometheus.GaugeValue, - Desc: prometheus.NewDesc( - prometheus.BuildFQName(namespace, "indices", "query_cache_memory_size_bytes"), - "Query cache memory usage in bytes", - defaultNodeLabels, nil, - ), - Value: func(node NodeStatsNodeResponse) float64 { - return float64(node.Indices.QueryCache.MemorySize) - }, - Labels: defaultNodeLabelValues, - }, - { - Type: prometheus.CounterValue, - Desc: prometheus.NewDesc( - prometheus.BuildFQName(namespace, "indices", "query_cache_evictions"), - "Evictions from query cache", - defaultNodeLabels, nil, - ), - Value: func(node NodeStatsNodeResponse) float64 { - return float64(node.Indices.QueryCache.Evictions) - }, - Labels: defaultNodeLabelValues, - }, - { - Type: prometheus.GaugeValue, - Desc: prometheus.NewDesc( - prometheus.BuildFQName(namespace, "indices", "request_cache_memory_size_bytes"), - "Request cache memory usage in bytes", - defaultNodeLabels, nil, - ), - Value: func(node NodeStatsNodeResponse) float64 { - return float64(node.Indices.RequestCache.MemorySize) - }, - Labels: defaultNodeLabelValues, - }, - { - Type: prometheus.CounterValue, - Desc: prometheus.NewDesc( - prometheus.BuildFQName(namespace, "indices", "request_cache_evictions"), - "Evictions from request cache", - defaultNodeLabels, nil, - ), - Value: func(node NodeStatsNodeResponse) float64 { - return float64(node.Indices.RequestCache.Evictions) - }, - Labels: defaultNodeLabelValues, - }, - { - Type: prometheus.CounterValue, - Desc: prometheus.NewDesc( - prometheus.BuildFQName(namespace, "indices", "translog_operations"), - "Total translog operations", - defaultNodeLabels, nil, - ), - Value: func(node NodeStatsNodeResponse) float64 { - return float64(node.Indices.Translog.Operations) - }, - Labels: defaultNodeLabelValues, - }, - { - Type: prometheus.CounterValue, - Desc: prometheus.NewDesc( - prometheus.BuildFQName(namespace, "indices", "translog_size_in_bytes"), - "Total translog size in bytes", - defaultNodeLabels, nil, - ), - Value: func(node NodeStatsNodeResponse) float64 { - return float64(node.Indices.Translog.Size) - }, - Labels: defaultNodeLabelValues, - }, - { - Type: prometheus.CounterValue, - Desc: prometheus.NewDesc( - prometheus.BuildFQName(namespace, "indices", "get_time_seconds"), - "Total get time in seconds", - defaultNodeLabels, nil, - ), - Value: func(node NodeStatsNodeResponse) float64 { - return float64(node.Indices.Get.Time / 1000) - }, - Labels: defaultNodeLabelValues, - }, - { - Type: prometheus.CounterValue, - Desc: prometheus.NewDesc( - prometheus.BuildFQName(namespace, "indices", "get_total"), - "Total get", - defaultNodeLabels, nil, - ), - Value: func(node NodeStatsNodeResponse) float64 { - return float64(node.Indices.Get.Total) - }, - Labels: defaultNodeLabelValues, - }, - { - Type: prometheus.CounterValue, - Desc: prometheus.NewDesc( - prometheus.BuildFQName(namespace, "indices", "get_missing_time_seconds"), - "Total time of get missing in seconds", - defaultNodeLabels, nil, - ), - Value: func(node NodeStatsNodeResponse) float64 { - return float64(node.Indices.Get.MissingTime / 1000) - }, - Labels: defaultNodeLabelValues, - }, - { - Type: prometheus.CounterValue, - Desc: prometheus.NewDesc( - prometheus.BuildFQName(namespace, "indices", "get_missing_total"), - "Total get missing", - defaultNodeLabels, nil, - ), - Value: func(node NodeStatsNodeResponse) float64 { - return float64(node.Indices.Get.MissingTotal) - }, - Labels: defaultNodeLabelValues, - }, - { - Type: prometheus.CounterValue, - Desc: prometheus.NewDesc( - prometheus.BuildFQName(namespace, "indices", "get_exists_time_seconds"), - "Total time get exists in seconds", - defaultNodeLabels, nil, - ), - Value: func(node NodeStatsNodeResponse) float64 { - return float64(node.Indices.Get.ExistsTime / 1000) - }, - Labels: defaultNodeLabelValues, - }, - { - Type: prometheus.CounterValue, - Desc: prometheus.NewDesc( - prometheus.BuildFQName(namespace, "indices", "get_exists_total"), - "Total get exists operations", - defaultNodeLabels, nil, - ), - Value: func(node NodeStatsNodeResponse) float64 { - return float64(node.Indices.Get.ExistsTotal) - }, - Labels: defaultNodeLabelValues, - }, - { - Type: prometheus.CounterValue, - Desc: prometheus.NewDesc( - prometheus.BuildFQName(namespace, "indices_refresh", "time_seconds_total"), - "Total refreshes", - defaultNodeLabels, nil, - ), - Value: func(node NodeStatsNodeResponse) float64 { - return float64(node.Indices.Refresh.TotalTime / 1000) - }, - Labels: defaultNodeLabelValues, - }, - { - Type: prometheus.CounterValue, - Desc: prometheus.NewDesc( - prometheus.BuildFQName(namespace, "indices_refresh", "total"), - "Total time spent refreshing in seconds", - defaultNodeLabels, nil, - ), - Value: func(node NodeStatsNodeResponse) float64 { - return float64(node.Indices.Refresh.Total) - }, - Labels: defaultNodeLabelValues, - }, - { - Type: prometheus.CounterValue, - Desc: prometheus.NewDesc( - prometheus.BuildFQName(namespace, "indices", "search_query_time_seconds"), - "Total search query time in seconds", - defaultNodeLabels, nil, - ), - Value: func(node NodeStatsNodeResponse) float64 { - return float64(node.Indices.Search.QueryTime / 1000) - }, - Labels: defaultNodeLabelValues, - }, - { - Type: prometheus.CounterValue, - Desc: prometheus.NewDesc( - prometheus.BuildFQName(namespace, "indices", "search_query_total"), - "Total number of queries", - defaultNodeLabels, nil, - ), - Value: func(node NodeStatsNodeResponse) float64 { - return float64(node.Indices.Search.QueryTotal) - }, - Labels: defaultNodeLabelValues, - }, - { - Type: prometheus.CounterValue, - Desc: prometheus.NewDesc( - prometheus.BuildFQName(namespace, "indices", "search_fetch_time_seconds"), - "Total search fetch time in seconds", - defaultNodeLabels, nil, - ), - Value: func(node NodeStatsNodeResponse) float64 { - return float64(node.Indices.Search.FetchTime / 1000) - }, - Labels: defaultNodeLabelValues, - }, - { - Type: prometheus.CounterValue, - Desc: prometheus.NewDesc( - prometheus.BuildFQName(namespace, "indices", "search_fetch_total"), - "Total number of fetches", - defaultNodeLabels, nil, - ), - Value: func(node NodeStatsNodeResponse) float64 { - return float64(node.Indices.Search.FetchTotal) - }, - Labels: defaultNodeLabelValues, - }, - { - Type: prometheus.GaugeValue, - Desc: prometheus.NewDesc( - prometheus.BuildFQName(namespace, "indices", "docs"), - "Count of documents on this node", - defaultNodeLabels, nil, - ), - Value: func(node NodeStatsNodeResponse) float64 { - return float64(node.Indices.Docs.Count) - }, - Labels: defaultNodeLabelValues, - }, - { - Type: prometheus.GaugeValue, - Desc: prometheus.NewDesc( - prometheus.BuildFQName(namespace, "indices", "docs_deleted"), - "Count of deleted documents on this node", - defaultNodeLabels, nil, - ), - Value: func(node NodeStatsNodeResponse) float64 { - return float64(node.Indices.Docs.Deleted) - }, - Labels: defaultNodeLabelValues, - }, - { - Type: prometheus.GaugeValue, - Desc: prometheus.NewDesc( - prometheus.BuildFQName(namespace, "indices", "store_size_bytes"), - "Current size of stored index data in bytes", - defaultNodeLabels, nil, - ), - Value: func(node NodeStatsNodeResponse) float64 { - return float64(node.Indices.Store.Size) - }, - Labels: defaultNodeLabelValues, - }, - { - Type: prometheus.CounterValue, - Desc: prometheus.NewDesc( - prometheus.BuildFQName(namespace, "indices", "store_throttle_time_seconds_total"), - "Throttle time for index store in seconds", - defaultNodeLabels, nil, - ), - Value: func(node NodeStatsNodeResponse) float64 { - return float64(node.Indices.Store.ThrottleTime / 1000) - }, - Labels: defaultNodeLabelValues, - }, - { - Type: prometheus.GaugeValue, - Desc: prometheus.NewDesc( - prometheus.BuildFQName(namespace, "indices", "segments_memory_bytes"), - "Current memory size of segments in bytes", - defaultNodeLabels, nil, - ), - Value: func(node NodeStatsNodeResponse) float64 { - return float64(node.Indices.Segments.Memory) - }, - Labels: defaultNodeLabelValues, - }, - { - Type: prometheus.GaugeValue, - Desc: prometheus.NewDesc( - prometheus.BuildFQName(namespace, "indices", "segments_count"), - "Count of index segments on this node", - defaultNodeLabels, nil, - ), - Value: func(node NodeStatsNodeResponse) float64 { - return float64(node.Indices.Segments.Count) - }, - Labels: defaultNodeLabelValues, - }, - { - Type: prometheus.CounterValue, - Desc: prometheus.NewDesc( - prometheus.BuildFQName(namespace, "indices", "flush_total"), - "Total flushes", - defaultNodeLabels, nil, - ), - Value: func(node NodeStatsNodeResponse) float64 { - return float64(node.Indices.Flush.Total) - }, - Labels: defaultNodeLabelValues, - }, - { - Type: prometheus.CounterValue, - Desc: prometheus.NewDesc( - prometheus.BuildFQName(namespace, "indices", "flush_time_seconds"), - "Cumulative flush time in seconds", - defaultNodeLabels, nil, - ), - Value: func(node NodeStatsNodeResponse) float64 { - return float64(node.Indices.Flush.Time / 1000) - }, - Labels: defaultNodeLabelValues, - }, - { - Type: prometheus.CounterValue, - Desc: prometheus.NewDesc( - prometheus.BuildFQName(namespace, "indices_indexing", "index_time_seconds_total"), - "Cumulative index time in seconds", - defaultNodeLabels, nil, - ), - Value: func(node NodeStatsNodeResponse) float64 { - return float64(node.Indices.Indexing.IndexTime / 1000) - }, - Labels: defaultNodeLabelValues, - }, - { - Type: prometheus.CounterValue, - Desc: prometheus.NewDesc( - prometheus.BuildFQName(namespace, "indices_indexing", "index_total"), - "Total index calls", - defaultNodeLabels, nil, - ), - Value: func(node NodeStatsNodeResponse) float64 { - return float64(node.Indices.Indexing.IndexTotal) - }, - Labels: defaultNodeLabelValues, - }, - { - Type: prometheus.CounterValue, - Desc: prometheus.NewDesc( - prometheus.BuildFQName(namespace, "indices_indexing", "delete_time_seconds_total"), - "Total time indexing delete in seconds", - defaultNodeLabels, nil, - ), - Value: func(node NodeStatsNodeResponse) float64 { - return float64(node.Indices.Indexing.DeleteTime / 1000) - }, - Labels: defaultNodeLabelValues, - }, - { - Type: prometheus.CounterValue, - Desc: prometheus.NewDesc( - prometheus.BuildFQName(namespace, "indices_indexing", "delete_total"), - "Total indexing deletes", - defaultNodeLabels, nil, - ), - Value: func(node NodeStatsNodeResponse) float64 { - return float64(node.Indices.Indexing.DeleteTotal) - }, - Labels: defaultNodeLabelValues, - }, - { - Type: prometheus.CounterValue, - Desc: prometheus.NewDesc( - prometheus.BuildFQName(namespace, "indices_merges", "total"), - "Total merges", - defaultNodeLabels, nil, - ), - Value: func(node NodeStatsNodeResponse) float64 { - return float64(node.Indices.Merges.Total) - }, - Labels: defaultNodeLabelValues, - }, - { - Type: prometheus.CounterValue, - Desc: prometheus.NewDesc( - prometheus.BuildFQName(namespace, "indices_merges", "docs_total"), - "Cumulative docs merged", - defaultNodeLabels, nil, - ), - Value: func(node NodeStatsNodeResponse) float64 { - return float64(node.Indices.Merges.TotalDocs) - }, - Labels: defaultNodeLabelValues, - }, - { - Type: prometheus.CounterValue, - Desc: prometheus.NewDesc( - prometheus.BuildFQName(namespace, "indices_merges", "total_size_bytes_total"), - "Total merge size in bytes", - defaultNodeLabels, nil, - ), - Value: func(node NodeStatsNodeResponse) float64 { - return float64(node.Indices.Merges.TotalSize) - }, - Labels: defaultNodeLabelValues, - }, - { - Type: prometheus.CounterValue, - Desc: prometheus.NewDesc( - prometheus.BuildFQName(namespace, "indices_merges", "total_time_seconds_total"), - "Total time spent merging in seconds", - defaultNodeLabels, nil, - ), - Value: func(node NodeStatsNodeResponse) float64 { - return float64(node.Indices.Merges.TotalTime / 1000) - }, - Labels: defaultNodeLabelValues, - }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( diff --git a/main.go b/main.go index d8875f99..dd5cea5d 100644 --- a/main.go +++ b/main.go @@ -55,7 +55,7 @@ func main() { prometheus.MustRegister(collector.NewClusterHealth(logger, httpClient, esURL)) prometheus.MustRegister(collector.NewNodes(logger, httpClient, esURL, *esAllNodes)) - prometheus.MustRegister(collector.NewIndices(logger, httpClient, esURL)) + prometheus.MustRegister(collector.NewIndices(logger, httpClient, esURL, *esAllNodes)) http.Handle(*metricsPath, prometheus.Handler()) http.HandleFunc("/", IndexHandler(*metricsPath)) From 911607596bd33fef1c19ffb12626a1c037e558dd Mon Sep 17 00:00:00 2001 From: Manabu Matsuzaki Date: Wed, 23 Aug 2017 21:49:28 +0900 Subject: [PATCH 04/12] gathered the metrics collect logic in metrics_collector --- collector/cluster_health.go | 40 +---- collector/cluster_health_test.go | 55 ------- collector/indices.go | 65 +------- collector/metrics_collector.go | 153 ++++++++++++++++++ ...ices_test.go => metrics_collector_test.go} | 135 ++++++++++++++++ collector/nodes.go | 42 +---- collector/nodes_test.go | 102 ------------ main.go | 4 +- 8 files changed, 293 insertions(+), 303 deletions(-) delete mode 100644 collector/cluster_health_test.go create mode 100644 collector/metrics_collector.go rename collector/{indices_test.go => metrics_collector_test.go} (65%) delete mode 100644 collector/nodes_test.go diff --git a/collector/cluster_health.go b/collector/cluster_health.go index dc301afd..d1af4631 100644 --- a/collector/cluster_health.go +++ b/collector/cluster_health.go @@ -1,13 +1,10 @@ package collector import ( - "encoding/json" - "fmt" "net/http" "net/url" "github.com/go-kit/kit/log" - "github.com/go-kit/kit/log/level" "github.com/prometheus/client_golang/prometheus" ) @@ -220,31 +217,7 @@ func (c *ClusterHealth) Describe(ch chan<- *prometheus.Desc) { ch <- c.jsonParseFailures.Desc() } -func (c *ClusterHealth) fetchAndDecodeClusterHealth() (clusterHealthResponse, error) { - var chr clusterHealthResponse - - u := *c.url - u.Path = "/_cluster/health" - res, err := c.client.Get(u.String()) - if err != nil { - return chr, fmt.Errorf("failed to get cluster health from %s://%s:%s/%s: %s", - u.Scheme, u.Hostname(), u.Port(), u.Path, err) - } - defer res.Body.Close() - - if res.StatusCode != http.StatusOK { - return chr, fmt.Errorf("HTTP Request failed with code %d", res.StatusCode) - } - - if err := json.NewDecoder(res.Body).Decode(&chr); err != nil { - c.jsonParseFailures.Inc() - return chr, err - } - - return chr, nil -} - -func (c *ClusterHealth) Collect(ch chan<- prometheus.Metric) { +func (c *ClusterHealth) Collect(ch chan<- prometheus.Metric, clusterHealthResponse clusterHealthResponse) { c.totalScrapes.Inc() defer func() { ch <- c.up @@ -252,17 +225,6 @@ func (c *ClusterHealth) Collect(ch chan<- prometheus.Metric) { ch <- c.jsonParseFailures }() - clusterHealthResponse, err := c.fetchAndDecodeClusterHealth() - if err != nil { - c.up.Set(0) - level.Warn(c.logger).Log( - "msg", "failed to fetch and decode cluster health", - "err", err, - ) - return - } - c.up.Set(1) - for _, metric := range c.metrics { ch <- prometheus.MustNewConstMetric( metric.Desc, diff --git a/collector/cluster_health_test.go b/collector/cluster_health_test.go deleted file mode 100644 index 03b153cf..00000000 --- a/collector/cluster_health_test.go +++ /dev/null @@ -1,55 +0,0 @@ -package collector - -import ( - "fmt" - "net/http" - "net/http/httptest" - "net/url" - "testing" - - "github.com/go-kit/kit/log" -) - -func TestClusterHealth(t *testing.T) { - // Testcases created using: - // docker run -d -p 9200:9200 elasticsearch:VERSION-alpine - // curl -XPUT http://localhost:9200/twitter - // curl http://localhost:9200/_cluster/health - tcs := map[string]string{ - "1.7.6": `{"cluster_name":"elasticsearch","status":"yellow","timed_out":false,"number_of_nodes":1,"number_of_data_nodes":1,"active_primary_shards":5,"active_shards":5,"relocating_shards":0,"initializing_shards":0,"unassigned_shards":5,"delayed_unassigned_shards":0,"number_of_pending_tasks":0,"number_of_in_flight_fetch":0}`, - "2.4.5": `{"cluster_name":"elasticsearch","status":"yellow","timed_out":false,"number_of_nodes":1,"number_of_data_nodes":1,"active_primary_shards":5,"active_shards":5,"relocating_shards":0,"initializing_shards":0,"unassigned_shards":5,"delayed_unassigned_shards":0,"number_of_pending_tasks":0,"number_of_in_flight_fetch":0,"task_max_waiting_in_queue_millis":0,"active_shards_percent_as_number":50.0}`, - "5.4.2": `{"cluster_name":"elasticsearch","status":"yellow","timed_out":false,"number_of_nodes":1,"number_of_data_nodes":1,"active_primary_shards":5,"active_shards":5,"relocating_shards":0,"initializing_shards":0,"unassigned_shards":5,"delayed_unassigned_shards":0,"number_of_pending_tasks":0,"number_of_in_flight_fetch":0,"task_max_waiting_in_queue_millis":0,"active_shards_percent_as_number":50.0}`, - } - for ver, out := range tcs { - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintln(w, out) - })) - defer ts.Close() - - u, err := url.Parse(ts.URL) - if err != nil { - t.Fatalf("Failed to parse URL: %s", err) - } - c := NewClusterHealth(log.NewNopLogger(), http.DefaultClient, u) - chr, err := c.fetchAndDecodeClusterHealth() - if err != nil { - t.Fatalf("Failed to fetch or decode cluster health: %s", err) - } - t.Logf("[%s] Cluster Health Response: %+v", ver, chr) - if chr.ClusterName != "elasticsearch" { - t.Errorf("Invalid cluster health response") - } - if chr.Status != "yellow" { - t.Errorf("Invalid cluster status") - } - if chr.TimedOut { - t.Errorf("Check didn't time out") - } - if chr.NumberOfNodes != 1 { - t.Errorf("Wrong number of nodes") - } - if chr.NumberOfDataNodes != 1 { - t.Errorf("Wrong number of data nodes") - } - } -} diff --git a/collector/indices.go b/collector/indices.go index 9ec034ca..1bb7575f 100644 --- a/collector/indices.go +++ b/collector/indices.go @@ -1,13 +1,10 @@ package collector import ( - "encoding/json" - "fmt" "net/http" "net/url" "github.com/go-kit/kit/log" - "github.com/go-kit/kit/log/level" "github.com/prometheus/client_golang/prometheus" ) @@ -567,30 +564,8 @@ func (i *Indices) Describe(ch chan<- *prometheus.Desc) { ch <- i.jsonParseFailures.Desc() } -func (c *Indices) fetchAndDecodeIndexStats() (indexStatsResponse, error) { - var isr indexStatsResponse - - u := *c.url - u.Path = "/_all/_stats" - - res, err := c.client.Get(u.String()) - if err != nil { - return isr, fmt.Errorf("failed to get index stats from %s://%s:%s/%s: %s", - u.Scheme, u.Hostname(), u.Port(), u.Path, err) - } - defer res.Body.Close() - if res.StatusCode != http.StatusOK { - return isr, fmt.Errorf("HTTP Request failed with code %d", res.StatusCode) - } - - if err := json.NewDecoder(res.Body).Decode(&isr); err != nil { - c.jsonParseFailures.Inc() - return isr, err - } - return isr, nil -} - -func (i *Indices) Collect(ch chan<- prometheus.Metric) { +func (i *Indices) Collect(ch chan<- prometheus.Metric, clusterHealthResponse clusterHealthResponse, + nodeStatsResponse nodeStatsResponse, indexStatsResponse indexStatsResponse) { i.totalScrapes.Inc() defer func() { ch <- i.up @@ -598,42 +573,6 @@ func (i *Indices) Collect(ch chan<- prometheus.Metric) { ch <- i.jsonParseFailures }() - // clusterHealth - clusterHealth := NewClusterHealth(i.logger, i.client, i.url) - clusterHealthResponse, err := clusterHealth.fetchAndDecodeClusterHealth() - if err != nil { - i.up.Set(0) - level.Warn(i.logger).Log( - "msg", "failed to fetch and decode cluster health", - "err", err, - ) - return - } - - // nodes - nodes := NewNodes(i.logger, i.client, i.url, i.all) - nodeStatsResponse, err := nodes.fetchAndDecodeNodeStats() - if err != nil { - i.up.Set(0) - level.Warn(i.logger).Log( - "msg", "failed to fetch and decode node stats", - "err", err, - ) - return - } - - // indices - indexStatsResponse, err := i.fetchAndDecodeIndexStats() - if err != nil { - i.up.Set(0) - level.Warn(i.logger).Log( - "msg", "failed to fetch and decode index stats", - "err", err, - ) - return - } - i.up.Set(1) - // Node stats for _, node := range nodeStatsResponse.Nodes { for _, metric := range i.nodeMetrics { diff --git a/collector/metrics_collector.go b/collector/metrics_collector.go new file mode 100644 index 00000000..a00b86f4 --- /dev/null +++ b/collector/metrics_collector.go @@ -0,0 +1,153 @@ +package collector + +import ( + "encoding/json" + "fmt" + "net/http" + "net/url" + + "github.com/go-kit/kit/log" + "github.com/go-kit/kit/log/level" + "github.com/prometheus/client_golang/prometheus" +) + +type MetricsCollector struct { + clusterHealth *ClusterHealth + nodes *Nodes + indices *Indices + + clusterHealthResponse *clusterHealthResponse + nodeStatsResponse *nodeStatsResponse + indexStatsResponse *indexStatsResponse +} + +func NewMetricsCollector(logger log.Logger, client *http.Client, url *url.URL, all bool) *MetricsCollector { + return &MetricsCollector{ + clusterHealth: NewClusterHealth(logger, client, url), + nodes: NewNodes(logger, client, url, all), + indices: NewIndices(logger, client, url, all), + } + + return nil +} + +func (c *MetricsCollector) Describe(ch chan<- *prometheus.Desc) { + c.clusterHealth.Describe(ch) + c.nodes.Describe(ch) + c.indices.Describe(ch) +} + +func (c *MetricsCollector) Collect(ch chan<- prometheus.Metric) { + // clusterHealth + clusterHealthResponse, err := c.clusterHealth.fetchAndDecodeClusterHealth() + if err != nil { + c.clusterHealth.up.Set(0) + level.Warn(c.clusterHealth.logger).Log( + "msg", "failed to fetch and decode cluster health", + "err", err, + ) + return + } + c.clusterHealth.up.Set(1) + + // nodes + nodeStatsResponse, err := c.nodes.fetchAndDecodeNodeStats() + if err != nil { + c.nodes.up.Set(0) + level.Warn(c.nodes.logger).Log( + "msg", "failed to fetch and decode node stats", + "err", err, + ) + return + } + c.nodes.up.Set(1) + + // indices + indexStatsResponse, err := c.indices.fetchAndDecodeIndexStats() + if err != nil { + c.indices.up.Set(0) + level.Warn(c.indices.logger).Log( + "msg", "failed to fetch and decode index stats", + "err", err, + ) + return + } + c.indices.up.Set(1) + + c.clusterHealth.Collect(ch, clusterHealthResponse) + c.nodes.Collect(ch, nodeStatsResponse) + c.indices.Collect(ch, clusterHealthResponse, nodeStatsResponse, indexStatsResponse) +} + +func (c *ClusterHealth) fetchAndDecodeClusterHealth() (clusterHealthResponse, error) { + var chr clusterHealthResponse + + u := *c.url + u.Path = "/_cluster/health" + res, err := c.client.Get(u.String()) + if err != nil { + return chr, fmt.Errorf("failed to get cluster health from %s://%s:%s/%s: %s", + u.Scheme, u.Hostname(), u.Port(), u.Path, err) + } + defer res.Body.Close() + + if res.StatusCode != http.StatusOK { + return chr, fmt.Errorf("HTTP Request failed with code %d", res.StatusCode) + } + + if err := json.NewDecoder(res.Body).Decode(&chr); err != nil { + c.jsonParseFailures.Inc() + return chr, err + } + + return chr, nil +} + +func (c *Nodes) fetchAndDecodeNodeStats() (nodeStatsResponse, error) { + var nsr nodeStatsResponse + + u := *c.url + u.Path = "/_nodes/_local/stats" + if c.all { + u.Path = "/_nodes/stats" + } + + res, err := c.client.Get(u.String()) + if err != nil { + return nsr, fmt.Errorf("failed to get cluster health from %s://%s:%s/%s: %s", + u.Scheme, u.Hostname(), u.Port(), u.Path, err) + } + defer res.Body.Close() + if res.StatusCode != http.StatusOK { + return nsr, fmt.Errorf("HTTP Request failed with code %d", res.StatusCode) + } + + if err := json.NewDecoder(res.Body).Decode(&nsr); err != nil { + c.jsonParseFailures.Inc() + return nsr, err + } + return nsr, nil +} + +func (c *Indices) fetchAndDecodeIndexStats() (indexStatsResponse, error) { + var isr indexStatsResponse + + u := *c.url + u.Path = "/_all/_stats" + + res, err := c.client.Get(u.String()) + if err != nil { + return isr, fmt.Errorf("failed to get index stats from %s://%s:%s/%s: %s", + u.Scheme, u.Hostname(), u.Port(), u.Path, err) + } + defer res.Body.Close() + if res.StatusCode != http.StatusOK { + return isr, fmt.Errorf("HTTP Request failed with code %d", res.StatusCode) + } + + if err := json.NewDecoder(res.Body).Decode(&isr); err != nil { + c.jsonParseFailures.Inc() + return isr, err + } + return isr, nil +} diff --git a/collector/indices_test.go b/collector/metrics_collector_test.go similarity index 65% rename from collector/indices_test.go rename to collector/metrics_collector_test.go index 6907f99e..c2ac2de4 100644 --- a/collector/indices_test.go +++ b/collector/metrics_collector_test.go @@ -1,15 +1,150 @@ package collector import ( + "encoding/base64" "fmt" "net/http" "net/http/httptest" "net/url" + "strings" "testing" "github.com/go-kit/kit/log" ) +type basicAuth struct { + User string + Pass string + Next http.Handler +} + +func (h *basicAuth) checkAuth(w http.ResponseWriter, r *http.Request) bool { + s := strings.SplitN(r.Header.Get("Authorization"), " ", 2) + if len(s) != 2 { + return false + } + + b, err := base64.StdEncoding.DecodeString(s[1]) + if err != nil { + return false + } + + pair := strings.SplitN(string(b), ":", 2) + if len(pair) != 2 { + return false + } + + if h.User == pair[0] && h.Pass == pair[1] { + return true + } + return false +} + +func (h *basicAuth) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if !h.checkAuth(w, r) { + w.Header().Set("WWW-Authenticate", "Basic realm=\"ES\"") + w.WriteHeader(401) + w.Write([]byte("401 Unauthorized\n")) + return + } + + h.Next.ServeHTTP(w, r) + return +} + +func TestClusterHealth(t *testing.T) { + // Testcases created using: + // docker run -d -p 9200:9200 elasticsearch:VERSION-alpine + // curl -XPUT http://localhost:9200/twitter + // curl http://localhost:9200/_cluster/health + tcs := map[string]string{ + "1.7.6": `{"cluster_name":"elasticsearch","status":"yellow","timed_out":false,"number_of_nodes":1,"number_of_data_nodes":1,"active_primary_shards":5,"active_shards":5,"relocating_shards":0,"initializing_shards":0,"unassigned_shards":5,"delayed_unassigned_shards":0,"number_of_pending_tasks":0,"number_of_in_flight_fetch":0}`, + "2.4.5": `{"cluster_name":"elasticsearch","status":"yellow","timed_out":false,"number_of_nodes":1,"number_of_data_nodes":1,"active_primary_shards":5,"active_shards":5,"relocating_shards":0,"initializing_shards":0,"unassigned_shards":5,"delayed_unassigned_shards":0,"number_of_pending_tasks":0,"number_of_in_flight_fetch":0,"task_max_waiting_in_queue_millis":0,"active_shards_percent_as_number":50.0}`, + "5.4.2": `{"cluster_name":"elasticsearch","status":"yellow","timed_out":false,"number_of_nodes":1,"number_of_data_nodes":1,"active_primary_shards":5,"active_shards":5,"relocating_shards":0,"initializing_shards":0,"unassigned_shards":5,"delayed_unassigned_shards":0,"number_of_pending_tasks":0,"number_of_in_flight_fetch":0,"task_max_waiting_in_queue_millis":0,"active_shards_percent_as_number":50.0}`, + } + for ver, out := range tcs { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, out) + })) + defer ts.Close() + + u, err := url.Parse(ts.URL) + if err != nil { + t.Fatalf("Failed to parse URL: %s", err) + } + c := NewClusterHealth(log.NewNopLogger(), http.DefaultClient, u) + chr, err := c.fetchAndDecodeClusterHealth() + if err != nil { + t.Fatalf("Failed to fetch or decode cluster health: %s", err) + } + t.Logf("[%s] Cluster Health Response: %+v", ver, chr) + if chr.ClusterName != "elasticsearch" { + t.Errorf("Invalid cluster health response") + } + if chr.Status != "yellow" { + t.Errorf("Invalid cluster status") + } + if chr.TimedOut { + t.Errorf("Check didn't time out") + } + if chr.NumberOfNodes != 1 { + t.Errorf("Wrong number of nodes") + } + if chr.NumberOfDataNodes != 1 { + t.Errorf("Wrong number of data nodes") + } + } +} + +func TestNodesStats(t *testing.T) { + // Testcases created using: + // docker run -d -p 9200:9200 elasticsearch:VERSION-alpine + // curl -XPUT http://localhost:9200/twitter + // curl http://localhost:9200/_nodes/stats + tcs := map[string]string{ + "1.7.6": `{"cluster_name":"elasticsearch","nodes":{"vKnRv2bPThCJIDTj92NvQg":{"timestamp":1498820641883,"name":"Overmind","transport_address":"inet[/172.17.0.4:9300]","host":"2603876b583f","ip":["inet[/172.17.0.4:9300]","NONE"],"indices":{"docs":{"count":0,"deleted":0},"store":{"size_in_bytes":575,"throttle_time_in_millis":0},"indexing":{"index_total":0,"index_time_in_millis":0,"index_current":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0},"refresh":{"total":0,"total_time_in_millis":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":10,"total_time_in_millis":52},"filter_cache":{"memory_size_in_bytes":0,"evictions":0},"id_cache":{"memory_size_in_bytes":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"percolate":{"total":0,"time_in_millis":0,"current":0,"memory_size_in_bytes":-1,"memory_size":"-1b","queries":0},"completion":{"size_in_bytes":0},"segments":{"count":0,"memory_in_bytes":0,"index_writer_memory_in_bytes":0,"index_writer_max_memory_in_bytes":335544320,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0},"translog":{"operations":0,"size_in_bytes":17},"suggest":{"total":0,"time_in_millis":0,"current":0},"query_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"os":{"timestamp":1498820641896},"process":{"timestamp":1498820641896,"open_file_descriptors":123},"jvm":{"timestamp":1498820641896,"uptime_in_millis":17871,"mem":{"heap_used_in_bytes":82958528,"heap_used_percent":7,"heap_committed_in_bytes":251002880,"heap_max_in_bytes":1056309248,"non_heap_used_in_bytes":44442536,"non_heap_committed_in_bytes":45400064,"pools":{"young":{"used_in_bytes":62150064,"max_in_bytes":139591680,"peak_used_in_bytes":139591680,"peak_max_in_bytes":139591680},"survivor":{"used_in_bytes":17432576,"max_in_bytes":17432576,"peak_used_in_bytes":17432576,"peak_max_in_bytes":17432576},"old":{"used_in_bytes":3375888,"max_in_bytes":899284992,"peak_used_in_bytes":10899920,"peak_max_in_bytes":899284992}}},"threads":{"count":36,"peak_count":36},"gc":{"collectors":{"young":{"collection_count":1,"collection_time_in_millis":40},"old":{"collection_count":1,"collection_time_in_millis":42}}},"buffer_pools":{"direct":{"count":27,"used_in_bytes":4981411,"total_capacity_in_bytes":4981411},"mapped":{"count":0,"used_in_bytes":0,"total_capacity_in_bytes":0}}},"thread_pool":{"percolate":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"fetch_shard_started":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"listener":{"threads":1,"queue":0,"active":0,"rejected":0,"largest":1,"completed":1},"index":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"refresh":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"suggest":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"generic":{"threads":4,"queue":0,"active":0,"rejected":0,"largest":4,"completed":14},"warmer":{"threads":1,"queue":0,"active":0,"rejected":0,"largest":1,"completed":5},"search":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"flush":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"optimize":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"fetch_shard_store":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"management":{"threads":1,"queue":0,"active":1,"rejected":0,"largest":1,"completed":1},"get":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"merge":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"bulk":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"snapshot":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0}},"network":{},"fs":{"timestamp":1498820641898,"total":{"total_in_bytes":63375708160,"free_in_bytes":59667320832,"available_in_bytes":56424390656},"data":[{"path":"/usr/share/elasticsearch/data/elasticsearch/nodes/0","mount":"/usr/share/elasticsearch/data (/dev/sda2)","type":"ext4","total_in_bytes":63375708160,"free_in_bytes":59667320832,"available_in_bytes":56424390656}]},"transport":{"server_open":13,"rx_count":6,"rx_size_in_bytes":1428,"tx_count":6,"tx_size_in_bytes":1428},"http":{"current_open":1,"total_opened":2},"breakers":{"request":{"limit_size_in_bytes":422523699,"limit_size":"402.9mb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.0,"tripped":0},"fielddata":{"limit_size_in_bytes":633785548,"limit_size":"604.4mb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.03,"tripped":0},"parent":{"limit_size_in_bytes":739416473,"limit_size":"705.1mb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.0,"tripped":0}}}}}`, + "2.4.5": `{"cluster_name":"elasticsearch","nodes":{"VnnrklnAQ7KRXhW2WkgPxA":{"timestamp":1498820602662,"name":"Battering Ram","transport_address":"172.17.0.4:9300","host":"172.17.0.4","ip":["172.17.0.4:9300","NONE"],"indices":{"docs":{"count":0,"deleted":0},"store":{"size_in_bytes":650,"throttle_time_in_millis":0},"indexing":{"index_total":0,"index_time_in_millis":0,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":104857600},"refresh":{"total":0,"total_time_in_millis":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":10,"total_time_in_millis":60},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"percolate":{"total":0,"time_in_millis":0,"current":0,"memory_size_in_bytes":-1,"memory_size":"-1b","queries":0},"completion":{"size_in_bytes":0},"segments":{"count":0,"memory_in_bytes":0,"terms_memory_in_bytes":0,"stored_fields_memory_in_bytes":0,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":0,"doc_values_memory_in_bytes":0,"index_writer_memory_in_bytes":0,"index_writer_max_memory_in_bytes":105630920,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0},"translog":{"operations":0,"size_in_bytes":215},"suggest":{"total":0,"time_in_millis":0,"current":0},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"os":{"timestamp":1498820602670,"cpu_percent":7,"load_average":0.32861328125,"mem":{"total_in_bytes":2096177152,"free_in_bytes":1624637440,"used_in_bytes":471539712,"free_percent":78,"used_percent":22},"swap":{"total_in_bytes":4195348480,"free_in_bytes":4005060608,"used_in_bytes":190287872}},"process":{"timestamp":1498820602670,"open_file_descriptors":116,"max_file_descriptors":1048576,"cpu":{"percent":7,"total_in_millis":10270},"mem":{"total_virtual_in_bytes":2681352192}},"jvm":{"timestamp":1498820602671,"uptime_in_millis":38833,"mem":{"heap_used_in_bytes":88986784,"heap_used_percent":8,"heap_committed_in_bytes":251002880,"heap_max_in_bytes":1056309248,"non_heap_used_in_bytes":49185912,"non_heap_committed_in_bytes":49987584,"pools":{"young":{"used_in_bytes":58281984,"max_in_bytes":139591680,"peak_used_in_bytes":139591680,"peak_max_in_bytes":139591680},"survivor":{"used_in_bytes":17432576,"max_in_bytes":17432576,"peak_used_in_bytes":17432576,"peak_max_in_bytes":17432576},"old":{"used_in_bytes":13272224,"max_in_bytes":899284992,"peak_used_in_bytes":13272224,"peak_max_in_bytes":899284992}}},"threads":{"count":35,"peak_count":38},"gc":{"collectors":{"young":{"collection_count":2,"collection_time_in_millis":87},"old":{"collection_count":1,"collection_time_in_millis":74}}},"buffer_pools":{"direct":{"count":23,"used_in_bytes":3416836,"total_capacity_in_bytes":3416836},"mapped":{"count":0,"used_in_bytes":0,"total_capacity_in_bytes":0}},"classes":{"current_loaded_count":7221,"total_loaded_count":7221,"total_unloaded_count":0}},"thread_pool":{"bulk":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"fetch_shard_started":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"fetch_shard_store":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"flush":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"force_merge":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"generic":{"threads":1,"queue":0,"active":0,"rejected":0,"largest":5,"completed":28},"get":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"index":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"listener":{"threads":1,"queue":0,"active":0,"rejected":0,"largest":1,"completed":2},"management":{"threads":2,"queue":0,"active":1,"rejected":0,"largest":2,"completed":5},"percolate":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"refresh":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"search":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"snapshot":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"suggest":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"warmer":{"threads":1,"queue":0,"active":0,"rejected":0,"largest":1,"completed":5}},"fs":{"timestamp":1498820602672,"total":{"total_in_bytes":63375708160,"free_in_bytes":59667599360,"available_in_bytes":56424669184,"spins":"true"},"data":[{"path":"/usr/share/elasticsearch/data/elasticsearch/nodes/0","mount":"/usr/share/elasticsearch/data (/dev/sda2)","type":"ext4","total_in_bytes":63375708160,"free_in_bytes":59667599360,"available_in_bytes":56424669184,"spins":"true"}]},"transport":{"server_open":0,"rx_count":6,"rx_size_in_bytes":2028,"tx_count":6,"tx_size_in_bytes":2028},"http":{"current_open":1,"total_opened":3},"breakers":{"request":{"limit_size_in_bytes":422523699,"limit_size":"402.9mb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.0,"tripped":0},"fielddata":{"limit_size_in_bytes":633785548,"limit_size":"604.4mb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.03,"tripped":0},"in_flight_requests":{"limit_size_in_bytes":1056309248,"limit_size":"1007.3mb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.0,"tripped":0},"parent":{"limit_size_in_bytes":739416473,"limit_size":"705.1mb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.0,"tripped":0}},"script":{"compilations":0,"cache_evictions":0}}}}`, + "5.4.2": `{"_nodes":{"total":1,"successful":1,"failed":0},"cluster_name":"elasticsearch","nodes":{"0hHcEFK1S7qMlk8hQCm7wQ":{"timestamp":1498820489394,"name":"0hHcEFK","transport_address":"127.0.0.1:9300","host":"127.0.0.1","ip":"127.0.0.1:9300","roles":["master","data","ingest"],"indices":{"docs":{"count":0,"deleted":0},"store":{"size_in_bytes":650,"throttle_time_in_millis":0},"indexing":{"index_total":0,"index_time_in_millis":0,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":104857600},"refresh":{"total":0,"total_time_in_millis":0,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":5,"total_time_in_millis":27},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":0,"memory_in_bytes":0,"terms_memory_in_bytes":0,"stored_fields_memory_in_bytes":0,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":0,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":0,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":0,"size_in_bytes":215},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"os":{"timestamp":1498820489400,"cpu":{"percent":0,"load_average":{"1m":0.35,"5m":0.28,"15m":0.12}},"mem":{"total_in_bytes":2096177152,"free_in_bytes":83501056,"used_in_bytes":2012676096,"free_percent":4,"used_percent":96},"swap":{"total_in_bytes":4195348480,"free_in_bytes":3487707136,"used_in_bytes":707641344}},"process":{"timestamp":1498820489400,"open_file_descriptors":139,"max_file_descriptors":1048576,"cpu":{"percent":0,"total_in_millis":26600},"mem":{"total_virtual_in_bytes":3823181824}},"jvm":{"timestamp":1498820489400,"uptime_in_millis":185693,"mem":{"heap_used_in_bytes":114959064,"heap_used_percent":5,"heap_committed_in_bytes":2130051072,"heap_max_in_bytes":2130051072,"non_heap_used_in_bytes":65471864,"non_heap_committed_in_bytes":69906432,"pools":{"young":{"used_in_bytes":82057312,"max_in_bytes":139591680,"peak_used_in_bytes":139591680,"peak_max_in_bytes":139591680},"survivor":{"used_in_bytes":17432576,"max_in_bytes":17432576,"peak_used_in_bytes":17432576,"peak_max_in_bytes":17432576},"old":{"used_in_bytes":15469176,"max_in_bytes":1973026816,"peak_used_in_bytes":15469176,"peak_max_in_bytes":1973026816}}},"threads":{"count":26,"peak_count":29},"gc":{"collectors":{"young":{"collection_count":4,"collection_time_in_millis":1618},"old":{"collection_count":1,"collection_time_in_millis":76}}},"buffer_pools":{"direct":{"count":16,"used_in_bytes":33776600,"total_capacity_in_bytes":33776599},"mapped":{"count":0,"used_in_bytes":0,"total_capacity_in_bytes":0}},"classes":{"current_loaded_count":9995,"total_loaded_count":9995,"total_unloaded_count":0}},"thread_pool":{"bulk":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"fetch_shard_started":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"fetch_shard_store":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"flush":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"force_merge":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"generic":{"threads":4,"queue":0,"active":0,"rejected":0,"largest":4,"completed":28},"get":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"index":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"listener":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"management":{"threads":2,"queue":0,"active":1,"rejected":0,"largest":2,"completed":20},"refresh":{"threads":1,"queue":0,"active":0,"rejected":0,"largest":1,"completed":31},"search":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"snapshot":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"warmer":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0}},"fs":{"timestamp":1498820489401,"total":{"total_in_bytes":63375708160,"free_in_bytes":59668226048,"available_in_bytes":56425295872,"spins":"true"},"data":[{"path":"/usr/share/elasticsearch/data/nodes/0","mount":"/usr/share/elasticsearch/data (/dev/sda2)","type":"ext4","total_in_bytes":63375708160,"free_in_bytes":59668226048,"available_in_bytes":56425295872,"spins":"true"}],"io_stats":{"devices":[{"device_name":"sda2","operations":3017,"read_operations":1800,"write_operations":1217,"read_kilobytes":24816,"write_kilobytes":6580}],"total":{"operations":3017,"read_operations":1800,"write_operations":1217,"read_kilobytes":24816,"write_kilobytes":6580}}},"transport":{"server_open":0,"rx_count":0,"rx_size_in_bytes":0,"tx_count":0,"tx_size_in_bytes":0},"http":{"current_open":2,"total_opened":3},"breakers":{"request":{"limit_size_in_bytes":1278030643,"limit_size":"1.1gb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.0,"tripped":0},"fielddata":{"limit_size_in_bytes":1278030643,"limit_size":"1.1gb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.03,"tripped":0},"in_flight_requests":{"limit_size_in_bytes":2130051072,"limit_size":"1.9gb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.0,"tripped":0},"parent":{"limit_size_in_bytes":1491035750,"limit_size":"1.3gb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.0,"tripped":0}},"script":{"compilations":0,"cache_evictions":0},"discovery":{"cluster_state_queue":{"total":0,"pending":0,"committed":0}},"ingest":{"total":{"count":0,"time_in_millis":0,"current":0,"failed":0},"pipelines":{}}}}}`, + } + for ver, out := range tcs { + for hn, handler := range map[string]http.Handler{ + "plain": http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, out) + }), + "basicauth": &basicAuth{ + User: "elastic", + Pass: "changeme", + Next: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, out) + }), + }, + } { + ts := httptest.NewServer(handler) + defer ts.Close() + + u, err := url.Parse(ts.URL) + if err != nil { + t.Fatalf("Failed to parse URL: %s", err) + } + u.User = url.UserPassword("elastic", "changeme") + c := NewNodes(log.NewNopLogger(), http.DefaultClient, u, true) + nsr, err := c.fetchAndDecodeNodeStats() + if err != nil { + t.Fatalf("Failed to fetch or decode node stats: %s", err) + } + t.Logf("[%s/%s] Node Stats Response: %+v", hn, ver, nsr) + if nsr.ClusterName != "elasticsearch" { + t.Errorf("Wrong cluster name") + } + for _, nsnr := range nsr.Nodes { + if nsnr.Indices.Docs.Count > 0 { + t.Errorf("Wrong doc count") + } + } + } + } +} + func TestIndices(t *testing.T) { // Testcases created using: // docker run -d -p 9200:9200 elasticsearch:VERSION-alpine diff --git a/collector/nodes.go b/collector/nodes.go index 2bdd8372..4b623f99 100644 --- a/collector/nodes.go +++ b/collector/nodes.go @@ -1,13 +1,10 @@ package collector import ( - "encoding/json" - "fmt" "net/http" "net/url" "github.com/go-kit/kit/log" - "github.com/go-kit/kit/log/level" "github.com/prometheus/client_golang/prometheus" ) @@ -528,33 +525,7 @@ func (c *Nodes) Describe(ch chan<- *prometheus.Desc) { ch <- c.jsonParseFailures.Desc() } -func (c *Nodes) fetchAndDecodeNodeStats() (nodeStatsResponse, error) { - var nsr nodeStatsResponse - - u := *c.url - u.Path = "/_nodes/_local/stats" - if c.all { - u.Path = "/_nodes/stats" - } - - res, err := c.client.Get(u.String()) - if err != nil { - return nsr, fmt.Errorf("failed to get cluster health from %s://%s:%s/%s: %s", - u.Scheme, u.Hostname(), u.Port(), u.Path, err) - } - defer res.Body.Close() - if res.StatusCode != http.StatusOK { - return nsr, fmt.Errorf("HTTP Request failed with code %d", res.StatusCode) - } - - if err := json.NewDecoder(res.Body).Decode(&nsr); err != nil { - c.jsonParseFailures.Inc() - return nsr, err - } - return nsr, nil -} - -func (c *Nodes) Collect(ch chan<- prometheus.Metric) { +func (c *Nodes) Collect(ch chan<- prometheus.Metric, nodeStatsResponse nodeStatsResponse) { c.totalScrapes.Inc() defer func() { ch <- c.up @@ -562,17 +533,6 @@ func (c *Nodes) Collect(ch chan<- prometheus.Metric) { ch <- c.jsonParseFailures }() - nodeStatsResponse, err := c.fetchAndDecodeNodeStats() - if err != nil { - c.up.Set(0) - level.Warn(c.logger).Log( - "msg", "failed to fetch and decode node stats", - "err", err, - ) - return - } - c.up.Set(1) - for _, node := range nodeStatsResponse.Nodes { for _, metric := range c.nodeMetrics { ch <- prometheus.MustNewConstMetric( diff --git a/collector/nodes_test.go b/collector/nodes_test.go deleted file mode 100644 index 46081272..00000000 --- a/collector/nodes_test.go +++ /dev/null @@ -1,102 +0,0 @@ -package collector - -import ( - "encoding/base64" - "fmt" - "net/http" - "net/http/httptest" - "net/url" - "strings" - "testing" - - "github.com/go-kit/kit/log" -) - -func TestNodesStats(t *testing.T) { - // Testcases created using: - // docker run -d -p 9200:9200 elasticsearch:VERSION-alpine - // curl -XPUT http://localhost:9200/twitter - // curl http://localhost:9200/_nodes/stats - tcs := map[string]string{ - "1.7.6": `{"cluster_name":"elasticsearch","nodes":{"vKnRv2bPThCJIDTj92NvQg":{"timestamp":1498820641883,"name":"Overmind","transport_address":"inet[/172.17.0.4:9300]","host":"2603876b583f","ip":["inet[/172.17.0.4:9300]","NONE"],"indices":{"docs":{"count":0,"deleted":0},"store":{"size_in_bytes":575,"throttle_time_in_millis":0},"indexing":{"index_total":0,"index_time_in_millis":0,"index_current":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0},"refresh":{"total":0,"total_time_in_millis":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":10,"total_time_in_millis":52},"filter_cache":{"memory_size_in_bytes":0,"evictions":0},"id_cache":{"memory_size_in_bytes":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"percolate":{"total":0,"time_in_millis":0,"current":0,"memory_size_in_bytes":-1,"memory_size":"-1b","queries":0},"completion":{"size_in_bytes":0},"segments":{"count":0,"memory_in_bytes":0,"index_writer_memory_in_bytes":0,"index_writer_max_memory_in_bytes":335544320,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0},"translog":{"operations":0,"size_in_bytes":17},"suggest":{"total":0,"time_in_millis":0,"current":0},"query_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"os":{"timestamp":1498820641896},"process":{"timestamp":1498820641896,"open_file_descriptors":123},"jvm":{"timestamp":1498820641896,"uptime_in_millis":17871,"mem":{"heap_used_in_bytes":82958528,"heap_used_percent":7,"heap_committed_in_bytes":251002880,"heap_max_in_bytes":1056309248,"non_heap_used_in_bytes":44442536,"non_heap_committed_in_bytes":45400064,"pools":{"young":{"used_in_bytes":62150064,"max_in_bytes":139591680,"peak_used_in_bytes":139591680,"peak_max_in_bytes":139591680},"survivor":{"used_in_bytes":17432576,"max_in_bytes":17432576,"peak_used_in_bytes":17432576,"peak_max_in_bytes":17432576},"old":{"used_in_bytes":3375888,"max_in_bytes":899284992,"peak_used_in_bytes":10899920,"peak_max_in_bytes":899284992}}},"threads":{"count":36,"peak_count":36},"gc":{"collectors":{"young":{"collection_count":1,"collection_time_in_millis":40},"old":{"collection_count":1,"collection_time_in_millis":42}}},"buffer_pools":{"direct":{"count":27,"used_in_bytes":4981411,"total_capacity_in_bytes":4981411},"mapped":{"count":0,"used_in_bytes":0,"total_capacity_in_bytes":0}}},"thread_pool":{"percolate":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"fetch_shard_started":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"listener":{"threads":1,"queue":0,"active":0,"rejected":0,"largest":1,"completed":1},"index":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"refresh":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"suggest":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"generic":{"threads":4,"queue":0,"active":0,"rejected":0,"largest":4,"completed":14},"warmer":{"threads":1,"queue":0,"active":0,"rejected":0,"largest":1,"completed":5},"search":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"flush":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"optimize":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"fetch_shard_store":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"management":{"threads":1,"queue":0,"active":1,"rejected":0,"largest":1,"completed":1},"get":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"merge":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"bulk":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"snapshot":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0}},"network":{},"fs":{"timestamp":1498820641898,"total":{"total_in_bytes":63375708160,"free_in_bytes":59667320832,"available_in_bytes":56424390656},"data":[{"path":"/usr/share/elasticsearch/data/elasticsearch/nodes/0","mount":"/usr/share/elasticsearch/data (/dev/sda2)","type":"ext4","total_in_bytes":63375708160,"free_in_bytes":59667320832,"available_in_bytes":56424390656}]},"transport":{"server_open":13,"rx_count":6,"rx_size_in_bytes":1428,"tx_count":6,"tx_size_in_bytes":1428},"http":{"current_open":1,"total_opened":2},"breakers":{"request":{"limit_size_in_bytes":422523699,"limit_size":"402.9mb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.0,"tripped":0},"fielddata":{"limit_size_in_bytes":633785548,"limit_size":"604.4mb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.03,"tripped":0},"parent":{"limit_size_in_bytes":739416473,"limit_size":"705.1mb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.0,"tripped":0}}}}}`, - "2.4.5": `{"cluster_name":"elasticsearch","nodes":{"VnnrklnAQ7KRXhW2WkgPxA":{"timestamp":1498820602662,"name":"Battering Ram","transport_address":"172.17.0.4:9300","host":"172.17.0.4","ip":["172.17.0.4:9300","NONE"],"indices":{"docs":{"count":0,"deleted":0},"store":{"size_in_bytes":650,"throttle_time_in_millis":0},"indexing":{"index_total":0,"index_time_in_millis":0,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":104857600},"refresh":{"total":0,"total_time_in_millis":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":10,"total_time_in_millis":60},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"percolate":{"total":0,"time_in_millis":0,"current":0,"memory_size_in_bytes":-1,"memory_size":"-1b","queries":0},"completion":{"size_in_bytes":0},"segments":{"count":0,"memory_in_bytes":0,"terms_memory_in_bytes":0,"stored_fields_memory_in_bytes":0,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":0,"doc_values_memory_in_bytes":0,"index_writer_memory_in_bytes":0,"index_writer_max_memory_in_bytes":105630920,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0},"translog":{"operations":0,"size_in_bytes":215},"suggest":{"total":0,"time_in_millis":0,"current":0},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"os":{"timestamp":1498820602670,"cpu_percent":7,"load_average":0.32861328125,"mem":{"total_in_bytes":2096177152,"free_in_bytes":1624637440,"used_in_bytes":471539712,"free_percent":78,"used_percent":22},"swap":{"total_in_bytes":4195348480,"free_in_bytes":4005060608,"used_in_bytes":190287872}},"process":{"timestamp":1498820602670,"open_file_descriptors":116,"max_file_descriptors":1048576,"cpu":{"percent":7,"total_in_millis":10270},"mem":{"total_virtual_in_bytes":2681352192}},"jvm":{"timestamp":1498820602671,"uptime_in_millis":38833,"mem":{"heap_used_in_bytes":88986784,"heap_used_percent":8,"heap_committed_in_bytes":251002880,"heap_max_in_bytes":1056309248,"non_heap_used_in_bytes":49185912,"non_heap_committed_in_bytes":49987584,"pools":{"young":{"used_in_bytes":58281984,"max_in_bytes":139591680,"peak_used_in_bytes":139591680,"peak_max_in_bytes":139591680},"survivor":{"used_in_bytes":17432576,"max_in_bytes":17432576,"peak_used_in_bytes":17432576,"peak_max_in_bytes":17432576},"old":{"used_in_bytes":13272224,"max_in_bytes":899284992,"peak_used_in_bytes":13272224,"peak_max_in_bytes":899284992}}},"threads":{"count":35,"peak_count":38},"gc":{"collectors":{"young":{"collection_count":2,"collection_time_in_millis":87},"old":{"collection_count":1,"collection_time_in_millis":74}}},"buffer_pools":{"direct":{"count":23,"used_in_bytes":3416836,"total_capacity_in_bytes":3416836},"mapped":{"count":0,"used_in_bytes":0,"total_capacity_in_bytes":0}},"classes":{"current_loaded_count":7221,"total_loaded_count":7221,"total_unloaded_count":0}},"thread_pool":{"bulk":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"fetch_shard_started":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"fetch_shard_store":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"flush":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"force_merge":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"generic":{"threads":1,"queue":0,"active":0,"rejected":0,"largest":5,"completed":28},"get":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"index":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"listener":{"threads":1,"queue":0,"active":0,"rejected":0,"largest":1,"completed":2},"management":{"threads":2,"queue":0,"active":1,"rejected":0,"largest":2,"completed":5},"percolate":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"refresh":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"search":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"snapshot":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"suggest":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"warmer":{"threads":1,"queue":0,"active":0,"rejected":0,"largest":1,"completed":5}},"fs":{"timestamp":1498820602672,"total":{"total_in_bytes":63375708160,"free_in_bytes":59667599360,"available_in_bytes":56424669184,"spins":"true"},"data":[{"path":"/usr/share/elasticsearch/data/elasticsearch/nodes/0","mount":"/usr/share/elasticsearch/data (/dev/sda2)","type":"ext4","total_in_bytes":63375708160,"free_in_bytes":59667599360,"available_in_bytes":56424669184,"spins":"true"}]},"transport":{"server_open":0,"rx_count":6,"rx_size_in_bytes":2028,"tx_count":6,"tx_size_in_bytes":2028},"http":{"current_open":1,"total_opened":3},"breakers":{"request":{"limit_size_in_bytes":422523699,"limit_size":"402.9mb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.0,"tripped":0},"fielddata":{"limit_size_in_bytes":633785548,"limit_size":"604.4mb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.03,"tripped":0},"in_flight_requests":{"limit_size_in_bytes":1056309248,"limit_size":"1007.3mb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.0,"tripped":0},"parent":{"limit_size_in_bytes":739416473,"limit_size":"705.1mb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.0,"tripped":0}},"script":{"compilations":0,"cache_evictions":0}}}}`, - "5.4.2": `{"_nodes":{"total":1,"successful":1,"failed":0},"cluster_name":"elasticsearch","nodes":{"0hHcEFK1S7qMlk8hQCm7wQ":{"timestamp":1498820489394,"name":"0hHcEFK","transport_address":"127.0.0.1:9300","host":"127.0.0.1","ip":"127.0.0.1:9300","roles":["master","data","ingest"],"indices":{"docs":{"count":0,"deleted":0},"store":{"size_in_bytes":650,"throttle_time_in_millis":0},"indexing":{"index_total":0,"index_time_in_millis":0,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":104857600},"refresh":{"total":0,"total_time_in_millis":0,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":5,"total_time_in_millis":27},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":0,"memory_in_bytes":0,"terms_memory_in_bytes":0,"stored_fields_memory_in_bytes":0,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":0,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":0,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":0,"size_in_bytes":215},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"os":{"timestamp":1498820489400,"cpu":{"percent":0,"load_average":{"1m":0.35,"5m":0.28,"15m":0.12}},"mem":{"total_in_bytes":2096177152,"free_in_bytes":83501056,"used_in_bytes":2012676096,"free_percent":4,"used_percent":96},"swap":{"total_in_bytes":4195348480,"free_in_bytes":3487707136,"used_in_bytes":707641344}},"process":{"timestamp":1498820489400,"open_file_descriptors":139,"max_file_descriptors":1048576,"cpu":{"percent":0,"total_in_millis":26600},"mem":{"total_virtual_in_bytes":3823181824}},"jvm":{"timestamp":1498820489400,"uptime_in_millis":185693,"mem":{"heap_used_in_bytes":114959064,"heap_used_percent":5,"heap_committed_in_bytes":2130051072,"heap_max_in_bytes":2130051072,"non_heap_used_in_bytes":65471864,"non_heap_committed_in_bytes":69906432,"pools":{"young":{"used_in_bytes":82057312,"max_in_bytes":139591680,"peak_used_in_bytes":139591680,"peak_max_in_bytes":139591680},"survivor":{"used_in_bytes":17432576,"max_in_bytes":17432576,"peak_used_in_bytes":17432576,"peak_max_in_bytes":17432576},"old":{"used_in_bytes":15469176,"max_in_bytes":1973026816,"peak_used_in_bytes":15469176,"peak_max_in_bytes":1973026816}}},"threads":{"count":26,"peak_count":29},"gc":{"collectors":{"young":{"collection_count":4,"collection_time_in_millis":1618},"old":{"collection_count":1,"collection_time_in_millis":76}}},"buffer_pools":{"direct":{"count":16,"used_in_bytes":33776600,"total_capacity_in_bytes":33776599},"mapped":{"count":0,"used_in_bytes":0,"total_capacity_in_bytes":0}},"classes":{"current_loaded_count":9995,"total_loaded_count":9995,"total_unloaded_count":0}},"thread_pool":{"bulk":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"fetch_shard_started":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"fetch_shard_store":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"flush":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"force_merge":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"generic":{"threads":4,"queue":0,"active":0,"rejected":0,"largest":4,"completed":28},"get":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"index":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"listener":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"management":{"threads":2,"queue":0,"active":1,"rejected":0,"largest":2,"completed":20},"refresh":{"threads":1,"queue":0,"active":0,"rejected":0,"largest":1,"completed":31},"search":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"snapshot":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"warmer":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0}},"fs":{"timestamp":1498820489401,"total":{"total_in_bytes":63375708160,"free_in_bytes":59668226048,"available_in_bytes":56425295872,"spins":"true"},"data":[{"path":"/usr/share/elasticsearch/data/nodes/0","mount":"/usr/share/elasticsearch/data (/dev/sda2)","type":"ext4","total_in_bytes":63375708160,"free_in_bytes":59668226048,"available_in_bytes":56425295872,"spins":"true"}],"io_stats":{"devices":[{"device_name":"sda2","operations":3017,"read_operations":1800,"write_operations":1217,"read_kilobytes":24816,"write_kilobytes":6580}],"total":{"operations":3017,"read_operations":1800,"write_operations":1217,"read_kilobytes":24816,"write_kilobytes":6580}}},"transport":{"server_open":0,"rx_count":0,"rx_size_in_bytes":0,"tx_count":0,"tx_size_in_bytes":0},"http":{"current_open":2,"total_opened":3},"breakers":{"request":{"limit_size_in_bytes":1278030643,"limit_size":"1.1gb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.0,"tripped":0},"fielddata":{"limit_size_in_bytes":1278030643,"limit_size":"1.1gb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.03,"tripped":0},"in_flight_requests":{"limit_size_in_bytes":2130051072,"limit_size":"1.9gb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.0,"tripped":0},"parent":{"limit_size_in_bytes":1491035750,"limit_size":"1.3gb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.0,"tripped":0}},"script":{"compilations":0,"cache_evictions":0},"discovery":{"cluster_state_queue":{"total":0,"pending":0,"committed":0}},"ingest":{"total":{"count":0,"time_in_millis":0,"current":0,"failed":0},"pipelines":{}}}}}`, - } - for ver, out := range tcs { - for hn, handler := range map[string]http.Handler{ - "plain": http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintln(w, out) - }), - "basicauth": &basicAuth{ - User: "elastic", - Pass: "changeme", - Next: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintln(w, out) - }), - }, - } { - ts := httptest.NewServer(handler) - defer ts.Close() - - u, err := url.Parse(ts.URL) - if err != nil { - t.Fatalf("Failed to parse URL: %s", err) - } - u.User = url.UserPassword("elastic", "changeme") - c := NewNodes(log.NewNopLogger(), http.DefaultClient, u, true) - nsr, err := c.fetchAndDecodeNodeStats() - if err != nil { - t.Fatalf("Failed to fetch or decode node stats: %s", err) - } - t.Logf("[%s/%s] Node Stats Response: %+v", hn, ver, nsr) - if nsr.ClusterName != "elasticsearch" { - t.Errorf("Wrong cluster name") - } - for _, nsnr := range nsr.Nodes { - if nsnr.Indices.Docs.Count > 0 { - t.Errorf("Wrong doc count") - } - } - } - } -} - -type basicAuth struct { - User string - Pass string - Next http.Handler -} - -func (h *basicAuth) checkAuth(w http.ResponseWriter, r *http.Request) bool { - s := strings.SplitN(r.Header.Get("Authorization"), " ", 2) - if len(s) != 2 { - return false - } - - b, err := base64.StdEncoding.DecodeString(s[1]) - if err != nil { - return false - } - - pair := strings.SplitN(string(b), ":", 2) - if len(pair) != 2 { - return false - } - - if h.User == pair[0] && h.Pass == pair[1] { - return true - } - return false -} - -func (h *basicAuth) ServeHTTP(w http.ResponseWriter, r *http.Request) { - if !h.checkAuth(w, r) { - w.Header().Set("WWW-Authenticate", "Basic realm=\"ES\"") - w.WriteHeader(401) - w.Write([]byte("401 Unauthorized\n")) - return - } - - h.Next.ServeHTTP(w, r) - return -} diff --git a/main.go b/main.go index dd5cea5d..ec93f15b 100644 --- a/main.go +++ b/main.go @@ -53,9 +53,7 @@ func main() { }, } - prometheus.MustRegister(collector.NewClusterHealth(logger, httpClient, esURL)) - prometheus.MustRegister(collector.NewNodes(logger, httpClient, esURL, *esAllNodes)) - prometheus.MustRegister(collector.NewIndices(logger, httpClient, esURL, *esAllNodes)) + prometheus.MustRegister(collector.NewMetricsCollector(logger, httpClient, esURL, *esAllNodes)) http.Handle(*metricsPath, prometheus.Handler()) http.HandleFunc("/", IndexHandler(*metricsPath)) From 5f4f6565fbd9d1245a808013cc587c9abb425c22 Mon Sep 17 00:00:00 2001 From: Manabu Matsuzaki Date: Wed, 23 Aug 2017 22:31:10 +0900 Subject: [PATCH 05/12] remove unreachable code --- collector/metrics_collector.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/collector/metrics_collector.go b/collector/metrics_collector.go index a00b86f4..a49823c0 100644 --- a/collector/metrics_collector.go +++ b/collector/metrics_collector.go @@ -27,8 +27,6 @@ func NewMetricsCollector(logger log.Logger, client *http.Client, url *url.URL, a nodes: NewNodes(logger, client, url, all), indices: NewIndices(logger, client, url, all), } - - return nil } func (c *MetricsCollector) Describe(ch chan<- *prometheus.Desc) { From 9bd010fb7fc2882fb5dee92e5f2d0a98cab81bd5 Mon Sep 17 00:00:00 2001 From: Manabu Matsuzaki Date: Wed, 23 Aug 2017 23:25:18 +0900 Subject: [PATCH 06/12] add test pattern --- collector/metrics_collector_test.go | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/collector/metrics_collector_test.go b/collector/metrics_collector_test.go index c2ac2de4..f338246f 100644 --- a/collector/metrics_collector_test.go +++ b/collector/metrics_collector_test.go @@ -61,6 +61,7 @@ func TestClusterHealth(t *testing.T) { "1.7.6": `{"cluster_name":"elasticsearch","status":"yellow","timed_out":false,"number_of_nodes":1,"number_of_data_nodes":1,"active_primary_shards":5,"active_shards":5,"relocating_shards":0,"initializing_shards":0,"unassigned_shards":5,"delayed_unassigned_shards":0,"number_of_pending_tasks":0,"number_of_in_flight_fetch":0}`, "2.4.5": `{"cluster_name":"elasticsearch","status":"yellow","timed_out":false,"number_of_nodes":1,"number_of_data_nodes":1,"active_primary_shards":5,"active_shards":5,"relocating_shards":0,"initializing_shards":0,"unassigned_shards":5,"delayed_unassigned_shards":0,"number_of_pending_tasks":0,"number_of_in_flight_fetch":0,"task_max_waiting_in_queue_millis":0,"active_shards_percent_as_number":50.0}`, "5.4.2": `{"cluster_name":"elasticsearch","status":"yellow","timed_out":false,"number_of_nodes":1,"number_of_data_nodes":1,"active_primary_shards":5,"active_shards":5,"relocating_shards":0,"initializing_shards":0,"unassigned_shards":5,"delayed_unassigned_shards":0,"number_of_pending_tasks":0,"number_of_in_flight_fetch":0,"task_max_waiting_in_queue_millis":0,"active_shards_percent_as_number":50.0}`, + "5.5.2": `{"cluster_name":"docker-cluster","status":"yellow","timed_out":false,"number_of_nodes":1,"number_of_data_nodes":1,"active_primary_shards":7,"active_shards":7,"relocating_shards":0,"initializing_shards":0,"unassigned_shards":7,"delayed_unassigned_shards":0,"number_of_pending_tasks":0,"number_of_in_flight_fetch":0,"task_max_waiting_in_queue_millis":0,"active_shards_percent_as_number":50.0}`, } for ver, out := range tcs { ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -78,7 +79,10 @@ func TestClusterHealth(t *testing.T) { t.Fatalf("Failed to fetch or decode cluster health: %s", err) } t.Logf("[%s] Cluster Health Response: %+v", ver, chr) - if chr.ClusterName != "elasticsearch" { + if ver != "5.5.2" && chr.ClusterName == "elasticsearch" || + ver == "5.5.2" && chr.ClusterName == "docker-cluster" { + // ok + } else { t.Errorf("Invalid cluster health response") } if chr.Status != "yellow" { @@ -105,6 +109,7 @@ func TestNodesStats(t *testing.T) { "1.7.6": `{"cluster_name":"elasticsearch","nodes":{"vKnRv2bPThCJIDTj92NvQg":{"timestamp":1498820641883,"name":"Overmind","transport_address":"inet[/172.17.0.4:9300]","host":"2603876b583f","ip":["inet[/172.17.0.4:9300]","NONE"],"indices":{"docs":{"count":0,"deleted":0},"store":{"size_in_bytes":575,"throttle_time_in_millis":0},"indexing":{"index_total":0,"index_time_in_millis":0,"index_current":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0},"refresh":{"total":0,"total_time_in_millis":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":10,"total_time_in_millis":52},"filter_cache":{"memory_size_in_bytes":0,"evictions":0},"id_cache":{"memory_size_in_bytes":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"percolate":{"total":0,"time_in_millis":0,"current":0,"memory_size_in_bytes":-1,"memory_size":"-1b","queries":0},"completion":{"size_in_bytes":0},"segments":{"count":0,"memory_in_bytes":0,"index_writer_memory_in_bytes":0,"index_writer_max_memory_in_bytes":335544320,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0},"translog":{"operations":0,"size_in_bytes":17},"suggest":{"total":0,"time_in_millis":0,"current":0},"query_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"os":{"timestamp":1498820641896},"process":{"timestamp":1498820641896,"open_file_descriptors":123},"jvm":{"timestamp":1498820641896,"uptime_in_millis":17871,"mem":{"heap_used_in_bytes":82958528,"heap_used_percent":7,"heap_committed_in_bytes":251002880,"heap_max_in_bytes":1056309248,"non_heap_used_in_bytes":44442536,"non_heap_committed_in_bytes":45400064,"pools":{"young":{"used_in_bytes":62150064,"max_in_bytes":139591680,"peak_used_in_bytes":139591680,"peak_max_in_bytes":139591680},"survivor":{"used_in_bytes":17432576,"max_in_bytes":17432576,"peak_used_in_bytes":17432576,"peak_max_in_bytes":17432576},"old":{"used_in_bytes":3375888,"max_in_bytes":899284992,"peak_used_in_bytes":10899920,"peak_max_in_bytes":899284992}}},"threads":{"count":36,"peak_count":36},"gc":{"collectors":{"young":{"collection_count":1,"collection_time_in_millis":40},"old":{"collection_count":1,"collection_time_in_millis":42}}},"buffer_pools":{"direct":{"count":27,"used_in_bytes":4981411,"total_capacity_in_bytes":4981411},"mapped":{"count":0,"used_in_bytes":0,"total_capacity_in_bytes":0}}},"thread_pool":{"percolate":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"fetch_shard_started":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"listener":{"threads":1,"queue":0,"active":0,"rejected":0,"largest":1,"completed":1},"index":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"refresh":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"suggest":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"generic":{"threads":4,"queue":0,"active":0,"rejected":0,"largest":4,"completed":14},"warmer":{"threads":1,"queue":0,"active":0,"rejected":0,"largest":1,"completed":5},"search":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"flush":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"optimize":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"fetch_shard_store":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"management":{"threads":1,"queue":0,"active":1,"rejected":0,"largest":1,"completed":1},"get":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"merge":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"bulk":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"snapshot":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0}},"network":{},"fs":{"timestamp":1498820641898,"total":{"total_in_bytes":63375708160,"free_in_bytes":59667320832,"available_in_bytes":56424390656},"data":[{"path":"/usr/share/elasticsearch/data/elasticsearch/nodes/0","mount":"/usr/share/elasticsearch/data (/dev/sda2)","type":"ext4","total_in_bytes":63375708160,"free_in_bytes":59667320832,"available_in_bytes":56424390656}]},"transport":{"server_open":13,"rx_count":6,"rx_size_in_bytes":1428,"tx_count":6,"tx_size_in_bytes":1428},"http":{"current_open":1,"total_opened":2},"breakers":{"request":{"limit_size_in_bytes":422523699,"limit_size":"402.9mb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.0,"tripped":0},"fielddata":{"limit_size_in_bytes":633785548,"limit_size":"604.4mb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.03,"tripped":0},"parent":{"limit_size_in_bytes":739416473,"limit_size":"705.1mb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.0,"tripped":0}}}}}`, "2.4.5": `{"cluster_name":"elasticsearch","nodes":{"VnnrklnAQ7KRXhW2WkgPxA":{"timestamp":1498820602662,"name":"Battering Ram","transport_address":"172.17.0.4:9300","host":"172.17.0.4","ip":["172.17.0.4:9300","NONE"],"indices":{"docs":{"count":0,"deleted":0},"store":{"size_in_bytes":650,"throttle_time_in_millis":0},"indexing":{"index_total":0,"index_time_in_millis":0,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":104857600},"refresh":{"total":0,"total_time_in_millis":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":10,"total_time_in_millis":60},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"percolate":{"total":0,"time_in_millis":0,"current":0,"memory_size_in_bytes":-1,"memory_size":"-1b","queries":0},"completion":{"size_in_bytes":0},"segments":{"count":0,"memory_in_bytes":0,"terms_memory_in_bytes":0,"stored_fields_memory_in_bytes":0,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":0,"doc_values_memory_in_bytes":0,"index_writer_memory_in_bytes":0,"index_writer_max_memory_in_bytes":105630920,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0},"translog":{"operations":0,"size_in_bytes":215},"suggest":{"total":0,"time_in_millis":0,"current":0},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"os":{"timestamp":1498820602670,"cpu_percent":7,"load_average":0.32861328125,"mem":{"total_in_bytes":2096177152,"free_in_bytes":1624637440,"used_in_bytes":471539712,"free_percent":78,"used_percent":22},"swap":{"total_in_bytes":4195348480,"free_in_bytes":4005060608,"used_in_bytes":190287872}},"process":{"timestamp":1498820602670,"open_file_descriptors":116,"max_file_descriptors":1048576,"cpu":{"percent":7,"total_in_millis":10270},"mem":{"total_virtual_in_bytes":2681352192}},"jvm":{"timestamp":1498820602671,"uptime_in_millis":38833,"mem":{"heap_used_in_bytes":88986784,"heap_used_percent":8,"heap_committed_in_bytes":251002880,"heap_max_in_bytes":1056309248,"non_heap_used_in_bytes":49185912,"non_heap_committed_in_bytes":49987584,"pools":{"young":{"used_in_bytes":58281984,"max_in_bytes":139591680,"peak_used_in_bytes":139591680,"peak_max_in_bytes":139591680},"survivor":{"used_in_bytes":17432576,"max_in_bytes":17432576,"peak_used_in_bytes":17432576,"peak_max_in_bytes":17432576},"old":{"used_in_bytes":13272224,"max_in_bytes":899284992,"peak_used_in_bytes":13272224,"peak_max_in_bytes":899284992}}},"threads":{"count":35,"peak_count":38},"gc":{"collectors":{"young":{"collection_count":2,"collection_time_in_millis":87},"old":{"collection_count":1,"collection_time_in_millis":74}}},"buffer_pools":{"direct":{"count":23,"used_in_bytes":3416836,"total_capacity_in_bytes":3416836},"mapped":{"count":0,"used_in_bytes":0,"total_capacity_in_bytes":0}},"classes":{"current_loaded_count":7221,"total_loaded_count":7221,"total_unloaded_count":0}},"thread_pool":{"bulk":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"fetch_shard_started":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"fetch_shard_store":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"flush":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"force_merge":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"generic":{"threads":1,"queue":0,"active":0,"rejected":0,"largest":5,"completed":28},"get":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"index":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"listener":{"threads":1,"queue":0,"active":0,"rejected":0,"largest":1,"completed":2},"management":{"threads":2,"queue":0,"active":1,"rejected":0,"largest":2,"completed":5},"percolate":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"refresh":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"search":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"snapshot":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"suggest":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"warmer":{"threads":1,"queue":0,"active":0,"rejected":0,"largest":1,"completed":5}},"fs":{"timestamp":1498820602672,"total":{"total_in_bytes":63375708160,"free_in_bytes":59667599360,"available_in_bytes":56424669184,"spins":"true"},"data":[{"path":"/usr/share/elasticsearch/data/elasticsearch/nodes/0","mount":"/usr/share/elasticsearch/data (/dev/sda2)","type":"ext4","total_in_bytes":63375708160,"free_in_bytes":59667599360,"available_in_bytes":56424669184,"spins":"true"}]},"transport":{"server_open":0,"rx_count":6,"rx_size_in_bytes":2028,"tx_count":6,"tx_size_in_bytes":2028},"http":{"current_open":1,"total_opened":3},"breakers":{"request":{"limit_size_in_bytes":422523699,"limit_size":"402.9mb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.0,"tripped":0},"fielddata":{"limit_size_in_bytes":633785548,"limit_size":"604.4mb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.03,"tripped":0},"in_flight_requests":{"limit_size_in_bytes":1056309248,"limit_size":"1007.3mb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.0,"tripped":0},"parent":{"limit_size_in_bytes":739416473,"limit_size":"705.1mb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.0,"tripped":0}},"script":{"compilations":0,"cache_evictions":0}}}}`, "5.4.2": `{"_nodes":{"total":1,"successful":1,"failed":0},"cluster_name":"elasticsearch","nodes":{"0hHcEFK1S7qMlk8hQCm7wQ":{"timestamp":1498820489394,"name":"0hHcEFK","transport_address":"127.0.0.1:9300","host":"127.0.0.1","ip":"127.0.0.1:9300","roles":["master","data","ingest"],"indices":{"docs":{"count":0,"deleted":0},"store":{"size_in_bytes":650,"throttle_time_in_millis":0},"indexing":{"index_total":0,"index_time_in_millis":0,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":104857600},"refresh":{"total":0,"total_time_in_millis":0,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":5,"total_time_in_millis":27},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":0,"memory_in_bytes":0,"terms_memory_in_bytes":0,"stored_fields_memory_in_bytes":0,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":0,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":0,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":0,"size_in_bytes":215},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"os":{"timestamp":1498820489400,"cpu":{"percent":0,"load_average":{"1m":0.35,"5m":0.28,"15m":0.12}},"mem":{"total_in_bytes":2096177152,"free_in_bytes":83501056,"used_in_bytes":2012676096,"free_percent":4,"used_percent":96},"swap":{"total_in_bytes":4195348480,"free_in_bytes":3487707136,"used_in_bytes":707641344}},"process":{"timestamp":1498820489400,"open_file_descriptors":139,"max_file_descriptors":1048576,"cpu":{"percent":0,"total_in_millis":26600},"mem":{"total_virtual_in_bytes":3823181824}},"jvm":{"timestamp":1498820489400,"uptime_in_millis":185693,"mem":{"heap_used_in_bytes":114959064,"heap_used_percent":5,"heap_committed_in_bytes":2130051072,"heap_max_in_bytes":2130051072,"non_heap_used_in_bytes":65471864,"non_heap_committed_in_bytes":69906432,"pools":{"young":{"used_in_bytes":82057312,"max_in_bytes":139591680,"peak_used_in_bytes":139591680,"peak_max_in_bytes":139591680},"survivor":{"used_in_bytes":17432576,"max_in_bytes":17432576,"peak_used_in_bytes":17432576,"peak_max_in_bytes":17432576},"old":{"used_in_bytes":15469176,"max_in_bytes":1973026816,"peak_used_in_bytes":15469176,"peak_max_in_bytes":1973026816}}},"threads":{"count":26,"peak_count":29},"gc":{"collectors":{"young":{"collection_count":4,"collection_time_in_millis":1618},"old":{"collection_count":1,"collection_time_in_millis":76}}},"buffer_pools":{"direct":{"count":16,"used_in_bytes":33776600,"total_capacity_in_bytes":33776599},"mapped":{"count":0,"used_in_bytes":0,"total_capacity_in_bytes":0}},"classes":{"current_loaded_count":9995,"total_loaded_count":9995,"total_unloaded_count":0}},"thread_pool":{"bulk":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"fetch_shard_started":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"fetch_shard_store":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"flush":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"force_merge":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"generic":{"threads":4,"queue":0,"active":0,"rejected":0,"largest":4,"completed":28},"get":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"index":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"listener":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"management":{"threads":2,"queue":0,"active":1,"rejected":0,"largest":2,"completed":20},"refresh":{"threads":1,"queue":0,"active":0,"rejected":0,"largest":1,"completed":31},"search":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"snapshot":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"warmer":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0}},"fs":{"timestamp":1498820489401,"total":{"total_in_bytes":63375708160,"free_in_bytes":59668226048,"available_in_bytes":56425295872,"spins":"true"},"data":[{"path":"/usr/share/elasticsearch/data/nodes/0","mount":"/usr/share/elasticsearch/data (/dev/sda2)","type":"ext4","total_in_bytes":63375708160,"free_in_bytes":59668226048,"available_in_bytes":56425295872,"spins":"true"}],"io_stats":{"devices":[{"device_name":"sda2","operations":3017,"read_operations":1800,"write_operations":1217,"read_kilobytes":24816,"write_kilobytes":6580}],"total":{"operations":3017,"read_operations":1800,"write_operations":1217,"read_kilobytes":24816,"write_kilobytes":6580}}},"transport":{"server_open":0,"rx_count":0,"rx_size_in_bytes":0,"tx_count":0,"tx_size_in_bytes":0},"http":{"current_open":2,"total_opened":3},"breakers":{"request":{"limit_size_in_bytes":1278030643,"limit_size":"1.1gb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.0,"tripped":0},"fielddata":{"limit_size_in_bytes":1278030643,"limit_size":"1.1gb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.03,"tripped":0},"in_flight_requests":{"limit_size_in_bytes":2130051072,"limit_size":"1.9gb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.0,"tripped":0},"parent":{"limit_size_in_bytes":1491035750,"limit_size":"1.3gb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.0,"tripped":0}},"script":{"compilations":0,"cache_evictions":0},"discovery":{"cluster_state_queue":{"total":0,"pending":0,"committed":0}},"ingest":{"total":{"count":0,"time_in_millis":0,"current":0,"failed":0},"pipelines":{}}}}}`, + "5.5.2": `{"_nodes":{"total":1,"successful":1,"failed":0},"cluster_name":"docker-cluster","nodes":{"8At5us-RRw6UofMRuJgFTw":{"timestamp":1503497399210,"name":"8At5us-","transport_address":"172.17.0.2:9300","host":"172.17.0.2","ip":"172.17.0.2:9300","roles":["master","data","ingest"],"attributes":{"ml.max_open_jobs":"10","ml.enabled":"true"},"indices":{"docs":{"count":19,"deleted":0},"store":{"size_in_bytes":91423,"throttle_time_in_millis":0},"indexing":{"index_total":19,"index_time_in_millis":357,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":146800640},"refresh":{"total":5,"total_time_in_millis":519,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":10,"total_time_in_millis":41},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":5,"memory_in_bytes":25435,"terms_memory_in_bytes":21179,"stored_fields_memory_in_bytes":1560,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":1600,"points_memory_in_bytes":292,"doc_values_memory_in_bytes":804,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":19,"size_in_bytes":23842},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"os":{"timestamp":1503497399218,"cpu":{"percent":20,"load_average":{"1m":1.19,"5m":0.43,"15m":0.27}},"mem":{"total_in_bytes":2095878144,"free_in_bytes":70742016,"used_in_bytes":2025136128,"free_percent":3,"used_percent":97},"swap":{"total_in_bytes":1073737728,"free_in_bytes":302432256,"used_in_bytes":771305472},"cgroup":{"cpuacct":{"control_group":"/","usage_nanos":33132630208},"cpu":{"control_group":"/","cfs_period_micros":100000,"cfs_quota_micros":-1,"stat":{"number_of_elapsed_periods":0,"number_of_times_throttled":0,"time_throttled_nanos":0}}}},"process":{"timestamp":1503497399218,"open_file_descriptors":221,"max_file_descriptors":1048576,"cpu":{"percent":21,"total_in_millis":32980},"mem":{"total_virtual_in_bytes":5950177280}},"jvm":{"timestamp":1503497399219,"uptime_in_millis":29213,"mem":{"heap_used_in_bytes":293256600,"heap_used_percent":13,"heap_committed_in_bytes":2112618496,"heap_max_in_bytes":2112618496,"non_heap_used_in_bytes":88549288,"non_heap_committed_in_bytes":94851072,"pools":{"young":{"used_in_bytes":210201424,"max_in_bytes":279183360,"peak_used_in_bytes":279183360,"peak_max_in_bytes":279183360},"survivor":{"used_in_bytes":34865152,"max_in_bytes":34865152,"peak_used_in_bytes":34865152,"peak_max_in_bytes":34865152},"old":{"used_in_bytes":48190024,"max_in_bytes":1798569984,"peak_used_in_bytes":48241264,"peak_max_in_bytes":1798569984}}},"threads":{"count":52,"peak_count":52},"gc":{"collectors":{"young":{"collection_count":3,"collection_time_in_millis":1258},"old":{"collection_count":2,"collection_time_in_millis":160}}},"buffer_pools":{"direct":{"count":24,"used_in_bytes":50664460,"total_capacity_in_bytes":50664459},"mapped":{"count":5,"used_in_bytes":86919,"total_capacity_in_bytes":86919}},"classes":{"current_loaded_count":13157,"total_loaded_count":13157,"total_unloaded_count":0}},"thread_pool":{"bulk":{"threads":4,"queue":0,"active":0,"rejected":0,"largest":4,"completed":6},"fetch_shard_started":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"fetch_shard_store":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"flush":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"force_merge":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"generic":{"threads":7,"queue":0,"active":0,"rejected":0,"largest":7,"completed":38},"get":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"index":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"listener":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"management":{"threads":4,"queue":0,"active":1,"rejected":0,"largest":4,"completed":15},"ml_autodetect":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"ml_datafeed":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"ml_utility":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"refresh":{"threads":2,"queue":0,"active":0,"rejected":0,"largest":2,"completed":14},"search":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"snapshot":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"warmer":{"threads":1,"queue":0,"active":0,"rejected":0,"largest":1,"completed":5},"watcher":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0}},"fs":{"timestamp":1503497399219,"total":{"total_in_bytes":67371577344,"free_in_bytes":64008867840,"available_in_bytes":60556169216,"spins":"true"},"data":[{"path":"/usr/share/elasticsearch/data/nodes/0","mount":"/ (overlay)","type":"overlay","total_in_bytes":67371577344,"free_in_bytes":64008867840,"available_in_bytes":60556169216,"spins":"true"}],"io_stats":{}},"transport":{"server_open":0,"rx_count":10,"rx_size_in_bytes":3710,"tx_count":10,"tx_size_in_bytes":3710},"http":{"current_open":1,"total_opened":2},"breakers":{"request":{"limit_size_in_bytes":1267571097,"limit_size":"1.1gb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.0,"tripped":0},"fielddata":{"limit_size_in_bytes":1267571097,"limit_size":"1.1gb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.03,"tripped":0},"in_flight_requests":{"limit_size_in_bytes":2112618496,"limit_size":"1.9gb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.0,"tripped":0},"parent":{"limit_size_in_bytes":1478832947,"limit_size":"1.3gb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.0,"tripped":0}},"script":{"compilations":9,"cache_evictions":0},"discovery":{"cluster_state_queue":{"total":0,"pending":0,"committed":0}},"ingest":{"total":{"count":0,"time_in_millis":0,"current":0,"failed":0},"pipelines":{"xpack_monitoring_6":{"count":0,"time_in_millis":0,"current":0,"failed":0},"xpack_monitoring_2":{"count":0,"time_in_millis":0,"current":0,"failed":0}}}}}}`, } for ver, out := range tcs { for hn, handler := range map[string]http.Handler{ @@ -133,14 +138,12 @@ func TestNodesStats(t *testing.T) { t.Fatalf("Failed to fetch or decode node stats: %s", err) } t.Logf("[%s/%s] Node Stats Response: %+v", hn, ver, nsr) - if nsr.ClusterName != "elasticsearch" { + if ver != "5.5.2" && nsr.ClusterName == "elasticsearch" || + ver == "5.5.2" && nsr.ClusterName == "docker-cluster" { + // ok + } else { t.Errorf("Wrong cluster name") } - for _, nsnr := range nsr.Nodes { - if nsnr.Indices.Docs.Count > 0 { - t.Errorf("Wrong doc count") - } - } } } } @@ -155,7 +158,10 @@ func TestIndices(t *testing.T) { // curl -XPUT http://localhost:9200/foo_2/type1/3 -d '{"title":"def003","content":"world003"}' // curl http://localhost:9200/_all/_stats ti := map[string]string{ - "5.5.1": `{"_shards":{"total":34,"successful":34,"failed":0},"_all":{"primaries":{"docs":{"count":256658,"deleted":148},"store":{"size_in_bytes":175959144,"throttle_time_in_millis":0},"indexing":{"index_total":413652,"index_time_in_millis":316231,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":88438,"time_in_millis":2799,"exists_total":88438,"exists_time_in_millis":2799,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":88458,"query_time_in_millis":2513,"query_current":0,"fetch_total":88446,"fetch_time_in_millis":2685,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":10275,"total_time_in_millis":5895937,"total_docs":80376844,"total_size_in_bytes":49005089428,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":274,"total_auto_throttle_in_bytes":354609338},"refresh":{"total":100264,"total_time_in_millis":818781,"listeners":0},"flush":{"total":5,"total_time_in_millis":124},"warmer":{"current":0,"total":100286,"total_time_in_millis":23851},"query_cache":{"memory_size_in_bytes":0,"total_count":22,"hit_count":0,"miss_count":22,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":38,"memory_in_bytes":720166,"terms_memory_in_bytes":481188,"stored_fields_memory_in_bytes":31920,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":3072,"points_memory_in_bytes":61858,"doc_values_memory_in_bytes":142128,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":137331,"size_in_bytes":167642357},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":1},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":513316,"deleted":324},"store":{"size_in_bytes":352143858,"throttle_time_in_millis":0},"indexing":{"index_total":827302,"index_time_in_millis":625030,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":176883,"time_in_millis":6265,"exists_total":176883,"exists_time_in_millis":6265,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":176913,"query_time_in_millis":5115,"query_current":0,"fetch_total":176891,"fetch_time_in_millis":5862,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":20547,"total_time_in_millis":11765629,"total_docs":160766745,"total_size_in_bytes":98024993696,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":522,"total_auto_throttle_in_bytes":709218676},"refresh":{"total":200378,"total_time_in_millis":1620051,"listeners":0},"flush":{"total":10,"total_time_in_millis":264},"warmer":{"current":0,"total":200427,"total_time_in_millis":46689},"query_cache":{"memory_size_in_bytes":0,"total_count":54,"hit_count":0,"miss_count":54,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":776,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":86,"memory_in_bytes":1496932,"terms_memory_in_bytes":1008932,"stored_fields_memory_in_bytes":66896,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":6784,"points_memory_in_bytes":124368,"doc_values_memory_in_bytes":289952,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":1502755202429,"file_sizes":{}},"translog":{"operations":274662,"size_in_bytes":335284714},"request_cache":{"memory_size_in_bytes":1794,"evictions":0,"hit_count":0,"miss_count":3},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}},"indices":{".monitoring-kibana-6-2017.08.16":{"primaries":{"docs":{"count":12459,"deleted":0},"store":{"size_in_bytes":4880578,"throttle_time_in_millis":0},"indexing":{"index_total":12459,"index_time_in_millis":20073,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":1372,"total_time_in_millis":275560,"total_docs":8619963,"total_size_in_bytes":3518738895,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":20971520},"refresh":{"total":12724,"total_time_in_millis":79600,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":12725,"total_time_in_millis":2653},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":2,"memory_in_bytes":34337,"terms_memory_in_bytes":30104,"stored_fields_memory_in_bytes":968,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":128,"points_memory_in_bytes":1393,"doc_values_memory_in_bytes":1744,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":12459,"size_in_bytes":12175093},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":24918,"deleted":0},"store":{"size_in_bytes":9682148,"throttle_time_in_millis":0},"indexing":{"index_total":24918,"index_time_in_millis":37852,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":1,"query_time_in_millis":10,"query_current":0,"fetch_total":1,"fetch_time_in_millis":1,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":2743,"total_time_in_millis":549689,"total_docs":17234974,"total_size_in_bytes":7034624631,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":41943040},"refresh":{"total":25390,"total_time_in_millis":155546,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":25392,"total_time_in_millis":5196},"query_cache":{"memory_size_in_bytes":0,"total_count":3,"hit_count":0,"miss_count":3,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":7,"memory_in_bytes":83565,"terms_memory_in_bytes":73488,"stored_fields_memory_in_bytes":2968,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":448,"points_memory_in_bytes":2897,"doc_values_memory_in_bytes":3764,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":24918,"size_in_bytes":24350186},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}},".monitoring-kibana-6-2017.08.15":{"primaries":{"docs":{"count":25917,"deleted":0},"store":{"size_in_bytes":9617967,"throttle_time_in_millis":0},"indexing":{"index_total":25917,"index_time_in_millis":36304,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":1,"query_time_in_millis":10,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":2055,"total_time_in_millis":746516,"total_docs":27797269,"total_size_in_bytes":10672726928,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":20971520},"refresh":{"total":19569,"total_time_in_millis":123083,"listeners":0},"flush":{"total":1,"total_time_in_millis":18},"warmer":{"current":0,"total":19571,"total_time_in_millis":3995},"query_cache":{"memory_size_in_bytes":0,"total_count":3,"hit_count":0,"miss_count":3,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":3,"memory_in_bytes":58226,"terms_memory_in_bytes":52997,"stored_fields_memory_in_bytes":1912,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":192,"points_memory_in_bytes":2721,"doc_values_memory_in_bytes":404,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":0,"size_in_bytes":43},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":51834,"deleted":0},"store":{"size_in_bytes":19309686,"throttle_time_in_millis":0},"indexing":{"index_total":51834,"index_time_in_millis":68258,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":1,"query_time_in_millis":10,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":4109,"total_time_in_millis":1477106,"total_docs":55598675,"total_size_in_bytes":21347372840,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":41943040},"refresh":{"total":39103,"total_time_in_millis":238886,"listeners":0},"flush":{"total":2,"total_time_in_millis":44},"warmer":{"current":0,"total":39108,"total_time_in_millis":7775},"query_cache":{"memory_size_in_bytes":0,"total_count":3,"hit_count":0,"miss_count":3,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":11,"memory_in_bytes":141204,"terms_memory_in_bytes":128132,"stored_fields_memory_in_bytes":5472,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":704,"points_memory_in_bytes":5628,"doc_values_memory_in_bytes":1268,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":1502755202429,"file_sizes":{}},"translog":{"operations":0,"size_in_bytes":86},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}},".monitoring-kibana-6-2017.08.14":{"primaries":{"docs":{"count":6147,"deleted":0},"store":{"size_in_bytes":2467088,"throttle_time_in_millis":0},"indexing":{"index_total":6147,"index_time_in_millis":9228,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":1,"query_time_in_millis":32,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":397,"total_time_in_millis":48142,"total_docs":1231276,"total_size_in_bytes":559560922,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":20971520},"refresh":{"total":3733,"total_time_in_millis":28841,"listeners":0},"flush":{"total":1,"total_time_in_millis":7},"warmer":{"current":0,"total":3735,"total_time_in_millis":1168},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":3,"memory_in_bytes":29099,"terms_memory_in_bytes":25724,"stored_fields_memory_in_bytes":1104,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":192,"points_memory_in_bytes":715,"doc_values_memory_in_bytes":1364,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":0,"size_in_bytes":43},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":12294,"deleted":0},"store":{"size_in_bytes":5029190,"throttle_time_in_millis":0},"indexing":{"index_total":12294,"index_time_in_millis":16557,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":1,"query_time_in_millis":32,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":792,"total_time_in_millis":96523,"total_docs":2467549,"total_size_in_bytes":1121117660,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":41943040},"refresh":{"total":7449,"total_time_in_millis":51544,"listeners":0},"flush":{"total":2,"total_time_in_millis":16},"warmer":{"current":0,"total":7453,"total_time_in_millis":1872},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":11,"memory_in_bytes":83924,"terms_memory_in_bytes":73590,"stored_fields_memory_in_bytes":3768,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":704,"points_memory_in_bytes":1618,"doc_values_memory_in_bytes":4244,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":0,"size_in_bytes":86},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}},".kibana":{"primaries":{"docs":{"count":1,"deleted":0},"store":{"size_in_bytes":3967,"throttle_time_in_millis":0},"indexing":{"index_total":1,"index_time_in_millis":2,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":88438,"time_in_millis":2799,"exists_total":88438,"exists_time_in_millis":2799,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":88441,"query_time_in_millis":2287,"query_current":0,"fetch_total":88441,"fetch_time_in_millis":2685,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":20971520},"refresh":{"total":2,"total_time_in_millis":19,"listeners":0},"flush":{"total":1,"total_time_in_millis":7},"warmer":{"current":0,"total":5,"total_time_in_millis":12},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":1,"memory_in_bytes":2219,"terms_memory_in_bytes":1751,"stored_fields_memory_in_bytes":312,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":64,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":92,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":0,"size_in_bytes":43},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":2,"deleted":0},"store":{"size_in_bytes":7934,"throttle_time_in_millis":0},"indexing":{"index_total":2,"index_time_in_millis":4,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":176883,"time_in_millis":6265,"exists_total":176883,"exists_time_in_millis":6265,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":176879,"query_time_in_millis":4703,"query_current":0,"fetch_total":176879,"fetch_time_in_millis":5860,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":41943040},"refresh":{"total":4,"total_time_in_millis":33,"listeners":0},"flush":{"total":2,"total_time_in_millis":14},"warmer":{"current":0,"total":10,"total_time_in_millis":26},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":2,"memory_in_bytes":4438,"terms_memory_in_bytes":3502,"stored_fields_memory_in_bytes":624,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":128,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":184,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":0,"size_in_bytes":86},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}},".monitoring-es-6-2017.08.15":{"primaries":{"docs":{"count":120970,"deleted":60},"store":{"size_in_bytes":90471073,"throttle_time_in_millis":0},"indexing":{"index_total":207360,"index_time_in_millis":142584,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":1,"query_time_in_millis":85,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":3781,"total_time_in_millis":2188959,"total_docs":19326607,"total_size_in_bytes":15981823826,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":274,"total_auto_throttle_in_bytes":19065018},"refresh":{"total":37364,"total_time_in_millis":337011,"listeners":0},"flush":{"total":1,"total_time_in_millis":67},"warmer":{"current":0,"total":37365,"total_time_in_millis":8943},"query_cache":{"memory_size_in_bytes":0,"total_count":9,"hit_count":0,"miss_count":9,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":9,"memory_in_bytes":297633,"terms_memory_in_bytes":184606,"stored_fields_memory_in_bytes":13736,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":576,"points_memory_in_bytes":31279,"doc_values_memory_in_bytes":67436,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":0,"size_in_bytes":43},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":241940,"deleted":120},"store":{"size_in_bytes":180988889,"throttle_time_in_millis":0},"indexing":{"index_total":414720,"index_time_in_millis":286230,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":2,"query_time_in_millis":135,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":7561,"total_time_in_millis":4425176,"total_docs":38722769,"total_size_in_bytes":32018268991,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":522,"total_auto_throttle_in_bytes":38130036},"refresh":{"total":74706,"total_time_in_millis":675900,"listeners":0},"flush":{"total":2,"total_time_in_millis":148},"warmer":{"current":0,"total":74710,"total_time_in_millis":18069},"query_cache":{"memory_size_in_bytes":0,"total_count":24,"hit_count":0,"miss_count":24,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":424,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":16,"memory_in_bytes":590531,"terms_memory_in_bytes":362031,"stored_fields_memory_in_bytes":26728,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":1024,"points_memory_in_bytes":62748,"doc_values_memory_in_bytes":138000,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":1502755201903,"file_sizes":{}},"translog":{"operations":0,"size_in_bytes":86},"request_cache":{"memory_size_in_bytes":897,"evictions":0,"hit_count":0,"miss_count":1},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}},".monitoring-es-6-2017.08.14":{"primaries":{"docs":{"count":24614,"deleted":12},"store":{"size_in_bytes":17680221,"throttle_time_in_millis":0},"indexing":{"index_total":36896,"index_time_in_millis":35441,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":1,"query_time_in_millis":24,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":927,"total_time_in_millis":1245791,"total_docs":11392506,"total_size_in_bytes":8435013687,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":20971520},"refresh":{"total":9432,"total_time_in_millis":90529,"listeners":0},"flush":{"total":1,"total_time_in_millis":25},"warmer":{"current":0,"total":9434,"total_time_in_millis":3098},"query_cache":{"memory_size_in_bytes":0,"total_count":2,"hit_count":0,"miss_count":2,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":7,"memory_in_bytes":83467,"terms_memory_in_bytes":59601,"stored_fields_memory_in_bytes":4040,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":448,"points_memory_in_bytes":7054,"doc_values_memory_in_bytes":12324,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":0,"size_in_bytes":43},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":49228,"deleted":18},"store":{"size_in_bytes":35172184,"throttle_time_in_millis":0},"indexing":{"index_total":73792,"index_time_in_millis":69939,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":2,"query_time_in_millis":61,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":1853,"total_time_in_millis":2468444,"total_docs":22769560,"total_size_in_bytes":16842830754,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":41943040},"refresh":{"total":18843,"total_time_in_millis":176757,"listeners":0},"flush":{"total":2,"total_time_in_millis":42},"warmer":{"current":0,"total":18847,"total_time_in_millis":5819},"query_cache":{"memory_size_in_bytes":0,"total_count":7,"hit_count":0,"miss_count":7,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":352,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":9,"memory_in_bytes":137869,"terms_memory_in_bytes":97131,"stored_fields_memory_in_bytes":6680,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":576,"points_memory_in_bytes":13662,"doc_values_memory_in_bytes":19820,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":0,"size_in_bytes":86},"request_cache":{"memory_size_in_bytes":897,"evictions":0,"hit_count":0,"miss_count":1},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}},"foo_2":{"primaries":{"docs":{"count":3,"deleted":0},"store":{"size_in_bytes":12138,"throttle_time_in_millis":0},"indexing":{"index_total":3,"index_time_in_millis":4,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":5,"query_time_in_millis":0,"query_current":0,"fetch_total":2,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":104857600},"refresh":{"total":3,"total_time_in_millis":12,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":8,"total_time_in_millis":0},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":3,"memory_in_bytes":7764,"terms_memory_in_bytes":5976,"stored_fields_memory_in_bytes":936,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":576,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":276,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":3,"size_in_bytes":524},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":6,"deleted":0},"store":{"size_in_bytes":24276,"throttle_time_in_millis":0},"indexing":{"index_total":5,"index_time_in_millis":6,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":10,"query_time_in_millis":0,"query_current":0,"fetch_total":5,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":209715200},"refresh":{"total":5,"total_time_in_millis":20,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":16,"total_time_in_millis":0},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":6,"memory_in_bytes":15528,"terms_memory_in_bytes":11952,"stored_fields_memory_in_bytes":1872,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":1152,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":552,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":6,"size_in_bytes":1048},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}},"foo_1":{"primaries":{"docs":{"count":2,"deleted":0},"store":{"size_in_bytes":8246,"throttle_time_in_millis":0},"indexing":{"index_total":4,"index_time_in_millis":7,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":7,"query_time_in_millis":8,"query_current":0,"fetch_total":3,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":104857600},"refresh":{"total":4,"total_time_in_millis":18,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":9,"total_time_in_millis":0},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":2,"memory_in_bytes":5176,"terms_memory_in_bytes":3984,"stored_fields_memory_in_bytes":624,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":384,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":184,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":4,"size_in_bytes":609},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":4,"deleted":0},"store":{"size_in_bytes":16492,"throttle_time_in_millis":0},"indexing":{"index_total":7,"index_time_in_millis":13,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":15,"query_time_in_millis":21,"query_current":0,"fetch_total":5,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":209715200},"refresh":{"total":7,"total_time_in_millis":32,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":18,"total_time_in_millis":0},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":4,"memory_in_bytes":10352,"terms_memory_in_bytes":7968,"stored_fields_memory_in_bytes":1248,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":768,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":368,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":8,"size_in_bytes":1218},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}},".monitoring-es-6-2017.08.16":{"primaries":{"docs":{"count":66545,"deleted":76},"store":{"size_in_bytes":50817866,"throttle_time_in_millis":0},"indexing":{"index_total":124865,"index_time_in_millis":72588,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":1,"query_time_in_millis":67,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":1743,"total_time_in_millis":1390969,"total_docs":12009223,"total_size_in_bytes":9837225170,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":20971520},"refresh":{"total":17433,"total_time_in_millis":159668,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":17434,"total_time_in_millis":3982},"query_cache":{"memory_size_in_bytes":0,"total_count":8,"hit_count":0,"miss_count":8,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":8,"memory_in_bytes":202245,"terms_memory_in_bytes":116445,"stored_fields_memory_in_bytes":8288,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":512,"points_memory_in_bytes":18696,"doc_values_memory_in_bytes":58304,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":124865,"size_in_bytes":155465916},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":1},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":133090,"deleted":186},"store":{"size_in_bytes":101913059,"throttle_time_in_millis":0},"indexing":{"index_total":249730,"index_time_in_millis":146171,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":2,"query_time_in_millis":143,"query_current":0,"fetch_total":1,"fetch_time_in_millis":1,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":3489,"total_time_in_millis":2748691,"total_docs":23973218,"total_size_in_bytes":19660778820,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":41943040},"refresh":{"total":34871,"total_time_in_millis":321333,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":34873,"total_time_in_millis":7932},"query_cache":{"memory_size_in_bytes":0,"total_count":17,"hit_count":0,"miss_count":17,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":20,"memory_in_bytes":429521,"terms_memory_in_bytes":251138,"stored_fields_memory_in_bytes":17536,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":1280,"points_memory_in_bytes":37815,"doc_values_memory_in_bytes":121752,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":249730,"size_in_bytes":310931832},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":1},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}}}}`, + "1.7.6": `{"_shards":{"total":20,"successful":10,"failed":0},"_all":{"primaries":{"docs":{"count":5,"deleted":0},"store":{"size_in_bytes":13798,"throttle_time_in_millis":0},"indexing":{"index_total":5,"index_time_in_millis":52,"index_current":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0},"refresh":{"total":5,"total_time_in_millis":163},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":30,"total_time_in_millis":42},"filter_cache":{"memory_size_in_bytes":0,"evictions":0},"id_cache":{"memory_size_in_bytes":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"percolate":{"total":0,"time_in_millis":0,"current":0,"memory_size_in_bytes":-1,"memory_size":"-1b","queries":0},"completion":{"size_in_bytes":0},"segments":{"count":5,"memory_in_bytes":18410,"index_writer_memory_in_bytes":0,"index_writer_max_memory_in_bytes":671088640,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0},"translog":{"operations":5,"size_in_bytes":102},"suggest":{"total":0,"time_in_millis":0,"current":0},"query_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":5,"deleted":0},"store":{"size_in_bytes":13798,"throttle_time_in_millis":0},"indexing":{"index_total":5,"index_time_in_millis":52,"index_current":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0},"refresh":{"total":5,"total_time_in_millis":163},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":30,"total_time_in_millis":42},"filter_cache":{"memory_size_in_bytes":0,"evictions":0},"id_cache":{"memory_size_in_bytes":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"percolate":{"total":0,"time_in_millis":0,"current":0,"memory_size_in_bytes":-1,"memory_size":"-1b","queries":0},"completion":{"size_in_bytes":0},"segments":{"count":5,"memory_in_bytes":18410,"index_writer_memory_in_bytes":0,"index_writer_max_memory_in_bytes":671088640,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0},"translog":{"operations":5,"size_in_bytes":102},"suggest":{"total":0,"time_in_millis":0,"current":0},"query_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}},"indices":{"foo_2":{"primaries":{"docs":{"count":3,"deleted":0},"store":{"size_in_bytes":8207,"throttle_time_in_millis":0},"indexing":{"index_total":3,"index_time_in_millis":6,"index_current":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0},"refresh":{"total":3,"total_time_in_millis":38},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":16,"total_time_in_millis":0},"filter_cache":{"memory_size_in_bytes":0,"evictions":0},"id_cache":{"memory_size_in_bytes":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"percolate":{"total":0,"time_in_millis":0,"current":0,"memory_size_in_bytes":-1,"memory_size":"-1b","queries":0},"completion":{"size_in_bytes":0},"segments":{"count":3,"memory_in_bytes":11046,"index_writer_memory_in_bytes":0,"index_writer_max_memory_in_bytes":335544320,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0},"translog":{"operations":3,"size_in_bytes":102},"suggest":{"total":0,"time_in_millis":0,"current":0},"query_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":3,"deleted":0},"store":{"size_in_bytes":8207,"throttle_time_in_millis":0},"indexing":{"index_total":3,"index_time_in_millis":6,"index_current":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0},"refresh":{"total":3,"total_time_in_millis":38},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":16,"total_time_in_millis":0},"filter_cache":{"memory_size_in_bytes":0,"evictions":0},"id_cache":{"memory_size_in_bytes":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"percolate":{"total":0,"time_in_millis":0,"current":0,"memory_size_in_bytes":-1,"memory_size":"-1b","queries":0},"completion":{"size_in_bytes":0},"segments":{"count":3,"memory_in_bytes":11046,"index_writer_memory_in_bytes":0,"index_writer_max_memory_in_bytes":335544320,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0},"translog":{"operations":3,"size_in_bytes":102},"suggest":{"total":0,"time_in_millis":0,"current":0},"query_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}},"foo_1":{"primaries":{"docs":{"count":2,"deleted":0},"store":{"size_in_bytes":5591,"throttle_time_in_millis":0},"indexing":{"index_total":2,"index_time_in_millis":46,"index_current":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0},"refresh":{"total":2,"total_time_in_millis":125},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":14,"total_time_in_millis":42},"filter_cache":{"memory_size_in_bytes":0,"evictions":0},"id_cache":{"memory_size_in_bytes":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"percolate":{"total":0,"time_in_millis":0,"current":0,"memory_size_in_bytes":-1,"memory_size":"-1b","queries":0},"completion":{"size_in_bytes":0},"segments":{"count":2,"memory_in_bytes":7364,"index_writer_memory_in_bytes":0,"index_writer_max_memory_in_bytes":335544320,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0},"translog":{"operations":2,"size_in_bytes":17},"suggest":{"total":0,"time_in_millis":0,"current":0},"query_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":2,"deleted":0},"store":{"size_in_bytes":5591,"throttle_time_in_millis":0},"indexing":{"index_total":2,"index_time_in_millis":46,"index_current":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0},"refresh":{"total":2,"total_time_in_millis":125},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":14,"total_time_in_millis":42},"filter_cache":{"memory_size_in_bytes":0,"evictions":0},"id_cache":{"memory_size_in_bytes":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"percolate":{"total":0,"time_in_millis":0,"current":0,"memory_size_in_bytes":-1,"memory_size":"-1b","queries":0},"completion":{"size_in_bytes":0},"segments":{"count":2,"memory_in_bytes":7364,"index_writer_memory_in_bytes":0,"index_writer_max_memory_in_bytes":335544320,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0},"translog":{"operations":2,"size_in_bytes":17},"suggest":{"total":0,"time_in_millis":0,"current":0},"query_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}}}}`, + "2.4.5": `{"_shards":{"total":20,"successful":10,"failed":0},"_all":{"primaries":{"docs":{"count":5,"deleted":0},"store":{"size_in_bytes":3610,"throttle_time_in_millis":0},"indexing":{"index_total":5,"index_time_in_millis":40,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":209715200},"refresh":{"total":5,"total_time_in_millis":171},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":30,"total_time_in_millis":12},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"percolate":{"total":0,"time_in_millis":0,"current":0,"memory_size_in_bytes":-1,"memory_size":"-1b","queries":0},"completion":{"size_in_bytes":0},"segments":{"count":5,"memory_in_bytes":10530,"terms_memory_in_bytes":7550,"stored_fields_memory_in_bytes":1560,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":960,"doc_values_memory_in_bytes":460,"index_writer_memory_in_bytes":0,"index_writer_max_memory_in_bytes":103887660,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0},"translog":{"operations":5,"size_in_bytes":843},"suggest":{"total":0,"time_in_millis":0,"current":0},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":5,"deleted":0},"store":{"size_in_bytes":3610,"throttle_time_in_millis":0},"indexing":{"index_total":5,"index_time_in_millis":40,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":209715200},"refresh":{"total":5,"total_time_in_millis":171},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":30,"total_time_in_millis":12},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"percolate":{"total":0,"time_in_millis":0,"current":0,"memory_size_in_bytes":-1,"memory_size":"-1b","queries":0},"completion":{"size_in_bytes":0},"segments":{"count":5,"memory_in_bytes":10530,"terms_memory_in_bytes":7550,"stored_fields_memory_in_bytes":1560,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":960,"doc_values_memory_in_bytes":460,"index_writer_memory_in_bytes":0,"index_writer_max_memory_in_bytes":103887660,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0},"translog":{"operations":5,"size_in_bytes":843},"suggest":{"total":0,"time_in_millis":0,"current":0},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}},"indices":{"foo_2":{"primaries":{"docs":{"count":3,"deleted":0},"store":{"size_in_bytes":3350,"throttle_time_in_millis":0},"indexing":{"index_total":3,"index_time_in_millis":6,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":104857600},"refresh":{"total":3,"total_time_in_millis":34},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":16,"total_time_in_millis":0},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"percolate":{"total":0,"time_in_millis":0,"current":0,"memory_size_in_bytes":-1,"memory_size":"-1b","queries":0},"completion":{"size_in_bytes":0},"segments":{"count":3,"memory_in_bytes":6318,"terms_memory_in_bytes":4530,"stored_fields_memory_in_bytes":936,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":576,"doc_values_memory_in_bytes":276,"index_writer_memory_in_bytes":0,"index_writer_max_memory_in_bytes":51943830,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0},"translog":{"operations":3,"size_in_bytes":470},"suggest":{"total":0,"time_in_millis":0,"current":0},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":3,"deleted":0},"store":{"size_in_bytes":3350,"throttle_time_in_millis":0},"indexing":{"index_total":3,"index_time_in_millis":6,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":104857600},"refresh":{"total":3,"total_time_in_millis":34},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":16,"total_time_in_millis":0},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"percolate":{"total":0,"time_in_millis":0,"current":0,"memory_size_in_bytes":-1,"memory_size":"-1b","queries":0},"completion":{"size_in_bytes":0},"segments":{"count":3,"memory_in_bytes":6318,"terms_memory_in_bytes":4530,"stored_fields_memory_in_bytes":936,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":576,"doc_values_memory_in_bytes":276,"index_writer_memory_in_bytes":0,"index_writer_max_memory_in_bytes":51943830,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0},"translog":{"operations":3,"size_in_bytes":470},"suggest":{"total":0,"time_in_millis":0,"current":0},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}},"foo_1":{"primaries":{"docs":{"count":2,"deleted":0},"store":{"size_in_bytes":260,"throttle_time_in_millis":0},"indexing":{"index_total":2,"index_time_in_millis":34,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":104857600},"refresh":{"total":2,"total_time_in_millis":137},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":14,"total_time_in_millis":12},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"percolate":{"total":0,"time_in_millis":0,"current":0,"memory_size_in_bytes":-1,"memory_size":"-1b","queries":0},"completion":{"size_in_bytes":0},"segments":{"count":2,"memory_in_bytes":4212,"terms_memory_in_bytes":3020,"stored_fields_memory_in_bytes":624,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":384,"doc_values_memory_in_bytes":184,"index_writer_memory_in_bytes":0,"index_writer_max_memory_in_bytes":51943830,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0},"translog":{"operations":2,"size_in_bytes":373},"suggest":{"total":0,"time_in_millis":0,"current":0},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":2,"deleted":0},"store":{"size_in_bytes":260,"throttle_time_in_millis":0},"indexing":{"index_total":2,"index_time_in_millis":34,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":104857600},"refresh":{"total":2,"total_time_in_millis":137},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":14,"total_time_in_millis":12},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"percolate":{"total":0,"time_in_millis":0,"current":0,"memory_size_in_bytes":-1,"memory_size":"-1b","queries":0},"completion":{"size_in_bytes":0},"segments":{"count":2,"memory_in_bytes":4212,"terms_memory_in_bytes":3020,"stored_fields_memory_in_bytes":624,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":384,"doc_values_memory_in_bytes":184,"index_writer_memory_in_bytes":0,"index_writer_max_memory_in_bytes":51943830,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0},"translog":{"operations":2,"size_in_bytes":373},"suggest":{"total":0,"time_in_millis":0,"current":0},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}}}}`, + "5.4.2": `{"_shards":{"total":26,"successful":13,"failed":0},"_all":{"primaries":{"docs":{"count":76,"deleted":0},"store":{"size_in_bytes":128534,"throttle_time_in_millis":0},"indexing":{"index_total":78,"index_time_in_millis":1598,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":272629760},"refresh":{"total":15,"total_time_in_millis":1361,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":26,"total_time_in_millis":124},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":13,"memory_in_bytes":56523,"terms_memory_in_bytes":44419,"stored_fields_memory_in_bytes":4056,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":2880,"points_memory_in_bytes":652,"doc_values_memory_in_bytes":4516,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":78,"size_in_bytes":56679},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":76,"deleted":0},"store":{"size_in_bytes":128534,"throttle_time_in_millis":0},"indexing":{"index_total":78,"index_time_in_millis":1598,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":272629760},"refresh":{"total":15,"total_time_in_millis":1361,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":26,"total_time_in_millis":124},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":13,"memory_in_bytes":56523,"terms_memory_in_bytes":44419,"stored_fields_memory_in_bytes":4056,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":2880,"points_memory_in_bytes":652,"doc_values_memory_in_bytes":4516,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":78,"size_in_bytes":56679},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}},"indices":{".monitoring-es-2-2017.08.23":{"primaries":{"docs":{"count":65,"deleted":0},"store":{"size_in_bytes":68917,"throttle_time_in_millis":0},"indexing":{"index_total":65,"index_time_in_millis":106,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":20971520},"refresh":{"total":3,"total_time_in_millis":390,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":4,"total_time_in_millis":15},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":3,"memory_in_bytes":23830,"terms_memory_in_bytes":18474,"stored_fields_memory_in_bytes":936,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":320,"points_memory_in_bytes":648,"doc_values_memory_in_bytes":3452,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":65,"size_in_bytes":37990},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":65,"deleted":0},"store":{"size_in_bytes":68917,"throttle_time_in_millis":0},"indexing":{"index_total":65,"index_time_in_millis":106,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":20971520},"refresh":{"total":3,"total_time_in_millis":390,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":4,"total_time_in_millis":15},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":3,"memory_in_bytes":23830,"terms_memory_in_bytes":18474,"stored_fields_memory_in_bytes":936,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":320,"points_memory_in_bytes":648,"doc_values_memory_in_bytes":3452,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":65,"size_in_bytes":37990},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}},".monitoring-data-2":{"primaries":{"docs":{"count":2,"deleted":0},"store":{"size_in_bytes":4226,"throttle_time_in_millis":0},"indexing":{"index_total":4,"index_time_in_millis":13,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":20971520},"refresh":{"total":2,"total_time_in_millis":74,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":3,"total_time_in_millis":2},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":1,"memory_in_bytes":1335,"terms_memory_in_bytes":787,"stored_fields_memory_in_bytes":312,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":0,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":236,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":4,"size_in_bytes":6738},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":2,"deleted":0},"store":{"size_in_bytes":4226,"throttle_time_in_millis":0},"indexing":{"index_total":4,"index_time_in_millis":13,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":20971520},"refresh":{"total":2,"total_time_in_millis":74,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":3,"total_time_in_millis":2},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":1,"memory_in_bytes":1335,"terms_memory_in_bytes":787,"stored_fields_memory_in_bytes":312,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":0,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":236,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":4,"size_in_bytes":6738},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}},"foo_2":{"primaries":{"docs":{"count":3,"deleted":0},"store":{"size_in_bytes":11909,"throttle_time_in_millis":0},"indexing":{"index_total":3,"index_time_in_millis":12,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":104857600},"refresh":{"total":3,"total_time_in_millis":42,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":8,"total_time_in_millis":4},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":3,"memory_in_bytes":7764,"terms_memory_in_bytes":5976,"stored_fields_memory_in_bytes":936,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":576,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":276,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":3,"size_in_bytes":494},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":3,"deleted":0},"store":{"size_in_bytes":11909,"throttle_time_in_millis":0},"indexing":{"index_total":3,"index_time_in_millis":12,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":104857600},"refresh":{"total":3,"total_time_in_millis":42,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":8,"total_time_in_millis":4},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":3,"memory_in_bytes":7764,"terms_memory_in_bytes":5976,"stored_fields_memory_in_bytes":936,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":576,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":276,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":3,"size_in_bytes":494},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}},"foo_1":{"primaries":{"docs":{"count":2,"deleted":0},"store":{"size_in_bytes":8038,"throttle_time_in_millis":0},"indexing":{"index_total":2,"index_time_in_millis":46,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":104857600},"refresh":{"total":2,"total_time_in_millis":84,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":7,"total_time_in_millis":94},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":2,"memory_in_bytes":5176,"terms_memory_in_bytes":3984,"stored_fields_memory_in_bytes":624,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":384,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":184,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":2,"size_in_bytes":389},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":2,"deleted":0},"store":{"size_in_bytes":8038,"throttle_time_in_millis":0},"indexing":{"index_total":2,"index_time_in_millis":46,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":104857600},"refresh":{"total":2,"total_time_in_millis":84,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":7,"total_time_in_millis":94},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":2,"memory_in_bytes":5176,"terms_memory_in_bytes":3984,"stored_fields_memory_in_bytes":624,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":384,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":184,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":2,"size_in_bytes":389},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}},".watches":{"primaries":{"docs":{"count":4,"deleted":0},"store":{"size_in_bytes":35444,"throttle_time_in_millis":0},"indexing":{"index_total":4,"index_time_in_millis":1421,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":20971520},"refresh":{"total":5,"total_time_in_millis":771,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":4,"total_time_in_millis":9},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":4,"memory_in_bytes":18418,"terms_memory_in_bytes":15198,"stored_fields_memory_in_bytes":1248,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":1600,"points_memory_in_bytes":4,"doc_values_memory_in_bytes":368,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":4,"size_in_bytes":11068},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":4,"deleted":0},"store":{"size_in_bytes":35444,"throttle_time_in_millis":0},"indexing":{"index_total":4,"index_time_in_millis":1421,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":20971520},"refresh":{"total":5,"total_time_in_millis":771,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":4,"total_time_in_millis":9},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":4,"memory_in_bytes":18418,"terms_memory_in_bytes":15198,"stored_fields_memory_in_bytes":1248,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":1600,"points_memory_in_bytes":4,"doc_values_memory_in_bytes":368,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":4,"size_in_bytes":11068},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}}}}`, + "5.5.2": `{"_shards":{"total":34,"successful":34,"failed":0},"_all":{"primaries":{"docs":{"count":256658,"deleted":148},"store":{"size_in_bytes":175959144,"throttle_time_in_millis":0},"indexing":{"index_total":413652,"index_time_in_millis":316231,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":88438,"time_in_millis":2799,"exists_total":88438,"exists_time_in_millis":2799,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":88458,"query_time_in_millis":2513,"query_current":0,"fetch_total":88446,"fetch_time_in_millis":2685,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":10275,"total_time_in_millis":5895937,"total_docs":80376844,"total_size_in_bytes":49005089428,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":274,"total_auto_throttle_in_bytes":354609338},"refresh":{"total":100264,"total_time_in_millis":818781,"listeners":0},"flush":{"total":5,"total_time_in_millis":124},"warmer":{"current":0,"total":100286,"total_time_in_millis":23851},"query_cache":{"memory_size_in_bytes":0,"total_count":22,"hit_count":0,"miss_count":22,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":38,"memory_in_bytes":720166,"terms_memory_in_bytes":481188,"stored_fields_memory_in_bytes":31920,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":3072,"points_memory_in_bytes":61858,"doc_values_memory_in_bytes":142128,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":137331,"size_in_bytes":167642357},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":1},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":513316,"deleted":324},"store":{"size_in_bytes":352143858,"throttle_time_in_millis":0},"indexing":{"index_total":827302,"index_time_in_millis":625030,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":176883,"time_in_millis":6265,"exists_total":176883,"exists_time_in_millis":6265,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":176913,"query_time_in_millis":5115,"query_current":0,"fetch_total":176891,"fetch_time_in_millis":5862,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":20547,"total_time_in_millis":11765629,"total_docs":160766745,"total_size_in_bytes":98024993696,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":522,"total_auto_throttle_in_bytes":709218676},"refresh":{"total":200378,"total_time_in_millis":1620051,"listeners":0},"flush":{"total":10,"total_time_in_millis":264},"warmer":{"current":0,"total":200427,"total_time_in_millis":46689},"query_cache":{"memory_size_in_bytes":0,"total_count":54,"hit_count":0,"miss_count":54,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":776,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":86,"memory_in_bytes":1496932,"terms_memory_in_bytes":1008932,"stored_fields_memory_in_bytes":66896,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":6784,"points_memory_in_bytes":124368,"doc_values_memory_in_bytes":289952,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":1502755202429,"file_sizes":{}},"translog":{"operations":274662,"size_in_bytes":335284714},"request_cache":{"memory_size_in_bytes":1794,"evictions":0,"hit_count":0,"miss_count":3},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}},"indices":{".monitoring-kibana-6-2017.08.16":{"primaries":{"docs":{"count":12459,"deleted":0},"store":{"size_in_bytes":4880578,"throttle_time_in_millis":0},"indexing":{"index_total":12459,"index_time_in_millis":20073,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":1372,"total_time_in_millis":275560,"total_docs":8619963,"total_size_in_bytes":3518738895,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":20971520},"refresh":{"total":12724,"total_time_in_millis":79600,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":12725,"total_time_in_millis":2653},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":2,"memory_in_bytes":34337,"terms_memory_in_bytes":30104,"stored_fields_memory_in_bytes":968,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":128,"points_memory_in_bytes":1393,"doc_values_memory_in_bytes":1744,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":12459,"size_in_bytes":12175093},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":24918,"deleted":0},"store":{"size_in_bytes":9682148,"throttle_time_in_millis":0},"indexing":{"index_total":24918,"index_time_in_millis":37852,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":1,"query_time_in_millis":10,"query_current":0,"fetch_total":1,"fetch_time_in_millis":1,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":2743,"total_time_in_millis":549689,"total_docs":17234974,"total_size_in_bytes":7034624631,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":41943040},"refresh":{"total":25390,"total_time_in_millis":155546,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":25392,"total_time_in_millis":5196},"query_cache":{"memory_size_in_bytes":0,"total_count":3,"hit_count":0,"miss_count":3,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":7,"memory_in_bytes":83565,"terms_memory_in_bytes":73488,"stored_fields_memory_in_bytes":2968,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":448,"points_memory_in_bytes":2897,"doc_values_memory_in_bytes":3764,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":24918,"size_in_bytes":24350186},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}},".monitoring-kibana-6-2017.08.15":{"primaries":{"docs":{"count":25917,"deleted":0},"store":{"size_in_bytes":9617967,"throttle_time_in_millis":0},"indexing":{"index_total":25917,"index_time_in_millis":36304,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":1,"query_time_in_millis":10,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":2055,"total_time_in_millis":746516,"total_docs":27797269,"total_size_in_bytes":10672726928,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":20971520},"refresh":{"total":19569,"total_time_in_millis":123083,"listeners":0},"flush":{"total":1,"total_time_in_millis":18},"warmer":{"current":0,"total":19571,"total_time_in_millis":3995},"query_cache":{"memory_size_in_bytes":0,"total_count":3,"hit_count":0,"miss_count":3,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":3,"memory_in_bytes":58226,"terms_memory_in_bytes":52997,"stored_fields_memory_in_bytes":1912,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":192,"points_memory_in_bytes":2721,"doc_values_memory_in_bytes":404,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":0,"size_in_bytes":43},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":51834,"deleted":0},"store":{"size_in_bytes":19309686,"throttle_time_in_millis":0},"indexing":{"index_total":51834,"index_time_in_millis":68258,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":1,"query_time_in_millis":10,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":4109,"total_time_in_millis":1477106,"total_docs":55598675,"total_size_in_bytes":21347372840,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":41943040},"refresh":{"total":39103,"total_time_in_millis":238886,"listeners":0},"flush":{"total":2,"total_time_in_millis":44},"warmer":{"current":0,"total":39108,"total_time_in_millis":7775},"query_cache":{"memory_size_in_bytes":0,"total_count":3,"hit_count":0,"miss_count":3,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":11,"memory_in_bytes":141204,"terms_memory_in_bytes":128132,"stored_fields_memory_in_bytes":5472,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":704,"points_memory_in_bytes":5628,"doc_values_memory_in_bytes":1268,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":1502755202429,"file_sizes":{}},"translog":{"operations":0,"size_in_bytes":86},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}},".monitoring-kibana-6-2017.08.14":{"primaries":{"docs":{"count":6147,"deleted":0},"store":{"size_in_bytes":2467088,"throttle_time_in_millis":0},"indexing":{"index_total":6147,"index_time_in_millis":9228,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":1,"query_time_in_millis":32,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":397,"total_time_in_millis":48142,"total_docs":1231276,"total_size_in_bytes":559560922,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":20971520},"refresh":{"total":3733,"total_time_in_millis":28841,"listeners":0},"flush":{"total":1,"total_time_in_millis":7},"warmer":{"current":0,"total":3735,"total_time_in_millis":1168},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":3,"memory_in_bytes":29099,"terms_memory_in_bytes":25724,"stored_fields_memory_in_bytes":1104,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":192,"points_memory_in_bytes":715,"doc_values_memory_in_bytes":1364,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":0,"size_in_bytes":43},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":12294,"deleted":0},"store":{"size_in_bytes":5029190,"throttle_time_in_millis":0},"indexing":{"index_total":12294,"index_time_in_millis":16557,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":1,"query_time_in_millis":32,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":792,"total_time_in_millis":96523,"total_docs":2467549,"total_size_in_bytes":1121117660,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":41943040},"refresh":{"total":7449,"total_time_in_millis":51544,"listeners":0},"flush":{"total":2,"total_time_in_millis":16},"warmer":{"current":0,"total":7453,"total_time_in_millis":1872},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":11,"memory_in_bytes":83924,"terms_memory_in_bytes":73590,"stored_fields_memory_in_bytes":3768,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":704,"points_memory_in_bytes":1618,"doc_values_memory_in_bytes":4244,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":0,"size_in_bytes":86},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}},".kibana":{"primaries":{"docs":{"count":1,"deleted":0},"store":{"size_in_bytes":3967,"throttle_time_in_millis":0},"indexing":{"index_total":1,"index_time_in_millis":2,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":88438,"time_in_millis":2799,"exists_total":88438,"exists_time_in_millis":2799,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":88441,"query_time_in_millis":2287,"query_current":0,"fetch_total":88441,"fetch_time_in_millis":2685,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":20971520},"refresh":{"total":2,"total_time_in_millis":19,"listeners":0},"flush":{"total":1,"total_time_in_millis":7},"warmer":{"current":0,"total":5,"total_time_in_millis":12},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":1,"memory_in_bytes":2219,"terms_memory_in_bytes":1751,"stored_fields_memory_in_bytes":312,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":64,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":92,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":0,"size_in_bytes":43},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":2,"deleted":0},"store":{"size_in_bytes":7934,"throttle_time_in_millis":0},"indexing":{"index_total":2,"index_time_in_millis":4,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":176883,"time_in_millis":6265,"exists_total":176883,"exists_time_in_millis":6265,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":176879,"query_time_in_millis":4703,"query_current":0,"fetch_total":176879,"fetch_time_in_millis":5860,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":41943040},"refresh":{"total":4,"total_time_in_millis":33,"listeners":0},"flush":{"total":2,"total_time_in_millis":14},"warmer":{"current":0,"total":10,"total_time_in_millis":26},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":2,"memory_in_bytes":4438,"terms_memory_in_bytes":3502,"stored_fields_memory_in_bytes":624,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":128,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":184,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":0,"size_in_bytes":86},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}},".monitoring-es-6-2017.08.15":{"primaries":{"docs":{"count":120970,"deleted":60},"store":{"size_in_bytes":90471073,"throttle_time_in_millis":0},"indexing":{"index_total":207360,"index_time_in_millis":142584,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":1,"query_time_in_millis":85,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":3781,"total_time_in_millis":2188959,"total_docs":19326607,"total_size_in_bytes":15981823826,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":274,"total_auto_throttle_in_bytes":19065018},"refresh":{"total":37364,"total_time_in_millis":337011,"listeners":0},"flush":{"total":1,"total_time_in_millis":67},"warmer":{"current":0,"total":37365,"total_time_in_millis":8943},"query_cache":{"memory_size_in_bytes":0,"total_count":9,"hit_count":0,"miss_count":9,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":9,"memory_in_bytes":297633,"terms_memory_in_bytes":184606,"stored_fields_memory_in_bytes":13736,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":576,"points_memory_in_bytes":31279,"doc_values_memory_in_bytes":67436,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":0,"size_in_bytes":43},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":241940,"deleted":120},"store":{"size_in_bytes":180988889,"throttle_time_in_millis":0},"indexing":{"index_total":414720,"index_time_in_millis":286230,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":2,"query_time_in_millis":135,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":7561,"total_time_in_millis":4425176,"total_docs":38722769,"total_size_in_bytes":32018268991,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":522,"total_auto_throttle_in_bytes":38130036},"refresh":{"total":74706,"total_time_in_millis":675900,"listeners":0},"flush":{"total":2,"total_time_in_millis":148},"warmer":{"current":0,"total":74710,"total_time_in_millis":18069},"query_cache":{"memory_size_in_bytes":0,"total_count":24,"hit_count":0,"miss_count":24,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":424,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":16,"memory_in_bytes":590531,"terms_memory_in_bytes":362031,"stored_fields_memory_in_bytes":26728,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":1024,"points_memory_in_bytes":62748,"doc_values_memory_in_bytes":138000,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":1502755201903,"file_sizes":{}},"translog":{"operations":0,"size_in_bytes":86},"request_cache":{"memory_size_in_bytes":897,"evictions":0,"hit_count":0,"miss_count":1},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}},".monitoring-es-6-2017.08.14":{"primaries":{"docs":{"count":24614,"deleted":12},"store":{"size_in_bytes":17680221,"throttle_time_in_millis":0},"indexing":{"index_total":36896,"index_time_in_millis":35441,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":1,"query_time_in_millis":24,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":927,"total_time_in_millis":1245791,"total_docs":11392506,"total_size_in_bytes":8435013687,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":20971520},"refresh":{"total":9432,"total_time_in_millis":90529,"listeners":0},"flush":{"total":1,"total_time_in_millis":25},"warmer":{"current":0,"total":9434,"total_time_in_millis":3098},"query_cache":{"memory_size_in_bytes":0,"total_count":2,"hit_count":0,"miss_count":2,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":7,"memory_in_bytes":83467,"terms_memory_in_bytes":59601,"stored_fields_memory_in_bytes":4040,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":448,"points_memory_in_bytes":7054,"doc_values_memory_in_bytes":12324,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":0,"size_in_bytes":43},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":49228,"deleted":18},"store":{"size_in_bytes":35172184,"throttle_time_in_millis":0},"indexing":{"index_total":73792,"index_time_in_millis":69939,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":2,"query_time_in_millis":61,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":1853,"total_time_in_millis":2468444,"total_docs":22769560,"total_size_in_bytes":16842830754,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":41943040},"refresh":{"total":18843,"total_time_in_millis":176757,"listeners":0},"flush":{"total":2,"total_time_in_millis":42},"warmer":{"current":0,"total":18847,"total_time_in_millis":5819},"query_cache":{"memory_size_in_bytes":0,"total_count":7,"hit_count":0,"miss_count":7,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":352,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":9,"memory_in_bytes":137869,"terms_memory_in_bytes":97131,"stored_fields_memory_in_bytes":6680,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":576,"points_memory_in_bytes":13662,"doc_values_memory_in_bytes":19820,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":0,"size_in_bytes":86},"request_cache":{"memory_size_in_bytes":897,"evictions":0,"hit_count":0,"miss_count":1},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}},"foo_2":{"primaries":{"docs":{"count":3,"deleted":0},"store":{"size_in_bytes":12138,"throttle_time_in_millis":0},"indexing":{"index_total":3,"index_time_in_millis":4,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":5,"query_time_in_millis":0,"query_current":0,"fetch_total":2,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":104857600},"refresh":{"total":3,"total_time_in_millis":12,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":8,"total_time_in_millis":0},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":3,"memory_in_bytes":7764,"terms_memory_in_bytes":5976,"stored_fields_memory_in_bytes":936,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":576,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":276,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":3,"size_in_bytes":524},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":6,"deleted":0},"store":{"size_in_bytes":24276,"throttle_time_in_millis":0},"indexing":{"index_total":5,"index_time_in_millis":6,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":10,"query_time_in_millis":0,"query_current":0,"fetch_total":5,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":209715200},"refresh":{"total":5,"total_time_in_millis":20,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":16,"total_time_in_millis":0},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":6,"memory_in_bytes":15528,"terms_memory_in_bytes":11952,"stored_fields_memory_in_bytes":1872,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":1152,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":552,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":6,"size_in_bytes":1048},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}},"foo_1":{"primaries":{"docs":{"count":2,"deleted":0},"store":{"size_in_bytes":8246,"throttle_time_in_millis":0},"indexing":{"index_total":4,"index_time_in_millis":7,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":7,"query_time_in_millis":8,"query_current":0,"fetch_total":3,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":104857600},"refresh":{"total":4,"total_time_in_millis":18,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":9,"total_time_in_millis":0},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":2,"memory_in_bytes":5176,"terms_memory_in_bytes":3984,"stored_fields_memory_in_bytes":624,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":384,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":184,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":4,"size_in_bytes":609},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":4,"deleted":0},"store":{"size_in_bytes":16492,"throttle_time_in_millis":0},"indexing":{"index_total":7,"index_time_in_millis":13,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":15,"query_time_in_millis":21,"query_current":0,"fetch_total":5,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":209715200},"refresh":{"total":7,"total_time_in_millis":32,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":18,"total_time_in_millis":0},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":4,"memory_in_bytes":10352,"terms_memory_in_bytes":7968,"stored_fields_memory_in_bytes":1248,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":768,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":368,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":8,"size_in_bytes":1218},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}},".monitoring-es-6-2017.08.16":{"primaries":{"docs":{"count":66545,"deleted":76},"store":{"size_in_bytes":50817866,"throttle_time_in_millis":0},"indexing":{"index_total":124865,"index_time_in_millis":72588,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":1,"query_time_in_millis":67,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":1743,"total_time_in_millis":1390969,"total_docs":12009223,"total_size_in_bytes":9837225170,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":20971520},"refresh":{"total":17433,"total_time_in_millis":159668,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":17434,"total_time_in_millis":3982},"query_cache":{"memory_size_in_bytes":0,"total_count":8,"hit_count":0,"miss_count":8,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":8,"memory_in_bytes":202245,"terms_memory_in_bytes":116445,"stored_fields_memory_in_bytes":8288,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":512,"points_memory_in_bytes":18696,"doc_values_memory_in_bytes":58304,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":124865,"size_in_bytes":155465916},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":1},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"total":{"docs":{"count":133090,"deleted":186},"store":{"size_in_bytes":101913059,"throttle_time_in_millis":0},"indexing":{"index_total":249730,"index_time_in_millis":146171,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":2,"query_time_in_millis":143,"query_current":0,"fetch_total":1,"fetch_time_in_millis":1,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":3489,"total_time_in_millis":2748691,"total_docs":23973218,"total_size_in_bytes":19660778820,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":41943040},"refresh":{"total":34871,"total_time_in_millis":321333,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":34873,"total_time_in_millis":7932},"query_cache":{"memory_size_in_bytes":0,"total_count":17,"hit_count":0,"miss_count":17,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":20,"memory_in_bytes":429521,"terms_memory_in_bytes":251138,"stored_fields_memory_in_bytes":17536,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":1280,"points_memory_in_bytes":37815,"doc_values_memory_in_bytes":121752,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":249730,"size_in_bytes":310931832},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":1},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}}}}}`, } for ver, out := range ti { ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -173,16 +179,13 @@ func TestIndices(t *testing.T) { t.Fatalf("Failed to fetch or decode indices stats: %s", err) } t.Logf("[%s] Index Response: %+v", ver, stats) - if len(stats.Indices) != 9 { - t.Errorf("Wrong number of indices") - } if stats.Indices["foo_1"].Primaries.Docs.Count != 2 { t.Errorf("Wrong number of primary docs") } - if stats.Indices["foo_1"].Primaries.Store.SizeInBytes != 8246 { + if stats.Indices["foo_1"].Primaries.Store.SizeInBytes == 0 { t.Errorf("Wrong number of primary store size in bytes") } - if stats.Indices["foo_1"].Total.Store.SizeInBytes != 16492 { + if stats.Indices["foo_1"].Total.Store.SizeInBytes == 0 { t.Errorf("Wrong number of total store size in bytes") } } From a3c322bf50ed13d01bd57cebfe260c017d5ba1d6 Mon Sep 17 00:00:00 2001 From: Manabu Matsuzaki Date: Sun, 27 Aug 2017 01:18:20 +0900 Subject: [PATCH 07/12] Make indices metrics optional --- collector/indices.go | 20 +++++++++------- collector/metrics_collector.go | 37 ++++++++++++++++------------- collector/metrics_collector_test.go | 2 +- main.go | 3 ++- 4 files changed, 34 insertions(+), 28 deletions(-) diff --git a/collector/indices.go b/collector/indices.go index 1bb7575f..5c7e8d8d 100644 --- a/collector/indices.go +++ b/collector/indices.go @@ -23,10 +23,11 @@ type indexMetric struct { } type Indices struct { - logger log.Logger - client *http.Client - url *url.URL - all bool + logger log.Logger + client *http.Client + url *url.URL + all bool + exportIndices bool up prometheus.Gauge totalScrapes prometheus.Counter @@ -36,12 +37,13 @@ type Indices struct { indexMetrics []*indexMetric } -func NewIndices(logger log.Logger, client *http.Client, url *url.URL, all bool) *Indices { +func NewIndices(logger log.Logger, client *http.Client, url *url.URL, all bool, exportIndices bool) *Indices { return &Indices{ - logger: logger, - client: client, - url: url, - all: all, + logger: logger, + client: client, + url: url, + all: all, + exportIndices: exportIndices, up: prometheus.NewGauge(prometheus.GaugeOpts{ Name: prometheus.BuildFQName(namespace, "index_stats", "up"), diff --git a/collector/metrics_collector.go b/collector/metrics_collector.go index a49823c0..ef1a778c 100644 --- a/collector/metrics_collector.go +++ b/collector/metrics_collector.go @@ -21,11 +21,11 @@ type MetricsCollector struct { indexStatsResponse *indexStatsResponse } -func NewMetricsCollector(logger log.Logger, client *http.Client, url *url.URL, all bool) *MetricsCollector { +func NewMetricsCollector(logger log.Logger, client *http.Client, url *url.URL, all bool, exportIndices bool) *MetricsCollector { return &MetricsCollector{ clusterHealth: NewClusterHealth(logger, client, url), nodes: NewNodes(logger, client, url, all), - indices: NewIndices(logger, client, url, all), + indices: NewIndices(logger, client, url, all, exportIndices), } } @@ -130,22 +130,25 @@ func (c *Nodes) fetchAndDecodeNodeStats() (nodeStatsResponse, error) { func (c *Indices) fetchAndDecodeIndexStats() (indexStatsResponse, error) { var isr indexStatsResponse - u := *c.url - u.Path = "/_all/_stats" - - res, err := c.client.Get(u.String()) - if err != nil { - return isr, fmt.Errorf("failed to get index stats from %s://%s:%s/%s: %s", - u.Scheme, u.Hostname(), u.Port(), u.Path, err) - } - defer res.Body.Close() - if res.StatusCode != http.StatusOK { - return isr, fmt.Errorf("HTTP Request failed with code %d", res.StatusCode) + if c.exportIndices { + u := *c.url + u.Path = "/_all/_stats" + + res, err := c.client.Get(u.String()) + if err != nil { + return isr, fmt.Errorf("failed to get index stats from %s://%s:%s/%s: %s", + u.Scheme, u.Hostname(), u.Port(), u.Path, err) + } + defer res.Body.Close() + if res.StatusCode != http.StatusOK { + return isr, fmt.Errorf("HTTP Request failed with code %d", res.StatusCode) + } + + if err := json.NewDecoder(res.Body).Decode(&isr); err != nil { + c.jsonParseFailures.Inc() + return isr, err + } } - if err := json.NewDecoder(res.Body).Decode(&isr); err != nil { - c.jsonParseFailures.Inc() - return isr, err - } return isr, nil } diff --git a/collector/metrics_collector_test.go b/collector/metrics_collector_test.go index f338246f..c44b803b 100644 --- a/collector/metrics_collector_test.go +++ b/collector/metrics_collector_test.go @@ -173,7 +173,7 @@ func TestIndices(t *testing.T) { if err != nil { t.Fatalf("Failed to parse URL: %s", err) } - i := NewIndices(log.NewNopLogger(), http.DefaultClient, u, true) + i := NewIndices(log.NewNopLogger(), http.DefaultClient, u, true, true) stats, err := i.fetchAndDecodeIndexStats() if err != nil { t.Fatalf("Failed to fetch or decode indices stats: %s", err) diff --git a/main.go b/main.go index ec93f15b..d4fa2aac 100644 --- a/main.go +++ b/main.go @@ -22,6 +22,7 @@ func main() { esURI = flag.String("es.uri", "http://localhost:9200", "HTTP API address of an Elasticsearch node.") esTimeout = flag.Duration("es.timeout", 5*time.Second, "Timeout for trying to get stats from Elasticsearch.") esAllNodes = flag.Bool("es.all", false, "Export stats for all nodes in the cluster.") + esExportIndices = flag.Bool("es.indices", false, "Export stats for indices in the cluster.") esCA = flag.String("es.ca", "", "Path to PEM file that conains trusted CAs for the Elasticsearch connection.") esClientPrivateKey = flag.String("es.client-private-key", "", "Path to PEM file that conains the private key for client auth when connecting to Elasticsearch.") esClientCert = flag.String("es.client-cert", "", "Path to PEM file that conains the corresponding cert for the private key to connect to Elasticsearch.") @@ -53,7 +54,7 @@ func main() { }, } - prometheus.MustRegister(collector.NewMetricsCollector(logger, httpClient, esURL, *esAllNodes)) + prometheus.MustRegister(collector.NewMetricsCollector(logger, httpClient, esURL, *esAllNodes, *esExportIndices)) http.Handle(*metricsPath, prometheus.Handler()) http.HandleFunc("/", IndexHandler(*metricsPath)) From 7af4dc8cf34727b59a41677c53758530e1c23919 Mon Sep 17 00:00:00 2001 From: Manabu Matsuzaki Date: Tue, 29 Aug 2017 01:25:34 +0900 Subject: [PATCH 08/12] revert --- collector/cluster_health.go | 40 ++++- collector/cluster_health_test.go | 55 +++++++ collector/indices.go | 65 +++++++- ...rics_collector_test.go => indices_test.go} | 138 ---------------- collector/metrics_collector.go | 154 ------------------ collector/nodes.go | 42 ++++- collector/nodes_test.go | 102 ++++++++++++ main.go | 4 +- 8 files changed, 303 insertions(+), 297 deletions(-) create mode 100644 collector/cluster_health_test.go rename collector/{metrics_collector_test.go => indices_test.go} (74%) delete mode 100644 collector/metrics_collector.go create mode 100644 collector/nodes_test.go diff --git a/collector/cluster_health.go b/collector/cluster_health.go index d1af4631..dc301afd 100644 --- a/collector/cluster_health.go +++ b/collector/cluster_health.go @@ -1,10 +1,13 @@ package collector import ( + "encoding/json" + "fmt" "net/http" "net/url" "github.com/go-kit/kit/log" + "github.com/go-kit/kit/log/level" "github.com/prometheus/client_golang/prometheus" ) @@ -217,7 +220,31 @@ func (c *ClusterHealth) Describe(ch chan<- *prometheus.Desc) { ch <- c.jsonParseFailures.Desc() } -func (c *ClusterHealth) Collect(ch chan<- prometheus.Metric, clusterHealthResponse clusterHealthResponse) { +func (c *ClusterHealth) fetchAndDecodeClusterHealth() (clusterHealthResponse, error) { + var chr clusterHealthResponse + + u := *c.url + u.Path = "/_cluster/health" + res, err := c.client.Get(u.String()) + if err != nil { + return chr, fmt.Errorf("failed to get cluster health from %s://%s:%s/%s: %s", + u.Scheme, u.Hostname(), u.Port(), u.Path, err) + } + defer res.Body.Close() + + if res.StatusCode != http.StatusOK { + return chr, fmt.Errorf("HTTP Request failed with code %d", res.StatusCode) + } + + if err := json.NewDecoder(res.Body).Decode(&chr); err != nil { + c.jsonParseFailures.Inc() + return chr, err + } + + return chr, nil +} + +func (c *ClusterHealth) Collect(ch chan<- prometheus.Metric) { c.totalScrapes.Inc() defer func() { ch <- c.up @@ -225,6 +252,17 @@ func (c *ClusterHealth) Collect(ch chan<- prometheus.Metric, clusterHealthRespon ch <- c.jsonParseFailures }() + clusterHealthResponse, err := c.fetchAndDecodeClusterHealth() + if err != nil { + c.up.Set(0) + level.Warn(c.logger).Log( + "msg", "failed to fetch and decode cluster health", + "err", err, + ) + return + } + c.up.Set(1) + for _, metric := range c.metrics { ch <- prometheus.MustNewConstMetric( metric.Desc, diff --git a/collector/cluster_health_test.go b/collector/cluster_health_test.go new file mode 100644 index 00000000..03b153cf --- /dev/null +++ b/collector/cluster_health_test.go @@ -0,0 +1,55 @@ +package collector + +import ( + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "testing" + + "github.com/go-kit/kit/log" +) + +func TestClusterHealth(t *testing.T) { + // Testcases created using: + // docker run -d -p 9200:9200 elasticsearch:VERSION-alpine + // curl -XPUT http://localhost:9200/twitter + // curl http://localhost:9200/_cluster/health + tcs := map[string]string{ + "1.7.6": `{"cluster_name":"elasticsearch","status":"yellow","timed_out":false,"number_of_nodes":1,"number_of_data_nodes":1,"active_primary_shards":5,"active_shards":5,"relocating_shards":0,"initializing_shards":0,"unassigned_shards":5,"delayed_unassigned_shards":0,"number_of_pending_tasks":0,"number_of_in_flight_fetch":0}`, + "2.4.5": `{"cluster_name":"elasticsearch","status":"yellow","timed_out":false,"number_of_nodes":1,"number_of_data_nodes":1,"active_primary_shards":5,"active_shards":5,"relocating_shards":0,"initializing_shards":0,"unassigned_shards":5,"delayed_unassigned_shards":0,"number_of_pending_tasks":0,"number_of_in_flight_fetch":0,"task_max_waiting_in_queue_millis":0,"active_shards_percent_as_number":50.0}`, + "5.4.2": `{"cluster_name":"elasticsearch","status":"yellow","timed_out":false,"number_of_nodes":1,"number_of_data_nodes":1,"active_primary_shards":5,"active_shards":5,"relocating_shards":0,"initializing_shards":0,"unassigned_shards":5,"delayed_unassigned_shards":0,"number_of_pending_tasks":0,"number_of_in_flight_fetch":0,"task_max_waiting_in_queue_millis":0,"active_shards_percent_as_number":50.0}`, + } + for ver, out := range tcs { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, out) + })) + defer ts.Close() + + u, err := url.Parse(ts.URL) + if err != nil { + t.Fatalf("Failed to parse URL: %s", err) + } + c := NewClusterHealth(log.NewNopLogger(), http.DefaultClient, u) + chr, err := c.fetchAndDecodeClusterHealth() + if err != nil { + t.Fatalf("Failed to fetch or decode cluster health: %s", err) + } + t.Logf("[%s] Cluster Health Response: %+v", ver, chr) + if chr.ClusterName != "elasticsearch" { + t.Errorf("Invalid cluster health response") + } + if chr.Status != "yellow" { + t.Errorf("Invalid cluster status") + } + if chr.TimedOut { + t.Errorf("Check didn't time out") + } + if chr.NumberOfNodes != 1 { + t.Errorf("Wrong number of nodes") + } + if chr.NumberOfDataNodes != 1 { + t.Errorf("Wrong number of data nodes") + } + } +} diff --git a/collector/indices.go b/collector/indices.go index 5c7e8d8d..4bfc9aff 100644 --- a/collector/indices.go +++ b/collector/indices.go @@ -1,10 +1,13 @@ package collector import ( + "encoding/json" + "fmt" "net/http" "net/url" "github.com/go-kit/kit/log" + "github.com/go-kit/kit/log/level" "github.com/prometheus/client_golang/prometheus" ) @@ -566,8 +569,30 @@ func (i *Indices) Describe(ch chan<- *prometheus.Desc) { ch <- i.jsonParseFailures.Desc() } -func (i *Indices) Collect(ch chan<- prometheus.Metric, clusterHealthResponse clusterHealthResponse, - nodeStatsResponse nodeStatsResponse, indexStatsResponse indexStatsResponse) { +func (c *Indices) fetchAndDecodeIndexStats() (indexStatsResponse, error) { + var isr indexStatsResponse + + u := *c.url + u.Path = "/_all/_stats" + + res, err := c.client.Get(u.String()) + if err != nil { + return isr, fmt.Errorf("failed to get index stats from %s://%s:%s/%s: %s", + u.Scheme, u.Hostname(), u.Port(), u.Path, err) + } + defer res.Body.Close() + if res.StatusCode != http.StatusOK { + return isr, fmt.Errorf("HTTP Request failed with code %d", res.StatusCode) + } + + if err := json.NewDecoder(res.Body).Decode(&isr); err != nil { + c.jsonParseFailures.Inc() + return isr, err + } + return isr, nil +} + +func (i *Indices) Collect(ch chan<- prometheus.Metric) { i.totalScrapes.Inc() defer func() { ch <- i.up @@ -575,6 +600,42 @@ func (i *Indices) Collect(ch chan<- prometheus.Metric, clusterHealthResponse clu ch <- i.jsonParseFailures }() + // clusterHealth + clusterHealth := NewClusterHealth(i.logger, i.client, i.url) + clusterHealthResponse, err := clusterHealth.fetchAndDecodeClusterHealth() + if err != nil { + i.up.Set(0) + level.Warn(i.logger).Log( + "msg", "failed to fetch and decode cluster health", + "err", err, + ) + return + } + + // nodes + nodes := NewNodes(i.logger, i.client, i.url, i.all) + nodeStatsResponse, err := nodes.fetchAndDecodeNodeStats() + if err != nil { + i.up.Set(0) + level.Warn(i.logger).Log( + "msg", "failed to fetch and decode node stats", + "err", err, + ) + return + } + + // indices + indexStatsResponse, err := i.fetchAndDecodeIndexStats() + if err != nil { + i.up.Set(0) + level.Warn(i.logger).Log( + "msg", "failed to fetch and decode index stats", + "err", err, + ) + return + } + i.up.Set(1) + // Node stats for _, node := range nodeStatsResponse.Nodes { for _, metric := range i.nodeMetrics { diff --git a/collector/metrics_collector_test.go b/collector/indices_test.go similarity index 74% rename from collector/metrics_collector_test.go rename to collector/indices_test.go index c44b803b..36a81d24 100644 --- a/collector/metrics_collector_test.go +++ b/collector/indices_test.go @@ -1,153 +1,15 @@ package collector import ( - "encoding/base64" "fmt" "net/http" "net/http/httptest" "net/url" - "strings" "testing" "github.com/go-kit/kit/log" ) -type basicAuth struct { - User string - Pass string - Next http.Handler -} - -func (h *basicAuth) checkAuth(w http.ResponseWriter, r *http.Request) bool { - s := strings.SplitN(r.Header.Get("Authorization"), " ", 2) - if len(s) != 2 { - return false - } - - b, err := base64.StdEncoding.DecodeString(s[1]) - if err != nil { - return false - } - - pair := strings.SplitN(string(b), ":", 2) - if len(pair) != 2 { - return false - } - - if h.User == pair[0] && h.Pass == pair[1] { - return true - } - return false -} - -func (h *basicAuth) ServeHTTP(w http.ResponseWriter, r *http.Request) { - if !h.checkAuth(w, r) { - w.Header().Set("WWW-Authenticate", "Basic realm=\"ES\"") - w.WriteHeader(401) - w.Write([]byte("401 Unauthorized\n")) - return - } - - h.Next.ServeHTTP(w, r) - return -} - -func TestClusterHealth(t *testing.T) { - // Testcases created using: - // docker run -d -p 9200:9200 elasticsearch:VERSION-alpine - // curl -XPUT http://localhost:9200/twitter - // curl http://localhost:9200/_cluster/health - tcs := map[string]string{ - "1.7.6": `{"cluster_name":"elasticsearch","status":"yellow","timed_out":false,"number_of_nodes":1,"number_of_data_nodes":1,"active_primary_shards":5,"active_shards":5,"relocating_shards":0,"initializing_shards":0,"unassigned_shards":5,"delayed_unassigned_shards":0,"number_of_pending_tasks":0,"number_of_in_flight_fetch":0}`, - "2.4.5": `{"cluster_name":"elasticsearch","status":"yellow","timed_out":false,"number_of_nodes":1,"number_of_data_nodes":1,"active_primary_shards":5,"active_shards":5,"relocating_shards":0,"initializing_shards":0,"unassigned_shards":5,"delayed_unassigned_shards":0,"number_of_pending_tasks":0,"number_of_in_flight_fetch":0,"task_max_waiting_in_queue_millis":0,"active_shards_percent_as_number":50.0}`, - "5.4.2": `{"cluster_name":"elasticsearch","status":"yellow","timed_out":false,"number_of_nodes":1,"number_of_data_nodes":1,"active_primary_shards":5,"active_shards":5,"relocating_shards":0,"initializing_shards":0,"unassigned_shards":5,"delayed_unassigned_shards":0,"number_of_pending_tasks":0,"number_of_in_flight_fetch":0,"task_max_waiting_in_queue_millis":0,"active_shards_percent_as_number":50.0}`, - "5.5.2": `{"cluster_name":"docker-cluster","status":"yellow","timed_out":false,"number_of_nodes":1,"number_of_data_nodes":1,"active_primary_shards":7,"active_shards":7,"relocating_shards":0,"initializing_shards":0,"unassigned_shards":7,"delayed_unassigned_shards":0,"number_of_pending_tasks":0,"number_of_in_flight_fetch":0,"task_max_waiting_in_queue_millis":0,"active_shards_percent_as_number":50.0}`, - } - for ver, out := range tcs { - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintln(w, out) - })) - defer ts.Close() - - u, err := url.Parse(ts.URL) - if err != nil { - t.Fatalf("Failed to parse URL: %s", err) - } - c := NewClusterHealth(log.NewNopLogger(), http.DefaultClient, u) - chr, err := c.fetchAndDecodeClusterHealth() - if err != nil { - t.Fatalf("Failed to fetch or decode cluster health: %s", err) - } - t.Logf("[%s] Cluster Health Response: %+v", ver, chr) - if ver != "5.5.2" && chr.ClusterName == "elasticsearch" || - ver == "5.5.2" && chr.ClusterName == "docker-cluster" { - // ok - } else { - t.Errorf("Invalid cluster health response") - } - if chr.Status != "yellow" { - t.Errorf("Invalid cluster status") - } - if chr.TimedOut { - t.Errorf("Check didn't time out") - } - if chr.NumberOfNodes != 1 { - t.Errorf("Wrong number of nodes") - } - if chr.NumberOfDataNodes != 1 { - t.Errorf("Wrong number of data nodes") - } - } -} - -func TestNodesStats(t *testing.T) { - // Testcases created using: - // docker run -d -p 9200:9200 elasticsearch:VERSION-alpine - // curl -XPUT http://localhost:9200/twitter - // curl http://localhost:9200/_nodes/stats - tcs := map[string]string{ - "1.7.6": `{"cluster_name":"elasticsearch","nodes":{"vKnRv2bPThCJIDTj92NvQg":{"timestamp":1498820641883,"name":"Overmind","transport_address":"inet[/172.17.0.4:9300]","host":"2603876b583f","ip":["inet[/172.17.0.4:9300]","NONE"],"indices":{"docs":{"count":0,"deleted":0},"store":{"size_in_bytes":575,"throttle_time_in_millis":0},"indexing":{"index_total":0,"index_time_in_millis":0,"index_current":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0},"refresh":{"total":0,"total_time_in_millis":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":10,"total_time_in_millis":52},"filter_cache":{"memory_size_in_bytes":0,"evictions":0},"id_cache":{"memory_size_in_bytes":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"percolate":{"total":0,"time_in_millis":0,"current":0,"memory_size_in_bytes":-1,"memory_size":"-1b","queries":0},"completion":{"size_in_bytes":0},"segments":{"count":0,"memory_in_bytes":0,"index_writer_memory_in_bytes":0,"index_writer_max_memory_in_bytes":335544320,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0},"translog":{"operations":0,"size_in_bytes":17},"suggest":{"total":0,"time_in_millis":0,"current":0},"query_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"os":{"timestamp":1498820641896},"process":{"timestamp":1498820641896,"open_file_descriptors":123},"jvm":{"timestamp":1498820641896,"uptime_in_millis":17871,"mem":{"heap_used_in_bytes":82958528,"heap_used_percent":7,"heap_committed_in_bytes":251002880,"heap_max_in_bytes":1056309248,"non_heap_used_in_bytes":44442536,"non_heap_committed_in_bytes":45400064,"pools":{"young":{"used_in_bytes":62150064,"max_in_bytes":139591680,"peak_used_in_bytes":139591680,"peak_max_in_bytes":139591680},"survivor":{"used_in_bytes":17432576,"max_in_bytes":17432576,"peak_used_in_bytes":17432576,"peak_max_in_bytes":17432576},"old":{"used_in_bytes":3375888,"max_in_bytes":899284992,"peak_used_in_bytes":10899920,"peak_max_in_bytes":899284992}}},"threads":{"count":36,"peak_count":36},"gc":{"collectors":{"young":{"collection_count":1,"collection_time_in_millis":40},"old":{"collection_count":1,"collection_time_in_millis":42}}},"buffer_pools":{"direct":{"count":27,"used_in_bytes":4981411,"total_capacity_in_bytes":4981411},"mapped":{"count":0,"used_in_bytes":0,"total_capacity_in_bytes":0}}},"thread_pool":{"percolate":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"fetch_shard_started":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"listener":{"threads":1,"queue":0,"active":0,"rejected":0,"largest":1,"completed":1},"index":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"refresh":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"suggest":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"generic":{"threads":4,"queue":0,"active":0,"rejected":0,"largest":4,"completed":14},"warmer":{"threads":1,"queue":0,"active":0,"rejected":0,"largest":1,"completed":5},"search":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"flush":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"optimize":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"fetch_shard_store":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"management":{"threads":1,"queue":0,"active":1,"rejected":0,"largest":1,"completed":1},"get":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"merge":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"bulk":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"snapshot":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0}},"network":{},"fs":{"timestamp":1498820641898,"total":{"total_in_bytes":63375708160,"free_in_bytes":59667320832,"available_in_bytes":56424390656},"data":[{"path":"/usr/share/elasticsearch/data/elasticsearch/nodes/0","mount":"/usr/share/elasticsearch/data (/dev/sda2)","type":"ext4","total_in_bytes":63375708160,"free_in_bytes":59667320832,"available_in_bytes":56424390656}]},"transport":{"server_open":13,"rx_count":6,"rx_size_in_bytes":1428,"tx_count":6,"tx_size_in_bytes":1428},"http":{"current_open":1,"total_opened":2},"breakers":{"request":{"limit_size_in_bytes":422523699,"limit_size":"402.9mb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.0,"tripped":0},"fielddata":{"limit_size_in_bytes":633785548,"limit_size":"604.4mb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.03,"tripped":0},"parent":{"limit_size_in_bytes":739416473,"limit_size":"705.1mb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.0,"tripped":0}}}}}`, - "2.4.5": `{"cluster_name":"elasticsearch","nodes":{"VnnrklnAQ7KRXhW2WkgPxA":{"timestamp":1498820602662,"name":"Battering Ram","transport_address":"172.17.0.4:9300","host":"172.17.0.4","ip":["172.17.0.4:9300","NONE"],"indices":{"docs":{"count":0,"deleted":0},"store":{"size_in_bytes":650,"throttle_time_in_millis":0},"indexing":{"index_total":0,"index_time_in_millis":0,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":104857600},"refresh":{"total":0,"total_time_in_millis":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":10,"total_time_in_millis":60},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"percolate":{"total":0,"time_in_millis":0,"current":0,"memory_size_in_bytes":-1,"memory_size":"-1b","queries":0},"completion":{"size_in_bytes":0},"segments":{"count":0,"memory_in_bytes":0,"terms_memory_in_bytes":0,"stored_fields_memory_in_bytes":0,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":0,"doc_values_memory_in_bytes":0,"index_writer_memory_in_bytes":0,"index_writer_max_memory_in_bytes":105630920,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0},"translog":{"operations":0,"size_in_bytes":215},"suggest":{"total":0,"time_in_millis":0,"current":0},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"os":{"timestamp":1498820602670,"cpu_percent":7,"load_average":0.32861328125,"mem":{"total_in_bytes":2096177152,"free_in_bytes":1624637440,"used_in_bytes":471539712,"free_percent":78,"used_percent":22},"swap":{"total_in_bytes":4195348480,"free_in_bytes":4005060608,"used_in_bytes":190287872}},"process":{"timestamp":1498820602670,"open_file_descriptors":116,"max_file_descriptors":1048576,"cpu":{"percent":7,"total_in_millis":10270},"mem":{"total_virtual_in_bytes":2681352192}},"jvm":{"timestamp":1498820602671,"uptime_in_millis":38833,"mem":{"heap_used_in_bytes":88986784,"heap_used_percent":8,"heap_committed_in_bytes":251002880,"heap_max_in_bytes":1056309248,"non_heap_used_in_bytes":49185912,"non_heap_committed_in_bytes":49987584,"pools":{"young":{"used_in_bytes":58281984,"max_in_bytes":139591680,"peak_used_in_bytes":139591680,"peak_max_in_bytes":139591680},"survivor":{"used_in_bytes":17432576,"max_in_bytes":17432576,"peak_used_in_bytes":17432576,"peak_max_in_bytes":17432576},"old":{"used_in_bytes":13272224,"max_in_bytes":899284992,"peak_used_in_bytes":13272224,"peak_max_in_bytes":899284992}}},"threads":{"count":35,"peak_count":38},"gc":{"collectors":{"young":{"collection_count":2,"collection_time_in_millis":87},"old":{"collection_count":1,"collection_time_in_millis":74}}},"buffer_pools":{"direct":{"count":23,"used_in_bytes":3416836,"total_capacity_in_bytes":3416836},"mapped":{"count":0,"used_in_bytes":0,"total_capacity_in_bytes":0}},"classes":{"current_loaded_count":7221,"total_loaded_count":7221,"total_unloaded_count":0}},"thread_pool":{"bulk":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"fetch_shard_started":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"fetch_shard_store":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"flush":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"force_merge":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"generic":{"threads":1,"queue":0,"active":0,"rejected":0,"largest":5,"completed":28},"get":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"index":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"listener":{"threads":1,"queue":0,"active":0,"rejected":0,"largest":1,"completed":2},"management":{"threads":2,"queue":0,"active":1,"rejected":0,"largest":2,"completed":5},"percolate":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"refresh":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"search":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"snapshot":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"suggest":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"warmer":{"threads":1,"queue":0,"active":0,"rejected":0,"largest":1,"completed":5}},"fs":{"timestamp":1498820602672,"total":{"total_in_bytes":63375708160,"free_in_bytes":59667599360,"available_in_bytes":56424669184,"spins":"true"},"data":[{"path":"/usr/share/elasticsearch/data/elasticsearch/nodes/0","mount":"/usr/share/elasticsearch/data (/dev/sda2)","type":"ext4","total_in_bytes":63375708160,"free_in_bytes":59667599360,"available_in_bytes":56424669184,"spins":"true"}]},"transport":{"server_open":0,"rx_count":6,"rx_size_in_bytes":2028,"tx_count":6,"tx_size_in_bytes":2028},"http":{"current_open":1,"total_opened":3},"breakers":{"request":{"limit_size_in_bytes":422523699,"limit_size":"402.9mb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.0,"tripped":0},"fielddata":{"limit_size_in_bytes":633785548,"limit_size":"604.4mb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.03,"tripped":0},"in_flight_requests":{"limit_size_in_bytes":1056309248,"limit_size":"1007.3mb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.0,"tripped":0},"parent":{"limit_size_in_bytes":739416473,"limit_size":"705.1mb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.0,"tripped":0}},"script":{"compilations":0,"cache_evictions":0}}}}`, - "5.4.2": `{"_nodes":{"total":1,"successful":1,"failed":0},"cluster_name":"elasticsearch","nodes":{"0hHcEFK1S7qMlk8hQCm7wQ":{"timestamp":1498820489394,"name":"0hHcEFK","transport_address":"127.0.0.1:9300","host":"127.0.0.1","ip":"127.0.0.1:9300","roles":["master","data","ingest"],"indices":{"docs":{"count":0,"deleted":0},"store":{"size_in_bytes":650,"throttle_time_in_millis":0},"indexing":{"index_total":0,"index_time_in_millis":0,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":104857600},"refresh":{"total":0,"total_time_in_millis":0,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":5,"total_time_in_millis":27},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":0,"memory_in_bytes":0,"terms_memory_in_bytes":0,"stored_fields_memory_in_bytes":0,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":0,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":0,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":0,"size_in_bytes":215},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"os":{"timestamp":1498820489400,"cpu":{"percent":0,"load_average":{"1m":0.35,"5m":0.28,"15m":0.12}},"mem":{"total_in_bytes":2096177152,"free_in_bytes":83501056,"used_in_bytes":2012676096,"free_percent":4,"used_percent":96},"swap":{"total_in_bytes":4195348480,"free_in_bytes":3487707136,"used_in_bytes":707641344}},"process":{"timestamp":1498820489400,"open_file_descriptors":139,"max_file_descriptors":1048576,"cpu":{"percent":0,"total_in_millis":26600},"mem":{"total_virtual_in_bytes":3823181824}},"jvm":{"timestamp":1498820489400,"uptime_in_millis":185693,"mem":{"heap_used_in_bytes":114959064,"heap_used_percent":5,"heap_committed_in_bytes":2130051072,"heap_max_in_bytes":2130051072,"non_heap_used_in_bytes":65471864,"non_heap_committed_in_bytes":69906432,"pools":{"young":{"used_in_bytes":82057312,"max_in_bytes":139591680,"peak_used_in_bytes":139591680,"peak_max_in_bytes":139591680},"survivor":{"used_in_bytes":17432576,"max_in_bytes":17432576,"peak_used_in_bytes":17432576,"peak_max_in_bytes":17432576},"old":{"used_in_bytes":15469176,"max_in_bytes":1973026816,"peak_used_in_bytes":15469176,"peak_max_in_bytes":1973026816}}},"threads":{"count":26,"peak_count":29},"gc":{"collectors":{"young":{"collection_count":4,"collection_time_in_millis":1618},"old":{"collection_count":1,"collection_time_in_millis":76}}},"buffer_pools":{"direct":{"count":16,"used_in_bytes":33776600,"total_capacity_in_bytes":33776599},"mapped":{"count":0,"used_in_bytes":0,"total_capacity_in_bytes":0}},"classes":{"current_loaded_count":9995,"total_loaded_count":9995,"total_unloaded_count":0}},"thread_pool":{"bulk":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"fetch_shard_started":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"fetch_shard_store":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"flush":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"force_merge":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"generic":{"threads":4,"queue":0,"active":0,"rejected":0,"largest":4,"completed":28},"get":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"index":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"listener":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"management":{"threads":2,"queue":0,"active":1,"rejected":0,"largest":2,"completed":20},"refresh":{"threads":1,"queue":0,"active":0,"rejected":0,"largest":1,"completed":31},"search":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"snapshot":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"warmer":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0}},"fs":{"timestamp":1498820489401,"total":{"total_in_bytes":63375708160,"free_in_bytes":59668226048,"available_in_bytes":56425295872,"spins":"true"},"data":[{"path":"/usr/share/elasticsearch/data/nodes/0","mount":"/usr/share/elasticsearch/data (/dev/sda2)","type":"ext4","total_in_bytes":63375708160,"free_in_bytes":59668226048,"available_in_bytes":56425295872,"spins":"true"}],"io_stats":{"devices":[{"device_name":"sda2","operations":3017,"read_operations":1800,"write_operations":1217,"read_kilobytes":24816,"write_kilobytes":6580}],"total":{"operations":3017,"read_operations":1800,"write_operations":1217,"read_kilobytes":24816,"write_kilobytes":6580}}},"transport":{"server_open":0,"rx_count":0,"rx_size_in_bytes":0,"tx_count":0,"tx_size_in_bytes":0},"http":{"current_open":2,"total_opened":3},"breakers":{"request":{"limit_size_in_bytes":1278030643,"limit_size":"1.1gb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.0,"tripped":0},"fielddata":{"limit_size_in_bytes":1278030643,"limit_size":"1.1gb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.03,"tripped":0},"in_flight_requests":{"limit_size_in_bytes":2130051072,"limit_size":"1.9gb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.0,"tripped":0},"parent":{"limit_size_in_bytes":1491035750,"limit_size":"1.3gb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.0,"tripped":0}},"script":{"compilations":0,"cache_evictions":0},"discovery":{"cluster_state_queue":{"total":0,"pending":0,"committed":0}},"ingest":{"total":{"count":0,"time_in_millis":0,"current":0,"failed":0},"pipelines":{}}}}}`, - "5.5.2": `{"_nodes":{"total":1,"successful":1,"failed":0},"cluster_name":"docker-cluster","nodes":{"8At5us-RRw6UofMRuJgFTw":{"timestamp":1503497399210,"name":"8At5us-","transport_address":"172.17.0.2:9300","host":"172.17.0.2","ip":"172.17.0.2:9300","roles":["master","data","ingest"],"attributes":{"ml.max_open_jobs":"10","ml.enabled":"true"},"indices":{"docs":{"count":19,"deleted":0},"store":{"size_in_bytes":91423,"throttle_time_in_millis":0},"indexing":{"index_total":19,"index_time_in_millis":357,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":146800640},"refresh":{"total":5,"total_time_in_millis":519,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":10,"total_time_in_millis":41},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":5,"memory_in_bytes":25435,"terms_memory_in_bytes":21179,"stored_fields_memory_in_bytes":1560,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":1600,"points_memory_in_bytes":292,"doc_values_memory_in_bytes":804,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":19,"size_in_bytes":23842},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"os":{"timestamp":1503497399218,"cpu":{"percent":20,"load_average":{"1m":1.19,"5m":0.43,"15m":0.27}},"mem":{"total_in_bytes":2095878144,"free_in_bytes":70742016,"used_in_bytes":2025136128,"free_percent":3,"used_percent":97},"swap":{"total_in_bytes":1073737728,"free_in_bytes":302432256,"used_in_bytes":771305472},"cgroup":{"cpuacct":{"control_group":"/","usage_nanos":33132630208},"cpu":{"control_group":"/","cfs_period_micros":100000,"cfs_quota_micros":-1,"stat":{"number_of_elapsed_periods":0,"number_of_times_throttled":0,"time_throttled_nanos":0}}}},"process":{"timestamp":1503497399218,"open_file_descriptors":221,"max_file_descriptors":1048576,"cpu":{"percent":21,"total_in_millis":32980},"mem":{"total_virtual_in_bytes":5950177280}},"jvm":{"timestamp":1503497399219,"uptime_in_millis":29213,"mem":{"heap_used_in_bytes":293256600,"heap_used_percent":13,"heap_committed_in_bytes":2112618496,"heap_max_in_bytes":2112618496,"non_heap_used_in_bytes":88549288,"non_heap_committed_in_bytes":94851072,"pools":{"young":{"used_in_bytes":210201424,"max_in_bytes":279183360,"peak_used_in_bytes":279183360,"peak_max_in_bytes":279183360},"survivor":{"used_in_bytes":34865152,"max_in_bytes":34865152,"peak_used_in_bytes":34865152,"peak_max_in_bytes":34865152},"old":{"used_in_bytes":48190024,"max_in_bytes":1798569984,"peak_used_in_bytes":48241264,"peak_max_in_bytes":1798569984}}},"threads":{"count":52,"peak_count":52},"gc":{"collectors":{"young":{"collection_count":3,"collection_time_in_millis":1258},"old":{"collection_count":2,"collection_time_in_millis":160}}},"buffer_pools":{"direct":{"count":24,"used_in_bytes":50664460,"total_capacity_in_bytes":50664459},"mapped":{"count":5,"used_in_bytes":86919,"total_capacity_in_bytes":86919}},"classes":{"current_loaded_count":13157,"total_loaded_count":13157,"total_unloaded_count":0}},"thread_pool":{"bulk":{"threads":4,"queue":0,"active":0,"rejected":0,"largest":4,"completed":6},"fetch_shard_started":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"fetch_shard_store":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"flush":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"force_merge":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"generic":{"threads":7,"queue":0,"active":0,"rejected":0,"largest":7,"completed":38},"get":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"index":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"listener":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"management":{"threads":4,"queue":0,"active":1,"rejected":0,"largest":4,"completed":15},"ml_autodetect":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"ml_datafeed":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"ml_utility":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"refresh":{"threads":2,"queue":0,"active":0,"rejected":0,"largest":2,"completed":14},"search":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"snapshot":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"warmer":{"threads":1,"queue":0,"active":0,"rejected":0,"largest":1,"completed":5},"watcher":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0}},"fs":{"timestamp":1503497399219,"total":{"total_in_bytes":67371577344,"free_in_bytes":64008867840,"available_in_bytes":60556169216,"spins":"true"},"data":[{"path":"/usr/share/elasticsearch/data/nodes/0","mount":"/ (overlay)","type":"overlay","total_in_bytes":67371577344,"free_in_bytes":64008867840,"available_in_bytes":60556169216,"spins":"true"}],"io_stats":{}},"transport":{"server_open":0,"rx_count":10,"rx_size_in_bytes":3710,"tx_count":10,"tx_size_in_bytes":3710},"http":{"current_open":1,"total_opened":2},"breakers":{"request":{"limit_size_in_bytes":1267571097,"limit_size":"1.1gb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.0,"tripped":0},"fielddata":{"limit_size_in_bytes":1267571097,"limit_size":"1.1gb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.03,"tripped":0},"in_flight_requests":{"limit_size_in_bytes":2112618496,"limit_size":"1.9gb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.0,"tripped":0},"parent":{"limit_size_in_bytes":1478832947,"limit_size":"1.3gb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.0,"tripped":0}},"script":{"compilations":9,"cache_evictions":0},"discovery":{"cluster_state_queue":{"total":0,"pending":0,"committed":0}},"ingest":{"total":{"count":0,"time_in_millis":0,"current":0,"failed":0},"pipelines":{"xpack_monitoring_6":{"count":0,"time_in_millis":0,"current":0,"failed":0},"xpack_monitoring_2":{"count":0,"time_in_millis":0,"current":0,"failed":0}}}}}}`, - } - for ver, out := range tcs { - for hn, handler := range map[string]http.Handler{ - "plain": http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintln(w, out) - }), - "basicauth": &basicAuth{ - User: "elastic", - Pass: "changeme", - Next: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintln(w, out) - }), - }, - } { - ts := httptest.NewServer(handler) - defer ts.Close() - - u, err := url.Parse(ts.URL) - if err != nil { - t.Fatalf("Failed to parse URL: %s", err) - } - u.User = url.UserPassword("elastic", "changeme") - c := NewNodes(log.NewNopLogger(), http.DefaultClient, u, true) - nsr, err := c.fetchAndDecodeNodeStats() - if err != nil { - t.Fatalf("Failed to fetch or decode node stats: %s", err) - } - t.Logf("[%s/%s] Node Stats Response: %+v", hn, ver, nsr) - if ver != "5.5.2" && nsr.ClusterName == "elasticsearch" || - ver == "5.5.2" && nsr.ClusterName == "docker-cluster" { - // ok - } else { - t.Errorf("Wrong cluster name") - } - } - } -} - func TestIndices(t *testing.T) { // Testcases created using: // docker run -d -p 9200:9200 elasticsearch:VERSION-alpine diff --git a/collector/metrics_collector.go b/collector/metrics_collector.go deleted file mode 100644 index ef1a778c..00000000 --- a/collector/metrics_collector.go +++ /dev/null @@ -1,154 +0,0 @@ -package collector - -import ( - "encoding/json" - "fmt" - "net/http" - "net/url" - - "github.com/go-kit/kit/log" - "github.com/go-kit/kit/log/level" - "github.com/prometheus/client_golang/prometheus" -) - -type MetricsCollector struct { - clusterHealth *ClusterHealth - nodes *Nodes - indices *Indices - - clusterHealthResponse *clusterHealthResponse - nodeStatsResponse *nodeStatsResponse - indexStatsResponse *indexStatsResponse -} - -func NewMetricsCollector(logger log.Logger, client *http.Client, url *url.URL, all bool, exportIndices bool) *MetricsCollector { - return &MetricsCollector{ - clusterHealth: NewClusterHealth(logger, client, url), - nodes: NewNodes(logger, client, url, all), - indices: NewIndices(logger, client, url, all, exportIndices), - } -} - -func (c *MetricsCollector) Describe(ch chan<- *prometheus.Desc) { - c.clusterHealth.Describe(ch) - c.nodes.Describe(ch) - c.indices.Describe(ch) -} - -func (c *MetricsCollector) Collect(ch chan<- prometheus.Metric) { - // clusterHealth - clusterHealthResponse, err := c.clusterHealth.fetchAndDecodeClusterHealth() - if err != nil { - c.clusterHealth.up.Set(0) - level.Warn(c.clusterHealth.logger).Log( - "msg", "failed to fetch and decode cluster health", - "err", err, - ) - return - } - c.clusterHealth.up.Set(1) - - // nodes - nodeStatsResponse, err := c.nodes.fetchAndDecodeNodeStats() - if err != nil { - c.nodes.up.Set(0) - level.Warn(c.nodes.logger).Log( - "msg", "failed to fetch and decode node stats", - "err", err, - ) - return - } - c.nodes.up.Set(1) - - // indices - indexStatsResponse, err := c.indices.fetchAndDecodeIndexStats() - if err != nil { - c.indices.up.Set(0) - level.Warn(c.indices.logger).Log( - "msg", "failed to fetch and decode index stats", - "err", err, - ) - return - } - c.indices.up.Set(1) - - c.clusterHealth.Collect(ch, clusterHealthResponse) - c.nodes.Collect(ch, nodeStatsResponse) - c.indices.Collect(ch, clusterHealthResponse, nodeStatsResponse, indexStatsResponse) -} - -func (c *ClusterHealth) fetchAndDecodeClusterHealth() (clusterHealthResponse, error) { - var chr clusterHealthResponse - - u := *c.url - u.Path = "/_cluster/health" - res, err := c.client.Get(u.String()) - if err != nil { - return chr, fmt.Errorf("failed to get cluster health from %s://%s:%s/%s: %s", - u.Scheme, u.Hostname(), u.Port(), u.Path, err) - } - defer res.Body.Close() - - if res.StatusCode != http.StatusOK { - return chr, fmt.Errorf("HTTP Request failed with code %d", res.StatusCode) - } - - if err := json.NewDecoder(res.Body).Decode(&chr); err != nil { - c.jsonParseFailures.Inc() - return chr, err - } - - return chr, nil -} - -func (c *Nodes) fetchAndDecodeNodeStats() (nodeStatsResponse, error) { - var nsr nodeStatsResponse - - u := *c.url - u.Path = "/_nodes/_local/stats" - if c.all { - u.Path = "/_nodes/stats" - } - - res, err := c.client.Get(u.String()) - if err != nil { - return nsr, fmt.Errorf("failed to get cluster health from %s://%s:%s/%s: %s", - u.Scheme, u.Hostname(), u.Port(), u.Path, err) - } - defer res.Body.Close() - if res.StatusCode != http.StatusOK { - return nsr, fmt.Errorf("HTTP Request failed with code %d", res.StatusCode) - } - - if err := json.NewDecoder(res.Body).Decode(&nsr); err != nil { - c.jsonParseFailures.Inc() - return nsr, err - } - return nsr, nil -} - -func (c *Indices) fetchAndDecodeIndexStats() (indexStatsResponse, error) { - var isr indexStatsResponse - - if c.exportIndices { - u := *c.url - u.Path = "/_all/_stats" - - res, err := c.client.Get(u.String()) - if err != nil { - return isr, fmt.Errorf("failed to get index stats from %s://%s:%s/%s: %s", - u.Scheme, u.Hostname(), u.Port(), u.Path, err) - } - defer res.Body.Close() - if res.StatusCode != http.StatusOK { - return isr, fmt.Errorf("HTTP Request failed with code %d", res.StatusCode) - } - - if err := json.NewDecoder(res.Body).Decode(&isr); err != nil { - c.jsonParseFailures.Inc() - return isr, err - } - } - - return isr, nil -} diff --git a/collector/nodes.go b/collector/nodes.go index 4b623f99..2bdd8372 100644 --- a/collector/nodes.go +++ b/collector/nodes.go @@ -1,10 +1,13 @@ package collector import ( + "encoding/json" + "fmt" "net/http" "net/url" "github.com/go-kit/kit/log" + "github.com/go-kit/kit/log/level" "github.com/prometheus/client_golang/prometheus" ) @@ -525,7 +528,33 @@ func (c *Nodes) Describe(ch chan<- *prometheus.Desc) { ch <- c.jsonParseFailures.Desc() } -func (c *Nodes) Collect(ch chan<- prometheus.Metric, nodeStatsResponse nodeStatsResponse) { +func (c *Nodes) fetchAndDecodeNodeStats() (nodeStatsResponse, error) { + var nsr nodeStatsResponse + + u := *c.url + u.Path = "/_nodes/_local/stats" + if c.all { + u.Path = "/_nodes/stats" + } + + res, err := c.client.Get(u.String()) + if err != nil { + return nsr, fmt.Errorf("failed to get cluster health from %s://%s:%s/%s: %s", + u.Scheme, u.Hostname(), u.Port(), u.Path, err) + } + defer res.Body.Close() + if res.StatusCode != http.StatusOK { + return nsr, fmt.Errorf("HTTP Request failed with code %d", res.StatusCode) + } + + if err := json.NewDecoder(res.Body).Decode(&nsr); err != nil { + c.jsonParseFailures.Inc() + return nsr, err + } + return nsr, nil +} + +func (c *Nodes) Collect(ch chan<- prometheus.Metric) { c.totalScrapes.Inc() defer func() { ch <- c.up @@ -533,6 +562,17 @@ func (c *Nodes) Collect(ch chan<- prometheus.Metric, nodeStatsResponse nodeStats ch <- c.jsonParseFailures }() + nodeStatsResponse, err := c.fetchAndDecodeNodeStats() + if err != nil { + c.up.Set(0) + level.Warn(c.logger).Log( + "msg", "failed to fetch and decode node stats", + "err", err, + ) + return + } + c.up.Set(1) + for _, node := range nodeStatsResponse.Nodes { for _, metric := range c.nodeMetrics { ch <- prometheus.MustNewConstMetric( diff --git a/collector/nodes_test.go b/collector/nodes_test.go new file mode 100644 index 00000000..46081272 --- /dev/null +++ b/collector/nodes_test.go @@ -0,0 +1,102 @@ +package collector + +import ( + "encoding/base64" + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" + + "github.com/go-kit/kit/log" +) + +func TestNodesStats(t *testing.T) { + // Testcases created using: + // docker run -d -p 9200:9200 elasticsearch:VERSION-alpine + // curl -XPUT http://localhost:9200/twitter + // curl http://localhost:9200/_nodes/stats + tcs := map[string]string{ + "1.7.6": `{"cluster_name":"elasticsearch","nodes":{"vKnRv2bPThCJIDTj92NvQg":{"timestamp":1498820641883,"name":"Overmind","transport_address":"inet[/172.17.0.4:9300]","host":"2603876b583f","ip":["inet[/172.17.0.4:9300]","NONE"],"indices":{"docs":{"count":0,"deleted":0},"store":{"size_in_bytes":575,"throttle_time_in_millis":0},"indexing":{"index_total":0,"index_time_in_millis":0,"index_current":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0},"refresh":{"total":0,"total_time_in_millis":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":10,"total_time_in_millis":52},"filter_cache":{"memory_size_in_bytes":0,"evictions":0},"id_cache":{"memory_size_in_bytes":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"percolate":{"total":0,"time_in_millis":0,"current":0,"memory_size_in_bytes":-1,"memory_size":"-1b","queries":0},"completion":{"size_in_bytes":0},"segments":{"count":0,"memory_in_bytes":0,"index_writer_memory_in_bytes":0,"index_writer_max_memory_in_bytes":335544320,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0},"translog":{"operations":0,"size_in_bytes":17},"suggest":{"total":0,"time_in_millis":0,"current":0},"query_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"os":{"timestamp":1498820641896},"process":{"timestamp":1498820641896,"open_file_descriptors":123},"jvm":{"timestamp":1498820641896,"uptime_in_millis":17871,"mem":{"heap_used_in_bytes":82958528,"heap_used_percent":7,"heap_committed_in_bytes":251002880,"heap_max_in_bytes":1056309248,"non_heap_used_in_bytes":44442536,"non_heap_committed_in_bytes":45400064,"pools":{"young":{"used_in_bytes":62150064,"max_in_bytes":139591680,"peak_used_in_bytes":139591680,"peak_max_in_bytes":139591680},"survivor":{"used_in_bytes":17432576,"max_in_bytes":17432576,"peak_used_in_bytes":17432576,"peak_max_in_bytes":17432576},"old":{"used_in_bytes":3375888,"max_in_bytes":899284992,"peak_used_in_bytes":10899920,"peak_max_in_bytes":899284992}}},"threads":{"count":36,"peak_count":36},"gc":{"collectors":{"young":{"collection_count":1,"collection_time_in_millis":40},"old":{"collection_count":1,"collection_time_in_millis":42}}},"buffer_pools":{"direct":{"count":27,"used_in_bytes":4981411,"total_capacity_in_bytes":4981411},"mapped":{"count":0,"used_in_bytes":0,"total_capacity_in_bytes":0}}},"thread_pool":{"percolate":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"fetch_shard_started":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"listener":{"threads":1,"queue":0,"active":0,"rejected":0,"largest":1,"completed":1},"index":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"refresh":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"suggest":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"generic":{"threads":4,"queue":0,"active":0,"rejected":0,"largest":4,"completed":14},"warmer":{"threads":1,"queue":0,"active":0,"rejected":0,"largest":1,"completed":5},"search":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"flush":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"optimize":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"fetch_shard_store":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"management":{"threads":1,"queue":0,"active":1,"rejected":0,"largest":1,"completed":1},"get":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"merge":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"bulk":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"snapshot":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0}},"network":{},"fs":{"timestamp":1498820641898,"total":{"total_in_bytes":63375708160,"free_in_bytes":59667320832,"available_in_bytes":56424390656},"data":[{"path":"/usr/share/elasticsearch/data/elasticsearch/nodes/0","mount":"/usr/share/elasticsearch/data (/dev/sda2)","type":"ext4","total_in_bytes":63375708160,"free_in_bytes":59667320832,"available_in_bytes":56424390656}]},"transport":{"server_open":13,"rx_count":6,"rx_size_in_bytes":1428,"tx_count":6,"tx_size_in_bytes":1428},"http":{"current_open":1,"total_opened":2},"breakers":{"request":{"limit_size_in_bytes":422523699,"limit_size":"402.9mb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.0,"tripped":0},"fielddata":{"limit_size_in_bytes":633785548,"limit_size":"604.4mb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.03,"tripped":0},"parent":{"limit_size_in_bytes":739416473,"limit_size":"705.1mb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.0,"tripped":0}}}}}`, + "2.4.5": `{"cluster_name":"elasticsearch","nodes":{"VnnrklnAQ7KRXhW2WkgPxA":{"timestamp":1498820602662,"name":"Battering Ram","transport_address":"172.17.0.4:9300","host":"172.17.0.4","ip":["172.17.0.4:9300","NONE"],"indices":{"docs":{"count":0,"deleted":0},"store":{"size_in_bytes":650,"throttle_time_in_millis":0},"indexing":{"index_total":0,"index_time_in_millis":0,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":104857600},"refresh":{"total":0,"total_time_in_millis":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":10,"total_time_in_millis":60},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"percolate":{"total":0,"time_in_millis":0,"current":0,"memory_size_in_bytes":-1,"memory_size":"-1b","queries":0},"completion":{"size_in_bytes":0},"segments":{"count":0,"memory_in_bytes":0,"terms_memory_in_bytes":0,"stored_fields_memory_in_bytes":0,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":0,"doc_values_memory_in_bytes":0,"index_writer_memory_in_bytes":0,"index_writer_max_memory_in_bytes":105630920,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0},"translog":{"operations":0,"size_in_bytes":215},"suggest":{"total":0,"time_in_millis":0,"current":0},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"os":{"timestamp":1498820602670,"cpu_percent":7,"load_average":0.32861328125,"mem":{"total_in_bytes":2096177152,"free_in_bytes":1624637440,"used_in_bytes":471539712,"free_percent":78,"used_percent":22},"swap":{"total_in_bytes":4195348480,"free_in_bytes":4005060608,"used_in_bytes":190287872}},"process":{"timestamp":1498820602670,"open_file_descriptors":116,"max_file_descriptors":1048576,"cpu":{"percent":7,"total_in_millis":10270},"mem":{"total_virtual_in_bytes":2681352192}},"jvm":{"timestamp":1498820602671,"uptime_in_millis":38833,"mem":{"heap_used_in_bytes":88986784,"heap_used_percent":8,"heap_committed_in_bytes":251002880,"heap_max_in_bytes":1056309248,"non_heap_used_in_bytes":49185912,"non_heap_committed_in_bytes":49987584,"pools":{"young":{"used_in_bytes":58281984,"max_in_bytes":139591680,"peak_used_in_bytes":139591680,"peak_max_in_bytes":139591680},"survivor":{"used_in_bytes":17432576,"max_in_bytes":17432576,"peak_used_in_bytes":17432576,"peak_max_in_bytes":17432576},"old":{"used_in_bytes":13272224,"max_in_bytes":899284992,"peak_used_in_bytes":13272224,"peak_max_in_bytes":899284992}}},"threads":{"count":35,"peak_count":38},"gc":{"collectors":{"young":{"collection_count":2,"collection_time_in_millis":87},"old":{"collection_count":1,"collection_time_in_millis":74}}},"buffer_pools":{"direct":{"count":23,"used_in_bytes":3416836,"total_capacity_in_bytes":3416836},"mapped":{"count":0,"used_in_bytes":0,"total_capacity_in_bytes":0}},"classes":{"current_loaded_count":7221,"total_loaded_count":7221,"total_unloaded_count":0}},"thread_pool":{"bulk":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"fetch_shard_started":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"fetch_shard_store":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"flush":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"force_merge":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"generic":{"threads":1,"queue":0,"active":0,"rejected":0,"largest":5,"completed":28},"get":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"index":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"listener":{"threads":1,"queue":0,"active":0,"rejected":0,"largest":1,"completed":2},"management":{"threads":2,"queue":0,"active":1,"rejected":0,"largest":2,"completed":5},"percolate":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"refresh":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"search":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"snapshot":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"suggest":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"warmer":{"threads":1,"queue":0,"active":0,"rejected":0,"largest":1,"completed":5}},"fs":{"timestamp":1498820602672,"total":{"total_in_bytes":63375708160,"free_in_bytes":59667599360,"available_in_bytes":56424669184,"spins":"true"},"data":[{"path":"/usr/share/elasticsearch/data/elasticsearch/nodes/0","mount":"/usr/share/elasticsearch/data (/dev/sda2)","type":"ext4","total_in_bytes":63375708160,"free_in_bytes":59667599360,"available_in_bytes":56424669184,"spins":"true"}]},"transport":{"server_open":0,"rx_count":6,"rx_size_in_bytes":2028,"tx_count":6,"tx_size_in_bytes":2028},"http":{"current_open":1,"total_opened":3},"breakers":{"request":{"limit_size_in_bytes":422523699,"limit_size":"402.9mb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.0,"tripped":0},"fielddata":{"limit_size_in_bytes":633785548,"limit_size":"604.4mb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.03,"tripped":0},"in_flight_requests":{"limit_size_in_bytes":1056309248,"limit_size":"1007.3mb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.0,"tripped":0},"parent":{"limit_size_in_bytes":739416473,"limit_size":"705.1mb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.0,"tripped":0}},"script":{"compilations":0,"cache_evictions":0}}}}`, + "5.4.2": `{"_nodes":{"total":1,"successful":1,"failed":0},"cluster_name":"elasticsearch","nodes":{"0hHcEFK1S7qMlk8hQCm7wQ":{"timestamp":1498820489394,"name":"0hHcEFK","transport_address":"127.0.0.1:9300","host":"127.0.0.1","ip":"127.0.0.1:9300","roles":["master","data","ingest"],"indices":{"docs":{"count":0,"deleted":0},"store":{"size_in_bytes":650,"throttle_time_in_millis":0},"indexing":{"index_total":0,"index_time_in_millis":0,"index_current":0,"index_failed":0,"delete_total":0,"delete_time_in_millis":0,"delete_current":0,"noop_update_total":0,"is_throttled":false,"throttle_time_in_millis":0},"get":{"total":0,"time_in_millis":0,"exists_total":0,"exists_time_in_millis":0,"missing_total":0,"missing_time_in_millis":0,"current":0},"search":{"open_contexts":0,"query_total":0,"query_time_in_millis":0,"query_current":0,"fetch_total":0,"fetch_time_in_millis":0,"fetch_current":0,"scroll_total":0,"scroll_time_in_millis":0,"scroll_current":0,"suggest_total":0,"suggest_time_in_millis":0,"suggest_current":0},"merges":{"current":0,"current_docs":0,"current_size_in_bytes":0,"total":0,"total_time_in_millis":0,"total_docs":0,"total_size_in_bytes":0,"total_stopped_time_in_millis":0,"total_throttled_time_in_millis":0,"total_auto_throttle_in_bytes":104857600},"refresh":{"total":0,"total_time_in_millis":0,"listeners":0},"flush":{"total":0,"total_time_in_millis":0},"warmer":{"current":0,"total":5,"total_time_in_millis":27},"query_cache":{"memory_size_in_bytes":0,"total_count":0,"hit_count":0,"miss_count":0,"cache_size":0,"cache_count":0,"evictions":0},"fielddata":{"memory_size_in_bytes":0,"evictions":0},"completion":{"size_in_bytes":0},"segments":{"count":0,"memory_in_bytes":0,"terms_memory_in_bytes":0,"stored_fields_memory_in_bytes":0,"term_vectors_memory_in_bytes":0,"norms_memory_in_bytes":0,"points_memory_in_bytes":0,"doc_values_memory_in_bytes":0,"index_writer_memory_in_bytes":0,"version_map_memory_in_bytes":0,"fixed_bit_set_memory_in_bytes":0,"max_unsafe_auto_id_timestamp":-1,"file_sizes":{}},"translog":{"operations":0,"size_in_bytes":215},"request_cache":{"memory_size_in_bytes":0,"evictions":0,"hit_count":0,"miss_count":0},"recovery":{"current_as_source":0,"current_as_target":0,"throttle_time_in_millis":0}},"os":{"timestamp":1498820489400,"cpu":{"percent":0,"load_average":{"1m":0.35,"5m":0.28,"15m":0.12}},"mem":{"total_in_bytes":2096177152,"free_in_bytes":83501056,"used_in_bytes":2012676096,"free_percent":4,"used_percent":96},"swap":{"total_in_bytes":4195348480,"free_in_bytes":3487707136,"used_in_bytes":707641344}},"process":{"timestamp":1498820489400,"open_file_descriptors":139,"max_file_descriptors":1048576,"cpu":{"percent":0,"total_in_millis":26600},"mem":{"total_virtual_in_bytes":3823181824}},"jvm":{"timestamp":1498820489400,"uptime_in_millis":185693,"mem":{"heap_used_in_bytes":114959064,"heap_used_percent":5,"heap_committed_in_bytes":2130051072,"heap_max_in_bytes":2130051072,"non_heap_used_in_bytes":65471864,"non_heap_committed_in_bytes":69906432,"pools":{"young":{"used_in_bytes":82057312,"max_in_bytes":139591680,"peak_used_in_bytes":139591680,"peak_max_in_bytes":139591680},"survivor":{"used_in_bytes":17432576,"max_in_bytes":17432576,"peak_used_in_bytes":17432576,"peak_max_in_bytes":17432576},"old":{"used_in_bytes":15469176,"max_in_bytes":1973026816,"peak_used_in_bytes":15469176,"peak_max_in_bytes":1973026816}}},"threads":{"count":26,"peak_count":29},"gc":{"collectors":{"young":{"collection_count":4,"collection_time_in_millis":1618},"old":{"collection_count":1,"collection_time_in_millis":76}}},"buffer_pools":{"direct":{"count":16,"used_in_bytes":33776600,"total_capacity_in_bytes":33776599},"mapped":{"count":0,"used_in_bytes":0,"total_capacity_in_bytes":0}},"classes":{"current_loaded_count":9995,"total_loaded_count":9995,"total_unloaded_count":0}},"thread_pool":{"bulk":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"fetch_shard_started":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"fetch_shard_store":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"flush":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"force_merge":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"generic":{"threads":4,"queue":0,"active":0,"rejected":0,"largest":4,"completed":28},"get":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"index":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"listener":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"management":{"threads":2,"queue":0,"active":1,"rejected":0,"largest":2,"completed":20},"refresh":{"threads":1,"queue":0,"active":0,"rejected":0,"largest":1,"completed":31},"search":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"snapshot":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0},"warmer":{"threads":0,"queue":0,"active":0,"rejected":0,"largest":0,"completed":0}},"fs":{"timestamp":1498820489401,"total":{"total_in_bytes":63375708160,"free_in_bytes":59668226048,"available_in_bytes":56425295872,"spins":"true"},"data":[{"path":"/usr/share/elasticsearch/data/nodes/0","mount":"/usr/share/elasticsearch/data (/dev/sda2)","type":"ext4","total_in_bytes":63375708160,"free_in_bytes":59668226048,"available_in_bytes":56425295872,"spins":"true"}],"io_stats":{"devices":[{"device_name":"sda2","operations":3017,"read_operations":1800,"write_operations":1217,"read_kilobytes":24816,"write_kilobytes":6580}],"total":{"operations":3017,"read_operations":1800,"write_operations":1217,"read_kilobytes":24816,"write_kilobytes":6580}}},"transport":{"server_open":0,"rx_count":0,"rx_size_in_bytes":0,"tx_count":0,"tx_size_in_bytes":0},"http":{"current_open":2,"total_opened":3},"breakers":{"request":{"limit_size_in_bytes":1278030643,"limit_size":"1.1gb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.0,"tripped":0},"fielddata":{"limit_size_in_bytes":1278030643,"limit_size":"1.1gb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.03,"tripped":0},"in_flight_requests":{"limit_size_in_bytes":2130051072,"limit_size":"1.9gb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.0,"tripped":0},"parent":{"limit_size_in_bytes":1491035750,"limit_size":"1.3gb","estimated_size_in_bytes":0,"estimated_size":"0b","overhead":1.0,"tripped":0}},"script":{"compilations":0,"cache_evictions":0},"discovery":{"cluster_state_queue":{"total":0,"pending":0,"committed":0}},"ingest":{"total":{"count":0,"time_in_millis":0,"current":0,"failed":0},"pipelines":{}}}}}`, + } + for ver, out := range tcs { + for hn, handler := range map[string]http.Handler{ + "plain": http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, out) + }), + "basicauth": &basicAuth{ + User: "elastic", + Pass: "changeme", + Next: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, out) + }), + }, + } { + ts := httptest.NewServer(handler) + defer ts.Close() + + u, err := url.Parse(ts.URL) + if err != nil { + t.Fatalf("Failed to parse URL: %s", err) + } + u.User = url.UserPassword("elastic", "changeme") + c := NewNodes(log.NewNopLogger(), http.DefaultClient, u, true) + nsr, err := c.fetchAndDecodeNodeStats() + if err != nil { + t.Fatalf("Failed to fetch or decode node stats: %s", err) + } + t.Logf("[%s/%s] Node Stats Response: %+v", hn, ver, nsr) + if nsr.ClusterName != "elasticsearch" { + t.Errorf("Wrong cluster name") + } + for _, nsnr := range nsr.Nodes { + if nsnr.Indices.Docs.Count > 0 { + t.Errorf("Wrong doc count") + } + } + } + } +} + +type basicAuth struct { + User string + Pass string + Next http.Handler +} + +func (h *basicAuth) checkAuth(w http.ResponseWriter, r *http.Request) bool { + s := strings.SplitN(r.Header.Get("Authorization"), " ", 2) + if len(s) != 2 { + return false + } + + b, err := base64.StdEncoding.DecodeString(s[1]) + if err != nil { + return false + } + + pair := strings.SplitN(string(b), ":", 2) + if len(pair) != 2 { + return false + } + + if h.User == pair[0] && h.Pass == pair[1] { + return true + } + return false +} + +func (h *basicAuth) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if !h.checkAuth(w, r) { + w.Header().Set("WWW-Authenticate", "Basic realm=\"ES\"") + w.WriteHeader(401) + w.Write([]byte("401 Unauthorized\n")) + return + } + + h.Next.ServeHTTP(w, r) + return +} diff --git a/main.go b/main.go index d4fa2aac..dbd233a0 100644 --- a/main.go +++ b/main.go @@ -54,7 +54,9 @@ func main() { }, } - prometheus.MustRegister(collector.NewMetricsCollector(logger, httpClient, esURL, *esAllNodes, *esExportIndices)) + prometheus.MustRegister(collector.NewClusterHealth(logger, httpClient, esURL)) + prometheus.MustRegister(collector.NewNodes(logger, httpClient, esURL, *esAllNodes)) + prometheus.MustRegister(collector.NewIndices(logger, httpClient, esURL, *esAllNodes, *esExportIndices)) http.Handle(*metricsPath, prometheus.Handler()) http.HandleFunc("/", IndexHandler(*metricsPath)) From 696489c56f5b987ad1c03e508db83b420b90b231 Mon Sep 17 00:00:00 2001 From: Manabu Matsuzaki Date: Tue, 29 Aug 2017 01:54:41 +0900 Subject: [PATCH 09/12] revert --- collector/indices.go | 483 ------------------------------------------- collector/nodes.go | 456 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 456 insertions(+), 483 deletions(-) diff --git a/collector/indices.go b/collector/indices.go index 4bfc9aff..06ac5af7 100644 --- a/collector/indices.go +++ b/collector/indices.go @@ -36,7 +36,6 @@ type Indices struct { totalScrapes prometheus.Counter jsonParseFailures prometheus.Counter - nodeMetrics []*nodeMetric indexMetrics []*indexMetric } @@ -61,464 +60,6 @@ func NewIndices(logger log.Logger, client *http.Client, url *url.URL, all bool, Help: "Number of errors while parsing JSON.", }), - nodeMetrics: []*nodeMetric{ - { - Type: prometheus.GaugeValue, - Desc: prometheus.NewDesc( - prometheus.BuildFQName(namespace, "indices", "fielddata_memory_size_bytes"), - "Field data cache memory usage in bytes", - defaultNodeLabels, nil, - ), - Value: func(node NodeStatsNodeResponse) float64 { - return float64(node.Indices.FieldData.MemorySize) - }, - Labels: defaultNodeLabelValues, - }, - { - Type: prometheus.CounterValue, - Desc: prometheus.NewDesc( - prometheus.BuildFQName(namespace, "indices", "fielddata_evictions"), - "Evictions from field data", - defaultNodeLabels, nil, - ), - Value: func(node NodeStatsNodeResponse) float64 { - return float64(node.Indices.FieldData.Evictions) - }, - Labels: defaultNodeLabelValues, - }, - { - Type: prometheus.GaugeValue, - Desc: prometheus.NewDesc( - prometheus.BuildFQName(namespace, "indices", "filter_cache_memory_size_bytes"), - "Filter cache memory usage in bytes", - defaultNodeLabels, nil, - ), - Value: func(node NodeStatsNodeResponse) float64 { - return float64(node.Indices.FilterCache.MemorySize) - }, - Labels: defaultNodeLabelValues, - }, - { - Type: prometheus.CounterValue, - Desc: prometheus.NewDesc( - prometheus.BuildFQName(namespace, "indices", "filter_cache_evictions"), - "Evictions from filter cache", - defaultNodeLabels, nil, - ), - Value: func(node NodeStatsNodeResponse) float64 { - return float64(node.Indices.FilterCache.Evictions) - }, - Labels: defaultNodeLabelValues, - }, - { - Type: prometheus.GaugeValue, - Desc: prometheus.NewDesc( - prometheus.BuildFQName(namespace, "indices", "query_cache_memory_size_bytes"), - "Query cache memory usage in bytes", - defaultNodeLabels, nil, - ), - Value: func(node NodeStatsNodeResponse) float64 { - return float64(node.Indices.QueryCache.MemorySize) - }, - Labels: defaultNodeLabelValues, - }, - { - Type: prometheus.CounterValue, - Desc: prometheus.NewDesc( - prometheus.BuildFQName(namespace, "indices", "query_cache_evictions"), - "Evictions from query cache", - defaultNodeLabels, nil, - ), - Value: func(node NodeStatsNodeResponse) float64 { - return float64(node.Indices.QueryCache.Evictions) - }, - Labels: defaultNodeLabelValues, - }, - { - Type: prometheus.GaugeValue, - Desc: prometheus.NewDesc( - prometheus.BuildFQName(namespace, "indices", "request_cache_memory_size_bytes"), - "Request cache memory usage in bytes", - defaultNodeLabels, nil, - ), - Value: func(node NodeStatsNodeResponse) float64 { - return float64(node.Indices.RequestCache.MemorySize) - }, - Labels: defaultNodeLabelValues, - }, - { - Type: prometheus.CounterValue, - Desc: prometheus.NewDesc( - prometheus.BuildFQName(namespace, "indices", "request_cache_evictions"), - "Evictions from request cache", - defaultNodeLabels, nil, - ), - Value: func(node NodeStatsNodeResponse) float64 { - return float64(node.Indices.RequestCache.Evictions) - }, - Labels: defaultNodeLabelValues, - }, - { - Type: prometheus.CounterValue, - Desc: prometheus.NewDesc( - prometheus.BuildFQName(namespace, "indices", "translog_operations"), - "Total translog operations", - defaultNodeLabels, nil, - ), - Value: func(node NodeStatsNodeResponse) float64 { - return float64(node.Indices.Translog.Operations) - }, - Labels: defaultNodeLabelValues, - }, - { - Type: prometheus.CounterValue, - Desc: prometheus.NewDesc( - prometheus.BuildFQName(namespace, "indices", "translog_size_in_bytes"), - "Total translog size in bytes", - defaultNodeLabels, nil, - ), - Value: func(node NodeStatsNodeResponse) float64 { - return float64(node.Indices.Translog.Size) - }, - Labels: defaultNodeLabelValues, - }, - { - Type: prometheus.CounterValue, - Desc: prometheus.NewDesc( - prometheus.BuildFQName(namespace, "indices", "get_time_seconds"), - "Total get time in seconds", - defaultNodeLabels, nil, - ), - Value: func(node NodeStatsNodeResponse) float64 { - return float64(node.Indices.Get.Time / 1000) - }, - Labels: defaultNodeLabelValues, - }, - { - Type: prometheus.CounterValue, - Desc: prometheus.NewDesc( - prometheus.BuildFQName(namespace, "indices", "get_total"), - "Total get", - defaultNodeLabels, nil, - ), - Value: func(node NodeStatsNodeResponse) float64 { - return float64(node.Indices.Get.Total) - }, - Labels: defaultNodeLabelValues, - }, - { - Type: prometheus.CounterValue, - Desc: prometheus.NewDesc( - prometheus.BuildFQName(namespace, "indices", "get_missing_time_seconds"), - "Total time of get missing in seconds", - defaultNodeLabels, nil, - ), - Value: func(node NodeStatsNodeResponse) float64 { - return float64(node.Indices.Get.MissingTime / 1000) - }, - Labels: defaultNodeLabelValues, - }, - { - Type: prometheus.CounterValue, - Desc: prometheus.NewDesc( - prometheus.BuildFQName(namespace, "indices", "get_missing_total"), - "Total get missing", - defaultNodeLabels, nil, - ), - Value: func(node NodeStatsNodeResponse) float64 { - return float64(node.Indices.Get.MissingTotal) - }, - Labels: defaultNodeLabelValues, - }, - { - Type: prometheus.CounterValue, - Desc: prometheus.NewDesc( - prometheus.BuildFQName(namespace, "indices", "get_exists_time_seconds"), - "Total time get exists in seconds", - defaultNodeLabels, nil, - ), - Value: func(node NodeStatsNodeResponse) float64 { - return float64(node.Indices.Get.ExistsTime / 1000) - }, - Labels: defaultNodeLabelValues, - }, - { - Type: prometheus.CounterValue, - Desc: prometheus.NewDesc( - prometheus.BuildFQName(namespace, "indices", "get_exists_total"), - "Total get exists operations", - defaultNodeLabels, nil, - ), - Value: func(node NodeStatsNodeResponse) float64 { - return float64(node.Indices.Get.ExistsTotal) - }, - Labels: defaultNodeLabelValues, - }, - { - Type: prometheus.CounterValue, - Desc: prometheus.NewDesc( - prometheus.BuildFQName(namespace, "indices_refresh", "time_seconds_total"), - "Total refreshes", - defaultNodeLabels, nil, - ), - Value: func(node NodeStatsNodeResponse) float64 { - return float64(node.Indices.Refresh.TotalTime / 1000) - }, - Labels: defaultNodeLabelValues, - }, - { - Type: prometheus.CounterValue, - Desc: prometheus.NewDesc( - prometheus.BuildFQName(namespace, "indices_refresh", "total"), - "Total time spent refreshing in seconds", - defaultNodeLabels, nil, - ), - Value: func(node NodeStatsNodeResponse) float64 { - return float64(node.Indices.Refresh.Total) - }, - Labels: defaultNodeLabelValues, - }, - { - Type: prometheus.CounterValue, - Desc: prometheus.NewDesc( - prometheus.BuildFQName(namespace, "indices", "search_query_time_seconds"), - "Total search query time in seconds", - defaultNodeLabels, nil, - ), - Value: func(node NodeStatsNodeResponse) float64 { - return float64(node.Indices.Search.QueryTime / 1000) - }, - Labels: defaultNodeLabelValues, - }, - { - Type: prometheus.CounterValue, - Desc: prometheus.NewDesc( - prometheus.BuildFQName(namespace, "indices", "search_query_total"), - "Total number of queries", - defaultNodeLabels, nil, - ), - Value: func(node NodeStatsNodeResponse) float64 { - return float64(node.Indices.Search.QueryTotal) - }, - Labels: defaultNodeLabelValues, - }, - { - Type: prometheus.CounterValue, - Desc: prometheus.NewDesc( - prometheus.BuildFQName(namespace, "indices", "search_fetch_time_seconds"), - "Total search fetch time in seconds", - defaultNodeLabels, nil, - ), - Value: func(node NodeStatsNodeResponse) float64 { - return float64(node.Indices.Search.FetchTime / 1000) - }, - Labels: defaultNodeLabelValues, - }, - { - Type: prometheus.CounterValue, - Desc: prometheus.NewDesc( - prometheus.BuildFQName(namespace, "indices", "search_fetch_total"), - "Total number of fetches", - defaultNodeLabels, nil, - ), - Value: func(node NodeStatsNodeResponse) float64 { - return float64(node.Indices.Search.FetchTotal) - }, - Labels: defaultNodeLabelValues, - }, - { - Type: prometheus.GaugeValue, - Desc: prometheus.NewDesc( - prometheus.BuildFQName(namespace, "indices", "docs"), - "Count of documents on this node", - defaultNodeLabels, nil, - ), - Value: func(node NodeStatsNodeResponse) float64 { - return float64(node.Indices.Docs.Count) - }, - Labels: defaultNodeLabelValues, - }, - { - Type: prometheus.GaugeValue, - Desc: prometheus.NewDesc( - prometheus.BuildFQName(namespace, "indices", "docs_deleted"), - "Count of deleted documents on this node", - defaultNodeLabels, nil, - ), - Value: func(node NodeStatsNodeResponse) float64 { - return float64(node.Indices.Docs.Deleted) - }, - Labels: defaultNodeLabelValues, - }, - { - Type: prometheus.GaugeValue, - Desc: prometheus.NewDesc( - prometheus.BuildFQName(namespace, "indices", "store_size_bytes"), - "Current size of stored index data in bytes", - defaultNodeLabels, nil, - ), - Value: func(node NodeStatsNodeResponse) float64 { - return float64(node.Indices.Store.Size) - }, - Labels: defaultNodeLabelValues, - }, - { - Type: prometheus.CounterValue, - Desc: prometheus.NewDesc( - prometheus.BuildFQName(namespace, "indices", "store_throttle_time_seconds_total"), - "Throttle time for index store in seconds", - defaultNodeLabels, nil, - ), - Value: func(node NodeStatsNodeResponse) float64 { - return float64(node.Indices.Store.ThrottleTime / 1000) - }, - Labels: defaultNodeLabelValues, - }, - { - Type: prometheus.GaugeValue, - Desc: prometheus.NewDesc( - prometheus.BuildFQName(namespace, "indices", "segments_memory_bytes"), - "Current memory size of segments in bytes", - defaultNodeLabels, nil, - ), - Value: func(node NodeStatsNodeResponse) float64 { - return float64(node.Indices.Segments.Memory) - }, - Labels: defaultNodeLabelValues, - }, - { - Type: prometheus.GaugeValue, - Desc: prometheus.NewDesc( - prometheus.BuildFQName(namespace, "indices", "segments_count"), - "Count of index segments on this node", - defaultNodeLabels, nil, - ), - Value: func(node NodeStatsNodeResponse) float64 { - return float64(node.Indices.Segments.Count) - }, - Labels: defaultNodeLabelValues, - }, - { - Type: prometheus.CounterValue, - Desc: prometheus.NewDesc( - prometheus.BuildFQName(namespace, "indices", "flush_total"), - "Total flushes", - defaultNodeLabels, nil, - ), - Value: func(node NodeStatsNodeResponse) float64 { - return float64(node.Indices.Flush.Total) - }, - Labels: defaultNodeLabelValues, - }, - { - Type: prometheus.CounterValue, - Desc: prometheus.NewDesc( - prometheus.BuildFQName(namespace, "indices", "flush_time_seconds"), - "Cumulative flush time in seconds", - defaultNodeLabels, nil, - ), - Value: func(node NodeStatsNodeResponse) float64 { - return float64(node.Indices.Flush.Time / 1000) - }, - Labels: defaultNodeLabelValues, - }, - { - Type: prometheus.CounterValue, - Desc: prometheus.NewDesc( - prometheus.BuildFQName(namespace, "indices_indexing", "index_time_seconds_total"), - "Cumulative index time in seconds", - defaultNodeLabels, nil, - ), - Value: func(node NodeStatsNodeResponse) float64 { - return float64(node.Indices.Indexing.IndexTime / 1000) - }, - Labels: defaultNodeLabelValues, - }, - { - Type: prometheus.CounterValue, - Desc: prometheus.NewDesc( - prometheus.BuildFQName(namespace, "indices_indexing", "index_total"), - "Total index calls", - defaultNodeLabels, nil, - ), - Value: func(node NodeStatsNodeResponse) float64 { - return float64(node.Indices.Indexing.IndexTotal) - }, - Labels: defaultNodeLabelValues, - }, - { - Type: prometheus.CounterValue, - Desc: prometheus.NewDesc( - prometheus.BuildFQName(namespace, "indices_indexing", "delete_time_seconds_total"), - "Total time indexing delete in seconds", - defaultNodeLabels, nil, - ), - Value: func(node NodeStatsNodeResponse) float64 { - return float64(node.Indices.Indexing.DeleteTime / 1000) - }, - Labels: defaultNodeLabelValues, - }, - { - Type: prometheus.CounterValue, - Desc: prometheus.NewDesc( - prometheus.BuildFQName(namespace, "indices_indexing", "delete_total"), - "Total indexing deletes", - defaultNodeLabels, nil, - ), - Value: func(node NodeStatsNodeResponse) float64 { - return float64(node.Indices.Indexing.DeleteTotal) - }, - Labels: defaultNodeLabelValues, - }, - { - Type: prometheus.CounterValue, - Desc: prometheus.NewDesc( - prometheus.BuildFQName(namespace, "indices_merges", "total"), - "Total merges", - defaultNodeLabels, nil, - ), - Value: func(node NodeStatsNodeResponse) float64 { - return float64(node.Indices.Merges.Total) - }, - Labels: defaultNodeLabelValues, - }, - { - Type: prometheus.CounterValue, - Desc: prometheus.NewDesc( - prometheus.BuildFQName(namespace, "indices_merges", "docs_total"), - "Cumulative docs merged", - defaultNodeLabels, nil, - ), - Value: func(node NodeStatsNodeResponse) float64 { - return float64(node.Indices.Merges.TotalDocs) - }, - Labels: defaultNodeLabelValues, - }, - { - Type: prometheus.CounterValue, - Desc: prometheus.NewDesc( - prometheus.BuildFQName(namespace, "indices_merges", "total_size_bytes_total"), - "Total merge size in bytes", - defaultNodeLabels, nil, - ), - Value: func(node NodeStatsNodeResponse) float64 { - return float64(node.Indices.Merges.TotalSize) - }, - Labels: defaultNodeLabelValues, - }, - { - Type: prometheus.CounterValue, - Desc: prometheus.NewDesc( - prometheus.BuildFQName(namespace, "indices_merges", "total_time_seconds_total"), - "Total time spent merging in seconds", - defaultNodeLabels, nil, - ), - Value: func(node NodeStatsNodeResponse) float64 { - return float64(node.Indices.Merges.TotalTime / 1000) - }, - Labels: defaultNodeLabelValues, - }, - }, indexMetrics: []*indexMetric{ { Type: prometheus.GaugeValue, @@ -612,18 +153,6 @@ func (i *Indices) Collect(ch chan<- prometheus.Metric) { return } - // nodes - nodes := NewNodes(i.logger, i.client, i.url, i.all) - nodeStatsResponse, err := nodes.fetchAndDecodeNodeStats() - if err != nil { - i.up.Set(0) - level.Warn(i.logger).Log( - "msg", "failed to fetch and decode node stats", - "err", err, - ) - return - } - // indices indexStatsResponse, err := i.fetchAndDecodeIndexStats() if err != nil { @@ -636,18 +165,6 @@ func (i *Indices) Collect(ch chan<- prometheus.Metric) { } i.up.Set(1) - // Node stats - for _, node := range nodeStatsResponse.Nodes { - for _, metric := range i.nodeMetrics { - ch <- prometheus.MustNewConstMetric( - metric.Desc, - metric.Type, - metric.Value(node), - metric.Labels(nodeStatsResponse.ClusterName, node)..., - ) - } - } - // Index stats for indexName, indexStats := range indexStatsResponse.Indices { for _, metric := range i.indexMetrics { diff --git a/collector/nodes.go b/collector/nodes.go index 2bdd8372..92f3d6e7 100644 --- a/collector/nodes.go +++ b/collector/nodes.go @@ -100,6 +100,462 @@ func NewNodes(logger log.Logger, client *http.Client, url *url.URL, all bool) *N }), nodeMetrics: []*nodeMetric{ + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "fielddata_memory_size_bytes"), + "Field data cache memory usage in bytes", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.FieldData.MemorySize) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "fielddata_evictions"), + "Evictions from field data", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.FieldData.Evictions) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "filter_cache_memory_size_bytes"), + "Filter cache memory usage in bytes", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.FilterCache.MemorySize) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "filter_cache_evictions"), + "Evictions from filter cache", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.FilterCache.Evictions) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "query_cache_memory_size_bytes"), + "Query cache memory usage in bytes", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.QueryCache.MemorySize) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "query_cache_evictions"), + "Evictions from query cache", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.QueryCache.Evictions) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "request_cache_memory_size_bytes"), + "Request cache memory usage in bytes", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.RequestCache.MemorySize) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "request_cache_evictions"), + "Evictions from request cache", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.RequestCache.Evictions) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "translog_operations"), + "Total translog operations", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Translog.Operations) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "translog_size_in_bytes"), + "Total translog size in bytes", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Translog.Size) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "get_time_seconds"), + "Total get time in seconds", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Get.Time / 1000) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "get_total"), + "Total get", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Get.Total) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "get_missing_time_seconds"), + "Total time of get missing in seconds", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Get.MissingTime / 1000) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "get_missing_total"), + "Total get missing", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Get.MissingTotal) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "get_exists_time_seconds"), + "Total time get exists in seconds", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Get.ExistsTime / 1000) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "get_exists_total"), + "Total get exists operations", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Get.ExistsTotal) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_refresh", "time_seconds_total"), + "Total refreshes", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Refresh.TotalTime / 1000) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_refresh", "total"), + "Total time spent refreshing in seconds", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Refresh.Total) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "search_query_time_seconds"), + "Total search query time in seconds", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Search.QueryTime / 1000) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "search_query_total"), + "Total number of queries", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Search.QueryTotal) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "search_fetch_time_seconds"), + "Total search fetch time in seconds", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Search.FetchTime / 1000) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "search_fetch_total"), + "Total number of fetches", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Search.FetchTotal) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "docs"), + "Count of documents on this node", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Docs.Count) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "docs_deleted"), + "Count of deleted documents on this node", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Docs.Deleted) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "store_size_bytes"), + "Current size of stored index data in bytes", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Store.Size) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "store_throttle_time_seconds_total"), + "Throttle time for index store in seconds", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Store.ThrottleTime / 1000) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "segments_memory_bytes"), + "Current memory size of segments in bytes", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Segments.Memory) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "segments_count"), + "Count of index segments on this node", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Segments.Count) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "flush_total"), + "Total flushes", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Flush.Total) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "flush_time_seconds"), + "Cumulative flush time in seconds", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Flush.Time / 1000) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_indexing", "index_time_seconds_total"), + "Cumulative index time in seconds", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Indexing.IndexTime / 1000) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_indexing", "index_total"), + "Total index calls", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Indexing.IndexTotal) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_indexing", "delete_time_seconds_total"), + "Total time indexing delete in seconds", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Indexing.DeleteTime / 1000) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_indexing", "delete_total"), + "Total indexing deletes", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Indexing.DeleteTotal) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_merges", "total"), + "Total merges", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Merges.Total) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_merges", "docs_total"), + "Cumulative docs merged", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Merges.TotalDocs) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_merges", "total_size_bytes_total"), + "Total merge size in bytes", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Merges.TotalSize) + }, + Labels: defaultNodeLabelValues, + }, + { + Type: prometheus.CounterValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices_merges", "total_time_seconds_total"), + "Total time spent merging in seconds", + defaultNodeLabels, nil, + ), + Value: func(node NodeStatsNodeResponse) float64 { + return float64(node.Indices.Merges.TotalTime / 1000) + }, + Labels: defaultNodeLabelValues, + }, { Type: prometheus.GaugeValue, Desc: prometheus.NewDesc( From 9c32cc17eef68fe63709eb38c9aeab82af525f00 Mon Sep 17 00:00:00 2001 From: Manabu Matsuzaki Date: Tue, 29 Aug 2017 02:13:48 +0900 Subject: [PATCH 10/12] Make index metrics optional --- collector/indices.go | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/collector/indices.go b/collector/indices.go index 06ac5af7..15fa8544 100644 --- a/collector/indices.go +++ b/collector/indices.go @@ -113,23 +113,26 @@ func (i *Indices) Describe(ch chan<- *prometheus.Desc) { func (c *Indices) fetchAndDecodeIndexStats() (indexStatsResponse, error) { var isr indexStatsResponse - u := *c.url - u.Path = "/_all/_stats" + if c.exportIndices { + u := *c.url + u.Path = "/_all/_stats" + + res, err := c.client.Get(u.String()) + if err != nil { + return isr, fmt.Errorf("failed to get index stats from %s://%s:%s/%s: %s", + u.Scheme, u.Hostname(), u.Port(), u.Path, err) + } + defer res.Body.Close() + if res.StatusCode != http.StatusOK { + return isr, fmt.Errorf("HTTP Request failed with code %d", res.StatusCode) + } - res, err := c.client.Get(u.String()) - if err != nil { - return isr, fmt.Errorf("failed to get index stats from %s://%s:%s/%s: %s", - u.Scheme, u.Hostname(), u.Port(), u.Path, err) - } - defer res.Body.Close() - if res.StatusCode != http.StatusOK { - return isr, fmt.Errorf("HTTP Request failed with code %d", res.StatusCode) + if err := json.NewDecoder(res.Body).Decode(&isr); err != nil { + c.jsonParseFailures.Inc() + return isr, err + } } - if err := json.NewDecoder(res.Body).Decode(&isr); err != nil { - c.jsonParseFailures.Inc() - return isr, err - } return isr, nil } From 9df62d376417f8129b0382fffe19bcf69a3d485a Mon Sep 17 00:00:00 2001 From: Manabu Matsuzaki Date: Tue, 29 Aug 2017 18:46:07 +0900 Subject: [PATCH 11/12] correct the places that were pointed out --- collector/indices.go | 70 ++++++++++++++++---------------------------- main.go | 4 ++- 2 files changed, 29 insertions(+), 45 deletions(-) diff --git a/collector/indices.go b/collector/indices.go index 15fa8544..0fb8f48e 100644 --- a/collector/indices.go +++ b/collector/indices.go @@ -12,9 +12,9 @@ import ( ) var ( - defaultIndexLabels = []string{"cluster", "index"} - defaultIndexLabelValues = func(clusterName string, indexName string) []string { - return []string{clusterName, indexName} + defaultIndexLabels = []string{"index"} + defaultIndexLabelValues = func(indexName string) []string { + return []string{indexName} } ) @@ -22,15 +22,13 @@ type indexMetric struct { Type prometheus.ValueType Desc *prometheus.Desc Value func(indexStats IndexStatsIndexResponse) float64 - Labels func(clusterName string, indexName string) []string + Labels func(indexName string) []string } type Indices struct { - logger log.Logger - client *http.Client - url *url.URL - all bool - exportIndices bool + logger log.Logger + client *http.Client + url *url.URL up prometheus.Gauge totalScrapes prometheus.Counter @@ -39,13 +37,11 @@ type Indices struct { indexMetrics []*indexMetric } -func NewIndices(logger log.Logger, client *http.Client, url *url.URL, all bool, exportIndices bool) *Indices { +func NewIndices(logger log.Logger, client *http.Client, url *url.URL) *Indices { return &Indices{ - logger: logger, - client: client, - url: url, - all: all, - exportIndices: exportIndices, + logger: logger, + client: client, + url: url, up: prometheus.NewGauge(prometheus.GaugeOpts{ Name: prometheus.BuildFQName(namespace, "index_stats", "up"), @@ -113,24 +109,22 @@ func (i *Indices) Describe(ch chan<- *prometheus.Desc) { func (c *Indices) fetchAndDecodeIndexStats() (indexStatsResponse, error) { var isr indexStatsResponse - if c.exportIndices { - u := *c.url - u.Path = "/_all/_stats" + u := *c.url + u.Path = "/_all/_stats" - res, err := c.client.Get(u.String()) - if err != nil { - return isr, fmt.Errorf("failed to get index stats from %s://%s:%s/%s: %s", - u.Scheme, u.Hostname(), u.Port(), u.Path, err) - } - defer res.Body.Close() - if res.StatusCode != http.StatusOK { - return isr, fmt.Errorf("HTTP Request failed with code %d", res.StatusCode) - } + res, err := c.client.Get(u.String()) + if err != nil { + return isr, fmt.Errorf("failed to get index stats from %s://%s:%s/%s: %s", + u.Scheme, u.Hostname(), u.Port(), u.Path, err) + } + defer res.Body.Close() + if res.StatusCode != http.StatusOK { + return isr, fmt.Errorf("HTTP Request failed with code %d", res.StatusCode) + } - if err := json.NewDecoder(res.Body).Decode(&isr); err != nil { - c.jsonParseFailures.Inc() - return isr, err - } + if err := json.NewDecoder(res.Body).Decode(&isr); err != nil { + c.jsonParseFailures.Inc() + return isr, err } return isr, nil @@ -144,18 +138,6 @@ func (i *Indices) Collect(ch chan<- prometheus.Metric) { ch <- i.jsonParseFailures }() - // clusterHealth - clusterHealth := NewClusterHealth(i.logger, i.client, i.url) - clusterHealthResponse, err := clusterHealth.fetchAndDecodeClusterHealth() - if err != nil { - i.up.Set(0) - level.Warn(i.logger).Log( - "msg", "failed to fetch and decode cluster health", - "err", err, - ) - return - } - // indices indexStatsResponse, err := i.fetchAndDecodeIndexStats() if err != nil { @@ -175,7 +157,7 @@ func (i *Indices) Collect(ch chan<- prometheus.Metric) { metric.Desc, metric.Type, metric.Value(indexStats), - metric.Labels(clusterHealthResponse.ClusterName, indexName)..., + metric.Labels(indexName)..., ) } } diff --git a/main.go b/main.go index dbd233a0..ec1a0495 100644 --- a/main.go +++ b/main.go @@ -56,7 +56,9 @@ func main() { prometheus.MustRegister(collector.NewClusterHealth(logger, httpClient, esURL)) prometheus.MustRegister(collector.NewNodes(logger, httpClient, esURL, *esAllNodes)) - prometheus.MustRegister(collector.NewIndices(logger, httpClient, esURL, *esAllNodes, *esExportIndices)) + if *esExportIndices { + prometheus.MustRegister(collector.NewIndices(logger, httpClient, esURL)) + } http.Handle(*metricsPath, prometheus.Handler()) http.HandleFunc("/", IndexHandler(*metricsPath)) From 6b509f3eac9a8fa724358e79e9427f8dda3cc92e Mon Sep 17 00:00:00 2001 From: Manabu Matsuzaki Date: Tue, 29 Aug 2017 18:48:52 +0900 Subject: [PATCH 12/12] fix test --- collector/indices_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/collector/indices_test.go b/collector/indices_test.go index 36a81d24..2151016e 100644 --- a/collector/indices_test.go +++ b/collector/indices_test.go @@ -35,7 +35,7 @@ func TestIndices(t *testing.T) { if err != nil { t.Fatalf("Failed to parse URL: %s", err) } - i := NewIndices(log.NewNopLogger(), http.DefaultClient, u, true, true) + i := NewIndices(log.NewNopLogger(), http.DefaultClient, u) stats, err := i.fetchAndDecodeIndexStats() if err != nil { t.Fatalf("Failed to fetch or decode indices stats: %s", err)