From 737dd57f172d3dbbc3cbf71bb580f587262afc0d Mon Sep 17 00:00:00 2001 From: Manabu Matsuzaki Date: Thu, 17 Aug 2017 00:21:01 +0900 Subject: [PATCH 1/5] add index metrics via alias --- collector/aliases.go | 166 ++++++++++++++++++++++++++++++++++ collector/aliases_response.go | 3 + collector/aliases_test.go | 48 ++++++++++ main.go | 1 + 4 files changed, 218 insertions(+) create mode 100644 collector/aliases.go create mode 100644 collector/aliases_response.go create mode 100644 collector/aliases_test.go diff --git a/collector/aliases.go b/collector/aliases.go new file mode 100644 index 00000000..29848a52 --- /dev/null +++ b/collector/aliases.go @@ -0,0 +1,166 @@ +package collector + +import ( + "encoding/json" + "fmt" + "net/http" + "net/url" + + "github.com/go-kit/kit/log" + "github.com/go-kit/kit/log/level" + "github.com/prometheus/client_golang/prometheus" +) + +var ( + defaultAliasLabels = []string{"cluster", "alias", "index"} + defaultAliasLabelValues = func(clusterName string, alias string, indexName string) []string { + return []string{clusterName, alias, indexName} + } +) + +type aliasMetric struct { + Type prometheus.ValueType + Desc *prometheus.Desc + Value func(indexStats IndexStatsIndexResponse) float64 + Labels func(clusterName string, alias string, indexName string) []string +} + +type Aliases struct { + logger log.Logger + client *http.Client + url *url.URL + + up prometheus.Gauge + totalScrapes prometheus.Counter + jsonParseFailures prometheus.Counter + + aliasMetrics []*aliasMetric +} + +func NewAliases(logger log.Logger, client *http.Client, url *url.URL) *Aliases { + return &Aliases{ + logger: logger, + client: client, + url: url, + + up: prometheus.NewGauge(prometheus.GaugeOpts{ + Name: prometheus.BuildFQName(namespace, "alias_stats", "up"), + Help: "Was the last scrape of the ElasticSearch alias endpoint successful.", + }), + totalScrapes: prometheus.NewCounter(prometheus.CounterOpts{ + Name: prometheus.BuildFQName(namespace, "alias_stats", "total_scrapes"), + Help: "Current total ElasticSearch alias scrapes.", + }), + jsonParseFailures: prometheus.NewCounter(prometheus.CounterOpts{ + Name: prometheus.BuildFQName(namespace, "alias_stats", "json_parse_failures"), + Help: "Number of errors while parsing JSON.", + }), + + aliasMetrics: []*aliasMetric{ + { + Type: prometheus.GaugeValue, + Desc: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "aliases", "docs_primary"), + "Count of documents which only primary shards via alias", + defaultAliasLabels, nil, + ), + Value: func(indexStats IndexStatsIndexResponse) float64 { + return float64(indexStats.Primaries.Docs.Count) + }, + Labels: defaultAliasLabelValues, + }, + }, + } +} + +func (i *Aliases) Describe(ch chan<- *prometheus.Desc) { + for _, metric := range i.aliasMetrics { + ch <- metric.Desc + } + ch <- i.up.Desc() + ch <- i.totalScrapes.Desc() + ch <- i.jsonParseFailures.Desc() +} + +func (c *Aliases) fetchAndDecodeStats() (aliasesResponse, error) { + var ar aliasesResponse + + u := *c.url + u.Path = "/_aliases" + + res, err := c.client.Get(u.String()) + if err != nil { + return ar, fmt.Errorf("failed to get alias from %s://%s:%s/%s: %s", + u.Scheme, u.Hostname(), u.Port(), u.Path, err) + } + defer res.Body.Close() + if res.StatusCode != http.StatusOK { + return ar, fmt.Errorf("HTTP Request failed with code %d", res.StatusCode) + } + + if err := json.NewDecoder(res.Body).Decode(&ar); err != nil { + c.jsonParseFailures.Inc() + return ar, err + } + return ar, nil +} + +func (a *Aliases) Collect(ch chan<- prometheus.Metric) { + a.totalScrapes.Inc() + defer func() { + ch <- a.up + ch <- a.totalScrapes + ch <- a.jsonParseFailures + }() + + // clusterHealth + clusterHealth := NewClusterHealth(a.logger, a.client, a.url) + clusterHealthResponse, err := clusterHealth.fetchAndDecodeClusterHealth() + if err != nil { + a.up.Set(0) + level.Warn(a.logger).Log( + "msg", "failed to fetch and decode cluster health", + "err", err, + ) + return + } + + // indices + indices := NewIndices(a.logger, a.client, a.url) + indexStatsResponse, err := indices.fetchAndDecodeStats() + if err != nil { + a.up.Set(0) + level.Warn(a.logger).Log( + "msg", "failed to fetch and decode index stats", + "err", err, + ) + return + } + + // aliases + aliasesResponse, err := a.fetchAndDecodeStats() + if err != nil { + a.up.Set(0) + level.Warn(a.logger).Log( + "msg", "failed to fetch and decode alias", + "err", err, + ) + return + } + + a.up.Set(1) + + for indexName, aliases := range aliasesResponse { + for alias := range aliases["aliases"] { + for _, metric := range a.aliasMetrics { + indexStatsIndexResponse := indexStatsResponse.Indices[indexName] + ch <- prometheus.MustNewConstMetric( + metric.Desc, + metric.Type, + metric.Value(indexStatsIndexResponse), + metric.Labels(clusterHealthResponse.ClusterName, alias, indexName)..., + ) + } + } + } +} diff --git a/collector/aliases_response.go b/collector/aliases_response.go new file mode 100644 index 00000000..580adf07 --- /dev/null +++ b/collector/aliases_response.go @@ -0,0 +1,3 @@ +package collector + +type aliasesResponse map[string]map[string]map[string]interface{} diff --git a/collector/aliases_test.go b/collector/aliases_test.go new file mode 100644 index 00000000..8229c1ee --- /dev/null +++ b/collector/aliases_test.go @@ -0,0 +1,48 @@ +package collector + +import ( + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "testing" + + "github.com/go-kit/kit/log" +) + +func TestAliases(t *testing.T) { + // Testcases created using: + // docker run -d -p 9200:9200 elasticsearch:VERSION-alpine + // curl -XPUT http://localhost:9200/foo_1 + // curl -XPUT http://localhost:9200/foo_2 + // curl -XPOST {"actions":[{"add":{"index":"foo_1","alias":"foo"}},{"add":{"index":"foo_1","alias":"fooA"}},{"add":{"index":"foo_2","alias":"fooB"}}]} + ta := map[string]string{ + "5.5.1": `{"foo_1":{"aliases":{"foo":{},"fooA":{}}},"foo_2":{"aliases":{"fooB":{}}}}`, + } + for ver, out := range ta { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, out) + })) + defer ts.Close() + + u, err := url.Parse(ts.URL) + if err != nil { + t.Fatalf("Failed to parse URL: %s", err) + } + a := NewAliases(log.NewNopLogger(), http.DefaultClient, u) + stats, err := a.fetchAndDecodeStats() + if err != nil { + t.Fatalf("Failed to fetch or decode indices stats: %s", err) + } + t.Logf("[%s] Index Response: %+v", ver, stats) + if len(stats) != 2 { + t.Errorf("Wrong number of indices") + } + if len(stats["foo_1"]["aliases"]) != 2 { + t.Errorf("Wrong number of aliases") + } + if len(stats["foo_2"]["aliases"]) != 1 { + t.Errorf("Wrong number of aliases") + } + } +} diff --git a/main.go b/main.go index d8875f99..0472cd16 100644 --- a/main.go +++ b/main.go @@ -56,6 +56,7 @@ func main() { prometheus.MustRegister(collector.NewClusterHealth(logger, httpClient, esURL)) prometheus.MustRegister(collector.NewNodes(logger, httpClient, esURL, *esAllNodes)) prometheus.MustRegister(collector.NewIndices(logger, httpClient, esURL)) + prometheus.MustRegister(collector.NewAliases(logger, httpClient, esURL)) http.Handle(*metricsPath, prometheus.Handler()) http.HandleFunc("/", IndexHandler(*metricsPath)) From 8e64ee2d8b7891af45f70164b6e7da21e05eaccf Mon Sep 17 00:00:00 2001 From: Manabu Matsuzaki Date: Tue, 29 Aug 2017 19:47:21 +0900 Subject: [PATCH 2/5] fix function name --- collector/aliases.go | 6 +++--- collector/aliases_test.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/collector/aliases.go b/collector/aliases.go index 29848a52..e24a2495 100644 --- a/collector/aliases.go +++ b/collector/aliases.go @@ -82,7 +82,7 @@ func (i *Aliases) Describe(ch chan<- *prometheus.Desc) { ch <- i.jsonParseFailures.Desc() } -func (c *Aliases) fetchAndDecodeStats() (aliasesResponse, error) { +func (c *Aliases) fetchAndDecodeAliasStats() (aliasesResponse, error) { var ar aliasesResponse u := *c.url @@ -127,7 +127,7 @@ func (a *Aliases) Collect(ch chan<- prometheus.Metric) { // indices indices := NewIndices(a.logger, a.client, a.url) - indexStatsResponse, err := indices.fetchAndDecodeStats() + indexStatsResponse, err := indices.fetchAndDecodeIndexStats() if err != nil { a.up.Set(0) level.Warn(a.logger).Log( @@ -138,7 +138,7 @@ func (a *Aliases) Collect(ch chan<- prometheus.Metric) { } // aliases - aliasesResponse, err := a.fetchAndDecodeStats() + aliasesResponse, err := a.fetchAndDecodeAliasStats() if err != nil { a.up.Set(0) level.Warn(a.logger).Log( diff --git a/collector/aliases_test.go b/collector/aliases_test.go index 8229c1ee..893b36df 100644 --- a/collector/aliases_test.go +++ b/collector/aliases_test.go @@ -30,7 +30,7 @@ func TestAliases(t *testing.T) { t.Fatalf("Failed to parse URL: %s", err) } a := NewAliases(log.NewNopLogger(), http.DefaultClient, u) - stats, err := a.fetchAndDecodeStats() + stats, err := a.fetchAndDecodeAliasStats() if err != nil { t.Fatalf("Failed to fetch or decode indices stats: %s", err) } From 541a5508fc52243faa2817d150a69a315432bcfb Mon Sep 17 00:00:00 2001 From: Manabu Matsuzaki Date: Tue, 29 Aug 2017 20:00:14 +0900 Subject: [PATCH 3/5] Add tests for each supported version --- collector/aliases_test.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/collector/aliases_test.go b/collector/aliases_test.go index 893b36df..5a929385 100644 --- a/collector/aliases_test.go +++ b/collector/aliases_test.go @@ -15,9 +15,13 @@ func TestAliases(t *testing.T) { // docker run -d -p 9200:9200 elasticsearch:VERSION-alpine // curl -XPUT http://localhost:9200/foo_1 // curl -XPUT http://localhost:9200/foo_2 - // curl -XPOST {"actions":[{"add":{"index":"foo_1","alias":"foo"}},{"add":{"index":"foo_1","alias":"fooA"}},{"add":{"index":"foo_2","alias":"fooB"}}]} + // curl -XPOST http://localhost:9200/_aliases -d '{"actions":[{"add":{"index":"foo_1","alias":"foo"}},{"add":{"index":"foo_1","alias":"fooA"}},{"add":{"index":"foo_2","alias":"fooB"}}]}' + // curl http://localhost:9200/_aliases ta := map[string]string{ - "5.5.1": `{"foo_1":{"aliases":{"foo":{},"fooA":{}}},"foo_2":{"aliases":{"fooB":{}}}}`, + "1.7.6": `{"foo_1":{"aliases":{"foo":{},"fooA":{}}},"foo_2":{"aliases":{"fooB":{}}}}`, + "2.4.5": `{"foo_2":{"aliases":{"fooB":{}}},"foo_1":{"aliases":{"foo":{},"fooA":{}}}}`, + "5.4.2": `{".monitoring-alerts-2":{"aliases":{}},".monitoring-data-2":{"aliases":{}},".watches":{"aliases":{}},".monitoring-es-2-2017.08.29":{"aliases":{}},".triggered_watches":{"aliases":{}},"foo_1":{"aliases":{"foo":{},"fooA":{}}},".watcher-history-3-2017.08.29":{"aliases":{}},"foo_2":{"aliases":{"fooB":{}}}}`, + "5.5.2": `{".watches":{"aliases":{}},".monitoring-alerts-6":{"aliases":{}},"foo_2":{"aliases":{"fooB":{}}},".monitoring-es-6-2017.08.29":{"aliases":{}},".watcher-history-3-2017.08.29":{"aliases":{}},".triggered_watches":{"aliases":{}},"foo_1":{"aliases":{"foo":{},"fooA":{}}}}`, } for ver, out := range ta { ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -35,9 +39,6 @@ func TestAliases(t *testing.T) { t.Fatalf("Failed to fetch or decode indices stats: %s", err) } t.Logf("[%s] Index Response: %+v", ver, stats) - if len(stats) != 2 { - t.Errorf("Wrong number of indices") - } if len(stats["foo_1"]["aliases"]) != 2 { t.Errorf("Wrong number of aliases") } From 8814a735ceaee3f471f1ea72cfc13f8f99872bf3 Mon Sep 17 00:00:00 2001 From: Manabu Matsuzaki Date: Tue, 29 Aug 2017 20:07:36 +0900 Subject: [PATCH 4/5] remove unused variable --- collector/aliases.go | 22 +++++----------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/collector/aliases.go b/collector/aliases.go index e24a2495..49e10a0c 100644 --- a/collector/aliases.go +++ b/collector/aliases.go @@ -12,9 +12,9 @@ import ( ) var ( - defaultAliasLabels = []string{"cluster", "alias", "index"} - defaultAliasLabelValues = func(clusterName string, alias string, indexName string) []string { - return []string{clusterName, alias, indexName} + defaultAliasLabels = []string{"alias", "index"} + defaultAliasLabelValues = func(alias string, indexName string) []string { + return []string{alias, indexName} } ) @@ -22,7 +22,7 @@ type aliasMetric struct { Type prometheus.ValueType Desc *prometheus.Desc Value func(indexStats IndexStatsIndexResponse) float64 - Labels func(clusterName string, alias string, indexName string) []string + Labels func(alias string, indexName string) []string } type Aliases struct { @@ -113,18 +113,6 @@ func (a *Aliases) Collect(ch chan<- prometheus.Metric) { ch <- a.jsonParseFailures }() - // clusterHealth - clusterHealth := NewClusterHealth(a.logger, a.client, a.url) - clusterHealthResponse, err := clusterHealth.fetchAndDecodeClusterHealth() - if err != nil { - a.up.Set(0) - level.Warn(a.logger).Log( - "msg", "failed to fetch and decode cluster health", - "err", err, - ) - return - } - // indices indices := NewIndices(a.logger, a.client, a.url) indexStatsResponse, err := indices.fetchAndDecodeIndexStats() @@ -158,7 +146,7 @@ func (a *Aliases) Collect(ch chan<- prometheus.Metric) { metric.Desc, metric.Type, metric.Value(indexStatsIndexResponse), - metric.Labels(clusterHealthResponse.ClusterName, alias, indexName)..., + metric.Labels(alias, indexName)..., ) } } From d6d7d1ba3c0a56399214ff372db84668ecc17ca7 Mon Sep 17 00:00:00 2001 From: Manabu Matsuzaki Date: Tue, 29 Aug 2017 20:12:18 +0900 Subject: [PATCH 5/5] Make aliases metrics optional --- main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.go b/main.go index b20e23cb..c3a300ef 100644 --- a/main.go +++ b/main.go @@ -58,8 +58,8 @@ func main() { prometheus.MustRegister(collector.NewNodes(logger, httpClient, esURL, *esAllNodes)) if *esExportIndices { prometheus.MustRegister(collector.NewIndices(logger, httpClient, esURL)) + prometheus.MustRegister(collector.NewAliases(logger, httpClient, esURL)) } - prometheus.MustRegister(collector.NewAliases(logger, httpClient, esURL)) http.Handle(*metricsPath, prometheus.Handler()) http.HandleFunc("/", IndexHandler(*metricsPath))