From b7662d3e3995d8f1a9bcd38762239122590649fd Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Fri, 26 Oct 2018 13:37:59 -0700 Subject: [PATCH 1/6] Add cluster_uuid to elasticsearch/node metricset data --- metricbeat/module/elasticsearch/node/_meta/data.json | 11 ++++++----- metricbeat/module/elasticsearch/node/data.go | 3 ++- metricbeat/module/elasticsearch/node/node.go | 11 ++++++++++- 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/metricbeat/module/elasticsearch/node/_meta/data.json b/metricbeat/module/elasticsearch/node/_meta/data.json index 2f2cdb4d9085..f878e335195a 100644 --- a/metricbeat/module/elasticsearch/node/_meta/data.json +++ b/metricbeat/module/elasticsearch/node/_meta/data.json @@ -6,6 +6,7 @@ }, "elasticsearch": { "cluster": { + "id": "91RpCx2xSQ21pVPTZfDK0Q", "name": "elasticsearch" }, "node": { @@ -16,25 +17,25 @@ "bytes": 1073741824 }, "max": { - "bytes": 1038876672 + "bytes": 1037959168 } }, "nonheap": { "init": { - "bytes": 2555904 + "bytes": 7667712 }, "max": { "bytes": 0 } } }, - "version": "1.8.0_144" + "version": "11.0.1" }, - "name": "523zXyT6TRWiqXcQItnkyQ", + "name": "DSiWcTyeThWtUXLB9J0BMw", "process": { "mlockall": false }, - "version": "6.3.0" + "version": "7.0.0-alpha1" } }, "metricset": { diff --git a/metricbeat/module/elasticsearch/node/data.go b/metricbeat/module/elasticsearch/node/data.go index aa1991895774..75ff395f2285 100644 --- a/metricbeat/module/elasticsearch/node/data.go +++ b/metricbeat/module/elasticsearch/node/data.go @@ -61,7 +61,7 @@ var ( } ) -func eventsMapping(r mb.ReporterV2, content []byte) error { +func eventsMapping(r mb.ReporterV2, info elasticsearch.Info, content []byte) error { nodesStruct := struct { ClusterName string `json:"cluster_name"` Nodes map[string]map[string]interface{} `json:"nodes"` @@ -83,6 +83,7 @@ func eventsMapping(r mb.ReporterV2, content []byte) error { event.ModuleFields = common.MapStr{} event.ModuleFields.Put("cluster.name", nodesStruct.ClusterName) + event.ModuleFields.Put("cluster.id", info.ClusterID) event.MetricSetFields, err = schema.Apply(node) if err != nil { diff --git a/metricbeat/module/elasticsearch/node/node.go b/metricbeat/module/elasticsearch/node/node.go index 2642b66ba285..11b27606bd1e 100644 --- a/metricbeat/module/elasticsearch/node/node.go +++ b/metricbeat/module/elasticsearch/node/node.go @@ -18,6 +18,8 @@ package node import ( + "github.com/pkg/errors" + "github.com/elastic/beats/libbeat/common/cfgwarn" "github.com/elastic/beats/metricbeat/helper/elastic" "github.com/elastic/beats/metricbeat/mb" @@ -72,7 +74,14 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) { return } - err = eventsMapping(r, content) + info, err := elasticsearch.GetInfo(m.HTTP, m.HostData().SanitizedURI+nodeStatsPath) + if err != nil { + err = errors.Wrap(err, "failed to get info from Elasticsearch") + elastic.ReportAndLogError(err, r, m.Log) + return + } + + err = eventsMapping(r, *info, content) if err != nil { elastic.ReportAndLogError(err, r, m.Log) return From eb18ec0e01824a71ff31a20395c3d5a70b9b66d6 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Fri, 26 Oct 2018 13:51:50 -0700 Subject: [PATCH 2/6] Fix unit tests --- metricbeat/module/elasticsearch/node/data_test.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/metricbeat/module/elasticsearch/node/data_test.go b/metricbeat/module/elasticsearch/node/data_test.go index de2537ddfa89..7e29e5b65c35 100644 --- a/metricbeat/module/elasticsearch/node/data_test.go +++ b/metricbeat/module/elasticsearch/node/data_test.go @@ -30,8 +30,13 @@ import ( "github.com/elastic/beats/metricbeat/module/elasticsearch" ) +var info = elasticsearch.Info{ + ClusterID: "1234", + ClusterName: "helloworld", +} + func TestGetMappings(t *testing.T) { - elasticsearch.TestMapper(t, "./_meta/test/node.*.json", eventsMapping) + elasticsearch.TestMapperWithInfo(t, "./_meta/test/node.*.json", eventsMapping) } func TestInvalid(t *testing.T) { @@ -41,6 +46,6 @@ func TestInvalid(t *testing.T) { assert.NoError(t, err) reporter := &mbtest.CapturingReporterV2{} - err = eventsMapping(reporter, content) + err = eventsMapping(reporter, info, content) assert.Error(t, err) } From 4c140eec88d852ea6951a5ad4e9437c4f8884d20 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Mon, 19 Nov 2018 19:59:53 -0800 Subject: [PATCH 3/6] Removing problematic assert None of the other test HTTP servers used in the Metricbeat codebase use an assertion like this so I thought it was safe to get rid of this one. --- metricbeat/module/elasticsearch/node/node_test.go | 1 - metricbeat/modules.d/elasticsearch.yml.disabled | 11 ----------- 2 files changed, 12 deletions(-) delete mode 100644 metricbeat/modules.d/elasticsearch.yml.disabled diff --git a/metricbeat/module/elasticsearch/node/node_test.go b/metricbeat/module/elasticsearch/node/node_test.go index ec8c805253e6..1c99f6169c51 100644 --- a/metricbeat/module/elasticsearch/node/node_test.go +++ b/metricbeat/module/elasticsearch/node/node_test.go @@ -48,7 +48,6 @@ func TestFetch(t *testing.T) { w.WriteHeader(200) w.Header().Set("Content-Type", "application/json;") w.Write([]byte(response)) - assert.Equal(t, "/_nodes/_local", r.RequestURI) })) defer server.Close() diff --git a/metricbeat/modules.d/elasticsearch.yml.disabled b/metricbeat/modules.d/elasticsearch.yml.disabled deleted file mode 100644 index 271f927e301e..000000000000 --- a/metricbeat/modules.d/elasticsearch.yml.disabled +++ /dev/null @@ -1,11 +0,0 @@ -# Module: elasticsearch -# Docs: https://www.elastic.co/guide/en/beats/metricbeat/master/metricbeat-module-elasticsearch.html - -- module: elasticsearch - #metricsets: - # - node - # - node_stats - period: 10s - hosts: ["http://localhost:9200"] - #username: "user" - #password: "secret" From 945dc02d413e1e03058e75f77e6c0fc5e2d24306 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Mon, 19 Nov 2018 20:14:46 -0800 Subject: [PATCH 4/6] Adding back accidentally deleted file --- metricbeat/modules.d/elasticsearch.yml.disabled | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 metricbeat/modules.d/elasticsearch.yml.disabled diff --git a/metricbeat/modules.d/elasticsearch.yml.disabled b/metricbeat/modules.d/elasticsearch.yml.disabled new file mode 100644 index 000000000000..271f927e301e --- /dev/null +++ b/metricbeat/modules.d/elasticsearch.yml.disabled @@ -0,0 +1,11 @@ +# Module: elasticsearch +# Docs: https://www.elastic.co/guide/en/beats/metricbeat/master/metricbeat-module-elasticsearch.html + +- module: elasticsearch + #metricsets: + # - node + # - node_stats + period: 10s + hosts: ["http://localhost:9200"] + #username: "user" + #password: "secret" From 92e014358995ef9cbc834340d348856fe5b0ac59 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Tue, 20 Nov 2018 07:23:51 -0800 Subject: [PATCH 5/6] Fixing test --- .../module/elasticsearch/node/node_test.go | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/metricbeat/module/elasticsearch/node/node_test.go b/metricbeat/module/elasticsearch/node/node_test.go index 1c99f6169c51..b6aaf0201891 100644 --- a/metricbeat/module/elasticsearch/node/node_test.go +++ b/metricbeat/module/elasticsearch/node/node_test.go @@ -45,9 +45,22 @@ func TestFetch(t *testing.T) { assert.NoError(t, err) server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(200) - w.Header().Set("Content-Type", "application/json;") - w.Write([]byte(response)) + switch r.RequestURI { + case "/_nodes/_local": + w.WriteHeader(200) + w.Header().Set("Content-Type", "application/json;") + w.Write([]byte(response)) + + case "/": + rootResponse := "{\"cluster_name\":\"es1\",\"cluster_uuid\":\"4heb1eiady103dxu71\",\"version\":{\"number\":\"7.0.0\"}}" + w.WriteHeader(200) + w.Header().Set("Content-Type", "application/json") + w.Write([]byte(rootResponse)) + + default: + t.FailNow() + } + })) defer server.Close() From 0a00aa1bf5243e8f7b76e2c38aadfab43a6e0024 Mon Sep 17 00:00:00 2001 From: Shaunak Kashyap Date: Wed, 21 Nov 2018 08:28:21 -0800 Subject: [PATCH 6/6] Adding CHANGELOG entry --- CHANGELOG.asciidoc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index f8a29cb79775..09fa233ab0d1 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -67,6 +67,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha1...master[Check the HEAD d *Metricbeat* +- The `elasticsearch/node` metricset now reports the Elasticsearch cluster UUID. {pull}8771[8771] + *Packetbeat* *Functionbeat*