From 23b80aa8c1727de7ef82cb4d8fbbfed8d994ccff Mon Sep 17 00:00:00 2001 From: Mario Castro Date: Mon, 28 Jan 2019 15:12:06 +0100 Subject: [PATCH] server Metricset for Zookeeper Metricbeat module (#10341) (cherry picked from commit 3895a23faa77e3e172db346f46a6500f2dc3f78a) # Conflicts: # metricbeat/module/zookeeper/fields.go # x-pack/metricbeat/module/mssql/performance/data_integration_test.go --- CHANGELOG.next.asciidoc | 1 + metricbeat/docs/fields.asciidoc | 124 ++++++++++ metricbeat/docs/modules/zookeeper.asciidoc | 8 +- .../docs/modules/zookeeper/server.asciidoc | 21 ++ metricbeat/docs/modules_list.asciidoc | 3 +- metricbeat/include/list.go | 1 + metricbeat/metricbeat.reference.yml | 2 +- .../zookeeper/_meta/config.reference.yml | 2 +- metricbeat/module/zookeeper/_meta/config.yml | 1 + .../module/zookeeper/_meta/docs.asciidoc | 2 +- metricbeat/module/zookeeper/fields.go | 2 +- .../module/zookeeper/server/_meta/data.json | 39 +++ .../zookeeper/server/_meta/docs.asciidoc | 15 ++ .../module/zookeeper/server/_meta/fields.yml | 47 ++++ metricbeat/module/zookeeper/server/data.go | 234 ++++++++++++++++++ .../zookeeper/server/data_integration_test.go | 54 ++++ metricbeat/module/zookeeper/server/server.go | 100 ++++++++ .../server/server_integration_test.go | 67 +++++ .../module/zookeeper/server/server_test.go | 70 ++++++ metricbeat/modules.d/zookeeper.yml.disabled | 1 + metricbeat/tests/system/test_zookeeper.py | 28 +++ x-pack/metricbeat/metricbeat.reference.yml | 2 +- 22 files changed, 816 insertions(+), 8 deletions(-) create mode 100644 metricbeat/docs/modules/zookeeper/server.asciidoc create mode 100644 metricbeat/module/zookeeper/server/_meta/data.json create mode 100644 metricbeat/module/zookeeper/server/_meta/docs.asciidoc create mode 100644 metricbeat/module/zookeeper/server/_meta/fields.yml create mode 100644 metricbeat/module/zookeeper/server/data.go create mode 100644 metricbeat/module/zookeeper/server/data_integration_test.go create mode 100644 metricbeat/module/zookeeper/server/server.go create mode 100644 metricbeat/module/zookeeper/server/server_integration_test.go create mode 100644 metricbeat/module/zookeeper/server/server_test.go diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index c7e2a7fe1c5..4096adebd30 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -203,6 +203,7 @@ https://github.com/elastic/beats/compare/v6.6.0...6.x[Check the HEAD diff] - Release use of xpack.enabled: true flag in Elasticsearch and Kibana modules as GA. {pull}10222[10222] - Making RabbitMQ Metricbeat module GA. {pull}10165[10165] - Release Elastic stack modules (Elasticsearch, Logstash, and Kibana) as GA. {pull}10094[10094] +- Added 'server' Metricset to Zookeeper Metricbeat module {issue}8938[8938] {pull}10341[10341] *Packetbeat* diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index e924c880b63..f7299d7e363 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -21596,3 +21596,127 @@ Number of znodes reported by the local ZooKeeper process. -- +[float] +== server fields + +server contains the metrics reported by the four-letter `srvr` command. + + +*`zookeeper.server.connections`*:: ++ +-- +type: long + +Connections established by the server + +-- + + +*`zookeeper.server.latency.avg`*:: ++ +-- +type: long + +Average latency of the server + +-- + +*`zookeeper.server.latency.max`*:: ++ +-- +type: long + +Max latency reached by the server + +-- + +*`zookeeper.server.latency.min`*:: ++ +-- +type: long + +Minimum latency that has been reached by the server + +-- + +*`zookeeper.server.mode`*:: ++ +-- +type: keyword + +Server mode + +-- + +*`zookeeper.server.node_count`*:: ++ +-- +type: long + +Total number of nodes + +-- + +*`zookeeper.server.outstanding`*:: ++ +-- +type: long + +Outstanding + +-- + +*`zookeeper.server.received`*:: ++ +-- +type: long + +Received requests to the server + +-- + +*`zookeeper.server.sent`*:: ++ +-- +type: long + +Requests sent by the server + +-- + +*`zookeeper.server.version_date`*:: ++ +-- +type: date + +Date of the Zookeeper release in use + +-- + +*`zookeeper.server.zxid`*:: ++ +-- +type: keyword + +Original value of the Zookeeper transaction ID + +-- + +*`zookeeper.server.count`*:: ++ +-- +type: long + +Total transactions of the leader in epoch + +-- + +*`zookeeper.server.epoch`*:: ++ +-- +type: long + +Epoch value of the Zookeeper transaction ID + +-- + diff --git a/metricbeat/docs/modules/zookeeper.asciidoc b/metricbeat/docs/modules/zookeeper.asciidoc index 30a7fa3cf48..32eb71cfc56 100644 --- a/metricbeat/docs/modules/zookeeper.asciidoc +++ b/metricbeat/docs/modules/zookeeper.asciidoc @@ -6,7 +6,7 @@ This file is generated! See scripts/docs_collector.py == ZooKeeper module The ZooKeeper module fetches statistics from the ZooKeeper service. The default -metricset is `mntr`. +metricset is `mntr` and `server`. [float] === Compatibility @@ -26,7 +26,7 @@ in <>. Here is an example configuration: metricbeat.modules: - module: zookeeper enabled: true - metricsets: ["mntr"] + metricsets: ["mntr", "server"] period: 10s hosts: ["localhost:2181"] ---- @@ -38,5 +38,9 @@ The following metricsets are available: * <> +* <> + include::zookeeper/mntr.asciidoc[] +include::zookeeper/server.asciidoc[] + diff --git a/metricbeat/docs/modules/zookeeper/server.asciidoc b/metricbeat/docs/modules/zookeeper/server.asciidoc new file mode 100644 index 00000000000..18097850acf --- /dev/null +++ b/metricbeat/docs/modules/zookeeper/server.asciidoc @@ -0,0 +1,21 @@ +//// +This file is generated! See scripts/docs_collector.py +//// + +[[metricbeat-metricset-zookeeper-server]] +=== ZooKeeper server metricset + +include::../../../module/zookeeper/server/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/zookeeper/server/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index 4149c8a5626..6a1b62a7da8 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -150,7 +150,8 @@ This file is generated! See scripts/docs_collector.py .2+| .2+| |<> beta[] |<> |<> |image:./images/icon-no.png[No prebuilt dashboards] | -.1+| .1+| |<> +.2+| .2+| |<> +|<> |================================ -- diff --git a/metricbeat/include/list.go b/metricbeat/include/list.go index f08861b04c2..229cf072e26 100644 --- a/metricbeat/include/list.go +++ b/metricbeat/include/list.go @@ -173,4 +173,5 @@ import ( _ "github.com/elastic/beats/metricbeat/module/windows/service" _ "github.com/elastic/beats/metricbeat/module/zookeeper" _ "github.com/elastic/beats/metricbeat/module/zookeeper/mntr" + _ "github.com/elastic/beats/metricbeat/module/zookeeper/server" ) diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index b5aa0f7ca26..0f441e958dc 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -688,7 +688,7 @@ metricbeat.modules: #------------------------------ ZooKeeper Module ----------------------------- - module: zookeeper enabled: true - metricsets: ["mntr"] + metricsets: ["mntr", "server"] period: 10s hosts: ["localhost:2181"] diff --git a/metricbeat/module/zookeeper/_meta/config.reference.yml b/metricbeat/module/zookeeper/_meta/config.reference.yml index 04742813c55..51b3e0b83bd 100644 --- a/metricbeat/module/zookeeper/_meta/config.reference.yml +++ b/metricbeat/module/zookeeper/_meta/config.reference.yml @@ -1,5 +1,5 @@ - module: zookeeper enabled: true - metricsets: ["mntr"] + metricsets: ["mntr", "server"] period: 10s hosts: ["localhost:2181"] diff --git a/metricbeat/module/zookeeper/_meta/config.yml b/metricbeat/module/zookeeper/_meta/config.yml index 17fb7135973..d6701a8b80f 100644 --- a/metricbeat/module/zookeeper/_meta/config.yml +++ b/metricbeat/module/zookeeper/_meta/config.yml @@ -1,5 +1,6 @@ - module: zookeeper #metricsets: # - mntr + # - server period: 10s hosts: ["localhost:2181"] diff --git a/metricbeat/module/zookeeper/_meta/docs.asciidoc b/metricbeat/module/zookeeper/_meta/docs.asciidoc index 34b3f950010..67f71737cad 100644 --- a/metricbeat/module/zookeeper/_meta/docs.asciidoc +++ b/metricbeat/module/zookeeper/_meta/docs.asciidoc @@ -1,5 +1,5 @@ The ZooKeeper module fetches statistics from the ZooKeeper service. The default -metricset is `mntr`. +metricset is `mntr` and `server`. [float] === Compatibility diff --git a/metricbeat/module/zookeeper/fields.go b/metricbeat/module/zookeeper/fields.go index 05725293014..1efe71cd2b8 100644 --- a/metricbeat/module/zookeeper/fields.go +++ b/metricbeat/module/zookeeper/fields.go @@ -31,5 +31,5 @@ func init() { // Asset returns asset data func Asset() string { - return "eJy0lsGSuzYMxu95Cs3/vnmAHDrTc6d76LEX1jEfiSfGorJIwj59xxAIpLCT7RJOGYj1/fRJtvxGJzQ7+mQ+ARVkQ6ROPXb062/mP9p3vzZEOaIVV6njsKPfNkREw3cqoeJsJMvewypy2jekR1DBtbx5qKY/cXDK4sKBLJelCXncbojikUUzy6Fwhx0VxkdsiAQeJmJHB7MhKhx8Hnet6hsFU2JKnB5tqvR34bq6vZlBTs/HsPKDLAc1LsQWts9CULHckhhyHJaP2dMzZhvzlUHvi+bwvkBsMVOAJwgfbe7WTULdkLejl1N/++cxl3E+R46afk0+9nmd0FxY8odvX2Q37Z8+9nZW2VSV8NWVRpHlRk0W3ec8hudw+B7D7/fYlMISFyOupDbP5I0i2GZrzo+C/5vkDDEH9JFpD70AgRAiyr1Ha1IkF6h03rsIy6MunNKhOqKEGB8zy3XQlRDf63IPSRYNAvQZOMcCRsHe8wUSV9cfIlNMHt22ga1FELR1ap6oNNescB5Zr8SyqkN/mqsr65LCndR50F0tkmnJcypYWuh7t1XCFnHBy77hSnNdmbVvuKc6a8BwYS0MF76PEeoyM96dkQZHgE3B1++yUWxSHlVKj0bJyNBxvqGWZh6WK4SXdt37crcl7X57PNlpXGtUE3IXDpngnxpR17d2JEK9SGdrAPLk9h495n3OWV9HhcxzV8aeoHErsHBnPA6jnzPf/QvQC8upl6Re8muwiBdUfBkqyS0AoatubIJdv7S36NRGT5W0RqRJFZ9uomGwDYf5PG2EnCFZVKMrXj7+Yo901kw3Rs+0QNIEizx73VjrBEbTbbjoXY4IZCgN24kh5CJ5mHxpU5wh0fH8Wf3DS9stNJmQ0752Pqeo0u3nDnqe6GLUHl90/LWxEUfHcoQSd1X2bI1/9hBsrzUvouyuTP+5xC/y/RsAAP//itrSOA==" + return "eJy0mM2O4zYMx+95CmIve9p5gBwKFN0eimJ3gWlPe/EyMhMLkUVXpPMxT1/Ijhw7sTOZfPg0sEf8/0iRIpUvsKb9HN6Y10QVhRmAWnU0h08/mf9u3n2aAeQkJthKLfs5/DYDAOi+Q0karBEw7BwZpRwWe9CCYMl1+OJINf4Te6scrF+B4bJEn8vLDEAKDpoZ9ku7msMSndAMIJAjFJrDCmcAS0sul3mj+gU8ljQkjo/uq/jvgevq8GYEOT6/upW/wLBXtF4a2ORFoIrDwYnOx255nz0+fbY+X+n1uGgM7wJigxkNXEF4GuZ23cDUAfml93IY3/Sc+tL3p2DR+NfgY/JrTfsth/zk2wXvhvmTbL+MKmNVBd7ZEpWyHBUzsW/jGI796mMMvx9tQzQLvOxxRbVxJodK3uxfcHMqeDPJhgKuKFmGBemWyAN5oXLhqAmSgPVQWueskOFeFg7pqCqopIBOMsO11wchfq/LBYUYok4A3jznNIGxZOd4S0Eert9ZBokxOpSBqUMgr02kxolK3GVL6yhLShweGqFvuLNlXYI/klpHcFQTwIY8hyWHBvqYbVVgQzIRy5RwJe4ezJoS7qrM6jCsfxSG9R/H8HWZobMbio3Dk4nGH59lPdug3NspLVABQ5dxbg8NzTgsV+SfmnXfp7MtaqfyuDLTuFZR9Ln1qyzQfzWJPj60PRFIIm1YPVEeo72ghHnsc8bVohTGuSs0a1J5CWTIbui0Gd3PfIyfJ91yWCdJSJKXwYSesOPTUFFuAoja3ZW9N4/f2oN1aKzHnTQYwj7u+LCIusbWHebjtEJhQyETRX3g8PHKjuJZMyyMxDRBsveG8ux5ba0V6HW3btDbFuQBITbbQUDACjjCfKooNhTE8vhZfefQdjAN6HNY1NblIBraem6hx4m2qKZ40vHX2CbpHctCCtzusmOD7tpDsBlrnkTZjkxnQ/y7fMOC+Njd4nO76IbrhIRNcw1prxCf77pD3Nus/+g1ZBLFhbNSHLnPAnM2t4yqngZuyou+vfO5/6IXZ56cTvu8nPIAhuPrXarfcNcpBkJzOXYD5bN574PKJ1Ne0+oLFFjE+f19lo6D85ubwD9tCZyZ6IbK+0r+X1Z0vcm/qfL3JqyblH5MrE/275qAXg+Le2MZv7crN082r0kjWrhi+w89J8unZoGRDwPBr7FnHortZ/odKJ1kcR6oZTw53nZ2PJ5XJN6PYFfWo4MNunpEXgN6weZcg7++Tpyc96VlT0ISQDs3RKepYlNM/JJw+uVa3T/jyis9/j8AAP//kA+yaQ==" } diff --git a/metricbeat/module/zookeeper/server/_meta/data.json b/metricbeat/module/zookeeper/server/_meta/data.json new file mode 100644 index 00000000000..0cc6c7b0b66 --- /dev/null +++ b/metricbeat/module/zookeeper/server/_meta/data.json @@ -0,0 +1,39 @@ +{ + "@timestamp": "2017-10-12T08:05:34.853Z", + "agent": { + "hostname": "host.example.com", + "name": "host.example.com" + }, + "event": { + "dataset": "zookeeper.server", + "duration": 115000, + "module": "zookeeper" + }, + "metricset": { + "name": "server" + }, + "service": { + "address": "localhost:2181", + "type": "zookeeper", + "version": "3.4.13-2d71af4dbe22557fda74f9a9b4309b15a7487f03" + }, + "zookeeper": { + "server": { + "connections": 1, + "count": 0, + "epoch": 0, + "latency": { + "avg": 0, + "max": 0, + "min": 0 + }, + "mode": "standalone", + "node_count": 4, + "outstanding": 0, + "received": 38, + "sent": 37, + "version_date": "06/29/2018 04:05 GMT", + "zxid": "0x0" + } + } +} \ No newline at end of file diff --git a/metricbeat/module/zookeeper/server/_meta/docs.asciidoc b/metricbeat/module/zookeeper/server/_meta/docs.asciidoc new file mode 100644 index 00000000000..fd30fbeebe2 --- /dev/null +++ b/metricbeat/module/zookeeper/server/_meta/docs.asciidoc @@ -0,0 +1,15 @@ +`server` Metricset fetches the data returned by the `srvr` admin keyword. + +* *connections*: Connections established by the server +* *latency.avg*: Average latency of the server +* *latency.max*: Max latency reached by the server +* *latency.min*: Minimum latency that has been reached by the server +* *mode*: Server mode +* *node_count*: Total number of nodes +* *outstanding*: Outstanding +* *received*: Received requests to the server +* *sent*: Requests sent by the server +* *version_date*: Date of the Zookeeper release in use +* *zxid*: Original value of the Zookeeper transaction ID +* *count*: Total transactions of the leader in epoch +* *epoch*: Epoch value of the Zookeeper transaction ID diff --git a/metricbeat/module/zookeeper/server/_meta/fields.yml b/metricbeat/module/zookeeper/server/_meta/fields.yml new file mode 100644 index 00000000000..2e691a459b7 --- /dev/null +++ b/metricbeat/module/zookeeper/server/_meta/fields.yml @@ -0,0 +1,47 @@ +- name: server + type: group + description: 'server contains the metrics reported by the four-letter `srvr` command.' + release: ga + fields: + - name: connections + type: long + description: Connections established by the server + - name: latency + type: group + fields: + - name: avg + type: long + description: Average latency of the server + - name: max + type: long + description: Max latency reached by the server + - name: min + type: long + description: Minimum latency that has been reached by the server + - name: mode + type: keyword + description: Server mode + - name: node_count + type: long + description: Total number of nodes + - name: outstanding + type: long + description: Outstanding + - name: received + type: long + description: Received requests to the server + - name: sent + type: long + description: Requests sent by the server + - name: version_date + type: date + description: Date of the Zookeeper release in use + - name: zxid + type: keyword + description: Original value of the Zookeeper transaction ID + - name: count + type: long + description: Total transactions of the leader in epoch + - name: epoch + type: long + description: Epoch value of the Zookeeper transaction ID diff --git a/metricbeat/module/zookeeper/server/data.go b/metricbeat/module/zookeeper/server/data.go new file mode 100644 index 00000000000..c768b87f1fb --- /dev/null +++ b/metricbeat/module/zookeeper/server/data.go @@ -0,0 +1,234 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package server + +import ( + "bufio" + "encoding/binary" + "io" + "regexp" + "strconv" + "strings" + + "github.com/pkg/errors" + + "github.com/elastic/beats/libbeat/common" +) + +var latencyCapturer = regexp.MustCompile(`(\d+)/(\d+)/(\d+)`) +var ipCapturer = regexp.MustCompile(`\d+\.\d+\.\d+\.\d+`) +var thatNumberCapturer = regexp.MustCompile(`\[(\d+)\]`) +var portCapturer = regexp.MustCompile(`:(\d+)\[`) +var dataCapturer = regexp.MustCompile(`(\w+)=(\d+)`) +var fieldsCapturer = regexp.MustCompile(`^([a-zA-Z\s]+):\s(\d+)`) +var versionCapturer = regexp.MustCompile(`:\s(.*),`) +var dateCapturer = regexp.MustCompile(`built on (.*)`) + +func parseSrvr(i io.Reader) (common.MapStr, string, error) { + scanner := bufio.NewScanner(i) + + //Get version + ok := scanner.Scan() + + if !ok { + return nil, "", errors.New("no initial successful text scan, aborting") + } + + output := common.MapStr{} + + version := versionCapturer.FindStringSubmatch(scanner.Text())[1] + output.Put("version_date", dateCapturer.FindStringSubmatch(scanner.Text())[1]) + + for scanner.Scan() { + line := scanner.Text() + + if strings.Contains(line, "Zxid") { + xid, err := parseZxid(line) + if err != nil { + err = errors.Wrapf(err, "error parsing 'zxid' line '%s'", line) + logger.Debug(err.Error()) + continue + } + + output.Update(xid) + + continue + } + + if strings.Contains(line, "Latency") { + latency, err := parseLatencyLine(line) + if err != nil { + err = errors.Wrapf(err, "error parsing 'latency values' line '%s'", line) + logger.Debug(err.Error()) + continue + } + + output.Put("latency", latency) + + continue + } + + if strings.Contains(line, "Proposal sizes") { + proposalSizes, err := parseProposalSizes(line) + if err != nil { + err = errors.Wrapf(err, "error parsing 'proposal sizes' line '%s'", line) + logger.Debug(err.Error()) + continue + } + + output.Put("proposal_sizes", proposalSizes) + + continue + } + + if strings.Contains(line, "Mode") { + modeSplit := strings.Split(line, " ") + if len(modeSplit) < 1 { + logger.Debugf("no tokens after splitting line '%s'", line) + continue + } + + output.Put("mode", modeSplit[1]) + continue + } + + // If code reaches here, just easy to parse lines or blank lines like the following are left: + // Received: 46 + // + // Sent: 45 + // Connections: 1 + // Outstanding: 0 + results := fieldsCapturer.FindAllStringSubmatch(line, -1) + if len(results) == 0 { + //probably a blank line + continue + } + + for _, result := range results { + // When submatching, the method returns the original value and the captured values, as you can see in the + // regexp of fieldsCapturer, they are 2 (so no less than 3, counting original value) + if len(result) < 3 { + logger.Debug("less than 3 tokens (%v) when regexp submatching '%s'", result, line) + continue + } + + val, err := strconv.ParseInt(result[2], 10, 64) + if err != nil { + err = errors.Wrapf(err, "error trying to parse value '%s' as int", result[2]) + logger.Debug(err.Error()) + continue + } + + output.Put(strings.ToLower(strings.Replace(result[1], " ", "_", -1)), val) + } + } + + return output, version, nil +} + +func parseZxid(line string) (common.MapStr, error) { + output := common.MapStr{} + + zxidSplit := strings.Split(line, " ") + if len(zxidSplit) < 2 { + return nil, errors.Errorf("less than 2 tokens (%v) after splitting", zxidSplit) + } + + zxidString := zxidSplit[1] + if len(zxidString) < 3 { + return nil, errors.Errorf("less than 3 characters on '%s'", zxidString) + } + zxid, err := strconv.ParseInt(zxidString[2:], 16, 64) + if err != nil { + return nil, errors.Wrapf(err, "error trying to parse value '%s' to int", zxidString[2:]) + } + + bs := make([]byte, 8) + binary.BigEndian.PutUint64(bs, uint64(zxid)) + + epoch := bs[:4] + count := bs[4:] + + output.Put("zxid", zxidString) + output.Put("epoch", binary.BigEndian.Uint32(epoch)) + output.Put("count", binary.BigEndian.Uint32(count)) + + return output, nil +} + +func parseProposalSizes(line string) (common.MapStr, error) { + output := common.MapStr{} + + initialSplit := strings.Split(line, " ") + if len(initialSplit) < 4 { + return nil, errors.Errorf("less than 4 tokens (%v) after splitting", initialSplit) + } + + values := strings.Split(initialSplit[3], "/") + if len(values) < 3 { + return nil, errors.Errorf("less than 3 tokens (%v) after splitting", values) + } + last, err := strconv.ParseInt(values[0], 10, 64) + if err != nil { + return nil, errors.Wrapf(err, "error trying to parse 'last' value as int from '%s'", values[0]) + } + output.Put("last", last) + + min, err := strconv.ParseInt(values[1], 10, 64) + if err != nil { + return nil, errors.Wrapf(err, "error trying to parse 'min' value as int from '%s'", values[1]) + } + output.Put("min", min) + + max, err := strconv.ParseInt(values[2], 10, 64) + if err != nil { + return nil, errors.Wrapf(err, "error trying to parse 'max' value as int from '%s'", values[2]) + } + output.Put("max", max) + + return output, nil +} + +func parseLatencyLine(line string) (common.MapStr, error) { + output := common.MapStr{} + + values := latencyCapturer.FindStringSubmatch(line) + if len(values) < 4 { + return nil, errors.Errorf("less than 4 fields (%v) after splitting", values) + } + + min, err := strconv.ParseInt(values[1], 10, 64) + if err != nil { + return nil, errors.Wrapf(err, "error trying to parse 'min' value '%s' as int", values[1]) + } + output.Put("min", min) + + avg, err := strconv.ParseInt(values[2], 10, 64) + if err != nil { + return nil, errors.Wrapf(err, "error trying to parse 'avg' value '%s' as int", values[2]) + } + output.Put("avg", avg) + + max, err := strconv.ParseInt(values[3], 10, 64) + if err != nil { + return nil, errors.Wrapf(err, "error trying to parse 'max' value '%s' as int", values[3]) + } + output.Put("max", max) + + return output, nil +} diff --git a/metricbeat/module/zookeeper/server/data_integration_test.go b/metricbeat/module/zookeeper/server/data_integration_test.go new file mode 100644 index 00000000000..843db0a7dca --- /dev/null +++ b/metricbeat/module/zookeeper/server/data_integration_test.go @@ -0,0 +1,54 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build integration + +package server + +import ( + "testing" + + "github.com/elastic/beats/metricbeat/module/zookeeper" + + _ "github.com/denisenkom/go-mssqldb" + "github.com/stretchr/testify/assert" + + mbtest "github.com/elastic/beats/metricbeat/mb/testing" +) + +func TestData(t *testing.T) { + t.Skip("Skipping `data.json` generation test") + + f := mbtest.NewReportingMetricSetV2(t, getDataConfig()) + events, errs := mbtest.ReportingFetchV2(f) + if len(errs) > 0 { + t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) + } + assert.NotEmpty(t, events) + + if err := mbtest.WriteEventsReporterV2(f, t, ""); err != nil { + t.Fatal("write", err) + } +} + +func getDataConfig() map[string]interface{} { + return map[string]interface{}{ + "module": "zookeeper", + "metricsets": []string{"server"}, + "hosts": []string{zookeeper.GetZookeeperEnvHost() + ":" + zookeeper.GetZookeeperEnvPort()}, + } +} diff --git a/metricbeat/module/zookeeper/server/server.go b/metricbeat/module/zookeeper/server/server.go new file mode 100644 index 00000000000..f73d6904fcd --- /dev/null +++ b/metricbeat/module/zookeeper/server/server.go @@ -0,0 +1,100 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +/* +Package server fetches metrics from ZooKeeper by using the srvr command + +See the srvr command documentation at +https://zookeeper.apache.org/doc/current/zookeeperAdmin.html + +ZooKeeper srvr Command Output + + $ echo srvr | nc localhost 2181 + Zookeeper version: 3.4.13-2d71af4dbe22557fda74f9a9b4309b15a7487f03, built on 06/29/2018 04:05 GMT +Latency min/avg/max: 1/2/3 +Received: 46 +Sent: 45 +Connections: 1 +Outstanding: 0 +Zxid: 0x700601132 +Mode: standalone +Node count: 4 +Proposal sizes last/min/max: -3/-999/-1 + + +*/ +package server + +import ( + "github.com/pkg/errors" + + "github.com/elastic/beats/libbeat/common" + + "github.com/elastic/beats/libbeat/logp" + + "github.com/elastic/beats/metricbeat/mb" + "github.com/elastic/beats/metricbeat/mb/parse" + "github.com/elastic/beats/metricbeat/module/zookeeper" +) + +var logger = logp.NewLogger("zookeeper.server") + +func init() { + mb.Registry.MustAddMetricSet("zookeeper", "server", New, + mb.WithHostParser(parse.PassThruHostParser), + mb.DefaultMetricSet(), + ) +} + +// MetricSet for fetching ZooKeeper health metrics. +type MetricSet struct { + mb.BaseMetricSet +} + +// New creates new instance of MetricSet. +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + return &MetricSet{ + BaseMetricSet: base, + }, nil +} + +// Fetch fetches metrics from ZooKeeper by making a tcp connection to the +// command port and sending the "srvr" command and parsing the output. +func (m *MetricSet) Fetch(reporter mb.ReporterV2) { + outputReader, err := zookeeper.RunCommand("srvr", m.Host(), m.Module().Config().Timeout) + if err != nil { + reporter.Error(errors.Wrap(err, "srvr command failed")) + return + } + + metricsetFields, version, err := parseSrvr(outputReader) + if err != nil { + reporter.Error(err) + return + } + + event := mb.Event{ + MetricSetFields: metricsetFields, + RootFields: common.MapStr{ + "service": common.MapStr{ + "version": version, + }, + }, + } + + reporter.Event(event) +} diff --git a/metricbeat/module/zookeeper/server/server_integration_test.go b/metricbeat/module/zookeeper/server/server_integration_test.go new file mode 100644 index 00000000000..7c5d4abecbc --- /dev/null +++ b/metricbeat/module/zookeeper/server/server_integration_test.go @@ -0,0 +1,67 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build integration + +package server + +import ( + "testing" + + "github.com/elastic/beats/libbeat/logp" + "github.com/elastic/beats/metricbeat/module/zookeeper" + + "github.com/stretchr/testify/assert" + + "github.com/elastic/beats/libbeat/tests/compose" + mbtest "github.com/elastic/beats/metricbeat/mb/testing" +) + +func TestFetch(t *testing.T) { + logp.TestingSetup() + + compose.EnsureUp(t, "zookeeper") + + f := mbtest.NewReportingMetricSetV2(t, getConfig()) + events, errs := mbtest.ReportingFetchV2(f) + if len(errs) > 0 { + t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) + } + assert.NotEmpty(t, events) + + for _, event := range events { + t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event) + metricsetFields := event.MetricSetFields + + // Check values + assert.Equal(t, "06/29/2018 04:05 GMT", metricsetFields["version_date"]) + + received := metricsetFields["received"].(int64) + assert.True(t, received >= 0) + + nodeCount := metricsetFields["node_count"].(int64) + assert.True(t, nodeCount >= 1) + } +} + +func getConfig() map[string]interface{} { + return map[string]interface{}{ + "module": "zookeeper", + "metricsets": []string{"server"}, + "hosts": []string{zookeeper.GetZookeeperEnvHost() + ":" + zookeeper.GetZookeeperEnvPort()}, + } +} diff --git a/metricbeat/module/zookeeper/server/server_test.go b/metricbeat/module/zookeeper/server/server_test.go new file mode 100644 index 00000000000..eec8e5a2494 --- /dev/null +++ b/metricbeat/module/zookeeper/server/server_test.go @@ -0,0 +1,70 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package server + +import ( + "bytes" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/elastic/beats/libbeat/common" +) + +var srvrTestInput = `Zookeeper version: 3.4.13-2d71af4dbe22557fda74f9a9b4309b15a7487f03, built on 06/29/2018 04:05 GMT +Latency min/avg/max: 1/2/3 +Received: 46 +Sent: 45 +Connections: 1 +Outstanding: 0 +Zxid: 0x700601132 +Mode: standalone +Node count: 4 +Proposal sizes last/min/max: -3/-999/-1 +` + +func TestParser(t *testing.T) { + mapStr, versionID, err := parseSrvr(bytes.NewReader([]byte(srvrTestInput))) + if err != nil { + t.Fatal(err) + } + + assert.Equal(t, "06/29/2018 04:05 GMT", mapStr["version_date"]) + assert.Equal(t, "3.4.13-2d71af4dbe22557fda74f9a9b4309b15a7487f03", versionID) + + latency := mapStr["latency"].(common.MapStr) + assert.Equal(t, int64(1), latency["min"]) + assert.Equal(t, int64(2), latency["avg"]) + assert.Equal(t, int64(3), latency["max"]) + + assert.Equal(t, int64(46), mapStr["received"]) + assert.Equal(t, int64(45), mapStr["sent"]) + assert.Equal(t, int64(1), mapStr["connections"]) + assert.Equal(t, int64(0), mapStr["outstanding"]) + assert.Equal(t, "standalone", mapStr["mode"]) + assert.Equal(t, int64(4), mapStr["node_count"]) + + proposalSizes := mapStr["proposal_sizes"].(common.MapStr) + assert.Equal(t, int64(-3), proposalSizes["last"]) + assert.Equal(t, int64(-999), proposalSizes["min"]) + assert.Equal(t, int64(-1), proposalSizes["max"]) + + assert.Equal(t, "0x700601132", mapStr["zxid"]) + assert.Equal(t, uint32(7), mapStr["epoch"]) + assert.Equal(t, uint32(0x601132), mapStr["count"]) +} diff --git a/metricbeat/modules.d/zookeeper.yml.disabled b/metricbeat/modules.d/zookeeper.yml.disabled index 1d13941cc91..2427a8f1f66 100644 --- a/metricbeat/modules.d/zookeeper.yml.disabled +++ b/metricbeat/modules.d/zookeeper.yml.disabled @@ -4,5 +4,6 @@ - module: zookeeper #metricsets: # - mntr + # - server period: 10s hosts: ["localhost:2181"] diff --git a/metricbeat/tests/system/test_zookeeper.py b/metricbeat/tests/system/test_zookeeper.py index 7871b2d1797..29b3ce1d4e3 100644 --- a/metricbeat/tests/system/test_zookeeper.py +++ b/metricbeat/tests/system/test_zookeeper.py @@ -50,6 +50,34 @@ def test_output(self): self.assert_fields_are_documented(evt) + @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") + @attr('integration') + def test_output(self): + """ + ZooKeeper server module outputs an event. + """ + self.render_config_template(modules=[{ + "name": "zookeeper", + "metricsets": ["server"], + "hosts": self.get_hosts(), + "period": "5s" + }]) + proc = self.start_beat() + self.wait_until(lambda: self.output_lines() > 0) + proc.check_kill_and_wait() + self.assert_no_logged_warnings() + + output = self.read_output_json() + self.assertEqual(len(output), 1) + evt = output[0] + + self.assertItemsEqual(self.de_dot(ZK_FIELDS), evt.keys()) + zk_srvr = evt["zookeeper"]["server"] + + assert zk_srvr["connections"] >= 0 + + self.assert_fields_are_documented(evt) + def get_hosts(self): return [os.getenv('ZOOKEEPER_HOST', 'localhost') + ':' + os.getenv('ZOOKEEPER_PORT', '2181')] diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 1624e8a6a7c..775e36af08c 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -688,7 +688,7 @@ metricbeat.modules: #------------------------------ ZooKeeper Module ------------------------------ - module: zookeeper enabled: true - metricsets: ["mntr"] + metricsets: ["mntr", "server"] period: 10s hosts: ["localhost:2181"]