From d87cc8e4fd616bf373fe716afc30acbf8f819fe7 Mon Sep 17 00:00:00 2001 From: Mario Castro Date: Tue, 29 Jun 2021 13:13:19 +0200 Subject: [PATCH] [Metricbeat] Add Couchbase's Sync Gateway module (#25599) (cherry picked from commit 77eb466253bd90103b62b8e3d8c7f50e6b1abce8) --- metricbeat/docs/fields.asciidoc | 1205 ++++++++++++++++ metricbeat/docs/modules/syncgateway.asciidoc | 57 + .../docs/modules/syncgateway/db.asciidoc | 25 + .../docs/modules/syncgateway/memory.asciidoc | 24 + .../modules/syncgateway/replication.asciidoc | 18 + .../modules/syncgateway/resources.asciidoc | 25 + metricbeat/docs/modules_list.asciidoc | 6 + x-pack/metricbeat/include/list.go | 5 + x-pack/metricbeat/metricbeat.reference.yml | 12 + .../module/syncgateway/_meta/config.yml | 10 + .../module/syncgateway/_meta/docs.asciidoc | 3 + .../module/syncgateway/_meta/fields.yml | 10 + .../syncgateway/_meta/testdata/config.json | 76 + .../_meta/testdata/expvar.282c.json | 1224 +++++++++++++++++ .../module/syncgateway/db/_meta/data.json | 206 +++ .../module/syncgateway/db/_meta/docs.asciidoc | 1 + .../module/syncgateway/db/_meta/fields.yml | 380 +++++ .../metricbeat/module/syncgateway/db/data.go | 200 +++ .../module/syncgateway/db/data_test.go | 44 + x-pack/metricbeat/module/syncgateway/db/db.go | 75 + x-pack/metricbeat/module/syncgateway/doc.go | 5 + .../metricbeat/module/syncgateway/fields.go | 23 + .../module/syncgateway/memory/_meta/data.json | 49 + .../syncgateway/memory/_meta/docs.asciidoc | 1 + .../syncgateway/memory/_meta/fields.yml | 64 + .../module/syncgateway/memory/data.go | 20 + .../module/syncgateway/memory/data_test.go | 45 + .../module/syncgateway/memory/memory.go | 74 + .../replication/_meta/docs.asciidoc | 1 + .../syncgateway/replication/_meta/fields.yml | 40 + .../module/syncgateway/replication/data.go | 41 + .../syncgateway/replication/data_test.go | 43 + .../syncgateway/replication/replication.go | 74 + .../syncgateway/resources/_meta/data.json | 57 + .../syncgateway/resources/_meta/docs.asciidoc | 1 + .../syncgateway/resources/_meta/fields.yml | 71 + .../module/syncgateway/resources/data.go | 51 + .../module/syncgateway/resources/data_test.go | 43 + .../module/syncgateway/resources/resources.go | 75 + .../metricbeat/module/syncgateway/response.go | 21 + .../module/syncgateway/syncgateway.go | 87 ++ .../metricbeat/module/syncgateway/testing.go | 33 + .../modules.d/syncgateway.yml.disabled | 13 + 43 files changed, 4538 insertions(+) create mode 100644 metricbeat/docs/modules/syncgateway.asciidoc create mode 100644 metricbeat/docs/modules/syncgateway/db.asciidoc create mode 100644 metricbeat/docs/modules/syncgateway/memory.asciidoc create mode 100644 metricbeat/docs/modules/syncgateway/replication.asciidoc create mode 100644 metricbeat/docs/modules/syncgateway/resources.asciidoc create mode 100644 x-pack/metricbeat/module/syncgateway/_meta/config.yml create mode 100644 x-pack/metricbeat/module/syncgateway/_meta/docs.asciidoc create mode 100644 x-pack/metricbeat/module/syncgateway/_meta/fields.yml create mode 100644 x-pack/metricbeat/module/syncgateway/_meta/testdata/config.json create mode 100644 x-pack/metricbeat/module/syncgateway/_meta/testdata/expvar.282c.json create mode 100644 x-pack/metricbeat/module/syncgateway/db/_meta/data.json create mode 100644 x-pack/metricbeat/module/syncgateway/db/_meta/docs.asciidoc create mode 100644 x-pack/metricbeat/module/syncgateway/db/_meta/fields.yml create mode 100644 x-pack/metricbeat/module/syncgateway/db/data.go create mode 100644 x-pack/metricbeat/module/syncgateway/db/data_test.go create mode 100644 x-pack/metricbeat/module/syncgateway/db/db.go create mode 100644 x-pack/metricbeat/module/syncgateway/doc.go create mode 100644 x-pack/metricbeat/module/syncgateway/fields.go create mode 100644 x-pack/metricbeat/module/syncgateway/memory/_meta/data.json create mode 100644 x-pack/metricbeat/module/syncgateway/memory/_meta/docs.asciidoc create mode 100644 x-pack/metricbeat/module/syncgateway/memory/_meta/fields.yml create mode 100644 x-pack/metricbeat/module/syncgateway/memory/data.go create mode 100644 x-pack/metricbeat/module/syncgateway/memory/data_test.go create mode 100644 x-pack/metricbeat/module/syncgateway/memory/memory.go create mode 100644 x-pack/metricbeat/module/syncgateway/replication/_meta/docs.asciidoc create mode 100644 x-pack/metricbeat/module/syncgateway/replication/_meta/fields.yml create mode 100644 x-pack/metricbeat/module/syncgateway/replication/data.go create mode 100644 x-pack/metricbeat/module/syncgateway/replication/data_test.go create mode 100644 x-pack/metricbeat/module/syncgateway/replication/replication.go create mode 100644 x-pack/metricbeat/module/syncgateway/resources/_meta/data.json create mode 100644 x-pack/metricbeat/module/syncgateway/resources/_meta/docs.asciidoc create mode 100644 x-pack/metricbeat/module/syncgateway/resources/_meta/fields.yml create mode 100644 x-pack/metricbeat/module/syncgateway/resources/data.go create mode 100644 x-pack/metricbeat/module/syncgateway/resources/data_test.go create mode 100644 x-pack/metricbeat/module/syncgateway/resources/resources.go create mode 100644 x-pack/metricbeat/module/syncgateway/response.go create mode 100644 x-pack/metricbeat/module/syncgateway/syncgateway.go create mode 100644 x-pack/metricbeat/module/syncgateway/testing.go create mode 100644 x-pack/metricbeat/modules.d/syncgateway.yml.disabled diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 1ef0c8612f20..a492cec0f509 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -75,6 +75,7 @@ grouped in the following categories: * <> * <> * <> +* <> * <> * <> * <> @@ -43342,6 +43343,1210 @@ type: object -- +[[exported-fields-syncgateway]] +== SyncGateway fields + +SyncGateway metrics + + +[float] +=== syncgateway + +`syncgateway` contains the information and statistics from SyncGateway. + + + +[float] +=== syncgateway + +Couchbase Sync Gateway metrics. + + + +*`syncgateway.syncgateway.name`*:: ++ +-- +Name of the database on when field `couchbase.syncgateway.type` is `db_stats`. + + +type: keyword + +-- + +[float] +=== metrics + +Metrics of all databases contained in the config file of the SyncGateway instance. + + + + +*`syncgateway.syncgateway.metrics.docs.writes.conflict.count`*:: ++ +-- +type: long + +-- + +*`syncgateway.syncgateway.metrics.docs.writes.count`*:: ++ +-- +type: long + +-- + +*`syncgateway.syncgateway.metrics.docs.writes.bytes`*:: ++ +-- +type: long + +-- + + +*`syncgateway.syncgateway.metrics.replications.active`*:: ++ +-- +Number of active replications + +type: long + +-- + +*`syncgateway.syncgateway.metrics.replications.total`*:: ++ +-- +Total number of replications (active or not) + +type: long + +-- + + + + + +*`syncgateway.syncgateway.gsi.views.tombstones.query.count`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.gsi.views.tombstones.query.time`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.gsi.views.tombstones.query.error.count`*:: ++ +-- +type: double + +-- + + + +*`syncgateway.syncgateway.gsi.views.access.query.count`*:: ++ +-- +type: double + +-- + + +*`syncgateway.syncgateway.gsi.views.access.query.error.count`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.gsi.views.access.query.time`*:: ++ +-- +type: double + +-- + + + +*`syncgateway.syncgateway.gsi.views.channels.query.count`*:: ++ +-- +type: double + +-- + + +*`syncgateway.syncgateway.gsi.views.channels.query.error.count`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.gsi.views.channels.query.time`*:: ++ +-- +type: double + +-- + + + +*`syncgateway.syncgateway.gsi.views.channels.star.query.time`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.gsi.views.channels.star.query.count`*:: ++ +-- +type: double + +-- + + +*`syncgateway.syncgateway.gsi.views.channels.star.query.error.count`*:: ++ +-- +type: double + +-- + + + +*`syncgateway.syncgateway.gsi.views.role_access.query.count`*:: ++ +-- +type: double + +-- + + +*`syncgateway.syncgateway.gsi.views.role_access.query.error.count`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.gsi.views.role_access.query.time`*:: ++ +-- +type: double + +-- + + + +*`syncgateway.syncgateway.gsi.views.sequences.query.count`*:: ++ +-- +type: double + +-- + + +*`syncgateway.syncgateway.gsi.views.sequences.query.error.count`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.gsi.views.sequences.query.time`*:: ++ +-- +type: double + +-- + + + +*`syncgateway.syncgateway.gsi.views.all_docs.query.count`*:: ++ +-- +type: double + +-- + + +*`syncgateway.syncgateway.gsi.views.all_docs.query.error.count`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.gsi.views.all_docs.query.time`*:: ++ +-- +type: double + +-- + + + +*`syncgateway.syncgateway.gsi.views.principals.query.count`*:: ++ +-- +type: double + +-- + + +*`syncgateway.syncgateway.gsi.views.principals.query.error.count`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.gsi.views.principals.query.time`*:: ++ +-- +type: double + +-- + + + +*`syncgateway.syncgateway.gsi.views.resync.query.count`*:: ++ +-- +type: double + +-- + + +*`syncgateway.syncgateway.gsi.views.resync.query.error.count`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.gsi.views.resync.query.time`*:: ++ +-- +type: double + +-- + + + +*`syncgateway.syncgateway.gsi.views.sessions.query.count`*:: ++ +-- +type: double + +-- + + +*`syncgateway.syncgateway.gsi.views.sessions.query.error.count`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.gsi.views.sessions.query.time`*:: ++ +-- +type: double + +-- + + + +*`syncgateway.syncgateway.security.access_errors.count`*:: ++ +-- +type: double + +-- + + + +*`syncgateway.syncgateway.security.auth.failed.count`*:: ++ +-- +type: double + +-- + + +*`syncgateway.syncgateway.security.docs_rejected.count`*:: ++ +-- +type: double + +-- + + + + +*`syncgateway.syncgateway.cache.channel.revs.active`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.cache.channel.revs.removal`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.cache.channel.revs.tombstone`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.cache.channel.hits`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.cache.channel.misses`*:: ++ +-- +type: double + +-- + + +*`syncgateway.syncgateway.cache.revs.hits`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.cache.revs.misses`*:: ++ +-- +type: double + +-- + + + + +*`syncgateway.syncgateway.cbl.replication.pull.caught_up`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.cbl.replication.pull.since_zero`*:: ++ +-- +type: double + +-- + + +*`syncgateway.syncgateway.cbl.replication.pull.total.continuous`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.cbl.replication.pull.total.one_shot`*:: ++ +-- +type: double + +-- + + +*`syncgateway.syncgateway.cbl.replication.pull.active.continuous`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.cbl.replication.pull.active.count`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.cbl.replication.pull.active.one_shot`*:: ++ +-- +type: double + +-- + + +*`syncgateway.syncgateway.cbl.replication.pull.attachment.bytes`*:: ++ +-- +type: long + +-- + +*`syncgateway.syncgateway.cbl.replication.pull.attachment.count`*:: ++ +-- +type: long + +-- + + +*`syncgateway.syncgateway.cbl.replication.pull.request_changes.count`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.cbl.replication.pull.request_changes.time`*:: ++ +-- +type: double + +-- + + +*`syncgateway.syncgateway.cbl.replication.pull.rev.processing_time`*:: ++ +-- +type: double + +-- + + +*`syncgateway.syncgateway.cbl.replication.pull.rev.send.count`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.cbl.replication.pull.rev.send.latency`*:: ++ +-- +type: double + +-- + + + +*`syncgateway.syncgateway.cbl.replication.push.attachment.bytes`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.cbl.replication.push.attachment.count`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.cbl.replication.push.doc_push_count`*:: ++ +-- +type: double + +-- + + +*`syncgateway.syncgateway.cbl.replication.push.propose_change.count`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.cbl.replication.push.propose_change.time`*:: ++ +-- +type: double + +-- + + +*`syncgateway.syncgateway.cbl.replication.push.sync_function.count`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.cbl.replication.push.sync_function.time`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.cbl.replication.push.write_processing_time`*:: ++ +-- +type: double + +-- + +[float] +=== memstats + +Dumps a large amount of information about the memory heap and garbage collector + + +*`syncgateway.syncgateway.memstats.BuckHashSys`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.memstats.Mallocs`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.memstats.PauseTotalNs`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.memstats.TotalAlloc`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.memstats.Alloc`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.memstats.GCSys`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.memstats.LastGC`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.memstats.MSpanSys`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.memstats.GCCPUFraction`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.memstats.HeapReleased`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.memstats.HeapSys`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.memstats.DebugGC`*:: ++ +-- +type: long + +-- + +*`syncgateway.syncgateway.memstats.HeapIdle`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.memstats.Lookups`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.memstats.HeapObjects`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.memstats.MSpanInuse`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.memstats.NumForcedGC`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.memstats.OtherSys`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.memstats.Frees`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.memstats.NextGC`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.memstats.StackInuse`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.memstats.Sys`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.memstats.NumGC`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.memstats.EnableGC`*:: ++ +-- +type: long + +-- + +*`syncgateway.syncgateway.memstats.HeapAlloc`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.memstats.MCacheInuse`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.memstats.MCacheSys`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.memstats.HeapInuse`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.memstats.StackSys`*:: ++ +-- +type: double + +-- + +[float] +=== memory + +SyncGateway memory metrics. It dumps a large amount of information about the memory heap and garbage collector + + + +*`syncgateway.memory.BuckHashSys`*:: ++ +-- +type: double + +-- + +*`syncgateway.memory.Mallocs`*:: ++ +-- +type: double + +-- + +*`syncgateway.memory.PauseTotalNs`*:: ++ +-- +type: double + +-- + +*`syncgateway.memory.TotalAlloc`*:: ++ +-- +type: double + +-- + +*`syncgateway.memory.Alloc`*:: ++ +-- +type: double + +-- + +*`syncgateway.memory.GCSys`*:: ++ +-- +type: double + +-- + +*`syncgateway.memory.LastGC`*:: ++ +-- +type: double + +-- + +*`syncgateway.memory.MSpanSys`*:: ++ +-- +type: double + +-- + +*`syncgateway.memory.GCCPUFraction`*:: ++ +-- +type: double + +-- + +*`syncgateway.memory.HeapReleased`*:: ++ +-- +type: double + +-- + +*`syncgateway.memory.HeapSys`*:: ++ +-- +type: double + +-- + +*`syncgateway.memory.DebugGC`*:: ++ +-- +type: long + +-- + +*`syncgateway.memory.HeapIdle`*:: ++ +-- +type: double + +-- + +*`syncgateway.memory.Lookups`*:: ++ +-- +type: double + +-- + +*`syncgateway.memory.HeapObjects`*:: ++ +-- +type: double + +-- + +*`syncgateway.memory.MSpanInuse`*:: ++ +-- +type: double + +-- + +*`syncgateway.memory.NumForcedGC`*:: ++ +-- +type: double + +-- + +*`syncgateway.memory.OtherSys`*:: ++ +-- +type: double + +-- + +*`syncgateway.memory.Frees`*:: ++ +-- +type: double + +-- + +*`syncgateway.memory.NextGC`*:: ++ +-- +type: double + +-- + +*`syncgateway.memory.StackInuse`*:: ++ +-- +type: double + +-- + +*`syncgateway.memory.Sys`*:: ++ +-- +type: double + +-- + +*`syncgateway.memory.NumGC`*:: ++ +-- +type: double + +-- + +*`syncgateway.memory.EnableGC`*:: ++ +-- +type: long + +-- + +*`syncgateway.memory.HeapAlloc`*:: ++ +-- +type: double + +-- + +*`syncgateway.memory.MCacheInuse`*:: ++ +-- +type: double + +-- + +*`syncgateway.memory.MCacheSys`*:: ++ +-- +type: double + +-- + +*`syncgateway.memory.HeapInuse`*:: ++ +-- +type: double + +-- + +*`syncgateway.memory.StackSys`*:: ++ +-- +type: double + +-- + +[float] +=== replication + +SyncGateway per replication metrics. + + + +[float] +=== metrics + +Metrics related with data replication. + + + + +*`syncgateway.replication.metrics.attachment.transferred.bytes`*:: ++ +-- +Number of attachment bytes transferred for this replica. + +type: long + +-- + +*`syncgateway.replication.metrics.attachment.transferred.count`*:: ++ +-- +The total number of attachments transferred since replication started. + +type: long + +-- + + +*`syncgateway.replication.metrics.docs.checked_sent`*:: ++ +-- +The total number of documents checked for changes since replication started. + +type: double + +-- + + +*`syncgateway.replication.metrics.docs.pushed.count`*:: ++ +-- +The total number of documents checked for changes since replication started. + +type: long + +-- + +*`syncgateway.replication.metrics.docs.pushed.failed`*:: ++ +-- +The total number of documents that failed to be pushed since replication started. + +type: long + +-- + +*`syncgateway.replication.id`*:: ++ +-- +ID of the replica. + +type: keyword + +-- + +[float] +=== resources + +SyncGateway global resource utilization + + + +*`syncgateway.resources.error_count`*:: ++ +-- +type: long + +-- + +*`syncgateway.resources.goroutines_high_watermark`*:: ++ +-- +type: long + +-- + +*`syncgateway.resources.num_goroutines`*:: ++ +-- +type: long + +-- + + +*`syncgateway.resources.process.cpu_percent_utilization`*:: ++ +-- +type: long + +-- + +*`syncgateway.resources.process.memory_resident`*:: ++ +-- +type: long + +-- + + + +*`syncgateway.resources.pub_net.recv.bytes`*:: ++ +-- +type: long + +-- + + +*`syncgateway.resources.pub_net.sent.bytes`*:: ++ +-- +type: long + +-- + + +*`syncgateway.resources.admin_net_bytes.recv`*:: ++ +-- +type: long + +-- + +*`syncgateway.resources.admin_net_bytes.sent`*:: ++ +-- +type: long + +-- + + + +*`syncgateway.resources.go_memstats.heap.alloc`*:: ++ +-- +type: long + +-- + +*`syncgateway.resources.go_memstats.heap.idle`*:: ++ +-- +type: long + +-- + +*`syncgateway.resources.go_memstats.heap.inuse`*:: ++ +-- +type: long + +-- + +*`syncgateway.resources.go_memstats.heap.released`*:: ++ +-- +type: long + +-- + + +*`syncgateway.resources.go_memstats.pause.ns`*:: ++ +-- +type: long + +-- + + +*`syncgateway.resources.go_memstats.stack.inuse`*:: ++ +-- +type: long + +-- + +*`syncgateway.resources.go_memstats.stack.sys`*:: ++ +-- +type: long + +-- + +*`syncgateway.resources.go_memstats.sys`*:: ++ +-- +type: long + +-- + +*`syncgateway.resources.system_memory_total`*:: ++ +-- +type: long + +-- + +*`syncgateway.resources.warn_count`*:: ++ +-- +type: long + +-- + [[exported-fields-system]] == System fields diff --git a/metricbeat/docs/modules/syncgateway.asciidoc b/metricbeat/docs/modules/syncgateway.asciidoc new file mode 100644 index 000000000000..a0688f60d229 --- /dev/null +++ b/metricbeat/docs/modules/syncgateway.asciidoc @@ -0,0 +1,57 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-module-syncgateway]] +[role="xpack"] +== SyncGateway module + +beta[] + +Sync Gateway is the synchronization server in a Couchbase for Mobile and Edge deployment. This metricset allows to monitor a Sync Gateway instance by using its REST API. + +Sync Gateway access `[host]:[port]/_expvar` on Sync Gateway nodes to fetch metrics data, ensure that the URL is accessible from the host where Metricbeat is running. + + +[float] +=== Example configuration + +The SyncGateway module supports the standard configuration options that are described +in <>. Here is an example configuration: + +[source,yaml] +---- +metricbeat.modules: +- module: syncgateway + metricsets: + - db +# - memory +# - replication +# - resources + period: 10s + + # SyncGateway hosts + hosts: ["127.0.0.1:4985"] +---- + +[float] +=== Metricsets + +The following metricsets are available: + +* <> + +* <> + +* <> + +* <> + +include::syncgateway/db.asciidoc[] + +include::syncgateway/memory.asciidoc[] + +include::syncgateway/replication.asciidoc[] + +include::syncgateway/resources.asciidoc[] + diff --git a/metricbeat/docs/modules/syncgateway/db.asciidoc b/metricbeat/docs/modules/syncgateway/db.asciidoc new file mode 100644 index 000000000000..4bc7a2964e7f --- /dev/null +++ b/metricbeat/docs/modules/syncgateway/db.asciidoc @@ -0,0 +1,25 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-syncgateway-db]] +[role="xpack"] +=== SyncGateway db metricset + +beta[] + +include::../../../../x-pack/metricbeat/module/syncgateway/db/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../../x-pack/metricbeat/module/syncgateway/db/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/syncgateway/memory.asciidoc b/metricbeat/docs/modules/syncgateway/memory.asciidoc new file mode 100644 index 000000000000..6ae4f0fedb61 --- /dev/null +++ b/metricbeat/docs/modules/syncgateway/memory.asciidoc @@ -0,0 +1,24 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-syncgateway-memory]] +[role="xpack"] +=== SyncGateway memory metricset + +beta[] + +include::../../../../x-pack/metricbeat/module/syncgateway/memory/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../../x-pack/metricbeat/module/syncgateway/memory/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/syncgateway/replication.asciidoc b/metricbeat/docs/modules/syncgateway/replication.asciidoc new file mode 100644 index 000000000000..0d73b130cfef --- /dev/null +++ b/metricbeat/docs/modules/syncgateway/replication.asciidoc @@ -0,0 +1,18 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-syncgateway-replication]] +[role="xpack"] +=== SyncGateway replication metricset + +beta[] + +include::../../../../x-pack/metricbeat/module/syncgateway/replication/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + diff --git a/metricbeat/docs/modules/syncgateway/resources.asciidoc b/metricbeat/docs/modules/syncgateway/resources.asciidoc new file mode 100644 index 000000000000..1860fe198142 --- /dev/null +++ b/metricbeat/docs/modules/syncgateway/resources.asciidoc @@ -0,0 +1,25 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-syncgateway-resources]] +[role="xpack"] +=== SyncGateway resources metricset + +beta[] + +include::../../../../x-pack/metricbeat/module/syncgateway/resources/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../../x-pack/metricbeat/module/syncgateway/resources/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index 2cbb0809b979..ce29ebeba8b3 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -254,6 +254,11 @@ This file is generated! See scripts/mage/docs_collector.go |<> |<> |image:./images/icon-no.png[No prebuilt dashboards] | .1+| .1+| |<> +|<> beta[] |image:./images/icon-no.png[No prebuilt dashboards] | +.4+| .4+| |<> beta[] +|<> beta[] +|<> beta[] +|<> beta[] |<> |image:./images/icon-yes.png[Prebuilt dashboards are available] | .18+| .18+| |<> |<> @@ -350,6 +355,7 @@ include::modules/redisenterprise.asciidoc[] include::modules/sql.asciidoc[] include::modules/stan.asciidoc[] include::modules/statsd.asciidoc[] +include::modules/syncgateway.asciidoc[] include::modules/system.asciidoc[] include::modules/tomcat.asciidoc[] include::modules/traefik.asciidoc[] diff --git a/x-pack/metricbeat/include/list.go b/x-pack/metricbeat/include/list.go index 856c33577462..9b0f402423a2 100644 --- a/x-pack/metricbeat/include/list.go +++ b/x-pack/metricbeat/include/list.go @@ -60,5 +60,10 @@ import ( _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/stan/subscriptions" _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/statsd" _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/statsd/server" + _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/syncgateway" + _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/syncgateway/db" + _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/syncgateway/memory" + _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/syncgateway/replication" + _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/syncgateway/resources" _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/tomcat" ) diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 783ff3ce3026..93027e06cf4c 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -1353,6 +1353,18 @@ metricbeat.modules: enabled: false #ttl: "30s" +#----------------------------- SyncGateway Module ----------------------------- +- module: syncgateway + metricsets: + - db +# - memory +# - replication +# - resources + period: 10s + + # SyncGateway hosts + hosts: ["127.0.0.1:4985"] + #-------------------------------- Tomcat Module -------------------------------- - module: tomcat metricsets: ['threading', 'cache', 'memory', 'requests'] diff --git a/x-pack/metricbeat/module/syncgateway/_meta/config.yml b/x-pack/metricbeat/module/syncgateway/_meta/config.yml new file mode 100644 index 000000000000..d47955d6da56 --- /dev/null +++ b/x-pack/metricbeat/module/syncgateway/_meta/config.yml @@ -0,0 +1,10 @@ +- module: syncgateway + metricsets: + - db +# - memory +# - replication +# - resources + period: 10s + + # SyncGateway hosts + hosts: ["127.0.0.1:4985"] diff --git a/x-pack/metricbeat/module/syncgateway/_meta/docs.asciidoc b/x-pack/metricbeat/module/syncgateway/_meta/docs.asciidoc new file mode 100644 index 000000000000..ce6c60d06826 --- /dev/null +++ b/x-pack/metricbeat/module/syncgateway/_meta/docs.asciidoc @@ -0,0 +1,3 @@ +Sync Gateway is the synchronization server in a Couchbase for Mobile and Edge deployment. This metricset allows to monitor a Sync Gateway instance by using its REST API. + +Sync Gateway access `[host]:[port]/_expvar` on Sync Gateway nodes to fetch metrics data, ensure that the URL is accessible from the host where Metricbeat is running. diff --git a/x-pack/metricbeat/module/syncgateway/_meta/fields.yml b/x-pack/metricbeat/module/syncgateway/_meta/fields.yml new file mode 100644 index 000000000000..1cba880d6d6d --- /dev/null +++ b/x-pack/metricbeat/module/syncgateway/_meta/fields.yml @@ -0,0 +1,10 @@ +- key: syncgateway + title: "SyncGateway" + description: SyncGateway metrics + release: beta + fields: + - name: syncgateway + type: group + description: > + `syncgateway` contains the information and statistics from SyncGateway. + fields: diff --git a/x-pack/metricbeat/module/syncgateway/_meta/testdata/config.json b/x-pack/metricbeat/module/syncgateway/_meta/testdata/config.json new file mode 100644 index 000000000000..c118d4641fba --- /dev/null +++ b/x-pack/metricbeat/module/syncgateway/_meta/testdata/config.json @@ -0,0 +1,76 @@ +{ + "interface": ":4984", + "logging": { + "log_file_path": "/var/tmp/sglogs", + "console": { + "log_level": "debug", + "log_keys": [ + "*" + ] + }, + "error": { + "enabled": true, + "rotation": { + "max_size": 20, + "max_age": 180 + } + }, + "warn": { + "enabled": true, + "rotation": { + "max_size": 20, + "max_age": 90 + } + }, + "info": { + "enabled": false + }, + "debug": { + "enabled": false + } + }, + "databases": { + "beer-sample": { + "import_docs": "continuous", + "enable_shared_bucket_access": true, + "bucket": "beer-sample", + "server": "http://172.17.0.2:8091", + "username": "admin", + "password": "123456", + "num_index_replicas": 0, + "users": { + "GUEST": { + "disabled": true + }, + "admin": { + "password": "123456", + "admin_channels": [ + "*" + ] + } + }, + "revs_limit": 20 + }, + "travel-sample": { + "import_docs": "continuous", + "enable_shared_bucket_access": true, + "bucket": "travel-sample", + "server": "http://172.17.0.2:8091", + "username": "admin", + "password": "123456", + "num_index_replicas": 0, + "users": { + "GUEST": { + "disabled": true + }, + "admin": { + "password": "123456", + "admin_channels": [ + "*" + ] + } + }, + "revs_limit": 20 + } + } +} diff --git a/x-pack/metricbeat/module/syncgateway/_meta/testdata/expvar.282c.json b/x-pack/metricbeat/module/syncgateway/_meta/testdata/expvar.282c.json new file mode 100644 index 000000000000..8b1f71fde745 --- /dev/null +++ b/x-pack/metricbeat/module/syncgateway/_meta/testdata/expvar.282c.json @@ -0,0 +1,1224 @@ +{ + "cb": { + "ops": {}, + "pools": {} + }, + "cmdline": [ + "sync_gateway", + "--defaultLogFilePath=/var/log/sync_gateway", + "-adminInterface", + ":4985", + "/etc/sync_gateway/sync_gateway.json" + ], + "goblip": {}, + "mc": { + "recv": { + "bytes": { + "0x1f": 1008, + "0x89": 864, + "SASL_AUTH": 864, + "SASL_LIST_MECHS": 2376, + "UPR_OPEN": 864, + "total": 5976 + }, + "errs": {}, + "ops": { + "0x1f": 36, + "0x89": 36, + "SASL_AUTH": 36, + "SASL_LIST_MECHS": 36, + "UPR_OPEN": 36, + "total": 180 + } + }, + "tap": { + "bytes": {}, + "errs": {}, + "ops": {} + }, + "xmit": { + "bytes": { + "0x1f": 3566, + "0x89": 1296, + "NOOP": 720, + "SASL_AUTH": 1512, + "SASL_LIST_MECHS": 864, + "UPR_BUFFERACK": 336, + "UPR_CONTROL": 4932, + "UPR_OPEN": 3170, + "UPR_STREAMREQ": 294912, + "total": 311308 + }, + "errs": {}, + "ops": { + "0x1f": 36, + "0x89": 36, + "NOOP": 30, + "SASL_AUTH": 36, + "SASL_LIST_MECHS": 36, + "UPR_BUFFERACK": 12, + "UPR_CONTROL": 108, + "UPR_OPEN": 36, + "UPR_STREAMREQ": 4096, + "total": 4426 + } + } + }, + "memstats": { + "Alloc": 159398816, + "TotalAlloc": 2719416432, + "Sys": 357708024, + "Lookups": 0, + "Mallocs": 57667615, + "Frees": 55790048, + "HeapAlloc": 159398816, + "HeapSys": 332169216, + "HeapIdle": 67780608, + "HeapInuse": 264388608, + "HeapReleased": 26927104, + "HeapObjects": 1877567, + "StackInuse": 3375104, + "StackSys": 3375104, + "MSpanInuse": 4030632, + "MSpanSys": 4358144, + "MCacheInuse": 20832, + "MCacheSys": 32768, + "BuckHashSys": 1759627, + "GCSys": 13555712, + "OtherSys": 2457453, + "NextGC": 265304208, + "LastGC": 1620310398040906500, + "PauseTotalNs": 88693194, + "PauseNs": [ + 67131, + 17720, + 49699, + 15125, + 26870, + 11396, + 17540, + 28606, + 64253, + 95491, + 26325, + 18928, + 255476, + 622721, + 378524, + 7860579, + 2189850, + 312269, + 160370, + 1127927, + 226473, + 714888, + 399169, + 3320206, + 7245241, + 2639040, + 1084283, + 3470662, + 975960, + 952710, + 166017, + 733559, + 6388460, + 276382, + 4632588, + 640706, + 146091, + 195839, + 4782434, + 79438, + 559445, + 1344800, + 9485458, + 2707508, + 1547026, + 220774, + 2629126, + 9999615, + 226927, + 7472682, + 82887, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "PauseEnd": [ + 1620310338920958200, + 1620310351426572000, + 1620310352681769500, + 1620310352686500000, + 1620310352693075200, + 1620310352700478000, + 1620310352745081600, + 1620310352759565600, + 1620310352763341800, + 1620310352767747600, + 1620310352793954800, + 1620310352846218000, + 1620310352905687000, + 1620310352982015700, + 1620310353051933000, + 1620310353138877000, + 1620310353255282200, + 1620310353399916800, + 1620310353720879600, + 1620310354061829600, + 1620310354374070800, + 1620310354696978400, + 1620310355152804000, + 1620310355628574500, + 1620310356137813000, + 1620310356627671300, + 1620310357016540400, + 1620310374534675200, + 1620310374809924900, + 1620310375047449300, + 1620310375474444000, + 1620310375924982000, + 1620310376440657000, + 1620310377146911000, + 1620310377906382000, + 1620310378854651000, + 1620310379714263000, + 1620310380508387800, + 1620310381220176100, + 1620310382398691600, + 1620310383426419200, + 1620310384538631700, + 1620310385624432000, + 1620310387400379100, + 1620310388572275200, + 1620310389941691000, + 1620310391488313600, + 1620310393008522000, + 1620310394713452500, + 1620310396453674500, + 1620310398040906500, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "NumGC": 51, + "NumForcedGC": 0, + "GCCPUFraction": 0.008328526565497294, + "EnableGC": true, + "DebugGC": false, + "BySize": [ + { + "Size": 0, + "Mallocs": 0, + "Frees": 0 + }, + { + "Size": 8, + "Mallocs": 901392, + "Frees": 896474 + }, + { + "Size": 16, + "Mallocs": 29531337, + "Frees": 28456937 + }, + { + "Size": 32, + "Mallocs": 6033284, + "Frees": 5643846 + }, + { + "Size": 48, + "Mallocs": 3201493, + "Frees": 3012460 + }, + { + "Size": 64, + "Mallocs": 797155, + "Frees": 786367 + }, + { + "Size": 80, + "Mallocs": 544250, + "Frees": 538869 + }, + { + "Size": 96, + "Mallocs": 712994, + "Frees": 706044 + }, + { + "Size": 112, + "Mallocs": 300043, + "Frees": 296594 + }, + { + "Size": 128, + "Mallocs": 454667, + "Frees": 453674 + }, + { + "Size": 144, + "Mallocs": 258222, + "Frees": 257167 + }, + { + "Size": 160, + "Mallocs": 197323, + "Frees": 185378 + }, + { + "Size": 176, + "Mallocs": 118510, + "Frees": 117754 + }, + { + "Size": 192, + "Mallocs": 44105, + "Frees": 43694 + }, + { + "Size": 208, + "Mallocs": 130830, + "Frees": 129313 + }, + { + "Size": 224, + "Mallocs": 42617, + "Frees": 42049 + }, + { + "Size": 240, + "Mallocs": 89888, + "Frees": 89150 + }, + { + "Size": 256, + "Mallocs": 300134, + "Frees": 297243 + }, + { + "Size": 288, + "Mallocs": 1747420, + "Frees": 1617371 + }, + { + "Size": 320, + "Mallocs": 4829, + "Frees": 3282 + }, + { + "Size": 352, + "Mallocs": 83681, + "Frees": 82264 + }, + { + "Size": 384, + "Mallocs": 3024, + "Frees": 2009 + }, + { + "Size": 416, + "Mallocs": 2534, + "Frees": 2134 + }, + { + "Size": 448, + "Mallocs": 2288, + "Frees": 1784 + }, + { + "Size": 480, + "Mallocs": 2686, + "Frees": 2434 + }, + { + "Size": 512, + "Mallocs": 277265, + "Frees": 272027 + }, + { + "Size": 576, + "Mallocs": 164320, + "Frees": 152902 + }, + { + "Size": 640, + "Mallocs": 14575, + "Frees": 14272 + }, + { + "Size": 704, + "Mallocs": 8716, + "Frees": 8364 + }, + { + "Size": 768, + "Mallocs": 5289, + "Frees": 5072 + }, + { + "Size": 896, + "Mallocs": 15547, + "Frees": 14356 + }, + { + "Size": 1024, + "Mallocs": 73044, + "Frees": 69817 + }, + { + "Size": 1152, + "Mallocs": 73955, + "Frees": 72310 + }, + { + "Size": 1280, + "Mallocs": 62537, + "Frees": 60565 + }, + { + "Size": 1408, + "Mallocs": 37251, + "Frees": 35700 + }, + { + "Size": 1536, + "Mallocs": 33999, + "Frees": 33097 + }, + { + "Size": 1792, + "Mallocs": 45383, + "Frees": 44658 + }, + { + "Size": 2048, + "Mallocs": 3753, + "Frees": 3688 + }, + { + "Size": 2304, + "Mallocs": 12732, + "Frees": 7985 + }, + { + "Size": 2688, + "Mallocs": 838, + "Frees": 761 + }, + { + "Size": 3072, + "Mallocs": 2908, + "Frees": 2903 + }, + { + "Size": 3200, + "Mallocs": 146, + "Frees": 143 + }, + { + "Size": 3456, + "Mallocs": 567, + "Frees": 492 + }, + { + "Size": 4096, + "Mallocs": 2241, + "Frees": 2136 + }, + { + "Size": 4864, + "Mallocs": 481, + "Frees": 432 + }, + { + "Size": 5376, + "Mallocs": 1494, + "Frees": 1410 + }, + { + "Size": 6144, + "Mallocs": 448, + "Frees": 440 + }, + { + "Size": 6528, + "Mallocs": 161, + "Frees": 158 + }, + { + "Size": 6784, + "Mallocs": 778, + "Frees": 778 + }, + { + "Size": 6912, + "Mallocs": 86, + "Frees": 83 + }, + { + "Size": 8192, + "Mallocs": 1377, + "Frees": 981 + }, + { + "Size": 9472, + "Mallocs": 995, + "Frees": 981 + }, + { + "Size": 9728, + "Mallocs": 2122, + "Frees": 72 + }, + { + "Size": 10240, + "Mallocs": 132, + "Frees": 130 + }, + { + "Size": 10880, + "Mallocs": 137, + "Frees": 137 + }, + { + "Size": 12288, + "Mallocs": 704, + "Frees": 629 + }, + { + "Size": 13568, + "Mallocs": 156, + "Frees": 147 + }, + { + "Size": 14336, + "Mallocs": 62, + "Frees": 60 + }, + { + "Size": 16384, + "Mallocs": 279, + "Frees": 264 + }, + { + "Size": 18432, + "Mallocs": 489, + "Frees": 222 + }, + { + "Size": 19072, + "Mallocs": 15, + "Frees": 15 + } + ] + }, + "syncGateway_changeCache": { + "maxPending": 0 + }, + "syncgateway": { + "global": { + "resource_utilization": { + "admin_net_bytes_recv": 186313933, + "admin_net_bytes_sent": 67278955, + "error_count": 0, + "go_memstats_heapalloc": 143386592, + "go_memstats_heapidle": 81616896, + "go_memstats_heapinuse": 250585088, + "go_memstats_heapreleased": 26927104, + "go_memstats_pausetotalns": 88693194, + "go_memstats_stackinuse": 3342336, + "go_memstats_stacksys": 3342336, + "go_memstats_sys": 357708024, + "goroutines_high_watermark": 311, + "num_goroutines": 308, + "process_cpu_percent_utilization": 0.48708528167446, + "process_memory_resident": 335306752, + "pub_net_bytes_recv": 186313933, + "pub_net_bytes_sent": 67278955, + "system_memory_total": 33291907072, + "warn_count": 6 + } + }, + "per_db": { + "beer-sample": { + "cache": { + "abandoned_seqs": 0, + "chan_cache_active_revs": 0, + "chan_cache_bypass_count": 0, + "chan_cache_channels_added": 0, + "chan_cache_channels_evicted_inactive": 0, + "chan_cache_channels_evicted_nru": 0, + "chan_cache_compact_count": 0, + "chan_cache_compact_time": 0, + "chan_cache_hits": 0, + "chan_cache_max_entries": 0, + "chan_cache_misses": 0, + "chan_cache_num_channels": 0, + "chan_cache_pending_queries": 0, + "chan_cache_removal_revs": 0, + "chan_cache_tombstone_revs": 0, + "high_seq_cached": 7304, + "high_seq_stable": 7305, + "num_active_channels": 0, + "num_skipped_seqs": 0, + "pending_seq_len": 0, + "rev_cache_bypass": 0, + "rev_cache_hits": 0, + "rev_cache_misses": 0, + "skipped_seq_len": 0 + }, + "cbl_replication_pull": { + "attachment_pull_bytes": 0, + "attachment_pull_count": 0, + "max_pending": 222, + "num_replications_active": 0, + "num_pull_repl_active_continuous": 0, + "num_pull_repl_active_one_shot": 0, + "num_pull_repl_caught_up": 0, + "num_pull_repl_since_zero": 0, + "num_pull_repl_total_continuous": 0, + "num_pull_repl_total_one_shot": 0, + "request_changes_count": 0, + "request_changes_time": 0, + "rev_processing_time": 0, + "rev_send_count": 0, + "rev_send_latency": 0 + }, + "cbl_replication_push": { + "attachment_push_bytes": 0, + "attachment_push_count": 0, + "doc_push_count": 0, + "propose_change_count": 0, + "propose_change_time": 0, + "sync_function_count": 0, + "sync_function_time": 0, + "write_processing_time": 0 + }, + "database": { + "abandoned_seqs": 0, + "conflict_write_count": 0, + "crc32c_match_count": 0, + "dcp_caching_count": 7303, + "dcp_caching_time": 47545220448, + "dcp_received_count": 7303, + "dcp_received_time": 71654930936, + "doc_reads_bytes_blip": 0, + "doc_writes_bytes": 2542924, + "doc_writes_bytes_blip": 0, + "doc_writes_xattr_bytes": 1822707, + "high_seq_feed": 14610, + "num_doc_reads_blip": 0, + "num_doc_reads_rest": 0, + "num_doc_writes": 7303, + "num_replications_active": 0, + "num_replications_total": 0, + "num_tombstones_compacted": 0, + "sequence_assigned_count": 7304, + "sequence_get_count": 2, + "sequence_incr_count": 733, + "sequence_released_count": 1, + "sequence_reserved_count": 7305, + "warn_channels_per_doc_count": 0, + "warn_grants_per_doc_count": 0, + "warn_xattr_size_count": 0, + "cache_feed": {}, + "import_feed": {} + }, + "gsi_views": { + "access_query_count": 1, + "access_query_error_count": 0, + "access_query_time": 214030402, + "allDocs_query_count": 0, + "allDocs_query_error_count": 0, + "allDocs_query_time": 0, + "channelsStar_query_count": 0, + "channelsStar_query_error_count": 0, + "channelsStar_query_time": 0, + "channels_query_count": 0, + "channels_query_error_count": 0, + "channels_query_time": 0, + "principals_query_count": 0, + "principals_query_error_count": 0, + "principals_query_time": 0, + "resync_query_count": 0, + "resync_query_error_count": 0, + "resync_query_time": 0, + "roleAccess_query_count": 1, + "roleAccess_query_error_count": 0, + "roleAccess_query_time": 197753953, + "sequences_query_count": 0, + "sequences_query_error_count": 0, + "sequences_query_time": 0, + "sessions_query_count": 0, + "sessions_query_error_count": 0, + "sessions_query_time": 0, + "tombstones_query_count": 0, + "tombstones_query_error_count": 0, + "tombstones_query_time": 0 + }, + "security": { + "auth_failed_count": 0, + "auth_success_count": 0, + "num_access_errors": 0, + "num_docs_rejected": 0, + "total_auth_time": 0 + }, + "shared_bucket_import": { + "import_count": 7303, + "import_cancel_cas": 0, + "import_error_count": 0, + "import_processing_time": 58213371539, + "import_high_seq": 7304, + "import_partitions": 16 + } + }, + "travel-sample": { + "cache": { + "abandoned_seqs": 0, + "chan_cache_active_revs": 0, + "chan_cache_bypass_count": 0, + "chan_cache_channels_added": 0, + "chan_cache_channels_evicted_inactive": 0, + "chan_cache_channels_evicted_nru": 0, + "chan_cache_compact_count": 0, + "chan_cache_compact_time": 0, + "chan_cache_hits": 0, + "chan_cache_max_entries": 0, + "chan_cache_misses": 0, + "chan_cache_num_channels": 0, + "chan_cache_pending_queries": 0, + "chan_cache_removal_revs": 0, + "chan_cache_tombstone_revs": 0, + "high_seq_cached": 31592, + "high_seq_stable": 31595, + "num_active_channels": 0, + "num_skipped_seqs": 0, + "pending_seq_len": 0, + "rev_cache_bypass": 0, + "rev_cache_hits": 0, + "rev_cache_misses": 0, + "skipped_seq_len": 0 + }, + "cbl_replication_pull": { + "attachment_pull_bytes": 0, + "attachment_pull_count": 0, + "max_pending": 260, + "num_replications_active": 0, + "num_pull_repl_active_continuous": 0, + "num_pull_repl_active_one_shot": 0, + "num_pull_repl_caught_up": 0, + "num_pull_repl_since_zero": 0, + "num_pull_repl_total_continuous": 0, + "num_pull_repl_total_one_shot": 0, + "request_changes_count": 0, + "request_changes_time": 0, + "rev_processing_time": 0, + "rev_send_count": 0, + "rev_send_latency": 0 + }, + "cbl_replication_push": { + "attachment_push_bytes": 0, + "attachment_push_count": 0, + "doc_push_count": 0, + "propose_change_count": 0, + "propose_change_time": 0, + "sync_function_count": 0, + "sync_function_time": 0, + "write_processing_time": 0 + }, + "database": { + "abandoned_seqs": 0, + "conflict_write_count": 0, + "crc32c_match_count": 0, + "dcp_caching_count": 31591, + "dcp_caching_time": 157384807512, + "dcp_received_count": 31591, + "dcp_received_time": 292348176079, + "doc_reads_bytes_blip": 0, + "doc_writes_bytes": 36196306, + "doc_writes_bytes_blip": 0, + "doc_writes_xattr_bytes": 7935238, + "high_seq_feed": 63187, + "num_doc_reads_blip": 0, + "num_doc_reads_rest": 0, + "num_doc_writes": 31591, + "num_replications_active": 0, + "num_replications_total": 0, + "num_tombstones_compacted": 0, + "sequence_assigned_count": 31592, + "sequence_get_count": 2, + "sequence_incr_count": 3162, + "sequence_released_count": 3, + "sequence_reserved_count": 31595, + "warn_channels_per_doc_count": 0, + "warn_grants_per_doc_count": 0, + "warn_xattr_size_count": 0, + "cache_feed": {}, + "import_feed": {} + }, + "gsi_views": { + "access_query_count": 1, + "access_query_error_count": 0, + "access_query_time": 116702828, + "allDocs_query_count": 0, + "allDocs_query_error_count": 0, + "allDocs_query_time": 0, + "channelsStar_query_count": 0, + "channelsStar_query_error_count": 0, + "channelsStar_query_time": 0, + "channels_query_count": 0, + "channels_query_error_count": 0, + "channels_query_time": 0, + "principals_query_count": 0, + "principals_query_error_count": 0, + "principals_query_time": 0, + "resync_query_count": 0, + "resync_query_error_count": 0, + "resync_query_time": 0, + "roleAccess_query_count": 1, + "roleAccess_query_error_count": 0, + "roleAccess_query_time": 120046831, + "sequences_query_count": 0, + "sequences_query_error_count": 0, + "sequences_query_time": 0, + "sessions_query_count": 0, + "sessions_query_error_count": 0, + "sessions_query_time": 0, + "tombstones_query_count": 0, + "tombstones_query_error_count": 0, + "tombstones_query_time": 0 + }, + "security": { + "auth_failed_count": 0, + "auth_success_count": 0, + "num_access_errors": 0, + "num_docs_rejected": 0, + "total_auth_time": 0 + }, + "shared_bucket_import": { + "import_count": 31591, + "import_cancel_cas": 0, + "import_error_count": 0, + "import_processing_time": 345830590715, + "import_high_seq": 31592, + "import_partitions": 16 + } + } + }, + "per_replication": { + "rep1": { + "sgr_num_docs_pushed": 0, + "sgr_num_attachment_bytes_transferred": 0, + "sgr_num_attachments_transferred": 0, + "sgr_docs_checked_sent": 0, + "sgr_num_docs_failed_to_push": 0 + } + } + } +} diff --git a/x-pack/metricbeat/module/syncgateway/db/_meta/data.json b/x-pack/metricbeat/module/syncgateway/db/_meta/data.json new file mode 100644 index 000000000000..b9abb0610641 --- /dev/null +++ b/x-pack/metricbeat/module/syncgateway/db/_meta/data.json @@ -0,0 +1,206 @@ +{ + "@timestamp": "2017-10-12T08:05:34.853Z", + "event": { + "dataset": "syncgateway.db", + "duration": 115000, + "module": "syncgateway" + }, + "metricset": { + "name": "db", + "period": 10000 + }, + "service": { + "address": "127.0.0.1:44949", + "type": "syncgateway" + }, + "syncgateway": { + "db": { + "cache": { + "channel": { + "hits": 0, + "misses": 0, + "revs": { + "active": 0, + "removal": 0, + "tombstone": 0 + } + }, + "revs": { + "hits": 0, + "misses": 0 + } + }, + "cbl": { + "replication": { + "pull": { + "active": { + "continuous": 0, + "count": 0, + "one_shot": 0 + }, + "attachment": { + "bytes": 0, + "count": 0 + }, + "caught_up": 0, + "request_changes": { + "count": 0, + "time": 0 + }, + "rev": { + "processing_time": 0, + "send": { + "count": 0, + "latency": 0 + } + }, + "since_zero": 0, + "total": { + "continuous": 0, + "one_shot": 0 + } + }, + "push": { + "attachment": { + "bytes": 0, + "count": 0 + }, + "doc_push_count": 0, + "propose_change": { + "count": 0, + "time": 0 + }, + "sync_function": { + "count": 0, + "time": 0 + }, + "write_processing_time": 0 + } + } + }, + "gsi": { + "views": { + "access": { + "query": { + "count": 1, + "error": { + "count": 0 + }, + "time": 214030402 + } + }, + "all_docs": { + "query": { + "count": 0, + "error": { + "count": 0 + }, + "time": 0 + } + }, + "channels": { + "query": { + "count": 0, + "error": { + "count": 0 + }, + "time": 0 + }, + "star": { + "query": { + "count": 0, + "error": { + "count": 0 + }, + "time": 0 + } + } + }, + "principals": { + "query": { + "count": 0, + "error": { + "count": 0 + }, + "time": 0 + } + }, + "resync": { + "query": { + "count": 0, + "error": { + "count": 0 + }, + "time": 0 + } + }, + "role_access": { + "query": { + "count": 1, + "error": { + "count": 0 + }, + "time": 197753953 + } + }, + "sequences": { + "query": { + "count": 0, + "error": { + "count": 0 + }, + "time": 0 + } + }, + "sessions": { + "query": { + "count": 0, + "error": { + "count": 0 + }, + "time": 0 + } + }, + "tombstones": { + "query": { + "count": 0, + "error": { + "count": 0 + }, + "time": 0 + } + } + } + }, + "metrics": { + "docs": { + "writes": { + "bytes": 2542924, + "conflict": { + "count": 0 + }, + "count": 7303 + } + }, + "replications": { + "active": 0, + "total": 0 + } + }, + "name": "beer-sample", + "security": { + "access_errors": { + "count": 0 + }, + "auth": { + "failed": { + "count": 0 + } + }, + "docs_rejected": { + "count": 0 + } + } + } + } +} \ No newline at end of file diff --git a/x-pack/metricbeat/module/syncgateway/db/_meta/docs.asciidoc b/x-pack/metricbeat/module/syncgateway/db/_meta/docs.asciidoc new file mode 100644 index 000000000000..f3185e37d4da --- /dev/null +++ b/x-pack/metricbeat/module/syncgateway/db/_meta/docs.asciidoc @@ -0,0 +1 @@ +SyncGateway `db` metriset contains the most relevant information of the module about the db metrics diff --git a/x-pack/metricbeat/module/syncgateway/db/_meta/fields.yml b/x-pack/metricbeat/module/syncgateway/db/_meta/fields.yml new file mode 100644 index 000000000000..4d7d4393e190 --- /dev/null +++ b/x-pack/metricbeat/module/syncgateway/db/_meta/fields.yml @@ -0,0 +1,380 @@ +- name: syncgateway + type: group + description: > + Couchbase Sync Gateway metrics. + release: beta + fields: + - name: name + type: keyword + description: > + Name of the database on when field `couchbase.syncgateway.type` is `db_stats`. + - name: metrics + type: group + description: Metrics of all databases contained in the config file of the SyncGateway instance. + fields: + - name: docs + type: group + fields: + - name: writes + type: group + fields: + - name: conflict.count + type: long + - name: count + type: long + - name: bytes + type: long + - name: replications + type: group + fields: + - name: active + type: long + description: Number of active replications + - name: total + type: long + description: Total number of replications (active or not) + - name: gsi + type: group + fields: + - name: views + type: group + fields: + - name: tombstones + type: group + fields: + - name: query + type: group + fields: + - name: count + type: double + - name: time + type: double + - name: error.count + type: double + - name: access + type: group + fields: + - name: query + type: group + fields: + - name: count + type: double + - name: error + type: group + fields: + - name: count + type: double + - name: time + type: double + - name: channels + type: group + fields: + - name: query + type: group + fields: + - name: count + type: double + - name: error + type: group + fields: + - name: count + type: double + - name: time + type: double + - name: star + type: group + fields: + - name: query + type: group + fields: + - name: time + type: double + - name: count + type: double + - name: error + type: group + fields: + - name: count + type: double + - name: role_access + type: group + fields: + - name: query + type: group + fields: + - name: count + type: double + - name: error + type: group + fields: + - name: count + type: double + - name: time + type: double + - name: sequences + type: group + fields: + - name: query + type: group + fields: + - name: count + type: double + - name: error + type: group + fields: + - name: count + type: double + - name: time + type: double + - name: all_docs + type: group + fields: + - name: query + type: group + fields: + - name: count + type: double + - name: error + type: group + fields: + - name: count + type: double + - name: time + type: double + - name: principals + type: group + fields: + - name: query + type: group + fields: + - name: count + type: double + - name: error + type: group + fields: + - name: count + type: double + - name: time + type: double + - name: resync + type: group + fields: + - name: query + type: group + fields: + - name: count + type: double + - name: error + type: group + fields: + - name: count + type: double + - name: time + type: double + - name: sessions + type: group + fields: + - name: query + type: group + fields: + - name: count + type: double + - name: error + type: group + fields: + - name: count + type: double + - name: time + type: double + - name: security + type: group + fields: + - name: access_errors + type: group + fields: + - name: count + type: double + - name: auth + type: group + fields: + - name: failed + type: group + fields: + - name: count + type: double + - name: docs_rejected + type: group + fields: + - name: count + type: double + - name: cache + type: group + fields: + - name: channel + type: group + fields: + - name: revs + type: group + fields: + - name: active + type: double + - name: removal + type: double + - name: tombstone + type: double + - name: hits + type: double + - name: misses + type: double + - name: revs + type: group + fields: + - name: hits + type: double + - name: misses + type: double + - name: cbl + type: group + fields: + - name: replication + type: group + fields: + - name: pull + type: group + fields: + - name: caught_up + type: double + - name: since_zero + type: double + - name: total + type: group + fields: + - name: continuous + type: double + - name: one_shot + type: double + - name: active + type: group + fields: + - name: continuous + type: double + - name: count + type: double + - name: one_shot + type: double + - name: attachment + type: group + fields: + - name: bytes + type: long + - name: count + type: long + - name: request_changes + type: group + fields: + - name: count + type: double + - name: time + type: double + - name: rev + type: group + fields: + - name: processing_time + type: double + - name: send + type: group + fields: + - name: count + type: double + - name: latency + type: double + - name: push + type: group + fields: + - name: attachment + type: group + fields: + - name: bytes + type: double + - name: count + type: double + - name: doc_push_count + type: double + - name: propose_change + type: group + fields: + - name: count + type: double + - name: time + type: double + - name: sync_function + type: group + fields: + - name: count + type: double + - name: time + type: double + - name: write_processing_time + type: double + - name: memstats + type: group + description: Dumps a large amount of information about the memory heap and garbage collector + fields: + - name: BuckHashSys + type: double + - name: Mallocs + type: double + - name: PauseTotalNs + type: double + - name: TotalAlloc + type: double + - name: Alloc + type: double + - name: GCSys + type: double + - name: LastGC + type: double + - name: MSpanSys + type: double + - name: GCCPUFraction + type: double + - name: HeapReleased + type: double + - name: HeapSys + type: double + - name: DebugGC + type: long + - name: HeapIdle + type: double + - name: Lookups + type: double + - name: HeapObjects + type: double + - name: MSpanInuse + type: double + - name: NumForcedGC + type: double + - name: OtherSys + type: double + - name: Frees + type: double + - name: NextGC + type: double + - name: StackInuse + type: double + - name: Sys + type: double + - name: NumGC + type: double + - name: EnableGC + type: long + - name: HeapAlloc + type: double + - name: MCacheInuse + type: double + - name: MCacheSys + type: double + - name: HeapInuse + type: double + - name: StackSys + type: double diff --git a/x-pack/metricbeat/module/syncgateway/db/data.go b/x-pack/metricbeat/module/syncgateway/db/data.go new file mode 100644 index 000000000000..d8fb8669ffce --- /dev/null +++ b/x-pack/metricbeat/module/syncgateway/db/data.go @@ -0,0 +1,200 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package db + +import ( + s "github.com/elastic/beats/v7/libbeat/common/schema" + c "github.com/elastic/beats/v7/libbeat/common/schema/mapstriface" + "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/elastic/beats/v7/x-pack/metricbeat/module/syncgateway" +) + +type SgResponse struct { + SyncgatewayChangeCache struct { + MaxPending float64 `json:"maxPending"` + } `json:"syncGateway_changeCache"` + Syncgateway Syncgateway `json:"syncgateway"` + MemStats map[string]interface{} `json:"memstats"` +} + +type Syncgateway struct { + Global struct { + ResourceUtilization map[string]interface{} `json:"resource_utilization"` + } `json:"global"` + PerDb map[string]map[string]interface{} `json:"per_db"` + PerReplication map[string]map[string]interface{} `json:"per_replication"` +} + +var ( + dbSchema = s.Schema{ + "cache": c.Dict("cache", s.Schema{ + "channel": s.Object{ + "revs": s.Object{ + "active": c.Float("chan_cache_active_revs"), + "removal": c.Float("chan_cache_removal_revs"), + "tombstone": c.Float("chan_cache_tombstone_revs"), + }, + "hits": c.Float("chan_cache_hits"), + "misses": c.Float("chan_cache_misses"), + }, + "revs": s.Object{ + "hits": c.Float("rev_cache_hits"), + "misses": c.Float("rev_cache_misses"), + }, + }), + "metrics": c.Dict("database", s.Schema{ + "replications": s.Object{ + "active": c.Float("num_replications_active"), + "total": c.Float("num_replications_total"), + }, + "docs": s.Object{ + "writes": s.Object{ + "count": c.Float("num_doc_writes"), + "bytes": c.Float("doc_writes_bytes"), + "conflict": s.Object{"count": c.Float("conflict_write_count")}, + }, + }, + }), + "security": c.Dict("security", s.Schema{ + "auth": s.Object{ + "failed": s.Object{"count": c.Float("auth_failed_count")}, + }, + "access_errors": s.Object{"count": c.Float("num_access_errors")}, + "docs_rejected": s.Object{"count": c.Float("num_docs_rejected")}, + }), + "cbl": s.Object{ + "replication": s.Object{ + "pull": c.Dict("cbl_replication_pull", s.Schema{ + "attachment": s.Object{ + "bytes": c.Float("attachment_pull_bytes"), + "count": c.Float("attachment_pull_count"), + }, + "active": s.Object{ + "count": c.Float("num_replications_active"), + "continuous": c.Float("num_pull_repl_active_continuous"), + "one_shot": c.Float("num_pull_repl_active_one_shot"), + }, + "total": s.Object{ + "continuous": c.Float("num_pull_repl_total_continuous"), + "one_shot": c.Float("num_pull_repl_total_one_shot"), + }, + "caught_up": c.Float("num_pull_repl_caught_up"), + "since_zero": c.Float("num_pull_repl_since_zero"), + "request_changes": s.Object{ + "count": c.Float("request_changes_count"), + "time": c.Float("request_changes_time"), + }, + "rev": s.Object{ + "processing_time": c.Float("rev_processing_time"), + "send": s.Object{ + "count": c.Float("rev_send_count"), + "latency": c.Float("rev_send_latency"), + }, + }, + }), + "push": c.Dict("cbl_replication_push", s.Schema{ + "write_processing_time": c.Float("write_processing_time"), + "doc_push_count": c.Float("doc_push_count"), + "attachment": s.Object{ + "bytes": c.Float("attachment_push_bytes"), + "count": c.Float("attachment_push_count"), + }, + "propose_change": s.Object{ + "count": c.Float("propose_change_count"), + "time": c.Float("propose_change_time"), + }, + "sync_function": s.Object{ + "count": c.Float("sync_function_count"), + "time": c.Float("sync_function_time"), + }, + }), + }, + }, + "gsi": s.Object{ + "views": c.Dict("gsi_views", s.Schema{ + "access": s.Object{ + "query": s.Object{ + "count": c.Float("access_query_count"), + "error": s.Object{"count": c.Float("access_query_error_count")}, + "time": c.Float("access_query_time"), + }, + }, + "all_docs": s.Object{ + "query": s.Object{ + "count": c.Float("allDocs_query_count"), + "error": s.Object{"count": c.Float("allDocs_query_error_count")}, + "time": c.Float("allDocs_query_time"), + }, + }, + "channels": s.Object{ + "star": s.Object{ + "query": s.Object{ + "count": c.Float("channelsStar_query_count"), + "error": s.Object{"count": c.Float("channelsStar_query_error_count")}, + "time": c.Float("channelsStar_query_time"), + }, + }, + "query": s.Object{ + "count": c.Float("channels_query_count"), + "error": s.Object{"count": c.Float("channels_query_error_count")}, + "time": c.Float("channels_query_time"), + }, + }, + "principals": s.Object{ + "query": s.Object{ + "count": c.Float("principals_query_count"), + "error": s.Object{"count": c.Float("principals_query_error_count")}, + "time": c.Float("principals_query_time"), + }, + }, + "resync": s.Object{ + "query": s.Object{ + "count": c.Float("resync_query_count"), + "error": s.Object{"count": c.Float("resync_query_error_count")}, + "time": c.Float("resync_query_time"), + }, + }, + "role_access": s.Object{ + "query": s.Object{ + "count": c.Float("roleAccess_query_count"), + "error": s.Object{"count": c.Float("roleAccess_query_error_count")}, + "time": c.Float("roleAccess_query_time"), + }, + }, + "sequences": s.Object{ + "query": s.Object{ + "count": c.Float("sequences_query_count"), + "error": s.Object{"count": c.Float("sequences_query_error_count")}, + "time": c.Float("sequences_query_time"), + }, + }, + "sessions": s.Object{ + "query": s.Object{ + "count": c.Float("sessions_query_count"), + "error": s.Object{"count": c.Float("sessions_query_error_count")}, + "time": c.Float("sessions_query_time"), + }, + }, + "tombstones": s.Object{ + "query": s.Object{ + "count": c.Float("tombstones_query_count"), + "error": s.Object{"count": c.Float("tombstones_query_error_count")}, + "time": c.Float("tombstones_query_time"), + }, + }, + }), + }, + } +) + +func eventMapping(r mb.ReporterV2, content *syncgateway.SgResponse) { + for dbName, db := range content.Syncgateway.PerDb { + dbData, _ := dbSchema.Apply(db) + dbData.Put("name", dbName) + r.Event(mb.Event{ + MetricSetFields: dbData, + }) + } +} diff --git a/x-pack/metricbeat/module/syncgateway/db/data_test.go b/x-pack/metricbeat/module/syncgateway/db/data_test.go new file mode 100644 index 000000000000..847d2a622595 --- /dev/null +++ b/x-pack/metricbeat/module/syncgateway/db/data_test.go @@ -0,0 +1,44 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package db + +import ( + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/assert" + + mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" + "github.com/elastic/beats/v7/x-pack/metricbeat/module/syncgateway" +) + +func TestData(t *testing.T) { + mux := syncgateway.CreateTestMuxer() + server := httptest.NewServer(mux) + defer server.Close() + + f := mbtest.NewReportingMetricSetV2Error(t, syncgateway.GetConfig([]string{"db"}, server.URL)) + if err := mbtest.WriteEventsReporterV2Error(f, t, ""); err != nil { + t.Fatal("write", err) + } +} + +func TestFetch(t *testing.T) { + t.Skip("Skipping test because it seems like 'TestMetricsetFieldsDocumented' function, used here in this test; has some kind of bug") + mux := syncgateway.CreateTestMuxer() + server := httptest.NewServer(mux) + defer server.Close() + + config := syncgateway.GetConfig([]string{"db"}, server.URL) + metricSet := mbtest.NewReportingMetricSetV2Error(t, config) + + events, errs := mbtest.ReportingFetchV2Error(metricSet) + if len(errs) > 0 { + t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) + } + + assert.NotEmpty(t, events) + mbtest.TestMetricsetFieldsDocumented(t, metricSet, events) +} diff --git a/x-pack/metricbeat/module/syncgateway/db/db.go b/x-pack/metricbeat/module/syncgateway/db/db.go new file mode 100644 index 000000000000..eae750f9badf --- /dev/null +++ b/x-pack/metricbeat/module/syncgateway/db/db.go @@ -0,0 +1,75 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package db + +import ( + "fmt" + + "github.com/elastic/beats/v7/metricbeat/helper" + "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/elastic/beats/v7/metricbeat/mb/parse" + "github.com/elastic/beats/v7/x-pack/metricbeat/module/syncgateway" +) + +const ( + defaultScheme = "http" + defaultPath = "/_expvar" +) + +var ( + hostParser = parse.URLHostParserBuilder{ + DefaultScheme: defaultScheme, + DefaultPath: defaultPath, + }.Build() +) + +// init registers the MetricSet with the central registry. +// The New method will be called after the setup of the module and before starting to fetch data +func init() { + mb.Registry.MustAddMetricSet("syncgateway", "db", New, + mb.WithHostParser(hostParser), + mb.DefaultMetricSet(), + ) +} + +// MetricSet type defines all fields of the MetricSet +type MetricSet struct { + mb.BaseMetricSet + http *helper.HTTP + mod syncgateway.Module +} + +// New create a new instance of the MetricSet +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + mod, ok := base.Module().(syncgateway.Module) + if !ok { + return nil, fmt.Errorf("error trying to type assert syncgateway.Module. Base module must be composed of syncgateway.Module") + } + + http, err := helper.NewHTTP(base) + if err != nil { + return nil, err + } + + return &MetricSet{ + BaseMetricSet: base, + http: http, + mod: mod, + }, nil +} + +// Fetch methods implements the data gathering and data conversion to the right +// format. It publishes the event which is then forwarded to the output. In case +// of an error set the Error field of mb.Event or simply call report.Error(). +func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { + response, err := m.mod.GetSyncgatewayResponse(m.http) + if err != nil { + return err + } + + eventMapping(reporter, response) + + return nil +} diff --git a/x-pack/metricbeat/module/syncgateway/doc.go b/x-pack/metricbeat/module/syncgateway/doc.go new file mode 100644 index 000000000000..2654d21ebd35 --- /dev/null +++ b/x-pack/metricbeat/module/syncgateway/doc.go @@ -0,0 +1,5 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package syncgateway diff --git a/x-pack/metricbeat/module/syncgateway/fields.go b/x-pack/metricbeat/module/syncgateway/fields.go new file mode 100644 index 000000000000..365091180c47 --- /dev/null +++ b/x-pack/metricbeat/module/syncgateway/fields.go @@ -0,0 +1,23 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// Code generated by beats/dev-tools/cmd/asset/asset.go - DO NOT EDIT. + +package syncgateway + +import ( + "github.com/elastic/beats/v7/libbeat/asset" +) + +func init() { + if err := asset.SetFields("metricbeat", "syncgateway", asset.ModuleFieldsPri, AssetSyncgateway); err != nil { + panic(err) + } +} + +// AssetSyncgateway returns asset data. +// This is the base64 encoded gzipped contents of module/syncgateway. +func AssetSyncgateway() string { + return "eJzsXMty27gS3fsrurK6dxF9gBe36o4zcVw1cVLjmTUNgi0SEQkweFijfP0UqRdFAQT4ij011CILKTg4fbqBbrz8Hja4uwW14zQlGrdkdwOgmc7xFt497Ti933/77gYgQUUlKzUT/BYav0GBWjKqbgAk5kgU3kKMmtwArBnmibq9AQB4D5wU2O6r+uhdibeQSmHKwzcXXf3v8CXAc6PtM1DBNWFcgc4QGF8LWZCqBRCegNJEM6UZVbCWomjyXR3wmuS6CdpJdhCtPnfC0CwmCuu+oSXWqvF/26odP22CTZLVvxc/HBlucLcVMmn91sGz+jySAkGsayUToklNW3DYZsj3NOCZHu1ZNQRaVZ0+A1PwnMRRJbp6XlkJn4PkmnNb1SvGn/eNK4okz08U1TEGMAHGa/ZU8DVLYc3yk0HNUGVcacIprlq92aRusk/EFfVu+l2gTeCtZBpt0H54XxfNbipZckb1igrDtfO/H7vMBU8DMCeDinduETxQRwiJZc5oPQHM4ylCNXtpj7lgWy/C+dEUMco6mmtQH/kzCS00ySfh8EeFBPzEpEkB/nPgJSRwof9rHdCpYsGD2Te8Xhhu5/GaFkWstOA/YYx9Nyh33iDu6imkN+gxAs+9JsLEuSt4L0E1u8osozFRSiG9c08Q9Hk4UlSLU/s4IAjUZwoEmgM9TYLXitUTyYxwjvkSUktITSHVaTWhSZdK0zvbF17hvYb2DP3kgz5ugTl9Dj3DGXpJBz3kgwFmQp/ZTYocoyVnLhPcpDlT4XeDnC7F9RJTk8UUyfPIsd0CS0hdgy4h5Q2pUjJOWUmW4n4JqsmCSqLacboE1BJQIzHPxZRSzg3gJaSuQJeQusI8hxI1kum2k4dv0O9XjlEt+Twb9V16dmp4omh0NguzNWE5tg9Uw+B9XUDv4zSPCFXhHEn8hlRbKb+Sn06NCc3sx9ZDgvKwYzuLmRJf5p+IO48Uoe9Oo8RCvDhPB3vDnU7OxgAewTKmu+UMwCiYUp6dBs/ocDh1fLS8nnmnsRC3HT98XDWOgmdRqzR59xn2JNMpMWmmo868328jn3GK0Q+UYroR5j7Mh5nqMK4ZN8J0XbYItqOJLDhGKhOTFHk9p8d/hkCzlMCzqK41oVmBAcXIlMr7rgBB2I0iGCR50CUlid8NKh1VZUcacF3pzS+gpj7qlPjyU0UppahWJYyn0RyXVhRyV9XfzyZ4C8vLJnhONHLqOyQOgj/ndGVbfcG09fJbn5peIx80ln5R5YVo1EKyjVpKUQqFh0lvmfPaeGrHabQ2nDqq5UvQf5089fXmKHyi9q5zCizqW+bBi52LS68fTFEqIJATmSKQopIQxPry9UAsjK4vjhdYCLmDDElZvylIiYxJikBFniPVV5uPvnXVL4ZuPhGVPe3cq1DPCvYzyfOuq+ie5l+JUVjf/H0cjFE3/39FYyjCqMb3dyPk+40ofX83WPynkvARvd/f3X3986MkrrkiCOQTkvL3/YMR9wZfAMYIOz5gbNIOGTvv6Vd9PyTWqSPMhUJsTDmYe9X9l/gbUuuGTXgYPHCjBtvwaIqPQlJMhsfiF52hHOHDjxKtNU0Yf/xrxDB60oRuRuk3wu5HUwxn/isncY5jIn/U1Pf5jtAMRym3hxihXz18R/mu8n4PAo3EL+SoB3mXTxbr1H58jAcPGpLZaoMh7/vctYK3RLLXCN5mHbWBt62zJvC2HNTIVgN4G1lzv19Me84PoOjO9d7GHTk+qO0Avvac7pjVPLnc7wprDg8yzZ67w9xom7e8Ld252tvUkaO97Wy52c/TlpO9rZy52N+yv1223Ott5Mi5AZE5aGpx59jApgN0ceXUMO8Fdth9uDcqmZYom7BTPXGf5MW4xJxoTGDLdFa/G28y7fsKvHP/cfy5qJaEqzVK+RNum4x4cX3+uB41n2Tad9M0DNZCgs6YOvqh7YNrqhO8M2+/fc5wf/jaeP98Jn3Jtz72vQhvpYnUmFwTb17CmSVAaIZ0g0mk3DvgAbtzXikSQc1eiEOHtdMOx159BIHWAcGbuUU1NlgmVQhC75rNwV9nRB96BS0gxoOjQo04Emf2QjXgr5E8fDj+oY6rGeGcs5Qw8vKZ0aiMleYiJvkJF4xmOfvRTopDklZ9Q9J6BOOpWFIhhdGMo4oylmbRlmiUBZGbfjDcFNEZql/bwzZ9cML13s8rTVSipMh1ZFfYQwyu1v+RRMWSrgTsts7EEUe7V4bdkqK24+7xs3xXag4QypEcXoXXqXBKCsYr9SMbyEw+GK5SxyCNeh9A+WzIkNg8Mt5fxLHX2GFnG4LZN+r7IDj2CntASPdphwflNPRJ14blGI09rzW6Q7Bavc3CagLRlXWHNtSyju1d59hSO6WxiA7zvO1upAdgSyT3Zd6/AwAA//8fni1d" +} diff --git a/x-pack/metricbeat/module/syncgateway/memory/_meta/data.json b/x-pack/metricbeat/module/syncgateway/memory/_meta/data.json new file mode 100644 index 000000000000..d10b990f9673 --- /dev/null +++ b/x-pack/metricbeat/module/syncgateway/memory/_meta/data.json @@ -0,0 +1,49 @@ +{ + "@timestamp": "2017-10-12T08:05:34.853Z", + "event": { + "dataset": "syncgateway.memory", + "duration": 115000, + "module": "syncgateway" + }, + "metricset": { + "name": "memory", + "period": 10000 + }, + "service": { + "address": "127.0.0.1:39195", + "type": "syncgateway" + }, + "syncgateway": { + "memory": { + "Alloc": 159398816, + "BuckHashSys": 1759627, + "DebugGC": false, + "EnableGC": true, + "Frees": 55790048, + "GCCPUFraction": 0.008328526565497294, + "GCSys": 13555712, + "HeapAlloc": 159398816, + "HeapIdle": 67780608, + "HeapInuse": 264388608, + "HeapObjects": 1877567, + "HeapReleased": 26927104, + "HeapSys": 332169216, + "LastGC": 1620310398040906500, + "Lookups": 0, + "MCacheInuse": 20832, + "MCacheSys": 32768, + "MSpanInuse": 4030632, + "MSpanSys": 4358144, + "Mallocs": 57667615, + "NextGC": 265304208, + "NumForcedGC": 0, + "NumGC": 51, + "OtherSys": 2457453, + "PauseTotalNs": 88693194, + "StackInuse": 3375104, + "StackSys": 3375104, + "Sys": 357708024, + "TotalAlloc": 2719416432 + } + } +} \ No newline at end of file diff --git a/x-pack/metricbeat/module/syncgateway/memory/_meta/docs.asciidoc b/x-pack/metricbeat/module/syncgateway/memory/_meta/docs.asciidoc new file mode 100644 index 000000000000..69772dee2ca3 --- /dev/null +++ b/x-pack/metricbeat/module/syncgateway/memory/_meta/docs.asciidoc @@ -0,0 +1 @@ +SyncGateway `memory` metriset contains detailed information about the memory usage of the SyncGateway Go's process. diff --git a/x-pack/metricbeat/module/syncgateway/memory/_meta/fields.yml b/x-pack/metricbeat/module/syncgateway/memory/_meta/fields.yml new file mode 100644 index 000000000000..824a590fbdb5 --- /dev/null +++ b/x-pack/metricbeat/module/syncgateway/memory/_meta/fields.yml @@ -0,0 +1,64 @@ +- name: memory + type: group + description: > + SyncGateway memory metrics. It dumps a large amount of information about the memory heap and garbage collector + release: beta + fields: + - name: BuckHashSys + type: double + - name: Mallocs + type: double + - name: PauseTotalNs + type: double + - name: TotalAlloc + type: double + - name: Alloc + type: double + - name: GCSys + type: double + - name: LastGC + type: double + - name: MSpanSys + type: double + - name: GCCPUFraction + type: double + - name: HeapReleased + type: double + - name: HeapSys + type: double + - name: DebugGC + type: long + - name: HeapIdle + type: double + - name: Lookups + type: double + - name: HeapObjects + type: double + - name: MSpanInuse + type: double + - name: NumForcedGC + type: double + - name: OtherSys + type: double + - name: Frees + type: double + - name: NextGC + type: double + - name: StackInuse + type: double + - name: Sys + type: double + - name: NumGC + type: double + - name: EnableGC + type: long + - name: HeapAlloc + type: double + - name: MCacheInuse + type: double + - name: MCacheSys + type: double + - name: HeapInuse + type: double + - name: StackSys + type: double diff --git a/x-pack/metricbeat/module/syncgateway/memory/data.go b/x-pack/metricbeat/module/syncgateway/memory/data.go new file mode 100644 index 000000000000..b51f91f30067 --- /dev/null +++ b/x-pack/metricbeat/module/syncgateway/memory/data.go @@ -0,0 +1,20 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package memory + +import ( + "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/elastic/beats/v7/x-pack/metricbeat/module/syncgateway" +) + +func eventMapping(r mb.ReporterV2, content *syncgateway.SgResponse) { + delete(content.MemStats, "BySize") + delete(content.MemStats, "PauseNs") + delete(content.MemStats, "PauseEnd") + + r.Event(mb.Event{ + MetricSetFields: content.MemStats, + }) +} diff --git a/x-pack/metricbeat/module/syncgateway/memory/data_test.go b/x-pack/metricbeat/module/syncgateway/memory/data_test.go new file mode 100644 index 000000000000..ad6ab9d98ff0 --- /dev/null +++ b/x-pack/metricbeat/module/syncgateway/memory/data_test.go @@ -0,0 +1,45 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package memory + +import ( + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/assert" + + mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" + "github.com/elastic/beats/v7/x-pack/metricbeat/module/syncgateway" +) + +func TestData(t *testing.T) { + mux := syncgateway.CreateTestMuxer() + server := httptest.NewServer(mux) + defer server.Close() + + config := syncgateway.GetConfig([]string{"memory"}, server.URL) + + f := mbtest.NewReportingMetricSetV2Error(t, config) + if err := mbtest.WriteEventsReporterV2Error(f, t, ""); err != nil { + t.Fatal("write", err) + } +} + +func TestFetch(t *testing.T) { + mux := syncgateway.CreateTestMuxer() + server := httptest.NewServer(mux) + defer server.Close() + + config := syncgateway.GetConfig([]string{"memory"}, server.URL) + metricSet := mbtest.NewReportingMetricSetV2Error(t, config) + + events, errs := mbtest.ReportingFetchV2Error(metricSet) + if len(errs) > 0 { + t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) + } + + assert.NotEmpty(t, events) + mbtest.TestMetricsetFieldsDocumented(t, metricSet, events) +} diff --git a/x-pack/metricbeat/module/syncgateway/memory/memory.go b/x-pack/metricbeat/module/syncgateway/memory/memory.go new file mode 100644 index 000000000000..0edc000d0f35 --- /dev/null +++ b/x-pack/metricbeat/module/syncgateway/memory/memory.go @@ -0,0 +1,74 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package memory + +import ( + "fmt" + + "github.com/elastic/beats/v7/metricbeat/helper" + "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/elastic/beats/v7/metricbeat/mb/parse" + "github.com/elastic/beats/v7/x-pack/metricbeat/module/syncgateway" +) + +const ( + defaultScheme = "http" + defaultPath = "/_expvar" +) + +var ( + hostParser = parse.URLHostParserBuilder{ + DefaultScheme: defaultScheme, + DefaultPath: defaultPath, + }.Build() +) + +// init registers the MetricSet with the central registry. +// The New method will be called after the setup of the module and before starting to fetch data +func init() { + mb.Registry.MustAddMetricSet("syncgateway", "memory", New, + mb.WithHostParser(hostParser), + ) +} + +// MetricSet type defines all fields of the MetricSet +type MetricSet struct { + mb.BaseMetricSet + http *helper.HTTP + mod syncgateway.Module +} + +// New create a new instance of the MetricSet +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + mod, ok := base.Module().(syncgateway.Module) + if !ok { + return nil, fmt.Errorf("error trying to type assert syncgateway.Module. Base module must be composed of syncgateway.Module") + } + + http, err := helper.NewHTTP(base) + if err != nil { + return nil, err + } + + return &MetricSet{ + BaseMetricSet: base, + http: http, + mod: mod, + }, nil +} + +// Fetch methods implements the data gathering and data conversion to the right +// format. It publishes the event which is then forwarded to the output. In case +// of an error set the Error field of mb.Event or simply call report.Error(). +func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { + response, err := m.mod.GetSyncgatewayResponse(m.http) + if err != nil { + return err + } + + eventMapping(reporter, response) + + return nil +} diff --git a/x-pack/metricbeat/module/syncgateway/replication/_meta/docs.asciidoc b/x-pack/metricbeat/module/syncgateway/replication/_meta/docs.asciidoc new file mode 100644 index 000000000000..691e7df6b779 --- /dev/null +++ b/x-pack/metricbeat/module/syncgateway/replication/_meta/docs.asciidoc @@ -0,0 +1 @@ +SyncGateway `replication` metriset contains information about the replicas. Note that you won't see any data if you don't have any replicas configured on your Couchbase buckets. diff --git a/x-pack/metricbeat/module/syncgateway/replication/_meta/fields.yml b/x-pack/metricbeat/module/syncgateway/replication/_meta/fields.yml new file mode 100644 index 000000000000..79f4e909efbb --- /dev/null +++ b/x-pack/metricbeat/module/syncgateway/replication/_meta/fields.yml @@ -0,0 +1,40 @@ +- name: replication + type: group + description: > + SyncGateway per replication metrics. + release: beta + fields: + - name: metrics + type: group + description: Metrics related with data replication. + fields: + - name: attachment + type: group + fields: + - name: transferred + type: group + fields: + - name: bytes + type: long + description: Number of attachment bytes transferred for this replica. + - name: count + type: long + description: The total number of attachments transferred since replication started. + - name: docs + type: group + fields: + - name: checked_sent + type: double + description: The total number of documents checked for changes since replication started. + - name: pushed + type: group + fields: + - name: count + type: long + description: The total number of documents checked for changes since replication started. + - name: failed + type: long + description: The total number of documents that failed to be pushed since replication started. + - name: id + type: keyword + description: ID of the replica. diff --git a/x-pack/metricbeat/module/syncgateway/replication/data.go b/x-pack/metricbeat/module/syncgateway/replication/data.go new file mode 100644 index 000000000000..55b68b7c213e --- /dev/null +++ b/x-pack/metricbeat/module/syncgateway/replication/data.go @@ -0,0 +1,41 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package replication + +import ( + "github.com/elastic/beats/v7/libbeat/common" + s "github.com/elastic/beats/v7/libbeat/common/schema" + c "github.com/elastic/beats/v7/libbeat/common/schema/mapstriface" + "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/elastic/beats/v7/x-pack/metricbeat/module/syncgateway" +) + +var replicationSchema = s.Schema{ + "docs": s.Object{ + "pushed": s.Object{ + "count": c.Int("sgr_num_docs_pushed"), + "failed": c.Int("sgr_num_docs_failed_to_push"), + }, + "checked_sent": c.Int("sgr_docs_checked_sent"), + }, + "attachment": s.Object{ + "transferred": s.Object{ + "bytes": c.Int("sgr_num_attachment_bytes_transferred"), + "count": c.Int("sgr_num_attachments_transferred"), + }, + }, +} + +func eventMapping(r mb.ReporterV2, content *syncgateway.SgResponse) { + for replID, replData := range content.Syncgateway.PerReplication { + replData, _ := replicationSchema.Apply(replData) + r.Event(mb.Event{ + MetricSetFields: common.MapStr{ + "id": replID, + "metrics": replData, + }, + }) + } +} diff --git a/x-pack/metricbeat/module/syncgateway/replication/data_test.go b/x-pack/metricbeat/module/syncgateway/replication/data_test.go new file mode 100644 index 000000000000..a622bb656e3a --- /dev/null +++ b/x-pack/metricbeat/module/syncgateway/replication/data_test.go @@ -0,0 +1,43 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package replication + +import ( + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/assert" + + mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" + "github.com/elastic/beats/v7/x-pack/metricbeat/module/syncgateway" +) + +func TestData(t *testing.T) { + mux := syncgateway.CreateTestMuxer() + server := httptest.NewServer(mux) + defer server.Close() + + f := mbtest.NewReportingMetricSetV2Error(t, syncgateway.GetConfig([]string{"replication"}, server.URL)) + if err := mbtest.WriteEventsReporterV2Error(f, t, ""); err != nil { + t.Fatal("write", err) + } +} + +func TestFetch(t *testing.T) { + mux := syncgateway.CreateTestMuxer() + server := httptest.NewServer(mux) + defer server.Close() + + config := syncgateway.GetConfig([]string{"replication"}, server.URL) + metricSet := mbtest.NewReportingMetricSetV2Error(t, config) + + events, errs := mbtest.ReportingFetchV2Error(metricSet) + if len(errs) > 0 { + t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) + } + + assert.NotEmpty(t, events) + mbtest.TestMetricsetFieldsDocumented(t, metricSet, events) +} diff --git a/x-pack/metricbeat/module/syncgateway/replication/replication.go b/x-pack/metricbeat/module/syncgateway/replication/replication.go new file mode 100644 index 000000000000..3ce9be667054 --- /dev/null +++ b/x-pack/metricbeat/module/syncgateway/replication/replication.go @@ -0,0 +1,74 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package replication + +import ( + "fmt" + + "github.com/elastic/beats/v7/metricbeat/helper" + "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/elastic/beats/v7/metricbeat/mb/parse" + "github.com/elastic/beats/v7/x-pack/metricbeat/module/syncgateway" +) + +const ( + defaultScheme = "http" + defaultPath = "/_expvar" +) + +var ( + hostParser = parse.URLHostParserBuilder{ + DefaultScheme: defaultScheme, + DefaultPath: defaultPath, + }.Build() +) + +// init registers the MetricSet with the central registry. +// The New method will be called after the setup of the module and before starting to fetch data +func init() { + mb.Registry.MustAddMetricSet("syncgateway", "replication", New, + mb.WithHostParser(hostParser), + ) +} + +// MetricSet type defines all fields of the MetricSet +type MetricSet struct { + mb.BaseMetricSet + http *helper.HTTP + mod syncgateway.Module +} + +// New create a new instance of the MetricSet +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + mod, ok := base.Module().(syncgateway.Module) + if !ok { + return nil, fmt.Errorf("error trying to type assert syncgateway.Module. Base module must be composed of syncgateway.Module") + } + + http, err := helper.NewHTTP(base) + if err != nil { + return nil, err + } + + return &MetricSet{ + BaseMetricSet: base, + http: http, + mod: mod, + }, nil +} + +// Fetch methods implements the data gathering and data conversion to the right +// format. It publishes the event which is then forwarded to the output. In case +// of an error set the Error field of mb.Event or simply call report.Error(). +func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { + response, err := m.mod.GetSyncgatewayResponse(m.http) + if err != nil { + return err + } + + eventMapping(reporter, response) + + return nil +} diff --git a/x-pack/metricbeat/module/syncgateway/resources/_meta/data.json b/x-pack/metricbeat/module/syncgateway/resources/_meta/data.json new file mode 100644 index 000000000000..d9968ce35239 --- /dev/null +++ b/x-pack/metricbeat/module/syncgateway/resources/_meta/data.json @@ -0,0 +1,57 @@ +{ + "@timestamp": "2017-10-12T08:05:34.853Z", + "event": { + "dataset": "syncgateway.resources", + "duration": 115000, + "module": "syncgateway" + }, + "metricset": { + "name": "resources", + "period": 10000 + }, + "service": { + "address": "127.0.0.1:37571", + "type": "syncgateway" + }, + "syncgateway": { + "resources": { + "admin_net_bytes": { + "recv": 186313933, + "sent": 67278955 + }, + "error_count": 0, + "go_memstats": { + "heap": { + "alloc": 143386592, + "idle": 81616896, + "inuse": 250585088, + "released": 26927104 + }, + "pause": { + "ns": 88693194 + }, + "stack": { + "inuse": 3342336, + "sys": 3342336 + }, + "sys": 357708024 + }, + "goroutines_high_watermark": 311, + "num_goroutines": 308, + "process": { + "cpu_percent_utilization": 0, + "memory_resident": 335306752 + }, + "pub_net": { + "recv": { + "bytes": 186313933 + }, + "sent": { + "bytes": 67278955 + } + }, + "system_memory_total": 33291907072, + "warn_count": 6 + } + } +} \ No newline at end of file diff --git a/x-pack/metricbeat/module/syncgateway/resources/_meta/docs.asciidoc b/x-pack/metricbeat/module/syncgateway/resources/_meta/docs.asciidoc new file mode 100644 index 000000000000..7f252b51dfd6 --- /dev/null +++ b/x-pack/metricbeat/module/syncgateway/resources/_meta/docs.asciidoc @@ -0,0 +1 @@ +SyncGateway `resources` metriset contains about the global resource utilization of the SyncGateway instance. diff --git a/x-pack/metricbeat/module/syncgateway/resources/_meta/fields.yml b/x-pack/metricbeat/module/syncgateway/resources/_meta/fields.yml new file mode 100644 index 000000000000..0de117b6fc1c --- /dev/null +++ b/x-pack/metricbeat/module/syncgateway/resources/_meta/fields.yml @@ -0,0 +1,71 @@ +- name: resources + type: group + description: > + SyncGateway global resource utilization + release: beta + fields: + - name: error_count + type: long + - name: goroutines_high_watermark + type: long + - name: num_goroutines + type: long + - name: process + type: group + fields: + - name: cpu_percent_utilization + type: long + - name: memory_resident + type: long + - name: pub_net + type: group + fields: + - name: recv + type: group + fields: + - name: bytes + type: long + - name: sent + type: group + fields: + - name: bytes + type: long + - name: admin_net_bytes + type: group + fields: + - name: recv + type: long + - name: sent + type: long + - name: go_memstats + type: group + fields: + - name: heap + type: group + fields: + - name: alloc + type: long + - name: idle + type: long + - name: inuse + type: long + - name: released + type: long + - name: pause + type: group + fields: + - name: ns + type: long + - name: stack + type: group + fields: + - name: inuse + type: long + - name: sys + type: long + - name: sys + type: long + - name: system_memory_total + type: long + - name: warn_count + type: long diff --git a/x-pack/metricbeat/module/syncgateway/resources/data.go b/x-pack/metricbeat/module/syncgateway/resources/data.go new file mode 100644 index 000000000000..46fcd2781daf --- /dev/null +++ b/x-pack/metricbeat/module/syncgateway/resources/data.go @@ -0,0 +1,51 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package resources + +import ( + s "github.com/elastic/beats/v7/libbeat/common/schema" + c "github.com/elastic/beats/v7/libbeat/common/schema/mapstriface" + "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/elastic/beats/v7/x-pack/metricbeat/module/syncgateway" +) + +var globalSchema = s.Schema{ + "go_memstats": s.Object{ + "heap": s.Object{ + "alloc": c.Int("go_memstats_heapalloc"), + "idle": c.Int("go_memstats_heapidle"), + "inuse": c.Int("go_memstats_heapinuse"), + "released": c.Int("go_memstats_heapreleased"), + }, + "pause": s.Object{"ns": c.Int("go_memstats_pausetotalns")}, + "stack": s.Object{ + "inuse": c.Int("go_memstats_stackinuse"), + "sys": c.Int("go_memstats_stacksys"), + }, + "sys": c.Int("go_memstats_sys"), + }, + "admin_net_bytes": s.Object{ + "recv": c.Int("admin_net_bytes_recv"), + "sent": c.Int("admin_net_bytes_sent"), + }, + "error_count": c.Int("error_count"), + "goroutines_high_watermark": c.Int("goroutines_high_watermark"), + "num_goroutines": c.Int("num_goroutines"), + "process": s.Object{ + "cpu_percent_utilization": c.Int("process_cpu_percent_utilization"), + "memory_resident": c.Int("process_memory_resident"), + }, + "pub_net": s.Object{ + "recv": s.Object{"bytes": c.Int("pub_net_bytes_recv")}, + "sent": s.Object{"bytes": c.Int("pub_net_bytes_sent")}, + }, + "system_memory_total": c.Int("system_memory_total"), + "warn_count": c.Int("warn_count"), +} + +func eventMapping(r mb.ReporterV2, content *syncgateway.SgResponse) { + globalData, _ := globalSchema.Apply(content.Syncgateway.Global.ResourceUtilization) + r.Event(mb.Event{MetricSetFields: globalData}) +} diff --git a/x-pack/metricbeat/module/syncgateway/resources/data_test.go b/x-pack/metricbeat/module/syncgateway/resources/data_test.go new file mode 100644 index 000000000000..7c0824770f47 --- /dev/null +++ b/x-pack/metricbeat/module/syncgateway/resources/data_test.go @@ -0,0 +1,43 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package resources + +import ( + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/assert" + + mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" + "github.com/elastic/beats/v7/x-pack/metricbeat/module/syncgateway" +) + +func TestData(t *testing.T) { + mux := syncgateway.CreateTestMuxer() + server := httptest.NewServer(mux) + defer server.Close() + + f := mbtest.NewReportingMetricSetV2Error(t, syncgateway.GetConfig([]string{"resources"}, server.URL)) + if err := mbtest.WriteEventsReporterV2Error(f, t, ""); err != nil { + t.Fatal("write", err) + } +} + +func TestFetch(t *testing.T) { + mux := syncgateway.CreateTestMuxer() + server := httptest.NewServer(mux) + defer server.Close() + + config := syncgateway.GetConfig([]string{"resources"}, server.URL) + metricSet := mbtest.NewReportingMetricSetV2Error(t, config) + + events, errs := mbtest.ReportingFetchV2Error(metricSet) + if len(errs) > 0 { + t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) + } + + assert.NotEmpty(t, events) + mbtest.TestMetricsetFieldsDocumented(t, metricSet, events) +} diff --git a/x-pack/metricbeat/module/syncgateway/resources/resources.go b/x-pack/metricbeat/module/syncgateway/resources/resources.go new file mode 100644 index 000000000000..2a4d8446ea7d --- /dev/null +++ b/x-pack/metricbeat/module/syncgateway/resources/resources.go @@ -0,0 +1,75 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package resources + +import ( + "fmt" + + "github.com/elastic/beats/v7/metricbeat/helper" + "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/elastic/beats/v7/metricbeat/mb/parse" + "github.com/elastic/beats/v7/x-pack/metricbeat/module/syncgateway" +) + +const ( + defaultScheme = "http" + defaultPath = "/_expvar" +) + +var ( + hostParser = parse.URLHostParserBuilder{ + DefaultScheme: defaultScheme, + DefaultPath: defaultPath, + }.Build() +) + +// init registers the MetricSet with the central registry. +// The New method will be called after the setup of the module and before starting to fetch data +func init() { + mb.Registry.MustAddMetricSet("syncgateway", "resources", New, + mb.WithHostParser(hostParser), + mb.DefaultMetricSet(), + ) +} + +// MetricSet type defines all fields of the MetricSet +type MetricSet struct { + mb.BaseMetricSet + http *helper.HTTP + mod syncgateway.Module +} + +// New create a new instance of the MetricSet +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + mod, ok := base.Module().(syncgateway.Module) + if !ok { + return nil, fmt.Errorf("error trying to type assert syncgateway.Module. Base module must be composed of syncgateway.Module") + } + + http, err := helper.NewHTTP(base) + if err != nil { + return nil, err + } + + return &MetricSet{ + BaseMetricSet: base, + http: http, + mod: mod, + }, nil +} + +// Fetch methods implements the data gathering and data conversion to the right +// format. It publishes the event which is then forwarded to the output. In case +// of an error set the Error field of mb.Event or simply call report.Error(). +func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { + response, err := m.mod.GetSyncgatewayResponse(m.http) + if err != nil { + return err + } + + eventMapping(reporter, response) + + return nil +} diff --git a/x-pack/metricbeat/module/syncgateway/response.go b/x-pack/metricbeat/module/syncgateway/response.go new file mode 100644 index 000000000000..a11b092f6298 --- /dev/null +++ b/x-pack/metricbeat/module/syncgateway/response.go @@ -0,0 +1,21 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package syncgateway + +type SgResponse struct { + SyncgatewayChangeCache struct { + MaxPending float64 `json:"maxPending"` + } `json:"syncGateway_changeCache"` + Syncgateway Syncgateway `json:"syncgateway"` + MemStats map[string]interface{} `json:"memstats"` +} + +type Syncgateway struct { + Global struct { + ResourceUtilization map[string]interface{} `json:"resource_utilization"` + } `json:"global"` + PerDb map[string]map[string]interface{} `json:"per_db"` + PerReplication map[string]map[string]interface{} `json:"per_replication"` +} diff --git a/x-pack/metricbeat/module/syncgateway/syncgateway.go b/x-pack/metricbeat/module/syncgateway/syncgateway.go new file mode 100644 index 000000000000..dfd2e16cb84e --- /dev/null +++ b/x-pack/metricbeat/module/syncgateway/syncgateway.go @@ -0,0 +1,87 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package syncgateway + +import ( + "encoding/json" + "sync" + "time" + + "github.com/pkg/errors" + + "github.com/elastic/beats/v7/metricbeat/helper" + "github.com/elastic/beats/v7/metricbeat/mb" +) + +func init() { + // Register the ModuleFactory function for the "kubernetes" module. + if err := mb.Registry.AddModule("syncgateway", ModuleBuilder()); err != nil { + panic(err) + } +} + +// FetchPeriodThreshold is the threshold to use when checking if the cluster state call in this cache should be +// fetched again. It's used as a percentage so 0.9 means that cache is invalidated if at least 90% of period time has +// passed. This small threshold is to avoid race conditions where one of the X number of calls that happen after each +// period do not fetch new data while other does, becoming unsync by [period] seconds. It does not fully solve the +// problem because the period waits aren't guaranteed but it should be mostly enough. +const FetchPeriodThreshold = 0.9 + +type Module interface { + mb.Module + GetSyncgatewayResponse(http *helper.HTTP) (*SgResponse, error) +} + +type exprVarCache struct { + lock sync.Mutex + lastFetchTimestamp time.Time + + cachedData SgResponse +} + +type module struct { + mb.BaseModule + + expvarCache *exprVarCache + cacheHash uint64 +} + +func ModuleBuilder() func(base mb.BaseModule) (mb.Module, error) { + expvarCache := &exprVarCache{} + + return func(base mb.BaseModule) (mb.Module, error) { + m := module{ + BaseModule: base, + expvarCache: expvarCache, + } + return &m, nil + } +} + +func (m *module) GetSyncgatewayResponse(http *helper.HTTP) (*SgResponse, error) { + m.expvarCache.lock.Lock() + defer m.expvarCache.lock.Unlock() + + elapsedFromLastFetch := time.Since(m.expvarCache.lastFetchTimestamp) + + //Check the lifetime of current response data + if elapsedFromLastFetch == 0 || elapsedFromLastFetch >= (time.Duration(float64(m.Config().Period)*FetchPeriodThreshold)) { + //Fetch data again + byt, err := http.FetchContent() + if err != nil { + return nil, err + } + + input := SgResponse{} + if err = json.Unmarshal(byt, &input); err != nil { + return nil, errors.Wrap(err, "error unmarshalling JSON of SyncGateway expvar response") + } + + m.expvarCache.cachedData = input + m.expvarCache.lastFetchTimestamp = time.Now() + } + + return &m.expvarCache.cachedData, nil +} diff --git a/x-pack/metricbeat/module/syncgateway/testing.go b/x-pack/metricbeat/module/syncgateway/testing.go new file mode 100644 index 000000000000..88c2d309c2a8 --- /dev/null +++ b/x-pack/metricbeat/module/syncgateway/testing.go @@ -0,0 +1,33 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package syncgateway + +import ( + "fmt" + "io/ioutil" + "net/http" +) + +func CreateTestMuxer() *http.ServeMux { + mux := http.NewServeMux() + + mux.Handle("/_expvar", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + input, _ := ioutil.ReadFile("../_meta/testdata/expvar.282c.json") + _, err := w.Write(input) + if err != nil { + fmt.Println("error writing response on mock server") + } + })) + + return mux +} + +func GetConfig(metricsets []string, host string) map[string]interface{} { + return map[string]interface{}{ + "module": "syncgateway", + "metricsets": metricsets, + "hosts": []string{host}, + } +} diff --git a/x-pack/metricbeat/modules.d/syncgateway.yml.disabled b/x-pack/metricbeat/modules.d/syncgateway.yml.disabled new file mode 100644 index 000000000000..54c42a11809c --- /dev/null +++ b/x-pack/metricbeat/modules.d/syncgateway.yml.disabled @@ -0,0 +1,13 @@ +# Module: syncgateway +# Docs: https://www.elastic.co/guide/en/beats/metricbeat/master/metricbeat-module-syncgateway.html + +- module: syncgateway + metricsets: + - db +# - memory +# - replication +# - resources + period: 10s + + # SyncGateway hosts + hosts: ["127.0.0.1:4985"]