From 5f586f062d8651a670eb70f5b0975a6ea57670d7 Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Fri, 20 Aug 2021 15:49:36 +0200 Subject: [PATCH] [7.15](backport #26824) [Metricbeat] gcp: add GKE metricset (#27522) Co-authored-by: Edoardo Tenani --- CHANGELOG.next.asciidoc | 1 + metricbeat/docs/fields.asciidoc | 381 ++++++++++++++++++ x-pack/metricbeat/module/gcp/constants.go | 1 + x-pack/metricbeat/module/gcp/fields.go | 2 +- .../module/gcp/gke/_meta/fields.yml | 170 ++++++++ x-pack/metricbeat/module/gcp/gke/gke_test.go | 21 + x-pack/metricbeat/module/gcp/gke/manifest.yml | 49 +++ .../metricbeat/module/gcp/metrics/helpers.go | 16 + .../module/gcp/metrics/helpers_test.go | 37 ++ .../module/gcp/metrics/metrics_requester.go | 21 + .../module/gcp/metrics/metricset_test.go | 5 + x-pack/metricbeat/module/gcp/module.yml | 1 + 12 files changed, 704 insertions(+), 1 deletion(-) create mode 100644 x-pack/metricbeat/module/gcp/gke/_meta/fields.yml create mode 100644 x-pack/metricbeat/module/gcp/gke/gke_test.go create mode 100644 x-pack/metricbeat/module/gcp/gke/manifest.yml create mode 100644 x-pack/metricbeat/module/gcp/metrics/helpers.go create mode 100644 x-pack/metricbeat/module/gcp/metrics/helpers_test.go diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 91b70e80852..9aa1a7d909d 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -518,6 +518,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Check fields are documented in aws metricsets. {pull}23887[23887] - Move openmetrics module to oss. {pull}26561[26561] - Fix release state of kubernetes metricsets. {pull}26864[26864] +- Add `gke` metricset collection to `gcp` module {pull}26824[26824] *Packetbeat* diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 2961809178b..7fcf2828465 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -26883,6 +26883,387 @@ type: long -- +[float] +=== gke + +`gke` contains the metrics that we scraped from GCP Stackdriver API containing monitoring metrics for GCP GKE + + + + +*`gcp.gke.container.cpu.core_usage_time.value`*:: ++ +-- +Cumulative CPU usage on all cores used by the container in seconds. Sampled every 60 seconds. + +type: double + +-- + +*`gcp.gke.container.cpu.limit_cores.value`*:: ++ +-- +CPU cores limit of the container. Sampled every 60 seconds. + +type: double + +-- + +*`gcp.gke.container.cpu.limit_utilization.value`*:: ++ +-- +The fraction of the CPU limit that is currently in use on the instance. This value cannot exceed 1 as usage cannot exceed the limit. Sampled every 60 seconds. After sampling, data is not visible for up to 240 seconds. + +type: double + +-- + +*`gcp.gke.container.cpu.request_cores.value`*:: ++ +-- +Number of CPU cores requested by the container. Sampled every 60 seconds. After sampling, data is not visible for up to 120 seconds. + +type: double + +-- + +*`gcp.gke.container.cpu.request_utilization.value`*:: ++ +-- +The fraction of the requested CPU that is currently in use on the instance. This value can be greater than 1 as usage can exceed the request. Sampled every 60 seconds. After sampling, data is not visible for up to 240 seconds. + +type: double + +-- + + +*`gcp.gke.container.ephemeral_storage.limit_bytes.value`*:: ++ +-- +Local ephemeral storage limit in bytes. Sampled every 60 seconds. + +type: long + +-- + +*`gcp.gke.container.ephemeral_storage.request_bytes.value`*:: ++ +-- +Local ephemeral storage request in bytes. Sampled every 60 seconds. + +type: long + +-- + +*`gcp.gke.container.ephemeral_storage.used_bytes.value`*:: ++ +-- +Local ephemeral storage usage in bytes. Sampled every 60 seconds. + +type: long + +-- + + +*`gcp.gke.container.memory.limit_bytes.value`*:: ++ +-- +Memory limit of the container in bytes. Sampled every 60 seconds. + +type: long + +-- + +*`gcp.gke.container.memory.limit_utilization.value`*:: ++ +-- +The fraction of the memory limit that is currently in use on the instance. This value cannot exceed 1 as usage cannot exceed the limit. Sampled every 60 seconds. After sampling, data is not visible for up to 120 seconds. + +type: double + +-- + +*`gcp.gke.container.memory.page_fault_count.value`*:: ++ +-- +Number of page faults, broken down by type, major and minor. + +type: long + +-- + +*`gcp.gke.container.memory.request_bytes.value`*:: ++ +-- +Memory request of the container in bytes. Sampled every 60 seconds. After sampling, data is not visible for up to 120 seconds. + +type: long + +-- + +*`gcp.gke.container.memory.request_utilization.value`*:: ++ +-- +The fraction of the requested memory that is currently in use on the instance. This value can be greater than 1 as usage can exceed the request. Sampled every 60 seconds. After sampling, data is not visible for up to 240 seconds. + +type: double + +-- + +*`gcp.gke.container.memory.used_bytes.value`*:: ++ +-- +Memory usage in bytes. Sampled every 60 seconds. + +type: long + +-- + +*`gcp.gke.container.restart_count.value`*:: ++ +-- +Number of times the container has restarted. Sampled every 60 seconds. After sampling, data is not visible for up to 120 seconds. + +type: long + +-- + +*`gcp.gke.container.uptime.value`*:: ++ +-- +Time in seconds that the container has been running. Sampled every 60 seconds. + +type: double + +-- + + + +*`gcp.gke.node.cpu.allocatable_cores.value`*:: ++ +-- +Number of allocatable CPU cores on the node. Sampled every 60 seconds. + +type: double + +-- + +*`gcp.gke.node.cpu.allocatable_utilization.value`*:: ++ +-- +The fraction of the allocatable CPU that is currently in use on the instance. Sampled every 60 seconds. After sampling, data is not visible for up to 240 seconds. + +type: double + +-- + +*`gcp.gke.node.cpu.core_usage_time.value`*:: ++ +-- +Cumulative CPU usage on all cores used on the node in seconds. Sampled every 60 seconds. + +type: double + +-- + +*`gcp.gke.node.cpu.total_cores.value`*:: ++ +-- +Total number of CPU cores on the node. Sampled every 60 seconds. + +type: double + +-- + + +*`gcp.gke.node.ephemeral_storage.allocatable_bytes.value`*:: ++ +-- +Local ephemeral storage bytes allocatable on the node. Sampled every 60 seconds. + +type: long + +-- + +*`gcp.gke.node.ephemeral_storage.inodes_free.value`*:: ++ +-- +Free number of inodes on local ephemeral storage. Sampled every 60 seconds. + +type: long + +-- + +*`gcp.gke.node.ephemeral_storage.inodes_total.value`*:: ++ +-- +Total number of inodes on local ephemeral storage. Sampled every 60 seconds. + +type: long + +-- + +*`gcp.gke.node.ephemeral_storage.total_bytes.value`*:: ++ +-- +Total ephemeral storage bytes on the node. Sampled every 60 seconds. + +type: long + +-- + +*`gcp.gke.node.ephemeral_storage.used_bytes.value`*:: ++ +-- +Local ephemeral storage bytes used by the node. Sampled every 60 seconds. + +type: long + +-- + + +*`gcp.gke.node.memory.allocatable_bytes.value`*:: ++ +-- +Cumulative memory bytes used by the node. Sampled every 60 seconds. + +type: long + +-- + +*`gcp.gke.node.memory.allocatable_utilization.value`*:: ++ +-- +The fraction of the allocatable memory that is currently in use on the instance. This value cannot exceed 1 as usage cannot exceed allocatable memory bytes. Sampled every 60 seconds. After sampling, data is not visible for up to 120 seconds. + +type: double + +-- + +*`gcp.gke.node.memory.total_bytes.value`*:: ++ +-- +Number of bytes of memory allocatable on the node. Sampled every 60 seconds. + +type: long + +-- + +*`gcp.gke.node.memory.used_bytes.value`*:: ++ +-- +Cumulative memory bytes used by the node. Sampled every 60 seconds. + +type: long + +-- + + +*`gcp.gke.node.network.received_bytes_count.value`*:: ++ +-- +Cumulative number of bytes received by the node over the network. Sampled every 60 seconds. + +type: long + +-- + +*`gcp.gke.node.network.sent_bytes_count.value`*:: ++ +-- +Cumulative number of bytes transmitted by the node over the network. Sampled every 60 seconds. + +type: long + +-- + +*`gcp.gke.node.pid_limit.value`*:: ++ +-- +The max PID of OS on the node. Sampled every 60 seconds. + +type: long + +-- + +*`gcp.gke.node.pid_used.value`*:: ++ +-- +The number of running process in the OS on the node. Sampled every 60 seconds. + +type: long + +-- + + + +*`gcp.gke.node_daemon.cpu.core_usage_time.value`*:: ++ +-- +Cumulative CPU usage on all cores used by the node level system daemon in seconds. Sampled every 60 seconds. + +type: double + +-- + + +*`gcp.gke.node_daemon.memory.used_bytes.value`*:: ++ +-- +Memory usage by the system daemon in bytes. Sampled every 60 seconds. + +type: long + +-- + + + +*`gcp.gke.pod.network.received_bytes_count.value`*:: ++ +-- +Cumulative number of bytes received by the pod over the network. Sampled every 60 seconds. + +type: long + +-- + +*`gcp.gke.pod.network.sent_bytes_count.value`*:: ++ +-- +Cumulative number of bytes transmitted by the pod over the network. Sampled every 60 seconds. + +type: long + +-- + + +*`gcp.gke.pod.volume.total_bytes.value`*:: ++ +-- +Total number of disk bytes available to the pod. Sampled every 60 seconds. After sampling, data is not visible for up to 120 seconds. + +type: long + +-- + +*`gcp.gke.pod.volume.used_bytes.value`*:: ++ +-- +Number of disk bytes used by the pod. Sampled every 60 seconds. + +type: long + +-- + +*`gcp.gke.pod.volume.utilization.value`*:: ++ +-- +The fraction of the volume that is currently being used by the instance. This value cannot be greater than 1 as usage cannot exceed the total available volume space. Sampled every 60 seconds. After sampling, data is not visible for up to 120 seconds. + +type: double + +-- + [float] === loadbalancing diff --git a/x-pack/metricbeat/module/gcp/constants.go b/x-pack/metricbeat/module/gcp/constants.go index 2b1723f9c7c..3776263b7b6 100644 --- a/x-pack/metricbeat/module/gcp/constants.go +++ b/x-pack/metricbeat/module/gcp/constants.go @@ -18,6 +18,7 @@ const ( const ( ServiceCloudFunctions = "cloudfunctions" ServiceCompute = "compute" + ServiceGKE = "gke" ServiceLoadBalancing = "loadbalancing" ServicePubsub = "pubsub" ServiceStorage = "storage" diff --git a/x-pack/metricbeat/module/gcp/fields.go b/x-pack/metricbeat/module/gcp/fields.go index 83e8e1322fc..1d12942e9eb 100644 --- a/x-pack/metricbeat/module/gcp/fields.go +++ b/x-pack/metricbeat/module/gcp/fields.go @@ -19,5 +19,5 @@ func init() { // AssetGcp returns asset data. // This is the base64 encoded zlib format compressed contents of module/gcp. func AssetGcp() string { - return "eJzcXN1u2zgWvu9THMxNp4vUBXbvisUAHRc7E6DZDZBMbwWKOrY5oUiVP/GoT78gKcmSLMmyLSnNtFdJJPL7zv85lPQenjD/CFuavQEwzHD8CG9/k3LLEdZc2gTuOTEbqdK3bwAUciQaP0KMhrwBSFBTxTLDpPgIv7wBAPhtfQ+pTCzHNwAbhjzRH/0f3oMgKZZbuX8mz9zPStryN/Xr6/dwEiPX1a/LW2X8J1JT+3UHnvJfwCWYkYqJLaRoFKP6eOU2hDoMq1Gt/tH4Uy8U9y/8MgpXPGG+lyrpXDhFQxJiyFyLO6qzrK1zbTCdbOly2Z8qzOH/T6dV31g3kTb2Btjx1yglWcbEtrj0p8biAwZ0V1iM2REDCo1VAhPYKJlCw18+3d/CN4sqXx3RihnnTGxry7Ydqkmy7hpH6Bqb/hpWLnU90p6p1EEmnerr0nsLw1pq46/WwATlNkFQuLWcqBsw5K8bIMmfVpsUhbkBIhJQ0orECR+VkmrViYqJZ8koRqkUZncpslIgCjOpDPi1urfLlPS2wdrrjd7rPqwAt59BbsDssFR0uXuMXIqtBiO7IRhpCO/cfcMlaXtQY+9Hd2u1H0mlFebY8KhMM2twBsNbh5XPNDwmtCGCdttde/O+xeoLbpjCPeFtIQ4vOrRwffFEySzDJIpzgzqiXsTPhNs2/OaOTuU9FzTEeSuoTJ3y/PLlZhDn3pIGiB0DzAh9QjMjxGKD0SAr+8vsLJpRqFE9YxJRqVCPYHyUGHo5/9emMSrn0H7taiuQwnPeudBXuHuPOTexWsM4+07c6pMCfXQKUIS6n0pAhHNJicEE1vd/hIzFNFCrFArDc2DC1TIllXHwNdliZFiKk6L/wy0LG6kc5kLUTIBGKkWiew0qYfppJosiczn62q3nNBQc3e0UyoceMi1QMpsRkoMQEN3+D2SGytvpsfzrqPaKGVxGVm4rgwKMPC2sAGt+afl9Toircp5s0HEGYTQg/C73/jrvt1/vYEc0xIgClBWCie3NGOcRaPZSzeU/FNnzbMnyyIfCbsGPnEz6uXVgnCtfVijLfHkZTo3CLCNHtxPIZ1TnYVtMfuPwHRreVKp8FbssKMU8Zk7SSLPvY3LhWNIuj/tGoKjiHf/AxPl08PcVPO6YLoptl9Kl4DmQZ8I4iXnIo1/vih41dB0uZrqb8Z+wISnj+eokMasxmZDYXSBxqD7c+kty0nuSRUzM5EpOb0ca86mUiQLn1qI2wfeZ0SD3wmMCnRGKC/GXdq5Y0imAMmNXAS/IwMjZJVDNDCVJYsKJoPOMXL5IksCv5QZnNsA7Y7J2gu6PSleA6ANSBxO7KCuSSOE3p6RRZjK+ZHHmIapmqpZviMuLfkcdzOT3x8f7Dw9ecRA05wK+LPHpYyvvYzAP9gpt0QnGeQXN/bkLfj/klxY25QyF0U6+lwFfTMbnwtOZFHpcYzKVYMOWJ8y4kHj3CJD/a8WEQSV6BoGLhYUSEG4Vaj1CigMyHCdBL7TbL7+WznSQFfzsQv/j+h42XO41MPNWg8dymJdJASTLOKO+BQNtFJLUZ5B3bSNpURtTvV5GrlG2DtArpjWOXB9WJpbVQ4HMyAbkefRQcltSEZ38eqbyNIu05lGm5F/5j5KrKZfazzuFQD/y6596jg9u9VlntW4ot/aoEAyqlAk/T/RN2OP6/sPDwxfwkukPxaNDyGVY26b79a7mWlb7efUwwPHONQ3Cg/F9vRuHUOB+YV1ThWcrWmYoJoa5Dt1izZelNdoQf3rYgq6k3e58JO3B+7636PcPPxx74nynYNWpo421jWdoS+5t/GDjM9sRbeNqvcsi3YOtfnBUvAldFuEIfYpS1P64YSqXXNvUcmLYM4a0FRpVv4d2Gwq555hsQ7356fBzVY7eBAmECxLk7BlV7gEMNyVcbkN0maIEDUe9uUHQ7DuW1YMVDQIVrZ/J6mlFViWM6g/vgAkgDZUPxB+bRjXHKxUzbSCqIBeCRV8jNSG+1ZBZvQMUSSaZMDcQWwNCGsjRNFQ4TMaKapN5yMyqDskT13r5PQ4EIrKd4mzh0xbh58PxwbvSwMKmPcT6eJ1FK7OcR3W/r45UZo0ANT4VncNhTsPlFWrLzQr+IxUQSHDDBCsPXLtu1ViLxx8OAbkhkA+pTDztBEnCmcBe/qclN3VXfkpenYGxkNIw2mV17LZ81cpdQrFeSBdqVO8Wgqh3QIzBNOuGCH8Izp7Qc9E3PnD5e/wcSQFLM44pChPa00Si9tkjJobu/IOdVURewYMEJHRXiiTMo6kUhjDhOl1s3LDyTWd9M+XMIfTE/jk3ZzMuoW3ZM4rGvUCJ75uRKEgtNyzjCIalODDsbMjcVVqCskmS2GemjWKxLW3fUyplUG3kc0TKqJJlojjLYvzB4aL1ne+/4rzlskaW7uoSdzGbu6TMC5MOVxy9UBp7KAHcOzf+GyS1DokuEWKqbZvxEPbM7EBI8d7FnrwhYJacFy5bzJa1kxa/pYzj31Qm+MtFJjJWjtXMf1HzKIb+ZxmAYxxxNAbVskEwszFnehdaK4cCAgowMmP0LA692luEyH4nNUK5N+yJBpslft4Q53AnE7bJP9Gnz+UF13Two0PZPJyPyYzz2fF8lgirI1VyCnvR+ip0RRh6Hkt3wJ39bwnogja+n0sU55HC7dCjwi/F6gZiJZ/QldB7cSirCrQXTC6W5Nozw5iKZEuhMw/+uudMl9hjF+5J1TIRgwvVohEXqiARnwab1NGF4csmnmZHcRzAL0w/p0j+QEX+MedwbHEWX2fvEZV6ci7hzZO6FlIk2qrgMyHshGfqmA5PWxpZXgLfrDSk8TZK/2GzFBu2jUJ9Nd0BaZdewlY2EAK6I8KFgo1UYRLTjAI1HVRC8PuHdwxPqKXMQYuFub4jgWvi2+HVwYzRy47sHt2tVx7WvdpGTKNIFh9F+xbs1U5l5njAckhO87QJP1hpXbTff9Oa+hp2P3JReg2vSauC60qBWhHgU60vDQZOHl66Hrg285ce4tQ9z4FJM8L7fbzveEW8eyUlybnWXWUIQTK9k+2x7dgniIq7r6xIiqC46GCoZ2pSEDoZwF8ocA+jvjS6LfPQU9+w4KTIG/iW8LQTSC8Us7DpzM9inSvYOqJJ5Toa24WifG1J7dB6SUW2c3zT5SGsfOYzpCRjowP/8KsnY5qMse8DfEZuyEEDn+5vgRL/hEhN6s5P3V9SNDuZeBSl6H1rCVQm2J3ziDW771fyrj/7yqOYaEyi4oNVhFLU0xllSxrV9Kv4ppUzNO9TovhgFsTo+uytIsJl5IAGtOTIc0gsumKyuPLT+stA5eiIHYrUyfiEr4c4va6/1IrgdiAYPuUrRK0zpGzDaOSwptZMOxBoSb4ccKUkqQuxxNAhzeGPN1xucmd9sOFivq3PNbRf5b/WG+GSryZcSabzmwRThZXj4H6dlgsbn+xtSF/r1N7N8MtryFBBbOkTmoYgCjZAOdFlO+pBgH/tu+hXpaDoV0hIHr4W5wRYXacwC2+mEFPMhEma+a+e+RcnnwkvXwWR1vhbEzLwzor/1IK3k6ioj1+11R/oTFd1l2Us4bzScPHZhh9Myf8PAAD//7kI98I=" + return "eJzkXVtz27a2fs+vWNOXNGcc57TnzH7I7OmM6+y2mR23mrGTVxYklyRUIMACoBXl1+/BhVeRFEWRdJydPFkigW/dL1ygXsMOD29hE6UvADTVDN/Cy1+F2DCEWyayGFaM6LWQycsXABIZEoVvIURNXgDEqCJJU00Ffws/vQAA+PV2BYmIM4YvANYUWaze2i9eAycJ5luZf/qQmr+lyPJPqtdX72EkRKaKj/NbRfgXRrrycQue/J/DxakWkvINJKgljdTxyk0IVRiZQnn9P7WvOqGYf+7DwF2xw8NeyLh14QQ1iYkmcy1uSJ1lbXVQGpPJls6X/a7A7P5/d1r0tXVjkYVWAVu+DRKSppRv/KXf1RbvUaA7rzF6SzRI1JnkGMNaigRq9nKzeg9/ZygP10dkhZQxyjeVZZsGVSeyahpH6Gqb/uxWzmU9UJ8joRxPWsXXJvcGhluhtL1aAeURy2IEiZuMEXkFmny+AhL/lSmdINdXQHgMUmQ8NsxHKYW8bkVF+aOgEQaJ4Ho7FlnOEImpkBrsWu3bpVJY3aDN9QbvtXIrwPt3INagt5gLOt89RCb4RoEW7RC00IS17r5mgjQtqLb3g7m12I8kIuP6WPEikaSZxhkU79atfKbiUa404VG73jU371qsuuCaStwT1mRi/6J9C1cXj6VIU4yD8KBRBZFl8SNhWRN+fUcj8o4Laux8zyORGOHZ5fPNIDxYTeoh7BhgSqId6hkh+g0Ggyz0L81mkYxEhfIR4yASEtUAio8CQyfNv2dJiNIYtF272AoEtzRvjevz5t6hznWsmaaMfiFm9UmBPhgBSBKZv3JAhDEREY0x3K4+uohFFUSZlMg1OwDlJpfJSRkGX5ENBpomOCn6j2ZZWAtpMHtWUw4KI8Fj1alQMVW7mTSKzGXot2Y9IyFn6GYnlz50ENMAJdIZIRkIDtH7P0CkKK2eHvO/imovqcZleGW20shBi9PMcrDm55bd5wS7CuNJew2nF0YNwm9ib6+zdvvpDrZEQYjIQWacU765GmI8HPVeyLnsJ0L6OFuwPLIht5uzI8OTbtpaMM4VLwuUebwch1Mh18vw0ewE4hHledgW498wfGXBmwh5uA5NFBR8HjUnSaDolyGxcCjRJo7bQsBn8YZ+R4mxaWfv1/Cwpcon2yakC84OQB4JZSRkLo5+uvM1qqs6jM80N+OPsCYJZYfrk4RlCuMJCbtzRJTZh1l/SZrUnqQB5TOZkpHbkcRsKKXc49xkqLSzfaoViD23mEClJMKF6BfZXL6klQF5xC4cnuOBFrNzoOgz7vBkXdtVBdfoe/nnZod/QiS4JpQru29SbQbtEVQkSVq0g25XcK9JtIslNT7rZvU+v9v2aI67kIZCc9ev//7Xy94Suizo7XIoXxzL7div9faAjkqzc6vwEpPE4GSZ0Fsg1KNAlmSMaPqItoqxK5uSpSwUrCPxNWjBkUoGdA33JEkZxoCPKA/wj/8tvumggdGE6t5q8gz8q48ep101r80KoGPRnS4kh2NsKx4Nbod4aOHoDdiCgYhwLjTg5wgxhh+AKC+6+hfmfrtLDxvgZq1RgjLf2+Q2JpoYQGadR6po7h2y1DiEH///JAcl/m380EQSLnsEpaz9Di2KOR2hP/w4mNC5laUk95xOQ1NhIETYSCTaZniEN9SmqjN+wwW0JmclpltMUBIWKC0k2Vzataxbs43JvaJpjcM1wXwQEWElTvA4vRVT7rL88/1NrkRzYvR7XIDSRIFZITo9PBdgvRr56nTGZ+XtoekCaSwTo5Iq+mcWpgZ479QkUWuSMd2brQ+VdRmnUtttNQurKwil2CGH2GTjJlgdUryChPwlpH1Wl1DefEQ3j2u4y2sm5wnGKOO3EEC9Tn9TMXQGL+215TKnLFFpIvusqxNOh2WZkkc1FHdLVL4Txgto74B2c4+y1lWVJlippJxWHlNX7T0Pk0HRhBYxfkXFq39iRkKGkxcHlbUrhYI3Z8OH86NsFe7cHqoJf7iDWtC9PFHzoSLEi/oOtvE7keK5kRDeUpuOULm5i6CqIs+Zw7vnHFVVvsj+qLlNBWuJ/ap2GvAvErEiLLewAcfaKRmN1arYhWCbqjUDWmcJU6iCA9ulCheJf/aS02Gs9jfHGO2kpefUllrxsD7zvYzopw+OFybwQ8rRlu2WL42mM9EyR/JGWTw/msxTT2aqU+tr/zTGWCs9Y/5iBOG8Ia9irqFC/9Fz+vNlNnDwYQICtCRcJVTriWgoGjg0DlzbaHxtadxNQj7Dys0V/3F/Sf5mAHU+1x+Op+SfL/oglSJCpfLn+WNQVkvCICaYNIY2/psfa1ptZPiIzJ+1AMegEdXGLFnBPG0dT/wRxWf0egrFF/HFyvQtuehUxM/dQ19CQk7Ao2BZMlURO3XNUvLATpr62rWYh/GzL6lYoqc4g6n/3kZe1en1U9YFb+ZM32lMS3YfoomDVfx9GX5/l77x5MnPAxaS9xj81NSssi+e5wkSh4QRHs1zcu2DIDH8nG9w5jmirdZpc86524gvANEFpAomJNEOeRzUHo2dmLYbPvldz74qY7ukmDlRbvzst4eH1Zt7KzhwkjO6JnJ8RwbUTcE82Au0/kBNeCigma/b4HdDfmpmR4wi18rwdxzwxXh8LjyVCq6Gne+YirFuyxNq7DneHtbZ/11TrlHyjvOUi7mFom++kajUhKlSJwct095/+Dk3ppJX8L3x9Q+3K1gzsVdA9UsFFkt57NAUIGnKaGQDKCgtkSR2EPdVV8D1pA05BDCOuNr0fw95PkAb4rp74MvKwSPTogZ5HjnktC0piFb6Og43R2mgFAtSKT4PrzpnjtURE8oeG+UcbZbXfXh0uHOrHhkt1s1HxCWCRplQbo9l2vLl4Xb15v7+A1jOdLviwS5kHNam6n66q5hWpnyPpw/gcOOaBmGpfJ/uhiHkuF9Y1pHN8c8TtEiRTwzz1lUpFVsWmTbFSWy4VocuRbbZWk/agfd1Z9Jv3yFzbInzvUyg6O5kocrCGcqSVRbeZ+GZ5YjKwmK9cZ7uPiv+MKQQNxc2xsORaBckqGzfciqTrLRpovK8j91DmQ252DOMNy7fvCn/LtLRK8cBd0GMjNrK1QDoL0qY2PR2Hs5KQd0bMw4aQdEvmGcPGa8RUJD1PbneXZPrHEbxxSugHEhN5D3+J0uCiuHlgpnWERWQPWPR5kh1iC8VpJnaAvI4FZTrKwgzbdsCB9Q1EfYTk/Fik3mImVUcgsV2tNPsURIQkM0UR7RvNgjfl935V7mCuU07COui6yyy0oyxoGr3xcn0WT1AhZ6CnPJMfM3kJaqM6Wv4RUggEOOacpr32NpuVVjxx29Kh1xjyJtExJbsGEnMKMdO+k9zbuqq/BS/Wh2j51I/2mVlbLZ81sJdQrCWSSMlqrYLQVRbIFpjkrZDhI+c0R1aWtSV68abe2wfSQJNUoYJcu3K01igayqHREdb+368wiNfw70AJNG2HOzn7FAeoxUcazdc26Kzupk06uBqYvu6MKMzJqBt6CPy2r12wJ2kKRIJScY0TRm6ieyBPDeZFo/oJEHsHVVa0jDLdd+SlPOg2MjGiIRGUuSB4iyNsQ/gFs3vbP0VHhomq0VuriZw+97cmDTPdTpMcvREYew+B7AyZvwNBLUWji7hYopt6/4Q9lRvgQv+2vieQ43BND7PXTYoW1ZPGvQtpRz/jESMP41SkaF8LHr+i6qHb/qfpQCG4oCh1iiXdYJpFjKqtq60MijAoQAtUhqdRUOn9BYhZL8VCiHfG/ZEQZbGxE9Z3ImYrg830e5dfsElFfxgVzYPzcfEDLPZ4fQs4VYHiuQUdl/6SrSHuCwdS1fArfVvDmhEGd9NSxAeAombvjcuPhVVR6dvndP3aEd0LpaktaOHMRWRDYHO3Phr7zON0cc23JOKZSIKRopFIS6UQSLueovUwYnh0waeekVx7MBHhp9TRH5FSf4xze6xxVn0Gn0PIqEmp8W9a6AqhQSJyqSzGT9rbCf4qJ9P1CK/BP7OhCa1ocPuh82Cr+kmcPnVdA9I2+TitsocQRBtCTeuYC2k68TUvUBFBgUT7P7uVe0nxJLHoMXcXNcjgUv8WznBm9Jo3CO7B3PrhQ/rnm0hppDHi7eibQn2bLsycwxY9vFpnjLhK0utffn9jebUl1D3NSell9A1aVZwWSpQSQJsqLWpQc+Th6fOBy6N/LmFGHHP88Ck7uHtPtZ2rCBePZOU5FztLiIEJ6naimbbdugEkb/7wozEO8VFG0MdXRNP0EkH/kSOux/1WO+2zNBTV7PgJMtr+JawtBNIR7KZZ8nMs1jnMraKaFK+DsY2kpXPLaiVpVfzdUZTjZDe+3e5nDdDSlI62PGfOmd8usgYeh7gHTJNSgnYN5oTOyFS4bqxU/NNgnorYosiZ70tLSESMbbHPJLp7ZcL6a7OvrIgJArjwP/uH4kiVNMpZYMbRffL/zSgUTRrU9z/7qA/IrqRhJuI7NCAEgzZAeLMnuj1V97cfujJHA1hZZI6GT3uR5iMXG8/VJLgtheE9kRhx2qVYkTXNAoM1iTT0zYEGpzPG1wJiatMzDG0cHPaI/2jDvVfQG/jSH/zEPyl1gjnn/C/mJjWn3aZyq1M8666Ix2f7DRk47S/W15BihLCLNqhrjEifzlYxIhStbPl9tczfL0qeIR2hZgc3I9u2tPr+XUSU3cyhWjfE/anxMEenHwkLD8KIjL3xs+Y9JxZKV99EORHyJ+z1p9+k8P58s3TWMJYIWH/tpyvTMj/CQAA//+Kgwc6" } diff --git a/x-pack/metricbeat/module/gcp/gke/_meta/fields.yml b/x-pack/metricbeat/module/gcp/gke/_meta/fields.yml new file mode 100644 index 00000000000..3c14872324c --- /dev/null +++ b/x-pack/metricbeat/module/gcp/gke/_meta/fields.yml @@ -0,0 +1,170 @@ +- name: gke + type: group + release: beta + description: '`gke` contains the metrics that we scraped from GCP Stackdriver API containing monitoring metrics for GCP GKE' + fields: + - name: container + type: group + fields: + - name: cpu + type: group + fields: + - name: core_usage_time.value + type: double + description: Cumulative CPU usage on all cores used by the container in seconds. Sampled every 60 seconds. + - name: limit_cores.value + type: double + description: CPU cores limit of the container. Sampled every 60 seconds. + - name: limit_utilization.value + type: double + description: The fraction of the CPU limit that is currently in use on the instance. This value cannot exceed 1 as usage cannot exceed the limit. Sampled every 60 seconds. After sampling, data is not visible for up to 240 seconds. + - name: request_cores.value + type: double + description: Number of CPU cores requested by the container. Sampled every 60 seconds. After sampling, data is not visible for up to 120 seconds. + - name: request_utilization.value + type: double + description: The fraction of the requested CPU that is currently in use on the instance. This value can be greater than 1 as usage can exceed the request. Sampled every 60 seconds. After sampling, data is not visible for up to 240 seconds. + - name: ephemeral_storage + type: group + fields: + - name: limit_bytes.value + type: long + description: Local ephemeral storage limit in bytes. Sampled every 60 seconds. + - name: request_bytes.value + type: long + description: Local ephemeral storage request in bytes. Sampled every 60 seconds. + - name: used_bytes.value + type: long + description: Local ephemeral storage usage in bytes. Sampled every 60 seconds. + - name: memory + type: group + fields: + - name: limit_bytes.value + type: long + description: Memory limit of the container in bytes. Sampled every 60 seconds. + - name: limit_utilization.value + type: double + description: The fraction of the memory limit that is currently in use on the instance. This value cannot exceed 1 as usage cannot exceed the limit. Sampled every 60 seconds. After sampling, data is not visible for up to 120 seconds. + - name: page_fault_count.value + type: long + description: Number of page faults, broken down by type, major and minor. + - name: request_bytes.value + type: long + description: Memory request of the container in bytes. Sampled every 60 seconds. After sampling, data is not visible for up to 120 seconds. + - name: request_utilization.value + type: double + description: The fraction of the requested memory that is currently in use on the instance. This value can be greater than 1 as usage can exceed the request. Sampled every 60 seconds. After sampling, data is not visible for up to 240 seconds. + - name: used_bytes.value + type: long + description: Memory usage in bytes. Sampled every 60 seconds. + - name: restart_count.value + type: long + description: Number of times the container has restarted. Sampled every 60 seconds. After sampling, data is not visible for up to 120 seconds. + - name: uptime.value + type: double + description: Time in seconds that the container has been running. Sampled every 60 seconds. + - name: node + type: group + fields: + - name: cpu + type: group + fields: + - name: allocatable_cores.value + type: double + description: Number of allocatable CPU cores on the node. Sampled every 60 seconds. + - name: allocatable_utilization.value + type: double + description: The fraction of the allocatable CPU that is currently in use on the instance. Sampled every 60 seconds. After sampling, data is not visible for up to 240 seconds. + - name: core_usage_time.value + type: double + description: Cumulative CPU usage on all cores used on the node in seconds. Sampled every 60 seconds. + - name: total_cores.value + type: double + description: Total number of CPU cores on the node. Sampled every 60 seconds. + - name: ephemeral_storage + type: group + fields: + - name: allocatable_bytes.value + type: long + description: Local ephemeral storage bytes allocatable on the node. Sampled every 60 seconds. + - name: inodes_free.value + type: long + description: Free number of inodes on local ephemeral storage. Sampled every 60 seconds. + - name: inodes_total.value + type: long + description: Total number of inodes on local ephemeral storage. Sampled every 60 seconds. + - name: total_bytes.value + type: long + description: Total ephemeral storage bytes on the node. Sampled every 60 seconds. + - name: used_bytes.value + type: long + description: Local ephemeral storage bytes used by the node. Sampled every 60 seconds. + - name: memory + type: group + fields: + - name: allocatable_bytes.value + type: long + description: Cumulative memory bytes used by the node. Sampled every 60 seconds. + - name: allocatable_utilization.value + type: double + description: The fraction of the allocatable memory that is currently in use on the instance. This value cannot exceed 1 as usage cannot exceed allocatable memory bytes. Sampled every 60 seconds. After sampling, data is not visible for up to 120 seconds. + - name: total_bytes.value + type: long + description: Number of bytes of memory allocatable on the node. Sampled every 60 seconds. + - name: used_bytes.value + type: long + description: Cumulative memory bytes used by the node. Sampled every 60 seconds. + - name: network + type: group + fields: + - name: received_bytes_count.value + type: long + description: Cumulative number of bytes received by the node over the network. Sampled every 60 seconds. + - name: sent_bytes_count.value + type: long + description: Cumulative number of bytes transmitted by the node over the network. Sampled every 60 seconds. + - name: pid_limit.value + type: long + description: The max PID of OS on the node. Sampled every 60 seconds. + - name: pid_used.value + type: long + description: The number of running process in the OS on the node. Sampled every 60 seconds. + - name: node_daemon + type: group + fields: + - name: cpu + type: group + fields: + - name: core_usage_time.value + type: double + description: Cumulative CPU usage on all cores used by the node level system daemon in seconds. Sampled every 60 seconds. + - name: memory + type: group + fields: + - name: used_bytes.value + type: long + description: Memory usage by the system daemon in bytes. Sampled every 60 seconds. + - name: pod + type: group + fields: + - name: network + type: group + fields: + - name: received_bytes_count.value + type: long + description: Cumulative number of bytes received by the pod over the network. Sampled every 60 seconds. + - name: sent_bytes_count.value + type: long + description: Cumulative number of bytes transmitted by the pod over the network. Sampled every 60 seconds. + - name: volume + type: group + fields: + - name: total_bytes.value + type: long + description: Total number of disk bytes available to the pod. Sampled every 60 seconds. After sampling, data is not visible for up to 120 seconds. + - name: used_bytes.value + type: long + description: Number of disk bytes used by the pod. Sampled every 60 seconds. + - name: utilization.value + type: double + description: The fraction of the volume that is currently being used by the instance. This value cannot be greater than 1 as usage cannot exceed the total available volume space. Sampled every 60 seconds. After sampling, data is not visible for up to 120 seconds. diff --git a/x-pack/metricbeat/module/gcp/gke/gke_test.go b/x-pack/metricbeat/module/gcp/gke/gke_test.go new file mode 100644 index 00000000000..3bad30c89f9 --- /dev/null +++ b/x-pack/metricbeat/module/gcp/gke/gke_test.go @@ -0,0 +1,21 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package gke + +import ( + "os" + + "github.com/elastic/beats/v7/metricbeat/mb" + + // Register input module and metricset + _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/gcp" + _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/gcp/metrics" +) + +func init() { + // To be moved to some kind of helper + os.Setenv("BEAT_STRICT_PERMS", "false") + mb.Registry.SetSecondarySource(mb.NewLightModulesSource("../../../module")) +} diff --git a/x-pack/metricbeat/module/gcp/gke/manifest.yml b/x-pack/metricbeat/module/gcp/gke/manifest.yml new file mode 100644 index 00000000000..de4030695cf --- /dev/null +++ b/x-pack/metricbeat/module/gcp/gke/manifest.yml @@ -0,0 +1,49 @@ +default: false +input: + module: gcp + metricset: metrics + defaults: + metrics: + - service: gke + service_metric_prefix: kubernetes.io/ + metric_types: + - "container/cpu/core_usage_time" + - "container/cpu/limit_cores" + - "container/cpu/limit_utilization" + - "container/cpu/request_cores" + - "container/cpu/request_utilization" + - "container/ephemeral_storage/limit_bytes" + - "container/ephemeral_storage/request_bytes" + - "container/ephemeral_storage/used_bytes" + - "container/memory/limit_bytes" + - "container/memory/limit_utilization" + - "container/memory/page_fault_count" + - "container/memory/request_bytes" + - "container/memory/request_utilization" + - "container/memory/used_bytes" + - "container/restart_count" + - "container/uptime" + - "node/cpu/allocatable_cores" + - "node/cpu/allocatable_utilization" + - "node/cpu/core_usage_time" + - "node/cpu/total_cores" + - "node/ephemeral_storage/allocatable_bytes" + - "node/ephemeral_storage/inodes_free" + - "node/ephemeral_storage/inodes_total" + - "node/ephemeral_storage/total_bytes" + - "node/ephemeral_storage/used_bytes" + - "node/memory/allocatable_bytes" + - "node/memory/allocatable_utilization" + - "node/memory/total_bytes" + - "node/memory/used_bytes" + - "node/network/received_bytes_count" + - "node/network/sent_bytes_count" + - "node/pid_limit" + - "node/pid_used" + - "node_daemon/cpu/core_usage_time" + - "node_daemon/memory/used_bytes" + - "pod/network/received_bytes_count" + - "pod/network/sent_bytes_count" + - "pod/volume/total_bytes" + - "pod/volume/used_bytes" + - "pod/volume/utilization" diff --git a/x-pack/metricbeat/module/gcp/metrics/helpers.go b/x-pack/metricbeat/module/gcp/metrics/helpers.go new file mode 100644 index 00000000000..296520be2ca --- /dev/null +++ b/x-pack/metricbeat/module/gcp/metrics/helpers.go @@ -0,0 +1,16 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package metrics + +import "strings" + +// withSuffix ensures a string end with the specified suffix. +func withSuffix(s, suffix string) string { + if strings.HasSuffix(s, suffix) { + return s + } + + return s + suffix +} diff --git a/x-pack/metricbeat/module/gcp/metrics/helpers_test.go b/x-pack/metricbeat/module/gcp/metrics/helpers_test.go new file mode 100644 index 00000000000..cdbb418be0d --- /dev/null +++ b/x-pack/metricbeat/module/gcp/metrics/helpers_test.go @@ -0,0 +1,37 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package metrics + +import "testing" + +func Test_withSuffix(t *testing.T) { + type args struct { + s string + suffix string + } + tests := []struct { + name string + args args + want string + }{ + { + name: "when suffix already present", + args: args{"foo/", "/"}, + want: "foo/", + }, + { + name: "when suffix missing", + args: args{"foo", "/"}, + want: "foo/", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := withSuffix(tt.args.s, tt.args.suffix); got != tt.want { + t.Errorf("withSuffix() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/x-pack/metricbeat/module/gcp/metrics/metrics_requester.go b/x-pack/metricbeat/module/gcp/metrics/metrics_requester.go index f195403fb1e..e5cf8c8b05d 100644 --- a/x-pack/metricbeat/module/gcp/metrics/metrics_requester.go +++ b/x-pack/metricbeat/module/gcp/metrics/metrics_requester.go @@ -104,6 +104,27 @@ func (r *metricsRequester) getFilterForMetric(serviceName, m string) (f string) } switch serviceName { + case gcp.ServiceGKE: + if r.config.Region != "" && r.config.Zone != "" { + r.logger.Warnf("when region %s and zone %s config parameter "+ + "both are provided, only use region", r.config.Region, r.config.Zone) + } + + region := r.config.Region + if region != "" { + if strings.HasSuffix(region, "*") { + region = strings.TrimSuffix(region, "*") + } + f = fmt.Sprintf("%s AND resource.label.location=starts_with(\"%s\")", f, region) + break + } + zone := r.config.Zone + if zone != "" { + if strings.HasSuffix(zone, "*") { + zone = strings.TrimSuffix(zone, "*") + } + f = fmt.Sprintf("%s AND resource.label.location=starts_with(\"%s\")", f, zone) + } case gcp.ServicePubsub, gcp.ServiceLoadBalancing, gcp.ServiceCloudFunctions: return case gcp.ServiceStorage: diff --git a/x-pack/metricbeat/module/gcp/metrics/metricset_test.go b/x-pack/metricbeat/module/gcp/metrics/metricset_test.go index 57d3cdd1f43..0e64210dab3 100644 --- a/x-pack/metricbeat/module/gcp/metrics/metricset_test.go +++ b/x-pack/metricbeat/module/gcp/metrics/metricset_test.go @@ -34,6 +34,11 @@ func Test_metricsConfig_AddPrefixTo(t *testing.T) { fields: fakeMetricsConfig[2], want: "foobar/" + metric, }, + { + name: "service metric prefix override (w/ dot)", + fields: metricsConfig{"billing", "foo.bar/", []string{}, ""}, + want: "foo.bar/" + metric, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/x-pack/metricbeat/module/gcp/module.yml b/x-pack/metricbeat/module/gcp/module.yml index 854a4af7154..527bd28b8fc 100644 --- a/x-pack/metricbeat/module/gcp/module.yml +++ b/x-pack/metricbeat/module/gcp/module.yml @@ -4,3 +4,4 @@ metricsets: - pubsub - loadbalancing - storage + - gke