diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index cb424dc4733..41d77146e0a 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -317,6 +317,7 @@ https://github.com/elastic/beats/compare/v6.2.3...master[Check the HEAD diff] - Release raid and socket metricset from system module as GA. {pull}7658[7658] - Release elasticsearch module and all its metricsets as beta. {pull}7662[7662] - Release munin and traefik module as beta. {pull}7660[7660] +- Add envoyproxy module. {pull}7569[7569] *Packetbeat* diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml index 0762a0aac20..0ddc7bf6cd1 100644 --- a/metricbeat/docker-compose.yml +++ b/metricbeat/docker-compose.yml @@ -17,6 +17,7 @@ services: - ./module/couchbase/_meta/env - ./module/dropwizard/_meta/env - ./module/elasticsearch/_meta/env + - ./module/envoyproxy/_meta/env - ./module/etcd/_meta/env - ./module/haproxy/_meta/env - ./module/http/_meta/env @@ -69,6 +70,9 @@ services: - "http.host=0.0.0.0" - "xpack.security.enabled=false" + envoyproxy: + build: ./module/envoyproxy/_meta + etcd: build: ./module/etcd/_meta diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 324d08b99db..88c86f2872b 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -23,6 +23,7 @@ grouped in the following categories: * <> * <> * <> +* <> * <> * <> * <> @@ -4301,6 +4302,455 @@ type: keyword The state of this shard. +-- + +[[exported-fields-envoyproxy]] +== envoyproxy fields + +experimental[] +envoyproxy module + + + +[float] +== envoyproxy fields + + + + +[float] +== server fields + +Contains envoy proxy server stats + + + + +*`envoyproxy.server.cluster_manager.active_clusters`*:: ++ +-- +type: integer + +Number of currently active (warmed) clusters + + +-- + +*`envoyproxy.server.cluster_manager.cluster_added`*:: ++ +-- +type: integer + +Total clusters added (either via static config or CDS) + + +-- + +*`envoyproxy.server.cluster_manager.cluster_modified`*:: ++ +-- +type: integer + +Total clusters modified (via CDS) + + +-- + +*`envoyproxy.server.cluster_manager.cluster_removed`*:: ++ +-- +type: integer + +Total clusters removed (via CDS) + + +-- + +*`envoyproxy.server.cluster_manager.warming_clusters`*:: ++ +-- +type: integer + +Number of currently warming (not active) clusters + + +-- + + +*`envoyproxy.server.filesystem.flushed_by_timer`*:: ++ +-- +type: integer + +Total number of times internal flush buffers are written to a file due to flush timeout + + +-- + +*`envoyproxy.server.filesystem.reopen_failed`*:: ++ +-- +type: integer + +Total number of times a file was failed to be opened + + +-- + +*`envoyproxy.server.filesystem.write_buffered`*:: ++ +-- +type: integer + +Total number of times file data is moved to Envoys internal flush buffer + + +-- + +*`envoyproxy.server.filesystem.write_completed`*:: ++ +-- +type: integer + +Total number of times a file was written + + +-- + +*`envoyproxy.server.filesystem.write_total_buffered`*:: ++ +-- +type: integer + +Current total size of internal flush buffer in bytes + + +-- + + +*`envoyproxy.server.runtime.load_error`*:: ++ +-- +type: integer + +Total number of load attempts that resulted in an error + + +-- + +*`envoyproxy.server.runtime.load_success`*:: ++ +-- +type: integer + +Total number of load attempts that were successful + + +-- + +*`envoyproxy.server.runtime.num_keys`*:: ++ +-- +type: integer + +Number of keys currently loaded + + +-- + +*`envoyproxy.server.runtime.override_dir_exists`*:: ++ +-- +type: integer + +Total number of loads that did use an override directory + + +-- + +*`envoyproxy.server.runtime.override_dir_not_exists`*:: ++ +-- +type: integer + +Total number of loads that did not use an override directory + + +-- + +*`envoyproxy.server.runtime.admin_overrides_active`*:: ++ +-- +type: integer + +-- + + +*`envoyproxy.server.listener_manager.listener_added`*:: ++ +-- +type: integer + +Total listeners added (either via static config or LDS) + + +-- + +*`envoyproxy.server.listener_manager.listener_create_failure`*:: ++ +-- +type: integer + +Total failed listener object additions to workers + + +-- + +*`envoyproxy.server.listener_manager.listener_create_success`*:: ++ +-- +type: integer + +Total listener objects successfully added to workers + + +-- + +*`envoyproxy.server.listener_manager.listener_modified`*:: ++ +-- +type: integer + +Total listeners modified (via LDS) + + +-- + +*`envoyproxy.server.listener_manager.listener_removed`*:: ++ +-- +type: integer + +Total listeners removed (via LDS) + + +-- + +*`envoyproxy.server.listener_manager.total_listeners_active`*:: ++ +-- +type: integer + +Number of currently active listeners + + +-- + +*`envoyproxy.server.listener_manager.total_listeners_draining`*:: ++ +-- +type: integer + +Number of currently draining listeners + + +-- + +*`envoyproxy.server.listener_manager.total_listeners_warming`*:: ++ +-- +type: integer + +Number of currently warming listeners + + +-- + + +*`envoyproxy.server.stats.overflow`*:: ++ +-- +type: integer + +Total number of times Envoy cannot allocate a statistic due to a shortage of shared memory + + +-- + + +*`envoyproxy.server.server.days_until_first_cert_expiring`*:: ++ +-- +type: integer + +Number of days until the next certificate being managed will expire + + +-- + +*`envoyproxy.server.server.live`*:: ++ +-- +type: integer + +1 if the server is not currently draining, 0 otherwise + + +-- + +*`envoyproxy.server.server.memory_allocated`*:: ++ +-- +type: integer + +Current amount of allocated memory in bytes + + +-- + +*`envoyproxy.server.server.memory_heap_size`*:: ++ +-- +type: integer + +Current reserved heap size in bytes + + +-- + +*`envoyproxy.server.server.parent_connections`*:: ++ +-- +type: integer + +Total connections of the old Envoy process on hot restart + + +-- + +*`envoyproxy.server.server.total_connections`*:: ++ +-- +type: integer + +Total connections of both new and old Envoy processes + + +-- + +*`envoyproxy.server.server.uptime`*:: ++ +-- +type: integer + +Current server uptime in seconds + + +-- + +*`envoyproxy.server.server.version`*:: ++ +-- +type: integer + +Integer represented version number based on SCM revision + + +-- + +*`envoyproxy.server.server.watchdog_mega_miss`*:: ++ +-- +type: integer + +-- + +*`envoyproxy.server.server.watchdog_miss`*:: ++ +-- +type: integer + +-- + +*`envoyproxy.server.server.hot_restart_epoch`*:: ++ +-- +type: integer + +Current hot restart epoch + + +-- + + +*`envoyproxy.server.http2.header_overflow`*:: ++ +-- +type: integer + +Total number of connections reset due to the headers being larger than Envoy::Http::Http2::ConnectionImpl::StreamImpl::MAX_HEADER_SIZE (63k) + + +-- + +*`envoyproxy.server.http2.headers_cb_no_stream`*:: ++ +-- +type: integer + +Total number of errors where a header callback is called without an associated stream. This tracks an unexpected occurrence due to an as yet undiagnosed bug + + +-- + +*`envoyproxy.server.http2.rx_messaging_error`*:: ++ +-- +type: integer + +Total number of invalid received frames that violated section 8 of the HTTP/2 spec. This will result in a tx_reset + + +-- + +*`envoyproxy.server.http2.rx_reset`*:: ++ +-- +type: integer + +Total number of reset stream frames received by Envoy + + +-- + +*`envoyproxy.server.http2.too_many_header_frames`*:: ++ +-- +type: integer + +Total number of times an HTTP2 connection is reset due to receiving too many headers frames. Envoy currently supports proxying at most one header frame for 100-Continue one non-100 response code header frame and one frame with trailers + + +-- + +*`envoyproxy.server.http2.trailers`*:: ++ +-- +type: integer + +Total number of trailers seen on requests coming from downstream + + +-- + +*`envoyproxy.server.http2.tx_reset`*:: ++ +-- +type: integer + +Total number of reset stream frames transmitted by Envoy + + -- [[exported-fields-etcd]] diff --git a/metricbeat/docs/modules/envoyproxy.asciidoc b/metricbeat/docs/modules/envoyproxy.asciidoc new file mode 100644 index 00000000000..030b71153cf --- /dev/null +++ b/metricbeat/docs/modules/envoyproxy.asciidoc @@ -0,0 +1,40 @@ +//// +This file is generated! See scripts/docs_collector.py +//// + +[[metricbeat-module-envoyproxy]] +== envoyproxy module + +experimental[] + +== envoyproxy module + +This is the envoyproxy module. + +The default metricset is `server`. + + +[float] +=== Example configuration + +The envoyproxy module supports the standard configuration options that are described +in <>. Here is an example configuration: + +[source,yaml] +---- +metricbeat.modules: +- module: envoyproxy + metricsets: ["server"] + period: 10s + hosts: ["localhost:9901"] +---- + +[float] +=== Metricsets + +The following metricsets are available: + +* <> + +include::envoyproxy/server.asciidoc[] + diff --git a/metricbeat/docs/modules/envoyproxy/server.asciidoc b/metricbeat/docs/modules/envoyproxy/server.asciidoc new file mode 100644 index 00000000000..66b3bf50212 --- /dev/null +++ b/metricbeat/docs/modules/envoyproxy/server.asciidoc @@ -0,0 +1,23 @@ +//// +This file is generated! See scripts/docs_collector.py +//// + +[[metricbeat-metricset-envoyproxy-server]] +=== envoyproxy server metricset + +experimental[] + +include::../../../module/envoyproxy/server/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/envoyproxy/server/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index cf7d64acb3d..3c129a04b1d 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -41,6 +41,8 @@ This file is generated! See scripts/docs_collector.py |<> beta[] |<> beta[] |<> beta[] +|<> experimental[] |image:./images/icon-no.png[No prebuilt dashboards] | +.1+| .1+| |<> experimental[] |<> beta[] |image:./images/icon-no.png[No prebuilt dashboards] | .3+| .3+| |<> beta[] |<> beta[] @@ -152,6 +154,7 @@ include::modules/couchbase.asciidoc[] include::modules/docker.asciidoc[] include::modules/dropwizard.asciidoc[] include::modules/elasticsearch.asciidoc[] +include::modules/envoyproxy.asciidoc[] include::modules/etcd.asciidoc[] include::modules/golang.asciidoc[] include::modules/graphite.asciidoc[] diff --git a/metricbeat/include/list.go b/metricbeat/include/list.go index 9ce42e5c9f6..04eadd6aeb8 100644 --- a/metricbeat/include/list.go +++ b/metricbeat/include/list.go @@ -62,6 +62,8 @@ import ( _ "github.com/elastic/beats/metricbeat/module/elasticsearch/node_stats" _ "github.com/elastic/beats/metricbeat/module/elasticsearch/pending_tasks" _ "github.com/elastic/beats/metricbeat/module/elasticsearch/shard" + _ "github.com/elastic/beats/metricbeat/module/envoyproxy" + _ "github.com/elastic/beats/metricbeat/module/envoyproxy/server" _ "github.com/elastic/beats/metricbeat/module/etcd" _ "github.com/elastic/beats/metricbeat/module/etcd/leader" _ "github.com/elastic/beats/metricbeat/module/etcd/self" diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 9e041845053..c2661613cc8 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -209,6 +209,12 @@ metricbeat.modules: # Set to false to fetch all entries #index_recovery.active_only: true +#----------------------------- envoyproxy Module ----------------------------- +- module: envoyproxy + metricsets: ["server"] + period: 10s + hosts: ["localhost:9901"] + #-------------------------------- Etcd Module -------------------------------- - module: etcd metricsets: ["leader", "self", "store"] diff --git a/metricbeat/module/envoyproxy/_meta/Dockerfile b/metricbeat/module/envoyproxy/_meta/Dockerfile new file mode 100644 index 00000000000..1de4e17c78c --- /dev/null +++ b/metricbeat/module/envoyproxy/_meta/Dockerfile @@ -0,0 +1,7 @@ +FROM envoyproxy/envoy:v1.7.0 +RUN apt-get update +COPY ./envoy.json /etc/envoy.json +EXPOSE 10000 9901 +HEALTHCHECK --interval=1s --retries=90 CMD wget -O - http://localhost:9901/clusters | grep health_flags | grep healthy +CMD /usr/local/bin/envoy -c /etc/envoy.json + diff --git a/metricbeat/module/envoyproxy/_meta/config.reference.yml b/metricbeat/module/envoyproxy/_meta/config.reference.yml new file mode 100644 index 00000000000..5ecc38dde07 --- /dev/null +++ b/metricbeat/module/envoyproxy/_meta/config.reference.yml @@ -0,0 +1,4 @@ +- module: envoyproxy + metricsets: ["server"] + period: 10s + hosts: ["localhost:9901"] diff --git a/metricbeat/module/envoyproxy/_meta/config.yml b/metricbeat/module/envoyproxy/_meta/config.yml new file mode 100644 index 00000000000..dde6380c98a --- /dev/null +++ b/metricbeat/module/envoyproxy/_meta/config.yml @@ -0,0 +1,5 @@ +- module: envoyproxy + #metricsets: + # - server + period: 10s + hosts: ["localhost:9901"] diff --git a/metricbeat/module/envoyproxy/_meta/docs.asciidoc b/metricbeat/module/envoyproxy/_meta/docs.asciidoc new file mode 100644 index 00000000000..45ba4494498 --- /dev/null +++ b/metricbeat/module/envoyproxy/_meta/docs.asciidoc @@ -0,0 +1,5 @@ +== envoyproxy module + +This is the envoyproxy module. + +The default metricset is `server`. diff --git a/metricbeat/module/envoyproxy/_meta/env b/metricbeat/module/envoyproxy/_meta/env new file mode 100644 index 00000000000..75a2386e7f6 --- /dev/null +++ b/metricbeat/module/envoyproxy/_meta/env @@ -0,0 +1,2 @@ +ENVOYPROXY_HOST=envoyproxy +ENVOYPROXY_PORT=9901 diff --git a/metricbeat/module/envoyproxy/_meta/envoy.json b/metricbeat/module/envoyproxy/_meta/envoy.json new file mode 100644 index 00000000000..dbd258d4876 --- /dev/null +++ b/metricbeat/module/envoyproxy/_meta/envoy.json @@ -0,0 +1,49 @@ +{ + "listeners": [{ + "address": "tcp://0.0.0.0:10000", + "filters": [{ + "name": "http_connection_manager", + "config": { + "codec_type": "auto", + "stat_prefix": "ingress_http", + "route_config": { + "virtual_hosts": [{ + "name": "local_service", + "domains": [ + "*" + ], + "routes": [{ + "timeout_ms": 0, + "prefix": "/", + "host_rewrite": "www.google.com", + "cluster": "service_google" + }] + }] + }, + "filters": [{ + "name": "router", + "config": {} + }] + } + }] + }], + "admin": { + "access_log_path": "/tmp/admin_access.log", + "address": "tcp://0.0.0.0:9901" + }, + "cluster_manager": { + "clusters": [{ + "name": "service_google", + "connect_timeout_ms": 250, + "type": "logical_dns", + "lb_type": "round_robin", + "hosts": [{ + "url": "tcp://google.com:443" + }], + "ssl_context": { + "sni": "www.google.com" + } + }] + } +} + diff --git a/metricbeat/module/envoyproxy/_meta/fields.yml b/metricbeat/module/envoyproxy/_meta/fields.yml new file mode 100644 index 00000000000..d3738a61ffc --- /dev/null +++ b/metricbeat/module/envoyproxy/_meta/fields.yml @@ -0,0 +1,11 @@ +- key: envoyproxy + title: "envoyproxy" + description: > + experimental[] + + envoyproxy module + fields: + - name: envoyproxy + type: group + description: > + fields: diff --git a/metricbeat/module/envoyproxy/_meta/test/serverstats b/metricbeat/module/envoyproxy/_meta/test/serverstats new file mode 100644 index 00000000000..34375203fae --- /dev/null +++ b/metricbeat/module/envoyproxy/_meta/test/serverstats @@ -0,0 +1,41 @@ +cluster_manager.active_clusters: 1 +cluster_manager.cluster_added: 1 +cluster_manager.cluster_modified: 0 +cluster_manager.cluster_removed: 0 +cluster_manager.warming_clusters: 0 +filesystem.flushed_by_timer: 389 +filesystem.reopen_failed: 0 +filesystem.write_buffered: 44 +filesystem.write_completed: 43 +filesystem.write_total_buffered: 0 +listener_manager.listener_added: 1 +listener_manager.listener_create_failure: 0 +listener_manager.listener_create_success: 4 +listener_manager.listener_modified: 0 +listener_manager.listener_removed: 0 +listener_manager.total_listeners_active: 1 +listener_manager.total_listeners_draining: 0 +listener_manager.total_listeners_warming: 0 +runtime.admin_overrides_active: 0 +runtime.load_error: 0 +runtime.load_success: 0 +runtime.num_keys: 0 +runtime.override_dir_exists: 0 +runtime.override_dir_not_exists: 0 +server.days_until_first_cert_expiring: 2147483647 +server.live: 1 +server.memory_allocated: 3120760 +server.memory_heap_size: 4194304 +server.parent_connections: 0 +server.total_connections: 0 +server.uptime: 5025 +server.version: 16364036 +server.watchdog_mega_miss: 4 +server.watchdog_miss: 4 +stats.overflow: 0 +http2.header_overflow: 0 +http2.headers_cb_no_stream: 0 +http2.rx_reset: 0 +http2.too_many_header_frames: 0 +http2.trailers: 0 +http2.tx_reset: 0 diff --git a/metricbeat/module/envoyproxy/doc.go b/metricbeat/module/envoyproxy/doc.go new file mode 100644 index 00000000000..380fe05f331 --- /dev/null +++ b/metricbeat/module/envoyproxy/doc.go @@ -0,0 +1,19 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Package envoyproxy is a Metricbeat module that contains MetricSets. +package envoyproxy diff --git a/metricbeat/module/envoyproxy/fields.go b/metricbeat/module/envoyproxy/fields.go new file mode 100644 index 00000000000..cd38beb5a27 --- /dev/null +++ b/metricbeat/module/envoyproxy/fields.go @@ -0,0 +1,35 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Code generated by beats/dev-tools/cmd/asset/asset.go - DO NOT EDIT. + +package envoyproxy + +import ( + "github.com/elastic/beats/libbeat/asset" +) + +func init() { + if err := asset.SetFields("metricbeat", "envoyproxy", Asset); err != nil { + panic(err) + } +} + +// Asset returns asset data +func Asset() string { + return "eJzEmV9v2zgSwN/zKQZ9SoFLL80Bh4MfDijSAC1wPSw2eVjsYkHQ5MjiRuJoyZEd76dfDCkpji07ThsrfigSV5z5zWj+Mhdwj+sZoF/Sugn0sD4DYMcVzuDd45fvzgAsRhNcw478DP57BgCADw0GV6NnXf32+1n+bjgENdm2wjOAwmFl4yz9/wV4XeOWRvnwusEZLAK1TffNiMansjblRQxLDMPXY/L2ysyfa/KsnY8ZDbINWSxE1hw3nt6m2CQxVRsZg6q114snSPux9snclKsNuyWqTnyEnQd74c4z7ip+xvr+8/+2nmMAKsC0IaDnat1phvOVDjXa972FcS9q7wJtLdrTcN4R62oggaQJztFxiQGWTqdX5gwY8oVbAAW4/nz7/lnimqwr3ETQvTI4F+Cj+ALWtJwIr9N1BJ0EhvMLtTcwThaanWY498RdnI7E52ieFq7CuI6M9aulaFG1sUSr5mvFrh6189Vekx+8IapiEhm8rjIEzNuiSIkREFbBMaMHJtDJbLAtym/5UTlPLe+1KiA16FWhXXXayNs2qYNd6QhZuTDPEQRnhGSIxuAYVXbAtMDZt5o1OMnuZSa+kX6y5wU9Y4ShuqmQ38ztXeQ8Q8kiZnD4iRrTdc55SMogur9QiEedCs7DfM34TAEIrReDXy37K9JWYQg0Ud6LPtDMWDccgUvNEDC2FaMVD2gP4zBPeGNrDMYTVewjiFcYEDqIoq32wvq2Vve4Pv3Yk5Q8NhhBPlBraIkhOIvKuqDwwUWe0JWdC62z0EaUF97jgHUBDVNYH0fuid+SXpr3yy3QtnZe9Qeiyu3/aANGa0LlIqM/wfQ+CD75TNxrOmoo/t+hsW5gNgE1Y5oA2nC8h78DvuvzvWag+R9oWCxxcjhKP11RuD+0fWxjT1DitnjjRkWT/Sm9h5eQT7GGPMbJ0z3kuJCYYBF5BHyyiRzky6PIcPKlNeFFlAdW5QHgaFAbtPPOL6ZD7TV+B2y3dU2/5u2ijlbx7esa+KHSLR2mqGg15QyedgYw2qe9tqrIaEboCniUGt6tcBpiSYH1Io3DsdQygddYbzbOcR9tX5gd5aQRgZtCrV5HJWN1pQoXIiuDQSaLxoUJ4kW0Q9IOXCJ4fGAQAFe45L45Sgzlzm5h5aoKEtpufXgsd6cqHh/BFYmyu2F0MU1Bu+n5D7gEkga+cnE/aH7jqo+UE9XlfgfTNbWexeWDwo5gd/XaQ1qibpRscaclDZj8a0H05a3xWcJGy1FlyHs0aeo46XXbo5qU/iUCVbYrAE0gGSOAPJSUrGEd9t/U5Dr9NuBz4hI8rkB7u2vBAXe3zcgW/kqkfRh0SZZVSQRENOTtfqglhuho9+LjVai+5sMQsJH49JI+ncK+Ecx1RCuv/fb6GwRculGax/tXNqWlhapxoVXtvmPY3RX1I1JKYtUFq8KGTHna17uRG/BU3WjrK5mbq1cbD0rUFoOadkrYTD6JIe7nAakfmSh2/a7SQYKNS+1zTs5mX5ib/O/VbHY9iPpaN9VsdssBdZ1//vbpF/Xl5tPnm5/V7ddfb+D83/+63z98d3qVmStPKiY507gjXXRFWJUYZErKIGB0Vc21uZfmKj+nps8ltQzag46RjEu9K6N+gLvSReCgzX2UJ1qPDw0aeYJM7sxmuDtPEmCNDK23Ti88ScbO290pZ7hvfFA1xqgXzi+mvCd0fqkrZyGgQSe9sAhaZsx0/bJ0VGUf5BiA//Rd6Mvd3U//vILYoOk8kyamfMWYLhiBH1QKvkMmjz9wEkNzIuSX2Rs5GD1f5+g/0DxJ1dqn0UQSOguYcujXPjn9aiO7JXSf5He2R/KaiWSWXQ/pnoE/9LvDMEnGtmkocMx/0pajmqGmyEC+Lxb5MBQU4OPl5cU1eXa+xfSEJ3/x8fJSOBryEcGQ3TqXer7H7jdJMkkjVx1cKvc9cBoPd9ogInpprAH/bDFyBENptSwC1WBp5ffUrQH7zUOag/axdsybUf13AAAA//+UGApY" +} diff --git a/metricbeat/module/envoyproxy/server/_meta/data.json b/metricbeat/module/envoyproxy/server/_meta/data.json new file mode 100644 index 00000000000..9c81813c7e9 --- /dev/null +++ b/metricbeat/module/envoyproxy/server/_meta/data.json @@ -0,0 +1,74 @@ +{ + "@timestamp": "2018-07-06T14:01:31.031Z", + "beat":{ + "hostname":"beathost", + "name":"beathost" + }, + "metricset":{ + "host":"localhost", + "module":"envoyproxy", + "name":"server", + "rtt":44269 + }, + "envoyproxy": { + "server": { + "filesystem": { + "write_buffered": 30, + "write_completed": 29, + "write_total_buffered": 0, + "flushed_by_timer": 311, + "reopen_failed": 0 + }, + "runtime": { + "override_dir_not_exists": 0, + "admin_overrides_active": 0, + "load_error": 0, + "load_success": 0, + "num_keys": 0, + "override_dir_exists": 0 + }, + "listener_manager": { + "listener_modified": 0, + "listener_removed": 0, + "total_listeners_active": 1, + "total_listeners_draining": 0, + "total_listeners_warming": 0, + "listener_added": 1, + "listener_create_failure": 0, + "listener_create_success": 4 + }, + "stats": { + "overflow": 0 + }, + "server": { + "version": 4151803, + "memory_allocated": 3170848, + "watchdog_miss": 0, + "uptime": 3146, + "watchdog_mega_miss": 0, + "hot_restart_epoch": 0, + "days_until_first_cert_expiring": 2147483647, + "live": 1, + "memory_heap_size": 4194304, + "parent_connections": 0, + "total_connections": 0 + }, + "http2": { + "header_overflow":0, + "headers_cb_no_stream":0, + "rx_reset":0, + "too_many_header_frames":0, + "trailers":0, + "tx_reset":0 + }, + "cluster_manager": { + "active_clusters": 1, + "cluster_added": 1, + "cluster_modified": 0, + "cluster_removed": 0, + "warming_clusters": 0 + } + } + }, + "type":"metricsets" +} diff --git a/metricbeat/module/envoyproxy/server/_meta/docs.asciidoc b/metricbeat/module/envoyproxy/server/_meta/docs.asciidoc new file mode 100644 index 00000000000..a0c6c79dd3d --- /dev/null +++ b/metricbeat/module/envoyproxy/server/_meta/docs.asciidoc @@ -0,0 +1,3 @@ +=== envoyproxy server MetricSet + +This is the server metricset of the module envoyproxy. diff --git a/metricbeat/module/envoyproxy/server/_meta/fields.yml b/metricbeat/module/envoyproxy/server/_meta/fields.yml new file mode 100644 index 00000000000..6c9efefe7a5 --- /dev/null +++ b/metricbeat/module/envoyproxy/server/_meta/fields.yml @@ -0,0 +1,198 @@ +- name: server + type: group + description: > + Contains envoy proxy server stats + fields: + - name: cluster_manager + type: group + fields: + - name: active_clusters + type: integer + description: > + Number of currently active (warmed) clusters + - name: cluster_added + type: integer + description: > + Total clusters added (either via static config or CDS) + - name: cluster_modified + type: integer + description: > + Total clusters modified (via CDS) + - name: cluster_removed + type: integer + description: > + Total clusters removed (via CDS) + - name: warming_clusters + type: integer + description: > + Number of currently warming (not active) clusters + + - name: filesystem + type: group + fields: + - name: flushed_by_timer + type: integer + description: > + Total number of times internal flush buffers are written to a file due to flush timeout + - name: reopen_failed + type: integer + description: > + Total number of times a file was failed to be opened + - name: write_buffered + type: integer + description: > + Total number of times file data is moved to Envoys internal flush buffer + - name: write_completed + type: integer + description: > + Total number of times a file was written + - name: write_total_buffered + type: integer + description: > + Current total size of internal flush buffer in bytes + + - name: runtime + type: group + fields: + - name: load_error + type: integer + description: > + Total number of load attempts that resulted in an error + - name: load_success + type: integer + description: > + Total number of load attempts that were successful + - name: num_keys + type: integer + description: > + Number of keys currently loaded + - name: override_dir_exists + type: integer + description: > + Total number of loads that did use an override directory + - name: override_dir_not_exists + type: integer + description: > + Total number of loads that did not use an override directory + - name: admin_overrides_active + type: integer + + - name: listener_manager + type: group + fields: + - name: listener_added + type: integer + description: > + Total listeners added (either via static config or LDS) + - name: listener_create_failure + type: integer + description: > + Total failed listener object additions to workers + - name: listener_create_success + type: integer + description: > + Total listener objects successfully added to workers + - name: listener_modified + type: integer + description: > + Total listeners modified (via LDS) + - name: listener_removed + type: integer + description: > + Total listeners removed (via LDS) + - name: total_listeners_active + type: integer + description: > + Number of currently active listeners + - name: total_listeners_draining + type: integer + description: > + Number of currently draining listeners + - name: total_listeners_warming + type: integer + description: > + Number of currently warming listeners + + - name: stats + type: group + fields: + - name: overflow + type: integer + description: > + Total number of times Envoy cannot allocate a statistic due to a shortage of shared memory + + - name: server + type: group + fields: + - name: days_until_first_cert_expiring + type: integer + description: > + Number of days until the next certificate being managed will expire + - name: live + type: integer + description: > + 1 if the server is not currently draining, 0 otherwise + - name: memory_allocated + type: integer + description: > + Current amount of allocated memory in bytes + - name: memory_heap_size + type: integer + description: > + Current reserved heap size in bytes + - name: parent_connections + type: integer + description: > + Total connections of the old Envoy process on hot restart + - name: total_connections + type: integer + description: > + Total connections of both new and old Envoy processes + - name: uptime + type: integer + description: > + Current server uptime in seconds + - name: version + type: integer + description: > + Integer represented version number based on SCM revision + - name: watchdog_mega_miss + type: integer + - name: watchdog_miss + type: integer + - name: hot_restart_epoch + type: integer + description: > + Current hot restart epoch + + - name: http2 + type: group + fields: + - name: header_overflow + type: integer + description: > + Total number of connections reset due to the headers being larger than Envoy::Http::Http2::ConnectionImpl::StreamImpl::MAX_HEADER_SIZE (63k) + - name: headers_cb_no_stream + type: integer + description: > + Total number of errors where a header callback is called without an associated stream. This tracks an unexpected occurrence due to an as yet undiagnosed bug + - name: rx_messaging_error + type: integer + description: > + Total number of invalid received frames that violated section 8 of the HTTP/2 spec. This will result in a tx_reset + - name: rx_reset + type: integer + description: > + Total number of reset stream frames received by Envoy + - name: too_many_header_frames + type: integer + description: > + Total number of times an HTTP2 connection is reset due to receiving too many headers frames. Envoy currently supports proxying at most one header frame for 100-Continue one non-100 response code header frame and one frame with trailers + - name: trailers + type: integer + description: > + Total number of trailers seen on requests coming from downstream + - name: tx_reset + type: integer + description: > + Total number of reset stream frames transmitted by Envoy diff --git a/metricbeat/module/envoyproxy/server/data.go b/metricbeat/module/envoyproxy/server/data.go new file mode 100644 index 00000000000..07f49a71464 --- /dev/null +++ b/metricbeat/module/envoyproxy/server/data.go @@ -0,0 +1,115 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package server + +import ( + "regexp" + "strings" + + "github.com/elastic/beats/libbeat/common" + s "github.com/elastic/beats/libbeat/common/schema" + c "github.com/elastic/beats/libbeat/common/schema/mapstrstr" +) + +var ( + schema = s.Schema{ + "cluster_manager": s.Object{ + "active_clusters": c.Int("active_clusters"), + "cluster_added": c.Int("cluster_added"), + "cluster_modified": c.Int("cluster_modified"), + "cluster_removed": c.Int("cluster_removed"), + "warming_clusters": c.Int("warming_clusters"), + }, + "filesystem": s.Object{ + "flushed_by_timer": c.Int("flushed_by_timer"), + "reopen_failed": c.Int("reopen_failed"), + "write_buffered": c.Int("write_buffered"), + "write_completed": c.Int("write_completed"), + "write_total_buffered": c.Int("write_total_buffered"), + }, + "runtime": s.Object{ + "load_error": c.Int("load_error"), + "load_success": c.Int("load_success"), + "num_keys": c.Int("num_keys"), + "override_dir_exists": c.Int("override_dir_exists"), + "override_dir_not_exists": c.Int("override_dir_not_exists"), + "admin_overrides_active": c.Int("admin_overrides_active", s.Optional), + }, + "listener_manager": s.Object{ + "listener_added": c.Int("listener_added"), + "listener_create_failure": c.Int("listener_create_failure"), + "listener_create_success": c.Int("listener_create_success"), + "listener_modified": c.Int("listener_modified"), + "listener_removed": c.Int("listener_removed"), + "total_listeners_active": c.Int("total_listeners_active"), + "total_listeners_draining": c.Int("total_listeners_draining"), + "total_listeners_warming": c.Int("total_listeners_warming"), + }, + "stats": s.Object{ + "overflow": c.Int("overflow"), + }, + "server": s.Object{ + "days_until_first_cert_expiring": c.Int("days_until_first_cert_expiring"), + "live": c.Int("live"), + "memory_allocated": c.Int("memory_allocated"), + "memory_heap_size": c.Int("memory_heap_size"), + "parent_connections": c.Int("parent_connections"), + "total_connections": c.Int("total_connections"), + "uptime": c.Int("uptime"), + "version": c.Int("version"), + "watchdog_mega_miss": c.Int("watchdog_mega_miss", s.Optional), + "watchdog_miss": c.Int("watchdog_miss", s.Optional), + "hot_restart_epoch": c.Int("hot_restart_epoch", s.Optional), + }, + "http2": s.Object{ + "header_overflow": c.Int("header_overflow", s.Optional), + "headers_cb_no_stream": c.Int("headers_cb_no_stream", s.Optional), + "rx_messaging_error": c.Int("rx_messaging_error", s.Optional), + "rx_reset": c.Int("rx_reset", s.Optional), + "too_many_header_frames": c.Int("too_many_header_frames", s.Optional), + "trailers": c.Int("trailers", s.Optional), + "tx_reset": c.Int("tx_reset", s.Optional), + }, + } +) +var reStats *regexp.Regexp = regexp.MustCompile(`cluster_manager.*|filesystem.*|runtime.*|listener_manager.*|stats.*|server.*|http2\..*`) + +func eventMapping(response []byte) (common.MapStr, error) { + data := map[string]interface{}{} + var events common.MapStr + var err error + + data = findStats(data, response) + events, err = schema.Apply(data) + if err != nil { + return nil, err + } + return events, nil +} + +func findStats(data common.MapStr, response []byte) common.MapStr { + matches := reStats.FindAllString(string(response), -1) + for i := 0; i < len(matches); i++ { + entries := strings.Split(matches[i], ": ") + if len(entries) == 2 { + temp := strings.Split(entries[0], ".") + data[temp[len(temp)-1]] = entries[1] + } + } + return data +} diff --git a/metricbeat/module/envoyproxy/server/server.go b/metricbeat/module/envoyproxy/server/server.go new file mode 100644 index 00000000000..f1fc8249a13 --- /dev/null +++ b/metricbeat/module/envoyproxy/server/server.go @@ -0,0 +1,80 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package server + +import ( + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/common/cfgwarn" + "github.com/elastic/beats/metricbeat/helper" + "github.com/elastic/beats/metricbeat/mb" + "github.com/elastic/beats/metricbeat/mb/parse" +) + +const ( + defaultScheme = "http" + defaultPath = "/stats" +) + +var ( + hostParser = parse.URLHostParserBuilder{ + DefaultScheme: defaultScheme, + DefaultPath: defaultPath, + }.Build() +) + +func init() { + mb.Registry.MustAddMetricSet("envoyproxy", "server", New, + mb.WithHostParser(hostParser), + mb.DefaultMetricSet(), + ) +} + +type MetricSet struct { + mb.BaseMetricSet + http *helper.HTTP +} + +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + cfgwarn.Experimental("The envoyproxy server metricset is experimental.") + + config := struct{}{} + if err := base.Module().UnpackConfig(&config); err != nil { + return nil, err + } + http, err := helper.NewHTTP(base) + if err != nil { + return nil, err + } + + return &MetricSet{ + base, + http, + }, nil +} + +func (m *MetricSet) Fetch() (common.MapStr, error) { + content, err := m.http.FetchContent() + if err != nil { + return nil, err + } + event, err := eventMapping(content) + if err != nil { + return nil, err + } + return event, nil +} diff --git a/metricbeat/module/envoyproxy/server/server_integration_test.go b/metricbeat/module/envoyproxy/server/server_integration_test.go new file mode 100644 index 00000000000..af001de4594 --- /dev/null +++ b/metricbeat/module/envoyproxy/server/server_integration_test.go @@ -0,0 +1,79 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build integration + +package server + +import ( + "os" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/elastic/beats/libbeat/tests/compose" + mbtest "github.com/elastic/beats/metricbeat/mb/testing" +) + +func TestData(t *testing.T) { + compose.EnsureUp(t, "envoyproxy") + + f := mbtest.NewEventFetcher(t, getConfig()) + err := mbtest.WriteEvent(f, t) + if err != nil { + t.Fatal("write", err) + } +} + +func TestFetch(t *testing.T) { + compose.EnsureUp(t, "envoyproxy") + + f := mbtest.NewEventFetcher(t, getConfig()) + event, err := f.Fetch() + if !assert.NoError(t, err) { + t.FailNow() + } + + assert.NotNil(t, event) + t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event) +} + +func getConfig() map[string]interface{} { + return map[string]interface{}{ + "module": "envoyproxy", + "metricsets": []string{"server"}, + "hosts": []string{GetEnvHost() + ":" + GetEnvPort()}, + } +} + +func GetEnvHost() string { + host := os.Getenv("ENVOYPROXY_HOST") + + if len(host) == 0 { + host = "127.0.0.1" + } + return host +} + +func GetEnvPort() string { + port := os.Getenv("ENVOYPROXY_PORT") + + if len(port) == 0 { + port = "9901" + } + return port +} diff --git a/metricbeat/module/envoyproxy/server/server_test.go b/metricbeat/module/envoyproxy/server/server_test.go new file mode 100644 index 00000000000..0c05fd668b1 --- /dev/null +++ b/metricbeat/module/envoyproxy/server/server_test.go @@ -0,0 +1,142 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package server + +import ( + "io/ioutil" + "net/http" + "net/http/httptest" + "path/filepath" + "testing" + "time" + + "github.com/elastic/beats/libbeat/common" + mbtest "github.com/elastic/beats/metricbeat/mb/testing" + + "github.com/stretchr/testify/assert" +) + +const testFile = "../_meta/test/serverstats" + +func TestEventMapping(t *testing.T) { + content, err := ioutil.ReadFile("../_meta/test/serverstats") + assert.NoError(t, err) + + event, err := eventMapping(content) + assert.NoError(t, err, "error mapping "+testFile) + + assert.Len(t, event, 7, "got wrong number of event") + + clusterManager := event["cluster_manager"].(common.MapStr) + assert.Equal(t, int64(1), clusterManager["active_clusters"]) + assert.Equal(t, int64(1), clusterManager["cluster_added"]) + assert.Equal(t, int64(0), clusterManager["cluster_modified"]) + assert.Equal(t, int64(0), clusterManager["cluster_removed"]) + assert.Equal(t, int64(0), clusterManager["warming_clusters"]) + + fileSystem := event["filesystem"].(common.MapStr) + assert.Equal(t, int64(389), fileSystem["flushed_by_timer"]) + assert.Equal(t, int64(0), fileSystem["reopen_failed"]) + assert.Equal(t, int64(44), fileSystem["write_buffered"]) + assert.Equal(t, int64(43), fileSystem["write_completed"]) + assert.Equal(t, int64(0), fileSystem["write_total_buffered"]) + + server := event["server"].(common.MapStr) + assert.Equal(t, int64(2147483647), server["days_until_first_cert_expiring"]) + assert.Equal(t, int64(1), server["live"]) + assert.Equal(t, int64(3120760), server["memory_allocated"]) + assert.Equal(t, int64(4194304), server["memory_heap_size"]) + assert.Equal(t, int64(0), server["parent_connections"]) + assert.Equal(t, int64(0), server["total_connections"]) + assert.Equal(t, int64(5025), server["uptime"]) + assert.Equal(t, int64(16364036), server["version"]) + assert.Equal(t, int64(4), server["watchdog_mega_miss"]) + assert.Equal(t, int64(4), server["watchdog_miss"]) + + stats := event["stats"].(common.MapStr) + assert.Equal(t, int64(0), stats["overflow"]) +} + +func TestFetchEventContent(t *testing.T) { + absPath, err := filepath.Abs("../_meta/test/") + assert.NoError(t, err) + + response, err := ioutil.ReadFile(absPath + "/serverstats") + assert.NoError(t, err) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(200) + w.Header().Set("Content-Type", "text/plain; charset=UTF-8") + w.Write([]byte(response)) + })) + defer server.Close() + + config := map[string]interface{}{ + "module": "envoyproxy", + "metricsets": []string{"server"}, + "hosts": []string{server.URL}, + } + + f := mbtest.NewEventFetcher(t, config) + event, err := f.Fetch() + if !assert.NoError(t, err) { + t.FailNow() + } + + t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), event) +} + +func testValue(t *testing.T, event common.MapStr, field string, value interface{}) { + data, err := event.GetValue(field) + assert.NoError(t, err, "Could not read field "+field) + assert.EqualValues(t, data, value, "Wrong value for field "+field) +} + +func TestFetchTimeout(t *testing.T) { + absPath, err := filepath.Abs("../_meta/test/") + assert.NoError(t, err) + + response, err := ioutil.ReadFile(absPath + "/serverstats") + assert.NoError(t, err) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(200) + w.Header().Set("Content-Type", "text/plain; charset=UTF-8") + w.Write([]byte(response)) + time.Sleep(100 * time.Millisecond) + })) + defer server.Close() + + config := map[string]interface{}{ + "module": "envoyproxy", + "metricsets": []string{"server"}, + "hosts": []string{server.URL}, + "timeout": "50ms", + } + + f := mbtest.NewEventFetcher(t, config) + + start := time.Now() + _, err = f.Fetch() + elapsed := time.Since(start) + if assert.Error(t, err) { + assert.Contains(t, err.Error(), "request canceled (Client.Timeout exceeded") + } + + assert.True(t, elapsed < 5*time.Second, "elapsed time: %s", elapsed.String()) +} diff --git a/metricbeat/modules.d/envoyproxy.yml.disabled b/metricbeat/modules.d/envoyproxy.yml.disabled new file mode 100644 index 00000000000..67d638f0b48 --- /dev/null +++ b/metricbeat/modules.d/envoyproxy.yml.disabled @@ -0,0 +1,8 @@ +# Module: envoyproxy +# Docs: https://www.elastic.co/guide/en/beats/metricbeat/master/metricbeat-module-envoyproxy.html + +- module: envoyproxy + #metricsets: + # - server + period: 10s + hosts: ["localhost:9901"] diff --git a/metricbeat/tests/system/test_envoyproxy.py b/metricbeat/tests/system/test_envoyproxy.py new file mode 100644 index 00000000000..40b7bce1a13 --- /dev/null +++ b/metricbeat/tests/system/test_envoyproxy.py @@ -0,0 +1,35 @@ +import os +import metricbeat +import unittest + + +class Test(metricbeat.BaseTest): + + COMPOSE_SERVICES = ['envoyproxy'] + + @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") + def test_stats(self): + """ + EnvoyProxy module outputs an event. + """ + self.render_config_template(modules=[{ + "name": "envoyproxy", + "metricsets": ["server"], + "hosts": self.get_hosts(), + "period": "5s", + }]) + proc = self.start_beat() + self.wait_until(lambda: self.output_lines() > 0, max_timeout=20) + proc.check_kill_and_wait() + self.assert_no_logged_warnings() + + output = self.read_output_json() + self.assertTrue(len(output) >= 1) + evt = output[0] + print(evt) + + self.assert_fields_are_documented(evt) + + def get_hosts(self): + return [os.getenv('ENVOYPROXY_HOST', 'localhost') + ':' + + os.getenv('ENVOYPROXY_PORT', '9901')]