diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 9fe64b4c2ad..edb5970cf42 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -134,6 +134,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - New module for Cisco ASA logs. {issue}9200[9200] {pull}11171[11171] - Added support for Cisco ASA fields to the netflow input. {pull}11201[11201] - Configurable line terminator. {pull}11015[11015] +- Add Filebeat envoyproxy module. {pull}11700[11700] *Heartbeat* diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index 4168ef88c79..b296f34bc8d 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -21,6 +21,7 @@ grouped in the following categories: * <> * <> * <> +* <> * <> * <> * <> @@ -5355,6 +5356,82 @@ example: doc Type +-- + +[[exported-fields-envoyproxy]] +== Envoyproxy fields + +Module for handling logs produced by envoy + + + +[float] +== envoyproxy fields + +Fields from envoy proxy logs after normalization + + + +*`envoyproxy.log_type`*:: ++ +-- +type: keyword + +Envoy log type, normally ACCESS + + +-- + +*`envoyproxy.response_flags`*:: ++ +-- +type: keyword + +Response flags + + +-- + +*`envoyproxy.upstream_service_time`*:: ++ +-- +type: long + +format: duration + +Upstream service time in nanoseconds + + +-- + +*`envoyproxy.request_id`*:: ++ +-- +type: keyword + +ID of the request + + +-- + +*`envoyproxy.authority`*:: ++ +-- +type: keyword + +Envoy proxy authority field + + +-- + +*`envoyproxy.proxy_type`*:: ++ +-- +type: keyword + +Envoy proxy type, tcp or http + + -- [[exported-fields-haproxy]] diff --git a/filebeat/docs/images/kibana-envoyproxy.jpg b/filebeat/docs/images/kibana-envoyproxy.jpg new file mode 100644 index 00000000000..ebdee56b99e Binary files /dev/null and b/filebeat/docs/images/kibana-envoyproxy.jpg differ diff --git a/filebeat/docs/modules/envoyproxy.asciidoc b/filebeat/docs/modules/envoyproxy.asciidoc new file mode 100644 index 00000000000..246ffb15f3a --- /dev/null +++ b/filebeat/docs/modules/envoyproxy.asciidoc @@ -0,0 +1,35 @@ +//// +This file is generated! See scripts/docs_collector.py +//// + +[[filebeat-module-envoyproxy]] +[role="xpack"] + +:modulename: envoyproxy +:has-dashboards: true + +== Envoyproxy Module + +This is a filebeat module for Envoy proxy access log (https://www.envoyproxy.io/docs/envoy/v1.10.0/configuration/access_log). It supports both standalone deployment and Envoy proxy deployment in Kubernetes. + +[float] +=== Compatibility + +Although this module has been developed against Envoy proxy 1.10.0 and Kubernetes v1.13.x, it is expected to work +with other versions of Envoy proxy and Kubernetes. + +[float] +=== Example dashboard + +This module comes with a sample dashboard. + +[role="screenshot"] +image::./images/kibana-envoyproxy.jpg[] + + +[float] +=== Fields + +For a description of each field in the module, see the +<> section. + diff --git a/filebeat/docs/modules_list.asciidoc b/filebeat/docs/modules_list.asciidoc index 79320965fd2..7fce59f6641 100644 --- a/filebeat/docs/modules_list.asciidoc +++ b/filebeat/docs/modules_list.asciidoc @@ -8,6 +8,7 @@ This file is generated! See scripts/docs_collector.py * <> * <> * <> + * <> * <> * <> * <> @@ -38,6 +39,7 @@ include::modules/auditd.asciidoc[] include::modules/cisco.asciidoc[] include::modules/coredns.asciidoc[] include::modules/elasticsearch.asciidoc[] +include::modules/envoyproxy.asciidoc[] include::modules/haproxy.asciidoc[] include::modules/icinga.asciidoc[] include::modules/iis.asciidoc[] diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index 5c42380a396..094af0e8f3c 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -156,6 +156,16 @@ filebeat.modules: #var.convert_timezone: false +#------------------------------ Envoyproxy Module ------------------------------ +- module: envoyproxy + # Fileset for native deployment + log: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: + #------------------------------- Haproxy Module ------------------------------- - module: haproxy # All logs diff --git a/x-pack/filebeat/include/list.go b/x-pack/filebeat/include/list.go index 2541405ea0d..be8a433b9e3 100644 --- a/x-pack/filebeat/include/list.go +++ b/x-pack/filebeat/include/list.go @@ -11,6 +11,7 @@ import ( _ "github.com/elastic/beats/x-pack/filebeat/input/netflow" _ "github.com/elastic/beats/x-pack/filebeat/module/cisco" _ "github.com/elastic/beats/x-pack/filebeat/module/coredns" + _ "github.com/elastic/beats/x-pack/filebeat/module/envoyproxy" _ "github.com/elastic/beats/x-pack/filebeat/module/iptables" _ "github.com/elastic/beats/x-pack/filebeat/module/netflow" _ "github.com/elastic/beats/x-pack/filebeat/module/suricata" diff --git a/x-pack/filebeat/module/envoyproxy/README.md b/x-pack/filebeat/module/envoyproxy/README.md new file mode 100644 index 00000000000..5690d87af4f --- /dev/null +++ b/x-pack/filebeat/module/envoyproxy/README.md @@ -0,0 +1,124 @@ +# Envoyproxy Module + +This is a filebeat module for Envoy proxy access log. + +## Caveats + +* Module is to be considered _beta_. + +## Download and install Filebeat + +Grab the filebeat binary from elastic.co, and install it by following the instructions. + +## Deployment Scenario #1: envoy native deployment + +This module assumes that envoy log entries will be written to /var/log/envoy.log. Should it be not the case, please point the module log path to the path of the log file. + +Update filebeat.yml to point to Elasticsearch and Kibana. +Setup Filebeat. +``` +./filebeat setup --modules envoyproxy -e +``` + +Enable the Filebeat envoyproxy module +``` +./filebeat modules enable envoyproxy +``` + +Start Filebeat +``` +./filebeat -e +``` + +Now, the Envoy logs and dashboard should appear in Kibana. + + +## Deployment Scenario #2: envoy for kubernetes + +For Kubernetes deployment, the filebeat daemon-set yaml file needs to be deployed to the Kubernetes cluster. Sample configuration files is provided under the `beats/deploy/filebeat` directory (https://github.com/elastic/beats/tree/master/deploy/kubernetes/filebeat), and can be deployed by doing the following: +``` +kubectl apply -f filebeat +``` + +#### Note the following section in the ConfigMap, make changes to the yaml file if necessary +``` + filebeat.autodiscover: + providers: + - type: kubernetes + hints.enabled: true + default.disable: true + + processors: + - add_kubernetes_metadata: + in_cluster: true +``` + +This enables auto-discovery and hints for filebeat. When default.disable is set to true (default value is false), it will disable log harvesting for the pod/container, unless it has specific annotations enabled. This gives users more granular control on kubernetes log ingestion. The `add_kubernetes_metadata` processor will add enrichment data for Kubernetes to the ingest logs. + +#### Note the following section in the DaemonSet, make changes to the yaml file if necessary +``` +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: filebeat + namespace: kube-system + labels: + k8s-app: filebeat +spec: + template: + metadata: + labels: + k8s-app: filebeat + spec: + serviceAccountName: filebeat + terminationGracePeriodSeconds: 30 + containers: + - name: filebeat + image: docker.elastic.co/beats/filebeat:%VERSION% + args: [ + "sh", "-c", "filebeat setup -e --modules envoyproxy -c /etc/filebeat.yml && filebeat -e -c /etc/filebeat.yml" + ] + env: + # Edit the following values to reflect your setup accordingly + - name: ELASTICSEARCH_HOST + value: 192.168.99.1 + - name: ELASTICSEARCH_USERNAME + value: elastic + - name: ELASTICSEARCH_PASSWORD + value: changeme + - name: KIBANA_HOST + value: 192.168.99.1 +``` + +The module setup step can also be done separately without Kubernetes if applicable, and in that case, the args can be simplified to: +``` + args: [ + "sh", "-c", "filebeat -e -c /etc/filebeat.yml" + ] +``` + +#### Sample Deployment for envoy, using ambassador as an example. Note the annotations. + +``` +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: ambassador +spec: + replicas: 3 + template: + metadata: + annotations: + "co.elastic.logs/module": "envoyproxy" + "co.elastic.logs/fileset": "log" + "co.elastic.logs/disable": "false" + labels: + service: ambassador + spec: + serviceAccountName: ambassador + containers: + - name: ambassador + image: quay.io/datawire/ambassador:0.50.0 + +``` + diff --git a/x-pack/filebeat/module/envoyproxy/_meta/config.yml b/x-pack/filebeat/module/envoyproxy/_meta/config.yml new file mode 100644 index 00000000000..c0fada4e3ae --- /dev/null +++ b/x-pack/filebeat/module/envoyproxy/_meta/config.yml @@ -0,0 +1,8 @@ +- module: envoyproxy + # Fileset for native deployment + log: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: diff --git a/x-pack/filebeat/module/envoyproxy/_meta/docs.asciidoc b/x-pack/filebeat/module/envoyproxy/_meta/docs.asciidoc new file mode 100644 index 00000000000..126b3f83c5e --- /dev/null +++ b/x-pack/filebeat/module/envoyproxy/_meta/docs.asciidoc @@ -0,0 +1,22 @@ +[role="xpack"] + +:modulename: envoyproxy +:has-dashboards: true + +== Envoyproxy Module + +This is a filebeat module for Envoy proxy access log (https://www.envoyproxy.io/docs/envoy/v1.10.0/configuration/access_log). It supports both standalone deployment and Envoy proxy deployment in Kubernetes. + +[float] +=== Compatibility + +Although this module has been developed against Envoy proxy 1.10.0 and Kubernetes v1.13.x, it is expected to work +with other versions of Envoy proxy and Kubernetes. + +[float] +=== Example dashboard + +This module comes with a sample dashboard. + +[role="screenshot"] +image::./images/kibana-envoyproxy.jpg[] diff --git a/x-pack/filebeat/module/envoyproxy/_meta/fields.yml b/x-pack/filebeat/module/envoyproxy/_meta/fields.yml new file mode 100644 index 00000000000..82fa9fefbf7 --- /dev/null +++ b/x-pack/filebeat/module/envoyproxy/_meta/fields.yml @@ -0,0 +1,45 @@ +- key: envoyproxy + title: Envoyproxy + description: > + Module for handling logs produced by envoy + fields: + - name: envoyproxy + type: group + description: > + Fields from envoy proxy logs after normalization + fields: + - name: log_type + type: keyword + description: > + Envoy log type, normally ACCESS + + - name: response_flags + type: keyword + description: > + Response flags + + - name: upstream_service_time + type: long + format: duration + input_format: nanoseconds + description: > + Upstream service time in nanoseconds + + - name: request_id + type: keyword + description: > + ID of the request + + - name: authority + type: keyword + description: > + Envoy proxy authority field + + - name: proxy_type + type: keyword + description: > + Envoy proxy type, tcp or http + + + + \ No newline at end of file diff --git a/x-pack/filebeat/module/envoyproxy/_meta/images/kibana-envoyproxy.jpg b/x-pack/filebeat/module/envoyproxy/_meta/images/kibana-envoyproxy.jpg new file mode 100644 index 00000000000..ebdee56b99e Binary files /dev/null and b/x-pack/filebeat/module/envoyproxy/_meta/images/kibana-envoyproxy.jpg differ diff --git a/x-pack/filebeat/module/envoyproxy/_meta/kibana/8/dashboard/Filebeat-Envoyproxy-Overview.json b/x-pack/filebeat/module/envoyproxy/_meta/kibana/8/dashboard/Filebeat-Envoyproxy-Overview.json new file mode 100644 index 00000000000..979c6c09cb6 --- /dev/null +++ b/x-pack/filebeat/module/envoyproxy/_meta/kibana/8/dashboard/Filebeat-Envoyproxy-Overview.json @@ -0,0 +1,910 @@ +{ + "objects": [ + { + "attributes": { + "description": "Filebeat Envoyproxy Overview Dashboard", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "optionsJSON": { + "hidePanelTitles": false, + "useMargins": true + }, + "panelsJSON": [ + { + "embeddableConfig": {}, + "gridData": { + "h": 7, + "i": "1", + "w": 22, + "x": 22, + "y": 0 + }, + "panelIndex": "1", + "panelRefName": "panel_0", + "version": "8.0.0-SNAPSHOT" + }, + { + "embeddableConfig": {}, + "gridData": { + "h": 10, + "i": "2", + "w": 22, + "x": 22, + "y": 7 + }, + "panelIndex": "2", + "panelRefName": "panel_1", + "version": "8.0.0-SNAPSHOT" + }, + { + "embeddableConfig": {}, + "gridData": { + "h": 10, + "i": "3", + "w": 22, + "x": 0, + "y": 7 + }, + "panelIndex": "3", + "panelRefName": "panel_2", + "version": "8.0.0-SNAPSHOT" + }, + { + "embeddableConfig": {}, + "gridData": { + "h": 7, + "i": "4", + "w": 22, + "x": 0, + "y": 0 + }, + "panelIndex": "4", + "panelRefName": "panel_3", + "version": "8.0.0-SNAPSHOT" + }, + { + "embeddableConfig": {}, + "gridData": { + "h": 10, + "i": "5", + "w": 22, + "x": 0, + "y": 17 + }, + "panelIndex": "5", + "panelRefName": "panel_4", + "version": "8.0.0-SNAPSHOT" + }, + { + "embeddableConfig": {}, + "gridData": { + "h": 10, + "i": "6", + "w": 22, + "x": 22, + "y": 17 + }, + "panelIndex": "6", + "panelRefName": "panel_5", + "version": "8.0.0-SNAPSHOT" + } + ], + "timeRestore": false, + "title": "Filebeat-Envoyproxy-Overview", + "version": 1 + }, + "id": "0c610510-5cbd-11e9-8477-077ec9664dbd", + "migrationVersion": { + "dashboard": "7.0.0" + }, + "references": [ + { + "id": "36f872a0-5c03-11e9-85b4-19d0072eb4f2", + "name": "panel_0", + "type": "visualization" + }, + { + "id": "80844540-5c97-11e9-8477-077ec9664dbd", + "name": "panel_1", + "type": "visualization" + }, + { + "id": "38f96190-5c99-11e9-8477-077ec9664dbd", + "name": "panel_2", + "type": "visualization" + }, + { + "id": "7e4084e0-5c99-11e9-8477-077ec9664dbd", + "name": "panel_3", + "type": "visualization" + }, + { + "id": "0a994af0-5c9d-11e9-8477-077ec9664dbd", + "name": "panel_4", + "type": "visualization" + }, + { + "id": "ab48c3f0-5ca6-11e9-8477-077ec9664dbd", + "name": "panel_5", + "type": "visualization" + } + ], + "type": "dashboard", + "updated_at": "2019-04-12T01:00:18.033Z", + "version": "WzExNjU4LDld" + }, + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [ + { + "$state": { + "store": "appState" + }, + "meta": { + "alias": null, + "disabled": false, + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", + "key": "event.dataset", + "negate": false, + "params": { + "query": "envoyproxy.log" + }, + "type": "phrase", + "value": "envoyproxy.log" + }, + "query": { + "match": { + "event.dataset": { + "query": "envoyproxy.log", + "type": "phrase" + } + } + } + } + ], + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index", + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "Top HTTP Response Codes [Filebeat Envoyproxy]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "field": "http.response.status_code", + "missingBucket": false, + "missingBucketLabel": "Missing", + "order": "desc", + "orderBy": "1", + "otherBucket": false, + "otherBucketLabel": "Other", + "size": 5 + }, + "schema": "segment", + "type": "terms" + } + ], + "params": { + "bucket": { + "accessor": 0, + "aggType": "terms", + "format": { + "id": "terms", + "params": { + "id": "number", + "missingBucketLabel": "Missing", + "otherBucketLabel": "Other" + } + }, + "params": {} + }, + "maxFontSize": 72, + "metric": { + "accessor": 1, + "aggType": "count", + "format": { + "id": "number" + }, + "params": {} + }, + "minFontSize": 18, + "orientation": "single", + "scale": "linear", + "showLabel": false + }, + "title": "Top HTTP Response Codes [Filebeat Envoyproxy]", + "type": "tagcloud" + } + }, + "id": "36f872a0-5c03-11e9-85b4-19d0072eb4f2", + "migrationVersion": { + "visualization": "7.0.0" + }, + "references": [ + { + "id": "filebeat-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + }, + { + "id": "filebeat-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", + "type": "index-pattern" + } + ], + "type": "visualization", + "updated_at": "2019-04-12T00:58:13.110Z", + "version": "WzExNjUzLDld" + }, + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [ + { + "$state": { + "store": "appState" + }, + "meta": { + "alias": null, + "disabled": false, + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", + "key": "event.dataset", + "negate": false, + "params": { + "query": "envoyproxy.log" + }, + "type": "phrase", + "value": "envoyproxy.log" + }, + "query": { + "match": { + "event.dataset": { + "query": "envoyproxy.log", + "type": "phrase" + } + } + } + } + ], + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index", + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "Top Domains [Filebeat Envoyproxy]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "field": "url.domain.keyword", + "missingBucket": false, + "missingBucketLabel": "Missing", + "order": "desc", + "orderBy": "1", + "otherBucket": false, + "otherBucketLabel": "Other", + "size": 5 + }, + "schema": "segment", + "type": "terms" + } + ], + "params": { + "addLegend": true, + "addTooltip": true, + "dimensions": { + "metric": { + "accessor": 0, + "aggType": "count", + "format": { + "id": "number" + }, + "params": {} + } + }, + "isDonut": true, + "labels": { + "last_level": true, + "show": false, + "truncate": 100, + "values": true + }, + "legendPosition": "right", + "type": "pie" + }, + "title": "Top Domains [Filebeat Envoyproxy]", + "type": "pie" + } + }, + "id": "80844540-5c97-11e9-8477-077ec9664dbd", + "migrationVersion": { + "visualization": "7.0.0" + }, + "references": [ + { + "id": "filebeat-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + }, + { + "id": "filebeat-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", + "type": "index-pattern" + } + ], + "type": "visualization", + "updated_at": "2019-04-12T00:58:53.299Z", + "version": "WzExNjU1LDld" + }, + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [ + { + "$state": { + "store": "appState" + }, + "meta": { + "alias": null, + "disabled": false, + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", + "key": "event.dataset", + "negate": false, + "params": { + "query": "envoyproxy.log" + }, + "type": "phrase", + "value": "envoyproxy.log" + }, + "query": { + "match": { + "event.dataset": { + "query": "envoyproxy.log", + "type": "phrase" + } + } + } + } + ], + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index", + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "Requests per Source [Filebeat Envoyproxy]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "field": "source.address.keyword", + "missingBucket": false, + "missingBucketLabel": "Missing", + "order": "desc", + "orderBy": "1", + "otherBucket": false, + "otherBucketLabel": "Other", + "size": 5 + }, + "schema": "segment", + "type": "terms" + } + ], + "params": { + "addLegend": true, + "addTimeMarker": false, + "addTooltip": true, + "categoryAxes": [ + { + "id": "CategoryAxis-1", + "labels": { + "show": true, + "truncate": 100 + }, + "position": "bottom", + "scale": { + "type": "linear" + }, + "show": true, + "style": {}, + "title": {}, + "type": "category" + } + ], + "dimensions": { + "x": null, + "y": [ + { + "accessor": 0, + "aggType": "count", + "format": { + "id": "number" + }, + "params": {} + } + ] + }, + "grid": { + "categoryLines": false + }, + "legendPosition": "right", + "seriesParams": [ + { + "data": { + "id": "1", + "label": "Count" + }, + "drawLinesBetweenPoints": true, + "mode": "stacked", + "show": "true", + "showCircles": true, + "type": "histogram", + "valueAxis": "ValueAxis-1" + } + ], + "times": [], + "type": "histogram", + "valueAxes": [ + { + "id": "ValueAxis-1", + "labels": { + "filter": false, + "rotate": 0, + "show": true, + "truncate": 100 + }, + "name": "LeftAxis-1", + "position": "left", + "scale": { + "mode": "normal", + "type": "linear" + }, + "show": true, + "style": {}, + "title": { + "text": "Count" + }, + "type": "value" + } + ] + }, + "title": "Requests per Source [Filebeat Envoyproxy]", + "type": "histogram" + } + }, + "id": "38f96190-5c99-11e9-8477-077ec9664dbd", + "migrationVersion": { + "visualization": "7.0.0" + }, + "references": [ + { + "id": "filebeat-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + }, + { + "id": "filebeat-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", + "type": "index-pattern" + } + ], + "type": "visualization", + "updated_at": "2019-04-12T00:58:36.398Z", + "version": "WzExNjU0LDld" + }, + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [ + { + "$state": { + "store": "appState" + }, + "meta": { + "alias": null, + "disabled": false, + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", + "key": "event.dataset", + "negate": false, + "params": { + "query": "envoyproxy.log" + }, + "type": "phrase", + "value": "envoyproxy.log" + }, + "query": { + "match": { + "event.dataset": { + "query": "envoyproxy.log", + "type": "phrase" + } + } + } + } + ], + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index", + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "Unique Domains [Filebeat Envoyproxy]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": { + "field": "url.domain.keyword" + }, + "schema": "metric", + "type": "cardinality" + } + ], + "params": { + "addLegend": false, + "addTooltip": true, + "metric": { + "bucket": { + "accessor": 0, + "aggType": "terms", + "format": { + "id": "terms", + "params": { + "id": "string", + "missingBucketLabel": "Missing", + "otherBucketLabel": "Other" + } + }, + "params": {} + }, + "colorSchema": "Green to Red", + "colorsRange": [ + { + "from": 0, + "to": 10000 + } + ], + "invertColors": false, + "labels": { + "show": false + }, + "metricColorMode": "None", + "metrics": [ + { + "accessor": 0, + "aggType": "cardinality", + "format": { + "id": "number" + }, + "params": {} + } + ], + "percentageMode": false, + "style": { + "bgColor": false, + "bgFill": "#000", + "fontSize": 60, + "labelColor": false, + "subText": "" + }, + "useRanges": false + }, + "type": "metric" + }, + "title": "Unique Domains [Filebeat Envoyproxy]", + "type": "metric" + } + }, + "id": "7e4084e0-5c99-11e9-8477-077ec9664dbd", + "migrationVersion": { + "visualization": "7.0.0" + }, + "references": [ + { + "id": "filebeat-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + }, + { + "id": "filebeat-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", + "type": "index-pattern" + } + ], + "type": "visualization", + "updated_at": "2019-04-12T00:57:42.389Z", + "version": "WzExNjUyLDld" + }, + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [ + { + "$state": { + "store": "appState" + }, + "meta": { + "alias": null, + "disabled": false, + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", + "key": "event.dataset", + "negate": false, + "params": { + "query": "envoyproxy.log" + }, + "type": "phrase", + "value": "envoyproxy.log" + }, + "query": { + "match": { + "event.dataset": { + "query": "envoyproxy.log", + "type": "phrase" + } + } + } + } + ], + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index", + "query": { + "language": "lucene", + "query": "" + } + } + }, + "title": "Top User Agents [Filebeat Envoyproxy]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "field": "user_agent.name.keyword", + "missingBucket": false, + "missingBucketLabel": "Missing", + "order": "desc", + "orderBy": "1", + "otherBucket": false, + "otherBucketLabel": "Other", + "size": 5 + }, + "schema": "segment", + "type": "terms" + } + ], + "params": { + "addLegend": true, + "addTooltip": true, + "dimensions": { + "metric": { + "accessor": 0, + "aggType": "count", + "format": { + "id": "number" + }, + "params": {} + } + }, + "isDonut": true, + "labels": { + "last_level": true, + "show": false, + "truncate": 100, + "values": true + }, + "legendPosition": "right", + "type": "pie" + }, + "title": "Top User Agents [Filebeat Envoyproxy]", + "type": "pie" + } + }, + "id": "0a994af0-5c9d-11e9-8477-077ec9664dbd", + "migrationVersion": { + "visualization": "7.0.0" + }, + "references": [ + { + "id": "filebeat-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + }, + { + "id": "filebeat-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", + "type": "index-pattern" + } + ], + "type": "visualization", + "updated_at": "2019-04-12T00:59:11.691Z", + "version": "WzExNjU2LDld" + }, + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [ + { + "$state": { + "store": "appState" + }, + "meta": { + "alias": null, + "disabled": false, + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", + "key": "event.dataset", + "negate": false, + "params": { + "query": "envoyproxy.log" + }, + "type": "phrase", + "value": "envoyproxy.log" + }, + "query": { + "match": { + "event.dataset": { + "query": "envoyproxy.log", + "type": "phrase" + } + } + } + } + ], + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index", + "query": { + "language": "kuery", + "query": "event.dataset:envoyproxy.log" + } + } + }, + "title": "Proxy Request Distribution [Filebeat Envoyproxy] ", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "field": "envoyproxy.proxy_type.keyword", + "missingBucket": false, + "missingBucketLabel": "Missing", + "order": "desc", + "orderBy": "1", + "otherBucket": false, + "otherBucketLabel": "Other", + "size": 5 + }, + "schema": "segment", + "type": "terms" + } + ], + "params": { + "addLegend": true, + "addTooltip": true, + "dimensions": { + "metric": { + "accessor": 0, + "aggType": "count", + "format": { + "id": "number" + }, + "params": {} + } + }, + "isDonut": true, + "labels": { + "last_level": true, + "show": false, + "truncate": 100, + "values": true + }, + "legendPosition": "right", + "type": "pie" + }, + "title": "Proxy Request Distribution [Filebeat Envoyproxy] ", + "type": "pie" + } + }, + "id": "ab48c3f0-5ca6-11e9-8477-077ec9664dbd", + "migrationVersion": { + "visualization": "7.0.0" + }, + "references": [ + { + "id": "filebeat-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + }, + { + "id": "filebeat-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", + "type": "index-pattern" + } + ], + "type": "visualization", + "updated_at": "2019-04-11T22:10:51.951Z", + "version": "WzExNjQ5LDld" + } + ], + "version": "8.0.0-SNAPSHOT" +} \ No newline at end of file diff --git a/x-pack/filebeat/module/envoyproxy/fields.go b/x-pack/filebeat/module/envoyproxy/fields.go new file mode 100644 index 00000000000..1b4aa69c83c --- /dev/null +++ b/x-pack/filebeat/module/envoyproxy/fields.go @@ -0,0 +1,23 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// Code generated by beats/dev-tools/cmd/asset/asset.go - DO NOT EDIT. + +package envoyproxy + +import ( + "github.com/elastic/beats/libbeat/asset" +) + +func init() { + if err := asset.SetFields("filebeat", "envoyproxy", asset.ModuleFieldsPri, AssetEnvoyproxy); err != nil { + panic(err) + } +} + +// AssetEnvoyproxy returns asset data. +// This is the base64 encoded gzipped contents of module/envoyproxy. +func AssetEnvoyproxy() string { + return "eJykk0Fu2zAQRfc8xT9AfQEtChSuC3TRTY2uBVYcyYQpDkMOnTCnDyTKdmQYAZwsNeJ/bzQabnCk0oD8iUuI/FIUIFYcNdi9rxlKXbRBLPsG3xUA/GGTHaHniIP2xlk/wPGQECKb3JHB/1LBCugtOZMaBWzg9Ug3SkBKoAZD5Bzm5ztC4NdMQR95rHnMgKrVvVCE5zhqZ1/1lJxjV/VV7nhoJ+PCrfIjlWeOZqndbQB1KlN+Dn1bfK7gx3a72+/VShMpBfaJ2t7pIT0u+7vkUfMrdg5JIumxTRRPtqNW7Lj+Hsd+WAr91KU0MDleBwNYH7K055dee07UsTfp47b+LWosakxqWL8i3AziKVOS1prHh/D7J7iHHOhMWaN1lgNHK+Wz/7Ku0AVT92XtmI98aWGqpK6MdAHTpREJSqnLQfUWAAD//zjwF5k=" +} diff --git a/x-pack/filebeat/module/envoyproxy/log/config/envoyproxy.yml b/x-pack/filebeat/module/envoyproxy/log/config/envoyproxy.yml new file mode 100644 index 00000000000..c085c9e3aab --- /dev/null +++ b/x-pack/filebeat/module/envoyproxy/log/config/envoyproxy.yml @@ -0,0 +1,7 @@ +type: log +paths: +{{ range $i, $path := .paths }} + - {{$path}} +{{ end }} +tags: {{.tags}} +processors: diff --git a/x-pack/filebeat/module/envoyproxy/log/ingest/pipeline-entry.json b/x-pack/filebeat/module/envoyproxy/log/ingest/pipeline-entry.json new file mode 100644 index 00000000000..600102c286a --- /dev/null +++ b/x-pack/filebeat/module/envoyproxy/log/ingest/pipeline-entry.json @@ -0,0 +1,42 @@ +{ + "description": "Pipeline for normalizing envoyproxy logs", + "processors": [ + { + "pipeline": { + "if": "ctx.message.charAt(0) != (char)(\"{\")", + "name": "{< IngestPipeline "pipeline-plaintext" >}" + } + }, + { + "pipeline": { + "if": "ctx.message.charAt(0) == (char)(\"{\")", + "name": "{< IngestPipeline "pipeline-json" >}" + } + }, + { + "set": { + "field": "event.created", + "value": "{{@timestamp}}" + } + }, + { + "set": { + "field": "@timestamp", + "value": "{{timestamp}}", + "if": "ctx.timestamp != null" + } + }, + { + "remove": { + "field": ["timestamp"], + "ignore_failure": true + } + } + ], + "on_failure" : [{ + "set" : { + "field" : "error.message", + "value" : "pipeline-entry: {{ _ingest.on_failure_message }}" + } + }] +} \ No newline at end of file diff --git a/x-pack/filebeat/module/envoyproxy/log/ingest/pipeline-http.json b/x-pack/filebeat/module/envoyproxy/log/ingest/pipeline-http.json new file mode 100644 index 00000000000..e38875cfeca --- /dev/null +++ b/x-pack/filebeat/module/envoyproxy/log/ingest/pipeline-http.json @@ -0,0 +1,94 @@ +{ + "description": "Pipeline for normalizing envoy HTTP ACCESS logs", + "processors": [ + { + "script": { + "lang": "painless", + "source": "ctx['http'] = new HashMap(); def p = ctx.proto.indexOf ('/'); def l = ctx.proto.length(); ctx.http.version = ctx.proto.substring(p+1, l);", + "ignore_failure" : true + } + }, + { + "rename": { + "field": "method", + "target_field": "http.request.method" + } + }, + { + "rename": { + "field": "path", + "target_field": "url.path" + } + }, + { + "convert" : { + "field" : "response_code", + "type": "long" + } + }, + { + "rename": { + "field": "response_code", + "target_field": "http.response.status_code" + } + }, + { + "rename": { + "field": "bytes_received", + "target_field": "http.response.body.bytes" + } + }, + { + "convert" : { + "field" : "http.response.body.bytes", + "type": "long" + } + }, + { + "rename": { + "field": "bytes_sent", + "target_field": "http.request.body.bytes" + } + }, + { + "convert" : { + "field" : "http.request.body.bytes", + "type": "long" + } + }, + { + "script": { + "lang": "painless", + "source": "ctx.envoyproxy.upstream_service_time = Math.round(Double.parseDouble(ctx.upstream_service_time) * params.scale)", + "params": { + "scale": 1000000 + }, + "if": "ctx.upstream_service_time != null && ctx.upstream_service_time != '-'" + } + }, + { + "set": { + "field": "envoyproxy.proxy_type", + "value": "http" + } + }, + { + "set": { + "field": "url.domain", + "value": "{{envoyproxy.authority}}" + } + }, + { + "user_agent": { + "field": "user_agent.original", + "ignore_missing": true + } + } + ], + "on_failure" : [{ + "set" : { + "field" : "error.message", + "value" : "pipeline-http: {{ _ingest.on_failure_message }}" + } + }] +} \ No newline at end of file diff --git a/x-pack/filebeat/module/envoyproxy/log/ingest/pipeline-json.json b/x-pack/filebeat/module/envoyproxy/log/ingest/pipeline-json.json new file mode 100644 index 00000000000..ce2244c8b06 --- /dev/null +++ b/x-pack/filebeat/module/envoyproxy/log/ingest/pipeline-json.json @@ -0,0 +1,47 @@ +{ + "description": "Pipeline for normalizing envoyproxy access logs", + "processors": [ + { + "json" : { + "field" : "message", + "target_field" : "json" + } + }, + { + "remove": { + "field": ["message"], + "ignore_failure" : true + } + }, + { + "rename": { + "field": "json.message", + "target_field": "message", + "ignore_failure" : true + } + }, + { + "rename": { + "field": "json.kubernetes", + "target_field": "kubernetes", + "ignore_failure" : true + } + }, + { + "remove": { + "field": ["json"] + } + }, + { + "pipeline": { + "name": "{< IngestPipeline "pipeline-plaintext" >}" + } + } + ], + "on_failure" : [{ + "set" : { + "field" : "error.message", + "value" : "pipeline-json: {{ _ingest.on_failure_message }}" + } + }] +} \ No newline at end of file diff --git a/x-pack/filebeat/module/envoyproxy/log/ingest/pipeline-plaintext.json b/x-pack/filebeat/module/envoyproxy/log/ingest/pipeline-plaintext.json new file mode 100644 index 00000000000..992b6c65ec2 --- /dev/null +++ b/x-pack/filebeat/module/envoyproxy/log/ingest/pipeline-plaintext.json @@ -0,0 +1,124 @@ +{ + "description": "Pipeline for normalizing envoy access logs", + "processors": [ + { + "script": { + "lang": "painless", + "source": "if (ctx.message.charAt(0) == (char)(\"[\")) { ctx.temp_message = \"ACCESS \" + ctx.message;} else if (ctx.message.substring(0, 7) == \"ACCESS \") { ctx.temp_message = ctx.message;} else { throw new Exception(\"Not a valid envoyproxy access log\");}" + } + }, + { + "dissect": { + "field": "temp_message", + "pattern": "%{envoyproxy.log_type} [%{timestamp}] \"%{method} %{path} %{proto}\" %{response_code} %{envoyproxy.response_flags} %{bytes_received} %{bytes_sent} %{duration} %{upstream_service_time} \"%{source.address}\" \"%{user_agent.original}\" \"%{envoyproxy.request_id}\" \"%{envoyproxy.authority}\" \"%{dest}\"", + "on_failure" : [{ + "script": { + "lang": "painless", + "source": "ctx.remove('temp_message'); throw new Exception(\"Dissect error: Not a valid envoyproxy access log\");" + } + }] + } + }, + { + "script": { + "lang": "painless", + "source": "if (ctx.dest == \"-\") { ctx.remove('dest');} else { ctx['destination'] = new HashMap(); def p = ctx.dest.indexOf (':'); def l = ctx.dest.length(); ctx.destination.address = ctx.dest.substring(0, p); ctx.destination.port = ctx.dest.substring(p+1, l);} ctx.remove('dest');", + "if": "ctx.dest != null" + } + }, + { + "convert" : { + "field" : "destination.port", + "type": "integer", + "if": "ctx.destination?.port != null" + } + }, + { + "convert" : { + "field" : "duration", + "type": "double", + "if": "ctx.duration != null" + } + }, + { + "script": { + "lang": "painless", + "source": "ctx.event.duration = Math.round(ctx.duration * params.scale)", + "params": { + "scale": 1000000 + }, + "if": "ctx.duration != null" + } + }, + { + "remove": { + "field": ["json", "duration", "time", "temp_message"], + "ignore_missing": true + } + }, + { + "pipeline": { + "if": "ctx.proto.charAt(0) != (char)(\"-\")", + "name": "{< IngestPipeline "pipeline-http" >}" + } + }, + { + "pipeline": { + "if": "ctx.proto.charAt(0) == (char)(\"-\")", + "name": "{< IngestPipeline "pipeline-tcp" >}" + } + }, + { + "remove": { + "field": ["proto", "upstream_service_time"], + "ignore_failure": true + } + }, + { + "remove": { + "field": "source.address", + "if": "ctx.source.address == '-'" + } + }, + { + "remove": { + "field": "envoyproxy.response_flags", + "if": "ctx.envoyproxy.response_flags == '-'" + } + }, + { + "split": { + "field": "envoyproxy.response_flags", + "separator": "," , + "if": "ctx.envoyproxy.response_flags != null" + } + }, + { + "set" : { + "field" : "destination.ip", + "value" : "{{destination.address}}", + "if": "ctx.destination?.address != null" + } + }, + { + "set" : { + "field" : "source.ip", + "value" : "{{source.address}}", + "if": "ctx.source?.address != null" + } + }, + { + "geoip": { + "field": "destination.ip", + "target_field": "destination.geo", + "if": "ctx.destination?.ip != null" + } + } + ], + "on_failure" : [{ + "set" : { + "field" : "error.message", + "value" : "pipeline-plaintext: {{ _ingest.on_failure_message }}" + } + }] +} \ No newline at end of file diff --git a/x-pack/filebeat/module/envoyproxy/log/ingest/pipeline-tcp.json b/x-pack/filebeat/module/envoyproxy/log/ingest/pipeline-tcp.json new file mode 100644 index 00000000000..8a84954dc23 --- /dev/null +++ b/x-pack/filebeat/module/envoyproxy/log/ingest/pipeline-tcp.json @@ -0,0 +1,46 @@ +{ + "description": "Pipeline for normalizing envoy TCP ACCESS logs", + "processors": [ + { + "remove": { + "field": ["upstream_service_time", "method", "user_agent", "path", "response_code"] + } + }, + { + "rename": { + "field": "bytes_received", + "target_field": "destination.bytes" + } + }, + { + "convert" : { + "field" : "destination.bytes", + "type": "long" + } + }, + { + "rename": { + "field": "bytes_sent", + "target_field": "source.bytes" + } + }, + { + "convert" : { + "field" : "source.bytes", + "type": "long" + } + }, + { + "set": { + "field": "envoyproxy.proxy_type", + "value": "tcp" + } + } + ], + "on_failure" : [{ + "set" : { + "field" : "error.message", + "value" : "pipeline-tcp: {{ _ingest.on_failure_message }}" + } + }] +} \ No newline at end of file diff --git a/x-pack/filebeat/module/envoyproxy/log/manifest.yml b/x-pack/filebeat/module/envoyproxy/log/manifest.yml new file mode 100644 index 00000000000..d52208df0fc --- /dev/null +++ b/x-pack/filebeat/module/envoyproxy/log/manifest.yml @@ -0,0 +1,17 @@ +module_version: 1.0 + +var: + - name: paths + default: + # Change this if it is not "/var/log/envoy.log" in native deployment mode + - /var/log/envoy.log + - name: tags + default: [envoyproxy] + +ingest_pipeline: + - ingest/pipeline-entry.json + - ingest/pipeline-json.json + - ingest/pipeline-plaintext.json + - ingest/pipeline-http.json + - ingest/pipeline-tcp.json +input: config/envoyproxy.yml diff --git a/x-pack/filebeat/module/envoyproxy/log/test/envoy-json.log b/x-pack/filebeat/module/envoyproxy/log/test/envoy-json.log new file mode 100644 index 00000000000..31cb674f4c7 --- /dev/null +++ b/x-pack/filebeat/module/envoyproxy/log/test/envoy-json.log @@ -0,0 +1,2 @@ +{"message":"ACCESS [2019-04-10T03:49:34.451Z] \"GET /httpbin/status/501 HTTP/1.1\" 501 - 0 0 180 179 \"172.17.0.3\" \"curl/7.59.0\" \"413bf460-bd56-4515-ada4-2a69c5e78e54\" \"httpbin.org\" \"52.71.234.219:80\"","stream":"stdout","time":"2019-02-12T18:37:43.139620629Z", "kubernetes": { "container": { "name": "ambassador" }, "node": { "name": "minikube" }, "pod": { "uid": "e57d545e-2a9d-11e9-995f-08002730e0dc", "name": "ambassador-76c58d9df4-jwhsg" }, "namespace": "default", "labels": { "service": "ambassador" }}} +{"message":"ACCESS [2019-04-06T06:20:05.972Z] \"- - -\" 0 UF,URX 0 0 0 - \"-\" \"-\" \"-\" \"-\" \"127.0.0.1:9200\"","stream":"stdout","time":"2019-02-12T18:37:43.139620629Z", "kubernetes": { "container": { "name": "ambassador" }, "node": { "name": "minikube" }, "pod": { "uid": "e57d545e-2a9d-11e9-995f-08002730e0dc", "name": "ambassador-76c58d9df4-jwhsg" }, "namespace": "default", "labels": { "service": "ambassador" }}}} diff --git a/x-pack/filebeat/module/envoyproxy/log/test/envoy-json.log-expected.json b/x-pack/filebeat/module/envoyproxy/log/test/envoy-json.log-expected.json new file mode 100644 index 00000000000..13a1ca6e4b0 --- /dev/null +++ b/x-pack/filebeat/module/envoyproxy/log/test/envoy-json.log-expected.json @@ -0,0 +1,85 @@ +[ + { + "@timestamp": "2019-04-10T03:49:34.451Z", + "destination.address": "52.71.234.219", + "destination.geo.city_name": "Ashburn", + "destination.geo.continent_name": "North America", + "destination.geo.country_iso_code": "US", + "destination.geo.location.lat": 39.0481, + "destination.geo.location.lon": -77.4728, + "destination.geo.region_iso_code": "US-VA", + "destination.geo.region_name": "Virginia", + "destination.ip": "52.71.234.219", + "destination.port": 80, + "ecs.version": "1.0.0", + "envoyproxy.authority": "httpbin.org", + "envoyproxy.log_type": "ACCESS", + "envoyproxy.proxy_type": "http", + "envoyproxy.request_id": "413bf460-bd56-4515-ada4-2a69c5e78e54", + "envoyproxy.upstream_service_time": 179000000, + "event.dataset": "envoyproxy.log", + "event.duration": 180000000, + "event.module": "envoyproxy", + "fileset.name": "log", + "http.request.body.bytes": 0, + "http.request.method": "GET", + "http.response.body.bytes": 0, + "http.response.status_code": 501, + "http.version": "1.1", + "input.type": "log", + "kubernetes.container.name": "ambassador", + "kubernetes.labels.service": "ambassador", + "kubernetes.namespace": "default", + "kubernetes.node.name": "minikube", + "kubernetes.pod.name": "ambassador-76c58d9df4-jwhsg", + "kubernetes.pod.uid": "e57d545e-2a9d-11e9-995f-08002730e0dc", + "log.offset": 0, + "message": "ACCESS [2019-04-10T03:49:34.451Z] \"GET /httpbin/status/501 HTTP/1.1\" 501 - 0 0 180 179 \"172.17.0.3\" \"curl/7.59.0\" \"413bf460-bd56-4515-ada4-2a69c5e78e54\" \"httpbin.org\" \"52.71.234.219:80\"", + "service.type": "envoyproxy", + "source.address": "172.17.0.3", + "source.ip": "172.17.0.3", + "tags": [ + "envoyproxy" + ], + "url.domain": "httpbin.org", + "url.path": "/httpbin/status/501", + "user_agent.device.name": "Other", + "user_agent.name": "curl", + "user_agent.original": "curl/7.59.0", + "user_agent.version": "7.59.0" + }, + { + "@timestamp": "2019-04-06T06:20:05.972Z", + "destination.address": "127.0.0.1", + "destination.bytes": 0, + "destination.ip": "127.0.0.1", + "destination.port": 9200, + "ecs.version": "1.0.0", + "envoyproxy.authority": "-", + "envoyproxy.log_type": "ACCESS", + "envoyproxy.proxy_type": "tcp", + "envoyproxy.request_id": "-", + "envoyproxy.response_flags": [ + "UF", + "URX" + ], + "event.dataset": "envoyproxy.log", + "event.duration": 0, + "event.module": "envoyproxy", + "fileset.name": "log", + "input.type": "log", + "kubernetes.container.name": "ambassador", + "kubernetes.labels.service": "ambassador", + "kubernetes.namespace": "default", + "kubernetes.node.name": "minikube", + "kubernetes.pod.name": "ambassador-76c58d9df4-jwhsg", + "kubernetes.pod.uid": "e57d545e-2a9d-11e9-995f-08002730e0dc", + "log.offset": 518, + "message": "ACCESS [2019-04-06T06:20:05.972Z] \"- - -\" 0 UF,URX 0 0 0 - \"-\" \"-\" \"-\" \"-\" \"127.0.0.1:9200\"", + "service.type": "envoyproxy", + "source.bytes": 0, + "tags": [ + "envoyproxy" + ] + } +] \ No newline at end of file diff --git a/x-pack/filebeat/module/envoyproxy/log/test/envoy.log b/x-pack/filebeat/module/envoyproxy/log/test/envoy.log new file mode 100644 index 00000000000..cd8cf20b66e --- /dev/null +++ b/x-pack/filebeat/module/envoyproxy/log/test/envoy.log @@ -0,0 +1,4 @@ +[2019-04-08T16:50:12.533Z] "- - -" 0 - 87 254 4 - "-" "-" "-" "-" "172.27.0.2:80" +[2019-04-08T16:16:55.931Z] "GET /service/1 HTTP/1.1" 200 - 0 89 5 4 "-" "curl/7.54.0" "c219f6da-2b7f-483e-9ced-ec323d9330a9" "localhost:8000" "172.27.0.3:80" +[2019-04-11T00:31:55.439Z] "GET /elastic HTTP/1.1" 404 NR 0 0 0 - "172.17.0.3" "curl/7.59.0" "58436667-bf70-4fd0-9fe9-cdadadecfd55" "192.168.99.107:30901" "-" +[2019-04-11T00:51:07.980Z] "GET /elastic/ HTTP/1.1" 301 - 0 0 41 39 "172.17.0.3" "curl/7.59.0" "078d1daa-b786-4d6d-85a5-7e4366adaa19" "www.elastic.co" "151.101.66.217:80" diff --git a/x-pack/filebeat/module/envoyproxy/log/test/envoy.log-expected.json b/x-pack/filebeat/module/envoyproxy/log/test/envoy.log-expected.json new file mode 100644 index 00000000000..8de6e767252 --- /dev/null +++ b/x-pack/filebeat/module/envoyproxy/log/test/envoy.log-expected.json @@ -0,0 +1,135 @@ +[ + { + "@timestamp": "2019-04-08T16:50:12.533Z", + "destination.address": "172.27.0.2", + "destination.bytes": 87, + "destination.ip": "172.27.0.2", + "destination.port": 80, + "ecs.version": "1.0.0", + "envoyproxy.authority": "-", + "envoyproxy.log_type": "ACCESS", + "envoyproxy.proxy_type": "tcp", + "envoyproxy.request_id": "-", + "event.dataset": "envoyproxy.log", + "event.duration": 4000000, + "event.module": "envoyproxy", + "fileset.name": "log", + "input.type": "log", + "log.offset": 0, + "message": "[2019-04-08T16:50:12.533Z] \"- - -\" 0 - 87 254 4 - \"-\" \"-\" \"-\" \"-\" \"172.27.0.2:80\"", + "service.type": "envoyproxy", + "source.bytes": 254, + "tags": [ + "envoyproxy" + ] + }, + { + "@timestamp": "2019-04-08T16:16:55.931Z", + "destination.address": "172.27.0.3", + "destination.ip": "172.27.0.3", + "destination.port": 80, + "ecs.version": "1.0.0", + "envoyproxy.authority": "localhost:8000", + "envoyproxy.log_type": "ACCESS", + "envoyproxy.proxy_type": "http", + "envoyproxy.request_id": "c219f6da-2b7f-483e-9ced-ec323d9330a9", + "envoyproxy.upstream_service_time": 4000000, + "event.dataset": "envoyproxy.log", + "event.duration": 5000000, + "event.module": "envoyproxy", + "fileset.name": "log", + "http.request.body.bytes": 89, + "http.request.method": "GET", + "http.response.body.bytes": 0, + "http.response.status_code": 200, + "http.version": "1.1", + "input.type": "log", + "log.offset": 82, + "message": "[2019-04-08T16:16:55.931Z] \"GET /service/1 HTTP/1.1\" 200 - 0 89 5 4 \"-\" \"curl/7.54.0\" \"c219f6da-2b7f-483e-9ced-ec323d9330a9\" \"localhost:8000\" \"172.27.0.3:80\"", + "service.type": "envoyproxy", + "tags": [ + "envoyproxy" + ], + "url.domain": "localhost:8000", + "url.path": "/service/1", + "user_agent.device.name": "Other", + "user_agent.name": "curl", + "user_agent.original": "curl/7.54.0", + "user_agent.version": "7.54.0" + }, + { + "@timestamp": "2019-04-11T00:31:55.439Z", + "ecs.version": "1.0.0", + "envoyproxy.authority": "192.168.99.107:30901", + "envoyproxy.log_type": "ACCESS", + "envoyproxy.proxy_type": "http", + "envoyproxy.request_id": "58436667-bf70-4fd0-9fe9-cdadadecfd55", + "envoyproxy.response_flags": [ + "NR" + ], + "event.dataset": "envoyproxy.log", + "event.duration": 0, + "event.module": "envoyproxy", + "fileset.name": "log", + "http.request.body.bytes": 0, + "http.request.method": "GET", + "http.response.body.bytes": 0, + "http.response.status_code": 404, + "http.version": "1.1", + "input.type": "log", + "log.offset": 240, + "message": "[2019-04-11T00:31:55.439Z] \"GET /elastic HTTP/1.1\" 404 NR 0 0 0 - \"172.17.0.3\" \"curl/7.59.0\" \"58436667-bf70-4fd0-9fe9-cdadadecfd55\" \"192.168.99.107:30901\" \"-\"", + "service.type": "envoyproxy", + "source.address": "172.17.0.3", + "source.ip": "172.17.0.3", + "tags": [ + "envoyproxy" + ], + "url.domain": "192.168.99.107:30901", + "url.path": "/elastic", + "user_agent.device.name": "Other", + "user_agent.name": "curl", + "user_agent.original": "curl/7.59.0", + "user_agent.version": "7.59.0" + }, + { + "@timestamp": "2019-04-11T00:51:07.980Z", + "destination.address": "151.101.66.217", + "destination.geo.continent_name": "North America", + "destination.geo.country_iso_code": "US", + "destination.geo.location.lat": 37.751, + "destination.geo.location.lon": -97.822, + "destination.ip": "151.101.66.217", + "destination.port": 80, + "ecs.version": "1.0.0", + "envoyproxy.authority": "www.elastic.co", + "envoyproxy.log_type": "ACCESS", + "envoyproxy.proxy_type": "http", + "envoyproxy.request_id": "078d1daa-b786-4d6d-85a5-7e4366adaa19", + "envoyproxy.upstream_service_time": 39000000, + "event.dataset": "envoyproxy.log", + "event.duration": 41000000, + "event.module": "envoyproxy", + "fileset.name": "log", + "http.request.body.bytes": 0, + "http.request.method": "GET", + "http.response.body.bytes": 0, + "http.response.status_code": 301, + "http.version": "1.1", + "input.type": "log", + "log.offset": 399, + "message": "[2019-04-11T00:51:07.980Z] \"GET /elastic/ HTTP/1.1\" 301 - 0 0 41 39 \"172.17.0.3\" \"curl/7.59.0\" \"078d1daa-b786-4d6d-85a5-7e4366adaa19\" \"www.elastic.co\" \"151.101.66.217:80\"", + "service.type": "envoyproxy", + "source.address": "172.17.0.3", + "source.ip": "172.17.0.3", + "tags": [ + "envoyproxy" + ], + "url.domain": "www.elastic.co", + "url.path": "/elastic/", + "user_agent.device.name": "Other", + "user_agent.name": "curl", + "user_agent.original": "curl/7.59.0", + "user_agent.version": "7.59.0" + } +] \ No newline at end of file diff --git a/x-pack/filebeat/module/envoyproxy/module.yml b/x-pack/filebeat/module/envoyproxy/module.yml new file mode 100644 index 00000000000..ec3662ad001 --- /dev/null +++ b/x-pack/filebeat/module/envoyproxy/module.yml @@ -0,0 +1,3 @@ +dashboards: +- id: 0c610510-5cbd-11e9-8477-077ec9664dbd + file: Filebeat-Envoyproxy-Overview.json \ No newline at end of file diff --git a/x-pack/filebeat/modules.d/envoyproxy.yml.disabled b/x-pack/filebeat/modules.d/envoyproxy.yml.disabled new file mode 100644 index 00000000000..a46cf279282 --- /dev/null +++ b/x-pack/filebeat/modules.d/envoyproxy.yml.disabled @@ -0,0 +1,11 @@ +# Module: envoyproxy +# Docs: https://www.elastic.co/guide/en/beats/filebeat/master/filebeat-module-envoyproxy.html + +- module: envoyproxy + # Fileset for native deployment + log: + enabled: true + + # Set custom paths for the log files. If left empty, + # Filebeat will choose the paths depending on your OS. + #var.paths: