From aed6cc680cde6f9079bca4d5c65d9e15eb2da423 Mon Sep 17 00:00:00 2001 From: Pier-Hugues Pellerin Date: Mon, 19 Nov 2018 14:35:21 -0500 Subject: [PATCH 1/4] Never default to a qualifier when none of them are set. (#9148) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove default version qualifier and rename the environment variable to set it from `BEAT_VERSION_QUALIFIER` to `VERSION_QUALIFIER` this will align with other parts of the stack. **Tested with filebeat.** ``` ❯ ./filebeat version [08:39:01] filebeat version 7.0.0 (amd64), libbeat 7.0.0 [0a0c267dd8c33dfce4b009a367b84297fc1a9b92 built 2018-11-19 13:38:15 +0000 UTC] ``` **Without the patch** ``` ❯ ./filebeat version [08:40:07] filebeat version 7.0.0-alpha1 (amd64), libbeat 7.0.0-alpha1 [b007837fde985832c257234b5a71bc863cbe2128 built 2018-11-19 13:39:59 +0000 UTC] ``` Fixes: #8384 --- .editorconfig | 5 +- .go-version | 2 +- .travis.yml | 10 +- CHANGELOG-developer.asciidoc | 3 + CHANGELOG.asciidoc | 17 +- Makefile | 16 +- NOTICE.txt | 8 +- README.md | 4 + auditbeat/Dockerfile | 2 +- auditbeat/docs/getting-started.asciidoc | 23 +- auditbeat/docs/index.asciidoc | 1 + auditbeat/magefile.go | 11 +- .../module/file_integrity/metricset_test.go | 4 +- .../file_integrity/monitor/monitor_test.go | 2 +- dev-tools/cmd/dashboards/export_dashboards.go | 171 +- dev-tools/ecs-migration.yml | 239 +++ dev-tools/jenkins_release.sh | 1 - dev-tools/mage/check.go | 180 ++ dev-tools/mage/common.go | 23 + dev-tools/mage/crossbuild.go | 2 +- dev-tools/mage/dashboard.go | 53 + dev-tools/mage/dockerbuilder.go | 42 +- dev-tools/mage/fmt.go | 130 ++ dev-tools/mage/gotest.go | 2 +- dev-tools/mage/integtest.go | 56 +- dev-tools/mage/pytest.go | 47 +- dev-tools/mage/settings.go | 4 +- dev-tools/make/mage.mk | 11 + dev-tools/make/xpack.mk | 43 + dev-tools/packaging/package_test.go | 13 +- docs/devguide/newdashboards.asciidoc | 46 +- filebeat/Dockerfile | 2 +- filebeat/_meta/fields.common.yml | 14 + filebeat/autodiscover/builder/hints/logs.go | 2 +- filebeat/docs/fields.asciidoc | 736 +++---- filebeat/docs/getting-started.asciidoc | 23 +- .../include/enable-modules-command.asciidoc | 7 + .../include/list-modules-command.asciidoc | 7 + filebeat/docs/include/run-command.asciidoc | 7 + filebeat/docs/include/set-paths.asciidoc | 7 + filebeat/docs/include/setup-command.asciidoc | 7 + filebeat/docs/index.asciidoc | 1 + filebeat/include/fields.go | 2 +- filebeat/input/log/harvester.go | 8 +- filebeat/input/log/input.go | 2 +- filebeat/magefile.go | 11 +- filebeat/module/haproxy/_meta/fields.yml | 126 +- .../module/haproxy/log/ingest/pipeline.json | 24 +- .../log/test/default.log-expected.json | 21 +- .../log/test/haproxy.log-expected.json | 17 +- .../haproxy/log/test/tcplog.log-expected.json | 9 +- filebeat/module/iis/_meta/fields.yml | 1 + filebeat/module/iis/access/_meta/fields.yml | 173 +- .../module/iis/access/ingest/default.json | 49 +- filebeat/module/iis/access/test/test.log | 2 +- .../iis/access/test/test.log-expected.json | 163 +- filebeat/module/nginx/access/_meta/fields.yml | 152 +- .../module/nginx/access/ingest/default.json | 54 +- filebeat/module/nginx/access/test/test.log | 2 +- .../nginx/access/test/test.log-expected.json | 281 +-- filebeat/module/system/_meta/fields.yml | 9 + .../module/system/syslog/_meta/fields.yml | 21 +- .../module/system/syslog/ingest/pipeline.json | 88 +- .../darwin-syslog-sample.log-expected.json | 21 +- filebeat/scripts/generator/fileset/main.go | 2 +- filebeat/scripts/generator/generator.go | 2 +- filebeat/scripts/generator/module/main.go | 11 +- filebeat/tests/system/test_crawler.py | 2 + filebeat/tests/system/test_harvester.py | 38 + filebeat/tests/system/test_registrar.py | 4 + generator/beat/{beat}/.travis.yml | 2 +- heartbeat/Dockerfile | 2 +- heartbeat/docs/getting-started.asciidoc | 21 +- heartbeat/docs/index.asciidoc | 1 + heartbeat/magefile.go | 2 +- .../6/dashboard/Heartbeat-http-monitor.json | 1405 +++++++------ journalbeat/Dockerfile | 2 +- journalbeat/_meta/beat.docker.yml | 2 +- journalbeat/checkpoint/file_windows.go | 4 +- journalbeat/docs/index.asciidoc | 1 + journalbeat/journalbeat.docker.yml | 2 +- libbeat/Dockerfile | 2 +- .../autodiscover/appenders/config/config.go | 6 +- libbeat/autodiscover/autodiscover.go | 2 +- .../providers/jolokia/discovery.go | 2 +- .../providers/kubernetes/kubernetes.go | 2 +- libbeat/cmd/export/dashboard.go | 44 +- libbeat/common/kubernetes/util.go | 8 +- libbeat/common/transport/tlscommon/tls.go | 6 +- libbeat/dashboards/decode.go | 70 + libbeat/dashboards/export.go | 106 + libbeat/docs/command-reference.asciidoc | 226 +- libbeat/docs/dashboards.asciidoc | 22 + libbeat/docs/reference-yml.asciidoc | 5 +- libbeat/docs/shared-template-load.asciidoc | 45 +- libbeat/docs/version.asciidoc | 2 +- libbeat/generator/fields/validate/mapping.go | 4 +- libbeat/kibana/client.go | 1 + libbeat/kibana/fields_transformer.go | 2 +- libbeat/outputs/elasticsearch/client.go | 2 +- .../outputs/transport/transptest/testing.go | 2 +- libbeat/plugin/cli.go | 2 +- .../add_kubernetes_metadata/kubernetes.go | 4 +- libbeat/reader/debug/debug.go | 178 ++ libbeat/reader/debug/debug_test.go | 146 ++ libbeat/reader/readfile/line.go | 2 +- libbeat/scripts/Makefile | 8 - libbeat/tests/files/dashboards.yml | 3 + libbeat/tests/system/test_base.py | 3 + libbeat/tests/system/test_dashboard.py | 176 +- libbeat/version/helper.go | 2 +- metricbeat/Dockerfile | 21 +- metricbeat/docs/fields.asciidoc | 34 +- metricbeat/docs/gettingstarted.asciidoc | 25 +- metricbeat/docs/index.asciidoc | 1 + metricbeat/magefile.go | 11 +- metricbeat/metricbeat.reference.yml | 3 + metricbeat/module/ceph/cluster_disk/data.go | 2 +- metricbeat/module/ceph/cluster_health/data.go | 2 +- metricbeat/module/ceph/cluster_status/data.go | 2 +- metricbeat/module/ceph/monitor_health/data.go | 2 +- metricbeat/module/ceph/osd_df/data.go | 2 +- metricbeat/module/ceph/osd_tree/data.go | 2 +- metricbeat/module/ceph/pool_disk/data.go | 2 +- metricbeat/module/couchbase/bucket/data.go | 2 +- metricbeat/module/couchbase/cluster/data.go | 2 +- metricbeat/module/couchbase/node/data.go | 2 +- .../module/docker/_meta/config.reference.yml | 3 + metricbeat/module/docker/cpu/cpu.go | 11 +- metricbeat/module/docker/cpu/helper.go | 14 +- metricbeat/module/docker/healthcheck/data.go | 6 + .../module/elasticsearch/_meta/fields.yml | 10 + .../module/elasticsearch/ccr/_meta/data.json | 20 +- .../module/elasticsearch/ccr/_meta/fields.yml | 4 +- .../ccr/_meta/test/ccr_stats.700.json | 138 +- metricbeat/module/elasticsearch/ccr/data.go | 40 +- .../module/elasticsearch/ccr/data_xpack.go | 27 +- .../elasticsearch_integration_test.go | 6 +- metricbeat/module/elasticsearch/fields.go | 2 +- .../index_recovery/_meta/data.json | 14 +- .../elasticsearch/index_recovery/data.go | 4 +- .../elasticsearch/index_recovery/data_test.go | 2 +- .../index_recovery/data_xpack.go | 7 +- .../index_recovery/index_recovery.go | 17 +- .../elasticsearch/ml_job/_meta/data.json | 9 +- .../module/elasticsearch/ml_job/data.go | 6 +- .../module/elasticsearch/ml_job/data_test.go | 30 + .../module/elasticsearch/ml_job/data_xpack.go | 9 +- .../module/elasticsearch/ml_job/ml_job.go | 16 +- .../module/elasticsearch/node/_meta/data.json | 11 +- .../elasticsearch/node/_meta/fields.yml | 4 - metricbeat/module/elasticsearch/node/data.go | 3 +- .../module/elasticsearch/node/data_test.go | 9 +- metricbeat/module/elasticsearch/node/node.go | 11 +- .../module/elasticsearch/node/node_test.go | 20 +- .../elasticsearch/node_stats/_meta/data.json | 56 +- .../module/elasticsearch/node_stats/data.go | 35 +- .../elasticsearch/node_stats/data_test.go | 2 +- .../elasticsearch/node_stats/data_xpack.go | 10 +- .../elasticsearch/node_stats/node_stats.go | 14 +- .../pending_tasks/_meta/data.json | 46 +- .../elasticsearch/pending_tasks/data.go | 6 +- .../elasticsearch/pending_tasks/data_test.go | 44 +- .../pending_tasks/pending_tasks.go | 14 +- .../module/envoyproxy/_meta/docs.asciidoc | 2 - metricbeat/module/envoyproxy/server/data.go | 20 +- metricbeat/module/http/_meta/Dockerfile | 2 +- metricbeat/module/kafka/_meta/Dockerfile | 2 +- ...n_test.go => logstash_integration_test.go} | 39 +- .../module/logstash/node/_meta/data.json | 13 +- .../logstash/node_stats/_meta/data.json | 11 +- .../node_stats/node_stats_integration_test.go | 56 - metricbeat/module/rabbitmq/connection/data.go | 2 +- metricbeat/module/rabbitmq/exchange/data.go | 2 +- metricbeat/module/rabbitmq/queue/data.go | 2 +- metricbeat/module/uwsgi/status/data.go | 2 +- metricbeat/module/uwsgi/status/status.go | 6 +- packetbeat/docs/gettingstarted.asciidoc | 23 +- packetbeat/docs/index.asciidoc | 1 + packetbeat/magefile.go | 2 +- packetbeat/protos/http/http_test.go | 2 +- packetbeat/protos/icmp/icmp.go | 2 +- packetbeat/protos/icmp/message.go | 10 +- packetbeat/protos/mysql/mysql.go | 2 +- packetbeat/protos/thrift/thrift_idl.go | 2 +- testing/environments/latest.yml | 6 +- .../fsnotify/fsevents/go_1_10_after.go | 27 + .../fsnotify/fsevents/go_1_10_before.go | 25 + .../fsnotify/fsevents/go_1_9_2_after.go | 11 + .../fsnotify/fsevents/go_1_9_2_before.go | 13 + vendor/github.com/fsnotify/fsevents/wrap.go | 36 +- .../fsnotify/fsevents/wrap_deprecated.go | 276 --- vendor/github.com/magefile/mage/Gopkg.lock | 9 - vendor/github.com/magefile/mage/Gopkg.toml | 22 - vendor/github.com/magefile/mage/README.md | 5 +- vendor/github.com/magefile/mage/go.mod | 1 + vendor/github.com/magefile/mage/go.sum | 0 vendor/github.com/magefile/mage/magefile.go | 32 +- vendor/golang.org/x/crypto/blake2b/blake2b.go | 68 + .../x/crypto/blake2b/blake2bAVX2_amd64.go | 26 +- .../x/crypto/blake2b/blake2bAVX2_amd64.s | 12 - .../x/crypto/blake2b/blake2b_amd64.go | 7 +- .../x/crypto/blake2b/blake2b_amd64.s | 9 - vendor/golang.org/x/sys/cpu/cpu.go | 38 + vendor/golang.org/x/sys/cpu/cpu_arm.go | 7 + vendor/golang.org/x/sys/cpu/cpu_arm64.go | 7 + vendor/golang.org/x/sys/cpu/cpu_gc_x86.go | 16 + vendor/golang.org/x/sys/cpu/cpu_gccgo.c | 43 + vendor/golang.org/x/sys/cpu/cpu_gccgo.go | 26 + vendor/golang.org/x/sys/cpu/cpu_mips64x.go | 9 + vendor/golang.org/x/sys/cpu/cpu_mipsx.go | 9 + vendor/golang.org/x/sys/cpu/cpu_ppc64x.go | 9 + vendor/golang.org/x/sys/cpu/cpu_s390x.go | 7 + vendor/golang.org/x/sys/cpu/cpu_x86.go | 55 + vendor/golang.org/x/sys/cpu/cpu_x86.s | 27 + vendor/vendor.json | 28 +- winlogbeat/checkpoint/file_windows.go | 4 +- winlogbeat/magefile.go | 2 +- x-pack/filebeat/Makefile | 9 +- x-pack/filebeat/magefile.go | 10 + x-pack/functionbeat/Dockerfile | 2 +- .../functionbeat/provider/aws/cli_manager.go | 2 +- x-pack/metricbeat/Makefile | 3 + x-pack/metricbeat/docker-compose.yml | 11 + x-pack/metricbeat/include/list.go | 13 + x-pack/metricbeat/magefile.go | 270 +++ x-pack/metricbeat/metricbeat.reference.yml | 1828 +++++++++++++++++ x-pack/metricbeat/metricbeat.yml | 148 ++ x-pack/metricbeat/module/foo/_meta/config.yml | 6 + .../metricbeat/module/foo/_meta/docs.asciidoc | 2 + x-pack/metricbeat/module/foo/_meta/fields.yml | 11 + .../metricbeat/module/foo/bar/_meta/data.json | 19 + .../module/foo/bar/_meta/docs.asciidoc | 1 + .../module/foo/bar/_meta/fields.yml | 9 + x-pack/metricbeat/module/foo/bar/bar.go | 56 + x-pack/metricbeat/module/foo/doc.go | 6 + x-pack/metricbeat/module/foo/fields.go | 22 + x-pack/metricbeat/packages.yml | 90 + 238 files changed, 7263 insertions(+), 2931 deletions(-) create mode 100644 dev-tools/mage/check.go create mode 100644 dev-tools/mage/dashboard.go create mode 100644 dev-tools/mage/fmt.go create mode 100644 dev-tools/make/mage.mk create mode 100644 dev-tools/make/xpack.mk create mode 100644 libbeat/dashboards/decode.go create mode 100644 libbeat/dashboards/export.go create mode 100644 libbeat/reader/debug/debug.go create mode 100644 libbeat/reader/debug/debug_test.go create mode 100644 libbeat/tests/files/dashboards.yml create mode 100644 metricbeat/module/elasticsearch/ml_job/data_test.go rename metricbeat/module/logstash/{node/node_integration_test.go => logstash_integration_test.go} (59%) delete mode 100644 metricbeat/module/logstash/node_stats/node_stats_integration_test.go create mode 100644 vendor/github.com/fsnotify/fsevents/go_1_10_after.go create mode 100644 vendor/github.com/fsnotify/fsevents/go_1_10_before.go create mode 100644 vendor/github.com/fsnotify/fsevents/go_1_9_2_after.go create mode 100644 vendor/github.com/fsnotify/fsevents/go_1_9_2_before.go delete mode 100644 vendor/github.com/fsnotify/fsevents/wrap_deprecated.go delete mode 100644 vendor/github.com/magefile/mage/Gopkg.lock delete mode 100644 vendor/github.com/magefile/mage/Gopkg.toml create mode 100644 vendor/github.com/magefile/mage/go.mod create mode 100644 vendor/github.com/magefile/mage/go.sum create mode 100644 vendor/golang.org/x/sys/cpu/cpu.go create mode 100644 vendor/golang.org/x/sys/cpu/cpu_arm.go create mode 100644 vendor/golang.org/x/sys/cpu/cpu_arm64.go create mode 100644 vendor/golang.org/x/sys/cpu/cpu_gc_x86.go create mode 100644 vendor/golang.org/x/sys/cpu/cpu_gccgo.c create mode 100644 vendor/golang.org/x/sys/cpu/cpu_gccgo.go create mode 100644 vendor/golang.org/x/sys/cpu/cpu_mips64x.go create mode 100644 vendor/golang.org/x/sys/cpu/cpu_mipsx.go create mode 100644 vendor/golang.org/x/sys/cpu/cpu_ppc64x.go create mode 100644 vendor/golang.org/x/sys/cpu/cpu_s390x.go create mode 100644 vendor/golang.org/x/sys/cpu/cpu_x86.go create mode 100644 vendor/golang.org/x/sys/cpu/cpu_x86.s create mode 100644 x-pack/metricbeat/Makefile create mode 100644 x-pack/metricbeat/docker-compose.yml create mode 100644 x-pack/metricbeat/include/list.go create mode 100644 x-pack/metricbeat/magefile.go create mode 100644 x-pack/metricbeat/metricbeat.reference.yml create mode 100644 x-pack/metricbeat/metricbeat.yml create mode 100644 x-pack/metricbeat/module/foo/_meta/config.yml create mode 100644 x-pack/metricbeat/module/foo/_meta/docs.asciidoc create mode 100644 x-pack/metricbeat/module/foo/_meta/fields.yml create mode 100644 x-pack/metricbeat/module/foo/bar/_meta/data.json create mode 100644 x-pack/metricbeat/module/foo/bar/_meta/docs.asciidoc create mode 100644 x-pack/metricbeat/module/foo/bar/_meta/fields.yml create mode 100644 x-pack/metricbeat/module/foo/bar/bar.go create mode 100644 x-pack/metricbeat/module/foo/doc.go create mode 100644 x-pack/metricbeat/module/foo/fields.go create mode 100644 x-pack/metricbeat/packages.yml diff --git a/.editorconfig b/.editorconfig index bd0cc74b875..7a1535ad1b4 100644 --- a/.editorconfig +++ b/.editorconfig @@ -23,7 +23,10 @@ indent_size = 4 indent_style = space indent_size = 2 -[Makefile] +[Makefile*] +indent_style = tab + +[*.mk] indent_style = tab [Vagrantfile] diff --git a/.go-version b/.go-version index 587c5f0c730..ca7176690dd 100644 --- a/.go-version +++ b/.go-version @@ -1 +1 @@ -1.10.3 +1.11.2 diff --git a/.travis.yml b/.travis.yml index 1faab6a1821..60282e2c955 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,7 +12,7 @@ env: global: # Cross-compile for amd64 only to speed up testing. - GOX_FLAGS="-arch amd64" - - DOCKER_COMPOSE_VERSION=1.11.1 + - DOCKER_COMPOSE_VERSION=1.21.0 - GO_VERSION="$(cat .go-version)" # Newer versions of minikube fail on travis, see: https://github.com/kubernetes/minikube/issues/2704 - TRAVIS_MINIKUBE_VERSION=v0.25.2 @@ -34,6 +34,10 @@ jobs: env: TARGETS="TEST_ENVIRONMENT=0 -C filebeat testsuite" go: $GO_VERSION stage: test + - os: linux + env: TARGETS="-C x-pack/filebeat testsuite" + go: $GO_VERSION + stage: test # Heartbeat - os: linux @@ -90,6 +94,10 @@ jobs: env: TARGETS="-C metricbeat crosscompile" go: $GO_VERSION stage: test + - os: linux + env: TARGETS="-C x-pack/metricbeat testsuite" + go: $GO_VERSION + stage: test # Packetbeat - os: linux diff --git a/CHANGELOG-developer.asciidoc b/CHANGELOG-developer.asciidoc index 6936595f881..0d9333b69c7 100644 --- a/CHANGELOG-developer.asciidoc +++ b/CHANGELOG-developer.asciidoc @@ -38,6 +38,7 @@ The list below covers the major changes between 6.3.0 and master only. - Fix permissions of generated Filebeat filesets. {pull}7140[7140] - Collect fields from _meta/fields.yml too. {pull}8397[8397] - Fix issue on asset generation that could lead to different results in Windows. {pull}8464[8464] +- Remove default version qualifier, you can use `VERSION_QUALIFIER` environment variable to set it. {pull}9148[9148] ==== Added @@ -66,3 +67,5 @@ The list below covers the major changes between 6.3.0 and master only. - Add `mage.AddPlatforms` to allow to specify dependent platforms when building a beat. {pull}8889[8889] - Add `cfgwarn.CheckRemoved6xSetting(s)` to display a warning for options removed in 7.0. {pull}8909[8909] - Add docker image building to `mage.Package`. {pull}8898[8898] +- Simplified exporting of dashboards. {pull}7730[7730] +- Update Beats to use go 1.11.2 {pull}8746[8746] diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 6b1abe53ca1..2d9e4d0fa19 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -23,8 +23,6 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha1...master[Check the HEAD d *Metricbeat* -- Fix issue preventing diskio metrics collection for idle disks. {issue}9124[9124] {pull}9125[9125] - *Packetbeat* *Winlogbeat* @@ -45,6 +43,10 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha1...master[Check the HEAD d *Metricbeat* +- Fix issue preventing diskio metrics collection for idle disks. {issue}9124[9124] {pull}9125[9125] +- Fix panic on docker healthcheck collection on dockers without healthchecks. {pull}9171[9171] +- Fix issue with not collecting Elasticsearch cross-cluster replication stats correctly. {pull}9179[9179] + *Packetbeat* *Winlogbeat* @@ -54,10 +56,12 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha1...master[Check the HEAD d ==== Added *Affecting all Beats* +- Unify dashboard exporter tools. {pull}9097[9097] *Auditbeat* *Filebeat* +- Added `detect_null_bytes` selector to detect null bytes from a io.reader. {pull}9210[9210] *Heartbeat* @@ -65,6 +69,9 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha1...master[Check the HEAD d *Metricbeat* +- Add setting to disable docker cpu metrics per core. {pull}9194[9194] +- The `elasticsearch/node` metricset now reports the Elasticsearch cluster UUID. {pull}8771[8771] + *Packetbeat* *Functionbeat* @@ -119,6 +126,11 @@ https://github.com/elastic/beats/compare/v6.5.0...v7.0.0-alpha1[View commits] - Rename `offset` to `log.offset`. {pull}8923[8923] - Rename `source_ecs` to `source` in the Filebeat Suricata module. {pull}8983[8983] - Remove warnings for deprecated options: "spool_size", "publish_async", "idle_timeout". {pull}9002[9002] +- Rename many `system.syslog.*` fields to map to ECS. {pull}9135[9135] +- Rename many `iis.access.*` fields to map to ECS. {pull}9084[9084] +- IIS module's user agent string is no longer encoded (`+` replaced with spaces). {pull}9084[9084] +- Rename many `haproxy.*` fields to map to ECS. {pull}9117[9117] +- Rename many `nginx.access.*` fields to map to ECS. {pull}9081[9081] *Metricbeat* @@ -314,6 +326,7 @@ https://github.com/elastic/beats/compare/v6.4.0...v6.5.0[View commits] - Added autodiscovery support {pull}8415[8415] - Added support for extra TLS/x509 metadata. {pull}7944[7944] - Added stats and state metrics for number of monitors and endpoints started. {pull}8621[8621] +- Add last monitor status to dashboard table. Further break out monitors in dashboard table by monitor.ip. {pull}9022[9022] *Journalbeat* diff --git a/Makefile b/Makefile index b9b5e274a26..20af5832d58 100644 --- a/Makefile +++ b/Makefile @@ -13,12 +13,18 @@ REVIEWDOG_OPTIONS?=-diff "git diff master" REVIEWDOG_REPO=github.com/haya14busa/reviewdog/cmd/reviewdog XPACK_SUFFIX=x-pack/ +# PROJECTS_XPACK_MAGE is a list of Beats whose primary build logic is based in +# Mage. For compatibility with CI testing these projects support a subset of the +# makefile targets. After all Beats converge to primarily using Mage we can +# remove this and treat all sub-projects the same. +PROJECTS_XPACK_MAGE=x-pack/filebeat x-pack/metricbeat + # Runs complete testsuites (unit, system, integration) for all beats with coverage and race detection. # Also it builds the docs and the generators .PHONY: testsuite testsuite: - @$(foreach var,$(PROJECTS),$(MAKE) -C $(var) testsuite || exit 1;) + @$(foreach var,$(PROJECTS) $(PROJECTS_XPACK_MAGE),$(MAKE) -C $(var) testsuite || exit 1;) .PHONY: setup-commit-hook setup-commit-hook: @@ -54,13 +60,13 @@ coverage-report: .PHONY: update update: notice - @$(foreach var,$(PROJECTS),$(MAKE) -C $(var) update || exit 1;) + @$(foreach var,$(PROJECTS) $(PROJECTS_XPACK_MAGE),$(MAKE) -C $(var) update || exit 1;) @$(MAKE) -C deploy/kubernetes all .PHONY: clean clean: @rm -rf build - @$(foreach var,$(PROJECTS),$(MAKE) -C $(var) clean || exit 1;) + @$(foreach var,$(PROJECTS) $(PROJECTS_XPACK_MAGE),$(MAKE) -C $(var) clean || exit 1;) @$(MAKE) -C generator clean @-mage -clean 2> /dev/null @@ -72,7 +78,7 @@ clean-vendor: .PHONY: check check: python-env - @$(foreach var,$(PROJECTS) dev-tools,$(MAKE) -C $(var) check || exit 1;) + @$(foreach var,$(PROJECTS) dev-tools $(PROJECTS_XPACK_MAGE),$(MAKE) -C $(var) check || exit 1;) @# Checks also python files which are not part of the beats @$(FIND) -name *.py -exec $(PYTHON_ENV)/bin/autopep8 -d --max-line-length 120 {} \; | (! grep . -q) || (echo "Code differs from autopep8's style" && false) @# Validate that all updates were committed @@ -107,7 +113,7 @@ misspell: .PHONY: fmt fmt: add-headers python-env - @$(foreach var,$(PROJECTS) dev-tools,$(MAKE) -C $(var) fmt || exit 1;) + @$(foreach var,$(PROJECTS) dev-tools $(PROJECTS_XPACK_MAGE),$(MAKE) -C $(var) fmt || exit 1;) @# Cleans also python files which are not part of the beats @$(FIND) -name "*.py" -exec $(PYTHON_ENV)/bin/autopep8 --in-place --max-line-length 120 {} \; diff --git a/NOTICE.txt b/NOTICE.txt index dcc3af7627c..b8629e99a51 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -648,7 +648,7 @@ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------- Dependency: github.com/fsnotify/fsevents -Revision: 4d5197691b054c220286f7c6ea167de9753d902d +Revision: e1d381a4d27063baac2e9d3c5887ceb4ab059287 License type (autodetected): BSD-3-Clause ./vendor/github.com/fsnotify/fsevents/LICENSE: -------------------------------------------------------------------- @@ -1713,7 +1713,7 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI -------------------------------------------------------------------- Dependency: github.com/magefile/mage -Revision: 5e51f9ad1ed0886c5d06a8c46a09703cfc4d9034 +Revision: 11d15918ff3aa6f75cd6d3340a9507bb5309c679 License type (autodetected): Apache-2.0 ./vendor/github.com/magefile/mage/LICENSE: -------------------------------------------------------------------- @@ -2849,7 +2849,7 @@ THE SOFTWARE. -------------------------------------------------------------------- Dependency: golang.org/x/crypto -Revision: 5119cf507ed5294cc409c092980c7497ee5d6fd2 +Revision: 85e1b3f9139abd58575d728a509643924e3b2ebf License type (autodetected): BSD-3-Clause ./vendor/golang.org/x/crypto/LICENSE: -------------------------------------------------------------------- @@ -2918,7 +2918,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------- Dependency: golang.org/x/sys -Revision: 0ffbfd41fbef8ffcf9b62b0b0aa3a5873ed7a4fe +Revision: d989b31c87461dc8ab2f1cac6792814e27fadea9 License type (autodetected): BSD-3-Clause ./vendor/golang.org/x/sys/LICENSE: -------------------------------------------------------------------- diff --git a/README.md b/README.md index aa735422e02..7bb72681625 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,9 @@ Beat | Description --- | --- [Auditbeat](https://github.com/elastic/beats/tree/master/auditbeat) | Collect your Linux audit framework data and monitor the integrity of your files. [Filebeat](https://github.com/elastic/beats/tree/master/filebeat) | Tails and ships log files +[Functionbeat](https://github.com/elastic/beats/tree/master/x-pack/functionbeat) | Read and ships events from serverless infrastructure. [Heartbeat](https://github.com/elastic/beats/tree/master/heartbeat) | Ping remote services for availability +[Journalbeat](https://github.com/elastic/beats/tree/master/journalbeat) | Read and ships event from Journald. [Metricbeat](https://github.com/elastic/beats/tree/master/metricbeat) | Fetches sets of metrics from the operating system and services [Packetbeat](https://github.com/elastic/beats/tree/master/packetbeat) | Monitors the network and applications by sniffing packets [Winlogbeat](https://github.com/elastic/beats/tree/master/winlogbeat) | Fetches and ships Windows Event logs @@ -40,7 +42,9 @@ on the [elastic.co site](https://www.elastic.co/guide/): * [Beats platform](https://www.elastic.co/guide/en/beats/libbeat/current/index.html) * [Auditbeat](https://www.elastic.co/guide/en/beats/auditbeat/current/index.html) * [Filebeat](https://www.elastic.co/guide/en/beats/filebeat/current/index.html) +* [Functionbeat](https://www.elastic.co/guide/en/beats/functionbeat/current/index.html) * [Heartbeat](https://www.elastic.co/guide/en/beats/heartbeat/current/index.html) +* [Journalbeat](https://www.elastic.co/guide/en/beats/journalbeat/current/index.html) * [Metricbeat](https://www.elastic.co/guide/en/beats/metricbeat/current/index.html) * [Packetbeat](https://www.elastic.co/guide/en/beats/packetbeat/current/index.html) * [Winlogbeat](https://www.elastic.co/guide/en/beats/winlogbeat/current/index.html) diff --git a/auditbeat/Dockerfile b/auditbeat/Dockerfile index 5cfb18d46a5..77eabf4b136 100644 --- a/auditbeat/Dockerfile +++ b/auditbeat/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.10.3 +FROM golang:1.11.2 MAINTAINER Nicolas Ruflin RUN set -x && \ diff --git a/auditbeat/docs/getting-started.asciidoc b/auditbeat/docs/getting-started.asciidoc index 1cedeaf864c..bd10a539d33 100644 --- a/auditbeat/docs/getting-started.asciidoc +++ b/auditbeat/docs/getting-started.asciidoc @@ -75,6 +75,25 @@ tar xzvf {beatname_lc}-{version}-darwin-x86_64.tar.gz endif::[] +[[linux]] +*linux:* + +ifeval::["{release-state}"=="unreleased"] + +Version {stack-version} of {beatname_uc} has not yet been released. + +endif::[] + +ifeval::["{release-state}"!="unreleased"] + +["source","sh",subs="attributes"] +------------------------------------------------ +curl -L -O https://artifacts.elastic.co/downloads/beats/{beatname_lc}/{beatname_lc}-{version}-linux-x86_64.tar.gz +tar xzvf {beatname_lc}-{version}-linux-x86_64.tar.gz +------------------------------------------------ + +endif::[] + [[docker]] *docker:* @@ -206,12 +225,12 @@ start {beatname_uc} in the foreground. sudo service {beatname_lc} start ---------------------------------------------------------------------- -*mac:* +*mac and linux:* ["source","sh",subs="attributes"] ---------------------------------------------------------------------- sudo chown root {beatname_lc}.yml <1> -sudo ./{beatname_lc} -e -c {beatname_lc}.yml +sudo ./{beatname_lc} -e ---------------------------------------------------------------------- <1> To monitor system files, you'll be running {beatname_uc} as root, so you need to change ownership of the configuration file, or run {beatname_uc} with diff --git a/auditbeat/docs/index.asciidoc b/auditbeat/docs/index.asciidoc index 28e01662b53..2bf016533ab 100644 --- a/auditbeat/docs/index.asciidoc +++ b/auditbeat/docs/index.asciidoc @@ -17,6 +17,7 @@ include::{asciidoc-dir}/../../shared/attributes.asciidoc[] :mac_os: :docker_platform: :win_os: +:linux_os: include::../../libbeat/docs/shared-beats-attributes.asciidoc[] diff --git a/auditbeat/magefile.go b/auditbeat/magefile.go index 5100db03533..8b6a0eedea1 100644 --- a/auditbeat/magefile.go +++ b/auditbeat/magefile.go @@ -75,7 +75,7 @@ func Clean() error { // Package packages the Beat for distribution. // Use SNAPSHOT=true to build snapshots. // Use PLATFORMS to control the target platforms. -// Use BEAT_VERSION_QUALIFIER to control the version qualifier. +// Use VERSION_QUALIFIER to control the version qualifier. func Package() { start := time.Now() defer func() { fmt.Println("package ran for", time.Since(start)) }() @@ -103,6 +103,15 @@ func Fields() error { return mage.GenerateFieldsYAML("module") } +// ExportDashboard exports a dashboard and writes it into the correct directory +// +// Required ENV variables: +// * MODULE: Name of the module +// * ID: Dashboard id +func ExportDashboard() error { + return mage.ExportDashboard() +} + // GoTestUnit executes the Go unit tests. // Use TEST_COVERAGE=true to enable code coverage profiling. // Use RACE_DETECTOR=true to enable the race detector. diff --git a/auditbeat/module/file_integrity/metricset_test.go b/auditbeat/module/file_integrity/metricset_test.go index 9f88490449f..c171deeee66 100644 --- a/auditbeat/module/file_integrity/metricset_test.go +++ b/auditbeat/module/file_integrity/metricset_test.go @@ -206,7 +206,7 @@ func TestExcludedFiles(t *testing.T) { } wanted := map[string]bool{ - dir: true, + dir: true, filepath.Join(dir, "FILE.TXT"): true, filepath.Join(dir, ".gitignore"): true, } @@ -271,7 +271,7 @@ func TestIncludedExcludedFiles(t *testing.T) { } wanted := map[string]bool{ - dir: true, + dir: true, filepath.Join(dir, ".ssh"): true, filepath.Join(dir, ".ssh/known_hosts"): true, } diff --git a/auditbeat/module/file_integrity/monitor/monitor_test.go b/auditbeat/module/file_integrity/monitor/monitor_test.go index 729184c9f2f..0b496c8e604 100644 --- a/auditbeat/module/file_integrity/monitor/monitor_test.go +++ b/auditbeat/module/file_integrity/monitor/monitor_test.go @@ -248,7 +248,7 @@ func TestRecursiveSubdirPermissions(t *testing.T) { // File "b/b" is missing because a watch to b couldn't be installed expected := map[string]fsnotify.Op{ - dest: fsnotify.Create, + dest: fsnotify.Create, filepath.Join(dest, "a"): fsnotify.Create, filepath.Join(dest, "a/a"): fsnotify.Create, filepath.Join(dest, "b"): fsnotify.Create, diff --git a/dev-tools/cmd/dashboards/export_dashboards.go b/dev-tools/cmd/dashboards/export_dashboards.go index a5338099f5f..22749237918 100644 --- a/dev-tools/cmd/dashboards/export_dashboards.go +++ b/dev-tools/cmd/dashboards/export_dashboards.go @@ -18,127 +18,27 @@ package main import ( - "crypto/tls" "encoding/json" "flag" - "fmt" "io/ioutil" "log" - "net/http" "net/url" "os" - "path" "path/filepath" - "strings" + "time" - "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/dashboards" "github.com/elastic/beats/libbeat/kibana" ) -var exportAPI = "/api/kibana/dashboards/export" - -type manifest struct { - Dashboards []map[string]string `config:"dashboards"` -} - -func makeURL(url, path string, params url.Values) string { - if len(params) == 0 { - return url + path - } - - return strings.Join([]string{url, path, "?", params.Encode()}, "") -} - -func Export(client *http.Client, conn string, spaceID string, dashboard string, out string) error { - params := url.Values{} - - params.Add("dashboard", dashboard) - - if spaceID != "" { - exportAPI = path.Join("/s", spaceID, exportAPI) - } - fullURL := makeURL(conn, exportAPI, params) - if !quiet { - log.Printf("Calling HTTP GET %v\n", fullURL) - } - - req, err := http.NewRequest("GET", fullURL, nil) - - resp, err := client.Do(req) - if err != nil { - return fmt.Errorf("GET HTTP request fails with: %v", err) - } - - defer resp.Body.Close() - - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - return fmt.Errorf("fail to read response %s", err) - } - - if resp.StatusCode != 200 { - return fmt.Errorf("HTTP GET %s fails with %s, %s", fullURL, resp.Status, body) - } - - data, err := kibana.RemoveIndexPattern(body) - if err != nil { - return fmt.Errorf("fail to extract the index pattern: %v", err) - } - - objects := data["objects"].([]interface{}) - for _, obj := range objects { - o := obj.(common.MapStr) - - decodeValue(o, "attributes.uiStateJSON") - decodeValue(o, "attributes.visState") - decodeValue(o, "attributes.optionsJSON") - decodeValue(o, "attributes.panelsJSON") - decodeValue(o, "attributes.kibanaSavedObjectMeta.searchSourceJSON") - } - - data["objects"] = objects - - // Create all missing directories - err = os.MkdirAll(filepath.Dir(out), 0755) - if err != nil { - return err - } - - err = ioutil.WriteFile(out, []byte(data.StringToPrint()), 0666) - if !quiet { - log.Printf("The dashboard %s was exported under the %s file\n", dashboard, out) - } - return err -} - -func decodeValue(data common.MapStr, key string) { - v, err := data.GetValue(key) - if err != nil { - return - } - s := v.(string) - var d interface{} - json.Unmarshal([]byte(s), &d) - - data.Put(key, d) -} - -func ReadManifest(file string) ([]map[string]string, error) { - cfg, err := common.LoadFile(file) - if err != nil { - return nil, fmt.Errorf("error reading manifest file: %v", err) - } - - var manifest manifest - err = cfg.Unpack(&manifest) - if err != nil { - return nil, fmt.Errorf("error unpacking manifest: %v", err) - } - return manifest.Dashboards, nil -} +var ( + indexPattern = false + quiet = false +) -var indexPattern = false -var quiet = false +const ( + kibanaTimeout = 90 * time.Second +) func main() { kibanaURL := flag.String("kibana", "http://localhost:5601", "Kibana URL") @@ -152,11 +52,20 @@ func main() { flag.Parse() log.SetFlags(0) - transCfg := &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, // ignore expired SSL certificates + u, err := url.Parse(*kibanaURL) + if err != nil { + log.Fatalf("Error parsing Kibana URL: %v", err) } - client := &http.Client{Transport: transCfg} + client, err := kibana.NewClientWithConfig(&kibana.ClientConfig{ + Protocol: u.Scheme, + Host: u.Host, + SpaceID: *spaceID, + Timeout: kibanaTimeout, + }) + if err != nil { + log.Fatalf("Error while connecting to Kibana: %+v", err) + } if len(*ymlFile) == 0 && len(*dashboard) == 0 { flag.Usage() @@ -164,30 +73,36 @@ func main() { } if len(*ymlFile) > 0 { - dashboards, err := ReadManifest(*ymlFile) - if err != nil { - log.Fatalf("%s", err) - } - - for _, dashboard := range dashboards { - log.Printf("id=%s, name=%s\n", dashboard["id"], dashboard["file"]) - directory := filepath.Join(filepath.Dir(*ymlFile), "_meta/kibana/6/dashboard") - err := os.MkdirAll(directory, 0755) - if err != nil { - log.Fatalf("fail to create directory %s: %v", directory, err) - } - err = Export(client, *kibanaURL, *spaceID, dashboard["id"], filepath.Join(directory, dashboard["file"])) + results, info, err := dashboards.ExportAllFromYml(client, *ymlFile) + for i, r := range results { + log.Printf("id=%s, name=%s\n", info.Dashboards[i].ID, info.Dashboards[i].File) + r = dashboards.DecodeExported(r) + err = dashboards.SaveToFile(r, info.Dashboards[i].File, filepath.Dir(*ymlFile), client.GetVersion()) if err != nil { - log.Fatalf("fail to export the dashboards: %s", err) + log.Fatalf("failed to export the dashboards: %s", err) } } os.Exit(0) } if len(*dashboard) > 0 { - err := Export(client, *kibanaURL, *spaceID, *dashboard, *fileOutput) + result, err := dashboards.Export(client, *dashboard) if err != nil { - log.Fatalf("fail to export the dashboards: %s", err) + log.Fatalf("Failed to export the dashboard: %s", err) + } + result = dashboards.DecodeExported(result) + bytes, err := json.Marshal(result) + if err != nil { + log.Fatalf("Failed to save the dashboard: %s", err) + } + + err = ioutil.WriteFile(*fileOutput, bytes, 0644) + if err != nil { + log.Fatalf("Failed to save the dashboard: %s", err) + + } + if !quiet { + log.Printf("The dashboard %s was exported under the %s file\n", *dashboard, *fileOutput) } } } diff --git a/dev-tools/ecs-migration.yml b/dev-tools/ecs-migration.yml index f87cb5757ad..53a70566b85 100644 --- a/dev-tools/ecs-migration.yml +++ b/dev-tools/ecs-migration.yml @@ -85,8 +85,247 @@ alias: true copy_to: false +- from: system.syslog.hostname + to: host.hostname + alias: true + copy_to: false + +- from: system.syslog.program + to: process.name + alias: true + copy_to: false + +- from: system.syslog.pid + to: process.pid + alias: true + copy_to: false + +- from: system.syslog.message + to: message + alias: true + copy_to: false + # From Auditbeat's auditd module. - from: source.hostname to: source.domain alias: true copy_to: false + +- from: iis.access.server_ip + to: destination.ip + alias: true + copy_to: false + +- from: iis.access.remote_ip + to: source.ip + alias: true + copy_to: false + +- from: iis.access.url + to: url.path + alias: true + copy_to: false + +- from: iis.access.query_string + to: url.query + alias: true + copy_to: false + +- from: iis.access.port + to: destination.port + alias: true + copy_to: false + +- from: iis.access.user_name + to: user.name + alias: true + copy_to: false + +- from: iis.access.hostname + to: destination.domain + alias: true + copy_to: false + +- from: iis.access.user_agent.original + to: user_agent.original + alias: true + copy_to: false + +- from: iis.access.geoip.continent_name + to: source.geo.continent_name + alias: true + copy_to: false + +- from: iis.access.geoip.country_iso_code + to: source.geo.country_iso_code + alias: true + copy_to: false + +- from: iis.access.geoip.location + to: source.geo.location + alias: true + copy_to: false + +- from: iis.access.geoip.region_name + to: source.geo.region_name + alias: true + copy_to: false + +- from: iis.access.geoip.city_name + to: source.geo.city_name + alias: true + copy_to: false + +- from: iis.access.geoip.region_iso_code + to: source.geo.region_iso_code + alias: true + copy_to: false + +# Note: `http` is not officially in ECS yet + +- from: iis.access.method + to: http.request.method + alias: true + copy_to: false + +- from: iis.access.response_code + to: http.response.status_code + alias: true + copy_to: false + +- from: iis.access.referrer + to: http.request.referrer + alias: true + copy_to: false + +- from: haproxy.client.port + to: source.port + alias: true + copy_to: false + +- from: haproxy.process_name + to: process.name + alias: true + copy_to: false + +- from: haproxy.pid + to: process.pid + alias: true + copy_to: false + +- from: haproxy.destination.ip + to: destination.ip + alias: true + copy_to: false + +- from: haproxy.destination.port + to: destination.port + alias: true + copy_to: false + +- from: haproxy.geoip.continent_name + to: source.geo.continent_name + alias: true + copy_to: false + +- from: haproxy.geoip.country_iso_code + to: source.geo.country_iso_code + alias: true + copy_to: false + +- from: haproxy.geoip.location + to: source.geo.location + alias: true + copy_to: false + +- from: haproxy.geoip.region_name + to: source.geo.region_name + alias: true + copy_to: false + +- from: haproxy.geoip.city_name + to: source.geo.city_name + alias: true + copy_to: false + +- from: haproxy.geoip.region_iso_code + to: source.geo.region_iso_code + alias: true + copy_to: false + +- from: nginx.access.remote_ip_list + to: network.forwarded_ip + alias: true + copy_to: false + +- from: nginx.access.user_name + to: user.name + alias: true + copy_to: false + +- from: nginx.access.url + to: url.original + alias: true + copy_to: false + +- from: nginx.access.agent + to: user_agent.original + alias: true + copy_to: false + + # Note: `http` is not officially in ECS yet + +- from: nginx.access.response_code + to: http.response.status_code + alias: true + copy_to: false + +- from: nginx.access.referrer + to: http.request.referrer + alias: true + copy_to: false + +- from: nginx.access.method + to: http.request.method + alias: true + copy_to: false + +- from: nginx.access.http_version + to: http.version + alias: true + copy_to: false + +- from: nginx.access.geoip.continent_name + to: source.geo.continent_name + alias: true + copy_to: false + +- from: nginx.access.geoip.country_iso_code + to: source.geo.country_iso_code + alias: true + copy_to: false + +- from: nginx.access.geoip.location + to: source.geo.location + alias: true + copy_to: false + +- from: nginx.access.geoip.region_name + to: source.geo.region_name + alias: true + copy_to: false + +- from: nginx.access.geoip.city_name + to: source.geo.city_name + alias: true + copy_to: false + +- from: nginx.access.geoip.region_iso_code + to: source.geo.region_iso_code + alias: true + copy_to: false + +- from: nginx.access.agent + to: user_agent.original + alias: true + copy_to: false diff --git a/dev-tools/jenkins_release.sh b/dev-tools/jenkins_release.sh index 3206ece6733..b8058df7355 100755 --- a/dev-tools/jenkins_release.sh +++ b/dev-tools/jenkins_release.sh @@ -18,7 +18,6 @@ trap cleanup EXIT # This controls the defaults used the Jenkins package job. They can be # overridden by setting them in the environement prior to running this script. -export BEAT_VERSION_QUALIFIER="${VERSION_QUALIFIER}" export SNAPSHOT="${SNAPSHOT:-true}" export PLATFORMS="${PLATFORMS:-+linux/armv7 +linux/ppc64le +linux/s390x +linux/mips64}" diff --git a/dev-tools/mage/check.go b/dev-tools/mage/check.go new file mode 100644 index 00000000000..acd07dfba29 --- /dev/null +++ b/dev-tools/mage/check.go @@ -0,0 +1,180 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package mage + +import ( + "bufio" + "bytes" + "fmt" + "log" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + + "github.com/magefile/mage/mg" + "github.com/magefile/mage/sh" + "github.com/pkg/errors" + + "github.com/elastic/beats/libbeat/processors/dissect" +) + +// Check looks for created/modified/deleted/renamed files and returns an error +// if it finds any modifications. If executed in in verbose mode it will write +// the results of 'git diff' to stdout to indicate what changes have been made. +// +// It also checks the file permissions of nosetests test cases and YAML files. +func Check() error { + fmt.Println(">> check: Checking for modified files or incorrect permissions") + + mg.Deps(CheckNosetestsNotExecutable, CheckYAMLNotExecutable) + + changes, err := GitDiffIndex() + if err != nil { + return errors.Wrap(err, "failed to diff the git index") + } + + if len(changes) > 0 { + if mg.Verbose() { + GitDiff() + } + + return errors.Errorf("some files are not up-to-date. "+ + "Run 'mage fmt update' then review and commit the changes. "+ + "Modified: %v", changes) + } + return nil +} + +// GitDiffIndex returns a list of files that differ from what is committed. +// These could file that were created, deleted, modified, or moved. +func GitDiffIndex() ([]string, error) { + // Ensure the index is updated so that diff-index gives accurate results. + if err := sh.Run("git", "update-index", "-q", "--refresh"); err != nil { + return nil, err + } + + // git diff-index provides a list of modified files. + // https://www.git-scm.com/docs/git-diff-index + out, err := sh.Output("git", "diff-index", "HEAD", "--", ".") + if err != nil { + return nil, err + } + + // Example formats. + // :100644 100644 bcd1234... 0123456... M file0 + // :100644 100644 abcd123... 1234567... R86 file1 file3 + d, err := dissect.New(":%{src_mode} %{dst_mode} %{src_sha1} %{dst_sha1} %{status}\t%{paths}") + if err != nil { + return nil, err + } + + // Parse lines. + var modified []string + s := bufio.NewScanner(bytes.NewBufferString(out)) + for s.Scan() { + m, err := d.Dissect(s.Text()) + if err != nil { + return nil, errors.Wrap(err, "failed to dissect git diff-index output") + } + + paths := strings.Split(m["paths"], "\t") + if len(paths) > 1 { + modified = append(modified, paths[1]) + } else { + modified = append(modified, paths[0]) + } + } + if err = s.Err(); err != nil { + return nil, err + } + + return modified, nil +} + +// GitDiff runs 'git diff' and writes the output to stdout. +func GitDiff() error { + c := exec.Command("git", "--no-pager", "diff", "--minimal") + c.Stdin = nil + c.Stdout = os.Stdout + c.Stderr = os.Stderr + log.Println("exec:", strings.Join(c.Args, " ")) + err := c.Run() + return err +} + +// CheckNosetestsNotExecutable checks that none of the nosetests files are +// executable. Nosetests silently skips executable .py files and we don't want +// this to happen. +func CheckNosetestsNotExecutable() error { + if runtime.GOOS == "windows" { + // Skip windows because it doesn't have POSIX permissions. + return nil + } + + tests, err := FindFiles(nosetestsTestFiles...) + if err != nil { + return err + } + + var executableTestFiles []string + for _, file := range tests { + info, err := os.Stat(file) + if err != nil { + return err + } + + if info.Mode().Perm()&0111 > 0 { + executableTestFiles = append(executableTestFiles, file) + } + } + + if len(executableTestFiles) > 0 { + return errors.Errorf("nosetests files cannot be executable because "+ + "they will be skipped. Fix permissions of %v", executableTestFiles) + } + return nil +} + +// CheckYAMLNotExecutable checks that no .yml or .yaml files are executable. +func CheckYAMLNotExecutable() error { + if runtime.GOOS == "windows" { + // Skip windows because it doesn't have POSIX permissions. + return nil + } + + executableYAMLFiles, err := FindFilesRecursive(func(path string, info os.FileInfo) bool { + switch filepath.Ext(path) { + default: + return false + case ".yml", ".yaml": + return info.Mode().Perm()&0111 > 0 + } + }) + if err != nil { + return errors.Wrap(err, "failed search for YAML files") + } + + if len(executableYAMLFiles) > 0 { + return errors.Errorf("YAML files cannot be executable. Fix "+ + "permissions of %v", executableYAMLFiles) + + } + return nil +} diff --git a/dev-tools/mage/common.go b/dev-tools/mage/common.go index 43a7b8de675..669e48736d7 100644 --- a/dev-tools/mage/common.go +++ b/dev-tools/mage/common.go @@ -520,6 +520,29 @@ func FindFiles(globs ...string) ([]string, error) { return configFiles, nil } +// FindFilesRecursive recursively traverses from the CWD and invokes the given +// match function on each regular file to determine if the given path should be +// returned as a match. +func FindFilesRecursive(match func(path string, info os.FileInfo) bool) ([]string, error) { + var matches []string + err := filepath.Walk(".", func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + if !info.Mode().IsRegular() { + // continue + return nil + } + + if match(filepath.ToSlash(path), info) { + matches = append(matches, path) + } + return nil + }) + return matches, err +} + // FileConcat concatenates files and writes the output to out. func FileConcat(out string, perm os.FileMode, files ...string) error { f, err := os.OpenFile(createDir(out), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, perm) diff --git a/dev-tools/mage/crossbuild.go b/dev-tools/mage/crossbuild.go index 4b981c927e3..3a7efad0a31 100644 --- a/dev-tools/mage/crossbuild.go +++ b/dev-tools/mage/crossbuild.go @@ -247,7 +247,7 @@ func (b GolangCrossBuilder) Build() error { ) } if versionQualified { - args = append(args, "--env", "BEAT_VERSION_QUALIFIER="+versionQualifier) + args = append(args, "--env", "VERSION_QUALIFIER="+versionQualifier) } args = append(args, "--rm", diff --git a/dev-tools/mage/dashboard.go b/dev-tools/mage/dashboard.go new file mode 100644 index 00000000000..4e4279738e2 --- /dev/null +++ b/dev-tools/mage/dashboard.go @@ -0,0 +1,53 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package mage + +import ( + "fmt" + "path/filepath" + + "github.com/magefile/mage/sh" +) + +// ExportDashboard exports a dashboard from Kibana and writes it into the given module. +func ExportDashboard() error { + module := EnvOr("MODULE", "") + if module == "" { + return fmt.Errorf("MODULE must be specified") + } + + id := EnvOr("ID", "") + if id == "" { + return fmt.Errorf("Dashboad ID must be specified") + } + + beatsDir, err := ElasticBeatsDir() + if err != nil { + return err + } + + // TODO: This is currently hardcoded for KB 6, we need to figure out what we do for KB 7 + file := CWD("module", module, "_meta/kibana/6/dashboard", id+".json") + + dashboardCmd := sh.RunCmd("go", "run", + filepath.Join(beatsDir, "dev-tools/cmd/dashboards/export_dashboards.go"), + "-output", file, "-dashboard", id, + ) + + return dashboardCmd() +} diff --git a/dev-tools/mage/dockerbuilder.go b/dev-tools/mage/dockerbuilder.go index 24baefd1eaf..44791010217 100644 --- a/dev-tools/mage/dockerbuilder.go +++ b/dev-tools/mage/dockerbuilder.go @@ -18,8 +18,12 @@ package mage import ( + "bytes" + "compress/gzip" "fmt" + "io" "os" + "os/exec" "path/filepath" "strings" @@ -141,13 +145,47 @@ func (b *dockerBuilder) dockerSave(tag string) error { // Save the container as artifact outputFile := b.OutputFile if outputFile == "" { - outputTar, err := b.Expand(defaultBinaryName + ".docker.tar") + outputTar, err := b.Expand(defaultBinaryName + ".docker.tar.gz") if err != nil { return err } outputFile = filepath.Join(distributionsDir, outputTar) } - if err := sh.Run("docker", "save", "-o", outputFile, tag); err != nil { + var stderr bytes.Buffer + cmd := exec.Command("docker", "save", tag) + cmd.Stderr = &stderr + stdout, err := cmd.StdoutPipe() + if err != nil { + return err + } + if err = cmd.Start(); err != nil { + return err + } + + err = func() error { + f, err := os.Create(outputFile) + if err != nil { + return err + } + defer f.Close() + + w := gzip.NewWriter(f) + defer w.Close() + + _, err = io.Copy(w, stdout) + if err != nil { + return err + } + return nil + }() + if err != nil { + return err + } + + if err = cmd.Wait(); err != nil { + if errmsg := strings.TrimSpace(stderr.String()); errmsg != "" { + err = errors.Wrap(errors.New(errmsg), err.Error()) + } return err } return errors.Wrap(CreateSHA512File(outputFile), "failed to create .sha512 file") diff --git a/dev-tools/mage/fmt.go b/dev-tools/mage/fmt.go new file mode 100644 index 00000000000..5e80a18e037 --- /dev/null +++ b/dev-tools/mage/fmt.go @@ -0,0 +1,130 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package mage + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/magefile/mage/mg" + "github.com/magefile/mage/sh" + "github.com/pkg/errors" +) + +var ( + // GoImportsImportPath controls the import path used to install goimports. + GoImportsImportPath = "github.com/elastic/beats/vendor/golang.org/x/tools/cmd/goimports" + + // GoImportsLocalPrefix is a string prefix matching imports that should be + // grouped after third-party packages. + GoImportsLocalPrefix = "github.com/elastic" + + // GoLicenserImportPath controls the import path used to install go-licenser. + GoLicenserImportPath = "github.com/elastic/go-licenser" +) + +// Format adds license headers, formats .go files with goimports, and formats +// .py files with autopep8. +func Format() { + // Don't run AddLicenseHeaders and GoImports concurrently because they + // both can modify the same files. + mg.Deps(AddLicenseHeaders) + mg.Deps(GoImports, PythonAutopep8) +} + +// GoImports executes goimports against all .go files in and below the CWD. It +// ignores vendor/ directories. +func GoImports() error { + goFiles, err := FindFilesRecursive(func(path string, _ os.FileInfo) bool { + return filepath.Ext(path) == ".go" && !strings.Contains(path, "vendor/") + }) + if err != nil { + return err + } + if len(goFiles) == 0 { + return nil + } + + fmt.Println(">> fmt - goimports: Formatting Go code") + if err := sh.Run("go", "get", GoImportsImportPath); err != nil { + return err + } + + args := append( + []string{"-local", GoImportsLocalPrefix, "-l", "-w"}, + goFiles..., + ) + + return sh.RunV("goimports", args...) +} + +// PythonAutopep8 executes autopep8 on all .py files in and below the CWD. It +// ignores build/ directories. +func PythonAutopep8() error { + pyFiles, err := FindFilesRecursive(func(path string, _ os.FileInfo) bool { + return filepath.Ext(path) == ".py" && !strings.Contains(path, "build/") + }) + if err != nil { + return err + } + if len(pyFiles) == 0 { + return nil + } + + fmt.Println(">> fmt - autopep8: Formatting Python code") + ve, err := PythonVirtualenv() + if err != nil { + return err + } + + autopep8, err := lookVirtualenvPath(ve, "autopep8") + if err != nil { + return err + } + + args := append( + []string{"--in-place", "--max-line-length", "120"}, + pyFiles..., + ) + + return sh.RunV(autopep8, args...) +} + +// AddLicenseHeaders adds license headers to .go files. It applies the +// appropriate license header based on the value of mage.BeatLicense. +func AddLicenseHeaders() error { + fmt.Println(">> fmt - go-licenser: Adding missing headers") + + if err := sh.Run("go", "get", GoLicenserImportPath); err != nil { + return err + } + + var license string + switch BeatLicense { + case "ASL 2.0": + license = "ASL2" + case "Elastic": + license = BeatLicense + default: + return errors.Errorf("unknown license type %v", BeatLicense) + } + + return sh.RunV("go-licenser", "-license", license) +} diff --git a/dev-tools/mage/gotest.go b/dev-tools/mage/gotest.go index 9b9e8a735dd..2de7aa8dbae 100644 --- a/dev-tools/mage/gotest.go +++ b/dev-tools/mage/gotest.go @@ -206,7 +206,7 @@ func GoTest(ctx context.Context, params GoTestArgs) error { } func makeCommand(ctx context.Context, env map[string]string, cmd string, args ...string) *exec.Cmd { - c := exec.CommandContext(ctx, "go", args...) + c := exec.CommandContext(ctx, cmd, args...) c.Env = os.Environ() for k, v := range env { c.Env = append(c.Env, k+"="+v) diff --git a/dev-tools/mage/integtest.go b/dev-tools/mage/integtest.go index 3d8adb4f486..55f5a25f66e 100644 --- a/dev-tools/mage/integtest.go +++ b/dev-tools/mage/integtest.go @@ -22,6 +22,7 @@ import ( "io/ioutil" "os" "path/filepath" + "strings" "sync" "github.com/pkg/errors" @@ -41,6 +42,8 @@ var ( integTestUseCountLock sync.Mutex // Lock to guard integTestUseCount. integTestLock sync.Mutex // Only allow one integration test at a time. + + integTestBuildImagesOnce sync.Once // Build images one time for all integ testing. ) // Integration Test Configuration @@ -142,6 +145,12 @@ func runInIntegTestEnv(mageTarget string, test func() error, passThroughEnvVars return test() } + var err error + integTestBuildImagesOnce.Do(func() { err = dockerComposeBuildImages() }) + if err != nil { + return err + } + // Test that we actually have Docker and docker-compose. if err := haveIntegTestEnvRequirements(); err != nil { return errors.Wrapf(err, "failed to run %v target in integration environment", mageTarget) @@ -231,10 +240,55 @@ func integTestDockerComposeEnvVars() (map[string]string, error) { }, nil } +// dockerComposeProjectName returns the project name to use with docker-compose. +// It is passed to docker-compose using the `-p` flag. And is passed to our +// Go and Python testing libraries through the DOCKER_COMPOSE_PROJECT_NAME +// environment variable. func dockerComposeProjectName() string { - projectName := "{{.BeatName}}{{.StackEnvironment}}{{ beat_version }}{{ commit }}" + commit, err := CommitHash() + if err != nil { + panic(errors.Wrap(err, "failed to construct docker compose project name")) + } + + version, err := BeatQualifiedVersion() + if err != nil { + panic(errors.Wrap(err, "failed to construct docker compose project name")) + } + version = strings.NewReplacer(".", "_").Replace(version) + + projectName := "{{.BeatName}}_{{.Version}}_{{.ShortCommit}}-{{.StackEnvironment}}" projectName = MustExpand(projectName, map[string]interface{}{ "StackEnvironment": StackEnvironment, + "ShortCommit": commit[:10], + "Version": version, }) return projectName } + +// dockerComposeBuildImages builds all images in the docker-compose.yml file. +func dockerComposeBuildImages() error { + fmt.Println(">> Building docker images") + + composeEnv, err := integTestDockerComposeEnvVars() + if err != nil { + return err + } + + args := []string{"-p", dockerComposeProjectName(), "build", "--pull", "--force-rm"} + if _, noCache := os.LookupEnv("DOCKER_NOCACHE"); noCache { + args = append(args, "--no-cache") + } + + out := ioutil.Discard + if mg.Verbose() { + out = os.Stderr + } + + _, err = sh.Exec( + composeEnv, + out, + os.Stderr, + "docker-compose", args..., + ) + return err +} diff --git a/dev-tools/mage/pytest.go b/dev-tools/mage/pytest.go index 72abf6562d1..0cef1b01be2 100644 --- a/dev-tools/mage/pytest.go +++ b/dev-tools/mage/pytest.go @@ -32,18 +32,31 @@ import ( ) const ( - libbeatRequirements = "libbeat/tests/system/requirements.txt" + libbeatRequirements = "{{ elastic_beats_dir}}/libbeat/tests/system/requirements.txt" ) var ( - pythonVirtualenvDir string + // VirtualenvReqs specifies a list of virtualenv requirements files to be + // used when calling PythonVirtualenv(). It defaults to the libbeat + // requirements.txt file. + VirtualenvReqs = []string{ + libbeatRequirements, + } + + pythonVirtualenvDir string // Location of python virtualenv (lazily set). + + // More globs may be needed in the future if tests are added in more places. + nosetestsTestFiles = []string{ + "tests/system/test_*.py", + "module/*/test_*.py", + "module/*/*/test_*.py", + } ) // PythonTestArgs are the arguments used for the "python*Test" targets and they // define how "nosetests" is invoked. type PythonTestArgs struct { TestName string // Test name used in logging. - VirtualenvReqs []string // requirements.txt files. Env map[string]string // Env vars to add to the current env. XUnitReportFile string // File to write the XUnit XML test report to. CoverageProfileFile string // Test coverage profile file. @@ -51,11 +64,9 @@ type PythonTestArgs struct { func makePythonTestArgs(name string) PythonTestArgs { fileName := fmt.Sprintf("build/TEST-python-%s", strings.Replace(strings.ToLower(name), " ", "_", -1)) - defaultReqsTxt := filepath.Join(MustExpand("{{ elastic_beats_dir }}"), libbeatRequirements) params := PythonTestArgs{ TestName: name, - VirtualenvReqs: []string{defaultReqsTxt}, Env: map[string]string{}, XUnitReportFile: fileName + ".xml", } @@ -78,7 +89,7 @@ func DefaultPythonTestIntegrationArgs() PythonTestArgs { return makePythonTestAr func PythonNoseTest(params PythonTestArgs) error { fmt.Println(">> python test:", params.TestName, "Testing") - ve, err := PythonVirtualenv(params.VirtualenvReqs...) + ve, err := PythonVirtualenv() if err != nil { return err } @@ -107,12 +118,8 @@ func PythonNoseTest(params PythonTestArgs) error { "--xunit-file="+createDir(params.XUnitReportFile), ) } - // More globs may be needed in the future if tests are added in more places. - testFiles, err := FindFiles( - "tests/system/test_*.py", - "module/*/test_*.py", - "module/*/*/test_*.py", - ) + + testFiles, err := FindFiles(nosetestsTestFiles...) if err != nil { return err } @@ -139,17 +146,19 @@ func PythonNoseTest(params PythonTestArgs) error { // PythonVirtualenv constructs a virtualenv that contains the given modules as // defined in the requirements file pointed to by requirementsTxt. It returns // the path to the virutalenv. -func PythonVirtualenv(requirementsTxt ...string) (string, error) { +func PythonVirtualenv() (string, error) { // Determine the location of the virtualenv. ve, err := pythonVirtualenvPath() if err != nil { return "", err } + reqs := expandVirtualenvReqs() + // Only execute if requirements.txt is newer than the virtualenv activate // script. activate := virtualenvPath(ve, "activate") - if IsUpToDate(activate, requirementsTxt...) { + if IsUpToDate(activate, reqs...) { return pythonVirtualenvDir, nil } @@ -171,7 +180,7 @@ func PythonVirtualenv(requirementsTxt ...string) (string, error) { if !mg.Verbose() { args = append(args, "--quiet") } - for _, req := range requirementsTxt { + for _, req := range reqs { args = append(args, "-Ur", req) } @@ -240,3 +249,11 @@ func lookVirtualenvPath(ve, file string) (string, error) { return exec.LookPath(file) } + +func expandVirtualenvReqs() []string { + out := make([]string, 0, len(VirtualenvReqs)) + for _, path := range VirtualenvReqs { + out = append(out, MustExpand(path)) + } + return out +} diff --git a/dev-tools/mage/settings.go b/dev-tools/mage/settings.go index ffaa54991d4..19f1e4829e6 100644 --- a/dev-tools/mage/settings.go +++ b/dev-tools/mage/settings.go @@ -103,7 +103,7 @@ func init() { panic(errors.Errorf("failed to parse SNAPSHOT env value", err)) } - versionQualifier, versionQualified = os.LookupEnv("BEAT_VERSION_QUALIFIER") + versionQualifier, versionQualified = os.LookupEnv("VERSION_QUALIFIER") } // EnvMap returns map containing the common settings variables and all variables @@ -319,7 +319,7 @@ var ( ) // BeatQualifiedVersion returns the Beat's qualified version. The value can be overwritten by -// setting BEAT_VERSION_QUALIFIER in the environment. +// setting VERSION_QUALIFIER in the environment. func BeatQualifiedVersion() (string, error) { version, err := beatVersion() if err != nil { diff --git a/dev-tools/make/mage.mk b/dev-tools/make/mage.mk new file mode 100644 index 00000000000..5875c1a4d78 --- /dev/null +++ b/dev-tools/make/mage.mk @@ -0,0 +1,11 @@ +MAGE_PRESENT := $(shell command -v mage 2> /dev/null) +MAGE_IMPORT_PATH?=github.com/elastic/beats/vendor/github.com/magefile/mage +export MAGE_IMPORT_PATH + +.PHONY: mage +mage: +ifndef MAGE_PRESENT + go install ${MAGE_IMPORT_PATH} + @-mage -clean 2> /dev/null +endif + @true diff --git a/dev-tools/make/xpack.mk b/dev-tools/make/xpack.mk new file mode 100644 index 00000000000..3210cb8fca0 --- /dev/null +++ b/dev-tools/make/xpack.mk @@ -0,0 +1,43 @@ +# This Makefile is for x-pack Beats. Its only responsibility is to provide +# compatibility with existing Jenkins and Travis setups. + +# +# Variables +# +.DEFAULT_GOAL := help +PWD := $(CURDIR) + +# +# Includes +# +include $(ES_BEATS)/dev-tools/make/mage.mk + +# +# Targets (alphabetically sorted). +# +.PHONY: check +check: mage + mage check + +.PHONY: clean +clean: mage + mage clean + +.PHONY: fmt +fmt: mage + mage fmt + +# Default target. +.PHONY: help +help: + @echo Use mage rather than make. Here are the available mage targets: + @mage -l + +.PHONY: testsuite +testsuite: mage + mage update build unitTest integTest + +.PHONY: update +update: mage + mage update + diff --git a/dev-tools/packaging/package_test.go b/dev-tools/packaging/package_test.go index d447e22ade7..cea1e9e9b69 100644 --- a/dev-tools/packaging/package_test.go +++ b/dev-tools/packaging/package_test.go @@ -80,7 +80,8 @@ func TestDeb(t *testing.T) { } func TestTar(t *testing.T) { - tars := getFiles(t, regexp.MustCompile(`\.tar\.gz$`)) + // Regexp matches *-arch.tar.gz, but not *-arch.docker.tar.gz + tars := getFiles(t, regexp.MustCompile(`-\w+\.tar\.gz$`)) for _, tar := range tars { checkTar(t, tar) } @@ -94,7 +95,7 @@ func TestZip(t *testing.T) { } func TestDocker(t *testing.T) { - dockers := getFiles(t, regexp.MustCompile(`\.docker.tar$`)) + dockers := getFiles(t, regexp.MustCompile(`\.docker\.tar\.gz$`)) for _, docker := range dockers { checkDocker(t, docker) } @@ -514,7 +515,13 @@ func readDocker(dockerFile string) (*packageFile, *dockerInfo, error) { var info *dockerInfo layers := make(map[string]*packageFile) - tarReader := tar.NewReader(file) + gzipReader, err := gzip.NewReader(file) + if err != nil { + return nil, nil, err + } + defer gzipReader.Close() + + tarReader := tar.NewReader(gzipReader) for { header, err := tarReader.Next() if err != nil { diff --git a/docs/devguide/newdashboards.asciidoc b/docs/devguide/newdashboards.asciidoc index b532655fc20..3c594fa4150 100644 --- a/docs/devguide/newdashboards.asciidoc +++ b/docs/devguide/newdashboards.asciidoc @@ -213,44 +213,33 @@ https://github.com/elastic/beats/tree/master/dev-tools/cmd/dashboards[dev-tools] for exporting Kibana 5.x dashboards. See the dev-tools https://github.com/elastic/beats/tree/master/dev-tools/README.md[readme] for more info. -///////////////////// -NOT YET IMPLEMENTED -NOTE: You can make use of the Makefile from the Beat GitHub repository to export all the Kibana dashboards for a Beat -from your Elasticsearch. If Elasticsearch is running on localhost, then you just need to run the following command from the Beat repository: - -[source,shell] ------------------------------ -make export-dashboards ------------------------------ - -If Elasticsearch is running on a different host, then you can use the `ES_URL` variable: - -[source,shell] ----------------------------- -ES_URL="http://192.168.3.206:9200" make export-dashboards ----------------------------- - -///////////////////// +Alternatively, if the scripts above are not available, you can use your Beat binary to export Kibana 6.0 dashboards or later. ==== Exporting Kibana 6.0 dashboards and newer The `dev-tools/cmd/export_dashboards.go` script helps you export your customized Kibana 6.0 dashboards and newer. You might need to export a single dashboard or all the dashboards available for a module or Beat. +It is also possible to use a Beat binary to export. ===== Export a single Kibana dashboard -You can export a single dashboard by passing the dashboard ID in the `-dashboard` flag. +To export a single dashboard for a module you can use the following command inside a Beat with modules: -NOTE: The dashboard ID is available in the dashboard URL. For example, in case the dashboard URL is -`app/kibana#/dashboard/7fea2930-478e-11e7-b1f0-cb29bac6bf8b?_g=()&_a=(description:'Overview%2...`, the dashboard ID is `7fea2930-478e-11e7-b1f0-cb29bac6bf8b`. +[source,shell] +--------------- +MODULE=redis ID=AV4REOpp5NkDleZmzKkE mage exportDashboard +--------------- [source,shell] --------------- -cd filebeat/module/redis/_meta/kibana/default/dashboard -go run ../../../../../../../dev-tools/cmd/dashboards/export_dashboards.go -id 7fea2930-478e-11e7-b1f0-cb29bac6bf8b -output Filebeat-redis.json +./filebeat export dashboard -id 7fea2930-478e-11e7-b1f0-cb29bac6bf8b >> Filebeat-redis.json --------------- -This generates the `Filebeat-redis.json` file that contains the dashboard for the Redis module of Filebeat, including the dependencies (visualizations and searches). +This generates a `AV4REOpp5NkDleZmzKkE.json` file inside dashboard directory in the redis module. +It contains all dependencies like visualizations and searches. + +NOTE: The dashboard ID is available in the dashboard URL. For example, in case the dashboard URL is +`app/kibana#/dashboard/AV4REOpp5NkDleZmzKkE?_g=()&_a=(description:'Overview%2...`, the dashboard ID is `AV4REOpp5NkDleZmzKkE`. ===== Export all module/Beat dashboards @@ -275,13 +264,18 @@ dashboards: Each dashboard is defined by an `id` and the name of json `file` where the dashboard is saved locally. -By passing the yml file to the `export_dashboards.go` script, you can export all the dashboards defined: +By passing the yml file to the `export_dashboards.go` script or to the Beat, you can export all the dashboards defined: [source,shell] ------------------- go run dev-tools/cmd/dashboards/export_dashboards.go -yml filebeat/module/system/module.yml ------------------- +[source,shell] +------------------- +./filebeat export dashboard -yml filebeat/module/system/module.yml +------------------- + ===== Export dashboards from a Kibana Space @@ -292,6 +286,8 @@ If you are using the Kibana Spaces feature and want to export dashboards from a go run dev-tools/cmd/dashboards/export_dashboards.go -space-id my-space [other-options] ------------------- +In case of running `export dashboard` of a Beat, you need to set the Space ID in `setup.kibana.space.id`. + ==== Exporting Kibana 5.x dashboards diff --git a/filebeat/Dockerfile b/filebeat/Dockerfile index 990d129eea0..6bc6144156c 100644 --- a/filebeat/Dockerfile +++ b/filebeat/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.10.3 +FROM golang:1.11.2 RUN \ apt-get update \ diff --git a/filebeat/_meta/fields.common.yml b/filebeat/_meta/fields.common.yml index d793c67f2f4..a5bf24c32af 100644 --- a/filebeat/_meta/fields.common.yml +++ b/filebeat/_meta/fields.common.yml @@ -101,6 +101,20 @@ - name: full_name type: keyword + # Temporary fix until ECS includes the HTTP object again + - name: http.request.referrer + type: keyword + description: > + Referrer for this HTTP request. + example: https://blog.example.com/ + + # Temporary fixes until ECS is reimported + - name: url.original + type: keyword + description: > + Full original url. The field is stored as keyword. + example: https://blog.example.com/ + # Temporary fix to get 7.0 dashboards working - name: fileset.name type: alias diff --git a/filebeat/autodiscover/builder/hints/logs.go b/filebeat/autodiscover/builder/hints/logs.go index 9b7d236e86a..9c74d999700 100644 --- a/filebeat/autodiscover/builder/hints/logs.go +++ b/filebeat/autodiscover/builder/hints/logs.go @@ -185,7 +185,7 @@ func (l *logHints) getFilesets(hints common.MapStr, module string) map[string]*f moduleFilesets, err := l.Registry.ModuleFilesets(module) if err != nil { - logp.Err("Error retrieving module filesets", err) + logp.Err("Error retrieving module filesets: %+v", err) return nil } diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index 0a03c51eea2..905a63e1ea3 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -3393,66 +3393,6 @@ haproxy Module -[float] -== destination fields - -Destination information - - -*`haproxy.destination.port`*:: -+ --- -type: long - -Port of the destination host - --- - -*`haproxy.destination.ip`*:: -+ --- -IP of the destination host - --- - -*`haproxy.process_name`*:: -+ --- -Name of the process - --- - -*`haproxy.pid`*:: -+ --- -type: long - -PID of the process - --- - -[float] -== client fields - -Information about the client doing the request - - -*`haproxy.client.ip`*:: -+ --- -IP address of the client which initiated the TCP connection to haproxy. - --- - -*`haproxy.client.port`*:: -+ --- -type: long - -TCP port of the client which initiated the connection. - --- - *`haproxy.frontend_name`*:: + -- @@ -3562,131 +3502,175 @@ The HAProxy source of the log -- +*`haproxy.termination_state`*:: ++ +-- +Condition the session was in when the session ended. + +-- + +*`haproxy.mode`*:: ++ +-- +type: text + +mode that the frontend is operating (TCP or HTTP) + +-- + [float] -== geoip fields +== connections fields -Contains GeoIP information gathered based on the client.ip field. Only present if the GeoIP Elasticsearch plugin is available and used. +Contains various counts of connections active in the process. +*`haproxy.connections.active`*:: ++ +-- +type: long + +Total number of concurrent connections on the process when the session was logged. + +-- -*`haproxy.geoip.continent_name`*:: +*`haproxy.connections.frontend`*:: + -- -type: keyword +type: long -Name of the continent. +Total number of concurrent connections on the frontend when the session was logged. -- -*`haproxy.geoip.country_iso_code`*:: +*`haproxy.connections.backend`*:: + -- -type: keyword +type: long -Country ISO code. +Total number of concurrent connections handled by the backend when the session was logged. -- -*`haproxy.geoip.location`*:: +*`haproxy.connections.server`*:: + -- -type: geo_point +type: long -Represents a geopoint with the longitude and latitude. +Total number of concurrent connections still active on the server when the session was logged. -- -*`haproxy.geoip.region_name`*:: +*`haproxy.connections.retries`*:: + -- -type: keyword +type: long -Name of the region +Number of connection retries experienced by this session when trying to connect to the server. -- -*`haproxy.geoip.city_name`*:: +[float] +== client fields + +Information about the client doing the request + + +*`haproxy.client.ip`*:: + -- -type: keyword +IP address of the client which initiated the TCP connection to haproxy. +If connection is via unix socket, socket path is in this field. -City name. -- -*`haproxy.geoip.region_iso_code`*:: +*`source.port`*:: + -- -type: keyword +type: alias + +-- -ISO code of the region +*`process.name`*:: ++ +-- +type: alias -- -*`haproxy.termination_state`*:: +*`process.pid`*:: + -- -Condition the session was in when the session ended. +type: alias -- [float] -== connections fields +== destination fields -Contains various counts of connections active in the process. +Destination information -*`haproxy.connections.active`*:: +*`destination.port`*:: + -- -type: long - -Total number of concurrent connections on the process when the session was logged. +type: alias -- -*`haproxy.connections.frontend`*:: +*`destination.ip`*:: + -- -type: long - -Total number of concurrent connections on the frontend when the session was logged. +type: alias -- -*`haproxy.connections.backend`*:: +[float] +== geoip fields + +Contains GeoIP information gathered based on the client.ip field. Only present if the GeoIP Elasticsearch plugin is available and used. + + + +*`source.geo.continent_name`*:: + -- -type: long - -Total number of concurrent connections handled by the backend when the session was logged. +type: alias -- -*`haproxy.connections.server`*:: +*`source.geo.country_iso_code`*:: + -- -type: long - -Total number of concurrent connections still active on the server when the session was logged. +type: alias -- -*`haproxy.connections.retries`*:: +*`source.geo.location`*:: + -- -type: long +type: alias -Number of connection retries experienced by this session when trying to connect to the server. +-- + +*`source.geo.region_name`*:: ++ +-- +type: alias -- -*`haproxy.mode`*:: +*`source.geo.city_name`*:: + -- -type: text +type: alias -mode that the frontend is operating (TCP or HTTP) +-- + +*`source.geo.region_iso_code`*:: ++ +-- +type: alias -- @@ -3976,98 +3960,6 @@ Contains fields for IIS access logs. -*`iis.access.server_ip`*:: -+ --- -type: keyword - -The server IP address. - - --- - -*`iis.access.method`*:: -+ --- -type: keyword - -example: GET - -The request HTTP method. - - --- - -*`iis.access.url`*:: -+ --- -type: keyword - -The request HTTP URL. - - --- - -*`iis.access.query_string`*:: -+ --- -type: keyword - -The request query string, if any. - - --- - -*`iis.access.port`*:: -+ --- -type: long - -The request port number. - - --- - -*`iis.access.user_name`*:: -+ --- -type: keyword - -The user name used when basic authentication is used. - - --- - -*`iis.access.remote_ip`*:: -+ --- -type: keyword - -The client IP address. - - --- - -*`iis.access.referrer`*:: -+ --- -type: keyword - -The HTTP referrer. - - --- - -*`iis.access.response_code`*:: -+ --- -type: long - -The HTTP response code. - - --- - *`iis.access.sub_status`*:: + -- @@ -4116,16 +4008,6 @@ type: keyword The name of the server on which the log file entry was generated. --- - -*`iis.access.http_version`*:: -+ --- -type: keyword - -The HTTP version. - - -- *`iis.access.cookie`*:: @@ -4136,16 +4018,6 @@ type: keyword The content of the cookie sent or received, if any. --- - -*`iis.access.hostname`*:: -+ --- -type: keyword - -The host header name, if any. - - -- *`iis.access.body_sent.bytes`*:: @@ -4172,191 +4044,194 @@ The number of bytes of the server request body. -- -*`iis.access.agent`*:: +*`destination.ip`*:: + -- -type: text - -Contains the un-parsed user agent string. Only present if the user agent Elasticsearch plugin is not available or not used. - +type: alias -- -[float] -== user_agent fields - -Contains the parsed user agent field. Only present if the user agent Elasticsearch plugin is available and used. - +*`http.request.method`*:: ++ +-- +type: alias +-- -*`iis.access.user_agent.device`*:: +*`url.path`*:: + -- -type: keyword +type: alias -The name of the physical device. +-- +*`url.query`*:: ++ +-- +type: alias -- -*`iis.access.user_agent.major`*:: +*`destination.port`*:: + -- -type: long +type: alias -The major version of the user agent. +-- +*`user.name`*:: ++ +-- +type: alias -- -*`iis.access.user_agent.minor`*:: +*`source.ip`*:: + -- -type: long +type: alias -The minor version of the user agent. +-- +*`http.request.referrer`*:: ++ +-- +type: alias -- -*`iis.access.user_agent.patch`*:: +*`http.response.status_code`*:: + -- -type: keyword +type: alias -The patch version of the user agent. +-- +*`http.version`*:: ++ +-- +type: alias -- -*`iis.access.user_agent.name`*:: +*`host.hostname`*:: + -- -type: keyword +type: alias -example: Chrome +-- -The name of the user agent. +*`user_agent.device`*:: ++ +-- +type: alias -- -*`iis.access.user_agent.os`*:: +*`user_agent.major`*:: + -- -type: keyword - -The name of the operating system. - +type: alias -- -*`iis.access.user_agent.os_major`*:: +*`user_agent.minor`*:: + -- -type: long - -The major version of the operating system. - +type: alias -- -*`iis.access.user_agent.os_minor`*:: +*`user_agent.patch`*:: + -- -type: long - -The minor version of the operating system. - +type: alias -- -*`iis.access.user_agent.os_name`*:: +*`user_agent.name`*:: + -- -type: keyword - -The name of the operating system. - +type: alias -- -*`iis.access.user_agent.original`*:: +*`user_agent.os.full_name`*:: + -- -type: text - -Original user agent value before parsing by ingest-user-agent plugin. - - -Field is not indexed. +type: alias -- -[float] -== geoip fields - -Contains GeoIP information gathered based on the remote_ip field. Only present if the GeoIP Elasticsearch plugin is available and used. - - - -*`iis.access.geoip.continent_name`*:: +*`user_agent.os.major`*:: + -- -type: keyword +type: alias -The name of the continent. +-- +*`user_agent.os.minor`*:: ++ +-- +type: alias -- -*`iis.access.geoip.country_iso_code`*:: +*`user_agent.os.name`*:: + -- -type: keyword - -Country ISO code. - +type: alias -- -*`iis.access.geoip.location`*:: +*`user_agent.original`*:: + -- -type: geo_point - -The longitude and latitude. - +type: alias -- -*`iis.access.geoip.region_name`*:: + +*`source.geo.continent_name`*:: + -- -type: keyword +type: alias -The region name. +-- +*`source.geo.country_iso_code`*:: ++ +-- +type: alias -- -*`iis.access.geoip.city_name`*:: +*`source.geo.location`*:: + -- -type: keyword +type: alias -The city name. +-- +*`source.geo.region_name`*:: ++ +-- +type: alias -- -*`iis.access.geoip.region_iso_code`*:: +*`source.geo.city_name`*:: + -- -type: keyword +type: alias -Region ISO code. +-- +*`source.geo.region_iso_code`*:: ++ +-- +type: alias -- @@ -4956,6 +4831,30 @@ Content length of the HTTP response body. -- type: keyword +-- + +*`http.request.referrer`*:: ++ +-- +type: keyword + +example: https://blog.example.com/ + +Referrer for this HTTP request. + + +-- + +*`url.original`*:: ++ +-- +type: keyword + +example: https://blog.example.com/ + +Full original url. The field is stored as keyword. + + -- *`event.dataset`*:: @@ -5404,285 +5303,192 @@ Contains fields for the Nginx access logs. -*`nginx.access.remote_ip_list`*:: +*`nginx.access.body_sent.bytes`*:: + -- -type: array +type: long + +format: bytes -An array of remote IP addresses. It is a list because it is common to include, besides the client IP address, IP addresses from headers like `X-Forwarded-For`. See also the `remote_ip` field. +The number of bytes of the server response body. -- -*`nginx.access.remote_ip`*:: +*`network.forwarded_ip`*:: + -- -type: keyword - -Client IP address. The first public IP address from the `remote_ip_list` array. If no public IP addresses are present, this field contains the first private IP address from the `remote_ip_list` array. - +type: alias -- -*`nginx.access.user_name`*:: +*`source.ip`*:: + -- -type: keyword - -The user name used when basic authentication is used. - +type: alias -- -*`nginx.access.method`*:: +*`user.name`*:: + -- -type: keyword - -example: GET - -The request HTTP method. - +type: alias -- -*`nginx.access.url`*:: +*`http.request.method`*:: + -- -type: keyword - -The request HTTP URL. - +type: alias -- -*`nginx.access.http_version`*:: +*`url.original`*:: + -- -type: keyword - -The HTTP version. - +type: alias -- -*`nginx.access.response_code`*:: +*`http.version`*:: + -- -type: long - -The HTTP response code. - +type: alias -- -*`nginx.access.body_sent.bytes`*:: +*`http.response.status_code`*:: + -- -type: long - -format: bytes - -The number of bytes of the server response body. - +type: alias -- -*`nginx.access.referrer`*:: +*`http.request.referrer`*:: + -- -type: keyword - -The HTTP referrer. - +type: alias -- -*`nginx.access.agent`*:: +*`user_agent.original`*:: + -- -type: text - -Contains the un-parsed user agent string. Only present if the user agent Elasticsearch plugin is not available or not used. - +type: alias -- -[float] -== user_agent fields - -Contains the parsed User agent field. Only present if the user agent Elasticsearch plugin is available and used. - - -*`nginx.access.user_agent.device`*:: +*`user_agent.device`*:: + -- -type: keyword - -The name of the physical device. - +type: alias -- -*`nginx.access.user_agent.major`*:: +*`user_agent.major`*:: + -- -type: long - -The major version of the user agent. - +type: alias -- -*`nginx.access.user_agent.minor`*:: +*`user_agent.minor`*:: + -- -type: long - -The minor version of the user agent. - +type: alias -- -*`nginx.access.user_agent.patch`*:: +*`user_agent.patch`*:: + -- -type: keyword - -The patch version of the user agent. - +type: alias -- -*`nginx.access.user_agent.name`*:: +*`user_agent.name`*:: + -- -type: keyword - -example: Chrome - -The name of the user agent. - +type: alias -- -*`nginx.access.user_agent.os`*:: +*`user_agent.os.full_name`*:: + -- -type: keyword - -The name of the operating system. - +type: alias -- -*`nginx.access.user_agent.os_major`*:: +*`user_agent.os.major`*:: + -- -type: long - -The major version of the operating system. - +type: alias -- -*`nginx.access.user_agent.os_minor`*:: +*`user_agent.os.minor`*:: + -- -type: long - -The minor version of the operating system. - +type: alias -- -*`nginx.access.user_agent.os_name`*:: +*`user_agent.os.name`*:: + -- -type: keyword - -The name of the operating system. - +type: alias -- -*`nginx.access.user_agent.original`*:: +*`user_agent.original`*:: + -- -type: text - -Original user agent value before parsing by ingest-user-agent plugin. - - -Field is not indexed. +type: alias -- -[float] -== geoip fields - -Contains GeoIP information gathered based on the remote_ip field. Only present if the GeoIP Elasticsearch plugin is available and used. - - -*`nginx.access.geoip.continent_name`*:: +*`source.geo.continent_name`*:: + -- -type: keyword - -The name of the continent. - +type: alias -- -*`nginx.access.geoip.country_iso_code`*:: +*`source.geo.country_iso_code`*:: + -- -type: keyword - -Country ISO code. - +type: alias -- -*`nginx.access.geoip.location`*:: +*`source.geo.location`*:: + -- -type: geo_point - -The longitude and latitude. - +type: alias -- -*`nginx.access.geoip.region_name`*:: +*`source.geo.region_name`*:: + -- -type: keyword - -The region name. - +type: alias -- -*`nginx.access.geoip.city_name`*:: +*`source.geo.city_name`*:: + -- -type: keyword - -The city name. - +type: alias -- -*`nginx.access.geoip.region_iso_code`*:: +*`source.geo.region_iso_code`*:: + -- -type: keyword - -Region ISO code. - +type: alias -- @@ -6039,6 +5845,13 @@ Module for parsing system log files. +*`host.hostname`*:: ++ +-- +type: keyword + +-- + [float] == system fields @@ -6363,45 +6176,38 @@ Contains fields from the syslog system logs. -*`system.syslog.timestamp`*:: +*`@timestamp`*:: + -- -The timestamp as read from the syslog message. - +type: alias -- -*`system.syslog.hostname`*:: +*`host.hostname`*:: + -- -The hostname as read from the syslog message. - +type: alias -- -*`system.syslog.program`*:: +*`process.name`*:: + -- -The process name as read from the syslog message. - +type: alias -- -*`system.syslog.pid`*:: +*`process.pid`*:: + -- -The PID of the process that sent the syslog message. - +type: alias -- -*`system.syslog.message`*:: +*`message`*:: + -- -type: text - -The message in the log line. - +type: alias -- diff --git a/filebeat/docs/getting-started.asciidoc b/filebeat/docs/getting-started.asciidoc index c3b02b99bef..9633fd1fd66 100644 --- a/filebeat/docs/getting-started.asciidoc +++ b/filebeat/docs/getting-started.asciidoc @@ -75,6 +75,25 @@ tar xzvf filebeat-{version}-darwin-x86_64.tar.gz endif::[] +[[linux]] +*linux:* + +ifeval::["{release-state}"=="unreleased"] + +Version {version} of {beatname_uc} has not yet been released. + +endif::[] + +ifeval::["{release-state}"!="unreleased"] + +["source","sh",subs="attributes,callouts"] +------------------------------------------------ +curl -L -O https://artifacts.elastic.co/downloads/beats/filebeat/filebeat-{version}-linux-x86_64.tar.gz +tar xzvf filebeat-{version}-linux-x86_64.tar.gz +------------------------------------------------ + +endif::[] + [[docker]] *docker:* @@ -223,12 +242,12 @@ sudo service {beatname_lc} start docker run {dockerimage} ---------------------------------------------------------------------- -*mac:* +*mac and linux:* [source,shell] ---------------------------------------------------------------------- sudo chown root filebeat.yml <1> -sudo ./filebeat -e -c filebeat.yml +sudo ./filebeat -e ---------------------------------------------------------------------- <1> You'll be running Filebeat as root, so you need to change ownership of the configuration file, or run Filebeat with `--strict.perms=false` diff --git a/filebeat/docs/include/enable-modules-command.asciidoc b/filebeat/docs/include/enable-modules-command.asciidoc index 98fac1dd486..243aef3d921 100644 --- a/filebeat/docs/include/enable-modules-command.asciidoc +++ b/filebeat/docs/include/enable-modules-command.asciidoc @@ -13,6 +13,13 @@ ./{beatname_lc} modules enable {modulename} ---- +*linux:* + +["source","sh",subs="attributes"] +---- +./{beatname_lc} modules enable {modulename} +---- + *win:* ["source","sh",subs="attributes"] diff --git a/filebeat/docs/include/list-modules-command.asciidoc b/filebeat/docs/include/list-modules-command.asciidoc index c3bbcff38a0..ca4cf789e34 100644 --- a/filebeat/docs/include/list-modules-command.asciidoc +++ b/filebeat/docs/include/list-modules-command.asciidoc @@ -13,6 +13,13 @@ ./{beatname_lc} modules list ---- +*linux:* + +["source","sh",subs="attributes"] +---- +./{beatname_lc} modules list +---- + *win:* ["source","sh",subs="attributes"] diff --git a/filebeat/docs/include/run-command.asciidoc b/filebeat/docs/include/run-command.asciidoc index 80f2bfb5e4f..158a66ef150 100644 --- a/filebeat/docs/include/run-command.asciidoc +++ b/filebeat/docs/include/run-command.asciidoc @@ -13,6 +13,13 @@ service {beatname_lc} start ./{beatname_lc} -e ---- +*linux:* + +["source","sh",subs="attributes"] +---- +./{beatname_lc} -e +---- + *win:* ["source","sh",subs="attributes"] diff --git a/filebeat/docs/include/set-paths.asciidoc b/filebeat/docs/include/set-paths.asciidoc index 8f4b1eb5089..07f9ffa0b49 100644 --- a/filebeat/docs/include/set-paths.asciidoc +++ b/filebeat/docs/include/set-paths.asciidoc @@ -28,6 +28,13 @@ must include the module and fileset name. For example: ./{beatname_lc} -e -M "nginx.access.var.paths=[/usr/local/var/log/nginx/access.log*]" ---- +*linux:* + +["source","sh",subs="attributes"] +---- +./{beatname_lc} -e -M "nginx.access.var.paths=[/usr/local/var/log/nginx/access.log*]" +---- + *win:* ["source","sh",subs="attributes"] diff --git a/filebeat/docs/include/setup-command.asciidoc b/filebeat/docs/include/setup-command.asciidoc index 66f4ce361a5..14642e1aa82 100644 --- a/filebeat/docs/include/setup-command.asciidoc +++ b/filebeat/docs/include/setup-command.asciidoc @@ -13,6 +13,13 @@ ./{beatname_lc} setup -e ---- +*linux:* + +["source","sh",subs="attributes"] +---- +./{beatname_lc} setup -e +---- + *win:* ["source","sh",subs="attributes"] diff --git a/filebeat/docs/index.asciidoc b/filebeat/docs/index.asciidoc index ba4bcce89a8..d7bc6f260eb 100644 --- a/filebeat/docs/index.asciidoc +++ b/filebeat/docs/index.asciidoc @@ -15,6 +15,7 @@ include::{asciidoc-dir}/../../shared/attributes.asciidoc[] :deb_os: :rpm_os: :mac_os: +:linux_os: :docker_platform: :win_os: diff --git a/filebeat/include/fields.go b/filebeat/include/fields.go index fdd26077269..25be0e14169 100644 --- a/filebeat/include/fields.go +++ b/filebeat/include/fields.go @@ -31,5 +31,5 @@ func init() { // Asset returns asset data func Asset() string { - return "eJzsvXuT27aSKP5/PgVqUvWLfX4aeWbsOMlsnXvXO35k9vi1nvHJ3XVSEkRCEs5QAAOAIyt797vfQgMgARKkKIljZ2vHp+rEpkh0o9FoNPp5jG7I5hxlfPENQoqqjJyj13yB5jQjKOFMEaa+QSglMhE0V5Szc/S/vkEIoQvOFKZM6m/N6xllRI6/QWhOSZbK82/gvWPE8IoAiLF+bZxjtYRfEFKbnJxrDNZcpPaZIL8XVJD0HM1xJol9GkFA/7leEgN7LvgKrZc0WSK1NKigNZZIEJyO0fWSSoMVzAnQ1q/hmeRZoQjSOCHF4SEgWUJ4yQUin/Eq15SZPrrF4lHGF4/kRiqyGmd8MR035yl5IRIypnkwz/KfO03xCsZCl+8bc+QLRG4JU+VE0SMk9b/1ixGs+HwuiQpQyjhbHER3MybgI0jOhSKpIb5UWCiJsKohsiJS4gUJsFDks0OLLhgXZIJn/Jaco5MGbkoUPVCzjIv4vOIGIBCQ0BHPLHSAnVSC4NVQ3Kk3khkRrZeEAQqULRwPEqHRkCOUYIZmBH0nVcoL9R3iAv5OhPguRI+yvFBjjdeehIEBYGJ1bqo4aUEYEViRcN9QiWChzR65xVlBkMxJQueUpCWMORfw+1SDmCIOSCDK4KEBLkkCD+3avKQZmRGsNFHmNLoommwTRVdEKrzKuyd5yVCCJbHwFkQqlNOcAAfkWEhiNn45WsgRlm/kCFGFpOKCyHJk/Q4XdEEZztD0n8sRpuiBILkgeufpxXXDmyV3IwcC6aGhCK0GBxrXeXEjQWbihGZUbYbbt3ZARD4rgRO9Y0si5IJyQdUmjor7dTBU3ICOGQwcw4txFCS5JfqLSYZnJBvyGFkWK2y2KJ5lBDlA3Yty52g4QDU0csETIuU4F3whhhNYGgENwK2HHT5ylswzvJDbBosfuvCpgxBb6qVS+VgQmXMmyZhkOJfECIA2xmvB4IX51GzHGVFrAkL490KLBcxS5IDo/biiWUa1dOIslZ0YWSkxyQhb1FSZ7Thd2KPJfOzI8PP19fsKmxlP6wteSCImeGG0sQreQvDCSUSndZl/uQ+5J8Ean9Q/qj6bF1k20X8tf6mz1zcIfYuuySrnAosNmtPP+nBYEIV+GJ+gFMvljGORSrTm4oZaspTD04xIosYeBDM6zih2GGul7NxySIoV1spLdJQVT4us3zj21W+s3qtPnkrx/Rfzry5lN+GrFWfwnaUdwreYZrBZKUM4y+xRqvEItGEfc/39WLPlH5z12JPuTQQHiCoEIymabcyZlOvTWp88RidFnHnHup4S8g4aUTBG2aLGXUfVcXYUkDHFqlPVmHOxwip4r1SWnxWLQip09lQt0dnJ6dMROj07f/z9+fePx48fn/WbtDmjy5PUUFYfEYIkXKQ1fSWclNoqoZ6JGVXAvfpdpJZaDTG6mNZ1ciIM/bSo0P9QAjOJQX0JNtW4zpZ6wQM68tk/SOK2rvnHJCatO4QGsJ8WA1ZTKoQ5IABYDQMiBBetgqJNWuqPHFNbia3ZCqcp1e/iDFE255rLQcXicwNHjqMCyFuISl9tipHqeQtaFWqG1HbnkkRWG/fFxVV83764uCopFCIY0CsQrGbIZ94jIN456sGzMFCNhsC5Wn4hPOOFuSvBe4+SjOr/yCXNgb2WWJWjJYJonvYPSbfluGJckWDpzJ6T51r/1V+4BdLsK0ErNzeNEjZIXgQHtNG/x3DTffb+Dei+FI7qcnwzLSs7nDqP8/yRJOKWJmTsTV7ziBYKWsdPOZGIcYWSJWYLgui8HBIIApcKLSuXgheLJfq9IEUlySTK6A1Bf8PzGzxCH0hK5Ujfjazy471YjiqLZKmF5Gu+kArLJTJzQldE3BIx7nl2VoxxS4T0d/te3Pt3M4g76w39KyY0f0qx+XR8Mj45FsnZHeyjt55utwUNxxcNLJZcqk7FoBcmP9tRatg0oNH0MDgfGf29IIim+nY2p0QYgFRabn1A50gfrOQzlUo+bNCj3FvnsD/MfoLv17zIUn1UwO6h6ThGxR/xk/n3JydpY14kX5IVETibHDrDF26kQyYJujpNEdNbN8s2dsNKhBPBpVY6jD1nhGaFQlOzWjSdlju8a/bzpsCd4fIq4nSv6okVt6fbxa0eBo7q8oKh9S8rfo0ShAXRGhHcynmOMnJLMhBXkpQKnCBOr7PT1aOA/gaHnJa+cnfZEVGqUEyxQm3Klf6TL7Ek5+hxjLxHWqs6Pvn++Ozx9cmP5yffnz9+Mv7x+8f/cdSPc55jRR6FpgqjYFkzB6hUDVZ5aQ4TSxbDZua4gElFBwzUtJHWp4Ih9QkBX9DKlhmB/MESyV4q9alWqtuy9n5ECUQd+6ui6adfj3LB0wK0vF+PRujXI8Juz349+q0nVV9TCYZHCwR0thTsZniBCE6W/nHewBfsCU2MA/0xQPg/b8jm9NwY5E5HGuqZ/dfZf/VD+G9k88jY83JMRZ2Q+s+F0YndRHCaohXRx7d31CvuFgJdLUE0wrlvVSBGpCLhopspyTF6lmUGYbMTweyW6kPcUrBLJk9TntwQMQUVfXrzo5xaCraQN7Q+o5gFGlW77jTKIT+TLOPoFy6ytCdLNLYMcYjE7CP6TftzZOqXDHG1JEKvBqh50fHCBUs4S7AiLJQ5CKV0PidCb1BL/0pkKr0d54KQbIMkwSJZ6tvGGF3O0arIFM2zcCgLX5ozBhTNjUMj4asZ1TdWyhSHg6g5PbdAScaLNDwZLrxH/TTxl0auC5IZFdrYrGForRBSNhdYKlEkqjBTtStT6bvmRNAapvGlmMdbVO85ekOUoMnM3LlLfVmfKwy9uDgD3QlYdU5UsiTSaMFgfaUeeP3ayMMZrl0BjwTXCSrRCidLysz6VEiUA4qCSUADCbLiirj3ES+UpCnxYMWxw8hq+v6Q/mUAPh5Z/1nA0mbYaijgVgvev2NYACHhdj91c8FvaUpEbOsST6k+WH8283Lgxo4RfFFGkrMRWiRE31pqG29BFc54QjBrkVTWqmSszJ6VKJhQIY8Jlur4NDlsXs88YAgMTbQyIlFp+LZamBaUBVn0uys18e+H5gcAsBdulEmFWULGvdTtEkF6fHr2+Mn3T3/48acTPEtSMj/ph+qlhYcunzuGAUTdRt2C5eEXrBIB/5bVAwX3a8/LZkkpdTZekZQWq37ovXESYJPvgh1OEl7A1WMX3J4+ffrDDz/8+OOPP/30Uz/0rit5aCDqc4OLBWb0D6Pv0LQ8Xu29a1Odp8FY+kdFiQTzsDk9j/VhzBQi7JYKzlaxm7h/tDz75apEhKYj9IrzRUbMyYjefXiFLlOwjFjNAO68wVDV1TB25jp3dO3crT3ud/aWX/m3K6CU1tcbamNlErPO5KSBDjKGWXvHMKEVmmW8YWoXuiXJcpRwYRQAc/boq2LFHCUMac83ttECRN9ddj9y7IeH7dcPZhC0wgwvjB+ZygrP6P3aKL9NKTKMzaSKUvCNGyWQlVbgDpdT/pEKY5rDtYSt74OzgmbK0wbqWCi8OAyJimktCnjRhHX4XCsweqwmhL6Xvw4HwhYMLmF6jSuSQyAlUumLf3WMW1nwvPFDP2ngfec2p3lzRlBKFKaZ9ESAB16zBC6HyXFyQ9SjwA7ef3/SvEHS4FEXvd7r264gsnRYezi235S1BqWlnb0pocv3t0/0g8v3t0/dgERGzJ05F6qBrOdU3oLuey5UFNHYMX8YL795dtFJmgbElK8w7aMdRi7fXUYsj2cMiAjsBeENwKEfPIQRQHhFeMYTy8NcNDnA/KlzX3i+UkaYqjvU22nQOeXaPcSN7s3bh10wJTYTKvkk4ekg0C/MmOjy6h3SY0YBO5JFAC4In+Sc1tSkTpCvOVtQVaQE7qcZVvCPKGBzCxmM1PbOYQR2jMD6fjYUsAt9/2oFZWc25FLa2dVWsjoOvCt/eRJ4z/oeAnCxr+uDirvbc93F3FQOA0zaFEIT8xcohaBCWT8NRnMqyBpn2QgxotZc3NhxR4ioZPdz5W5kaDDROzrCwGfbAHI3nr02aLeEpYFZJGqJ7ZT8wFZmnGDhI7AGcOOW8GCsJhBJBMXZhBWrGWnOax9QZkRkRmwCdFFBNrB7LEmTH/vrDtcuxsiGdPuXcsqqwDhU13neAnr6ffuOMbzSW6K3+MfrC7BKQqiSGZlKdHxyev74JLD/6T/GDbGmWaY37PH3T05Oohcf+KVJj4P94xB25FkkDO9WFlcQJzWzcH0AASZMpoUbSckcDN+Z9Qm58SA0DF3xFXFzArkYDDUlLIVTcjpCUye59N9pKuE/OfwnF/zzZhqlkvuoqecH8UE2hMZ71DvepbpyJxC0asOgbVwQ6PBsg24oS8foo4nKXoEOZV8IIl6WOM8JmPYyYkzQmtDWZwI73Po71kDkyrtIlSTZ3PMBMzN+sD47XBcGDzlwUfFNrHb2TG0Nkop7jip1MB0kFEuP425yLui9PruS2W4bwVUvbvcJrjKrHTMrQUTuZ9WmPMDWBSbZ4/I4DDdcPtfCsLz7NqK6UGfUSORSVK4oVmTBxebAVQXSurHaAkSsPw+bMEQn3MKvalNZgTNKxrnxcIH9zIjrBb0lzPj5qAR5UwZuWFeB7xHVHANL33QXlFMFEW5jYdxEbcCtnnx0rmxB2edjqbCSx53zroWQ7n1UmXFQgnNViApBw1jBYWbfhJP1FsK1BV8F49ngYcXd32YFnNQZvSHZBszcLMngBgZjSQ1NkqSA/A3rvJOjcEwbjTfLeHIDDj2Bfi+wwPrGStnin/SPa5Jl+r8rLogJEqFJCUOPEAyJId+QMnsujEwWGn3EbWDg541e3jUWaXV4xM9pq2zss9CClAa5phz3Y9H3XFv/em/GM4zdVwfR/OtJwvALb1Qbm0KZjWvjogycjG/mjfw9i0/bxecPNm87YMvaJZwlJAedCqOpfXeKHtiMAPTICR6iHur5h/PE0rMtGkadWZXXEmaMLlXocfcJakSKJmshBGEq24SjmQgWyiokTLgtZqn3yK4sxFxX2RBRwoNMiRPe5Slt0/w7Q1p+6BnIcmWBlQeZvYK7x0FSD/pF39JhLaN+sfIr6zFfEcxATt8S4fnSysSdMuBFL853EhU5UjwY0fgQ8oysCFNEaKG1wjcEyUKUSFLiAv6YpBJScWzQX2ccmcv868HgEUp/iz5q9lEFwwqkqd6ilvxGAikkl3zNjNcqUdkGbYjSjPp/UcpNgBwXN8GQlCGFZ3oXaxEa/HQp0f/37enZk39yRpJSNS+N6/8Xgu24uNGIwF4CRapSsIMBjcGGJjcyyp9HVyRHpz+hkx/Pz56en56YW+PFi5fnJwaPK3tQmH8Fi6aXTRCswPFFhHnjdGw/PD05iX6z5mKlT4eESDkvtPCWiuc5Sd1n5r9SJH89PRnr/53WRkil+uvZ+HR8Nj6Tufrr6dnjs567AKEPeA2KeRl2pbUNpqgoef+jtXClZMWZVAIrE9hFmSILk1eI6oIN1fJt9apTlpLPxITlpDyZeNElKZV6+VMjqzDTr89IbUQTu0VSE7hLy/wWocUQuXW5n9OJMaMFF0mAHeYNAmVKNPzfGjtmieVyv91SsVUVfBH727N/uXjee8l+xnKJHuRELHEOOoTJD5hTtiAiF5Sph3oVBV7bBVAcdN2ZPnx5nXd6ruru9qfWQOAtqqCFEIsndD9h5m5QXEBiDE71PpdI8TYtwowml86Eau21EJ2ZY+NrqkJaS3lLFcq5lHRWCxKE/aBIAm+aQ1Tj0UBwRvThFdPbzO5yH1AJEW1BVDCcsYVUJhAxSMmDg+ObcB3dMdbEprIvbKETcWoA8vA6GZ+O47Yr+KVFiSpE3WeyqxXvuR0iOIo1FRhmPG7DK2+SJuOoAbwWqt4B3KyOy1yqByxGo8Lty20MWOUAavWXSkVZYupboH/2fmPGI+A9csAb+oFNHrJZwfDy2AXoAqqSILXm1a/ltTeuxeB6QQFaS/rXDFmbOLXlOuZVbZFgzNmmKougJT0cBGBOSnA2DosPAK/7mWb1QgmO/Rq5/g7DUW3dmnULqB+S7zO+1FqtcbDgPDfXxBwnN/pINLdSfesw9rrI4jTsv9UrEXydz8YB0ISNY95kyi285leNqC2+pn9J+5E/i0osau2oLSaSypuJTLhoXgnnGcc9TXsfqLxBMIq55oZVE8wMH5DxYuzdyHlWwB36YbhsHyVBG14Ie83/TlZFLcyFWC/W1slM9J35kBm9hTs3/YOkMOqWyY1M8LJMMJScQSea0U6dcyBqvVlhyrKNXpp5kSE615OGKwTYGdQSM4jScGYPLT6wlHRRExkVchLyVmCYNTaHnSQEYWs+gKkYCnpJRDY/MWIV1Xc+C6lmAbU20pfVC61h7mX+b+lJDYNq4Gz26xn1rRrg1WiKGqIDjN63Fk+CAJhJa9wcXh9mL2gALldfrwo7xgxnmz9K1cB5jQ1PBCNBLtFiIcgCTs/wiKxyicSCqMlOtLmGb0x9KQ1EblYZZf41Kk6jNirt7ekfjlY9qUU+K8JkPVW+iXkr1sDe5SiNrQ7oWxmMs4yvEcFyo+emCBw7s40xDpZDeEQvtbHcKlb1pfYt0z3wBlzB2AomqBFKqYCIXLveD6Mkqkc1bIfz3Dkk2+Ifqv1Xg0WZ7/rpAepSf4AapY2MvZWVf7eF0GIgC893suPaX1vzK7p8jh58vHz+EGjpzjbPtfbgCn6sJo/4mhERxQd+2XlV4avvTOmFykBXG3qx21TfC7rCYmMEMczxVW0acShByNrOcPyojFYYq+1sUl1lnj45iQN+o3nHXxXKEE8UzmqWqCgKkv5RRyG4ADXXSH+hQcw2iki9Ba0FhWsVAKep0w1tWTIanvFTjeE0vkVXQWR35EIUIPMaS2WqDfn1E0H5XPEUSqVFoSSHQFkRhcEzYHK204iyUcU/WuXiVfmgn/v1FeG+pz/BQmz8JDRche+XsZJe+p272ZfjcaFxCozqcKgwqHNoiqH5xOjjqW0Nszw00asKsGyCbI2u3Cs8vB5XWYcXCapsD6nsylFuCaesw4vHUu6T3eBHUTaoGAmh3Id8VfAkqm+AJZe1EISfqyf9toD+oK5t+/zrszvAG6Nnxg7u3OblUPlyI/V10iU7jRBGt1Sown+ktwN6Dhke9TSQcqC3znPpRWoFfr9aCmyZ9snqlR/LIYNU/UcJzzKSKGc/9rN6wSVQ2kSyjb5jMUJSssfW/R8XydZl9a6C2xp0OnyTAGO62j+OKvUUwZiFxLCxMzSttQI6dd9ObVEyyDH+yOhnd++1CcFFVvOQ/l7gDE5DG7IPE7MsD8iUlUUDX7yxOREWpvfq+SY0LY24hvSK629aad4gba84n91SE2zoj+G7mNnpmazIb/09OFvjjbQpfCMwWFiXjzFRCAJ+UsoW9WsZZcau0yun8DywWxfOhzWFWjawpJFcq/1jkEF20twFIrennh7G3D/bBNItcAaIE7VhNS2b5SUXNjfTpYfbOilWdAYp8HooqHM1LVNop6HJ7nKOblcjlxBobY5BltzINyV7maDeaRCMWLFQO9uYP/FN8y16V1YdvDIWtBioqgblOM+wmsdshjvR/V291qEbFj1ICFNcjlAxK5gqRmhNWcrX0oT2P4zJ2RSLtU1IimHcU9ZWzso3OEHvrtD/6emSbMylcbkM0JnjFc36RPlVCKVkRjHri84VMiDQA0HSJVYjZL4fQRmQmUyjNI2h2t/b6Xl6T8anZ+On+9IuCMpv4IRFsqSKQLmPnbD6/OPTydMn+yLlg43ppErlNZ30+vr9Tjpps9CJLSQLJW5lUON2jwpWdpzxiqglPzAO9mel8rL2rhkw6h599eJ6hN6/u9L///E6gpItwisVVoWM37r6q4oWK1t314xZu3t5uD05edKO0Iynze3ZP3r72ipKwBb1UsARXEwVojUXWbO43CDpLkCaRrKLh8Hp+LTJ1KaRBEJ+M4lePFyVHiotCYp7VZN2514o9XYYDV7zhRmm7CRQVSdH9VO/kc6Bpr88+/B2OkLTFx8+6P9cvn35Lp6q8eLDh6YkPSjkrD02K+MJzkApfbPRE/LF204hP63kqzF2VSCudDV6Na5ASIUFwvU28N4IhpuROQcmyagCYUsVKsDrXmZb51hEg34vzf1FgPnMXIinFsTUuj2qYHF308HM80XrkYMhPbawI1k9LRKH4yY/akxwHLtqLfEtQTgTBKcbJDVvGROisQBJcLhTyC26IYiwhKc2wpqR0GEEbVmg8NOtLQeWEcwgfHJrtbG9AtKQ5DbS7LtGRNrvBRFwrbO5Geay1isoLZAzNhgglDVvg4f7HqFlbihWeHepE1Ub+x8DYHg06Qyzja3tDZlS3DXOcGmcVDhM4+coHLS/0Dn1fm3zNbZ7G7v8jVs8jodMpkHWXHDFE36gPH/rQkjsaKg14tpTzjx/HRVkgNSN524YJz4cxymB53OaRPbhB5Lw1Yqw1AUZwI47r1H8L4iyGS9YfZn+gnih4j8U7IbxNYuRwB+rQQqbZEHSyaFmAS8/uYw8kmXPI/eTPUAgwyOujfx0Nj4dn47PQny/teXwZGMGdnpj8BkdoEI6nrLjGR9UHMUfm+qjw8JUOBkSDztiHJNmcWnHIYPRww24I0FKPIajSInJjiRRXOFsMHrAaJYYxpBZrEwZK4/u6P+vLUQU18dPf2xB9g6JFsPZ/uZj3cSgRPvsSfMc92uqhYf5u+Yv/VNFg1Jt1mlDmNDKHXgt11QtW7JFE77KMdtoTQoqt1WXOj8NHEvJE2qiDqlaxgqQbXiBsBBQ+N4k+SgizABVhhBmRqOCAzKsGlTC9Sezxz3oQI3EX4cuG9XdpU378x+H3CNrPFOzSu7MN++u6s0b4kxSb7oy9kcJK4vzuTLJS3q9odiqsc3mgszpZyJHZZok+FPGXI7/MtV8MK16/piHuy/9nVtdAfUW0+vDeM26yuq6lUm/jLXVR+MLWlndqm+ztj48pJxJw8B6LJK+aU5tRlZIn4REGalEmULt43dDBOtleqnQezJ+Mj45Pj09O7YpwPsiaWB34xrIEJsQEAqS98HDfephtIoPXDZva/lS3/3d+VEVsbR5o2Eeqj7FyvEQTR8F28hWbvZv+EbKTcv2cTSdWgElFd5IF9hngLnCGvqq74VMJTynVUjBIuMznHkl+R3KdXN8f6mFRa+a/V2BwZYiWCyKVUsK+Bu8QTNij+WyHBVkJ0nCJAW3f7SqkMe3n46Os6MROtKiWv/X5Ro+PfptXxHXY1qRUxhZAySkJ6AEZxlJXfs+G/gnkKQrmuF4Trv0svXKrRE503coRliyZQiwA94wAHMMXu2Gy72KNlGHZug7UDBUS1aY3mTw+8huMeUyZrAs92xLvFJYbd0KpavgYX+lxlVWrxfgVP5vUN+43hTW6MrY3/s2HqhN4Z1TllqLrpNckFgF0X2lab8cz4HXX8R8eF+zao81zrhi9K7VVWyxTfMcG4xuYjeyTVUXGizCXqssSE+5IbIrUbJGP690gFkr5jlK2lErwz0u5/Y+QhD5nBNBCUvAei4lNH7QJwmHLskpVI8wxcNH+qNgQH062ZsMt1l3NHW5MA5BCCp0qw7vSMoWEAVs65vXMa3Uw8c/kO/JbE5OMHmaPPnph7N0Rn6an5z+8ASfPn38w2z249mTH+ZPvW+743p6St1ODwrJsFQ0MbnUPRUTP4LUcXlVv8Puoo4yYkZo1xp5mDjuyPYK2EPv4bBhAOrJIjCWKdNtFhIKJfjIujZsUzegif9yzbCCkafATNPDonB2C7myIhJGa4ErVZjPOgzgCxtKBaPX1v0QBb6TLx+Pz8Z9oxNqTegcS/pSvg9fUmmSbaTxzvIbhLVKa6waRJmI+1DYl7p4UNIZ1ZnSp88X6o7miDB4fzQ3sf4d0sLTH8zftcPff9YyYfNOj0LbYc6Q34O5f4qgZ+QHF/k5CrJcG56A1kVqFig16AU1cPcrrB0tq92ObXuWiY+vX2S7hmkskrEdXO9sqEid2BbAtSLbA8C2PFUrrR0rrN1knNai2vWS2m427vevmOIRgXm3OR4NgHed5NEAeDdZHk1CDp7m0TaTYZaquzZ2IbJQQH/88LpbOn/88LqeP4LB25ARRfSvI6OGy0QfWSPbBQx6T2PrYfCAuC4QVeyEq3HWbV4uRDb+y1TvunIgexqN0d8IMUEhVXM0r0zWekkYuSWizKSvJrTnnW0pyLyxRv09Ey+LLNPrYEhTRqn0aSA41Z9p8FOTAf0JThQzxm8Plkrl8vzRo/V6Pba6/zjhjxYFTckjwh4FQwWXg0eCQD5MQh49HZ+FL5rOP5ZgS7XKvp348RgTvfgTd7JNbD62kA/N9OzdIdSf6jP156UZRxGp4vMeu3zvae0mTxiUPNJrrLi+/CIMQTsbhBdY399ag6AKkSGpaJbZsmJViJYNNdL8ou+LWnEyCYyxlalWhaFaUro0JsccC8PqlSXUpVglprZLeJm2zaWn4bz1VjHRSE3r4BeOkyljPz9+eH1IXn5bZr5lVD+2RbN3xdrnT548fmQ4+H///teAo79VvBkIY0TUYeL1CsYorSwmMriSVkeA5VEsSws6MIId+3zqwtJcNSqQXjBy+9SbcuhOCt83p1QR/ChQbiE0EUL8TP09DFtlhTcIxInNoNV6MksfcQHqrA1Gyjbm1ADPQjCkl1k1Nm3hIQFFEpOU5YfdwOV9wcugyCqvK0jFDShZzaVBzmgTGyiRFuRtdZlXPRW7QcYnTx7Ho7OfPG6i4tfq2P2EgaIZrctpd8zR+OtJDs0nRjt4Nqi0cMiC5D+AgHqXmtPDIBTWDTW/GM9cnczhQedIXhNOMfEAguF/g2Agn6FisVdDyocIyZxmq0WrhTGux4HdUtb09+bickHNbxhg6su/e2tUO4RCQhhTg/XgMURWuarwgimYN6bBKGaEmlGwzMGlWJGyWqorZWUqpn5dDjVoaxF9V3w6F3ixCkuz7ePV4cIPy9QKDZ5DIVm9IN9Ovb2veN7KfN9GTyWHYhN5V1nkMOQ/2lFqG6kJLsdS1obdq/aSGSUK7pv69GpXJbljU8myHEzdtBWPzoFXHU8JkpFb7LGG4sivUvzSc7vjW2NiInBH9w1N+gmF0sO+ERMALV3x8rKoGE1H1RWPQRDYxuJjaqib4mC8uvyoZRVD9OV8Xu9q1rSi7gMrzU1hKfThPNq+EaaC0dhS5d0Ou+HNxRjy603dW6T4DWH0DxLpVUlWmO6ZRrNlw5mhw3xjNEgR3O2uSsd8y9Bd2KipYl6EWEPONisoVKdfidD6Y1ktD4LPwH7tItGsp8cFtiSczQ2j1Jt21aLMy8rE9TKJvnwwYW5NKYH857vJCjOkkxiV4V6r2TYyZib4WgNxskt/uzHO+nI4ueRrm2C0JrPSZQCesnpVfXsxLUrEaxFS/Xd2a+5Xf9XrI7Po3IaeHy+qsAG2VpDs4C1dFjppbwI2QKZizbW1FegK/yPSeKx/nMkb/X2MrKiFrCvKDgOov98FYI5V0kfudF99kuUuMHeN4LxYCr7qWVi4fky04dA/bb8nsPY4373y3fszcS/Ad8LI/SDfBUc3IX+D0LfomqxyLqBpDf0MURBEoR/GJyjFcjnjWKQSLI5W0H5rA2wKqdCCu4hGksjxZgWNZsA+vqaSgHlUopSz70wrhDC4vKyFEkhvnNEyHkrfvM8tL0I4Q/P7mmmpe4zy5W+ONfOcmxb031Qd2YOaImXht2+ipH7jysLB8ZTUypGYoni2uA5O0wm8MClrydn4NNPQy51SwfT0q2P4auyGtVOqbETJlrM68MEFGI7RexsY5eW76QFHaJGYEicpXVCFM54QzMatuLmQoyqSoAWXS/siunweVIyyhVp6QPDWeRsMVqu4tB2KfWHihc2UdC4Lx3RDf+OXnNkJOL7FNMMzmlG1mfxRxRaVGBTymGCpjk+TbhSeeQMhqNFFq1JkVNpaSdIF3bVjlAsOPdDLVa0quppfjj/3Zz37icblFeeLjJid1g7d+Bm7AVj34Zb52Y1edvJ3XXbdvyOD25pw0BOoHp1lftN7Vi65UBPQ0hdV0j5myZILB++43OXfhKpoFU5g0UA7V2I05eoGispMqv78keudB24Va6G49+kPw4WVliBOalbQTKFYD9gKlQEc3SXMeK5KBcv0829AMw5i77F5MNkDl0ughIFTMq0tqhgWVIwMcsnm3GdUm9YVip6KN/XzrZzpF3Tsf6uS496pKVuM2nVd5TtZyzwpqXRTzPQPJonU0upv/rMIpOr3qrprcGJXgyKfUt2bvvpoK3kDpHcjcs7TAZjfo0DOU3O5iIIqDhUxHqT3PEUfL583Aen/lzk+9GrsgapGbALjKRmWglDJO07CvqKjHyAzGlrhvAkJzEDGfj8UOG/IOMwhxbEHNwkkcxfYAQ6kKFwzrpUwOMfJkpxV4uXomXlyFJcu9lf0xrWqDMWGbTwVEwsVJBSVCW1anwXoBwt1pH8lXupdHM4Wkl24sjvW+OdOHIfHz9fX759bOGDJ8/1jnZ4xsuKKhCUvupZ1C56Aa0YJ86tgjKOQwWpXY+IDIZdOFbiMgDcRCnDMsKQJwoVamq43ytYSrsy3deQa9eG2YeZXe9sdaVdFDmrdNUvJxWL0BiJXAPnjh9dxsEul8knTKDQAfIAbqcnWLEdXj39EbUaaXSCXVenqFfI8owdPNxNJmGrUzejEwLgPzlHsox7YsbJws6mi4SU5gHm/pZieT7Y5EaJWVniY5XJDxwH7/o0Qas3S3wPkhV9trGDHre4I9I5lG+Sa9dPKrtoY0nz2wo+5QHlWLCgrCxabG71pZKsftIuJhqMnnHBdwO86Yzvdym3k4jQGmm01U8zSyDTjRwfqcq+EBIjzWQ8yID9lbJsXpI5U3Ygc4hTZqzsgtOrrKWkgVbMvD4pUX29KHam6VyVE6uDVy/t6XOp4RdSCPmi1e2J2RJxt8c7U8eV1Cd8H2z1wafdkNDGafPFtsBt2X3o/7ITdngx4t0va9KCHeEXOV9ReAXUHnN+5IqfewWvKXNrapfqgsg3KTCz8sX7z2Lxpzpv4AbogvOXqccjZ+Yrwy/dBCMcCQ1xjqq8CJEW2GW55+WlGBduJR85ZM3ivI7Ux3j5HbGuGWEivQZky0iCoiVZrEtmAiLU3EKrjE8kxC/GI5ZrtSKJsS5OhOk7xNLQByWNucJFGRHVMYplqA+ORhM2KOuhxtyzTSHKrY0GEiHj6dzDG2PjKMkjKGGHMuDsZX+r1vLdRoMcFTZJbAh0FgyLf0arMJW+AvWZYNJoZ0AaKiRktuzGZQAZNtzYzTLNc9v63SbOFFwuSdhMkrNmDDrQzuLo4l8/j0NSg0NQS+nC3AVs5A2kT3t5rbcbU80yLxDXVrNPZGXaLlKrUt+vCgxazrjHngrHTaRhmgHKX9bfzOsBoFzNvfafXoKOuWv18MaSMeU1Z8dnA10OP0VtorZT5dRlTnkBtMJJCy2s0Iwkuajof9DuDlzcMr2hi6hRhsdG6mxneNN+u6o3E54kCuZ5wkU5q9U96sk8XUE/5zdIJLhpbZcv4tgs+tc0abc8iV7o/Sy3wy+fGWOys6qDmQttupHhjUBgDRo2jysh6aFQZWZeojj2qXT6vpc40kRU4IWheQOqzG5lXs9SPrGZLhY2TV1UxkAcZvWme0zNi44QF5+ph+4LJXS2fW9dLEgl3uuFXbFhc9YJVuI5dBwQvx0lRUktvRC79QvHags02/mDRKUjye0FYwxR3yFHib0w3vLVLt1h+k2SPE9ncKRO4T5hLSK0SsV/RJQa2eVz3UVCee6FSSauTKr/Lwakiq4NcAzCArbncQSBa1Q3eAYz+yqWpsJQmWJX9COEnXpSJYKYQeA2v5jUAoljtW1SiP4jgx3Af/yeErT2Bz9EJ9HKDSnJ2M82pkAoGbeG7k91nZ8Z01TSdSLQVVxOcZa3OqN1hCSKLrMrlrmBAyVxQDrlAc0yzQpAWcfp1DSVTo/iMteah9fppY8gOx8S9weRLXcEDjKjafGXLBAtyNxcNDy+6NyftY076wuYTe3MLstS9C1yjkmQERvBOFaQTu6fVwaD+17XWbgGR6LNggCM/Qla/7RcgrGoT3v7y9l/lfzw+alzr6vQuz12Wks/dkC+hU4J+JQ5zTjMyI1gdKyLVMWV5oXaFT1ujsix0msZh43evFs/Xs48f5hd///6HZ1fJ77OLxbo/eLnEIu0EX6Zaw6txLE76A4RDav9Ld6elDm8a3vVwMrCh9VthxWEqrQZtvTeKpKBFqJG+mjEJhU+4QDSfmCK0RzUoFSX0V/Vf2zd8uaE09K1Xc0DfZVHYu/gSK8QT6KicnttUW17IiYkym6SEUZKOamFVE63GwOPaW+afC4EZ9F5POGOmZVT0mftM4VWu1ZFJWatEFGyCvYHsv80H7cQL4e9ORrN82+n4C1hevPIBjYVHD5q/uPTdDy+urtGz95fu44c+l5TfraHHQkLobaWhVa/pqzsj2cORaYs4gVDZB8Yml2g1Xf8biiSnPp4P22lXjbM33awxeCsLenbjWuHuJtHaEYbOWU9/HJ+On5zFUa7p0uV1T1CW0LzhY20iWr6JHrhiFw/NljEboLYt2nGdlBtrd+Lieuu2OK6+HmY+MZhqPiKfSVJ0EjPJCqmIOF9xRhUXj1aYNqazHdVC0K14AvcTloJahT5+uGxF6tHkc46Tm0eSJIWgavNo4pG7v3m7UqyAt3oLSMeLO1DxIiNYXCWCZ9kH8/XuNLRgJ7W+vHFc9UuNQk90bqvhdWCqP4zjFnhcqgiwXJC28p77Hr3lrbfZPn8HG/qrC6T1J0lULU47BtIHC8UrhrpsW0u+3wP41YUBsevN9u6ua74G/OrCpTZpSRFF1Fv+QphzX5KkFbV5xvGe96SLGiYlQDAZCugQa403/4pvMbqlQhU487Ow4ojLRBSzidysZjybKL0nJoquyF3NA73HhSRQIhJRhiRJOEulaVxrC0oaXBDgErGe1RCHuNcvgHgPvE21s214rwm+mQgylxNrFAX87xDza42zzCEEqYQIaKCypqn0JtUVJilwlpFsIohMMPtSWHv0XmGoBoAyektsbhEYYzNiuiZXOQ1S8TxvGs18dz+WclKwjOP0S83EQAN+YeAAASR6Uj/JC8Cz3RQTEco9cXRdby7efzQ8bvmFiDkX4OKqRGEExXaRjepR4nEio62E7jkR/ac2CV4oaM4H2ZWQyRmbgCdYNvIrYGk7CHpIok4sBcHZl0DzGnwaJMO55tca0lCeypa5Nubf8pSCawu0TgA/HmVULuMm/X/criaiYC1bsH0ifaJAqCsJ9a9/f2OxMRWf7G4bmTZyMLzmcqNydzn3TGCJnICvZ6KlTJvw2BvzV1jM8CKgpoVqPUwaql2GmNAoGVmLQDhdHM5Dk1ijoDi/gTZBgJTFsxMvhRfx9KG9Qm9eXUCQjTl6Fy0glwQP5jX6meAcWqUkZVk0ty70j511Wf3N5GbWKtSb1XV7oonKzasnD3A049/QjDf6D4co6ZPpzlD6KCEsB+cdyPixEwsSz7TbY+HeZakLuYN4+CQpcsySzZ9/BWHx+BxCP7wZ/AmWs5Wm21d3wwu2GHJ9/10P+N98hTf1OfwJ1riDrnHsqmAccdteGPzoyiRxZnwB9ommg6POA811qtymq5yzevhuCO41X1TvhZadyurDx2ScjFfjN0Th51jhC0GwIuAgsh3Iwi/bDq6o5aaOkTm6YgM2ub/LTgNM07VXjswSvrpoN3fFTV2xXRjfLaXMZs0LSohLHVIXFh2RW6U2sW4Gug0OsFrOCb8lYklw2rGubcwVW+kAULlxMr4OA2drO8f87uLiQMN90dbKsIL/6ezk9Mfjk6fHZz9dn56cnzw9P30y+unx498+Xb59+Q799sl4Ss0QY4vEGGp8/4Y+3U7+/q/Lf/z9N/TJNBEEf+zT8ePxybEed3zydHz29LdPJ7+BSvjpyfj7lfxtBP+YrGiWUfnpCfxbK85LquSn05+ePP5eP9rkRH76bWQqX8FfAAVwM336t48vPvz75PrnF28nL19cX/xcjgHeUvnpVL8PbaY+/eevR4Dtr0fn//nr0QqrZDnBWWb+OeNcql+Pzk/HJ//1X//12+gQeQNh3aJb2CxsAYY2bogSe05UuHrbRYwmcAcmoKRTVerp1kZf1a1vw+/xyclKxlCpZRyUeOhV7EJE/77L1mifMvBJB6grhRWF3bALvJZ5ebzYBdIEdei32mDWGXnHOQOLT+p9G2KioXtdd9gkO1CJfFYCT4L+hjH0XujXXJdCL+BugHXyBM227QB7gTJk3jZ31RYMnpztuBmddOvCwVzLqBoUqBGHW8GazkipiTVpQ+BsNwQELxStndAh7A/mjbZllienP//H2b/9y81P/1g/WagFfqnYbtuDdhzIl+kgUmeLBLju2PopT7pgubJ7OBf888aLKrNPWuLJ7K+NSLKwCns1Kto9iKyhmqREKsrqfs5gjOfVK/4WP+C4rfUdqsHzewp56PmVBH2woHHFOCjv4KD3rQAaFLJOlnpQaTBerUYplMVqDtQeM/a+EVrfoG44zdYkx3Ca0aYbNmUx5S6prRknsOuCbqF1PGfSBt0zqmiZNHl98d6Ll9L6jWX3cWyJu/lIj5V7vNQBtgI53sphDvhccKYIS3szhvsAPeACZdCrkYiHFp8y8AnaSBoeiCDXwGKGk5tdkLDvR3FYY4kkscU/FUcrzLyyqt6aVCWbIhiZH3ojpK85rv6T4l5klAfSIBZt+mrOyjWmqvS7BsknIUfAiemcvr62YB0gdhxXi/wWC8oLqc/YgvTeklW4nx5uMJxcTli4FEQqPMuo6S1iskoYzpqmmi6MwdwzEeElOIJkvXgXBOetqLLsEuwx29hK32qpRMS8Ne6JEJAMiH43K7kTHm6HWbp/jXUc2XZEkJtIVNUJc/sU7G7sQcxqde154NKi1kQQTyTZ2igQA2zrgnsl3ABQX+Qcae8eOwvpO4kWGZ8ZtXkHPOkOEtZIVUjqdm0bQwm/VaZD5YBJs0hAAPIFlGWwL7m89NkG/fzsPSiRlEFHOQh8rlciiN67mqxTv/xFPwvXaklKBMLW9b75qwGpngW2c0jczplfRk6N4yVyhsz2qmd6ddl9OzO8ertUW9qlxwF2Zif1Btmdf7Ql96g77yiA84HYVZEI6y+Me79Mpu2TeLQ9U2wvQgd9BmpEbkmQ60/deAWafulTvaG4tavNqHkmErGyF6aJVGFYco0pWEpttAMpE721KgedFkj4nLCUpNGm/k5QtmpQ5eZ3hzywNdw2vK8hRPy2PBAiGcx9Lz1hoPptZ+JE/fBKOIPME6YC3HiAVJM6mmpGxNd9hvE7cPOWMhyS5TVmQCzt8TwAkkvM0qwqv++uOwPi2lCt90XVtGC3bMkDLWpAdK2+2CnIfEydTmq/Q+RzTgQlLHFEpbLCCtAUGxutbL+u3xBb8a2jugrF13Z1Q39gshUCxtSqX1l+8MH1xXvEBZT4fdgAuVSqVcV4nxGtQOE09Z/3FRioqe4E+fZQ6LjdeGHD602ryPJ6FRR3jlgqdvMza/FdyNipEaACQM275oQQRBWC1S99PRY6fj7iXBWC6AsWv6E7Vtp5B7/gDB3pwf4K1RaObFdYW97BWLlwYPta4tS2WQeY7lBwlB33xHhJcErEjqUTXlMJJin7cTlaHQmUFsRR2Kg9lTZ9ZD+qXjajHcE6kZUNs/Z3RXx5gt0VZdRmClF/Pm1+uw+bflEGsQJ4CbmIls2xYxPQNKmSzZbNX4VPTLZWPzYx7w7MJR6f4LXLZZ1ktBEzUtPXbIixzyVIf+XbOUx9P7Xk6ah8R6v1fmX9WtfWDqQ7UAcjjzmFJ911a4yv0SO9ICtMgUNK3dKaq0fOzCy9SjJR45ALxfbbwZsqLrONqvcID3NkIQrOmVDd26VEqIymMepsl8tAF7BgarbnhZqkWOFtJNrVClZZfqU+ujGaF1kWnnMjqFYPSr3+UmNx+JwGnEaIs1mnssCdPXAeBFOY8XTz0OtY7623v8C7zLKcYdKqz2hFCILzwDayx8Wn5hGLG9xbQ7z6Eji2IXCSkFyFDJ9kPNCBWuzvfwosrUOYJpQtsOcPvoQHLe5g82N3XZFyxPg67lRQJCWz4qDijm3dfOxEYPydCsnOcQINKPe1aTQP1SuTrgolv7Cq4ke15LRoWlPq1vqyriTtcMjFGqkeAdWORuiIcUUTov/mx9mM0NEaC0bZ4ghFCs8fJYIqmuDs6GtXoi0hYnpARvVWJtPD3/PY/3Aeg8ywYhiPQpzNLIR7TvsfxmnuIKfSP8Uvr/pXer68vCpTJIB1osc6bW/n2YK1X1m5AQN98S5+GoU9+vZZd/WQffuuq1vGtt599+3xArAQVmxLMtwNfIBgb/FQxgWzlr5vjVCvjlSSHRCA+LCu/OA/dTvHO+hyeV3dWbftlq/Wiu9rt06UUKgEq6J318S+wGUx8wztcehryh6fDQ//F8pSvpZoK3xn5otdqofZlLELd8tKUEXuYHfqYc3u1Hd5yqTC24qEx6MeB8CFeVEG9hgDz5sLk3TnvLVpr7Gs+pC0JM9/xa6nURP+oeIKTNdVmK+x1UtTnbI0pnUfL0su1fBrB43zjc0d4HTj8N+0IyugXZos/0yoG2nSjvl9S9dD0vNjLV2L+5au6r6l631L161o3bd09TC6b+l639IV/ty3dL1v6bojU963dI2S6L6l631L1w7L/O49Xb+2qRGgD2wEtsC32oC/rlPCQh947hb41rl/TWPRvTsmAPu1zd6CYMnZJF+KtjLyhxr99fjIjN/qkSruwuAL3kqv4HTOedaRcnWvC97rgve64L0uOCAubf3pbvD8xo8Y/Zv+d0u0CfxW9UKPBZa44dDh4aIHdgI3yGZ8AZH/vfVQRVdEKrzaUci66uHwaRWd7cC3JCyTW1I/6asaUL88+/C2Xm6yX0SRGfhrB8uhQCzG6q0edKxelMFoXh0Q219b078FkQw32mntO3lonAID7oQCtBYf6nBH6Bo6lVPWwW89TtMIWdAwgqdGJdNYvYtOaCu3om12vh5oIfTGlrbIcVXCCbBrR2deZDvbHHvhAq2Viyxz5KmvphPWdIaZL63NgxZxbX7sju8vR0T/bQX2oN0B/mZotr1DQL00wYFwL2z+NgyrudEg0npvrbfAN6BNh5vaT+bhJFpBMOMLqbD0G9O6Ry1M5X7uZitvXDQ4Y1lEX3uIhmTYgen8qFa95dygO9muhj1TWzz9emPEAHUpEwfeWktVwolHC3/kcpuFudVDWu5rvnjyD/N6W/Sr45gBUTRjIi7sEbMu+4jW2sd2dWcZaOHi1RBFwZjJN9WgPAQ1dbegl/HFBObRf7dvwfGGmOYGxmcF0fILU8asxD2SdFwKvUad8Z03XHOI+511v7O++M5q31W7Y/cBr1FarPLSj+18xE0gZbQJWMYGNjQGRWIBQBds1WzJfAjH2BavFexzdMnyQskRegkNuuUIvSuUfqJ56oKnJGnr98T5zYSyWG3u/Q3RL6CMPVSLgiZfNt3KmSj7BAM7vBhmjSiXO0MLgHVhZZczxwK3BEvvztFXpjWlOSSCVUUJZ3O6sM1EtyM0iR5Sh51fx/8rxCxAyeQ72OpM9XiLXn+xqvGKswVPZ55mbJ/0T8V6oz94/i/b07EqWGiXlKxQffWgbc3JOvAQjzh+2zCIYbElK3Abc9pvqgM0dniXdrTL4HGbiOs2VG3B6GXBoB4AzlCCFVlwQf+wnY+2IHfx7s2bZ2+f74gia+zoHooP+ay2okMZVZilpsLoTkjFhu2jZFgbTKf5ypNibm9u5O+ZtzPfbK7+7XX/falBwSfhzpRLLtTESJNzpETRdrt14NG++ZMtCKCOHTt8qEaIyO4RG1/SUm5UvAmNK5S7H7vPIJjfzPz78Q/jM6t4u3I6RqOk6Ri95MK+Z0MJJMoF5VBRxvuyAQEoB3u1imG31Rdpi9t/izvA5i13TLT7qvG1/QEDXiK38LKGsBMrRxIGekzUAINAUCjvlUDvN5MLD4mn7alAuwODVB+YZ3XP6QDtVqEt2rQRXtAniKFqtDAcIiYHWAuE8dDNfKviOhU2WocfHdTQN+PJzZ3gi1e8sFlmIc5rTDVJ3d1AI6Clz4xUYRVjPUJjVKMlU3nQfAVfS8gaG0j0holVevSqEJ5V2zs2D2CjhSJlZKjDIIKRTDDrh1DbKXgIMgWjn70zUuEbwioZN716cV39Ou1Crtn8q1/sXtkTrEV4DEl5ryTs5fOSyS10q++xBWWfPX3vrf73bvoefLKnvufAo0P0vQgC6ItXzagQ2aN2RhkXNtEXhCgLYCHwjgz3jJmvTG1BDcE7aIgco0sFEWTQ9wDNSIILCQ0KjQ95ZdpbmLqOZIRmRNKUSK82XgNiNfwoAGXWypXDzOgNQdP/c/ySizUWKUn136ZjdEUIwpk0BTGnJU2msWC5OwxuvmgENhsnMlT6y4tZRpPGgR1iDKs4NcQfo8s5Yrz6sAGvohIWrhCoslpzRNe1eAh6i1VTc4gh0oQIiLXqa3/aohn3UcUB2K8Z4P21I5r/m2bcf7XCK/cJ80MnzH+8T5i/T5i/T5i/T5i/T5i/T5i/T5i/T5KK0es+Seo+Seo+SeqrJcxXRrndnbADxyaapp8msOIBGS/GBqURcoWRH7YEIQ1mEn5fOkkJU3ROiUAP3l8+b4GrBjRFW5evA9uWyFQ2fxgM9EVlAd8GfnhvLfHbvDp7O5fOc+As7u/Mkxabu7V1k885F6pym0ztONPunMEKGjo8V0AQWWTbm490blEwKs/jczLjoxVRQh/hqu9GHd5a6R+61rm5xKoqzmlssxCD2mJtSSKH3gFIveQCUZYIaPyk79pY4RFaYXED0cNaizLxw2UhUZymDS8eMkU1V/yWpGD8TzBDMwLNlvkcHcE3RyN0ZN85GukPjiTDuVxy1VK5fcmlmlS7a9iV8GSVk+fgrg/qqFoutyowlS58uXnkvdWqZ5ZtyoGaJ2NpRGL0MzijBxJFH0PPo+Uu4CHfa44kZYkNBs95shyjj9J6qBO+ygvlvG7Tf/YclQnPilVb3VacEZZiEZ1Msffq2EBW4fr6llF5RlPNMtcgna4IuMaN2m/3u12y0g2Zc6kWgoSxZ+/Nw50D0Krv9vRKBtig/eNGQ0TuOnS07hZtI4P786eJQKMr8gfvbjzXDuoPK71KsF8mzM1Xp+Lyo2n4rSLOcLqibKd4M5eB0Bi2tPlihWfN6i4VzNXGBFjvDDI6cr/IupfPrp+9HjquLo2FyHdFCFX4PD4Zn+yEznMX+87nCO8aD1LBvXrx+sXFNfoLevnh3RtYQ/lPO+Hxb7bbgu39+LUCDq20FiQNuqh80P9ukdHwW3dKqxsOffVEaYNsKS17CsvhrmjXXizr5XN3mhqsYj2Zq9itoXPU9IghfFdLf4wuArVxusJSETEdoanM8C3Rf0mWNEun6IE+mT88f/no2buXaC1M30X47eEopptOtSJBGcmm/cN4h0oXbEwLMjj1ZG6JmHEJ8zKtj6agF09tu6MWXO9kMzZGHTDy98qF9kIYiukhfqtVT32KGxa4pRhhxIhac3HjXdj7ahXJapfgjV4RbqsVZikikOvV5g92B8Z4sK4bPwOp2AJRBXGvSHGHg9V/DV6Q/JaI7jSzQaVHJTU6DqsbMmCzMA31hmzCK5kjgL6Kdi8OFkMWmYBoX7Eo9CEpTdPnOFIJzjKNkj3RjJfHO9Ku4EH/e4cZYM/7RgkdHRIGGUMBdcVBFmo55H3jNWXFZxi1ytL64lkv0IMXpxVWGp/uCkotDUR6Zg6ArWgPqLngC4FX++sHewMeVN68rwSOQwxsZdKVj9qO0PAnZa/ct8MyVMCcUyVnVAZBE4clkeKR5FgfrpT1GI+9Xax2J0rTUzLRp9HV1c963pQZrGQ//2ZXDn+PK7EmTA1wXa06egadlo2d8SWmWWlmvGS3OKPp0dh7JwJjRTCTCCNZQJj1vMgMuHE1gn3HLoyNFbFhZC6juXQ3R0BYl3+JX328aopYKbLKFVpiiebwcp3OnaGrO5C0FiZro1HrxM2xlPrQPAKKmpDjG7I5asOq4eV3TBj5oReqVVHoWh5TSC99Aq9w00lbamyC5zlJm2HdA+OnKVupsXaJtfrLc8JMB7HViqQUK5JtHFZtSEfKPHcGzuyCMBR7Poikki4YVoVoMnwvPMrPSxOvRcyEtd+QTRvgWDBJl6zrgdDOISVTu6X1Lhq3ZBSYP0PHlsSjS9rjS3aIMNnul+/lmd8pzqRf7MLdYUZVg89Q79COO0PLgO2k1va4nMGw2x6d0ys+p0+Ezg706huls0tcymAka41O8fGRRcrvUGMzelqZ5usc/Rrq1F1dd9TiaiE15k9plQa16O27a/A+Fiknohkv2+tsCAId9GgJluaI0sOW1+5uBUk1Gpj3hH59/e/eoRhApG3GB+/QXu+plCW2rGRKBUkUF5sDkIgmCZTrJDjfUxdXWCyIstcU7llC6gjKNVXJMuIy94q3rGLHWz9S1ax0YEfUKGy5IWm8cRq/rd7pnrOA99x20dOnF6GqLLkZoWxhgjhamaZxj++tbXaBv3zeqsgNDhAWsQPiMpYu0GNc/R2a8yz1wkYYWcMEW/XjJYlUIO4BLCVzXGTKDNABLsriQIGvwuMO8hdncl9x0lQCRO6A51oRqCxWEfCeSfauKqmYoT1z7Ve2kFp8vriNtA/cO7KS9gLdYL0hzKF9IH9Bg6h1fyiByZzeeP6Pa/Nkt8Ar+9H2qnwVPHSIxyMKD32V0g8OlUOKP0QXfKASBq0K1n2y/32y/32yfwy7+2R/dJ/sf5/sz+6T/e+T/XujdZ/sf5/sf5/sX/9zn+x/n+x/n+x/n+z/Pz3ZP8QErr0T4OIBL5VevVkDQUbBzwVnirC03f6xn6nN38MOBgid+M0WJzcaiTajwhYc4uYXUfY+ssNb16QzNFAwW5nSm9/8vwAAAP//ppxhlw==" + return "eJzsvX1z2ziSOPx/PgXKU/VMso+s2E4mM+Or/f0u57z5Nm8XOzvPXXZKgkhIwpoCOABoRXN33/0pNAASIEGKlORktjbZqp2EIrsbQKPR3eiXY3RDNuco44t7CCmqMnKOXvMFmtOMoIQzRZi6h1BKZCJorihn5+j/3EMIoQvOFKZM6m/N6xllRI7vITSnJEvl+T147xgxvCKAYqxfG+dYLeEXhNQmJ+eagjUXqX0myG8FFSQ9R3OcSWKfRgjQf66XxOCeC75C6yVNlkgtDSlojSUSBKdjdL2k0lAFYwKy9Wt4JnlWKII0TUhxeAhElhhecIHIZ7zK9cxMH95i8TDji4dyIxVZjTO+mI6b45S8EAkZ0zwYZ/nPQUO8Aljo8n1jjHyByC1hqhwoeoik/rd+MUIVn88lUQFJGWeLvebdwAR6BMm5UCQ1ky8VFkoirGqErIiUeEECKhT57MiiC8YFmeAZvyXn6KRBmxJFD9Is4yI+r7gBJgim0E2eWeiAOqkEwatDcafeSAYiWi8JAxIoWzgeJEKTIUcowQzNCPpeqpQX6nvEBfydCPF9SB5leaHGmq4dJwYAwMDq3FRx0oIwIrAi4b6hEsFCmz1yi7OCIJmThM4pSUsccy7g96lGMUUciECUwUODXJIEHtq1eUEzMiNY6UmZ0+ii6GmbKLoiUuFV3j3IS4YSLInFtyBSoZzmBDggx0ISs/FLaCFHWL6RI0QVkooLIkvI+h0u6IIynKHpv5YQpui+ILkgeufpxXXgzZI7yIFAemBmhFbAYY7rvLiRIDNxQjOqNofbtxYgIp+VwIneseUk5IJyQdUmTor79WCkOICOGQwew4txEiS5JfqLSYZnJDvkMbIsVthsUTzLCHKIuhflzslwiGpk5IInRMpxLvhCHE5gaQI0ArceFnzkLJlneCG3AYsfuvCpwxBb6qVS+VgQmXMmyZhkOJfECIA2xmuh4Ln51GzHGVFrAkL4t0KLBcxS5JDo/biiWUa1dOIslZ0UWSkxyQhb1FSZ7TRd2KPJfOym4dX19fuKmhlP6wteSCImeGG0sQrfQvDCSUSndZl/uQ+5J8Ean9Q/qj6bF1k20X8tf6mz1z2EvkPXZJVzgcUGzelnVDBFM/T84gpRlmRFaoUtDI7P/k4ShfACUxabW1iUsSBzIgQRHQzdMq8f7Jf2CKLSzakBbN8rNTmNVZ4/fDjTzGyfjhO+ehgZF5H+yLQIpyuj6oRLJLKxOyCG0/+iyLLqfNGwrJLlDl99GqUISwduryHpc3xBFPpxfIJSLJczjkUq0ZqLG2o5uOQEmhFJ1NhjBjMsnFHsmEvrz+d2M6dYYa1nRqGseFpk/eDYV+9ZE0UrCZWN8m/mX112ScJXK87gO8vmCN9imoFcpQzhLLNaj6YjMFx8yvX3Yy1Bfuesh/h0byI461UhGEnRbGPUh1wrVlpJMOYD4szTwPSQkKcTiIIxyhY1QXBUaR5HwTSmWHVqhXMuVlgF75Ws87RYFFKhsydqic5OTp+M0OnZ+aMfzn94NH706KzfoI06VSo9Zmb1aS5IwkVaUy3DQamth8lTMaMKuFe/i9RSa4xGbdZqaU6EmT8t1fU/lMBMYtA0A/k3rrOlXvBgHo2gso/MPyYD9nHJflpiW6W2EOYsB2Q1CogQXLTK9LaDTX/kmNoerpqtcJpS/S7OEGVzrrkctGE+N3jkOHpWeAtRmRZt8quDrIo0M9V255JEVhv3+cVVfN9q4epmKCQwmK/gDDQgn3qPYPLOUQ+eBUC1OQTO1fIL4RkvjFkL7z1MMqr/I5c0B/ZaYlVCSwTRPO3rM27LccW4IsHSmT0nz7Wpor9wC6TZV8LpZYzCEjdIXgS6lDGVxuCUePr+DZgpFA7ZEr4ZlpUdzvLCef5QEnFLEzL2Bq95RAsFbY6lnEjEuELJErMFQXRegoQJAftPy8ql4MViiX4rSFFJMokyekPQX/D8Bo/QB5JSOdJmrNVTvRdLqLJIllpIvuYLqbBcIjMmdEXELRHjnmpOxRi3REh/t+/EvX81QJxaZua/YkLzpxSbT8Yn45NjkZzdwT5666nhW8hwfNGgYsml6tThelHyykKpUdPARtP98Hxk9LeCIJpqQ3pOiTAIqbTcep/OkT5YyWcqlXzQmI9yb53D/jD7Cb5f8yJL9VEBu4em49gs/oQfz384OUkb4yL5kqyIwNlk3xE+d5D2GSSYVTRFTG/dLNvYDSsRTgSXWukwrrcRmhUKTc1q0XRa7vCu0c+bAneGS6vR6V7VEytuT7eLWw0GjurSFtT6lxW/RgnCgmiNCBwoPEcZuSVGC5akVOAEcXqdHa6GAvobHHJa+srhsiOiVKGYYoXalCv9J19iSc7Ro9j0Hmmt6vjkh+OzR9cnP52f/HD+6PH4px8e/ddRP855hhV5GHqVjIJlLQZQqRqs8sIcJnZaDJuZ4wIGFQUYqGkjrU8FIPUJAV/Qyu0cwfzBTpK1X/SpVqrbsvZ+RAlEHfurmtNPfzvKBU8L0PL+djRCfzsi7Pbsb0e/9pzV11SCj9giAZ0tBRcnXiCCk6V/nDfoBddPk+JAfwwI/u8bsjk9N77T05HGemb/dfa//Qj+C9k8NK7XHFNRn0j958LoxG4gOE3Riujj2zvqFXcLga6WIBrh3LcqECNSkXDRzZDkGD3NMkOw2YktNmlcJk9TntwQMQUVfXrzk5zaGWyZ3vCiAMUuC1C1606jHPKKZBlHv3CRpT1ZorFliCMk5srSb9qfI0O/ZIirJRF6NUDNi8ILFyzhLMGKsFDmIJTS+ZwIvUHt/FciU+ntOBeEZBskCRbJUlsbY3Q5R6siUzTPQlAWvzRnDCiaG0dGwlczqi1WyhSHg6g5PLdAScaLNDwZLrxH/TTxF0auC5IZFdpcLwBorRBSNhdYKlEkqjBDtStT6bvmRNAaprn2Mo+3qN5z9IYoQZOZsblLfVmfKww9vzgD3QlYdU5UsiTSaMHgKKceev3ayKMZzK6ARwJzgkq0wsmSMrM+FRElQFEwCWQgQVZcEfc+4oWSNCUerjh1GFlN3wfpGwPw8chedQYsbcBWoIBbLXrfxrAIwokbfurmgt/StHT2BVuXeEr13vqzGZdDN3aM4IsykpyN0CIh2mqpbbwFVTjjCcGsRVJZr5K5EPC8RMGACnlMsFTHp8l+43rqIUPgaKKVE4lKw7fVwrSQLMiin63UpL8fmR8AwU60USYVZgkZ91K3SwLp8enZo8c/PPnxp59P8CxJyfykH6mXFh+6fOYYBgh1G3ULlfsbWCUBvpXVgwT3a09js5wpdTZekZQWq37kvXESYJMPoQ4nCS/A9BhC25MnT3788ceffvrp559/7kfedSUPDUZ9bnCxwIz+bvQdmpbHq7W7NtV5GsDSPypKJLiHzel5rA9jphBht1RwtopZ4v7R8vSXq5IQmo7QS84XGTEnI3r34SW6TMEzYjUDsHkDUJVpGDtzXeRA7dytPe539pZf+dYVzJTW1xtqY+USs/f+SYMcZByz1sYwUTCaZTwwNYNuSbIcJVwYBcCcPdpUrJijxCHt+cY2WoBo22X4kWM/3G+/fjBA0AozvDBX/lRWdEbta6P8NqXIYXwmVUCJ79wokay0Are/nPKPVIBpDtcSt7YHZwXNlKcN1KlQeLEfERXTWhLwoolr/7FWaDSsJoa+xl/HBcIWCi5heA0TyRGQEqm04V8d41YWPGv80E8aeN+5zWnenBGUEoVpJj0R4KHXLIFLMDlOboh6GPjB++9PmjemNHjUNV/vtbUriCxjCzwa2y1lrUFpaWctJXT5/vaxfnD5/vaJA0hkxN2Zc6EaxHr3/1vIfc+FihIaO+b34+U3Ty86p6aBMeUrd0vfjTRifHc5sTyeMSgiuBeENxCHIQshjgDDS8Iznlge5qLJAeZPnfvC85UywlQ99qF9DjqHXLNDHHRv3D7ugimxmVDJJwlPD4L9wsBEl1fvkIYZReymLIJwQfgk57SmJnWifM3ZgqoiJWCfZljBP6KIjRVysKm2NocR2LEJ1vbZoZBdaPurFZUd2SGX0o6utpLVceCZ/OVJ4D3rewiAYV/XBxV31nP9irmpHAaUtCmEJjwzUApBhbL3NBjNqSBrnGUjxIhac3Fj4Y4QUcnwc+VuZGgw0Ds6wuDOtoHkbm722rDdEpYGbpGoJ7ZT8gNbGTjBwkdwHeAat8QHsJpIJBEUZxNWrGakOa5dUBmIyEBsInRRQTYGfyxJkx/76w7XLsbIRt/7RjllVQwjqus8b4E8/b59xzhe6S3RW/zj9QV4JSFUyUCmEh2fnJ4/Ogn8f/qPuYZY0yzTG/b4h8cnJ1HDB35pzsfe9+MQduR5JAzvVh5XECc1t3AdgAAXJtPCjaRkDo7vzN4JOXgQGoau+Iq4MYFcDEBNCUvhlJyO0NRJLv13mkr4Tw7/yQX/vJlGZ8l91NTzg/ggG0LjPeod71KZ3AnEF9uIdRsXBDo826AbytIx+mgC6FegQ9kXgoiXJc5zAq69jBgXtJ5oe2cCO9zed6xhkqvbRaokyebeHTAz8IP1GWAuHDzkwCUwNKkafDO1NUgqfnNUqYPpQUKxNBxnybn8hProSma7bQRXPb/dJbjKrHbMrQTB059Vm/IAWxeYZAfj8TDccPlMC8PS9m1EdaHOqJGIUVSuKFZkwcVmz1WFqXWw2gJE7H0eNmGITriFX9WGsoLLKBnnxv0F9lMjrhf0ljBzz0clyJsycMNeFfg3oppjYOmb1wXlUEGE21gYN1AbcKsHHx0rW1D2+VgqrORx57hrIaQ7H1UGDkpwrgpREWgYKzjM7Jtwst5CuLbgqwCeDR5W3P1tVsBJndEbkm3AzW3C7y0sqbFJkhSQamMv7+QohGmj8WYZT27gQk+g3wossLZYKVv8i/5xTbJM/3fFBTFBIjQpcWgIAUgMqaGU2XNhZBIG6UNuAwM/b/TyrrFIq8Mjfk5bZWOXhRakdMg15bgfi77j2vrmvYFnGLuvDqL515OE4RceVBubQpmNa+OiDJyMb+aN/C2LD9vF5x9s3BZgy9olnCUkB50Ko6l9d4ru24wA9NAJHqIe6PGH48TS8y0aRp1ZlddOzBhdqvDG3Z9QI1L0tBZCEKayTQjNRLBQVhFhwm0xS71HdmUh5rrKhohOPMiU+MS7lLJtmn9nSMuPPQNZriyy8iCzJrh7HORfoV+0lQ5rGb0XK7+yN+YrghnI6VsivLu0MseqDHjRi/O9REWOFA8gmjuEPCMrwhQRWmit8A1BshAlkZS4gD8mqYSsKRv01xlHVsvB6WLwyEx/hz5q9lEFwwqkqd6idvqNBFJILvmamVurRGUbtCFKM+r/oJSbADkubgKQlCGFZ3oXaxEa/HQp0f/z3enZ439xTpJSNS+d6/8DwXZc3GhCYC+BIlUp2AFA47ChyY2M8ufRFcnR6c/o5KfzsyfnpyfGarx4/uL8xNBxZQ8K869g0fSyCYIVXHwRYd44HdsPT09Oot+suVjp0yEhUs4LLbyl4nlOUveZ+a8UyZ9PT8b6f6c1CKlUfz4bn47PxmcyV38+PXt01nMXIPQBr0ExL8OutLbBFBUl73+0Hq6UrDiTSmBlArsoU2RhUkBRXbChWmq0XnXKUvKZmLCclCcTL7okpVIvf2pkFWb69RmpQTSxWyQ1gbu0zG8RWgyRW5emO50YN1pgSALuMMUTZqYkw/+tsWOWWC532y0VW1XBF7G/Pf23i2e9l+wVlkt0PydiiXPQIUx+wJyyBRG5oEw90Kso8NougOKg68704cvrvNNzVYf7n1oDgbeoghZDLJ7Q/YSZs6C4gMQYnOp9LpHibVqEgSaXzoVq/bUQnZljc9dUhbSW8pYqlHMp6awWJAj7QZEE3jSHqKajQeCM6MMrpreZ3eU+oBIi2oKoYDhjC6lMIGKQkgcHx71wHd0x1qSm8i9smSfi1ADk0XUyPh3HfVfwS4sSVYj6nclQL94zCyI4ivUsMMx43IdXWpIm46iBvBaq3oHcrI7LXKoHLEajwu3LbQxY5QBq9ZdKRVliSpGgf/V+Y+ZGwHvkkDf0A5s8ZBO44eWxC9AFUiVBas2rX0uzN67F4HrtB1qrz6AZsjZwaiurzKsyMAHM2aaqYKElPRwE4E5KcDYO60QAr/uZZvWaFo79GmUZHIWj2ro1S0xQPyTfZ3yptVpzwYLz3JiJOU5u9JForFJtdRh/XWRxGv7f6pUIve7OxiHQExunvMmUW3jNL/BRW3w9/+Xcj/xRVGJRa0dtMZFU3kxkwkXTJJxnHPd07X2g8gYBFGPmhgUuzAjvk/Fi7FnkPCvAhn4QLttHSdCGF8Ka+d/LKj/cGMR6sbYOZqJt5n1G9BZsbvo7SQHqlsGNTPCyTDBUB0InmtFO3eVA1HuzwpRlG7008yJDdK4HDSYE+BnUEjOI0nBuDy0+sJR0URMZFXES8lYAzBqbw04SgrB1H8BQzAx6SUQ2PzHiFdU2n8VU84BaH+mL6oXWMPcy/7e8SQ2DauBs9ktP9S3w4JXTijqiA4ret9a5ggCYSWvcHF7v5y9oIC5XX68KO8YMZ5vfS9XA3RobngggQS7RYiHIAk7P8IisconEgqjJoLm5hm9MKTCNRG5WGWW+GRWfo7ZZ2vmm/3Bz1XO2yGdFmKynyjcpb6Ua2LuE0tjqQL6VwTjL+BoRLDd6bIrAsTPbGOdgCcKb9FIby61iVV9q3zPdg26gFZyt4IIaoZQKiMi16/0gOkX1qIbteJ65C8m2+Idq/9VwUeZf/fRAdak/QI0qVMbfysq/25p1MZSFd3cycO2vrfsVXT5D9z9ePnsAc+nONu9q7f4V/FgNHvE1IyJKD/wyeFXhq+9N6YXKQVcDvRg21PeCrrDYGEEMY3xZG0YcSxCyNhiPH5XRimO1nU0qU+bJ45M44jead/xVoQzxROGs5omKkiDp73USAgOouUb6C41itlFE6i1oPShcqwA4TZ1uaCvI0fCMn2oKp/EtugoiuyMGUUDMayyVKQzll7oE5XPFU6hqF8WS7INlRRSGmwGTs51GlI0q/tEqFy/LB/2uX18S7t/0J1iIjZ+Ehqvw/TJW0ku/c5Z9CY8LTVPgVIdDhUFJSlO3zp+MPje1rWGW+yZ6VQGWTZSt0ZU7hYfX4yrr+CJBle0hlV05yi3hlHV88VjKXbIb/CjKxixGQih3mb4qeBLVN8CSy1oIwqvqSb8toD+oa9s+//rsDvjG6Knxg7tr8xJUvtxIbU66ZKcRwuiWClX4j/R2QM8gw6OeBlICeutuLr1IreDer5YCW6Z9snqRzhJkkKr/MOFZRhLl/Md+Vi9cCZQ+kWyjbSxGSEp22Lr/dJFsXV7vKritMU/7bxJgTFf7x81KPUUw5iExbOwcTWutgE7dt1NblAxyjD8y+tnZvTYhuMhqN6S/FTiD09CG7MPALMsDMWUR2OAu3vicCAvTe/V4E5qWTlwz9Yrrb1rnvDG1veJ8hqUm2NAfw3cxt9NTWU2/ve/B2RpvpE3hG4HDwl75GBeFIHBPStmibpZRZvw6vXIKzwO/deHusKZQywaWNJJrtXsMMshOmrtA5PbU0/2Y+5VNIN2C5wBxojaspmWzvODC5ma69HBbJ8WKziAFXoOCOlfTMoV2GrrsLufodjVyCYHW5xhkyY18V7KXCeqdBgHEioXa2cb8iW+a79C7surglfGgxVBV5ULHeYbVPOYzHDTv7+q1Dh1YdD8hTHE5QsWsYKoYoTVlKV9LE9r/ICZnUyzWNiEpRnFPWVtdVr7BCXp3hf6/nleSjbE0jMuAnDle0axPlF9FUEpmFLO+5FwhgwLdFyRdYjVC5vsRlAGZyTQ6pzFS+992eje9J+PTs/GTXecuCMpv0IRFsqSKQLmPQVR9/unJ5MnjXYny0cZ0UqXymk56ff1+kE7aLHTi16eVQTniHSpYuTq3K6KWfM842FdK5WWZZAMwej368vn1CL1/d6X//+N1hCRbL1kqrAoZt7r6q4qWKlsi2cCs2V4ebY9PHrcTNONpc3v2j96+tooSsEW9anOEFlOFaM1F1iwud5B0F5iaRrKLR8Hp+LTJ1KbnB0J+349ePFyVHio9CYp7VZOGcy+UettvDl7zhQFTNn2oCsmj+qnfSOdA01+efng7HaHp8w8f9H8u3754F0/VeP7hQ1OS7hVy1h6blfEEZ6CUvtnoAfnibVDIT+v01Ri7KhBXXjV6Na5ASIW13PU28N4IwM3InAOTZFSBsKUKFXDrXmZb51hEg34vjf0iwH1mDOKpRTG11x5VsLizdDDz7qI15ACkxxYWktXTInE4bvCjxgDHMVNriW8JwpkgON0gqXnLuBCNB0jChTuF3KIbgghLeGojrBkJL4yggw4Ufrq15cAyghmET26tNrZTQBqS3Eaafd+ISPutIALMOpubYYy1XkFpgZyxwQChrHkbPNz1CC1zQ7HCw6VOVG3sfwyA49GkM8w2trY3ZEpx1+PEpXFS4SiNn6Nw0P5C59T7te2usf22seu+ccuN4z6DaUxrLrjiCd9Tnr91ISQWGmqNuPaUM+++jgpygNSNZw6MEx+O45TA8zlNIvvwA0n4akVY6oIMYMed12b8T4iyGS9YfZn+hHih4j8U7IbxNYtNgQ+rMRU2yYKkk33dAl5+chl5JMv2VO4ne4BAhkdcG/n5bHw6Ph2fhfR+Z8vhycYI7PDGcGe0hwrpeMrCM3dQcRJ/aqqPjgpT4eSQdFiIcUqaxaUdhxxsPhzAgRNS0nG4GSkpGTgliiucHWw+AJqdDOPILFamjJU37+j/rS1ElNZHT35qIfYOJy1Gs/3Np7pJQUn22ePmOe7XVAsP83fNX/qnigal2uylDWFCK3dwa7mmatmSLZrwVY7ZRmtSULmtMur8NHAsJU+oiTqkahkrQLbhBcJCQOF7k+SjiDAAqgwhzIxGBQdkWDWoxOsPZgc7aE+NxF+HLh/V3aVN++Mfh9wjazxT80oO5pt3V/XmDXEmqTddGftQwsrifK5M8pJebyi2anyzuSDQBmhUpknCfcqYy/GfppoPplV7JvNw+NLfudcVSG9xvT6I16yrvK5bmfTLeFt9Mr6gl9Wt+jZv64N9ypk0HKzHIumb5tTmZIX0SUiUkUqUKdQ+fTdEsF6ul4q8x+PH45Pj09OzY5sCvCuRBnc3rYEMsQkBoSB5HzzcpR5Gq/jAZZ+9li+17e/Oj6qIpc0bDfNQ9SlWwkM0fRhsI1u52bfwjZSblp3+aDq1AkoqvJEusM8gc4U1tKnvhUwlPKdVSMEi4zOceSX5Hcl1d3x/qYVFr5r9XYHBdkawWBSrlhTwN3iDZsQey2U5KshOkoRJCtf+0apCHt9+OjrOjkboSItq/V+Xa/jk6NddRVyPYUVOYWQdkJCegBKcZSR1nRZt4J9Akq5ohuM57dLL1iu3RuRMH1CMsGTLEGEHvsMgzDHcajeu3KtoE7Vvhr5DBaBassL0JoPfR3aLKZcxg2W5Z1vilcJq61YoXQUP+ys1rrJ6vQCn8n+D+sb1/r1GV8b+3rfxQG0K75yy1Hp0neSCxCqI7itd+yU8h15/EbvD+5pVe6xzxhWjd62uYottmufYYHQTu5FtqrrQ4BH2WmVBesoNkV2JkrX580oHmLVi3kVJO2lluMfl3NojBJHPORGUsAS851JC4wd9knBoaJ1C9QhTPHykPwoA6tPJWjLcZt3R1OXCOAIhqNCtOrwjKVtAFLCtb16ntFIPH/1IfiCzOTnB5Eny+Ocfz9IZ+Xl+cvrjY3z65NGPs9lPZ49/nD/xvu2O6+kpdTtvUEiGpaKJyaXuqZj4EaSOy6v6HXYXdZQRM0K71sjDxHFHtlfAHnoPhw0DUE8WAVimTLdZSCiU4BPr2rBNHUAT/+WaYQWQp8BM0/2icIaFXFkRCdBa8EoV5rMeBvGFDaUC6LV130eB7+TLR+Ozcd/ohFoTOseSvpTvw5dUmmQbaW5n+Q3CWqU1Xg2iTMR9KOxLXTwo6YzqTOnPzxfqjuYm4eD90dzA+ndIC09/cH/XDn//WcuAzTs9Cm2HOUN+u+z+KYKekx+uyM9RkOXauAloXaRmgVJDXlADd7fC2tGy2u3UtmeZ+PT6RbZrlMYiGdvR9c6GitSJbUFcK7J9ANyWp2qltWOFtZuM01pUu15S243G/f4VUzwiOO82x6OB8K6TPBoI7ybLozmRB0/zaBvJYZaquzZ2IbJQQH/88LpbOn/88LqeP4LhtiEjiuhfR0YNl4k+ska2Cxj0nsb2hsFD4rpAVLETrsZZt3u5ENn4T1O960pArq07+gshJiikao7mlclaLwkjt0SUmfTVgHa02ZaCzBtr1P9mArrb925qH6qm+jONfmoyoD/BiWJg/HrfNb1fr9djq/uPE/5wUdCUPCTsYQAqMA4eCgL5MAl5+GR8Fr5oOv/YCVuqVfbdxI/HmOjFn7iTbWLzsYV8YIZnbYdQf6qP1B+XZhxFpIqPe+zyvac1S54wKHmk11hxbfwiDEE7G4QXWNtvrUFQhciQVDTLbFmxKkTLhhppftH2olacTAJjbGWqVWGolpQujcsxx8KweuUJdSlWiantEhrTtrn0NBy33iomGqnpHfzCcTJl7OfHD6/3yctvy8y3jOrHtmj2rlj7/PHjRw8NB//f3/4ccPR3ijcDYYyI2k+8XgGM0stiIoMraXUEVB7FsrSgAyP4sc+nLizNVaMC6QWQ24felEN3Uvi+OaRqwo8C5RZCEyHEz9Tfw7BVVniDQJzYDFqtJ7P0IRegztpgpGxjTg24WQhAeplVY9MWHhJQJDFJWX7YDRjvC14GRVZ5XUEqbjCT1Vga0xltYgMl0oK8rS73qqdiN6bx8eNH8ejsx4+apPi1OoafMFA0o3U57Y45Gn89yaH5xGgHTw8qLRyxIPn3mEC9S83pYQgK64aaX8zNXH2aw4POTXlNOMXEAwiG/wuCgXyGisVeDSkfIyRzmq0WrRbGuIYDu6Ws6e+NxeWCmt8w4NTGv3trVDuEwokwrgZ7g8cQWeWqoguGYN6YBlAMhJpTsMzBpViRslqqK2VlKqZ+XQ41ZGsRfVd8Ohd4sQpLs+1yq8OFH5apFRo8h0KyekG+m3p7X/G8lfm+i55KjsQm8a6yyH7Ef7RQahupiS7HUtbA7lR7yUCJortXH17NVJIDm0qW5WDqrq14dA686nhKkIzcYo81FEd+leIX3rU7vjUuJgI2uu9o0k8olB72nZiAaOmKl5dFxWg6qkw8BkFgG0uPqaFuioPxyvhRyyqG6Mvdeb2redOK+h1Y6W4KS6Ef7kbbd8JUOBpbqrTtsANvDGPIrzd1b5HiN4TR30mkVyVZYbpjGs2WDWdAh/nG6CBFcLdfVTrmW4bXhY2aKuZFiDXkbLOCQnX6lchcfyyr5UHwGfivXSSavelxgS0JZ3PDKPWmXbUo87Iycb1Moi8fTJhbU0og//kwWWFAOolROe61mm0jY2aCrzUSJ7v0txtzWV+Ck0u+tglGazIrrwzgpqxeVd8apkVJeC1Cqv/Obs396q96fWSWnNvw5seLKmygrRUk23tLl4VO2puAHSBTsXa1tRXpCv890nisf5zJG/19bFpRy7SuKNsPof5+CMIcq6SP3Ok2fZLlEJxDIzgvloKvehYWrh8TbTT0T9vviaw9znenfPf+TNwL8Z0wcj/Md8HRTcz3EPoOXZNVzgU0raGfIQqCKPTj+ASlWC5nHItUgsfRCtrvbIBNIRVacBfRSBI53qyg0Qz4x9dUEnCPSpRy9r1phRAGl5e1UALpjTNaxkNpy/vc8iKEMzS/r7mWumGUL9871sxzblrQ36s6sgc1RcrCb/eiU/3GlYWD4ymplSMxRfFscR2cphN4YVLWkrPxaaahlzulguHpV8fw1diBtUOqfETJlrM6uIMLKByj9zYwyst30wBHaJGYEicpXVCFM54QzMattLmQoyqSoIWWS/siunwWVIyyhVp6YPDWeRsOVqu4tB2LfWHihc2U81wWjunG/sYvOTMIOb7FNMMzmlG1mfxexRaVFBTymGCpjk+TbhKeeoAQ1OiiVSkyKm2tJOmC7topygWHHujlqlYVXc0vx5/7s579RNPykvNFRsxOa8du7hm7Edjrwy3jsxu97OTvuuy6f0eA25pw0BOoHp1lftN7Vi65UBPQ0hdV0j5myZILh++43OX3QlW0CiewZKDBlRhNuboDRWUmVX/+iHnnoVvFWijufPoDuLDSEsRJzQqaKRTrAVuRcoCL7hJnPFelwmX6+TewmQti77F5MNmBlkuYCYOnZFpbVDEsqBgBcsnm3GdUm9YVip6KN/XzrZzpF3Tsb1XJce/UlC1O7bqu8r2sZZ6Us3RTzPQPJonUztVf/GcRTNXvVXXX4MSugCJ/pro3ffXR1ukNiB42yTlPD8D83gzkPDXGRRRVsa+I8TC95yn6ePmsiUj/v8zxvqaxh6qC2ETGU3LYGYRK3vEp7Cs6+iEy0NAK501M4AYy/vtDofNAxnEeUhx7eJNAMnehPcCBFMVr4FoJg3OcLMlZJV6OnponR3HpYn9Fb1yrylBs2MZTMbFQYUJRmdCm9VmEfrBQR/pX4qXexfFsmbILV3bHOv/ciePoeHV9/f6ZxQOePP9+rPNmjKy4ImHJi65l3UIn0JpRwvwqGOMoZvDa1Zh4T8zlpQoYI3CbCAU4ZljSBOFCLU3XG2VrCVfu2zpxjfpw2yjzq70NJ9pVkYNad81ScrEYvQNNV4D544fXcbRLpfJJ0yl0APyAN1KTrVmOrh7/iNqcNEMwl1Xp6hXyPKcHTzcTSZhq1M3opMBcH5yj2Ec9qGNl4WZTRcNLcgD3fksxPX/a5kSIWlnhwyyXAx1H7N9vhFhrnv4eKC/8amMFO269jkDvWLZBrlk/rfyqDZDms+d+zAXKs2JBWVmw2Fj0ppGtftAuJhoXPeGA6wJ+6IjtcKtrIxencaDRViPFLI0MM350oK7rlXAC4nzWYxqQnzK27RakTlTdiRzSFNmrAwha9b0paRBV8y8flKi+tyl1ouq3KiFRe69e3vfGpU5XRC3oQ1b7TcxAwtmW25k6vbwu4ftQuwMt7TcZTYomX3wbDKPuS++HQdTtyIB3u6TNG/SQrsj5itoroA6g+Z0rcuodvKbMpa1dqg8q26DMxMIf6zePzZvmvIkfoAvCW0yPfc7Ol4Rfvg9COBYY4hpTbQqQFNlmuKXx04wKtgOPnLMGeK8jtQFvlyO2NUMsnK+DMmWkQVCTrNYksgMS1t5AqE5PJMcspCOWazZwirItTYbqNMXT0A44PcaCizQiqlMSy1Q7MB1J2KyoYz7ulmUaSW51KogQkZv+Ac4YG19ZBkkZJ4yBO8j5Uq/nvW0GehhoktwS6CgYFPmOVmUueQP8NYclo5kBbbCYmNGyG5MJZNDz1uaGaZbL3t2aNFt4sSBp94SENXvQnn4GVxfn8lkcmzooNrWEPtxtyFbOQdrEt/NaG5h6nGmRuKaa9Xl2jt0ipSr1/brwoMWta9y54Ox0GoYBUO6y/n5ehxgNcfPWd3oNO+qq1c8Xh5QxrykrPhv8GvQYvYXWSplflzHlCdQGIym0vEYzkuCipvNBvzN4ecPwiiamThEWG627GfCm+XZVbyQ+ThTI9YSLdFKrf9KTfbqQespvlk5w0dgqW+DbLvjUNmu0PYtc6f4stcgvnxlnsfOqg5oLbbuR4g2gAAOgxkllZH1oUhlZl6SOvVm7fFZLnWkSK3BC0LyA1GcHmVej1I+sZkuFjZNXVTGQ+xm9aZ7TM2LjhAXn6kH7gsmhns+t6yWJBJvu8Ct2WFr1glW0jl0HBC/HSVFSS29ELv1C8dqCzTY+sOgQJPmtIKzhitvnKPE3pgNv/dItnt8k2eFENjZlAvaEMUJqlYj9ii4xtM3juo+C8swLlUpaL6nyuwROFVntdTUAAGzN5Y4JolXd4AFo9FcuTYWlNMGq7EcIP/GiTAQzhcBrdDXNAIhitW9RiX4ngh+DPf4vCFt/Ap+jE+jlBpXk7GaaUyEVAG3hu5PhozMwXTVNJxJtxdUEZ1nrZdRwXILIIqtyuSscUDIXlEMu0BzTrBCkRZx+XUfJ1Cg+Y615aL1+2gDZcTHxzWHypUzwgCKqNl/ZM8GC3M1F44YXfXMn7eJO+sLuE2u5BVnqngHXqCQZwRG8UwXpxOy0OhrU31xr7RYQiT4LABz5EbL6bb8AYVWb8PaXt/8u/+vRUcOsq893ee6ylHzuxnwJnRL0K3Gcc5qRGcHqWBGpjinLCzUUP22NyrLYaRrHjd+9XDxbzz5+mF/89Ycfn14lv80uFuv+6OUSi7QTfZlqDa/GqTjpjxAOqd2N7k5PHd40btfDwcCG1m+FFYeptBq0vb1RJAUtQo20acYkFD7hAtF8YorQHtWwVDOhv6r/2r7hyw2lsW81zYF8l0VhbfElVogn0FE5PbeptryQExNlNkkJoyQd1cKqJlqNgce1t8w/FwIz6L2ecMZMy6joM/eZwqtcqyOTslaJKNgEe4Dsv80H7ZMX4h8+jWb5ts/jL+B58coHNBYe3W/+4tJ3Pzy/ukZP31+6jx/4XFJ+t4YeCwmht5WGVr2mTXdGsgcj0xZxAqGy941PLtFquv43FElOfToftM9dBWfnebPO4K0s6PmNa4W7m5PWTjB0znry0/h0/PgsTnJNly7NPUFZQvPGHWuT0PJNdN8Vu3hgtozZALVt0U7rpNxYwycX11u3xWn19TDziaFU8xH5TJKiczKTrJCKiPMVZ1Rx8XCFaWM420ktBN1KJ3A/YSmoVejjh8tWoh5OPuc4uXkoSVIIqjYPJ95093dvV4oV8FZvAel4ccAsXmQEi6tE8Cz7YL4ePocW7aTWlzdOq36pUeiJzm01vA5K9Ydx2oIblyoCLBekrbznrkdvafU22+cP8KG/vEBaf5JE1eK0Yyh9tFC84lDGtvXk+z2AX14YFEMt27sz13wN+OWFS23SkiJKqLf8hTDnviRJK2nzjOMd7aSLGiUlQnAZCugQa503/45vMbqlQhU487Ow4oTLRBSzidysZjybKL0nJoquyF2NA73HhSRQIhJRhiRJOEulaVxrC0oaWhDQEvGe1QiHuNcvQHgPuk21s210rwm+mQgylxPrFAX675Dya02zzCEEqcQIZKCypqn0BtUVJilwlpFsIohMMPtSVHvzvcJQDQBl9JbY3CJwxmbEdE2uchqk4nnedJr51/1YyknBMo7TLzUSgw34hcEFCBDRc/aTvAA6210xEaHck0bX9ebi/UfD45ZfiJhzAVdclSiMkNguslE9Sjw+yWjrRPcciP5TGwQvFDTng+xKyOSMDcATLBv5Fai0HQQ9IlEnlYLg7EuQeQ13GiTDuebXGtFQnsqWuTbu3/KUArMFWifAPR5lVC7jLv2/364momAtW7B9IH2iQKgrCfXvf31jqTEVn+xuG5k2cgBec7lRubsu90xgiZzAXc9ES5k24bEz5S+xmOFFMJsWq71h0ljtMsSERsnIWgTC6eJoPvQUaxIU5zfQJgiIsnR20qXwIp4+tFPozcsLCLIxR++iBeWS4IPdGr0iOIdWKUlZFs2tC/19sC6rv5nczFqFerO6bk8yUbl59eABj2b8G5rxRv/hkCR9Mt0ZSR8lhOXgvIMYP3ZiQeKZdjss3LssdSF3EA+fJEWOWbL5468gLB6fQ+iHN4I/wHK2zun21d3wgi0Oub7/qQH+g6/wpj6GP8Aad8xrnLoqGEfcthcGP7oySZwZX4B/onnBUeeB5jpV16arnLN6+G6I7jVfVO+Fnp3K68PHZJyMV+M3ROFnWOELQbAicEFkO5CFX7YdXFHPTZ0ic3TFADa5v8tPA0zTtVeOzBK+vGh3d8VdXbFdGN8tpcxmTQMlpKWOqYuKjsitUptYNwPdDo6wWs4JvyViSXDasa5tzBVb6QBRuXEyvg4DZ2s7x/zu4uJAw33e1sqwwv/p7OT0p+OTJ8dnP1+fnpyfPDk/fTz6+dGjXz9dvn3xDv36ydyUGhBjS8QYanz/ij7dTv7678u///VX9Mk0EYT72CfjR+OTYw13fPJkfPbk108nv4JK+Onx+IeV/HUE/5isaJZR+ekx/Fsrzkuq5KfTnx8/+kE/2uREfvp1ZCpfwV+ABLhm+vQfH59/+M/J9avnbycvnl9fvCphwG2p/HSq34c2U5/++29HQO3fjs7/+29HK6yS5QRnmfnnjHOp/nZ0fjo++d///d9fR/vIGwjrFt3CZmELMLRxQ3Sy50SFq7ddxOgJ7qAElHSqSj3d+uiruvVt9D06OVnJGCm1jIOSDr2KXYTo34dsjfYhA590oLpSWFHYDUPwtYzL48UulCaoQ7/VhrPOyAPHDCw+qfdtiImG7nUdsEkGzBL5rASeBP0NY+Q916+5LoVewN0B1skTNNu2A+wFypB529iqLRQ8Phu4GZ1066LBmGVUHRSpEYdb0ZrOSKmJNWkj4GwYAYIXitZO6BD3B/NG2zLLk9NX/3X2H/928/Pf148XaoFfKDZse9COA/kyPYjU2SIBrju2fsqTLlyu7B7OBf+88aLK7JOWeDL7a3ckWQUUDY8ha2gmc8GZIiyth0wGUPxbNPcBus8FyqDhHBEPbEBDGb0BvfCMD9qGY1RBMJFugDOc3Awhwr4fpWGNJZLEVjBUHK0w82pDMueA417dmQhF5ofeBGldzRWxUdwL7/BQGsKinSvNhl9jqsrLoyCCPsBrtr27ufJFnvXiWjiuoPItFpQXUguKgjSrdYHC2qDIi1nS4A5Gk0tsCZeCSIVnGTUNEkxoPNNGdN3e7KIYbNaJCDX5CJH1CkQQYbSiyrKLl9MJNbZs0h80KTJvjXsSBFMGk343KzmIDrfD7Lx/jXUc2Z4qkGBFVNXOb/sQ7G7sMZnV6trgEJfbsSaCeCLJFniAQEZb3NirQwWI+hLnpvbuqbOYvpdokfGZOfsH0EkHSFgjVSEz1fWeCyX8VpkO6c+TZqZzgPI55Jbbl1xy7WyDXj19DychZdAWC6I36+nUnvLYZJi63hrVNMMVWpISbdh1W1vuzV1FxIoyGyNSa1QfwL3gLKX20oeU+W76MICC0yR8TlhK0shsrmJNcruGoz8wgVzBqa0ZqqzMcv/64j3iAqqfPeg6A1rlfplx40QT5DtAcrz3NUTn3ZZsHEke62uvhzGCt50xq/Utl3CW2N7/Pm08IKq5InqlDGPW3bWxzYYiutXhiCyX8YBUWqFyACKXmKVZVfnYKWkHpLWhEOxKqul+a9mSB7L/gOTaU66L3rc+pe4ktd8h8jnXth1L3KRSWVEFZIqNDRSzX9f12u30tlbLCC2uaPc2qyel3FVHaAacDnXE1QKYO28zWotx2GxORhUtq3FoURcqK9aQqvV3RegyWAsq0S3FqGD0M5I8uSFqZP8LDUH079S2bG0UWyrjrHitD7OZFb+5iPljWozYHvvw1b1ILW0jqeIN2uswDUQncVm8OHekMHc3HP1FA0xKpLInYhsbPate8Z1Fe/DLwKn1aBzXPm3hwP7waKSsdT29dXCs7+CUVrMFxvHaX4dMY62nsHYd1p2pq20zXNsOC8LHHXB6JlwOQtYBaUs+5QA0UQjb81kHYGgDsi2Dd8hcRUH0S/AcPpIGoNI3plTrvnqfEW1J4DT1n/cVPVXwXkRkuLLF7UeXDZY3jR9LP0NQqjkidYbdGmsrpJCxGQ5IAaTmXciE1dpGIVjd+9GhO0RmwmMmnKtCkHSScH5DB9bNeQe/4AwdaWB/htoJR7bHqy3WYFQNHCggS5zapumA09kZbmbHPSleEpwSMbAQwmsqoQSC/biEVicCpQVxM2wkcGVgHtmPqpcNtCNYJ7KyQdO+BRBfnsAgjDJqMyGoP582v92FTb8og1idfgmZhZbNsWMTKMJClWw2YP4qfGJyr/qxiXn3wFzi8Qleu8zUSUYbESA1F4ANGPa5BOmvfIefqdanljwdle9glgZ18ms9WDuI7iAdvJ3GsJt0V6ExN4fe1AuywhQ4pHRXWBth5EwK6dWFiXpJXWC139zd1GSZbVS943eY8Qoxbe4uwb1dSoTq9iA2O9vlMswLuPI12/NCTVKs8LYpGuoOrq5ApDb9MZoXWRaecyOoPQ/Klf5SU7H/mA44jJBms05luTp74NwPhjDj6eaB13/eW29/gYeMshxh0qrPaIMWQu3AINjHhKqlLdXmsiVgq+8ExzYEThKSq5Dhk4wHOlDLRdQfgkp7vUsTyhbYu929hActl7vmx+673RJifB0HlQdJyazYq1RjW28eOxCAP6gs7Bwn0E4yug92yhW4MsmnUMALqyoaVEtOS6a9U9haLdYVmD0ccbG2qEcwa0cjdMS4ognRf/OjZkboaI0Fo2xxhCJl5I8SQRVNcHb0tevKlhgx3SM/eiuTafDfeOyfnMcgz6s4jBstzmYWwzdO+yfjNHeQU+mf4pdX/es2X15elQkPctx2rNP25pwtVPt1khs4UFdE19005dM07NCGT0JKP1ZF7/5iPVbReLCKmefEijPQmrJHZ4fH/wtlKV9LtBW/M6FjCut+JJSVWiLKbItEoIrcQVtCDdYU4tF6Mi27pndk3MZD6w5AC/PiV6whypkXi+f2kPUXadusrNjf0kIg5qLak8oEXDOqqrAJvihpaqmVxiLUtMGspQneP2jvPiC7NIf/SKSb3RRS3sW8LTVr2y8xaheFMdAdvTnb4S6VysfOfxmBUNWpijfkaIdciGys/xYFBwkHtljLDnCbSQ+t97nDprbxeVCrYoc7Lv1dvdSlL+C7usxuvdJqYYTOFpc9WSEKo35ftcttnMVib1ba7pz8C7nOBqtbEMU+LWFzqXZYUf3ZOPptwCvDul/u2U8yRi0KeNDQM26BU1njXa3xemOJgymRdHa4648kCqaUA50dHHsjiYNxSCLcswOOKJQyfba9g2JvBFyO50WWxataV4i2NEYcgq6bAbZ2ORyEqpMNtrUsHIKpe/q2NSHsjykGqS0eCO0lVXpWhO8mvV94DeofYrMT4i0Qt4Tb7ICyFdL20JsdsHUB2xaGswO6dlD9QnJ2H2FraM7e3enavBLD29N1a227G3bmbrCKSm1zCgD2VkV3N4eARQ6VknsY34ceuzWkto3dYj/w2C3yrWP/mt3+O4y7ONYy1/Tl8+vhBDmLFggzuFs6vbfYhTtPRYD544fXrf1mtlgge7gGS09El2cOS84m+VK0VcTdjxcMfGTgx0mA3K078IKBp9arnZlzngHKP2IDnm+diu2fb52Ke03Rt07F/4idiu012w2e3/jhMn/R/265aoPfqrausVs1Bw7tHyuzZ1NTQ2zGFxD22FsPVXRFpMKrgULWFUKFT6vQNIc+LuZjLZmrcha/PP3wtl45q991qgH8tSMFUCAWY6Xj9jpWL8qbeK8agG0Vque/rf80bnQG2XXwUAMeAA4iAbqkHupwR+gamq5S1sFvPU7TyLSgwwie2iyZHrFd84S2civqWrSeZCH0xia451hUzSU1de3kzIus3Rm1Dy3QJbLIMjc99dV0wprOMPOltXnQIq7Nj93BjSVE9A8rsA9a6PgvZs62FzuulxfYE++FzYcGsJobDSGtdmu9m69BbYr1134yDychcZahMr6QCku/x5571MJU7udutvLgooMzliX0tUdoOA37tD53wxviuzrsmVraa9Brmh3nWGjjTG+MGKIuZWJPq7VUJZx4tPhHLrFLGKsecpJe88Xjv5vXW7ZMqTgekEQDE3Fhj5h12RKt1gmvq9D8gRYuno8vCsZMso1G5RGoZ3cLeRlfTGAc/Xf7FhpvyMZ24c8KYsLFF6aYUUl7JNiiFHqNkqmDN1wTxLed9W1nffGd1b6rhlP3Aa9RWqxyt5busjGCpLxJB8/YgR2Nfqkog6ALt2p2l9yHY2y3ugr3ObpkeaHkCL2AXqNyhN4VSj/RPHXBU5K0ta7g/GZCWazM6O6O6OdQkReqjEC/Ehtr7lyUfaI1HV0Ms0YEwZ2RBci6qLLLmWOBW6JZh3P0lemyVbbJ90hKOJvThe2Ltp2gSfSQ2u/8Ov4/IWUBSeBMdtWODCmRA63rL1Y1XnG24OnM04ztk/5x6G/0B8/+bXsseoULDYlHD9VXD1sjIL2+Rnse4pGL3zYKYlRsSYnYxpz2m+oAjR3epR/tMnjcJuK6HVVbKHpRsMSmwCdYkQUX9HfbxGELcRfv3rx5+vbZQBJZY0f3UHzIZ7WVHMqowiw1dQYHERUD20fJsD6YTveVJ8Xc3tzI3zJvZ77ZXP3H6/77UqOCT8KdKZdcqImRJudIiaLNunXo0a7JIy0EoI4de/hQjZCQ4REbX9JTblS8CY0rlMOP3acQbW5G/sP4x/GZVbxdLQGjUdJ0jF5wYd+zoQQS5YJySKf3vmxggJmDvVpaHK6aIW259t9yHWCTtjoG2m1qfO37gAMakVt4WWMYxMqF7C5gGB+oQaa/NbVNEmhjk1ZNH1oiXnh39+c4Mv2Va1Pu7JwO1G4VaEvoSyO8oE8QQ6Nf+gEIMUkLWiCMD92XsKosUFGjdfjRXr0JM57c3Am9eMULm4IU0rzGFNr2W9tAE6Clz4xUYRVjDaEB1WjJVO41XsHXEtKaDiR6w8wfDb2qAmTV9o7NA9RooUgZOdRhEKFIJpj1I6jtFNyHGKhrWZ2RCt8QVsm46dXz6+rXaRdxzZSefrF7ZXuTFuFxyJn3ynpePiuZ3GK3+h5bUPbZ0/fe6n8P0/fgkx31PYce7aPvRQhAHfre3WQMV4TskDf8D5TgGJcdLq5tog2cKPntGUqMqDUXN+M5F2ssUpLWw3XvNv3sblLl/oDZjZ2ZGneZufbl8u/uPJWwPU2um1HakmXaOfJbQl4rkm8Jed8S8r4l5LVi6iVjvmXmtSD+lpn3LTNvH+1/uLf3wEEQpseQucG5T8aLsSFphFz5qQctt50Hsz3fl95YwhSdUyLQ/feXz1rwqgPavNa37NC2RUyXJTYPhvrC66CxBf3h3cLE7yrlDHsunYvCmfbvZNlXNQLUGtXkc86FqvwzUwtn2p2cUGFD+wclCiKLbHuJ184tulpx5jZofUwGPjJtoCVRfTfq4ZPX/BQo60VdYlWVaTJXJxDsEmcmnESOjj2IesEFoiwRUF4bZ1CueIRWWNxAmJJa2pq/VUkpnKYNdyEypZ1W/JakY3SpUIIZmhHo7cbn6Ai+ORqhI/vO0Uh/cCQZzuWSq5b6eEsu1aTaXYddCU9WOXkO9wJBRS3L5cYKRFS6OKlmWPtbLlY4yzYloGaeUmntMfoZvN4HEkUfQxen5S7gId89jyRliY06y3myHKOP0rrCE77KC+Xce9N/9TyiCc+KVct9Q4IzwlIsooMpdl4dGzEjiE2LLK//TYBKlrkWR3RFwAdvvGJ2v9slK/2dOZdqIUh4yf3ePBx80119t6P7M6AG7R6gEhJy1zEqdf9r2zS4P3+Yq266Ir/z7vL+7ah+t9KrRPtl7tN9dSouP5o3rNXVNk5XlA262Hahjg2wpSsJKzxrppFXOFcbE8k1GGUUcr8r/BdPr5++PvQFfhqLxeu6iqzoeXQyPhlEzjMXZMfnCA+9eKrwXj1//fziGv0Jvfjw7g2sofyXQXT8BxxwZYeNr1ypVpA0qFX7Qf+7RUbDb925Mw4c+uoZWYbYUlr2FJaHM9GuvaCZy2fuNDVUxZopVpfEhw6G1xBD/K6q6hhdBGrjdIWlImI6QlOZ4Vui/5IsaZZO0X19Mn949uLh03cv0FqY7hbw24NRTDedakWCMpJN+8cLHSovoTEsSBXRg7klYsYljMsUmJ6CXjy1RaVbaL2TzdiAesAQoysXQwRFtU3zz1uteupT3LDALcUIu4vBoO9yBaaz6dJqSO2ZXlfpqxVmKSIQVN5WP9cdGOOD1V9+BVPFFogqCLCBruOGBqv/Grogyj4R3fHsB5UeldToOKxuyAFLsmusN2QTmmRuArQp2r04WBwymxXCisSi0IekNK214kQlOMs0SfZEkxupyMo70q7gQX+7wwDY0d6IFQONTUJKYmN/ZT90Cw/Qqors1s63gZmmqP/UIZvaUCTo7QwmqhmIHJkeQkWWVazyW4EzbYunKOXQEkNDAFmIjctE2tYBpUsHLuNtdSyWepprShKaEtfO3YRtKa7ft5SXgs0tDNonFCW2OqgrFqUIKg7vn2RLWfEZoFaR8l888hiaQOG0okrT013FoqW4bc/oTeCJHbDmgi8EXu2uOu2M+KCi+H0lix1h4EaUroTHdoIOr0T0yj/YL0oYxEgVIFv5Sk0ciESKbytvLuuX8zuXC7E70UqmRAuZq6tXetyUGapkv9IhXXmUfYqWLEkdcV3jPHoKrb6MC/YFplnpgb1ktzij6dHYeyeCY0UwkwgjWUCo27zIDLpxBcG+U3amhGWi0jbDM1llZV20CAojSMclfXV41RCxUmSVK2h6OYeX6/PcGZM1YEr1NtJ2g600Z+sM1ic3x1Lqo/QIZjQvZhlNbsjmqI2qxjW8Y8LID71IrQpz1mLJw/nSyskKN6uJlcqs4HneDMY7OH16ZisN3y6xtgx4Tphps7FakZRiRbKNo6qN6EipzQ7ZOoxgKLi515RKumBYFaLJ8L3oKD+v9f83PKZVuDbEsWiPLlnXg6DBtQ+ndkvrXTSm+bStBOLhiyDGyyC2R7UMiGvZXsdpQJki1q8gYr9AjbujjKoGn6He4Sp3RpZB2zlb26OFDkbd9jKSvUKJ+pSSHDBffctJDonOOdiUNcooxuiRRcrvUGMzelqZauViIDTWqbPqB2pxtWgj86d02INa9PbdNVzMFilv9tnueTYEMSAaWoKlOaI02NIj0a0gqUYHvZ7Yr6//0zsUA4y0zS/jHdrrHZWyxJb2SqkgieJiswcRERPEWyfB+Y66uMJiQZQ1U7jnJKoTKNdUJctINIGXQL+KHW/9pqrmwAQXqyZhi4Wk6cZp3Fq90z1nEe+47aKnT6+JgoWCg2VGKFuY+JZWpmnY8b21zS70l89aFbmDI4RF7MC45FtmsgWu/g7NeZZ6ETWMrI2nrk0/XpJIFcgeyFIyx0WmDIAOdFEWhxn4KjzuMH9xJvcVJz1LQMgd8FwrAZXHKoLec8neVTa7Ae25a/f2kG7LAvq+CqP6Pgr4zvp7xb2c2+BaT2J7hlubD3M7yPqXfZyQ7VBrca9KYDKnN94Ny7V5Miy0y360vcBQhQ/tc3EQxYe+SharI2WHPNY7aDVz0a/NTHcW585XfJU2AMWv1kvC0AxLmtSdgVRGnAw9kkH/GbuhfM3GNF+7E8s/UKY36ptYu2/vGAO65da+NQ32YEVKYY+bkG7T4XUc9UBGDUTzWZtbknHluSa5gAftYmKHvN8hI7bD/VgN13hgDzXawAm7U+eZLVnKB+04ky83kiY4s0g7CuV3ZrVGVeDeBAFsJ7AcZRU/dhDVmf+6J1Ea9i5EdedM7716AH4XujpTGdvJqkoJLkXMCN6J7frQ25GzfdAtwHMiTK0iezXTQdGW5O472AbDqPvS+2EQdTsy4N0u6baE85amG5Sl5PM5muOskQLQk+Z3roizd/CawKwZmXNBSjtstkGULYhUx/rNY/NmvfAs2jmpve/Z+a2524BZQ9+au/Wdom/N3Tro+MM2d4tTAmbvBLj4gEalVzrPYJBR9HPBmSIsbfd/7BbC5+9hhwOETtyyxcmNJqLNqbCFhrj7RZRtHCx4e8PnHA0U3FambNi9/z8AAP//eicJHQ==" } diff --git a/filebeat/input/log/harvester.go b/filebeat/input/log/harvester.go index cdb0aa78aa9..07408ffe316 100644 --- a/filebeat/input/log/harvester.go +++ b/filebeat/input/log/harvester.go @@ -51,6 +51,7 @@ import ( "github.com/elastic/beats/filebeat/input/file" "github.com/elastic/beats/filebeat/util" "github.com/elastic/beats/libbeat/reader" + "github.com/elastic/beats/libbeat/reader/debug" "github.com/elastic/beats/libbeat/reader/multiline" "github.com/elastic/beats/libbeat/reader/readfile" "github.com/elastic/beats/libbeat/reader/readfile/encoding" @@ -558,7 +559,12 @@ func (h *Harvester) newLogFileReader() (reader.Reader, error) { return nil, err } - r, err = readfile.NewEncodeReader(h.log, h.encoding, h.config.BufferSize) + reader, err := debug.AppendReaders(h.log) + if err != nil { + return nil, err + } + + r, err = readfile.NewEncodeReader(reader, h.encoding, h.config.BufferSize) if err != nil { return nil, err } diff --git a/filebeat/input/log/input.go b/filebeat/input/log/input.go index afba5fb4220..875c3a6b104 100644 --- a/filebeat/input/log/input.go +++ b/filebeat/input/log/input.go @@ -510,7 +510,7 @@ func (p *Input) harvestExistingFile(newState file.State, oldState file.State) { // File size was reduced -> truncated file if oldState.Finished && newState.Fileinfo.Size() < oldState.Offset { - logp.Debug("input", "Old file was truncated. Starting from the beginning: %s, offset: %d, new size: %d ", newState.Source, newState.Fileinfo.Size()) + logp.Debug("input", "Old file was truncated. Starting from the beginning: %s, offset: %d, new size: %d ", newState.Source, newState.Offset, newState.Fileinfo.Size()) err := p.startHarvester(newState, 0) if err != nil { logp.Err("Harvester could not be started on truncated file: %s, Err: %s", newState.Source, err) diff --git a/filebeat/magefile.go b/filebeat/magefile.go index bc31b29c174..984a3fdc960 100644 --- a/filebeat/magefile.go +++ b/filebeat/magefile.go @@ -75,7 +75,7 @@ func Clean() error { // Package packages the Beat for distribution. // Use SNAPSHOT=true to build snapshots. // Use PLATFORMS to control the target platforms. -// Use BEAT_VERSION_QUALIFIER to control the version qualifier. +// Use VERSION_QUALIFIER to control the version qualifier. func Package() { start := time.Now() defer func() { fmt.Println("package ran for", time.Since(start)) }() @@ -125,6 +125,15 @@ func GoTestIntegration(ctx context.Context) error { return mage.GoTest(ctx, mage.DefaultGoTestIntegrationArgs()) } +// ExportDashboard exports a dashboard and writes it into the correct directory +// +// Required ENV variables: +// * MODULE: Name of the module +// * ID: Dashboard id +func ExportDashboard() error { + return mage.ExportDashboard() +} + // ----------------------------------------------------------------------------- // Customizations specific to Filebeat. // - Include modules directory in packages (minus _meta and test files). diff --git a/filebeat/module/haproxy/_meta/fields.yml b/filebeat/module/haproxy/_meta/fields.yml index f38fecdf0ce..ce81845be61 100644 --- a/filebeat/module/haproxy/_meta/fields.yml +++ b/filebeat/module/haproxy/_meta/fields.yml @@ -2,41 +2,12 @@ title: "haproxy" description: > haproxy Module - fields: + fields: - name: haproxy type: group description: > fields: - - name: destination - description: Destination information - type: group - fields: - - name: port - description: Port of the destination host - type: long - - - name: ip - description: IP of the destination host - - - name: process_name - description: Name of the process - - - name: pid - description: PID of the process - type: long - - - name: client - description: Information about the client doing the request - type: group - fields: - - name: ip - description: IP address of the client which initiated the TCP connection to haproxy. - - - name: port - description: TCP port of the client which initiated the connection. - type: long - - name: frontend_name description: Name of the frontend (or listener) which received and processed the connection. @@ -80,40 +51,18 @@ - name: error_message description: Error message logged by HAProxy in case of error. type: text - + - name: source type: text description: The HAProxy source of the log - - - name: geoip - type: group - description: > - Contains GeoIP information gathered based on the client.ip field. - Only present if the GeoIP Elasticsearch plugin is available and - used. - fields: - - name: continent_name - type: keyword - description: Name of the continent. - - name: country_iso_code - type: keyword - description: Country ISO code. - - name: location - type: geo_point - description: Represents a geopoint with the longitude and latitude. - - name: region_name - type: keyword - description: Name of the region - - name: city_name - type: keyword - description: City name. - - name: region_iso_code - type: keyword - description: ISO code of the region - name: termination_state description: Condition the session was in when the session ended. + - name: mode + type: text + description: mode that the frontend is operating (TCP or HTTP) + - name: connections description: Contains various counts of connections active in the process. type: group @@ -137,6 +86,63 @@ - name: retries description: Number of connection retries experienced by this session when trying to connect to the server. type: long - - name: mode - type: text - description: mode that the frontend is operating (TCP or HTTP) + + - name: client + description: Information about the client doing the request + type: group + fields: + - name: ip + description: > + IP address of the client which initiated the TCP connection to haproxy. + + If connection is via unix socket, socket path is in this field. + - name: port + type: alias + path: source.port + + + + - name: process_name + type: alias + path: process.name + + - name: pid + type: alias + path: process.pid + + - name: destination + description: Destination information + type: group + fields: + - name: port + type: alias + path: destination.port + - name: ip + type: alias + path: destination.ip + + - name: geoip + type: group + description: > + Contains GeoIP information gathered based on the client.ip field. + Only present if the GeoIP Elasticsearch plugin is available and + used. + fields: + - name: continent_name + type: alias + path: source.geo.continent_name + - name: country_iso_code + type: alias + path: source.geo.country_iso_code + - name: location + type: alias + path: source.geo.location + - name: region_name + type: alias + path: source.geo.region_name + - name: city_name + type: alias + path: source.geo.city_name + - name: region_iso_code + type: alias + path: source.geo.region_iso_code diff --git a/filebeat/module/haproxy/log/ingest/pipeline.json b/filebeat/module/haproxy/log/ingest/pipeline.json index b0ae63df4a8..23ac50c334c 100644 --- a/filebeat/module/haproxy/log/ingest/pipeline.json +++ b/filebeat/module/haproxy/log/ingest/pipeline.json @@ -5,13 +5,13 @@ "grok": { "field": "message", "patterns": [ - "%{HAPROXY_DATE:haproxy.request_date} %{IPORHOST:haproxy.source} %{PROG:haproxy.process_name}(?:\\[%{POSINT:haproxy.pid}\\])?: %{GREEDYDATA} %{IPORHOST:haproxy.client.ip}:%{POSINT:haproxy.client.port} %{WORD} %{IPORHOST:haproxy.destination.ip}:%{POSINT:haproxy.destination.port} \\(%{WORD:haproxy.frontend_name}/%{WORD:haproxy.mode}\\)", + "%{HAPROXY_DATE:haproxy.request_date} %{IPORHOST:haproxy.source} %{PROG:process.name}(?:\\[%{POSINT:process.pid:int}\\])?: %{GREEDYDATA} %{IPORHOST:haproxy.client.ip}:%{POSINT:source.port:int} %{WORD} %{IPORHOST:destination.ip}:%{POSINT:destination.port:int} \\(%{WORD:haproxy.frontend_name}/%{WORD:haproxy.mode}\\)", - "(%{NOTSPACE:haproxy.process_name}\\[%{NUMBER:haproxy.pid:int}\\]: )?%{IP:haproxy.client.ip}:%{NUMBER:haproxy.client.port:int} \\[%{NOTSPACE:haproxy.request_date}\\] %{NOTSPACE:haproxy.frontend_name} %{NOTSPACE:haproxy.backend_name}/%{NOTSPACE:haproxy.server_name} %{NUMBER:haproxy.http.request.time_wait_ms:int}/%{NUMBER:haproxy.total_waiting_time_ms:int}/%{NUMBER:haproxy.connection_wait_time_ms:int}/%{NUMBER:haproxy.http.request.time_wait_without_data_ms:int}/%{NUMBER:haproxy.http.request.time_active_ms:int} %{NUMBER:haproxy.http.response.status_code:int} %{NUMBER:haproxy.bytes_read:int} %{NOTSPACE:haproxy.http.request.captured_cookie} %{NOTSPACE:haproxy.http.response.captured_cookie} %{NOTSPACE:haproxy.termination_state} %{NUMBER:haproxy.connections.active:int}/%{NUMBER:haproxy.connections.frontend:int}/%{NUMBER:haproxy.connections.backend:int}/%{NUMBER:haproxy.connections.server:int}/%{NUMBER:haproxy.connections.retries:int} %{NUMBER:haproxy.server_queue:int}/%{NUMBER:haproxy.backend_queue:int} \\{%{DATA:haproxy.http.request.captured_headers}\\} \\{%{DATA:haproxy.http.response.captured_headers}\\} \"%{GREEDYDATA:haproxy.http.request.raw_request_line}\"", + "(%{NOTSPACE:process.name}\\[%{NUMBER:process.pid:int}\\]: )?%{IP:haproxy.client.ip}:%{NUMBER:source.port:int} \\[%{NOTSPACE:haproxy.request_date}\\] %{NOTSPACE:haproxy.frontend_name} %{NOTSPACE:haproxy.backend_name}/%{NOTSPACE:haproxy.server_name} %{NUMBER:haproxy.http.request.time_wait_ms:int}/%{NUMBER:haproxy.total_waiting_time_ms:int}/%{NUMBER:haproxy.connection_wait_time_ms:int}/%{NUMBER:haproxy.http.request.time_wait_without_data_ms:int}/%{NUMBER:haproxy.http.request.time_active_ms:int} %{NUMBER:haproxy.http.response.status_code:int} %{NUMBER:haproxy.bytes_read:int} %{NOTSPACE:haproxy.http.request.captured_cookie} %{NOTSPACE:haproxy.http.response.captured_cookie} %{NOTSPACE:haproxy.termination_state} %{NUMBER:haproxy.connections.active:int}/%{NUMBER:haproxy.connections.frontend:int}/%{NUMBER:haproxy.connections.backend:int}/%{NUMBER:haproxy.connections.server:int}/%{NUMBER:haproxy.connections.retries:int} %{NUMBER:haproxy.server_queue:int}/%{NUMBER:haproxy.backend_queue:int} \\{%{DATA:haproxy.http.request.captured_headers}\\} \\{%{DATA:haproxy.http.response.captured_headers}\\} \"%{GREEDYDATA:haproxy.http.request.raw_request_line}\"", - "(%{NOTSPACE:haproxy.process_name}\\[%{NUMBER:haproxy.pid:int}\\]: )?%{IP:haproxy.client.ip}:%{NUMBER:haproxy.client.port:int} \\[%{NOTSPACE:haproxy.request_date}\\] %{NOTSPACE:haproxy.frontend_name}/%{NOTSPACE:haproxy.bind_name} %{GREEDYDATA:haproxy.error_message}", + "(%{NOTSPACE:process.name}\\[%{NUMBER:process.pid:int}\\]: )?%{IP:haproxy.client.ip}:%{NUMBER:source.port:int} \\[%{NOTSPACE:haproxy.request_date}\\] %{NOTSPACE:haproxy.frontend_name}/%{NOTSPACE:haproxy.bind_name} %{GREEDYDATA:haproxy.error_message}", - "%{HAPROXY_DATE} %{IPORHOST:haproxy.source} (%{NOTSPACE:haproxy.process_name}\\[%{NUMBER:haproxy.pid:int}\\]: )?%{IP:haproxy.client.ip}:%{NUMBER:haproxy.client.port:int} \\[%{NOTSPACE:haproxy.request_date}\\] %{NOTSPACE:haproxy.frontend_name} %{NOTSPACE:haproxy.backend_name}/%{NOTSPACE:haproxy.server_name} %{NUMBER:haproxy.total_waiting_time_ms:int}/%{NUMBER:haproxy.connection_wait_time_ms:int}/%{NUMBER:haproxy.tcp.processing_time_ms:int} %{NUMBER:haproxy.bytes_read:int} %{NOTSPACE:haproxy.termination_state} %{NUMBER:haproxy.connections.active:int}/%{NUMBER:haproxy.connections.frontend:int}/%{NUMBER:haproxy.connections.backend:int}/%{NUMBER:haproxy.connections.server:int}/%{NUMBER:haproxy.connections.retries:int} %{NUMBER:haproxy.server_queue:int}/%{NUMBER:haproxy.backend_queue:int}" + "%{HAPROXY_DATE} %{IPORHOST:haproxy.source} (%{NOTSPACE:process.name}\\[%{NUMBER:process.pid:int}\\]: )?%{IP:haproxy.client.ip}:%{NUMBER:source.port:int} \\[%{NOTSPACE:haproxy.request_date}\\] %{NOTSPACE:haproxy.frontend_name} %{NOTSPACE:haproxy.backend_name}/%{NOTSPACE:haproxy.server_name} %{NUMBER:haproxy.total_waiting_time_ms:int}/%{NUMBER:haproxy.connection_wait_time_ms:int}/%{NUMBER:haproxy.tcp.processing_time_ms:int} %{NUMBER:haproxy.bytes_read:int} %{NOTSPACE:haproxy.termination_state} %{NUMBER:haproxy.connections.active:int}/%{NUMBER:haproxy.connections.frontend:int}/%{NUMBER:haproxy.connections.backend:int}/%{NUMBER:haproxy.connections.server:int}/%{NUMBER:haproxy.connections.retries:int} %{NUMBER:haproxy.server_queue:int}/%{NUMBER:haproxy.backend_queue:int}" ], "ignore_missing": false, "pattern_definitions": { @@ -40,9 +40,19 @@ } }, { - "geoip": { + "grok": { "field": "haproxy.client.ip", - "target_field": "haproxy.geoip" + "ignore_missing": true, + "patterns": [ + "^(%{IP:source.ip}|%{HOSTNAME:source.domain})$" + ] + } + }, + { + "geoip": { + "field": "source.ip", + "target_field": "source.geo", + "ignore_missing": true } }, { @@ -68,4 +78,4 @@ } } ] -} \ No newline at end of file +} diff --git a/filebeat/module/haproxy/log/test/default.log-expected.json b/filebeat/module/haproxy/log/test/default.log-expected.json index 99a36d83793..3bbeb68f8bd 100644 --- a/filebeat/module/haproxy/log/test/default.log-expected.json +++ b/filebeat/module/haproxy/log/test/default.log-expected.json @@ -1,22 +1,23 @@ [ { "@timestamp": "2018-09-20T15:42:59.000Z", + "destination.ip": "1.2.3.4", + "destination.port": 5000, "event.dataset": "log", "event.module": "haproxy", "haproxy.client.ip": "1.2.3.4", - "haproxy.client.port": "40780", - "haproxy.destination.ip": "1.2.3.4", - "haproxy.destination.port": "5000", "haproxy.frontend_name": "main", - "haproxy.geoip.continent_name": "North America", - "haproxy.geoip.country_iso_code": "US", - "haproxy.geoip.location.lat": 37.751, - "haproxy.geoip.location.lon": -97.822, "haproxy.mode": "HTTP", - "haproxy.pid": "24551", - "haproxy.process_name": "haproxy", "haproxy.source": "1.2.3.4", "input.type": "log", - "log.offset": 0 + "log.offset": 0, + "process.name": "haproxy", + "process.pid": 24551, + "source.geo.continent_name": "North America", + "source.geo.country_iso_code": "US", + "source.geo.location.lat": 37.751, + "source.geo.location.lon": -97.822, + "source.ip": "1.2.3.4", + "source.port": 40780 } ] \ No newline at end of file diff --git a/filebeat/module/haproxy/log/test/haproxy.log-expected.json b/filebeat/module/haproxy/log/test/haproxy.log-expected.json index 13c503f8b0d..e989d5f5b76 100644 --- a/filebeat/module/haproxy/log/test/haproxy.log-expected.json +++ b/filebeat/module/haproxy/log/test/haproxy.log-expected.json @@ -7,7 +7,6 @@ "haproxy.backend_queue": 0, "haproxy.bytes_read": 168, "haproxy.client.ip": "1.2.3.4", - "haproxy.client.port": 38862, "haproxy.connection_wait_time_ms": 1, "haproxy.connections.active": 6, "haproxy.connections.backend": 0, @@ -15,10 +14,6 @@ "haproxy.connections.retries": 0, "haproxy.connections.server": 0, "haproxy.frontend_name": "incoming~", - "haproxy.geoip.continent_name": "North America", - "haproxy.geoip.country_iso_code": "US", - "haproxy.geoip.location.lat": 37.751, - "haproxy.geoip.location.lon": -97.822, "haproxy.http.request.captured_cookie": "-", "haproxy.http.request.captured_headers": [ "docs.example.internal" @@ -30,13 +25,19 @@ "haproxy.http.response.captured_cookie": "-", "haproxy.http.response.captured_headers": [], "haproxy.http.response.status_code": 304, - "haproxy.pid": 32450, - "haproxy.process_name": "haproxy", "haproxy.server_name": "docs", "haproxy.server_queue": 0, "haproxy.termination_state": "----", "haproxy.total_waiting_time_ms": 0, "input.type": "log", - "log.offset": 0 + "log.offset": 0, + "process.name": "haproxy", + "process.pid": 32450, + "source.geo.continent_name": "North America", + "source.geo.country_iso_code": "US", + "source.geo.location.lat": 37.751, + "source.geo.location.lon": -97.822, + "source.ip": "1.2.3.4", + "source.port": 38862 } ] \ No newline at end of file diff --git a/filebeat/module/haproxy/log/test/tcplog.log-expected.json b/filebeat/module/haproxy/log/test/tcplog.log-expected.json index f9ef4f67ece..68ebe5798c6 100644 --- a/filebeat/module/haproxy/log/test/tcplog.log-expected.json +++ b/filebeat/module/haproxy/log/test/tcplog.log-expected.json @@ -7,7 +7,6 @@ "haproxy.backend_queue": 0, "haproxy.bytes_read": 212, "haproxy.client.ip": "127.0.0.1", - "haproxy.client.port": 40962, "haproxy.connection_wait_time_ms": -1, "haproxy.connections.active": 1, "haproxy.connections.backend": 0, @@ -15,8 +14,6 @@ "haproxy.connections.retries": 0, "haproxy.connections.server": 0, "haproxy.frontend_name": "main", - "haproxy.pid": 25457, - "haproxy.process_name": "haproxy", "haproxy.server_name": "", "haproxy.server_queue": 0, "haproxy.source": "127.0.0.1", @@ -24,6 +21,10 @@ "haproxy.termination_state": "SC", "haproxy.total_waiting_time_ms": -1, "input.type": "log", - "log.offset": 0 + "log.offset": 0, + "process.name": "haproxy", + "process.pid": 25457, + "source.ip": "127.0.0.1", + "source.port": 40962 } ] \ No newline at end of file diff --git a/filebeat/module/iis/_meta/fields.yml b/filebeat/module/iis/_meta/fields.yml index 686d29271ed..66710a75974 100644 --- a/filebeat/module/iis/_meta/fields.yml +++ b/filebeat/module/iis/_meta/fields.yml @@ -8,3 +8,4 @@ description: > Fields from IIS log files. fields: + diff --git a/filebeat/module/iis/access/_meta/fields.yml b/filebeat/module/iis/access/_meta/fields.yml index 6167860b515..89d5f91ab00 100644 --- a/filebeat/module/iis/access/_meta/fields.yml +++ b/filebeat/module/iis/access/_meta/fields.yml @@ -3,43 +3,6 @@ description: > Contains fields for IIS access logs. fields: - - name: server_ip - type: keyword - description: > - The server IP address. - - name: method - type: keyword - example: GET - description: > - The request HTTP method. - - name: url - type: keyword - description: > - The request HTTP URL. - - name: query_string - type: keyword - description: > - The request query string, if any. - - name: port - type: long - description: > - The request port number. - - name: user_name - type: keyword - description: > - The user name used when basic authentication is used. - - name: remote_ip - type: keyword - description: > - The client IP address. - - name: referrer - type: keyword - description: > - The HTTP referrer. - - name: response_code - type: long - description: > - The HTTP response code. - name: sub_status type: long description: > @@ -60,18 +23,10 @@ type: keyword description: > The name of the server on which the log file entry was generated. - - name: http_version - type: keyword - description: > - The HTTP version. - name: cookie type: keyword description: > The content of the cookie sent or received, if any. - - name: hostname - type: keyword - description: > - The host header name, if any. - name: body_sent.bytes type: long format: bytes @@ -82,87 +37,91 @@ format: bytes description: > The number of bytes of the server request body. - - name: agent - type: text - description: > - Contains the un-parsed user agent string. Only present if the user - agent Elasticsearch plugin is not available or not used. + + - name: server_ip + type: alias + path: destination.ip + - name: method + type: alias + path: http.request.method + - name: url + type: alias + path: url.path + - name: query_string + type: alias + path: url.query + - name: port + type: alias + path: destination.port + - name: user_name + type: alias + path: user.name + - name: remote_ip + type: alias + path: source.ip + - name: referrer + type: alias + path: http.request.referrer + - name: response_code + type: alias + path: http.response.status_code + - name: http_version + type: alias + path: http.version + - name: hostname + type: alias + path: host.hostname - name: user_agent type: group - description: > - Contains the parsed user agent field. Only present if the user - agent Elasticsearch plugin is available and used. fields: - name: device - type: keyword - description: > - The name of the physical device. + type: alias + path: user_agent.device - name: major - type: long - description: > - The major version of the user agent. + type: alias + path: user_agent.major - name: minor - type: long - description: > - The minor version of the user agent. + type: alias + path: user_agent.minor - name: patch - type: keyword - description: > - The patch version of the user agent. + type: alias + path: user_agent.patch - name: name - type: keyword - example: Chrome - description: > - The name of the user agent. + type: alias + path: user_agent.name - name: os - type: keyword - description: > - The name of the operating system. + type: alias + path: user_agent.os.full_name - name: os_major - type: long - description: > - The major version of the operating system. + type: alias + path: user_agent.os.major - name: os_minor - type: long - description: > - The minor version of the operating system. + type: alias + path: user_agent.os.minor - name: os_name - type: keyword - description: > - The name of the operating system. + type: alias + path: user_agent.os.name - name: original - type: text - index: false - description: > - Original user agent value before parsing by ingest-user-agent plugin. + type: alias + path: user_agent.original - name: geoip type: group - description: > - Contains GeoIP information gathered based on the remote_ip field. - Only present if the GeoIP Elasticsearch plugin is available and - used. fields: - name: continent_name - type: keyword - description: > - The name of the continent. + type: alias + path: source.geo.continent_name - name: country_iso_code - type: keyword - description: > - Country ISO code. + type: alias + path: source.geo.country_iso_code - name: location - type: geo_point - description: > - The longitude and latitude. + type: alias + path: source.geo.location - name: region_name - type: keyword - description: > - The region name. + type: alias + path: source.geo.region_name - name: city_name - type: keyword - description: > - The city name. + type: alias + path: source.geo.city_name - name: region_iso_code - type: keyword - description: > - Region ISO code. + type: alias + path: source.geo.region_iso_code diff --git a/filebeat/module/iis/access/ingest/default.json b/filebeat/module/iis/access/ingest/default.json index 4cbe512f5c4..26d4973c868 100644 --- a/filebeat/module/iis/access/ingest/default.json +++ b/filebeat/module/iis/access/ingest/default.json @@ -4,9 +4,9 @@ "grok": { "field": "message", "patterns":[ - "%{TIMESTAMP_ISO8601:iis.access.time} %{IPORHOST:iis.access.server_ip} %{WORD:iis.access.method} %{URIPATH:iis.access.url} %{NOTSPACE:iis.access.query_string} %{NUMBER:iis.access.port} %{NOTSPACE:iis.access.user_name} %{IPORHOST:iis.access.remote_ip} %{NOTSPACE:iis.access.agent} %{NOTSPACE:iis.access.referrer} %{NUMBER:iis.access.response_code} %{NUMBER:iis.access.sub_status} %{NUMBER:iis.access.win32_status} %{NUMBER:iis.access.request_time_ms}", - "%{TIMESTAMP_ISO8601:iis.access.time} %{NOTSPACE:iis.access.site_name} %{WORD:iis.access.method} %{URIPATH:iis.access.url} %{NOTSPACE:iis.access.query_string} %{NUMBER:iis.access.port} %{NOTSPACE:iis.access.user_name} %{IPORHOST:iis.access.remote_ip} %{NOTSPACE:iis.access.agent} %{NOTSPACE:iis.access.cookie} %{NOTSPACE:iis.access.referrer} %{NOTSPACE:iis.access.hostname} %{NUMBER:iis.access.response_code} %{NUMBER:iis.access.sub_status} %{NUMBER:iis.access.win32_status} %{NUMBER:iis.access.body_sent.bytes} %{NUMBER:iis.access.body_received.bytes} %{NUMBER:iis.access.request_time_ms}", - "%{TIMESTAMP_ISO8601:iis.access.time} %{NOTSPACE:iis.access.site_name} %{NOTSPACE:iis.access.server_name} %{IPORHOST:iis.access.server_ip} %{WORD:iis.access.method} %{URIPATH:iis.access.url} %{NOTSPACE:iis.access.query_string} %{NUMBER:iis.access.port} %{NOTSPACE:iis.access.user_name} %{IPORHOST:iis.access.remote_ip} HTTP/%{NUMBER:iis.access.http_version} %{NOTSPACE:iis.access.agent} %{NOTSPACE:iis.access.cookie} %{NOTSPACE:iis.access.referrer} %{NOTSPACE:iis.access.hostname} %{NUMBER:iis.access.response_code} %{NUMBER:iis.access.sub_status} %{NUMBER:iis.access.win32_status} %{NUMBER:iis.access.body_sent.bytes} %{NUMBER:iis.access.body_received.bytes} %{NUMBER:iis.access.request_time_ms}" + "%{TIMESTAMP_ISO8601:iis.access.time} %{IPORHOST:destination.ip} %{WORD:http.request.method} %{URIPATH:url.path} %{NOTSPACE:url.query} %{NUMBER:destination.port:int} %{NOTSPACE:user.name} %{IPORHOST:source.ip} %{NOTSPACE:iis.access.agent} %{NOTSPACE:http.request.referrer} %{NUMBER:http.response.status_code:int} %{NUMBER:iis.access.sub_status:int} %{NUMBER:iis.access.win32_status:int} %{NUMBER:iis.access.request_time_ms:int}", + "%{TIMESTAMP_ISO8601:iis.access.time} %{NOTSPACE:iis.access.site_name} %{WORD:http.request.method} %{URIPATH:url.path} %{NOTSPACE:url.query} %{NUMBER:destination.port:int} %{NOTSPACE:user.name} %{IPORHOST:source.ip} %{NOTSPACE:iis.access.agent} %{NOTSPACE:iis.access.cookie} %{NOTSPACE:http.request.referrer} %{NOTSPACE:destination.domain} %{NUMBER:http.response.status_code:int} %{NUMBER:iis.access.sub_status:int} %{NUMBER:iis.access.win32_status:int} %{NUMBER:iis.access.body_sent.bytes:int} %{NUMBER:iis.access.body_received.bytes:int} %{NUMBER:iis.access.request_time_ms:int}", + "%{TIMESTAMP_ISO8601:iis.access.time} %{NOTSPACE:iis.access.site_name} %{NOTSPACE:iis.access.server_name} %{IPORHOST:destination.ip} %{WORD:http.request.method} %{URIPATH:url.path} %{NOTSPACE:url.query} %{NUMBER:destination.port:int} %{NOTSPACE:user.name} %{IPORHOST:source.ip} HTTP/%{NUMBER:http.version} %{NOTSPACE:iis.access.agent} %{NOTSPACE:iis.access.cookie} %{NOTSPACE:http.request.referrer} %{NOTSPACE:destination.domain} %{NUMBER:http.response.status_code:int} %{NUMBER:iis.access.sub_status:int} %{NUMBER:iis.access.win32_status:int} %{NUMBER:iis.access.body_sent.bytes:int} %{NUMBER:iis.access.body_received.bytes:int} %{NUMBER:iis.access.request_time_ms:int}" ], "ignore_missing": true } @@ -29,20 +29,53 @@ "remove": { "field": "iis.access.time" } + }, { + "urldecode": { + "field": "iis.access.agent" + } }, { "user_agent": { - "field": "iis.access.agent", - "target_field": "iis.access.user_agent" + "field": "iis.access.agent" + } + }, { + "rename": { + "field": "user_agent.os", + "target_field": "user_agent.os.full_name", + "ignore_missing": true + } + }, { + "rename": { + "field": "user_agent.os_name", + "target_field": "user_agent.os.name", + "ignore_missing": true + } + }, { + "rename": { + "field": "user_agent.os_major", + "target_field": "user_agent.os.major", + "ignore_missing": true + } + }, { + "rename": { + "field": "user_agent.os_minor", + "target_field": "user_agent.os.minor", + "ignore_missing": true + } + }, { + "rename": { + "field": "user_agent.os_patch", + "target_field": "user_agent.os.patch", + "ignore_missing": true } }, { "rename": { "field": "iis.access.agent", - "target_field": "iis.access.user_agent.original" + "target_field": "user_agent.original" } }, { "geoip": { - "field": "iis.access.remote_ip", - "target_field": "iis.access.geoip" + "field": "source.ip", + "target_field": "source.geo" } }], "on_failure" : [{ diff --git a/filebeat/module/iis/access/test/test.log b/filebeat/module/iis/access/test/test.log index a0c219151d3..d5f30c9f0c2 100644 --- a/filebeat/module/iis/access/test/test.log +++ b/filebeat/module/iis/access/test/test.log @@ -12,4 +12,4 @@ #Version: 1.0 #Date: 2018-01-01 10:11:12 #Fields: date time s-sitename s-computername s-ip cs-method cs-uri-stem cs-uri-query s-port cs-username c-ip cs-version cs(User-Agent) cs(Cookie) cs(Referer) cs-host sc-status sc-substatus sc-win32-status sc-bytes cs-bytes time-taken -2018-01-01 10:11:12 W3SVC1 MACHINE-NAME 127.0.0.1 GET / - 80 - 85.181.35.98 HTTP/1.1 Mozilla/5.0+(Windows+NT+6.1;+Win64;+x64;+rv:57.0)+Gecko/20100101+Firefox/57.0 - - example.com 200 0 0 123 456 789 +2018-01-01 10:11:12 W3SVC1 MACHINE-NAME 127.0.0.1 GET / - 80 - 85.181.35.98 HTTP/1.1 Mozilla/5.0+(Macintosh;+Intel+Mac+OS+X+10_14_0)+AppleWebKit/537.36+(KHTML,+like+Gecko)+Chrome/70.0.3538.102+Safari/537.36 - - example.com 200 0 0 123 456 789 diff --git a/filebeat/module/iis/access/test/test.log-expected.json b/filebeat/module/iis/access/test/test.log-expected.json index 83f130f4783..7e01cd9613d 100644 --- a/filebeat/module/iis/access/test/test.log-expected.json +++ b/filebeat/module/iis/access/test/test.log-expected.json @@ -1,105 +1,108 @@ [ { "@timestamp": "2018-01-01T08:09:10.000Z", + "destination.ip": "127.0.0.1", + "destination.port": 80, "event.dataset": "access", "event.module": "iis", - "iis.access.geoip.city_name": "Berlin", - "iis.access.geoip.continent_name": "Europe", - "iis.access.geoip.country_iso_code": "DE", - "iis.access.geoip.location.lat": 52.4908, - "iis.access.geoip.location.lon": 13.3275, - "iis.access.geoip.region_iso_code": "DE-BE", - "iis.access.geoip.region_name": "Land Berlin", - "iis.access.method": "GET", - "iis.access.port": "80", - "iis.access.query_string": "q=100", - "iis.access.referrer": "-", - "iis.access.remote_ip": "85.181.35.98", - "iis.access.request_time_ms": "123", - "iis.access.response_code": "200", - "iis.access.server_ip": "127.0.0.1", - "iis.access.sub_status": "0", - "iis.access.url": "/", - "iis.access.user_agent.device": "Other", - "iis.access.user_agent.major": "57", - "iis.access.user_agent.minor": "0", - "iis.access.user_agent.name": "Firefox", - "iis.access.user_agent.original": "Mozilla/5.0+(Windows+NT+6.1;+Win64;+x64;+rv:57.0)+Gecko/20100101+Firefox/57.0", - "iis.access.user_agent.os": "Windows", - "iis.access.user_agent.os_name": "Windows", - "iis.access.user_name": "-", - "iis.access.win32_status": "0", + "http.request.method": "GET", + "http.request.referrer": "-", + "http.response.status_code": 200, + "iis.access.request_time_ms": 123, + "iis.access.sub_status": 0, + "iis.access.win32_status": 0, "input.type": "log", - "log.offset": 257 + "log.offset": 257, + "source.geo.city_name": "Berlin", + "source.geo.continent_name": "Europe", + "source.geo.country_iso_code": "DE", + "source.geo.location.lat": 52.4908, + "source.geo.location.lon": 13.3275, + "source.geo.region_iso_code": "DE-BE", + "source.geo.region_name": "Land Berlin", + "source.ip": "85.181.35.98", + "url.path": "/", + "url.query": "q=100", + "user.name": "-", + "user_agent.device": "Other", + "user_agent.major": "57", + "user_agent.minor": "0", + "user_agent.name": "Firefox", + "user_agent.original": "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:57.0) Gecko/20100101 Firefox/57.0", + "user_agent.os.full_name": "Windows 7", + "user_agent.os.name": "Windows 7" }, { "@timestamp": "2018-01-01T09:10:11.000Z", + "destination.domain": "example.com", + "destination.port": 80, "event.dataset": "access", "event.module": "iis", - "iis.access.body_received.bytes": "456", - "iis.access.body_sent.bytes": "123", + "http.request.method": "GET", + "http.request.referrer": "-", + "http.response.status_code": 200, + "iis.access.body_received.bytes": 456, + "iis.access.body_sent.bytes": 123, "iis.access.cookie": "-", - "iis.access.hostname": "example.com", - "iis.access.method": "GET", - "iis.access.port": "80", - "iis.access.query_string": "-", - "iis.access.referrer": "-", - "iis.access.remote_ip": "127.0.0.1", - "iis.access.request_time_ms": "789", - "iis.access.response_code": "200", + "iis.access.request_time_ms": 789, "iis.access.site_name": "W3SVC1", - "iis.access.sub_status": "0", - "iis.access.url": "/", - "iis.access.user_agent.device": "Other", - "iis.access.user_agent.major": "57", - "iis.access.user_agent.minor": "0", - "iis.access.user_agent.name": "Firefox", - "iis.access.user_agent.original": "Mozilla/5.0+(Windows+NT+6.1;+Win64;+x64;+rv:57.0)+Gecko/20100101+Firefox/57.0", - "iis.access.user_agent.os": "Windows", - "iis.access.user_agent.os_name": "Windows", - "iis.access.user_name": "-", - "iis.access.win32_status": "0", + "iis.access.sub_status": 0, + "iis.access.win32_status": 0, "input.type": "log", - "log.offset": 709 + "log.offset": 709, + "source.ip": "127.0.0.1", + "url.path": "/", + "url.query": "-", + "user.name": "-", + "user_agent.device": "Other", + "user_agent.major": "57", + "user_agent.minor": "0", + "user_agent.name": "Firefox", + "user_agent.original": "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:57.0) Gecko/20100101 Firefox/57.0", + "user_agent.os.full_name": "Windows 7", + "user_agent.os.name": "Windows 7" }, { "@timestamp": "2018-01-01T10:11:12.000Z", + "destination.domain": "example.com", + "destination.ip": "127.0.0.1", + "destination.port": 80, "event.dataset": "access", "event.module": "iis", - "iis.access.body_received.bytes": "456", - "iis.access.body_sent.bytes": "123", + "http.request.method": "GET", + "http.request.referrer": "-", + "http.response.status_code": 200, + "http.version": "1.1", + "iis.access.body_received.bytes": 456, + "iis.access.body_sent.bytes": 123, "iis.access.cookie": "-", - "iis.access.geoip.city_name": "Berlin", - "iis.access.geoip.continent_name": "Europe", - "iis.access.geoip.country_iso_code": "DE", - "iis.access.geoip.location.lat": 52.4908, - "iis.access.geoip.location.lon": 13.3275, - "iis.access.geoip.region_iso_code": "DE-BE", - "iis.access.geoip.region_name": "Land Berlin", - "iis.access.hostname": "example.com", - "iis.access.http_version": "1.1", - "iis.access.method": "GET", - "iis.access.port": "80", - "iis.access.query_string": "-", - "iis.access.referrer": "-", - "iis.access.remote_ip": "85.181.35.98", - "iis.access.request_time_ms": "789", - "iis.access.response_code": "200", - "iis.access.server_ip": "127.0.0.1", + "iis.access.request_time_ms": 789, "iis.access.server_name": "MACHINE-NAME", "iis.access.site_name": "W3SVC1", - "iis.access.sub_status": "0", - "iis.access.url": "/", - "iis.access.user_agent.device": "Other", - "iis.access.user_agent.major": "57", - "iis.access.user_agent.minor": "0", - "iis.access.user_agent.name": "Firefox", - "iis.access.user_agent.original": "Mozilla/5.0+(Windows+NT+6.1;+Win64;+x64;+rv:57.0)+Gecko/20100101+Firefox/57.0", - "iis.access.user_agent.os": "Windows", - "iis.access.user_agent.os_name": "Windows", - "iis.access.user_name": "-", - "iis.access.win32_status": "0", + "iis.access.sub_status": 0, + "iis.access.win32_status": 0, "input.type": "log", - "log.offset": 1204 + "log.offset": 1204, + "source.geo.city_name": "Berlin", + "source.geo.continent_name": "Europe", + "source.geo.country_iso_code": "DE", + "source.geo.location.lat": 52.4908, + "source.geo.location.lon": 13.3275, + "source.geo.region_iso_code": "DE-BE", + "source.geo.region_name": "Land Berlin", + "source.ip": "85.181.35.98", + "url.path": "/", + "url.query": "-", + "user.name": "-", + "user_agent.device": "Other", + "user_agent.major": "70", + "user_agent.minor": "0", + "user_agent.name": "Chrome", + "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36", + "user_agent.os.full_name": "Mac OS X 10.14.0", + "user_agent.os.major": "10", + "user_agent.os.minor": "14", + "user_agent.os.name": "Mac OS X", + "user_agent.patch": "3538" } ] \ No newline at end of file diff --git a/filebeat/module/nginx/access/_meta/fields.yml b/filebeat/module/nginx/access/_meta/fields.yml index 58fd4d6d04b..c4801bef1ec 100644 --- a/filebeat/module/nginx/access/_meta/fields.yml +++ b/filebeat/module/nginx/access/_meta/fields.yml @@ -3,128 +3,92 @@ description: > Contains fields for the Nginx access logs. fields: - - name: remote_ip_list - type: array + - name: body_sent.bytes + type: long + format: bytes description: > - An array of remote IP addresses. It is a list because it is common to include, besides the client - IP address, IP addresses from headers like `X-Forwarded-For`. See also the `remote_ip` field. + The number of bytes of the server response body. + + - name: remote_ip_list + type: alias + path: network.forwarded_ip - name: remote_ip - type: keyword - description: > - Client IP address. The first public IP address from the `remote_ip_list` array. If no public IP - addresses are present, this field contains the first private IP address from the `remote_ip_list` - array. + type: alias + path: source.ip - name: user_name - type: keyword - description: > - The user name used when basic authentication is used. + type: alias + path: user.name - name: method - type: keyword - example: GET - description: > - The request HTTP method. + type: alias + path: http.request.method - name: url - type: keyword - description: > - The request HTTP URL. + type: alias + path: url.original - name: http_version - type: keyword - description: > - The HTTP version. + type: alias + path: http.version - name: response_code - type: long - description: > - The HTTP response code. - - name: body_sent.bytes - type: long - format: bytes - description: > - The number of bytes of the server response body. + type: alias + path: http.response.status_code - name: referrer - type: keyword - description: > - The HTTP referrer. + type: alias + path: http.request.referrer - name: agent - type: text - description: > - Contains the un-parsed user agent string. Only present if the user - agent Elasticsearch plugin is not available or not used. + type: alias + path: user_agent.original + - name: user_agent type: group - description: > - Contains the parsed User agent field. Only present if the user - agent Elasticsearch plugin is available and used. fields: - name: device - type: keyword - description: > - The name of the physical device. + type: alias + path: user_agent.device - name: major - type: long - description: > - The major version of the user agent. + type: alias + path: user_agent.major - name: minor - type: long - description: > - The minor version of the user agent. + type: alias + path: user_agent.minor - name: patch - type: keyword - description: > - The patch version of the user agent. + type: alias + path: user_agent.patch - name: name - type: keyword - example: Chrome - description: > - The name of the user agent. + type: alias + path: user_agent.name - name: os - type: keyword - description: > - The name of the operating system. + type: alias + path: user_agent.os.full_name - name: os_major - type: long - description: > - The major version of the operating system. + type: alias + path: user_agent.os.major - name: os_minor - type: long - description: > - The minor version of the operating system. + type: alias + path: user_agent.os.minor - name: os_name - type: keyword - description: > - The name of the operating system. + type: alias + path: user_agent.os.name - name: original - type: text - index: false - description: > - Original user agent value before parsing by ingest-user-agent plugin. + type: alias + path: user_agent.original + - name: geoip type: group - description: > - Contains GeoIP information gathered based on the remote_ip field. - Only present if the GeoIP Elasticsearch plugin is available and - used. fields: - name: continent_name - type: keyword - description: > - The name of the continent. + type: alias + path: source.geo.continent_name - name: country_iso_code - type: keyword - description: > - Country ISO code. + type: alias + path: source.geo.country_iso_code - name: location - type: geo_point - description: > - The longitude and latitude. + type: alias + path: source.geo.location - name: region_name - type: keyword - description: > - The region name. + type: alias + path: source.geo.region_name - name: city_name - type: keyword - description: > - The city name. + type: alias + path: source.geo.city_name - name: region_iso_code - type: keyword - description: > - Region ISO code. + type: alias + path: source.geo.region_iso_code diff --git a/filebeat/module/nginx/access/ingest/default.json b/filebeat/module/nginx/access/ingest/default.json index dfeae281f08..be4398272a9 100644 --- a/filebeat/module/nginx/access/ingest/default.json +++ b/filebeat/module/nginx/access/ingest/default.json @@ -4,7 +4,7 @@ "grok": { "field": "message", "patterns":[ - "\"?%{IP_LIST:nginx.access.remote_ip_list} - %{DATA:nginx.access.user_name} \\[%{HTTPDATE:nginx.access.time}\\] \"%{GREEDYDATA:nginx.access.info}\" %{NUMBER:nginx.access.response_code} %{NUMBER:nginx.access.body_sent.bytes} \"%{DATA:nginx.access.referrer}\" \"%{DATA:nginx.access.agent}\"" + "\"?%{IP_LIST:network.forwarded_ip} - %{DATA:user.name} \\[%{HTTPDATE:nginx.access.time}\\] \"%{GREEDYDATA:nginx.access.info}\" %{NUMBER:http.response.status_code:int} %{NUMBER:nginx.access.body_sent.bytes:int} \"%{DATA:http.request.referrer}\" \"%{DATA:nginx.access.agent}\"" ], "pattern_definitions": { "IP_LIST": "%{IP}(\"?,?\\s*%{IP})*" @@ -15,7 +15,7 @@ "grok": { "field": "nginx.access.info", "patterns": [ - "%{WORD:nginx.access.method} %{DATA:nginx.access.url} HTTP/%{NUMBER:nginx.access.http_version}", + "%{WORD:http.request.method} %{DATA:url.original} HTTP/%{NUMBER:http.version}", "" ], "ignore_missing": true @@ -26,13 +26,18 @@ } }, { "split": { - "field": "nginx.access.remote_ip_list", + "field": "network.forwarded_ip", "separator": "\"?,?\\s+" } + }, { + "set": { + "field": "source.ip", + "value": "" + } }, { "script": { "lang": "painless", - "inline": "boolean isPrivate(def ip) { try { StringTokenizer tok = new StringTokenizer(ip, '.'); int firstByte = Integer.parseInt(tok.nextToken()); int secondByte = Integer.parseInt(tok.nextToken()); if (firstByte == 10) { return true; } if (firstByte == 192 && secondByte == 168) { return true; } if (firstByte == 172 && secondByte >= 16 && secondByte <= 31) { return true; } if (firstByte == 127) { return true; } return false; } catch (Exception e) { return false; } } def found = false; for (def item : ctx.nginx.access.remote_ip_list) { if (!isPrivate(item)) { ctx.nginx.access.remote_ip = item; found = true; break; } } if (!found) { ctx.nginx.access.remote_ip = ctx.nginx.access.remote_ip_list[0]; }" + "inline": "boolean isPrivate(def ip) { try { StringTokenizer tok = new StringTokenizer(ip, '.'); int firstByte = Integer.parseInt(tok.nextToken()); int secondByte = Integer.parseInt(tok.nextToken()); if (firstByte == 10) { return true; } if (firstByte == 192 && secondByte == 168) { return true; } if (firstByte == 172 && secondByte >= 16 && secondByte <= 31) { return true; } if (firstByte == 127) { return true; } return false; } catch (Exception e) { return false; } } def found = false; for (def item : ctx.network.forwarded_ip) { if (!isPrivate(item)) { ctx.source.ip = item; found = true; break; } } if (!found) { ctx.source.ip = ctx.network.forwarded_ip[0]; }" } }, { "remove":{ @@ -53,20 +58,49 @@ "remove": { "field": "nginx.access.time" } + + }, { "user_agent": { "field": "nginx.access.agent" } }, { - "user_agent": { + "rename": { "field": "nginx.access.agent", - "target_field": "nginx.access.user_agent" + "target_field": "user_agent.original" } }, { "rename": { - "field": "nginx.access.agent", - "target_field": "nginx.access.user_agent.original" + "field": "user_agent.os", + "target_field": "user_agent.os.full_name", + "ignore_missing": true } }, { + "rename": { + "field": "user_agent.os_name", + "target_field": "user_agent.os.name", + "ignore_missing": true + } + }, { + "rename": { + "field": "user_agent.os_major", + "target_field": "user_agent.os.major", + "ignore_missing": true + } + }, { + "rename": { + "field": "user_agent.os_minor", + "target_field": "user_agent.os.minor", + "ignore_missing": true + } + }, { + "rename": { + "field": "user_agent.os_patch", + "target_field": "user_agent.os.patch", + "ignore_missing": true + } + + },{ "geoip": { - "field": "nginx.access.remote_ip", - "target_field": "nginx.access.geoip" + "field": "source.ip", + "target_field": "source.geo", + "ignore_missing": true } }], "on_failure" : [{ diff --git a/filebeat/module/nginx/access/test/test.log b/filebeat/module/nginx/access/test/test.log index d107d245269..3e19abfd670 100644 --- a/filebeat/module/nginx/access/test/test.log +++ b/filebeat/module/nginx/access/test/test.log @@ -1,7 +1,7 @@ 10.0.0.2, 10.0.0.1, 127.0.0.1 - - [07/Dec/2016:11:05:07 +0100] "GET /ocelot HTTP/1.1" 200 571 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0" 172.17.0.1 - - [29/May/2017:19:02:48 +0000] "GET /stringpatch HTTP/1.1" 404 612 "-" "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2" "-" 10.0.0.2, 10.0.0.1, 85.181.35.98 - - [07/Dec/2016:11:05:07 +0100] "GET /ocelot HTTP/1.1" 200 571 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0" -85.181.35.98 - - [07/Dec/2016:11:05:07 +0100] "GET /ocelot HTTP/1.1" 200 571 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0" +85.181.35.98 - - [07/Dec/2016:11:05:07 +0100] "GET /ocelot HTTP/1.1" 200 571 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36" "10.5.102.222, 199.96.1.1, 204.246.1.1" 10.2.1.185 - - [22/Jan/2016:13:18:29 +0000] "GET /assets/xxxx?q=100 HTTP/1.1" 200 25507 "-" "Amazon CloudFront" 2a03:0000:10ff:f00f:0000:0000:0:8000, 10.225.192.17 10.2.2.121 - - [30/Dec/2016:06:47:09 +0000] "GET /test.html HTTP/1.1" 404 8571 "-" "Mozilla/5.0 (compatible; Facebot 1.0; https://developers.facebook.com/docs/sharing/webmasters/crawler)" 127.0.0.1 - - [12/Apr/2018:09:48:40 +0200] "" 400 0 "-" "-" diff --git a/filebeat/module/nginx/access/test/test.log-expected.json b/filebeat/module/nginx/access/test/test.log-expected.json index 97e9f2df1a0..317cbd88939 100644 --- a/filebeat/module/nginx/access/test/test.log-expected.json +++ b/filebeat/module/nginx/access/test/test.log-expected.json @@ -3,209 +3,210 @@ "@timestamp": "2016-12-07T10:05:07.000Z", "event.dataset": "access", "event.module": "nginx", + "http.request.method": "GET", + "http.request.referrer": "-", + "http.response.status_code": 200, + "http.version": "1.1", "input.type": "log", "log.offset": 0, - "nginx.access.body_sent.bytes": "571", - "nginx.access.http_version": "1.1", - "nginx.access.method": "GET", - "nginx.access.referrer": "-", - "nginx.access.remote_ip": "10.0.0.2", - "nginx.access.remote_ip_list": [ + "network.forwarded_ip": [ "10.0.0.2", "10.0.0.1", "127.0.0.1" ], - "nginx.access.response_code": "200", - "nginx.access.url": "/ocelot", - "nginx.access.user_agent.device": "Other", - "nginx.access.user_agent.major": "49", - "nginx.access.user_agent.minor": "0", - "nginx.access.user_agent.name": "Firefox", - "nginx.access.user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0", - "nginx.access.user_agent.os": "Mac OS X 10.12", - "nginx.access.user_agent.os_major": "10", - "nginx.access.user_agent.os_minor": "12", - "nginx.access.user_agent.os_name": "Mac OS X", - "nginx.access.user_name": "-" + "nginx.access.body_sent.bytes": 571, + "source.ip": "10.0.0.2", + "url.original": "/ocelot", + "user.name": "-", + "user_agent.device": "Other", + "user_agent.major": "49", + "user_agent.minor": "0", + "user_agent.name": "Firefox", + "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0", + "user_agent.os.full_name": "Mac OS X 10.12", + "user_agent.os.major": "10", + "user_agent.os.minor": "12", + "user_agent.os.name": "Mac OS X" }, { "@timestamp": "2017-05-29T19:02:48.000Z", "event.dataset": "access", "event.module": "nginx", + "http.request.method": "GET", + "http.request.referrer": "-", + "http.response.status_code": 404, + "http.version": "1.1", "input.type": "log", "log.offset": 183, - "nginx.access.body_sent.bytes": "612", - "nginx.access.http_version": "1.1", - "nginx.access.method": "GET", - "nginx.access.referrer": "-", - "nginx.access.remote_ip": "172.17.0.1", - "nginx.access.remote_ip_list": [ + "network.forwarded_ip": [ "172.17.0.1" ], - "nginx.access.response_code": "404", - "nginx.access.url": "/stringpatch", - "nginx.access.user_agent.device": "Other", - "nginx.access.user_agent.major": "15", - "nginx.access.user_agent.minor": "0", - "nginx.access.user_agent.name": "Firefox Alpha", - "nginx.access.user_agent.original": "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2", - "nginx.access.user_agent.os": "Windows 7", - "nginx.access.user_agent.os_name": "Windows 7", - "nginx.access.user_agent.patch": "a2", - "nginx.access.user_name": "-" + "nginx.access.body_sent.bytes": 612, + "source.ip": "172.17.0.1", + "url.original": "/stringpatch", + "user.name": "-", + "user_agent.device": "Other", + "user_agent.major": "15", + "user_agent.minor": "0", + "user_agent.name": "Firefox Alpha", + "user_agent.original": "Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2", + "user_agent.os.full_name": "Windows 7", + "user_agent.os.name": "Windows 7", + "user_agent.patch": "a2" }, { "@timestamp": "2016-12-07T10:05:07.000Z", "event.dataset": "access", "event.module": "nginx", + "http.request.method": "GET", + "http.request.referrer": "-", + "http.response.status_code": 200, + "http.version": "1.1", "input.type": "log", "log.offset": 341, - "nginx.access.body_sent.bytes": "571", - "nginx.access.geoip.city_name": "Berlin", - "nginx.access.geoip.continent_name": "Europe", - "nginx.access.geoip.country_iso_code": "DE", - "nginx.access.geoip.location.lat": 52.4908, - "nginx.access.geoip.location.lon": 13.3275, - "nginx.access.geoip.region_iso_code": "DE-BE", - "nginx.access.geoip.region_name": "Land Berlin", - "nginx.access.http_version": "1.1", - "nginx.access.method": "GET", - "nginx.access.referrer": "-", - "nginx.access.remote_ip": "85.181.35.98", - "nginx.access.remote_ip_list": [ + "network.forwarded_ip": [ "10.0.0.2", "10.0.0.1", "85.181.35.98" ], - "nginx.access.response_code": "200", - "nginx.access.url": "/ocelot", - "nginx.access.user_agent.device": "Other", - "nginx.access.user_agent.major": "49", - "nginx.access.user_agent.minor": "0", - "nginx.access.user_agent.name": "Firefox", - "nginx.access.user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0", - "nginx.access.user_agent.os": "Mac OS X 10.12", - "nginx.access.user_agent.os_major": "10", - "nginx.access.user_agent.os_minor": "12", - "nginx.access.user_agent.os_name": "Mac OS X", - "nginx.access.user_name": "-" + "nginx.access.body_sent.bytes": 571, + "source.geo.city_name": "Berlin", + "source.geo.continent_name": "Europe", + "source.geo.country_iso_code": "DE", + "source.geo.location.lat": 52.4908, + "source.geo.location.lon": 13.3275, + "source.geo.region_iso_code": "DE-BE", + "source.geo.region_name": "Land Berlin", + "source.ip": "85.181.35.98", + "url.original": "/ocelot", + "user.name": "-", + "user_agent.device": "Other", + "user_agent.major": "49", + "user_agent.minor": "0", + "user_agent.name": "Firefox", + "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0", + "user_agent.os.full_name": "Mac OS X 10.12", + "user_agent.os.major": "10", + "user_agent.os.minor": "12", + "user_agent.os.name": "Mac OS X" }, { "@timestamp": "2016-12-07T10:05:07.000Z", "event.dataset": "access", "event.module": "nginx", + "http.request.method": "GET", + "http.request.referrer": "-", + "http.response.status_code": 200, + "http.version": "1.1", "input.type": "log", "log.offset": 527, - "nginx.access.body_sent.bytes": "571", - "nginx.access.geoip.city_name": "Berlin", - "nginx.access.geoip.continent_name": "Europe", - "nginx.access.geoip.country_iso_code": "DE", - "nginx.access.geoip.location.lat": 52.4908, - "nginx.access.geoip.location.lon": 13.3275, - "nginx.access.geoip.region_iso_code": "DE-BE", - "nginx.access.geoip.region_name": "Land Berlin", - "nginx.access.http_version": "1.1", - "nginx.access.method": "GET", - "nginx.access.referrer": "-", - "nginx.access.remote_ip": "85.181.35.98", - "nginx.access.remote_ip_list": [ + "network.forwarded_ip": [ "85.181.35.98" ], - "nginx.access.response_code": "200", - "nginx.access.url": "/ocelot", - "nginx.access.user_agent.device": "Other", - "nginx.access.user_agent.major": "49", - "nginx.access.user_agent.minor": "0", - "nginx.access.user_agent.name": "Firefox", - "nginx.access.user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0", - "nginx.access.user_agent.os": "Mac OS X 10.12", - "nginx.access.user_agent.os_major": "10", - "nginx.access.user_agent.os_minor": "12", - "nginx.access.user_agent.os_name": "Mac OS X", - "nginx.access.user_name": "-" + "nginx.access.body_sent.bytes": 571, + "source.geo.city_name": "Berlin", + "source.geo.continent_name": "Europe", + "source.geo.country_iso_code": "DE", + "source.geo.location.lat": 52.4908, + "source.geo.location.lon": 13.3275, + "source.geo.region_iso_code": "DE-BE", + "source.geo.region_name": "Land Berlin", + "source.ip": "85.181.35.98", + "url.original": "/ocelot", + "user.name": "-", + "user_agent.device": "Other", + "user_agent.major": "70", + "user_agent.minor": "0", + "user_agent.name": "Chrome", + "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36", + "user_agent.os.full_name": "Mac OS X 10.14.0", + "user_agent.os.major": "10", + "user_agent.os.minor": "14", + "user_agent.os.name": "Mac OS X", + "user_agent.patch": "3538" }, { "@timestamp": "2016-01-22T13:18:29.000Z", "event.dataset": "access", "event.module": "nginx", + "http.request.method": "GET", + "http.request.referrer": "-", + "http.response.status_code": 200, + "http.version": "1.1", "input.type": "log", - "log.offset": 693, - "nginx.access.body_sent.bytes": "25507", - "nginx.access.geoip.city_name": "Springfield", - "nginx.access.geoip.continent_name": "North America", - "nginx.access.geoip.country_iso_code": "US", - "nginx.access.geoip.location.lat": 39.772, - "nginx.access.geoip.location.lon": -89.6859, - "nginx.access.geoip.region_iso_code": "US-IL", - "nginx.access.geoip.region_name": "Illinois", - "nginx.access.http_version": "1.1", - "nginx.access.method": "GET", - "nginx.access.referrer": "-", - "nginx.access.remote_ip": "199.96.1.1", - "nginx.access.remote_ip_list": [ + "log.offset": 732, + "network.forwarded_ip": [ "10.5.102.222", "199.96.1.1", "204.246.1.1", "10.2.1.185" ], - "nginx.access.response_code": "200", - "nginx.access.url": "/assets/xxxx?q=100", - "nginx.access.user_agent.device": "Other", - "nginx.access.user_agent.name": "Other", - "nginx.access.user_agent.original": "Amazon CloudFront", - "nginx.access.user_agent.os": "Other", - "nginx.access.user_agent.os_name": "Other", - "nginx.access.user_name": "-" + "nginx.access.body_sent.bytes": 25507, + "source.geo.city_name": "Springfield", + "source.geo.continent_name": "North America", + "source.geo.country_iso_code": "US", + "source.geo.location.lat": 39.772, + "source.geo.location.lon": -89.6859, + "source.geo.region_iso_code": "US-IL", + "source.geo.region_name": "Illinois", + "source.ip": "199.96.1.1", + "url.original": "/assets/xxxx?q=100", + "user.name": "-", + "user_agent.device": "Other", + "user_agent.name": "Other", + "user_agent.original": "Amazon CloudFront", + "user_agent.os.full_name": "Other", + "user_agent.os.name": "Other" }, { "@timestamp": "2016-12-30T06:47:09.000Z", "event.dataset": "access", "event.module": "nginx", + "http.request.method": "GET", + "http.request.referrer": "-", + "http.response.status_code": 404, + "http.version": "1.1", "input.type": "log", - "log.offset": 845, - "nginx.access.body_sent.bytes": "8571", - "nginx.access.geoip.continent_name": "Europe", - "nginx.access.geoip.country_iso_code": "PT", - "nginx.access.geoip.location.lat": 39.5, - "nginx.access.geoip.location.lon": -8.0, - "nginx.access.http_version": "1.1", - "nginx.access.method": "GET", - "nginx.access.referrer": "-", - "nginx.access.remote_ip": "2a03:0000:10ff:f00f:0000:0000:0:8000", - "nginx.access.remote_ip_list": [ + "log.offset": 884, + "network.forwarded_ip": [ "2a03:0000:10ff:f00f:0000:0000:0:8000", "10.225.192.17", "10.2.2.121" ], - "nginx.access.response_code": "404", - "nginx.access.url": "/test.html", - "nginx.access.user_agent.device": "Spider", - "nginx.access.user_agent.major": "1", - "nginx.access.user_agent.minor": "0", - "nginx.access.user_agent.name": "Facebot", - "nginx.access.user_agent.original": "Mozilla/5.0 (compatible; Facebot 1.0; https://developers.facebook.com/docs/sharing/webmasters/crawler)", - "nginx.access.user_agent.os": "Other", - "nginx.access.user_agent.os_name": "Other", - "nginx.access.user_name": "-" + "nginx.access.body_sent.bytes": 8571, + "source.geo.continent_name": "Europe", + "source.geo.country_iso_code": "PT", + "source.geo.location.lat": 39.5, + "source.geo.location.lon": -8.0, + "source.ip": "2a03:0000:10ff:f00f:0000:0000:0:8000", + "url.original": "/test.html", + "user.name": "-", + "user_agent.device": "Spider", + "user_agent.major": "1", + "user_agent.minor": "0", + "user_agent.name": "Facebot", + "user_agent.original": "Mozilla/5.0 (compatible; Facebot 1.0; https://developers.facebook.com/docs/sharing/webmasters/crawler)", + "user_agent.os.full_name": "Other", + "user_agent.os.name": "Other" }, { "@timestamp": "2018-04-12T07:48:40.000Z", "event.dataset": "access", "event.module": "nginx", + "http.request.referrer": "-", + "http.response.status_code": 400, "input.type": "log", - "log.offset": 1085, - "nginx.access.body_sent.bytes": "0", - "nginx.access.referrer": "-", - "nginx.access.remote_ip": "127.0.0.1", - "nginx.access.remote_ip_list": [ + "log.offset": 1124, + "network.forwarded_ip": [ "127.0.0.1" ], - "nginx.access.response_code": "400", - "nginx.access.user_agent.device": "Other", - "nginx.access.user_agent.name": "Other", - "nginx.access.user_agent.original": "-", - "nginx.access.user_agent.os": "Other", - "nginx.access.user_agent.os_name": "Other", - "nginx.access.user_name": "-" + "nginx.access.body_sent.bytes": 0, + "source.ip": "127.0.0.1", + "user.name": "-", + "user_agent.device": "Other", + "user_agent.name": "Other", + "user_agent.original": "-", + "user_agent.os.full_name": "Other", + "user_agent.os.name": "Other" } ] \ No newline at end of file diff --git a/filebeat/module/system/_meta/fields.yml b/filebeat/module/system/_meta/fields.yml index a6fb8d16c8b..87f33ac8a95 100644 --- a/filebeat/module/system/_meta/fields.yml +++ b/filebeat/module/system/_meta/fields.yml @@ -4,6 +4,15 @@ Module for parsing system log files. short_config: true fields: + - name: host.hostname + type: keyword + decription: > + Hostname of the host. + + It can contain what `hostname` returns on Unix systems, the fully + qualified domain name, or a name specified by the user. The sender + decides which value to use. + - name: system type: group description: > diff --git a/filebeat/module/system/syslog/_meta/fields.yml b/filebeat/module/system/syslog/_meta/fields.yml index 667222cca98..222bd93aa22 100644 --- a/filebeat/module/system/syslog/_meta/fields.yml +++ b/filebeat/module/system/syslog/_meta/fields.yml @@ -4,18 +4,17 @@ Contains fields from the syslog system logs. fields: - name: timestamp - description: > - The timestamp as read from the syslog message. + type: alias + path: '@timestamp' - name: hostname - description: > - The hostname as read from the syslog message. + type: alias + path: host.hostname - name: program - description: > - The process name as read from the syslog message. + type: alias + path: process.name - name: pid - description: > - The PID of the process that sent the syslog message. + type: alias + path: process.pid - name: message - type: text - description: > - The message in the log line. + type: alias + path: message diff --git a/filebeat/module/system/syslog/ingest/pipeline.json b/filebeat/module/system/syslog/ingest/pipeline.json index 1f0126aadb7..ff228a66c30 100644 --- a/filebeat/module/system/syslog/ingest/pipeline.json +++ b/filebeat/module/system/syslog/ingest/pipeline.json @@ -1,41 +1,53 @@ { - "description": "Pipeline for parsing Syslog messages.", - "processors": [ - { - "grok": { - "field": "message", - "patterns": [ - "%{SYSLOGTIMESTAMP:system.syslog.timestamp} %{SYSLOGHOST:system.syslog.hostname} %{DATA:system.syslog.program}(?:\\[%{POSINT:system.syslog.pid}\\])?: %{GREEDYMULTILINE:system.syslog.message}", - "%{SYSLOGTIMESTAMP:system.syslog.timestamp} %{GREEDYMULTILINE:system.syslog.message}" - ], - "pattern_definitions" : { - "GREEDYMULTILINE" : "(.|\n)*" + "description": "Pipeline for parsing Syslog messages.", + "processors": [ + { + "grok": { + "field": "message", + "patterns": [ + "%{SYSLOGTIMESTAMP:system.syslog.timestamp} %{SYSLOGHOST:host.hostname} %{DATA:process.name}(?:\\[%{POSINT:process.pid:int}\\])?: %{GREEDYMULTILINE:system.syslog.message}", + "%{SYSLOGTIMESTAMP:system.syslog.timestamp} %{GREEDYMULTILINE:system.syslog.message}" + ], + "pattern_definitions" : { + "GREEDYMULTILINE" : "(.|\n)*" + }, + "ignore_missing": true + } }, - "ignore_missing": true - } - }, - { - "remove": { - "field": "message" - } - }, - { - "date": { - "field": "system.syslog.timestamp", - "target_field": "@timestamp", - "formats": [ - "MMM d HH:mm:ss", - "MMM dd HH:mm:ss" - ], - {< if .convert_timezone >}"timezone": "{{ beat.timezone }}",{< end >} - "ignore_failure": true - } - } - ], - "on_failure" : [{ - "set" : { - "field" : "error.message", - "value" : "{{ _ingest.on_failure_message }}" - } - }] + { + "remove": { + "field": "message" + } + }, + { + "rename": { + "field": "system.syslog.message", + "target_field": "message", + "ignore_missing": true + } + }, + { + "date": { + "field": "system.syslog.timestamp", + "target_field": "@timestamp", + "formats": [ + "MMM d HH:mm:ss", + "MMM dd HH:mm:ss" + ], + {< if .convert_timezone >}"timezone": "{{ beat.timezone }}",{< end >} + "ignore_failure": true + } + }, + { + "remove": { + "field": "system.syslog.timestamp" + } + } + ], + "on_failure" : [{ + "set" : { + "field" : "error.message", + "value" : "{{ _ingest.on_failure_message }}" + } + }] } diff --git a/filebeat/module/system/syslog/test/darwin-syslog-sample.log-expected.json b/filebeat/module/system/syslog/test/darwin-syslog-sample.log-expected.json index b77691a5a29..dd9286fb24d 100644 --- a/filebeat/module/system/syslog/test/darwin-syslog-sample.log-expected.json +++ b/filebeat/module/system/syslog/test/darwin-syslog-sample.log-expected.json @@ -3,28 +3,26 @@ "@timestamp": "2018-12-13T11:35:28.000Z", "event.dataset": "syslog", "event.module": "system", + "host.hostname": "a-mac-with-esc-key", "input.type": "log", "log.flags": [ "multiline" ], "log.offset": 0, - "system.syslog.hostname": "a-mac-with-esc-key", - "system.syslog.message": "2016-12-13 11:35:28.420 GoogleSoftwareUpdateAgent[21412/0x700007399000] [lvl=2] -[KSAgentApp updateProductWithProductID:usingEngine:] Checking for updates for \"All Products\" using engine \n\t\t>>\n\t\tprocessor=\n\t\t\tisProcessing=NO actionsCompleted=0 progress=0.00\n\t\t\terrors=0 currentActionErrors=0\n\t\t\tevents=0 currentActionEvents=0\n\t\t\tactionQueue=( )\n\t\t>\n\t\tdelegate=(null)\n\t\tserverInfoStore=(null)\n\t\terrors=0\n\t>", - "system.syslog.pid": "21412", - "system.syslog.program": "GoogleSoftwareUpdateAgent", - "system.syslog.timestamp": "Dec 13 11:35:28" + "message": "2016-12-13 11:35:28.420 GoogleSoftwareUpdateAgent[21412/0x700007399000] [lvl=2] -[KSAgentApp updateProductWithProductID:usingEngine:] Checking for updates for \"All Products\" using engine \n\t\t>>\n\t\tprocessor=\n\t\t\tisProcessing=NO actionsCompleted=0 progress=0.00\n\t\t\terrors=0 currentActionErrors=0\n\t\t\tevents=0 currentActionEvents=0\n\t\t\tactionQueue=( )\n\t\t>\n\t\tdelegate=(null)\n\t\tserverInfoStore=(null)\n\t\terrors=0\n\t>", + "process.name": "GoogleSoftwareUpdateAgent", + "process.pid": 21412 }, { "@timestamp": "2018-12-13T11:35:28.000Z", "event.dataset": "syslog", "event.module": "system", + "host.hostname": "a-mac-with-esc-key", "input.type": "log", "log.offset": 907, - "system.syslog.hostname": "a-mac-with-esc-key", - "system.syslog.message": "2016-12-13 11:35:28.421 GoogleSoftwareUpdateAgent[21412/0x700007399000] [lvl=2] -[KSUpdateEngine updateAllExceptProduct:] KSUpdateEngine updating all installed products, except:'com.google.Keystone'.", - "system.syslog.pid": "21412", - "system.syslog.program": "GoogleSoftwareUpdateAgent", - "system.syslog.timestamp": "Dec 13 11:35:28" + "message": "2016-12-13 11:35:28.421 GoogleSoftwareUpdateAgent[21412/0x700007399000] [lvl=2] -[KSUpdateEngine updateAllExceptProduct:] KSUpdateEngine updating all installed products, except:'com.google.Keystone'.", + "process.name": "GoogleSoftwareUpdateAgent", + "process.pid": 21412 }, { "@timestamp": "2018-04-04T03:39:57.000Z", @@ -32,7 +30,6 @@ "event.module": "system", "input.type": "log", "log.offset": 1176, - "system.syslog.message": "--- last message repeated 1 time ---", - "system.syslog.timestamp": "Apr 4 03:39:57" + "message": "--- last message repeated 1 time ---" } ] \ No newline at end of file diff --git a/filebeat/scripts/generator/fileset/main.go b/filebeat/scripts/generator/fileset/main.go index 94d0ac6b6a2..76de5168d05 100644 --- a/filebeat/scripts/generator/fileset/main.go +++ b/filebeat/scripts/generator/fileset/main.go @@ -32,7 +32,7 @@ func generateFileset(module, fileset, modulesPath, beatsPath string) error { return fmt.Errorf("fileset already exists: %s", fileset) } - err := generator.CreateDirectories(filesetPath, []string{"", "_meta", "test", "config", "ingest"}) + err := generator.CreateDirectories(filesetPath, "_meta", "test", "config", "ingest") if err != nil { return err } diff --git a/filebeat/scripts/generator/generator.go b/filebeat/scripts/generator/generator.go index e5aaf771ab4..40df8311ede 100644 --- a/filebeat/scripts/generator/generator.go +++ b/filebeat/scripts/generator/generator.go @@ -35,7 +35,7 @@ func DirExists(dir string) bool { } // CreateDirectories create directories in baseDir -func CreateDirectories(baseDir string, directories []string) error { +func CreateDirectories(baseDir string, directories ...string) error { for _, d := range directories { p := path.Join(baseDir, d) err := os.MkdirAll(p, 0750) diff --git a/filebeat/scripts/generator/module/main.go b/filebeat/scripts/generator/module/main.go index dedf1f3e279..b88ed2d207b 100644 --- a/filebeat/scripts/generator/module/main.go +++ b/filebeat/scripts/generator/module/main.go @@ -32,15 +32,20 @@ func generateModule(module, modulesPath, beatsPath string) error { return fmt.Errorf("module already exists: %s", module) } - err := generator.CreateDirectories(modulePath, []string{path.Join("_meta", "kibana", "6")}) + err := generator.CreateDirectories(modulePath, "_meta") if err != nil { return err } replace := map[string]string{"module": module} templatesPath := path.Join(beatsPath, "scripts", "module") - filesToCopy := []string{path.Join("_meta", "fields.yml"), path.Join("_meta", "docs.asciidoc"), path.Join("_meta", "config.yml"), path.Join("module.yml")} - generator.CopyTemplates(templatesPath, modulePath, filesToCopy, replace) + filesToCopy := []string{ + path.Join("_meta", "fields.yml"), + path.Join("_meta", "docs.asciidoc"), + path.Join("_meta", "config.yml"), + "module.yml", + } + err = generator.CopyTemplates(templatesPath, modulePath, filesToCopy, replace) if err != nil { return err } diff --git a/filebeat/tests/system/test_crawler.py b/filebeat/tests/system/test_crawler.py index 19714387741..315198f2841 100644 --- a/filebeat/tests/system/test_crawler.py +++ b/filebeat/tests/system/test_crawler.py @@ -5,6 +5,7 @@ import codecs import os import time +import unittest from nose.plugins.skip import Skip, SkipTest import shutil @@ -263,6 +264,7 @@ def test_file_disappear(self): assert len(output) == 5 + 6 + @unittest.skipIf(os.name == 'nt', 'flaky test https://github.com/elastic/beats/issues/9213') def test_file_disappear_appear(self): """ Checks that filebeat keeps running in case a log files is deleted diff --git a/filebeat/tests/system/test_harvester.py b/filebeat/tests/system/test_harvester.py index 0f8ed244c59..8f394f38b81 100644 --- a/filebeat/tests/system/test_harvester.py +++ b/filebeat/tests/system/test_harvester.py @@ -4,7 +4,10 @@ import os import codecs import time +import base64 import io +import re +import unittest from parameterized import parameterized """ @@ -75,6 +78,7 @@ def test_close_renamed(self): data = self.get_registry() assert len(data) == 2 + @unittest.skipIf(os.name == 'nt', 'flaky test https://github.com/elastic/beats/issues/9214') def test_close_removed(self): """ Checks that a file is closed if removed @@ -818,3 +822,37 @@ def test_decode_error(self): output = self.read_output_json() assert output[2]["message"] == "hello world2" + + def test_debug_reader(self): + """ + Test that you can enable a debug reader. + """ + self.render_config_template( + path=os.path.abspath(self.working_dir) + "/log/*", + ) + + os.mkdir(self.working_dir + "/log/") + + logfile = self.working_dir + "/log/test.log" + + file = open(logfile, 'w', 0) + file.write("hello world1") + file.write("\n") + file.write("\x00\x00\x00\x00") + file.write("\n") + file.write("hello world2") + file.write("\n") + file.write("\x00\x00\x00\x00") + file.write("Hello World\n") + # Write some more data to hit the 16k min buffer size. + # Make it web safe. + file.write(base64.b64encode(os.urandom(16 * 1024))) + file.close() + + filebeat = self.start_beat() + + # 13 on unix, 14 on windows. + self.wait_until(lambda: self.log_contains(re.compile( + 'Matching null byte found at offset (13|14)')), max_timeout=5) + + filebeat.check_kill_and_wait() diff --git a/filebeat/tests/system/test_registrar.py b/filebeat/tests/system/test_registrar.py index ade0f2657e5..5927a586d91 100644 --- a/filebeat/tests/system/test_registrar.py +++ b/filebeat/tests/system/test_registrar.py @@ -6,6 +6,7 @@ import time import shutil import stat +import unittest from filebeat import BaseTest from nose.plugins.skip import SkipTest @@ -847,10 +848,12 @@ def test_clean_inactive(self): else: assert data[0]["offset"] == 2 + @unittest.skip("Skipped as flaky: https://github.com/elastic/beats/issues/7690") def test_clean_removed(self): """ Checks that files which were removed, the state is removed """ + self.render_config_template( path=os.path.abspath(self.working_dir) + "/log/input*", scan_frequency="0.1s", @@ -909,6 +912,7 @@ def test_clean_removed(self): else: assert data[0]["offset"] == len("make sure registry is written\n" + "2\n") + @unittest.skipIf(os.name == 'nt', 'flaky test https://github.com/elastic/beats/issues/9215') def test_clean_removed_with_clean_inactive(self): """ Checks that files which were removed, the state is removed diff --git a/generator/beat/{beat}/.travis.yml b/generator/beat/{beat}/.travis.yml index 0138ce5b670..88c5ee623a2 100644 --- a/generator/beat/{beat}/.travis.yml +++ b/generator/beat/{beat}/.travis.yml @@ -6,7 +6,7 @@ services: language: go go: - - 1.10.3 + - 1.11.2 os: - linux diff --git a/heartbeat/Dockerfile b/heartbeat/Dockerfile index 222f4944ba8..89272c54c6c 100644 --- a/heartbeat/Dockerfile +++ b/heartbeat/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.10.3 +FROM golang:1.11.2 MAINTAINER Nicolas Ruflin RUN set -x && \ diff --git a/heartbeat/docs/getting-started.asciidoc b/heartbeat/docs/getting-started.asciidoc index 28ded322eae..6008ecec6cf 100644 --- a/heartbeat/docs/getting-started.asciidoc +++ b/heartbeat/docs/getting-started.asciidoc @@ -81,6 +81,23 @@ tar xzvf heartbeat-{version}-darwin-x86_64.tar.gz endif::[] +[[linux]] +*linux:* + +ifeval::["{release-state}"=="unreleased"] + +Version {version} of {beatname_uc} has not yet been released. + +endif::[] + +ifeval::["{release-state}"!="unreleased"] + +["source","sh",subs="attributes"] +------------------------------------------------ +curl -L -O {downloads}/heartbeat/heartbeat-{version}-linux-x86_64.tar.gz +tar xzvf heartbeat-{version}-linux-x86_64.tar.gz +------------------------------------------------ +endif::[] [[docker]] *docker:* @@ -230,12 +247,12 @@ start Heartbeat in the foreground. sudo service {beatname_lc}-elastic start ---------------------------------------------------------------------- -*mac:* +*mac and linux:* ["source","sh",subs="attributes"] ---------------------------------------------------------------------- sudo chown root heartbeat.yml <1> -sudo ./heartbeat -e -c heartbeat.yml +sudo ./heartbeat -e ---------------------------------------------------------------------- <1> You'll be running Heartbeat as root, so you need to change ownership of the configuration file, or run Heartbeat with `--strict.perms=false` specified. See diff --git a/heartbeat/docs/index.asciidoc b/heartbeat/docs/index.asciidoc index 0176b007290..53f2cc9a48c 100644 --- a/heartbeat/docs/index.asciidoc +++ b/heartbeat/docs/index.asciidoc @@ -15,6 +15,7 @@ include::{asciidoc-dir}/../../shared/attributes.asciidoc[] :deb_os: :rpm_os: :mac_os: +:linux_os: :docker_platform: :win_os: diff --git a/heartbeat/magefile.go b/heartbeat/magefile.go index b7d08244051..78233688487 100644 --- a/heartbeat/magefile.go +++ b/heartbeat/magefile.go @@ -75,7 +75,7 @@ func Clean() error { // Package packages the Beat for distribution. // Use SNAPSHOT=true to build snapshots. // Use PLATFORMS to control the target platforms. -// Use BEAT_VERSION_QUALIFIER to control the version qualifier. +// Use VERSION_QUALIFIER to control the version qualifier. func Package() { start := time.Now() defer func() { fmt.Println("package ran for", time.Since(start)) }() diff --git a/heartbeat/monitors/active/http/_meta/kibana/6/dashboard/Heartbeat-http-monitor.json b/heartbeat/monitors/active/http/_meta/kibana/6/dashboard/Heartbeat-http-monitor.json index e877a919132..0a5e4dcbb8b 100644 --- a/heartbeat/monitors/active/http/_meta/kibana/6/dashboard/Heartbeat-http-monitor.json +++ b/heartbeat/monitors/active/http/_meta/kibana/6/dashboard/Heartbeat-http-monitor.json @@ -1,702 +1,745 @@ { - "objects": [ - { - "attributes": { - "description": "", - "kibanaSavedObjectMeta": { - "searchSourceJSON": { - "filter": [] - } - }, - "savedSearchId": "02014c80-29d2-11e7-a68f-bfaa2341cc52", - "title": "HTTP ping times", - "uiStateJSON": {}, - "version": 1, - "visState": { - "aggs": [ - { - "enabled": true, - "id": "1", - "params": { - "customLabel": "", - "field": "resolve.rtt.us" - }, - "schema": "metric", - "type": "max" - }, - { - "enabled": true, - "id": "3", - "params": { - "field": "tcp.rtt.connect.us" - }, - "schema": "metric", - "type": "max" - }, - { - "enabled": true, - "id": "5", - "params": { - "field": "tls.rtt.handshake.us" - }, - "schema": "metric", - "type": "max" - }, - { - "enabled": true, - "id": "4", - "params": { - "field": "http.rtt.response_header.us" - }, - "schema": "metric", - "type": "max" - }, - { - "enabled": true, - "id": "2", - "params": { - "customInterval": "2h", - "extended_bounds": {}, - "field": "@timestamp", - "interval": "auto", - "min_doc_count": 1 - }, - "schema": "segment", - "type": "date_histogram" - } - ], - "listeners": {}, - "params": { - "addLegend": true, - "addTimeMarker": false, - "addTooltip": true, - "defaultYExtents": false, - "interpolate": "linear", - "legendPosition": "right", - "mode": "stacked", - "scale": "linear", - "setYExtents": false, - "times": [] - }, - "title": "HTTP ping times", - "type": "area" - } - }, - "id": "c65ef340-eb19-11e6-be20-559646f8b9ba", - "type": "visualization", - "version": 1 + "objects": [ + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } }, - { - "attributes": { - "description": "", - "kibanaSavedObjectMeta": { - "searchSourceJSON": { - "filter": [] - } - }, - "savedSearchId": "02014c80-29d2-11e7-a68f-bfaa2341cc52", - "title": "HTTP monitors status", - "uiStateJSON": { - "vis": { - "colors": { - "200": "#B7DBAB", - "monitor.status: down": "#E24D42", - "monitor.status: up": "#629E51" - }, - "legendOpen": true - } - }, - "version": 1, - "visState": { - "aggs": [ - { - "enabled": true, - "id": "1", - "params": { - "field": "monitor.id" - }, - "schema": "metric", - "type": "cardinality" - }, - { - "enabled": true, - "id": "3", - "params": { - "filters": [ - { - "input": { - "query": { - "query_string": { - "analyze_wildcard": true, - "query": "monitor.status: up" - } - } - }, - "label": "" - }, - { - "input": { - "query": { - "query_string": { - "analyze_wildcard": true, - "query": "monitor.status: down" - } - } - } - } - ] - }, - "schema": "segment", - "type": "filters" - }, - { - "enabled": true, - "id": "2", - "params": { - "field": "http.response.status_code", - "order": "desc", - "orderBy": "1", - "size": 5 - }, - "schema": "segment", - "type": "terms" - } - ], - "listeners": {}, - "params": { - "addLegend": true, - "addTooltip": true, - "isDonut": false, - "legendPosition": "bottom" - }, - "title": "HTTP monitors status", - "type": "pie" - } + "savedSearchId": "02014c80-29d2-11e7-a68f-bfaa2341cc52", + "title": "HTTP ping times", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": { + "customLabel": "", + "field": "resolve.rtt.us" + }, + "schema": "metric", + "type": "max" }, - "id": "920e8140-eb1a-11e6-be20-559646f8b9ba", - "type": "visualization", - "version": 1 + { + "enabled": true, + "id": "3", + "params": { + "field": "tcp.rtt.connect.us" + }, + "schema": "metric", + "type": "max" + }, + { + "enabled": true, + "id": "5", + "params": { + "field": "tls.rtt.handshake.us" + }, + "schema": "metric", + "type": "max" + }, + { + "enabled": true, + "id": "4", + "params": { + "field": "http.rtt.response_header.us" + }, + "schema": "metric", + "type": "max" + }, + { + "enabled": true, + "id": "2", + "params": { + "customInterval": "2h", + "extended_bounds": {}, + "field": "@timestamp", + "interval": "auto", + "min_doc_count": 1 + }, + "schema": "segment", + "type": "date_histogram" + } + ], + "listeners": {}, + "params": { + "addLegend": true, + "addTimeMarker": false, + "addTooltip": true, + "defaultYExtents": false, + "interpolate": "linear", + "legendPosition": "right", + "mode": "stacked", + "scale": "linear", + "setYExtents": false, + "times": [] + }, + "title": "HTTP ping times", + "type": "area" + } + }, + "id": "c65ef340-eb19-11e6-be20-559646f8b9ba", + "type": "visualization", + "updated_at": "2018-11-07T19:12:21.881Z", + "version": 2 + }, + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } }, - { - "attributes": { - "description": "", - "kibanaSavedObjectMeta": { - "searchSourceJSON": { - "filter": [] - } - }, - "savedSearchId": "02014c80-29d2-11e7-a68f-bfaa2341cc52", - "title": "HTTP monitors", - "uiStateJSON": { - "vis": { - "params": { - "sort": { - "columnIndex": null, - "direction": null - } - } - } - }, - "version": 1, - "visState": { - "aggs": [ - { - "enabled": true, - "id": "1", - "params": { - "field": "monitor.duration.us" - }, - "schema": "metric", - "type": "max" - }, - { - "enabled": true, - "id": "2", - "params": { - "field": "monitor.id", - "order": "desc", - "orderBy": "1", - "size": 5 - }, - "schema": "bucket", - "type": "terms" - }, - { - "enabled": true, - "id": "5", - "params": { - "field": "resolve.rtt.us" - }, - "schema": "metric", - "type": "max" - }, - { - "enabled": true, - "id": "6", - "params": { - "field": "tcp.rtt.connect.us" - }, - "schema": "metric", - "type": "max" - }, - { - "enabled": true, - "id": "7", - "params": { - "field": "tls.rtt.handshake.us" - }, - "schema": "metric", - "type": "max" - }, - { - "enabled": true, - "id": "8", - "params": { - "field": "http.rtt.response_header.us" - }, - "schema": "metric", - "type": "max" - } - ], - "listeners": {}, - "params": { - "perPage": 10, - "showMeticsAtAllLevels": false, - "showPartialRows": false, - "showTotal": false, - "sort": { - "columnIndex": null, - "direction": null - }, - "totalFunc": "sum" - }, - "title": "HTTP monitors", - "type": "table" - } + "savedSearchId": "02014c80-29d2-11e7-a68f-bfaa2341cc52", + "title": "HTTP monitors status", + "uiStateJSON": { + "vis": { + "colors": { + "200": "#B7DBAB", + "monitor.status: down": "#E24D42", + "monitor.status: up": "#629E51" }, - "id": "1738dbc0-eb1d-11e6-be20-559646f8b9ba", - "type": "visualization", - "version": 1 + "legendOpen": true + } }, - { - "attributes": { - "description": "", - "kibanaSavedObjectMeta": { - "searchSourceJSON": { - "filter": [], - "query": { - "language": "lucene", - "query": "" + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": { + "field": "monitor.id" + }, + "schema": "metric", + "type": "cardinality" + }, + { + "enabled": true, + "id": "3", + "params": { + "filters": [ + { + "input": { + "query": { + "query_string": { + "analyze_wildcard": true, + "query": "monitor.status: up" } - } - }, - "savedSearchId": "02014c80-29d2-11e7-a68f-bfaa2341cc52", - "title": "HTTP up status", - "uiStateJSON": { - "vis": { - "colors": { - "monitor.status: down": "#E24D42", - "monitor.status: up": "#629E51" + } + }, + "label": "" + }, + { + "input": { + "query": { + "query_string": { + "analyze_wildcard": true, + "query": "monitor.status: down" } + } } - }, - "version": 1, - "visState": { - "aggs": [ - { - "enabled": true, - "id": "1", - "params": {}, - "schema": "metric", - "type": "count" - }, - { - "enabled": true, - "id": "2", - "params": { - "customInterval": "2h", - "extended_bounds": {}, - "field": "@timestamp", - "interval": "auto", - "min_doc_count": 1 - }, - "schema": "segment", - "type": "date_histogram" - }, - { - "enabled": true, - "id": "3", - "params": { - "filters": [ - { - "input": { - "query": "monitor.status: down" - }, - "label": "" - }, - { - "input": { - "query": "monitor.status: up" - } - } - ] - }, - "schema": "group", - "type": "filters" - } - ], - "params": { - "addLegend": true, - "addTimeMarker": false, - "addTooltip": true, - "categoryAxes": [ - { - "id": "CategoryAxis-1", - "labels": { - "show": true, - "truncate": 100 - }, - "position": "bottom", - "scale": { - "type": "linear" - }, - "show": true, - "style": {}, - "title": {}, - "type": "category" - } - ], - "defaultYExtents": false, - "grid": { - "categoryLines": false, - "style": { - "color": "#eee" - } - }, - "interpolate": "linear", - "legendPosition": "right", - "mode": "percentage", - "scale": "linear", - "seriesParams": [ - { - "data": { - "id": "1", - "label": "Percentage of Count" - }, - "interpolate": "linear", - "mode": "stacked", - "show": "true", - "type": "area", - "valueAxis": "ValueAxis-1" - } - ], - "setYExtents": true, - "times": [], - "type": "area", - "valueAxes": [ - { - "id": "ValueAxis-1", - "labels": { - "filter": false, - "rotate": 0, - "show": true, - "truncate": 100 - }, - "name": "LeftAxis-1", - "position": "left", - "scale": { - "defaultYExtents": false, - "max": 100, - "min": 0, - "mode": "percentage", - "setYExtents": true, - "type": "linear" - }, - "show": true, - "style": {}, - "title": { - "text": "Percentage of Count" - }, - "type": "value" - } - ], - "yAxis": { - "max": 100, - "min": 0 - } - }, - "title": "HTTP up status", - "type": "area" - } + } + ] + }, + "schema": "segment", + "type": "filters" }, - "id": "091c3a90-eb1e-11e6-be20-559646f8b9ba", - "type": "visualization", - "version": 2 + { + "enabled": true, + "id": "2", + "params": { + "field": "http.response.status_code", + "order": "desc", + "orderBy": "1", + "size": 5 + }, + "schema": "segment", + "type": "terms" + } + ], + "listeners": {}, + "params": { + "addLegend": true, + "addTooltip": true, + "isDonut": false, + "legendPosition": "bottom" + }, + "title": "HTTP monitors status", + "type": "pie" + } + }, + "id": "920e8140-eb1a-11e6-be20-559646f8b9ba", + "type": "visualization", + "updated_at": "2018-11-07T19:12:21.881Z", + "version": 2 + }, + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } }, - { - "attributes": { - "description": "", - "kibanaSavedObjectMeta": { - "searchSourceJSON": { - "filter": [] - } - }, - "savedSearchId": "02014c80-29d2-11e7-a68f-bfaa2341cc52", - "title": "HTTP duration heatmap", - "uiStateJSON": { - "vis": { - "defaultColors": { - "0 - 2": "rgb(247,251,255)", - "10 - 11": "rgb(23,100,171)", - "11 - 12": "rgb(8,74,145)", - "2 - 3": "rgb(227,238,249)", - "3 - 4": "rgb(208,225,242)", - "4 - 5": "rgb(182,212,233)", - "5 - 6": "rgb(148,196,223)", - "6 - 8": "rgb(107,174,214)", - "8 - 9": "rgb(74,152,201)", - "9 - 10": "rgb(46,126,188)" - } - } - }, - "version": 1, - "visState": { - "aggs": [ - { - "enabled": true, - "id": "1", - "params": {}, - "schema": "metric", - "type": "count" - }, - { - "enabled": true, - "id": "2", - "params": { - "customInterval": "2h", - "extended_bounds": {}, - "field": "@timestamp", - "interval": "auto", - "min_doc_count": 1 - }, - "schema": "segment", - "type": "date_histogram" - }, - { - "enabled": true, - "id": "3", - "params": { - "extended_bounds": {}, - "field": "monitor.duration.us", - "interval": 50000 - }, - "schema": "group", - "type": "histogram" - } - ], - "listeners": {}, - "params": { - "addLegend": true, - "addTooltip": true, - "colorSchema": "Blues", - "colorsNumber": 10, - "colorsRange": [], - "enableHover": false, - "invertColors": false, - "legendPosition": "right", - "percentageMode": false, - "setColorRange": false, - "times": [], - "valueAxes": [ - { - "id": "ValueAxis-1", - "labels": { - "color": "#555", - "rotate": 0, - "show": false - }, - "scale": { - "defaultYExtents": false, - "type": "linear" - }, - "show": false, - "type": "value" - } - ] - }, - "title": "HTTP duration heatmap", - "type": "heatmap" - } + "savedSearchId": "02014c80-29d2-11e7-a68f-bfaa2341cc52", + "title": "HTTP monitors", + "uiStateJSON": { + "vis": { + "params": { + "sort": { + "columnIndex": null, + "direction": null + } + } + } + }, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "2", + "params": { + "field": "monitor.id", + "missingBucket": false, + "missingBucketLabel": "Missing", + "order": "desc", + "orderBy": "1", + "otherBucket": false, + "otherBucketLabel": "Other", + "size": 5 + }, + "schema": "bucket", + "type": "terms" }, - "id": "0f4c0560-eb20-11e6-9f11-159ff202874a", - "type": "visualization", - "version": 1 + { + "enabled": true, + "id": "9", + "params": { + "aggregate": "concat", + "field": "monitor.status", + "size": 1, + "sortField": "@timestamp", + "sortOrder": "desc" + }, + "schema": "metric", + "type": "top_hits" + }, + { + "enabled": true, + "id": "1", + "params": { + "field": "monitor.duration.us" + }, + "schema": "metric", + "type": "max" + }, + { + "enabled": true, + "id": "5", + "params": { + "field": "resolve.rtt.us" + }, + "schema": "metric", + "type": "max" + }, + { + "enabled": true, + "id": "6", + "params": { + "field": "tcp.rtt.connect.us" + }, + "schema": "metric", + "type": "max" + }, + { + "enabled": true, + "id": "7", + "params": { + "field": "tls.rtt.handshake.us" + }, + "schema": "metric", + "type": "max" + }, + { + "enabled": true, + "id": "8", + "params": { + "field": "http.rtt.response_header.us" + }, + "schema": "metric", + "type": "max" + }, + { + "enabled": true, + "id": "10", + "params": { + "field": "monitor.ip", + "missingBucket": false, + "missingBucketLabel": "Missing", + "order": "desc", + "orderBy": "1", + "otherBucket": false, + "otherBucketLabel": "Other", + "size": 5 + }, + "schema": "bucket", + "type": "terms" + } + ], + "params": { + "perPage": 10, + "showMetricsAtAllLevels": false, + "showPartialRows": false, + "showTotal": false, + "sort": { + "columnIndex": null, + "direction": null + }, + "totalFunc": "sum" + }, + "title": "HTTP monitors", + "type": "table" + } + }, + "id": "1738dbc0-eb1d-11e6-be20-559646f8b9ba", + "type": "visualization", + "updated_at": "2018-11-07T19:16:14.694Z", + "version": 3 + }, + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "lucene", + "query": "" + } + } }, - { - "attributes": { - "columns": [ - "monitor.id", - "http.url", - "monitor.status", - "http.response.status_code", - "monitor.duration.us", - "tcp.rtt.connect.us", - "tls.rtt.handshake.us", - "resolve.rtt.us", - "http.rtt.content.us" - ], - "description": "", - "hits": 0, - "kibanaSavedObjectMeta": { - "searchSourceJSON": { - "filter": [ - { - "$state": { - "store": "appState" - }, - "meta": { - "alias": null, - "disabled": false, - "index": "heartbeat-*", - "key": "monitor.type", - "negate": false, - "value": "http" - }, - "query": { - "match": { - "monitor.type": { - "query": "http", - "type": "phrase" - } - } - } - } - ], - "highlightAll": true, - "index": "heartbeat-*", - "query": { - "query_string": { - "analyze_wildcard": true, - "query": "*" - } - } + "savedSearchId": "02014c80-29d2-11e7-a68f-bfaa2341cc52", + "title": "HTTP up status", + "uiStateJSON": { + "vis": { + "colors": { + "monitor.status: down": "#E24D42", + "monitor.status: up": "#629E51" + } + } + }, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "customInterval": "2h", + "extended_bounds": {}, + "field": "@timestamp", + "interval": "auto", + "min_doc_count": 1 + }, + "schema": "segment", + "type": "date_histogram" + }, + { + "enabled": true, + "id": "3", + "params": { + "filters": [ + { + "input": { + "query": "monitor.status: down" + }, + "label": "" + }, + { + "input": { + "query": "monitor.status: up" } + } + ] + }, + "schema": "group", + "type": "filters" + } + ], + "params": { + "addLegend": true, + "addTimeMarker": false, + "addTooltip": true, + "categoryAxes": [ + { + "id": "CategoryAxis-1", + "labels": { + "show": true, + "truncate": 100 + }, + "position": "bottom", + "scale": { + "type": "linear" }, - "sort": [ - "@timestamp", - "desc" - ], - "title": "Heartbeat HTTP pings", - "version": 1 + "show": true, + "style": {}, + "title": {}, + "type": "category" + } + ], + "defaultYExtents": false, + "grid": { + "categoryLines": false, + "style": { + "color": "#eee" + } }, - "id": "02014c80-29d2-11e7-a68f-bfaa2341cc52", - "type": "search", - "version": 1 - }, - { - "attributes": { - "description": "", - "hits": 0, - "kibanaSavedObjectMeta": { - "searchSourceJSON": { - "filter": [ - { - "query": { - "query_string": { - "analyze_wildcard": true, - "query": "*" - } - } - } - ] - } + "interpolate": "linear", + "legendPosition": "right", + "mode": "percentage", + "scale": "linear", + "seriesParams": [ + { + "data": { + "id": "1", + "label": "Percentage of Count" }, - "optionsJSON": { - "darkTheme": false + "interpolate": "linear", + "mode": "stacked", + "show": "true", + "type": "area", + "valueAxis": "ValueAxis-1" + } + ], + "setYExtents": true, + "times": [], + "type": "area", + "valueAxes": [ + { + "id": "ValueAxis-1", + "labels": { + "filter": false, + "rotate": 0, + "show": true, + "truncate": 100 }, - "panelsJSON": [ - { - "col": 1, - "id": "c65ef340-eb19-11e6-be20-559646f8b9ba", - "panelIndex": 1, - "row": 7, - "size_x": 12, - "size_y": 4, - "type": "visualization" - }, - { - "col": 9, - "id": "920e8140-eb1a-11e6-be20-559646f8b9ba", - "panelIndex": 2, - "row": 1, - "size_x": 4, - "size_y": 4, - "type": "visualization" - }, - { - "col": 1, - "id": "1738dbc0-eb1d-11e6-be20-559646f8b9ba", - "panelIndex": 3, - "row": 1, - "size_x": 8, - "size_y": 4, - "type": "visualization" - }, - { - "col": 1, - "id": "091c3a90-eb1e-11e6-be20-559646f8b9ba", - "panelIndex": 4, - "row": 5, - "size_x": 12, - "size_y": 2, - "type": "visualization" - }, - { - "col": 1, - "id": "0f4c0560-eb20-11e6-9f11-159ff202874a", - "panelIndex": 5, - "row": 11, - "size_x": 12, - "size_y": 5, - "type": "visualization" - } - ], - "timeRestore": false, - "title": "Heartbeat HTTP monitoring", - "uiStateJSON": { - "P-3": { - "vis": { - "params": { - "sort": { - "columnIndex": null, - "direction": null - } - } - } - }, - "P-5": { - "vis": { - "defaultColors": { - "0 - 2": "rgb(247,251,255)", - "10 - 11": "rgb(23,100,171)", - "11 - 12": "rgb(8,74,145)", - "2 - 3": "rgb(227,238,249)", - "3 - 4": "rgb(208,225,242)", - "4 - 5": "rgb(182,212,233)", - "5 - 6": "rgb(148,196,223)", - "6 - 8": "rgb(107,174,214)", - "8 - 9": "rgb(74,152,201)", - "9 - 10": "rgb(46,126,188)" - } - } - } + "name": "LeftAxis-1", + "position": "left", + "scale": { + "defaultYExtents": false, + "max": 100, + "min": 0, + "mode": "percentage", + "setYExtents": true, + "type": "linear" + }, + "show": true, + "style": {}, + "title": { + "text": "Percentage of Count" }, - "version": 1 + "type": "value" + } + ], + "yAxis": { + "max": 100, + "min": 0 + } + }, + "title": "HTTP up status", + "type": "area" + } + }, + "id": "091c3a90-eb1e-11e6-be20-559646f8b9ba", + "type": "visualization", + "updated_at": "2018-11-07T19:12:21.881Z", + "version": 2 + }, + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [] + } + }, + "savedSearchId": "02014c80-29d2-11e7-a68f-bfaa2341cc52", + "title": "HTTP duration heatmap", + "uiStateJSON": { + "vis": { + "defaultColors": { + "0 - 2": "rgb(247,251,255)", + "10 - 11": "rgb(23,100,171)", + "11 - 12": "rgb(8,74,145)", + "2 - 3": "rgb(227,238,249)", + "3 - 4": "rgb(208,225,242)", + "4 - 5": "rgb(182,212,233)", + "5 - 6": "rgb(148,196,223)", + "6 - 8": "rgb(107,174,214)", + "8 - 9": "rgb(74,152,201)", + "9 - 10": "rgb(46,126,188)" + } + } + }, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" }, - "id": "f3e771c0-eb19-11e6-be20-559646f8b9ba", - "type": "dashboard", - "version": 1 + { + "enabled": true, + "id": "2", + "params": { + "customInterval": "2h", + "extended_bounds": {}, + "field": "@timestamp", + "interval": "auto", + "min_doc_count": 1 + }, + "schema": "segment", + "type": "date_histogram" + }, + { + "enabled": true, + "id": "3", + "params": { + "extended_bounds": {}, + "field": "monitor.duration.us", + "interval": 50000 + }, + "schema": "group", + "type": "histogram" + } + ], + "listeners": {}, + "params": { + "addLegend": true, + "addTooltip": true, + "colorSchema": "Blues", + "colorsNumber": 10, + "colorsRange": [], + "enableHover": false, + "invertColors": false, + "legendPosition": "right", + "percentageMode": false, + "setColorRange": false, + "times": [], + "valueAxes": [ + { + "id": "ValueAxis-1", + "labels": { + "color": "#555", + "rotate": 0, + "show": false + }, + "scale": { + "defaultYExtents": false, + "type": "linear" + }, + "show": false, + "type": "value" + } + ] + }, + "title": "HTTP duration heatmap", + "type": "heatmap" } - ], - "version": "6.0.0-SNAPSHOT" -} + }, + "id": "0f4c0560-eb20-11e6-9f11-159ff202874a", + "type": "visualization", + "updated_at": "2018-11-07T19:12:21.881Z", + "version": 2 + }, + { + "attributes": { + "columns": [ + "monitor.id", + "http.url", + "monitor.status", + "http.response.status_code", + "monitor.duration.us", + "tcp.rtt.connect.us", + "tls.rtt.handshake.us", + "resolve.rtt.us", + "http.rtt.content.us" + ], + "description": "", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [ + { + "$state": { + "store": "appState" + }, + "meta": { + "alias": null, + "disabled": false, + "index": "heartbeat-*", + "key": "monitor.type", + "negate": false, + "value": "http" + }, + "query": { + "match": { + "monitor.type": { + "query": "http", + "type": "phrase" + } + } + } + } + ], + "highlightAll": true, + "index": "heartbeat-*", + "query": { + "query_string": { + "analyze_wildcard": true, + "query": "*" + } + } + } + }, + "sort": [ + "@timestamp", + "desc" + ], + "title": "Heartbeat HTTP pings", + "version": 1 + }, + "id": "02014c80-29d2-11e7-a68f-bfaa2341cc52", + "type": "search", + "updated_at": "2018-11-07T19:12:21.881Z", + "version": 2 + }, + { + "attributes": { + "description": "", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [ + { + "query": { + "query_string": { + "analyze_wildcard": true, + "query": "*" + } + } + } + ] + } + }, + "optionsJSON": { + "darkTheme": false + }, + "panelsJSON": [ + { + "col": 1, + "id": "c65ef340-eb19-11e6-be20-559646f8b9ba", + "panelIndex": 1, + "row": 7, + "size_x": 12, + "size_y": 4, + "type": "visualization" + }, + { + "col": 9, + "id": "920e8140-eb1a-11e6-be20-559646f8b9ba", + "panelIndex": 2, + "row": 1, + "size_x": 4, + "size_y": 4, + "type": "visualization" + }, + { + "col": 1, + "id": "1738dbc0-eb1d-11e6-be20-559646f8b9ba", + "panelIndex": 3, + "row": 1, + "size_x": 8, + "size_y": 4, + "type": "visualization" + }, + { + "col": 1, + "id": "091c3a90-eb1e-11e6-be20-559646f8b9ba", + "panelIndex": 4, + "row": 5, + "size_x": 12, + "size_y": 2, + "type": "visualization" + }, + { + "col": 1, + "id": "0f4c0560-eb20-11e6-9f11-159ff202874a", + "panelIndex": 5, + "row": 11, + "size_x": 12, + "size_y": 5, + "type": "visualization" + } + ], + "timeRestore": false, + "title": "Heartbeat HTTP monitoring", + "uiStateJSON": { + "P-3": { + "vis": { + "params": { + "sort": { + "columnIndex": null, + "direction": null + } + } + } + }, + "P-5": { + "vis": { + "defaultColors": { + "0 - 2": "rgb(247,251,255)", + "10 - 11": "rgb(23,100,171)", + "11 - 12": "rgb(8,74,145)", + "2 - 3": "rgb(227,238,249)", + "3 - 4": "rgb(208,225,242)", + "4 - 5": "rgb(182,212,233)", + "5 - 6": "rgb(148,196,223)", + "6 - 8": "rgb(107,174,214)", + "8 - 9": "rgb(74,152,201)", + "9 - 10": "rgb(46,126,188)" + } + } + } + }, + "version": 1 + }, + "id": "f3e771c0-eb19-11e6-be20-559646f8b9ba", + "type": "dashboard", + "updated_at": "2018-11-07T19:12:21.881Z", + "version": 2 + } + ], + "version": "6.5.0" +} \ No newline at end of file diff --git a/journalbeat/Dockerfile b/journalbeat/Dockerfile index 975fce8f382..7e187972feb 100644 --- a/journalbeat/Dockerfile +++ b/journalbeat/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.10.3 +FROM golang:1.11.2 MAINTAINER Noémi Ványi RUN set -x && \ diff --git a/journalbeat/_meta/beat.docker.yml b/journalbeat/_meta/beat.docker.yml index 2c87c0346b6..b7d83bd0111 100644 --- a/journalbeat/_meta/beat.docker.yml +++ b/journalbeat/_meta/beat.docker.yml @@ -1,4 +1,4 @@ journalbeat.inputs: - paths: [] - seek: tail + seek: cursor diff --git a/journalbeat/checkpoint/file_windows.go b/journalbeat/checkpoint/file_windows.go index 6644d751096..267086398f3 100644 --- a/journalbeat/checkpoint/file_windows.go +++ b/journalbeat/checkpoint/file_windows.go @@ -45,8 +45,8 @@ func createWriteThroughFile(path string) (*os.File, error) { pathp, // Path syscall.GENERIC_READ|syscall.GENERIC_WRITE, // Access Mode uint32(syscall.FILE_SHARE_READ|syscall.FILE_SHARE_WRITE), // Share Mode - nil, // Security Attributes - syscall.CREATE_ALWAYS, // Create Mode + nil, // Security Attributes + syscall.CREATE_ALWAYS, // Create Mode uint32(syscall.FILE_ATTRIBUTE_NORMAL|_FILE_FLAG_WRITE_THROUGH), // Flags and Attributes 0) // Template File diff --git a/journalbeat/docs/index.asciidoc b/journalbeat/docs/index.asciidoc index ac818e5b412..3c99a47bdd1 100644 --- a/journalbeat/docs/index.asciidoc +++ b/journalbeat/docs/index.asciidoc @@ -14,6 +14,7 @@ include::{asciidoc-dir}/../../shared/attributes.asciidoc[] :libbeat-docs: Beats Platform Reference :deb_os: :rpm_os: +:linux_os: :no_dashboards: include::../../libbeat/docs/shared-beats-attributes.asciidoc[] diff --git a/journalbeat/journalbeat.docker.yml b/journalbeat/journalbeat.docker.yml index aa709469a72..a1e67c0d961 100644 --- a/journalbeat/journalbeat.docker.yml +++ b/journalbeat/journalbeat.docker.yml @@ -1,6 +1,6 @@ journalbeat.inputs: - paths: [] - seek: tail + seek: cursor processors: - add_cloud_metadata: ~ diff --git a/libbeat/Dockerfile b/libbeat/Dockerfile index 4c9a303bd4d..9e1c4ec4d0e 100644 --- a/libbeat/Dockerfile +++ b/libbeat/Dockerfile @@ -1,5 +1,5 @@ # Beats dockerfile used for testing -FROM golang:1.10.3 +FROM golang:1.11.2 MAINTAINER Nicolas Ruflin RUN set -x && \ diff --git a/libbeat/autodiscover/appenders/config/config.go b/libbeat/autodiscover/appenders/config/config.go index 070d5380689..d95e6c7bbd4 100644 --- a/libbeat/autodiscover/appenders/config/config.go +++ b/libbeat/autodiscover/appenders/config/config.go @@ -56,7 +56,7 @@ func NewConfigAppender(cfg *common.Config) (autodiscover.Appender, error) { confs := configs{} err := cfg.Unpack(&confs) if err != nil { - return nil, fmt.Errorf("unable to unpack config appender due to error: %v", err) + return nil, fmt.Errorf("unable to unpack config appender due to error: %+v", err) } var configMaps []configMap @@ -66,7 +66,7 @@ func NewConfigAppender(cfg *common.Config) (autodiscover.Appender, error) { if conf.ConditionConfig != nil { cond, err = conditions.NewCondition(conf.ConditionConfig) if err != nil { - logp.Warn("config", "unable to create condition due to error: %v", err) + logp.Warn("unable to create condition due to error: %+v", err) continue } } @@ -76,7 +76,7 @@ func NewConfigAppender(cfg *common.Config) (autodiscover.Appender, error) { cf := common.MapStr{} err = conf.Config.Unpack(&cf) if err != nil { - logp.Warn("config", "unable to unpack config due to error: %v", err) + logp.Warn("unable to unpack config due to error: %+v", err) continue } cm.config = cf diff --git a/libbeat/autodiscover/autodiscover.go b/libbeat/autodiscover/autodiscover.go index 695f02567d9..28afd967680 100644 --- a/libbeat/autodiscover/autodiscover.go +++ b/libbeat/autodiscover/autodiscover.go @@ -225,7 +225,7 @@ func (a *Autodiscover) handleStop(event bus.Event) bool { delete(a.configs, hash) updated = true } else { - logp.Debug(debugK, "Runner not found for stopping: %s", hash) + logp.Debug(debugK, "Runner not found for stopping: %d", hash) } } diff --git a/libbeat/autodiscover/providers/jolokia/discovery.go b/libbeat/autodiscover/providers/jolokia/discovery.go index fc704dafb16..6492ab44151 100644 --- a/libbeat/autodiscover/providers/jolokia/discovery.go +++ b/libbeat/autodiscover/providers/jolokia/discovery.go @@ -192,7 +192,7 @@ var queryMessage = []byte(`{"type":"query"}`) func (d *Discovery) sendProbe(config InterfaceConfig) { interfaces, err := d.interfaces(config.Name) if err != nil { - logp.Err("failed to get interfaces: ", err) + logp.Err("failed to get interfaces: %+v", err) return } diff --git a/libbeat/autodiscover/providers/kubernetes/kubernetes.go b/libbeat/autodiscover/providers/kubernetes/kubernetes.go index c2f3b133c93..7f35e522295 100644 --- a/libbeat/autodiscover/providers/kubernetes/kubernetes.go +++ b/libbeat/autodiscover/providers/kubernetes/kubernetes.go @@ -73,7 +73,7 @@ func AutodiscoverBuilder(bus bus.Bus, c *common.Config) (autodiscover.Provider, Namespace: config.Namespace, }) if err != nil { - logp.Err("kubernetes: Couldn't create watcher for %t", &kubernetes.Pod{}) + logp.Err("kubernetes: Couldn't create watcher for %T", &kubernetes.Pod{}) return nil, err } diff --git a/libbeat/cmd/export/dashboard.go b/libbeat/cmd/export/dashboard.go index 59a6fa04cc3..697da10061e 100644 --- a/libbeat/cmd/export/dashboard.go +++ b/libbeat/cmd/export/dashboard.go @@ -20,11 +20,13 @@ package export import ( "fmt" "os" + "path/filepath" "github.com/spf13/cobra" "github.com/elastic/beats/libbeat/cmd/instance" "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/dashboards" "github.com/elastic/beats/libbeat/kibana" ) @@ -35,6 +37,8 @@ func GenDashboardCmd(name, idxPrefix, beatVersion string) *cobra.Command { Short: "Export defined dashboard to stdout", Run: func(cmd *cobra.Command, args []string) { dashboard, _ := cmd.Flags().GetString("id") + yml, _ := cmd.Flags().GetString("yml") + decode, _ := cmd.Flags().GetBool("decode") b, err := instance.NewBeat(name, idxPrefix, beatVersion) if err != nil { @@ -58,16 +62,46 @@ func GenDashboardCmd(name, idxPrefix, beatVersion string) *cobra.Command { os.Exit(1) } - result, err := client.GetDashboard(dashboard) - if err != nil { - fmt.Fprintf(os.Stderr, "Error getting dashboard: %+v\n", err) - os.Exit(1) + // Export dashboards from yml file + if yml != "" { + results, info, err := dashboards.ExportAllFromYml(client, yml) + if err != nil { + fmt.Fprintf(os.Stderr, "Error getting dashboards from yml: %+v\n", err) + os.Exit(1) + } + for i, r := range results { + if decode { + r = dashboards.DecodeExported(r) + } + err = dashboards.SaveToFile(r, info.Dashboards[i].File, filepath.Dir(yml), client.GetVersion()) + if err != nil { + fmt.Fprintf(os.Stderr, "Error saving dashboard '%s' to file '%s' : %+v\n", + info.Dashboards[i].ID, info.Dashboards[i].File, err) + os.Exit(1) + } + } + return + } + + // Export single dashboard + if dashboard != "" { + result, err := dashboards.Export(client, dashboard) + if err != nil { + fmt.Fprintf(os.Stderr, "Error getting dashboard: %+v\n", err) + os.Exit(1) + } + + if decode { + result = dashboards.DecodeExported(result) + } + fmt.Println(result.StringToPrint()) } - fmt.Println(result.StringToPrint()) }, } genTemplateConfigCmd.Flags().String("id", "", "Dashboard id") + genTemplateConfigCmd.Flags().String("yml", "", "Yaml file containing list of dashboard ID and filename pairs") + genTemplateConfigCmd.Flags().Bool("decode", false, "Decode exported dashboard") return genTemplateConfigCmd } diff --git a/libbeat/common/kubernetes/util.go b/libbeat/common/kubernetes/util.go index eb4654143d3..38ee7c8c6a3 100644 --- a/libbeat/common/kubernetes/util.go +++ b/libbeat/common/kubernetes/util.go @@ -75,19 +75,19 @@ func DiscoverKubernetesNode(host string, inCluster bool, client *k8s.Client) (no if inCluster { ns, err := inClusterNamespace() if err != nil { - logp.Err("kubernetes: Couldn't get namespace when beat is in cluster with error: ", err.Error()) + logp.Err("kubernetes: Couldn't get namespace when beat is in cluster with error: %+v", err.Error()) return defaultNode } podName, err := os.Hostname() if err != nil { - logp.Err("kubernetes: Couldn't get hostname as beat pod name in cluster with error: ", err.Error()) + logp.Err("kubernetes: Couldn't get hostname as beat pod name in cluster with error: %+v", err.Error()) return defaultNode } logp.Info("kubernetes: Using pod name %s and namespace %s to discover kubernetes node", podName, ns) pod := v1.Pod{} err = client.Get(context.TODO(), ns, podName, &pod) if err != nil { - logp.Err("kubernetes: Querying for pod failed with error: %v", err.Error()) + logp.Err("kubernetes: Querying for pod failed with error: %+v", err.Error()) return defaultNode } logp.Info("kubernetes: Using node %s discovered by in cluster pod node query", pod.Spec.GetNodeName()) @@ -103,7 +103,7 @@ func DiscoverKubernetesNode(host string, inCluster bool, client *k8s.Client) (no nodes := v1.NodeList{} err := client.List(context.TODO(), k8s.AllNamespaces, &nodes) if err != nil { - logp.Err("kubernetes: Querying for nodes failed with error: ", err.Error()) + logp.Err("kubernetes: Querying for nodes failed with error: %+v", err.Error()) return defaultNode } for _, n := range nodes.Items { diff --git a/libbeat/common/transport/tlscommon/tls.go b/libbeat/common/transport/tlscommon/tls.go index 6fe79e8df1d..43eb371cdb2 100644 --- a/libbeat/common/transport/tlscommon/tls.go +++ b/libbeat/common/transport/tlscommon/tls.go @@ -48,19 +48,19 @@ func LoadCertificate(config *CertificateConfig) (*tls.Certificate, error) { certPEM, err := ReadPEMFile(certificate, config.Passphrase) if err != nil { - logp.Critical("Failed reading certificate file %v: %v", certificate, err) + logp.Critical("Failed reading certificate file %v: %+v", certificate, err) return nil, fmt.Errorf("%v %v", err, certificate) } keyPEM, err := ReadPEMFile(key, config.Passphrase) if err != nil { - logp.Critical("Failed reading key file %v: %v", key, err) + logp.Critical("Failed reading key file %v: %+v", key, err) return nil, fmt.Errorf("%v %v", err, key) } cert, err := tls.X509KeyPair(certPEM, keyPEM) if err != nil { - logp.Critical("Failed loading client certificate", err) + logp.Critical("Failed loading client certificate %+v", err) return nil, err } diff --git a/libbeat/dashboards/decode.go b/libbeat/dashboards/decode.go new file mode 100644 index 00000000000..2b659d7251e --- /dev/null +++ b/libbeat/dashboards/decode.go @@ -0,0 +1,70 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package dashboards + +import ( + "encoding/json" + "fmt" + + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/logp" +) + +var ( + responseToDecode = []string{ + "attributes.uiStateJSON", + "attributes.visState", + "attributes.optionsJSON", + "attributes.panelsJSON", + "attributes.kibanaSavedObjectMeta.searchSourceJSON", + } +) + +// DecodeExported decodes an exported dashboard +func DecodeExported(result common.MapStr) common.MapStr { + // remove unsupported chars + objects := result["objects"].([]interface{}) + for _, obj := range objects { + o := obj.(common.MapStr) + for _, key := range responseToDecode { + // All fields are optional, so errors are not caught + err := decodeValue(o, key) + if err != nil { + logp.Debug("dashboards", "Error while decoding dashboard objects: %+v", err) + } + } + } + result["objects"] = objects + return result +} + +func decodeValue(data common.MapStr, key string) error { + v, err := data.GetValue(key) + if err != nil { + return err + } + s := v.(string) + var d interface{} + err = json.Unmarshal([]byte(s), &d) + if err != nil { + return fmt.Errorf("error decoding %s: %v", key, err) + } + + data.Put(key, d) + return nil +} diff --git a/libbeat/dashboards/export.go b/libbeat/dashboards/export.go new file mode 100644 index 00000000000..d74a1d84b7b --- /dev/null +++ b/libbeat/dashboards/export.go @@ -0,0 +1,106 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package dashboards + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "path/filepath" + "strconv" + + yaml "gopkg.in/yaml.v2" + + "github.com/elastic/beats/filebeat/scripts/generator" + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/kibana" +) + +const ( + dashboardPerm = 0644 +) + +// ListYML is the yaml file which contains list of available dashboards. +type ListYML struct { + Dashboards []YMLElement `yaml:"dashboards"` +} + +// YMLElement contains the data of a dashboard: +// * its uuid in Kibana +// * filename to be saved as +type YMLElement struct { + ID string `yaml:"id"` + File string `yaml:"file"` +} + +// Export wraps GetDashboard call to provide a more descriptive API +func Export(client *kibana.Client, id string) (common.MapStr, error) { + return client.GetDashboard(id) +} + +// ExportAllFromYml exports all dashboards found in the YML file +func ExportAllFromYml(client *kibana.Client, ymlPath string) ([]common.MapStr, ListYML, error) { + b, err := ioutil.ReadFile(ymlPath) + if err != nil { + return nil, ListYML{}, fmt.Errorf("error opening the list of dashboards: %+v", err) + } + var list ListYML + err = yaml.Unmarshal(b, &list) + if err != nil { + return nil, ListYML{}, fmt.Errorf("error reading the list of dashboards: %+v", err) + } + + results, err := ExportAll(client, list) + + return results, list, err +} + +// ExportAll exports all dashboards from an opened and parsed dashboards YML. +func ExportAll(client *kibana.Client, list ListYML) ([]common.MapStr, error) { + var results []common.MapStr + for _, e := range list.Dashboards { + result, err := Export(client, e.ID) + if err != nil { + return nil, err + } + results = append(results, result) + } + return results, nil +} + +// SaveToFile creates the required directories if needed and saves dashboard. +func SaveToFile(dashboard common.MapStr, filename, root, versionStr string) error { + version, err := common.NewVersion(versionStr) + if err != nil { + return err + } + + dashboardsPath := "_meta/kibana/" + strconv.Itoa(version.Major) + "/dashboard" + err = generator.CreateDirectories(root, dashboardsPath) + if err != nil { + return err + } + + out := filepath.Join(root, dashboardsPath, filename) + bytes, err := json.Marshal(dashboard) + if err != nil { + return err + } + + return ioutil.WriteFile(out, bytes, dashboardPerm) +} diff --git a/libbeat/docs/command-reference.asciidoc b/libbeat/docs/command-reference.asciidoc index 97834b75ba4..e30cabe3b5d 100644 --- a/libbeat/docs/command-reference.asciidoc +++ b/libbeat/docs/command-reference.asciidoc @@ -19,25 +19,25 @@ :help-command-short-desc: Shows help for any command :keystore-command-short-desc: Manages the <> :modules-command-short-desc: Manages configured modules -:package-command-short-desc: Packages the configuration and executable in a zip file +:package-command-short-desc: Packages the configuration and executable into a zip file :remove-command-short-desc: Removes the specified function from your serverless environment :run-command-short-desc: Runs {beatname_uc}. This command is used by default if you start {beatname_uc} without specifying a command ifndef::deprecate_dashboard_loading[] ifdef::has_ml_jobs[] -:setup-command-short-desc: Sets up the initial environment, including the index template, Kibana dashboards (when available), and machine learning jobs (when available) +:setup-command-short-desc: Sets up the initial environment, including the index template, {kib} dashboards (when available), and machine learning jobs (when available) endif::[] ifndef::has_ml_jobs[] -:setup-command-short-desc: Sets up the initial environment, including the index template and Kibana dashboards (when available) +:setup-command-short-desc: Sets up the initial environment, including the index template and {kib} dashboards (when available) endif::[] endif::[] ifdef::deprecate_dashboard_loading[] -:setup-command-short-desc: Sets up the initial environment, including the ES index template and Kibana dashboards (deprecated). +:setup-command-short-desc: Sets up the initial environment, including the ES index template and {kib} dashboards (deprecated). endif::[] @@ -67,7 +67,6 @@ The command-line also supports <> for controlling global behaviors. ifeval::["{beatname_lc}"!="winlogbeat"] - [TIP] ========================= Use `sudo` to run the following commands if: @@ -76,7 +75,6 @@ Use `sudo` to run the following commands if: * {beatname_uc} is configured to capture data that requires `root` access ========================= - endif::[] [options="header"] @@ -110,8 +108,33 @@ ifeval::[("{beatname_lc}"=="functionbeat")] [[deploy-command]] ==== `deploy` command -{deploy-command-short-desc}. +{deploy-command-short-desc}. Before deploying functions, make sure the user has +the credentials required by your cloud service provider. + +*SYNOPSIS* + +["source","sh",subs="attributes"] +---- +{beatname_lc} deploy FUNCTION_NAME [FLAGS] +---- + +*`FUNCTION_NAME`*:: +Specifies the name of the function to deploy. + +*FLAGS* + +*`-h, --help`*:: +Shows help for the `deploy` command. + +{global-flags} + +*EXAMPLES* +["source","sh",subs="attributes"] +----- +{beatname_lc} deploy cloudwatch +{beatname_lc} deploy sqs +----- endif::[] [[export-command]] @@ -119,7 +142,7 @@ endif::[] {export-command-short-desc}. You can use this command to quickly view your configuration, see the contents of the index -template, or export a dashboard from Kibana. +template, or export a dashboard from {kib}. *SYNOPSIS* @@ -128,55 +151,55 @@ template, or export a dashboard from Kibana. {beatname_lc} export SUBCOMMAND [FLAGS] ---- - *SUBCOMMANDS* *`config`*:: Exports the current configuration to stdout. If you use the `-c` flag, this command exports the configuration that's defined in the specified file. - - -*`dashboard`*:: -Exporting a dashboard allows to store a dashboard on disk in a -module and load it automatically. The following command can be used: +[[dashboard-subcommand]]*`dashboard`*:: +Exports a dashboard. You can use this option to store a dashboard on disk in a +module and load it automatically. For example, to export the dashboard to a JSON +file, run: + ["source","shell",subs="attributes"] ---- -{beatname_lc} export dashboard --id="dashboard-id" > dashboard.json +{beatname_lc} export dashboard --id="DASHBOARD_ID" > dashboard.json ---- + -The `dashboard-id` can be found in the Kibana URL. By default `export dashboard` -will write the dashboard to stdout. Above it's written into `dashboard.json` so -it can later imported again. The file contains the dashboard with all -visualizations and searches. The index pattern is removed as it is -expected to be loaded separately for {beatname_uc}. +To find the `DASHBOARD_ID`, look at the URL for the dashboard in {kib}. By +default, `export dashboard` writes the dashboard to stdout. The example shows +how to write the dashboard to a JSON file so that you can import it later. The +JSON file will contain the dashboard with all visualizations and searches. You +must load the index pattern separately for {beatname_uc}. + -The generated `dashboard.json` file can be copied into the `kibana/6/dashboard` -directory of {beatname_lc} and next time +{beatname_lc} setup dashboards+ is -run the dashboard will be imported. +To load the dashboard, copy the generated `dashboard.json` file into the +`kibana/6/dashboard` directory of {beatname_uc}, and run ++{beatname_lc} setup --dashboards+ to import the dashboard. + -In case Kibana is not running on `localhost:5061` the {beatname_uc} -configuration under `setup.kibana` must be adjusted. +If {kib} is not running on `localhost:5061`, you must also adjust the +{beatname_uc} configuration under `setup.kibana`. -[[template-subcommand]] -*`template`*:: +[[template-subcommand]]*`template`*:: Exports the index template to stdout. You can specify the `--es.version` and `--index` flags to further define what gets exported. *FLAGS* *`--es.version VERSION`*:: -When specified along with <>, exports an index +When used with <>, exports an index template that is compatible with the specified version. *`-h, --help`*:: Shows help for the `export` command. *`--index BASE_NAME`*:: -When specified along with <>, sets the base name -to use for the index template. If this flag is not specified, the default base -name is +{beatname_lc}+. +When used with <>, sets the base name to use for +the index template. If this flag is not specified, the default base name is ++{beatname_lc}+. + +*`--id DASHBOARD_ID`*:: +When used with <>, specifies the dashboard ID. {global-flags} @@ -186,6 +209,7 @@ name is +{beatname_lc}+. ----- {beatname_lc} export config {beatname_lc} export template --es.version {stack-version} --index myindexname +{beatname_lc} export dashboard --id="a7b35890-8baa-11e8-9676-ef67484126fb" > dashboard.json ----- @@ -272,7 +296,7 @@ Shows help for the `keystore` command. {beatname_lc} keystore list ----- -see <> for more examples. +See <> for more examples. ifeval::[("{beatname_lc}"=="functionbeat")] [[package-command]] @@ -280,15 +304,63 @@ ifeval::[("{beatname_lc}"=="functionbeat")] {package-command-short-desc}. +*SYNOPSIS* + +["source","sh",subs="attributes"] +---- +{beatname_lc} package [FLAGS] +---- + +*FLAGS* + +*`-h, --help`*:: +Shows help for the `package` command. + +*`-o, --output`*:: +Specifies the full path to the zip file that will contain the package. + +{global-flags} + +*EXAMPLES* + +["source","sh",subs="attributes"] +----- +{beatname_lc} package /path/to/file.zip +----- + [[remove-command]] ==== `remove` command -{remove-command-short-desc}. +{remove-command-short-desc}. Before removing functions, make sure the user has +the credentials required by your cloud service provider. + +*SYNOPSIS* + +["source","sh",subs="attributes"] +---- +{beatname_lc} remove FUNCTION_NAME [FLAGS] +---- + +*`FUNCTION_NAME`*:: +Specifies the name of the function to remove. + +*FLAGS* +*`-h, --help`*:: +Shows help for the `remove` command. + +{global-flags} + +*EXAMPLES* + +["source","sh",subs="attributes"] +----- +{beatname_lc} remove cloudwatch +{beatname_lc} remove sqs +----- endif::[] ifeval::[("{beatname_lc}"=="filebeat") or ("{beatname_lc}"=="metricbeat")] - [[modules-command]] ==== `modules` command @@ -330,26 +402,20 @@ Shows help for the `export` command. *EXAMPLES* ifeval::["{beatname_lc}"=="filebeat"] - ["source","sh",subs="attributes"] ----- {beatname_lc} modules list {beatname_lc} modules enable apache2 auditd mysql ----- - endif::[] ifeval::["{beatname_lc}"=="metricbeat"] - ["source","sh",subs="attributes"] ----- {beatname_lc} modules list {beatname_lc} modules enable apache nginx system ----- - - endif::[] - endif::[] @@ -375,7 +441,6 @@ Or: *FLAGS* ifeval::["{beatname_lc}"=="packetbeat"] - *`-I, --I FILE`*:: Reads packet data from the specified file instead of reading packets from the network. This option is useful only for testing {beatname_uc}. @@ -384,19 +449,17 @@ network. This option is useful only for testing {beatname_uc}. ----- {beatname_lc} run -I ~/pcaps/network_traffic.pcap ----- - endif::[] -*`-N, --N`*:: -Disables the publishing of events to the defined output. This option is useful -only for testing {beatname_uc}. +*`-N, --N`*:: Disables publishing for testing purposes. +ifndef::only-elasticsearch[] +This option disables all outputs except the <>. +endif::only-elasticsearch[] ifeval::["{beatname_lc}"=="packetbeat"] - *`-O, --O`*:: Read packets one by one by pressing _Enter_ after each. This option is useful only for testing {beatname_uc}. - endif::[] *`--cpuprofile FILE`*:: @@ -404,18 +467,14 @@ Writes CPU profile data to the specified file. This option is useful for troubleshooting {beatname_uc}. ifeval::["{beatname_lc}"=="packetbeat"] - *`-devices`*:: Prints the list of devices that are available for sniffing and then exits. - endif::[] ifeval::["{beatname_lc}"=="packetbeat"] - *`-dump FILE`*:: Writes all captured packets to the specified file. This option is useful for troubleshooting {beatname_uc}. - endif::[] *`-h, --help`*:: @@ -426,12 +485,10 @@ Starts an http server for profiling. This option is useful for troubleshooting and profiling {beatname_uc}. ifeval::["{beatname_lc}"=="packetbeat"] - *`-l N`*:: Reads the pcap file `N` number of times. The default is 1. Use this option in combination with the `-I` option. For an infinite loop, use _0_. The `-l` option is useful only for testing {beatname_uc}. - endif::[] *`--memprofile FILE`*:: @@ -439,7 +496,6 @@ Writes memory profile data to the specified output file. This option is useful for troubleshooting {beatname_uc}. ifeval::["{beatname_lc}"=="filebeat"] - *`--modules MODULE_LIST`*:: Specifies a comma-separated list of modules to run. For example: + @@ -452,18 +508,15 @@ Rather than specifying the list of modules every time you run {beatname_uc}, you can use the <> command to enable and disable specific modules. Then when you run {beatname_uc}, it will run any modules that are enabled. - endif::[] ifeval::["{beatname_lc}"=="filebeat"] - *`--once`*:: When the `--once` flag is used, {beatname_uc} starts all configured harvesters and inputs, and runs each input until the harvesters are closed. If you set the `--once` flag, you should also set `close_eof` so the harvester is closed when the end of the file is reached. By default harvesters are closed after `close_inactive` is reached. - endif::[] *`--setup`*:: @@ -472,31 +525,27 @@ deprecated[{deprecate_dashboard_loading}] endif::[] + ifdef::has_ml_jobs[] -Loads the initial setup, including Elasticsearch template, Kibana index pattern, -Kibana dashboards (when available), and Machine learning jobs. +Loads the initial setup, including Elasticsearch template, {kib} index pattern, +{kib} dashboards (when available), and Machine learning jobs. endif::[] ifndef::has_ml_jobs[] -Loads the initial setup, including Elasticsearch template, Kibana index pattern, and Kibana dashboards (when available). +Loads the initial setup, including Elasticsearch template, {kib} index pattern, and {kib} dashboards (when available). endif::[] If you want to use the command without running {beatname_uc}, use the <> command instead. ifeval::["{beatname_lc}"=="metricbeat"] - *`--system.hostfs MOUNT_POINT`*:: Specifies the mount point of the host's filesystem for use in monitoring a host from within a container. - endif::[] ifeval::["{beatname_lc}"=="packetbeat"] - *`-t`*:: Reads packets from the pcap file as fast as possible without sleeping. Use this option in combination with the `-I` option. The `-t` option is useful only for testing Packetbeat. - endif::[] {global-flags} @@ -522,8 +571,8 @@ Or: * The index template ensures that fields are mapped correctly in Elasticsearch. -* The Kibana dashboards make it easier for you to visualize {beatname_uc} data -in Kibana. +* The {kib} dashboards make it easier for you to visualize {beatname_uc} data +in {kib}. ifdef::has_ml_jobs[] * The machine learning jobs contain the configuration information and metadata @@ -545,7 +594,7 @@ environment without actually running {beatname_uc} and ingesting data. ifndef::deprecate_dashboard_loading[] *`--dashboards`*:: -Sets up the Kibana dashboards (when available). This option loads the dashboards +Sets up the {kib} dashboards (when available). This option loads the dashboards from the {beatname_uc} package. For more options, such as loading customized dashboards, see {beatsdevguide}/import-dashboards.html[Importing Existing Beat Dashboards] in the _Beats Developer Guide_. @@ -556,28 +605,24 @@ ifdef::deprecate_dashboard_loading[] deprecated[{deprecate_dashboard_loading}] + -Sets up the Kibana dashboards only. +Sets up the {kib} dashboards only. endif::[] *`-h, --help`*:: Shows help for the `setup` command. ifdef::has_ml_jobs[] - *`--machine-learning`*:: Sets up machine learning job configurations only. - endif::[] ifeval::["{beatname_lc}"=="filebeat"] - *`--modules MODULE_LIST`*:: Specifies a comma-separated list of modules. Use this flag to avoid errors when there are no modules defined in the +{beatname_lc}.yml+ file. *`--pipelines`*:: Sets up ingest pipelines for configured filesets. - endif::[] *`--template`*:: @@ -611,14 +656,12 @@ Sets up the index template only. Tests the configuration settings. ifeval::["{beatname_lc}"=="metricbeat"] - *`modules [MODULE_NAME] [METRICSET_NAME]`*:: Tests module settings for all configured modules. When you run this command, {beatname_uc} does a test run that applies the current settings, retrieves the metrics, and shows them as output. To test the settings for a specific module, specify `MODULE_NAME`. To test the settings for a specific metricset in the module, also specify `METRICSET_NAME`. - endif::[] *`output`*:: @@ -632,18 +675,15 @@ current settings. {global-flags} ifeval::["{beatname_lc}"!="metricbeat"] - *EXAMPLE* ["source","sh",subs="attributes"] ----- {beatname_lc} test config ----- - endif::[] ifeval::["{beatname_lc}"=="metricbeat"] - *EXAMPLES* ["source","sh",subs="attributes"] @@ -651,15 +691,39 @@ ifeval::["{beatname_lc}"=="metricbeat"] {beatname_lc} test config {beatname_lc} test modules system cpu ----- - endif::[] ifeval::[("{beatname_lc}"=="functionbeat")] [[update-command]] ==== `update` command -{update-command-short-desc}. +{update-command-short-desc}. Before updating functions, make sure the user has +the credentials required by your cloud service provider. +*SYNOPSIS* + +["source","sh",subs="attributes"] +---- +{beatname_lc} update FUNCTION_NAME [FLAGS] +---- + +*`FUNCTION_NAME`*:: +Specifies the name of the function to update. + +*FLAGS* + +*`-h, --help`*:: +Shows help for the `update` command. + +{global-flags} + +*EXAMPLES* + +["source","sh",subs="attributes"] +----- +{beatname_lc} update cloudwatch +{beatname_lc} update sqs +----- endif::[] [[version-command]] @@ -686,7 +750,7 @@ endif::[] ["source","sh",subs="attributes"] ----- {beatname_lc} version ----- +----- [float] @@ -708,7 +772,6 @@ This setting is applied to the currently running {beatname_uc} process. The {beatname_uc} configuration file is not changed. ifeval::["{beatname_lc}"=="filebeat"] - *`-M, --M "VAR_NAME=VALUE"`*:: Overrides the default configuration for a {beatname_uc} module. You can specify multiple variable overrides. For example: + @@ -716,7 +779,6 @@ ifeval::["{beatname_lc}"=="filebeat"] ---------------------------------------------------------------------- {beatname_lc} -modules=nginx -M "nginx.access.var.paths=['/var/log/nginx/access.log*']" -M "nginx.access.var.pipeline=no_plugins" ---------------------------------------------------------------------- - endif::[] *`-c, --c FILE`*:: diff --git a/libbeat/docs/dashboards.asciidoc b/libbeat/docs/dashboards.asciidoc index ea210d56c95..96bf17b5527 100644 --- a/libbeat/docs/dashboards.asciidoc +++ b/libbeat/docs/dashboards.asciidoc @@ -58,6 +58,15 @@ ifdef::mac_os[] ---------------------------------------------------------------------- endif::mac_os[] +ifdef::linux_os[] +*linux:* + +["source","sh",subs="attributes"] +---------------------------------------------------------------------- +./{beatname_lc} setup --dashboards +---------------------------------------------------------------------- +endif::linux_os[] + ifdef::docker_platform[] *docker:* @@ -125,6 +134,19 @@ ifdef::mac_os[] ---- endif::mac_os[] +ifdef::linux_os[] +*linux:* + +["source","sh",subs="attributes"] +---- +./{beatname_lc} setup -e \ + -E output.logstash.enabled=false \ + -E output.elasticsearch.hosts=['localhost:9200'] \ + -E output.elasticsearch.username={beat_default_index_prefix}_internal \ + -E output.elasticsearch.password={pwd} \ + -E setup.kibana.host=localhost:5601 +---- +endif::linux_os[] ifdef::docker_platform[] *docker:* diff --git a/libbeat/docs/reference-yml.asciidoc b/libbeat/docs/reference-yml.asciidoc index f506d61c337..44361b4fe55 100644 --- a/libbeat/docs/reference-yml.asciidoc +++ b/libbeat/docs/reference-yml.asciidoc @@ -5,9 +5,8 @@ The following reference file is available with your {beatname_uc} installation. shows all non-deprecated {beatname_uc} options. You can copy from this file and paste configurations into the +{beatname_lc}.yml+ file to customize it. -TIP: For rpm and deb, you'll find the reference configuration file at +/etc/{beatname_lc}/{beatname_lc}.reference.yml+. Under -Docker, it's located at +/usr/share/{beatname_lc}/{beatname_lc}.reference.yml+. For mac and win, -look in the archive that you just extracted. +TIP: The reference file is located in the same directory as the ++{beatname_lc}.yml+ file. To locate the file, see <>. The contents of the file are included here for your convenience. diff --git a/libbeat/docs/shared-template-load.asciidoc b/libbeat/docs/shared-template-load.asciidoc index f3d23d4d767..f09aec8305e 100644 --- a/libbeat/docs/shared-template-load.asciidoc +++ b/libbeat/docs/shared-template-load.asciidoc @@ -155,14 +155,24 @@ ifdef::mac_os[] ---- endif::mac_os[] -ifdef::mac_os[] +ifdef::linux_os[] +*linux:* + +["source","sh",subs="attributes"] +---- +./{beatname_lc} setup --template{disable_logstash} -E 'output.elasticsearch.hosts=["localhost:9200"]' +---- +endif::linux_os[] + + +ifdef::docker_platform[] *docker:* ["source","sh",subs="attributes"] ---------------------------------------------------------------------- docker run {dockerimage} setup --template{disable_logstash} -E 'output.elasticsearch.hosts=["localhost:9200"]' ---------------------------------------------------------------------- -endif::mac_os[] +endif::docker_platform[] ifdef::win_os[] ifndef::win_only[] @@ -187,7 +197,9 @@ endif::win_os[] If you've already used {beatname_uc} to index data into Elasticsearch, the index may contain old documents. After you load the index template, you can delete the old documents from +{beatname_lc}-*+ to force Kibana to look -at the newest documents. Use this command: +at the newest documents. + +Use this command: ifdef::deb_os,rpm_os[] *deb and rpm:* @@ -207,6 +219,15 @@ curl -XDELETE 'http://localhost:9200/{beatname_lc}-*' ---------------------------------------------------------------------- endif::mac_os[] +ifdef::linux_os[] +*linux:* + +["source","sh",subs="attributes"] +---------------------------------------------------------------------- +curl -XDELETE 'http://localhost:9200/{beatname_lc}-*' +---------------------------------------------------------------------- +endif::linux_os[] + ifdef::win_os[] ifndef::win_only[] *win:* @@ -249,6 +270,15 @@ ifdef::mac_os[] ---- endif::mac_os[] +ifdef::linux_os[] +*linux:* + +["source","sh",subs="attributes"] +---- +./{beatname_lc} export template > {beatname_lc}.template.json +---- +endif::linux_os[] + ifdef::win_os[] ifndef::win_only[] *win:* @@ -280,6 +310,15 @@ curl -XPUT -H 'Content-Type: application/json' http://localhost:9200/_template/{ ---- endif::mac_os[] +ifdef::linux_os[] +*linux:* + +["source","sh",subs="attributes"] +---- +curl -XPUT -H 'Content-Type: application/json' http://localhost:9200/_template/{beatname_lc}-{stack-version} -d@{beatname_lc}.template.json +---- +endif::linux_os[] + ifdef::win_os[] ifndef::win_only[] *win:* diff --git a/libbeat/docs/version.asciidoc b/libbeat/docs/version.asciidoc index 2874d8bccca..d16c330f597 100644 --- a/libbeat/docs/version.asciidoc +++ b/libbeat/docs/version.asciidoc @@ -1,6 +1,6 @@ :stack-version: 7.0.0-alpha1 :doc-branch: master -:go-version: 1.10.3 +:go-version: 1.11.2 :release-state: prerelease :python: 2.7.9 :docker: 1.12 diff --git a/libbeat/generator/fields/validate/mapping.go b/libbeat/generator/fields/validate/mapping.go index 4a9e03079d9..a364511a0c9 100644 --- a/libbeat/generator/fields/validate/mapping.go +++ b/libbeat/generator/fields/validate/mapping.go @@ -198,7 +198,7 @@ func recursiveFattenFields(fields interface{}, prefix Prefix, mapping *Mapping, if hasRequired { required, ok = requiredIf.(bool) if !ok { - return errors.Errorf("field [%s](%s) has 'required' property but is not a boolean, but %T (value=%v)", requiredIf, requiredIf) + return errors.Errorf("field [%s](%s) has 'required' property but is not a boolean, but %T (value=%v)", key, prefix, requiredIf, requiredIf) } } @@ -235,7 +235,7 @@ func recursiveFattenFields(fields interface{}, prefix Prefix, mapping *Mapping, if fieldsIf != nil { innerFields, ok := fieldsIf.([]interface{}) if !ok { - return errors.Errorf("field [%s](%s) has a 'fields' tag of unexpected type (type=%T value=%v)", nameIf, nameIf) + return errors.Errorf("field [%s](%s) has a 'fields' tag of unexpected type (type=%T value=%v)", key, prefix, nameIf, nameIf) } for _, field := range innerFields { if err := recursiveFattenFields(field, prefix, mapping, key); err != nil { diff --git a/libbeat/kibana/client.go b/libbeat/kibana/client.go index 75323a3b655..fe21af40dcb 100644 --- a/libbeat/kibana/client.go +++ b/libbeat/kibana/client.go @@ -283,6 +283,7 @@ func (client *Client) GetDashboard(id string) (common.MapStr, error) { if err != nil { return nil, fmt.Errorf("error removing index pattern: %+v", err) } + return result, nil } diff --git a/libbeat/kibana/fields_transformer.go b/libbeat/kibana/fields_transformer.go index 8f4f7005a38..581f4bcb1e8 100644 --- a/libbeat/kibana/fields_transformer.go +++ b/libbeat/kibana/fields_transformer.go @@ -41,7 +41,7 @@ func newFieldsTransformer(version *common.Version, fields common.Fields) (*field version: version, transformedFields: []common.MapStr{}, transformedFieldFormatMap: common.MapStr{}, - keys: common.MapStr{}, + keys: common.MapStr{}, }, nil } diff --git a/libbeat/outputs/elasticsearch/client.go b/libbeat/outputs/elasticsearch/client.go index 308767b1eaa..83bf3fe83a5 100644 --- a/libbeat/outputs/elasticsearch/client.go +++ b/libbeat/outputs/elasticsearch/client.go @@ -761,7 +761,7 @@ func (conn *Connection) execRequest( ) (int, []byte, error) { req, err := http.NewRequest(method, url, body) if err != nil { - logp.Warn("Failed to create request", err) + logp.Warn("Failed to create request %+v", err) return 0, nil, err } if body != nil { diff --git a/libbeat/outputs/transport/transptest/testing.go b/libbeat/outputs/transport/transptest/testing.go index f657b173278..402be992d1d 100644 --- a/libbeat/outputs/transport/transptest/testing.go +++ b/libbeat/outputs/transport/transptest/testing.go @@ -236,7 +236,7 @@ func GenCertForTestingPurpose(t *testing.T, host, name, keyPassword string) erro NotAfter: time.Now().AddDate(10, 0, 0), SubjectKeyId: []byte("12345"), BasicConstraintsValid: true, - IsCA: true, + IsCA: true, ExtKeyUsage: []x509.ExtKeyUsage{ x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, diff --git a/libbeat/plugin/cli.go b/libbeat/plugin/cli.go index 6171bef7e6b..425b2b42934 100644 --- a/libbeat/plugin/cli.go +++ b/libbeat/plugin/cli.go @@ -39,7 +39,7 @@ func (p *pluginList) String() string { func (p *pluginList) Set(v string) error { for _, path := range p.paths { if path == v { - logp.Warn("%s is already a registered plugin") + logp.Warn("%s is already a registered plugin", path) return nil } } diff --git a/libbeat/processors/add_kubernetes_metadata/kubernetes.go b/libbeat/processors/add_kubernetes_metadata/kubernetes.go index 63915e4b6ea..414d98b0b44 100644 --- a/libbeat/processors/add_kubernetes_metadata/kubernetes.go +++ b/libbeat/processors/add_kubernetes_metadata/kubernetes.go @@ -103,7 +103,7 @@ func newKubernetesAnnotator(cfg *common.Config) (processors.Processor, error) { config.Host = kubernetes.DiscoverKubernetesNode(config.Host, config.InCluster, client) - logp.Debug("kubernetes", "Using host ", config.Host) + logp.Debug("kubernetes", "Using host: %s", config.Host) logp.Debug("kubernetes", "Initializing watcher") watcher, err := kubernetes.NewWatcher(client, &kubernetes.Pod{}, kubernetes.WatchOptions{ @@ -112,7 +112,7 @@ func newKubernetesAnnotator(cfg *common.Config) (processors.Processor, error) { Namespace: config.Namespace, }) if err != nil { - logp.Err("kubernetes: Couldn't create watcher for %t", &kubernetes.Pod{}) + logp.Err("kubernetes: Couldn't create watcher for %T", &kubernetes.Pod{}) return nil, err } diff --git a/libbeat/reader/debug/debug.go b/libbeat/reader/debug/debug.go new file mode 100644 index 00000000000..31012158dc5 --- /dev/null +++ b/libbeat/reader/debug/debug.go @@ -0,0 +1,178 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package debug + +import ( + "bytes" + "io" + + "github.com/elastic/beats/libbeat/logp" +) + +const ( + offsetStart = 100 + offsetEnd = 100 + defaultMinBuffer = 16 * 1024 + defaultMaxFailures = 100 +) + +type state int + +const ( + initial state = iota + running + stopped +) + +// CheckFunc func receive a slice of bytes and returns true if it match the predicate. +type CheckFunc func(offset int64, buf []byte) bool + +// Reader is a debug reader used to check the values of specific bytes from an io.Reader, +// Is is useful is you want to detect if you have received garbage from a network volume. +type Reader struct { + log *logp.Logger + reader io.Reader + buffer bytes.Buffer + minBufferSize int + maxFailures int + failures int + predicate CheckFunc + state state + offset int64 +} + +// NewReader returns a debug reader. +func NewReader( + log *logp.Logger, + reader io.Reader, + minBufferSize int, + maxFailures int, + predicate CheckFunc, +) (*Reader, error) { + return &Reader{ + log: log, + minBufferSize: minBufferSize, + reader: reader, + maxFailures: maxFailures, + predicate: predicate, + }, nil +} + +// Read will proxy the read call to the original reader and will periodically checks the values of +// bytes in the buffer. +func (r *Reader) Read(p []byte) (int, error) { + if r.state == stopped { + return r.reader.Read(p) + } + + if r.state == running && r.failures > r.maxFailures { + // cleanup any remaining bytes in the buffer. + if r.buffer.Len() > 0 { + r.predicate(r.offset, r.buffer.Bytes()) + } + r.buffer = bytes.Buffer{} + r.log.Info("Stopping debug reader, max execution reached") + r.state = stopped + return r.reader.Read(p) + } + + if r.state == initial { + r.log.Infof( + "Starting debug reader with a buffer size of %d and max failures of %d", + r.minBufferSize, + r.maxFailures, + ) + r.state = running + } + + n, err := r.reader.Read(p) + + if n != 0 { + r.buffer.Write(p[:n]) + if r.buffer.Len() >= r.minBufferSize { + if r.failures < r.maxFailures && r.predicate(r.offset, r.buffer.Bytes()) { + r.failures++ + } + r.buffer.Reset() + } + r.offset += int64(n) + } + return n, err +} + +func makeNullCheck(log *logp.Logger, minSize int) CheckFunc { + // create a slice with null bytes to match on the buffer. + pattern := make([]byte, minSize, minSize) + return func(offset int64, buf []byte) bool { + idx := bytes.Index(buf, pattern) + if idx <= 0 { + offset += int64(len(buf)) + return false + } + reportNull(log, offset+int64(idx), idx, buf) + return true + } +} + +func reportNull(log *logp.Logger, offset int64, idx int, buf []byte) { + relativePos, surround := summarizeBufferInfo(idx, buf) + log.Debugf( + "Matching null byte found at offset %d (position %d in surrounding string: %s, bytes: %+v", + offset, + relativePos, + string(surround), + surround) +} + +func summarizeBufferInfo(idx int, buf []byte) (int, []byte) { + startAt := idx - offsetStart + var relativePos int + if startAt < 0 { + startAt = 0 + relativePos = idx + } else { + relativePos = offsetStart + } + + endAt := idx + offsetEnd + if endAt >= len(buf) { + endAt = len(buf) + } + surround := buf[startAt:endAt] + return relativePos, surround +} + +// AppendReaders look into the current enabled log selector and will add any debug reader that match +// the selectors. +func AppendReaders(reader io.Reader) (io.Reader, error) { + var err error + + if logp.HasSelector("detect_null_bytes") || logp.HasSelector("*") { + log := logp.NewLogger("detect_null_bytes") + if reader, err = NewReader( + log, + reader, + defaultMinBuffer, + defaultMaxFailures, + makeNullCheck(log, 4), + ); err != nil { + return nil, err + } + } + return reader, nil +} diff --git a/libbeat/reader/debug/debug_test.go b/libbeat/reader/debug/debug_test.go new file mode 100644 index 00000000000..deb89eb0834 --- /dev/null +++ b/libbeat/reader/debug/debug_test.go @@ -0,0 +1,146 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package debug + +import ( + "bytes" + "io" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/logp" +) + +func TestMakeNullCheck(t *testing.T) { + t.Run("return true if null byte is received", func(t *testing.T) { + check := makeNullCheck(logp.NewLogger("detect-null"), 1) + assert.True(t, check(100, []byte{'a', 'b', 'c', 0x0, 'd'})) + }) + + t.Run("return false on anything other bytes", func(t *testing.T) { + check := makeNullCheck(logp.NewLogger("detect-null"), 1) + assert.False(t, check(100, []byte{'a', 'b', 'c', 'd'})) + }) + + t.Run("return true when a slice of bytes is present", func(t *testing.T) { + check := makeNullCheck(logp.NewLogger("detect-null"), 3) + assert.True(t, check(100, []byte{'a', 'b', 'c', 0x0, 0x0, 0x0, 'd'})) + }) +} + +func TestSummarizeBufferInfo(t *testing.T) { + t.Run("when position is the start of the buffer", func(t *testing.T) { + relativePos, surround := summarizeBufferInfo(0, []byte("hello world")) + assert.Equal(t, 0, relativePos) + assert.Equal(t, []byte("hello world"), surround) + }) + + t.Run("when position is not the start of the buffer", func(t *testing.T) { + c, _ := common.RandomBytes(10000) + relativePos, surround := summarizeBufferInfo(200, c) + assert.Equal(t, 100, relativePos) + assert.Equal(t, 200, len(surround)) + }) +} + +func TestReader(t *testing.T) { + t.Run("check that we check the content of byte", testCheckContent) + t.Run("consume all bytes", testConsumeAll) + t.Run("empty buffer", testEmptyBuffer) + t.Run("should become silent after hitting max failures", testSilent) +} + +func testCheckContent(t *testing.T) { + var c int + check := func(_ int64, _ []byte) bool { + c++ + return true + } + + var s bytes.Buffer + s.WriteString("hello world") + s.WriteByte(0x00) + s.WriteString("hello world") + + reader, _ := NewReader(logp.L(), &s, 5, 3, check) + + _, err := reader.Read(make([]byte, 20)) + if !assert.NoError(t, err) { + return + } + + assert.Equal(t, 1, c) +} + +func testConsumeAll(t *testing.T) { + c, _ := common.RandomBytes(2000) + reader := bytes.NewReader(c) + var buf bytes.Buffer + consumed := 0 + debug, _ := NewReader(logp.L(), reader, 8, 20, makeNullCheck(logp.L(), 1)) + for consumed < 2000 { + data := make([]byte, 33) + n, _ := debug.Read(data) + buf.Write(data[:n]) + consumed += n + } + assert.Equal(t, len(c), consumed) + assert.Equal(t, c, buf.Bytes()) +} + +func testEmptyBuffer(t *testing.T) { + var buf bytes.Buffer + debug, _ := NewReader(logp.L(), &buf, 8, 20, makeNullCheck(logp.L(), 1)) + data := make([]byte, 33) + n, err := debug.Read(data) + assert.Equal(t, io.EOF, err) + assert.Equal(t, 0, n) +} + +func testSilent(t *testing.T) { + var c int + check := func(_ int64, buf []byte) bool { + pattern := make([]byte, 1, 1) + idx := bytes.Index(buf, pattern) + if idx <= 0 { + return false + } + c++ + return true + } + + var b bytes.Buffer + b.Write([]byte{'a', 'b', 'c', 'd', 0x00, 'e'}) + b.Write([]byte{'a', 'b', 'c', 'd', 0x00, 'e'}) + b.Write([]byte{'a', 'b', 'c', 'd', 0x00, 'e'}) + b.Write([]byte{'a', 'b', 'c', 'd', 0x00, 'e'}) + b.Write([]byte{'a', 'b', 'c', 'd', 0x00, 'e'}) + b.Write([]byte{'a', 'b', 'c', 'd', 0x00, 'e'}) + b.Write([]byte{'a', 'b', 'c', 'd', 0x00, 'e'}) + + debug, _ := NewReader(logp.L(), &b, 3, 2, check) + consumed := 0 + for consumed < b.Len() { + n, _ := debug.Read(make([]byte, 3)) + consumed += n + } + assert.Equal(t, 2, c) + assert.Equal(t, consumed, b.Len()) +} diff --git a/libbeat/reader/readfile/line.go b/libbeat/reader/readfile/line.go index be2714b498f..07e8590b02e 100644 --- a/libbeat/reader/readfile/line.go +++ b/libbeat/reader/readfile/line.go @@ -87,7 +87,7 @@ func (r *LineReader) Next() ([]byte, int, error) { if buf[len(buf)-1] == '\n' { break } else { - logp.Debug("line", "Line ending char found which wasn't one: %s", buf[len(buf)-1]) + logp.Debug("line", "Line ending char found which wasn't one: %c", buf[len(buf)-1]) } } diff --git a/libbeat/scripts/Makefile b/libbeat/scripts/Makefile index 1655bf3131f..4a5b00734da 100755 --- a/libbeat/scripts/Makefile +++ b/libbeat/scripts/Makefile @@ -211,14 +211,6 @@ system-tests: prepare-tests ${BEAT_NAME}.test python-env system-tests-environment: ## @testing Runs the system tests inside a virtual environment. This can be run on any docker-machine (local, remote) system-tests-environment: prepare-tests build-image ${DOCKER_COMPOSE} run -e INTEGRATION_TESTS=1 -e TESTING_ENVIRONMENT=${TESTING_ENVIRONMENT} -e DOCKER_COMPOSE_PROJECT_NAME=${DOCKER_COMPOSE_PROJECT_NAME} beat make system-tests - #This is a hack to run x-pack/filebeat module tests - @XPACKBEAT="${ES_BEATS}/x-pack/${BEAT_NAME}" ; \ - if [ -e "$$XPACKBEAT/tests/system" ] && [ "$(BEAT_NAME)" != "libbeat" ] && [ $(XPACK_ONLY) = false ]; then \ - $(MAKE) -C ../x-pack/${BEAT_NAME} fields; \ - ${DOCKER_COMPOSE} run -e INTEGRATION_TESTS=1 -e MODULES_PATH="../../x-pack/${BEAT_NAME}/module" -e TESTING_ENVIRONMENT=${TESTING_ENVIRONMENT} -e DOCKER_COMPOSE_PROJECT_NAME=${DOCKER_COMPOSE_PROJECT_NAME} beat make -C "$$XPACKBEAT" ${BEAT_NAME}.test system-tests ; \ - $(MAKE) -C ../x-pack/${BEAT_NAME} fix-permissions; \ - fi - .PHONY: fast-system-tests fast-system-tests: ## @testing Runs system tests without coverage reports and in parallel diff --git a/libbeat/tests/files/dashboards.yml b/libbeat/tests/files/dashboards.yml new file mode 100644 index 00000000000..8278dd7b319 --- /dev/null +++ b/libbeat/tests/files/dashboards.yml @@ -0,0 +1,3 @@ +dashboards: +- id: Metricbeat-system-overview + file: Metricbeat-system-test-overview.json diff --git a/libbeat/tests/system/test_base.py b/libbeat/tests/system/test_base.py index c775f0b644b..0a3c8585036 100644 --- a/libbeat/tests/system/test_base.py +++ b/libbeat/tests/system/test_base.py @@ -4,6 +4,8 @@ import os import shutil import subprocess +import sys +import unittest class Test(BaseTest): @@ -177,6 +179,7 @@ def test_logging_metrics(self): lambda: self.log_contains("Total non-zero metrics"), max_timeout=2) + @unittest.skipIf(sys.platform == 'darwin', 'flaky test https://github.com/elastic/beats/issues/9216') def test_persistent_uuid(self): self.render_config_template() diff --git a/libbeat/tests/system/test_dashboard.py b/libbeat/tests/system/test_dashboard.py index 7386334bf82..ad445d1779f 100644 --- a/libbeat/tests/system/test_dashboard.py +++ b/libbeat/tests/system/test_dashboard.py @@ -125,9 +125,130 @@ def test_load_only_index_patterns(self): @unittest.skipUnless(INTEGRATION_TESTS, "integration test") @attr('integration') - def test_export_dashboard(self): + def test_export_dashboard_cmd_export_dashboard_by_id_and_decoding(self): """ - Test export dashboards and remove unsupported characters + Test testbeat export dashboard can export dashboards + and removes unsupported characters + """ + self.render_config_template() + self.test_load_dashboard() + beat = self.start_beat( + logging_args=["-e", "-d", "*"], + extra_args=["export", + "dashboard", + "-E", "setup.kibana.protocol=http", + "-E", "setup.kibana.host=" + self.get_kibana_host(), + "-E", "setup.kibana.port=" + self.get_kibana_port(), + "-decode", + "-id", "Metricbeat-system-overview"] + ) + + beat.check_wait(exit_code=0) + + assert self.log_contains("\"id\": \"Metricbeat-system-overview\",") is True + + @unittest.skipUnless(INTEGRATION_TESTS, "integration test") + @attr('integration') + def test_export_dashboard_cmd_export_dashboard_by_id(self): + """ + Test testbeat export dashboard can export dashboards + """ + self.render_config_template() + self.test_load_dashboard() + beat = self.start_beat( + logging_args=["-e", "-d", "*"], + extra_args=["export", + "dashboard", + "-E", "setup.kibana.protocol=http", + "-E", "setup.kibana.host=" + self.get_kibana_host(), + "-E", "setup.kibana.port=" + self.get_kibana_port(), + "-id", "Metricbeat-system-overview"] + ) + + beat.check_wait(exit_code=0) + + assert self.log_contains("\"id\": \"Metricbeat-system-overview\",") is True + + @unittest.skipUnless(INTEGRATION_TESTS, "integration test") + @attr('integration') + def test_export_dashboard_cmd_export_dashboard_by_id_unknown_id(self): + """ + Test testbeat export dashboard fails gracefully when dashboard with unknown ID is requested + """ + self.render_config_template() + beat = self.start_beat( + logging_args=["-e", "-d", "*"], + extra_args=["export", + "dashboard", + "-E", "setup.kibana.protocol=http", + "-E", "setup.kibana.host=" + self.get_kibana_host(), + "-E", "setup.kibana.port=" + self.get_kibana_port(), + "-id", "No-such-dashboard"] + ) + + beat.check_wait(exit_code=1) + + assert self.log_contains("Error getting dashboard: error exporting dashboard: Not found") is True + + @unittest.skipUnless(INTEGRATION_TESTS, "integration test") + @attr('integration') + def test_export_dashboard_cmd_export_dashboard_from_yml(self): + """ + Test testbeat export dashboard can export dashboards from dashboards YAML file + and removes unsupported characters + """ + + self.render_config_template() + self.test_load_dashboard() + beat = self.start_beat( + logging_args=["-e", "-d", "*"], + extra_args=["export", + "dashboard", + "-E", "setup.kibana.protocol=http", + "-E", "setup.kibana.host=" + self.get_kibana_host(), + "-E", "setup.kibana.port=" + self.get_kibana_port(), + "-yml", os.path.join(self.beat_path, "tests", "files", "dashboards.yml")] + ) + + beat.check_wait(exit_code=0) + + version = self.get_version() + kibana_semver = semver.VersionInfo.parse(version) + exported_dashboard_path = os.path.join(self.beat_path, "tests", "files", "_meta", + "kibana", str(kibana_semver.major), "dashboard", "Metricbeat-system-test-overview.json") + + with open(exported_dashboard_path) as f: + content = f.read() + assert "Metricbeat-system-overview" in content + + os.remove(exported_dashboard_path) + + @unittest.skipUnless(INTEGRATION_TESTS, "integration test") + @attr('integration') + def test_export_dashboard_cmd_export_dashboard_from_not_existent_yml(self): + """ + Test testbeat export dashboard fails gracefully when cannot find YAML file + """ + + self.render_config_template() + beat = self.start_beat( + logging_args=["-e", "-d", "*"], + extra_args=["export", + "dashboard", + "-E", "setup.kibana.protocol=http", + "-E", "setup.kibana.host=" + self.get_kibana_host(), + "-E", "setup.kibana.port=" + self.get_kibana_port(), + "-yml", os.path.join(self.beat_path, "tests", "files", "no-such-file.yml")] + ) + + beat.check_wait(exit_code=1) + assert self.log_contains("Error getting dashboards from yml: error opening the list of dashboards:") is True + + @unittest.skipUnless(INTEGRATION_TESTS, "integration test") + @attr('integration') + def test_dev_tool_export_dashboard_by_id(self): + """ + Test dev-tools/cmd/dashboards exports dashboard and removes unsupported characters """ self.test_load_dashboard() @@ -151,9 +272,26 @@ def test_export_dashboard(self): @unittest.skipUnless(INTEGRATION_TESTS, "integration test") @attr('integration') - def test_export_dashboard_from_space(self): + def test_dev_tool_export_dashboard_by_id_unknown_id(self): """ - Test export dashboards from Kibana space and remove unsupported characters + Test dev-tools/cmd/dashboards fails gracefully when dashboard with unknown ID is requested + """ + + path = os.path.normpath(self.beat_path + "/../dev-tools/cmd/dashboards/export_dashboards.go") + command = path + " -kibana http://" + self.get_kibana_host() + ":" + self.get_kibana_port() + command = "go run " + command + " -dashboard No-such-dashboard" + + p = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + content, err = p.communicate() + + assert p.returncode != 0 + + @unittest.skipUnless(INTEGRATION_TESTS, "integration test") + @attr('integration') + def test_dev_tool_export_dashboard_by_id_from_space(self): + """ + Test dev-tools/cmd/dashboards exports dashboard from Kibana space + and removes unsupported characters """ version = self.get_version() if semver.compare(version, "6.5.0") == -1: @@ -179,6 +317,36 @@ def test_export_dashboard_from_space(self): os.remove("output.json") + @unittest.skipUnless(INTEGRATION_TESTS, "integration test") + @attr('integration') + def test_dev_tool_export_dashboard_from_yml(self): + """ + Test dev-tools/cmd/dashboards exports dashboard from dashboards YAML file + and removes unsupported characters + """ + + self.test_load_dashboard() + + path = os.path.normpath(self.beat_path + "/../dev-tools/cmd/dashboards/export_dashboards.go") + command = path + " -kibana http://" + self.get_kibana_host() + ":" + self.get_kibana_port() + command = "go run " + command + " -yml " + os.path.join(self.beat_path, "tests", "files", "dashboards.yml") + + p = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + content, err = p.communicate() + + assert p.returncode == 0 + + version = self.get_version() + kibana_semver = semver.VersionInfo.parse(version) + exported_dashboard_path = os.path.join(self.beat_path, "tests", "files", "_meta", + "kibana", str(kibana_semver.major), "dashboard", "Metricbeat-system-test-overview.json") + + with open(exported_dashboard_path) as f: + content = f.read() + assert "Metricbeat-system-overview" in content + + os.remove(exported_dashboard_path) + def get_host(self): return os.getenv('ES_HOST', 'localhost') + ':' + os.getenv('ES_PORT', '9200') diff --git a/libbeat/version/helper.go b/libbeat/version/helper.go index b301d0b1639..5ed206d8a6c 100644 --- a/libbeat/version/helper.go +++ b/libbeat/version/helper.go @@ -31,7 +31,7 @@ func GetDefaultVersion() string { var ( buildTime = "unknown" commit = "unknown" - qualifier = "alpha1" + qualifier = "" ) // BuildTime exposes the compile-time build time information. diff --git a/metricbeat/Dockerfile b/metricbeat/Dockerfile index 8ab5e06e414..e40961df042 100644 --- a/metricbeat/Dockerfile +++ b/metricbeat/Dockerfile @@ -1,21 +1,16 @@ FROM golang:1.10.3 -MAINTAINER Nicolas Ruflin -RUN set -x && \ - apt-get update && \ - apt-get install -y --no-install-recommends \ - netcat python-pip virtualenv && \ - apt-get clean +RUN \ + apt-get update \ + && apt-get install -y --no-install-recommends \ + netcat \ + python-pip \ + virtualenv \ + && rm -rf /var/lib/apt/lists/* RUN pip install --upgrade pip RUN pip install --upgrade setuptools RUN pip install --upgrade docker-compose==1.21.0 -# Setup work environment -ENV METRICBEAT_PATH /go/src/github.com/elastic/beats/metricbeat - -RUN mkdir -p $METRICBEAT_PATH/build/coverage -WORKDIR $METRICBEAT_PATH - -# Add healthcheck for docker/healthcheck metricset to check during testing +# Add healthcheck for the docker/healthcheck metricset to check during testing. HEALTHCHECK CMD exit 0 diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index b3364abc322..1faa2447212 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -5389,6 +5389,26 @@ type: keyword Elasticsearch state id. +-- + +*`elasticsearch.node.id`*:: ++ +-- +type: keyword + +Node ID + + +-- + +*`elasticsearch.node.name`*:: ++ +-- +type: keyword + +Node name. + + -- [float] @@ -5440,7 +5460,7 @@ Number of the shard within the index -- -*`elasticsearch.ccr.follower.operations_indexed`*:: +*`elasticsearch.ccr.follower.operations_written`*:: + -- type: long @@ -5450,7 +5470,7 @@ Number of operations indexed (replicated) into the follower shard from the leade -- -*`elasticsearch.ccr.follower.time_since_last_fetch.ms`*:: +*`elasticsearch.ccr.follower.time_since_last_read.ms`*:: + -- type: long @@ -5928,16 +5948,6 @@ node -*`elasticsearch.node.name`*:: -+ --- -type: keyword - -Node name. - - --- - *`elasticsearch.node.version`*:: + -- diff --git a/metricbeat/docs/gettingstarted.asciidoc b/metricbeat/docs/gettingstarted.asciidoc index d4fef139c0b..75065798e8e 100644 --- a/metricbeat/docs/gettingstarted.asciidoc +++ b/metricbeat/docs/gettingstarted.asciidoc @@ -84,6 +84,25 @@ tar xzvf {beatname_lc}-{version}-darwin-x86_64.tar.gz endif::[] +[[linux]] +*linux:* + +ifeval::["{release-state}"=="unreleased"] + +Version {stack-version} of {beatname_uc} has not yet been released. + +endif::[] + +ifeval::["{release-state}"!="unreleased"] + +["source","sh",subs="attributes"] +------------------------------------------------ +curl -L -O https://artifacts.elastic.co/downloads/beats/{beatname_lc}/{beatname_lc}-{version}-linux-x86_64.tar.gz +tar xzvf {beatname_lc}-{version}-linux-x86_64.tar.gz +------------------------------------------------ + +endif::[] + [[docker]] *docker:* @@ -178,7 +197,7 @@ The following examples enable the `apache` and `mysql` configs in the {beatname_lc} modules enable apache mysql ---- + -*mac:* +*mac and linux:* + ["source","sh",subs="attributes"] ---- @@ -243,13 +262,13 @@ sudo service {beatname_lc} start See <>. -*mac:* +*mac and linux:* ["source","sh",subs="attributes,callouts"] ---------------------------------------------------------------------- sudo chown root {beatname_lc}.yml <1> sudo chown root modules.d/system.yml <1> -sudo ./{beatname_lc} -e -c {beatname_lc}.yml +sudo ./{beatname_lc} -e ---------------------------------------------------------------------- <1> You'll be running {beatname_uc} as root, so you need to change ownership of the configuration file and any configurations enabled in the `modules.d` directory, diff --git a/metricbeat/docs/index.asciidoc b/metricbeat/docs/index.asciidoc index 7075cb96d88..bcf37922597 100644 --- a/metricbeat/docs/index.asciidoc +++ b/metricbeat/docs/index.asciidoc @@ -15,6 +15,7 @@ include::{asciidoc-dir}/../../shared/attributes.asciidoc[] :deb_os: :rpm_os: :mac_os: +:linux_os: :docker_platform: :win_os: diff --git a/metricbeat/magefile.go b/metricbeat/magefile.go index b9a8f79644b..8e47869d968 100644 --- a/metricbeat/magefile.go +++ b/metricbeat/magefile.go @@ -75,7 +75,7 @@ func Clean() error { // Package packages the Beat for distribution. // Use SNAPSHOT=true to build snapshots. // Use PLATFORMS to control the target platforms. -// Use BEAT_VERSION_QUALIFIER to control the version qualifier. +// Use VERSION_QUALIFIER to control the version qualifier. func Package() { start := time.Now() defer func() { fmt.Println("package ran for", time.Since(start)) }() @@ -117,6 +117,15 @@ func GoTestIntegration(ctx context.Context) error { return mage.GoTest(ctx, mage.DefaultGoTestIntegrationArgs()) } +// ExportDashboard exports a dashboard and writes it into the correct directory +// +// Required ENV variables: +// * MODULE: Name of the module +// * ID: Dashboard id +func ExportDashboard() error { + return mage.ExportDashboard() +} + // ----------------------------------------------------------------------------- // Customizations specific to Metricbeat. // - Include modules.d directory in packages. diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 4db8f49b71f..24449a776bf 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -178,6 +178,9 @@ metricbeat.modules: # If set to true, replace dots in labels with `_`. #labels.dedot: false + # If set to true, collects metrics per core. + #cpu.cores: true + # To connect to Docker over TLS you must specify a client and CA certificate. #ssl: #certificate_authority: "/etc/pki/root/ca.pem" diff --git a/metricbeat/module/ceph/cluster_disk/data.go b/metricbeat/module/ceph/cluster_disk/data.go index d4245aa89af..7a2c600cc53 100644 --- a/metricbeat/module/ceph/cluster_disk/data.go +++ b/metricbeat/module/ceph/cluster_disk/data.go @@ -43,7 +43,7 @@ func eventMapping(content []byte) common.MapStr { var d DfRequest err := json.Unmarshal(content, &d) if err != nil { - logp.Err("Error: ", err) + logp.Err("Error: %+v", err) } return common.MapStr{ diff --git a/metricbeat/module/ceph/cluster_health/data.go b/metricbeat/module/ceph/cluster_health/data.go index e2dd642ebb2..10ecfbcd389 100644 --- a/metricbeat/module/ceph/cluster_health/data.go +++ b/metricbeat/module/ceph/cluster_health/data.go @@ -44,7 +44,7 @@ func eventMapping(content []byte) common.MapStr { var d HealthRequest err := json.Unmarshal(content, &d) if err != nil { - logp.Err("Error: ", err) + logp.Err("Error: %+v", err) } return common.MapStr{ diff --git a/metricbeat/module/ceph/cluster_status/data.go b/metricbeat/module/ceph/cluster_status/data.go index ad49607a9cf..55a860281b0 100644 --- a/metricbeat/module/ceph/cluster_status/data.go +++ b/metricbeat/module/ceph/cluster_status/data.go @@ -81,7 +81,7 @@ func eventsMapping(content []byte) ([]common.MapStr, error) { var d HealthRequest err := json.Unmarshal(content, &d) if err != nil { - logp.Err("Error: ", err) + logp.Err("Error: %+v", err) return nil, err } diff --git a/metricbeat/module/ceph/monitor_health/data.go b/metricbeat/module/ceph/monitor_health/data.go index aa461e74ac7..c4ed630c385 100644 --- a/metricbeat/module/ceph/monitor_health/data.go +++ b/metricbeat/module/ceph/monitor_health/data.go @@ -89,7 +89,7 @@ func eventsMapping(content []byte) []common.MapStr { var d HealthRequest err := json.Unmarshal(content, &d) if err != nil { - logp.Err("Error: ", err) + logp.Err("Error: %+v", err) } events := []common.MapStr{} diff --git a/metricbeat/module/ceph/osd_df/data.go b/metricbeat/module/ceph/osd_df/data.go index 2e341fb99da..b22d1dd593a 100644 --- a/metricbeat/module/ceph/osd_df/data.go +++ b/metricbeat/module/ceph/osd_df/data.go @@ -47,7 +47,7 @@ func eventsMapping(content []byte) ([]common.MapStr, error) { var d OsdDfRequest err := json.Unmarshal(content, &d) if err != nil { - logp.Err("Error: ", err) + logp.Err("Error: %+v", err) return nil, err } diff --git a/metricbeat/module/ceph/osd_tree/data.go b/metricbeat/module/ceph/osd_tree/data.go index 25dee1840be..27a3ea14278 100644 --- a/metricbeat/module/ceph/osd_tree/data.go +++ b/metricbeat/module/ceph/osd_tree/data.go @@ -55,7 +55,7 @@ func eventsMapping(content []byte) ([]common.MapStr, error) { var d OsdTreeRequest err := json.Unmarshal(content, &d) if err != nil { - logp.Err("Error: ", err) + logp.Err("Error: %+v", err) return nil, err } diff --git a/metricbeat/module/ceph/pool_disk/data.go b/metricbeat/module/ceph/pool_disk/data.go index 6a89f0d4210..7dba27044c6 100644 --- a/metricbeat/module/ceph/pool_disk/data.go +++ b/metricbeat/module/ceph/pool_disk/data.go @@ -50,7 +50,7 @@ func eventsMapping(content []byte) []common.MapStr { var d DfRequest err := json.Unmarshal(content, &d) if err != nil { - logp.Err("Error: ", err) + logp.Err("Error: %+v", err) } events := []common.MapStr{} diff --git a/metricbeat/module/couchbase/bucket/data.go b/metricbeat/module/couchbase/bucket/data.go index 95fd950eead..432bcf38cd8 100644 --- a/metricbeat/module/couchbase/bucket/data.go +++ b/metricbeat/module/couchbase/bucket/data.go @@ -50,7 +50,7 @@ func eventsMapping(content []byte) []common.MapStr { var d Buckets err := json.Unmarshal(content, &d) if err != nil { - logp.Err("Error: ", err) + logp.Err("Error: %+v", err) } events := []common.MapStr{} diff --git a/metricbeat/module/couchbase/cluster/data.go b/metricbeat/module/couchbase/cluster/data.go index 064160f2672..e099c3e5a9f 100644 --- a/metricbeat/module/couchbase/cluster/data.go +++ b/metricbeat/module/couchbase/cluster/data.go @@ -61,7 +61,7 @@ func eventMapping(content []byte) common.MapStr { var d Data err := json.Unmarshal(content, &d) if err != nil { - logp.Err("Error: ", err) + logp.Err("Error: %+v", err) } logp.Info("Printing Data:") diff --git a/metricbeat/module/couchbase/node/data.go b/metricbeat/module/couchbase/node/data.go index 990887a2a87..046226bce5f 100644 --- a/metricbeat/module/couchbase/node/data.go +++ b/metricbeat/module/couchbase/node/data.go @@ -77,7 +77,7 @@ func eventsMapping(content []byte) []common.MapStr { var d Data err := json.Unmarshal(content, &d) if err != nil { - logp.Err("Error: ", err) + logp.Err("Error: %+v", err) } events := []common.MapStr{} diff --git a/metricbeat/module/docker/_meta/config.reference.yml b/metricbeat/module/docker/_meta/config.reference.yml index b06bafddfa1..2ac7c913ab3 100644 --- a/metricbeat/module/docker/_meta/config.reference.yml +++ b/metricbeat/module/docker/_meta/config.reference.yml @@ -15,6 +15,9 @@ # If set to true, replace dots in labels with `_`. #labels.dedot: false + # If set to true, collects metrics per core. + #cpu.cores: true + # To connect to Docker over TLS you must specify a client and CA certificate. #ssl: #certificate_authority: "/etc/pki/root/ca.pem" diff --git a/metricbeat/module/docker/cpu/cpu.go b/metricbeat/module/docker/cpu/cpu.go index 4e78f031ac1..954eb37d6d2 100644 --- a/metricbeat/module/docker/cpu/cpu.go +++ b/metricbeat/module/docker/cpu/cpu.go @@ -51,10 +51,19 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { return nil, err } + cpuConfig := struct { + Cores bool `config:"cpu.cores"` + }{ + Cores: true, + } + if err := base.Module().UnpackConfig(&cpuConfig); err != nil { + return nil, err + } + return &MetricSet{ BaseMetricSet: base, dockerClient: client, - cpuService: &CPUService{}, + cpuService: &CPUService{Cores: cpuConfig.Cores}, dedot: config.DeDot, }, nil } diff --git a/metricbeat/module/docker/cpu/helper.go b/metricbeat/module/docker/cpu/helper.go index 03accc2245a..cfee7ac0d08 100644 --- a/metricbeat/module/docker/cpu/helper.go +++ b/metricbeat/module/docker/cpu/helper.go @@ -38,7 +38,10 @@ type CPUStats struct { SystemUsagePercentage float64 } -type CPUService struct{} +// CPUService is a helper to collect docker CPU metrics +type CPUService struct { + Cores bool +} func NewCpuService() *CPUService { return &CPUService{} @@ -57,10 +60,9 @@ func (c *CPUService) getCPUStatsList(rawStats []docker.Stat, dedot bool) []CPUSt func (c *CPUService) getCPUStats(myRawStat *docker.Stat, dedot bool) CPUStats { usage := cpuUsage{Stat: myRawStat} - return CPUStats{ + stats := CPUStats{ Time: common.Time(myRawStat.Stats.Read), Container: docker.NewContainer(myRawStat.Container, dedot), - PerCpuUsage: usage.PerCPU(), TotalUsage: usage.Total(), UsageInKernelmode: myRawStat.Stats.CPUStats.CPUUsage.UsageInKernelmode, UsageInKernelmodePercentage: usage.InKernelMode(), @@ -69,6 +71,12 @@ func (c *CPUService) getCPUStats(myRawStat *docker.Stat, dedot bool) CPUStats { SystemUsage: myRawStat.Stats.CPUStats.SystemUsage, SystemUsagePercentage: usage.System(), } + + if c.Cores { + stats.PerCpuUsage = usage.PerCPU() + } + + return stats } // TODO: These helper should be merged with the cpu helper in system/cpu diff --git a/metricbeat/module/docker/healthcheck/data.go b/metricbeat/module/docker/healthcheck/data.go index 0b745827600..dc91093c918 100644 --- a/metricbeat/module/docker/healthcheck/data.go +++ b/metricbeat/module/docker/healthcheck/data.go @@ -50,6 +50,12 @@ func eventMapping(cont *types.Container, m *MetricSet) common.MapStr { logp.Err("Error inspecting container %v: %v", cont.ID, err) return nil } + + // Check if the container has any health check + if container.State.Health == nil { + return nil + } + lastEvent := len(container.State.Health.Log) - 1 // Checks if a healthcheck already happened diff --git a/metricbeat/module/elasticsearch/_meta/fields.yml b/metricbeat/module/elasticsearch/_meta/fields.yml index 6824317ff74..d457c6f0d43 100644 --- a/metricbeat/module/elasticsearch/_meta/fields.yml +++ b/metricbeat/module/elasticsearch/_meta/fields.yml @@ -24,3 +24,13 @@ type: keyword description: > Elasticsearch state id. + + - name: node.id + type: keyword + description: > + Node ID + + - name: node.name + type: keyword + description: > + Node name. diff --git a/metricbeat/module/elasticsearch/ccr/_meta/data.json b/metricbeat/module/elasticsearch/ccr/_meta/data.json index 993aa63a607..a33929b9195 100644 --- a/metricbeat/module/elasticsearch/ccr/_meta/data.json +++ b/metricbeat/module/elasticsearch/ccr/_meta/data.json @@ -1,30 +1,30 @@ { "@timestamp": "2017-10-12T08:05:34.853Z", - "beat": { + "agent": { "hostname": "host.example.com", "name": "host.example.com" }, "elasticsearch": { "ccr": { "follower": { - "global_checkpoint": 1, - "index": "my_follower", - "operations_indexed": 2, + "global_checkpoint": -1, + "index": "my_index_f", + "operations_written": 0, "shard": { "number": 0 }, - "time_since_last_fetch": { - "ms": 4926 + "time_since_last_read": { + "ms": 42294 } }, "leader": { - "index": "my_leader", - "max_seq_no": 1 + "index": "my_index", + "max_seq_no": -1 } }, "cluster": { - "id": "KSGkOjOuSg6whAgtpPyhQw", - "name": "elasticsearch" + "id": "3LbUkLkURz--FR-YO0wLNA", + "name": "es1" } }, "metricset": { diff --git a/metricbeat/module/elasticsearch/ccr/_meta/fields.yml b/metricbeat/module/elasticsearch/ccr/_meta/fields.yml index a4119ec30b1..15b9db640b7 100644 --- a/metricbeat/module/elasticsearch/ccr/_meta/fields.yml +++ b/metricbeat/module/elasticsearch/ccr/_meta/fields.yml @@ -26,11 +26,11 @@ type: long description: > Number of the shard within the index - - name: operations_indexed + - name: operations_written type: long description: > Number of operations indexed (replicated) into the follower shard from the leader shard - - name: time_since_last_fetch.ms + - name: time_since_last_read.ms type: long description: > Time, in ms, since the follower last fetched from the leader diff --git a/metricbeat/module/elasticsearch/ccr/_meta/test/ccr_stats.700.json b/metricbeat/module/elasticsearch/ccr/_meta/test/ccr_stats.700.json index 1905f466d8f..1971e4df9ae 100644 --- a/metricbeat/module/elasticsearch/ccr/_meta/test/ccr_stats.700.json +++ b/metricbeat/module/elasticsearch/ccr/_meta/test/ccr_stats.700.json @@ -1,98 +1,46 @@ { - "my_follower": [ - { - "leader_index": "my_leader", - "follower_index": "my_follower", - "shard_id": 0, - "leader_global_checkpoint": 1, - "leader_max_seq_no": 1, - "follower_global_checkpoint": 1, - "follower_max_seq_no": 1, - "last_requested_seq_no": 1, - "number_of_concurrent_reads": 1, - "number_of_concurrent_writes": 0, - "number_of_queued_writes": 0, - "mapping_version": 2, - "total_fetch_time_millis": 21, - "number_of_successful_fetches": 142, - "number_of_failed_fetches": 6743, - "operations_received": 2, - "total_transferred_bytes": 166, - "total_index_time_millis": 48, - "number_of_successful_bulk_operations": 2, - "number_of_failed_bulk_operations": 0, - "number_of_operations_indexed": 2, - "fetch_exceptions": [ - { - "from_seq_no": 2, - "retries": 6743, - "exception": { - "type": "exception", - "reason": "NoShardAvailableActionException[No shard available for [Request{fromSeqNo=2, maxOperationCount=1024, shardId=[leader][0], expectedHistoryUUID=ki9Do9c-QQKVydB-6Txkdg, maxOperationSizeInBytes=9223372036854775807}]]; nested: RemoteTransportException[[d0XH9XU][127.0.0.1:9300][indices:data/read/xpack/ccr/shard_changes[s]]]; nested: IndexNotFoundException[no such index];", - "caused_by": { - "type": "no_shard_available_action_exception", - "reason": "No shard available for [Request{fromSeqNo=2, maxOperationCount=1024, shardId=[leader][0], expectedHistoryUUID=ki9Do9c-QQKVydB-6Txkdg, maxOperationSizeInBytes=9223372036854775807}]", - "caused_by": { - "type": "index_not_found_exception", - "reason": "no such index", - "index_uuid": "RH8_j_w0Q0GGmpY0HMVQ_A", - "index": "my_leader" - } - } + "auto_follow_stats": { + "number_of_failed_follow_indices": 0, + "number_of_failed_remote_cluster_state_requests": 0, + "number_of_successful_follow_indices": 1, + "recent_auto_follow_errors": [] + }, + "follow_stats": { + "indices": [ + { + "index": "follower_index", + "shards": [ + { + "remote_cluster": "remote_cluster", + "leader_index": "leader_index", + "follower_index": "follower_index", + "shard_id": 0, + "leader_global_checkpoint": 1024, + "leader_max_seq_no": 1536, + "follower_global_checkpoint": 768, + "follower_max_seq_no": 896, + "last_requested_seq_no": 897, + "outstanding_read_requests": 8, + "outstanding_write_requests": 2, + "write_buffer_operation_count": 64, + "follower_mapping_version": 4, + "follower_settings_version": 2, + "total_read_time_millis": 32768, + "total_read_remote_exec_time_millis": 16384, + "successful_read_requests": 32, + "failed_read_requests": 0, + "operations_read": 896, + "bytes_read": 32768, + "total_write_time_millis": 16384, + "write_buffer_size_in_bytes": 1536, + "successful_write_requests": 16, + "failed_write_requests": 0, + "operations_written": 832, + "read_exceptions": [], + "time_since_last_read_millis": 8 } - } - ], - "time_since_last_fetch_millis": 470 - }, - { - "leader_index": "my_leader", - "follower_index": "my_follower", - "shard_id": 1, - "leader_global_checkpoint": -1, - "leader_max_seq_no": -1, - "follower_global_checkpoint": -1, - "follower_max_seq_no": -1, - "last_requested_seq_no": -1, - "number_of_concurrent_reads": 0, - "number_of_concurrent_writes": 0, - "number_of_queued_writes": 0, - "mapping_version": 2, - "total_fetch_time_millis": 0, - "number_of_successful_fetches": 336, - "number_of_failed_fetches": 0, - "operations_received": 0, - "total_transferred_bytes": 0, - "total_index_time_millis": 0, - "number_of_successful_bulk_operations": 0, - "number_of_failed_bulk_operations": 0, - "number_of_operations_indexed": 0, - "fetch_exceptions": [], - "time_since_last_fetch_millis": 4323 - }, - { - "leader_index": "my_leader", - "follower_index": "my_follower", - "shard_id": 2, - "leader_global_checkpoint": 1, - "leader_max_seq_no": 1, - "follower_global_checkpoint": 1, - "follower_max_seq_no": 1, - "last_requested_seq_no": 1, - "number_of_concurrent_reads": 0, - "number_of_concurrent_writes": 0, - "number_of_queued_writes": 0, - "mapping_version": 2, - "total_fetch_time_millis": 0, - "number_of_successful_fetches": 372, - "number_of_failed_fetches": 0, - "operations_received": 2, - "total_transferred_bytes": 166, - "total_index_time_millis": 32, - "number_of_successful_bulk_operations": 2, - "number_of_failed_bulk_operations": 0, - "number_of_operations_indexed": 2, - "fetch_exceptions": [], - "time_since_last_fetch_millis": 4323 - } - ] + ] + } + ] + } } diff --git a/metricbeat/module/elasticsearch/ccr/data.go b/metricbeat/module/elasticsearch/ccr/data.go index 0a5548547b3..78872e1cf61 100644 --- a/metricbeat/module/elasticsearch/ccr/data.go +++ b/metricbeat/module/elasticsearch/ccr/data.go @@ -19,7 +19,6 @@ package ccr import ( "encoding/json" - "fmt" "github.com/joeshaw/multierror" "github.com/pkg/errors" @@ -42,17 +41,25 @@ var ( "shard": s.Object{ "number": c.Int("shard_id"), }, - "operations_indexed": c.Int("number_of_operations_indexed"), - "time_since_last_fetch": s.Object{ - "ms": c.Int("time_since_last_fetch_millis"), + "operations_written": c.Int("operations_written"), + "time_since_last_read": s.Object{ + "ms": c.Int("time_since_last_read_millis"), }, "global_checkpoint": c.Int("follower_global_checkpoint"), }, } ) +type response struct { + FollowStats struct { + Indices []struct { + Shards []map[string]interface{} `json:"shards"` + } `json:"indices"` + } `json:"follow_stats"` +} + func eventsMapping(r mb.ReporterV2, info elasticsearch.Info, content []byte) error { - var data map[string]interface{} + var data response err := json.Unmarshal(content, &data) if err != nil { err = errors.Wrap(err, "failure parsing Elasticsearch CCR Stats API response") @@ -61,17 +68,8 @@ func eventsMapping(r mb.ReporterV2, info elasticsearch.Info, content []byte) err } var errs multierror.Errors - for _, followerShards := range data { - - shards, ok := followerShards.([]interface{}) - if !ok { - err := fmt.Errorf("shards is not an array") - errs = append(errs, err) - r.Error(err) - continue - } - - for _, s := range shards { + for _, followerIndex := range data.FollowStats.Indices { + for _, followerShard := range followerIndex.Shards { event := mb.Event{} event.RootFields = common.MapStr{} event.RootFields.Put("service.name", elasticsearch.ModuleName) @@ -80,15 +78,7 @@ func eventsMapping(r mb.ReporterV2, info elasticsearch.Info, content []byte) err event.ModuleFields.Put("cluster.name", info.ClusterName) event.ModuleFields.Put("cluster.id", info.ClusterID) - shard, ok := s.(map[string]interface{}) - if !ok { - event.Error = fmt.Errorf("shard is not an object") - r.Event(event) - errs = append(errs, event.Error) - continue - } - - event.MetricSetFields, err = schema.Apply(shard) + event.MetricSetFields, err = schema.Apply(followerShard) if err != nil { event.Error = errors.Wrap(err, "failure applying shard schema") r.Event(event) diff --git a/metricbeat/module/elasticsearch/ccr/data_xpack.go b/metricbeat/module/elasticsearch/ccr/data_xpack.go index db5688caa9e..eb97de72cd4 100644 --- a/metricbeat/module/elasticsearch/ccr/data_xpack.go +++ b/metricbeat/module/elasticsearch/ccr/data_xpack.go @@ -19,7 +19,6 @@ package ccr import ( "encoding/json" - "fmt" "time" "github.com/joeshaw/multierror" @@ -32,36 +31,24 @@ import ( ) func eventsMappingXPack(r mb.ReporterV2, m *MetricSet, info elasticsearch.Info, content []byte) error { - var data map[string]interface{} + var data response err := json.Unmarshal(content, &data) if err != nil { - return errors.Wrap(err, "failure parsing Elasticsearch CCR Stats API response") + err = errors.Wrap(err, "failure parsing Elasticsearch CCR Stats API response") + r.Error(err) + return err } var errors multierror.Errors - for _, followerShards := range data { - - shards, ok := followerShards.([]interface{}) - if !ok { - err := fmt.Errorf("shards is not an array") - errors = append(errors, err) - continue - } - - for _, s := range shards { - shard, ok := s.(map[string]interface{}) - if !ok { - err := fmt.Errorf("shard is not an object") - errors = append(errors, err) - continue - } + for _, followerIndex := range data.FollowStats.Indices { + for _, followerShard := range followerIndex.Shards { event := mb.Event{} event.RootFields = common.MapStr{ "cluster_uuid": info.ClusterID, "timestamp": common.Time(time.Now()), "interval_ms": m.Module().Config().Period / time.Millisecond, "type": "ccr_stats", - "ccr_stats": shard, + "ccr_stats": followerShard, } event.Index = elastic.MakeXPackMonitoringIndexName(elastic.Elasticsearch) diff --git a/metricbeat/module/elasticsearch/elasticsearch_integration_test.go b/metricbeat/module/elasticsearch/elasticsearch_integration_test.go index ccd7084a313..ee99b790620 100644 --- a/metricbeat/module/elasticsearch/elasticsearch_integration_test.go +++ b/metricbeat/module/elasticsearch/elasticsearch_integration_test.go @@ -127,9 +127,9 @@ func getEnvPort() string { // GetConfig returns config for elasticsearch module func getConfig(metricset string) map[string]interface{} { return map[string]interface{}{ - "module": elasticsearch.ModuleName, - "metricsets": []string{metricset}, - "hosts": []string{getEnvHost() + ":" + getEnvPort()}, + "module": elasticsearch.ModuleName, + "metricsets": []string{metricset}, + "hosts": []string{getEnvHost() + ":" + getEnvPort()}, "index_recovery.active_only": false, } } diff --git a/metricbeat/module/elasticsearch/fields.go b/metricbeat/module/elasticsearch/fields.go index 3a71d23866a..8ea1f9417a6 100644 --- a/metricbeat/module/elasticsearch/fields.go +++ b/metricbeat/module/elasticsearch/fields.go @@ -31,5 +31,5 @@ func init() { // Asset returns asset data func Asset() string { - return "eJzsW91u2zoSvvdTDHLVAokewBd7U2y7WaDZYpsusFgsVFoaW0z4o5CUG5+nPyAlOZREWbItxy6OfRfLme+b4fyS1B0842YOyIg2NNFIVJLNAAw1DOdw83f/+5sZQIo6UTQ3VIo5/G0GAND4DXCZFgxnAAoZEo1zWKAhMwCNxlCx0nP4343W7OYWbjJj8pv/22eZVCZOpFjS1RyWhGkrYUmRpXruQO5AEI5dovZjNjnOYaVkkVffBFg2xfkiE1Zogyqyf20f1lKfcfNLqtT7Pii7/DQtUcl1KNGsF5ampwCl6Q5IbYjBiYGdTAfbQU1UB8hfrAGYT0pqfVfrpTBnNCH2hw5Se79te1z9aS+8T44hSVE1HvVx7BPli6MixdfO037zjtC//jwQjiCXFeMepJoHJ6+xxpdYyF4yTIrVYUy+klfKCw4aXwoUCYIo+AKVJSdzVOXiSAEmw5qtzkhL75rpUjImf/1eS1BzHlgEp3RUGucEy/Cwtbo1tAODX9RktLT8bm7bhdKx+yGGTDIdwzc4qODgQx3KmH4EKox0rLemLfVZKsl3+5GvlKEcY01FgrHNTfESTZJFXJ9AtUfK8RaoAK5vwUE26Vt8cPjY0aKX/4rJBWFxkmHynEsqzAmIf3EY8IYBa8IKtPHatP3O6qGPyuhVLp8if1sZRXuBd4X6gIl8boWGDyuFKG5hg9Y0t6Aw/RgFiQiZYphHKIkNsHiwwhwH6kpsNCoHbhdKFgHX2eE4g27zKA1hXpp3ylrvr32ihwkn9umkVN5ySin8Dhld0QXD0aRS0vCuKSlZ0QM8vDJFk+k85r4Ud8E+Uyk8YoFc+ukm7X7bjGDz3ZWTfusM9xVhC01QSVp2KrXfYaY3SrminCja8aJT0CqxNsP0to2dtacNiIgjl2oTLTYmwPQYF/vqBEOhbY2VyoPsFK92N7R30WoLOKRYtSZMOK5U3VtG1XQZQjN2BSdrq1OZ6Oi9YiCVScFRGOdn22a2PxAcuRQZmpP0sR16JdLeNLWRCiNN/8CeYBigupSKEzOHvn8erYqlUA8OjrNVwEndQR5XTtd3c4KS2BZ2mNjOPPNepn3jz73stNiUQ1pFNWDucMaKFCZyjWpz7tRF23HVa80BS5WFuFarsWHk41nx06XKFqiVFIatqlwQeSElQyL2Q35UBQJtlc8wtjZkNaHO/661dXIDjuZjG6JWaKKeVT4I/9GJdN1w/yqXsJnU7YRSAtM9u2Af0woFkqYKtd6JPm1V9ik0a3N40WWhEpzU8N+dyN2Gr2AnM7yPOWz4Cn1aw/sUmoZvZlRd8FaInyWh9vXu1w5tEnrXDu3aoQ3y369Dg+tsdY3ca+T+hpG73Y5m0ZNcHFP5OTvNHHVQx/ND0JcCgTN4kov+Vs8QM2Gb9U+5KEWG0VJiSOy8WEe5kglqjWlsJy+VxiHvPnSO/FYLLzfdcd115BAnKtaE0TROicFJ+Txm/jl4qbB257GA1GSogACnWlOxsoSw9BKQ9nv3t8mIgUQWLAUhDSwQcqI0poEBouPYtuk9xq1b/3/+vc2Hbhfvg61RaSrbo/ixeJXUMOTTmo+u90Mh9J+vcC+WctwJzZDWQ5qPIFSTChrAZ1BVgQxJHlFBzdnqwT+Q5GAZNEqA1WG42PpKcPJ6Xh04eT1cBSHF+ZfiQYq7CZaj1uWcK7JVZfyqvI3zrhxFnMnkmbDwbHDQDuL9shYOVjam7tZJabRgITj+goaVEk9xO2PiQ26Xpal30t2WfFGDFr5SbVy5r0eZyx2yLmg0+d2GknreG9g5gAln1YNmUQc6kMWe1twaNcqlZJNFrU2e1bxm5R4UuJL1h0X4RsgoK/2LpV1q3XXr5wfNO7e9ywlDSwojXLKtVM9P3AXdPhf0CedIni+E8Tckz2Mpx5djaEebj7O27ScuhPaPsrXZmaM2sggSOTrm/msFX6PuEhhfo+7Sok4Xak3Xsv91gSMC73sl+xp7l8D4Gnvnjr3eDniVRIlkDBMj1WRd8JdPsBUajroRPXDNa9cu4JHt8BsCrJJDE0PfrAhjFn0E0W0COtizBkAaXdCJrV42RH9tuwdjcakDl3XgmCD8TBmC3miDHMKih4LQHfyfZcdhaxWF5znirgmQNaGMLNj7smi/hZajSKlYxYbo51kbeo+9zp8hgT8hkcIQKjQQqB6AfeBL8oP0sM1RjcrEUvW9gLz/IeS9Ewldkd6tM6moCQfUIYewAXHNW35BpCMu+EXwWSrAV8JzZhUqzB0neU5b1BuvhFIRvxRYYPdV0INPeyl3e2lObMdH22+p7u2U1cuvznmO9LKT3Wg2GdVAtdtbHHG7Ofga9DSH7Y7J7ovVU96AeHSbqcTgGGyFTCbE2MTizmYmvmycVRdevTfAia5BqzePo9mfAQAA//9vz37G" + return "eJzsXN1u2zgWvvdTHOSqBRI9gC/2prvtZoFmi226wGAwUGnx2GbCH4WknHiefkBKcmSJshRJjl2MfVfL+c53fnkOSfUGHnE7B+TEWJYYJDpZzwAssxzncPWv6vdXMwCKJtEstUzJOfxjBgCw9xsQimYcZwAaORKDc1igJTMAg9YyuTJz+P3KGH51DVdra9OrP9yztdI2TpRcstUcloQbh7BkyKmZeyE3IInAJlH3sdsU57DSKkuLbwIs9+GqkAnPjEUduX/tHpaoj7h9VppWvg9i5599SxS4Xko0axXL6DGEMnpApLHE4sSCPWZYrFR0vLQ7RRFu/9mCPt53Hj93VcNsiW5gV6OtA/mTVsbclI7RmHKWEPdDbzNT+W09ZcpPPXKr5DgSinrvURvHNqgqHJMUXxpP2y3aQ//yc0cEgloWjFsklTwEeYkNPsVStZLhSq6GMflKXpjIBBh8ylAmCDITC9SOnEpR585REuwaS7ZmTWp6l0yXinP1/Gu5oOTc4QSvdJQb5whuuNtZ3RnaC4NnZtcst/xhbjtHmfhZM2tRHpXhq7icF1L4UKYy0o/ApFWe9c60uT5LrcThOKoqZZnA2DCZYOyKa6yR0EiYI2h2zwReA5MgzDV4ifvsnXhYok3W2FCilf6KqwXhcbLG5DFVTNojEP/iZcCrDNgQnqFL133TH1z9zKiCXpTyKcq3w8jqDj6U6R0mqnLLDHxYaUR5DVt0prkGjfRjFCTiltEwj1AN62DhFlPjOTDfIkS9SuDOUSoLhM6BwOkMm3tlCa9Uea+si/4yJlqYCOKeTkrltaTk4DfI2YotOPYmRcledE1JyUF38KisUiyZLmJuc7gzjplC4R4O8uWnWbTbbdODzXe/mrRbp7utCFtogpWkZqdc+wNmeqWUaiaIZo0oOgatXNa2m96ur3P2dAkRCRRKb6PF1gaYjgmxrx4YMuPWWKUrIhuLV70ZevOiVQcYsljVpiwYt1TdOka1kasqzToPTtZVU5WY6L1ygKokEyitj7NdL9ueCJ4cRY4WQ5395PRySW+maazSGBn2J7YkQwfVpdKC2Dm0/XFvVRyFcm7wnJ0CHvUAeVx5Xd8tCHJiO7HdxA7Wmfcy7St/UalOi20+oxVUA+YOV6xIY6I2qLenLl2snlet1uywVL4Ql2r5Da+QPAc/XamsCXVIYbHFKheUvFCKI6mPyR2S73WGwGrLZ1i2sWQ1oc7/K7X1uIFAq8q2RK/QRi1eHiT/3kP6brjdy7nYtTL1gpILZm/sgqsyHSgQSjUac1D6tKtylcL+2hx2usp0gpMa/ruHPGz4Quxkhq/K7DZ8IX1aw1cphPeh84pqMlFL8ZMU1Lbe/dKhTULv0qFdOrRO/m/r0OAyW10y95K5v2Dm7rajefSgFmNWfsGPM0cN6nh+SPaUIQgOD2rR3upZYidss/6jFjlkWBollsQ+ik2UapWgMUhjN3lpGoeie+gc+a0EzzfdcdMM5BAnJjeEMxpTYnFSPvfr6jF4rrDxx7GAzK5RAwHBjGFy5QhhHiWg3Pf+33ZNLCQq4xSksrBASIk2SAMDRCOwXdM7Jqxrfz8ksDeoDVP16XhEoPkLJQVq2K8PG9F7Ce6K6v9/hVu5VP0OTbq07tK8B6GSVNAAVQZFYV4jSSMmmT1Zif43khQcg72q7HToXv+qSgjyclodBHkZroJU8vSuuFPyZgJ3lLqc0iM7Vfp75XXC9itEJLhKHgkPt+uDNvVulyU4OGyk/iJIbrRgbR5/Z8KhxFNcmJj43NlXaVY5fK4jn9Xsgy/MWL8Cl9PF+c49ZzQt/GpzQjmCdQzzMOH4OGg89EI7qtjDRjijRqlSfLKsdcWzGKEc7qDEVbw9LcKXNHpZ6b+cNqk1/dbOD/Zvwba6E7pcCj1Csq5Uy0/8ldm2EKwSTpE8ngnjb0ge+1KOz8fQnrboZ23XT5wJ7R95a3OwRm1VFiQyOud+c8CXrDsHxpesO7esM5nesI1qv8A/IvG+F9iX3DsHxpfcO3XutXbAqyRKFOeYWKUn64K/fIIdaDjrevTAJa9Du4Aj2+FXCbBKhhaGtlkR+ji9B9FdARocWR1C9rqgI1s9b4j+3nYP5uLSBO7PwJgk/Mw4gtkaiwLC0F1J6M/iT7LjsLOKxtOcOpcEyIYwThb8fVnUXwxLUVImV7El5nFWF/2Gvc6fIcCfkChpCZMGCBQPwD2oIlWTdNjmqEFtY6XbXgl++7ngrYeEJmTlIpjSzIYTasi5aABu/+JdUNKIO3cRfFYa8IWIlDuFMnsjSJqyGvW9lzSZjJ8yzLD5dubgA1gm/F6ah23EaP290TcHZfE6qg+ekVF2tEvGds0MMOP3FntcOA6+mDzN+bdncviu85SXEu79Ziqx2Ee2Rq4SYl1hCf3fBxNQ8XdQK+9kE1MKLV4GjmZ/BQAA///0i5UN" } diff --git a/metricbeat/module/elasticsearch/index_recovery/_meta/data.json b/metricbeat/module/elasticsearch/index_recovery/_meta/data.json index 2320ddc116a..a22b96172e1 100644 --- a/metricbeat/module/elasticsearch/index_recovery/_meta/data.json +++ b/metricbeat/module/elasticsearch/index_recovery/_meta/data.json @@ -1,12 +1,16 @@ { "@timestamp": "2017-10-12T08:05:34.853Z", - "beat": { + "agent": { "hostname": "host.example.com", "name": "host.example.com" }, "elasticsearch": { + "cluster": { + "id": "3LbUkLkURz--FR-YO0wLNA", + "name": "es1" + }, "index": { - "name": "test-index", + "name": ".monitoring-es-6-2018.11.20", "recovery": { "id": 0, "primary": true, @@ -14,8 +18,8 @@ "stage": "DONE", "target": { "host": "127.0.0.1", - "id": "ODWb5m6zT9q0lf8YTPyWWg", - "name": "ODWb5m6" + "id": "FMRmkE3HTU6xxxoFK-06Ww", + "name": "es1_1" }, "type": "EMPTY_STORE" } @@ -31,4 +35,4 @@ "service": { "name": "elasticsearch" } -} +} \ No newline at end of file diff --git a/metricbeat/module/elasticsearch/index_recovery/data.go b/metricbeat/module/elasticsearch/index_recovery/data.go index 2400fa6a075..ee9152ff8c1 100644 --- a/metricbeat/module/elasticsearch/index_recovery/data.go +++ b/metricbeat/module/elasticsearch/index_recovery/data.go @@ -55,7 +55,7 @@ var ( } ) -func eventsMapping(r mb.ReporterV2, content []byte) error { +func eventsMapping(r mb.ReporterV2, info elasticsearch.Info, content []byte) error { var data map[string]map[string][]map[string]interface{} @@ -82,6 +82,8 @@ func eventsMapping(r mb.ReporterV2, content []byte) error { event.RootFields.Put("service.name", elasticsearch.ModuleName) event.ModuleFields = common.MapStr{} + event.ModuleFields.Put("cluster.name", info.ClusterName) + event.ModuleFields.Put("cluster.id", info.ClusterID) event.ModuleFields.Put("index.name", indexName) event.MetricSetFields, err = schema.Apply(data) diff --git a/metricbeat/module/elasticsearch/index_recovery/data_test.go b/metricbeat/module/elasticsearch/index_recovery/data_test.go index 7133f2f0b4c..c06e01ca23b 100644 --- a/metricbeat/module/elasticsearch/index_recovery/data_test.go +++ b/metricbeat/module/elasticsearch/index_recovery/data_test.go @@ -26,5 +26,5 @@ import ( ) func TestMapper(t *testing.T) { - elasticsearch.TestMapper(t, "./_meta/test/recovery.*.json", eventsMapping) + elasticsearch.TestMapperWithInfo(t, "./_meta/test/recovery.*.json", eventsMapping) } diff --git a/metricbeat/module/elasticsearch/index_recovery/data_xpack.go b/metricbeat/module/elasticsearch/index_recovery/data_xpack.go index 77b504e704d..bbef1654a84 100644 --- a/metricbeat/module/elasticsearch/index_recovery/data_xpack.go +++ b/metricbeat/module/elasticsearch/index_recovery/data_xpack.go @@ -30,7 +30,7 @@ import ( "github.com/elastic/beats/metricbeat/module/elasticsearch" ) -func eventsMappingXPack(r mb.ReporterV2, m *MetricSet, content []byte) error { +func eventsMappingXPack(r mb.ReporterV2, m *MetricSet, info elasticsearch.Info, content []byte) error { var data map[string]interface{} err := json.Unmarshal(content, &data) if err != nil { @@ -68,11 +68,6 @@ func eventsMappingXPack(r mb.ReporterV2, m *MetricSet, content []byte) error { indexRecovery := common.MapStr{} indexRecovery["shards"] = results - info, err := elasticsearch.GetInfo(m.HTTP, m.HTTP.GetURI()) - if err != nil { - return errors.Wrap(err, "failed to retrieve info from Elasticsearch") - } - event := mb.Event{} event.RootFields = common.MapStr{ "cluster_uuid": info.ClusterID, diff --git a/metricbeat/module/elasticsearch/index_recovery/index_recovery.go b/metricbeat/module/elasticsearch/index_recovery/index_recovery.go index 1a39f893663..49e6f55a4b5 100644 --- a/metricbeat/module/elasticsearch/index_recovery/index_recovery.go +++ b/metricbeat/module/elasticsearch/index_recovery/index_recovery.go @@ -72,7 +72,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // Fetch gathers stats for each index from the _stats API func (m *MetricSet) Fetch(r mb.ReporterV2) { - isMaster, err := elasticsearch.IsMaster(m.HTTP, m.HostData().SanitizedURI+m.recoveryPath) + isMaster, err := elasticsearch.IsMaster(m.HTTP, m.getServiceURI()) if err != nil { err = errors.Wrap(err, "error determining if connected Elasticsearch node is master") elastic.ReportAndLogError(err, r, m.Log) @@ -85,6 +85,12 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) { return } + info, err := elasticsearch.GetInfo(m.HTTP, m.getServiceURI()) + if err != nil { + elastic.ReportAndLogError(err, r, m.Log) + return + } + content, err := m.HTTP.FetchContent() if err != nil { elastic.ReportAndLogError(err, r, m.Log) @@ -92,9 +98,9 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) { } if m.MetricSet.XPack { - err = eventsMappingXPack(r, m, content) + err = eventsMappingXPack(r, m, *info, content) } else { - err = eventsMapping(r, content) + err = eventsMapping(r, *info, content) } if err != nil { @@ -102,3 +108,8 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) { return } } + +func (m *MetricSet) getServiceURI() string { + return m.HostData().SanitizedURI + m.recoveryPath + +} diff --git a/metricbeat/module/elasticsearch/ml_job/_meta/data.json b/metricbeat/module/elasticsearch/ml_job/_meta/data.json index 93a053224f0..7605e12e76a 100644 --- a/metricbeat/module/elasticsearch/ml_job/_meta/data.json +++ b/metricbeat/module/elasticsearch/ml_job/_meta/data.json @@ -1,16 +1,21 @@ { "@timestamp": "2017-10-12T08:05:34.853Z", - "beat": { + "agent": { "hostname": "host.example.com", "name": "host.example.com" }, "elasticsearch": { + "cluster": { + "id": "3LbUkLkURz--FR-YO0wLNA", + "name": "es1" + }, "ml": { "job": { "data_counts": { + "invalid_date_count": 0, "processed_record_count": 0 }, - "id": "filebeat-apache2-access-low_request_rate", + "id": "total-requests", "state": "closed" } } diff --git a/metricbeat/module/elasticsearch/ml_job/data.go b/metricbeat/module/elasticsearch/ml_job/data.go index df86ebfe89b..796d1faf987 100644 --- a/metricbeat/module/elasticsearch/ml_job/data.go +++ b/metricbeat/module/elasticsearch/ml_job/data.go @@ -45,7 +45,7 @@ type jobsStruct struct { Jobs []map[string]interface{} `json:"jobs"` } -func eventsMapping(r mb.ReporterV2, content []byte) error { +func eventsMapping(r mb.ReporterV2, info elasticsearch.Info, content []byte) error { jobsData := &jobsStruct{} err := json.Unmarshal(content, jobsData) @@ -63,6 +63,10 @@ func eventsMapping(r mb.ReporterV2, content []byte) error { event.RootFields = common.MapStr{} event.RootFields.Put("service.name", elasticsearch.ModuleName) + event.ModuleFields = common.MapStr{} + event.ModuleFields.Put("cluster.name", info.ClusterName) + event.ModuleFields.Put("cluster.id", info.ClusterID) + event.MetricSetFields, err = schema.Apply(job) if err != nil { event.Error = errors.Wrap(err, "failure applying ml job schema") diff --git a/metricbeat/module/elasticsearch/ml_job/data_test.go b/metricbeat/module/elasticsearch/ml_job/data_test.go new file mode 100644 index 00000000000..b6081642b9f --- /dev/null +++ b/metricbeat/module/elasticsearch/ml_job/data_test.go @@ -0,0 +1,30 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build !integration + +package ml_job + +import ( + "testing" + + "github.com/elastic/beats/metricbeat/module/elasticsearch" +) + +func TestMapper(t *testing.T) { + elasticsearch.TestMapperWithInfo(t, "./_meta/test/ml.*.json", eventsMapping) +} diff --git a/metricbeat/module/elasticsearch/ml_job/data_xpack.go b/metricbeat/module/elasticsearch/ml_job/data_xpack.go index 894c87c35d8..a68d956824b 100644 --- a/metricbeat/module/elasticsearch/ml_job/data_xpack.go +++ b/metricbeat/module/elasticsearch/ml_job/data_xpack.go @@ -31,14 +31,9 @@ import ( "github.com/elastic/beats/metricbeat/module/elasticsearch" ) -func eventsMappingXPack(r mb.ReporterV2, m *MetricSet, content []byte) error { - info, err := elasticsearch.GetInfo(m.HTTP, m.HTTP.GetURI()) - if err != nil { - return errors.Wrap(err, "failed to get info from Elasticsearch") - } - +func eventsMappingXPack(r mb.ReporterV2, m *MetricSet, info elasticsearch.Info, content []byte) error { var data map[string]interface{} - err = json.Unmarshal(content, &data) + err := json.Unmarshal(content, &data) if err != nil { return errors.Wrap(err, "failure parsing Elasticsearch ML Job Stats API response") } diff --git a/metricbeat/module/elasticsearch/ml_job/ml_job.go b/metricbeat/module/elasticsearch/ml_job/ml_job.go index cc568747d9c..453bea1a221 100644 --- a/metricbeat/module/elasticsearch/ml_job/ml_job.go +++ b/metricbeat/module/elasticsearch/ml_job/ml_job.go @@ -58,7 +58,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // Fetch methods implements the data gathering and data conversion to the right format func (m *MetricSet) Fetch(r mb.ReporterV2) { - isMaster, err := elasticsearch.IsMaster(m.HTTP, m.HostData().SanitizedURI+jobPath) + isMaster, err := elasticsearch.IsMaster(m.HTTP, m.getServiceURI()) if err != nil { err = errors.Wrap(err, "error determining if connected Elasticsearch node is master") elastic.ReportAndLogError(err, r, m.Log) @@ -71,6 +71,12 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) { return } + info, err := elasticsearch.GetInfo(m.HTTP, m.getServiceURI()) + if err != nil { + elastic.ReportAndLogError(err, r, m.Log) + return + } + content, err := m.HTTP.FetchContent() if err != nil { elastic.ReportAndLogError(err, r, m.Log) @@ -78,9 +84,9 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) { } if m.XPack { - err = eventsMappingXPack(r, m, content) + err = eventsMappingXPack(r, m, *info, content) } else { - err = eventsMapping(r, content) + err = eventsMapping(r, *info, content) } if err != nil { @@ -88,3 +94,7 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) { return } } + +func (m *MetricSet) getServiceURI() string { + return m.HostData().SanitizedURI + jobPath +} diff --git a/metricbeat/module/elasticsearch/node/_meta/data.json b/metricbeat/module/elasticsearch/node/_meta/data.json index 2f2cdb4d908..f878e335195 100644 --- a/metricbeat/module/elasticsearch/node/_meta/data.json +++ b/metricbeat/module/elasticsearch/node/_meta/data.json @@ -6,6 +6,7 @@ }, "elasticsearch": { "cluster": { + "id": "91RpCx2xSQ21pVPTZfDK0Q", "name": "elasticsearch" }, "node": { @@ -16,25 +17,25 @@ "bytes": 1073741824 }, "max": { - "bytes": 1038876672 + "bytes": 1037959168 } }, "nonheap": { "init": { - "bytes": 2555904 + "bytes": 7667712 }, "max": { "bytes": 0 } } }, - "version": "1.8.0_144" + "version": "11.0.1" }, - "name": "523zXyT6TRWiqXcQItnkyQ", + "name": "DSiWcTyeThWtUXLB9J0BMw", "process": { "mlockall": false }, - "version": "6.3.0" + "version": "7.0.0-alpha1" } }, "metricset": { diff --git a/metricbeat/module/elasticsearch/node/_meta/fields.yml b/metricbeat/module/elasticsearch/node/_meta/fields.yml index 1874c97162d..c1fc72631ad 100644 --- a/metricbeat/module/elasticsearch/node/_meta/fields.yml +++ b/metricbeat/module/elasticsearch/node/_meta/fields.yml @@ -4,10 +4,6 @@ node release: beta fields: - - name: name - type: keyword - description: > - Node name. - name: version type: keyword description: > diff --git a/metricbeat/module/elasticsearch/node/data.go b/metricbeat/module/elasticsearch/node/data.go index aa199189577..75ff395f228 100644 --- a/metricbeat/module/elasticsearch/node/data.go +++ b/metricbeat/module/elasticsearch/node/data.go @@ -61,7 +61,7 @@ var ( } ) -func eventsMapping(r mb.ReporterV2, content []byte) error { +func eventsMapping(r mb.ReporterV2, info elasticsearch.Info, content []byte) error { nodesStruct := struct { ClusterName string `json:"cluster_name"` Nodes map[string]map[string]interface{} `json:"nodes"` @@ -83,6 +83,7 @@ func eventsMapping(r mb.ReporterV2, content []byte) error { event.ModuleFields = common.MapStr{} event.ModuleFields.Put("cluster.name", nodesStruct.ClusterName) + event.ModuleFields.Put("cluster.id", info.ClusterID) event.MetricSetFields, err = schema.Apply(node) if err != nil { diff --git a/metricbeat/module/elasticsearch/node/data_test.go b/metricbeat/module/elasticsearch/node/data_test.go index de2537ddfa8..7e29e5b65c3 100644 --- a/metricbeat/module/elasticsearch/node/data_test.go +++ b/metricbeat/module/elasticsearch/node/data_test.go @@ -30,8 +30,13 @@ import ( "github.com/elastic/beats/metricbeat/module/elasticsearch" ) +var info = elasticsearch.Info{ + ClusterID: "1234", + ClusterName: "helloworld", +} + func TestGetMappings(t *testing.T) { - elasticsearch.TestMapper(t, "./_meta/test/node.*.json", eventsMapping) + elasticsearch.TestMapperWithInfo(t, "./_meta/test/node.*.json", eventsMapping) } func TestInvalid(t *testing.T) { @@ -41,6 +46,6 @@ func TestInvalid(t *testing.T) { assert.NoError(t, err) reporter := &mbtest.CapturingReporterV2{} - err = eventsMapping(reporter, content) + err = eventsMapping(reporter, info, content) assert.Error(t, err) } diff --git a/metricbeat/module/elasticsearch/node/node.go b/metricbeat/module/elasticsearch/node/node.go index 2642b66ba28..11b27606bd1 100644 --- a/metricbeat/module/elasticsearch/node/node.go +++ b/metricbeat/module/elasticsearch/node/node.go @@ -18,6 +18,8 @@ package node import ( + "github.com/pkg/errors" + "github.com/elastic/beats/libbeat/common/cfgwarn" "github.com/elastic/beats/metricbeat/helper/elastic" "github.com/elastic/beats/metricbeat/mb" @@ -72,7 +74,14 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) { return } - err = eventsMapping(r, content) + info, err := elasticsearch.GetInfo(m.HTTP, m.HostData().SanitizedURI+nodeStatsPath) + if err != nil { + err = errors.Wrap(err, "failed to get info from Elasticsearch") + elastic.ReportAndLogError(err, r, m.Log) + return + } + + err = eventsMapping(r, *info, content) if err != nil { elastic.ReportAndLogError(err, r, m.Log) return diff --git a/metricbeat/module/elasticsearch/node/node_test.go b/metricbeat/module/elasticsearch/node/node_test.go index ec8c805253e..b6aaf020189 100644 --- a/metricbeat/module/elasticsearch/node/node_test.go +++ b/metricbeat/module/elasticsearch/node/node_test.go @@ -45,10 +45,22 @@ func TestFetch(t *testing.T) { assert.NoError(t, err) server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(200) - w.Header().Set("Content-Type", "application/json;") - w.Write([]byte(response)) - assert.Equal(t, "/_nodes/_local", r.RequestURI) + switch r.RequestURI { + case "/_nodes/_local": + w.WriteHeader(200) + w.Header().Set("Content-Type", "application/json;") + w.Write([]byte(response)) + + case "/": + rootResponse := "{\"cluster_name\":\"es1\",\"cluster_uuid\":\"4heb1eiady103dxu71\",\"version\":{\"number\":\"7.0.0\"}}" + w.WriteHeader(200) + w.Header().Set("Content-Type", "application/json") + w.Write([]byte(rootResponse)) + + default: + t.FailNow() + } + })) defer server.Close() diff --git a/metricbeat/module/elasticsearch/node_stats/_meta/data.json b/metricbeat/module/elasticsearch/node_stats/_meta/data.json index a654b83e14c..cb33996adf0 100644 --- a/metricbeat/module/elasticsearch/node_stats/_meta/data.json +++ b/metricbeat/module/elasticsearch/node_stats/_meta/data.json @@ -1,43 +1,45 @@ { "@timestamp": "2017-10-12T08:05:34.853Z", - "beat": { + "agent": { "hostname": "host.example.com", "name": "host.example.com" }, "elasticsearch": { "cluster": { - "name": "elasticsearch" + "id": "3LbUkLkURz--FR-YO0wLNA", + "name": "es1" }, "node": { - "name": "523zXyT6TRWiqXcQItnkyQ", + "id": "FMRmkE3HTU6xxxoFK-06Ww", + "name": "es1_1", "stats": { "fs": { "summary": { "available": { - "bytes": 47081816064 + "bytes": 350828584960 }, "free": { - "bytes": 52054323200 + "bytes": 354770468864 }, "total": { - "bytes": 250790436864 + "bytes": 499963170816 } } }, "indices": { "docs": { - "count": 231, - "deleted": 56 + "count": 30880, + "deleted": 124 }, "segments": { - "count": 16, + "count": 39, "memory": { - "bytes": 105245 + "bytes": 300797 } }, "store": { "size": { - "bytes": 444882 + "bytes": 15205991 } } }, @@ -46,14 +48,14 @@ "collectors": { "old": { "collection": { - "count": 2, - "ms": 139 + "count": 3, + "ms": 219 } }, "young": { "collection": { - "count": 56, - "ms": 1288 + "count": 505, + "ms": 2439 } } } @@ -62,44 +64,44 @@ "pools": { "old": { "max": { - "bytes": 724828160 + "bytes": 715849728 }, "peak": { - "bytes": 225180248 + "bytes": 543519960 }, "peak_max": { - "bytes": 724828160 + "bytes": 715849728 }, "used": { - "bytes": 225180248 + "bytes": 382281744 } }, "survivor": { "max": { - "bytes": 34865152 + "bytes": 35782656 }, "peak": { - "bytes": 34865152 + "bytes": 35782656 }, "peak_max": { - "bytes": 34865152 + "bytes": 35782656 }, "used": { - "bytes": 5280456 + "bytes": 6418816 } }, "young": { "max": { - "bytes": 279183360 + "bytes": 286326784 }, "peak": { - "bytes": 279183360 + "bytes": 286326784 }, "peak_max": { - "bytes": 279183360 + "bytes": 286326784 }, "used": { - "bytes": 188134984 + "bytes": 118870448 } } } diff --git a/metricbeat/module/elasticsearch/node_stats/data.go b/metricbeat/module/elasticsearch/node_stats/data.go index 41423d7caa6..8002d93873f 100644 --- a/metricbeat/module/elasticsearch/node_stats/data.go +++ b/metricbeat/module/elasticsearch/node_stats/data.go @@ -19,6 +19,9 @@ package node_stats import ( "encoding/json" + "fmt" + + "github.com/elastic/beats/metricbeat/helper/elastic" "github.com/joeshaw/multierror" "github.com/pkg/errors" @@ -32,6 +35,7 @@ import ( var ( schema = s.Schema{ + "name": c.Str("name"), "jvm": c.Dict("jvm", s.Schema{ "mem": c.Dict("mem", s.Schema{ "pools": c.Dict("pools", s.Schema{ @@ -103,11 +107,10 @@ var ( ) type nodesStruct struct { - ClusterName string `json:"cluster_name"` - Nodes map[string]map[string]interface{} `json:"nodes"` + Nodes map[string]map[string]interface{} `json:"nodes"` } -func eventsMapping(r mb.ReporterV2, content []byte) error { +func eventsMapping(r mb.ReporterV2, info elasticsearch.Info, content []byte) error { nodeData := &nodesStruct{} err := json.Unmarshal(content, nodeData) @@ -118,7 +121,7 @@ func eventsMapping(r mb.ReporterV2, content []byte) error { } var errs multierror.Errors - for name, node := range nodeData.Nodes { + for id, node := range nodeData.Nodes { event := mb.Event{} event.RootFields = common.MapStr{} @@ -126,10 +129,11 @@ func eventsMapping(r mb.ReporterV2, content []byte) error { event.ModuleFields = common.MapStr{ "node": common.MapStr{ - "name": name, + "id": id, }, "cluster": common.MapStr{ - "name": nodeData.ClusterName, + "name": info.ClusterName, + "id": info.ClusterID, }, } @@ -138,7 +142,26 @@ func eventsMapping(r mb.ReporterV2, content []byte) error { event.Error = errors.Wrap(err, "failure to apply node schema") r.Event(event) errs = append(errs, event.Error) + continue + } + + name, err := event.MetricSetFields.GetValue("name") + if err != nil { + event.Error = elastic.MakeErrorForMissingField("name", elastic.Elasticsearch) + r.Event(event) + errs = append(errs, event.Error) + continue + } + + nameStr, ok := name.(string) + if !ok { + event.Error = fmt.Errorf("name is not a string") + r.Event(event) + errs = append(errs, event.Error) + continue } + event.ModuleFields.Put("node.name", nameStr) + event.MetricSetFields.Delete("name") r.Event(event) } diff --git a/metricbeat/module/elasticsearch/node_stats/data_test.go b/metricbeat/module/elasticsearch/node_stats/data_test.go index cb73845d9ec..670b67cf289 100644 --- a/metricbeat/module/elasticsearch/node_stats/data_test.go +++ b/metricbeat/module/elasticsearch/node_stats/data_test.go @@ -26,5 +26,5 @@ import ( ) func TestStats(t *testing.T) { - elasticsearch.TestMapper(t, "./_meta/test/node_stats.*.json", eventsMapping) + elasticsearch.TestMapperWithInfo(t, "./_meta/test/node_stats.*.json", eventsMapping) } diff --git a/metricbeat/module/elasticsearch/node_stats/data_xpack.go b/metricbeat/module/elasticsearch/node_stats/data_xpack.go index 6510fc3cfbf..bb4dcb5b3af 100644 --- a/metricbeat/module/elasticsearch/node_stats/data_xpack.go +++ b/metricbeat/module/elasticsearch/node_stats/data_xpack.go @@ -163,7 +163,7 @@ var ( } ) -func eventsMappingXPack(r mb.ReporterV2, m *MetricSet, content []byte) error { +func eventsMappingXPack(r mb.ReporterV2, m *MetricSet, info elasticsearch.Info, content []byte) error { nodesStruct := struct { ClusterName string `json:"cluster_name"` Nodes map[string]map[string]interface{} `json:"nodes"` @@ -181,12 +181,6 @@ func eventsMappingXPack(r mb.ReporterV2, m *MetricSet, content []byte) error { // of ES and it's not know if the request will be routed to the same node as before. var errs multierror.Errors for nodeID, node := range nodesStruct.Nodes { - clusterID, err := elasticsearch.GetClusterID(m.HTTP, m.HTTP.GetURI(), nodeID) - if err != nil { - errs = append(errs, errors.Wrap(err, "could not fetch cluster id")) - continue - } - isMaster, err := elasticsearch.IsMaster(m.HTTP, m.HTTP.GetURI()) if err != nil { errs = append(errs, errors.Wrap(err, "error determining if connected Elasticsearch node is master")) @@ -205,7 +199,7 @@ func eventsMappingXPack(r mb.ReporterV2, m *MetricSet, content []byte) error { event.RootFields = common.MapStr{ "timestamp": time.Now(), - "cluster_uuid": clusterID, + "cluster_uuid": info.ClusterID, "interval_ms": m.Module().Config().Period.Nanoseconds() / 1000 / 1000, "type": "node_stats", "node_stats": nodeData, diff --git a/metricbeat/module/elasticsearch/node_stats/node_stats.go b/metricbeat/module/elasticsearch/node_stats/node_stats.go index c355fe51fd9..a8d81082292 100644 --- a/metricbeat/module/elasticsearch/node_stats/node_stats.go +++ b/metricbeat/module/elasticsearch/node_stats/node_stats.go @@ -63,17 +63,27 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) { return } + info, err := elasticsearch.GetInfo(m.HTTP, m.getServiceURI()) + if err != nil { + elastic.ReportAndLogError(err, r, m.Log) + return + } + if m.XPack { - err = eventsMappingXPack(r, m, content) + err = eventsMappingXPack(r, m, *info, content) if err != nil { m.Log.Error(err) return } } else { - err = eventsMapping(r, content) + err = eventsMapping(r, *info, content) if err != nil { elastic.ReportAndLogError(err, r, m.Log) return } } } + +func (m *MetricSet) getServiceURI() string { + return m.HostData().SanitizedURI + nodeStatsPath +} diff --git a/metricbeat/module/elasticsearch/pending_tasks/_meta/data.json b/metricbeat/module/elasticsearch/pending_tasks/_meta/data.json index c27dffcdcc9..8892c9e1efa 100644 --- a/metricbeat/module/elasticsearch/pending_tasks/_meta/data.json +++ b/metricbeat/module/elasticsearch/pending_tasks/_meta/data.json @@ -1,24 +1,26 @@ { - "@timestamp": "2018-05-03T20:58:17.379Z", - "metricset": { - "rtt": 2487, - "namespace": "elasticsearch.cluster.pending_task", - "name": "pending_tasks", - "module": "elasticsearch", - "host": "localhost:9200" - }, - "elasticsearch": { - "cluster": { - "pending_task": { - "insert_order": 47, - "priority": "HIGH", - "source": "put-mapping", - "time_in_queue.ms": 34 - } + "@timestamp": "2018-05-03T20:58:17.379Z", + "metricset": { + "rtt": 2487, + "namespace": "elasticsearch.cluster.pending_task", + "name": "pending_tasks", + "module": "elasticsearch", + "host": "localhost:9200" + }, + "elasticsearch": { + "cluster": { + "id": "3LbUkLkURz--FR-YO0wLNA", + "name": "es1", + "pending_task": { + "insert_order": 47, + "priority": "HIGH", + "source": "put-mapping", + "time_in_queue.ms": 34 + } + } + }, + "agent": { + "name": "host.example.com", + "hostname": "host.example.com" } - }, - "beat": { - "name": "host.example.com", - "hostname": "host.example.com" - } -} \ No newline at end of file +} diff --git a/metricbeat/module/elasticsearch/pending_tasks/data.go b/metricbeat/module/elasticsearch/pending_tasks/data.go index d35207db884..e87fe3eee4d 100644 --- a/metricbeat/module/elasticsearch/pending_tasks/data.go +++ b/metricbeat/module/elasticsearch/pending_tasks/data.go @@ -40,7 +40,7 @@ var ( } ) -func eventsMapping(r mb.ReporterV2, content []byte) error { +func eventsMapping(r mb.ReporterV2, info elasticsearch.Info, content []byte) error { tasksStruct := struct { Tasks []map[string]interface{} `json:"tasks"` }{} @@ -63,6 +63,10 @@ func eventsMapping(r mb.ReporterV2, content []byte) error { event.RootFields = common.MapStr{} event.RootFields.Put("service.name", elasticsearch.ModuleName) + event.ModuleFields = common.MapStr{} + event.ModuleFields.Put("cluster.name", info.ClusterName) + event.ModuleFields.Put("cluster.id", info.ClusterID) + event.MetricSetFields, err = schema.Apply(task) if err != nil { event.Error = errors.Wrap(err, "failure applying task schema") diff --git a/metricbeat/module/elasticsearch/pending_tasks/data_test.go b/metricbeat/module/elasticsearch/pending_tasks/data_test.go index 733a0d54ace..dd59e41c337 100644 --- a/metricbeat/module/elasticsearch/pending_tasks/data_test.go +++ b/metricbeat/module/elasticsearch/pending_tasks/data_test.go @@ -30,8 +30,14 @@ import ( "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/metricbeat/mb" mbtest "github.com/elastic/beats/metricbeat/mb/testing" + "github.com/elastic/beats/metricbeat/module/elasticsearch" ) +var info = elasticsearch.Info{ + ClusterID: "1234", + ClusterName: "helloworld", +} + //Events Mapping func TestEmptyQueueShouldGiveNoError(t *testing.T) { @@ -40,7 +46,7 @@ func TestEmptyQueueShouldGiveNoError(t *testing.T) { assert.NoError(t, err) reporter := &mbtest.CapturingReporterV2{} - err = eventsMapping(reporter, content) + err = eventsMapping(reporter, info, content) assert.NoError(t, err) } @@ -50,7 +56,7 @@ func TestNotEmptyQueueShouldGiveNoError(t *testing.T) { assert.NoError(t, err) reporter := &mbtest.CapturingReporterV2{} - err = eventsMapping(reporter, content) + err = eventsMapping(reporter, info, content) assert.NoError(t, err) assert.True(t, len(reporter.GetEvents()) >= 1) assert.Zero(t, len(reporter.GetErrors())) @@ -62,7 +68,7 @@ func TestEmptyQueueShouldGiveZeroEvent(t *testing.T) { assert.NoError(t, err) reporter := &mbtest.CapturingReporterV2{} - err = eventsMapping(reporter, content) + err = eventsMapping(reporter, info, content) assert.Zero(t, len(reporter.GetEvents())) assert.Zero(t, len(reporter.GetErrors())) } @@ -73,7 +79,7 @@ func TestNotEmptyQueueShouldGiveSeveralEvents(t *testing.T) { assert.NoError(t, err) reporter := &mbtest.CapturingReporterV2{} - err = eventsMapping(reporter, content) + err = eventsMapping(reporter, info, content) assert.Equal(t, 3, len(reporter.GetEvents())) assert.Zero(t, len(reporter.GetErrors())) } @@ -84,7 +90,7 @@ func TestInvalidJsonForRequiredFieldShouldThrowError(t *testing.T) { assert.NoError(t, err) reporter := &mbtest.CapturingReporterV2{} - err = eventsMapping(reporter, content) + err = eventsMapping(reporter, info, content) assert.Error(t, err) } @@ -94,7 +100,7 @@ func TestInvalidJsonForBadFormatShouldThrowError(t *testing.T) { assert.NoError(t, err) reporter := &mbtest.CapturingReporterV2{} - err = eventsMapping(reporter, content) + err = eventsMapping(reporter, info, content) assert.Error(t, err) } @@ -111,6 +117,12 @@ func TestEventsMappedMatchToContentReceived(t *testing.T) { "name": "elasticsearch", }, }, + ModuleFields: common.MapStr{ + "cluster": common.MapStr{ + "id": "1234", + "name": "helloworld", + }, + }, MetricSetFields: common.MapStr{ "priority": "URGENT", "source": "create-index [foo_9], cause [api]", @@ -128,6 +140,12 @@ func TestEventsMappedMatchToContentReceived(t *testing.T) { "name": "elasticsearch", }, }, + ModuleFields: common.MapStr{ + "cluster": common.MapStr{ + "id": "1234", + "name": "helloworld", + }, + }, MetricSetFields: common.MapStr{ "priority": "URGENT", "source": "create-index [foo_9], cause [api]", @@ -143,6 +161,12 @@ func TestEventsMappedMatchToContentReceived(t *testing.T) { "name": "elasticsearch", }, }, + ModuleFields: common.MapStr{ + "cluster": common.MapStr{ + "id": "1234", + "name": "helloworld", + }, + }, MetricSetFields: common.MapStr{"priority": "HIGH", "source": "shard-started ([foo_2][1], node[tMTocMvQQgGCkj7QDHl3OA], [P], s[INITIALIZING]), reason [after recovery from shard_store]", "time_in_queue.ms": int64(842), @@ -156,6 +180,12 @@ func TestEventsMappedMatchToContentReceived(t *testing.T) { "name": "elasticsearch", }, }, + ModuleFields: common.MapStr{ + "cluster": common.MapStr{ + "id": "1234", + "name": "helloworld", + }, + }, MetricSetFields: common.MapStr{ "priority": "HIGH", "source": "shard-started ([foo_2][0], node[tMTocMvQQgGCkj7QDHl3OA], [P], s[INITIALIZING]), reason [after recovery from shard_store]", @@ -172,7 +202,7 @@ func TestEventsMappedMatchToContentReceived(t *testing.T) { assert.NoError(t, err) reporter := &mbtest.CapturingReporterV2{} - err = eventsMapping(reporter, content) + err = eventsMapping(reporter, info, content) events := reporter.GetEvents() if !reflect.DeepEqual(testCase.expected, events) { diff --git a/metricbeat/module/elasticsearch/pending_tasks/pending_tasks.go b/metricbeat/module/elasticsearch/pending_tasks/pending_tasks.go index 07f354586ba..52b03cf97b3 100644 --- a/metricbeat/module/elasticsearch/pending_tasks/pending_tasks.go +++ b/metricbeat/module/elasticsearch/pending_tasks/pending_tasks.go @@ -63,7 +63,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // Fetch methods implements the data gathering and data conversion to the right format func (m *MetricSet) Fetch(r mb.ReporterV2) { - isMaster, err := elasticsearch.IsMaster(m.HTTP, m.HostData().SanitizedURI+pendingTasksPath) + isMaster, err := elasticsearch.IsMaster(m.HTTP, m.getServiceURI()) if err != nil { err := errors.Wrap(err, "error determining if connected Elasticsearch node is master") elastic.ReportAndLogError(err, r, m.Log) @@ -76,15 +76,25 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) { return } + info, err := elasticsearch.GetInfo(m.HTTP, m.getServiceURI()) + if err != nil { + elastic.ReportAndLogError(err, r, m.Log) + return + } + content, err := m.HTTP.FetchContent() if err != nil { elastic.ReportAndLogError(err, r, m.Log) return } - err = eventsMapping(r, content) + err = eventsMapping(r, *info, content) if err != nil { m.Log.Error(err) return } } + +func (m *MetricSet) getServiceURI() string { + return m.HostData().SanitizedURI + pendingTasksPath +} diff --git a/metricbeat/module/envoyproxy/_meta/docs.asciidoc b/metricbeat/module/envoyproxy/_meta/docs.asciidoc index cc900202a11..87f3e7e77aa 100644 --- a/metricbeat/module/envoyproxy/_meta/docs.asciidoc +++ b/metricbeat/module/envoyproxy/_meta/docs.asciidoc @@ -1,5 +1,3 @@ -== envoyproxy module - This is the envoyproxy module. The default metricset is `server`. diff --git a/metricbeat/module/envoyproxy/server/data.go b/metricbeat/module/envoyproxy/server/data.go index 07f49a71464..16688ce75d2 100644 --- a/metricbeat/module/envoyproxy/server/data.go +++ b/metricbeat/module/envoyproxy/server/data.go @@ -65,16 +65,16 @@ var ( }, "server": s.Object{ "days_until_first_cert_expiring": c.Int("days_until_first_cert_expiring"), - "live": c.Int("live"), - "memory_allocated": c.Int("memory_allocated"), - "memory_heap_size": c.Int("memory_heap_size"), - "parent_connections": c.Int("parent_connections"), - "total_connections": c.Int("total_connections"), - "uptime": c.Int("uptime"), - "version": c.Int("version"), - "watchdog_mega_miss": c.Int("watchdog_mega_miss", s.Optional), - "watchdog_miss": c.Int("watchdog_miss", s.Optional), - "hot_restart_epoch": c.Int("hot_restart_epoch", s.Optional), + "live": c.Int("live"), + "memory_allocated": c.Int("memory_allocated"), + "memory_heap_size": c.Int("memory_heap_size"), + "parent_connections": c.Int("parent_connections"), + "total_connections": c.Int("total_connections"), + "uptime": c.Int("uptime"), + "version": c.Int("version"), + "watchdog_mega_miss": c.Int("watchdog_mega_miss", s.Optional), + "watchdog_miss": c.Int("watchdog_miss", s.Optional), + "hot_restart_epoch": c.Int("hot_restart_epoch", s.Optional), }, "http2": s.Object{ "header_overflow": c.Int("header_overflow", s.Optional), diff --git a/metricbeat/module/http/_meta/Dockerfile b/metricbeat/module/http/_meta/Dockerfile index d67abee2cd0..6a3fe8a18ac 100644 --- a/metricbeat/module/http/_meta/Dockerfile +++ b/metricbeat/module/http/_meta/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.10.3 +FROM golang:1.11.2 COPY test/main.go main.go diff --git a/metricbeat/module/kafka/_meta/Dockerfile b/metricbeat/module/kafka/_meta/Dockerfile index aab10dbed24..35d19a8bc5c 100644 --- a/metricbeat/module/kafka/_meta/Dockerfile +++ b/metricbeat/module/kafka/_meta/Dockerfile @@ -11,7 +11,7 @@ ENV TERM=linux RUN apt-get update && apt-get install -y curl openjdk-8-jre-headless netcat dnsutils RUN mkdir -p ${KAFKA_LOGS_DIR} && mkdir -p ${KAFKA_HOME} && curl -s -o $INSTALL_DIR/kafka.tgz \ - "http://ftp.wayne.edu/apache/kafka/${KAFKA_VERSION}/kafka_2.11-${KAFKA_VERSION}.tgz" && \ + "https://archive.apache.org/dist/kafka/${KAFKA_VERSION}/kafka_2.11-${KAFKA_VERSION}.tgz" && \ tar xzf ${INSTALL_DIR}/kafka.tgz -C ${KAFKA_HOME} --strip-components 1 ADD run.sh /run.sh diff --git a/metricbeat/module/logstash/node/node_integration_test.go b/metricbeat/module/logstash/logstash_integration_test.go similarity index 59% rename from metricbeat/module/logstash/node/node_integration_test.go rename to metricbeat/module/logstash/logstash_integration_test.go index 2cf56681fdc..6987b44535c 100644 --- a/metricbeat/module/logstash/node/node_integration_test.go +++ b/metricbeat/module/logstash/logstash_integration_test.go @@ -17,7 +17,7 @@ // +build integration -package node +package logstash_test import ( "testing" @@ -27,30 +27,41 @@ import ( "github.com/elastic/beats/libbeat/tests/compose" mbtest "github.com/elastic/beats/metricbeat/mb/testing" "github.com/elastic/beats/metricbeat/module/logstash" + _ "github.com/elastic/beats/metricbeat/module/logstash/node" + _ "github.com/elastic/beats/metricbeat/module/logstash/node_stats" ) +var metricSets = []string{ + "node", + "node_stats", +} + func TestFetch(t *testing.T) { compose.EnsureUp(t, "logstash") - f := mbtest.NewReportingMetricSetV2(t, logstash.GetConfig("node")) - events, errs := mbtest.ReportingFetchV2(f) + for _, metricSet := range metricSets { + f := mbtest.NewReportingMetricSetV2(t, logstash.GetConfig(metricSet)) + events, errs := mbtest.ReportingFetchV2(f) - assert.Empty(t, errs) - if !assert.NotEmpty(t, events) { - t.FailNow() - } + assert.Empty(t, errs) + if !assert.NotEmpty(t, events) { + t.FailNow() + } - t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), - events[0].BeatEvent("logstash", "node").Fields.StringToPrint()) + t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), + events[0].BeatEvent("logstash", metricSet).Fields.StringToPrint()) + } } func TestData(t *testing.T) { compose.EnsureUp(t, "logstash") - config := logstash.GetConfig("node") - f := mbtest.NewReportingMetricSetV2(t, config) - err := mbtest.WriteEventsReporterV2(f, t, "") - if err != nil { - t.Fatal("write", err) + for _, metricSet := range metricSets { + config := logstash.GetConfig(metricSet) + f := mbtest.NewReportingMetricSetV2(t, config) + err := mbtest.WriteEventsReporterV2(f, t, metricSet) + if err != nil { + t.Fatal("write", err) + } } } diff --git a/metricbeat/module/logstash/node/_meta/data.json b/metricbeat/module/logstash/node/_meta/data.json index 5dcd8bca57b..3059f1d20e2 100644 --- a/metricbeat/module/logstash/node/_meta/data.json +++ b/metricbeat/module/logstash/node/_meta/data.json @@ -1,17 +1,17 @@ { "@timestamp": "2017-10-12T08:05:34.853Z", - "beat": { + "agent": { "hostname": "host.example.com", "name": "host.example.com" }, "logstash": { "node": { - "host": "5256c6d0f05a", + "host": "Shaunaks-MBP-2", "jvm": { - "pid": 1, - "version": "1.8.0_161" + "pid": 3674, + "version": "1.8.0_171" }, - "version": "6.2.3" + "version": "7.0.0-alpha1" } }, "metricset": { @@ -19,5 +19,8 @@ "module": "logstash", "name": "node", "rtt": 115 + }, + "service": { + "name": "logstash" } } \ No newline at end of file diff --git a/metricbeat/module/logstash/node_stats/_meta/data.json b/metricbeat/module/logstash/node_stats/_meta/data.json index 0d2378cda85..d36b978c579 100644 --- a/metricbeat/module/logstash/node_stats/_meta/data.json +++ b/metricbeat/module/logstash/node_stats/_meta/data.json @@ -1,6 +1,6 @@ { "@timestamp": "2017-10-12T08:05:34.853Z", - "beat": { + "agent": { "hostname": "host.example.com", "name": "host.example.com" }, @@ -8,9 +8,9 @@ "node": { "stats": { "events": { - "filtered": 0, - "in": 0, - "out": 0 + "filtered": 30750, + "in": 1, + "out": 30750 } } } @@ -21,5 +21,8 @@ "name": "node_stats", "namespace": "logstash.node.stats", "rtt": 115 + }, + "service": { + "name": "logstash" } } \ No newline at end of file diff --git a/metricbeat/module/logstash/node_stats/node_stats_integration_test.go b/metricbeat/module/logstash/node_stats/node_stats_integration_test.go deleted file mode 100644 index 3e3839405d9..00000000000 --- a/metricbeat/module/logstash/node_stats/node_stats_integration_test.go +++ /dev/null @@ -1,56 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -// +build integration - -package node_stats - -import ( - "testing" - - "github.com/stretchr/testify/assert" - - "github.com/elastic/beats/libbeat/tests/compose" - mbtest "github.com/elastic/beats/metricbeat/mb/testing" - "github.com/elastic/beats/metricbeat/module/logstash" -) - -func TestFetch(t *testing.T) { - compose.EnsureUp(t, "logstash") - - f := mbtest.NewReportingMetricSetV2(t, logstash.GetConfig("node_stats")) - events, errs := mbtest.ReportingFetchV2(f) - - assert.Empty(t, errs) - if !assert.NotEmpty(t, events) { - t.FailNow() - } - - t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), - events[0].BeatEvent("logstash", "node_stats").Fields.StringToPrint()) -} - -func TestData(t *testing.T) { - compose.EnsureUp(t, "logstash") - - config := logstash.GetConfig("node_stats") - f := mbtest.NewReportingMetricSetV2(t, config) - err := mbtest.WriteEventsReporterV2(f, t, "") - if err != nil { - t.Fatal("write", err) - } -} diff --git a/metricbeat/module/rabbitmq/connection/data.go b/metricbeat/module/rabbitmq/connection/data.go index b4ee6d20a44..68e2c15964a 100644 --- a/metricbeat/module/rabbitmq/connection/data.go +++ b/metricbeat/module/rabbitmq/connection/data.go @@ -60,7 +60,7 @@ func eventsMapping(content []byte) ([]common.MapStr, error) { var connections []map[string]interface{} err := json.Unmarshal(content, &connections) if err != nil { - logp.Err("Error: ", err) + logp.Err("Error: %+v", err) return nil, err } diff --git a/metricbeat/module/rabbitmq/exchange/data.go b/metricbeat/module/rabbitmq/exchange/data.go index 72e869fb595..a322bb8a8de 100644 --- a/metricbeat/module/rabbitmq/exchange/data.go +++ b/metricbeat/module/rabbitmq/exchange/data.go @@ -59,7 +59,7 @@ func eventsMapping(content []byte) ([]common.MapStr, error) { var exchanges []map[string]interface{} err := json.Unmarshal(content, &exchanges) if err != nil { - logp.Err("Error: ", err) + logp.Err("Error: %+v", err) return nil, err } diff --git a/metricbeat/module/rabbitmq/queue/data.go b/metricbeat/module/rabbitmq/queue/data.go index 56b977eab12..5b98742c88d 100644 --- a/metricbeat/module/rabbitmq/queue/data.go +++ b/metricbeat/module/rabbitmq/queue/data.go @@ -87,7 +87,7 @@ func eventsMapping(content []byte) ([]common.MapStr, error) { var queues []map[string]interface{} err := json.Unmarshal(content, &queues) if err != nil { - logp.Err("Error: ", err) + logp.Err("Error: %+v", err) return nil, err } diff --git a/metricbeat/module/uwsgi/status/data.go b/metricbeat/module/uwsgi/status/data.go index dc94f0530ef..26bc46a0e01 100644 --- a/metricbeat/module/uwsgi/status/data.go +++ b/metricbeat/module/uwsgi/status/data.go @@ -79,7 +79,7 @@ func eventsMapping(content []byte) ([]common.MapStr, error) { var stats uwsgiStat err := json.Unmarshal(content, &stats) if err != nil { - logp.Err("uwsgi statistics parsing failed with error: ", err) + logp.Err("uwsgi statistics parsing failed with error: %+v", err) return nil, err } diff --git a/metricbeat/module/uwsgi/status/status.go b/metricbeat/module/uwsgi/status/status.go index 5bd9ef8bcf8..efb88dec987 100644 --- a/metricbeat/module/uwsgi/status/status.go +++ b/metricbeat/module/uwsgi/status/status.go @@ -57,7 +57,7 @@ func fetchStatData(URL string) ([]byte, error) { u, err := url.Parse(URL) if err != nil { - logp.Err("parsing uwsgi stats url failed: ", err) + logp.Err("parsing uwsgi stats url failed: %+v", err) return nil, err } @@ -85,7 +85,7 @@ func fetchStatData(URL string) ([]byte, error) { defer res.Body.Close() if res.StatusCode != 200 { - logp.Err("failed to fetch uwsgi status with code: ", res.StatusCode) + logp.Err("failed to fetch uwsgi status with code: %d", res.StatusCode) return nil, errors.New("http failed") } reader = res.Body @@ -95,7 +95,7 @@ func fetchStatData(URL string) ([]byte, error) { data, err := ioutil.ReadAll(reader) if err != nil { - logp.Err("uwsgi data read failed: ", err) + logp.Err("uwsgi data read failed: %+v", err) return nil, err } diff --git a/packetbeat/docs/gettingstarted.asciidoc b/packetbeat/docs/gettingstarted.asciidoc index ba7ec090a73..60c3b978ce0 100644 --- a/packetbeat/docs/gettingstarted.asciidoc +++ b/packetbeat/docs/gettingstarted.asciidoc @@ -96,6 +96,25 @@ tar xzvf packetbeat-{version}-darwin-x86_64.tar.gz endif::[] +[[linux]] +*linux:* + +ifeval::["{release-state}"=="unreleased"] + +Version {stack-version} of {beatname_uc} has not yet been released. + +endif::[] + +ifeval::["{release-state}"!="unreleased"] + +["source","sh",subs="attributes,callouts"] +---------------------------------------------------------------------- +curl -L -O https://artifacts.elastic.co/downloads/beats/packetbeat/packetbeat-{version}-linux-x86_64.tar.gz +tar xzvf packetbeat-{version}-linux-x86_64.tar.gz +---------------------------------------------------------------------- + +endif::[] + [[win]] *win:* @@ -272,12 +291,12 @@ sudo service {beatname_lc} start docker run {dockerimage} ---------------------------------------------------------------------- -*mac:* +*mac and linux:* [source,shell] ---------------------------------------------------------------------- sudo chown root packetbeat.yml <1> -sudo ./packetbeat -e -c packetbeat.yml +sudo ./packetbeat -e ---------------------------------------------------------------------- <1> You'll be running Packetbeat as root, so you need to change ownership of the configuration file, or run Packetbeat with `--strict.perms=false` specified. See diff --git a/packetbeat/docs/index.asciidoc b/packetbeat/docs/index.asciidoc index 920377164a6..e1fb05c3459 100644 --- a/packetbeat/docs/index.asciidoc +++ b/packetbeat/docs/index.asciidoc @@ -15,6 +15,7 @@ include::{asciidoc-dir}/../../shared/attributes.asciidoc[] :deb_os: :rpm_os: :mac_os: +:linux_os: :docker_platform: :win_os: diff --git a/packetbeat/magefile.go b/packetbeat/magefile.go index bb2f6d924e3..810dd74f778 100644 --- a/packetbeat/magefile.go +++ b/packetbeat/magefile.go @@ -113,7 +113,7 @@ func Clean() error { // Package packages the Beat for distribution. // Use SNAPSHOT=true to build snapshots. // Use PLATFORMS to control the target platforms. -// Use BEAT_VERSION_QUALIFIER to control the version qualifier. +// Use VERSION_QUALIFIER to control the version qualifier. func Package() { start := time.Now() defer func() { fmt.Println("package ran for", time.Since(start)) }() diff --git a/packetbeat/protos/http/http_test.go b/packetbeat/protos/http/http_test.go index 9318bfb4358..7033b9e6539 100644 --- a/packetbeat/protos/http/http_test.go +++ b/packetbeat/protos/http/http_test.go @@ -720,7 +720,7 @@ func TestEatBodyChunkedWaitCRLF(t *testing.T) { t.Error("Unexpected state", st.parseState) } - logp.Debug("http", "parseOffset", st.parseOffset) + logp.Debug("http", "parseOffset: %d", st.parseOffset) ok, complete = parser.parseBodyChunkedWaitFinalCRLF(st, msg) if ok != true || complete != false { diff --git a/packetbeat/protos/icmp/icmp.go b/packetbeat/protos/icmp/icmp.go index 00740290708..b64de3852fa 100644 --- a/packetbeat/protos/icmp/icmp.go +++ b/packetbeat/protos/icmp/icmp.go @@ -94,7 +94,7 @@ func (icmp *icmpPlugin) init(results protos.Reporter, config *icmpConfig) error var err error icmp.localIps, err = common.LocalIPAddrs() if err != nil { - logp.Err("icmp", "Error getting local IP addresses: %s", err) + logp.Err("Error getting local IP addresses: %+v", err) icmp.localIps = []net.IP{} } logp.Debug("icmp", "Local IP addresses: %s", icmp.localIps) diff --git a/packetbeat/protos/icmp/message.go b/packetbeat/protos/icmp/message.go index ec1c9181a22..0a8b857c2df 100644 --- a/packetbeat/protos/icmp/message.go +++ b/packetbeat/protos/icmp/message.go @@ -90,7 +90,7 @@ func isRequest(tuple *icmpTuple, msg *icmpMessage) bool { if tuple.icmpVersion == 6 { return !icmp6ResponseTypes[msg.Type] } - logp.WTF("icmp", "Invalid ICMP version[%d]", tuple.icmpVersion) + logp.WTF("Invalid ICMP version[%d]", tuple.icmpVersion) return true } @@ -101,7 +101,7 @@ func isError(tuple *icmpTuple, msg *icmpMessage) bool { if tuple.icmpVersion == 6 { return icmp6ErrorTypes[msg.Type] } - logp.WTF("icmp", "Invalid ICMP version[%d]", tuple.icmpVersion) + logp.WTF("Invalid ICMP version[%d]", tuple.icmpVersion) return true } @@ -112,7 +112,7 @@ func requiresCounterpart(tuple *icmpTuple, msg *icmpMessage) bool { if tuple.icmpVersion == 6 { return icmp6PairTypes[msg.Type] } - logp.WTF("icmp", "Invalid ICMP version[%d]", tuple.icmpVersion) + logp.WTF("Invalid ICMP version[%d]", tuple.icmpVersion) return false } @@ -133,7 +133,7 @@ func extractTrackingData(icmpVersion uint8, msgType uint8, baseLayer *layers.Bas } return 0, 0 } - logp.WTF("icmp", "Invalid ICMP version[%d]", icmpVersion) + logp.WTF("Invalid ICMP version[%d]", icmpVersion) return 0, 0 } @@ -144,6 +144,6 @@ func humanReadable(tuple *icmpTuple, msg *icmpMessage) string { if tuple.icmpVersion == 6 { return layers.ICMPv6TypeCode(binary.BigEndian.Uint16([]byte{msg.Type, msg.code})).String() } - logp.WTF("icmp", "Invalid ICMP version[%d]", tuple.icmpVersion) + logp.WTF("Invalid ICMP version[%d]", tuple.icmpVersion) return "" } diff --git a/packetbeat/protos/mysql/mysql.go b/packetbeat/protos/mysql/mysql.go index 89bd0221a77..38e6959e940 100644 --- a/packetbeat/protos/mysql/mysql.go +++ b/packetbeat/protos/mysql/mysql.go @@ -801,7 +801,7 @@ func (mysql *mysqlPlugin) parseMysqlResponse(data []byte) ([]string, [][]string) var complete bool text, off, complete, err = readLstring(data, off) if err != nil || !complete { - logp.Debug("mysql", "Error parsing rows: %s %b", err, complete) + logp.Debug("mysql", "Error parsing rows: %+v %t", err, complete) // nevertheless, return what we have so far return fields, rows } diff --git a/packetbeat/protos/thrift/thrift_idl.go b/packetbeat/protos/thrift/thrift_idl.go index dd4ecc4279e..8d4d891bc86 100644 --- a/packetbeat/protos/thrift/thrift_idl.go +++ b/packetbeat/protos/thrift/thrift_idl.go @@ -68,7 +68,7 @@ func buildMethodsMap(thriftFiles map[string]parser.Thrift) map[string]*thriftIdl for _, method := range service.Methods { if _, exists := output[method.Name]; exists { logp.Warn("Thrift IDL: Method %s is defined in more services: %s and %s", - output[method.Name].service.Name, service.Name) + method.Name, output[method.Name].service.Name, service.Name) } output[method.Name] = &thriftIdlMethod{ service: service, diff --git a/testing/environments/latest.yml b/testing/environments/latest.yml index 9ad1c7a04d6..3780f08b385 100644 --- a/testing/environments/latest.yml +++ b/testing/environments/latest.yml @@ -3,7 +3,7 @@ version: '2.1' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:6.4.3 + image: docker.elastic.co/elasticsearch/elasticsearch:6.5.1 healthcheck: test: ["CMD", "curl", "-f", "http://localhost:9200"] retries: 300 @@ -16,7 +16,7 @@ services: - "xpack.security.enabled=false" logstash: - image: docker.elastic.co/logstash/logstash:6.4.3 + image: docker.elastic.co/logstash/logstash:6.5.1 healthcheck: test: ["CMD", "curl", "-f", "http://localhost:9600/_node/stats"] retries: 300 @@ -26,7 +26,7 @@ services: - ./docker/logstash/pki:/etc/pki:ro kibana: - image: docker.elastic.co/kibana/kibana:6.4.3 + image: docker.elastic.co/kibana/kibana:6.5.1 healthcheck: test: ["CMD", "curl", "-f", "http://localhost:5601"] retries: 300 diff --git a/vendor/github.com/fsnotify/fsevents/go_1_10_after.go b/vendor/github.com/fsnotify/fsevents/go_1_10_after.go new file mode 100644 index 00000000000..db7824c6691 --- /dev/null +++ b/vendor/github.com/fsnotify/fsevents/go_1_10_after.go @@ -0,0 +1,27 @@ +// +build darwin,go1.10 + +package fsevents + +/* +#include +*/ +import "C" + +const ( + nullCFStringRef = C.CFStringRef(0) + nullCFUUIDRef = C.CFUUIDRef(0) +) + +// NOTE: The following code is identical between go_1_10_after and go_1_10_before, +// however versions of Go 1.10.x prior to 1.10.4 fail to compile when the code utilizing +// the above constants is in a different file (wrap.go). + +// GetDeviceUUID retrieves the UUID required to identify an EventID +// in the FSEvents database +func GetDeviceUUID(deviceID int32) string { + uuid := C.FSEventsCopyUUIDForDevice(C.dev_t(deviceID)) + if uuid == nullCFUUIDRef { + return "" + } + return cfStringToGoString(C.CFUUIDCreateString(C.kCFAllocatorDefault, uuid)) +} diff --git a/vendor/github.com/fsnotify/fsevents/go_1_10_before.go b/vendor/github.com/fsnotify/fsevents/go_1_10_before.go new file mode 100644 index 00000000000..4ddd444130a --- /dev/null +++ b/vendor/github.com/fsnotify/fsevents/go_1_10_before.go @@ -0,0 +1,25 @@ +// +build darwin,!go1.10 + +package fsevents + +/* +#include +*/ +import "C" + +var ( + nullCFStringRef = C.CFStringRef(nil) + nullCFUUIDRef = C.CFUUIDRef(nil) +) + +// NOTE: The following code is identical between go_1_10_after and go_1_10_before. + +// GetDeviceUUID retrieves the UUID required to identify an EventID +// in the FSEvents database +func GetDeviceUUID(deviceID int32) string { + uuid := C.FSEventsCopyUUIDForDevice(C.dev_t(deviceID)) + if uuid == nullCFUUIDRef { + return "" + } + return cfStringToGoString(C.CFUUIDCreateString(C.kCFAllocatorDefault, uuid)) +} diff --git a/vendor/github.com/fsnotify/fsevents/go_1_9_2_after.go b/vendor/github.com/fsnotify/fsevents/go_1_9_2_after.go new file mode 100644 index 00000000000..4164f9c7e91 --- /dev/null +++ b/vendor/github.com/fsnotify/fsevents/go_1_9_2_after.go @@ -0,0 +1,11 @@ +// +build darwin,go1.9.2 + +package fsevents + +/* +#include +*/ +import "C" + +// eventIDSinceNow is a sentinel to begin watching events "since now". +const eventIDSinceNow = uint64(C.kFSEventStreamEventIdSinceNow) diff --git a/vendor/github.com/fsnotify/fsevents/go_1_9_2_before.go b/vendor/github.com/fsnotify/fsevents/go_1_9_2_before.go new file mode 100644 index 00000000000..e1806e97c93 --- /dev/null +++ b/vendor/github.com/fsnotify/fsevents/go_1_9_2_before.go @@ -0,0 +1,13 @@ +// +build darwin,!go1.9.2 + +package fsevents + +// Prior to Go 1.9.2, converting C.kFSEventStreamEventIdSinceNow to a uint64 +// results in the error: "constant -1 overflows uint64". +// Related Go issue: https://github.com/golang/go/issues/21708 + +// Hardcoding the value here from FSEvents.h: +// kFSEventStreamEventIdSinceNow = 0xFFFFFFFFFFFFFFFFULL + +// eventIDSinceNow is a sentinel to begin watching events "since now". +const eventIDSinceNow = uint64(0xFFFFFFFFFFFFFFFF) diff --git a/vendor/github.com/fsnotify/fsevents/wrap.go b/vendor/github.com/fsnotify/fsevents/wrap.go index 6e4e418f18d..b3ce5d95ec7 100644 --- a/vendor/github.com/fsnotify/fsevents/wrap.go +++ b/vendor/github.com/fsnotify/fsevents/wrap.go @@ -1,4 +1,4 @@ -// +build darwin,go1.10 +// +build darwin package fsevents @@ -34,9 +34,6 @@ import ( "unsafe" ) -// eventIDSinceNow is a sentinel to begin watching events "since now". -const eventIDSinceNow = uint64(C.kFSEventStreamEventIdSinceNow) - // LatestEventID returns the most recently generated event ID, system-wide. func LatestEventID() uint64 { return uint64(C.FSEventsGetCurrentEventId()) @@ -105,21 +102,11 @@ func GetStreamRefPaths(f FSEventStreamRef) []string { return ss } -// GetDeviceUUID retrieves the UUID required to identify an EventID -// in the FSEvents database -func GetDeviceUUID(deviceID int32) string { - uuid := C.FSEventsCopyUUIDForDevice(C.dev_t(deviceID)) - if uuid == C.CFUUIDRef(0) { - return "" - } - return cfStringToGoString(C.CFUUIDCreateString(nil, uuid)) -} - func cfStringToGoString(cfs C.CFStringRef) string { - if cfs == 0 { + if cfs == nullCFStringRef { return "" } - cfStr := C.CFStringCreateCopy(nil, cfs) + cfStr := copyCFString(cfs) length := C.CFStringGetLength(cfStr) if length == 0 { // short-cut for empty strings @@ -148,6 +135,11 @@ func cfStringToGoString(cfs C.CFStringRef) string { return *(*string)(unsafe.Pointer(strHeader)) } +// copyCFString makes an immutable copy of a string with CFStringCreateCopy. +func copyCFString(cfs C.CFStringRef) C.CFStringRef { + return C.CFStringCreateCopy(C.kCFAllocatorDefault, cfs) +} + // CFRunLoopRef wraps C.CFRunLoopRef type CFRunLoopRef C.CFRunLoopRef @@ -169,10 +161,7 @@ func createPaths(paths []string) (C.CFArrayRef, error) { // because of them errs = append(errs, err) } - cpath := C.CString(p) - defer C.free(unsafe.Pointer(cpath)) - - str := C.CFStringCreateWithCString(nil, cpath, C.kCFStringEncodingUTF8) + str := makeCFString(p) C.CFArrayAppendValue(C.CFMutableArrayRef(cPaths), unsafe.Pointer(str)) } var err error @@ -182,6 +171,13 @@ func createPaths(paths []string) (C.CFArrayRef, error) { return cPaths, err } +// makeCFString makes an immutable string with CFStringCreateWithCString. +func makeCFString(str string) C.CFStringRef { + s := C.CString(str) + defer C.free(unsafe.Pointer(s)) + return C.CFStringCreateWithCString(C.kCFAllocatorDefault, s, C.kCFStringEncodingUTF8) +} + // CFArrayLen retrieves the length of CFArray type // See https://developer.apple.com/library/mac/documentation/CoreFoundation/Reference/CFArrayRef/#//apple_ref/c/func/CFArrayGetCount func cfArrayLen(ref C.CFArrayRef) int { diff --git a/vendor/github.com/fsnotify/fsevents/wrap_deprecated.go b/vendor/github.com/fsnotify/fsevents/wrap_deprecated.go deleted file mode 100644 index 5493428bd71..00000000000 --- a/vendor/github.com/fsnotify/fsevents/wrap_deprecated.go +++ /dev/null @@ -1,276 +0,0 @@ -// +build darwin,!go1.10 - -package fsevents - -/* -#cgo LDFLAGS: -framework CoreServices -#include -#include - -static CFArrayRef ArrayCreateMutable(int len) { - return CFArrayCreateMutable(NULL, len, &kCFTypeArrayCallBacks); -} - -extern void fsevtCallback(FSEventStreamRef p0, uintptr_t info, size_t p1, char** p2, FSEventStreamEventFlags* p3, FSEventStreamEventId* p4); - -static FSEventStreamRef EventStreamCreateRelativeToDevice(FSEventStreamContext * context, uintptr_t info, dev_t dev, CFArrayRef paths, FSEventStreamEventId since, CFTimeInterval latency, FSEventStreamCreateFlags flags) { - context->info = (void*) info; - return FSEventStreamCreateRelativeToDevice(NULL, (FSEventStreamCallback) fsevtCallback, context, dev, paths, since, latency, flags); -} - -static FSEventStreamRef EventStreamCreate(FSEventStreamContext * context, uintptr_t info, CFArrayRef paths, FSEventStreamEventId since, CFTimeInterval latency, FSEventStreamCreateFlags flags) { - context->info = (void*) info; - return FSEventStreamCreate(NULL, (FSEventStreamCallback) fsevtCallback, context, paths, since, latency, flags); -} -*/ -import "C" -import ( - "fmt" - "log" - "path/filepath" - "reflect" - "runtime" - "time" - "unsafe" -) - -// eventIDSinceNow is a sentinel to begin watching events "since now". -// NOTE: Go 1.9.2 broke compatibility here, for 1.9.1 and earlier we did: -// uint64(C.kFSEventStreamEventIdSinceNow + (1 << 64)) -// But 1.9.2+ complains about overflow and requires: -// uint64(C.kFSEventStreamEventIdSinceNow) -// There does not seem to be an easy way to rectify, so hardcoding the value -// here from FSEvents.h: -// kFSEventStreamEventIdSinceNow = 0xFFFFFFFFFFFFFFFFULL -const eventIDSinceNow = uint64(0xFFFFFFFFFFFFFFFF) - -// LatestEventID returns the most recently generated event ID, system-wide. -func LatestEventID() uint64 { - return uint64(C.FSEventsGetCurrentEventId()) -} - -// arguments are released by C at the end of the callback. Ensure copies -// are made if data is expected to persist beyond this function ending. -// -//export fsevtCallback -func fsevtCallback(stream C.FSEventStreamRef, info uintptr, numEvents C.size_t, cpaths **C.char, cflags *C.FSEventStreamEventFlags, cids *C.FSEventStreamEventId) { - l := int(numEvents) - events := make([]Event, l) - - es := registry.Get(info) - if es == nil { - log.Printf("failed to retrieve registry %d", info) - return - } - // These slices are backed by C data. Ensure data is copied out - // if it expected to exist outside of this function. - paths := (*[1 << 30]*C.char)(unsafe.Pointer(cpaths))[:l:l] - ids := (*[1 << 30]C.FSEventStreamEventId)(unsafe.Pointer(cids))[:l:l] - flags := (*[1 << 30]C.FSEventStreamEventFlags)(unsafe.Pointer(cflags))[:l:l] - for i := range events { - events[i] = Event{ - Path: C.GoString(paths[i]), - Flags: EventFlags(flags[i]), - ID: uint64(ids[i]), - } - es.EventID = uint64(ids[i]) - } - - es.Events <- events -} - -// FSEventStreamRef wraps C.FSEventStreamRef -type FSEventStreamRef C.FSEventStreamRef - -// GetStreamRefEventID retrieves the last EventID from the ref -func GetStreamRefEventID(f FSEventStreamRef) uint64 { - return uint64(C.FSEventStreamGetLatestEventId(f)) -} - -// GetStreamRefDeviceID retrieves the device ID the stream is watching -func GetStreamRefDeviceID(f FSEventStreamRef) int32 { - return int32(C.FSEventStreamGetDeviceBeingWatched(f)) -} - -// GetStreamRefDescription retrieves debugging description information -// about the StreamRef -func GetStreamRefDescription(f FSEventStreamRef) string { - return cfStringToGoString(C.FSEventStreamCopyDescription(f)) -} - -// GetStreamRefPaths returns a copy of the paths being watched by -// this stream -func GetStreamRefPaths(f FSEventStreamRef) []string { - arr := C.FSEventStreamCopyPathsBeingWatched(f) - l := cfArrayLen(arr) - - ss := make([]string, l) - for i := range ss { - void := C.CFArrayGetValueAtIndex(arr, C.CFIndex(i)) - ss[i] = cfStringToGoString(C.CFStringRef(void)) - } - return ss -} - -// GetDeviceUUID retrieves the UUID required to identify an EventID -// in the FSEvents database -func GetDeviceUUID(deviceID int32) string { - uuid := C.FSEventsCopyUUIDForDevice(C.dev_t(deviceID)) - if uuid == nil { - return "" - } - return cfStringToGoString(C.CFUUIDCreateString(nil, uuid)) -} - -func cfStringToGoString(cfs C.CFStringRef) string { - if cfs == nil { - return "" - } - cfStr := C.CFStringCreateCopy(nil, cfs) - length := C.CFStringGetLength(cfStr) - if length == 0 { - // short-cut for empty strings - return "" - } - cfRange := C.CFRange{0, length} - enc := C.CFStringEncoding(C.kCFStringEncodingUTF8) - // first find the buffer size necessary - var usedBufLen C.CFIndex - if C.CFStringGetBytes(cfStr, cfRange, enc, 0, C.false, nil, 0, &usedBufLen) == 0 { - return "" - } - - bs := make([]byte, usedBufLen) - buf := (*C.UInt8)(unsafe.Pointer(&bs[0])) - if C.CFStringGetBytes(cfStr, cfRange, enc, 0, C.false, buf, usedBufLen, nil) == 0 { - return "" - } - - // Create a string (byte array) backed by C byte array - header := (*reflect.SliceHeader)(unsafe.Pointer(&bs)) - strHeader := &reflect.StringHeader{ - Data: header.Data, - Len: header.Len, - } - return *(*string)(unsafe.Pointer(strHeader)) -} - -// CFRunLoopRef wraps C.CFRunLoopRef -type CFRunLoopRef C.CFRunLoopRef - -// EventIDForDeviceBeforeTime returns an event ID before a given time. -func EventIDForDeviceBeforeTime(dev int32, before time.Time) uint64 { - tm := C.CFAbsoluteTime(before.Unix()) - return uint64(C.FSEventsGetLastEventIdForDeviceBeforeTime(C.dev_t(dev), tm)) -} - -// createPaths accepts the user defined set of paths and returns FSEvents -// compatible array of paths -func createPaths(paths []string) (C.CFArrayRef, error) { - cPaths := C.ArrayCreateMutable(C.int(len(paths))) - var errs []error - for _, path := range paths { - p, err := filepath.Abs(path) - if err != nil { - // hack up some reporting errors, but don't prevent execution - // because of them - errs = append(errs, err) - } - cpath := C.CString(p) - defer C.free(unsafe.Pointer(cpath)) - - str := C.CFStringCreateWithCString(nil, cpath, C.kCFStringEncodingUTF8) - C.CFArrayAppendValue(cPaths, unsafe.Pointer(str)) - } - var err error - if len(errs) > 0 { - err = fmt.Errorf("%q", errs) - } - return cPaths, err -} - -// CFArrayLen retrieves the length of CFArray type -// See https://developer.apple.com/library/mac/documentation/CoreFoundation/Reference/CFArrayRef/#//apple_ref/c/func/CFArrayGetCount -func cfArrayLen(ref C.CFArrayRef) int { - // FIXME: this will probably crash on 32bit, untested - // requires OS X v10.0 - return int(C.CFArrayGetCount(ref)) -} - -func setupStream(paths []string, flags CreateFlags, callbackInfo uintptr, eventID uint64, latency time.Duration, deviceID int32) FSEventStreamRef { - cPaths, err := createPaths(paths) - if err != nil { - log.Printf("Error creating paths: %s", err) - } - defer C.CFRelease(C.CFTypeRef(cPaths)) - - since := C.FSEventStreamEventId(eventID) - context := C.FSEventStreamContext{} - info := C.uintptr_t(callbackInfo) - cfinv := C.CFTimeInterval(float64(latency) / float64(time.Second)) - - var ref C.FSEventStreamRef - if deviceID != 0 { - ref = C.EventStreamCreateRelativeToDevice(&context, info, - C.dev_t(deviceID), cPaths, since, cfinv, - C.FSEventStreamCreateFlags(flags)) - } else { - ref = C.EventStreamCreate(&context, info, cPaths, since, - cfinv, C.FSEventStreamCreateFlags(flags)) - } - - return FSEventStreamRef(ref) -} - -func (es *EventStream) start(paths []string, callbackInfo uintptr) { - - since := eventIDSinceNow - if es.Resume { - since = es.EventID - } - - es.stream = setupStream(paths, es.Flags, callbackInfo, since, es.Latency, es.Device) - - started := make(chan struct{}) - - go func() { - runtime.LockOSThread() - es.rlref = CFRunLoopRef(C.CFRunLoopGetCurrent()) - C.FSEventStreamScheduleWithRunLoop(es.stream, es.rlref, C.kCFRunLoopDefaultMode) - C.FSEventStreamStart(es.stream) - close(started) - C.CFRunLoopRun() - }() - - if !es.hasFinalizer { - // TODO: There is no guarantee this run before program exit - // and could result in panics at exit. - runtime.SetFinalizer(es, finalizer) - es.hasFinalizer = true - } - - <-started -} - -func finalizer(es *EventStream) { - // If an EventStream is freed without Stop being called it will - // cause a panic. This avoids that, and closes the stream instead. - es.Stop() -} - -// flush drains the event stream of undelivered events -func flush(stream FSEventStreamRef, sync bool) { - if sync { - C.FSEventStreamFlushSync(stream) - } else { - C.FSEventStreamFlushAsync(stream) - } -} - -// stop requests fsevents stops streaming events -func stop(stream FSEventStreamRef, rlref CFRunLoopRef) { - C.FSEventStreamStop(stream) - C.FSEventStreamInvalidate(stream) - C.FSEventStreamRelease(stream) - C.CFRunLoopStop(rlref) -} diff --git a/vendor/github.com/magefile/mage/Gopkg.lock b/vendor/github.com/magefile/mage/Gopkg.lock deleted file mode 100644 index bef2d0092eb..00000000000 --- a/vendor/github.com/magefile/mage/Gopkg.lock +++ /dev/null @@ -1,9 +0,0 @@ -# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. - - -[solve-meta] - analyzer-name = "dep" - analyzer-version = 1 - inputs-digest = "ab4fef131ee828e96ba67d31a7d690bd5f2f42040c6766b1b12fe856f87e0ff7" - solver-name = "gps-cdcl" - solver-version = 1 diff --git a/vendor/github.com/magefile/mage/Gopkg.toml b/vendor/github.com/magefile/mage/Gopkg.toml deleted file mode 100644 index 9425a542966..00000000000 --- a/vendor/github.com/magefile/mage/Gopkg.toml +++ /dev/null @@ -1,22 +0,0 @@ - -# Gopkg.toml example -# -# Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md -# for detailed Gopkg.toml documentation. -# -# required = ["github.com/user/thing/cmd/thing"] -# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"] -# -# [[constraint]] -# name = "github.com/user/project" -# version = "1.0.0" -# -# [[constraint]] -# name = "github.com/user/project2" -# branch = "dev" -# source = "github.com/myfork/project2" -# -# [[override]] -# name = "github.com/x/y" -# version = "2.4.0" - diff --git a/vendor/github.com/magefile/mage/README.md b/vendor/github.com/magefile/mage/README.md index 63826bb7106..b757a92c690 100644 --- a/vendor/github.com/magefile/mage/README.md +++ b/vendor/github.com/magefile/mage/README.md @@ -1,6 +1,9 @@ +[![Built with Mage](https://magefile.org/badge.svg)](https://magefile.org) +[![Build Status](https://travis-ci.org/magefile/mage.svg?branch=master)](https://travis-ci.org/magefile/mage) [![Build status](https://ci.appveyor.com/api/projects/status/n6h146y79xgxkidl/branch/master?svg=true)](https://ci.appveyor.com/project/natefinch/mage/branch/master) +

-## About [![Build Status](https://travis-ci.org/magefile/mage.svg?branch=master)](https://travis-ci.org/magefile/mage) +## About Mage is a make/rake-like build tool using Go. You write plain-old go functions, and Mage automatically uses them as Makefile-like runnable targets. diff --git a/vendor/github.com/magefile/mage/go.mod b/vendor/github.com/magefile/mage/go.mod new file mode 100644 index 00000000000..d702af13eb7 --- /dev/null +++ b/vendor/github.com/magefile/mage/go.mod @@ -0,0 +1 @@ +module github.com/magefile/mage diff --git a/vendor/github.com/magefile/mage/go.sum b/vendor/github.com/magefile/mage/go.sum new file mode 100644 index 00000000000..e69de29bb2d diff --git a/vendor/github.com/magefile/mage/magefile.go b/vendor/github.com/magefile/mage/magefile.go index 2bb3ace6fee..12e9a48daab 100644 --- a/vendor/github.com/magefile/mage/magefile.go +++ b/vendor/github.com/magefile/mage/magefile.go @@ -1,5 +1,8 @@ //+build mage +// This is the build script for Mage. The install target is all you really need. +// The release target is for generating offial releases and is really only +// useful to project admins. package main import ( @@ -7,25 +10,24 @@ import ( "fmt" "os" "path/filepath" + "regexp" "runtime" "strings" "time" + "github.com/magefile/mage/mg" "github.com/magefile/mage/sh" ) // Runs "go install" for mage. This generates the version info the binary. func Install() error { - ldf, err := flags() - if err != nil { - return err - } - name := "mage" if runtime.GOOS == "windows" { name += ".exe" } - gopath, err := sh.Output("go", "env", "GOPATH") + + gocmd := mg.GoCmd() + gopath, err := sh.Output(gocmd, "env", "GOPATH") if err != nil { return fmt.Errorf("can't determine GOPATH: %v", err) } @@ -42,19 +44,23 @@ func Install() error { // install` turns into a no-op, and `go install -a` fails on people's // machines that have go installed in a non-writeable directory (such as // normal OS installs in /usr/bin) - return sh.RunV("go", "build", "-o", path, "-ldflags="+ldf, "github.com/magefile/mage") + return sh.RunV(gocmd, "build", "-o", path, "-ldflags="+flags(), "github.com/magefile/mage") } +var releaseTag = regexp.MustCompile(`^v1\.[0-9]+\.[0-9]+$`) + // Generates a new release. Expects the TAG environment variable to be set, // which will create a new tag with that name. func Release() (err error) { - if os.Getenv("TAG") == "" { - return errors.New("MSG and TAG environment variables are required") + tag := os.Getenv("TAG") + if !releaseTag.MatchString(tag) { + return errors.New("TAG environment variable must be in semver v1.x.x format, but was " + tag) } - if err := sh.RunV("git", "tag", "-a", "$TAG"); err != nil { + + if err := sh.RunV("git", "tag", "-a", tag, "-m", tag); err != nil { return err } - if err := sh.RunV("git", "push", "origin", "$TAG"); err != nil { + if err := sh.RunV("git", "push", "origin", tag); err != nil { return err } defer func() { @@ -71,14 +77,14 @@ func Clean() error { return sh.Rm("dist") } -func flags() (string, error) { +func flags() string { timestamp := time.Now().Format(time.RFC3339) hash := hash() tag := tag() if tag == "" { tag = "dev" } - return fmt.Sprintf(`-X "github.com/magefile/mage/mage.timestamp=%s" -X "github.com/magefile/mage/mage.commitHash=%s" -X "github.com/magefile/mage/mage.gitTag=%s"`, timestamp, hash, tag), nil + return fmt.Sprintf(`-X "github.com/magefile/mage/mage.timestamp=%s" -X "github.com/magefile/mage/mage.commitHash=%s" -X "github.com/magefile/mage/mage.gitTag=%s"`, timestamp, hash, tag) } // tag returns the git tag for the current branch or "" if none. diff --git a/vendor/golang.org/x/crypto/blake2b/blake2b.go b/vendor/golang.org/x/crypto/blake2b/blake2b.go index 6dedb89467a..58ea8753618 100644 --- a/vendor/golang.org/x/crypto/blake2b/blake2b.go +++ b/vendor/golang.org/x/crypto/blake2b/blake2b.go @@ -92,6 +92,8 @@ func New256(key []byte) (hash.Hash, error) { return newDigest(Size256, key) } // values equal or greater than: // - 32 if BLAKE2b is used as a hash function (The key is zero bytes long). // - 16 if BLAKE2b is used as a MAC function (The key is at least 16 bytes long). +// When the key is nil, the returned hash.Hash implements BinaryMarshaler +// and BinaryUnmarshaler for state (de)serialization as documented by hash.Hash. func New(size int, key []byte) (hash.Hash, error) { return newDigest(size, key) } func newDigest(hashSize int, key []byte) (*digest, error) { @@ -150,6 +152,50 @@ type digest struct { keyLen int } +const ( + magic = "b2b" + marshaledSize = len(magic) + 8*8 + 2*8 + 1 + BlockSize + 1 +) + +func (d *digest) MarshalBinary() ([]byte, error) { + if d.keyLen != 0 { + return nil, errors.New("crypto/blake2b: cannot marshal MACs") + } + b := make([]byte, 0, marshaledSize) + b = append(b, magic...) + for i := 0; i < 8; i++ { + b = appendUint64(b, d.h[i]) + } + b = appendUint64(b, d.c[0]) + b = appendUint64(b, d.c[1]) + // Maximum value for size is 64 + b = append(b, byte(d.size)) + b = append(b, d.block[:]...) + b = append(b, byte(d.offset)) + return b, nil +} + +func (d *digest) UnmarshalBinary(b []byte) error { + if len(b) < len(magic) || string(b[:len(magic)]) != magic { + return errors.New("crypto/blake2b: invalid hash state identifier") + } + if len(b) != marshaledSize { + return errors.New("crypto/blake2b: invalid hash state size") + } + b = b[len(magic):] + for i := 0; i < 8; i++ { + b, d.h[i] = consumeUint64(b) + } + b, d.c[0] = consumeUint64(b) + b, d.c[1] = consumeUint64(b) + d.size = int(b[0]) + b = b[1:] + copy(d.block[:], b[:BlockSize]) + b = b[BlockSize:] + d.offset = int(b[0]) + return nil +} + func (d *digest) BlockSize() int { return BlockSize } func (d *digest) Size() int { return d.size } @@ -219,3 +265,25 @@ func (d *digest) finalize(hash *[Size]byte) { binary.LittleEndian.PutUint64(hash[8*i:], v) } } + +func appendUint64(b []byte, x uint64) []byte { + var a [8]byte + binary.BigEndian.PutUint64(a[:], x) + return append(b, a[:]...) +} + +func appendUint32(b []byte, x uint32) []byte { + var a [4]byte + binary.BigEndian.PutUint32(a[:], x) + return append(b, a[:]...) +} + +func consumeUint64(b []byte) ([]byte, uint64) { + x := binary.BigEndian.Uint64(b) + return b[8:], x +} + +func consumeUint32(b []byte) ([]byte, uint32) { + x := binary.BigEndian.Uint32(b) + return b[4:], x +} diff --git a/vendor/golang.org/x/crypto/blake2b/blake2bAVX2_amd64.go b/vendor/golang.org/x/crypto/blake2b/blake2bAVX2_amd64.go index 8c41cf6c79e..4d31dd0fdcd 100644 --- a/vendor/golang.org/x/crypto/blake2b/blake2bAVX2_amd64.go +++ b/vendor/golang.org/x/crypto/blake2b/blake2bAVX2_amd64.go @@ -6,21 +6,14 @@ package blake2b +import "golang.org/x/sys/cpu" + func init() { - useAVX2 = supportsAVX2() - useAVX = supportsAVX() - useSSE4 = supportsSSE4() + useAVX2 = cpu.X86.HasAVX2 + useAVX = cpu.X86.HasAVX + useSSE4 = cpu.X86.HasSSE41 } -//go:noescape -func supportsSSE4() bool - -//go:noescape -func supportsAVX() bool - -//go:noescape -func supportsAVX2() bool - //go:noescape func hashBlocksAVX2(h *[8]uint64, c *[2]uint64, flag uint64, blocks []byte) @@ -31,13 +24,14 @@ func hashBlocksAVX(h *[8]uint64, c *[2]uint64, flag uint64, blocks []byte) func hashBlocksSSE4(h *[8]uint64, c *[2]uint64, flag uint64, blocks []byte) func hashBlocks(h *[8]uint64, c *[2]uint64, flag uint64, blocks []byte) { - if useAVX2 { + switch { + case useAVX2: hashBlocksAVX2(h, c, flag, blocks) - } else if useAVX { + case useAVX: hashBlocksAVX(h, c, flag, blocks) - } else if useSSE4 { + case useSSE4: hashBlocksSSE4(h, c, flag, blocks) - } else { + default: hashBlocksGeneric(h, c, flag, blocks) } } diff --git a/vendor/golang.org/x/crypto/blake2b/blake2bAVX2_amd64.s b/vendor/golang.org/x/crypto/blake2b/blake2bAVX2_amd64.s index 784bce6a9c4..5593b1b3dce 100644 --- a/vendor/golang.org/x/crypto/blake2b/blake2bAVX2_amd64.s +++ b/vendor/golang.org/x/crypto/blake2b/blake2bAVX2_amd64.s @@ -748,15 +748,3 @@ noinc: MOVQ BP, SP RET - -// func supportsAVX2() bool -TEXT ·supportsAVX2(SB), 4, $0-1 - MOVQ runtime·support_avx2(SB), AX - MOVB AX, ret+0(FP) - RET - -// func supportsAVX() bool -TEXT ·supportsAVX(SB), 4, $0-1 - MOVQ runtime·support_avx(SB), AX - MOVB AX, ret+0(FP) - RET diff --git a/vendor/golang.org/x/crypto/blake2b/blake2b_amd64.go b/vendor/golang.org/x/crypto/blake2b/blake2b_amd64.go index 2ab7c30fc2b..30e2fcd581f 100644 --- a/vendor/golang.org/x/crypto/blake2b/blake2b_amd64.go +++ b/vendor/golang.org/x/crypto/blake2b/blake2b_amd64.go @@ -6,13 +6,12 @@ package blake2b +import "golang.org/x/sys/cpu" + func init() { - useSSE4 = supportsSSE4() + useSSE4 = cpu.X86.HasSSE41 } -//go:noescape -func supportsSSE4() bool - //go:noescape func hashBlocksSSE4(h *[8]uint64, c *[2]uint64, flag uint64, blocks []byte) diff --git a/vendor/golang.org/x/crypto/blake2b/blake2b_amd64.s b/vendor/golang.org/x/crypto/blake2b/blake2b_amd64.s index 64530740b40..578e947b3bf 100644 --- a/vendor/golang.org/x/crypto/blake2b/blake2b_amd64.s +++ b/vendor/golang.org/x/crypto/blake2b/blake2b_amd64.s @@ -279,12 +279,3 @@ noinc: MOVQ BP, SP RET - -// func supportsSSE4() bool -TEXT ·supportsSSE4(SB), 4, $0-1 - MOVL $1, AX - CPUID - SHRL $19, CX // Bit 19 indicates SSE4 support - ANDL $1, CX // CX != 0 if support SSE4 - MOVB CX, ret+0(FP) - RET diff --git a/vendor/golang.org/x/sys/cpu/cpu.go b/vendor/golang.org/x/sys/cpu/cpu.go new file mode 100644 index 00000000000..3d88f866739 --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/cpu.go @@ -0,0 +1,38 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package cpu implements processor feature detection for +// various CPU architectures. +package cpu + +// CacheLinePad is used to pad structs to avoid false sharing. +type CacheLinePad struct{ _ [cacheLineSize]byte } + +// X86 contains the supported CPU features of the +// current X86/AMD64 platform. If the current platform +// is not X86/AMD64 then all feature flags are false. +// +// X86 is padded to avoid false sharing. Further the HasAVX +// and HasAVX2 are only set if the OS supports XMM and YMM +// registers in addition to the CPUID feature bit being set. +var X86 struct { + _ CacheLinePad + HasAES bool // AES hardware implementation (AES NI) + HasADX bool // Multi-precision add-carry instruction extensions + HasAVX bool // Advanced vector extension + HasAVX2 bool // Advanced vector extension 2 + HasBMI1 bool // Bit manipulation instruction set 1 + HasBMI2 bool // Bit manipulation instruction set 2 + HasERMS bool // Enhanced REP for MOVSB and STOSB + HasFMA bool // Fused-multiply-add instructions + HasOSXSAVE bool // OS supports XSAVE/XRESTOR for saving/restoring XMM registers. + HasPCLMULQDQ bool // PCLMULQDQ instruction - most often used for AES-GCM + HasPOPCNT bool // Hamming weight instruction POPCNT. + HasSSE2 bool // Streaming SIMD extension 2 (always available on amd64) + HasSSE3 bool // Streaming SIMD extension 3 + HasSSSE3 bool // Supplemental streaming SIMD extension 3 + HasSSE41 bool // Streaming SIMD extension 4 and 4.1 + HasSSE42 bool // Streaming SIMD extension 4 and 4.2 + _ CacheLinePad +} diff --git a/vendor/golang.org/x/sys/cpu/cpu_arm.go b/vendor/golang.org/x/sys/cpu/cpu_arm.go new file mode 100644 index 00000000000..d93036f7522 --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/cpu_arm.go @@ -0,0 +1,7 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cpu + +const cacheLineSize = 32 diff --git a/vendor/golang.org/x/sys/cpu/cpu_arm64.go b/vendor/golang.org/x/sys/cpu/cpu_arm64.go new file mode 100644 index 00000000000..1d2ab2902a7 --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/cpu_arm64.go @@ -0,0 +1,7 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cpu + +const cacheLineSize = 64 diff --git a/vendor/golang.org/x/sys/cpu/cpu_gc_x86.go b/vendor/golang.org/x/sys/cpu/cpu_gc_x86.go new file mode 100644 index 00000000000..f7cb46971cb --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/cpu_gc_x86.go @@ -0,0 +1,16 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build 386 amd64 amd64p32 +// +build !gccgo + +package cpu + +// cpuid is implemented in cpu_x86.s for gc compiler +// and in cpu_gccgo.c for gccgo. +func cpuid(eaxArg, ecxArg uint32) (eax, ebx, ecx, edx uint32) + +// xgetbv with ecx = 0 is implemented in cpu_x86.s for gc compiler +// and in cpu_gccgo.c for gccgo. +func xgetbv() (eax, edx uint32) diff --git a/vendor/golang.org/x/sys/cpu/cpu_gccgo.c b/vendor/golang.org/x/sys/cpu/cpu_gccgo.c new file mode 100644 index 00000000000..e363c7d1319 --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/cpu_gccgo.c @@ -0,0 +1,43 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build 386 amd64 amd64p32 +// +build gccgo + +#include +#include + +// Need to wrap __get_cpuid_count because it's declared as static. +int +gccgoGetCpuidCount(uint32_t leaf, uint32_t subleaf, + uint32_t *eax, uint32_t *ebx, + uint32_t *ecx, uint32_t *edx) +{ + return __get_cpuid_count(leaf, subleaf, eax, ebx, ecx, edx); +} + +// xgetbv reads the contents of an XCR (Extended Control Register) +// specified in the ECX register into registers EDX:EAX. +// Currently, the only supported value for XCR is 0. +// +// TODO: Replace with a better alternative: +// +// #include +// +// #pragma GCC target("xsave") +// +// void gccgoXgetbv(uint32_t *eax, uint32_t *edx) { +// unsigned long long x = _xgetbv(0); +// *eax = x & 0xffffffff; +// *edx = (x >> 32) & 0xffffffff; +// } +// +// Note that _xgetbv is defined starting with GCC 8. +void +gccgoXgetbv(uint32_t *eax, uint32_t *edx) +{ + __asm(" xorl %%ecx, %%ecx\n" + " xgetbv" + : "=a"(*eax), "=d"(*edx)); +} diff --git a/vendor/golang.org/x/sys/cpu/cpu_gccgo.go b/vendor/golang.org/x/sys/cpu/cpu_gccgo.go new file mode 100644 index 00000000000..ba49b91bd39 --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/cpu_gccgo.go @@ -0,0 +1,26 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build 386 amd64 amd64p32 +// +build gccgo + +package cpu + +//extern gccgoGetCpuidCount +func gccgoGetCpuidCount(eaxArg, ecxArg uint32, eax, ebx, ecx, edx *uint32) + +func cpuid(eaxArg, ecxArg uint32) (eax, ebx, ecx, edx uint32) { + var a, b, c, d uint32 + gccgoGetCpuidCount(eaxArg, ecxArg, &a, &b, &c, &d) + return a, b, c, d +} + +//extern gccgoXgetbv +func gccgoXgetbv(eax, edx *uint32) + +func xgetbv() (eax, edx uint32) { + var a, d uint32 + gccgoXgetbv(&a, &d) + return a, d +} diff --git a/vendor/golang.org/x/sys/cpu/cpu_mips64x.go b/vendor/golang.org/x/sys/cpu/cpu_mips64x.go new file mode 100644 index 00000000000..6165f121249 --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/cpu_mips64x.go @@ -0,0 +1,9 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build mips64 mips64le + +package cpu + +const cacheLineSize = 32 diff --git a/vendor/golang.org/x/sys/cpu/cpu_mipsx.go b/vendor/golang.org/x/sys/cpu/cpu_mipsx.go new file mode 100644 index 00000000000..1269eee88d0 --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/cpu_mipsx.go @@ -0,0 +1,9 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build mips mipsle + +package cpu + +const cacheLineSize = 32 diff --git a/vendor/golang.org/x/sys/cpu/cpu_ppc64x.go b/vendor/golang.org/x/sys/cpu/cpu_ppc64x.go new file mode 100644 index 00000000000..d10759a524f --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/cpu_ppc64x.go @@ -0,0 +1,9 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ppc64 ppc64le + +package cpu + +const cacheLineSize = 128 diff --git a/vendor/golang.org/x/sys/cpu/cpu_s390x.go b/vendor/golang.org/x/sys/cpu/cpu_s390x.go new file mode 100644 index 00000000000..684c4f005d0 --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/cpu_s390x.go @@ -0,0 +1,7 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cpu + +const cacheLineSize = 256 diff --git a/vendor/golang.org/x/sys/cpu/cpu_x86.go b/vendor/golang.org/x/sys/cpu/cpu_x86.go new file mode 100644 index 00000000000..71e288b0622 --- /dev/null +++ b/vendor/golang.org/x/sys/cpu/cpu_x86.go @@ -0,0 +1,55 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build 386 amd64 amd64p32 + +package cpu + +const cacheLineSize = 64 + +func init() { + maxID, _, _, _ := cpuid(0, 0) + + if maxID < 1 { + return + } + + _, _, ecx1, edx1 := cpuid(1, 0) + X86.HasSSE2 = isSet(26, edx1) + + X86.HasSSE3 = isSet(0, ecx1) + X86.HasPCLMULQDQ = isSet(1, ecx1) + X86.HasSSSE3 = isSet(9, ecx1) + X86.HasFMA = isSet(12, ecx1) + X86.HasSSE41 = isSet(19, ecx1) + X86.HasSSE42 = isSet(20, ecx1) + X86.HasPOPCNT = isSet(23, ecx1) + X86.HasAES = isSet(25, ecx1) + X86.HasOSXSAVE = isSet(27, ecx1) + + osSupportsAVX := false + // For XGETBV, OSXSAVE bit is required and sufficient. + if X86.HasOSXSAVE { + eax, _ := xgetbv() + // Check if XMM and YMM registers have OS support. + osSupportsAVX = isSet(1, eax) && isSet(2, eax) + } + + X86.HasAVX = isSet(28, ecx1) && osSupportsAVX + + if maxID < 7 { + return + } + + _, ebx7, _, _ := cpuid(7, 0) + X86.HasBMI1 = isSet(3, ebx7) + X86.HasAVX2 = isSet(5, ebx7) && osSupportsAVX + X86.HasBMI2 = isSet(8, ebx7) + X86.HasERMS = isSet(9, ebx7) + X86.HasADX = isSet(19, ebx7) +} + +func isSet(bitpos uint, value uint32) bool { + return value&(1< RUN set -x && \ diff --git a/x-pack/functionbeat/provider/aws/cli_manager.go b/x-pack/functionbeat/provider/aws/cli_manager.go index c70b73b2686..ef5f6f12fb9 100644 --- a/x-pack/functionbeat/provider/aws/cli_manager.go +++ b/x-pack/functionbeat/provider/aws/cli_manager.go @@ -153,7 +153,7 @@ func (c *CLIManager) template(function installer, name, templateLoc string) *clo Handler: handlerName, MemorySize: lambdaConfig.MemorySize.Megabytes(), ReservedConcurrentExecutions: lambdaConfig.Concurrency, - Timeout: int(lambdaConfig.Timeout.Seconds()), + Timeout: int(lambdaConfig.Timeout.Seconds()), }, DependsOn: []string{"IAMRoleLambdaExecution"}, } diff --git a/x-pack/metricbeat/Makefile b/x-pack/metricbeat/Makefile new file mode 100644 index 00000000000..56633e2b3e5 --- /dev/null +++ b/x-pack/metricbeat/Makefile @@ -0,0 +1,3 @@ +ES_BEATS ?= ../.. + +include $(ES_BEATS)/dev-tools/make/xpack.mk diff --git a/x-pack/metricbeat/docker-compose.yml b/x-pack/metricbeat/docker-compose.yml new file mode 100644 index 00000000000..324849d7c1d --- /dev/null +++ b/x-pack/metricbeat/docker-compose.yml @@ -0,0 +1,11 @@ +version: '2.1' +services: + beat: + build: ../../metricbeat + environment: + - TEST_ENVIRONMENT=false + working_dir: /go/src/github.com/elastic/beats/x-pack/metricbeat + volumes: + - ${PWD}/../..:/go/src/github.com/elastic/beats/ + - /var/run/docker.sock:/var/run/docker.sock + command: make diff --git a/x-pack/metricbeat/include/list.go b/x-pack/metricbeat/include/list.go new file mode 100644 index 00000000000..601cf3a1220 --- /dev/null +++ b/x-pack/metricbeat/include/list.go @@ -0,0 +1,13 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// Code generated by beats/dev-tools/module_include_list/module_include_list.go - DO NOT EDIT. + +package include + +import ( + // Import modules. + _ "github.com/elastic/beats/x-pack/metricbeat/module/foo" + _ "github.com/elastic/beats/x-pack/metricbeat/module/foo/bar" +) diff --git a/x-pack/metricbeat/magefile.go b/x-pack/metricbeat/magefile.go new file mode 100644 index 00000000000..6d54031a64a --- /dev/null +++ b/x-pack/metricbeat/magefile.go @@ -0,0 +1,270 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// +build mage + +package main + +import ( + "context" + "fmt" + "os" + "path/filepath" + "regexp" + "strings" + "time" + + "github.com/magefile/mage/mg" + + "github.com/elastic/beats/dev-tools/mage" +) + +func init() { + mage.BeatDescription = "Metricbeat is a lightweight shipper for metrics." + mage.BeatLicense = "Elastic" +} + +// Build builds the Beat binary. +func Build() error { + return mage.Build(mage.DefaultBuildArgs()) +} + +// GolangCrossBuild build the Beat binary inside of the golang-builder. +// Do not use directly, use crossBuild instead. +func GolangCrossBuild() error { + return mage.GolangCrossBuild(mage.DefaultGolangCrossBuildArgs()) +} + +// CrossBuild cross-builds the beat for all target platforms. +func CrossBuild() error { + return mage.CrossBuild() +} + +// BuildGoDaemon builds the go-daemon binary (use crossBuildGoDaemon). +func BuildGoDaemon() error { + return mage.BuildGoDaemon() +} + +// CrossBuildGoDaemon cross-builds the go-daemon binary using Docker. +func CrossBuildGoDaemon() error { + return mage.CrossBuildGoDaemon() +} + +// Clean cleans all generated files and build artifacts. +func Clean() error { + return mage.Clean() +} + +// Package packages the Beat for distribution. +// Use SNAPSHOT=true to build snapshots. +// Use PLATFORMS to control the target platforms. +// Use BEAT_VERSION_QUALIFIER to control the version qualifier. +func Package() { + start := time.Now() + defer func() { fmt.Println("package ran for", time.Since(start)) }() + + mage.LoadLocalNamedSpec("xpack") + + mg.Deps(Update, prepareModulePackaging) + mg.Deps(CrossBuild, CrossBuildGoDaemon) + mg.SerialDeps(mage.Package, TestPackages) +} + +// TestPackages tests the generated packages (i.e. file modes, owners, groups). +func TestPackages() error { + return mage.TestPackages(mage.WithModulesD()) +} + +// Fields generates a fields.yml and fields.go for each module. +func Fields() { + mg.Deps(fieldsYML, mage.GenerateModuleFieldsGo) +} + +// fieldsYML generates a fields.yml based on filebeat + x-pack/filebeat/modules. +func fieldsYML() error { + return mage.GenerateFieldsYAML(mage.OSSBeatDir("module"), "module") +} + +// Dashboards collects all the dashboards and generates index patterns. +func Dashboards() error { + return mage.KibanaDashboards(mage.OSSBeatDir("module"), "module") +} + +// Config generates both the short and reference configs. +func Config() { + mg.Deps(shortConfig, referenceConfig, createDirModulesD) +} + +// Update is an alias for running fields, dashboards, config. +func Update() { + mg.SerialDeps(Fields, Dashboards, Config, prepareModulePackaging, + mage.GenerateModuleIncludeListGo) +} + +// Fmt formats source code and adds file headers. +func Fmt() { + mg.Deps(mage.Format) +} + +// Check runs fmt and update then returns an error if any modifications are found. +func Check() { + mg.SerialDeps(mage.Format, Update, mage.Check) +} + +// IntegTest executes integration tests (it uses Docker to run the tests). +func IntegTest() { + mage.AddIntegTestUsage() + defer mage.StopIntegTestEnv() + mg.SerialDeps(GoIntegTest, PythonIntegTest) +} + +// UnitTest executes the unit tests. +func UnitTest() { + mg.SerialDeps(GoUnitTest, PythonUnitTest) +} + +// GoUnitTest executes the Go unit tests. +// Use TEST_COVERAGE=true to enable code coverage profiling. +// Use RACE_DETECTOR=true to enable the race detector. +func GoUnitTest(ctx context.Context) error { + return mage.GoTest(ctx, mage.DefaultGoTestUnitArgs()) +} + +// GoIntegTest executes the Go integration tests. +// Use TEST_COVERAGE=true to enable code coverage profiling. +// Use RACE_DETECTOR=true to enable the race detector. +func GoIntegTest(ctx context.Context) error { + return mage.RunIntegTest("goIntegTest", func() error { + return mage.GoTest(ctx, mage.DefaultGoTestIntegrationArgs()) + }) +} + +// PythonUnitTest executes the python system tests. +func PythonUnitTest() error { + mg.Deps(mage.BuildSystemTestBinary) + return mage.PythonNoseTest(mage.DefaultPythonTestUnitArgs()) +} + +// PythonIntegTest executes the python system tests in the integration environment (Docker). +func PythonIntegTest(ctx context.Context) error { + if !mage.IsInIntegTestEnv() { + mg.Deps(Fields) + } + return mage.RunIntegTest("pythonIntegTest", func() error { + mg.Deps(mage.BuildSystemTestBinary) + return mage.PythonNoseTest(mage.DefaultPythonTestIntegrationArgs()) + }) +} + +// ----------------------------------------------------------------------------- +// Customizations specific to Metricbeat. +// - Include modules.d directory in packages. + +const ( + dirModulesDGenerated = "build/package/modules.d" +) + +// prepareModulePackaging generates modules and modules.d directories +// for an x-pack distribution, excluding _meta and test files so that they are +// not included in packages. +func prepareModulePackaging() error { + mg.Deps(createDirModulesD) + + err := mage.Clean([]string{ + dirModulesDGenerated, + }) + if err != nil { + return err + } + + for _, copyAction := range []struct { + src, dst string + }{ + {mage.OSSBeatDir("modules.d"), dirModulesDGenerated}, + {"modules.d", dirModulesDGenerated}, + } { + err := (&mage.CopyTask{ + Source: copyAction.src, + Dest: copyAction.dst, + Mode: 0644, + DirMode: 0755, + }).Execute() + if err != nil { + return err + } + } + return nil +} + +func shortConfig() error { + var configParts = []string{ + mage.OSSBeatDir("_meta/common.yml"), + mage.OSSBeatDir("_meta/setup.yml"), + "{{ elastic_beats_dir }}/libbeat/_meta/config.yml", + } + + for i, f := range configParts { + configParts[i] = mage.MustExpand(f) + } + + configFile := mage.BeatName + ".yml" + mage.MustFileConcat(configFile, 0640, configParts...) + mage.MustFindReplace(configFile, regexp.MustCompile("beatname"), mage.BeatName) + mage.MustFindReplace(configFile, regexp.MustCompile("beat-index-prefix"), mage.BeatIndexPrefix) + return nil +} + +func referenceConfig() error { + const modulesConfigYml = "build/config.modules.yml" + err := mage.GenerateModuleReferenceConfig(modulesConfigYml, mage.OSSBeatDir("module"), "module") + if err != nil { + return err + } + defer os.Remove(modulesConfigYml) + + var configParts = []string{ + mage.OSSBeatDir("_meta/common.reference.yml"), + modulesConfigYml, + "{{ elastic_beats_dir }}/libbeat/_meta/config.reference.yml", + } + + for i, f := range configParts { + configParts[i] = mage.MustExpand(f) + } + + configFile := mage.BeatName + ".reference.yml" + mage.MustFileConcat(configFile, 0640, configParts...) + mage.MustFindReplace(configFile, regexp.MustCompile("beatname"), mage.BeatName) + mage.MustFindReplace(configFile, regexp.MustCompile("beat-index-prefix"), mage.BeatIndexPrefix) + return nil +} + +func createDirModulesD() error { + if err := os.RemoveAll("modules.d"); err != nil { + return err + } + + shortConfigs, err := filepath.Glob("module/*/_meta/config.yml") + if err != nil { + return err + } + + for _, f := range shortConfigs { + parts := strings.Split(filepath.ToSlash(f), "/") + if len(parts) < 2 { + continue + } + moduleName := parts[1] + + cp := mage.CopyTask{ + Source: f, + Dest: filepath.Join("modules.d", moduleName+".yml.disabled"), + Mode: 0644, + } + if err = cp.Execute(); err != nil { + return err + } + } + return nil +} diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml new file mode 100644 index 00000000000..dc716223806 --- /dev/null +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -0,0 +1,1828 @@ +########################## Metricbeat Configuration ########################### + +# This file is a full configuration example documenting all non-deprecated +# options in comments. For a shorter configuration example, that contains only +# the most common options, please see metricbeat.yml in the same directory. +# +# You can find the full configuration reference here: +# https://www.elastic.co/guide/en/beats/metricbeat/index.html + +#============================ Config Reloading =============================== + +# Config reloading allows to dynamically load modules. Each file which is +# monitored must contain one or multiple modules as a list. +metricbeat.config.modules: + + # Glob pattern for configuration reloading + path: ${path.config}/conf.d/*.yml + + # Period on which files under path should be checked for changes + reload.period: 10s + + # Set to true to enable config reloading + reload.enabled: false + +# Maximum amount of time to randomly delay the start of a metricset. Use 0 to +# disable startup delay. +metricbeat.max_start_delay: 10s + +#============================== Autodiscover =================================== + +# Autodiscover allows you to detect changes in the system and spawn new modules +# as they happen. + +#metricbeat.autodiscover: + # List of enabled autodiscover providers +# providers: +# - type: docker +# templates: +# - condition: +# equals.docker.container.image: etcd +# config: +# - module: etcd +# metricsets: ["leader", "self", "store"] +# period: 10s +# hosts: ["${host}:2379"] + +#========================== Modules configuration ============================= +metricbeat.modules: + +#-------------------------------- System Module -------------------------------- +- module: system + metricsets: + - cpu # CPU usage + - load # CPU load averages + - memory # Memory usage + - network # Network IO + - process # Per process metrics + - process_summary # Process summary + - uptime # System Uptime + #- core # Per CPU core usage + #- diskio # Disk IO + #- filesystem # File system usage for each mountpoint + #- fsstat # File system summary metrics + #- raid # Raid + #- socket # Sockets and connection info (linux only) + enabled: true + period: 10s + processes: ['.*'] + + # Configure the metric types that are included by these metricsets. + cpu.metrics: ["percentages"] # The other available options are normalized_percentages and ticks. + core.metrics: ["percentages"] # The other available option is ticks. + + # A list of filesystem types to ignore. The filesystem metricset will not + # collect data from filesystems matching any of the specified types, and + # fsstats will not include data from these filesystems in its summary stats. + # If not set, types associated to virtual filesystems are automatically + # added when this information is available in the system (e.g. the list of + # `nodev` types in `/proc/filesystem`). + #filesystem.ignore_types: [] + + # These options allow you to filter out all processes that are not + # in the top N by CPU or memory, in order to reduce the number of documents created. + # If both the `by_cpu` and `by_memory` options are used, the union of the two sets + # is included. + #process.include_top_n: + + # Set to false to disable this feature and include all processes + #enabled: true + + # How many processes to include from the top by CPU. The processes are sorted + # by the `system.process.cpu.total.pct` field. + #by_cpu: 0 + + # How many processes to include from the top by memory. The processes are sorted + # by the `system.process.memory.rss.bytes` field. + #by_memory: 0 + + # If false, cmdline of a process is not cached. + #process.cmdline.cache.enabled: true + + # Enable collection of cgroup metrics from processes on Linux. + #process.cgroups.enabled: true + + # A list of regular expressions used to whitelist environment variables + # reported with the process metricset's events. Defaults to empty. + #process.env.whitelist: [] + + # Include the cumulative CPU tick values with the process metrics. Defaults + # to false. + #process.include_cpu_ticks: false + + # Raid mount point to monitor + #raid.mount_point: '/' + + # Configure reverse DNS lookup on remote IP addresses in the socket metricset. + #socket.reverse_lookup.enabled: false + #socket.reverse_lookup.success_ttl: 60s + #socket.reverse_lookup.failure_ttl: 60s + + # Diskio configurations + #diskio.include_devices: [] + +#------------------------------ Aerospike Module ------------------------------ +- module: aerospike + metricsets: ["namespace"] + enabled: true + period: 10s + hosts: ["localhost:3000"] + +#-------------------------------- Apache Module -------------------------------- +- module: apache + metricsets: ["status"] + period: 10s + enabled: true + + # Apache hosts + hosts: ["http://127.0.0.1"] + + # Path to server status. Default server-status + #server_status_path: "server-status" + + # Username of hosts. Empty by default + #username: username + + # Password of hosts. Empty by default + #password: password + +#--------------------------------- Ceph Module --------------------------------- +- module: ceph + metricsets: ["cluster_disk", "cluster_health", "monitor_health", "pool_disk", "osd_tree"] + period: 10s + hosts: ["localhost:5000"] + enabled: true + +#------------------------------ Couchbase Module ------------------------------ +- module: couchbase + metricsets: ["bucket", "cluster", "node"] + period: 10s + hosts: ["localhost:8091"] + enabled: true + +#-------------------------------- Docker Module -------------------------------- +- module: docker + metricsets: + - "container" + - "cpu" + - "diskio" + - "healthcheck" + - "info" + #- "image" + - "memory" + - "network" + hosts: ["unix:///var/run/docker.sock"] + period: 10s + enabled: true + + # If set to true, replace dots in labels with `_`. + #labels.dedot: false + + # If set to true, collects metrics per core. + #cpu.cores: true + + # To connect to Docker over TLS you must specify a client and CA certificate. + #ssl: + #certificate_authority: "/etc/pki/root/ca.pem" + #certificate: "/etc/pki/client/cert.pem" + #key: "/etc/pki/client/cert.key" + +#------------------------------ Dropwizard Module ------------------------------ +- module: dropwizard + metricsets: ["collector"] + period: 10s + hosts: ["localhost:8080"] + metrics_path: /metrics/metrics + namespace: example + enabled: true + +#---------------------------- Elasticsearch Module ---------------------------- +- module: elasticsearch + metricsets: + - node + - node_stats + #- index + #- index_recovery + #- index_summary + #- shard + #- ml_job + period: 10s + hosts: ["http://localhost:9200"] + #username: "elastic" + #password: "changeme" + #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] + + # Set to false to fetch all entries + #index_recovery.active_only: true + +#------------------------------ Envoyproxy Module ------------------------------ +- module: envoyproxy + metricsets: ["server"] + period: 10s + hosts: ["localhost:9901"] + +#--------------------------------- Etcd Module --------------------------------- +- module: etcd + metricsets: ["leader", "self", "store"] + period: 10s + hosts: ["localhost:2379"] + +#--------------------------------- Foo Module --------------------------------- +- module: foo + metricsets: ["bar"] + enabled: false + period: 10s + hosts: ["localhost"] + + +#-------------------------------- Golang Module -------------------------------- +- module: golang + #metricsets: + # - expvar + # - heap + period: 10s + hosts: ["localhost:6060"] + heap.path: "/debug/vars" + expvar: + namespace: "example" + path: "/debug/vars" + +#------------------------------- Graphite Module ------------------------------- +- module: graphite + metricsets: ["server"] + enabled: true + + # Host address to listen on. Default localhost. + #host: localhost + + # Listening port. Default 2003. + #port: 2003 + + # Protocol to listen on. This can be udp or tcp. Default udp. + #protocol: "udp" + + # Receive buffer size in bytes + #receive_buffer_size: 1024 + + #templates: + # - filter: "test.*.bash.*" # This would match metrics like test.localhost.bash.stats + # namespace: "test" + # template: ".host.shell.metric*" # test.localhost.bash.stats would become metric=stats and tags host=localhost,shell=bash + # delimiter: "_" + + +#------------------------------- HAProxy Module ------------------------------- +- module: haproxy + metricsets: ["info", "stat"] + period: 10s + hosts: ["tcp://127.0.0.1:14567"] + enabled: true + +#--------------------------------- HTTP Module --------------------------------- +- module: http + #metricsets: + # - json + period: 10s + hosts: ["localhost:80"] + namespace: "json_namespace" + path: "/" + #body: "" + #method: "GET" + #username: "user" + #password: "secret" + #request.enabled: false + #response.enabled: false + #json.is_array: false + #dedot.enabled: false + +- module: http + #metricsets: + # - server + host: "localhost" + port: "8080" + enabled: false + #paths: + # - path: "/foo" + # namespace: "foo" + # fields: # added to the the response in root. overwrites existing fields + # key: "value" + +#------------------------------- Jolokia Module ------------------------------- +- module: jolokia + #metricsets: ["jmx"] + period: 10s + hosts: ["localhost"] + namespace: "metrics" + #path: "/jolokia/?ignoreErrors=true&canonicalNaming=false" + #username: "user" + #password: "secret" + jmx.mappings: + #- mbean: 'java.lang:type=Runtime' + # attributes: + # - attr: Uptime + # field: uptime + #- mbean: 'java.lang:type=Memory' + # attributes: + # - attr: HeapMemoryUsage + # field: memory.heap_usage + # - attr: NonHeapMemoryUsage + # field: memory.non_heap_usage + # GC Metrics - this depends on what is available on your JVM + #- mbean: 'java.lang:type=GarbageCollector,name=ConcurrentMarkSweep' + # attributes: + # - attr: CollectionTime + # field: gc.cms_collection_time + # - attr: CollectionCount + # field: gc.cms_collection_count + + jmx.application: + jmx.instance: + +#-------------------------------- Kafka Module -------------------------------- +- module: kafka + metricsets: ["consumergroup", "partition"] + period: 10s + hosts: ["localhost:9092"] + enabled: true + + #client_id: metricbeat + #retries: 3 + #backoff: 250ms + + # List of Topics to query metadata for. If empty, all topics will be queried. + #topics: [] + + # Optional SSL. By default is off. + # List of root certificates for HTTPS server verifications + #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] + + # Certificate for SSL client authentication + #ssl.certificate: "/etc/pki/client/cert.pem" + + # Client Certificate Key + #ssl.key: "/etc/pki/client/cert.key" + + # SASL authentication + #username: "" + #password: "" + +#-------------------------------- Kibana Module -------------------------------- +- module: kibana + metricsets: ["status"] + period: 10s + hosts: ["localhost:5601"] + basepath: "" + enabled: true + +#------------------------------ Kubernetes Module ------------------------------ +# Node metrics, from kubelet: +- module: kubernetes + metricsets: + - container + - node + - pod + - system + - volume + period: 10s + hosts: ["localhost:10255"] + enabled: true + #bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + #ssl.certificate_authorities: + # - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt + #ssl.certificate: "/etc/pki/client/cert.pem" + #ssl.key: "/etc/pki/client/cert.key" + + # Enriching parameters: + add_metadata: true + in_cluster: true + # When used outside the cluster: + #host: node_name + #kube_config: ~/.kube/config + +# State metrics from kube-state-metrics service: +- module: kubernetes + enabled: true + metricsets: + - state_node + - state_deployment + - state_replicaset + - state_statefulset + - state_pod + - state_container + period: 10s + hosts: ["kube-state-metrics:8080"] + + # Enriching parameters: + add_metadata: true + in_cluster: true + # When used outside the cluster: + #host: node_name + #kube_config: ~/.kube/config + +# Kubernetes events +- module: kubernetes + enabled: true + metricsets: + - event + +# Kubernetes API server +- module: kubernetes + enabled: true + metricsets: + - apiserver + hosts: ["https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT}"] + +#--------------------------------- Kvm Module --------------------------------- +- module: kvm + metricsets: ["dommemstat"] + enabled: true + period: 10s + hosts: ["unix:///var/run/libvirt/libvirt-sock"] + # For remote hosts, setup network access in libvirtd.conf + # and use the tcp scheme: + # hosts: [ "tcp://:16509" ] + + # Timeout to connect to Libvirt server + #timeout: 1s + +#------------------------------- Logstash Module ------------------------------- +- module: logstash + metricsets: ["node", "node_stats"] + enabled: true + period: 10s + hosts: ["localhost:9600"] + +#------------------------------ Memcached Module ------------------------------ +- module: memcached + metricsets: ["stats"] + period: 10s + hosts: ["localhost:11211"] + enabled: true + +#------------------------------- MongoDB Module ------------------------------- +- module: mongodb + metricsets: ["dbstats", "status", "collstats", "metrics", "replstatus"] + period: 10s + enabled: true + + # The hosts must be passed as MongoDB URLs in the format: + # [mongodb://][user:pass@]host[:port]. + # The username and password can also be set using the respective configuration + # options. The credentials in the URL take precedence over the username and + # password configuration options. + hosts: ["localhost:27017"] + + # Optional SSL. By default is off. + #ssl.enabled: true + + # Mode of verification of server certificate ('none' or 'full') + #ssl.verification_mode: 'full' + + # List of root certificates for TLS server verifications + #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] + + # Certificate for SSL client authentication + #ssl.certificate: "/etc/pki/client/cert.pem" + + # Client Certificate Key + #ssl.key: "/etc/pki/client/cert.key" + + # Username to use when connecting to MongoDB. Empty by default. + #username: user + + # Password to use when connecting to MongoDB. Empty by default. + #password: pass + +#-------------------------------- Munin Module -------------------------------- +- module: munin + metricsets: ["node"] + enabled: true + period: 10s + hosts: ["localhost:4949"] + node.namespace: node + +#-------------------------------- MySQL Module -------------------------------- +- module: mysql + metricsets: + - "status" + # - "galera_status" + period: 10s + + # Host DSN should be defined as "user:pass@tcp(127.0.0.1:3306)/" + # The username and password can either be set in the DSN or using the username + # and password config options. Those specified in the DSN take precedence. + hosts: ["root:secret@tcp(127.0.0.1:3306)/"] + + # Username of hosts. Empty by default. + #username: root + + # Password of hosts. Empty by default. + #password: secret + + # By setting raw to true, all raw fields from the status metricset will be added to the event. + #raw: false + +#-------------------------------- Nginx Module -------------------------------- +- module: nginx + metricsets: ["stubstatus"] + enabled: true + period: 10s + + # Nginx hosts + hosts: ["http://127.0.0.1"] + + # Path to server status. Default server-status + server_status_path: "server-status" + +#------------------------------- PHP_FPM Module ------------------------------- +- module: php_fpm + metricsets: + - pool + #- process + enabled: true + period: 10s + status_path: "/status" + hosts: ["localhost:8080"] + +#------------------------------ PostgreSQL Module ------------------------------ +- module: postgresql + enabled: true + metricsets: + # Stats about every PostgreSQL database + - database + + # Stats about the background writer process's activity + - bgwriter + + # Stats about every PostgreSQL process + - activity + + period: 10s + + # The host must be passed as PostgreSQL URL. Example: + # postgres://localhost:5432?sslmode=disable + # The available parameters are documented here: + # https://godoc.org/github.com/lib/pq#hdr-Connection_String_Parameters + hosts: ["postgres://localhost:5432"] + + # Username to use when connecting to PostgreSQL. Empty by default. + #username: user + + # Password to use when connecting to PostgreSQL. Empty by default. + #password: pass + +#------------------------------ Prometheus Module ------------------------------ +- module: prometheus + metricsets: ["stats"] + enabled: true + period: 10s + hosts: ["localhost:9090"] + #metrics_path: /metrics + #namespace: example + +- module: prometheus + metricsets: ["collector"] + enabled: true + period: 10s + hosts: ["localhost:9090"] + #metrics_path: /metrics + #namespace: example + + # This can be used for service account based authorization: + # bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + #ssl.certificate_authorities: + # - /var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt + +#------------------------------- RabbitMQ Module ------------------------------- +- module: rabbitmq + metricsets: ["node", "queue", "connection"] + enabled: true + period: 10s + hosts: ["localhost:15672"] + + # Management path prefix, if `management.path_prefix` is set in RabbitMQ + # configuration, it has to be set to the same value. + #management_path_prefix: "" + + #username: guest + #password: guest + +#-------------------------------- Redis Module -------------------------------- +- module: redis + metricsets: ["info", "keyspace"] + enabled: true + period: 10s + + # Redis hosts + hosts: ["127.0.0.1:6379"] + + # Timeout after which time a metricset should return an error + # Timeout is by default defined as period, as a fetch of a metricset + # should never take longer then period, as otherwise calls can pile up. + #timeout: 1s + + # Optional fields to be added to each event + #fields: + # datacenter: west + + # Network type to be used for redis connection. Default: tcp + #network: tcp + + # Max number of concurrent connections. Default: 10 + #maxconn: 10 + + # Filters can be used to reduce the number of fields sent. + #processors: + # - include_fields: + # fields: ["beat", "metricset", "redis.info.stats"] + + # Redis AUTH password. Empty by default. + #password: foobared + +#------------------------------- Traefik Module ------------------------------- +- module: traefik + metricsets: ["health"] + period: 10s + hosts: ["localhost:8080"] + +#-------------------------------- Uwsgi Module -------------------------------- +- module: uwsgi + metricsets: ["status"] + enable: true + period: 10s + hosts: ["tcp://127.0.0.1:9191"] + +#------------------------------- VSphere Module ------------------------------- +- module: vsphere + enabled: true + metricsets: ["datastore", "host", "virtualmachine"] + period: 10s + hosts: ["https://localhost/sdk"] + + username: "user" + password: "password" + # If insecure is true, don't verify the server's certificate chain + insecure: false + # Get custom fields when using virtualmachine metric set. Default false. + # get_custom_fields: false + +#------------------------------- Windows Module ------------------------------- +- module: windows + metricsets: ["perfmon"] + enabled: true + period: 10s + perfmon.ignore_non_existent_counters: true + perfmon.counters: + # - instance_label: processor.name + # instance_name: total + # measurement_label: processor.time.total.pct + # query: '\Processor Information(_Total)\% Processor Time' + +- module: windows + metricsets: ["service"] + enabled: true + period: 60s + +#------------------------------ ZooKeeper Module ------------------------------ +- module: zookeeper + enabled: true + metricsets: ["mntr"] + period: 10s + hosts: ["localhost:2181"] + + + +#================================ General ====================================== + +# The name of the shipper that publishes the network data. It can be used to group +# all the transactions sent by a single shipper in the web interface. +# If this options is not defined, the hostname is used. +#name: + +# The tags of the shipper are included in their own field with each +# transaction published. Tags make it easy to group servers by different +# logical properties. +#tags: ["service-X", "web-tier"] + +# Optional fields that you can specify to add additional information to the +# output. Fields can be scalar values, arrays, dictionaries, or any nested +# combination of these. +#fields: +# env: staging + +# If this option is set to true, the custom fields are stored as top-level +# fields in the output document instead of being grouped under a fields +# sub-dictionary. Default is false. +#fields_under_root: false + +# Internal queue configuration for buffering events to be published. +#queue: + # Queue type by name (default 'mem') + # The memory queue will present all available events (up to the outputs + # bulk_max_size) to the output, the moment the output is ready to server + # another batch of events. + #mem: + # Max number of events the queue can buffer. + #events: 4096 + + # Hints the minimum number of events stored in the queue, + # before providing a batch of events to the outputs. + # The default value is set to 2048. + # A value of 0 ensures events are immediately available + # to be sent to the outputs. + #flush.min_events: 2048 + + # Maximum duration after which events are available to the outputs, + # if the number of events stored in the queue is < min_flush_events. + #flush.timeout: 1s + + # The spool queue will store events in a local spool file, before + # forwarding the events to the outputs. + # + # Beta: spooling to disk is currently a beta feature. Use with care. + # + # The spool file is a circular buffer, which blocks once the file/buffer is full. + # Events are put into a write buffer and flushed once the write buffer + # is full or the flush_timeout is triggered. + # Once ACKed by the output, events are removed immediately from the queue, + # making space for new events to be persisted. + #spool: + # The file namespace configures the file path and the file creation settings. + # Once the file exists, the `size`, `page_size` and `prealloc` settings + # will have no more effect. + #file: + # Location of spool file. The default value is ${path.data}/spool.dat. + #path: "${path.data}/spool.dat" + + # Configure file permissions if file is created. The default value is 0600. + #permissions: 0600 + + # File size hint. The spool blocks, once this limit is reached. The default value is 100 MiB. + #size: 100MiB + + # The files page size. A file is split into multiple pages of the same size. The default value is 4KiB. + #page_size: 4KiB + + # If prealloc is set, the required space for the file is reserved using + # truncate. The default value is true. + #prealloc: true + + # Spool writer settings + # Events are serialized into a write buffer. The write buffer is flushed if: + # - The buffer limit has been reached. + # - The configured limit of buffered events is reached. + # - The flush timeout is triggered. + #write: + # Sets the write buffer size. + #buffer_size: 1MiB + + # Maximum duration after which events are flushed, if the write buffer + # is not full yet. The default value is 1s. + #flush.timeout: 1s + + # Number of maximum buffered events. The write buffer is flushed once the + # limit is reached. + #flush.events: 16384 + + # Configure the on-disk event encoding. The encoding can be changed + # between restarts. + # Valid encodings are: json, ubjson, and cbor. + #codec: cbor + #read: + # Reader flush timeout, waiting for more events to become available, so + # to fill a complete batch, as required by the outputs. + # If flush_timeout is 0, all available events are forwarded to the + # outputs immediately. + # The default value is 0s. + #flush.timeout: 0s + +# Sets the maximum number of CPUs that can be executing simultaneously. The +# default is the number of logical CPUs available in the system. +#max_procs: + +#================================ Processors =================================== + +# Processors are used to reduce the number of fields in the exported event or to +# enhance the event with external metadata. This section defines a list of +# processors that are applied one by one and the first one receives the initial +# event: +# +# event -> filter1 -> event1 -> filter2 ->event2 ... +# +# The supported processors are drop_fields, drop_event, include_fields, +# decode_json_fields, and add_cloud_metadata. +# +# For example, you can use the following processors to keep the fields that +# contain CPU load percentages, but remove the fields that contain CPU ticks +# values: +# +#processors: +#- include_fields: +# fields: ["cpu"] +#- drop_fields: +# fields: ["cpu.user", "cpu.system"] +# +# The following example drops the events that have the HTTP response code 200: +# +#processors: +#- drop_event: +# when: +# equals: +# http.code: 200 +# +# The following example renames the field a to b: +# +#processors: +#- rename: +# fields: +# - from: "a" +# to: "b" +# +# The following example tokenizes the string into fields: +# +#processors: +#- dissect: +# tokenizer: "%{key1} - %{key2}" +# field: "message" +# target_prefix: "dissect" +# +# The following example enriches each event with metadata from the cloud +# provider about the host machine. It works on EC2, GCE, DigitalOcean, +# Tencent Cloud, and Alibaba Cloud. +# +#processors: +#- add_cloud_metadata: ~ +# +# The following example enriches each event with the machine's local time zone +# offset from UTC. +# +#processors: +#- add_locale: +# format: offset +# +# The following example enriches each event with docker metadata, it matches +# given fields to an existing container id and adds info from that container: +# +#processors: +#- add_docker_metadata: +# host: "unix:///var/run/docker.sock" +# match_fields: ["system.process.cgroup.id"] +# match_pids: ["process.pid", "process.ppid"] +# match_source: true +# match_source_index: 4 +# match_short_id: false +# cleanup_timeout: 60 +# # To connect to Docker over TLS you must specify a client and CA certificate. +# #ssl: +# # certificate_authority: "/etc/pki/root/ca.pem" +# # certificate: "/etc/pki/client/cert.pem" +# # key: "/etc/pki/client/cert.key" +# +# The following example enriches each event with docker metadata, it matches +# container id from log path available in `source` field (by default it expects +# it to be /var/lib/docker/containers/*/*.log). +# +#processors: +#- add_docker_metadata: ~ +# +# The following example enriches each event with host metadata. +# +#processors: +#- add_host_metadata: +# netinfo.enabled: false +# +# The following example enriches each event with process metadata using +# process IDs included in the event. +# +#processors: +#- add_process_metadata: +# match_pids: ["system.process.ppid"] +# target: system.process.parent +# +# The following example decodes fields containing JSON strings +# and replaces the strings with valid JSON objects. +# +#processors: +#- decode_json_fields: +# fields: ["field1", "field2", ...] +# process_array: false +# max_depth: 1 +# target: "" +# overwrite_keys: false + +#============================= Elastic Cloud ================================== + +# These settings simplify using metricbeat with the Elastic Cloud (https://cloud.elastic.co/). + +# The cloud.id setting overwrites the `output.elasticsearch.hosts` and +# `setup.kibana.host` options. +# You can find the `cloud.id` in the Elastic Cloud web UI. +#cloud.id: + +# The cloud.auth setting overwrites the `output.elasticsearch.username` and +# `output.elasticsearch.password` settings. The format is `:`. +#cloud.auth: + +#================================ Outputs ====================================== + +# Configure what output to use when sending the data collected by the beat. + +#-------------------------- Elasticsearch output ------------------------------- +output.elasticsearch: + # Boolean flag to enable or disable the output module. + #enabled: true + + # Array of hosts to connect to. + # Scheme and port can be left out and will be set to the default (http and 9200) + # In case you specify and additional path, the scheme is required: http://localhost:9200/path + # IPv6 addresses should always be defined as: https://[2001:db8::1]:9200 + hosts: ["localhost:9200"] + + # Set gzip compression level. + #compression_level: 0 + + # Configure escaping html symbols in strings. + #escape_html: true + + # Optional protocol and basic auth credentials. + #protocol: "https" + #username: "elastic" + #password: "changeme" + + # Dictionary of HTTP parameters to pass within the url with index operations. + #parameters: + #param1: value1 + #param2: value2 + + # Number of workers per Elasticsearch host. + #worker: 1 + + # Optional index name. The default is "metricbeat" plus date + # and generates [metricbeat-]YYYY.MM.DD keys. + # In case you modify this pattern you must update setup.template.name and setup.template.pattern accordingly. + #index: "metricbeat-%{[beat.version]}-%{+yyyy.MM.dd}" + + # Optional ingest node pipeline. By default no pipeline will be used. + #pipeline: "" + + # Optional HTTP Path + #path: "/elasticsearch" + + # Custom HTTP headers to add to each request + #headers: + # X-My-Header: Contents of the header + + # Proxy server url + #proxy_url: http://proxy:3128 + + # The number of times a particular Elasticsearch index operation is attempted. If + # the indexing operation doesn't succeed after this many retries, the events are + # dropped. The default is 3. + #max_retries: 3 + + # The maximum number of events to bulk in a single Elasticsearch bulk API index request. + # The default is 50. + #bulk_max_size: 50 + + # The number of seconds to wait before trying to reconnect to Elasticsearch + # after a network error. After waiting backoff.init seconds, the Beat + # tries to reconnect. If the attempt fails, the backoff timer is increased + # exponentially up to backoff.max. After a successful connection, the backoff + # timer is reset. The default is 1s. + #backoff.init: 1s + + # The maximum number of seconds to wait before attempting to connect to + # Elasticsearch after a network error. The default is 60s. + #backoff.max: 60s + + # Configure http request timeout before failing a request to Elasticsearch. + #timeout: 90 + + # Use SSL settings for HTTPS. + #ssl.enabled: true + + # Configure SSL verification mode. If `none` is configured, all server hosts + # and certificates will be accepted. In this mode, SSL based connections are + # susceptible to man-in-the-middle attacks. Use only for testing. Default is + # `full`. + #ssl.verification_mode: full + + # List of supported/valid TLS versions. By default all TLS versions 1.0 up to + # 1.2 are enabled. + #ssl.supported_protocols: [TLSv1.0, TLSv1.1, TLSv1.2] + + # SSL configuration. By default is off. + # List of root certificates for HTTPS server verifications + #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] + + # Certificate for SSL client authentication + #ssl.certificate: "/etc/pki/client/cert.pem" + + # Client Certificate Key + #ssl.key: "/etc/pki/client/cert.key" + + # Optional passphrase for decrypting the Certificate Key. + #ssl.key_passphrase: '' + + # Configure cipher suites to be used for SSL connections + #ssl.cipher_suites: [] + + # Configure curve types for ECDHE based cipher suites + #ssl.curve_types: [] + + # Configure what types of renegotiation are supported. Valid options are + # never, once, and freely. Default is never. + #ssl.renegotiation: never + + +#----------------------------- Logstash output --------------------------------- +#output.logstash: + # Boolean flag to enable or disable the output module. + #enabled: true + + # The Logstash hosts + #hosts: ["localhost:5044"] + + # Number of workers per Logstash host. + #worker: 1 + + # Set gzip compression level. + #compression_level: 3 + + # Configure escaping html symbols in strings. + #escape_html: true + + # Optional maximum time to live for a connection to Logstash, after which the + # connection will be re-established. A value of `0s` (the default) will + # disable this feature. + # + # Not yet supported for async connections (i.e. with the "pipelining" option set) + #ttl: 30s + + # Optional load balance the events between the Logstash hosts. Default is false. + #loadbalance: false + + # Number of batches to be sent asynchronously to Logstash while processing + # new batches. + #pipelining: 2 + + # If enabled only a subset of events in a batch of events is transferred per + # transaction. The number of events to be sent increases up to `bulk_max_size` + # if no error is encountered. + #slow_start: false + + # The number of seconds to wait before trying to reconnect to Logstash + # after a network error. After waiting backoff.init seconds, the Beat + # tries to reconnect. If the attempt fails, the backoff timer is increased + # exponentially up to backoff.max. After a successful connection, the backoff + # timer is reset. The default is 1s. + #backoff.init: 1s + + # The maximum number of seconds to wait before attempting to connect to + # Logstash after a network error. The default is 60s. + #backoff.max: 60s + + # Optional index name. The default index name is set to metricbeat + # in all lowercase. + #index: 'metricbeat' + + # SOCKS5 proxy server URL + #proxy_url: socks5://user:password@socks5-server:2233 + + # Resolve names locally when using a proxy server. Defaults to false. + #proxy_use_local_resolver: false + + # Enable SSL support. SSL is automatically enabled, if any SSL setting is set. + #ssl.enabled: true + + # Configure SSL verification mode. If `none` is configured, all server hosts + # and certificates will be accepted. In this mode, SSL based connections are + # susceptible to man-in-the-middle attacks. Use only for testing. Default is + # `full`. + #ssl.verification_mode: full + + # List of supported/valid TLS versions. By default all TLS versions 1.0 up to + # 1.2 are enabled. + #ssl.supported_protocols: [TLSv1.0, TLSv1.1, TLSv1.2] + + # Optional SSL configuration options. SSL is off by default. + # List of root certificates for HTTPS server verifications + #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] + + # Certificate for SSL client authentication + #ssl.certificate: "/etc/pki/client/cert.pem" + + # Client Certificate Key + #ssl.key: "/etc/pki/client/cert.key" + + # Optional passphrase for decrypting the Certificate Key. + #ssl.key_passphrase: '' + + # Configure cipher suites to be used for SSL connections + #ssl.cipher_suites: [] + + # Configure curve types for ECDHE based cipher suites + #ssl.curve_types: [] + + # Configure what types of renegotiation are supported. Valid options are + # never, once, and freely. Default is never. + #ssl.renegotiation: never + + # The number of times to retry publishing an event after a publishing failure. + # After the specified number of retries, the events are typically dropped. + # Some Beats, such as Filebeat and Winlogbeat, ignore the max_retries setting + # and retry until all events are published. Set max_retries to a value less + # than 0 to retry until all events are published. The default is 3. + #max_retries: 3 + + # The maximum number of events to bulk in a single Logstash request. The + # default is 2048. + #bulk_max_size: 2048 + + # The number of seconds to wait for responses from the Logstash server before + # timing out. The default is 30s. + #timeout: 30s + +#------------------------------- Kafka output ---------------------------------- +#output.kafka: + # Boolean flag to enable or disable the output module. + #enabled: true + + # The list of Kafka broker addresses from where to fetch the cluster metadata. + # The cluster metadata contain the actual Kafka brokers events are published + # to. + #hosts: ["localhost:9092"] + + # The Kafka topic used for produced events. The setting can be a format string + # using any event field. To set the topic from document type use `%{[type]}`. + #topic: beats + + # The Kafka event key setting. Use format string to create unique event key. + # By default no event key will be generated. + #key: '' + + # The Kafka event partitioning strategy. Default hashing strategy is `hash` + # using the `output.kafka.key` setting or randomly distributes events if + # `output.kafka.key` is not configured. + #partition.hash: + # If enabled, events will only be published to partitions with reachable + # leaders. Default is false. + #reachable_only: false + + # Configure alternative event field names used to compute the hash value. + # If empty `output.kafka.key` setting will be used. + # Default value is empty list. + #hash: [] + + # Authentication details. Password is required if username is set. + #username: '' + #password: '' + + # Kafka version metricbeat is assumed to run against. Defaults to the "1.0.0". + #version: '1.0.0' + + # Configure JSON encoding + #codec.json: + # Pretty print json event + #pretty: false + + # Configure escaping html symbols in strings. + #escape_html: true + + # Metadata update configuration. Metadata do contain leader information + # deciding which broker to use when publishing. + #metadata: + # Max metadata request retry attempts when cluster is in middle of leader + # election. Defaults to 3 retries. + #retry.max: 3 + + # Waiting time between retries during leader elections. Default is 250ms. + #retry.backoff: 250ms + + # Refresh metadata interval. Defaults to every 10 minutes. + #refresh_frequency: 10m + + # The number of concurrent load-balanced Kafka output workers. + #worker: 1 + + # The number of times to retry publishing an event after a publishing failure. + # After the specified number of retries, the events are typically dropped. + # Some Beats, such as Filebeat, ignore the max_retries setting and retry until + # all events are published. Set max_retries to a value less than 0 to retry + # until all events are published. The default is 3. + #max_retries: 3 + + # The maximum number of events to bulk in a single Kafka request. The default + # is 2048. + #bulk_max_size: 2048 + + # The number of seconds to wait for responses from the Kafka brokers before + # timing out. The default is 30s. + #timeout: 30s + + # The maximum duration a broker will wait for number of required ACKs. The + # default is 10s. + #broker_timeout: 10s + + # The number of messages buffered for each Kafka broker. The default is 256. + #channel_buffer_size: 256 + + # The keep-alive period for an active network connection. If 0s, keep-alives + # are disabled. The default is 0 seconds. + #keep_alive: 0 + + # Sets the output compression codec. Must be one of none, snappy and gzip. The + # default is gzip. + #compression: gzip + + # Set the compression level. Currently only gzip provides a compression level + # between 0 and 9. The default value is chosen by the compression algorithm. + #compression_level: 4 + + # The maximum permitted size of JSON-encoded messages. Bigger messages will be + # dropped. The default value is 1000000 (bytes). This value should be equal to + # or less than the broker's message.max.bytes. + #max_message_bytes: 1000000 + + # The ACK reliability level required from broker. 0=no response, 1=wait for + # local commit, -1=wait for all replicas to commit. The default is 1. Note: + # If set to 0, no ACKs are returned by Kafka. Messages might be lost silently + # on error. + #required_acks: 1 + + # The configurable ClientID used for logging, debugging, and auditing + # purposes. The default is "beats". + #client_id: beats + + # Enable SSL support. SSL is automatically enabled, if any SSL setting is set. + #ssl.enabled: true + + # Optional SSL configuration options. SSL is off by default. + # List of root certificates for HTTPS server verifications + #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] + + # Configure SSL verification mode. If `none` is configured, all server hosts + # and certificates will be accepted. In this mode, SSL based connections are + # susceptible to man-in-the-middle attacks. Use only for testing. Default is + # `full`. + #ssl.verification_mode: full + + # List of supported/valid TLS versions. By default all TLS versions 1.0 up to + # 1.2 are enabled. + #ssl.supported_protocols: [TLSv1.0, TLSv1.1, TLSv1.2] + + # Certificate for SSL client authentication + #ssl.certificate: "/etc/pki/client/cert.pem" + + # Client Certificate Key + #ssl.key: "/etc/pki/client/cert.key" + + # Optional passphrase for decrypting the Certificate Key. + #ssl.key_passphrase: '' + + # Configure cipher suites to be used for SSL connections + #ssl.cipher_suites: [] + + # Configure curve types for ECDHE based cipher suites + #ssl.curve_types: [] + + # Configure what types of renegotiation are supported. Valid options are + # never, once, and freely. Default is never. + #ssl.renegotiation: never + +#------------------------------- Redis output ---------------------------------- +#output.redis: + # Boolean flag to enable or disable the output module. + #enabled: true + + # Configure JSON encoding + #codec.json: + # Pretty print json event + #pretty: false + + # Configure escaping html symbols in strings. + #escape_html: true + + # The list of Redis servers to connect to. If load balancing is enabled, the + # events are distributed to the servers in the list. If one server becomes + # unreachable, the events are distributed to the reachable servers only. + #hosts: ["localhost:6379"] + + # The Redis port to use if hosts does not contain a port number. The default + # is 6379. + #port: 6379 + + # The name of the Redis list or channel the events are published to. The + # default is metricbeat. + #key: metricbeat + + # The password to authenticate with. The default is no authentication. + #password: + + # The Redis database number where the events are published. The default is 0. + #db: 0 + + # The Redis data type to use for publishing events. If the data type is list, + # the Redis RPUSH command is used. If the data type is channel, the Redis + # PUBLISH command is used. The default value is list. + #datatype: list + + # The number of workers to use for each host configured to publish events to + # Redis. Use this setting along with the loadbalance option. For example, if + # you have 2 hosts and 3 workers, in total 6 workers are started (3 for each + # host). + #worker: 1 + + # If set to true and multiple hosts or workers are configured, the output + # plugin load balances published events onto all Redis hosts. If set to false, + # the output plugin sends all events to only one host (determined at random) + # and will switch to another host if the currently selected one becomes + # unreachable. The default value is true. + #loadbalance: true + + # The Redis connection timeout in seconds. The default is 5 seconds. + #timeout: 5s + + # The number of times to retry publishing an event after a publishing failure. + # After the specified number of retries, the events are typically dropped. + # Some Beats, such as Filebeat, ignore the max_retries setting and retry until + # all events are published. Set max_retries to a value less than 0 to retry + # until all events are published. The default is 3. + #max_retries: 3 + + # The number of seconds to wait before trying to reconnect to Redis + # after a network error. After waiting backoff.init seconds, the Beat + # tries to reconnect. If the attempt fails, the backoff timer is increased + # exponentially up to backoff.max. After a successful connection, the backoff + # timer is reset. The default is 1s. + #backoff.init: 1s + + # The maximum number of seconds to wait before attempting to connect to + # Redis after a network error. The default is 60s. + #backoff.max: 60s + + # The maximum number of events to bulk in a single Redis request or pipeline. + # The default is 2048. + #bulk_max_size: 2048 + + # The URL of the SOCKS5 proxy to use when connecting to the Redis servers. The + # value must be a URL with a scheme of socks5://. + #proxy_url: + + # This option determines whether Redis hostnames are resolved locally when + # using a proxy. The default value is false, which means that name resolution + # occurs on the proxy server. + #proxy_use_local_resolver: false + + # Enable SSL support. SSL is automatically enabled, if any SSL setting is set. + #ssl.enabled: true + + # Configure SSL verification mode. If `none` is configured, all server hosts + # and certificates will be accepted. In this mode, SSL based connections are + # susceptible to man-in-the-middle attacks. Use only for testing. Default is + # `full`. + #ssl.verification_mode: full + + # List of supported/valid TLS versions. By default all TLS versions 1.0 up to + # 1.2 are enabled. + #ssl.supported_protocols: [TLSv1.0, TLSv1.1, TLSv1.2] + + # Optional SSL configuration options. SSL is off by default. + # List of root certificates for HTTPS server verifications + #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] + + # Certificate for SSL client authentication + #ssl.certificate: "/etc/pki/client/cert.pem" + + # Client Certificate Key + #ssl.key: "/etc/pki/client/cert.key" + + # Optional passphrase for decrypting the Certificate Key. + #ssl.key_passphrase: '' + + # Configure cipher suites to be used for SSL connections + #ssl.cipher_suites: [] + + # Configure curve types for ECDHE based cipher suites + #ssl.curve_types: [] + + # Configure what types of renegotiation are supported. Valid options are + # never, once, and freely. Default is never. + #ssl.renegotiation: never + +#------------------------------- File output ----------------------------------- +#output.file: + # Boolean flag to enable or disable the output module. + #enabled: true + + # Configure JSON encoding + #codec.json: + # Pretty print json event + #pretty: false + + # Configure escaping html symbols in strings. + #escape_html: true + + # Path to the directory where to save the generated files. The option is + # mandatory. + #path: "/tmp/metricbeat" + + # Name of the generated files. The default is `metricbeat` and it generates + # files: `metricbeat`, `metricbeat.1`, `metricbeat.2`, etc. + #filename: metricbeat + + # Maximum size in kilobytes of each file. When this size is reached, and on + # every metricbeat restart, the files are rotated. The default value is 10240 + # kB. + #rotate_every_kb: 10000 + + # Maximum number of files under path. When this number of files is reached, + # the oldest file is deleted and the rest are shifted from last to first. The + # default is 7 files. + #number_of_files: 7 + + # Permissions to use for file creation. The default is 0600. + #permissions: 0600 + + +#----------------------------- Console output --------------------------------- +#output.console: + # Boolean flag to enable or disable the output module. + #enabled: true + + # Configure JSON encoding + #codec.json: + # Pretty print json event + #pretty: false + + # Configure escaping html symbols in strings. + #escape_html: true + +#================================= Paths ====================================== + +# The home path for the metricbeat installation. This is the default base path +# for all other path settings and for miscellaneous files that come with the +# distribution (for example, the sample dashboards). +# If not set by a CLI flag or in the configuration file, the default for the +# home path is the location of the binary. +#path.home: + +# The configuration path for the metricbeat installation. This is the default +# base path for configuration files, including the main YAML configuration file +# and the Elasticsearch template file. If not set by a CLI flag or in the +# configuration file, the default for the configuration path is the home path. +#path.config: ${path.home} + +# The data path for the metricbeat installation. This is the default base path +# for all the files in which metricbeat needs to store its data. If not set by a +# CLI flag or in the configuration file, the default for the data path is a data +# subdirectory inside the home path. +#path.data: ${path.home}/data + +# The logs path for a metricbeat installation. This is the default location for +# the Beat's log files. If not set by a CLI flag or in the configuration file, +# the default for the logs path is a logs subdirectory inside the home path. +#path.logs: ${path.home}/logs + +#================================ Keystore ========================================== +# Location of the Keystore containing the keys and their sensitive values. +#keystore.path: "${path.config}/beats.keystore" + +#============================== Dashboards ===================================== +# These settings control loading the sample dashboards to the Kibana index. Loading +# the dashboards are disabled by default and can be enabled either by setting the +# options here, or by using the `-setup` CLI flag or the `setup` command. +#setup.dashboards.enabled: false + +# The directory from where to read the dashboards. The default is the `kibana` +# folder in the home path. +#setup.dashboards.directory: ${path.home}/kibana + +# The URL from where to download the dashboards archive. It is used instead of +# the directory if it has a value. +#setup.dashboards.url: + +# The file archive (zip file) from where to read the dashboards. It is used instead +# of the directory when it has a value. +#setup.dashboards.file: + +# In case the archive contains the dashboards from multiple Beats, this lets you +# select which one to load. You can load all the dashboards in the archive by +# setting this to the empty string. +#setup.dashboards.beat: metricbeat + +# The name of the Kibana index to use for setting the configuration. Default is ".kibana" +#setup.dashboards.kibana_index: .kibana + +# The Elasticsearch index name. This overwrites the index name defined in the +# dashboards and index pattern. Example: testbeat-* +#setup.dashboards.index: + +# Always use the Kibana API for loading the dashboards instead of autodetecting +# how to install the dashboards by first querying Elasticsearch. +#setup.dashboards.always_kibana: false + +# If true and Kibana is not reachable at the time when dashboards are loaded, +# it will retry to reconnect to Kibana instead of exiting with an error. +#setup.dashboards.retry.enabled: false + +# Duration interval between Kibana connection retries. +#setup.dashboards.retry.interval: 1s + +# Maximum number of retries before exiting with an error, 0 for unlimited retrying. +#setup.dashboards.retry.maximum: 0 + + +#============================== Template ===================================== + +# A template is used to set the mapping in Elasticsearch +# By default template loading is enabled and the template is loaded. +# These settings can be adjusted to load your own template or overwrite existing ones. + +# Set to false to disable template loading. +#setup.template.enabled: true + +# Template name. By default the template name is "metricbeat-%{[beat.version]}" +# The template name and pattern has to be set in case the elasticsearch index pattern is modified. +#setup.template.name: "metricbeat-%{[beat.version]}" + +# Template pattern. By default the template pattern is "-%{[beat.version]}-*" to apply to the default index settings. +# The first part is the version of the beat and then -* is used to match all daily indices. +# The template name and pattern has to be set in case the elasticsearch index pattern is modified. +#setup.template.pattern: "metricbeat-%{[beat.version]}-*" + +# Path to fields.yml file to generate the template +#setup.template.fields: "${path.config}/fields.yml" + +# A list of fields to be added to the template and Kibana index pattern. Also +# specify setup.template.overwrite: true to overwrite the existing template. +# This setting is experimental. +#setup.template.append_fields: +#- name: field_name +# type: field_type + +# Enable json template loading. If this is enabled, the fields.yml is ignored. +#setup.template.json.enabled: false + +# Path to the json template file +#setup.template.json.path: "${path.config}/template.json" + +# Name under which the template is stored in Elasticsearch +#setup.template.json.name: "" + +# Overwrite existing template +#setup.template.overwrite: false + +# Elasticsearch template settings +setup.template.settings: + + # A dictionary of settings to place into the settings.index dictionary + # of the Elasticsearch template. For more details, please check + # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping.html + #index: + #number_of_shards: 1 + #codec: best_compression + #number_of_routing_shards: 30 + + # A dictionary of settings for the _source field. For more details, please check + # https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-source-field.html + #_source: + #enabled: false + +#============================== Kibana ===================================== + +# Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API. +# This requires a Kibana endpoint configuration. +setup.kibana: + + # Kibana Host + # Scheme and port can be left out and will be set to the default (http and 5601) + # In case you specify and additional path, the scheme is required: http://localhost:5601/path + # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601 + #host: "localhost:5601" + + # Optional protocol and basic auth credentials. + #protocol: "https" + #username: "elastic" + #password: "changeme" + + # Optional HTTP Path + #path: "" + + # Use SSL settings for HTTPS. Default is true. + #ssl.enabled: true + + # Configure SSL verification mode. If `none` is configured, all server hosts + # and certificates will be accepted. In this mode, SSL based connections are + # susceptible to man-in-the-middle attacks. Use only for testing. Default is + # `full`. + #ssl.verification_mode: full + + # List of supported/valid TLS versions. By default all TLS versions 1.0 up to + # 1.2 are enabled. + #ssl.supported_protocols: [TLSv1.0, TLSv1.1, TLSv1.2] + + # SSL configuration. By default is off. + # List of root certificates for HTTPS server verifications + #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] + + # Certificate for SSL client authentication + #ssl.certificate: "/etc/pki/client/cert.pem" + + # Client Certificate Key + #ssl.key: "/etc/pki/client/cert.key" + + # Optional passphrase for decrypting the Certificate Key. + #ssl.key_passphrase: '' + + # Configure cipher suites to be used for SSL connections + #ssl.cipher_suites: [] + + # Configure curve types for ECDHE based cipher suites + #ssl.curve_types: [] + + + +#================================ Logging ====================================== +# There are four options for the log output: file, stderr, syslog, eventlog +# The file output is the default. + +# Sets log level. The default log level is info. +# Available log levels are: error, warning, info, debug +#logging.level: info + +# Enable debug output for selected components. To enable all selectors use ["*"] +# Other available selectors are "beat", "publish", "service" +# Multiple selectors can be chained. +#logging.selectors: [ ] + +# Send all logging output to syslog. The default is false. +#logging.to_syslog: false + +# Send all logging output to Windows Event Logs. The default is false. +#logging.to_eventlog: false + +# If enabled, metricbeat periodically logs its internal metrics that have changed +# in the last period. For each metric that changed, the delta from the value at +# the beginning of the period is logged. Also, the total values for +# all non-zero internal metrics are logged on shutdown. The default is true. +#logging.metrics.enabled: true + +# The period after which to log the internal metrics. The default is 30s. +#logging.metrics.period: 30s + +# Logging to rotating files. Set logging.to_files to false to disable logging to +# files. +logging.to_files: true +logging.files: + # Configure the path where the logs are written. The default is the logs directory + # under the home path (the binary location). + #path: /var/log/metricbeat + + # The name of the files where the logs are written to. + #name: metricbeat + + # Configure log file size limit. If limit is reached, log file will be + # automatically rotated + #rotateeverybytes: 10485760 # = 10MB + + # Number of rotated log files to keep. Oldest files will be deleted first. + #keepfiles: 7 + + # The permissions mask to apply when rotating log files. The default value is 0600. + # Must be a valid Unix-style file permissions mask expressed in octal notation. + #permissions: 0600 + + # Enable log file rotation on time intervals in addition to size-based rotation. + # Intervals must be at least 1s. Values of 1m, 1h, 24h, 7*24h, 30*24h, and 365*24h + # are boundary-aligned with minutes, hours, days, weeks, months, and years as + # reported by the local system clock. All other intervals are calculated from the + # unix epoch. Defaults to disabled. + #interval: 0 + +# Set to true to log messages in json format. +#logging.json: false + + +#============================== Xpack Monitoring ===================================== +# metricbeat can export internal metrics to a central Elasticsearch monitoring cluster. +# This requires xpack monitoring to be enabled in Elasticsearch. +# The reporting is disabled by default. + +# Set to true to enable the monitoring reporter. +#xpack.monitoring.enabled: false + +# Uncomment to send the metrics to Elasticsearch. Most settings from the +# Elasticsearch output are accepted here as well. Any setting that is not set is +# automatically inherited from the Elasticsearch output configuration, so if you +# have the Elasticsearch output configured, you can simply uncomment the +# following line, and leave the rest commented out. +#xpack.monitoring.elasticsearch: + + # Array of hosts to connect to. + # Scheme and port can be left out and will be set to the default (http and 9200) + # In case you specify and additional path, the scheme is required: http://localhost:9200/path + # IPv6 addresses should always be defined as: https://[2001:db8::1]:9200 + #hosts: ["localhost:9200"] + + # Set gzip compression level. + #compression_level: 0 + + # Optional protocol and basic auth credentials. + #protocol: "https" + #username: "beats_system" + #password: "changeme" + + # Dictionary of HTTP parameters to pass within the url with index operations. + #parameters: + #param1: value1 + #param2: value2 + + # Custom HTTP headers to add to each request + #headers: + # X-My-Header: Contents of the header + + # Proxy server url + #proxy_url: http://proxy:3128 + + # The number of times a particular Elasticsearch index operation is attempted. If + # the indexing operation doesn't succeed after this many retries, the events are + # dropped. The default is 3. + #max_retries: 3 + + # The maximum number of events to bulk in a single Elasticsearch bulk API index request. + # The default is 50. + #bulk_max_size: 50 + + # The number of seconds to wait before trying to reconnect to Elasticsearch + # after a network error. After waiting backoff.init seconds, the Beat + # tries to reconnect. If the attempt fails, the backoff timer is increased + # exponentially up to backoff.max. After a successful connection, the backoff + # timer is reset. The default is 1s. + #backoff.init: 1s + + # The maximum number of seconds to wait before attempting to connect to + # Elasticsearch after a network error. The default is 60s. + #backoff.max: 60s + + # Configure http request timeout before failing an request to Elasticsearch. + #timeout: 90 + + # Use SSL settings for HTTPS. + #ssl.enabled: true + + # Configure SSL verification mode. If `none` is configured, all server hosts + # and certificates will be accepted. In this mode, SSL based connections are + # susceptible to man-in-the-middle attacks. Use only for testing. Default is + # `full`. + #ssl.verification_mode: full + + # List of supported/valid TLS versions. By default all TLS versions 1.0 up to + # 1.2 are enabled. + #ssl.supported_protocols: [TLSv1.0, TLSv1.1, TLSv1.2] + + # SSL configuration. By default is off. + # List of root certificates for HTTPS server verifications + #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] + + # Certificate for SSL client authentication + #ssl.certificate: "/etc/pki/client/cert.pem" + + # Client Certificate Key + #ssl.key: "/etc/pki/client/cert.key" + + # Optional passphrase for decrypting the Certificate Key. + #ssl.key_passphrase: '' + + # Configure cipher suites to be used for SSL connections + #ssl.cipher_suites: [] + + # Configure curve types for ECDHE based cipher suites + #ssl.curve_types: [] + + # Configure what types of renegotiation are supported. Valid options are + # never, once, and freely. Default is never. + #ssl.renegotiation: never + + #metrics.period: 10s + #state.period: 1m + +#================================ HTTP Endpoint ====================================== +# Each beat can expose internal metrics through a HTTP endpoint. For security +# reasons the endpoint is disabled by default. This feature is currently experimental. +# Stats can be access through http://localhost:5066/stats . For pretty JSON output +# append ?pretty to the URL. + +# Defines if the HTTP endpoint is enabled. +#http.enabled: false + +# The HTTP endpoint will bind to this hostname or IP address. It is recommended to use only localhost. +#http.host: localhost + +# Port on which the HTTP endpoint will bind. Default is 5066. +#http.port: 5066 + +#============================= Process Security ================================ + +# Enable or disable seccomp system call filtering on Linux. Default is enabled. +#seccomp.enabled: true diff --git a/x-pack/metricbeat/metricbeat.yml b/x-pack/metricbeat/metricbeat.yml new file mode 100644 index 00000000000..259092c66df --- /dev/null +++ b/x-pack/metricbeat/metricbeat.yml @@ -0,0 +1,148 @@ +###################### Metricbeat Configuration Example ####################### + +# This file is an example configuration file highlighting only the most common +# options. The metricbeat.reference.yml file from the same directory contains all the +# supported options with more comments. You can use it as a reference. +# +# You can find the full configuration reference here: +# https://www.elastic.co/guide/en/beats/metricbeat/index.html + +#========================== Modules configuration ============================ + +metricbeat.config.modules: + # Glob pattern for configuration loading + path: ${path.config}/modules.d/*.yml + + # Set to true to enable config reloading + reload.enabled: false + + # Period on which files under path should be checked for changes + #reload.period: 10s + +#==================== Elasticsearch template setting ========================== + +setup.template.settings: + index.number_of_shards: 1 + index.codec: best_compression + #_source.enabled: false + +#================================ General ===================================== + +# The name of the shipper that publishes the network data. It can be used to group +# all the transactions sent by a single shipper in the web interface. +#name: + +# The tags of the shipper are included in their own field with each +# transaction published. +#tags: ["service-X", "web-tier"] + +# Optional fields that you can specify to add additional information to the +# output. +#fields: +# env: staging + + +#============================== Dashboards ===================================== +# These settings control loading the sample dashboards to the Kibana index. Loading +# the dashboards is disabled by default and can be enabled either by setting the +# options here, or by using the `-setup` CLI flag or the `setup` command. +#setup.dashboards.enabled: false + +# The URL from where to download the dashboards archive. By default this URL +# has a value which is computed based on the Beat name and version. For released +# versions, this URL points to the dashboard archive on the artifacts.elastic.co +# website. +#setup.dashboards.url: + +#============================== Kibana ===================================== + +# Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API. +# This requires a Kibana endpoint configuration. +setup.kibana: + + # Kibana Host + # Scheme and port can be left out and will be set to the default (http and 5601) + # In case you specify and additional path, the scheme is required: http://localhost:5601/path + # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601 + #host: "localhost:5601" + + # Kibana Space ID + # ID of the Kibana Space into which the dashboards should be loaded. By default, + # the Default Space will be used. + #space.id: + +#============================= Elastic Cloud ================================== + +# These settings simplify using metricbeat with the Elastic Cloud (https://cloud.elastic.co/). + +# The cloud.id setting overwrites the `output.elasticsearch.hosts` and +# `setup.kibana.host` options. +# You can find the `cloud.id` in the Elastic Cloud web UI. +#cloud.id: + +# The cloud.auth setting overwrites the `output.elasticsearch.username` and +# `output.elasticsearch.password` settings. The format is `:`. +#cloud.auth: + +#================================ Outputs ===================================== + +# Configure what output to use when sending the data collected by the beat. + +#-------------------------- Elasticsearch output ------------------------------ +output.elasticsearch: + # Array of hosts to connect to. + hosts: ["localhost:9200"] + + # Optional protocol and basic auth credentials. + #protocol: "https" + #username: "elastic" + #password: "changeme" + +#----------------------------- Logstash output -------------------------------- +#output.logstash: + # The Logstash hosts + #hosts: ["localhost:5044"] + + # Optional SSL. By default is off. + # List of root certificates for HTTPS server verifications + #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] + + # Certificate for SSL client authentication + #ssl.certificate: "/etc/pki/client/cert.pem" + + # Client Certificate Key + #ssl.key: "/etc/pki/client/cert.key" + +#================================ Procesors ===================================== + +# Configure processors to enhance or manipulate events generated by the beat. + +processors: + - add_host_metadata: ~ + - add_cloud_metadata: ~ + +#================================ Logging ===================================== + +# Sets log level. The default log level is info. +# Available log levels are: error, warning, info, debug +#logging.level: debug + +# At debug level, you can selectively enable logging only for some components. +# To enable all selectors use ["*"]. Examples of other selectors are "beat", +# "publish", "service". +#logging.selectors: ["*"] + +#============================== Xpack Monitoring =============================== +# metricbeat can export internal metrics to a central Elasticsearch monitoring +# cluster. This requires xpack monitoring to be enabled in Elasticsearch. The +# reporting is disabled by default. + +# Set to true to enable the monitoring reporter. +#xpack.monitoring.enabled: false + +# Uncomment to send the metrics to Elasticsearch. Most settings from the +# Elasticsearch output are accepted here as well. Any setting that is not set is +# automatically inherited from the Elasticsearch output configuration, so if you +# have the Elasticsearch output configured, you can simply uncomment the +# following line. +#xpack.monitoring.elasticsearch: diff --git a/x-pack/metricbeat/module/foo/_meta/config.yml b/x-pack/metricbeat/module/foo/_meta/config.yml new file mode 100644 index 00000000000..2e6466ec30e --- /dev/null +++ b/x-pack/metricbeat/module/foo/_meta/config.yml @@ -0,0 +1,6 @@ +- module: foo + metricsets: ["bar"] + enabled: false + period: 10s + hosts: ["localhost"] + diff --git a/x-pack/metricbeat/module/foo/_meta/docs.asciidoc b/x-pack/metricbeat/module/foo/_meta/docs.asciidoc new file mode 100644 index 00000000000..7111494befd --- /dev/null +++ b/x-pack/metricbeat/module/foo/_meta/docs.asciidoc @@ -0,0 +1,2 @@ +This is the foo module. + diff --git a/x-pack/metricbeat/module/foo/_meta/fields.yml b/x-pack/metricbeat/module/foo/_meta/fields.yml new file mode 100644 index 00000000000..b4aa91edc51 --- /dev/null +++ b/x-pack/metricbeat/module/foo/_meta/fields.yml @@ -0,0 +1,11 @@ +- key: foo + title: "foo" + description: > + experimental[] + + foo module + fields: + - name: foo + type: group + description: > + fields: diff --git a/x-pack/metricbeat/module/foo/bar/_meta/data.json b/x-pack/metricbeat/module/foo/bar/_meta/data.json new file mode 100644 index 00000000000..f86e348898c --- /dev/null +++ b/x-pack/metricbeat/module/foo/bar/_meta/data.json @@ -0,0 +1,19 @@ +{ + "@timestamp":"2016-05-23T08:05:34.853Z", + "beat":{ + "hostname":"beathost", + "name":"beathost" + }, + "metricset":{ + "host":"localhost", + "module":"foo", + "name":"bar", + "rtt":44269 + }, + "foo":{ + "bar":{ + "example": "bar" + } + }, + "type":"metricsets" +} diff --git a/x-pack/metricbeat/module/foo/bar/_meta/docs.asciidoc b/x-pack/metricbeat/module/foo/bar/_meta/docs.asciidoc new file mode 100644 index 00000000000..d1fc97c6d4c --- /dev/null +++ b/x-pack/metricbeat/module/foo/bar/_meta/docs.asciidoc @@ -0,0 +1 @@ +This is the bar metricset of the module foo. diff --git a/x-pack/metricbeat/module/foo/bar/_meta/fields.yml b/x-pack/metricbeat/module/foo/bar/_meta/fields.yml new file mode 100644 index 00000000000..481f212d17d --- /dev/null +++ b/x-pack/metricbeat/module/foo/bar/_meta/fields.yml @@ -0,0 +1,9 @@ +- name: bar + type: group + description: > + bar + fields: + - name: example + type: keyword + description: > + Example field diff --git a/x-pack/metricbeat/module/foo/bar/bar.go b/x-pack/metricbeat/module/foo/bar/bar.go new file mode 100644 index 00000000000..2102f957644 --- /dev/null +++ b/x-pack/metricbeat/module/foo/bar/bar.go @@ -0,0 +1,56 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package bar + +import ( + "github.com/elastic/beats/libbeat/common" + "github.com/elastic/beats/libbeat/common/cfgwarn" + "github.com/elastic/beats/metricbeat/mb" +) + +// init registers the MetricSet with the central registry as soon as the program +// starts. The New function will be called later to instantiate an instance of +// the MetricSet for each host defined in the module's configuration. After the +// MetricSet has been created then Fetch will begin to be called periodically. +func init() { + mb.Registry.MustAddMetricSet("foo", "bar", New) +} + +// MetricSet holds any configuration or state information. It must implement +// the mb.MetricSet interface. And this is best achieved by embedding +// mb.BaseMetricSet because it implements all of the required mb.MetricSet +// interface methods except for Fetch. +type MetricSet struct { + mb.BaseMetricSet + counter int +} + +// New creates a new instance of the MetricSet. New is responsible for unpacking +// any MetricSet specific configuration options if there are any. +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + cfgwarn.Experimental("The foo bar metricset is experimental.") + + config := struct{}{} + if err := base.Module().UnpackConfig(&config); err != nil { + return nil, err + } + + return &MetricSet{ + BaseMetricSet: base, + counter: 1, + }, nil +} + +// Fetch methods implements the data gathering and data conversion to the right +// format. It publishes the event which is then forwarded to the output. In case +// of an error set the Error field of mb.Event or simply call report.Error(). +func (m *MetricSet) Fetch(report mb.ReporterV2) { + report.Event(mb.Event{ + MetricSetFields: common.MapStr{ + "counter": m.counter, + }, + }) + m.counter++ +} diff --git a/x-pack/metricbeat/module/foo/doc.go b/x-pack/metricbeat/module/foo/doc.go new file mode 100644 index 00000000000..bf4f382834a --- /dev/null +++ b/x-pack/metricbeat/module/foo/doc.go @@ -0,0 +1,6 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// Package foo is a Metricbeat module that contains MetricSets. +package foo diff --git a/x-pack/metricbeat/module/foo/fields.go b/x-pack/metricbeat/module/foo/fields.go new file mode 100644 index 00000000000..744f92643b1 --- /dev/null +++ b/x-pack/metricbeat/module/foo/fields.go @@ -0,0 +1,22 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// Code generated by beats/dev-tools/cmd/asset/asset.go - DO NOT EDIT. + +package foo + +import ( + "github.com/elastic/beats/libbeat/asset" +) + +func init() { + if err := asset.SetFields("metricbeat", "foo", Asset); err != nil { + panic(err) + } +} + +// Asset returns asset data +func Asset() string { + return "eJx8j0EOwiAQRfdzip/uewEW7jyFcYEyGFLoEEpje3vTooY21bf8E94LLTqeFawIAdllzwqNFWkIMDzck4vZSa9wIgDgKXJygfus/eVK62ZFEMSMngmwjr0Z1Hpo0evAH/lCniMrPJKM8b0cNLaSWnTT6bsdyX4KC9vn+0gd4kmHuP6npgQ7np+SzO72J7twLsISpVcAAAD//+HhYIk=" +} diff --git a/x-pack/metricbeat/packages.yml b/x-pack/metricbeat/packages.yml new file mode 100644 index 00000000000..a0cf5267533 --- /dev/null +++ b/x-pack/metricbeat/packages.yml @@ -0,0 +1,90 @@ +specs: + xpack: + - os: windows + types: [zip] + spec: + <<: *windows_binary_spec + <<: *elastic_license_for_binaries + files: + modules.d: + mode: 0644 + source: build/package/modules.d + config: true + kibana: + source: build/kibana + mode: 0644 + + - os: darwin + types: [tgz] + spec: + <<: *binary_spec + <<: *elastic_license_for_binaries + files: + modules.d: + mode: 0644 + source: build/package/modules.d + config: true + kibana: + source: build/kibana + mode: 0644 + + - os: darwin + types: [dmg] + spec: + <<: *macos_beat_pkg_spec + <<: *elastic_license_for_macos_pkg + files: + /etc/{{.BeatName}}/modules.d: + mode: 0644 + source: build/package/modules.d + config: true + '/Library/Application Support/{{.BeatVendor}}/{{.BeatName}}/kibana': + source: build/kibana + mode: 0644 + + - os: linux + types: [tgz] + spec: + <<: *binary_spec + <<: *elastic_license_for_binaries + files: + modules.d: + mode: 0644 + source: build/package/modules.d + config: true + kibana: + source: build/kibana + mode: 0644 + + - os: linux + types: [deb, rpm] + spec: + <<: *deb_rpm_spec + <<: *elastic_license_for_deb_rpm + files: + '/etc/{{.BeatName}}/modules.d': + mode: 0644 + source: build/package/modules.d + config: true + '/usr/share/{{.BeatName}}/kibana': + source: build/kibana + mode: 0644 + + - os: linux + types: [docker] + spec: + <<: *docker_spec + <<: *elastic_docker_spec + <<: *elastic_license_for_binaries + files: + '{{.BeatName}}.yml': + source: '../../metricbeat/metricbeat.docker.yml' + mode: 0600 + config: true + modules.d: + mode: 0644 + source: build/package/modules.d + config: true + kibana: + source: build/kibana + mode: 0644 From 83f2977eb8743be220d7df5e094ab7d19898e181 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Fri, 11 Jan 2019 18:43:37 -0700 Subject: [PATCH 2/4] Add grok pattern to support different timestamp fmt in redis log --- filebeat/module/redis/log/ingest/pipeline.json | 7 +++++-- filebeat/module/redis/log/test/test-timestamp.log | 1 + .../log/test/test-timestamp.log-expected.json | 14 ++++++++++++++ 3 files changed, 20 insertions(+), 2 deletions(-) create mode 100644 filebeat/module/redis/log/test/test-timestamp.log create mode 100644 filebeat/module/redis/log/test/test-timestamp.log-expected.json diff --git a/filebeat/module/redis/log/ingest/pipeline.json b/filebeat/module/redis/log/ingest/pipeline.json index f932547d61b..c325087a33c 100644 --- a/filebeat/module/redis/log/ingest/pipeline.json +++ b/filebeat/module/redis/log/ingest/pipeline.json @@ -5,12 +5,14 @@ "grok": { "field": "message", "patterns": [ - "(%{POSINT:process.pid:long}:%{CHAR:redis.log.role} )?%{REDISTIMESTAMP:redis.log.timestamp} %{REDISLEVEL:log.level} %{GREEDYDATA:message}", + "(%{POSINT:process.pid:long}:%{CHAR:redis.log.role} )?(%{REDISTIMESTAMP1:redis.log.timestamp}||%{REDISTIMESTAMP2:redis.log.timestamp}) %{REDISLEVEL:log.level} %{GREEDYDATA:message}", "%{POSINT:process.pid:long}:signal-handler \\(%{POSINT:redis.log.timestamp}\\) %{GREEDYDATA:message}" ], "pattern_definitions": { "CHAR": "[a-zA-Z]", - "REDISLEVEL": "[.\\-*#]" + "REDISLEVEL": "[.\\-*#]", + "REDISTIMESTAMP1": "%{MONTHDAY} %{MONTH} %{TIME}", + "REDISTIMESTAMP2": "%{MONTHDAY} %{MONTH} %{YEAR} %{TIME}" } } }, @@ -57,6 +59,7 @@ "field": "redis.log.timestamp", "target_field": "@timestamp", "formats": [ + "dd MMM YYYY H:m:s.SSS", "dd MMM H:m:s.SSS", "dd MMM H:m:s", "UNIX" diff --git a/filebeat/module/redis/log/test/test-timestamp.log b/filebeat/module/redis/log/test/test-timestamp.log new file mode 100644 index 00000000000..e59b6eeab43 --- /dev/null +++ b/filebeat/module/redis/log/test/test-timestamp.log @@ -0,0 +1 @@ +26571:M 27 Dec 2018 11:19:18.874 * Synchronization with replica 10.114.208.18:6023 succeeded diff --git a/filebeat/module/redis/log/test/test-timestamp.log-expected.json b/filebeat/module/redis/log/test/test-timestamp.log-expected.json new file mode 100644 index 00000000000..50883fdf85e --- /dev/null +++ b/filebeat/module/redis/log/test/test-timestamp.log-expected.json @@ -0,0 +1,14 @@ +[ + { + "ecs.version": "1.0.0-beta2", + "event.dataset": "redis.log", + "event.module": "redis", + "fileset.name": "log", + "input.type": "log", + "log.level": "notice", + "log.offset": 0, + "message": "Synchronization with replica 10.114.208.18:6023 succeeded", + "process.pid": 26571, + "redis.log.role": "master" + } +] \ No newline at end of file From f037ec20b179f0d3e814a86fa0b3415cee56c943 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Fri, 11 Jan 2019 18:49:22 -0700 Subject: [PATCH 3/4] Add changelog --- CHANGELOG.next.asciidoc | 1 + .../redis/log/test/{test-timestamp.log => redis-5.0.3.log} | 0 ...timestamp.log-expected.json => redis-5.0.3.log-expected.json} | 0 3 files changed, 1 insertion(+) rename filebeat/module/redis/log/test/{test-timestamp.log => redis-5.0.3.log} (100%) rename filebeat/module/redis/log/test/{test-timestamp.log-expected.json => redis-5.0.3.log-expected.json} (100%) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index d401f8589b2..af0ce207e1a 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -33,6 +33,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Rename a few `mysql.*` fields to map to ECS. {pull}10008[10008] - Rename a few `nginx.error.*` fields to map to ECS. {pull}10007[10007] - Filesets with multiple ingest pipelines added in {pull}8914[8914] only work with Elasticsearch >= 6.5.0 {pull}10001[10001] +- Add grok pattern to support redis 5.0.3 log timestamp. {issue}9819[9819] {pull}10033[10033] *Heartbeat* diff --git a/filebeat/module/redis/log/test/test-timestamp.log b/filebeat/module/redis/log/test/redis-5.0.3.log similarity index 100% rename from filebeat/module/redis/log/test/test-timestamp.log rename to filebeat/module/redis/log/test/redis-5.0.3.log diff --git a/filebeat/module/redis/log/test/test-timestamp.log-expected.json b/filebeat/module/redis/log/test/redis-5.0.3.log-expected.json similarity index 100% rename from filebeat/module/redis/log/test/test-timestamp.log-expected.json rename to filebeat/module/redis/log/test/redis-5.0.3.log-expected.json From bfb8d2c319c37d27f463e74bf6b2f79fd3933c42 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Sun, 13 Jan 2019 18:14:57 -0700 Subject: [PATCH 4/4] Fix rebase issue --- CHANGELOG.asciidoc | 2 -- packetbeat/docs/gettingstarted.asciidoc | 19 ------------------- 2 files changed, 21 deletions(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index f92ff257059..91616dc790e 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -222,7 +222,6 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha1...v7.0.0-alpha2[Check the ==== Added *Affecting all Beats* -- Unify dashboard exporter tools. {pull}9097[9097] - Unify dashboard exporter tools. {pull}9097[9097] - Add cache.ttl to add_host_metadata. {pull}9359[9359] @@ -232,7 +231,6 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha1...v7.0.0-alpha2[Check the - Add geo fields to `add_host_metadata` processor. {pull}9392[9392] *Filebeat* -- Added `detect_null_bytes` selector to detect null bytes from a io.reader. {pull}9210[9210] - Added the `redirect_stderr` option that allows panics to be logged to log files. {pull}8430[8430] - Added `detect_null_bytes` selector to detect null bytes from a io.reader. {pull}9210[9210] diff --git a/packetbeat/docs/gettingstarted.asciidoc b/packetbeat/docs/gettingstarted.asciidoc index f95913500b2..7c06c435074 100644 --- a/packetbeat/docs/gettingstarted.asciidoc +++ b/packetbeat/docs/gettingstarted.asciidoc @@ -102,25 +102,6 @@ tar xzvf packetbeat-{version}-linux-x86_64.tar.gz endif::[] -[[linux]] -*linux:* - -ifeval::["{release-state}"=="unreleased"] - -Version {stack-version} of {beatname_uc} has not yet been released. - -endif::[] - -ifeval::["{release-state}"!="unreleased"] - -["source","sh",subs="attributes,callouts"] ----------------------------------------------------------------------- -curl -L -O https://artifacts.elastic.co/downloads/beats/packetbeat/packetbeat-{version}-linux-x86_64.tar.gz -tar xzvf packetbeat-{version}-linux-x86_64.tar.gz ----------------------------------------------------------------------- - -endif::[] - [[win]] *win:*