diff --git a/filebeat/docs/command-line.asciidoc b/filebeat/docs/command-line.asciidoc index 32bf1b78d38c..60eabbbca35c 100644 --- a/filebeat/docs/command-line.asciidoc +++ b/filebeat/docs/command-line.asciidoc @@ -1,4 +1,4 @@ -[[filebeat-command-line]] +[[command-line-options]] === Command Line Options The following command line option is specific to Filebeat. diff --git a/filebeat/docs/getting-started.asciidoc b/filebeat/docs/getting-started.asciidoc index 25fdf6c6a755..1de38f458411 100644 --- a/filebeat/docs/getting-started.asciidoc +++ b/filebeat/docs/getting-started.asciidoc @@ -18,7 +18,7 @@ After installing the Elastic Stack, read the following topics to learn how to in * <> * <> * <> -* <> +* <> * <> [[filebeat-installation]] @@ -27,17 +27,7 @@ After installing the Elastic Stack, read the following topics to learn how to in Before running Filebeat, you need to install and configure the Elastic stack. See {libbeat}/getting-started.html[Getting Started with Beats and the Elastic Stack]. -To download and install Filebeat, use the commands that work with your system -(<> for Debian/Ubuntu, <> for Redhat/Centos/Fedora, <> for OS X, and <> for Windows). - -[NOTE] -================================================== -If you use Apt or Yum, you can <> to update to the newest version more easily. - -See our https://www.elastic.co/downloads/beats/filebeat[download page] for other installation options, such as 32-bit images. - -================================================== +include::../../libbeat/docs/shared-download-and-install.asciidoc[] [[deb]] *deb:* @@ -96,6 +86,24 @@ tar xzvf filebeat-{version}-darwin-x86_64.tar.gz endif::[] +[[docker]] +*docker:* + +ifeval::["{release-state}"=="unreleased"] + +Version {stack-version} of {beatname_uc} has not yet been released. + +endif::[] + +ifeval::["{release-state}"!="unreleased"] + +["source", "shell", subs="attributes"] +------------------------------------------------ +docker pull {dockerimage} +------------------------------------------------ + +endif::[] + [[win]] *win:* @@ -139,15 +147,7 @@ started, you can skip the content in this section, including the remaining getting started steps, and go directly to the <> page. -To configure Filebeat manually, you edit the configuration file. For rpm and deb, -you'll find the configuration file at `/etc/filebeat/filebeat.yml`. For mac and -win, look in the archive that you just extracted. There’s also a full example -configuration file called `filebeat.full.yml` that shows all non-deprecated -options. - -See the -{libbeat}/config-file-format.html[Config File Format] section of the -_Beats Platform Reference_ for more about the structure of the config file. +include::../../libbeat/docs/shared-configuring.asciidoc[] Here is a sample of the `filebeat` section of the `filebeat.yml` file. Filebeat uses predefined default values for most configuration options. @@ -223,7 +223,7 @@ include::../../libbeat/docs/shared-template-load.asciidoc[] Start Filebeat by issuing the appropriate command for your platform. NOTE: If you use an init.d script to start Filebeat on deb or rpm, you can't -specify command line flags (see <>). To specify flags, +specify command line flags (see <>). To specify flags, start Filebeat in the foreground. *deb:* @@ -240,6 +240,13 @@ sudo /etc/init.d/filebeat start sudo /etc/init.d/filebeat start ---------------------------------------------------------------------- +*docker:* + +["source", "shell", subs="attributes"] +---------------------------------------------------------------------- +docker run {dockerimage} +---------------------------------------------------------------------- + *mac:* [source,shell] diff --git a/filebeat/docs/images/false-after-multi.png b/filebeat/docs/images/false-after-multi.png new file mode 100644 index 000000000000..1918c531d239 Binary files /dev/null and b/filebeat/docs/images/false-after-multi.png differ diff --git a/filebeat/docs/images/false-before-multi.png b/filebeat/docs/images/false-before-multi.png new file mode 100644 index 000000000000..ecb949b77ae0 Binary files /dev/null and b/filebeat/docs/images/false-before-multi.png differ diff --git a/filebeat/docs/images/true-after-multi.png b/filebeat/docs/images/true-after-multi.png new file mode 100644 index 000000000000..a77af69d974a Binary files /dev/null and b/filebeat/docs/images/true-after-multi.png differ diff --git a/filebeat/docs/images/true-before-multi.png b/filebeat/docs/images/true-before-multi.png new file mode 100644 index 000000000000..dc6bfcafbe9f Binary files /dev/null and b/filebeat/docs/images/true-before-multi.png differ diff --git a/filebeat/docs/index.asciidoc b/filebeat/docs/index.asciidoc index f33c2b3059ce..a51a550f42a9 100644 --- a/filebeat/docs/index.asciidoc +++ b/filebeat/docs/index.asciidoc @@ -15,6 +15,7 @@ include::../../libbeat/docs/version.asciidoc[] :beatname_lc: filebeat :beatname_uc: Filebeat :security: X-Pack Security +:dockerimage: docker.elastic.co/beats/{beatname_lc}:{version} include::./overview.asciidoc[] @@ -28,6 +29,8 @@ include::../../libbeat/docs/shared-directory-layout.asciidoc[] include::../../libbeat/docs/repositories.asciidoc[] +include::./running-on-docker.asciidoc[] + include::./upgrading.asciidoc[] include::./how-filebeat-works.asciidoc[] @@ -40,6 +43,7 @@ include::./multiline.asciidoc[] include::../../libbeat/docs/shared-config-ingest.asciidoc[] +:standalone: include::../../libbeat/docs/shared-env-vars.asciidoc[] include::./multiple-prospectors.asciidoc[] diff --git a/filebeat/docs/migration.asciidoc b/filebeat/docs/migration.asciidoc index 8f4b18ba6799..254f87f9305f 100644 --- a/filebeat/docs/migration.asciidoc +++ b/filebeat/docs/migration.asciidoc @@ -304,7 +304,7 @@ options with Logstash Forwarder, make sure that you add your options to the configuration file. For naming changes, see <>. Filebeat does provide command line options that are common to all Beats. For more details about -these options, see <>. +these options, see <>. [[renamed-options]] [float] diff --git a/filebeat/docs/modules-getting-started.asciidoc b/filebeat/docs/modules-getting-started.asciidoc index 1b3c11564aac..926133d47489 100644 --- a/filebeat/docs/modules-getting-started.asciidoc +++ b/filebeat/docs/modules-getting-started.asciidoc @@ -15,7 +15,7 @@ modules, see <>. If you are using a log file type that isn't supported by one of the available Filebeat modules, you'll need to set up and configure Filebeat manually by -following the numbered steps under <>. +following the numbered steps under <>. ==== Prerequisites @@ -116,4 +116,4 @@ Open the dashboard and explore the visualizations for your parsed logs. Here's an example of the syslog dashboard: -image:./images/kibana-system.png[Sylog dashboard] \ No newline at end of file +image:./images/kibana-system.png[Syslog dashboard] diff --git a/filebeat/docs/modules-overview.asciidoc b/filebeat/docs/modules-overview.asciidoc index 204f797c00aa..8ed17b80565f 100644 --- a/filebeat/docs/modules-overview.asciidoc +++ b/filebeat/docs/modules-overview.asciidoc @@ -32,6 +32,8 @@ NOTE: At the moment, Filebeat modules require using the Elasticsearch be able to also configure Logstash as a more powerful alternative to Ingest Node. +Filebeat modules require Elasticsearch 5.2 or later. + === Tutorial This tutorial assumes you have Elasticsearch and Kibana installed and diff --git a/filebeat/docs/modules.asciidoc b/filebeat/docs/modules.asciidoc index b4e1c16e1534..0a1a2111fc8b 100644 --- a/filebeat/docs/modules.asciidoc +++ b/filebeat/docs/modules.asciidoc @@ -7,6 +7,8 @@ This section contains an <> of the Filebeat modules feature as well as details about each of the currently supported modules. +Filebeat modules require Elasticsearch 5.2 or later. + //pass macro block used here to remove Edit links from modules documentation because it is generated pass::[] include::modules_list.asciidoc[] diff --git a/filebeat/docs/multiline.asciidoc b/filebeat/docs/multiline.asciidoc index 34ae182c275a..a7011305d4dd 100644 --- a/filebeat/docs/multiline.asciidoc +++ b/filebeat/docs/multiline.asciidoc @@ -3,26 +3,26 @@ The files harvested by {beatname_uc} may contain messages that span multiple lines of text. In order to correctly handle these multiline events, you need to configure `multiline` settings in the +{beatname_lc}.yml+ file to specify which -lines are part of a single event. +lines are part of a single event. IMPORTANT: If you are sending multiline events to Logstash, use the options described here to handle multiline events before sending the event data to Logstash. Trying to implement multiline event handling in Logstash (for example, by -using the Logstash multiline codec) may result in the mixing of streams and corrupted data. +using the Logstash multiline codec) may result in the mixing of streams and corrupted data. At a minimum, you need to configure these `multiline` options: -* the `pattern` option, which specifies a regular expression. Depending on how you configure other multiline options, +* the `pattern` option, which specifies a regular expression. Depending on how you configure other multiline options, lines that match the specified regular expression are considered either continuations of a previous line or the start of a new multiline event. You can set the `negate` option to negate the pattern. * the `match` option, which specifies how Filebeat combines matching lines into an event. You can specify `before` or `after`. -See the full documentation for <> to learn more about these options. Also read <> and +See the full documentation for <> to learn more about these options. Also read <> and <> to avoid common mistakes. [float] === Testing Your Regexp Pattern for Multiline -To make it easier for you to test the regexp patterns in your multiline config, we've created a +To make it easier for you to test the regexp patterns in your multiline config, we've created a https://play.golang.org/p/uAd5XHxscu[Go Playground]. You can simply plug in the regexp pattern along with the `negate` setting that you plan to use, and paste a sample message between the content backticks (` `). Then click Run, and you'll see which lines in the message match your specified configuration. For example: @@ -63,7 +63,7 @@ multiline.match: after This configuration merges any line that begins with whitespace up to the previous line. -Here's a Java stack trace that presents a slightly more complex example: +Here's a Java stack trace that presents a slightly more complex example: ["source","sh",subs="attributes,callouts"] ------------------------------------------------------------------------------------- @@ -93,7 +93,7 @@ In this example, the pattern matches the following lines: [float] ==== Line Continuations -Several programming languages use the backslash (`\`) character at the end of a line to denote that the line continues, +Several programming languages use the backslash (`\`) character at the end of a line to denote that the line continues, as in this example: [source,c] @@ -134,7 +134,7 @@ multiline.negate: true multiline.match: after ------------------------------------------------------------------------------------- -This configuration uses the `negate: true` and `match: after` settings to specify that any line that does not match the +This configuration uses the `negate: true` and `match: after` settings to specify that any line that does not match the specified pattern belongs to the previous line. diff --git a/filebeat/docs/reference/configuration/filebeat-options.asciidoc b/filebeat/docs/reference/configuration/filebeat-options.asciidoc index 34263eb077b1..1aa06e9d6e7b 100644 --- a/filebeat/docs/reference/configuration/filebeat-options.asciidoc +++ b/filebeat/docs/reference/configuration/filebeat-options.asciidoc @@ -390,11 +390,11 @@ somewhat from the patterns supported by Logstash. See <> for a l + [options="header"] |======================= -|Setting for `negate` | Setting for `match` | Result -|`false` | `after` | Consecutive lines that match the pattern are appended to the previous line that doesn't match. -|`false` | `before` | Consecutive lines that match the pattern are prepended to the next line that doesn't match. -|`true` | `after` | Consecutive lines that don't match the pattern are appended to the previous line that does match. -|`true` | `before` | Consecutive lines that don't match the pattern are prepended to the next line that does match. +|Setting for `negate` | Setting for `match` | Result | Example `pattern: ^b` +|`false` | `after` | Consecutive lines that match the pattern are appended to the previous line that doesn't match. | image:./images/false-after-multi.png[Lines a b b c b b become "abb" and "cbb"] +|`false` | `before` | Consecutive lines that match the pattern are prepended to the next line that doesn't match. | image:./images/false-before-multi.png[Lines b b a b b c become "bba" and "bbc"] +|`true` | `after` | Consecutive lines that don't match the pattern are appended to the previous line that does match. | image:./images/true-after-multi.png[Lines b a c b d e become "bac" and "bde"] +|`true` | `before` | Consecutive lines that don't match the pattern are prepended to the next line that does match. | image:./images/true-before-multi.png[Lines a c b d e b become "acb" and "deb"] |======================= + NOTE: The `after` setting is equivalent to `previous` in https://www.elastic.co/guide/en/logstash/current/plugins-codecs-multiline.html[Logstash], and `before` is equivalent to `next`. @@ -433,7 +433,7 @@ Because this option may lead to data loss, it is disabled by default. ===== backoff -The backoff options specify how aggressively Filebeat crawls new files for updates. +The backoff options specify how aggressively Filebeat crawls open files for updates. You can use the default values in most cases. The `backoff` option defines how long Filebeat diff --git a/filebeat/docs/running-on-docker.asciidoc b/filebeat/docs/running-on-docker.asciidoc new file mode 100644 index 000000000000..6bbc976ad853 --- /dev/null +++ b/filebeat/docs/running-on-docker.asciidoc @@ -0,0 +1 @@ +include::../../libbeat/docs/shared-docker.asciidoc[] diff --git a/filebeat/filebeat.template-es2x.json b/filebeat/filebeat.template-es2x.json index bbc3dd67500f..ea56f69c5832 100644 --- a/filebeat/filebeat.template-es2x.json +++ b/filebeat/filebeat.template-es2x.json @@ -303,9 +303,6 @@ "index": "not_analyzed", "type": "string" }, - "fields": { - "properties": {} - }, "fileset": { "properties": { "module": { diff --git a/filebeat/filebeat.template-es6x.json b/filebeat/filebeat.template-es6x.json index aa16fd612e9b..d57671c52bf6 100644 --- a/filebeat/filebeat.template-es6x.json +++ b/filebeat/filebeat.template-es6x.json @@ -253,9 +253,6 @@ "ignore_above": 1024, "type": "keyword" }, - "fields": { - "properties": {} - }, "fileset": { "properties": { "module": { diff --git a/filebeat/filebeat.template.json b/filebeat/filebeat.template.json index 2cc5af97cef3..9e618d31c98b 100644 --- a/filebeat/filebeat.template.json +++ b/filebeat/filebeat.template.json @@ -256,9 +256,6 @@ "ignore_above": 1024, "type": "keyword" }, - "fields": { - "properties": {} - }, "fileset": { "properties": { "module": { diff --git a/glide.yaml b/glide.yaml index 7d3248c48452..4ce9691a1ae0 100644 --- a/glide.yaml +++ b/glide.yaml @@ -25,7 +25,7 @@ import: subpackages: - /difflib - package: github.com/elastic/gosigar - version: v0.2.0 + version: v0.2.1 - package: github.com/elastic/procfs version: abf152e5f3e97f2fafac028d2cc06c1feb87ffa5 - package: github.com/samuel/go-parser diff --git a/heartbeat/docs/command-line.asciidoc b/heartbeat/docs/command-line.asciidoc index e04b93370123..4e3fb5a71fa8 100644 --- a/heartbeat/docs/command-line.asciidoc +++ b/heartbeat/docs/command-line.asciidoc @@ -1,4 +1,4 @@ -[[heartbeat-command-line]] +[[command-line-options]] === Command Line Options Heartbeat does not have any Heartbeat-specific command line options. diff --git a/heartbeat/docs/getting-started.asciidoc b/heartbeat/docs/getting-started.asciidoc index 2bd228c464e8..f0f76a0be7f0 100644 --- a/heartbeat/docs/getting-started.asciidoc +++ b/heartbeat/docs/getting-started.asciidoc @@ -18,7 +18,7 @@ install, configure, and run Heartbeat: * <> * <> * <> -* <> +* <> * <> @@ -32,17 +32,7 @@ monitor are running. //TODO: Add a separate topic that explores deployment scenarios in more detail (like installing on a sub-network where there's a firewall etc. -To download and install Heartbeat, use the commands that work with your -system (<> for Debian/Ubuntu, <> for Redhat/Centos/Fedora, -<> for OS X, and <> for Windows). - -[NOTE] -================================================== -If you use Apt or Yum, you can <> to update to the newest version more easily. - -See our https://www.elastic.co/downloads/beats/heartbeat[download page] for other installation options, such as 32-bit images. - -================================================== +include::../../libbeat/docs/shared-download-and-install.asciidoc[] [[deb]] *deb:* @@ -101,6 +91,25 @@ tar xzvf heartbeat-{version}-darwin-x86_64.tar.gz endif::[] + +[[docker]] +*docker:* + +ifeval::["{release-state}"=="unreleased"] + +Version {stack-version} of {beatname_uc} has not yet been released. + +endif::[] + +ifeval::["{release-state}"!="unreleased"] + +["source", "shell", subs="attributes"] +------------------------------------------------ +docker pull {dockerimage} +------------------------------------------------ + +endif::[] + [[win]] *win:* @@ -147,15 +156,7 @@ options, see <>. [[heartbeat-configuration]] === Step 2: Configuring Heartbeat -To configure Heartbeat, you edit the configuration file. For rpm and deb, -you'll find the configuration file at +/etc/heartbeat/heartbeat.yml+. -For mac and win, look in the archive that you just extracted. There’s also a -full example configuration file called `heartbeat.full.yml` that shows all -non-deprecated options. - -See the -{libbeat}/config-file-format.html[Config File Format] section of the -_Beats Platform Reference_ for more about the structure of the config file. +include::../../libbeat/docs/shared-configuring.asciidoc[] Heartbeat provides monitors to check the status of hosts at set intervals. You configure each monitor individually. Heartbeat currently provides monitors @@ -233,7 +234,7 @@ include::../../libbeat/docs/shared-template-load.asciidoc[] Start Heartbeat by issuing the appropriate command for your platform. NOTE: If you use an init.d script to start Heartbeat on deb or rpm, you can't -specify command line flags (see <>). To specify flags, +specify command line flags (see <>). To specify flags, start Heartbeat in the foreground. *deb:* diff --git a/heartbeat/docs/index.asciidoc b/heartbeat/docs/index.asciidoc index 680217e1d8f1..ef8420fa9d82 100644 --- a/heartbeat/docs/index.asciidoc +++ b/heartbeat/docs/index.asciidoc @@ -15,6 +15,7 @@ include::../../libbeat/docs/version.asciidoc[] :beatname_lc: heartbeat :beatname_uc: Heartbeat :security: X-Pack Security +:dockerimage: docker.elastic.co/beats/{beatname_lc}:{version} include::./overview.asciidoc[] @@ -26,6 +27,8 @@ include::../../libbeat/docs/shared-directory-layout.asciidoc[] include::../../libbeat/docs/repositories.asciidoc[] +include::./running-on-docker.asciidoc[] + // //include::./upgrading.asciidoc[] @@ -41,6 +44,7 @@ include::../../libbeat/docs/shared-config-ingest.asciidoc[] //points to shared topic because configuring-logstash.asciidoc is just a wrapper include::./configuring-logstash.asciidoc[] +:standalone: include::../../libbeat/docs/shared-env-vars.asciidoc[] :standalone: @@ -57,5 +61,5 @@ include::./troubleshooting.asciidoc[] include::./faq.asciidoc[] -// +// //include::./heartbeat-devguide.asciidoc[] diff --git a/heartbeat/docs/running-on-docker.asciidoc b/heartbeat/docs/running-on-docker.asciidoc new file mode 100644 index 000000000000..6bbc976ad853 --- /dev/null +++ b/heartbeat/docs/running-on-docker.asciidoc @@ -0,0 +1 @@ +include::../../libbeat/docs/shared-docker.asciidoc[] diff --git a/heartbeat/heartbeat.template-es2x.json b/heartbeat/heartbeat.template-es2x.json index 84721834b047..1b17ff29cbb4 100644 --- a/heartbeat/heartbeat.template-es2x.json +++ b/heartbeat/heartbeat.template-es2x.json @@ -68,9 +68,6 @@ } } }, - "fields": { - "properties": {} - }, "host": { "ignore_above": 1024, "index": "not_analyzed", diff --git a/heartbeat/heartbeat.template-es6x.json b/heartbeat/heartbeat.template-es6x.json index 8bf9aa5377ed..a35baa0d7f18 100644 --- a/heartbeat/heartbeat.template-es6x.json +++ b/heartbeat/heartbeat.template-es6x.json @@ -55,9 +55,6 @@ } } }, - "fields": { - "properties": {} - }, "host": { "ignore_above": 1024, "type": "keyword" diff --git a/heartbeat/heartbeat.template.json b/heartbeat/heartbeat.template.json index 5be16187d7bb..1baace5a885c 100644 --- a/heartbeat/heartbeat.template.json +++ b/heartbeat/heartbeat.template.json @@ -58,9 +58,6 @@ } } }, - "fields": { - "properties": {} - }, "host": { "ignore_above": 1024, "type": "keyword" diff --git a/libbeat/docs/communitybeats.asciidoc b/libbeat/docs/communitybeats.asciidoc index db42d9c7d21f..660222247965 100644 --- a/libbeat/docs/communitybeats.asciidoc +++ b/libbeat/docs/communitybeats.asciidoc @@ -38,6 +38,7 @@ https://github.com/eskibars/lmsensorsbeat[lmsensorsbeat]:: Collects data from lm https://github.com/consulthys/logstashbeat[logstashbeat]:: Collects data from Logstash monitoring API (v5 onwards) and indexes them in Elasticsearch. https://github.com/yedamao/mcqbeat[mcqbeat]:: Reads the status of queues from memcacheq. https://github.com/scottcrespo/mongobeat[mongobeat]:: Monitors MongoDB instances and can be configured to send multiple document types to Elasticsearch. +https://github.com/nathan-K-/mqttbeat[mqttbeat]:: Add messages from mqtt topics to Elasticsearch. https://github.com/adibendahan/mysqlbeat[mysqlbeat]:: Run any query on MySQL and send results to Elasticsearch. https://github.com/PhaedrusTheGreek/nagioscheckbeat[nagioscheckbeat]:: For Nagios checks and performance data. https://github.com/mrkschan/nginxbeat[nginxbeat]:: Reads status from Nginx. @@ -50,6 +51,7 @@ https://github.com/kozlice/phpfpmbeat[phpfpmbeat]:: Reads status from PHP-FPM. https://github.com/joshuar/pingbeat[pingbeat]:: Sends ICMP pings to a list of targets and stores the round trip time (RTT) in Elasticsearch. https://github.com/carlpett/prombeat[prombeat]:: Indexes https://prometheus.io[Prometheus] metrics. +https://github.com/infonova/prometheusbeat[prometheusbeat]:: Send Prometheus metrics to Elasticsearch via the remote write feature. https://github.com/hartfordfive/protologbeat[protologbeat]:: Accepts structured and unstructured logs via UDP or TCP. Can also be used to receive syslog messages or GELF formatted messages. (To be used as a successor to udplogbeat) https://github.com/voigt/redditbeat[redditbeat]:: Collects new Reddit Submissions of one or multiple Subreddits. https://github.com/chrsblck/redisbeat[redisbeat]:: Used for Redis monitoring. diff --git a/libbeat/docs/config-file-format.asciidoc b/libbeat/docs/config-file-format.asciidoc index dad5ada79701..4d0ca4288956 100644 --- a/libbeat/docs/config-file-format.asciidoc +++ b/libbeat/docs/config-file-format.asciidoc @@ -110,7 +110,7 @@ Simple filebeat example with partially collapsed setting names and use of compac filebeat.prospectors: - input_type: log - paths: ["/var/log/*.log"] + paths: ["/var/log/*.log"] multiline.pattern: '^[' multiline.match: after @@ -213,44 +213,7 @@ format-string-with-date: '%{[fieldname]}-%{+yyyy.MM.dd}' [[config-file-format-env-vars]] === Environment Variables -Beats support use of environment variables in config files to set values that -need to be configurable during deployment. Environment variable expansion is -introduced using `${VAR}`, where `VAR` is the name of the environment variable. - -Note: Only values can be set using environment variables. Environment variables -usage in namespace and setting names are not supported. - -Variable references are replaced when settings are read by beats. The -replacement is case-sensitive and occurs after the YAML file itself has been -parsed. References to undefined variables will lead to errors when dereferenced -and no default value is specified. To specify a default value, use: - -`${VAR:default_value}` - -Where `default_value` is the value to use if the environment variable is -undefined. - -If you need to use a literal `${` in your configuration file then you can write -`$${` to escape the expansion. The `$` symbol can be used to escape other -characters in the default_value like using `$}` in order to generate a `}` -character without closing the variable expansion. - -After changing the value of an environment variable, the beat needs to be -restarted to pick up the new value. - -[float] -==== Examples - -Here are some examples of configurations that use environment variables -and what each configuration looks like after replacement: - -[options="header"] -|================================== -|Config source |Environment setting |Config after replacement -|`name: ${NAME}` |`export NAME=elastic` |`name: elastic` -|`name: ${NAME:beats}` |no setting |`name: beats` -|`name: ${NAME:beats}` |`export NAME=elastic` |`name: elastic` -|================================== +include::shared-env-vars.asciidoc[] [[config-gile-format-refs]] === Reference Variables @@ -333,7 +296,7 @@ When installed via an RPM or DEB package, the config file at `/etc/{beatname}/{beatname}.yml` will have the proper owner and permissions. The file is owned by `root` and has file permissions of `0644` (`-rw-r--r--`). -You may encounter the following errors if your config file fails these checks: +You may encounter the following errors if your config file fails these checks: ["source","sh"] -------------------------------------------------------------------------------- @@ -341,7 +304,7 @@ Exiting: error loading config file: config file ("{beatname}.yml") must be owned by the beat user (uid=501) or root -------------------------------------------------------------------------------- -To correct this problem you can use either `chown root {beatname}.yml` or +To correct this problem you can use either `chown root {beatname}.yml` or `chown 501 {beatname}.yml` to change the owner of the configuration file. ["source","sh"] @@ -370,7 +333,43 @@ use this, for example, for setting defaults in a base configuration file, and overwrite settings via local configuration files. In addition to overwriting settings using multiple configuration files, -individual settings can be overwritten using `-E =`. +individual settings can be overwritten using `-E =`. The +`` can be either a single value or a complex object, such as a list or +dictionary. + +For example, given the following configuration: + +["source","yaml"] +-------------------------------------------------------------------------------- +output.elasticsearch: + hosts: ["http://localhost:9200"] + username: username + password: password +-------------------------------------------------------------------------------- + +You can disable the Elasticsearch output and write all events to the console by +setting: + +["source","sh"] +-------------------------------------------------------------------------------- +-E output='{elasticsearch.enabled: false, console.pretty: true}' +-------------------------------------------------------------------------------- + +Any complex objects that you specify at the command line are merged with the +original configuration, and the following configuration is passed to the Beat: + +["source","yaml"] +-------------------------------------------------------------------------------- +output.elasticsearch: + enabled: false + hosts: ["http://localhost:9200"] + username: username + password: password + +output.console: + pretty: true +-------------------------------------------------------------------------------- + [[config-file-format-tips]] === YAML Tips and Gotchas diff --git a/libbeat/docs/dashboards.asciidoc b/libbeat/docs/dashboards.asciidoc index 210649156b91..d4ac6e2684db 100644 --- a/libbeat/docs/dashboards.asciidoc +++ b/libbeat/docs/dashboards.asciidoc @@ -16,7 +16,7 @@ {beatname_uc} comes packaged with the `scripts/import_dashboards` script that you can use to import the example dashboards, visualizations, and searches for {beatname_uc}. The script also creates an index pattern, -+{beatname_lc}-*+, for {beatname_uc}. ++{beatname_lc}-*+, for {beatname_uc}. The steps in this section show how to import {beatname_uc} dashboards. You may want to import dashboards for more than one Beat or specify import options that aren't described here. See {libbeat}/import-dashboards.html[Importing Existing Beat Dashboards] @@ -28,21 +28,28 @@ ifdef::allplatforms[] *deb, rpm, and mac:* -From the directory where you installed {beatname_uc}, run the `import_dashboards` script. +From the directory where you installed {beatname_uc}, run the `import_dashboards` script. ["source","sh",subs="attributes,callouts"] ---------------------------------------------------------------------- ./scripts/import_dashboards ---------------------------------------------------------------------- -On deb and rpm, the `scripts` folder is located under the home path, which is +/usr/share/{beatname_lc}/+ unless you change it. +*docker:* + +["source","sh",subs="attributes"] +---------------------------------------------------------------------- +docker run {dockerimage} ./scripts/import_dashboards +---------------------------------------------------------------------- + +On deb, rpm, and docker, the `scripts` folder is located under the home path, which is +/usr/share/{beatname_lc}/+ unless you change it. By default, the script assumes that you are running Elasticsearch on `127.0.0.1:9200`. Use the `-es` option -to specify a different location. For example: +to specify a different location. For example: ["source","sh",subs="attributes,callouts"] ---------------------------------------------------------------------- -./scripts/import_dashboards -es http://192.168.33.60:9200 +./scripts/import_dashboards -es http://192.168.33.60:9200 ---------------------------------------------------------------------- Use the `-user` option to specify the username and password to use for Elasticsearch authentication. There are a few ways to pass @@ -51,7 +58,7 @@ in the username and password. For example: ["source","sh",subs="attributes,callouts"] ----------------------------------------------------------------------- ./scripts/import_dashboards -es https://xyz.found.io -user user -pass password <1> -./scripts/import_dashboards -es https://xyz.found.io -user admin -pass $(cat ~/pass-file) <2> +./scripts/import_dashboards -es https://xyz.found.io -user admin -pass $(cat ~/pass-file) <2> ----------------------------------------------------------------------- <1> Specify the username and password as options. @@ -63,7 +70,7 @@ endif::allplatforms[] Open a PowerShell prompt as an Administrator (right-click the PowerShell icon and select *Run As Administrator*). If you are running Windows XP, you may need -to download and install PowerShell. +to download and install PowerShell. From the PowerShell prompt, change to the directory where you installed {beatname_uc}, and run the `import_dashboards.exe` script: @@ -103,6 +110,6 @@ pattern is selected to see {beatname_uc} data. image:./images/kibana-created-indexes.png[Discover tab with index selected] To open the loaded dashboards, go to the *Dashboard* page and select the -dashboard that you want to open. +dashboard that you want to open. image:./images/kibana-navigation-vis.png[Navigation widget in Kibana] diff --git a/libbeat/docs/gettingstarted.asciidoc b/libbeat/docs/gettingstarted.asciidoc index e508c5fe8827..fd10d933f5e3 100644 --- a/libbeat/docs/gettingstarted.asciidoc +++ b/libbeat/docs/gettingstarted.asciidoc @@ -1,5 +1,5 @@ [[getting-started]] -== Getting Started with Beats and the Elastic Stack +== Getting Started with Beats and the Elastic Stack Looking for an "ELK tutorial" that shows how to set up the Elastic stack for Beats? You've come to the right place. The topics in this section describe how to install and configure @@ -12,8 +12,8 @@ A regular _Beats setup_ consists of: * Kibana for the UI. See <>. * One or more Beats. You install the Beats on your servers to capture operational data. See <>. * Kibana dashboards for visualizing the data. - -See the https://www.elastic.co/support/matrix[Elastic Support Matrix] for information + +See the https://www.elastic.co/support/matrix[Elastic Support Matrix] for information about supported operating systems and product compatibility. NOTE: To get started, you can install Elasticsearch and Kibana on a @@ -173,10 +173,10 @@ The simplest architecture for the Beats platform setup consists of one or more B Elasticsearch, and Kibana. This architecture is easy to get started with and sufficient for networks with low traffic. It also uses the minimum amount of servers: a single machine running Elasticsearch and Kibana. The Beats -insert the transactions directly into the Elasticsearch instance. +insert the transactions directly into the Elasticsearch instance. If you want to perform additional processing or buffering on the data, however, -you'll want to install Logstash. +you'll want to install Logstash. An important advantage to this approach is that you can use Logstash to modify the data captured by Beats in any way you like. You can also @@ -267,44 +267,17 @@ endif::[] ==== Setting Up Logstash In this setup, the Beat sends events to Logstash. Logstash receives -these events by using the {logstashdoc}/plugins-inputs-beats.html[Beats input plugin for Logstash] and then sends the transaction to Elasticsearch by using the -{logstashdoc}/plugins-outputs-elasticsearch.html[Elasticsearch -output plugin for Logstash]. The Elasticsearch output plugin uses the bulk API, making -indexing very efficient. - -To set up Logstash: +these events by using the +{logstashdoc}/plugins-inputs-beats.html[Beats input plugin for Logstash] +and then sends the transaction to Elasticsearch by using the +{logstashdoc}/plugins-outputs-elasticsearch.html[Elasticsearch output plugin for Logstash]. +The Elasticsearch output plugin uses the bulk API, making indexing very efficient. -. Make sure you have the latest compatible version of the Beats input plugin for -Logstash installed. -+ -The Beats input plugin requires Logstash 1.5.4 or later. If you are using -Logstash 1.5.4, you must install the Beats input plugin before applying this -configuration because the plugin is not shipped with 1.5.4. -+ -To install -the required plugin, run the following command inside the logstash directory -(for deb and rpm installs, the directory is `/opt/logstash`). -+ -*deb, rpm, and mac:* -+ -["source","sh",subs="attributes,callouts"] ----------------------------------------------------------------------- -./bin/logstash-plugin install logstash-input-beats ----------------------------------------------------------------------- -+ -*win:* -+ -["source","sh",subs="attributes,callouts"] ----------------------------------------------------------------------- -bin\logstash-plugin install logstash-input-beats ----------------------------------------------------------------------- +To set up Logstash, you create a Logstash pipeline configuration file that +configures Logstash to listen on port 5044 for incoming Beats connections +and to index into Elasticsearch. For example, you can save the following +example configuration to a file called `logstash.conf`: -. Configure Logstash to listen on port 5044 for incoming Beats connections -and to index into Elasticsearch. You configure Logstash by creating a -configuration file. For example, you can save the following example configuration -to a file called `logstash.conf`: -+ --- [source,ruby] ------------------------------------------------------------------------------ input { @@ -313,6 +286,12 @@ input { } } +# The filter part of this file is commented out to indicate that it is +# optional. +# filter { +# +# } + output { elasticsearch { hosts => "localhost:9200" @@ -329,19 +308,21 @@ name to a date based on the Logstash `@timestamp` field. For example: <2> `%{[@metadata][type]}` sets the document type based on the value of the `type` metadata field. -Logstash uses this configuration to index events in Elasticsearch in the same -way that the Beat would, but you get additional buffering and other capabilities -provided by Logstash. --- +When you run Logstash with this configuration, it indexes events into +Elasticsearch in the same way that the Beat would, but you get access to other +capabilities provided by Logstash for collecting, enriching, and transforming +data. See the {logstashdoc}/introduction.html[Logstash introduction] for more +information about these capabilities. -To use this setup, you'll also need to configure your Beat to use Logstash. For more information, see the documentation for the Beat. +To use this setup, you'll also need to configure your Beat to use Logstash. +For more information, see the documentation for the Beat. [[logstash-input-update]] -==== Updating the Beats Input Plugin for Logstash +===== Updating the Beats Input Plugin for Logstash Plugins have their own release cycle and are often released independent of Logstash’s core release cycle. To ensure that you have the latest version of -the https://www.elastic.co/guide/en/logstash/current/plugins-inputs-beats.html[Beats input plugin for Logstash], +the https://www.elastic.co/guide/en/logstash/current/plugins-inputs-beats.html[Beats input plugin for Logstash], run the following command from your Logstash installation: *deb, rpm, and mac:* diff --git a/libbeat/docs/loggingconfig.asciidoc b/libbeat/docs/loggingconfig.asciidoc index df7d7e11f12b..2a27cf7d5c04 100644 --- a/libbeat/docs/loggingconfig.asciidoc +++ b/libbeat/docs/loggingconfig.asciidoc @@ -15,8 +15,8 @@ The `logging` section of the +{beatname_lc}.yml+ config file contains options for configuring the Beats logging output. The logging system can write logs to -syslog or rotate log files. If logging is not explicitly configured, file output -is used on Windows systems, and syslog output is used on Linux and OS X. +the syslog or rotate log files. If logging is not explicitly configured, file +output is used on Windows systems, and syslog output is used on Linux and OS X. [source,yaml] ------------------------------------------------------------------------------ @@ -29,8 +29,9 @@ logging.files: keepfiles: 7 ------------------------------------------------------------------------------ -In addition to the logging system, the logging output configuration can be -modified from the command line. +TIP: In addition to setting logging options in the config file, you can modify +the logging output configuration from the command line. See +<>. ==== Logging Options @@ -38,20 +39,40 @@ You can specify the following options in the `logging` section of the +{beatname ===== to_syslog -If enabled, sends all logging output to syslog. The default -value is false. +When true, writes all logging output to the syslog. ===== to_files -Writes all logging output to files subject to file rotation. The -default value is true. +When true, writes all logging output to files. The log files are automatically +rotated when the log file size limit is reached. +NOTE: {beatname_uc} only creates a log file if there is logging output. For +example, if you set the log <> to `error` and there are no errors, +there will be no log file in the directory specified for logs. + +[[level]] ===== level -Minimum log level. One of debug, info, warning, error or critical. If debug is -used, but no selectors are configured, the `*` selector will be used. -The default log level is "info". +Minimum log level. One of `debug`, `info`, `warning`, `error`, or `critical`. +The default log level is `info`. + +`debug`:: Logs debug messages, including a detailed printout of all events +flushed by the Beat. Also logs informational messages, warnings, errors, and +critical errors. When the log level is `debug`, you can specify a list of +<> to display debug messages for specific components. +If no selectors are specified, the `*` selector is used to display debug +messages for all components. + +`info`:: Logs informational messages, including the number of events +that are published. Also logs any warnings, errors, or critical errors. + +`warning`:: Logs warnings, errors, and critical errors. + +`error`:: Logs errors and critical errors. + +`critical`:: Logs critical errors only. +[[selectors]] ===== selectors The list of debugging-only selector tags used by different Beats components. Use `*` diff --git a/libbeat/docs/newbeat.asciidoc b/libbeat/docs/newbeat.asciidoc index 7a6a7b7186eb..952402d68fc2 100644 --- a/libbeat/docs/newbeat.asciidoc +++ b/libbeat/docs/newbeat.asciidoc @@ -352,7 +352,7 @@ countbeat: The config file is generated when you run `make setup` to <>. The file contains basic configuration information. To add configuration options to your Beat, you need to update the Go structures in -`config/config.go` and add the corresponding config options to `etc/beat.yml`. +`config/config.go` and add the corresponding config options to `_meta/beat.yml`. For example, if you add a config option called `path` to the Go structures: @@ -370,7 +370,7 @@ var DefaultConfig = Config{ ---------------------------------------------------------------------- -You also need to add `path` to `etc/beat.yml`: +You also need to add `path` to `_meta/beat.yml`: [source,yml] ---------------------------------------------------------------------- @@ -428,7 +428,7 @@ through the `client` variable. The `event := common.MapStr{}` stores the event in a json format, and `bt.client.PublishEvent(event)` publishes data to Elasticsearch. In the generated Beat, there are three fields in the event: @timestamp, type, and counter. -When you add fields to the event object, you also need to add them to the `etc/fields.yml` file: +When you add fields to the event object, you also need to add them to the `_meta/fields.yml` file: [source,yaml] ---------------------------------------------------------------------- diff --git a/libbeat/docs/outputconfig.asciidoc b/libbeat/docs/outputconfig.asciidoc index de0e55a2668f..d6201281a2d4 100644 --- a/libbeat/docs/outputconfig.asciidoc +++ b/libbeat/docs/outputconfig.asciidoc @@ -234,10 +234,10 @@ output.elasticsearch: pipelines: - pipeline: critical_pipeline when.equals: - type: "critical" + fields.type: "critical" - pipeline: normal_pipeline when.equals: - type: "normal" + fields.type: "normal" ------------------------------------------------------------------------------ ===== template diff --git a/libbeat/docs/regexp.asciidoc b/libbeat/docs/regexp.asciidoc index e8f1777aa5ee..379551272686 100644 --- a/libbeat/docs/regexp.asciidoc +++ b/libbeat/docs/regexp.asciidoc @@ -12,7 +12,21 @@ [[regexp-support]] == Regular Expression Support -{beatname_uc} regular expression support is based on https://godoc.org/regexp/syntax[RE2]. +{beatname_uc} regular expression support is based on https://godoc.org/regexp/syntax[RE2]. + +ifeval::["{beatname_lc}"=="filebeat"] + +{beatname_uc} has several configuration options that accept regular expressions. +For example, <>, +<>, <>, and +<> all accept regular expressions. Some options, +however, such as the prospector <> option, accept only +glob-based paths. + +endif::[] + +Before using a regular expression in the config file, refer to the documentation +to verify that the option you are setting accepts a regular expression. NOTE: We recommend that you wrap regular expressions in single quotation marks to work around YAML's string escaping rules. For example, `'^\[?[0-9][0-9]:?[0-9][0-9]|^[[:graph:]]+'`. @@ -33,7 +47,7 @@ The following patterns are supported: [options="header"] |======================= |Pattern |Description -|[[single-characters]]*Single Characters* 1+| +|[[single-characters]]*Single Characters* 1+| |`x` |single character |`.` |any character |`[xyz]` |character class @@ -49,7 +63,7 @@ The following patterns are supported: |[[composites]]*Composites* 1+| |`xy` |`x` followed by `y` |`x\|y` |`x` or `y` (prefer `x`) -|[[repetitions]]*Repetitions* 1+| +|[[repetitions]]*Repetitions* 1+| |`x*` |zero or more `x` |`x+` |one or more `x` |`x?` |zero or one `x` diff --git a/libbeat/docs/release.asciidoc b/libbeat/docs/release.asciidoc index d363fb22ba11..110d9271392e 100644 --- a/libbeat/docs/release.asciidoc +++ b/libbeat/docs/release.asciidoc @@ -6,6 +6,8 @@ -- This section summarizes the changes in each release. +* <> +* <> * <> * <> * <> diff --git a/libbeat/docs/shared-configuring.asciidoc b/libbeat/docs/shared-configuring.asciidoc new file mode 100644 index 000000000000..265bbc2b5468 --- /dev/null +++ b/libbeat/docs/shared-configuring.asciidoc @@ -0,0 +1,10 @@ +To configure {beatname_uc}, you edit the configuration file. For rpm and deb, +you'll find the configuration file at +/etc/{beatname_lc}/{beatname_lc}.yml+. Under +Docker, it's located at +/usr/share/{beatname_lc}/{beatname_lc}.yml+. For mac and win, +look in the archive that you just extracted. There’s also a full example +configuration file called +{beatname_lc}.full.yml+ that shows all non-deprecated +options. + +See the +{libbeat}/config-file-format.html[Config File Format] section of the +_Beats Platform Reference_ for more about the structure of the config file. diff --git a/libbeat/docs/shared-directory-layout.asciidoc b/libbeat/docs/shared-directory-layout.asciidoc index fcc6218ec286..0d9b1887b1a2 100644 --- a/libbeat/docs/shared-directory-layout.asciidoc +++ b/libbeat/docs/shared-directory-layout.asciidoc @@ -16,12 +16,12 @@ The directory layout of an installation is as follows: [cols="> in the configuration @@ -29,18 +29,18 @@ file. ==== Default paths -{beatname_uc} uses the following default paths unless you explicitly change them. +{beatname_uc} uses the following default paths unless you explicitly change them. [float] ===== deb and rpm [cols="> for Debian/Ubuntu, <> for Redhat/Centos/Fedora, <> for OS X, <> for any Docker platform, and <> for +Windows). + +[NOTE] +================================================== +If you use Apt or Yum, you can <> to update to the newest version more easily. + +See our https://www.elastic.co/downloads/beats/{beatname_lc}[download page] for +other installation options, such as 32-bit images. +================================================== diff --git a/libbeat/docs/shared-env-vars.asciidoc b/libbeat/docs/shared-env-vars.asciidoc index ae55ac7f6f71..8400d54e93ef 100644 --- a/libbeat/docs/shared-env-vars.asciidoc +++ b/libbeat/docs/shared-env-vars.asciidoc @@ -6,13 +6,20 @@ //// Use the appropriate variables defined in the index.asciidoc file to //// resolve Beat names: beatname_uc and beatname_lc. //// Use the following include to pull this content into a doc file: +//// :standalone: //// include::../../libbeat/docs/shared-env-vars.asciidoc[] +//// Specify :standalone: when this file is pulled into and index. When +//// the file is embedded in another file, do no specify :standalone: ////////////////////////////////////////////////////////////////////////// +ifdef::standalone[] + [[using-environ-vars]] == Using Environment Variables in the Configuration -You can use environment variable references in the +{beatname_lc}.yml+ file to +endif::[] + +You can use environment variable references in the config file to set values that need to be configurable during deployment. To do this, use: `${VAR}` @@ -22,18 +29,36 @@ Where `VAR` is the name of the environment variable. Each variable reference is replaced at startup by the value of the environment variable. The replacement is case-sensitive and occurs before the YAML file is parsed. References to undefined variables are replaced by empty strings unless -you specify a default value. To specify a default value, use: +you specify a default value or custom error text. + +To specify a default value, use: `${VAR:default_value}` Where `default_value` is the value to use if the environment variable is undefined. +To specify custom error text, use: + +`${VAR:?error_text}` + +Where `error_text` is custom text that will be prepended to the error +message if the environment variable cannot be expanded. + If you need to use a literal `${` in your configuration file then you can write `$${` to escape the expansion. After changing the value of an environment variable, you need to restart -{beatname_uc} to pick up the new value. +the Beat to pick up the new value. + +[NOTE] +================================== +You can also specify environment variables when you override a config +setting from the command line by using the `-E` option. For example: + +`-E name=${NAME}` + +================================== [float] === Examples @@ -43,9 +68,43 @@ and what each configuration looks like after replacement: [options="header"] |================================== -|Config source |Environment setting |Config after replacement -|`name: ${NAME}` |`export NAME=elastic` |`name: elastic` -|`name: ${NAME}` |no setting |`name:` -|`name: ${NAME:beats}` |no setting |`name: beats` -|`name: ${NAME:beats}` |`export NAME=elastic` |`name: elastic` +|Config source |Environment setting |Config after replacement +|`name: ${NAME}` |`export NAME=elastic` |`name: elastic` +|`name: ${NAME}` |no setting |`name:` +|`name: ${NAME:beats}` |no setting |`name: beats` +|`name: ${NAME:beats}` |`export NAME=elastic` |`name: elastic` +|`name: ${NAME:?You need to set the NAME environment variable}` |no setting | None. Returns an error message that's prepended with the custom text. +|`name: ${NAME:?You need to set the NAME environment variable}` |`export NAME=elastic` | `name: elastic` |================================== + +[float] +=== Specifying Complex Objects in Environment Variables + +You can specify complex objects, such as lists or dictionaries, in environment +variables by using a JSON-like syntax. + +As with JSON, dictionaries and lists are constructed using `{}` and `[]`. But +unlike JSON, the syntax allows for trailing commas and slightly different string +quotation rules. Strings can be unquoted, single-quoted, or double-quoted, as a +convenience for simple settings and to make it easier for you to mix quotation +usage in the shell. Arrays at the top-level do not require brackets (`[]`). + +For example, the following environment variable is set to a list: + +[source,yaml] +------------------------------------------------------------------------------- +ES_HOSTS="10.45.3.2:9220,10.45.3.1:9230" +------------------------------------------------------------------------------- + +You can reference this variable in the config file: + +[source,yaml] +------------------------------------------------------------------------------- +output.elasticsearch: + hosts: '${ES_HOSTS}' +------------------------------------------------------------------------------- + +When the Beat loads the config file, it resolves the environment variable and +replaces it with the specified list before reading the `hosts` setting. + +NOTE: Do not use double-quotes (`"`) to wrap regular expressions, or the backslash (`\`) will be interpreted as an escape character. diff --git a/libbeat/docs/shared-path-config.asciidoc b/libbeat/docs/shared-path-config.asciidoc index eaadea3179bc..b29e5addcd81 100644 --- a/libbeat/docs/shared-path-config.asciidoc +++ b/libbeat/docs/shared-path-config.asciidoc @@ -26,12 +26,12 @@ Here is an example configuration: [source,yaml] ------------------------------------------------------------------------------ path.home: /usr/share/beat -path.conf: /etc/beat +path.config: /etc/beat path.data: /var/lib/beat path.logs: /var/log/ ------------------------------------------------------------------------------ -Note that it is possible to override these options by using command line flags. +Note that it is possible to override these options by using command line flags. ==== Path Options @@ -51,7 +51,7 @@ Example: path.home: /usr/share/beats ------------------------------------------------------------------------------ -===== conf +===== config The configuration path for the {beatname_uc} installation. This is the default base path for configuration files, including the main YAML configuration file and the @@ -62,7 +62,7 @@ Example: [source,yaml] ------------------------------------------------------------------------------ -path.conf: /usr/share/beats/config +path.config: /usr/share/beats/config ------------------------------------------------------------------------------ ===== data diff --git a/libbeat/docs/shared-ssl-logstash-config.asciidoc b/libbeat/docs/shared-ssl-logstash-config.asciidoc index fb008889c709..0434b2ba830e 100644 --- a/libbeat/docs/shared-ssl-logstash-config.asciidoc +++ b/libbeat/docs/shared-ssl-logstash-config.asciidoc @@ -18,6 +18,9 @@ To use SSL mutual authentication: . Create a certificate authority (CA) and use it to sign the certificates that you plan to use for {beatname_uc} and Logstash. Creating a correct SSL/TLS infrastructure is outside the scope of this document. There are many online resources available that describe how to create certificates. ++ +TIP: If you are using X-Pack, you can use the +{securitydoc}/ssl-tls.html#generating-signed-certificates[certgen tool] to generate certificates. . Configure {beatname_uc} to use SSL. In the +{beatname_lc}.yml+ config file, specify the following settings under `ssl`: @@ -94,7 +97,7 @@ If the test is successful, you'll receive an empty response error: > Host: logs.mycompany.com:5044 > User-Agent: curl/7.43.0 > Accept: */* -> +> * Empty reply from server * Connection #0 to host logs.mycompany.com left intact curl: (52) Empty reply from server diff --git a/libbeat/docs/shared-template-load.asciidoc b/libbeat/docs/shared-template-load.asciidoc index 3ce8def274c5..6970581d18d0 100644 --- a/libbeat/docs/shared-template-load.asciidoc +++ b/libbeat/docs/shared-template-load.asciidoc @@ -78,6 +78,13 @@ cd {beatname_lc}-{version}-darwin-x86_64 curl -H 'Content-Type: application/json' -XPUT 'http://localhost:9200/_template/{beatname_lc}' -d@{beatname_lc}.template.json ---------------------------------------------------------------------- +*docker:* + +["source", "sh", subs="attributes"] +---------------------------------------------------------------------- +docker run --rm {dockerimage} curl -H 'Content-Type: application/json' -XPUT 'http://localhost:9200/_template/{beatname_lc}' -d@{beatname_lc}.template.json +---------------------------------------------------------------------- + *win:* endif::allplatforms[] diff --git a/libbeat/docs/version.asciidoc b/libbeat/docs/version.asciidoc index 5930aae83637..ae993c9d5510 100644 --- a/libbeat/docs/version.asciidoc +++ b/libbeat/docs/version.asciidoc @@ -1,4 +1,4 @@ -:stack-version: 5.4.1 -:doc-branch: 5.4 +:stack-version: 5.5.0 +:doc-branch: 5.5 :go-version: 1.7.6 :release-state: released diff --git a/libbeat/docs/yaml.asciidoc b/libbeat/docs/yaml.asciidoc index 7a24bdab9a37..a11e3e38e61d 100644 --- a/libbeat/docs/yaml.asciidoc +++ b/libbeat/docs/yaml.asciidoc @@ -6,7 +6,10 @@ //// Use the appropriate variables defined in the index.asciidoc file to //// resolve Beat names: beatname_uc and beatname_lc. //// Use the following include to pull this content into a doc file: +//// :standalone: //// include::../../libbeat/docs/yaml.asciidoc[] +//// Specify :standalone: when this file is pulled into and index. When +//// the file is embedded in another file, do no specify :standalone: ////////////////////////////////////////////////////////////////////////// ifdef::standalone[] @@ -40,7 +43,7 @@ simply uncomment the line and change the values. You can test your configuration file to verify that the structure is valid. Simply change to the directory where the binary is installed, and run -the Beat in the foreground with the `-configtest` flag specified. For example: +the Beat in the foreground with the `-configtest` flag specified. For example: ifdef::allplatforms[] @@ -65,7 +68,7 @@ You'll see a message if the Beat finds an error in the file. [float] === Wrap Regular Expressions in Single Quotation Marks -If you need to specify a regular expression in a YAML file, it's a good idea to wrap the regular expression in single quotation marks to work around YAML's tricky rules for string escaping. +If you need to specify a regular expression in a YAML file, it's a good idea to wrap the regular expression in single quotation marks to work around YAML's tricky rules for string escaping. For more information about YAML, see http://yaml.org/. @@ -74,9 +77,9 @@ For more information about YAML, see http://yaml.org/. === Wrap Paths in Single Quotation Marks Windows paths in particular sometimes contain spaces or characters, such as drive -letters or triple dots, that may be misinterpreted by the YAML parser. +letters or triple dots, that may be misinterpreted by the YAML parser. -To avoid this problem, it's a good idea to wrap paths in single quotation marks. +To avoid this problem, it's a good idea to wrap paths in single quotation marks. [float] [[avoid-leading-zeros]] @@ -85,7 +88,7 @@ To avoid this problem, it's a good idea to wrap paths in single quotation marks. If you use a leading zero (for example, `09`) in a numeric field without wrapping the value in single quotation marks, the value may be interpreted incorrectly by the YAML parser. If the value is a valid octal, it's converted -to an integer. If not, it's converted to a float. +to an integer. If not, it's converted to a float. To prevent unwanted type conversions, avoid using leading zeros in field values, or wrap the values in single quotation marks. diff --git a/libbeat/libbeat.template-es6x.json b/libbeat/libbeat.template-es6x.json index 0723038e70f4..d240d005de9b 100644 --- a/libbeat/libbeat.template-es6x.json +++ b/libbeat/libbeat.template-es6x.json @@ -36,9 +36,6 @@ } } }, - "fields": { - "properties": {} - }, "meta": { "properties": { "cloud": { diff --git a/libbeat/scripts/generate_template.py b/libbeat/scripts/generate_template.py index 9dfeb1447063..71b21d7c6e9a 100644 --- a/libbeat/scripts/generate_template.py +++ b/libbeat/scripts/generate_template.py @@ -305,13 +305,6 @@ def fill_field_properties(args, field, defaults, path): } }) - - properties[field["name"]] = { - "properties": {} - } - - - elif field.get("type") == "group": if len(path) > 0: path = path + "." + field["name"] diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 6119e5459cee..a00533b610b3 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -5601,7 +5601,7 @@ The amount of CPU time spent in involuntary wait by the virtual CPU while the hy type: long -The number of CPU cores. +The number of CPU cores. The CPU percentages can range from `[0, 100% * cores]`. [float] @@ -5611,7 +5611,7 @@ type: scaled_float format: percent -The percentage of CPU time spent in user space. On multi-core systems, you can have percentages that are greater than 100%. For example, if 3 cores are at 60% use, then the `cpu.user_p` will be 180%. +The percentage of CPU time spent in user space. On multi-core systems, you can have percentages that are greater than 100%. For example, if 3 cores are at 60% use, then the `system.cpu.user.pct` will be 180%. [float] diff --git a/metricbeat/docs/gettingstarted.asciidoc b/metricbeat/docs/gettingstarted.asciidoc index d80d88081c7e..81336491bf66 100644 --- a/metricbeat/docs/gettingstarted.asciidoc +++ b/metricbeat/docs/gettingstarted.asciidoc @@ -34,19 +34,7 @@ traffic or prevent Metricbeat from collecting metrics when there are network problems. Metrics from multiple Metricbeat instances will be combined on the Elasticsearch server. -To download and install Metricbeat, use the commands that work with your system -(<> for Debian/Ubuntu, <> for Redhat/Centos/Fedora, <> for OS X, and <> for Windows). - -[NOTE] -================================================== -If you use Apt or Yum, you can -<> to -update to the newest version more easily. - -See our https://www.elastic.co/downloads/beats/metricbeat[download page] for -other installation options, such as 32-bit images. -================================================== +include::../../libbeat/docs/shared-download-and-install.asciidoc[] [[deb]] *deb:* @@ -105,6 +93,24 @@ tar xzvf metricbeat-{version}-darwin-x86_64.tar.gz endif::[] +[[docker]] +*docker:* + +ifeval::["{release-state}"=="unreleased"] + +Version {stack-version} of {beatname_uc} has not yet been released. + +endif::[] + +ifeval::["{release-state}"!="unreleased"] + +["source", "shell", subs="attributes"] +------------------------------------------------ +docker pull {dockerimage} +------------------------------------------------ + +endif::[] + [[win]] *win:* @@ -151,15 +157,7 @@ For more information about these options, see [[metricbeat-configuration]] === Step 2: Configuring Metricbeat -To configure Metricbeat, you edit the configuration file. For rpm and deb, -you'll find the configuration file at `/etc/metricbeat/metricbeat.yml`. For mac -and win, look in the archive that you just extracted. There’s also a full -example configuration file called `metricbeat.full.yml` that shows all -non-deprecated options. - -See the -{libbeat}/config-file-format.html[Config File Format] section of the -_Beats Platform Reference_ for more about the structure of the config file. +include::../../libbeat/docs/shared-configuring.asciidoc[] Metricbeat uses <> to collect metrics. You configure each module individually. The following example shows the default configuration @@ -249,6 +247,7 @@ start Metricbeat in the foreground. ---------------------------------------------------------------------- sudo /etc/init.d/metricbeat start ---------------------------------------------------------------------- + *rpm:* [source,shell] @@ -256,6 +255,13 @@ sudo /etc/init.d/metricbeat start sudo /etc/init.d/metricbeat start ---------------------------------------------------------------------- +*docker:* + +["source", "shell", subs="attributes"] +---------------------------------------------------------------------- +docker run {dockerimage} +---------------------------------------------------------------------- + *mac:* [source,shell] @@ -309,4 +315,3 @@ image:./images/metricbeat_system_dashboard.png[Metricbeat Dashboard] :allplatforms: include::../../libbeat/docs/dashboards.asciidoc[] - diff --git a/metricbeat/docs/index.asciidoc b/metricbeat/docs/index.asciidoc index 9454e1709310..c768252b0741 100644 --- a/metricbeat/docs/index.asciidoc +++ b/metricbeat/docs/index.asciidoc @@ -11,6 +11,7 @@ include::../../libbeat/docs/version.asciidoc[] :beatname_lc: metricbeat :beatname_uc: Metricbeat :security: X-Pack Security +:dockerimage: docker.elastic.co/beats/{beatname_lc}:{version} include::./overview.asciidoc[] @@ -22,12 +23,12 @@ include::../../libbeat/docs/shared-directory-layout.asciidoc[] include::../../libbeat/docs/repositories.asciidoc[] +include::./running-on-docker.asciidoc[] + include::./upgrading.asciidoc[] include::./how-metricbeat-works.asciidoc[] -include::./metricbeat-in-a-container.asciidoc[] - include::./configuring-howto.asciidoc[] include::./metricbeat-filtering.asciidoc[] @@ -36,6 +37,7 @@ include::../../libbeat/docs/shared-config-ingest.asciidoc[] include::./configuring-logstash.asciidoc[] +:standalone: include::../../libbeat/docs/shared-env-vars.asciidoc[] :standalone: diff --git a/metricbeat/docs/metricbeat-in-a-container.asciidoc b/metricbeat/docs/running-on-docker.asciidoc similarity index 69% rename from metricbeat/docs/metricbeat-in-a-container.asciidoc rename to metricbeat/docs/running-on-docker.asciidoc index d15a43b36f40..f9c5c7d50a41 100644 --- a/metricbeat/docs/metricbeat-in-a-container.asciidoc +++ b/metricbeat/docs/running-on-docker.asciidoc @@ -1,45 +1,25 @@ -[[running-in-container]] -== Running Metricbeat in a Container - -ifeval::["{release-state}"=="released"] - -[NOTE] -================================================== -The https://github.com/elastic/beats-docker[official Docker images] for Beats -are available from the Elastic Docker registry. To retrieve the images, simply -issue the `docker pull` command: - -+docker pull docker.elastic.co/beats/metricbeat:{stack-version}+. - -The images are currently under development and should be considered -alpha-quality. We strongly recommend that you do not run these images -in a production environment. - -================================================== - -endif::[] +include::../../libbeat/docs/shared-docker.asciidoc[] +[float] +[[monitoring-host]] +=== Monitoring the Host Machine When executing Metricbeat in a container, there are some important things to be aware of if you want to monitor the host machine or other containers. Let's walk-through some examples using Docker as our container orchestration tool. -[float] -[[monitoring-host]] -=== Monitoring the Host Machine - This example highlights the changes required to make the system module work properly inside of a container. This enables Metricbeat to monitor the host machine from within the container. ["source","sh",subs="attributes"] ---- -sudo docker run \ +docker run \ --volume=/proc:/hostfs/proc:ro \ <1> --volume=/sys/fs/cgroup:/hostfs/sys/fs/cgroup:ro \ <2> --volume=/:/hostfs:ro \ <3> --net=host <4> - docker.elastic.co/beats/metricbeat:{stack-version} -system.hostfs=/hostfs + {dockerimage} -system.hostfs=/hostfs ---- <1> Metricbeat's <> collects much of its data through the Linux proc @@ -49,8 +29,8 @@ container's `/proc` is different than the host's `/proc`. To account for this, y can mount the host's `/proc` filesystem inside of the container and tell Metricbeat to look inside the `/hostfs` directory when looking for `/proc` by using the `-system.hostfs=/hostfs` CLI flag. -<2> If cgroup reporting is enabled for the -<>, then you need +<2> By default, cgroup reporting is enabled for the +<>, so you need to mount the host's cgroup mountpoints within the container. They need to be mounted inside the directory specified by the `-system.hostfs` CLI flag. <3> If you want to be able to monitor filesystems from the host by using the @@ -62,23 +42,28 @@ to make this file contain the host's network devices is to use the `--net=host` flag. This is due to Linux namespacing; simply bind mounting the host's `/proc` to `/hostfs/proc` is not sufficient. +NOTE: The special filesystems +/proc+ and +/sys+ are only available if the +host system is running Linux. Attempts to bind-mount these filesystems will +fail on Windows and MacOS. + [float] [[monitoring-service]] === Monitoring a Service in Another Container -Next let's look at an example of monitoring a containerized service from a +Next, let's look at an example of monitoring a containerized service from a Metricbeat container. ["source","sh",subs="attributes"] ---- -sudo docker run \ - --link some-mysql:mysql \ <1> +docker run \ + --network=mysqlnet \ <1> -e MYSQL_PASSWORD=secret \ <2> - docker.elastic.co/beats/metricbeat:{stack-version}  + {dockerimage} ---- -<1> Linking the containers enables Metricbeat access the exposed ports of the -mysql container, and it makes the hostname `mysql` resolvable to Metricbeat. +<1> Placing the Metricbeat and MySQL containers on the same Docker network +allows Metricbeat access to the exposed ports of the MySQL container, and +makes the hostname `mysql` resolvable to Metricbeat. <2> If you do not want to hardcode certain values into your Metricbeat configuration, then you can pass them into the container either as environment variables or as command line flags to Metricbeat (see the `-E` CLI flag in <>). @@ -95,7 +80,7 @@ metricbeat.modules: password: ${MYSQL_PASSWORD} <2> ---- -<1> The `mysql` hostname will resolve to the `some-mysql` container's address. +<1> The `mysql` hostname will resolve to the address of a container +named `mysql` on the `mysqlnet` Docker network. <2> The `MYSQL_PASSWORD` variable will be evaluated at startup. If the variable is not set, this will lead to an error at startup. - diff --git a/metricbeat/metricbeat.template-es2x.json b/metricbeat/metricbeat.template-es2x.json index a1c7207bd246..14d8ea974432 100644 --- a/metricbeat/metricbeat.template-es2x.json +++ b/metricbeat/metricbeat.template-es2x.json @@ -771,9 +771,6 @@ "index": "not_analyzed", "type": "string" }, - "labels": { - "properties": {} - }, "name": { "ignore_above": 1024, "index": "not_analyzed", @@ -793,9 +790,6 @@ "ignore_above": 1024, "index": "not_analyzed", "type": "string" - }, - "tags": { - "properties": {} } } }, @@ -902,9 +896,6 @@ } } }, - "labels": { - "properties": {} - }, "size": { "properties": { "regular": { @@ -914,9 +905,6 @@ "type": "long" } } - }, - "tags": { - "properties": {} } } }, @@ -1028,9 +1016,6 @@ } } }, - "fields": { - "properties": {} - }, "haproxy": { "properties": { "info": { @@ -1672,9 +1657,6 @@ "insync_replica": { "type": "boolean" }, - "isr": { - "properties": {} - }, "leader": { "type": "long" }, @@ -3641,9 +3623,6 @@ "index": "not_analyzed", "type": "string" }, - "percpu": { - "properties": {} - }, "stats": { "properties": { "system": { @@ -3937,9 +3916,6 @@ } } }, - "env": { - "properties": {} - }, "fd": { "properties": { "limit": { diff --git a/metricbeat/metricbeat.template-es6x.json b/metricbeat/metricbeat.template-es6x.json index 1981b5da20cb..8264859c2168 100644 --- a/metricbeat/metricbeat.template-es6x.json +++ b/metricbeat/metricbeat.template-es6x.json @@ -763,9 +763,6 @@ "ignore_above": 1024, "type": "keyword" }, - "labels": { - "properties": {} - }, "name": { "ignore_above": 1024, "type": "keyword" @@ -783,9 +780,6 @@ "status": { "ignore_above": 1024, "type": "keyword" - }, - "tags": { - "properties": {} } } }, @@ -895,9 +889,6 @@ } } }, - "labels": { - "properties": {} - }, "size": { "properties": { "regular": { @@ -907,9 +898,6 @@ "type": "long" } } - }, - "tags": { - "properties": {} } } }, @@ -1024,9 +1012,6 @@ } } }, - "fields": { - "properties": {} - }, "haproxy": { "properties": { "info": { @@ -1656,9 +1641,6 @@ "insync_replica": { "type": "boolean" }, - "isr": { - "properties": {} - }, "leader": { "type": "long" }, @@ -3605,9 +3587,6 @@ "ignore_above": 1024, "type": "keyword" }, - "percpu": { - "properties": {} - }, "stats": { "properties": { "system": { @@ -3897,9 +3876,6 @@ } } }, - "env": { - "properties": {} - }, "fd": { "properties": { "limit": { diff --git a/metricbeat/metricbeat.template.json b/metricbeat/metricbeat.template.json index d357f3aabf84..36e3bfd612df 100644 --- a/metricbeat/metricbeat.template.json +++ b/metricbeat/metricbeat.template.json @@ -766,9 +766,6 @@ "ignore_above": 1024, "type": "keyword" }, - "labels": { - "properties": {} - }, "name": { "ignore_above": 1024, "type": "keyword" @@ -786,9 +783,6 @@ "status": { "ignore_above": 1024, "type": "keyword" - }, - "tags": { - "properties": {} } } }, @@ -898,9 +892,6 @@ } } }, - "labels": { - "properties": {} - }, "size": { "properties": { "regular": { @@ -910,9 +901,6 @@ "type": "long" } } - }, - "tags": { - "properties": {} } } }, @@ -1027,9 +1015,6 @@ } } }, - "fields": { - "properties": {} - }, "haproxy": { "properties": { "info": { @@ -1659,9 +1644,6 @@ "insync_replica": { "type": "boolean" }, - "isr": { - "properties": {} - }, "leader": { "type": "long" }, @@ -3608,9 +3590,6 @@ "ignore_above": 1024, "type": "keyword" }, - "percpu": { - "properties": {} - }, "stats": { "properties": { "system": { @@ -3900,9 +3879,6 @@ } } }, - "env": { - "properties": {} - }, "fd": { "properties": { "limit": { diff --git a/metricbeat/module/system/cpu/_meta/data.json b/metricbeat/module/system/cpu/_meta/data.json index c0e041c9fa2c..17d7e58cde39 100644 --- a/metricbeat/module/system/cpu/_meta/data.json +++ b/metricbeat/module/system/cpu/_meta/data.json @@ -11,13 +11,14 @@ }, "system": { "cpu": { + "cores": 8, "idle": { - "pct": 0.852, - "ticks": 44421033 + "pct": 7.0854, + "ticks": 1617015818 }, "iowait": { "pct": 0, - "ticks": 159735 + "ticks": 0 }, "irq": { "pct": 0, @@ -29,19 +30,19 @@ }, "softirq": { "pct": 0, - "ticks": 14070 + "ticks": 0 }, "steal": { "pct": 0, "ticks": 0 }, "system": { - "pct": 0.0408, - "ticks": 305704 + "pct": 0.3317, + "ticks": 40488863 }, "user": { - "pct": 0.1071, - "ticks": 841974 + "pct": 0.5829, + "ticks": 48194733 } } }, diff --git a/metricbeat/module/system/cpu/_meta/fields.yml b/metricbeat/module/system/cpu/_meta/fields.yml index abb3e03ed3f8..3d874af492fa 100644 --- a/metricbeat/module/system/cpu/_meta/fields.yml +++ b/metricbeat/module/system/cpu/_meta/fields.yml @@ -6,14 +6,14 @@ - name: cores type: long description: > - The number of CPU cores. + The number of CPU cores. The CPU percentages can range from `[0, 100% * cores]`. - name: user.pct type: scaled_float format: percent description: > The percentage of CPU time spent in user space. On multi-core systems, you can have percentages that are greater than 100%. - For example, if 3 cores are at 60% use, then the `cpu.user_p` will be 180%. + For example, if 3 cores are at 60% use, then the `system.cpu.user.pct` will be 180%. - name: system.pct type: scaled_float diff --git a/metricbeat/module/system/cpu/helper.go b/metricbeat/module/system/cpu/helper.go index 391fed94b2c4..952346e517f7 100644 --- a/metricbeat/module/system/cpu/helper.go +++ b/metricbeat/module/system/cpu/helper.go @@ -9,6 +9,9 @@ import ( sigar "github.com/elastic/gosigar" ) +// NumCPU is the number of CPU cores the system has. +var NumCPU = runtime.NumCPU() + type CPU struct { CpuPerCore bool LastCpuTimes *CpuTimes @@ -72,7 +75,7 @@ func GetCpuPercentage(last *CpuTimes, current *CpuTimes) *CpuTimes { perc := 0.0 delta := int64(field2 - field1) perc = float64(delta) / float64(allDelta) - return system.Round(perc, .5, 4) + return system.Round(perc*float64(NumCPU), .5, 4) } current.UserPercent = calculate(current.Cpu.User, last.Cpu.User) diff --git a/metricbeat/module/system/cpu/helper_test.go b/metricbeat/module/system/cpu/helper_test.go index af9f22a260ba..a25bbe560f49 100644 --- a/metricbeat/module/system/cpu/helper_test.go +++ b/metricbeat/module/system/cpu/helper_test.go @@ -4,6 +4,7 @@ package cpu import ( + "runtime" "testing" "github.com/elastic/gosigar" @@ -21,6 +22,8 @@ func TestGetCpuTimes(t *testing.T) { } func TestCpuPercentage(t *testing.T) { + NumCPU = 1 + defer func() { NumCPU = runtime.NumCPU() }() cpu := CPU{} diff --git a/metricbeat/module/system/process/_meta/docs.asciidoc b/metricbeat/module/system/process/_meta/docs.asciidoc index 7c37c8860a56..3e885a7ff13b 100644 --- a/metricbeat/module/system/process/_meta/docs.asciidoc +++ b/metricbeat/module/system/process/_meta/docs.asciidoc @@ -15,7 +15,7 @@ This metricset is available on: On Linux this metricset will collect metrics from any cgroups that the process is a member of. This feature is enabled by default and can be disabled by adding -`process.cgroup.enabled: false` to the system module configuration. +`process.cgroups.enabled: false` to the system module configuration. [float] === Process Environment Variables diff --git a/packetbeat/docs/command-line.asciidoc b/packetbeat/docs/command-line.asciidoc index e9738a2948c6..f3d347dd0331 100644 --- a/packetbeat/docs/command-line.asciidoc +++ b/packetbeat/docs/command-line.asciidoc @@ -1,4 +1,4 @@ -[[packetbeat-command]] +[[command-line-options]] === Command Line Options The following command line options are available for Packetbeat. To use these options, diff --git a/packetbeat/docs/flows.asciidoc b/packetbeat/docs/flows.asciidoc index ae1dea1de485..4f7dde72c7ee 100644 --- a/packetbeat/docs/flows.asciidoc +++ b/packetbeat/docs/flows.asciidoc @@ -77,4 +77,4 @@ contains an intermediate report about a flow that it's tracking. When the flow completes, Packetbeat sends one last event with `final` set to `true`. If you want to aggregate sums of traffic, you need to filter on `final:true`, or use some other technique, so that you get only the latest update from each flow. -You can disable intermediate reports by setting `period: -1`. +You can disable intermediate reports by setting `period: -1s`. diff --git a/packetbeat/docs/gettingstarted.asciidoc b/packetbeat/docs/gettingstarted.asciidoc index f08e8ece675f..2024ffb5a19b 100644 --- a/packetbeat/docs/gettingstarted.asciidoc +++ b/packetbeat/docs/gettingstarted.asciidoc @@ -19,22 +19,13 @@ After installing the Elastic Stack, read the following topics to learn how to in * <> * <> * <> -* <> +* <> * <> [[packetbeat-installation]] === Step 1: Installing Packetbeat -To download and install Packetbeat on your application servers, use the commands -that work with your system (<> for Debian/Ubuntu, <> for -Redhat/Centos/Fedora, <> for OS X, and <> for Windows). - -[NOTE] -================================================== -If you use Apt or Yum, you can <> to update to the newest version more easily. - -See our https://www.elastic.co/downloads/beats/packetbeat[download page] for other installation options, such as 32-bit images. -================================================== +include::../../libbeat/docs/shared-download-and-install.asciidoc[] [[deb]] *deb:* @@ -76,6 +67,24 @@ sudo rpm -vi packetbeat-{version}-x86_64.rpm endif::[] +[[docker]] +*docker:* + +ifeval::["{release-state}"=="unreleased"] + +Version {stack-version} of {beatname_uc} has not yet been released. + +endif::[] + +ifeval::["{release-state}"!="unreleased"] + +["source", "shell", subs="attributes"] +------------------------------------------------ +docker pull {dockerimage} +------------------------------------------------ + +endif::[] + [[mac]] *mac:* @@ -138,14 +147,7 @@ more information about these options, see <>. [[configuring-packetbeat]] === Step 2: Configuring Packetbeat -To configure Packetbeat, you edit the configuration file. For rpm and deb, you'll -find the configuration file at `/etc/packetbeat/packetbeat.yml`. For mac and win, look in -the archive that you just extracted. There’s also a full example configuration file called -`packetbeat.full.yml` that shows all non-deprecated options. - -See the -{libbeat}/config-file-format.html[Config File Format] section of the -_Beats Platform Reference_ for more about the structure of the config file. +include::../../libbeat/docs/shared-configuring.asciidoc[] To configure Packetbeat: @@ -255,7 +257,7 @@ include::../../libbeat/docs/shared-template-load.asciidoc[] Run Packetbeat by issuing the command that is appropriate for your platform. NOTE: If you use an init.d script to start Packetbeat on deb or rpm, you can't -specify command line flags (see <>). To specify flags, +specify command line flags (see <>). To specify flags, start Packetbeat in the foreground. *deb:* @@ -272,6 +274,13 @@ sudo /etc/init.d/packetbeat start sudo /etc/init.d/packetbeat start ---------------------------------------------------------------------- +*docker:* + +["source", "shell", subs="attributes"] +---------------------------------------------------------------------- +docker run {dockerimage} +---------------------------------------------------------------------- + *mac:* [source,shell] @@ -327,4 +336,3 @@ image:./images/packetbeat-statistics.png[Packetbeat statistics] :allplatforms: include::../../libbeat/docs/dashboards.asciidoc[] - diff --git a/packetbeat/docs/index.asciidoc b/packetbeat/docs/index.asciidoc index d79960ea100f..9cf45644caf9 100644 --- a/packetbeat/docs/index.asciidoc +++ b/packetbeat/docs/index.asciidoc @@ -16,6 +16,7 @@ include::../../libbeat/docs/version.asciidoc[] :beatname_lc: packetbeat :beatname_uc: Packetbeat :security: X-Pack Security +:dockerimage: docker.elastic.co/beats/{beatname_lc}:{version} include::./overview.asciidoc[] @@ -28,6 +29,8 @@ include::../../libbeat/docs/shared-directory-layout.asciidoc[] include::../../libbeat/docs/repositories.asciidoc[] +include::./running-on-docker.asciidoc[] + include::./upgrading.asciidoc[] include::./configuring-howto.asciidoc[] @@ -46,6 +49,9 @@ include::./configuring-logstash.asciidoc[] include::./flows.asciidoc[] +:standalone: +include::../../libbeat/docs/shared-env-vars.asciidoc[] + include::./thrift.asciidoc[] include::./maintaining-topology.asciidoc[] diff --git a/packetbeat/docs/reference/configuration/packetbeat-options.asciidoc b/packetbeat/docs/reference/configuration/packetbeat-options.asciidoc index 8267357974b7..b32b8efe9578 100644 --- a/packetbeat/docs/reference/configuration/packetbeat-options.asciidoc +++ b/packetbeat/docs/reference/configuration/packetbeat-options.asciidoc @@ -211,6 +211,8 @@ packetbeat.flows: period: 10s ------------------------------------------------------------------------------ +See <> for more information. + ==== Options You can specify the following options in the `flows` section of the +{beatname_lc}.yml+ config file: diff --git a/packetbeat/docs/running-on-docker.asciidoc b/packetbeat/docs/running-on-docker.asciidoc new file mode 100644 index 000000000000..fd939cc46f4d --- /dev/null +++ b/packetbeat/docs/running-on-docker.asciidoc @@ -0,0 +1,29 @@ +include::../../libbeat/docs/shared-docker.asciidoc[] + +=== Required Network Capabilities + +Under Docker, Packetbeat runs as a non-root user, but requires some privileged +network capabilities to operate correctly. Ensure that the +NET_ADMIN+ +capability is available to the container. + +["source","sh",subs="attributes"] +---- +docker run --cap-add=NET_ADMIN {dockerimage} +---- + +=== Capturing Traffic from the Host System + +By default, Docker networking will connect the Packetbeat container to an +isolated virtual network, with a limited view of network traffic. You may wish +to connect the container directly to the host network in order to see traffic +destined for, and originating from, the host system. With +docker run+, this can +be achieved by specifying +--network=host+. + +["source","sh",subs="attributes"] +---- +docker run --cap-add=NET_ADMIN --network=host {dockerimage} +---- + +NOTE: On Windows and MacOS, specifying +--network=host+ will bind the +container's network interface to the virtual interface of Docker's embedded +Linux virtual machine, not to the physical interface of the host system. diff --git a/packetbeat/packetbeat.template-es2x.json b/packetbeat/packetbeat.template-es2x.json index 8be5460114bc..0ebd86cbf9dc 100644 --- a/packetbeat/packetbeat.template-es2x.json +++ b/packetbeat/packetbeat.template-es2x.json @@ -33,9 +33,6 @@ "index": "not_analyzed", "type": "string" }, - "arguments": { - "properties": {} - }, "auto-delete": { "type": "boolean" }, @@ -94,9 +91,6 @@ "index": "not_analyzed", "type": "string" }, - "headers": { - "properties": {} - }, "if-empty": { "type": "boolean" }, @@ -574,9 +568,6 @@ } } }, - "supported": { - "properties": {} - }, "warnings": { "ignore_above": 1024, "index": "not_analyzed", @@ -864,9 +855,6 @@ "domloadtime": { "type": "long" }, - "fields": { - "properties": {} - }, "final": { "ignore_above": 1024, "index": "not_analyzed", @@ -888,9 +876,6 @@ }, "type": "string" }, - "headers": { - "properties": {} - }, "params": { "ignore_above": 1024, "index": "not_analyzed", @@ -910,9 +895,6 @@ "index": "not_analyzed", "type": "string" }, - "headers": { - "properties": {} - }, "phrase": { "ignore_above": 1024, "index": "not_analyzed", @@ -1020,9 +1002,6 @@ "initial": { "type": "long" }, - "keys": { - "properties": {} - }, "line": { "ignore_above": 1024, "index": "not_analyzed", @@ -1061,9 +1040,6 @@ "index": "not_analyzed", "type": "string" }, - "values": { - "properties": {} - }, "vbucket": { "type": "long" }, @@ -1096,9 +1072,6 @@ "flags": { "type": "long" }, - "keys": { - "properties": {} - }, "opaque": { "type": "long" }, @@ -1110,9 +1083,6 @@ "opcode_value": { "type": "long" }, - "stats": { - "properties": {} - }, "status": { "ignore_above": 1024, "index": "not_analyzed", @@ -1129,9 +1099,6 @@ "value": { "type": "long" }, - "values": { - "properties": {} - }, "version": { "ignore_above": 1024, "index": "not_analyzed", diff --git a/packetbeat/packetbeat.template-es6x.json b/packetbeat/packetbeat.template-es6x.json index afb1dd1e78d9..359f593337fa 100644 --- a/packetbeat/packetbeat.template-es6x.json +++ b/packetbeat/packetbeat.template-es6x.json @@ -26,9 +26,6 @@ "ignore_above": 1024, "type": "keyword" }, - "arguments": { - "properties": {} - }, "auto-delete": { "type": "boolean" }, @@ -79,9 +76,6 @@ "ignore_above": 1024, "type": "keyword" }, - "headers": { - "properties": {} - }, "if-empty": { "type": "boolean" }, @@ -498,9 +492,6 @@ } } }, - "supported": { - "properties": {} - }, "warnings": { "ignore_above": 1024, "type": "keyword" @@ -755,9 +746,6 @@ "domloadtime": { "type": "long" }, - "fields": { - "properties": {} - }, "final": { "ignore_above": 1024, "type": "keyword" @@ -774,9 +762,6 @@ "norms": false, "type": "text" }, - "headers": { - "properties": {} - }, "params": { "ignore_above": 1024, "type": "keyword" @@ -793,9 +778,6 @@ "ignore_above": 1024, "type": "keyword" }, - "headers": { - "properties": {} - }, "phrase": { "ignore_above": 1024, "type": "keyword" @@ -894,9 +876,6 @@ "initial": { "type": "long" }, - "keys": { - "properties": {} - }, "line": { "ignore_above": 1024, "type": "keyword" @@ -931,9 +910,6 @@ "ignore_above": 1024, "type": "keyword" }, - "values": { - "properties": {} - }, "vbucket": { "type": "long" }, @@ -964,9 +940,6 @@ "flags": { "type": "long" }, - "keys": { - "properties": {} - }, "opaque": { "type": "long" }, @@ -977,9 +950,6 @@ "opcode_value": { "type": "long" }, - "stats": { - "properties": {} - }, "status": { "ignore_above": 1024, "type": "keyword" @@ -994,9 +964,6 @@ "value": { "type": "long" }, - "values": { - "properties": {} - }, "version": { "ignore_above": 1024, "type": "keyword" diff --git a/packetbeat/packetbeat.template.json b/packetbeat/packetbeat.template.json index 264a7a462ca7..0460b6d3625d 100644 --- a/packetbeat/packetbeat.template.json +++ b/packetbeat/packetbeat.template.json @@ -29,9 +29,6 @@ "ignore_above": 1024, "type": "keyword" }, - "arguments": { - "properties": {} - }, "auto-delete": { "type": "boolean" }, @@ -82,9 +79,6 @@ "ignore_above": 1024, "type": "keyword" }, - "headers": { - "properties": {} - }, "if-empty": { "type": "boolean" }, @@ -501,9 +495,6 @@ } } }, - "supported": { - "properties": {} - }, "warnings": { "ignore_above": 1024, "type": "keyword" @@ -758,9 +749,6 @@ "domloadtime": { "type": "long" }, - "fields": { - "properties": {} - }, "final": { "ignore_above": 1024, "type": "keyword" @@ -777,9 +765,6 @@ "norms": false, "type": "text" }, - "headers": { - "properties": {} - }, "params": { "ignore_above": 1024, "type": "keyword" @@ -796,9 +781,6 @@ "ignore_above": 1024, "type": "keyword" }, - "headers": { - "properties": {} - }, "phrase": { "ignore_above": 1024, "type": "keyword" @@ -897,9 +879,6 @@ "initial": { "type": "long" }, - "keys": { - "properties": {} - }, "line": { "ignore_above": 1024, "type": "keyword" @@ -934,9 +913,6 @@ "ignore_above": 1024, "type": "keyword" }, - "values": { - "properties": {} - }, "vbucket": { "type": "long" }, @@ -967,9 +943,6 @@ "flags": { "type": "long" }, - "keys": { - "properties": {} - }, "opaque": { "type": "long" }, @@ -980,9 +953,6 @@ "opcode_value": { "type": "long" }, - "stats": { - "properties": {} - }, "status": { "ignore_above": 1024, "type": "keyword" @@ -997,9 +967,6 @@ "value": { "type": "long" }, - "values": { - "properties": {} - }, "version": { "ignore_above": 1024, "type": "keyword" diff --git a/packetbeat/procs/procs.go b/packetbeat/procs/procs.go index 6b34cf7d48cc..233cfafaf45c 100644 --- a/packetbeat/procs/procs.go +++ b/packetbeat/procs/procs.go @@ -244,7 +244,7 @@ func hexToIpv4(word string) (net.IP, error) { func hexToIpv6(word string) (net.IP, error) { p := make(net.IP, net.IPv6len) for i := 0; i < 4; i++ { - part, err := strconv.ParseInt(word[i*8:(i+1)*8], 16, 32) + part, err := strconv.ParseUint(word[i*8:(i+1)*8], 16, 32) if err != nil { return nil, err } @@ -324,12 +324,12 @@ func (proc *ProcessesWatcher) updateMap() { } func socketsFromProc(filename string, ipv6 bool) ([]*socketInfo, error) { - file, err := os.Open("/proc/net/tcp") + file, err := os.Open(filename) if err != nil { return nil, err } defer file.Close() - return parseProcNetTCP(file, false) + return parseProcNetTCP(file, ipv6) } // Parses the /proc/net/tcp file diff --git a/vendor/github.com/elastic/gosigar/CHANGELOG.md b/vendor/github.com/elastic/gosigar/CHANGELOG.md index 548b1ceee4a7..2d1e9ade6861 100644 --- a/vendor/github.com/elastic/gosigar/CHANGELOG.md +++ b/vendor/github.com/elastic/gosigar/CHANGELOG.md @@ -2,15 +2,10 @@ All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](http://semver.org/). -## [Unreleased] - -### Added +## [0.2.1] ### Changed - -### Deprecated - -### Removed +- Fixed Windows issue that caused a hang during `init()` if WMI wasn't ready. #74 ## [0.2.0] diff --git a/vendor/github.com/elastic/gosigar/sigar_windows.go b/vendor/github.com/elastic/gosigar/sigar_windows.go index c5f665d920d4..79105aebaf8b 100644 --- a/vendor/github.com/elastic/gosigar/sigar_windows.go +++ b/vendor/github.com/elastic/gosigar/sigar_windows.go @@ -7,6 +7,7 @@ import ( "path/filepath" "runtime" "strings" + "sync" "syscall" "time" @@ -41,7 +42,8 @@ var ( // bootTime is the time when the OS was last booted. This value may be nil // on operating systems that do not support the WMI query used to obtain it. - bootTime *time.Time + bootTime *time.Time + bootTimeLock sync.Mutex ) func init() { @@ -49,14 +51,6 @@ func init() { // PROCESS_QUERY_LIMITED_INFORMATION cannot be used on 2003 or XP. processQueryLimitedInfoAccess = syscall.PROCESS_QUERY_INFORMATION } - - if version.IsWindowsVistaOrGreater() { - // The minimum supported client for Win32_OperatingSystem is Windows Vista. - os, err := getWin32OperatingSystem() - if err == nil { - bootTime = &os.LastBootUpTime - } - } } func (self *LoadAverage) Get() error { @@ -80,11 +74,21 @@ func (self *ProcFDUsage) Get(pid int) error { } func (self *Uptime) Get() error { - if bootTime == nil { - // Minimum supported OS is Windows Vista. + // Minimum supported OS is Windows Vista. + if !version.IsWindowsVistaOrGreater() { return ErrNotImplemented{runtime.GOOS} } + bootTimeLock.Lock() + defer bootTimeLock.Unlock() + if bootTime == nil { + os, err := getWin32OperatingSystem() + if err != nil { + return errors.Wrap(err, "failed to get boot time using WMI") + } + bootTime = &os.LastBootUpTime + } + self.Length = time.Since(*bootTime).Seconds() return nil } diff --git a/winlogbeat/docs/command-line.asciidoc b/winlogbeat/docs/command-line.asciidoc index 32b63d7c6b0c..a45e37b844da 100644 --- a/winlogbeat/docs/command-line.asciidoc +++ b/winlogbeat/docs/command-line.asciidoc @@ -1,4 +1,4 @@ -[[winlogbeat-command-line-options]] +[[command-line-options]] === Command Line Options Winlogbeat does not have any Winlogbeat-specific command line options. Instead, diff --git a/winlogbeat/docs/getting-started.asciidoc b/winlogbeat/docs/getting-started.asciidoc index 0696f8e096b3..bc5f5b656d7d 100644 --- a/winlogbeat/docs/getting-started.asciidoc +++ b/winlogbeat/docs/getting-started.asciidoc @@ -19,7 +19,7 @@ After installing the Elastic Stack, read the following topics to learn how to in * <> * <> * <> -* <> +* <> * <> [[winlogbeat-installation]] diff --git a/winlogbeat/docs/index.asciidoc b/winlogbeat/docs/index.asciidoc index bb6c2073f22e..eef8ae0491fe 100644 --- a/winlogbeat/docs/index.asciidoc +++ b/winlogbeat/docs/index.asciidoc @@ -31,6 +31,7 @@ include::./winlogbeat-filtering.asciidoc[] include::../../libbeat/docs/shared-config-ingest.asciidoc[] +:standalone: include::../../libbeat/docs/shared-env-vars.asciidoc[] :standalone: diff --git a/winlogbeat/winlogbeat.template-es2x.json b/winlogbeat/winlogbeat.template-es2x.json index 6bd255438efc..26d93f02ac5c 100644 --- a/winlogbeat/winlogbeat.template-es2x.json +++ b/winlogbeat/winlogbeat.template-es2x.json @@ -55,15 +55,9 @@ "index": "not_analyzed", "type": "string" }, - "event_data": { - "properties": {} - }, "event_id": { "type": "long" }, - "fields": { - "properties": {} - }, "keywords": { "ignore_above": 1024, "index": "not_analyzed", @@ -199,9 +193,6 @@ } } }, - "user_data": { - "properties": {} - }, "version": { "type": "long" }, diff --git a/winlogbeat/winlogbeat.template-es6x.json b/winlogbeat/winlogbeat.template-es6x.json index 0422a942af6c..67f3e5567939 100644 --- a/winlogbeat/winlogbeat.template-es6x.json +++ b/winlogbeat/winlogbeat.template-es6x.json @@ -44,15 +44,9 @@ "ignore_above": 1024, "type": "keyword" }, - "event_data": { - "properties": {} - }, "event_id": { "type": "long" }, - "fields": { - "properties": {} - }, "keywords": { "ignore_above": 1024, "type": "keyword" @@ -163,9 +157,6 @@ } } }, - "user_data": { - "properties": {} - }, "version": { "type": "long" }, diff --git a/winlogbeat/winlogbeat.template.json b/winlogbeat/winlogbeat.template.json index e783cc098e1e..256f12c8e544 100644 --- a/winlogbeat/winlogbeat.template.json +++ b/winlogbeat/winlogbeat.template.json @@ -47,15 +47,9 @@ "ignore_above": 1024, "type": "keyword" }, - "event_data": { - "properties": {} - }, "event_id": { "type": "long" }, - "fields": { - "properties": {} - }, "keywords": { "ignore_above": 1024, "type": "keyword" @@ -166,9 +160,6 @@ } } }, - "user_data": { - "properties": {} - }, "version": { "type": "long" },