diff --git a/filebeat/docs/getting-started.asciidoc b/filebeat/docs/getting-started.asciidoc index 831a6e36a45..789dde80c1e 100644 --- a/filebeat/docs/getting-started.asciidoc +++ b/filebeat/docs/getting-started.asciidoc @@ -23,6 +23,7 @@ mac>> for OS X, and <> for Windows). If you use Apt or Yum, you can {libbeat}/setup-repositories.html[install Filebeat from our repositories] to update to the newest version more easily. See our https://www.elastic.co/downloads/beats/filebeat[download page] for other installation options, such as 32-bit images. + ================================================== [[deb]] @@ -91,17 +92,10 @@ default values for most configuration options. [source,yaml] ------------------------------------------------------------------------------------- filebeat.prospectors: -# Each - is a prospector. Most options can be set at the prospector level, so -# you can use different prospectors for various configurations. -# Below are the prospector specific configurations. - - input_type: log - - # Paths that should be crawled and fetched. Glob based paths. paths: - /var/log/*.log #- c:\programdata\elasticsearch\logs\* - ------------------------------------------------------------------------------------- To configure Filebeat: @@ -115,7 +109,7 @@ For the most basic Filebeat configuration, you can define a single prospector wi filebeat.prospectors: - input_type: log paths: - - "/var/log/*.log" + - /var/log/*.log ------------------------------------------------------------------------------------- + The prospector in this example harvests all files in the path `/var/log/*.log`, which means @@ -131,11 +125,7 @@ fetch all files in all subdirectories of a directory. + [source,yaml] ---------------------------------------------------------------------- -# Configure what outputs to use when sending the data collected by the beat. -# Multiple outputs may be used. - output.elasticsearch: - # Array of hosts to connect to. hosts: ["192.168.1.42:9200"] ---------------------------------------------------------------------- + @@ -197,12 +187,12 @@ Filebeat is now ready to send log files to your defined output. We don't offer prebuilt dashboards for visualizing Filebeat data. However, to make it easier for you to explore Filebeat data in Kibana, we've created a Filebeat index pattern: `filebeat-*`. To load this pattern, you can use the script -that's provided for loading dashboards. +that's provided for loading dashboards. For more information about running the script, see {libbeat}/load-kibana-dashboards.html[Loading the Beats Dashboards]. After you've created the index pattern, you can select the `filebeat-*` index pattern in Kibana to explore -Filebeat data. +Filebeat data. image:./images/filebeat-discover-tab.png[] diff --git a/filebeat/docs/migration.asciidoc b/filebeat/docs/migration.asciidoc index 836dce99316..99cdf216195 100644 --- a/filebeat/docs/migration.asciidoc +++ b/filebeat/docs/migration.asciidoc @@ -35,13 +35,13 @@ Beats input plugin on the same Logstash instances, but set up the Beats input pl all the machines to Filebeat, you can remove the Lumberjack plugin. We realize that opening additional ports may not be feasible in your organization. Another option for phased migration -to Filebeat is to ship data from Logstash Forwarder directly to the Beats input plugin. +to Filebeat is to ship data from Logstash Forwarder directly to the Beats input plugin. IMPORTANT: This data shipping path is only supported for migrating to Filebeat and will no longer be supported when Logstash Forwarder reaches https://www.elastic.co/support/eol[End of Life]. What's required? -* The https://www.elastic.co/guide/en/logstash/current/plugins-inputs-beats.html[Beats input plugin for Logstash] +* The https://www.elastic.co/guide/en/logstash/current/plugins-inputs-beats.html[Beats input plugin for Logstash] version 2.2.8 or later. * SSL must be explicitly enabled in the Beats input plugin (`ssl => true`) because SSL is on by default with Logstash Forwarder. The SSL/TLS configs should be the same for both the Logstash Forwarder and Filebeat instances. @@ -125,11 +125,10 @@ The equivalent `prospectors` section would look like this: [source,yaml] ------------------------------------------------------------------------------------- filebeat.prospectors: -# Each - is a prospector. Below are the prospector specific configurations - input_type: log paths: - /var/log/messages - - "/var/log/*.log" + - /var/log/*.log document_type: syslog <1> fields: service: apache @@ -139,7 +138,7 @@ filebeat.prospectors: document_type: stdin - input_type: log paths: - - "/var/log/apache2/httpd-*.log" + - /var/log/apache2/httpd-*.log document_type: apache ------------------------------------------------------------------------------------- @@ -201,22 +200,12 @@ The equivalent in Filebeat would look like this: [source,yaml] ------------------------------------------------------------------------------------- output.logstash: - # The Logstash hosts. <1> - hosts: + hosts: <1> - localhost:5043 - - # Network timeout in seconds. timeout: 15 - - # Optional SSL settings <2> - # List of root certificates for HTTPS server verifications - ssl.certificate_authorities: + ssl.certificate_authorities: <2> - ./logstash-forwarder.crt - - # Certificate for SSL client authentication ssl.certificate: ./logstash-forwarder.crt - - # Client Certificate Key ssl.key: ./logstash-forwarder.key ------------------------------------------------------------------------------------- @@ -287,7 +276,7 @@ Filebeat configuration: filebeat.prospectors: - input_type: log paths: - - "/var/log/*.log" + - /var/log/*.log document_type: syslog fields: service: test01 @@ -374,7 +363,7 @@ you can use the <> configuration option: filebeat.prospectors: - input_type: log paths: - - "/var/log/*.log" + - /var/log/*.log document_type: syslog fields: service: test01 diff --git a/filebeat/docs/reference/configuration/filebeat-options.asciidoc b/filebeat/docs/reference/configuration/filebeat-options.asciidoc index d3a3d44da7e..f724588e56b 100644 --- a/filebeat/docs/reference/configuration/filebeat-options.asciidoc +++ b/filebeat/docs/reference/configuration/filebeat-options.asciidoc @@ -11,22 +11,15 @@ Here is a sample configuration: [source,yaml] ------------------------------------------------------------------------------------- filebeat.prospectors: -# Each - is a prospector. Below are the prospector specific configurations - input_type: log - # Paths that should be crawled and fetched. Glob based paths. - # For each file found under this path, a harvester is started. paths: - - "/var/log/apache/httpd-*.log" - # Type to be published in the 'type' field. For Elasticsearch output, - # the type defines the document type these entries should be stored - # in. Default: log + - /var/log/apache/httpd-*.log document_type: apache - input_type: log paths: - /var/log/messages - - "/var/log/*.log" - + - /var/log/*.log ------------------------------------------------------------------------------------- ==== Options @@ -433,14 +426,14 @@ lines. The `backoff` value will be multiplied each time with the `backoff_factor ===== harvester_limit -EXPERIMENTAL +experimental[] -harvester_limit limits the number of harvesters that are started in parallel for one prospector. This directly relates +`harvester_limit` limits the number of harvesters that are started in parallel for one prospector. This directly relates to the maximum number of file handlers that are opened. The default is 0 which means there is no limit. This configuration is useful if the number of files to be harvested exceeds the open file handler limit of the operating system. As setting a limit on harvester means that potentially not all files are opened in parallel, it is recommended to use -this option in combination with the close_* options to make sure harvesters are stopped more often so new files can be +this option in combination with the `close_*` options to make sure harvesters are stopped more often so new files can be picked up. Currently if a new harvester can be started again, the new harvester to be started is picked randomly. This means it can diff --git a/libbeat/docs/generalconfig.asciidoc b/libbeat/docs/generalconfig.asciidoc index 9d6c398b703..08c406519f5 100644 --- a/libbeat/docs/generalconfig.asciidoc +++ b/libbeat/docs/generalconfig.asciidoc @@ -20,16 +20,8 @@ Here is an example configuration: [source,yaml] ------------------------------------------------------------------------------ -# The name of the shipper that publishes the network data. It can be used to group -# all the transactions sent by a single shipper in the web interface. -# If this options is not defined, the hostname is used. -#name: - -# The tags of the shipper are included in their own field with each -# transaction published. Tags make it easy to group servers by different -# logical properties. +name: "my-shipper" tags: ["service-X", "web-tier"] - ------------------------------------------------------------------------------ ==== General Options @@ -144,12 +136,12 @@ that if finds. Then, for each transaction, the Beat exports the GeoIP location o The recommended values for geoip.paths are `/usr/share/GeoIP/GeoLiteCity.dat` and `/usr/local/var/GeoIP/GeoLiteCity.dat`. +If no paths are configured, GeoIP is disabled. + Example: [source,yaml] ------------------------------------------------------------------------------ -# Configure local GeoIP database support. -# If no paths are not configured geoip is disabled. geoip: paths: - "/usr/share/GeoIP/GeoLiteCity.dat" diff --git a/libbeat/docs/loggingconfig.asciidoc b/libbeat/docs/loggingconfig.asciidoc index 16208dbfd86..c03201b2b82 100644 --- a/libbeat/docs/loggingconfig.asciidoc +++ b/libbeat/docs/loggingconfig.asciidoc @@ -21,13 +21,8 @@ is used on Windows systems, and syslog output is used on Linux and OS X. [source,yaml] ------------------------------------------------------------------------------ logging.level: warning - -# enable file rotation with default configuration logging.to_files: true - -# do not log to syslog logging.to_syslog: false - logging.files: path: /var/log/mybeat name: mybeat.log diff --git a/libbeat/docs/newbeat.asciidoc b/libbeat/docs/newbeat.asciidoc index 266eb85cd65..9690d72b227 100644 --- a/libbeat/docs/newbeat.asciidoc +++ b/libbeat/docs/newbeat.asciidoc @@ -355,7 +355,6 @@ You also need to add `path` to `etc/beat.yml`: [source,yml] ---------------------------------------------------------------------- countbeat: - # Defines how often an event is sent to the output period: 10s path: "." ---------------------------------------------------------------------- diff --git a/libbeat/docs/outputconfig.asciidoc b/libbeat/docs/outputconfig.asciidoc index d16b25fb025..9e4aaf6efb3 100644 --- a/libbeat/docs/outputconfig.asciidoc +++ b/libbeat/docs/outputconfig.asciidoc @@ -21,33 +21,14 @@ Example configuration: ------------------------------------------------------------------------------ output.elasticsearch: - # The Elasticsearch cluster hosts: ["http://localhost:9200"] - - # Load the Elasticsearch template. template.enabled: true - - # Path to template file template.path: "{beatname_lc}.template.json" - - # Overwrite existing template template.overwrite: false - - # Optional index name. The default is "{beatname_lc}" and generates - # [{beatname_lc}-]YYYY.MM.DD keys. index: "{beatname_lc}" - - # Optional SSL configuration options. SSL is off by default. - - # List of root certificates for HTTPS server verifications ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] - - # Certificate for SSL client authentication ssl.certificate: "/etc/pki/client/cert.pem" - - # Client Certificate Key ssl.key: "/etc/pki/client/cert.key" - ------------------------------------------------------------------------------ To enable SSL, just add `https` to all URLs defined under __hosts__. @@ -56,28 +37,18 @@ To enable SSL, just add `https` to all URLs defined under __hosts__. ------------------------------------------------------------------------------ output.elasticsearch: - # Array of hosts to connect to. hosts: ["https://localhost:9200"] - - # HTTP basic auth username: "admin" password: "s3cr3t" - ------------------------------------------------------------------------------ If the Elasticsearch nodes are defined by `IP:PORT`, then add `protocol: https` to the yaml file. [source,yaml] ------------------------------------------------------------------------------ - output.elasticsearch: - # The Elasticsearch cluster hosts: ["localhost"] - - # Optional http or https. Default is http protocol: "https" - - # HTTP basic auth username: "admin" password: "s3cr3t" @@ -110,13 +81,8 @@ NOTE: When a node is defined as an `IP:PORT`, the _scheme_ and _path_ are taken [source,yaml] ------------------------------------------------------------------------------ output.elasticsearch: - # The Elasticsearch cluster hosts: ["10.45.3.2:9220", "10.45.3.1:9230"] - - # Optional http or https. Default is http protocol: https - - # HTTP Path at which each Elasticsearch server lives path: /elasticsearch ------------------------------------------------------------------------------ @@ -372,11 +338,6 @@ Here is an example of how to configure the Beat to use Logstash: ------------------------------------------------------------------------------ output.logstash: hosts: ["localhost:5044"] - - # index configures '@metadata.beat' field to be used by Logstash for - # indexing. The default index name depends on the each beat. - # For Packetbeat, the default is set to packetbeat, for Metricbeat to - # metricbeat and for Filebeat to filebeat. index: {beatname_lc} ------------------------------------------------------------------------------ @@ -424,11 +385,7 @@ will switch to another host if the selected one becomes unresponsive. The defaul ------------------------------------------------------------------------------ output.logstash: hosts: ["localhost:5044", "localhost:5045"] - - # configure logstash plugin to loadbalance events between the logstash instances loadbalance: true - - # configure index prefix name index: {beatname_lc} ------------------------------------------------------------------------------ @@ -647,22 +604,10 @@ Example configuration: ["source","yaml",subs="attributes"] ------------------------------------------------------------------------------ output.redis: - # Set the host and port where to find Redis. hosts: ["localhost"] - - # Optional password to authenticate with. By default, no - # password is set. password: "my_password" - - # Optional key name. The default is {beatname_lc} key: "{beatname_lc}" - - # Optional Redis database number where the events are stored - # The default is 0. db: 0 - - # Optional Redis initial connection timeout in seconds. - # The default is 5 seconds. timeout: 5 ------------------------------------------------------------------------------ @@ -813,20 +758,9 @@ Logstash. ["source","yaml",subs="attributes"] ------------------------------------------------------------------------------ output.file: - # Path to the directory where to save the generated files. The option is mandatory. path: "/tmp/{beatname_lc}" - - # Name of the generated files. The default is `beatname` and it generates - # files: `beatname`, `beatname.1`, `beatname.2`, etc. filename: {beatname_lc} - - # Maximum size in kilobytes of each file. When this size is reached, the files are - # rotated. The default value is 10240 kB. #rotate_every_kb: 10000 - - # Maximum number of files under path. When this number of files is reached, the - # oldest file is deleted and the rest are shifted from last to first. The default - # is 7 files. #number_of_files: 7 ------------------------------------------------------------------------------ @@ -910,14 +844,8 @@ Example configuration: ------------------------------------------------------------------------------ output.elasticsearch: hosts: ["192.168.1.42:9200"] - - # List of root certificates for HTTPS server verifications ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] - - # Certificate for SSL client authentication ssl.certificate: "/etc/pki/client/cert.pem" - - # Client Certificate Key ssl.key: "/etc/pki/client/cert.key" ------------------------------------------------------------------------------ diff --git a/packetbeat/docs/gettingstarted.asciidoc b/packetbeat/docs/gettingstarted.asciidoc index 7398790ab26..f2f455b3027 100644 --- a/packetbeat/docs/gettingstarted.asciidoc +++ b/packetbeat/docs/gettingstarted.asciidoc @@ -104,8 +104,6 @@ server on which Packetbeat is installed. For this, use `any` as the device: + [source,yaml] ---------------------------------------------------------------------- -# Select the network interface to sniff the data. You can use the "any" -# keyword to sniff on all connected interfaces. packetbeat.interfaces.device: any ---------------------------------------------------------------------- @@ -176,12 +174,8 @@ installation: + [source,yaml] ---------------------------------------------------------------------- -# Configure what outputs to use when sending the data collected by the beat. -# Multiple outputs may be used. - #-------------------------- Elasticsearch output ------------------------------ output.elasticsearch: - # Array of hosts to connect to. hosts: ["192.168.1.42:9200"] ---------------------------------------------------------------------- + diff --git a/packetbeat/docs/maintaining-topology.asciidoc b/packetbeat/docs/maintaining-topology.asciidoc index 1bf1fbd9fc8..802a5849be9 100644 --- a/packetbeat/docs/maintaining-topology.asciidoc +++ b/packetbeat/docs/maintaining-topology.asciidoc @@ -18,6 +18,5 @@ For example: ------------------------------------------------------------------------------ output.elasticsearch: hosts: ["localhost:9200"] - save_topology: true ------------------------------------------------------------------------------ diff --git a/packetbeat/docs/reference/configuration/packetbeat-options.asciidoc b/packetbeat/docs/reference/configuration/packetbeat-options.asciidoc index f774204d7f8..7acf41b6ae2 100644 --- a/packetbeat/docs/reference/configuration/packetbeat-options.asciidoc +++ b/packetbeat/docs/reference/configuration/packetbeat-options.asciidoc @@ -5,17 +5,9 @@ The `interfaces` section of the +{beatname_lc}.yml+ config file configures the s [source,yaml] ------------------------------------------------------------------------------ -# Select the network interfaces to sniff the data. You can use the "any" -# keyword to sniff on all connected interfaces. packetbeat.interfaces.device: any - -# The maximum capture size of a single packet. packetbeat.interfaces.snaplen: 1514 - -# The type of the sniffer to use packetbeat.interfaces.type: af_packet - -# The size of the sniffing buffer packetbeat.interfaces.buffer_size_mb: 100 ------------------------------------------------------------------------------ @@ -347,15 +339,7 @@ The `dns` section of the +{beatname_lc}.yml+ config file specifies configuration ------------------------------------------------------------------------------ packetbeat.protocols.dns: ports: [53] - - # include_authorities controls whether or not the dns.authorities field - # (authority resource records) is added to messages. - # Default: false include_authorities: true - - # include_additionals controls whether or not the dns.additionals field - # (additional resource records) is added to messages. - # Default: false include_additionals: true ------------------------------------------------------------------------------ @@ -377,27 +361,10 @@ sample configuration for the `http` section of the +{beatname_lc}.yml+ config fi [source,yaml] ------------------------------------------------------------------------------ packetbeat.protocols.http: - - # Configure the ports where to listen for HTTP traffic. You can disable - # the http protocol by commenting the list of ports. ports: [80, 8080, 8000, 5000, 8002] - - # Uncomment the following to hide certain parameters in URL or forms attached - # to HTTP requests. The names of the parameters are case insensitive. - # The value of the parameters will be replaced with the 'xxxxx' string. - # This is generally useful for avoiding storing user passwords or other - # sensitive information. hide_keywords: ["pass", "password", "passwd"] - - # A list of header names to capture and send to Elasticsearch. These headers - # are placed under the `headers` dictionary in the resulting JSON. send_headers: ["User-Agent", "Cookie", "Set-Cookie"] - - # If the Cookie or Set-Cookie headers are sent, this option controls whether - # they are split into individual values. split_coookie: true - - # Configure the HTTP header that contains the real IP address. real_ip_header: "X-Forwarded-For" ------------------------------------------------------------------------------