diff --git a/auditbeat/docs/configuring-howto.asciidoc b/auditbeat/docs/configuring-howto.asciidoc index 40efe9693071..5d7dd5477859 100644 --- a/auditbeat/docs/configuring-howto.asciidoc +++ b/auditbeat/docs/configuring-howto.asciidoc @@ -49,7 +49,6 @@ include::./auditbeat-general-options.asciidoc[] include::./reload-configuration.asciidoc[] -:allplatforms: include::../../libbeat/docs/queueconfig.asciidoc[] include::../../libbeat/docs/outputconfig.asciidoc[] @@ -70,10 +69,11 @@ include::../../libbeat/docs/loggingconfig.asciidoc[] :standalone: include::../../libbeat/docs/shared-env-vars.asciidoc[] +:standalone!: :standalone: -:allplatforms: include::../../libbeat/docs/yaml.asciidoc[] +:standalone!: include::../../libbeat/docs/regexp.asciidoc[] diff --git a/auditbeat/docs/getting-started.asciidoc b/auditbeat/docs/getting-started.asciidoc index b330ba204cf2..a6efca0f3dd3 100644 --- a/auditbeat/docs/getting-started.asciidoc +++ b/auditbeat/docs/getting-started.asciidoc @@ -181,13 +181,11 @@ include::../../libbeat/docs/step-look-at-config.asciidoc[] [id="{beatname_lc}-template"] === Step 3: Load the index template in {es} -:allplatforms: include::../../libbeat/docs/shared-template-load.asciidoc[] [[load-kibana-dashboards]] === Step 4: Set up the {kib} dashboards -:allplatforms: include::../../libbeat/docs/dashboards.asciidoc[] [id="{beatname_lc}-starting"] @@ -201,14 +199,7 @@ NOTE: If you use an init.d script to start {beatname_uc} on deb or rpm, you can' specify command line flags (see <>). To specify flags, start {beatname_uc} in the foreground. -*deb:* - -["source","sh",subs="attributes"] ----------------------------------------------------------------------- -sudo service {beatname_lc} start ----------------------------------------------------------------------- - -*rpm:* +*deb and rpm:* ["source","sh",subs="attributes"] ---------------------------------------------------------------------- diff --git a/auditbeat/docs/index.asciidoc b/auditbeat/docs/index.asciidoc index 508755b92c99..28e01662b530 100644 --- a/auditbeat/docs/index.asciidoc +++ b/auditbeat/docs/index.asciidoc @@ -12,6 +12,11 @@ include::{asciidoc-dir}/../../shared/attributes.asciidoc[] :discuss_forum: beats/{beatname_lc} :beat_default_index_prefix: {beatname_lc} :has_ml_jobs: yes +:deb_os: +:rpm_os: +:mac_os: +:docker_platform: +:win_os: include::../../libbeat/docs/shared-beats-attributes.asciidoc[] diff --git a/filebeat/docs/configuring-howto.asciidoc b/filebeat/docs/configuring-howto.asciidoc index 77d3bcfcb93b..94538d344261 100644 --- a/filebeat/docs/configuring-howto.asciidoc +++ b/filebeat/docs/configuring-howto.asciidoc @@ -54,7 +54,6 @@ include::./filebeat-general-options.asciidoc[] include::./reload-configuration.asciidoc[] -:allplatforms: include::../../libbeat/docs/queueconfig.asciidoc[] include::../../libbeat/docs/outputconfig.asciidoc[] @@ -77,14 +76,15 @@ include::../../libbeat/docs/loggingconfig.asciidoc[] :standalone: include::../../libbeat/docs/shared-env-vars.asciidoc[] +:standalone!: :autodiscoverJolokia: :autodiscoverHints: include::../../libbeat/docs/shared-autodiscover.asciidoc[] :standalone: -:allplatforms: include::../../libbeat/docs/yaml.asciidoc[] +:standalone!: include::../../libbeat/docs/regexp.asciidoc[] diff --git a/filebeat/docs/getting-started.asciidoc b/filebeat/docs/getting-started.asciidoc index 81f88a92d3af..f1371eaadf54 100644 --- a/filebeat/docs/getting-started.asciidoc +++ b/filebeat/docs/getting-started.asciidoc @@ -186,19 +186,16 @@ include::../../libbeat/docs/step-look-at-config.asciidoc[] [[config-filebeat-logstash]] === Step 3: Configure Filebeat to use Logstash -:allplatforms: include::../../libbeat/docs/shared-logstash-config.asciidoc[] [[filebeat-template]] === Step 4: Load the index template in Elasticsearch -:allplatforms: include::../../libbeat/docs/shared-template-load.asciidoc[] [[load-kibana-dashboards]] === Step 5: Set up the Kibana dashboards -:allplatforms: include::../../libbeat/docs/dashboards.asciidoc[] [[filebeat-starting]] @@ -212,14 +209,7 @@ NOTE: If you use an init.d script to start Filebeat on deb or rpm, you can't specify command line flags (see <>). To specify flags, start Filebeat in the foreground. -*deb:* - -["source","sh",subs="attributes"] ----------------------------------------------------------------------- -sudo service {beatname_lc} start ----------------------------------------------------------------------- - -*rpm:* +*deb and rpm:* ["source","sh",subs="attributes"] ---------------------------------------------------------------------- diff --git a/filebeat/docs/index.asciidoc b/filebeat/docs/index.asciidoc index 17dbcdfe8706..3ecf85403fd7 100644 --- a/filebeat/docs/index.asciidoc +++ b/filebeat/docs/index.asciidoc @@ -12,6 +12,11 @@ include::{asciidoc-dir}/../../shared/attributes.asciidoc[] :discuss_forum: beats/{beatname_lc} :beat_default_index_prefix: {beatname_lc} :has_ml_jobs: yes +:deb_os: +:rpm_os: +:mac_os: +:docker_platform: +:win_os: include::../../libbeat/docs/shared-beats-attributes.asciidoc[] diff --git a/heartbeat/docs/configuring-howto.asciidoc b/heartbeat/docs/configuring-howto.asciidoc index ff23dc75e633..dfa21e81c3e5 100644 --- a/heartbeat/docs/configuring-howto.asciidoc +++ b/heartbeat/docs/configuring-howto.asciidoc @@ -46,7 +46,6 @@ include::./heartbeat-options.asciidoc[] include::./heartbeat-general-options.asciidoc[] -:allplatforms: include::../../libbeat/docs/queueconfig.asciidoc[] include::../../libbeat/docs/outputconfig.asciidoc[] @@ -67,12 +66,13 @@ include::../../libbeat/docs/loggingconfig.asciidoc[] :standalone: include::../../libbeat/docs/shared-env-vars.asciidoc[] +:standalone!: include::../../libbeat/docs/shared-autodiscover.asciidoc[] :standalone: -:allplatforms: include::../../libbeat/docs/yaml.asciidoc[] +:standalone!: include::../../libbeat/docs/regexp.asciidoc[] diff --git a/heartbeat/docs/getting-started.asciidoc b/heartbeat/docs/getting-started.asciidoc index 18c3ed4b89c4..2a435afd1180 100644 --- a/heartbeat/docs/getting-started.asciidoc +++ b/heartbeat/docs/getting-started.asciidoc @@ -205,13 +205,11 @@ include::../../libbeat/docs/step-look-at-config.asciidoc[] [[heartbeat-template]] === Step 3: Load the index template in Elasticsearch -:allplatforms: include::../../libbeat/docs/shared-template-load.asciidoc[] [[load-kibana-dashboards]] === Step 4: Set up the Kibana dashboards -:allplatforms: include::../../libbeat/docs/dashboards.asciidoc[] [[heartbeat-starting]] @@ -225,14 +223,7 @@ NOTE: If you use an init.d script to start Heartbeat on deb or rpm, you can't specify command line flags (see <>). To specify flags, start Heartbeat in the foreground. -*deb:* - -["source","sh",subs="attributes"] ----------------------------------------------------------------------- -sudo service {beatname_lc}-elastic start ----------------------------------------------------------------------- - -*rpm:* +*deb and rpm:* ["source","sh",subs="attributes"] ---------------------------------------------------------------------- diff --git a/heartbeat/docs/index.asciidoc b/heartbeat/docs/index.asciidoc index e0ece667f7a9..0176b0072904 100644 --- a/heartbeat/docs/index.asciidoc +++ b/heartbeat/docs/index.asciidoc @@ -12,6 +12,11 @@ include::{asciidoc-dir}/../../shared/attributes.asciidoc[] :discuss_forum: beats/{beatname_lc} :beat_default_index_prefix: {beatname_lc} :has_ml_jobs: yes +:deb_os: +:rpm_os: +:mac_os: +:docker_platform: +:win_os: include::../../libbeat/docs/shared-beats-attributes.asciidoc[] diff --git a/journalbeat/docs/config-options.asciidoc b/journalbeat/docs/config-options.asciidoc index 722d9b9534a2..4cfdb7d9e093 100644 --- a/journalbeat/docs/config-options.asciidoc +++ b/journalbeat/docs/config-options.asciidoc @@ -5,38 +5,35 @@ Configure inputs ++++ -IMPORTANT: This documentation is placeholder content. It has not yet been reviewed. - By default, {beatname_uc} reads log events from the default systemd journals. To specify other journal files, set the <<{beatname_lc}-paths,`paths`>> option in -the +{beatname_lc}.inputs+ section of the +{beatname_lc}.yml+ file. - -The list of paths is a YAML array, so each path begins with a dash (-). Each -path can be a directory path (to collect events from all journals in a -directory), or a file path. For example: +the +{beatname_lc}.inputs+ section of the +{beatname_lc}.yml+ file. Each path +can be a directory path (to collect events from all journals in a directory), or +a file path. For example: ["source","sh",subs="attributes"] ---- {beatname_lc}.inputs: - paths: - "/dev/log" - - "/var/log/messages/my-journal-file" + - "/var/log/messages/my-journal-file.journal" ---- -Within the +{beatname_lc}.inputs+ section, you can also specify options that -control the position where {beatname_uc} starts reading the journal file, and -set filters to reduce the fields that {beatname_uc} needs to process. See -<<{beatname_lc}-options>> for a list of available options. - -[float] -=== Configuration examples - -The following example shows how to monitor multiple journals under the -same directory. {beatname_uc} merges all journals under the directory into a -single journal and reads them. With `seek` set to `cursor`, {beatname_uc} -starts reading at the beginning of the journal, but will continue reading where -it left off after a reload or restart. - +Within the configuration file, you can also specify options that control how +{beatname_uc} reads the journal files and which fields are sent to the +configured output. See <<{beatname_lc}-options>> for a list of available +options. + +The following examples show how to configure {beatname_uc} for some common use +cases. + +[[monitor-multiple-journals]] +.Example 1: Monitor multiple journals under the same directory +This example configures {beatname_uc} to read from multiple journals that are +stored under the same directory. {beatname_uc} merges all journals under the +directory into a single event stream and reads the events. With `seek` set to +`cursor`, {beatname_uc} starts reading at the beginning of the journal, but will +continue reading at the last known position after a reload or restart. ["source","sh",subs="attributes"] ---- {beatname_lc}.inputs: @@ -44,31 +41,32 @@ it left off after a reload or restart. seek: cursor ---- -The following examples show how to get Redis events from a Docker container that -is tagged as `redis`. - -//TODO: Add a better explanation of the options. - -This example uses the translated fields by Journald: - +[[filter-using-field-names]] +.Example 2: Fetch log events for Redis running on Docker (uses field names from systemd) +This example configures {beatname_uc} to fetch log events for Redis running in a +Docker container. The fields are matched using field names from the systemd +journal. ["source","sh",subs="attributes"] ---- {beatname_lc}.inputs: - paths: [] include_matches: - - "container.image.tag=redis" - - "process.name=redis" + - "CONTAINER_TAG=redis" + - "_COMM=redis" ---- -This example uses the field names from the systemd journal: - +[[filter-using-translated-names]] +.Example 3: Fetch log events for Redis running on Docker (uses translated field names) +This example also configures {beatname_uc} to fetch log events for Redis running +in a Docker container. However, in this example the fields are matched using the +<> provided by {beatname_uc}. ["source","sh",subs="attributes"] ---- {beatname_lc}.inputs: - paths: [] include_matches: - - "CONTAINER_TAG=redis" - - "_COMM=redis" + - "container.image.tag=redis" + - "process.name=redis" ---- [id="{beatname_lc}-options"] @@ -86,7 +84,21 @@ path (to collect events from all journals in a directory), or a file path. If you specify a directory, {beatname_uc} merges all journals under the directory into a single journal and reads them. -//QUESTION: Are globs supported? If so, I need to add more detail here. +If no paths are specified, {beatname_uc} reads from the default journal. + +[float] +[id="{beatname_lc}-backoff"] +==== `backoff` + +The number of seconds to wait before trying to read again from journals. The +default is 1s. + +[float] +[id="{beatname_lc}-max-backoff"] +==== `max_backoff` + +The maximum number of seconds to wait before attempting to read again from +journals. The default is 60s. [float] [id="{beatname_lc}-seek"] @@ -94,9 +106,112 @@ into a single journal and reads them. The position to start reading the journal from. Valid settings are: -* `head`: Starts reading at the beginning of the file. -* `tail`: Starts reading at the end of the file. -* `cursor`: Initially starts reading at the beginning of the file, but continues -reading where it left off after a reload or restart. +* `head`: Starts reading at the beginning of the journal. After a restart, +{beatname_uc} resends all log messages in the journal. +* `tail`: Starts reading at the end of the journal. After a restart, +{beatname_uc} resends the last message, which might result in duplicates. If +multiple log messages are written to a journal while {beatname_uc} is down, +only the last log message is sent on restart. +* `cursor`: On first read, starts reading at the beginning of the journal. After a +reload or restart, continues reading at the last known position. + +When specified under `paths`, the `seek` setting applies to all journals under +the configured paths. When specified directly under the +{beatname_lc}+ +namespace, the setting applies to all journals read by {beatname_uc}. -//TODO: ADD OTHER OPTIONS HERE. \ No newline at end of file +If you have old log files and want to skip lines, start {beatname_uc} with +`seek: tail` specified. Then stop {beatname_uc}, set `seek: cursor`, and restart +{beatname_uc}. + +[float] +[id="{beatname_lc}-include-matches"] +==== `include_matches` + +A list of filter expressions used to match fields. The format of the expression +is `field=value`. {beatname_uc} fetches all events that exactly match the +expressions. Pattern matching is not supported. + +To reference fields, use one of the following: + +* The field name used by the systemd journal. For example, +`CONTAINER_TAG=redis` (<>). +* The <> used by +{beatname_uc}. For example, `container.image.tag=redis` +(<>). {beatname_uc} +does not translate all fields from the journal. For custom fields, use the name +specified in the systemd journal. + +When specified under `paths`, the `include_matches` filter is applied to all +journals under the configured paths. When specified directly under the ++{beatname_lc}+ namespace, the setting applies to all journals read by +{beatname_uc}. + +[float] +[[translated-fields]] +=== Translated field names + +You can use the following translated names in filter expressions to reference +journald fields: + +[horizontal] +*Journald field name*:: *Translated name* +`COREDUMP_UNIT`:: `journald.coredump.unit` +`COREDUMP_USER_UNIT`:: `journald.coredump.user_unit` +`OBJECT_AUDIT_LOGINUID`:: `journald.object.audit.login_uid` +`OBJECT_AUDIT_SESSION`:: `journald.object.audit.session` +`OBJECT_CMDLINE`:: `journald.object.cmd` +`OBJECT_COMM`:: `journald.object.name` +`OBJECT_EXE`:: `journald.object.executable` +`OBJECT_GID`:: `journald.object.gid` +`OBJECT_PID`:: `journald.object.pid` +`OBJECT_SYSTEMD_OWNER_UID`:: `journald.object.systemd.owner_uid` +`OBJECT_SYSTEMD_SESSION`:: `journald.object.systemd.session` +`OBJECT_SYSTEMD_UNIT`:: `journald.object.systemd.unit` +`OBJECT_SYSTEMD_USER_UNIT`:: `journald.object.systemd.user_unit` +`OBJECT_UID`:: `journald.object.uid` +`_AUDIT_LOGINUID`:: `process.audit.login_uid` +`_AUDIT_SESSION`:: `process.audit.session` +`_BOOT_ID`:: `host.boot_id` +`_CAP_EFFECTIVE`:: `process.capabilites` +`_CMDLINE`:: `process.cmd` +`_CODE_FILE`:: `journald.code.file` +`_CODE_FUNC`:: `journald.code.func` +`_CODE_LINE`:: `journald.code.line` +`_COMM`:: `process.name` +`_EXE`:: `process.executable` +`_GID`:: `process.uid` +`_HOSTNAME`:: `host.name` +`_KERNEL_DEVICE`:: `journald.kernel.device` +`_KERNEL_SUBSYSTEM`:: `journald.kernel.subsystem` +`_MACHINE_ID`:: `host.id` +`_MESSAGE`:: `message` +`_PID`:: `process.pid` +`_PRIORITY`:: `syslog.priority` +`_SYSLOG_FACILITY`:: `syslog.facility` +`_SYSLOG_IDENTIFIER`:: `syslog.identifier` +`_SYSLOG_PID`:: `syslog.pid` +`_SYSTEMD_CGROUP`:: `systemd.cgroup` +`_SYSTEMD_INVOCATION_ID`:: `systemd.invocation_id` +`_SYSTEMD_OWNER_UID`:: `systemd.owner_uid` +`_SYSTEMD_SESSION`:: `systemd.session` +`_SYSTEMD_SLICE`:: `systemd.slice` +`_SYSTEMD_UNIT`:: `systemd.unit` +`_SYSTEMD_USER_SLICE`:: `systemd.user_slice` +`_SYSTEMD_USER_UNIT`:: `systemd.user_unit` +`_TRANSPORT`:: `systemd.transport` +`_UDEV_DEVLINK`:: `journald.kernel.device_symlinks` +`_UDEV_DEVNODE`:: `journald.kernel.device_node_path` +`_UDEV_SYSNAME`:: `journald.kernel.device_name` +`_UID`:: `process.uid` + + +The following translated fields for +https://docs.docker.com/config/containers/logging/journald/[Docker] are also +available: + +[horizontal] +`CONTAINER_ID`:: `conatiner.id_truncated` +`CONTAINER_ID_FULL`:: `container.id` +`CONTAINER_NAME`:: `container.name` +`CONTAINER_PARTIAL_MESSAGE`:: `container.partial` +`CONTAINER_TAG`:: `container.image.tag` diff --git a/journalbeat/docs/configuring-howto.asciidoc b/journalbeat/docs/configuring-howto.asciidoc index 95d56d427f28..c23d40d50e24 100644 --- a/journalbeat/docs/configuring-howto.asciidoc +++ b/journalbeat/docs/configuring-howto.asciidoc @@ -4,21 +4,11 @@ [partintro] -- -IMPORTANT: This documentation is placeholder content. It has not yet been reviewed. - Before modifying configuration settings, make sure you've completed the <<{beatname_lc}-configuration,configuration steps>> in the Getting Started. This section describes some common use cases for changing configuration options. -To configure {beatname_uc}, you edit the configuration file. For rpm and deb, -you’ll find the configuration file at +/etc/{beatname_lc}/{beatname_lc}.yml+. -There's also a full example configuration file at -+/etc/{beatname_lc}/{beatname_lc}.reference.yml+ that shows all non-deprecated -options. For mac and win, look in the archive that you extracted. - -The {beatname_uc} configuration file uses http://yaml.org/[YAML] for its syntax. -See the {libbeat}/config-file-format.html[Config File Format] section of the -_{libbeat_docs}_ for more about the structure of the config file. +include::../../libbeat/docs/shared-configuring.asciidoc[] The following topics describe how to configure {beatname_uc}: @@ -31,15 +21,13 @@ The following topics describe how to configure {beatname_uc}: * <> * <> * <> -* <> * <> * <> * <> -//* <> * <> * <> * <> -//* <<{beatname_lc}-reference-yml>> +* <<{beatname_lc}-reference-yml>> -- @@ -47,9 +35,7 @@ include::./config-options.asciidoc[] include::./general-options.asciidoc[] -:allplatforms: include::../../libbeat/docs/queueconfig.asciidoc[] -:allplatforms!: include::../../libbeat/docs/outputconfig.asciidoc[] @@ -71,20 +57,12 @@ include::../../libbeat/docs/loggingconfig.asciidoc[] include::../../libbeat/docs/shared-env-vars.asciidoc[] :standalone!: -//OPEN ISSUE: DO WE NEED AUTODISCOVER? -//include::../../libbeat/docs/shared-autodiscover.asciidoc[] - :standalone: -:allplatforms: include::../../libbeat/docs/yaml.asciidoc[] :standalone!: -:allplatforms!: include::../../libbeat/docs/regexp.asciidoc[] include::../../libbeat/docs/http-endpoint.asciidoc[] -// TODO: Uncomment the following include statement when the reference yaml file -// is available in the repo. Also uncomment the link in the jump list at the top -// of this file. -//include::../../libbeat/docs/reference-yml.asciidoc[] +include::../../libbeat/docs/reference-yml.asciidoc[] diff --git a/journalbeat/docs/faq.asciidoc b/journalbeat/docs/faq.asciidoc index 6d5b4ed296ec..9e0d0a0c1585 100644 --- a/journalbeat/docs/faq.asciidoc +++ b/journalbeat/docs/faq.asciidoc @@ -1,24 +1,10 @@ [[faq]] == Frequently asked questions -IMPORTANT: This documentation is placeholder content. It has not yet been reviewed. - This section contains frequently asked questions about {beatname_uc}. Also check out the https://discuss.elastic.co/c/beats/{beatname_lc}[{beatname_uc} discussion forum]. -[float] -[id="{beatname_lc}-sometext"] -=== Question 1? - -ADD DESCRIPTION HERE - -[float] -[id="{beatname_lc}-sometext2"] -=== Question 2? - -ADD DESCRIPTION HERE - include::../../libbeat/docs/faq-limit-bandwidth.asciidoc[] include::../../libbeat/docs/shared-faq.asciidoc[] diff --git a/journalbeat/docs/filtering.asciidoc b/journalbeat/docs/filtering.asciidoc index e75b4e73bd76..c9182ebe46d4 100644 --- a/journalbeat/docs/filtering.asciidoc +++ b/journalbeat/docs/filtering.asciidoc @@ -1,20 +1,15 @@ [[filtering-and-enhancing-data]] == Filter and enhance the exported data -IMPORTANT: This documentation is placeholder content. It has not yet been reviewed. - Your use case might require only a subset of the data exported by {beatname_uc}, or you might need to enhance the exported data (for example, by adding metadata). {beatname_uc} provides a couple of options for filtering and enhancing exported data. -You can configure each input to include or exclude specific lines or files. This -allows you to specify different filtering criteria for each input. To do this, -you use the `include_lines`, `exclude_lines`, and `exclude_files` options under -the +{beatname_lc}.inputs+ section of the config file (see -<>). The disadvantage of this approach is -that you need to implement a configuration option for each filtering criteria -that you need. +You can configure {beatname_uc} to include events that match specific filtering +criteria. To do this, use the <<{beatname_lc}-include-matches,`include_matches`>> +option. The advantage of this approach is that you can reduce the number of +fields that {beatname_uc} needs to process. Another approach (the one described here) is to define processors to configure global processing across all data exported by {beatname_uc}. @@ -26,12 +21,6 @@ global processing across all data exported by {beatname_uc}. include::../../libbeat/docs/processors.asciidoc[] -[float] -[[specific-example]] -==== XYZ example - -ADD EXAMPLES SPECIFIC TO THE BEAT, OR DELETE THIS SECTION - // You must set the processor-scope attribute to resolve the attribute reference // defined in processors-using.asciidoc. The attribute is used to indicate where // processors are valid. If processors are valid in more than two locations diff --git a/journalbeat/docs/general-options.asciidoc b/journalbeat/docs/general-options.asciidoc index 97367b71aac9..71ab82fc54c8 100644 --- a/journalbeat/docs/general-options.asciidoc +++ b/journalbeat/docs/general-options.asciidoc @@ -1,10 +1,58 @@ [[configuration-general-options]] == Specify general settings -IMPORTANT: This documentation is placeholder content. It has not yet been reviewed. - You can specify settings in the +{beatname_lc}.yml+ config file to control the -general behavior of {beatname_uc}. +general behavior of {beatname_uc}. This includes: + +* <> that control things like +publisher behavior and the location of some files. + +* <> that are supported by all Elastic +Beats. + +[float] +[[configuration-global-options]] +=== Global {beatname_uc} configuration options + +These options are in the +{beatname_lc}+ namespace. + +[float] +[id="{beatname_lc}-registry-file"] +==== `registry_file` + +The name of the registry file. If a relative path is used, it is considered relative to the +data path. See the <> section for details. The default is `${path.data}/registry`. + +["source","sh",subs="attributes"] +---- +{beatname_lc}.registry_file: registry +---- + +[float] +==== `backoff` +This option is valid as a global setting under the +{beatname_lc}+ namespace +or under `paths`. For a description of this option, see +<<{beatname_lc}-backoff,`backoff`>>. + +[float] +==== `max_backoff` +This option is valid as a global setting under the +{beatname_lc}+ namespace +or under `paths`. For a description of this option, see +<<{beatname_lc}-max-backoff,`max_backoff`>>. + +[float] +==== `seek` + +This option is valid as a global setting under the +{beatname_lc}+ namespace +or under `paths`. For a description of this option, see +<<{beatname_lc}-seek,`seek`>>. + +[float] +==== `include_matches` + +This option is valid as a global setting under the +{beatname_lc}+ namespace +or under `paths`. For a description of this option, see +<<{beatname_lc}-include-matches,`include_matches`>>. include::../../libbeat/docs/generalconfig.asciidoc[] diff --git a/journalbeat/docs/getting-started.asciidoc b/journalbeat/docs/getting-started.asciidoc index 6fffeff4b5e5..37b835b96d0e 100644 --- a/journalbeat/docs/getting-started.asciidoc +++ b/journalbeat/docs/getting-started.asciidoc @@ -1,24 +1,17 @@ [id="{beatname_lc}-getting-started"] -== Getting Started With {beatname_uc} - -IMPORTANT: This documentation is placeholder content. It has not yet been reviewed. +== Getting started with {beatname_uc} include::../../libbeat/docs/shared-getting-started-intro.asciidoc[] * <<{beatname_lc}-installation>> * <<{beatname_lc}-configuration>> * <<{beatname_lc}-template>> -* <> * <<{beatname_lc}-starting>> -* <> * <> [id="{beatname_lc}-installation"] === Step 1: Install {beatname_uc} -IMPORTANT: This documentation is placeholder content. It has not yet been reviewed. - -:no-docker: include::../../libbeat/docs/shared-download-and-install.asciidoc[] [[deb]] @@ -59,8 +52,8 @@ sudo rpm -vi {beatname_lc}-{version}-x86_64.rpm endif::[] -[[mac]] -*mac:* +[[linux]] +*linux:* ifeval::["{release-state}"=="unreleased"] @@ -72,51 +65,19 @@ ifeval::["{release-state}"!="unreleased"] ["source","sh",subs="attributes"] ------------------------------------------------ -curl -L -O https://artifacts.elastic.co/downloads/beats/{beatname_lc}/{beatname_lc}-{version}-darwin-x86_64.tar.gz -tar xzvf {beatname_lc}-{version}-darwin-x86_64.tar.gz +curl -L -O https://artifacts.elastic.co/downloads/beats/{beatname_lc}/{beatname_lc}-{version}-linux-x86_64.tar.gz +tar xzvf {beatname_lc}-{version}-linux-x86_64.tar.gz ------------------------------------------------ endif::[] -[[win]] -*win:* - -ifeval::["{release-state}"=="unreleased"] - -Version {version} of {beatname_uc} has not yet been released. - -endif::[] - -ifeval::["{release-state}"!="unreleased"] - -. Download the {beatname_uc} Windows zip file from the -https://www.elastic.co/downloads/beats/{beatname_lc}[downloads page]. - -. Extract the contents of the zip file into `C:\Program Files`. - -. Rename the +{beatname_lc}--windows+ directory to +{beatname_uc}+. - -. Open a PowerShell prompt as an Administrator (right-click the PowerShell icon and select *Run As Administrator*). - -. From the PowerShell prompt, run the following commands to install {beatname_uc} as a -Windows service: -+ -["source","sh",subs="attributes"] ----------------------------------------------------------------------- -PS > cd 'C:{backslash}Program Files{backslash}{beatname_uc}' -PS C:{backslash}Program Files{backslash}{beatname_uc}> .{backslash}install-service-{beatname_lc}.ps1 ----------------------------------------------------------------------- - -NOTE: If script execution is disabled on your system, you need to set the execution policy for the current session to allow the script to run. For example: +PowerShell.exe -ExecutionPolicy UnRestricted -File .\install-service-{beatname_lc}.ps1+. - -endif::[] - [id="{beatname_lc}-configuration"] === Step 2: Configure {beatname_uc} -IMPORTANT: This documentation is placeholder content. It has not yet been reviewed. +Before running {beatname_uc}, you can specify the location of the systemd +journal files and configure how you want the files to be read. If you accept the +default configuration, {beatname_uc} reads from the local journal. -:no-docker: include::../../libbeat/docs/shared-configuring.asciidoc[] Here is a sample of the +{beatname_lc}+ section of the +{beatname_lc}.yml+ file. @@ -126,7 +87,7 @@ Here is a sample of the +{beatname_lc}+ section of the +{beatname_lc}.yml+ file. ---------------------------------------------------------------------- journalbeat.inputs: - paths: ["/path/to/journal/directory"] - seek: cursor + seek: head ---------------------------------------------------------------------- To configure {beatname_uc}: @@ -140,20 +101,23 @@ path. For example: {beatname_lc}.inputs: - paths: - "/dev/log" - - "/var/log/messages/my-journal-file" + - "/var/log/messages/my-journal-file.journal" ---- + If no paths are specified, {beatname_uc} reads from the default journal. -. Set the `seek` option to control the position where {beatname_uc} starts -reading the journal. The available options are `head`, `tail`, and `cursor`. -Typically, you'll set `seek: cursor` so {beatname_uc} can continue reading -where it left off after a reload or restart. - -. Optional: Set the `include_matches` option to filter entries in journald -before collecting any log events. This reduces the number of fields that the -Beat needs to process. For example, to fetch only Redis events from a Docker -container tagged as `redis`, use: +. Set the <<{beatname_lc}-seek,`seek`>> option to control the position where +{beatname_uc} starts reading the journal. The available options are `head`, +`tail`, and `cursor`. The default is `cursor`, which means that on first read, +{beatname_uc} starts reading at the beginning of the file, but continues reading +at the last known position after a reload or restart. For more detail about +the settings, see the reference docs for the +<<{beatname_lc}-seek,`seek` option>>. + +. (Optional) Set the <<{beatname_lc}-include-matches,`include_matches`>> option +to filter entries in journald before collecting any log events. This reduces the +number of events that {beatname_uc} needs to process. For example, to fetch only +Redis events from a Docker container tagged as `redis`, use: + ["source","sh",subs="attributes"] ---- @@ -163,8 +127,6 @@ container tagged as `redis`, use: - "CONTAINER_TAG=redis" - "_COMM=redis" ---- -+ -See <> for more about this setting. include::../../libbeat/docs/step-configure-output.asciidoc[] @@ -180,23 +142,10 @@ include::../../libbeat/docs/step-look-at-config.asciidoc[] [id="{beatname_lc}-template"] === Step 3: Load the index template in Elasticsearch -IMPORTANT: This documentation is placeholder content. It has not yet been reviewed. - -:allplatforms: include::../../libbeat/docs/shared-template-load.asciidoc[] -[[load-kibana-dashboards]] -=== Step 4: Set up the Kibana dashboards - -IMPORTANT: This documentation is placeholder content. It has not yet been reviewed. - -:allplatforms: -include::../../libbeat/docs/dashboards.asciidoc[] - [id="{beatname_lc}-starting"] -=== Step 5: Start {beatname_uc} - -IMPORTANT: This documentation is placeholder content. It has not yet been reviewed. +=== Step 4: Start {beatname_uc} Start {beatname_uc} by issuing the appropriate command for your platform. If you are accessing a secured Elasticsearch cluster, make sure you've configured @@ -206,26 +155,19 @@ NOTE: If you use an init.d script to start {beatname_uc} on deb or rpm, you can' specify command line flags (see <>). To specify flags, start {beatname_uc} in the foreground. -*deb:* - -["source","sh",subs="attributes"] ----------------------------------------------------------------------- -sudo service {beatname_lc} start ----------------------------------------------------------------------- - -*rpm:* +*deb and rpm:* ["source","sh",subs="attributes"] ---------------------------------------------------------------------- sudo service {beatname_lc} start ---------------------------------------------------------------------- -*mac:* +*linux:* ["source","sh",subs="attributes"] ---------------------------------------------------------------------- sudo chown root {beatname_lc}.yml <1> -sudo ./{beatname_lc} -e -c {beatname_lc}.yml -d "publish" +sudo ./{beatname_lc} -e ---------------------------------------------------------------------- <1> You'll be running {beatname_uc} as root, so you need to change ownership of the configuration file, or run {beatname_uc} with `--strict.perms=false` @@ -233,38 +175,28 @@ specified. See {libbeat}/config-file-permissions.html[Config File Ownership and Permissions] in the _Beats Platform Reference_. -*win:* - -["source","sh",subs="attributes"] ----------------------------------------------------------------------- -PS C:{backslash}Program Files{backslash}{beatname_uc}> Start-Service {beatname_lc} ----------------------------------------------------------------------- - - -By default, Windows log files are stored in +C:\ProgramData\{beatname_lc}\Logs+. - -{beatname_uc} is now ready to send log files to your defined output. +{beatname_uc} is now ready to send journal events to the defined output. [[view-kibana-dashboards]] -=== Step 6: View the sample Kibana dashboards - -IMPORTANT: This documentation is placeholder content. It has not yet been reviewed. +=== Step 5: View your data in Kibana -To make it easier for you to explore {beatname_uc} data in Kibana, we've created -example {beatname_uc} dashboards. You loaded the dashboards earlier when you -ran the `setup` command. +There are currently no example dashboards available for {beatname_uc}. -include::../../libbeat/docs/opendashboards.asciidoc[] +To learn how to view and explore your data, see the +_{kibana-ref}/index.html[{kib} User Guide]_. -You can use these dashboards as examples and -{kibana-ref}/dashboard.html[customize] them to meet your needs. +[NOTE] +===== +By default, the Logs UI in {kib} only shows logs from `filebeat-*` +indexes. To show {beatname_uc} indexes, add the following settings to the {kib} +configuration: -To populate the example dashboards with data, you need to either -<> or use Logstash to -parse the data into the fields expected by the dashboards. - -Here is an example of the {beatname_uc} ADD DASHBOARD NAME dashboard: +[source,yaml] +---- +xpack.infra: + sources: + default: + logAlias: "filebeat-*,journalbeat-*" +---- -// Add an example of the dashboard -//[role="screenshot"] -//image:./images/add-image-name.png[] +===== diff --git a/journalbeat/docs/how-it-works.asciidoc b/journalbeat/docs/how-it-works.asciidoc deleted file mode 100644 index 375c55507e31..000000000000 --- a/journalbeat/docs/how-it-works.asciidoc +++ /dev/null @@ -1,6 +0,0 @@ -[id="how-{beatname_lc}-works"] -== How {beatname_uc} works - -IMPORTANT: This documentation is placeholder content. It has not yet been reviewed. - -DESCRIBE HOW THE BEAT WORKS. diff --git a/journalbeat/docs/index.asciidoc b/journalbeat/docs/index.asciidoc index be27ac9a406c..2c6febb64175 100644 --- a/journalbeat/docs/index.asciidoc +++ b/journalbeat/docs/index.asciidoc @@ -11,8 +11,10 @@ include::{asciidoc-dir}/../../shared/attributes.asciidoc[] :github_repo_name: beats :discuss_forum: beats/{beatname_lc} :beat_default_index_prefix: {beatname_lc} -:has_ml_jobs: no :libbeat-docs: Beats Platform Reference +:deb_os: +:rpm_os: +:no_dashboards: include::../../libbeat/docs/shared-beats-attributes.asciidoc[] @@ -26,9 +28,6 @@ include::../../libbeat/docs/repositories.asciidoc[] include::./setting-up-running.asciidoc[] -//TODO: Decide whether this requires a separate topic -//include::./how-it-works.asciidoc[] - include::./configuring-howto.asciidoc[] include::./fields.asciidoc[] diff --git a/journalbeat/docs/overview.asciidoc b/journalbeat/docs/overview.asciidoc index 8ed7b4553887..d8c018c38a49 100644 --- a/journalbeat/docs/overview.asciidoc +++ b/journalbeat/docs/overview.asciidoc @@ -5,8 +5,6 @@ Overview ++++ -IMPORTANT: This documentation is placeholder content. It has not yet been reviewed. - {beatname_uc} is a lightweight shipper for forwarding and centralizing log data from https://www.freedesktop.org/software/systemd/man/systemd-journald.service.html[systemd journals]. Installed as an agent on your servers, {beatname_uc} monitors the journal diff --git a/journalbeat/docs/running-on-kubernetes.asciidoc b/journalbeat/docs/running-on-kubernetes.asciidoc deleted file mode 100644 index 16b53e8e3af3..000000000000 --- a/journalbeat/docs/running-on-kubernetes.asciidoc +++ /dev/null @@ -1,6 +0,0 @@ -[id="running-{beatname_lc}-on-kubernetes"] -=== Running {beatname_uc} on Kubernetes - -IMPORTANT: This documentation is placeholder content. It has not yet been reviewed. - -ADD CONTENT HERE. diff --git a/journalbeat/docs/setting-up-running.asciidoc b/journalbeat/docs/setting-up-running.asciidoc index 8f7ccba3bb8e..aeed49f80515 100644 --- a/journalbeat/docs/setting-up-running.asciidoc +++ b/journalbeat/docs/setting-up-running.asciidoc @@ -7,8 +7,6 @@ [[setting-up-and-running]] == Setting up and running {beatname_uc} -IMPORTANT: This documentation is placeholder content. It has not yet been reviewed. - Before reading this section, see the <<{beatname_lc}-getting-started,getting started documentation>> for basic installation instructions to get you started. @@ -17,15 +15,8 @@ This section includes additional information on how to set up and run {beatname_uc}, including: * <> - * <> - * <> - -//* <> - -//* <> - * <> @@ -37,8 +28,4 @@ include::../../libbeat/docs/keystore.asciidoc[] include::../../libbeat/docs/command-reference.asciidoc[] -//include::./running-on-docker.asciidoc[] - -//include::./running-on-kubernetes.asciidoc[] - include::../../libbeat/docs/shared-shutdown.asciidoc[] diff --git a/journalbeat/docs/troubleshooting.asciidoc b/journalbeat/docs/troubleshooting.asciidoc index fa25622e7f1e..3c14416e3dce 100644 --- a/journalbeat/docs/troubleshooting.asciidoc +++ b/journalbeat/docs/troubleshooting.asciidoc @@ -4,8 +4,6 @@ [partintro] -- -IMPORTANT: This documentation is placeholder content. It has not yet been reviewed. - If you have issues installing or running {beatname_uc}, read the following tips: diff --git a/libbeat/docs/command-reference.asciidoc b/libbeat/docs/command-reference.asciidoc index a6d666e78b73..97834b75ba40 100644 --- a/libbeat/docs/command-reference.asciidoc +++ b/libbeat/docs/command-reference.asciidoc @@ -14,10 +14,13 @@ :global-flags: Also see <>. -:export-command-short-desc: Exports the configuration, index template or a dashboard to stdout +:deploy-command-short-desc: Deploys the specified function to your serverless environment +:export-command-short-desc: Exports the configuration, index template, or a dashboard to stdout :help-command-short-desc: Shows help for any command :keystore-command-short-desc: Manages the <> :modules-command-short-desc: Manages configured modules +:package-command-short-desc: Packages the configuration and executable in a zip file +:remove-command-short-desc: Removes the specified function from your serverless environment :run-command-short-desc: Runs {beatname_uc}. This command is used by default if you start {beatname_uc} without specifying a command ifndef::deprecate_dashboard_loading[] @@ -38,6 +41,7 @@ ifdef::deprecate_dashboard_loading[] endif::[] +:update-command-short-desc: Updates the specified function :test-command-short-desc: Tests the configuration :version-command-short-desc: Shows information about the current version @@ -78,26 +82,44 @@ endif::[] [options="header"] |======================= |Commands | +ifeval::[("{beatname_lc}"=="functionbeat")] +|<> | {deploy-command-short-desc}. +endif::[] |<> |{export-command-short-desc}. |<> |{help-command-short-desc}. |<> |{keystore-command-short-desc}. +ifeval::[("{beatname_lc}"=="functionbeat")] +|<> |{package-command-short-desc}. +|<> |{remove-command-short-desc}. +endif::[] ifeval::[("{beatname_lc}"=="filebeat") or ("{beatname_lc}"=="metricbeat")] |<> |{modules-command-short-desc}. endif::[] |<> |{run-command-short-desc}. |<> |{setup-command-short-desc}. |<> |{test-command-short-desc}. +ifeval::[("{beatname_lc}"=="functionbeat")] +|<> |{update-command-short-desc}. +endif::[] |<> |{version-command-short-desc}. |======================= Also see <>. +ifeval::[("{beatname_lc}"=="functionbeat")] +[[deploy-command]] +==== `deploy` command + +{deploy-command-short-desc}. + +endif::[] + [[export-command]] ==== `export` command {export-command-short-desc}. You can use this command to quickly view your configuration, see the contents of the index -template or export a dashboard from Kibana. +template, or export a dashboard from Kibana. *SYNOPSIS* @@ -252,6 +274,18 @@ Shows help for the `keystore` command. see <> for more examples. +ifeval::[("{beatname_lc}"=="functionbeat")] +[[package-command]] +==== `package` command + +{package-command-short-desc}. + +[[remove-command]] +==== `remove` command + +{remove-command-short-desc}. + +endif::[] ifeval::[("{beatname_lc}"=="filebeat") or ("{beatname_lc}"=="metricbeat")] @@ -439,10 +473,10 @@ endif::[] + ifdef::has_ml_jobs[] Loads the initial setup, including Elasticsearch template, Kibana index pattern, -Kibana dashboards and Machine learning jobs. +Kibana dashboards (when available), and Machine learning jobs. endif::[] ifndef::has_ml_jobs[] -Loads the initial setup, including Elasticsearch template, Kibana index pattern and Kibana dashboards. +Loads the initial setup, including Elasticsearch template, Kibana index pattern, and Kibana dashboards (when available). endif::[] If you want to use the command without running {beatname_uc}, use the <> command instead. @@ -511,10 +545,10 @@ environment without actually running {beatname_uc} and ingesting data. ifndef::deprecate_dashboard_loading[] *`--dashboards`*:: -Sets up the Kibana dashboards only. This option loads the dashboards from the -{beatname_uc} package. For more options, such as loading customized dashboards, -see {beatsdevguide}/import-dashboards.html[Importing Existing Beat Dashboards] -in the _Beats Developer Guide_. +Sets up the Kibana dashboards (when available). This option loads the dashboards +from the {beatname_uc} package. For more options, such as loading customized +dashboards, see {beatsdevguide}/import-dashboards.html[Importing Existing Beat +Dashboards] in the _Beats Developer Guide_. endif::[] ifdef::deprecate_dashboard_loading[] @@ -620,6 +654,14 @@ ifeval::["{beatname_lc}"=="metricbeat"] endif::[] +ifeval::[("{beatname_lc}"=="functionbeat")] +[[update-command]] +==== `update` command + +{update-command-short-desc}. + +endif::[] + [[version-command]] ==== `version` command diff --git a/libbeat/docs/config-file-format.asciidoc b/libbeat/docs/config-file-format.asciidoc index efbfb34aad8f..d72b37ba10b9 100644 --- a/libbeat/docs/config-file-format.asciidoc +++ b/libbeat/docs/config-file-format.asciidoc @@ -378,5 +378,4 @@ output.console: [[config-file-format-tips]] === YAML tips and gotchas -:allplatforms: include::yaml.asciidoc[] diff --git a/libbeat/docs/dashboards.asciidoc b/libbeat/docs/dashboards.asciidoc index def0ddd9808e..ea210d56c955 100644 --- a/libbeat/docs/dashboards.asciidoc +++ b/libbeat/docs/dashboards.asciidoc @@ -32,9 +32,7 @@ the _Beats Developer Guide_. ifndef::only-elasticsearch[] If you've configured the Logstash output, see <>. -endif::[] - -ifdef::allplatforms[] +endif::only-elasticsearch[] ifeval::["{requires-sudo}"=="yes"] @@ -42,32 +40,37 @@ include::../../libbeat/docs/shared-note-sudo.asciidoc[] endif::[] +ifdef::deb_os,rpm_os[] *deb and rpm:* ["source","sh",subs="attributes"] ---------------------------------------------------------------------- {beatname_lc} setup --dashboards ---------------------------------------------------------------------- +endif::deb_os,rpm_os[] - +ifdef::mac_os[] *mac:* ["source","sh",subs="attributes"] ---------------------------------------------------------------------- ./{beatname_lc} setup --dashboards ---------------------------------------------------------------------- +endif::mac_os[] - +ifdef::docker_platform[] *docker:* ["source","sh",subs="attributes"] ---------------------------------------------------------------------- docker run --net="host" {dockerimage} setup --dashboards ---------------------------------------------------------------------- +endif::docker_platform[] +ifdef::win_os[] +ifndef::win_only[] *win:* - -endif::allplatforms[] +endif::win_only[] Open a PowerShell prompt as an Administrator (right-click the PowerShell icon and select *Run As Administrator*). @@ -79,6 +82,7 @@ and run: ---------------------------------------------------------------------- PS > .{backslash}{beatname_lc}.exe setup --dashboards ---------------------------------------------------------------------- +endif::win_os[] ifndef::only-elasticsearch[] [[load-dashboards-logstash]] @@ -93,8 +97,7 @@ credentials. TIP: The example shows a hard-coded password, but you should store sensitive values in the <>. -ifdef::allplatforms[] - +ifdef::deb_os,rpm_os[] *deb and rpm:* ["source","sh",subs="attributes"] @@ -106,8 +109,9 @@ ifdef::allplatforms[] -E output.elasticsearch.password={pwd} \ -E setup.kibana.host=localhost:5601 ---- +endif::deb_os,rpm_os[] - +ifdef::mac_os[] *mac:* ["source","sh",subs="attributes"] @@ -119,8 +123,10 @@ ifdef::allplatforms[] -E output.elasticsearch.password={pwd} \ -E setup.kibana.host=localhost:5601 ---- +endif::mac_os[] +ifdef::docker_platform[] *docker:* ["source","sh",subs="attributes"] @@ -132,11 +138,12 @@ docker run --net="host" {dockerimage} setup -e \ -E output.elasticsearch.password={pwd} \ -E setup.kibana.host=localhost:5601 ---- +endif::docker_platform[] - +ifdef::win_os[] +ifndef::win_only[] *win:* - -endif::allplatforms[] +endif::win_only[] Open a PowerShell prompt as an Administrator (right-click the PowerShell icon and select *Run As Administrator*). @@ -153,7 +160,7 @@ PS > .{backslash}{beatname_lc}.exe setup -e ` -E output.elasticsearch.password={pwd} ` -E setup.kibana.host=localhost:5601 ---- - +endif::win_os[] endif::only-elasticsearch[] diff --git a/libbeat/docs/outputconfig.asciidoc b/libbeat/docs/outputconfig.asciidoc index fc949ed7979f..35e1eb658790 100644 --- a/libbeat/docs/outputconfig.asciidoc +++ b/libbeat/docs/outputconfig.asciidoc @@ -215,9 +215,12 @@ The index name to write events to. The default is +"{beatname_lc}-%\{[beat.version]\}-%\{+yyyy.MM.dd\}"+ (for example, +"{beatname_lc}-{version}-{localdate}"+). If you change this setting, you also need to configure the `setup.template.name` and `setup.template.pattern` options -(see <>). If you are using the pre-built Kibana +(see <>). +ifndef::no_dashboards[] +If you are using the pre-built Kibana dashboards, you also need to set the `setup.dashboards.index` option (see <>). +endif::no_dashboards[] You can set the index dynamically by using a format string to access any event field. For example, this configuration uses a custom field, `fields.log_type`, @@ -485,6 +488,7 @@ Elasticsearch. See <> for more information. +//begin outer exclude for ls, kafka, redis, file, and console ifndef::only-elasticsearch[] [[logstash-output]] @@ -994,6 +998,7 @@ Note: If set to 0, no ACKs are returned by Kafka. Messages might be lost silentl Configuration options for SSL parameters like the root CA for Kafka connections. See <> for more information. +//begin inner exclude for redis ifndef::no-redis-output[] [[redis-output]] === Configure the Redis output @@ -1250,6 +1255,7 @@ client. You can change this behavior by setting the This option determines whether Redis hostnames are resolved locally when using a proxy. The default value is false, which means that name resolution occurs on the proxy server. +//end inner exclude for redis endif::[] [[file-output]] @@ -1364,6 +1370,9 @@ Setting `bulk_max_size` to values less than or equal to 0 disables the splitting of batches. When splitting is disabled, the queue decides on the number of events to be contained in a batch. +//end outer exclude for ls, kafka, redis, file, and console +endif::[] + [[configure-cloud-id]] === Configure the output for the Elastic Cloud @@ -1405,6 +1414,9 @@ When specified, the `cloud.auth` overwrites the `output.elasticsearch.username` the username and password from the Elasticsearch output, this can also be used to set the `setup.kibana.username` and `setup.kibana.password` options. +//begin exclude for output codec +ifndef::only-elasticsearch[] + [[configuration-output-codec]] === Change the output codec @@ -1437,4 +1449,5 @@ output.console: string: '%{[@timestamp]} %{[message]}' ------------------------------------------------------------------------------ +//end exclude for output codec endif::[] diff --git a/libbeat/docs/reference-yml.asciidoc b/libbeat/docs/reference-yml.asciidoc index e869825a601e..f506d61c3377 100644 --- a/libbeat/docs/reference-yml.asciidoc +++ b/libbeat/docs/reference-yml.asciidoc @@ -11,7 +11,16 @@ look in the archive that you just extracted. The contents of the file are included here for your convenience. +ifndef::has_xpack[] [source,yaml] -- include::../../{beatname_lc}/{beatname_lc}.reference.yml[] -- +endif::has_xpack[] + +ifdef::has_xpack[] +[source,yaml] +-- +include::../../x-pack/{beatname_lc}/{beatname_lc}.reference.yml[] +-- +endif::has_xpack[] diff --git a/libbeat/docs/security/securing-beats.asciidoc b/libbeat/docs/security/securing-beats.asciidoc index 29eff583df6b..3d8ecda61759 100644 --- a/libbeat/docs/security/securing-beats.asciidoc +++ b/libbeat/docs/security/securing-beats.asciidoc @@ -67,12 +67,16 @@ ifeval::["{beatname_lc}"=="filebeat"] |Run Filebeat modules | ++{beat_default_index_prefix}_writer++ endif::[] |Load index templates | ++{beat_default_index_prefix}_writer++ and `kibana_user` +ifndef::no_dashboards[] |Load {beatname_uc} dashboards into {kib} | ++{beat_default_index_prefix}_writer++ and `kibana_user` +endif::[] ifdef::has_ml_jobs[] |Load machine learning jobs | `machine_learning_admin` endif::[] |Read indices created by {beatname_uc} | ++{beat_default_index_prefix}_reader++ +ifndef::no_dashboards[] |View {beatname_uc} dashboards in {kib} | `kibana_user` +endif::[] |======= include::basic-auth.asciidoc[] diff --git a/libbeat/docs/setup-config.asciidoc b/libbeat/docs/setup-config.asciidoc index 2e8b942c2e3c..2fa6373f72c8 100644 --- a/libbeat/docs/setup-config.asciidoc +++ b/libbeat/docs/setup-config.asciidoc @@ -1,4 +1,6 @@ - +ifndef::no_dashboards[] include::./dashboardsconfig.asciidoc[] +endif::no_dashboards[] + include::./template-config.asciidoc[] diff --git a/libbeat/docs/shared-configuring.asciidoc b/libbeat/docs/shared-configuring.asciidoc index c4008c4dc7f1..e47afd25be1b 100644 --- a/libbeat/docs/shared-configuring.asciidoc +++ b/libbeat/docs/shared-configuring.asciidoc @@ -1,15 +1,8 @@ -ifndef::no-docker[] -To configure {beatname_uc}, you edit the configuration file. For rpm and deb, -you'll find the configuration file at +/etc/{beatname_lc}/{beatname_lc}.yml+. Under -Docker, it's located at +/usr/share/{beatname_lc}/{beatname_lc}.yml+. For mac and win, -look in the archive that you just extracted. -endif::[] -ifdef::no-docker[] -To configure {beatname_uc}, you edit the configuration file. For rpm and deb, -you'll find the configuration file at +/etc/{beatname_lc}/{beatname_lc}.yml+. -For mac and win, look in the archive that you just extracted. -endif::[] +To configure {beatname_uc}, you edit the configuration file. The default +configuration file is called +{beatname_lc}.yml+. The location of the file +varies by platform. To locate the file, see <>. + ifeval::["{beatname_lc}"!="apm-server"] There’s also a full example configuration file called +{beatname_lc}.reference.yml+ that shows all non-deprecated options. @@ -17,4 +10,4 @@ endif::[] TIP: See the {libbeat}/config-file-format.html[Config File Format] section of the -_Beats Platform Reference_ for more about the structure of the config file. +_Beats Platform Reference_ for more about the structure of the config file. \ No newline at end of file diff --git a/libbeat/docs/shared-directory-layout.asciidoc b/libbeat/docs/shared-directory-layout.asciidoc index 8dce21cc9884..4c9c30ad144d 100644 --- a/libbeat/docs/shared-directory-layout.asciidoc +++ b/libbeat/docs/shared-directory-layout.asciidoc @@ -24,15 +24,14 @@ The directory layout of an installation is as follows: | logs | The location for the logs created by {beatname_uc}. | {path.home}/logs | path.logs |======================================================================= -You can change these settings by using CLI flags or setting <> in the configuration -file. +You can change these settings by using CLI flags or setting +<> in the configuration file. ==== Default paths {beatname_uc} uses the following default paths unless you explicitly change them. -ifeval::["{beatname_lc}"!="winlogbeat"] - +ifdef::deb_os,rpm_os[] [float] ===== deb and rpm [cols="> for Debian/Ubuntu, <> for Redhat/Centos/Fedora, <> for OS X, <> for any Docker platform, and <> for -Windows). -endif::[] - -ifdef::no-docker[] -To download and install {beatname_uc}, use the commands that work with your system -(<> for Debian/Ubuntu, <> for Redhat/Centos/Fedora, <> for OS X, and <> for Windows). -endif::[] +To download and install {beatname_uc}, use the commands that work with your +system. +ifeval::["{release-state}"!="unreleased"] [NOTE] ================================================== @@ -24,3 +15,5 @@ repositories>> to update to the newest version more easily. See our https://www.elastic.co/downloads/beats/{beatname_lc}[download page] for other installation options, such as 32-bit images. ================================================== + +endif::[] diff --git a/libbeat/docs/shared-getting-started-intro.asciidoc b/libbeat/docs/shared-getting-started-intro.asciidoc index 84c6f07f1ae4..ff2ca9dd6e54 100644 --- a/libbeat/docs/shared-getting-started-intro.asciidoc +++ b/libbeat/docs/shared-getting-started-intro.asciidoc @@ -7,7 +7,7 @@ related products: * {ls} (optional) for parsing and enhancing the data. See {stack-gs}/get-started-elastic-stack.html[Getting started with the {stack}] -for more information. +for more information about installing these products. [TIP] ============== diff --git a/libbeat/docs/shared-logstash-config.asciidoc b/libbeat/docs/shared-logstash-config.asciidoc index d5a16ec6a5a4..ae22a44a981d 100644 --- a/libbeat/docs/shared-logstash-config.asciidoc +++ b/libbeat/docs/shared-logstash-config.asciidoc @@ -39,16 +39,16 @@ Beats connections. For this configuration, you must <> because the options for auto loading the template are only available for the Elasticsearch output. -ifdef::allplatforms[] +ifndef::win-only[] include::../../libbeat/docs/step-test-config.asciidoc[] -endif::allplatforms[] +endif::win-only[] -ifdef::win[] +ifdef::win-only[] TIP: To test your configuration file, change to the directory where the {beatname_uc} binary is installed, and run {beatname_uc} in the foreground with the following options specified: +.\winlogbeat.exe test config -c .\winlogbeat.yml -e+. -endif::win[] +endif::win-only[] diff --git a/libbeat/docs/shared-template-load.asciidoc b/libbeat/docs/shared-template-load.asciidoc index 6f4f1c7da8ea..f3d23d4d767e 100644 --- a/libbeat/docs/shared-template-load.asciidoc +++ b/libbeat/docs/shared-template-load.asciidoc @@ -7,14 +7,14 @@ //// resolve Beat names: beatname_uc and beatname_lc //// Use the following include to pull this content into a doc file: //// include::../../libbeat/docs/shared-template-load.asciidoc[] -//// If you want to include conditional content, you also need to -//// add the following doc attribute definition before the -//// include statement so that you have: -//// :allplatforms: -//// include::../../libbeat/docs/shared-template-load.asciidoc[] //// This content must be embedded underneath a level 3 heading. ////////////////////////////////////////////////////////////////////////// +ifndef::only-elasticsearch[] +NOTE: A connection to Elasticsearch is required to load the index template. If +the output is not Elasticsearch, you must +<>. +endif::only-elasticsearch[] In Elasticsearch, {elasticsearch}/indices-templates.html[index templates] are used to define settings and mappings that determine how fields @@ -26,35 +26,14 @@ The recommended index template file for {beatname_uc} is installed by the after successfully connecting to Elasticsearch. If the template already exists, it's not overwritten unless you configure {beatname_uc} to do so. -You can disable automatic template loading, or load your own template, by -configuring template loading options in the {beatname_uc} configuration file. - -You can also set options to change the name of the index and index template. - -ifndef::only-elasticsearch[] -NOTE: A connection to Elasticsearch is required to load the index template. If -the output is not Elasticsearch, you must -<>. -endif::[] - -For more information, see: - -ifdef::only-elasticsearch[] -* <> -* <> -endif::[] - -ifndef::only-elasticsearch[] -* <> -* <> - required for non-Elasticsearch output -endif::[] - [[load-template-auto]] ==== Configure template loading By default, {beatname_uc} automatically loads the recommended template file, -+fields.yml+, if the Elasticsearch output is enabled. You can change the -defaults in the +{beatname_lc}.yml+ config file to: ++fields.yml+, if the Elasticsearch output is enabled. If you want to use the +default index template, no additional configuration is required. Otherwise, you +can change the defaults in the +{beatname_lc}.yml+ config file +to: * **Load a different template** + @@ -94,35 +73,26 @@ that you specify should include the root name of the index plus version and date information. You also need to configure the `setup.template.name` and `setup.template.pattern` options to match the new name. For example: + -ifndef::deprecate_dashboard_loading[] - ["source","sh",subs="attributes,callouts"] ----- output.elasticsearch.index: "customname-%{[beat.version]}-%{+yyyy.MM.dd}" setup.template.name: "customname" setup.template.pattern: "customname-*" -setup.dashboards.index: "customname-*" <1> ----- - -<1> If you plan to -<>, also set -this option to overwrite the index name defined in the dashboards and index -pattern. - +ifndef::deprecate_dashboard_loading,no_dashboards[] ++ +If you're using pre-built Kibana dashboards, also set the +`setup.dashboards.index` option. For example: ++ +[source, yaml] +---- +setup.dashboards.index: "customname-*" +---- endif::[] ifdef::deprecate_dashboard_loading[] - -["source","sh",subs="attributes,callouts"] ------ -output.elasticsearch.index: "customname-%{[beat.version]}-%{+yyyy.MM.dd}" -setup.template.name: "customname" -setup.template.pattern: "customname-*" ------ + -Also ensure to change the index name accordingly in the Kibana dashboards, -when loading via the Kibana UI. - +Remember to change the index name when you load dashboards via the Kibana UI. endif::[] See <> for the full list of configuration options. @@ -138,7 +108,7 @@ If another output is enabled, you need to temporarily disable that output and enable Elasticsearch by using the `-E` option. The examples here assume that Logstash output is enabled. You can omit the `-E` flags if Elasticsearch output is already enabled. -endif::[] +endif::only-elasticsearch[] If you are connecting to a secured Elasticsearch cluster, make sure you've configured credentials as described in <<{beatname_lc}-configuration>>. @@ -146,17 +116,21 @@ configured credentials as described in <<{beatname_lc}-configuration>>. If the host running {beatname_uc} does not have direct connectivity to Elasticsearch, see <>. +ifndef::win_only[] To load the template, use the appropriate command for your system. +endif::win_only[] + +ifdef::win_only[] +To load the template: +endif::win_only[] ifndef::only-elasticsearch[] :disable_logstash: {sp}-E output.logstash.enabled=false -endif::[] +endif::only-elasticsearch[] ifdef::only-elasticsearch[] :disable_logstash: -endif::[] - -ifdef::allplatforms[] +endif::only-elasticsearch[] ifeval::["{requires-sudo}"=="yes"] @@ -164,31 +138,36 @@ include::./shared-note-sudo.asciidoc[] endif::[] +ifdef::deb_os,rpm_os[] *deb and rpm:* ["source","sh",subs="attributes"] ---- {beatname_lc} setup --template{disable_logstash} -E 'output.elasticsearch.hosts=["localhost:9200"]' ---- +endif::deb_os,rpm_os[] +ifdef::mac_os[] *mac:* ["source","sh",subs="attributes"] ---- ./{beatname_lc} setup --template{disable_logstash} -E 'output.elasticsearch.hosts=["localhost:9200"]' ---- +endif::mac_os[] - +ifdef::mac_os[] *docker:* ["source","sh",subs="attributes"] ---------------------------------------------------------------------- docker run {dockerimage} setup --template{disable_logstash} -E 'output.elasticsearch.hosts=["localhost:9200"]' ---------------------------------------------------------------------- +endif::mac_os[] - +ifdef::win_os[] +ifndef::win_only[] *win:* - -endif::allplatforms[] +endif::win_only[] Open a PowerShell prompt as an Administrator (right-click the PowerShell icon and select *Run As Administrator*). @@ -200,7 +179,7 @@ and run: ---------------------------------------------------------------------- PS > .{backslash}{beatname_lc}.exe setup --template{disable_logstash} -E 'output.elasticsearch.hosts=["localhost:9200"]' ---------------------------------------------------------------------- - +endif::win_os[] [[force-kibana-new]] ===== Force Kibana to look at newest documents @@ -210,20 +189,34 @@ the index may contain old documents. After you load the index template, you can delete the old documents from +{beatname_lc}-*+ to force Kibana to look at the newest documents. Use this command: -*deb, rpm, and mac:* +ifdef::deb_os,rpm_os[] +*deb and rpm:* + +["source","sh",subs="attributes"] +---------------------------------------------------------------------- +curl -XDELETE 'http://localhost:9200/{beatname_lc}-*' +---------------------------------------------------------------------- +endif::deb_os,rpm_os[] + +ifdef::mac_os[] +*mac:* ["source","sh",subs="attributes"] ---------------------------------------------------------------------- curl -XDELETE 'http://localhost:9200/{beatname_lc}-*' ---------------------------------------------------------------------- +endif::mac_os[] +ifdef::win_os[] +ifndef::win_only[] *win:* +endif::win_only[] ["source","sh",subs="attributes"] ---------------------------------------------------------------------- PS > Invoke-RestMethod -Method Delete "http://localhost:9200/{beatname_lc}-*" ---------------------------------------------------------------------- - +endif::win_os[] This command deletes all indices that match the pattern +{beat_default_index_prefix}-*+. Before running this command, make sure you want to delete all indices that match @@ -236,43 +229,64 @@ If the host running {beatname_uc} does not have direct connectivity to Elasticsearch, you can export the index template to a file, move it to a machine that does have connectivity, and then install the template manually. -. Export the index template: -+ -ifdef::allplatforms[] +To export the index template, run: + +ifdef::deb_os,rpm_os[] *deb and rpm:* -+ + ["source","sh",subs="attributes"] ---- {beatname_lc} export template > {beatname_lc}.template.json ---- -+ +endif::deb_os,rpm_os[] + +ifdef::mac_os[] *mac:* -+ + ["source","sh",subs="attributes"] ---- ./{beatname_lc} export template > {beatname_lc}.template.json ---- -+ +endif::mac_os[] + +ifdef::win_os[] +ifndef::win_only[] *win:* -+ -endif::allplatforms[] +endif::win_only[] + ["source","sh",subs="attributes"] ---- PS > .{backslash}{beatname_lc}.exe export template --es.version {stack-version} | Out-File -Encoding UTF8 {beatname_lc}.template.json ---- +endif::win_os[] + +To install the template, run: + +ifdef::deb_os,rpm_os[] +*deb and rpm:* -. Install the template: -+ -*deb, rpm, and mac:* -+ ["source","sh",subs="attributes"] ---- curl -XPUT -H 'Content-Type: application/json' http://localhost:9200/_template/{beatname_lc}-{stack-version} -d@{beatname_lc}.template.json ---- -+ +endif::deb_os,rpm_os[] + +ifdef::mac_os[] +*mac:* + +["source","sh",subs="attributes"] +---- +curl -XPUT -H 'Content-Type: application/json' http://localhost:9200/_template/{beatname_lc}-{stack-version} -d@{beatname_lc}.template.json +---- +endif::mac_os[] + +ifdef::win_os[] +ifndef::win_only[] *win:* -+ +endif::win_only[] + ["source","sh",subs="attributes"] ---- PS > Invoke-RestMethod -Method Put -ContentType "application/json" -InFile {beatname_lc}.template.json -Uri http://localhost:9200/_template/{beatname_lc}-{stack-version} ---- +endif::win_os[] diff --git a/libbeat/docs/step-configure-credentials.asciidoc b/libbeat/docs/step-configure-credentials.asciidoc index 6da1e51eb38b..5b43e51fffef 100644 --- a/libbeat/docs/step-configure-credentials.asciidoc +++ b/libbeat/docs/step-configure-credentials.asciidoc @@ -30,8 +30,8 @@ values in the <>. <2> The `username` and `password` settings for {kib} are optional. If you don't specify credentials for {kib}, {beatname_uc} uses the `username` and `password` specified for the {es} output. -<3> If you are planning to <>, -the user must have the `kibana_user` -{xpack-ref}/built-in-roles.html[built-in role] or equivalent privileges. +<3> To use the pre-built Kibana dashboards, this user must have the +`kibana_user` {xpack-ref}/built-in-roles.html[built-in role] or equivalent +privileges. + For more information, see <>. diff --git a/libbeat/docs/step-configure-output.asciidoc b/libbeat/docs/step-configure-output.asciidoc index a9dfb316d591..d607ff074d63 100644 --- a/libbeat/docs/step-configure-output.asciidoc +++ b/libbeat/docs/step-configure-output.asciidoc @@ -1,4 +1,5 @@ ifndef::has_module_steps[] +ifndef::only-elasticsearch[] . Configure the output. {beatname_uc} supports a variety of <>, but typically you'll either send events directly to {es}, or to {ls} for additional processing. @@ -7,7 +8,13 @@ To send output directly to {es} (without using {ls}), set the location of the {es} installation: + -- -endif::[] +endif::only-elasticsearch[] +ifdef::only-elasticsearch[] +. Configure the {es} output by setting the location of the {es} installation: ++ +-- +endif::only-elasticsearch[] +endif::has_module_steps[] * If you're running our https://www.elastic.co/cloud/elasticsearch-service[hosted {es} Service] on Elastic Cloud, specify your <>. For example: @@ -28,6 +35,7 @@ output.elasticsearch: ifndef::has_module_steps[] -- + +ifndef::only-elasticsearch[] ifeval::["{beatname_lc}"!="filebeat" and "{beatname_lc}"!="winlogbeat"] To send output to {ls}, <> instead. For all other @@ -37,4 +45,8 @@ ifeval::[("{beatname_lc}"=="filebeat") or ("{beatname_lc}"=="winlogbeat")] To send output to {ls}, make sure you configure the Logstash output in <>. For all other outputs, see <>. endif::[] -endif::[] +endif::only-elasticsearch[] +ifdef::only-elasticsearch[] +{es} is currently the only output supported by {beatname_uc}. +endif::only-elasticsearch[] +endif::has_module_steps[] diff --git a/libbeat/docs/yaml.asciidoc b/libbeat/docs/yaml.asciidoc index 1266115ba625..15997c4acf03 100644 --- a/libbeat/docs/yaml.asciidoc +++ b/libbeat/docs/yaml.asciidoc @@ -46,23 +46,23 @@ Simply change to the directory where the binary is installed, and run the Beat in the foreground with the `test config` command specified. For example: -ifdef::allplatforms[] +ifndef::win-only[] ["source","shell",subs="attributes"] ---------------------------------------------------------------------- {beatname_lc} test config -c {beatname_lc}.yml ---------------------------------------------------------------------- -endif::allplatforms[] +endif::win-only[] -ifdef::win[] +ifdef::win-only[] ["source","shell",subs="attributes"] ---------------------------------------------------------------------- .\winlogbeat.exe test config -c .\winlogbeat.yml -e ---------------------------------------------------------------------- -endif::win[] +endif::win-only[] You'll see a message if the Beat finds an error in the file. diff --git a/metricbeat/docs/configuring-howto.asciidoc b/metricbeat/docs/configuring-howto.asciidoc index e20e78f7d948..c0fa9058025e 100644 --- a/metricbeat/docs/configuring-howto.asciidoc +++ b/metricbeat/docs/configuring-howto.asciidoc @@ -47,7 +47,6 @@ include::./metricbeat-general-options.asciidoc[] include::./reload-configuration.asciidoc[] -:allplatforms: include::../../libbeat/docs/queueconfig.asciidoc[] include::../../libbeat/docs/outputconfig.asciidoc[] @@ -68,14 +67,15 @@ include::../../libbeat/docs/loggingconfig.asciidoc[] :standalone: include::../../libbeat/docs/shared-env-vars.asciidoc[] +:standalone!: :autodiscoverJolokia: :autodiscoverHints: include::../../libbeat/docs/shared-autodiscover.asciidoc[] :standalone: -:allplatforms: include::../../libbeat/docs/yaml.asciidoc[] +:standalone!: include::../../libbeat/docs/regexp.asciidoc[] diff --git a/metricbeat/docs/gettingstarted.asciidoc b/metricbeat/docs/gettingstarted.asciidoc index 4861f0ed7783..1bb8036309fa 100644 --- a/metricbeat/docs/gettingstarted.asciidoc +++ b/metricbeat/docs/gettingstarted.asciidoc @@ -214,13 +214,11 @@ include::../../libbeat/docs/step-look-at-config.asciidoc[] [id="{beatname_lc}-template"] === Step 3: Load the index template in Elasticsearch -:allplatforms: include::../../libbeat/docs/shared-template-load.asciidoc[] [[load-kibana-dashboards]] === Step 4: Set up the Kibana dashboards -:allplatforms: include::../../libbeat/docs/dashboards.asciidoc[] [id="{beatname_lc}-starting"] @@ -234,14 +232,7 @@ NOTE: If you use an init.d script to start {beatname_uc} on deb or rpm, you can' specify command line flags (see <>). To specify flags, start {beatname_uc} in the foreground. -*deb:* - -["source","sh",subs="attributes"] ----------------------------------------------------------------------- -sudo service {beatname_lc} start ----------------------------------------------------------------------- - -*rpm:* +*deb and rpm:* ["source","sh",subs="attributes"] ---------------------------------------------------------------------- diff --git a/metricbeat/docs/index.asciidoc b/metricbeat/docs/index.asciidoc index 6168cb2f80e9..913bdcf0e904 100644 --- a/metricbeat/docs/index.asciidoc +++ b/metricbeat/docs/index.asciidoc @@ -12,6 +12,11 @@ include::{asciidoc-dir}/../../shared/attributes.asciidoc[] :discuss_forum: beats/{beatname_lc} :beat_default_index_prefix: {beatname_lc} :has_ml_jobs: yes +:deb_os: +:rpm_os: +:mac_os: +:docker_platform: +:win_os: include::../../libbeat/docs/shared-beats-attributes.asciidoc[] diff --git a/packetbeat/docs/configuring-howto.asciidoc b/packetbeat/docs/configuring-howto.asciidoc index 72a22e81fe79..74fb08fc36cf 100644 --- a/packetbeat/docs/configuring-howto.asciidoc +++ b/packetbeat/docs/configuring-howto.asciidoc @@ -46,7 +46,6 @@ include::./packetbeat-options.asciidoc[] include::./packetbeat-general-options.asciidoc[] -:allplatforms: include::../../libbeat/docs/queueconfig.asciidoc[] include::../../libbeat/docs/outputconfig.asciidoc[] @@ -69,10 +68,11 @@ include::../../libbeat/docs/loggingconfig.asciidoc[] :standalone: include::../../libbeat/docs/shared-env-vars.asciidoc[] +:standalone!: :standalone: -:allplatforms: include::../../libbeat/docs/yaml.asciidoc[] +:standalone!: include::../../libbeat/docs/http-endpoint.asciidoc[] diff --git a/packetbeat/docs/gettingstarted.asciidoc b/packetbeat/docs/gettingstarted.asciidoc index dd38968bddba..d749029d3cc0 100644 --- a/packetbeat/docs/gettingstarted.asciidoc +++ b/packetbeat/docs/gettingstarted.asciidoc @@ -239,14 +239,12 @@ include::../../libbeat/docs/step-look-at-config.asciidoc[] === Step 3: Load the index template in Elasticsearch :requires-sudo: yes -:allplatforms: include::../../libbeat/docs/shared-template-load.asciidoc[] [[load-kibana-dashboards]] === Step 4: Set up the Kibana dashboards :requires-sudo: yes -:allplatforms: include::../../libbeat/docs/dashboards.asciidoc[] [[packetbeat-starting]] @@ -260,14 +258,7 @@ NOTE: If you use an init.d script to start Packetbeat on deb or rpm, you can't specify command line flags (see <>). To specify flags, start Packetbeat in the foreground. -*deb:* - -["source","sh",subs="attributes"] ----------------------------------------------------------------------- -sudo service {beatname_lc} start ----------------------------------------------------------------------- - -*rpm:* +*deb and rpm:* ["source","sh",subs="attributes"] ---------------------------------------------------------------------- diff --git a/packetbeat/docs/index.asciidoc b/packetbeat/docs/index.asciidoc index 105c7663ae34..920377164a6c 100644 --- a/packetbeat/docs/index.asciidoc +++ b/packetbeat/docs/index.asciidoc @@ -12,6 +12,11 @@ include::{asciidoc-dir}/../../shared/attributes.asciidoc[] :discuss_forum: beats/{beatname_lc} :beat_default_index_prefix: {beatname_lc} :has_ml_jobs: yes +:deb_os: +:rpm_os: +:mac_os: +:docker_platform: +:win_os: include::../../libbeat/docs/shared-beats-attributes.asciidoc[] diff --git a/winlogbeat/docs/configuring-howto.asciidoc b/winlogbeat/docs/configuring-howto.asciidoc index 146564db5a17..2ae8ab527daa 100644 --- a/winlogbeat/docs/configuring-howto.asciidoc +++ b/winlogbeat/docs/configuring-howto.asciidoc @@ -40,7 +40,6 @@ include::./winlogbeat-options.asciidoc[] include::./winlogbeat-general-options.asciidoc[] -:win: include::../../libbeat/docs/queueconfig.asciidoc[] include::../../libbeat/docs/outputconfig.asciidoc[] @@ -61,10 +60,11 @@ include::../../libbeat/docs/loggingconfig.asciidoc[] :standalone: include::../../libbeat/docs/shared-env-vars.asciidoc[] +:standalone!: :standalone: -:win: include::../../libbeat/docs/yaml.asciidoc[] +:standalone!: include::../../libbeat/docs/http-endpoint.asciidoc[] diff --git a/winlogbeat/docs/getting-started.asciidoc b/winlogbeat/docs/getting-started.asciidoc index dca176fbd856..ecfc3162f314 100644 --- a/winlogbeat/docs/getting-started.asciidoc +++ b/winlogbeat/docs/getting-started.asciidoc @@ -111,7 +111,6 @@ PS C:\Program Files\Winlogbeat> .\winlogbeat.exe test config -c .\winlogbeat.yml [[config-winlogbeat-logstash]] === Step 3: Configure Winlogbeat to use Logstash -:win: include::../../libbeat/docs/shared-logstash-config.asciidoc[] [[winlogbeat-template]] @@ -122,7 +121,6 @@ include::../../libbeat/docs/shared-template-load.asciidoc[] [[load-kibana-dashboards]] === Step 5: Set up the Kibana dashboards -:win: include::../../libbeat/docs/dashboards.asciidoc[] [[winlogbeat-starting]] diff --git a/winlogbeat/docs/index.asciidoc b/winlogbeat/docs/index.asciidoc index 638e4614af13..2417d31ff3d9 100644 --- a/winlogbeat/docs/index.asciidoc +++ b/winlogbeat/docs/index.asciidoc @@ -12,6 +12,8 @@ include::{asciidoc-dir}/../../shared/attributes.asciidoc[] :discuss_forum: beats/{beatname_lc} :beat_default_index_prefix: {beatname_lc} :has_ml_jobs: yes +:win_os: +:win_only: include::../../libbeat/docs/shared-beats-attributes.asciidoc[] diff --git a/x-pack/functionbeat/docs/config-options.asciidoc b/x-pack/functionbeat/docs/config-options.asciidoc new file mode 100644 index 000000000000..44c90fedaad1 --- /dev/null +++ b/x-pack/functionbeat/docs/config-options.asciidoc @@ -0,0 +1,125 @@ +[id="configuration-{beatname_lc}-options"] +== Configure functions + +++++ +Configure functions +++++ + +{beatname_uc} runs as a function in your serverless environment. + +Before deploying {beatname_uc}, you need to configure one or more functions and +specify details about the services that will trigger the functions. + +You configure the functions in the the +{beatname_lc}.yml+ configuration file. +When you're done, you can <<{beatname_lc}-deploying,deploy the functions>> +to your serverless environment. + +The following example configures two functions: `cloudwatch` and `sqs`. The +`cloudwatch` function collects events from CloudWatch Logs. The `sqs` function +collects messages from Amazon Simple Queue Service (SQS). Both functions forward +the events to {es}. + +["source","sh",subs="attributes"] +---- +{beatname_lc}.provider.aws.deploy_bucket: "functionbeat-deploy" +{beatname_lc}.provider.aws.functions: + - name: cloudwatch + enabled: true + type: cloudwatch_logs + description: "lambda function for cloudwatch logs" + triggers: + - log_group_name: /aws/lambda/my-lambda-function + #filter_pattern: mylog_ + - name: sqs + enabled: true + type: sqs + description: "lambda function for SQS events" + triggers: + - event_source_arn: arn:aws:sqs:us-east-1:123456789012:myevents +output.elasticsearch: + cloud.id: "MyESDeployment:SomeLongString==" + cloud.auth: "elastic:SomeLongString" +processors: + - add_host_metadata: ~ + - add_cloud_metadata: ~ +---- + + +[id="{beatname_lc}-options"] +[float] +=== Configuration options +You can specify the following options to configure the functions that you want +to deploy. + +TIP: If you change the configuration after deploying the function, use +the <> to update your deployment. + +[float] +[id="{beatname_lc}-deploy-bucket"] +==== `provider.aws.deploy_bucket` + +A unique name for the S3 bucket that the Lambda artifact will be uploaded to. + +[float] +[id="{beatname_lc}-name"] +==== `name` + +A unique name for the Lambda function. This is the name of the function as it +will appear in the Lambda console on AWS. + +[float] +[id="{beatname_lc}-type"] +==== `type` + +The type of service to monitor. For this release, the supported types +are: + +`cloudwatch_logs`:: Collects events from CloudWatch logs. +`sqs`:: Collects data from Amazon Simple Queue Service (SQS). + +[float] +[id="{beatname_lc}-description"] +==== `description` + +A description of the function. This description is useful when you are running +multiple functions and need more context about how each function is used. + +[float] +[id="{beatname_lc}-triggers"] +==== `triggers` + +A list of triggers that will cause the function to execute. The list of valid +triggers depends on the `type`. If `type` is `cloudwatch_logs` logs, specify a +list of log groups. If `type` is `sqs`, specify a list of Amazon Resource Names +(ARNs). + +[float] +[id="{beatname_lc}-filter_pattern"] +==== `filter_pattern` + +A regular expression that matches the events you want to collect. Setting this +option may reduce execution costs because the function only executes if there is +data that matches the pattern. + +[float] +[id="{beatname_lc}-concurrency"] +==== `concurrency` + +The reserved number of instances for the function. Setting this option may +reduce execution costs by limiting the number of functions that can execute in +your serverless environment. The default is unreserved. + +[float] +[id="{beatname_lc}-memory-size"] +==== `memory_size` + +The maximum amount of memory to allocate for this function. Specify a value that +is a factor of 64. There is a hard limit of 3008 MiB for each function. The +default is 128 MiB. + +[float] +[id="{beatname_lc}-dead-letter-config"] +==== `dead_letter_config.target_arn` + +The dead letter queue to use for messages that can't be processed successfully. +Set this option to an ARN that points to an SQS queue. diff --git a/x-pack/functionbeat/docs/configuring-howto.asciidoc b/x-pack/functionbeat/docs/configuring-howto.asciidoc new file mode 100644 index 000000000000..499fd2a976e1 --- /dev/null +++ b/x-pack/functionbeat/docs/configuring-howto.asciidoc @@ -0,0 +1,71 @@ +[id="configuring-howto-{beatname_lc}"] += Configuring {beatname_uc} + +[partintro] +-- +Before modifying configuration settings, make sure you've completed the +<<{beatname_lc}-configuration,configuration steps>> in the Getting Started. +This section describes some common use cases for changing configuration options. + +include::{libbeat-dir}/docs/shared-configuring.asciidoc[] + +The following topics describe how to configure {beatname_uc}: + +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <<{beatname_lc}-reference-yml>> + +-- + +include::./config-options.asciidoc[] + +include::./general-options.asciidoc[] + +:allplatforms: +include::{libbeat-dir}/docs/queueconfig.asciidoc[] +:allplatforms!: + +include::{libbeat-dir}/docs/outputconfig.asciidoc[] + +include::{libbeat-dir}/docs/shared-ssl-config.asciidoc[] + +include::./filtering.asciidoc[] + +include::{libbeat-dir}/docs/shared-config-ingest.asciidoc[] + +include::{libbeat-dir}/docs/shared-path-config.asciidoc[] + +include::{libbeat-dir}/docs/shared-kibana-config.asciidoc[] + +include::{libbeat-dir}/docs/setup-config.asciidoc[] + +include::{libbeat-dir}/docs/loggingconfig.asciidoc[] + +:standalone: +include::{libbeat-dir}/docs/shared-env-vars.asciidoc[] +:standalone!: + +:standalone: +:allplatforms: +include::{libbeat-dir}/docs/yaml.asciidoc[] +:standalone!: +:allplatforms!: + +include::{libbeat-dir}/docs/regexp.asciidoc[] + +include::{libbeat-dir}/docs/http-endpoint.asciidoc[] + +include::{libbeat-dir}/docs/reference-yml.asciidoc[] diff --git a/x-pack/functionbeat/docs/faq.asciidoc b/x-pack/functionbeat/docs/faq.asciidoc new file mode 100644 index 000000000000..2ca1763168ec --- /dev/null +++ b/x-pack/functionbeat/docs/faq.asciidoc @@ -0,0 +1,11 @@ +[[faq]] +== Frequently asked questions + +This section contains frequently asked questions about {beatname_uc}. Also check +out the https://discuss.elastic.co/c/beats/{beatname_lc}[{beatname_uc} +discussion forum]. + + +include::{libbeat-dir}/docs/faq-limit-bandwidth.asciidoc[] + +include::{libbeat-dir}/docs/shared-faq.asciidoc[] diff --git a/x-pack/functionbeat/docs/filtering.asciidoc b/x-pack/functionbeat/docs/filtering.asciidoc new file mode 100644 index 000000000000..cfb7cff64514 --- /dev/null +++ b/x-pack/functionbeat/docs/filtering.asciidoc @@ -0,0 +1,25 @@ +[[filtering-and-enhancing-data]] +== Filter and enhance the exported data + +Your use case might require only a subset of the data exported by {beatname_uc}, +or you might need to enhance the exported data (for example, by adding +metadata). {beatname_uc} provides a couple of options for filtering and +enhancing exported data. + +You can specify a <<{beatname_lc}-filter_pattern,`filter_pattern`>> to match the +data you want to send. This approach may reduce execution costs because the +function running {beatname_uc} only executes if there is data that matches the +pattern. + +Another approach (the one described here) is to define processors. + + +[float] +[[using-processors]] +=== Processors + +include::{libbeat-dir}/docs/processors.asciidoc[] + +:processor-scope: function +include::{libbeat-dir}/docs/processors-using.asciidoc[] +:processor-scope!: diff --git a/x-pack/functionbeat/docs/general-options.asciidoc b/x-pack/functionbeat/docs/general-options.asciidoc new file mode 100644 index 000000000000..60e017c7a29d --- /dev/null +++ b/x-pack/functionbeat/docs/general-options.asciidoc @@ -0,0 +1,8 @@ +[[configuration-general-options]] +== Specify general settings + +You can specify settings in the +{beatname_lc}.yml+ config file to control the +general behavior of {beatname_uc}. + +include::{libbeat-dir}/docs/generalconfig.asciidoc[] + diff --git a/x-pack/functionbeat/docs/getting-started.asciidoc b/x-pack/functionbeat/docs/getting-started.asciidoc new file mode 100644 index 000000000000..d611baf4bdac --- /dev/null +++ b/x-pack/functionbeat/docs/getting-started.asciidoc @@ -0,0 +1,218 @@ +[id="{beatname_lc}-getting-started"] +== Getting Started With {beatname_uc} + +include::{libbeat-dir}/docs/shared-getting-started-intro.asciidoc[] + +* <<{beatname_lc}-installation>> +* <<{beatname_lc}-configuration>> +* <<{beatname_lc}-template>> +* <<{beatname_lc}-deploying>> +* <> +* <> + +[id="{beatname_lc}-installation"] +=== Step 1: Download the {beatname_uc} package + +The {beatname_uc} package contains the command line tools, configuration file, +and binary code required to run {beatname_uc} in your serverless environment. + +To download and extract the package, use the commands that work with your +system. + +[[linux]] +*linux:* + +ifeval::["{release-state}"=="unreleased"] + +Version {version} of {beatname_uc} has not yet been released. + +endif::[] + +ifeval::["{release-state}"!="unreleased"] + +["source","sh",subs="attributes"] +------------------------------------------------ +curl -L -O https://artifacts.elastic.co/downloads/beats/x-pack/{beatname_lc}/{beatname_lc}-{version}-linux-x86_64.tar.gz +tar xzvf {beatname_lc}-{version}-linux-x86_64.tar.gz +------------------------------------------------ + +endif::[] + +[[mac]] +*mac:* + +ifeval::["{release-state}"=="unreleased"] + +Version {version} of {beatname_uc} has not yet been released. + +endif::[] + +ifeval::["{release-state}"!="unreleased"] + +["source","sh",subs="attributes"] +------------------------------------------------ +curl -L -O https://artifacts.elastic.co/downloads/beats/x-pack/{beatname_lc}/{beatname_lc}-{version}-darwin-x86_64.tar.gz +tar xzvf {beatname_lc}-{version}-darwin-x86_64.tar.gz +------------------------------------------------ + +endif::[] + +[[win]] +*win:* + +ifeval::["{release-state}"=="unreleased"] + +Version {version} of {beatname_uc} has not yet been released. + +endif::[] + +ifeval::["{release-state}"!="unreleased"] + +. Download the {beatname_uc} Windows zip file from the +https://www.elastic.co/downloads/beats/{beatname_lc}[downloads page]. + +. Extract the contents of the zip file. + +endif::[] + +[id="{beatname_lc}-configuration"] +=== Step 2: Configure {beatname_uc} + +Before deploying {beatname_uc} to your serverless environment, you need to +specify details about the functions that you want to deploy, including the +function name, type, and triggers that will cause the function to execute. +You also need to specify connection details for your {es} cluster. + +You specify settings in the +{beatname_lc}.yml+ configuration file. This file +is located in the archive that you extracted earlier. + +TIP: See the +{libbeat}/config-file-format.html[Config File Format] section of the +_Beats Platform Reference_ for more about the structure of the config file. + +The following example configures a function called `cloudwatch` that collects +events from CloudWatch Logs and forwards the events to {es}. + +["source","sh",subs="attributes"] +------------------------------------------------------------------------------------- +{beatname_lc}.provider.aws.deploy_bucket: "functionbeat-deploy" +{beatname_lc}.provider.aws.functions: + - name: cloudwatch + enabled: true + type: cloudwatch_logs + description: "lambda function for cloudwatch logs" + triggers: + - log_group_name: /aws/lambda/my-lambda-function +output.elasticsearch: + cloud.id: "MyESDeployment:SomeLongString==" + cloud.auth: "elastic:SomeLongString" +------------------------------------------------------------------------------------- + +To configure {beatname_uc}: + +. Specify a unique name for the S3 bucket to which the functions will be +uploaded. For example: ++ +["source","sh",subs="attributes"] +---- +{beatname_lc}.provider.aws.deploy_bucket: "functionbeat-deploy" +---- + +. Define the functions that want to deploy. For each function, you must specify: ++ +[horizontal] +`name`:: A unique name for the Lambda function. +`type`:: The type of service to monitor. For this release, the supported types +are: +* `cloudwatch_logs` to collect data from CloudWatch logs +* `sqs` to collect messages from Amazon Simple Queue Service (SQS) +`triggers`:: The triggers that will cause the function to execute. If `type` +is `cloudwatch_logs` logs, specify a list of log groups. If `type` is `sqs`, +specify a list of Amazon Resource Names (ARNs). ++ +When a message is sent to the specified log group or queue, the Lambda function +executes and sends message events to the output configured for {beatname_uc}. ++ +The following example configures a function called `sqs` that collects data +from Amazon SQS: ++ +["source","sh",subs="attributes"] +---- +- name: sqs + enabled: true + type: sqs + triggers: + - event_source_arn: arn:aws:sqs:us-east-1:123456789012:myevents +---- + +include::{libbeat-dir}/docs/step-configure-output.asciidoc[] + +include::{libbeat-dir}/docs/step-configure-credentials.asciidoc[] + +include::{libbeat-dir}/docs/step-test-config.asciidoc[] + +include::{libbeat-dir}/docs/step-look-at-config.asciidoc[] + +[id="{beatname_lc}-template"] +=== Step 3: Load the index template in Elasticsearch + +:allplatforms: +include::{libbeat-dir}/docs/shared-template-load.asciidoc[] + +[id="{beatname_lc}-deploying"] +=== Step 4: Deploy {beatname_uc} code to your serverless environment + +Before deploying functions to your serverless environment, make sure your user +has the credentials required by your cloud service provider. For example, if you +are deploying an AWS Lambda function, you can set environment variables that +contain your credentials: + +*linux and mac*: + +[source, shell] +---- +export AWS_ACCESS_KEY_ID=ABCDEFGHIJKLMNOPUSER +export AWS_SECRET_ACCESS_KEY=EXAMPLE567890devgHIJKMLOPNQRSTUVZ1234KEY +export AWS_DEFAULT_REGION=us-east-1 +---- + +*win*: + +[source, shell] +---- +set AWS_ACCESS_KEY_ID=ABCDEFGHIJKLMNOPUSER +set AWS_SECRET_ACCESS_KEY=EXAMPLE567890devgHIJKMLOPNQRSTUVZ1234KEY +set AWS_DEFAULT_REGION=us-east-1 +---- + +After setting credentials, run the `deploy` command to deploy the function. +The following command deploys a function called `cloudwatch`: + +*linux and mac:* + +["source","sh",subs="attributes"] +---------------------------------------------------------------------- +./{beatname_lc} -v -e -d "*" deploy cloudwatch +---------------------------------------------------------------------- + +*win:* + +["source","sh",subs="attributes"] +---------------------------------------------------------------------- +.{backslash}{beatname_lc}.exe -v -e -d "*" deploy cloudwatch +---------------------------------------------------------------------- + +{beatname_uc} is now deployed in your serverless environment and ready to send +log events to the configured output. + +TIP: If you change the configuration after deploying the function, use +the <> to update your deployment. + +[[view-kibana-dashboards]] +=== Step 5: View your data in Kibana + +There are currently no example dashboards available for {beatname_uc}. + +To learn how to view and explore your data, see the +_{kibana-ref}/index.html[{kib} User Guide]_. + diff --git a/x-pack/functionbeat/docs/index.asciidoc b/x-pack/functionbeat/docs/index.asciidoc index 4bdd7d1c2df0..5c091f4fc740 100644 --- a/x-pack/functionbeat/docs/index.asciidoc +++ b/x-pack/functionbeat/docs/index.asciidoc @@ -1,6 +1,8 @@ = Functionbeat Reference -include::../../../libbeat/docs/version.asciidoc[] +:libbeat-dir: ../../../libbeat + +include::{libbeat-dir}/docs/version.asciidoc[] include::{asciidoc-dir}/../../shared/attributes.asciidoc[] @@ -12,9 +14,36 @@ include::{asciidoc-dir}/../../shared/attributes.asciidoc[] :discuss_forum: beats/{beatname_lc} :beat_default_index_prefix: {beatname_lc} :has_ml_jobs: no +:libbeat-docs: Beats Platform Reference +:only-elasticsearch: +:has_xpack: +:serverless: +:mac_os: +:win_os: +:linux_os: +:no_dashboards: +include::{libbeat-dir}/docs/shared-beats-attributes.asciidoc[] -include::../../../libbeat/docs/shared-beats-attributes.asciidoc[] +:release-state: released include::./overview.asciidoc[] +include::./getting-started.asciidoc[] + +include::{libbeat-dir}/docs/repositories.asciidoc[] + +include::./setting-up-running.asciidoc[] + +include::./configuring-howto.asciidoc[] + +include::./fields.asciidoc[] + +include::{libbeat-dir}/docs/monitoring/monitoring-beats.asciidoc[] + +include::{libbeat-dir}/docs/shared-securing-beat.asciidoc[] + +include::./troubleshooting.asciidoc[] + +include::./faq.asciidoc[] + diff --git a/x-pack/functionbeat/docs/overview.asciidoc b/x-pack/functionbeat/docs/overview.asciidoc index 74ae9bbbe20c..17f0a7816ba4 100644 --- a/x-pack/functionbeat/docs/overview.asciidoc +++ b/x-pack/functionbeat/docs/overview.asciidoc @@ -5,6 +5,15 @@ Overview ++++ -{beatname_uc} is a lightweight shipper that you can install as a function on -a serverless provider.For example, you can use {beatname_uc} to collect and -centralize log events from the Cloudwatchlogs to your Elasticsearch cluster. +{beatname_uc} is an Elastic https://www.elastic.co/products/beats[Beat] that you +deploy on your serverless environment to collect events generated by cloud +services and ship the events to {es}. + +The beta version supports deploying {beatname_uc} as an AWS Lambda service and +responds to the triggers defined for the following event sources: + +* https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/WhatIsCloudWatchLogs.html[CloudWatch Logs] +* https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/welcome.html[Amazon Simple Queue Service (SQS)] + +{beatname_uc} is based on the `libbeat` framework. For more information, see the +{libbeat}/index.html[{libbeat-docs}]. diff --git a/x-pack/functionbeat/docs/page_header.html b/x-pack/functionbeat/docs/page_header.html new file mode 100644 index 000000000000..73213d675743 --- /dev/null +++ b/x-pack/functionbeat/docs/page_header.html @@ -0,0 +1,4 @@ +This functionality is in beta and is subject to change. The design and code is +less mature than official GA features and is being provided as-is with no +warranties. Beta features are not subject to the support SLA of official GA +features. \ No newline at end of file diff --git a/x-pack/functionbeat/docs/setting-up-running.asciidoc b/x-pack/functionbeat/docs/setting-up-running.asciidoc new file mode 100644 index 000000000000..3a620425759c --- /dev/null +++ b/x-pack/functionbeat/docs/setting-up-running.asciidoc @@ -0,0 +1,30 @@ +///// +// NOTE: +// Each beat has its own setup overview to allow for the addition of content +// that is unique to each beat. +///// + +[[setting-up-and-running]] +== Setting up and running {beatname_uc} + +Before reading this section, see the +<<{beatname_lc}-getting-started,getting started documentation>> for basic +installation instructions to get you started. + +This section includes additional information on how to set up and run +{beatname_uc}, including: + +* <> + +* <> + +* <> + + +//MAINTAINERS: If you add a new file to this section, make sure you update the bulleted list ^^ too. + +include::{libbeat-dir}/docs/shared-directory-layout.asciidoc[] + +include::{libbeat-dir}/docs/keystore.asciidoc[] + +include::{libbeat-dir}/docs/command-reference.asciidoc[] diff --git a/x-pack/functionbeat/docs/troubleshooting.asciidoc b/x-pack/functionbeat/docs/troubleshooting.asciidoc new file mode 100644 index 000000000000..223fd3023c54 --- /dev/null +++ b/x-pack/functionbeat/docs/troubleshooting.asciidoc @@ -0,0 +1,30 @@ +[[troubleshooting]] += Troubleshooting + +[partintro] +-- +If you have issues installing or running {beatname_uc}, read the +following tips: + +* <> +* <> +* <> + +//sets block macro for getting-help.asciidoc included in next section + +-- + +[[getting-help]] +== Get help + +include::{libbeat-dir}/docs/getting-help.asciidoc[] + +//sets block macro for debugging.asciidoc included in next section + +[id="enable-{beatname_lc}-debugging"] +== Debug + +include::{libbeat-dir}/docs/debugging.asciidoc[] + + +