From be9a996830be63c7341014f01fe3bc0a0808ff31 Mon Sep 17 00:00:00 2001 From: dedemorton Date: Thu, 1 Nov 2018 15:46:06 -0700 Subject: [PATCH 1/4] Docs for journalbeat review --- journalbeat/docs/config-options.asciidoc | 124 + journalbeat/docs/configuring-howto.asciidoc | 68 + journalbeat/docs/faq.asciidoc | 10 + journalbeat/docs/fields.asciidoc | 2536 ++++++++++++++++++ journalbeat/docs/filtering.asciidoc | 32 + journalbeat/docs/general-options.asciidoc | 66 + journalbeat/docs/getting-started.asciidoc | 185 ++ journalbeat/docs/index.asciidoc | 42 + journalbeat/docs/overview.asciidoc | 17 + journalbeat/docs/page_header.html | 4 + journalbeat/docs/running-on-docker.asciidoc | 1 + journalbeat/docs/setting-up-running.asciidoc | 31 + journalbeat/docs/troubleshooting.asciidoc | 31 + 13 files changed, 3147 insertions(+) create mode 100644 journalbeat/docs/config-options.asciidoc create mode 100644 journalbeat/docs/configuring-howto.asciidoc create mode 100644 journalbeat/docs/faq.asciidoc create mode 100644 journalbeat/docs/fields.asciidoc create mode 100644 journalbeat/docs/filtering.asciidoc create mode 100644 journalbeat/docs/general-options.asciidoc create mode 100644 journalbeat/docs/getting-started.asciidoc create mode 100644 journalbeat/docs/index.asciidoc create mode 100644 journalbeat/docs/overview.asciidoc create mode 100644 journalbeat/docs/page_header.html create mode 100644 journalbeat/docs/running-on-docker.asciidoc create mode 100644 journalbeat/docs/setting-up-running.asciidoc create mode 100644 journalbeat/docs/troubleshooting.asciidoc diff --git a/journalbeat/docs/config-options.asciidoc b/journalbeat/docs/config-options.asciidoc new file mode 100644 index 0000000..7ca7a1e --- /dev/null +++ b/journalbeat/docs/config-options.asciidoc @@ -0,0 +1,124 @@ +[id="configuration-{beatname_lc}-options"] +== Configure inputs + +++++ +Configure inputs +++++ + +By default, {beatname_uc} reads log events from the default systemd journals. To +specify other journal files, set the <<{beatname_lc}-paths,`paths`>> option in +the +{beatname_lc}.inputs+ section of the +{beatname_lc}.yml+ file. Each path +can be a directory path (to collect events from all journals in a directory), or +a file path. For example: + +["source","sh",subs="attributes"] +---- +{beatname_lc}.inputs: +- paths: + - "/dev/log" + - "/var/log/messages/my-journal-file.journal" +---- + +Within the configuration file, you can also specify options that control how +{beatname_uc} reads the journal files and which fields are sent to the +configured output. See <<{beatname_lc}-options>> for a list of available +options. + +The following examples show how to configure {beatname_uc} for some common use +cases. + +[[monitor-multiple-journals]] +.Example 1: Monitor multiple journals under the same directory +This example configures {beatname_uc} to read from multiple journals that +are stored under the same directory. {beatname_uc} merges all journals under the +directory into a single journal and reads them. With `seek` set to `cursor`, +{beatname_uc} starts reading at the beginning of the journal, but will continue +reading where it left off after a reload or restart. +["source","sh",subs="attributes"] +---- +{beatname_lc}.inputs: +- paths: ["/path/to/journal/directory"] + seek: cursor +---- + +[[filter-using-field-names]] +.Example 2: Fetch log events for Redis running on Docker (uses field names from systemd) +This example configures {beatname_uc} to fetch log events for Redis running in a +Docker container. The fields are matched using field names from the systemd +journal. +["source","sh",subs="attributes"] +---- +{beatname_lc}.inputs: +- paths: [] + include_matches: + - "CONTAINER_TAG=redis" + - "_COMM=redis" +---- + +[[filter-using-translated-names]] +.Example 3: Fetch log events for Redis running on Docker (uses translated field names) +This example also configures {beatname_uc} to fetch log events for Redis running in a +Docker container. However, in this example the fields are matched using the +translated field names provided by {beatname_uc}. +["source","sh",subs="attributes"] +---- +{beatname_lc}.inputs: +- paths: [] + include_matches: + - "container.image.tag=redis" + - "process.name=redis" +---- + +[id="{beatname_lc}-options"] +[float] +=== Configuration options +You can specify the following options to configure how {beatname_uc} reads the +journal files. + +[float] +[id="{beatname_lc}-paths"] +==== `paths` + +A list of paths that will be crawled and fetched. Each path can be a directory +path (to collect events from all journals in a directory), or a file path. If +you specify a directory, {beatname_uc} merges all journals under the directory +into a single journal and reads them. + +[float] +[id="{beatname_lc}-seek"] +==== `seek` + +The position to start reading the journal from. Valid settings are: + +* `head`: Starts reading at the beginning of the file. +* `tail`: Starts reading at the end of the file. +* `cursor`: On first read, starts reading at the beginning of the file. After a +reload or restart, continues reading where it left off. + +When specified under `paths`, the `seek` setting applies to all journals under +the configured paths. When specified directly under the +{beatname_lc}+ +namespace, the setting applies to all journals read by {beatname_uc}. + +[float] +[id="{beatname_lc}-include-matches"] +==== `include_matches` + +A list of filter expressions used to match fields. The format of the expression +is `field=value`. {beatname_uc} fetches all events that exactly match the +expressions. Pattern matching is not supported. + +To reference fields, use one of the following: + +* The field name used by the systemd journal. For example, +`CONTAINER_TAG=redis` (<>). +* The translated field name used by {beatname_uc}. For example, +`container.image.tag=redis` +(<>). {beatname_uc} +does not translate all fields from the journal. For custom fields, use the name +specified in the systemd journal. + +When specified under `paths`, the `include_matches` filter is applied to all +journals under the configured paths. When specified directly under the ++{beatname_lc}+ namespace, the setting applies to all journals read by +{beatname_uc}. + diff --git a/journalbeat/docs/configuring-howto.asciidoc b/journalbeat/docs/configuring-howto.asciidoc new file mode 100644 index 0000000..c23d40d --- /dev/null +++ b/journalbeat/docs/configuring-howto.asciidoc @@ -0,0 +1,68 @@ +[id="configuring-howto-{beatname_lc}"] += Configuring {beatname_uc} + +[partintro] +-- + +Before modifying configuration settings, make sure you've completed the +<<{beatname_lc}-configuration,configuration steps>> in the Getting Started. +This section describes some common use cases for changing configuration options. + +include::../../libbeat/docs/shared-configuring.asciidoc[] + +The following topics describe how to configure {beatname_uc}: + +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <> +* <<{beatname_lc}-reference-yml>> + +-- + +include::./config-options.asciidoc[] + +include::./general-options.asciidoc[] + +include::../../libbeat/docs/queueconfig.asciidoc[] + +include::../../libbeat/docs/outputconfig.asciidoc[] + +include::../../libbeat/docs/shared-ssl-config.asciidoc[] + +include::./filtering.asciidoc[] + +include::../../libbeat/docs/shared-config-ingest.asciidoc[] + +include::../../libbeat/docs/shared-path-config.asciidoc[] + +include::../../libbeat/docs/shared-kibana-config.asciidoc[] + +include::../../libbeat/docs/setup-config.asciidoc[] + +include::../../libbeat/docs/loggingconfig.asciidoc[] + +:standalone: +include::../../libbeat/docs/shared-env-vars.asciidoc[] +:standalone!: + +:standalone: +include::../../libbeat/docs/yaml.asciidoc[] +:standalone!: + +include::../../libbeat/docs/regexp.asciidoc[] + +include::../../libbeat/docs/http-endpoint.asciidoc[] + +include::../../libbeat/docs/reference-yml.asciidoc[] diff --git a/journalbeat/docs/faq.asciidoc b/journalbeat/docs/faq.asciidoc new file mode 100644 index 0000000..9e0d0a0 --- /dev/null +++ b/journalbeat/docs/faq.asciidoc @@ -0,0 +1,10 @@ +[[faq]] +== Frequently asked questions + +This section contains frequently asked questions about {beatname_uc}. Also check +out the https://discuss.elastic.co/c/beats/{beatname_lc}[{beatname_uc} +discussion forum]. + +include::../../libbeat/docs/faq-limit-bandwidth.asciidoc[] + +include::../../libbeat/docs/shared-faq.asciidoc[] diff --git a/journalbeat/docs/fields.asciidoc b/journalbeat/docs/fields.asciidoc new file mode 100644 index 0000000..e40fa96 --- /dev/null +++ b/journalbeat/docs/fields.asciidoc @@ -0,0 +1,2536 @@ + +//// +This file is generated! See _meta/fields.yml and scripts/generate_field_docs.py +//// + +[[exported-fields]] += Exported fields + +[partintro] + +-- +This document describes the fields that are exported by Journalbeat. They are +grouped in the following categories: + +* <> +* <> +* <> +* <> +* <> +* <> +* <> + +-- +[[exported-fields-beat]] +== Beat fields + +Contains common beat fields available in all event types. + + + +*`beat.name`*:: ++ +-- +The name of the Beat sending the log messages. If the Beat name is set in the configuration file, then that value is used. If it is not set, the hostname is used. To set the Beat name, use the `name` option in the configuration file. + + +-- + +*`beat.hostname`*:: ++ +-- +The hostname as returned by the operating system on which the Beat is running. + + +-- + +*`beat.timezone`*:: ++ +-- +The timezone as returned by the operating system on which the Beat is running. + + +-- + +*`beat.version`*:: ++ +-- +The version of the beat that generated this event. + + +-- + +*`@timestamp`*:: ++ +-- +type: date + +example: August 26th 2016, 12:35:53.332 + +format: date + +required: True + +The timestamp when the event log record was generated. + + +-- + +*`tags`*:: ++ +-- +Arbitrary tags that can be set per Beat and per transaction type. + + +-- + +*`fields`*:: ++ +-- +type: object + +Contains user configurable fields. + + +-- + +[float] +== error fields + +Error fields containing additional info in case of errors. + + + +*`error.type`*:: ++ +-- +type: keyword + +Error type. + + +-- + +[[exported-fields-cloud]] +== Cloud provider metadata fields + +Metadata from cloud providers added by the add_cloud_metadata processor. + + + +*`meta.cloud.provider`*:: ++ +-- +example: ec2 + +Name of the cloud provider. Possible values are ec2, gce, or digitalocean. + + +-- + +*`meta.cloud.instance_id`*:: ++ +-- +Instance ID of the host machine. + + +-- + +*`meta.cloud.instance_name`*:: ++ +-- +Instance name of the host machine. + + +-- + +*`meta.cloud.machine_type`*:: ++ +-- +example: t2.medium + +Machine type of the host machine. + + +-- + +*`meta.cloud.availability_zone`*:: ++ +-- +example: us-east-1c + +Availability zone in which this host is running. + + +-- + +*`meta.cloud.project_id`*:: ++ +-- +example: project-x + +Name of the project in Google Cloud. + + +-- + +*`meta.cloud.region`*:: ++ +-- +Region in which this host is running. + + +-- + +[[exported-fields-common]] +== Common Journalbeat fields + +Contains common fields available in all event types. + + + +*`read_timestamp`*:: ++ +-- +The time when Journalbeat read the journal entry. + + +-- + +[float] +== coredump fields + +Fields used by systemd-coredump kernel helper. + + + +*`coredump.unit`*:: ++ +-- +type: keyword + +Annotations of messages containing coredumps from system units. + + +-- + +*`coredump.user_unit`*:: ++ +-- +type: keyword + +Annotations of messages containing coredumps from user units. + + +-- + +[float] +== journald fields + +Fields to log on behalf of a different program. + + + +[float] +== audit fields + +Audit fields of event. + + + +*`journald.audit.loginuid`*:: ++ +-- +type: long + +example: 1000 + +required: False + +The login UID of the source process. + + +-- + +*`journald.audit.session`*:: ++ +-- +type: long + +example: 3 + +required: False + +The audit session of the source process. + + +-- + +*`journald.cmd`*:: ++ +-- +type: keyword + +example: /lib/systemd/systemd --user + +required: False + +The command line of the process. + + +-- + +*`journald.name`*:: ++ +-- +type: keyword + +example: /lib/systemd/systemd + +required: False + +Name of the executable. + + +-- + +*`journald.executable`*:: ++ +-- +type: keyword + +example: /lib/systemd/systemd + +required: False + +Path to the the executable. + + +-- + +*`journald.pid`*:: ++ +-- +type: long + +example: 1 + +required: False + +The ID of the process which logged the message. + + +-- + +*`journald.gid`*:: ++ +-- +type: long + +example: 1 + +required: False + +The ID of the group which runs the process. + + +-- + +*`journald.uid`*:: ++ +-- +type: long + +example: 1 + +required: False + +The ID of the user which runs the process. + + +-- + +*`journald.capabilites`*:: ++ +-- +required: False + +The effective capabilites of the process. + + +-- + +[float] +== systemd fields + +Fields of systemd. + + + +*`systemd.invocation_id`*:: ++ +-- +type: keyword + +example: 8450f1672de646c88cd133aadd4f2d70 + +required: False + +The invocation ID for the runtime cycle of the unit the message was generated in. + + +-- + +*`systemd.cgroup`*:: ++ +-- +type: keyword + +example: /user.slice/user-1234.slice/session-2.scope + +required: False + +The control group path in the systemd hierarchy. + + +-- + +*`systemd.owner_uid`*:: ++ +-- +type: long + +required: False + +The owner UID of the systemd user unit or systemd session. + + +-- + +*`systemd.session`*:: ++ +-- +type: keyword + +required: False + +The ID of the systemd session. + + +-- + +*`systemd.slice`*:: ++ +-- +type: keyword + +example: user-1234.slice + +required: False + +The systemd slice unit. + + +-- + +*`systemd.user_slice`*:: ++ +-- +type: keyword + +required: False + +The systemd user slice unit. + + +-- + +*`systemd.unit`*:: ++ +-- +type: keyword + +example: nginx.service + +required: False + +The name of the systemd unit. + + +-- + +*`systemd.user_unit`*:: ++ +-- +type: keyword + +example: user-1234.slice + +required: False + +The name of the systemd user unit. + + +-- + +*`systemd.transport`*:: ++ +-- +type: keyword + +example: syslog + +required: True + +How the log message was received by journald. + + +-- + +[float] +== host fields + +Fields of the host. + + + +*`host.boot_id`*:: ++ +-- +type: text + +example: dd8c974asdf01dbe2ef26d7fasdf264c9 + +required: False + +The boot ID for the boot the log was generated in. + + +-- + +[float] +== syslog fields + +Fields of the code generating the event. + + + +*`syslog.priority`*:: ++ +-- +type: long + +example: 1 + +required: False + +The priority of the message. A syslog compatibility field. + + +-- + +*`syslog.facility`*:: ++ +-- +type: long + +example: 1 + +required: False + +The facility of the message. A syslog compatibility field. + + +-- + +*`syslog.identifier`*:: ++ +-- +type: text + +example: su + +required: False + +The identifier of the message. A syslog compatibility field. + + +-- + +*`message`*:: ++ +-- +type: text + +required: True + +The logged message. + + +-- + +*`custom`*:: ++ +-- +type: nested + +required: False + +Arbitrary fields coming from processes. + + +-- + +[[exported-fields-docker-processor]] +== Docker fields + +Docker stats collected from Docker. + + + + +*`docker.container.id`*:: ++ +-- +type: keyword + +Unique container id. + + +-- + +*`docker.container.image`*:: ++ +-- +type: keyword + +Name of the image the container was built on. + + +-- + +*`docker.container.name`*:: ++ +-- +type: keyword + +Container name. + + +-- + +*`docker.container.labels`*:: ++ +-- +type: object + +Image labels. + + +-- + +[[exported-fields-ecs]] +== ECS fields + +ECS fields. + + + +[float] +== agent fields + +The agent fields contain the data about the agent/client/shipper that created the event. + + + +*`agent.version`*:: ++ +-- +type: keyword + +example: 6.0.0-rc2 + +Version of the agent. + + +-- + +*`agent.name`*:: ++ +-- +type: keyword + +example: filebeat + +Name of the agent. + + +-- + +*`agent.id`*:: ++ +-- +type: keyword + +example: 8a4f500d + +Unique identifier of this agent (if one exists). +Example: For Beats this would be beat.id. + + +-- + +*`agent.ephemeral_id`*:: ++ +-- +type: keyword + +example: 8a4f500f + +Ephemeral identifier of this agent (if one exists). +This id normally changes across restarts, but `agent.id` does not. + + +-- + +[float] +== base fields + +The base set contains all fields which are on the top level. These fields are common across all types of events. + + + +*`base.@timestamp`*:: ++ +-- +type: date + +example: 2016-05-23T08:05:34.853Z + +required: True + +Date/time when the event originated. +For log events this is the date/time when the event was generated, and not when it was read. +Required field for all events. + + +-- + +*`base.tags`*:: ++ +-- +type: keyword + +example: ["production", "env2"] + +List of keywords used to tag each event. + + +-- + +*`base.labels`*:: ++ +-- +type: object + +example: {'key2': 'value2', 'key1': 'value1'} + +Key/value pairs. +Can be used to add meta information to events. Should not contain nested objects. All values are stored as keyword. +Example: `docker` and `k8s` labels. + + +-- + +*`base.message`*:: ++ +-- +type: text + +example: Hello World + +For log events the message field contains the log message. +In other use cases the message field can be used to concatenate different values which are then freely searchable. If multiple messages exist, they can be combined into one message. + + +-- + +[float] +== cloud fields + +Fields related to the cloud or infrastructure the events are coming from. + + + +*`cloud.provider`*:: ++ +-- +type: keyword + +example: ec2 + +Name of the cloud provider. Example values are ec2, gce, or digitalocean. + + +-- + +*`cloud.availability_zone`*:: ++ +-- +type: keyword + +example: us-east-1c + +Availability zone in which this host is running. + + +-- + +*`cloud.region`*:: ++ +-- +type: keyword + +example: us-east-1 + +Region in which this host is running. + + +-- + +*`cloud.instance.id`*:: ++ +-- +type: keyword + +example: i-1234567890abcdef0 + +Instance ID of the host machine. + + +-- + +*`cloud.instance.name`*:: ++ +-- +type: keyword + +Instance name of the host machine. + + +-- + +*`cloud.machine.type`*:: ++ +-- +type: keyword + +example: t2.medium + +Machine type of the host machine. + + +-- + +*`cloud.account.id`*:: ++ +-- +type: keyword + +example: 666777888999 + +The cloud account or organization id used to identify different entities in a multi-tenant environment. +Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. + + +-- + +[float] +== container fields + +Container fields are used for meta information about the specific container that is the source of information. These fields help correlate data based containers from any runtime. + + + +*`container.runtime`*:: ++ +-- +type: keyword + +example: docker + +Runtime managing this container. + + +-- + +*`container.id`*:: ++ +-- +type: keyword + +Unique container id. + + +-- + +*`container.image.name`*:: ++ +-- +type: keyword + +Name of the image the container was built on. + + +-- + +*`container.image.tag`*:: ++ +-- +type: keyword + +Container image tag. + + +-- + +*`container.name`*:: ++ +-- +type: keyword + +Container name. + + +-- + +*`container.labels`*:: ++ +-- +type: object + +Image labels. + + +-- + +[float] +== destination fields + +Destination fields describe details about the destination of a packet/event. + + + +*`destination.ip`*:: ++ +-- +type: ip + +IP address of the destination. +Can be one or multiple IPv4 or IPv6 addresses. + + +-- + +*`destination.hostname`*:: ++ +-- +type: keyword + +Hostname of the destination. + + +-- + +*`destination.port`*:: ++ +-- +type: long + +Port of the destination. + + +-- + +*`destination.mac`*:: ++ +-- +type: keyword + +MAC address of the destination. + + +-- + +*`destination.domain`*:: ++ +-- +type: keyword + +Destination domain. + + +-- + +*`destination.subdomain`*:: ++ +-- +type: keyword + +Destination subdomain. + + +-- + +[float] +== device fields + +Device fields are used to provide additional information about the device that is the source of the information. This could be a firewall, network device, etc. + + + +*`device.mac`*:: ++ +-- +type: keyword + +MAC address of the device + + +-- + +*`device.ip`*:: ++ +-- +type: ip + +IP address of the device. + + +-- + +*`device.hostname`*:: ++ +-- +type: keyword + +Hostname of the device. + + +-- + +*`device.vendor`*:: ++ +-- +type: text + +Device vendor information. + + +-- + +*`device.version`*:: ++ +-- +type: keyword + +Device version. + + +-- + +*`device.serial_number`*:: ++ +-- +type: keyword + +Device serial number. + + +-- + +*`device.timezone.offset.sec`*:: ++ +-- +type: long + +example: -5400 + +Timezone offset of the host in seconds. +Number of seconds relative to UTC. If the offset is -01:30 the value will be -5400. + + +-- + +*`device.type`*:: ++ +-- +type: keyword + +example: firewall + +The type of the device the data is coming from. +There is no predefined list of device types. Some examples are `endpoint`, `firewall`, `ids`, `ips`, `proxy`. + + +-- + +[float] +== error fields + +These fields can represent errors of any kind. Use them for errors that happen while fetching events or in cases where the event itself contains an error. + + + +*`error.id`*:: ++ +-- +type: keyword + +Unique identifier for the error. + + +-- + +*`error.message`*:: ++ +-- +type: text + +Error message. + + +-- + +*`error.code`*:: ++ +-- +type: keyword + +Error code describing the error. + + +-- + +[float] +== event fields + +The event fields are used for context information about the data itself. + + + +*`event.id`*:: ++ +-- +type: keyword + +example: 8a4f500d + +Unique ID to describe the event. + + +-- + +*`event.category`*:: ++ +-- +type: keyword + +example: metrics + +Event category. +This can be a user defined category. + + +-- + +*`event.type`*:: ++ +-- +type: keyword + +example: nginx-stats-metrics + +A type given to this kind of event which can be used for grouping. +This is normally defined by the user. + + +-- + +*`event.action`*:: ++ +-- +type: keyword + +example: reject + +The action captured by the event. The type of action will vary from system to system but is likely to include actions by security services, such as blocking or quarantining; as well as more generic actions such as login events, file i/o or proxy forwarding events. +The value is normally defined by the user. + + +-- + +*`event.module`*:: ++ +-- +type: keyword + +example: mysql + +Name of the module this data is coming from. +This information is coming from the modules used in Beats or Logstash. + + +-- + +*`event.dataset`*:: ++ +-- +type: keyword + +example: stats + +Name of the dataset. +The concept of a `dataset` (fileset / metricset) is used in Beats as a subset of modules. It contains the information which is currently stored in metricset.name and metricset.module or fileset.name. + + +-- + +*`event.severity`*:: ++ +-- +type: long + +example: 7 + +Severity describes the severity of the event. What the different severity values mean can very different between use cases. It's up to the implementer to make sure severities are consistent across events. + + +-- + +*`event.original`*:: ++ +-- +type: keyword + +example: Sep 19 08:26:10 host CEF:0|Security| threatmanager|1.0|100| worm successfully stopped|10|src=10.0.0.1 dst=2.1.2.2spt=1232 + +Raw text message of entire event. Used to demonstrate log integrity. +This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. + + +Field is not indexed. + +-- + +*`event.hash`*:: ++ +-- +type: keyword + +example: 123456789012345678901234567890ABCD + +Hash (perhaps logstash fingerprint) of raw field to be able to demonstrate log integrity. + + +-- + +*`event.version`*:: ++ +-- +type: keyword + +example: 0.1.0 + +required: True + +The version field contains the version an event for ECS adheres to. +This field should be provided as part of each event to make it possible to detect to which ECS version an event belongs. +event.version is a required field and must exist in all events. It describes which ECS version the event adheres to. +The current version is 0.1.0. + + +-- + +*`event.duration`*:: ++ +-- +type: long + +Duration of the event in nanoseconds. + + +-- + +*`event.created`*:: ++ +-- +type: date + +event.created contains the date when the event was created. +This timestamp is distinct from @timestamp in that @timestamp contains the processed timestamp. For logs these two timestamps can be different as the timestamp in the log line and when the event is read for example by Filebeat are not identical. `@timestamp` must contain the timestamp extracted from the log line, event.created when the log line is read. The same could apply to package capturing where @timestamp contains the timestamp extracted from the network package and event.created when the event was created. +In case the two timestamps are identical, @timestamp should be used. + + +-- + +*`event.risk_score`*:: ++ +-- +type: float + +Risk score or priority of the event (e.g. security solutions). Use your system's original value here. + + +-- + +*`event.risk_score_norm`*:: ++ +-- +type: float + +Normalized risk score or priority of the event, on a scale of 0 to 100. +This is mainly useful if you use more than one system that assigns risk scores, and you want to see a normalized value across all systems. + + +-- + +[float] +== file fields + +File fields provide details about each file. + + + +*`file.path`*:: ++ +-- +type: text + +Path to the file. + +*`file.path.raw`*:: ++ +-- +type: keyword + +Path to the file. This is a non-analyzed field that is useful for aggregations. + + +-- + +-- + +*`file.target_path`*:: ++ +-- +type: text + +Target path for symlinks. + +*`file.target_path.raw`*:: ++ +-- +type: keyword + +Path to the file. This is a non-analyzed field that is useful for aggregations. + + +-- + +-- + +*`file.extension`*:: ++ +-- +type: keyword + +example: png + +File extension. +This should allow easy filtering by file extensions. + + +-- + +*`file.type`*:: ++ +-- +type: keyword + +File type (file, dir, or symlink). + +-- + +*`file.device`*:: ++ +-- +type: keyword + +Device that is the source of the file. + +-- + +*`file.inode`*:: ++ +-- +type: keyword + +Inode representing the file in the filesystem. + +-- + +*`file.uid`*:: ++ +-- +type: keyword + +The user ID (UID) or security identifier (SID) of the file owner. + + +-- + +*`file.owner`*:: ++ +-- +type: keyword + +File owner's username. + +-- + +*`file.gid`*:: ++ +-- +type: keyword + +Primary group ID (GID) of the file. + +-- + +*`file.group`*:: ++ +-- +type: keyword + +Primary group name of the file. + +-- + +*`file.mode`*:: ++ +-- +type: keyword + +example: 416 + +Mode of the file in octal representation. + +-- + +*`file.size`*:: ++ +-- +type: long + +File size in bytes (field is only added when `type` is `file`). + +-- + +*`file.mtime`*:: ++ +-- +type: date + +Last time file content was modified. + +-- + +*`file.ctime`*:: ++ +-- +type: date + +Last time file metadata changed. + +-- + +[float] +== geo fields + +Geo fields can carry data about a specific location related to an event or geo information for an IP field. + + + +*`geo.continent_name`*:: ++ +-- +type: keyword + +Name of the continent. + + +-- + +*`geo.country_iso_code`*:: ++ +-- +type: keyword + +Country ISO code. + + +-- + +*`geo.location`*:: ++ +-- +type: geo_point + +Longitude and latitude. + + +-- + +*`geo.region_name`*:: ++ +-- +type: keyword + +Region name. + + +-- + +*`geo.city_name`*:: ++ +-- +type: keyword + +City name. + + +-- + +[float] +== host fields + +Host fields provide information related to a host. A host can be a physical machine, a virtual machine, or a Docker container. +Normally the host information is related to the machine on which the event was generated/collected, but they can be used differently if needed. + + + +*`host.timezone.offset.sec`*:: ++ +-- +type: long + +example: -5400 + +Timezone offset of the host in seconds. +Number of seconds relative to UTC. If the offset is -01:30 the value will be -5400. + + +-- + +*`host.name`*:: ++ +-- +type: keyword + +host.name is the hostname of the host. +It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. + + +-- + +*`host.id`*:: ++ +-- +type: keyword + +Unique host id. +As hostname is not always unique, use values that are meaningful in your environment. +Example: The current usage of `beat.name`. + + +-- + +*`host.ip`*:: ++ +-- +type: ip + +Host ip address. + + +-- + +*`host.mac`*:: ++ +-- +type: keyword + +Host mac address. + + +-- + +*`host.type`*:: ++ +-- +type: keyword + +Type of host. +For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. + + +-- + +*`host.os.platform`*:: ++ +-- +type: keyword + +example: darwin + +Operating system platform (centos, ubuntu, windows, etc.) + + +-- + +*`host.os.name`*:: ++ +-- +type: keyword + +example: Mac OS X + +Operating system name. + + +-- + +*`host.os.family`*:: ++ +-- +type: keyword + +example: debian + +OS family (redhat, debian, freebsd, windows, etc.) + + +-- + +*`host.os.version`*:: ++ +-- +type: keyword + +example: 10.12.6 + +Operating system version. + + +-- + +*`host.architecture`*:: ++ +-- +type: keyword + +example: x86_64 + +Operating system architecture. + + +-- + +[float] +== http fields + +Fields related to HTTP requests and responses. + + + +*`http.request.method`*:: ++ +-- +type: keyword + +example: GET, POST, PUT + +Http request method. + + +-- + +*`http.response.status_code`*:: ++ +-- +type: long + +example: 404 + +Http response status code. + + +-- + +*`http.response.body`*:: ++ +-- +type: text + +example: Hello world + +The full http response body. + + +-- + +*`http.version`*:: ++ +-- +type: keyword + +example: 1.1 + +Http version. + + +-- + +[float] +== log fields + +Fields which are specific to log events. + + + +*`log.level`*:: ++ +-- +type: keyword + +example: ERR + +Log level of the log event. +Some examples are `WARN`, `ERR`, `INFO`. + + +-- + +*`log.original`*:: ++ +-- +type: keyword + +example: Sep 19 08:26:10 localhost My log + + +This is the original log message and contains the full log message before splitting it up in multiple parts. +In contrast to the `message` field which can contain an extracted part of the log message, this field contains the original, full log message. It can have already some modifications applied like encoding or new lines removed to clean up the log message. +This field is not indexed and doc_values are disabled so it can't be queried but the value can be retrieved from `_source`. + + +Field is not indexed. + +-- + +[float] +== network fields + +Fields related to network data. + + + +*`network.name`*:: ++ +-- +type: text + +example: Guest Wifi + +Name given by operators to sections of their network. + + +*`network.name.raw`*:: ++ +-- +type: keyword + +Name given by operators to sections of their network. + + +-- + +-- + +*`network.protocol`*:: ++ +-- +type: keyword + +example: http + +Network protocol name. + + +-- + +*`network.direction`*:: ++ +-- +type: keyword + +example: inbound + +Direction of the network traffic. +Recommended values are: + * inbound + * outbound + * unknown + + +-- + +*`network.forwarded_ip`*:: ++ +-- +type: ip + +example: 192.1.1.2 + +Host IP address when the source IP address is the proxy. + + +-- + +*`network.inbound.bytes`*:: ++ +-- +type: long + +example: 184 + +Network inbound bytes. + + +-- + +*`network.inbound.packets`*:: ++ +-- +type: long + +example: 12 + +Network inbound packets. + + +-- + +*`network.outbound.bytes`*:: ++ +-- +type: long + +example: 184 + +Network outbound bytes. + + +-- + +*`network.outbound.packets`*:: ++ +-- +type: long + +example: 12 + +Network outbound packets. + + +-- + +*`network.total.bytes`*:: ++ +-- +type: long + +example: 368 + +Network total bytes. The sum of inbound.bytes + outbound.bytes. + + +-- + +*`network.total.packets`*:: ++ +-- +type: long + +example: 24 + +Network outbound packets. The sum of inbound.packets + outbound.packets + + +-- + +[float] +== organization fields + +The organization fields enrich data with information about the company or entity the data is associated with. These fields help you arrange or filter data stored in an index by one or multiple organizations. + + + +*`organization.name`*:: ++ +-- +type: text + +Organization name. + + +-- + +*`organization.id`*:: ++ +-- +type: keyword + +Unique identifier for the organization. + + +-- + +[float] +== os fields + +The OS fields contain information about the operating system. These fields are often used inside other prefixes, such as `host.os.*` or `user_agent.os.*`. + + + +*`os.platform`*:: ++ +-- +type: keyword + +example: darwin + +Operating system platform (such centos, ubuntu, windows). + + +-- + +*`os.name`*:: ++ +-- +type: keyword + +example: Mac OS X + +Operating system name. + + +-- + +*`os.family`*:: ++ +-- +type: keyword + +example: debian + +OS family (such as redhat, debian, freebsd, windows). + + +-- + +*`os.version`*:: ++ +-- +type: keyword + +example: 10.12.6-rc2 + +Operating system version as a raw string. + + +-- + +*`os.kernel`*:: ++ +-- +type: keyword + +example: 4.4.0-112-generic + +Operating system kernel version as a raw string. + + +-- + +[float] +== process fields + +These fields contain information about a process. These fields can help you correlate metrics information with a process id/name from a log message. The `process.pid` often stays in the metric itself and is copied to the global field for correlation. + + + +*`process.args`*:: ++ +-- +type: keyword + +example: ['-l', 'user', '10.0.0.16'] + +Process arguments. +May be filtered to protect sensitive information. + + +-- + +*`process.name`*:: ++ +-- +type: keyword + +example: ssh + +Process name. +Sometimes called program name or similar. + + +-- + +*`process.pid`*:: ++ +-- +type: long + +Process id. + + +-- + +*`process.ppid`*:: ++ +-- +type: long + +Process parent id. + + +-- + +*`process.title`*:: ++ +-- +type: keyword + +Process title. +The proctitle, often the same as process name. + + +-- + +[float] +== service fields + +The service fields describe the service for or from which the data was collected. These fields help you find and correlate logs for a specific service and version. + + + +*`service.id`*:: ++ +-- +type: keyword + +example: d37e5ebfe0ae6c4972dbe9f0174a1637bb8247f6 + +Unique identifier of the running service. +This id should uniquely identify this service. This makes it possible to correlate logs and metrics for one specific service. +Example: If you are experiencing issues with one redis instance, you can filter on that id to see metrics and logs for that single instance. + + +-- + +*`service.name`*:: ++ +-- +type: keyword + +example: elasticsearch + +Name of the service data is collected from. +The name can be used to group and correlate logs and metrics from one service. +Example: If logs or metrics are collected from Redis, `service.name` would be `redis`. + + +-- + +*`service.type`*:: ++ +-- +type: keyword + +Service type. + + +-- + +*`service.state`*:: ++ +-- +type: keyword + +Current state of the service. + + +-- + +*`service.version`*:: ++ +-- +type: keyword + +example: 3.2.4 + +Version of the service the data was collected from. +This allows to look at a data set only for a specific version of a service. + + +-- + +*`service.ephemeral_id`*:: ++ +-- +type: keyword + +example: 8a4f500f + +Ephemeral identifier of this service (if one exists). +This id normally changes across restarts, but `service.id` does not. + + +-- + +[float] +== url fields + +URL fields provide a complete URL, with scheme, host, and path. The URL object can be reused in other prefixes, such as `host.url.*` for example. Keep the structure consistent whenever you use URL fields. + + + +*`url.href`*:: ++ +-- +type: text + +example: https://elastic.co:443/search?q=elasticsearch#top + +Full url. The field is stored as keyword. +`url.href` is a [multi field](https://www.elastic.co/guide/en/ elasticsearch/reference/6.2/ multi-fields.html#_multi_fields_with_multiple_analyzers). The data is stored as keyword `url.href` and test `url.href.analyzed`. These fields enable you to run a query against part of the url still works splitting up the URL at ingest time. +`href` is an analyzed field so the parsed information can be accessed through `href.analyzed` in queries. + + +*`url.href.raw`*:: ++ +-- +type: keyword + +The full URL. This is a non-analyzed field that is useful for aggregations. + + +-- + +-- + +*`url.scheme`*:: ++ +-- +type: keyword + +example: https + +Scheme of the request, such as "https". +Note: The `:` is not part of the scheme. + + +-- + +*`url.hostname`*:: ++ +-- +type: keyword + +example: elastic.co + +Hostname of the request, such as "elastic.co". +In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `hostname` field. + + +-- + +*`url.port`*:: ++ +-- +type: integer + +example: 443 + +Port of the request, such as 443. + + +-- + +*`url.path`*:: ++ +-- +type: text + +Path of the request, such as "/search". + + +*`url.path.raw`*:: ++ +-- +type: keyword + +URL path. A non-analyzed field that is useful for aggregations. + + +-- + +-- + +*`url.query`*:: ++ +-- +type: text + +The query field describes the query string of the request, such as "q=elasticsearch". +The `?` is excluded from the query string. If a URL contains no `?`, there is no query field. If there is a `?` but no query, the query field exists with an empty string. The `exists` query can be used to differentiate between the two cases. + + +*`url.query.raw`*:: ++ +-- +type: keyword + +URL query part. A non-analyzed field that is useful for aggregations. + + +-- + +-- + +*`url.fragment`*:: ++ +-- +type: keyword + +Portion of the url after the `#`, such as "top". +The `#` is not part of the fragment. + + +-- + +*`url.username`*:: ++ +-- +type: keyword + +Username of the request. + + +-- + +*`url.password`*:: ++ +-- +type: keyword + +Password of the request. + + +-- + +[float] +== user fields + +The user fields describe information about the user that is relevant to the event. Fields can have one entry or multiple entries. If a user has more than one id, provide an array that includes all of them. + + + +*`user.id`*:: ++ +-- +type: keyword + +One or multiple unique identifiers of the user. + + +-- + +*`user.name`*:: ++ +-- +type: keyword + +Name of the user. +The field is a keyword, and will not be tokenized. + + +-- + +*`user.email`*:: ++ +-- +type: keyword + +User email address. + + +-- + +*`user.hash`*:: ++ +-- +type: keyword + +Unique user hash to correlate information for a user in anonymized form. +Useful if `user.id` or `user.name` contain confidential information and cannot be used. + + +-- + +[float] +== user_agent fields + +The user_agent fields normally come from a browser request. They often show up in web service logs coming from the parsed user agent string. + + + +*`user_agent.original`*:: ++ +-- +type: text + +Unparsed version of the user_agent. + + +-- + +*`user_agent.device`*:: ++ +-- +type: keyword + +Name of the physical device. + + +-- + +*`user_agent.version`*:: ++ +-- +type: keyword + +Version of the physical device. + + +-- + +*`user_agent.major`*:: ++ +-- +type: long + +Major version of the user agent. + + +-- + +*`user_agent.minor`*:: ++ +-- +type: long + +Minor version of the user agent. + + +-- + +*`user_agent.patch`*:: ++ +-- +type: keyword + +Patch version of the user agent. + + +-- + +*`user_agent.name`*:: ++ +-- +type: keyword + +example: Chrome + +Name of the user agent. + + +-- + +*`user_agent.os.name`*:: ++ +-- +type: keyword + +Name of the operating system. + + +-- + +*`user_agent.os.version`*:: ++ +-- +type: keyword + +Version of the operating system. + + +-- + +*`user_agent.os.major`*:: ++ +-- +type: long + +Major version of the operating system. + + +-- + +*`user_agent.os.minor`*:: ++ +-- +type: long + +Minor version of the operating system. + + +-- + +[[exported-fields-host-processor]] +== Host fields + +Info collected for the host machine. + + + + +*`host.os.kernel`*:: ++ +-- +type: keyword + +The operating system's kernel version. + + +-- + +[[exported-fields-kubernetes-processor]] +== Kubernetes fields + +Kubernetes metadata added by the kubernetes processor + + + + +*`kubernetes.pod.name`*:: ++ +-- +type: keyword + +Kubernetes pod name + + +-- + +*`kubernetes.pod.uid`*:: ++ +-- +type: keyword + +Kubernetes Pod UID + + +-- + +*`kubernetes.namespace`*:: ++ +-- +type: keyword + +Kubernetes namespace + + +-- + +*`kubernetes.node.name`*:: ++ +-- +type: keyword + +Kubernetes node name + + +-- + +*`kubernetes.labels`*:: ++ +-- +type: object + +Kubernetes labels map + + +-- + +*`kubernetes.annotations`*:: ++ +-- +type: object + +Kubernetes annotations map + + +-- + +*`kubernetes.container.name`*:: ++ +-- +type: keyword + +Kubernetes container name + + +-- + +*`kubernetes.container.image`*:: ++ +-- +type: keyword + +Kubernetes container image + + +-- + diff --git a/journalbeat/docs/filtering.asciidoc b/journalbeat/docs/filtering.asciidoc new file mode 100644 index 0000000..c9182eb --- /dev/null +++ b/journalbeat/docs/filtering.asciidoc @@ -0,0 +1,32 @@ +[[filtering-and-enhancing-data]] +== Filter and enhance the exported data + +Your use case might require only a subset of the data exported by {beatname_uc}, +or you might need to enhance the exported data (for example, by adding +metadata). {beatname_uc} provides a couple of options for filtering and +enhancing exported data. + +You can configure {beatname_uc} to include events that match specific filtering +criteria. To do this, use the <<{beatname_lc}-include-matches,`include_matches`>> +option. The advantage of this approach is that you can reduce the number of +fields that {beatname_uc} needs to process. + +Another approach (the one described here) is to define processors to configure +global processing across all data exported by {beatname_uc}. + + +[float] +[[using-processors]] +=== Processors + +include::../../libbeat/docs/processors.asciidoc[] + +// You must set the processor-scope attribute to resolve the attribute reference +// defined in processors-using.asciidoc. The attribute is used to indicate where +// processors are valid. If processors are valid in more than two locations +// (root and :processor-scope:), you need to add a conditionally coded section +// to processors-using.asciidoc. + +:processor-scope: input +include::../../libbeat/docs/processors-using.asciidoc[] +:processor-scope!: diff --git a/journalbeat/docs/general-options.asciidoc b/journalbeat/docs/general-options.asciidoc new file mode 100644 index 0000000..d5f1c21 --- /dev/null +++ b/journalbeat/docs/general-options.asciidoc @@ -0,0 +1,66 @@ +[[configuration-general-options]] +== Specify general settings + +You can specify settings in the +{beatname_lc}.yml+ config file to control the +general behavior of {beatname_uc}. This includes: + +* <> that control things like +publisher behavior and the location of some files. + +* <> that are supported by all Elastic +Beats. + +[float] +[[configuration-global-options]] +=== Global {beatname_uc} configuration options + +These options are in the +{beatname_lc}+ namespace. + +[float] +[id="{beatname_lc}-registry-file"] +==== `registry_file` + +The name of the registry file. If a relative path is used, it is considered relative to the +data path. See the <> section for details. The default is `${path.data}/registry`. + +["source","sh",subs="attributes"] +---- +{beatname_lc}.registry_file: registry +---- + +[float] +[id="{beatname_lc}-backoff"] +==== `backoff` + +The number of seconds to wait before trying to read again from journals. The +default is 1s. + +[float] +[id="{beatname_lc}-backoff-factor"] +==== `backoff_factor` + +Multiplier of the backoff value. The default is 1s. + +[float] +[id="{beatname_lc}-max-backoff"] +==== `max_backoff` + +The maximum number of seconds to wait before attempting to read again from +journals. The default is 60s. + +[float] +==== `seek` + +This option is valid as a global setting under the +{beatname_lc}+ namespace +or under `paths`. For a description of this option, see +<<{beatname_lc}-seek,`seek`>>. + +[float] +==== `include_matches` + +This option is valid as a global setting under the +{beatname_lc}+ namespace +or under `paths`. For a description of this option, see +<<{beatname_lc}-include-matches,`include_matches`>>. + +include::../../libbeat/docs/generalconfig.asciidoc[] + diff --git a/journalbeat/docs/getting-started.asciidoc b/journalbeat/docs/getting-started.asciidoc new file mode 100644 index 0000000..5290913 --- /dev/null +++ b/journalbeat/docs/getting-started.asciidoc @@ -0,0 +1,185 @@ +[id="{beatname_lc}-getting-started"] +== Getting started with {beatname_uc} + +include::../../libbeat/docs/shared-getting-started-intro.asciidoc[] + +* <<{beatname_lc}-installation>> +* <<{beatname_lc}-configuration>> +* <<{beatname_lc}-template>> +* <<{beatname_lc}-starting>> +* <> + +[id="{beatname_lc}-installation"] +=== Step 1: Install {beatname_uc} + +include::../../libbeat/docs/shared-download-and-install.asciidoc[] + +[[deb]] +*deb:* + +ifeval::["{release-state}"=="unreleased"] + +Version {version} of {beatname_uc} has not yet been released. + +endif::[] + +ifeval::["{release-state}"!="unreleased"] + +["source","sh",subs="attributes"] +------------------------------------------------ +curl -L -O https://artifacts.elastic.co/downloads/beats/{beatname_lc}/{beatname_lc}-{version}-amd64.deb +sudo dpkg -i {beatname_lc}-{version}-amd64.deb +------------------------------------------------ + +endif::[] + +[[rpm]] +*rpm:* + +ifeval::["{release-state}"=="unreleased"] + +Version {version} of {beatname_uc} has not yet been released. + +endif::[] + +ifeval::["{release-state}"!="unreleased"] + +["source","sh",subs="attributes"] +------------------------------------------------ +curl -L -O https://artifacts.elastic.co/downloads/beats/{beatname_lc}/{beatname_lc}-{version}-x86_64.rpm +sudo rpm -vi {beatname_lc}-{version}-x86_64.rpm +------------------------------------------------ + +endif::[] + +[[linux]] +*linux:* + +ifeval::["{release-state}"=="unreleased"] + +Version {version} of {beatname_uc} has not yet been released. + +endif::[] + +ifeval::["{release-state}"!="unreleased"] + +["source","sh",subs="attributes"] +------------------------------------------------ +curl -L -O https://artifacts.elastic.co/downloads/beats/{beatname_lc}/{beatname_lc}-{version}-linux-x86_64.tar.gz +tar xzvf {beatname_lc}-{version}-linux-x86_64.tar.gz +------------------------------------------------ + +endif::[] + +[id="{beatname_lc}-configuration"] +=== Step 2: Configure {beatname_uc} + +Before running {beatname_uc}, you need to specify the location of the systemd +journal files and configure how you want the files to be read. + +include::../../libbeat/docs/shared-configuring.asciidoc[] + +Here is a sample of the +{beatname_lc}+ section of the +{beatname_lc}.yml+ file. +{beatname_uc} uses predefined default values for most configuration options. + +["source","sh",subs="attributes"] +---------------------------------------------------------------------- +journalbeat.inputs: +- paths: ["/path/to/journal/directory"] + seek: head +---------------------------------------------------------------------- + +To configure {beatname_uc}: + +. Specify a list of paths to your systemd journal files. Each path can be a +directory path (to collect events from all journals in a directory), or a file +path. For example: ++ +["source","sh",subs="attributes"] +---- +{beatname_lc}.inputs: +- paths: + - "/dev/log" + - "/var/log/messages/my-journal-file.journal" +---- ++ +If no paths are specified, {beatname_uc} reads from the default journal. + +. Set the <<{beatname_lc}-seek,`seek`>> option to control the position where +{beatname_uc} starts reading the journal. The available options are `head`, +`tail`, and `cursor`. The default is `cursor`, which means that +{beatname_uc} will continue reading where it left off after a reload or restart. + +. (Optional) Set the <<{beatname_lc}-include-matches,`include_matches`>> option +to filter entries in journald before collecting any log events. This reduces the +number of fields that {beatname_uc} needs to process. For example, to fetch only +Redis events from a Docker container tagged as `redis`, use: ++ +["source","sh",subs="attributes"] +---- +{beatname_lc}.inputs: +- paths: [] + include_matches: + - "CONTAINER_TAG=redis" + - "_COMM=redis" +---- + +include::../../libbeat/docs/step-configure-output.asciidoc[] + +include::../../libbeat/docs/step-configure-kibana-endpoint.asciidoc[] + +include::../../libbeat/docs/step-configure-credentials.asciidoc[] + +include::../../libbeat/docs/step-test-config.asciidoc[] + +include::../../libbeat/docs/step-look-at-config.asciidoc[] + + +[id="{beatname_lc}-template"] +=== Step 3: Load the index template in Elasticsearch + +include::../../libbeat/docs/shared-template-load.asciidoc[] + +[id="{beatname_lc}-starting"] +=== Step 4: Start {beatname_uc} + +Start {beatname_uc} by issuing the appropriate command for your platform. If you +are accessing a secured Elasticsearch cluster, make sure you've configured +credentials as described in <<{beatname_lc}-configuration>>. + +NOTE: If you use an init.d script to start {beatname_uc} on deb or rpm, you can't +specify command line flags (see <>). To specify flags, +start {beatname_uc} in the foreground. + +*deb and rpm:* + +["source","sh",subs="attributes"] +---------------------------------------------------------------------- +sudo service {beatname_lc} start +---------------------------------------------------------------------- + +*linux:* + +["source","sh",subs="attributes"] +---------------------------------------------------------------------- +sudo chown root {beatname_lc}.yml <1> +sudo ./{beatname_lc} -e -d "publish" +---------------------------------------------------------------------- +<1> You'll be running {beatname_uc} as root, so you need to change ownership +of the configuration file, or run {beatname_uc} with `--strict.perms=false` +specified. See +{libbeat}/config-file-permissions.html[Config File Ownership and Permissions] +in the _Beats Platform Reference_. + +// REVIEWERS: Do you think it's better to show the run command with or without +// -c journalbeat.yml included? I'm inclined to show it without. WDYT? + +{beatname_uc} is now ready to send journal events to the defined output. + +[[view-kibana-dashboards]] +=== Step 5: View your data in Kibana + +There are currently no example dashboards available for {beatname_uc}. + +To learn how to view and explore your data, see the +_{kibana-ref}/index.html[{kib} User Guide]_. diff --git a/journalbeat/docs/index.asciidoc b/journalbeat/docs/index.asciidoc new file mode 100644 index 0000000..2c6febb --- /dev/null +++ b/journalbeat/docs/index.asciidoc @@ -0,0 +1,42 @@ += Journalbeat Reference + +include::../../libbeat/docs/version.asciidoc[] + +include::{asciidoc-dir}/../../shared/attributes.asciidoc[] + +:version: {stack-version} +:beatname_lc: journalbeat +:beatname_uc: Journalbeat +:beatname_pkg: {beatname_lc} +:github_repo_name: beats +:discuss_forum: beats/{beatname_lc} +:beat_default_index_prefix: {beatname_lc} +:libbeat-docs: Beats Platform Reference +:deb_os: +:rpm_os: +:no_dashboards: + +include::../../libbeat/docs/shared-beats-attributes.asciidoc[] + +:release-state: released + +include::./overview.asciidoc[] + +include::./getting-started.asciidoc[] + +include::../../libbeat/docs/repositories.asciidoc[] + +include::./setting-up-running.asciidoc[] + +include::./configuring-howto.asciidoc[] + +include::./fields.asciidoc[] + +include::../../libbeat/docs/monitoring/monitoring-beats.asciidoc[] + +include::../../libbeat/docs/shared-securing-beat.asciidoc[] + +include::./troubleshooting.asciidoc[] + +include::./faq.asciidoc[] + diff --git a/journalbeat/docs/overview.asciidoc b/journalbeat/docs/overview.asciidoc new file mode 100644 index 0000000..d8c018c --- /dev/null +++ b/journalbeat/docs/overview.asciidoc @@ -0,0 +1,17 @@ +[id="{beatname_lc}-overview"] +== {beatname_uc} overview + +++++ +Overview +++++ + +{beatname_uc} is a lightweight shipper for forwarding and centralizing log data +from https://www.freedesktop.org/software/systemd/man/systemd-journald.service.html[systemd journals]. +Installed as an agent on your servers, {beatname_uc} monitors the journal +locations that you specify, collects log events, and forwards them to either to +https://www.elastic.co/products/elasticsearch[Elasticsearch] or +https://www.elastic.co/products/logstash[Logstash]. + +{beatname_uc} is an Elastic https://www.elastic.co/products/beats[Beat]. It's +based on the `libbeat` framework. For more information, see the +{libbeat}/index.html[{libbeat-docs}]. diff --git a/journalbeat/docs/page_header.html b/journalbeat/docs/page_header.html new file mode 100644 index 0000000..0f01f3b --- /dev/null +++ b/journalbeat/docs/page_header.html @@ -0,0 +1,4 @@ +This functionality is experimental and may be changed or removed completely in a +future release. Elastic will take a best effort approach to fix any issues, but +experimental features are not subject to the support SLA of official GA +features. \ No newline at end of file diff --git a/journalbeat/docs/running-on-docker.asciidoc b/journalbeat/docs/running-on-docker.asciidoc new file mode 100644 index 0000000..6bbc976 --- /dev/null +++ b/journalbeat/docs/running-on-docker.asciidoc @@ -0,0 +1 @@ +include::../../libbeat/docs/shared-docker.asciidoc[] diff --git a/journalbeat/docs/setting-up-running.asciidoc b/journalbeat/docs/setting-up-running.asciidoc new file mode 100644 index 0000000..aeed49f --- /dev/null +++ b/journalbeat/docs/setting-up-running.asciidoc @@ -0,0 +1,31 @@ +///// +// NOTE: +// Each beat has its own setup overview to allow for the addition of content +// that is unique to each beat. +///// + +[[setting-up-and-running]] +== Setting up and running {beatname_uc} + +Before reading this section, see the +<<{beatname_lc}-getting-started,getting started documentation>> for basic +installation instructions to get you started. + +This section includes additional information on how to set up and run +{beatname_uc}, including: + +* <> +* <> +* <> +* <> + + +//MAINTAINERS: If you add a new file to this section, make sure you update the bulleted list ^^ too. + +include::../../libbeat/docs/shared-directory-layout.asciidoc[] + +include::../../libbeat/docs/keystore.asciidoc[] + +include::../../libbeat/docs/command-reference.asciidoc[] + +include::../../libbeat/docs/shared-shutdown.asciidoc[] diff --git a/journalbeat/docs/troubleshooting.asciidoc b/journalbeat/docs/troubleshooting.asciidoc new file mode 100644 index 0000000..3c14416 --- /dev/null +++ b/journalbeat/docs/troubleshooting.asciidoc @@ -0,0 +1,31 @@ +[[troubleshooting]] += Troubleshooting + +[partintro] +-- + +If you have issues installing or running {beatname_uc}, read the +following tips: + +* <> +* <> +* <> + +//sets block macro for getting-help.asciidoc included in next section + +-- + +[[getting-help]] +== Get help + +include::../../libbeat/docs/getting-help.asciidoc[] + +//sets block macro for debugging.asciidoc included in next section + +[id="enable-{beatname_lc}-debugging"] +== Debug + +include::../../libbeat/docs/debugging.asciidoc[] + + + From e742a4495a5c448449337998640908f8bb6504ac Mon Sep 17 00:00:00 2001 From: dedemorton Date: Tue, 6 Nov 2018 17:42:38 -0800 Subject: [PATCH 2/4] Add changes from review --- journalbeat/docs/config-options.asciidoc | 25 +++++++++++++++-------- journalbeat/docs/getting-started.asciidoc | 19 +++++++++-------- 2 files changed, 26 insertions(+), 18 deletions(-) diff --git a/journalbeat/docs/config-options.asciidoc b/journalbeat/docs/config-options.asciidoc index 7ca7a1e..53785c2 100644 --- a/journalbeat/docs/config-options.asciidoc +++ b/journalbeat/docs/config-options.asciidoc @@ -29,11 +29,11 @@ cases. [[monitor-multiple-journals]] .Example 1: Monitor multiple journals under the same directory -This example configures {beatname_uc} to read from multiple journals that -are stored under the same directory. {beatname_uc} merges all journals under the -directory into a single journal and reads them. With `seek` set to `cursor`, -{beatname_uc} starts reading at the beginning of the journal, but will continue -reading where it left off after a reload or restart. +This example configures {beatname_uc} to read from multiple journals that are +stored under the same directory. {beatname_uc} merges all journals under the +directory into a single event stream and reads the events. With `seek` set to +`cursor`, {beatname_uc} starts reading at the beginning of the journal, but will +continue reading at the last known position after a reload or restart. ["source","sh",subs="attributes"] ---- {beatname_lc}.inputs: @@ -82,7 +82,9 @@ journal files. A list of paths that will be crawled and fetched. Each path can be a directory path (to collect events from all journals in a directory), or a file path. If you specify a directory, {beatname_uc} merges all journals under the directory -into a single journal and reads them. +into a single journal and reads them. + +If no paths are specified, {beatname_uc} reads from the default journal. [float] [id="{beatname_lc}-seek"] @@ -90,15 +92,20 @@ into a single journal and reads them. The position to start reading the journal from. Valid settings are: -* `head`: Starts reading at the beginning of the file. -* `tail`: Starts reading at the end of the file. +* `head`: Starts reading at the beginning of the file, even after a reload or +restart. +* `tail`: Starts reading at the end of the file, even after a reload or restart. * `cursor`: On first read, starts reading at the beginning of the file. After a -reload or restart, continues reading where it left off. +reload or restart, continues reading at the last known position. When specified under `paths`, the `seek` setting applies to all journals under the configured paths. When specified directly under the +{beatname_lc}+ namespace, the setting applies to all journals read by {beatname_uc}. +If you have old log files and want to skip lines, start {beatname_uc} with +`seek: tail` specified. Then stop {beatname_uc}, set `seek: cursor`, and restart +{beatname_uc}. + [float] [id="{beatname_lc}-include-matches"] ==== `include_matches` diff --git a/journalbeat/docs/getting-started.asciidoc b/journalbeat/docs/getting-started.asciidoc index 5290913..4a72f3d 100644 --- a/journalbeat/docs/getting-started.asciidoc +++ b/journalbeat/docs/getting-started.asciidoc @@ -74,8 +74,9 @@ endif::[] [id="{beatname_lc}-configuration"] === Step 2: Configure {beatname_uc} -Before running {beatname_uc}, you need to specify the location of the systemd -journal files and configure how you want the files to be read. +Before running {beatname_uc}, you can specify the location of the systemd +journal files and configure how you want the files to be read. If you accept the +default configuration, {beatname_uc} reads from the local journal. include::../../libbeat/docs/shared-configuring.asciidoc[] @@ -107,12 +108,15 @@ If no paths are specified, {beatname_uc} reads from the default journal. . Set the <<{beatname_lc}-seek,`seek`>> option to control the position where {beatname_uc} starts reading the journal. The available options are `head`, -`tail`, and `cursor`. The default is `cursor`, which means that -{beatname_uc} will continue reading where it left off after a reload or restart. +`tail`, and `cursor`. The default is `cursor`, which means that on first read, +{beatname_uc} starts reading at the beginning of the file, but continues reading +at the last known position after a reload or restart. For more detail about +the settings, see the reference docs for the +<<{beatname_lc}-seek,`seek` option>>. . (Optional) Set the <<{beatname_lc}-include-matches,`include_matches`>> option to filter entries in journald before collecting any log events. This reduces the -number of fields that {beatname_uc} needs to process. For example, to fetch only +number of events that {beatname_uc} needs to process. For example, to fetch only Redis events from a Docker container tagged as `redis`, use: + ["source","sh",subs="attributes"] @@ -163,7 +167,7 @@ sudo service {beatname_lc} start ["source","sh",subs="attributes"] ---------------------------------------------------------------------- sudo chown root {beatname_lc}.yml <1> -sudo ./{beatname_lc} -e -d "publish" +sudo ./{beatname_lc} -e ---------------------------------------------------------------------- <1> You'll be running {beatname_uc} as root, so you need to change ownership of the configuration file, or run {beatname_uc} with `--strict.perms=false` @@ -171,9 +175,6 @@ specified. See {libbeat}/config-file-permissions.html[Config File Ownership and Permissions] in the _Beats Platform Reference_. -// REVIEWERS: Do you think it's better to show the run command with or without -// -c journalbeat.yml included? I'm inclined to show it without. WDYT? - {beatname_uc} is now ready to send journal events to the defined output. [[view-kibana-dashboards]] From edfee802a5fb97b0024ce76e609793069dc645a1 Mon Sep 17 00:00:00 2001 From: dedemorton Date: Wed, 7 Nov 2018 22:13:11 -0800 Subject: [PATCH 3/4] Changes from second round of reviews --- journalbeat/docs/config-options.asciidoc | 104 ++++++++++++++++++++-- journalbeat/docs/general-options.asciidoc | 20 ----- 2 files changed, 96 insertions(+), 28 deletions(-) diff --git a/journalbeat/docs/config-options.asciidoc b/journalbeat/docs/config-options.asciidoc index 53785c2..43a78bb 100644 --- a/journalbeat/docs/config-options.asciidoc +++ b/journalbeat/docs/config-options.asciidoc @@ -57,9 +57,9 @@ journal. [[filter-using-translated-names]] .Example 3: Fetch log events for Redis running on Docker (uses translated field names) -This example also configures {beatname_uc} to fetch log events for Redis running in a -Docker container. However, in this example the fields are matched using the -translated field names provided by {beatname_uc}. +This example also configures {beatname_uc} to fetch log events for Redis running +in a Docker container. However, in this example the fields are matched using the +<> provided by {beatname_uc}. ["source","sh",subs="attributes"] ---- {beatname_lc}.inputs: @@ -86,15 +86,34 @@ into a single journal and reads them. If no paths are specified, {beatname_uc} reads from the default journal. +[float] +[id="{beatname_lc}-backoff"] +==== `backoff` + +The number of seconds to wait before trying to read again from journals. The +default is 1s. + +[float] +[id="{beatname_lc}-max-backoff"] +==== `max_backoff` + +The maximum number of seconds to wait before attempting to read again from +journals. The default is 60s. + [float] [id="{beatname_lc}-seek"] ==== `seek` The position to start reading the journal from. Valid settings are: -* `head`: Starts reading at the beginning of the file, even after a reload or -restart. -* `tail`: Starts reading at the end of the file, even after a reload or restart. +// REVIEWERS: Not sure if I've gotten this quite right. + +* `head`: Starts reading at the beginning of the file. After a restart, +{beatname_uc} resends all log messages in the journal. +* `tail`: Starts reading at the end of the file. After a restart, +{beatname_uc} resends the last message, which might result in duplicates. If +multiple log messages are written to a journal while {beatname_uc} is down, +only the last log message is sent on restart. * `cursor`: On first read, starts reading at the beginning of the file. After a reload or restart, continues reading at the last known position. @@ -118,8 +137,8 @@ To reference fields, use one of the following: * The field name used by the systemd journal. For example, `CONTAINER_TAG=redis` (<>). -* The translated field name used by {beatname_uc}. For example, -`container.image.tag=redis` +* The <> used by +{beatname_uc}. For example, `container.image.tag=redis` (<>). {beatname_uc} does not translate all fields from the journal. For custom fields, use the name specified in the systemd journal. @@ -129,3 +148,72 @@ journals under the configured paths. When specified directly under the +{beatname_lc}+ namespace, the setting applies to all journals read by {beatname_uc}. +[float] +[[translated-fields]] +=== Translated field names + +You can use the following translated names in filter expressions to reference +journald fields: + +[horizontal] +*Journald field name*:: *Translated name* +`COREDUMP_UNIT`:: `journald.coredump.unit` +`COREDUMP_USER_UNIT`:: `journald.coredump.user_unit` +`OBJECT_AUDIT_LOGINUID`:: `journald.object.audit.login_uid` +`OBJECT_AUDIT_SESSION`:: `journald.object.audit.session` +`OBJECT_CMDLINE`:: `journald.object.cmd` +`OBJECT_COMM`:: `journald.object.name` +`OBJECT_EXE`:: `journald.object.executable` +`OBJECT_GID`:: `journald.object.gid` +`OBJECT_PID`:: `journald.object.pid` +`OBJECT_SYSTEMD_OWNER_UID`:: `journald.object.systemd.owner_uid` +`OBJECT_SYSTEMD_SESSION`:: `journald.object.systemd.session` +`OBJECT_SYSTEMD_UNIT`:: `journald.object.systemd.unit` +`OBJECT_SYSTEMD_USER_UNIT`:: `journald.object.systemd.user_unit` +`OBJECT_UID`:: `journald.object.uid` +`_AUDIT_LOGINUID`:: `process.audit.login_uid` +`_AUDIT_SESSION`:: `process.audit.session` +`_BOOT_ID`:: `host.boot_id` +`_CAP_EFFECTIVE`:: `process.capabilites` +`_CMDLINE`:: `process.cmd` +`_CODE_FILE`:: `journald.code.file` +`_CODE_FUNC`:: `journald.code.func` +`_CODE_LINE`:: `journald.code.line` +`_COMM`:: `process.name` +`_EXE`:: `process.executable` +`_GID`:: `process.uid` +`_HOSTNAME`:: `host.name` +`_KERNEL_DEVICE`:: `journald.kernel.device` +`_KERNEL_SUBSYSTEM`:: `journald.kernel.subsystem` +`_MACHINE_ID`:: `host.id` +`_MESSAGE`:: `message` +`_PID`:: `process.pid` +`_PRIORITY`:: `syslog.priority` +`_SYSLOG_FACILITY`:: `syslog.facility` +`_SYSLOG_IDENTIFIER`:: `syslog.identifier` +`_SYSLOG_PID`:: `syslog.pid` +`_SYSTEMD_CGROUP`:: `systemd.cgroup` +`_SYSTEMD_INVOCATION_ID`:: `systemd.invocation_id` +`_SYSTEMD_OWNER_UID`:: `systemd.owner_uid` +`_SYSTEMD_SESSION`:: `systemd.session` +`_SYSTEMD_SLICE`:: `systemd.slice` +`_SYSTEMD_UNIT`:: `systemd.unit` +`_SYSTEMD_USER_SLICE`:: `systemd.user_slice` +`_SYSTEMD_USER_UNIT`:: `systemd.user_unit` +`_TRANSPORT`:: `systemd.transport` +`_UDEV_DEVLINK`:: `journald.kernel.device_symlinks` +`_UDEV_DEVNODE`:: `journald.kernel.device_node_path` +`_UDEV_SYSNAME`:: `journald.kernel.device_name` +`_UID`:: `process.uid` + + +The following translated fields for +https://docs.docker.com/config/containers/logging/journald/[Docker] are also +available: + +[horizontal] +`CONTAINER_ID`:: `conatiner.id_truncated` +`CONTAINER_ID_FULL`:: `container.id` +`CONTAINER_NAME`:: `container.name` +`CONTAINER_PARTIAL_MESSAGE`:: `container.partial` +`CONTAINER_TAG`:: `container.image.tag` diff --git a/journalbeat/docs/general-options.asciidoc b/journalbeat/docs/general-options.asciidoc index d5f1c21..aa8c495 100644 --- a/journalbeat/docs/general-options.asciidoc +++ b/journalbeat/docs/general-options.asciidoc @@ -28,26 +28,6 @@ data path. See the <> section for details. The default is `${p {beatname_lc}.registry_file: registry ---- -[float] -[id="{beatname_lc}-backoff"] -==== `backoff` - -The number of seconds to wait before trying to read again from journals. The -default is 1s. - -[float] -[id="{beatname_lc}-backoff-factor"] -==== `backoff_factor` - -Multiplier of the backoff value. The default is 1s. - -[float] -[id="{beatname_lc}-max-backoff"] -==== `max_backoff` - -The maximum number of seconds to wait before attempting to read again from -journals. The default is 60s. - [float] ==== `seek` From 2d4360255787543a9f713b787bf06285d771e7b5 Mon Sep 17 00:00:00 2001 From: dedemorton Date: Thu, 8 Nov 2018 10:58:09 -0800 Subject: [PATCH 4/4] Remove backoff options --- journalbeat/docs/config-options.asciidoc | 9 --------- journalbeat/docs/general-options.asciidoc | 14 -------------- 2 files changed, 23 deletions(-) diff --git a/journalbeat/docs/config-options.asciidoc b/journalbeat/docs/config-options.asciidoc index 43a78bb..0ec0f10 100644 --- a/journalbeat/docs/config-options.asciidoc +++ b/journalbeat/docs/config-options.asciidoc @@ -117,10 +117,6 @@ only the last log message is sent on restart. * `cursor`: On first read, starts reading at the beginning of the file. After a reload or restart, continues reading at the last known position. -When specified under `paths`, the `seek` setting applies to all journals under -the configured paths. When specified directly under the +{beatname_lc}+ -namespace, the setting applies to all journals read by {beatname_uc}. - If you have old log files and want to skip lines, start {beatname_uc} with `seek: tail` specified. Then stop {beatname_uc}, set `seek: cursor`, and restart {beatname_uc}. @@ -143,11 +139,6 @@ To reference fields, use one of the following: does not translate all fields from the journal. For custom fields, use the name specified in the systemd journal. -When specified under `paths`, the `include_matches` filter is applied to all -journals under the configured paths. When specified directly under the -+{beatname_lc}+ namespace, the setting applies to all journals read by -{beatname_uc}. - [float] [[translated-fields]] === Translated field names diff --git a/journalbeat/docs/general-options.asciidoc b/journalbeat/docs/general-options.asciidoc index aa8c495..adfd452 100644 --- a/journalbeat/docs/general-options.asciidoc +++ b/journalbeat/docs/general-options.asciidoc @@ -28,19 +28,5 @@ data path. See the <> section for details. The default is `${p {beatname_lc}.registry_file: registry ---- -[float] -==== `seek` - -This option is valid as a global setting under the +{beatname_lc}+ namespace -or under `paths`. For a description of this option, see -<<{beatname_lc}-seek,`seek`>>. - -[float] -==== `include_matches` - -This option is valid as a global setting under the +{beatname_lc}+ namespace -or under `paths`. For a description of this option, see -<<{beatname_lc}-include-matches,`include_matches`>>. - include::../../libbeat/docs/generalconfig.asciidoc[]