diff --git a/NOTICE.txt b/NOTICE.txt index 7ccbb30e3df..5de53b53cda 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -173,7 +173,8 @@ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------- Dependency: github.com/coreos/go-systemd -Revision: eee3db372b31153ca0b90702e165948699803fd0 +Version: v17 +Revision: 39ca1b05acc7ad1220e09f133283b8859a8b71ab License type (autodetected): Apache-2.0 ./vendor/github.com/coreos/go-systemd/LICENSE: -------------------------------------------------------------------- diff --git a/journalbeat/_meta/beat.yml b/journalbeat/_meta/beat.yml index d0a9f974ee3..597f1ada844 100644 --- a/journalbeat/_meta/beat.yml +++ b/journalbeat/_meta/beat.yml @@ -20,8 +20,6 @@ journalbeat.inputs: # The number of seconds to wait before trying to read again from journals. #backoff: 1s - # Multiplier of backoff value. - #backoff_factor: 2 # The maximum number of seconds to wait before attempting to read again from journals. #max_backoff: 60s @@ -47,8 +45,6 @@ journalbeat.inputs: # The number of seconds to wait before trying to read again from journals. #backoff: 1s - # Multiplier of backoff value. - #backoff_factor: 2 # The maximum number of seconds to wait before attempting to read again from journals. #max_backoff: 60s diff --git a/journalbeat/_meta/fields.common.yml b/journalbeat/_meta/fields.common.yml index 8faa06cc623..7f0cedd4875 100644 --- a/journalbeat/_meta/fields.common.yml +++ b/journalbeat/_meta/fields.common.yml @@ -34,7 +34,7 @@ description: > Audit fields of event. fields: - - name: loginuid + - name: login_uid type: long required: false example: 1000 @@ -314,3 +314,8 @@ required: true description: > The logged message. + - name: custom + type: nested + required: false + description: > + Arbitrary fields coming from processes. diff --git a/journalbeat/config/config.go b/journalbeat/config/config.go index 85ba8ef1ab8..bcbed93df37 100644 --- a/journalbeat/config/config.go +++ b/journalbeat/config/config.go @@ -28,20 +28,18 @@ import ( // Config stores the configuration of Journalbeat type Config struct { - Inputs []*common.Config `config:"inputs"` - RegistryFile string `config:"registry"` - Backoff time.Duration `config:"backoff" validate:"min=0,nonzero"` - BackoffFactor int `config:"backoff_factor" validate:"min=1"` - MaxBackoff time.Duration `config:"max_backoff" validate:"min=0,nonzero"` - Seek string `config:"seek"` - Matches []string `config:"include_matches"` + Inputs []*common.Config `config:"inputs"` + RegistryFile string `config:"registry_file"` + Backoff time.Duration `config:"backoff" validate:"min=0,nonzero"` + MaxBackoff time.Duration `config:"max_backoff" validate:"min=0,nonzero"` + Seek string `config:"seek"` + Matches []string `config:"include_matches"` } // DefaultConfig are the defaults of a Journalbeat instance var DefaultConfig = Config{ - RegistryFile: "registry", - Backoff: 1 * time.Second, - BackoffFactor: 2, - MaxBackoff: 60 * time.Second, - Seek: "tail", + RegistryFile: "registry", + Backoff: 1 * time.Second, + MaxBackoff: 60 * time.Second, + Seek: "tail", } diff --git a/journalbeat/docs/fields.asciidoc b/journalbeat/docs/fields.asciidoc index 61dc08cb084..b548ba877e1 100644 --- a/journalbeat/docs/fields.asciidoc +++ b/journalbeat/docs/fields.asciidoc @@ -16,6 +16,7 @@ grouped in the following categories: * <> * <> * <> +* <> * <> * <> @@ -100,26 +101,6 @@ Error fields containing additional info in case of errors. -*`error.message`*:: -+ --- -type: text - -Error message. - - --- - -*`error.code`*:: -+ --- -type: long - -Error code. - - --- - *`error.type`*:: + -- @@ -586,6 +567,18 @@ required: True The logged message. +-- + +*`custom`*:: ++ +-- +type: nested + +required: False + +Arbitrary fields coming from processes. + + -- [[exported-fields-docker-processor]] @@ -636,100 +629,1819 @@ Image labels. -- -[[exported-fields-host-processor]] -== Host fields +[[exported-fields-ecs]] +== ECS fields + +ECS fields. -Info collected for the host machine. +[float] +== agent fields +The agent fields contain the data about the agent/client/shipper that created the event. -*`host.name`*:: + + +*`agent.version`*:: + -- type: keyword -Hostname. +example: 6.0.0-rc2 + +Version of the agent. -- -*`host.id`*:: +*`agent.name`*:: + -- type: keyword -Unique host id. +example: filebeat + +Name of the agent. -- -*`host.architecture`*:: +*`agent.id`*:: + -- type: keyword -Host architecture (e.g. x86_64, arm, ppc, mips). +example: 8a4f500d + +Unique identifier of this agent (if one exists). +Example: For Beats this would be beat.id. -- -*`host.os.platform`*:: +*`agent.ephemeral_id`*:: + -- type: keyword -OS platform (e.g. centos, ubuntu, windows). +example: 8a4f500f + +Ephemeral identifier of this agent (if one exists). +This id normally changes across restarts, but `agent.id` does not. -- -*`host.os.version`*:: +[float] +== base fields + +The base set contains all fields which are on the top level. These fields are common across all types of events. + + + +*`base.@timestamp`*:: ++ +-- +type: date + +example: 2016-05-23T08:05:34.853Z + +required: True + +Date/time when the event originated. +For log events this is the date/time when the event was generated, and not when it was read. +Required field for all events. + + +-- + +*`base.tags`*:: + -- type: keyword -OS version. +example: ["production", "env2"] + +List of keywords used to tag each event. -- -*`host.os.family`*:: +*`base.labels`*:: ++ +-- +type: object + +example: {'key2': 'value2', 'key1': 'value1'} + +Key/value pairs. +Can be used to add meta information to events. Should not contain nested objects. All values are stored as keyword. +Example: `docker` and `k8s` labels. + + +-- + +*`base.message`*:: ++ +-- +type: text + +example: Hello World + +For log events the message field contains the log message. +In other use cases the message field can be used to concatenate different values which are then freely searchable. If multiple messages exist, they can be combined into one message. + + +-- + +[float] +== cloud fields + +Fields related to the cloud or infrastructure the events are coming from. + + + +*`cloud.provider`*:: + -- type: keyword -OS family (e.g. redhat, debian, freebsd, windows). +example: ec2 + +Name of the cloud provider. Example values are ec2, gce, or digitalocean. -- -*`host.os.kernel`*:: +*`cloud.availability_zone`*:: + -- type: keyword -The operating system's kernel version. +example: us-east-1c + +Availability zone in which this host is running. -- -*`host.ip`*:: +*`cloud.region`*:: ++ +-- +type: keyword + +example: us-east-1 + +Region in which this host is running. + + +-- + +*`cloud.instance.id`*:: ++ +-- +type: keyword + +example: i-1234567890abcdef0 + +Instance ID of the host machine. + + +-- + +*`cloud.instance.name`*:: ++ +-- +type: keyword + +Instance name of the host machine. + + +-- + +*`cloud.machine.type`*:: ++ +-- +type: keyword + +example: t2.medium + +Machine type of the host machine. + + +-- + +*`cloud.account.id`*:: ++ +-- +type: keyword + +example: 666777888999 + +The cloud account or organization id used to identify different entities in a multi-tenant environment. +Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. + + +-- + +[float] +== container fields + +Container fields are used for meta information about the specific container that is the source of information. These fields help correlate data based containers from any runtime. + + + +*`container.runtime`*:: ++ +-- +type: keyword + +example: docker + +Runtime managing this container. + + +-- + +*`container.id`*:: ++ +-- +type: keyword + +Unique container id. + + +-- + +*`container.image.name`*:: ++ +-- +type: keyword + +Name of the image the container was built on. + + +-- + +*`container.image.tag`*:: ++ +-- +type: keyword + +Container image tag. + + +-- + +*`container.name`*:: ++ +-- +type: keyword + +Container name. + + +-- + +*`container.labels`*:: ++ +-- +type: object + +Image labels. + + +-- + +[float] +== destination fields + +Destination fields describe details about the destination of a packet/event. + + + +*`destination.ip`*:: + -- type: ip -List of IP-addresses. +IP address of the destination. +Can be one or multiple IPv4 or IPv6 addresses. -- -*`host.mac`*:: +*`destination.hostname`*:: ++ +-- +type: keyword + +Hostname of the destination. + + +-- + +*`destination.port`*:: ++ +-- +type: long + +Port of the destination. + + +-- + +*`destination.mac`*:: ++ +-- +type: keyword + +MAC address of the destination. + + +-- + +*`destination.domain`*:: ++ +-- +type: keyword + +Destination domain. + + +-- + +*`destination.subdomain`*:: ++ +-- +type: keyword + +Destination subdomain. + + +-- + +[float] +== device fields + +Device fields are used to provide additional information about the device that is the source of the information. This could be a firewall, network device, etc. + + + +*`device.mac`*:: + -- type: keyword -List of hardware-addresses, usually MAC-addresses. +MAC address of the device + + +-- + +*`device.ip`*:: ++ +-- +type: ip + +IP address of the device. + + +-- + +*`device.hostname`*:: ++ +-- +type: keyword + +Hostname of the device. + + +-- + +*`device.vendor`*:: ++ +-- +type: text + +Device vendor information. + + +-- + +*`device.version`*:: ++ +-- +type: keyword + +Device version. + + +-- + +*`device.serial_number`*:: ++ +-- +type: keyword + +Device serial number. + + +-- + +*`device.timezone.offset.sec`*:: ++ +-- +type: long + +example: -5400 + +Timezone offset of the host in seconds. +Number of seconds relative to UTC. If the offset is -01:30 the value will be -5400. + + +-- + +*`device.type`*:: ++ +-- +type: keyword + +example: firewall + +The type of the device the data is coming from. +There is no predefined list of device types. Some examples are `endpoint`, `firewall`, `ids`, `ips`, `proxy`. + + +-- + +[float] +== error fields + +These fields can represent errors of any kind. Use them for errors that happen while fetching events or in cases where the event itself contains an error. + + + +*`error.id`*:: ++ +-- +type: keyword + +Unique identifier for the error. + + +-- + +*`error.message`*:: ++ +-- +type: text + +Error message. + + +-- + +*`error.code`*:: ++ +-- +type: keyword + +Error code describing the error. + + +-- + +[float] +== event fields + +The event fields are used for context information about the data itself. + + + +*`event.id`*:: ++ +-- +type: keyword + +example: 8a4f500d + +Unique ID to describe the event. + + +-- + +*`event.category`*:: ++ +-- +type: keyword + +example: metrics + +Event category. +This can be a user defined category. + + +-- + +*`event.type`*:: ++ +-- +type: keyword + +example: nginx-stats-metrics + +A type given to this kind of event which can be used for grouping. +This is normally defined by the user. + + +-- + +*`event.action`*:: ++ +-- +type: keyword + +example: reject + +The action captured by the event. The type of action will vary from system to system but is likely to include actions by security services, such as blocking or quarantining; as well as more generic actions such as login events, file i/o or proxy forwarding events. +The value is normally defined by the user. + + +-- + +*`event.module`*:: ++ +-- +type: keyword + +example: mysql + +Name of the module this data is coming from. +This information is coming from the modules used in Beats or Logstash. + + +-- + +*`event.dataset`*:: ++ +-- +type: keyword + +example: stats + +Name of the dataset. +The concept of a `dataset` (fileset / metricset) is used in Beats as a subset of modules. It contains the information which is currently stored in metricset.name and metricset.module or fileset.name. + + +-- + +*`event.severity`*:: ++ +-- +type: long + +example: 7 + +Severity describes the severity of the event. What the different severity values mean can very different between use cases. It's up to the implementer to make sure severities are consistent across events. + + +-- + +*`event.original`*:: ++ +-- +type: keyword + +example: Sep 19 08:26:10 host CEF:0|Security| threatmanager|1.0|100| worm successfully stopped|10|src=10.0.0.1 dst=2.1.2.2spt=1232 + +Raw text message of entire event. Used to demonstrate log integrity. +This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. + + +Field is not indexed. + +-- + +*`event.hash`*:: ++ +-- +type: keyword + +example: 123456789012345678901234567890ABCD + +Hash (perhaps logstash fingerprint) of raw field to be able to demonstrate log integrity. + + +-- + +*`event.version`*:: ++ +-- +type: keyword + +example: 0.1.0 + +required: True + +The version field contains the version an event for ECS adheres to. +This field should be provided as part of each event to make it possible to detect to which ECS version an event belongs. +event.version is a required field and must exist in all events. It describes which ECS version the event adheres to. +The current version is 0.1.0. + + +-- + +*`event.duration`*:: ++ +-- +type: long + +Duration of the event in nanoseconds. + + +-- + +*`event.created`*:: ++ +-- +type: date + +event.created contains the date when the event was created. +This timestamp is distinct from @timestamp in that @timestamp contains the processed timestamp. For logs these two timestamps can be different as the timestamp in the log line and when the event is read for example by Filebeat are not identical. `@timestamp` must contain the timestamp extracted from the log line, event.created when the log line is read. The same could apply to package capturing where @timestamp contains the timestamp extracted from the network package and event.created when the event was created. +In case the two timestamps are identical, @timestamp should be used. + + +-- + +*`event.risk_score`*:: ++ +-- +type: float + +Risk score or priority of the event (e.g. security solutions). Use your system's original value here. + + +-- + +*`event.risk_score_norm`*:: ++ +-- +type: float + +Normalized risk score or priority of the event, on a scale of 0 to 100. +This is mainly useful if you use more than one system that assigns risk scores, and you want to see a normalized value across all systems. + + +-- + +[float] +== file fields + +File fields provide details about each file. + + + +*`file.path`*:: ++ +-- +type: text + +Path to the file. + +*`file.path.raw`*:: ++ +-- +type: keyword + +Path to the file. This is a non-analyzed field that is useful for aggregations. + + +-- + +-- + +*`file.target_path`*:: ++ +-- +type: text + +Target path for symlinks. + +*`file.target_path.raw`*:: ++ +-- +type: keyword + +Path to the file. This is a non-analyzed field that is useful for aggregations. + + +-- + +-- + +*`file.extension`*:: ++ +-- +type: keyword + +example: png + +File extension. +This should allow easy filtering by file extensions. + + +-- + +*`file.type`*:: ++ +-- +type: keyword + +File type (file, dir, or symlink). + +-- + +*`file.device`*:: ++ +-- +type: keyword + +Device that is the source of the file. + +-- + +*`file.inode`*:: ++ +-- +type: keyword + +Inode representing the file in the filesystem. + +-- + +*`file.uid`*:: ++ +-- +type: keyword + +The user ID (UID) or security identifier (SID) of the file owner. + + +-- + +*`file.owner`*:: ++ +-- +type: keyword + +File owner's username. + +-- + +*`file.gid`*:: ++ +-- +type: keyword + +Primary group ID (GID) of the file. + +-- + +*`file.group`*:: ++ +-- +type: keyword + +Primary group name of the file. + +-- + +*`file.mode`*:: ++ +-- +type: keyword + +example: 416 + +Mode of the file in octal representation. + +-- + +*`file.size`*:: ++ +-- +type: long + +File size in bytes (field is only added when `type` is `file`). + +-- + +*`file.mtime`*:: ++ +-- +type: date + +Last time file content was modified. + +-- + +*`file.ctime`*:: ++ +-- +type: date + +Last time file metadata changed. + +-- + +[float] +== geo fields + +Geo fields can carry data about a specific location related to an event or geo information for an IP field. + + + +*`geo.continent_name`*:: ++ +-- +type: keyword + +Name of the continent. + + +-- + +*`geo.country_iso_code`*:: ++ +-- +type: keyword + +Country ISO code. + + +-- + +*`geo.location`*:: ++ +-- +type: geo_point + +Longitude and latitude. + + +-- + +*`geo.region_name`*:: ++ +-- +type: keyword + +Region name. + + +-- + +*`geo.city_name`*:: ++ +-- +type: keyword + +City name. + + +-- + +[float] +== host fields + +Host fields provide information related to a host. A host can be a physical machine, a virtual machine, or a Docker container. +Normally the host information is related to the machine on which the event was generated/collected, but they can be used differently if needed. + + + +*`host.timezone.offset.sec`*:: ++ +-- +type: long + +example: -5400 + +Timezone offset of the host in seconds. +Number of seconds relative to UTC. If the offset is -01:30 the value will be -5400. + + +-- + +*`host.name`*:: ++ +-- +type: keyword + +host.name is the hostname of the host. +It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. + + +-- + +*`host.id`*:: ++ +-- +type: keyword + +Unique host id. +As hostname is not always unique, use values that are meaningful in your environment. +Example: The current usage of `beat.name`. + + +-- + +*`host.ip`*:: ++ +-- +type: ip + +Host ip address. + + +-- + +*`host.mac`*:: ++ +-- +type: keyword + +Host mac address. + + +-- + +*`host.type`*:: ++ +-- +type: keyword + +Type of host. +For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. + + +-- + +*`host.os.platform`*:: ++ +-- +type: keyword + +example: darwin + +Operating system platform (centos, ubuntu, windows, etc.) + + +-- + +*`host.os.name`*:: ++ +-- +type: keyword + +example: Mac OS X + +Operating system name. + + +-- + +*`host.os.family`*:: ++ +-- +type: keyword + +example: debian + +OS family (redhat, debian, freebsd, windows, etc.) + + +-- + +*`host.os.version`*:: ++ +-- +type: keyword + +example: 10.12.6 + +Operating system version. + + +-- + +*`host.architecture`*:: ++ +-- +type: keyword + +example: x86_64 + +Operating system architecture. + + +-- + +[float] +== http fields + +Fields related to HTTP requests and responses. + + + +*`http.request.method`*:: ++ +-- +type: keyword + +example: GET, POST, PUT + +Http request method. + + +-- + +*`http.response.status_code`*:: ++ +-- +type: long + +example: 404 + +Http response status code. + + +-- + +*`http.response.body`*:: ++ +-- +type: text + +example: Hello world + +The full http response body. + + +-- + +*`http.version`*:: ++ +-- +type: keyword + +example: 1.1 + +Http version. + + +-- + +[float] +== log fields + +Fields which are specific to log events. + + + +*`log.level`*:: ++ +-- +type: keyword + +example: ERR + +Log level of the log event. +Some examples are `WARN`, `ERR`, `INFO`. + + +-- + +*`log.original`*:: ++ +-- +type: keyword + +example: Sep 19 08:26:10 localhost My log + + +This is the original log message and contains the full log message before splitting it up in multiple parts. +In contrast to the `message` field which can contain an extracted part of the log message, this field contains the original, full log message. It can have already some modifications applied like encoding or new lines removed to clean up the log message. +This field is not indexed and doc_values are disabled so it can't be queried but the value can be retrieved from `_source`. + + +Field is not indexed. + +-- + +[float] +== network fields + +Fields related to network data. + + + +*`network.name`*:: ++ +-- +type: text + +example: Guest Wifi + +Name given by operators to sections of their network. + + +*`network.name.raw`*:: ++ +-- +type: keyword + +Name given by operators to sections of their network. + + +-- + +-- + +*`network.protocol`*:: ++ +-- +type: keyword + +example: http + +Network protocol name. + + +-- + +*`network.direction`*:: ++ +-- +type: keyword + +example: inbound + +Direction of the network traffic. +Recommended values are: + * inbound + * outbound + * unknown + + +-- + +*`network.forwarded_ip`*:: ++ +-- +type: ip + +example: 192.1.1.2 + +Host IP address when the source IP address is the proxy. + + +-- + +*`network.inbound.bytes`*:: ++ +-- +type: long + +example: 184 + +Network inbound bytes. + + +-- + +*`network.inbound.packets`*:: ++ +-- +type: long + +example: 12 + +Network inbound packets. + + +-- + +*`network.outbound.bytes`*:: ++ +-- +type: long + +example: 184 + +Network outbound bytes. + + +-- + +*`network.outbound.packets`*:: ++ +-- +type: long + +example: 12 + +Network outbound packets. + + +-- + +*`network.total.bytes`*:: ++ +-- +type: long + +example: 368 + +Network total bytes. The sum of inbound.bytes + outbound.bytes. + + +-- + +*`network.total.packets`*:: ++ +-- +type: long + +example: 24 + +Network outbound packets. The sum of inbound.packets + outbound.packets + + +-- + +[float] +== organization fields + +The organization fields enrich data with information about the company or entity the data is associated with. These fields help you arrange or filter data stored in an index by one or multiple organizations. + + + +*`organization.name`*:: ++ +-- +type: text + +Organization name. + + +-- + +*`organization.id`*:: ++ +-- +type: keyword + +Unique identifier for the organization. + + +-- + +[float] +== os fields + +The OS fields contain information about the operating system. These fields are often used inside other prefixes, such as `host.os.*` or `user_agent.os.*`. + + + +*`os.platform`*:: ++ +-- +type: keyword + +example: darwin + +Operating system platform (such centos, ubuntu, windows). + + +-- + +*`os.name`*:: ++ +-- +type: keyword + +example: Mac OS X + +Operating system name. + + +-- + +*`os.family`*:: ++ +-- +type: keyword + +example: debian + +OS family (such as redhat, debian, freebsd, windows). + + +-- + +*`os.version`*:: ++ +-- +type: keyword + +example: 10.12.6-rc2 + +Operating system version as a raw string. + + +-- + +*`os.kernel`*:: ++ +-- +type: keyword + +example: 4.4.0-112-generic + +Operating system kernel version as a raw string. + + +-- + +[float] +== process fields + +These fields contain information about a process. These fields can help you correlate metrics information with a process id/name from a log message. The `process.pid` often stays in the metric itself and is copied to the global field for correlation. + + + +*`process.args`*:: ++ +-- +type: keyword + +example: ['-l', 'user', '10.0.0.16'] + +Process arguments. +May be filtered to protect sensitive information. + + +-- + +*`process.name`*:: ++ +-- +type: keyword + +example: ssh + +Process name. +Sometimes called program name or similar. + + +-- + +*`process.pid`*:: ++ +-- +type: long + +Process id. + + +-- + +*`process.ppid`*:: ++ +-- +type: long + +Process parent id. + + +-- + +*`process.title`*:: ++ +-- +type: keyword + +Process title. +The proctitle, often the same as process name. + + +-- + +[float] +== service fields + +The service fields describe the service for or from which the data was collected. These fields help you find and correlate logs for a specific service and version. + + + +*`service.id`*:: ++ +-- +type: keyword + +example: d37e5ebfe0ae6c4972dbe9f0174a1637bb8247f6 + +Unique identifier of the running service. +This id should uniquely identify this service. This makes it possible to correlate logs and metrics for one specific service. +Example: If you are experiencing issues with one redis instance, you can filter on that id to see metrics and logs for that single instance. + + +-- + +*`service.name`*:: ++ +-- +type: keyword + +example: elasticsearch + +Name of the service data is collected from. +The name can be used to group and correlate logs and metrics from one service. +Example: If logs or metrics are collected from Redis, `service.name` would be `redis`. + + +-- + +*`service.type`*:: ++ +-- +type: keyword + +Service type. + + +-- + +*`service.state`*:: ++ +-- +type: keyword + +Current state of the service. + + +-- + +*`service.version`*:: ++ +-- +type: keyword + +example: 3.2.4 + +Version of the service the data was collected from. +This allows to look at a data set only for a specific version of a service. + + +-- + +*`service.ephemeral_id`*:: ++ +-- +type: keyword + +example: 8a4f500f + +Ephemeral identifier of this service (if one exists). +This id normally changes across restarts, but `service.id` does not. + + +-- + +[float] +== url fields + +URL fields provide a complete URL, with scheme, host, and path. The URL object can be reused in other prefixes, such as `host.url.*` for example. Keep the structure consistent whenever you use URL fields. + + + +*`url.href`*:: ++ +-- +type: text + +example: https://elastic.co:443/search?q=elasticsearch#top + +Full url. The field is stored as keyword. +`url.href` is a [multi field](https://www.elastic.co/guide/en/ elasticsearch/reference/6.2/ multi-fields.html#_multi_fields_with_multiple_analyzers). The data is stored as keyword `url.href` and test `url.href.analyzed`. These fields enable you to run a query against part of the url still works splitting up the URL at ingest time. +`href` is an analyzed field so the parsed information can be accessed through `href.analyzed` in queries. + + +*`url.href.raw`*:: ++ +-- +type: keyword + +The full URL. This is a non-analyzed field that is useful for aggregations. + + +-- + +-- + +*`url.scheme`*:: ++ +-- +type: keyword + +example: https + +Scheme of the request, such as "https". +Note: The `:` is not part of the scheme. + + +-- + +*`url.host.name`*:: ++ +-- +type: keyword + +example: elastic.co + +Hostname of the request, such as "example.com". +For correlation the this field can be copied into the `host.name` field. + + +-- + +*`url.port`*:: ++ +-- +type: integer + +example: 443 + +Port of the request, such as 443. + + +-- + +*`url.path`*:: ++ +-- +type: text + +Path of the request, such as "/search". + + +*`url.path.raw`*:: ++ +-- +type: keyword + +URL path. A non-analyzed field that is useful for aggregations. + + +-- + +-- + +*`url.query`*:: ++ +-- +type: text + +The query field describes the query string of the request, such as "q=elasticsearch". +The `?` is excluded from the query string. If a URL contains no `?`, there is no query field. If there is a `?` but no query, the query field exists with an empty string. The `exists` query can be used to differentiate between the two cases. + + +*`url.query.raw`*:: ++ +-- +type: keyword + +URL query part. A non-analyzed field that is useful for aggregations. + + +-- + +-- + +*`url.fragment`*:: ++ +-- +type: keyword + +Portion of the url after the `#`, such as "top". +The `#` is not part of the fragment. + + +-- + +*`url.username`*:: ++ +-- +type: keyword + +Username of the request. + + +-- + +*`url.password`*:: ++ +-- +type: keyword + +Password of the request. + + +-- + +[float] +== user fields + +The user fields describe information about the user that is relevant to the event. Fields can have one entry or multiple entries. If a user has more than one id, provide an array that includes all of them. + + + +*`user.id`*:: ++ +-- +type: keyword + +One or multiple unique identifiers of the user. + + +-- + +*`user.name`*:: ++ +-- +type: keyword + +Name of the user. +The field is a keyword, and will not be tokenized. + + +-- + +*`user.email`*:: ++ +-- +type: keyword + +User email address. + + +-- + +*`user.hash`*:: ++ +-- +type: keyword + +Unique user hash to correlate information for a user in anonymized form. +Useful if `user.id` or `user.name` contain confidential information and cannot be used. + + +-- + +[float] +== user_agent fields + +The user_agent fields normally come from a browser request. They often show up in web service logs coming from the parsed user agent string. + + + +*`user_agent.original`*:: ++ +-- +type: text + +Unparsed version of the user_agent. + + +-- + +*`user_agent.device`*:: ++ +-- +type: keyword + +Name of the physical device. + + +-- + +*`user_agent.version`*:: ++ +-- +type: keyword + +Version of the physical device. + + +-- + +*`user_agent.major`*:: ++ +-- +type: long + +Major version of the user agent. + + +-- + +*`user_agent.minor`*:: ++ +-- +type: long + +Minor version of the user agent. + + +-- + +*`user_agent.patch`*:: ++ +-- +type: keyword + +Patch version of the user agent. + + +-- + +*`user_agent.name`*:: ++ +-- +type: keyword + +example: Chrome + +Name of the user agent. + + +-- + +*`user_agent.os.name`*:: ++ +-- +type: keyword + +Name of the operating system. + + +-- + +*`user_agent.os.version`*:: ++ +-- +type: keyword + +Version of the operating system. + + +-- + +*`user_agent.os.major`*:: ++ +-- +type: long + +Major version of the operating system. + + +-- + +*`user_agent.os.minor`*:: ++ +-- +type: long + +Minor version of the operating system. + + +-- + +[[exported-fields-host-processor]] +== Host fields + +Info collected for the host machine. + + + + +*`host.os.kernel`*:: ++ +-- +type: keyword + +The operating system's kernel version. -- diff --git a/journalbeat/include/fields.go b/journalbeat/include/fields.go index aee02e903a4..c1a0d8ef27a 100644 --- a/journalbeat/include/fields.go +++ b/journalbeat/include/fields.go @@ -31,5 +31,5 @@ func init() { // Asset returns asset data func Asset() string { - return "eJzMW91z47YRf/dfsXMvbWck2pYd38UPnbqXNnHbtDfN5VmBgCWFmAQYALSs/vUdgOCXBH5ZOl/ykIQiuPvbxX4DXsIT7u+ByiyT4gLAcJPiPbz76H6Af8hCCZJukJh3FwAMNVU8N1yKe/jzBQDARykM4UJ7EhBzTJkG8kx4SjYpAhdA0hTwGYUBs89RRxfgl907EksQJMN7UEjY2vAMtSFZ7l4FWZb/fN4i2MWw22IHqKMDZovwa/kjoDBqH3V4UamQFTUXi+seEiWLUb5/LwUsNDLY7EHvtcGMLSuC8IRKYApbTHNUkf+wLW4bRiG4adEuYTzhficVa/3eCwbgQQhpiH2jQcaQodYkQbsdbl+4SGpZNcRKZh6xY62jY0Qa1fprwLKM26AqQH4T2at3KlfymbNytypiY/siN78iPVZBm+2IAmr2RkIqE5ACNrglaWzVQYDxOEZlXSJXMlEki1pfH6JqIyMF6+xNP7hRgG6fLLnKaWVcuml0sDCEp40plQkXBWcB+iW0VIok8FLhbwVXyO4hJqnGwAp8IVluA9L11dVV4P2ofGWccADh58fvrIg2MpT7a3VPUetDeRvJNGrNXWD8UoLdvF4qZwoVxCmS1cEvO9yqPg8fF6YW5N1lyjeXPhxW/4Xl0vr1u3lmaYWz6YQIBikXWMk2KJT99xtJNVOcf5OsFgFfkBbGJsawFM37s8kygu4TMVsbpCy6IYTzlFKnk6Oo0Os4J0kx0bcrWMkbwfp+Hqz8jWB9mgfL7/D5ss5PPjqcmHfkTthi5UsknokhuG12FszrM0nYs88KtsFaRWcPqh/1QSH45pBFK3RWoC2mAcSB+vX3Abuqb49r7rJl+D1WmwyfOX2zVGRV59unkrFD0ROTik2p2a8AruY9gK8UYK33WcrFkw6iNPhyaKYnQXxgjNt3JAXP14HRkEsujO25fJ6vtCuZ688vGT4PSmEXrnNitm8ixucDhJZx6U1cVy8moe6rCL8E4GO7BaLB1udbudNQ5BZyS/dGIcIGU7kDW0EdBwUqGZ4lJPg4ZOlBggIVKU3B1nqHWXcoFsS8pyidrc6mkIyiS63oJZUKLzMiSIIqoq/oFjoBVxaKooMLuy2qsqy1UZHrSgHIwpYTF4Ka4zR9spy/ys06lclaG2IKvfbjkBMFrcDCbsvpthGtltezCYtqG6vzFJtNk7467GYnSOQaPFFkG1QdU50sVF08lyXsRVuWOXOimamzb3R0OKB5heOODmWG3LRnGHPy3h4PYKZs7uHgxTvncLsRLJVPFeAVphkcsPRJEB6u9JUj/aBnD1QGBJk8SOkZopwb/UTYYwOTwWHJfMwDSKYNR6YKHW7zg4Y9QcPX022g8T+/8z66pjJJfGjtDavJ1wPrgqaHqgqhh223+HpAXW83FSclOdnwlBvUp5qnRYFxjNTwZ2wTDjp6eJwzP1HKuCIxlgu5eJbUnQKtA5vzurjy4fabq/j67v2K4d3tHf3wgbLrmxtCGLuNV+z91YzQ2MCzexlL5XSmCuHOFemepnUYso17209gR1rFJHAR2OfDnH9SNLUmFumUU3T/u7xe3dz6Z5+glqtIU5njrNwgjJKpdzTXZ/kupUo3W46KKLrdH8sXmr7N9LoReI5Dp3o4nKWAVP3DrP564qw5YuJorUaTdicqp1jFgSXM2Pkapv2uZyzlRmmnw52IxO3pIJxph9JT9CYSLl4ijep5ntbGx5GvOUb/sjs9cxZpFBE6l2oecKOKMG6916lMJsL9Qe4O2zwXZxVS5M+hE/wK9VZqc0o+s1wtjbGEtpHShFLZwWhgyo4y9oF++/6WaBZfXbMNrjBe3bH3sf1hdXdLv52xxxZWO4W550qT4UzVKgZSmZyqu9HhUp9Cc8Wl4mb/tpVbxbWCXxW/8OD1YbumnBjuyql9if7YV2JC3fu3BV9xPRE8ZygMjzmqc1izLubUXTXrV8hQ4fefXPQAD8amHlx+UmFboboNuvA34zZITHMv7q/l04R7cO4m2szLcB35LIGo1Y8PYG+HeIsQNApWuWErmOoIHlur3Ge8aUI0mqr8o1LEPClUWRzHPMWF/d2+JAaeSVrYL90lOEeTG/sopGkTW9Rh1XPy6z9Lx6qDY2HfuZ9+sY+/1HSkk7gfV3SstIrjuOJqbC7JmEKJMsm489y8Cmb+rKcetNbAW7pThRBcJAE0tpf4nxQT0FQrvySaZ1StWngAjF9YmZUzZ7f57Yks11WU73B695f6Fue7jn8yYnDIP2OpMmI66+oQ81AkhTawujNbWF1d3y3genV/8839NzfRzc1qmnYdpPLCaJ2hnIMopFKxbqo8EMqQRA9zeVAbbhRRe7e21BYlNhQ4e89RlRtFBHMPrtQi3fMGq6cDxmV06Oixcz2xfFiHirQeoHWscnVg7VM2QJXMDhCgUlLNLRH+Zj+qImDr2idpDim5iKX1bEq0i1+Ojx4rGbqBH/qy1kAOKqH1TryCJ28HKX2UuiUSqK73+ZRmapR6aSbV5e1UFqzJUR/tY3Xr1YppCCOGhNPWj/5teQeXdj7Vdq+aEEQYW7sF64pkNV6SqjeL2aWR+yqqyB46NtIR722Pg7sII/gktebWcF1O0kAUWoILSCguQCpgPOGGpJIiEVEvNi60IYJiU9v3YHn0C1sdv00ikBG65eLQdUMcxjNTzaOd16dx8QvWLTur9WxWUYaMF9kw9x9LEs7E5jH3ZY6r2NatlFcjKPQSiTbLazoSSFuEwGVE3mQ7rks4XDdpbsDkXGysd7WG4t8sX6abnv/EYvleyiTF0tP6uStMRlPtf92aMfm8ozNJn5z/eE//rnoOEC/fgTbE2PCbpkhtznZuXr6zPqu3Upl1mQGa8p4IupWq4resvbzn7zZqWBDMD31x3OcEVNGk2fBATPxZ8N8KbAgCD/Q7LXZZKH3M4ti2C0euqk49AFtIbAqeGgiNABsoE8/bBpB8rHl27yId80rJBlN9xO3oTx0G6okRLI9OEyWf2mj9aMab7A/lU4DIoy0GWobqRxnd0NPYpv191DIHx0J9dnn6nvzg24pQ030WSy8DRMDIiaJbbpCaQp1Bhg45+CNGSQQvH+7Wd7cLICpbQJ7TBWQ8138KHA/oKE+JsSX9aUj+8xNUhDwGisJIvYBiUwhTLGDHBZO7HhDdjuf1GDydII+YZDw9HgPNZVGS8UIqZFtiFsBww4lYQKwQN5qNSNtzlXQWks+BfvMPurrh1qsHfnzWxSdeefkX18aG08dPS8KYQq0xcHCaEXqaYBWbLVFsRxQ2zBZQ6IKk6R5+fPjYxlBFsadiY8Uvz2x9LPtn+7cA2+Z9XYR3K+qGKLQj2XBSbj4aDX8d0DArCOaSnSE5tTSQS1ZG1iCr0Cniazl9kgx+fvzumJG7C5uTScdZ01g1FI+Z2f7vrBp0d2HDKpya2qcxKqlBRvJjTqT5u85zsWuRDPM8Z7nU4ks7ldMQ2zMUjEG+Jd3/BwAA//87xgOV" + return "eJzkff1z2ziy4O/5K1CZqptkT6Zt2XESv9p7Ly8fM75NJqk4ubm63S0LIiEJYwrgAKAVzb3736/Q+CBAghJlKZ5X9bJVm4lIdjeARnejv3CEbsn6EuV8ueTsEUKKqpJcosev4Qf0P3ktGC6nBKvHjxAqiMwFrRTl7BL9j0cIIfSaM4UpkxYEmlFSFhLhO0xLPC0JogzhskTkjjCF1LoiMnuE7GuXAOIIMbwkl0gQXNwouiRS4WUFj5IozZ8vC4L0y2i1IBGhAAepBUG/mR8RYUqsswhXzgUpao9F03WJ5oLXW/G+MwOsJSnQdI3kWiqyLI4cQHRLBCMlWpCyIiKzH4bDDcmoGVUBbEPGLVmvuCiC33uJQegVY1xh/UQiPkNLIiWeE70csC6Uzf1YJZoJvrQUA2qZdSmSRNz8GWRpxCFRjiC7iMW9V6oS/I4WZrUcsG3rwqe/kbw7BSHaLRPg0SuOSj5HnKEpWeBypqcDo4LOZkToLVEJPhd4mQVft6kKKcN1Ea1NP3FbCYR10uDcpuUzs02z1ospekKaSj6n7KamRQKBoa3kbJ54KMjvNRWkuEQzXEqSeIN8w8tKS6TTk5OTxPOtAzSCAihEX6/e6DFq0WAWWE9+TqRsD7gZmiRSUpCM32tgZ/cfFfCCI3HIyLz0W7aXqm+Lbx+MH8jj45JOj608dH+joyO9sR/vxpd6cFqfYFagkjLixrZxUPr/H2hUOw7nF7z0QyDfSF4rrRnTo2ieH2wsW6j7hNVCSylN3SYKd5sUr086UqF34+w1ioF725E1fyCyftqNrOqByPq0G1l2hQ+ndq6tdNhT8fAV09bK91A8A0VwyHaamPtrkvTOPiixDa1OOlui+qluWYIPTjILRKcjWtO0geKEAfufg2xn4HaNbnNm+M9obhbkjuYPpor01Nnzk0EMVPTIpHpqZvZPIM7j3kCfGcCNXC9Lym5lkkpFvrXZdC8SXxUF1c9wiSxeIEaiilOm9KHL6nk3u7yAA/pxQe42jkK/eFNhtXiQYXxpUagRm91EpXswiOo+i/B7ENzlW4Ql0vb5gq8kqitNcjD3ShCCpqTkK6QtqK5QyHlBDiISrBzS8NCcMCKwYQVt67W17iZZMKM9RunO09kYkll2LEV+nHNBjpeY4TkRWX6P00IkcHktcgLkotWCCGPWaqlIpZsAUqQ5Z1azXHXV9N7j/I1Pb0o+v5EKq1reWH/IngN1xKLVguaLZmh+vBZNeqj6YHUYY7M5pI/bp9kBI4IDHquXUyIiVh08KG88GxP2UTiWXRxFO6rOPt9R20Nzj4271SuzaZuCr+Ngx65NDpghi9t2vNjNufm4kTSV9x3APVgz6WDpG0HaudJnjvQTvbNDZcNABjtSepwoh6Z+INnbHCYbnSW707yBkmHOkaGDTh/zk4w9YIZPh/NAs//sylvpWvL53IrWXrE6//OIBaFpSRU1k5t5t/7zCIWz3VA6c1zhKS2pInJf9tRUkNmM5IrekRBwcqOn3Tm7K0o+cyC26ULK7ngOYaCbxOLcT668OH92Mju9eD4uyMX5Rf7iRV6cnp1hXBTns3Hx/GQH0diQp9dyxgXMmagZBBbzdV56MaQP7uE+QSscGJOIssQ6t3X+XtJUs1gmS5oT+M+j0/HZuf23VVBH40zmvCI76QamBC/tRoNzlj2lOHWzoERgkS/W3fGlvG877rot5AGGyHpo+1IQF/3OrH574qA6YqBrzVNTxh6VfbiixQk7rLwnU3/X45YCV9r+5A6kBNZ0IznDotJD5o3NKfuWSSLudpu17e7I+8TRv+9K7+iLVAIzWXGxG+FK1Gm65VqWfD6Q3J/5qn3MAzkrSE7oXSqE76hecKn20Wcaq4axTaFNOVcpVdZyDQxZ0aJ4kb98fo5lMTs5LaZkTGbji+L5TP8wvjjPX+6wxpqsUIXBv91MpjVVYAyUfL7v3G11LvVNaCUoF1StH9Zyc1gd+c74Ra/sfOhTU4UVBXNqbajv7pUZzuH5wxLvsO5JPC0IU3RGiTgEN8t6F7vLo77HGBz99pNHPYQnZVMPXdZToY9C0THIW3K1VHwZYWJEKlJ0cIUT1IPslZhSJbBYOwdPzpd6y0AWlLXXicwePbK5eVOCVZOZ9+/mXwMy8SAXbsd0vGjUGkAWOAQ2TF6oYzSFSBJWODkQSHOZoavgLfiMNqcgSZSzP3POZnReC2Odz2hJRvp3/RArdIfLWn8JaXgAkyr9T8ZVCGzk5brFZN//wgFVRMdIP4OfJvqfEw+Hw4j76cq6k+Ywbp84TxtoOVULZrQcBJQrJ01tsMl7ej3hwdyJmjHK5glq9GHmD84GUOPe/J7U3BERGOMbiLEvOrYCdobFD13CVDo1E2F6/G8+j/RxtG0LrMgmATHjYolV9J6Xca/qeS0VGl+oBRqfnF6M0On48uzZ5bOz7OxsPGx2gSSTsupVJGwQQXIuilhXtwal8FwOFS36XTNbOdaiAPi9IsIsFGYF/ANsPRwHPPQ8tRAb6RDNY5Qgaf5xk7ISewj1sgoMUb+ntIAyyFoUECG42NVGeas/aoSsTzzFTZSUshnXOzvHEuQX4JHbbBZNwQDreIMSNKSZqbZinuSykfJvX1+nhfzb19d+hmICo/nCc8K8PWxAvgp+gsm7RAN4FgC15tBEMbHCCE95bcQovHecl1T/JRe0AvZa4EYc54L4ME5sGXKuGFckWjqz5+QlurJi1y6QZl8JZm7J53LU4M6chNciGQQNescFevXpw8jqBrVoFs0MywopJ9pxVR3bI2EWDF7ziBYKWhYVnICSQfkCszlBdOZBwoRQqTWfnhLB6/kC/V6TuhGZEpX0lqC/4dktHqHPpKByhLhwSj94sVFidb7Q0vg9n0uF5QKZMaFrIu6abKPuluhj3Vj63pN7/1csmc38N0xo/nixeZGdZCdHIh93iBkYbBgYKNhChuOLhB28Hw1fGf297tq0VFr+eUJnSOtU8o1KJZ92KPTcfgkcazgcvl/xuiy08AZ+pkUy5PACn8+enZwUnXGRakGWROBymOt1k7RykPYZ5Bf9Mi0Q05upLNd2C0mEc8GltjekwkLJEZrWCk3MatJi4vfcptHPuiJwir0h7kzn5hcrAE+3C0ANBpRn7lSWNp+tQDT2DxZEG0MQIuIVKskdKUGASOLtb0GcWW6Hq6GA+e3ju3L33Zwwc1DK1EGbfDXVAstWeLQ51mk75+jk2dH47MvJi8uTZ5dn59mLZ2f/Z+Bh7w1W5Lip0mlMHi7onLImDyP66p0R73ZaDJsZAQ6DSgKMDKeRtnAikFpmwxdUWY8STmH+bCfJzDjoGX9akq33E2YZ2rC/mjn9+z8eV4IXNdhd/3g8Qv94TNjd+B+P/zlwVt9TqTTbWCS2GElxTQoiOF+ECrZDb4mnpOxS3Cl58QT/31uyPr00Z67TkcY6tv8a/79hBP+NrI/Nka3CVLQnUv95baxUNxBc6JO4VqiB8lXcLQS6XoBoBE1sjZLoQG7+mCHJDL0qS0Ow2YlScb3GWLoZ3CSTJwXPb4mYgNE8uX0hJ3YGe6Y3dkugPp+K3XWnSQ75mZQlR79yUQ6NmHe2TBOoMqzsxVcyjSaCdcUQVwsi4ECsDa8kvHjBcs5yrAiLZQ4Ksmfs/DciE07zM0FIuUaSYJEvIKyuD/PLulS0amWb+Roy0DFg+q0dGTlfTikD76bioIi6w/POnJLXRawZXgc/DbONrfdTkNIYtSZDAEBrE42ymcBSiTpXtc0/MyvTWKBGIzjHz0BjeIY+ECVoPjXHbW/Bar3C0NvXY3AoAKvOiMoXRBq7FHxLNECvXxsFNMNBKOKRyMCnEi1xvqDMrE9DRHjgl0AGEmTJFXHvI14rSQsS4EpTh5G1vUOQoXkOHxuaWyxtwDaggFst+tDqtwjiidtd69qCQpHauiQwc/e2aM24HLrMMUIoykg+HqF5TvQ5orXx5lThkucEsx5JZZ2C4GC9CRxE0YBqeUSwVEen+X7jehUgQ+Bjoo3/iErDt83C9JAsyHzY6aVL/zAyPwOCe9FGmVSY5SQbZG57AilE9Z5dPH/x8gRP84LMToaRemXxBbFnINRt1C1U7n/48gSE3t8BJLinA90ofqbUOFuSgtbLYeR9cBJgXe1CHc5zXsPRYxfaLi4unj9//uLFi5cvXw4j70sjDw1GrTe4mGNG/zD2Di28erXnrnWjTyNY+qGiRIJ332jPI62MmUKE3VHB2TJ1Ng5Vy6tfrz0htBihnzifl8RoRvTx80/oqgBfhbUM4MwbgWqOhimda0S1l5lO77Z+HqZ7/Vfh6QpmStvrHbOxcVLJiuR0RvMOOci4Su0ZwyZ48lkIpnWgW5CyQjkXxgAwukcfFRvm8DhsfTtma5dZtLvKsR/ut18/27wmyLM3wRkqGzqT52tj/HalyGF8Jh43oqkA5VIbcId1EgFMF8ixuPV5cFrTUqFUuo6hQuH5fkQ0TGtJwPN7Z98OQhNXJu16+Nvg0t9CwRUMr3NEakpzpNIH/0aNW1nwpvNgmDQIvnOb07w5JaggCtNSBiIgQA/p/R5MhfNboo4jz/Tw/Um7aX50YLb/1Sd92hVE+lyKgMb+k7K2oLS0sycldPXp7lz/cPXp7sIBJInk01Zo8p5s9rMLW/aRHKJMJhO1MiQ2pWBzoQbhWeI9bdQPr15vXYsQYcGXmA6xRhOH/U1Os4BHDYpEEmE9fQDsHktyIweHNb+Hg9+Gbl84krU1ueLu3NMO13XVeqtGNa3KQfjH6hyUn/WwYzSjgqxwWY4QI2rFxa2FO0JE5btLhO/DjNFAv5PwgfjXg8mNNLY7wgq+NS9pIxcDWxk40cIncB0gJObxib4UZEFxeWPK3A6CykC0hXOJNE6bypHx2UwSlUnS5cfhMviLSwwx0KLjFGVIkpyzIuXX/cXX9dl3jMuM3hG9xb9+ee0TgixkKtHRyenl2UnkudF/jAN5RctSb9ijZ+cnJ0mTFZ5052PvmD2kcARnSVfH63xlVHYdem0AgpgsJVQJUpAZuCxL68138CAnC13zJXFjArkYgZoQVkBN92SEJk5y6f+mhYS/KvirEvzbepKcJfdRV7BHuRY2HSH4aXDuQHNYyjFDglSCQGzc5FiA9cXW6JayIkNfTerVEk5w9oUoe2CBq4qAU6YkxnmoJ9p6u2GHW091U/Br4kJUSVLOgugdM/Cj9dnB0Dt4sNil7Hap2jmmsDXhJO3z31hwfo+0FsgFtja4zwVujc4z210nUeXt3X0SVcxqpxwCeunJN9VnPMDWBSa5h9l/GG64eqOFoT+1JAvz++P9CQOvqTRTZM5FNz95t1WFqXWw+kL7NhKDTUqXE27xV62hLCGMINPcuL/AfmXE9ZzeEWYiNFSCvPEhd+vkDWNZmmNg6buOXj9UEOE2i8EN1GZJQpVWaqxQcXIkFVbyaOO4cbv/wD1VlYGDclypWjQEGsaKlJl9EzTrHeQlCx53VbEZn4q7/5rWoKlLekvKNTgoWV7WhcMqoTMmyWvIsbdhFzmKYdrMpmnJ81sIxQj0e40FZgrS9P5FP1yRstR/L7mw5QU09zg0hAgklrbe3OiFkWkBQY+5TbL6ttbLu8KiaJRHWk83CcY7L7Qg3pXSleO8qAfVKg/0Zhl4ti/KQBtE828gCeMvAqg2q4Aym5HEhU9CS2/mtfy9TA9bkybJnh1Fw3FbgD1rl3OWk0qZ9g0T++4EPdHcoE3MYyd4iHrqcsKbcWIZeIUMo06tyWsnJkNXKo6VhhNqRIqe1loIwlS5jqGZ3APKGiJM6iJmRfCTXVnIXwWqs3anIT/xIFPSEy/JHRlSXLMxGeH5wBSEa4vMKzJ7BHc/u3p+I4B+XWCrgJMRDf+VjXUuCWYgp++ICKIgaErUihDWpCroxfkR2u0oHkE03t+qJEvCFBFaaC3xLUGyFp5ISlyqFpNUKo3ApmttzACyyUzdBl6DfDE/oK+afVTNsAJpCkntNtRrahqghxAz8YZclWu0Jkoz6n+ggpvUJi5uI5CUIehSYERo9OhKov/2w+n4/F+ck8Sb5t4t+h+QJsXFrSYE9hIYUo2BHQE0Dhua38p0T4RrUqHTl+jkxeX44vL0xJwaX799d3li6Li2isL8K1o0vWyCYGVbA5k3TjP74enJSfKbFRdLrR1yIuWs1sJbKl5VpHCfmb+lyP96epLp/522IBRS/XWcnWbjbCwr9dfT8dl44C5A6DNegWHuE2a0tcEUFZ73v1oPV0GWnEklsLLtiZgicz0TCcFmRbfJfLBcQVlBvhGTUFHw/CbICyio1MtfGFmFmX592m6NYrJuSGFSLqmvFRBaDJE7bQ1pnTC5MW606CAJuBO1vp6M8FlnxyywXNxvtzRs1YTNU//16t9fvxm8ZD9juUBPKiIWuAIbwuRazyibE1EJytRTvYoCr+wCKA627lQrX97mnYGrurv/qTeFc4sp6CppEplg7hFm7gTFBRQZ4ELvc4kU77MiDDS5cC5U32cbS1Rh47NvkhG9vKUKVVxK2u5wC/tBkRzeNEpU09EhcEq08krZbWZ3uQ+ohFykKJ8TdGwtlUkhi2rhQHE8itfRqbEuNY1/Ycs8EWcGoICuk+w0S/uu4EmPEWVrzrbp8k2eQ1e2FqpiPQsMM5724fmTpKne6CBvJRlvQG5Wx1WBtFPNkvm89uU+BmzqqbT5S6WiLFdGZP1b8MyWDAY/OeQd+8BVXxYN7MylVgKpkiC14s1Tf+xNWzHYjK9FjBEL0I5JM2Rr4NQkJxtPmOGLCOZ0jd7ZUgaQ9KAIwJ2U4zJDk2acE8PrYdVO+3YFx35K4Fw5eR9SOGqtmyfWD4GGydQh40tt1ZoAC64qc0yscH6rVaI5lepTh/HXJRan4/9tXknQ62I2DoGe2DTlXabcwmtXtjwM5i9efD3/fu5H4SgasQgVp+lNJai8vZE5F90j4azkeKBr7zOVtwigmGNuXNluRviEZPMsOJHzsoYz9NN42b5Kgta8di1VfpTetLUHYr1YWwdzo8/M+4zoFzhz0z9IAVC3DG5k0k5ljk23nhPNaKcuOJD03iwxZeVaL82sLhGd6UHDEQL8DGqBGcTXndtDiw8sJZ23REZDnISKAwCzwkbZSUIQtu4DGIqZwaD8w9Z6JbyiQafNlgfU+kjfNS/0Jij7WkofSY3TIUA3NwXM2/2ePp4f9oFNOKIjisIGaiEyZFIXbnoznvBqP39BB7Fffb0q7AgzXK7/8KaBixobnoggQRXIfC7I3FxoEjFWUwUi5kTd7DQ3X+Ab03ZpBo2MTK/gbXPUN0ubmh9vtH8PN1cDZ4t8U4TJdtlxl/IN3UdK0kDpbHUg38pgXJZ8hQiWaz02RUDtTNfGOehBBJPurbHKGlbtpQ490wPoBlrB2frE9DEoqBihZr2fJqeondWwHc8bF5Dsy39o9l8LF2Vh6GcAqivoy+wdBy7KY/ytzP+3kXBJlGHbsB3X/otruHf1Bj35evXmKcyl021BaO3JNTxsBh/26G/RA092XlX46kdTxt446Fqg57sN9ZOgSyzWtiebHuNPrWGksUTN5nbGE2Zl9OJYbmeT5ihzcX6SRvxB8064KpQhnitctjxRSRIk/aNNQnQA6q6R/kKjmK4VkXoLWg8K1yYALgpnG040tEnYxEL/mWgKJ+ktuoxychMHooiY91gqc2MZDBrCktb4XPJCc2yRxJLvg2VJFIbIgKm2LRLGxpzw2Lj4yf8wLPz6E+FhpD/HQqzD8iHcJF6Xrt1iUDjlTvYeHheapsipDkqFoatPcTOj4ZFaPduUEabaTeH3LdFxcBPtH3nNlFjfUMlv9g+tvzbQ0NX1RwiwJ1J77dx28MwJv4FkkWGY3nM2pwqCeaxAJVbwjy4+U4tzgPm0NTfphOWcqvUBcLzWqqEloVO948wO+Ln5ZdgW0B+0re2Qf0N2N33m0CvjB3dhcw+qWqylPk66MpURwuiOClWHP+ntgN5Abn47gd8D+sVFLoNMrSju1ype9AV7YV+feGdGRdbHOS9LkivnPw7rMSEk4H0i5VqfsRghBbnH1v0vl8m2yevdJLd15mn/TQKM6fqoRJ2zOh0So+9MiME7mlbaAJ24bye2kxRUh35l9Js799pSzrpsRUh/r3EJ2tAmP9vuXMDyQIzVJq1YvPE5ERYXZurx5rTwTlwz9Yrrb3rnvDO1g/J8dkuztqk/hu9SbqdXMmpcxrhCuFzhtbTFV6ZfmQ35GBeFIBAnpWzePpZRZvw6g6rBLiO/de1iWBPfD26SqJK5fw4yyE5auUTk/qLB/Zj7Z1v6twXPAfJEbVpNz2Z5x4WtqnOFvbbDhRWdUfGyBgU9gya++HESu+yuZuhuOXKlXNbnGNU3jUJXclDDF2iDCGLDQv1sY/6kN80P6KNvFWfueUui8gcvmVUlVrOUz3Cnef/YblDnwKInOWGKyxGqpzVT9QitKCv4SprU/qcpOVtgsbLFFSmKB8raJlj5Aefo4zX63wNDkp2xdA6XETkzvKTlkCy/oPssmVLMhpJzjQwK9ESQYoHVCJnvR9DAYSqL5JymSB0e7QwivSfZ6Ti7uO/cRUn5HZqwyBdUEWjUsBNV315c3Fyc35eoEG3KJlWqatmkX7582skm7bao0CAgJEqkkmDdCyIrzoJCsR1KUg2cbEnUgu+ZB/uzUpUDiAzAZHj0p7dfRujTx2v9/1+/JEgyo8nsLUvJU9dwU9FSZWAiA7N19gpoOz857ydoyovu9hyevf3FGkrAFg1JGmqSFtM/ZsVF2W0LdpByF5iaTrFL2M44O+0yddBn2vD0e//DZh5umsZ4T4K9rOm+nbOgSdd+c/Cezw0YZx17ehJav1POgSa/vvr8y2SEJm8/f9Z/Xf3y7mO6VOPt589dSbpXyll/blbJc1yCUfphjVqN23dK+emdvhZjN629fKgx7P6uhVSUKwDbIHgjAjclMw5MUlIFwpYqe/Ger5OtsEgm/V6Z84sA95k5EE8siokNezTJ4u6kg1kQi9aQI5ABW1hI1k5L5OG4wY86A8xSR60FviMIl4LgYo2k5i3jQsztlfu4qkoKtUW3BBGW88JmWDMSB4xKyoiElj13tpFTSTCD9MmtfaLulZCGJLeZZj92MtJ+r4mAY52tzTCHtUFJaZGcsckAsaz5JfrxvirU14ZihXeXOkmzcbgaAMejKWeYrm1DZqiU4kgSmxRvmI4KR2laj4Ki/ZXOaPC0L9bYH23cFG/cEnHcZzCdaa0EVzzne8rzX1wKiYWGejOuA+MsiNdRQQ5QuvHGgXHiw3GcEng2o3liH34mOV8uCStckgHsuPat2X9BlE15zdrL9BfEa5V+ULNbxlcsNQUhrM5U2CILUtzs6xYI6pN95pGNaQaPqL9w61vaHjp9Oc5Os9NsHNP7g21kJjsjsMPLIGa0hwnpeMrCMzGoNIkvuuajo8L0pjgkHRZimpJuo17HIQebDwdwxwnxdBxuRjwlO06J4gqXB5sPgGYnwzgy66VpQBTMO/rvrYVI0np28aKH2O84aSma7bOQ6i4FnuzxeVePh92wYmX+sftkeKlo1GTLBm0IE9q4g6jlisI1aKlqUbiUhK21JQU9t5pDXVgGjqXkOTVZh1QtUq2j1rxGWAhoIm6KfBQRBkBTIYSZsahAQcb9XjzecDD3OAftaZGE67DJR/X9yqbD8Wcx98gWz7S8kjvzzcfrdiP8NJO0b8rIQihxT2g+U6Z4Sa83tMk0vtlKkBn9RuTIl0lCPCXjMvvLRPPBBO4XM02y4cfdl/67e12B9B7X69N0t7HG67qVSR/G2xqS8YBeVrfq27ytT/dpZ9JxsB6JfGiZU5+TFconoVBGKuFLqEP6zD35O5F3np1nJ0enp+MjWwJ8XyLtHf0baY1kSOtKcSNIPkU/3qcfRq/4wN1bYzudNLz+aNoP2rrRuA5VazEPD9HiONpGtudueMI3Um7iKKhoMbECSiq8li6xzyBzjTX0UT9Imcp5RZuUgnnJp7gMmqk7ktvu+OFSC4tB3dY3JQbbGcFiXi97SsA/4DWaEquWfTsqqE6ShEkKYf9kV6GAb//++Kh8PDJXN+q/Xa3hxeN/3lfEDRhWQgsj64CE8gSU47IkhbvN3ib+CSTpkpY4XdMug2o9vzW23/s6gNr2pRYb8B0GYYUhqt0JuTfZJmrfCn2HCkD1VIXpTQbPR3aLKVcxg6Xfsz35SnGfbCuUrqMfhxs1rid2u3WiCp9BZ1ojMprUIGMr43Dv23ygPoN3RllhPbpOckFhFWT3ede+h+fQ6y9SMbw/s2uPdc64NuLu2qDUYptrT2wyusndKNdNR1/wCAfXDkF5yi2RmwolW/MXtA4wa8WCQEk/aT7d42pmzyMEkW8VEZSwHLznUkLLfq1JNExBCugeYdo+j/RHEUCtnexJhtuqO1q4WhhHICQVulWHdyRlc8gCtp2p25Q25uHZc/KMTGfkBJOL/Pzl83ExJS9nJ6fPz/Hpxdnz6fTF+Pz57CL4dnNez0CpuzGCQkosFc1NLfVAwyTMIHVc3vTvsLtoQxsxI7RbVzCYPO7E9orYQ+/huNU7GsgiAMs0WDYLCY0SQmLdlVYTB9Dkf7lrjCLIE2CmyX5ZOLulXFkRCdB68EoV17MeBvFrm0oF0Fvrvo8Bv5Evz7JxNjQ7oXWhl2PJUMoP4UsqTbGNNNFZfouwNmmNV4Mok3EfC/vwlkfcz5Th/DzQvVZuEg5+s5Ub2B53W9WijLX/18/vN6v6r5/ft/OTMXizSqKIfjoyYl7mekpG9n4QuCcSWw9WgMT1h25ic66Hzmb3RS3K7C8TzQKPWqPN0N8IMUHH5tqUoA3LakEYuSPCV2o2A7qnTbAQZNZhn+Ger3d1Wep1MFPjo6BDrhaa6M80+ompsPs7OPUMjH8+WShVycvj49VqlVndkuX8eF7TghwTdhyBipTPsSCQb52T44tsHL9o7gSwE7ZQy/KHmzDed6MX/8Y5F29svZ+QT83wrG6K92d7pOG4NOMoIlV63JmrJ5y0LEXCoKWGXmPFtXGFMASF1wjPsbYPeoPstSiRVLQsbduaJgXAhrI1v2h7RG9MUyCTWplmVRhqFT1Kc6StsDCs3py0XQp/bnoHxMaavQhyEo9bbxUT7e6ePh84Dutzi75+fr9P3Wdf5adl1DB2qtm7Ye3L8/OzY8PB//r7XyOO/kHxbqDViKj9JP81wPBWvMk8a6TVY6DycaoKAO5mAj/J5cSlPbhuJyC9AHL/0LtyyCXd7zeidmfl7picpM35MjWyd7F7xrQ7CLJV3EVb4N+Ba7YgR8aTP4mgte6cD29I8gvfPeKnGsVD+5wop39gr/jODJyfn6Uz987PuqSEddy7awcoqO5dCcvtj7M/b9drcWg0+6uD7nRHLEjtPSZQ7zAj+Q1BcU8588R4bdvTHCspN+UtwZLaALCp/xU2NfkG3SyD/iIhRij0wTCFyU4yjGs4UN7i+z0HY3F1QuYZBpzaMHRvjVoKJJ4IY4Za7y5DZFmphi4Ygnkj3o4GQuvA6OuzqD6WuE56rs2J6ab353KoIVuL1+/FpzOB58u4bc99PH5chCk72hjBM2gyqBfkh0mw9xWvepnvh6RGcSR2iXdV5/sR/9VCaW2kLroKS9kCe6++HAZKEt2j9vBaxxy541VRvlVA272ZjtzCq46nBCnJHQ5YQ3EUdrB8F4Rk8J25kYVAjW54L4v+hUJbyvCAC4gWrrGtbzhDi1FzPGOQILC29Jj+uqZxDG8OLmrRxJcfzh/6sXX5TN32j/rrJOI2uYeLdoQetAZHZ0v5cxl24M2hFmovTU9EpPgtYfQPkriBiiwxvWeK9ZYNZ0DHtWjoIA0St7uxHfMtYldyp97evAh5KJytl9DESL+SmOuvvpMSJCaAb8NlKVgvoAt65pzNDKO0L3RpZSD6rpXtFlqhfDApEF0pgcLfd5MVBqSTGI1Thy+Ji5pOBV9pJE526W/XJpDjwckFX9nk8xWZencSeFHbHZftobL2hLei58N3dm9dwHDT6yuz5NzFXsEg46SDttWsZu8t7Yvg+y+IOUAVS8vtuRXpEv+WuJRmeAzyg/4+Na2oZ1qXlO2HUH+/C8IKq3yI3Nl89MkXu+DcNbvn9ULw5cCmk2010UfD8JLOgcj6c8DuVQs5nIkHIf4ujDwM8/fg6C7mIz2HzVXf4TXfvuza98Z5lMT4wXXOASmdtyq2Td8g238AF8UNvHDj2+3YEL6588QJ60h76Vcz+CprXSiduEy6Z0Y2XRb9ycaO+26Lbt0Q3Uebi8o2wZYeWrZeR7wVQ7ADt+FoN6XYjsW+cBNEFnsvFu7Bvv1C4T7kfTdt99+y3UPCzrdob2A5uODTr2rT9M48Ofo2nPXsJ5qW8NrgfuzRNd49CIbdxG03ur+m1l1E6P6dAG7b5sC1Ce0Atnmm96xccKFuwFidN3WNmOULLhy+I7/LH8UWmTeLwttzd2tWZTr6PMg9uwG6ZeqWqQe8bLchZX91vO0q3AbXQ1+Ka5nW9p2Ke04lgFyxGQ8Z1Wa+t+8yd7ypf9/KmWHPq+GHC5kNzt7d4tttq+wfZSs518/SbT3VD0ydjZ2rv4W/JTA1z5sGeJHGboCicKY2b/rmo63TGxG92yRXvDgA8wczUPHC2NhJVPW+IibA9IkX6OvVmy4i/f+ywvueEANUDcQuMl4c4tbwEBkvSM8UDhUdwxAZaGiJqy4m8IYYN/ah0AUg0zgPKY4DvHkkmTehPYBCSuI1cP9/AAAA//83s451" } diff --git a/journalbeat/input/input.go b/journalbeat/input/input.go index 1a42d2cd986..42d8a0ea394 100644 --- a/journalbeat/input/input.go +++ b/journalbeat/input/input.go @@ -21,7 +21,7 @@ import ( "fmt" "sync" - uuid "github.com/satori/go.uuid" + "github.com/gofrs/uuid" "github.com/elastic/beats/journalbeat/checkpoint" "github.com/elastic/beats/journalbeat/reader" @@ -56,18 +56,21 @@ func New( return nil, err } - id := uuid.NewV4() + id, err := uuid.NewV4() + if err != nil { + return nil, fmt.Errorf("error while generating ID for input: %v", err) + } + logger := logp.NewLogger("input").With("id", id) var readers []*reader.Reader if len(config.Paths) == 0 { cfg := reader.Config{ - Path: reader.LocalSystemJournalID, // used to identify the state in the registry - Backoff: config.Backoff, - MaxBackoff: config.MaxBackoff, - BackoffFactor: config.BackoffFactor, - Seek: config.Seek, - Matches: config.Matches, + Path: reader.LocalSystemJournalID, // used to identify the state in the registry + Backoff: config.Backoff, + MaxBackoff: config.MaxBackoff, + Seek: config.Seek, + Matches: config.Matches, } state := states[reader.LocalSystemJournalID] @@ -80,12 +83,11 @@ func New( for _, p := range config.Paths { cfg := reader.Config{ - Path: p, - Backoff: config.Backoff, - MaxBackoff: config.MaxBackoff, - BackoffFactor: config.BackoffFactor, - Seek: config.Seek, - Matches: config.Matches, + Path: p, + Backoff: config.Backoff, + MaxBackoff: config.MaxBackoff, + Seek: config.Seek, + Matches: config.Matches, } state := states[p] r, err := reader.New(cfg, done, state, logger) @@ -99,7 +101,6 @@ func New( if err != nil { return nil, err } - logp.Info(">>> %v", config.EventMetadata) logger.Debugf("New input is created for paths %v", config.Paths) @@ -137,46 +138,50 @@ func (i *Input) Run() { i.publishAll(client) } +// publishAll reads events from all readers and publishes them. func (i *Input) publishAll(client beat.Client) { out := make(chan *beat.Event) defer close(out) var wg sync.WaitGroup - merge := func(in chan *beat.Event) { + defer wg.Wait() + for _, r := range i.readers { wg.Add(1) - - go func(c chan *beat.Event) { + r := r + go func() { defer wg.Done() + for { select { case <-i.done: return - case v, ok := <-c: - if !ok { - return + default: + } + + event, err := r.Next() + if event == nil { + if err != nil { + i.logger.Errorf("Error while reading event: %v", err) } - out <- v + continue } - } - }(in) - } - // merge channels of readers into a single output channel - for _, r := range i.readers { - c := r.Follow() - merge(c) + select { + case <-i.done: + case out <- event: + } + } + }() } -loop: for { select { case <-i.done: - break loop + return case e := <-out: client.Publish(*e) } } - wg.Wait() } // Stop stops all readers of the input. diff --git a/journalbeat/journalbeat.reference.yml b/journalbeat/journalbeat.reference.yml index 27c12f0c4a2..2fcc03bf832 100644 --- a/journalbeat/journalbeat.reference.yml +++ b/journalbeat/journalbeat.reference.yml @@ -20,8 +20,6 @@ journalbeat.inputs: # The number of seconds to wait before trying to read again from journals. #backoff: 1s - # Multiplier of backoff value. - #backoff_factor: 2 # The maximum number of seconds to wait before attempting to read again from journals. #max_backoff: 60s @@ -47,8 +45,6 @@ journalbeat.inputs: # The number of seconds to wait before trying to read again from journals. #backoff: 1s - # Multiplier of backoff value. - #backoff_factor: 2 # The maximum number of seconds to wait before attempting to read again from journals. #max_backoff: 60s @@ -501,11 +497,11 @@ output.elasticsearch: # and retry until all events are published. Set max_retries to a value less # than 0 to retry until all events are published. The default is 3. #max_retries: 3 - + # The maximum number of events to bulk in a single Logstash request. The # default is 2048. #bulk_max_size: 2048 - + # The number of seconds to wait for responses from the Logstash server before # timing out. The default is 30s. #timeout: 30s @@ -1060,6 +1056,13 @@ logging.files: # Must be a valid Unix-style file permissions mask expressed in octal notation. #permissions: 0600 + # Enable log file rotation on time intervals in addition to size-based rotation. + # Intervals must be at least 1s. Values of 1m, 1h, 24h, 7*24h, 30*24h, and 365*24h + # are boundary-aligned with minutes, hours, days, weeks, months, and years as + # reported by the local system clock. All other intervals are calculated from the + # unix epoch. Defaults to disabled. + #interval: 0 + # Set to true to log messages in json format. #logging.json: false diff --git a/journalbeat/journalbeat.yml b/journalbeat/journalbeat.yml index 617b70e3832..81468062a2d 100644 --- a/journalbeat/journalbeat.yml +++ b/journalbeat/journalbeat.yml @@ -20,8 +20,6 @@ journalbeat.inputs: # The number of seconds to wait before trying to read again from journals. #backoff: 1s - # Multiplier of backoff value. - #backoff_factor: 2 # The maximum number of seconds to wait before attempting to read again from journals. #max_backoff: 60s @@ -47,8 +45,6 @@ journalbeat.inputs: # The number of seconds to wait before trying to read again from journals. #backoff: 1s - # Multiplier of backoff value. - #backoff_factor: 2 # The maximum number of seconds to wait before attempting to read again from journals. #max_backoff: 60s @@ -99,6 +95,11 @@ setup.kibana: # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601 #host: "localhost:5601" + # Kibana Space ID + # ID of the Kibana Space into which the dashboards should be loaded. By default, + # the Default Space will be used. + #space.id: + #============================= Elastic Cloud ================================== # These settings simplify using journalbeat with the Elastic Cloud (https://cloud.elastic.co/). diff --git a/journalbeat/reader/fields.go b/journalbeat/reader/fields.go index 10ee1fbd63e..0744f6fda49 100644 --- a/journalbeat/reader/fields.go +++ b/journalbeat/reader/fields.go @@ -19,69 +19,75 @@ package reader import "github.com/coreos/go-systemd/sdjournal" +type fieldConversion struct { + name string + isInteger bool + dropped bool +} + var ( - journaldEventFields = map[string]string{ + journaldEventFields = map[string]fieldConversion{ // provided by systemd journal - "COREDUMP_UNIT": "journald.coredump.unit", - "COREDUMP_USER_UNIT": "journald.coredump.user_unit", - "OBJECT_AUDIT_LOGINUID": "journald.object.audit.login_uid", - "OBJECT_AUDIT_SESSION": "journald.object.audit.session", - "OBJECT_CMDLINE": "journald.object.cmd", - "OBJECT_COMM": "journald.object.name", - "OBJECT_EXE": "journald.object.executable", - "OBJECT_GID": "journald.object.gid", - "OBJECT_PID": "journald.object.pid", - "OBJECT_SYSTEMD_OWNER_UID": "journald.object.systemd.owner_uid", - "OBJECT_SYSTEMD_SESSION": "journald.object.systemd.session", - "OBJECT_SYSTEMD_UNIT": "journald.object.systemd.unit", - "OBJECT_SYSTEMD_USER_UNIT": "journald.object.systemd.user_unit", - "OBJECT_UID": "journald.object.uid", - "_KERNEL_DEVICE": "journald.kernel.device", - "_KERNEL_SUBSYSTEM": "journald.kernel.subsystem", - "_SYSTEMD_INVOCATION_ID": "systemd.invocation_id", - "_SYSTEMD_USER_SLICE": "systemd.user_slice", - "_UDEV_DEVLINK": "journald.kernel.device_symlinks", // TODO aggregate multiple elements - "_UDEV_DEVNODE": "journald.kernel.device_node_path", - "_UDEV_SYSNAME": "journald.kernel.device_name", - sdjournal.SD_JOURNAL_FIELD_AUDIT_LOGINUID: "process.audit.login_uid", - sdjournal.SD_JOURNAL_FIELD_AUDIT_SESSION: "process.audit.session", - sdjournal.SD_JOURNAL_FIELD_BOOT_ID: "host.boot_id", - sdjournal.SD_JOURNAL_FIELD_CAP_EFFECTIVE: "process.capabilites", - sdjournal.SD_JOURNAL_FIELD_CMDLINE: "process.cmd", - sdjournal.SD_JOURNAL_FIELD_CODE_FILE: "journald.code.file", - sdjournal.SD_JOURNAL_FIELD_CODE_FUNC: "journald.code.func", - sdjournal.SD_JOURNAL_FIELD_CODE_LINE: "journald.code.line", - sdjournal.SD_JOURNAL_FIELD_COMM: "process.name", - sdjournal.SD_JOURNAL_FIELD_EXE: "process.executable", - sdjournal.SD_JOURNAL_FIELD_GID: "process.uid", - sdjournal.SD_JOURNAL_FIELD_HOSTNAME: "host.name", - sdjournal.SD_JOURNAL_FIELD_MACHINE_ID: "host.id", - sdjournal.SD_JOURNAL_FIELD_MESSAGE: "message", - sdjournal.SD_JOURNAL_FIELD_PID: "process.pid", - sdjournal.SD_JOURNAL_FIELD_PRIORITY: "syslog.priority", - sdjournal.SD_JOURNAL_FIELD_SYSLOG_FACILITY: "syslog.facility", - sdjournal.SD_JOURNAL_FIELD_SYSLOG_IDENTIFIER: "syslog.identifier", - sdjournal.SD_JOURNAL_FIELD_SYSLOG_PID: "syslog.pid", - sdjournal.SD_JOURNAL_FIELD_SYSTEMD_CGROUP: "systemd.cgroup", - sdjournal.SD_JOURNAL_FIELD_SYSTEMD_OWNER_UID: "systemd.owner_uid", - sdjournal.SD_JOURNAL_FIELD_SYSTEMD_SESSION: "systemd.session", - sdjournal.SD_JOURNAL_FIELD_SYSTEMD_SLICE: "systemd.slice", - sdjournal.SD_JOURNAL_FIELD_SYSTEMD_UNIT: "systemd.unit", - sdjournal.SD_JOURNAL_FIELD_SYSTEMD_USER_UNIT: "systemd.user_unit", - sdjournal.SD_JOURNAL_FIELD_TRANSPORT: "systemd.transport", - sdjournal.SD_JOURNAL_FIELD_UID: "process.uid", + "COREDUMP_UNIT": fieldConversion{"journald.coredump.unit", false, false}, + "COREDUMP_USER_UNIT": fieldConversion{"journald.coredump.user_unit", false, false}, + "OBJECT_AUDIT_LOGINUID": fieldConversion{"journald.object.audit.login_uid", true, false}, + "OBJECT_AUDIT_SESSION": fieldConversion{"journald.object.audit.session", true, false}, + "OBJECT_CMDLINE": fieldConversion{"journald.object.cmd", false, false}, + "OBJECT_COMM": fieldConversion{"journald.object.name", false, false}, + "OBJECT_EXE": fieldConversion{"journald.object.executable", false, false}, + "OBJECT_GID": fieldConversion{"journald.object.gid", true, false}, + "OBJECT_PID": fieldConversion{"journald.object.pid", true, false}, + "OBJECT_SYSTEMD_OWNER_UID": fieldConversion{"journald.object.systemd.owner_uid", true, false}, + "OBJECT_SYSTEMD_SESSION": fieldConversion{"journald.object.systemd.session", false, false}, + "OBJECT_SYSTEMD_UNIT": fieldConversion{"journald.object.systemd.unit", false, false}, + "OBJECT_SYSTEMD_USER_UNIT": fieldConversion{"journald.object.systemd.user_unit", false, false}, + "OBJECT_UID": fieldConversion{"journald.object.uid", true, false}, + "_KERNEL_DEVICE": fieldConversion{"journald.kernel.device", false, false}, + "_KERNEL_SUBSYSTEM": fieldConversion{"journald.kernel.subsystem", false, false}, + "_SYSTEMD_INVOCATION_ID": fieldConversion{"systemd.invocation_id", false, false}, + "_SYSTEMD_USER_SLICE": fieldConversion{"systemd.user_slice", false, false}, + "_UDEV_DEVLINK": fieldConversion{"journald.kernel.device_symlinks", false, false}, // TODO aggregate multiple elements + "_UDEV_DEVNODE": fieldConversion{"journald.kernel.device_node_path", false, false}, + "_UDEV_SYSNAME": fieldConversion{"journald.kernel.device_name", false, false}, + sdjournal.SD_JOURNAL_FIELD_AUDIT_LOGINUID: fieldConversion{"process.audit.login_uid", true, false}, + sdjournal.SD_JOURNAL_FIELD_AUDIT_SESSION: fieldConversion{"process.audit.session", false, false}, + sdjournal.SD_JOURNAL_FIELD_BOOT_ID: fieldConversion{"host.boot_id", false, false}, + sdjournal.SD_JOURNAL_FIELD_CAP_EFFECTIVE: fieldConversion{"process.capabilites", false, false}, + sdjournal.SD_JOURNAL_FIELD_CMDLINE: fieldConversion{"process.cmd", false, false}, + sdjournal.SD_JOURNAL_FIELD_CODE_FILE: fieldConversion{"journald.code.file", false, false}, + sdjournal.SD_JOURNAL_FIELD_CODE_FUNC: fieldConversion{"journald.code.func", false, false}, + sdjournal.SD_JOURNAL_FIELD_CODE_LINE: fieldConversion{"journald.code.line", true, false}, + sdjournal.SD_JOURNAL_FIELD_COMM: fieldConversion{"process.name", false, false}, + sdjournal.SD_JOURNAL_FIELD_EXE: fieldConversion{"process.executable", false, false}, + sdjournal.SD_JOURNAL_FIELD_GID: fieldConversion{"process.uid", true, false}, + sdjournal.SD_JOURNAL_FIELD_HOSTNAME: fieldConversion{"host.name", false, false}, + sdjournal.SD_JOURNAL_FIELD_MACHINE_ID: fieldConversion{"host.id", false, false}, + sdjournal.SD_JOURNAL_FIELD_MESSAGE: fieldConversion{"message", false, false}, + sdjournal.SD_JOURNAL_FIELD_PID: fieldConversion{"process.pid", true, false}, + sdjournal.SD_JOURNAL_FIELD_PRIORITY: fieldConversion{"syslog.priority", true, false}, + sdjournal.SD_JOURNAL_FIELD_SYSLOG_FACILITY: fieldConversion{"syslog.facility", true, false}, + sdjournal.SD_JOURNAL_FIELD_SYSLOG_IDENTIFIER: fieldConversion{"syslog.identifier", false, false}, + sdjournal.SD_JOURNAL_FIELD_SYSLOG_PID: fieldConversion{"syslog.pid", true, false}, + sdjournal.SD_JOURNAL_FIELD_SYSTEMD_CGROUP: fieldConversion{"systemd.cgroup", false, false}, + sdjournal.SD_JOURNAL_FIELD_SYSTEMD_OWNER_UID: fieldConversion{"systemd.owner_uid", true, false}, + sdjournal.SD_JOURNAL_FIELD_SYSTEMD_SESSION: fieldConversion{"systemd.session", false, false}, + sdjournal.SD_JOURNAL_FIELD_SYSTEMD_SLICE: fieldConversion{"systemd.slice", false, false}, + sdjournal.SD_JOURNAL_FIELD_SYSTEMD_UNIT: fieldConversion{"systemd.unit", false, false}, + sdjournal.SD_JOURNAL_FIELD_SYSTEMD_USER_UNIT: fieldConversion{"systemd.user_unit", false, false}, + sdjournal.SD_JOURNAL_FIELD_TRANSPORT: fieldConversion{"systemd.transport", false, false}, + sdjournal.SD_JOURNAL_FIELD_UID: fieldConversion{"process.uid", true, false}, // docker journald fields from: https://docs.docker.com/config/containers/logging/journald/ - "CONTAINER_ID": "conatiner.id_truncated", - "CONTAINER_ID_FULL": "container.id", - "CONTAINER_NAME": "container.name", - "CONTAINER_TAG": "container.image.tag", - "CONTAINER_PARTIAL_MESSAGE": "container.partial", + "CONTAINER_ID": fieldConversion{"conatiner.id_truncated", false, false}, + "CONTAINER_ID_FULL": fieldConversion{"container.id", false, false}, + "CONTAINER_NAME": fieldConversion{"container.name", false, false}, + "CONTAINER_TAG": fieldConversion{"container.image.tag", false, false}, + "CONTAINER_PARTIAL_MESSAGE": fieldConversion{"container.partial", false, false}, // dropped fields - sdjournal.SD_JOURNAL_FIELD_MONOTONIC_TIMESTAMP: "", // saved in the registry - sdjournal.SD_JOURNAL_FIELD_SOURCE_REALTIME_TIMESTAMP: "", // saved in the registry - sdjournal.SD_JOURNAL_FIELD_CURSOR: "", // saved in the registry - "_SOURCE_MONOTONIC_TIMESTAMP": "", // received timestamp stored in @timestamp + sdjournal.SD_JOURNAL_FIELD_MONOTONIC_TIMESTAMP: fieldConversion{"", false, true}, // saved in the registry + sdjournal.SD_JOURNAL_FIELD_SOURCE_REALTIME_TIMESTAMP: fieldConversion{"", false, true}, // saved in the registry + sdjournal.SD_JOURNAL_FIELD_CURSOR: fieldConversion{"", false, true}, // saved in the registry + "_SOURCE_MONOTONIC_TIMESTAMP": fieldConversion{"", false, true}, // received timestamp stored in @timestamp } ) diff --git a/journalbeat/reader/journal.go b/journalbeat/reader/journal.go index 7cfcb1802b8..756b2087ec2 100644 --- a/journalbeat/reader/journal.go +++ b/journalbeat/reader/journal.go @@ -21,6 +21,7 @@ import ( "fmt" "io" "os" + "strconv" "strings" "time" @@ -51,8 +52,6 @@ type Config struct { // Backoff is the current interval to wait before // attemting to read again from the journal. Backoff time.Duration - // BackoffFactor is the multiplier of Backoff. - BackoffFactor int // Matches store the key value pairs to match entries. Matches []string } @@ -61,9 +60,9 @@ type Config struct { type Reader struct { journal *sdjournal.Journal config Config - changes chan int done chan struct{} logger *logp.Logger + backoff *common.Backoff } // New creates a new journal reader and moves the FP to the configured position. @@ -73,8 +72,6 @@ func New(c Config, done chan struct{}, state checkpoint.JournalState, logger *lo return nil, errors.Wrap(err, "failed to open file") } - logger = logger.With("path", c.Path) - var j *sdjournal.Journal if f.IsDir() { j, err = sdjournal.NewJournalFromDir(c.Path) @@ -88,25 +85,10 @@ func New(c Config, done chan struct{}, state checkpoint.JournalState, logger *lo } } - err = setupMatches(j, c.Matches) - if err != nil { - return nil, err - } - - r := &Reader{ - journal: j, - changes: make(chan int), - config: c, - done: done, - logger: logger, - } - r.seek(state.Cursor) - - instance.AddJournalToMonitor(c.Path, j) + l := logger.With("path", c.Path) + l.Debug("New journal is opened for reading") - r.logger.Debug("New journal is opened for reading") - - return r, nil + return newReader(l, done, c, j, state) } // NewLocal creates a reader to read form the local journal and moves the FP @@ -117,25 +99,28 @@ func NewLocal(c Config, done chan struct{}, state checkpoint.JournalState, logge return nil, errors.Wrap(err, "failed to open local journal") } - c.Path = LocalSystemJournalID - logger = logger.With("path", "local") - logger.Debug("New local journal is opened for reading") + l := logger.With("path", "local") + l.Debug("New local journal is opened for reading") - err = setupMatches(j, c.Matches) + return newReader(l, done, c, j, state) +} + +func newReader(logger *logp.Logger, done chan struct{}, c Config, journal *sdjournal.Journal, state checkpoint.JournalState) (*Reader, error) { + err := setupMatches(journal, c.Matches) if err != nil { return nil, err } r := &Reader{ - journal: j, - changes: make(chan int), + journal: journal, config: c, done: done, logger: logger, + backoff: common.NewBackoff(done, c.Backoff, c.MaxBackoff), } r.seek(state.Cursor) - instance.AddJournalToMonitor(c.Path, j) + instance.AddJournalToMonitor(c.Path, journal) return r, nil } @@ -148,8 +133,8 @@ func setupMatches(j *sdjournal.Journal, matches []string) error { } var p string - for journalKey, eventKey := range journaldEventFields { - if elems[0] == eventKey { + for journalKey, eventField := range journaldEventFields { + if elems[0] == eventField.name { p = journalKey + "=" + elems[1] } } @@ -197,72 +182,45 @@ func (r *Reader) seek(cursor string) { } } -// Follow reads entries from journals. -func (r *Reader) Follow() chan *beat.Event { - out := make(chan *beat.Event) - go r.readEntriesFromJournal(out) - - return out -} - -func (r *Reader) readEntriesFromJournal(entries chan *beat.Event) { - defer close(entries) - -process: +// Next waits until a new event shows up and returns it. +// It blocks until an event is returned or an error occurs. +func (r *Reader) Next() (*beat.Event, error) { for { select { case <-r.done: - return + return nil, nil default: - err := r.readUntilNotNull(entries) + event, err := r.readEvent() if err != nil { - r.logger.Error("Unexpected error while reading from journal: %v", err) + return nil, err } - } - for { - go r.stopOrWait() - - select { - case <-r.done: - return - case e := <-r.changes: - switch e { - case sdjournal.SD_JOURNAL_NOP: - r.wait() - case sdjournal.SD_JOURNAL_APPEND, sdjournal.SD_JOURNAL_INVALIDATE: - continue process - default: - if e < 0 { - //r.logger.Error("Unexpected error: %v", syscall.Errno(-e)) - } - r.wait() - } + if event == nil { + r.backoff.Wait() + continue } + + r.backoff.Reset() + return event, nil } } } -func (r *Reader) readUntilNotNull(entries chan<- *beat.Event) error { +func (r *Reader) readEvent() (*beat.Event, error) { n, err := r.journal.Next() if err != nil && err != io.EOF { - return err + return nil, err } - for n != 0 { + for n == 1 { entry, err := r.journal.GetEntry() if err != nil { - return err + return nil, err } event := r.toEvent(entry) - entries <- event - - n, err = r.journal.Next() - if err != nil && err != io.EOF { - return err - } + return event, nil } - return nil + return nil, nil } // toEvent creates a beat.Event from journal entries. @@ -270,14 +228,13 @@ func (r *Reader) toEvent(entry *sdjournal.JournalEntry) *beat.Event { fields := common.MapStr{} custom := common.MapStr{} - for k, v := range entry.Fields { - if kk, ok := journaldEventFields[k]; !ok { - normalized := strings.ToLower(strings.TrimLeft(k, "_")) + for entryKey, v := range entry.Fields { + if fieldConversionInfo, ok := journaldEventFields[entryKey]; !ok { + normalized := strings.ToLower(strings.TrimLeft(entryKey, "_")) custom.Put(normalized, v) - } else { - if isKept(kk) { - fields.Put(kk, v) - } + } else if !fieldConversionInfo.dropped { + value := r.convertNamedField(fieldConversionInfo, v) + fields.Put(fieldConversionInfo.name, value) } } @@ -303,32 +260,16 @@ func (r *Reader) toEvent(entry *sdjournal.JournalEntry) *beat.Event { return &event } -func isKept(key string) bool { - return key != "" -} - -// stopOrWait waits for a journal event. -func (r *Reader) stopOrWait() { - select { - case <-r.done: - case r.changes <- r.journal.Wait(100 * time.Millisecond): - } -} - -func (r *Reader) wait() { - select { - case <-r.done: - return - case <-time.After(r.config.Backoff): - } - - if r.config.Backoff < r.config.MaxBackoff { - r.config.Backoff = r.config.Backoff * time.Duration(r.config.BackoffFactor) - if r.config.Backoff > r.config.MaxBackoff { - r.config.Backoff = r.config.MaxBackoff +func (r *Reader) convertNamedField(fc fieldConversion, value string) interface{} { + if fc.isInteger { + v, err := strconv.ParseInt(value, 10, 64) + if err != nil { + r.logger.Debugf("Failed to convert field: %s \"%v\" to int: %v", fc.name, value, err) + return value } - r.logger.Debugf("Increasing backoff time to: %v factor: %v", r.config.Backoff, r.config.BackoffFactor) + return v } + return value } // Close closes the underlying journal reader. diff --git a/vendor/vendor.json b/vendor/vendor.json index 4596c110942..701ee22306d 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -129,8 +129,10 @@ { "checksumSHA1": "cEszpxh1szqTb440ze4hm/Vfm40=", "path": "github.com/coreos/go-systemd/sdjournal", - "revision": "eee3db372b31153ca0b90702e165948699803fd0", - "revisionTime": "2018-08-28T14:03:53Z" + "revision": "39ca1b05acc7ad1220e09f133283b8859a8b71ab", + "revisionTime": "2018-05-11T13:34:05Z", + "version": "v17", + "versionExact": "v17" }, { "checksumSHA1": "nwll5KjH9SKrWxB45OmN09wdipI=",