From ac35a91f18cf7117bd84ffbde94d1f4708d0f3a3 Mon Sep 17 00:00:00 2001 From: ruflin Date: Fri, 12 May 2017 10:17:14 +0200 Subject: [PATCH] Rename input_type to type in config and input_type to prospector.type in event Document type was removed recently because _type does not exist anymore in Elasticsearch. As input_type is actually the type of the prospector it makes more sense to have it as type config options instead of type. Renaming it in the config means it should also be renamed in the event. In the event the field was renamed to `prospector.type`. This change is on the config side backward compatible as input_type was only deprecated. On the event side the old field does not exist anymore. * Cleanup expected test json files. Indentation was standardised and `input_type` replaced by `prospector.type`. * Update changelog * Add system tests for deprecated message --- CHANGELOG.asciidoc | 3 + filebeat/_meta/common.full.p2.yml | 4 +- filebeat/_meta/common.p2.yml | 2 +- filebeat/_meta/fields.common.yml | 4 +- filebeat/config/config.go | 13 +- filebeat/docs/fields.asciidoc | 4 +- filebeat/docs/filebeat-filtering.asciidoc | 4 +- filebeat/docs/getting-started.asciidoc | 6 +- filebeat/docs/how-filebeat-works.asciidoc | 14 +- filebeat/docs/load-balancing.asciidoc | 4 +- filebeat/docs/migration.asciidoc | 14 +- filebeat/docs/multiple-prospectors.asciidoc | 8 +- .../configuration/filebeat-options.asciidoc | 8 +- .../reload-configuration.asciidoc | 14 +- filebeat/docs/system-module-note.asciidoc | 4 +- filebeat/filebeat.full.yml | 4 +- filebeat/filebeat.yml | 2 +- .../module/apache2/access/config/access.yml | 2 +- .../access/test/test.log-expected.json | 269 +++---- .../module/apache2/error/config/error.yml | 2 +- .../apache2/error/test/test.log-expected.json | 185 ++--- filebeat/module/auditd/log/config/log.yml | 2 +- .../auditd/log/test/test.log-expected.json | 10 +- filebeat/module/icinga/debug/config/debug.yml | 2 +- .../icinga/debug/test/test.log-expected.json | 221 +++--- filebeat/module/icinga/main/config/main.yml | 2 +- .../icinga/main/test/test.log-expected.json | 221 +++--- .../module/icinga/startup/config/startup.yml | 2 +- .../startup/test/test.log-expected.json | 148 ++-- filebeat/module/mysql/error/config/error.yml | 2 +- .../module/mysql/slowlog/config/slowlog.yml | 2 +- .../nginx/access/config/nginx-access.yml | 2 +- .../module/nginx/error/config/nginx-error.yml | 2 +- filebeat/module/system/auth/config/auth.yml | 2 +- .../system/auth/test/test.log-expected.json | 706 +++++++++--------- .../module/system/syslog/config/syslog.yml | 2 +- .../darwin-syslog-sample.log-expected.json | 171 ++--- filebeat/prospector/config.go | 12 +- filebeat/prospector/log/config.go | 14 +- filebeat/prospector/log/config_test.go | 2 +- filebeat/prospector/log/harvester.go | 7 +- filebeat/prospector/log/log.go | 1 - filebeat/prospector/log/prospector.go | 4 +- filebeat/prospector/prospector.go | 10 +- .../scripts/module/fileset/config/config.yml | 2 +- filebeat/tests/load/filebeat.yml | 2 +- .../tests/open-file-handlers/filebeat.yml | 6 +- filebeat/tests/system/config/filebeat.yml.j2 | 3 +- filebeat/tests/system/test_deprecated.py | 40 + filebeat/tests/system/test_prospector.py | 4 +- filebeat/tests/system/test_reload.py | 2 +- filebeat/tests/system/test_shutdown.py | 2 +- 52 files changed, 1135 insertions(+), 1043 deletions(-) create mode 100644 filebeat/tests/system/test_deprecated.py diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 6c9a3c4513af..46c893e8cd1d 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -15,6 +15,7 @@ https://github.com/elastic/beats/compare/v6.0.0-alpha1...master[Check the HEAD d *Affecting all Beats* *Filebeat* +- Rename `input_type` field to `prospector.type` {pull}4294[4294] *Heartbeat* @@ -66,6 +67,8 @@ https://github.com/elastic/beats/compare/v6.0.0-alpha1...master[Check the HEAD d *Affecting all Beats* *Filebeat* +- Deprecate `input_type` prospector config. Use `type` config option instead. {pull}4294[4294] + *Heartbeat* diff --git a/filebeat/_meta/common.full.p2.yml b/filebeat/_meta/common.full.p2.yml index 1a7b64c46589..6cccbd8f7e8e 100644 --- a/filebeat/_meta/common.full.p2.yml +++ b/filebeat/_meta/common.full.p2.yml @@ -14,7 +14,7 @@ filebeat.prospectors: # * stdin: Reads the standard in #------------------------------ Log prospector -------------------------------- -- input_type: log +- type: log # Change to true to enable this prospector configuration. enabled: false @@ -208,7 +208,7 @@ filebeat.prospectors: #----------------------------- Stdin prospector ------------------------------- # Configuration to use stdin input -#- input_type: stdin +#- type: stdin #========================= Filebeat global options ============================ diff --git a/filebeat/_meta/common.p2.yml b/filebeat/_meta/common.p2.yml index 80d7d03cc69d..6afa1b2d946d 100644 --- a/filebeat/_meta/common.p2.yml +++ b/filebeat/_meta/common.p2.yml @@ -9,7 +9,7 @@ filebeat.prospectors: # you can use different prospectors for various configurations. # Below are the prospector specific configurations. -- input_type: log +- type: log # Change to true to enable this prospector configuration. enabled: false diff --git a/filebeat/_meta/fields.common.yml b/filebeat/_meta/fields.common.yml index 0c515594660a..14258c621497 100644 --- a/filebeat/_meta/fields.common.yml +++ b/filebeat/_meta/fields.common.yml @@ -23,10 +23,10 @@ description: > The content of the line read from the log file. - - name: input_type + - name: prospector.type required: true description: > - The input type from which the event was generated. This field is set to the value specified for the `input_type` option in the prospector section of the Filebeat config file. + The prospector type from which the event was generated. This field is set to the value specified for the `type` option in the prospector section of the Filebeat config file. - name: read_timestamp description: > diff --git a/filebeat/config/config.go b/filebeat/config/config.go index c66be16db10f..07cac05abc2e 100644 --- a/filebeat/config/config.go +++ b/filebeat/config/config.go @@ -15,7 +15,7 @@ import ( // Defaults for config variables which are not set const ( - DefaultInputType = "log" + DefaultType = "log" ) type Config struct { @@ -39,15 +39,16 @@ var ( } ) +// Contains available prospector types const ( - LogInputType = "log" - StdinInputType = "stdin" + LogType = "log" + StdinType = "stdin" ) // List of valid input types -var ValidInputType = map[string]struct{}{ - StdinInputType: {}, - LogInputType: {}, +var ValidType = map[string]struct{}{ + StdinType: {}, + LogType: {}, } // getConfigFiles returns list of config files. diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index d8b6c07bf7d5..14c5059373db 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -768,11 +768,11 @@ The content of the line read from the log file. [float] -=== input_type +=== prospector.type required: True -The input type from which the event was generated. This field is set to the value specified for the `input_type` option in the prospector section of the Filebeat config file. +The prospector type from which the event was generated. This field is set to the value specified for the `type` option in the prospector section of the Filebeat config file. [float] diff --git a/filebeat/docs/filebeat-filtering.asciidoc b/filebeat/docs/filebeat-filtering.asciidoc index 113e8c91c1cd..00c345948296 100644 --- a/filebeat/docs/filebeat-filtering.asciidoc +++ b/filebeat/docs/filebeat-filtering.asciidoc @@ -93,7 +93,9 @@ The resulting output looks something like this: "inner": { "data": "value" }, - "input_type": "log", + "prospector": { + "type": "log", + }, "offset": 55, "outer": "value", "source": "input.json", diff --git a/filebeat/docs/getting-started.asciidoc b/filebeat/docs/getting-started.asciidoc index 3c8ba64ab698..7bf843d4953d 100644 --- a/filebeat/docs/getting-started.asciidoc +++ b/filebeat/docs/getting-started.asciidoc @@ -137,7 +137,7 @@ started experience for common log formats. See <> to learn how to get started with modules. If you use Filebeat modules to get started, you can skip the content in this section, including the remaining getting started steps, and go directly to the <> -page. +page. To configure Filebeat manually, you edit the configuration file. For rpm and deb, you'll find the configuration file at `/etc/filebeat/filebeat.yml`. For mac and @@ -155,7 +155,7 @@ default values for most configuration options. [source,yaml] ------------------------------------------------------------------------------------- filebeat.prospectors: -- input_type: log +- type: log paths: - /var/log/*.log #- c:\programdata\elasticsearch\logs\* @@ -170,7 +170,7 @@ For the most basic Filebeat configuration, you can define a single prospector wi [source,yaml] ------------------------------------------------------------------------------------- filebeat.prospectors: -- input_type: log +- type: log paths: - /var/log/*.log ------------------------------------------------------------------------------------- diff --git a/filebeat/docs/how-filebeat-works.asciidoc b/filebeat/docs/how-filebeat-works.asciidoc index 3130b2c4077a..1f7faea7739e 100644 --- a/filebeat/docs/how-filebeat-works.asciidoc +++ b/filebeat/docs/how-filebeat-works.asciidoc @@ -1,9 +1,9 @@ [[how-filebeat-works]] == How Filebeat Works -In this topic, you learn about the key building blocks of Filebeat and how they work together. Understanding these concepts will help you make informed decisions about configuring Filebeat for specific use cases. +In this topic, you learn about the key building blocks of Filebeat and how they work together. Understanding these concepts will help you make informed decisions about configuring Filebeat for specific use cases. -Filebeat consists of two main components: <> and <>. These components work together to tail files and send event data to the output that you specify. +Filebeat consists of two main components: <> and <>. These components work together to tail files and send event data to the output that you specify. [float] @@ -26,14 +26,14 @@ To control when a harvester is closed, use the <> confi A prospector is responsible for managing the harvesters and finding all sources to read from. -If the input type is `log`, the prospector finds all files on the drive that match the defined glob paths and starts a harvester for each file. Each prospector runs in its own Go routine. +If the input type is `log`, the prospector finds all files on the drive that match the defined glob paths and starts a harvester for each file. Each prospector runs in its own Go routine. The following example configures Filebeat to harvest lines from all log files that match the specified glob patterns: [source,yaml] ------------------------------------------------------------------------------------- filebeat.prospectors: -- input_type: log +- type: log paths: - /var/log/*.log - /var/path2/*.log @@ -50,7 +50,7 @@ Filebeat keeps the state of each file and frequently flushes the state to disk i Each prospector keeps a state for each file it finds. Because files can be renamed or moved, the filename and path are not enough to identify a file. For each file, Filebeat stores unique identifiers to detect whether a file was harvested previously. -If your use case involves creating a large number of new files every day, you might find that the registry file grows to be too large. See <> for details about configuration options that you can set to resolve this issue. +If your use case involves creating a large number of new files every day, you might find that the registry file grows to be too large. See <> for details about configuration options that you can set to resolve this issue. [float] [[at-least-once-delivery]] @@ -58,11 +58,11 @@ If your use case involves creating a large number of new files every day, you mi Filebeat guarantees that events will be delivered to the configured output at least once and with no data loss. Filebeat is able to achieve this behavior -because it stores the delivery state of each event in the registry file. +because it stores the delivery state of each event in the registry file. In situations where the defined output is blocked and has not confirmed all events, Filebeat will keep trying to send events until the output acknowledges -that it has received the events. +that it has received the events. If Filebeat shuts down while it's in the process of sending events, it does not wait for the output to acknowledge all events before shutting down. Any events diff --git a/filebeat/docs/load-balancing.asciidoc b/filebeat/docs/load-balancing.asciidoc index d890562190ce..c024b406c576 100644 --- a/filebeat/docs/load-balancing.asciidoc +++ b/filebeat/docs/load-balancing.asciidoc @@ -37,7 +37,7 @@ Example: [source,yaml] ------------------------------------------------------------------------------- filebeat.prospectors: -- input_type: log +- type: log paths: - /var/log/*.log output.logstash: @@ -63,7 +63,7 @@ Example: [source,yaml] ------------------------------------------------------------------------------- filebeat.prospectors: -- input_type: log +- type: log paths: - /var/log/*.log filebeat.spool_size: 4096 diff --git a/filebeat/docs/migration.asciidoc b/filebeat/docs/migration.asciidoc index 47ef518ae26a..d765e5cf5233 100644 --- a/filebeat/docs/migration.asciidoc +++ b/filebeat/docs/migration.asciidoc @@ -135,7 +135,7 @@ The equivalent `prospectors` section would look like this: [source,yaml] ------------------------------------------------------------------------------------- filebeat.prospectors: -- input_type: log +- type: log paths: - /var/log/messages - /var/log/*.log @@ -143,16 +143,16 @@ filebeat.prospectors: service: apache zone: us-east-1 fields_under_root: true -- input_type: stdin <1> -- input_type: log +- type: stdin <1> +- type: log paths: - /var/log/apache2/httpd-*.log ------------------------------------------------------------------------------------- -<1> The explicit `input_type` option was introduced to differentiate between normal files and +<1> The explicit `type` option was introduced to differentiate between normal files and stdin. In the future, additional types might be supported. -As you can see, apart from the new `input_type` options, +As you can see, apart from the new `type` options, which were before implicitly defined via the `type` custom field, the remaining options can be migrated mechanically. @@ -279,7 +279,7 @@ Filebeat configuration: [source,yaml] ------------------------------------------------------------------------------------- filebeat.prospectors: -- input_type: log +- type: log paths: - /var/log/*.log fields: @@ -366,7 +366,7 @@ you can use the <> configuration option: [source,yaml] ------------------------------------------------------------------------------------- filebeat.prospectors: -- input_type: log +- type: log paths: - /var/log/*.log fields: diff --git a/filebeat/docs/multiple-prospectors.asciidoc b/filebeat/docs/multiple-prospectors.asciidoc index 42b0760eb3de..57ae3dbbe511 100644 --- a/filebeat/docs/multiple-prospectors.asciidoc +++ b/filebeat/docs/multiple-prospectors.asciidoc @@ -4,10 +4,10 @@ When you need to collect lines from multiple files, you can simply configure a single prospector and specify multiple paths to start a harvester for each file. However, if you want to apply additional prospector-specific <> (such as `fields`, `include_lines`, `exclude_lines`, `multiline`, and so on) -to the lines harvested from specific files, you need to define multiple prospectors in the Filebeat config file. +to the lines harvested from specific files, you need to define multiple prospectors in the Filebeat config file. Within the config file, you can specify multiple prospectors, and each prospector can define multiple paths to crawl, as -shown in the following example. +shown in the following example. NOTE: Make sure a file is not defined more than once across all prospectors because this can lead to unexpected behaviour. @@ -15,11 +15,11 @@ to unexpected behaviour. [source,yaml] ------------------------------------------------------------------------------------- filebeat.prospectors: -- input_type: log +- type: log paths: - /var/log/system.log - /var/log/wifi.log -- input_type: log +- type: log paths: - "/var/log/apache2/*" fields: diff --git a/filebeat/docs/reference/configuration/filebeat-options.asciidoc b/filebeat/docs/reference/configuration/filebeat-options.asciidoc index b3c823cffea7..0014000a3b28 100644 --- a/filebeat/docs/reference/configuration/filebeat-options.asciidoc +++ b/filebeat/docs/reference/configuration/filebeat-options.asciidoc @@ -11,11 +11,11 @@ Here is a sample configuration: [source,yaml] ------------------------------------------------------------------------------------- filebeat.prospectors: -- input_type: log +- type: log paths: - /var/log/apache/httpd-*.log -- input_type: log +- type: log paths: - /var/log/messages - /var/log/*.log @@ -23,14 +23,14 @@ filebeat.prospectors: ==== Options -===== input_type +===== type One of the following input types: * log: Reads every line of the log file (default) * stdin: Reads the standard in -The value that you specify here is used as the `input_type` for each event published to Logstash and Elasticsearch. +The value that you specify here is used as the `type` for each event published to Logstash and Elasticsearch. [[prospector-paths]] ===== paths diff --git a/filebeat/docs/reference/configuration/reload-configuration.asciidoc b/filebeat/docs/reference/configuration/reload-configuration.asciidoc index aa6f7560f911..ef7932a11087 100644 --- a/filebeat/docs/reference/configuration/reload-configuration.asciidoc +++ b/filebeat/docs/reference/configuration/reload-configuration.asciidoc @@ -4,7 +4,7 @@ beta[] You can configure Filebeat to dynamically reload prospector configuration files -when there are changes. To do this, you specify a path +when there are changes. To do this, you specify a path (https://golang.org/pkg/path/filepath/#Glob[Glob]) to watch for prospector configuration changes. When the files found by the Glob change, new prospectors are started/stopped according to changes in the configuration files. @@ -13,7 +13,7 @@ This feature is especially useful in container environments where one container is used to tail logs for services running in other containers on the same host. To enable dynamic config reloading, you specify the `path` and `reload` options -in the main `filebeat.yml` config file. For example: +in the main `filebeat.yml` config file. For example: [source,yaml] ------------------------------------------------------------------------------ @@ -23,24 +23,24 @@ filebeat.config.prospectors: reload.period: 10s ------------------------------------------------------------------------------ -`path`:: A Glob that defines the files to check for changes. -`reload.enabled`:: When set to `true`, enables dynamic config reload. +`path`:: A Glob that defines the files to check for changes. +`reload.enabled`:: When set to `true`, enables dynamic config reload. `reload.period`:: Specifies how often the files are checked for changes. Do not set the `period` to less than 1s because the modification time of files is often stored in seconds. Setting the `period` to less than 1s will result in unnecessary overhead. Each file found by the Glob must contain a list of one or more prospector -definitions. For example: +definitions. For example: [source,yaml] ------------------------------------------------------------------------------ -- input_type: log +- type: log paths: - /var/log/mysql.log scan_frequency: 10s -- input_type: log +- type: log paths: - /var/log/apache.log scan_frequency: 5s diff --git a/filebeat/docs/system-module-note.asciidoc b/filebeat/docs/system-module-note.asciidoc index 1774a041533a..5d38e17a4084 100644 --- a/filebeat/docs/system-module-note.asciidoc +++ b/filebeat/docs/system-module-note.asciidoc @@ -7,7 +7,7 @@ you plan to run the `system` module, edit the Filebeat configuration file, [source,yaml] ---------------------------------------------------------------------- -#- input_type: log +#- type: log #paths: #- /var/log/*.log ---------------------------------------------------------------------- @@ -16,4 +16,4 @@ For rpm and deb, you'll find the configuration file at `/etc/filebeat/filebeat.yml`. For mac and win, look in the archive that you extracted when you installed Filebeat. -=============================================================================== \ No newline at end of file +=============================================================================== diff --git a/filebeat/filebeat.full.yml b/filebeat/filebeat.full.yml index 06036fc779ad..f20d047894ae 100644 --- a/filebeat/filebeat.full.yml +++ b/filebeat/filebeat.full.yml @@ -183,7 +183,7 @@ filebeat.prospectors: # * stdin: Reads the standard in #------------------------------ Log prospector -------------------------------- -- input_type: log +- type: log # Change to true to enable this prospector configuration. enabled: false @@ -377,7 +377,7 @@ filebeat.prospectors: #----------------------------- Stdin prospector ------------------------------- # Configuration to use stdin input -#- input_type: stdin +#- type: stdin #========================= Filebeat global options ============================ diff --git a/filebeat/filebeat.yml b/filebeat/filebeat.yml index 790eee22a016..1adfa33ee239 100644 --- a/filebeat/filebeat.yml +++ b/filebeat/filebeat.yml @@ -105,7 +105,7 @@ filebeat.prospectors: # you can use different prospectors for various configurations. # Below are the prospector specific configurations. -- input_type: log +- type: log # Change to true to enable this prospector configuration. enabled: false diff --git a/filebeat/module/apache2/access/config/access.yml b/filebeat/module/apache2/access/config/access.yml index 56e3dfd95b82..0afd17317d4f 100644 --- a/filebeat/module/apache2/access/config/access.yml +++ b/filebeat/module/apache2/access/config/access.yml @@ -1,4 +1,4 @@ -input_type: log +type: log paths: {{ range $i, $path := .paths }} - {{$path}} diff --git a/filebeat/module/apache2/access/test/test.log-expected.json b/filebeat/module/apache2/access/test/test.log-expected.json index 0997759acc72..7a7e78f4633e 100644 --- a/filebeat/module/apache2/access/test/test.log-expected.json +++ b/filebeat/module/apache2/access/test/test.log-expected.json @@ -1,140 +1,143 @@ [ -{ - "_index": "filebeat-2016.12.27", - "_type": "log", - "_id": "AVlBCaYsqYg9cc5KQfcT", - "_score": null, - "_source": { - "@timestamp": "2016-12-26T14:16:29.000Z", - "offset": 73, - "apache2": { - "access": { - "response_code": "404", - "remote_ip": "::1", - "method": "GET", - "user_name": "-", - "http_version": "1.1", - "body_sent": { - "bytes": "209" + { + "_index": "filebeat-2016.12.27", + "_type": "log", + "_id": "AVlBCaYsqYg9cc5KQfcT", + "_score": null, + "_source": { + "@timestamp": "2016-12-26T14:16:29.000Z", + "offset": 73, + "apache2": { + "access": { + "response_code": "404", + "remote_ip": "::1", + "method": "GET", + "user_name": "-", + "http_version": "1.1", + "body_sent": { + "bytes": "209" + }, + "url": "/favicon.ico" + } + }, + "beat": { + "hostname": "192-168-0-7.rdsnet.ro", + "name": "192-168-0-7.rdsnet.ro", + "version": "6.0.0-alpha1" + }, + "read_timestamp": "2016-12-27T15:52:23.304Z", + "source": "module/apache2/access/test/test.log", + "fields": { + "pipeline_id": "apache2-access-with_plugins", + "source_type": "apache2-access" + }, + "prospector": { + "type": "log" + } }, - "url": "/favicon.ico" - } - }, - "beat": { - "hostname": "192-168-0-7.rdsnet.ro", - "name": "192-168-0-7.rdsnet.ro", - "version": "6.0.0-alpha1" - }, - "input_type": "log", - "read_timestamp": "2016-12-27T15:52:23.304Z", - "source": "module/apache2/access/test/test.log", - "fields": { - "pipeline_id": "apache2-access-with_plugins", - "source_type": "apache2-access" - }, - "type": "log" - }, - "fields": { - "@timestamp": [ - 1482761789000 - ] - }, - "sort": [ - 1482761789000 - ] -}, -{ - "_index": "filebeat-2016.12.27", - "_type": "log", - "_id": "AVlBCaYsqYg9cc5KQfcU", - "_score": null, - "_source": { - "@timestamp": "2016-12-26T16:22:13.000Z", - "offset": 238, - "apache2": { - "access": { - "referrer": "-", - "response_code": "404", - "remote_ip": "192.168.33.1", - "method": "GET", - "user_name": "-", - "http_version": "1.1", - "body_sent": { - "bytes": "499" + "fields": { + "@timestamp": [ + 1482761789000 + ] }, - "url": "/hello", - "user_agent": { - "major": "50", - "minor": "0", - "os": "Mac OS X 10.12", - "os_minor": "12", - "os_major": "10", - "name": "Firefox", - "os_name": "Mac OS X", - "device": "Other" - } - } - }, - "beat": { - "hostname": "192-168-0-7.rdsnet.ro", - "name": "192-168-0-7.rdsnet.ro", - "version": "6.0.0-alpha1" - }, - "input_type": "log", - "read_timestamp": "2016-12-27T15:52:23.304Z", - "source": "module/apache2/access/test/test.log", - "fields": { - "pipeline_id": "apache2-access-with_plugins", - "source_type": "apache2-access" - }, - "type": "log" - }, - "fields": { - "@timestamp": [ - 1482769333000 - ] - }, - "sort": [ - 1482769333000 - ] -}, -{ - "_index": "filebeat-2016.12.27", - "_type": "log", - "_id": "AVlBCaYsqYg9cc5KQfc-", - "_score": null, - "_source": { - "@timestamp": "2016-12-26T14:16:48.000Z", - "offset": 285, - "apache2": { - "access": { - "response_code": "408", - "remote_ip": "::1", - "user_name": "-" - } + "sort": [ + 1482761789000 + ] }, - "beat": { - "hostname": "192-168-0-7.rdsnet.ro", - "name": "192-168-0-7.rdsnet.ro", - "version": "6.0.0-alpha1" - }, - "input_type": "log", - "read_timestamp": "2016-12-27T16:04:58.319Z", - "source": "module/apache2/access/test/test.log", - "fields": { - "pipeline_id": "apache2-access-with_plugins", - "source_type": "apache2-access" + { + "_index": "filebeat-2016.12.27", + "_type": "log", + "_id": "AVlBCaYsqYg9cc5KQfcU", + "_score": null, + "_source": { + "@timestamp": "2016-12-26T16:22:13.000Z", + "offset": 238, + "apache2": { + "access": { + "referrer": "-", + "response_code": "404", + "remote_ip": "192.168.33.1", + "method": "GET", + "user_name": "-", + "http_version": "1.1", + "body_sent": { + "bytes": "499" + }, + "url": "/hello", + "user_agent": { + "major": "50", + "minor": "0", + "os": "Mac OS X 10.12", + "os_minor": "12", + "os_major": "10", + "name": "Firefox", + "os_name": "Mac OS X", + "device": "Other" + } + } + }, + "beat": { + "hostname": "192-168-0-7.rdsnet.ro", + "name": "192-168-0-7.rdsnet.ro", + "version": "6.0.0-alpha1" + }, + "read_timestamp": "2016-12-27T15:52:23.304Z", + "source": "module/apache2/access/test/test.log", + "fields": { + "pipeline_id": "apache2-access-with_plugins", + "source_type": "apache2-access" + }, + "prospector": { + "type": "log" + } + }, + "fields": { + "@timestamp": [ + 1482769333000 + ] + }, + "sort": [ + 1482769333000 + ] }, - "type": "log" - }, - "fields": { - "@timestamp": [ - 1482761808000 - ] - }, - "sort": [ - 1482761808000 - ] -} + { + "_index": "filebeat-2016.12.27", + "_type": "log", + "_id": "AVlBCaYsqYg9cc5KQfc-", + "_score": null, + "_source": { + "@timestamp": "2016-12-26T14:16:48.000Z", + "offset": 285, + "apache2": { + "access": { + "response_code": "408", + "remote_ip": "::1", + "user_name": "-" + } + }, + "beat": { + "hostname": "192-168-0-7.rdsnet.ro", + "name": "192-168-0-7.rdsnet.ro", + "version": "6.0.0-alpha1" + }, + "read_timestamp": "2016-12-27T16:04:58.319Z", + "source": "module/apache2/access/test/test.log", + "fields": { + "pipeline_id": "apache2-access-with_plugins", + "source_type": "apache2-access" + }, + "prospector": { + "type": "log" + } + }, + "fields": { + "@timestamp": [ + 1482761808000 + ] + }, + "sort": [ + 1482761808000 + ] + } ] diff --git a/filebeat/module/apache2/error/config/error.yml b/filebeat/module/apache2/error/config/error.yml index 56e3dfd95b82..0afd17317d4f 100644 --- a/filebeat/module/apache2/error/config/error.yml +++ b/filebeat/module/apache2/error/config/error.yml @@ -1,4 +1,4 @@ -input_type: log +type: log paths: {{ range $i, $path := .paths }} - {{$path}} diff --git a/filebeat/module/apache2/error/test/test.log-expected.json b/filebeat/module/apache2/error/test/test.log-expected.json index 050086035959..d72695adc646 100644 --- a/filebeat/module/apache2/error/test/test.log-expected.json +++ b/filebeat/module/apache2/error/test/test.log-expected.json @@ -1,93 +1,96 @@ [ - { - "_index": "filebeat-2016.12.29", - "_type": "log", - "_id": "AVlGiYcNqYg9cc5KQfdz", - "_score": 1, - "_source": { - "@timestamp": "2016-12-26T16:15:55.103Z", - "offset": 229, - "apache2": { - "error": { - "level": "notice", - "module": "core", - "pid": "11379", - "message": "AH00094: Command line: '/usr/local/Cellar/httpd24/2.4.23_2/bin/httpd'" - } - }, - "beat": { - "hostname": "192-168-0-7.rdsnet.ro", - "name": "192-168-0-7.rdsnet.ro", - "version": "6.0.0-alpha1" - }, - "input_type": "log", - "source": "module/apache2/error/test/test.log", - "fields": { - "pipeline_id": "apache2-error-pipeline", - "source_type": "apache2-error" - }, - "type": "log" - } - }, - { - "_index": "filebeat-2016.12.29", - "_type": "log", - "_id": "AVlGiYcNqYg9cc5KQfd0", - "_score": 1, - "_source": { - "@timestamp": "2011-09-09T10:42:29.902Z", - "offset": 384, - "apache2": { - "error": { - "level": "error", - "module": "core", - "client": "72.15.99.187", - "pid": "35708", - "message": "File does not exist: /usr/local/apache2/htdocs/favicon.ico", - "tid": "4328636416" - } - }, - "beat": { - "hostname": "192-168-0-7.rdsnet.ro", - "name": "192-168-0-7.rdsnet.ro", - "version": "6.0.0-alpha1" - }, - "input_type": "log", - "source": "module/apache2/error/test/test.log", - "fields": { - "pipeline_id": "apache2-error-pipeline", - "source_type": "apache2-error" - }, - "type": "log" - } - }, - { - "_index": "filebeat-2016.12.29", - "_type": "log", - "_id": "AVlGiYcMqYg9cc5KQfdy", - "_score": 1, - "_source": { - "@timestamp": "2016-12-26T16:22:08.000Z", - "offset": 99, - "apache2": { - "error": { - "level": "error", - "client": "192.168.33.1", - "message": "File does not exist: /var/www/favicon.ico" - } - }, - "beat": { - "hostname": "192-168-0-7.rdsnet.ro", - "name": "192-168-0-7.rdsnet.ro", - "version": "6.0.0-alpha1" - }, - "input_type": "log", - "source": "module/apache2/error/test/test.log", - "fields": { - "pipeline_id": "apache2-error-pipeline", - "source_type": "apache2-error" - }, - "type": "log" - } - } + { + "_index": "filebeat-2016.12.29", + "_type": "log", + "_id": "AVlGiYcNqYg9cc5KQfdz", + "_score": 1, + "_source": { + "@timestamp": "2016-12-26T16:15:55.103Z", + "offset": 229, + "apache2": { + "error": { + "level": "notice", + "module": "core", + "pid": "11379", + "message": "AH00094: Command line: '/usr/local/Cellar/httpd24/2.4.23_2/bin/httpd'" + } + }, + "beat": { + "hostname": "192-168-0-7.rdsnet.ro", + "name": "192-168-0-7.rdsnet.ro", + "version": "6.0.0-alpha1" + }, + "prospector": { + "type": "log" + }, + "source": "module/apache2/error/test/test.log", + "fields": { + "pipeline_id": "apache2-error-pipeline", + "source_type": "apache2-error" + } + } + }, + { + "_index": "filebeat-2016.12.29", + "_type": "log", + "_id": "AVlGiYcNqYg9cc5KQfd0", + "_score": 1, + "_source": { + "@timestamp": "2011-09-09T10:42:29.902Z", + "offset": 384, + "apache2": { + "error": { + "level": "error", + "module": "core", + "client": "72.15.99.187", + "pid": "35708", + "message": "File does not exist: /usr/local/apache2/htdocs/favicon.ico", + "tid": "4328636416" + } + }, + "beat": { + "hostname": "192-168-0-7.rdsnet.ro", + "name": "192-168-0-7.rdsnet.ro", + "version": "6.0.0-alpha1" + }, + "prospector": { + "type": "log" + }, + "source": "module/apache2/error/test/test.log", + "fields": { + "pipeline_id": "apache2-error-pipeline", + "source_type": "apache2-error" + } + } + }, + { + "_index": "filebeat-2016.12.29", + "_type": "log", + "_id": "AVlGiYcMqYg9cc5KQfdy", + "_score": 1, + "_source": { + "@timestamp": "2016-12-26T16:22:08.000Z", + "offset": 99, + "apache2": { + "error": { + "level": "error", + "client": "192.168.33.1", + "message": "File does not exist: /var/www/favicon.ico" + } + }, + "beat": { + "hostname": "192-168-0-7.rdsnet.ro", + "name": "192-168-0-7.rdsnet.ro", + "version": "6.0.0-alpha1" + }, + "prospector": { + "type": "log" + }, + "source": "module/apache2/error/test/test.log", + "fields": { + "pipeline_id": "apache2-error-pipeline", + "source_type": "apache2-error" + } + } + } ] diff --git a/filebeat/module/auditd/log/config/log.yml b/filebeat/module/auditd/log/config/log.yml index 56e3dfd95b82..0afd17317d4f 100644 --- a/filebeat/module/auditd/log/config/log.yml +++ b/filebeat/module/auditd/log/config/log.yml @@ -1,4 +1,4 @@ -input_type: log +type: log paths: {{ range $i, $path := .paths }} - {{$path}} diff --git a/filebeat/module/auditd/log/test/test.log-expected.json b/filebeat/module/auditd/log/test/test.log-expected.json index a0a661eb5d89..65590185ca21 100644 --- a/filebeat/module/auditd/log/test/test.log-expected.json +++ b/filebeat/module/auditd/log/test/test.log-expected.json @@ -7,13 +7,14 @@ "_score": null, "_source": { "offset": 172, - "input_type": "log", + "prospector": { + "type": "log" + }, "source": "/Users/me/go/src/github.com/elastic/beats/filebeat/module/auditd/log/test/test.log", "fileset": { "module": "auditd", "name": "log" }, - "type": "log", "error": "", "@timestamp": "2017-01-31T20:18:46.912Z", "auditd": { @@ -45,13 +46,14 @@ "_score": null, "_source": { "offset": 534, - "input_type": "log", + "prospector": { + "type": "log" + }, "source": "/Users/me/go/src/github.com/elastic/beats/filebeat/module/auditd/log/test/test.log", "fileset": { "module": "auditd", "name": "log" }, - "type": "log", "error": "", "@timestamp": "2017-01-31T20:18:46.912Z", "auditd": { diff --git a/filebeat/module/icinga/debug/config/debug.yml b/filebeat/module/icinga/debug/config/debug.yml index 8deb67e81317..98c684ef52f6 100644 --- a/filebeat/module/icinga/debug/config/debug.yml +++ b/filebeat/module/icinga/debug/config/debug.yml @@ -1,4 +1,4 @@ -input_type: log +type: log paths: {{ range $i, $path := .paths }} - {{$path}} diff --git a/filebeat/module/icinga/debug/test/test.log-expected.json b/filebeat/module/icinga/debug/test/test.log-expected.json index d9636d71c83a..95084f423c82 100644 --- a/filebeat/module/icinga/debug/test/test.log-expected.json +++ b/filebeat/module/icinga/debug/test/test.log-expected.json @@ -1,113 +1,116 @@ [ -{ - "_index": "filebeat-2017.04.04", - "_type": "log", - "_id": "AVs46val2BrA1a1-fCdy", - "_score": null, - "_source": { - "@timestamp": "2017-04-04T11:43:09.000Z", - "offset": 1763, - "beat": { - "hostname": "blerims-mbp.int.netways.de", - "name": "blerims-mbp.int.netways.de", - "version": "5.3.0" + { + "_index": "filebeat-2017.04.04", + "_type": "log", + "_id": "AVs46val2BrA1a1-fCdy", + "_score": null, + "_source": { + "@timestamp": "2017-04-04T11:43:09.000Z", + "offset": 1763, + "beat": { + "hostname": "blerims-mbp.int.netways.de", + "name": "blerims-mbp.int.netways.de", + "version": "5.3.0" + }, + "prospector": { + "type": "log" + }, + "icinga": { + "debug": { + "severity": "debug", + "message": "Query: UPDATE icinga_servicestatus SET acknowledgement_type = '0', active_checks_enabled = '1', check_command = 'mysql_health', check_source = 'demo', check_type = '0', current_check_attempt = '1', current_notification_number = '180', current_state = '2', endpoint_object_id = 242, event_handler = '', event_handler_enabled = '1', execution_time = '0.355594', flap_detection_enabled = '0', has_been_checked = '1', instance_id = 1, is_flapping = '0', is_reachable = '1', last_check = FROM_UNIXTIME(1491306189), last_hard_state = '2', last_hard_state_change = FROM_UNIXTIME(1491290599), last_notification = FROM_UNIXTIME(1491304989), last_state_change = FROM_UNIXTIME(1491290599), last_time_critical = FROM_UNIXTIME(1491306189), last_time_unknown = FROM_UNIXTIME(1491290589), latency = '0.001466', long_output = '', max_check_attempts = '5', next_check = FROM_UNIXTIME(1491306198), next_notification = FROM_UNIXTIME(1491306789), normal_check_interval = '0.166667', notifications_enabled = '1', original_attributes = 'null', output = 'CRITICAL - cannot connect to information_schema. Access denied for user \\'test1\\'@\\'blerims-mbp.int.netways.de\\' (using password: YES)', passive_checks_enabled = '1', percent_state_change = '0', perfdata = '', problem_has_been_acknowledged = '0', process_performance_data = '1', retry_check_interval = '0.166667', scheduled_downtime_depth = '0', service_object_id = 333, should_be_scheduled = '1', state_type = '1', status_update_time = FROM_UNIXTIME(1491306189) WHERE service_object_id = 333", + "facility": "IdoMysqlConnection" + } + }, + "source": "/usr/local/icinga2/var/log/icinga2/debug.log", + "fileset": { + "module": "icinga", + "name": "debug" + } + }, + "fields": { + "@timestamp": [ + 1491306189000 + ] + }, + "sort": [ + 1491306189000 + ] }, - "input_type": "log", - "icinga": { - "debug": { - "severity": "debug", - "message": "Query: UPDATE icinga_servicestatus SET acknowledgement_type = '0', active_checks_enabled = '1', check_command = 'mysql_health', check_source = 'demo', check_type = '0', current_check_attempt = '1', current_notification_number = '180', current_state = '2', endpoint_object_id = 242, event_handler = '', event_handler_enabled = '1', execution_time = '0.355594', flap_detection_enabled = '0', has_been_checked = '1', instance_id = 1, is_flapping = '0', is_reachable = '1', last_check = FROM_UNIXTIME(1491306189), last_hard_state = '2', last_hard_state_change = FROM_UNIXTIME(1491290599), last_notification = FROM_UNIXTIME(1491304989), last_state_change = FROM_UNIXTIME(1491290599), last_time_critical = FROM_UNIXTIME(1491306189), last_time_unknown = FROM_UNIXTIME(1491290589), latency = '0.001466', long_output = '', max_check_attempts = '5', next_check = FROM_UNIXTIME(1491306198), next_notification = FROM_UNIXTIME(1491306789), normal_check_interval = '0.166667', notifications_enabled = '1', original_attributes = 'null', output = 'CRITICAL - cannot connect to information_schema. Access denied for user \\'test1\\'@\\'blerims-mbp.int.netways.de\\' (using password: YES)', passive_checks_enabled = '1', percent_state_change = '0', perfdata = '', problem_has_been_acknowledged = '0', process_performance_data = '1', retry_check_interval = '0.166667', scheduled_downtime_depth = '0', service_object_id = 333, should_be_scheduled = '1', state_type = '1', status_update_time = FROM_UNIXTIME(1491306189) WHERE service_object_id = 333", - "facility": "IdoMysqlConnection" - } + { + "_index": "filebeat-2017.04.04", + "_type": "log", + "_id": "AVs46val2BrA1a1-fCdx", + "_score": null, + "_source": { + "@timestamp": "2017-04-04T11:43:09.000Z", + "offset": 141, + "beat": { + "hostname": "blerims-mbp.int.netways.de", + "name": "blerims-mbp.int.netways.de", + "version": "5.3.0" + }, + "prospector": { + "type": "log" + }, + "icinga": { + "debug": { + "severity": "debug", + "message": "Add to metric list:'icinga2.demo.services.procs.procs.perfdata.procs.warn 250 1491306189'.", + "facility": "GraphiteWriter" + } + }, + "source": "/usr/local/icinga2/var/log/icinga2/debug.log", + "fileset": { + "module": "icinga", + "name": "debug" + } + }, + "fields": { + "@timestamp": [ + 1491306189000 + ] + }, + "sort": [ + 1491306189000 + ] }, - "source": "/usr/local/icinga2/var/log/icinga2/debug.log", - "fileset": { - "module": "icinga", - "name": "debug" - }, - "type": "log" - }, - "fields": { - "@timestamp": [ - 1491306189000 - ] - }, - "sort": [ - 1491306189000 - ] -}, -{ - "_index": "filebeat-2017.04.04", - "_type": "log", - "_id": "AVs46val2BrA1a1-fCdx", - "_score": null, - "_source": { - "@timestamp": "2017-04-04T11:43:09.000Z", - "offset": 141, - "beat": { - "hostname": "blerims-mbp.int.netways.de", - "name": "blerims-mbp.int.netways.de", - "version": "5.3.0" - }, - "input_type": "log", - "icinga": { - "debug": { - "severity": "debug", - "message": "Add to metric list:'icinga2.demo.services.procs.procs.perfdata.procs.warn 250 1491306189'.", - "facility": "GraphiteWriter" - } - }, - "source": "/usr/local/icinga2/var/log/icinga2/debug.log", - "fileset": { - "module": "icinga", - "name": "debug" - }, - "type": "log" - }, - "fields": { - "@timestamp": [ - 1491306189000 - ] - }, - "sort": [ - 1491306189000 - ] -}, -{ - "_index": "filebeat-2017.04.04", - "_type": "log", - "_id": "AVs46wjD2BrA1a1-fCdz", - "_score": null, - "_source": { - "@timestamp": "2017-04-04T11:43:11.000Z", - "offset": 1927, - "beat": { - "hostname": "blerims-mbp.int.netways.de", - "name": "blerims-mbp.int.netways.de", - "version": "5.3.0" - }, - "input_type": "log", - "icinga": { - "debug": { - "severity": "notice", - "message": "Running command '/usr/lib/nagios/plugins/check_ping' '-H' 'mysql.icinga.com' '-c' '5000,100%' '-w' '3000,80%': PID 8288", - "facility": "Process" - } - }, - "source": "/usr/local/icinga2/var/log/icinga2/debug.log", - "fileset": { - "module": "icinga", - "name": "debug" - }, - "type": "log" - }, - "fields": { - "@timestamp": [ - 1491306191000 - ] - }, - "sort": [ - 1491306191000 - ] -} + { + "_index": "filebeat-2017.04.04", + "_type": "log", + "_id": "AVs46wjD2BrA1a1-fCdz", + "_score": null, + "_source": { + "@timestamp": "2017-04-04T11:43:11.000Z", + "offset": 1927, + "beat": { + "hostname": "blerims-mbp.int.netways.de", + "name": "blerims-mbp.int.netways.de", + "version": "5.3.0" + }, + "prospector": { + "type": "log" + }, + "icinga": { + "debug": { + "severity": "notice", + "message": "Running command '/usr/lib/nagios/plugins/check_ping' '-H' 'mysql.icinga.com' '-c' '5000,100%' '-w' '3000,80%': PID 8288", + "facility": "Process" + } + }, + "source": "/usr/local/icinga2/var/log/icinga2/debug.log", + "fileset": { + "module": "icinga", + "name": "debug" + }, + }, + "fields": { + "@timestamp": [ + 1491306191000 + ] + }, + "sort": [ + 1491306191000 + ] + } ] diff --git a/filebeat/module/icinga/main/config/main.yml b/filebeat/module/icinga/main/config/main.yml index 8deb67e81317..98c684ef52f6 100644 --- a/filebeat/module/icinga/main/config/main.yml +++ b/filebeat/module/icinga/main/config/main.yml @@ -1,4 +1,4 @@ -input_type: log +type: log paths: {{ range $i, $path := .paths }} - {{$path}} diff --git a/filebeat/module/icinga/main/test/test.log-expected.json b/filebeat/module/icinga/main/test/test.log-expected.json index b5592b6c1409..9b30dad5255b 100644 --- a/filebeat/module/icinga/main/test/test.log-expected.json +++ b/filebeat/module/icinga/main/test/test.log-expected.json @@ -1,113 +1,116 @@ [ -{ - "_index": "filebeat-2017.04.04", - "_type": "log", - "_id": "AVs4ouOJ2BrA1a1-e_FX", - "_score": null, - "_source": { - "@timestamp": "2017-04-04T09:16:34.000Z", - "offset": 518, - "beat": { - "hostname": "blerims-mbp.int.netways.de", - "name": "blerims-mbp.int.netways.de", - "version": "5.3.0" + { + "_index": "filebeat-2017.04.04", + "_type": "log", + "_id": "AVs4ouOJ2BrA1a1-e_FX", + "_score": null, + "_source": { + "@timestamp": "2017-04-04T09:16:34.000Z", + "offset": 518, + "beat": { + "hostname": "blerims-mbp.int.netways.de", + "name": "blerims-mbp.int.netways.de", + "version": "5.3.0" + }, + "prospector": { + "type": "log" + }, + "icinga": { + "main": { + "severity": "warning", + "message": "Notification command for object 'demo!load' (PID: 19401, arguments: '/etc/icinga2/scripts/mail-service-notification.sh') terminated with exit code 127, output: /etc/icinga2/scripts/mail-service-notification.sh: 20: /etc/icinga2/scripts/mail-service-notification.sh: mail: not found\n/usr/bin/printf: write error: Broken pipe\n", + "facility": "PluginNotificationTask" + } + }, + "source": "/usr/local/icinga2/var/log/icinga2/icinga2.log", + "fileset": { + "module": "icinga", + "name": "main" + } + }, + "fields": { + "@timestamp": [ + 1491297394000 + ] + }, + "sort": [ + 1491297394000 + ] }, - "input_type": "log", - "icinga": { - "main": { - "severity": "warning", - "message": "Notification command for object 'demo!load' (PID: 19401, arguments: '/etc/icinga2/scripts/mail-service-notification.sh') terminated with exit code 127, output: /etc/icinga2/scripts/mail-service-notification.sh: 20: /etc/icinga2/scripts/mail-service-notification.sh: mail: not found\n/usr/bin/printf: write error: Broken pipe\n", - "facility": "PluginNotificationTask" - } + { + "_index": "filebeat-2017.04.04", + "_type": "log", + "_id": "AVs4ouOJ2BrA1a1-e_FW", + "_score": null, + "_source": { + "@timestamp": "2017-04-04T09:16:34.000Z", + "offset": 133, + "beat": { + "hostname": "blerims-mbp.int.netways.de", + "name": "blerims-mbp.int.netways.de", + "version": "5.3.0" + }, + "prospector": { + "type": "log" + }, + "icinga": { + "main": { + "severity": "information", + "message": "Sending 'Recovery' notification 'demo!load!mail-icingaadmin for user 'on-call'", + "facility": "Notification" + } + }, + "source": "/usr/local/icinga2/var/log/icinga2/icinga2.log", + "fileset": { + "module": "icinga", + "name": "main" + } + }, + "fields": { + "@timestamp": [ + 1491297394000 + ] + }, + "sort": [ + 1491297394000 + ] }, - "source": "/usr/local/icinga2/var/log/icinga2/icinga2.log", - "fileset": { - "module": "icinga", - "name": "main" - }, - "type": "log" - }, - "fields": { - "@timestamp": [ - 1491297394000 - ] - }, - "sort": [ - 1491297394000 - ] -}, -{ - "_index": "filebeat-2017.04.04", - "_type": "log", - "_id": "AVs4ouOJ2BrA1a1-e_FW", - "_score": null, - "_source": { - "@timestamp": "2017-04-04T09:16:34.000Z", - "offset": 133, - "beat": { - "hostname": "blerims-mbp.int.netways.de", - "name": "blerims-mbp.int.netways.de", - "version": "5.3.0" - }, - "input_type": "log", - "icinga": { - "main": { - "severity": "information", - "message": "Sending 'Recovery' notification 'demo!load!mail-icingaadmin for user 'on-call'", - "facility": "Notification" - } - }, - "source": "/usr/local/icinga2/var/log/icinga2/icinga2.log", - "fileset": { - "module": "icinga", - "name": "main" - }, - "type": "log" - }, - "fields": { - "@timestamp": [ - 1491297394000 - ] - }, - "sort": [ - 1491297394000 - ] -}, -{ - "_index": "filebeat-2017.04.04", - "_type": "log", - "_id": "AVs4ovY82BrA1a1-e_FY", - "_score": null, - "_source": { - "@timestamp": "2017-04-04T09:16:48.000Z", - "offset": 654, - "beat": { - "hostname": "blerims-mbp.int.netways.de", - "name": "blerims-mbp.int.netways.de", - "version": "5.3.0" - }, - "input_type": "log", - "icinga": { - "main": { - "severity": "information", - "message": "Query queue items: 0, query rate: 5.38333/s (323/min 1610/5min 4778/15min);", - "facility": "IdoMysqlConnection" - } - }, - "source": "/usr/local/icinga2/var/log/icinga2/icinga2.log", - "fileset": { - "module": "icinga", - "name": "main" - }, - "type": "log" - }, - "fields": { - "@timestamp": [ - 1491297408000 - ] - }, - "sort": [ - 1491297408000 - ] -} + { + "_index": "filebeat-2017.04.04", + "_type": "log", + "_id": "AVs4ovY82BrA1a1-e_FY", + "_score": null, + "_source": { + "@timestamp": "2017-04-04T09:16:48.000Z", + "offset": 654, + "beat": { + "hostname": "blerims-mbp.int.netways.de", + "name": "blerims-mbp.int.netways.de", + "version": "5.3.0" + }, + "prospector": { + "type": "log" + }, + "icinga": { + "main": { + "severity": "information", + "message": "Query queue items: 0, query rate: 5.38333/s (323/min 1610/5min 4778/15min);", + "facility": "IdoMysqlConnection" + } + }, + "source": "/usr/local/icinga2/var/log/icinga2/icinga2.log", + "fileset": { + "module": "icinga", + "name": "main" + } + }, + "fields": { + "@timestamp": [ + 1491297408000 + ] + }, + "sort": [ + 1491297408000 + ] + } ] diff --git a/filebeat/module/icinga/startup/config/startup.yml b/filebeat/module/icinga/startup/config/startup.yml index e9ef3077e6d5..a86ec3f1f312 100644 --- a/filebeat/module/icinga/startup/config/startup.yml +++ b/filebeat/module/icinga/startup/config/startup.yml @@ -1,4 +1,4 @@ -input_type: log +type: log paths: {{ range $i, $path := .paths }} - {{$path}} diff --git a/filebeat/module/icinga/startup/test/test.log-expected.json b/filebeat/module/icinga/startup/test/test.log-expected.json index 0023201b0dde..567c1b03c8e1 100644 --- a/filebeat/module/icinga/startup/test/test.log-expected.json +++ b/filebeat/module/icinga/startup/test/test.log-expected.json @@ -1,76 +1,78 @@ [ -{ - "_index": "filebeat-2017.04.04", - "_type": "log", - "_id": "AVs5obub2BrA1a1-fVkV", - "_score": null, - "_source": { - "@timestamp": "2017-04-04T15:42:27.138Z", - "offset": 63, - "beat": { - "hostname": "blerims-mbp.int.netways.de", - "name": "blerims-mbp.int.netways.de", - "version": "5.3.0" + { + "_index": "filebeat-2017.04.04", + "_type": "log", + "_id": "AVs5obub2BrA1a1-fVkV", + "_score": null, + "_source": { + "@timestamp": "2017-04-04T15:42:27.138Z", + "offset": 63, + "beat": { + "hostname": "blerims-mbp.int.netways.de", + "name": "blerims-mbp.int.netways.de", + "version": "5.3.0" + }, + "prospector": { + "type": "log" + }, + "icinga": { + "startup": { + "severity": "information", + "message": "Icinga application loader (version: r2.6.3-1)", + "facility": "cli" + } + }, + "source": "/usr/local/icinga2/var/log/icinga2/startup.log", + "fileset": { + "module": "icinga", + "name": "startup" + } + }, + "fields": { + "@timestamp": [ + 1491320547138 + ] + }, + "sort": [ + 1491320547138 + ] }, - "input_type": "log", - "icinga": { - "startup": { - "severity": "information", - "message": "Icinga application loader (version: r2.6.3-1)", - "facility": "cli" - } - }, - "source": "/usr/local/icinga2/var/log/icinga2/startup.log", - "fileset": { - "module": "icinga", - "name": "startup" - }, - "type": "log" - }, - "fields": { - "@timestamp": [ - 1491320547138 - ] - }, - "sort": [ - 1491320547138 - ] -}, -{ - "_index": "filebeat-2017.04.04", - "_type": "log", - "_id": "AVs5obub2BrA1a1-fVkW", - "_score": null, - "_source": { - "@timestamp": "2017-04-04T15:42:27.138Z", - "offset": 111, - "beat": { - "hostname": "blerims-mbp.int.netways.de", - "name": "blerims-mbp.int.netways.de", - "version": "5.3.0" - }, - "input_type": "log", - "icinga": { - "startup": { - "severity": "information", - "message": "Loading configuration file(s).", - "facility": "cli" - } - }, - "source": "/usr/local/icinga2/var/log/icinga2/startup.log", - "fileset": { - "module": "icinga", - "name": "startup" - }, - "type": "log" - }, - "fields": { - "@timestamp": [ - 1491320547138 - ] - }, - "sort": [ - 1491320547138 - ] -} + { + "_index": "filebeat-2017.04.04", + "_type": "log", + "_id": "AVs5obub2BrA1a1-fVkW", + "_score": null, + "_source": { + "@timestamp": "2017-04-04T15:42:27.138Z", + "offset": 111, + "beat": { + "hostname": "blerims-mbp.int.netways.de", + "name": "blerims-mbp.int.netways.de", + "version": "5.3.0" + }, + "prospector": { + "type": "log" + }, + "icinga": { + "startup": { + "severity": "information", + "message": "Loading configuration file(s).", + "facility": "cli" + } + }, + "source": "/usr/local/icinga2/var/log/icinga2/startup.log", + "fileset": { + "module": "icinga", + "name": "startup" + }, + }, + "fields": { + "@timestamp": [ + 1491320547138 + ] + }, + "sort": [ + 1491320547138 + ] + } ] diff --git a/filebeat/module/mysql/error/config/error.yml b/filebeat/module/mysql/error/config/error.yml index 56e3dfd95b82..0afd17317d4f 100644 --- a/filebeat/module/mysql/error/config/error.yml +++ b/filebeat/module/mysql/error/config/error.yml @@ -1,4 +1,4 @@ -input_type: log +type: log paths: {{ range $i, $path := .paths }} - {{$path}} diff --git a/filebeat/module/mysql/slowlog/config/slowlog.yml b/filebeat/module/mysql/slowlog/config/slowlog.yml index 923f55654a6b..ec998f5e646b 100644 --- a/filebeat/module/mysql/slowlog/config/slowlog.yml +++ b/filebeat/module/mysql/slowlog/config/slowlog.yml @@ -1,4 +1,4 @@ -input_type: log +type: log paths: {{ range $i, $path := .paths }} - {{$path}} diff --git a/filebeat/module/nginx/access/config/nginx-access.yml b/filebeat/module/nginx/access/config/nginx-access.yml index 56e3dfd95b82..0afd17317d4f 100644 --- a/filebeat/module/nginx/access/config/nginx-access.yml +++ b/filebeat/module/nginx/access/config/nginx-access.yml @@ -1,4 +1,4 @@ -input_type: log +type: log paths: {{ range $i, $path := .paths }} - {{$path}} diff --git a/filebeat/module/nginx/error/config/nginx-error.yml b/filebeat/module/nginx/error/config/nginx-error.yml index 56e3dfd95b82..0afd17317d4f 100644 --- a/filebeat/module/nginx/error/config/nginx-error.yml +++ b/filebeat/module/nginx/error/config/nginx-error.yml @@ -1,4 +1,4 @@ -input_type: log +type: log paths: {{ range $i, $path := .paths }} - {{$path}} diff --git a/filebeat/module/system/auth/config/auth.yml b/filebeat/module/system/auth/config/auth.yml index 1af7dee5e13c..003b5d06b3ec 100644 --- a/filebeat/module/system/auth/config/auth.yml +++ b/filebeat/module/system/auth/config/auth.yml @@ -1,4 +1,4 @@ -input_type: log +type: log paths: {{ range $i, $path := .paths }} - {{$path}} diff --git a/filebeat/module/system/auth/test/test.log-expected.json b/filebeat/module/system/auth/test/test.log-expected.json index 9d24032c6b43..f601ddb62773 100644 --- a/filebeat/module/system/auth/test/test.log-expected.json +++ b/filebeat/module/system/auth/test/test.log-expected.json @@ -1,362 +1,372 @@ [ - { - "_index" : "test-filebeat-modules", - "_type" : "log", - "_id" : "AVpwYhxbd3MQAzSWoGjF", - "_score" : 1.0, - "_source" : { - "@timestamp" : "2017-02-23T00:13:35.000Z", - "system" : { - "auth" : { - "hostname" : "localhost", - "ssh" : { - "method" : "password", - "port" : "58803", - "ip" : "192.168.33.1", - "event" : "Accepted" - }, - "pid" : "7483", - "user" : "vagrant", - "timestamp" : "Feb 23 00:13:35" + { + "_index": "test-filebeat-modules", + "_type": "log", + "_id": "AVpwYhxbd3MQAzSWoGjF", + "_score": 1.0, + "_source": { + "@timestamp": "2017-02-23T00:13:35.000Z", + "system": { + "auth": { + "hostname": "localhost", + "ssh": { + "method": "password", + "port": "58803", + "ip": "192.168.33.1", + "event": "Accepted" + }, + "pid": "7483", + "user": "vagrant", + "timestamp": "Feb 23 00:13:35" + } + }, + "offset": 254, + "beat": { + "hostname": "a-mac-with-esc-key.local", + "name": "a-mac-with-esc-key.local", + "version": "6.0.0-alpha1" + }, + "prospector": { + "type": "log" + }, + "source": "/Users/tsg/src/github.com/elastic/beats/filebeat/module/system/auth/test/test.log", + "fileset": { + "module": "system", + "name": "auth" + } } - }, - "offset" : 254, - "beat" : { - "hostname" : "a-mac-with-esc-key.local", - "name" : "a-mac-with-esc-key.local", - "version" : "6.0.0-alpha1" - }, - "input_type" : "log", - "source" : "/Users/tsg/src/github.com/elastic/beats/filebeat/module/system/auth/test/test.log", - "fileset" : { - "module" : "system", - "name" : "auth" - }, - "type" : "log" - } - }, - { - "_index" : "test-filebeat-modules", - "_type" : "log", - "_id" : "AVpwYhxbd3MQAzSWoGjK", - "_score" : 1.0, - "_source" : { - "@timestamp" : "2017-02-23T00:08:48.000Z", - "system" : { - "auth" : { - "hostname" : "localhost", - "sudo" : { - "tty" : "pts/1", - "pwd" : "/home/vagrant", - "user" : "root", - "command" : "/bin/cat /var/log/secure" - }, - "user" : "vagrant", - "timestamp" : "Feb 23 00:08:48" + }, + { + "_index": "test-filebeat-modules", + "_type": "log", + "_id": "AVpwYhxbd3MQAzSWoGjK", + "_score": 1.0, + "_source": { + "@timestamp": "2017-02-23T00:08:48.000Z", + "system": { + "auth": { + "hostname": "localhost", + "sudo": { + "tty": "pts/1", + "pwd": "/home/vagrant", + "user": "root", + "command": "/bin/cat /var/log/secure" + }, + "user": "vagrant", + "timestamp": "Feb 23 00:08:48" + } + }, + "offset": 736, + "beat": { + "hostname": "a-mac-with-esc-key.local", + "name": "a-mac-with-esc-key.local", + "version": "6.0.0-alpha1" + }, + "prospector": { + "type": "log" + }, + "source": "/Users/tsg/src/github.com/elastic/beats/filebeat/module/system/auth/test/test.log", + "fileset": { + "module": "system", + "name": "auth" + } } - }, - "offset" : 736, - "beat" : { - "hostname" : "a-mac-with-esc-key.local", - "name" : "a-mac-with-esc-key.local", - "version" : "6.0.0-alpha1" - }, - "input_type" : "log", - "source" : "/Users/tsg/src/github.com/elastic/beats/filebeat/module/system/auth/test/test.log", - "fileset" : { - "module" : "system", - "name" : "auth" - }, - "type" : "log" - } - }, - { - "_index" : "test-filebeat-modules", - "_type" : "log", - "_id" : "AVpwYhxbd3MQAzSWoGjG", - "_score" : 1.0, - "_source" : { - "@timestamp" : "2017-02-21T21:56:12.000Z", - "system" : { - "auth" : { - "hostname" : "localhost", - "ssh" : { - "ip" : "10.0.2.2", - "event" : "Invalid" - }, - "pid" : "3430", - "user" : "test", - "timestamp" : "Feb 21 21:56:12" + }, + { + "_index": "test-filebeat-modules", + "_type": "log", + "_id": "AVpwYhxbd3MQAzSWoGjG", + "_score": 1.0, + "_source": { + "@timestamp": "2017-02-21T21:56:12.000Z", + "system": { + "auth": { + "hostname": "localhost", + "ssh": { + "ip": "10.0.2.2", + "event": "Invalid" + }, + "pid": "3430", + "user": "test", + "timestamp": "Feb 21 21:56:12" + } + }, + "offset": 324, + "beat": { + "hostname": "a-mac-with-esc-key.local", + "name": "a-mac-with-esc-key.local", + "version": "6.0.0-alpha1" + }, + "prospector": { + "type": "log" + }, + "source": "/Users/tsg/src/github.com/elastic/beats/filebeat/module/system/auth/test/test.log", + "fileset": { + "module": "system", + "name": "auth" + } } - }, - "offset" : 324, - "beat" : { - "hostname" : "a-mac-with-esc-key.local", - "name" : "a-mac-with-esc-key.local", - "version" : "6.0.0-alpha1" - }, - "input_type" : "log", - "source" : "/Users/tsg/src/github.com/elastic/beats/filebeat/module/system/auth/test/test.log", - "fileset" : { - "module" : "system", - "name" : "auth" - }, - "type" : "log" - } - }, - { - "_index" : "test-filebeat-modules", - "_type" : "log", - "_id" : "AVpwYhxbd3MQAzSWoGjJ", - "_score" : 1.0, - "_source" : { - "@timestamp" : "2017-02-19T15:30:04.000Z", - "system" : { - "auth" : { - "hostname" : "slave22", - "ssh" : { - "dropped_ip" : "123.57.245.163" - }, - "pid" : "18406", - "timestamp" : "Feb 19 15:30:04" + }, + { + "_index": "test-filebeat-modules", + "_type": "log", + "_id": "AVpwYhxbd3MQAzSWoGjJ", + "_score": 1.0, + "_source": { + "@timestamp": "2017-02-19T15:30:04.000Z", + "system": { + "auth": { + "hostname": "slave22", + "ssh": { + "dropped_ip": "123.57.245.163" + }, + "pid": "18406", + "timestamp": "Feb 19 15:30:04" + } + }, + "offset": 617, + "beat": { + "hostname": "a-mac-with-esc-key.local", + "name": "a-mac-with-esc-key.local", + "version": "6.0.0-alpha1" + }, + "prospector": { + "type": "log" + }, + "source": "/Users/tsg/src/github.com/elastic/beats/filebeat/module/system/auth/test/test.log", + "fileset": { + "module": "system", + "name": "auth" + } } - }, - "offset" : 617, - "beat" : { - "hostname" : "a-mac-with-esc-key.local", - "name" : "a-mac-with-esc-key.local", - "version" : "6.0.0-alpha1" - }, - "input_type" : "log", - "source" : "/Users/tsg/src/github.com/elastic/beats/filebeat/module/system/auth/test/test.log", - "fileset" : { - "module" : "system", - "name" : "auth" - }, - "type" : "log" - } - }, - { - "_index" : "test-filebeat-modules", - "_type" : "log", - "_id" : "AVpwYhxbd3MQAzSWoGjL", - "_score" : 1.0, - "_source" : { - "@timestamp" : "2017-02-24T00:13:02.000Z", - "system" : { - "auth" : { - "hostname" : "precise32", - "sudo" : { - "tty" : "pts/1", - "pwd" : "/home/vagrant", - "error" : "user NOT in sudoers", - "user" : "root", - "command" : "/bin/ls" - }, - "user" : "tsg", - "timestamp" : "Feb 24 00:13:02" + }, + { + "_index": "test-filebeat-modules", + "_type": "log", + "_id": "AVpwYhxbd3MQAzSWoGjL", + "_score": 1.0, + "_source": { + "@timestamp": "2017-02-24T00:13:02.000Z", + "system": { + "auth": { + "hostname": "precise32", + "sudo": { + "tty": "pts/1", + "pwd": "/home/vagrant", + "error": "user NOT in sudoers", + "user": "root", + "command": "/bin/ls" + }, + "user": "tsg", + "timestamp": "Feb 24 00:13:02" + } + }, + "offset": 861, + "beat": { + "hostname": "a-mac-with-esc-key.local", + "name": "a-mac-with-esc-key.local", + "version": "6.0.0-alpha1" + }, + "prospector": { + "type": "log" + }, + "source": "/Users/tsg/src/github.com/elastic/beats/filebeat/module/system/auth/test/test.log", + "fileset": { + "module": "system", + "name": "auth" + } } - }, - "offset" : 861, - "beat" : { - "hostname" : "a-mac-with-esc-key.local", - "name" : "a-mac-with-esc-key.local", - "version" : "6.0.0-alpha1" - }, - "input_type" : "log", - "source" : "/Users/tsg/src/github.com/elastic/beats/filebeat/module/system/auth/test/test.log", - "fileset" : { - "module" : "system", - "name" : "auth" - }, - "type" : "log" - } - }, - { - "_index" : "test-filebeat-modules", - "_type" : "log", - "_id" : "AVpwYhxbd3MQAzSWoGjM", - "_score" : 1.0, - "_source" : { - "@timestamp" : "2017-02-22T11:47:05.000Z", - "system" : { - "auth" : { - "hostname" : "localhost", - "pid" : "6991", - "groupadd" : { - "gid" : "48", - "name" : "apache" - }, - "timestamp" : "Feb 22 11:47:05" + }, + { + "_index": "test-filebeat-modules", + "_type": "log", + "_id": "AVpwYhxbd3MQAzSWoGjM", + "_score": 1.0, + "_source": { + "@timestamp": "2017-02-22T11:47:05.000Z", + "system": { + "auth": { + "hostname": "localhost", + "pid": "6991", + "groupadd": { + "gid": "48", + "name": "apache" + }, + "timestamp": "Feb 22 11:47:05" + } + }, + "offset": 934, + "beat": { + "hostname": "a-mac-with-esc-key.local", + "name": "a-mac-with-esc-key.local", + "version": "6.0.0-alpha1" + }, + "prospector": { + "type": "log" + }, + "source": "/Users/tsg/src/github.com/elastic/beats/filebeat/module/system/auth/test/test.log", + "fileset": { + "module": "system", + "name": "auth" + } } - }, - "offset" : 934, - "beat" : { - "hostname" : "a-mac-with-esc-key.local", - "name" : "a-mac-with-esc-key.local", - "version" : "6.0.0-alpha1" - }, - "input_type" : "log", - "source" : "/Users/tsg/src/github.com/elastic/beats/filebeat/module/system/auth/test/test.log", - "fileset" : { - "module" : "system", - "name" : "auth" - }, - "type" : "log" - } - }, - { - "_index" : "test-filebeat-modules", - "_type" : "log", - "_id" : "AVpwYhxbd3MQAzSWoGjN", - "_score" : 1.0, - "_source" : { - "@timestamp" : "2017-02-22T11:47:05.000Z", - "system" : { - "auth" : { - "hostname" : "localhost", - "pid" : "6995", - "useradd" : { - "uid" : "48", - "gid" : "48", - "shell" : "/sbin/nologin", - "name" : "apache", - "home" : "/usr/share/httpd" - }, - "timestamp" : "Feb 22 11:47:05" + }, + { + "_index": "test-filebeat-modules", + "_type": "log", + "_id": "AVpwYhxbd3MQAzSWoGjN", + "_score": 1.0, + "_source": { + "@timestamp": "2017-02-22T11:47:05.000Z", + "system": { + "auth": { + "hostname": "localhost", + "pid": "6995", + "useradd": { + "uid": "48", + "gid": "48", + "shell": "/sbin/nologin", + "name": "apache", + "home": "/usr/share/httpd" + }, + "timestamp": "Feb 22 11:47:05" + } + }, + "offset": 1057, + "beat": { + "hostname": "a-mac-with-esc-key.local", + "name": "a-mac-with-esc-key.local", + "version": "6.0.0-alpha1" + }, + "prospector": { + "type": "log" + }, + "source": "/Users/tsg/src/github.com/elastic/beats/filebeat/module/system/auth/test/test.log", + "fileset": { + "module": "system", + "name": "auth" + } } - }, - "offset" : 1057, - "beat" : { - "hostname" : "a-mac-with-esc-key.local", - "name" : "a-mac-with-esc-key.local", - "version" : "6.0.0-alpha1" - }, - "input_type" : "log", - "source" : "/Users/tsg/src/github.com/elastic/beats/filebeat/module/system/auth/test/test.log", - "fileset" : { - "module" : "system", - "name" : "auth" - }, - "type" : "log" - } - }, - { - "_index" : "test-filebeat-modules", - "_type" : "log", - "_id" : "AVpwYhxbd3MQAzSWoGjE", - "_score" : 1.0, - "_source" : { - "@timestamp" : "2017-02-21T21:54:44.000Z", - "system" : { - "auth" : { - "hostname" : "localhost", - "ssh" : { - "method" : "publickey", - "signature" : "RSA 39:33:99:e9:a0:dc:f2:33:a3:e5:72:3b:7c:3a:56:84", - "port" : "63673", - "ip" : "10.0.2.2", - "event" : "Accepted" - }, - "pid" : "3402", - "user" : "vagrant", - "timestamp" : "Feb 21 21:54:44" + }, + { + "_index": "test-filebeat-modules", + "_type": "log", + "_id": "AVpwYhxbd3MQAzSWoGjE", + "_score": 1.0, + "_source": { + "@timestamp": "2017-02-21T21:54:44.000Z", + "system": { + "auth": { + "hostname": "localhost", + "ssh": { + "method": "publickey", + "signature": "RSA 39:33:99:e9:a0:dc:f2:33:a3:e5:72:3b:7c:3a:56:84", + "port": "63673", + "ip": "10.0.2.2", + "event": "Accepted" + }, + "pid": "3402", + "user": "vagrant", + "timestamp": "Feb 21 21:54:44" + } + }, + "offset": 152, + "beat": { + "hostname": "a-mac-with-esc-key.local", + "name": "a-mac-with-esc-key.local", + "version": "6.0.0-alpha1" + }, + "prospector": { + "type": "log" + }, + "source": "/Users/tsg/src/github.com/elastic/beats/filebeat/module/system/auth/test/test.log", + "fileset": { + "module": "system", + "name": "auth" + } } - }, - "offset" : 152, - "beat" : { - "hostname" : "a-mac-with-esc-key.local", - "name" : "a-mac-with-esc-key.local", - "version" : "6.0.0-alpha1" - }, - "input_type" : "log", - "source" : "/Users/tsg/src/github.com/elastic/beats/filebeat/module/system/auth/test/test.log", - "fileset" : { - "module" : "system", - "name" : "auth" - }, - "type" : "log" - } - }, - { - "_index" : "test-filebeat-modules", - "_type" : "log", - "_id" : "AVpwYhxbd3MQAzSWoGjH", - "_score" : 1.0, - "_source" : { - "@timestamp" : "2017-02-20T08:35:22.000Z", - "system" : { - "auth" : { - "hostname" : "slave22", - "ssh" : { - "geoip" : { - "continent_name" : "Asia", - "city_name" : "Guangzhou", - "country_iso_code" : "CN", - "region_name" : "Guangdong", - "location" : { - "lon" : 113.25, - "lat" : 23.1167 - } + }, + { + "_index": "test-filebeat-modules", + "_type": "log", + "_id": "AVpwYhxbd3MQAzSWoGjH", + "_score": 1.0, + "_source": { + "@timestamp": "2017-02-20T08:35:22.000Z", + "system": { + "auth": { + "hostname": "slave22", + "ssh": { + "geoip": { + "continent_name": "Asia", + "city_name": "Guangzhou", + "country_iso_code": "CN", + "region_name": "Guangdong", + "location": { + "lon": 113.25, + "lat": 23.1167 + } + }, + "method": "password", + "port": "29160", + "ip": "116.31.116.24", + "event": "Failed" + }, + "pid": "5774", + "user": "root", + "timestamp": "Feb 20 08:35:22" + } + }, + "offset": 420, + "beat": { + "hostname": "a-mac-with-esc-key.local", + "name": "a-mac-with-esc-key.local", + "version": "6.0.0-alpha1" + }, + "prospector": { + "type": "log" }, - "method" : "password", - "port" : "29160", - "ip" : "116.31.116.24", - "event" : "Failed" - }, - "pid" : "5774", - "user" : "root", - "timestamp" : "Feb 20 08:35:22" + "source": "/Users/tsg/src/github.com/elastic/beats/filebeat/module/system/auth/test/test.log", + "fileset": { + "module": "system", + "name": "auth" + } } - }, - "offset" : 420, - "beat" : { - "hostname" : "a-mac-with-esc-key.local", - "name" : "a-mac-with-esc-key.local", - "version" : "6.0.0-alpha1" - }, - "input_type" : "log", - "source" : "/Users/tsg/src/github.com/elastic/beats/filebeat/module/system/auth/test/test.log", - "fileset" : { - "module" : "system", - "name" : "auth" - }, - "type" : "log" - } - }, - { - "_index" : "test-filebeat-modules", - "_type" : "log", - "_id" : "AVpwYhxbd3MQAzSWoGjI", - "_score" : 1.0, - "_source" : { - "@timestamp" : "2017-02-21T23:35:33.000Z", - "system" : { - "auth" : { - "hostname" : "localhost", - "sudo" : { - "tty" : "pts/0", - "pwd" : "/home/vagrant", - "user" : "root", - "command" : "/bin/ls" - }, - "user" : "vagrant", - "timestamp" : "Feb 21 23:35:33" + }, + { + "_index": "test-filebeat-modules", + "_type": "log", + "_id": "AVpwYhxbd3MQAzSWoGjI", + "_score": 1.0, + "_source": { + "@timestamp": "2017-02-21T23:35:33.000Z", + "system": { + "auth": { + "hostname": "localhost", + "sudo": { + "tty": "pts/0", + "pwd": "/home/vagrant", + "user": "root", + "command": "/bin/ls" + }, + "user": "vagrant", + "timestamp": "Feb 21 23:35:33" + } + }, + "offset": 522, + "beat": { + "hostname": "a-mac-with-esc-key.local", + "name": "a-mac-with-esc-key.local", + "version": "6.0.0-alpha1" + }, + "prospector": { + "type": "log" + }, + "source": "/Users/tsg/src/github.com/elastic/beats/filebeat/module/system/auth/test/test.log", + "fileset": { + "module": "system", + "name": "auth" + } } - }, - "offset" : 522, - "beat" : { - "hostname" : "a-mac-with-esc-key.local", - "name" : "a-mac-with-esc-key.local", - "version" : "6.0.0-alpha1" - }, - "input_type" : "log", - "source" : "/Users/tsg/src/github.com/elastic/beats/filebeat/module/system/auth/test/test.log", - "fileset" : { - "module" : "system", - "name" : "auth" - }, - "type" : "log" } - } ] diff --git a/filebeat/module/system/syslog/config/syslog.yml b/filebeat/module/system/syslog/config/syslog.yml index 1af7dee5e13c..003b5d06b3ec 100644 --- a/filebeat/module/system/syslog/config/syslog.yml +++ b/filebeat/module/system/syslog/config/syslog.yml @@ -1,4 +1,4 @@ -input_type: log +type: log paths: {{ range $i, $path := .paths }} - {{$path}} diff --git a/filebeat/module/system/syslog/test/darwin-syslog-sample.log-expected.json b/filebeat/module/system/syslog/test/darwin-syslog-sample.log-expected.json index cf6e1ae645ff..fdf1f34aaa0a 100644 --- a/filebeat/module/system/syslog/test/darwin-syslog-sample.log-expected.json +++ b/filebeat/module/system/syslog/test/darwin-syslog-sample.log-expected.json @@ -1,90 +1,93 @@ [ - { - "_index" : "filebeat-2017.01.27", - "_type" : "log", - "_id" : "AVngMBcpUmQau8zc_UuM", - "_score" : 1.0, - "_source" : { - "@timestamp" : "2017-12-13T11:35:28.000Z", - "system" : { - "syslog" : { - "hostname" : "a-mac-with-esc-key", - "pid" : "21412", - "program" : "GoogleSoftwareUpdateAgent", - "message" : "2016-12-13 11:35:28.420 GoogleSoftwareUpdateAgent[21412/0x700007399000] [lvl=2] -[KSAgentApp updateProductWithProductID:usingEngine:] Checking for updates for \"All Products\" using engine \n\t\t>>\n\t\tprocessor=\n\t\t\tisProcessing=NO actionsCompleted=0 progress=0.00\n\t\t\terrors=0 currentActionErrors=0\n\t\t\tevents=0 currentActionEvents=0\n\t\t\tactionQueue=( )\n\t\t>\n\t\tdelegate=(null)\n\t\tserverInfoStore=(null)\n\t\terrors=0\n\t>", - "timestamp" : "Dec 13 11:35:28" + { + "_index": "filebeat-2017.01.27", + "_type": "log", + "_id": "AVngMBcpUmQau8zc_UuM", + "_score": 1.0, + "_source": { + "@timestamp": "2017-12-13T11:35:28.000Z", + "system": { + "syslog": { + "hostname": "a-mac-with-esc-key", + "pid": "21412", + "program": "GoogleSoftwareUpdateAgent", + "message": "2016-12-13 11:35:28.420 GoogleSoftwareUpdateAgent[21412/0x700007399000] [lvl=2] -[KSAgentApp updateProductWithProductID:usingEngine:] Checking for updates for \"All Products\" using engine \n\t\t>>\n\t\tprocessor=\n\t\t\tisProcessing=NO actionsCompleted=0 progress=0.00\n\t\t\terrors=0 currentActionErrors=0\n\t\t\tevents=0 currentActionEvents=0\n\t\t\tactionQueue=( )\n\t\t>\n\t\tdelegate=(null)\n\t\tserverInfoStore=(null)\n\t\terrors=0\n\t>", + "timestamp": "Dec 13 11:35:28" + } + }, + "offset": 907, + "beat": { + "hostname": "a-mac-with-esc-key.local", + "name": "a-mac-with-esc-key.local", + "version": "6.0.0-alpha1" + }, + "prospector": { + "type": "log" + }, + "source": "module/system/syslog/test/darwin-syslog-sample.log", + "fields": { + "source_type": "system-syslog" + } } - }, - "offset" : 907, - "beat" : { - "hostname" : "a-mac-with-esc-key.local", - "name" : "a-mac-with-esc-key.local", - "version" : "6.0.0-alpha1" - }, - "input_type" : "log", - "source" : "module/system/syslog/test/darwin-syslog-sample.log", - "fields" : { - "source_type" : "system-syslog" - }, - "type" : "log" - } - }, - { - "_index" : "filebeat-2017.01.27", - "_type" : "log", - "_id" : "AVngMCdNUmQau8zc_UuN", - "_score" : 1.0, - "_source" : { - "@timestamp" : "2017-12-13T11:35:28.000Z", - "system" : { - "syslog" : { - "hostname" : "a-mac-with-esc-key", - "pid" : "21412", - "program" : "GoogleSoftwareUpdateAgent", - "message" : "2016-12-13 11:35:28.421 GoogleSoftwareUpdateAgent[21412/0x700007399000] [lvl=2] -[KSUpdateEngine updateAllExceptProduct:] KSUpdateEngine updating all installed products, except:'com.google.Keystone'.", - "timestamp" : "Dec 13 11:35:28" + }, + { + "_index": "filebeat-2017.01.27", + "_type": "log", + "_id": "AVngMCdNUmQau8zc_UuN", + "_score": 1.0, + "_source": { + "@timestamp": "2017-12-13T11:35:28.000Z", + "system": { + "syslog": { + "hostname": "a-mac-with-esc-key", + "pid": "21412", + "program": "GoogleSoftwareUpdateAgent", + "message": "2016-12-13 11:35:28.421 GoogleSoftwareUpdateAgent[21412/0x700007399000] [lvl=2] -[KSUpdateEngine updateAllExceptProduct:] KSUpdateEngine updating all installed products, except:'com.google.Keystone'.", + "timestamp": "Dec 13 11:35:28" + } + }, + "offset": 1176, + "beat": { + "hostname": "a-mac-with-esc-key.local", + "name": "a-mac-with-esc-key.local", + "version": "6.0.0-alpha1" + }, + "prospector": { + "type": "log" + }, + "source": "module/system/syslog/test/darwin-syslog-sample.log", + "fields": { + "source_type": "system-syslog" + } } - }, - "offset" : 1176, - "beat" : { - "hostname" : "a-mac-with-esc-key.local", - "name" : "a-mac-with-esc-key.local", - "version" : "6.0.0-alpha1" - }, - "input_type" : "log", - "source" : "module/system/syslog/test/darwin-syslog-sample.log", - "fields" : { - "source_type" : "system-syslog" - }, - "type" : "log" - } - }, - { - "_index": "test-filebeat-modules", - "_type": "doc", - "_id": "AVtFVFY1nbkdi71WgGxo", - "_score": 1, - "_source": { - "@timestamp": "2017-04-04T03:39:57.000Z", - "system": { - "syslog": { - "message": "--- last message repeated 1 time ---", - "timestamp": "Apr 4 03:39:57" + }, + { + "_index": "test-filebeat-modules", + "_type": "doc", + "_id": "AVtFVFY1nbkdi71WgGxo", + "_score": 1, + "_source": { + "@timestamp": "2017-04-04T03:39:57.000Z", + "system": { + "syslog": { + "message": "--- last message repeated 1 time ---", + "timestamp": "Apr 4 03:39:57" + } + }, + "offset": 1228, + "beat": { + "hostname": "a-mac-with-esc-key-2.local", + "name": "a-mac-with-esc-key-2.local", + "version": "6.0.0-alpha1" + }, + "prospector": { + "type": "log" + }, + "source": "/Users/tsg/src/github.com/elastic/beats/filebeat/module/system/syslog/test/darwin-syslog-sample.log", + "fileset": { + "module": "system", + "name": "syslog" + } } - }, - "offset": 1228, - "beat": { - "hostname": "a-mac-with-esc-key-2.local", - "name": "a-mac-with-esc-key-2.local", - "version": "6.0.0-alpha1" - }, - "input_type": "log", - "source": "/Users/tsg/src/github.com/elastic/beats/filebeat/module/system/syslog/test/darwin-syslog-sample.log", - "fileset": { - "module": "system", - "name": "syslog" - }, - "type": "log" } - } ] diff --git a/filebeat/prospector/config.go b/filebeat/prospector/config.go index 3658c77fa8e4..99d8d2ad41b3 100644 --- a/filebeat/prospector/config.go +++ b/filebeat/prospector/config.go @@ -4,16 +4,26 @@ import ( "time" cfg "github.com/elastic/beats/filebeat/config" + "github.com/elastic/beats/libbeat/logp" ) var ( defaultConfig = prospectorConfig{ ScanFrequency: 10 * time.Second, - InputType: cfg.DefaultInputType, + Type: cfg.DefaultType, } ) type prospectorConfig struct { ScanFrequency time.Duration `config:"scan_frequency" validate:"min=0,nonzero"` + Type string `config:"type"` InputType string `config:"input_type"` } + +func (c *prospectorConfig) Validate() error { + if c.InputType != "" { + logp.Deprecate("6.0.0", "input_type prospector config is deprecated. Use type instead.") + c.Type = c.InputType + } + return nil +} diff --git a/filebeat/prospector/log/config.go b/filebeat/prospector/log/config.go index f040f82ef466..cee66710bcbb 100644 --- a/filebeat/prospector/log/config.go +++ b/filebeat/prospector/log/config.go @@ -15,7 +15,7 @@ import ( var ( defaultConfig = config{ // Common - InputType: cfg.DefaultInputType, + Type: cfg.DefaultType, CleanInactive: 0, // Prospector @@ -44,6 +44,7 @@ var ( type config struct { // Common + Type string `config:"type"` InputType string `config:"input_type"` CleanInactive time.Duration `config:"clean_inactive" validate:"min=0"` @@ -84,8 +85,13 @@ type config struct { func (c *config) Validate() error { + // DEPRECATED 6.0.0: warning is already outputted on propsector level + if c.InputType != "" { + c.Type = c.InputType + } + // Prospector - if c.InputType == cfg.LogInputType && len(c.Paths) == 0 { + if c.Type == cfg.LogType && len(c.Paths) == 0 { return fmt.Errorf("No paths were defined for prospector") } @@ -99,8 +105,8 @@ func (c *config) Validate() error { // Harvester // Check input type - if _, ok := cfg.ValidInputType[c.InputType]; !ok { - return fmt.Errorf("Invalid input type: %v", c.InputType) + if _, ok := cfg.ValidType[c.Type]; !ok { + return fmt.Errorf("Invalid input type: %v", c.Type) } if c.JSON != nil && len(c.JSON.MessageKey) == 0 && diff --git a/filebeat/prospector/log/config_test.go b/filebeat/prospector/log/config_test.go index c79a0d7361bc..6537fbcecf3d 100644 --- a/filebeat/prospector/log/config_test.go +++ b/filebeat/prospector/log/config_test.go @@ -47,7 +47,7 @@ func TestCleanOlderIgnoreOlder(t *testing.T) { config := config{ CleanInactive: 10*time.Hour + defaultConfig.ScanFrequency + 1*time.Second, IgnoreOlder: 10 * time.Hour, - InputType: "log", + Type: "log", Paths: []string{"hello"}, } diff --git a/filebeat/prospector/log/harvester.go b/filebeat/prospector/log/harvester.go index 6b61dab9ed45..854e72dd3a5f 100644 --- a/filebeat/prospector/log/harvester.go +++ b/filebeat/prospector/log/harvester.go @@ -108,10 +108,10 @@ func NewHarvester( // open does open the file given under h.Path and assigns the file handler to h.file func (h *Harvester) open() error { - switch h.config.InputType { - case cfg.StdinInputType: + switch h.config.Type { + case cfg.StdinType: return h.openStdin() - case cfg.LogInputType: + case cfg.LogType: return h.openFile() default: return fmt.Errorf("Invalid harvester type: %+v", h.config) @@ -129,6 +129,7 @@ func (h *Harvester) forwardEvent(data *util.Data) error { if data.HasEvent() { data.Event[common.EventMetadataKey] = h.config.EventMetadata + data.Event.Put("prospector.type", h.config.Type) // run the filters before sending to spooler data.Event = h.processors.Run(data.Event) diff --git a/filebeat/prospector/log/log.go b/filebeat/prospector/log/log.go index 6e1fc81a1829..2a4639dfd12f 100644 --- a/filebeat/prospector/log/log.go +++ b/filebeat/prospector/log/log.go @@ -149,7 +149,6 @@ func (h *Harvester) Start() { "@timestamp": common.Time(message.Ts), "source": state.Source, "offset": state.Offset, // Offset here is the offset before the starting char. - "input_type": h.config.InputType, } data.Event.DeepUpdate(message.Fields) diff --git a/filebeat/prospector/log/prospector.go b/filebeat/prospector/log/prospector.go index 4aa913a8b05c..944d7b951304 100644 --- a/filebeat/prospector/log/prospector.go +++ b/filebeat/prospector/log/prospector.go @@ -142,7 +142,7 @@ func (p *Prospector) Run() { } } else { // Check if existing source on disk and state are the same. Remove if not the case. - newState := file.NewState(stat, state.Source, p.config.InputType) + newState := file.NewState(stat, state.Source, p.config.Type) if !newState.FileStateOS.IsSame(state.FileStateOS) { p.removeState(state) logp.Debug("prospector", "Remove state for file as file removed or renamed: %s", state.Source) @@ -284,7 +284,7 @@ func (p *Prospector) scan() { logp.Debug("prospector", "Check file for harvesting: %s", path) // Create new state for comparison - newState := file.NewState(info, path, p.config.InputType) + newState := file.NewState(info, path, p.config.Type) // Load last state lastState := p.states.FindPrevious(newState) diff --git a/filebeat/prospector/prospector.go b/filebeat/prospector/prospector.go index c405ceeb7595..8c9ecb2b982b 100644 --- a/filebeat/prospector/prospector.go +++ b/filebeat/prospector/prospector.go @@ -69,13 +69,13 @@ func (p *Prospector) initProspectorer(outlet channel.Outleter, states []file.Sta var prospectorer Prospectorer var err error - switch p.config.InputType { - case cfg.StdinInputType: + switch p.config.Type { + case cfg.StdinType: prospectorer, err = stdin.NewProspector(config, outlet) - case cfg.LogInputType: + case cfg.LogType: prospectorer, err = log.NewProspector(config, states, outlet, p.done) default: - return fmt.Errorf("invalid prospector type: %v. Change input_type", p.config.InputType) + return fmt.Errorf("invalid prospector type: %v. Change type", p.config.Type) } if err != nil { @@ -90,7 +90,7 @@ func (p *Prospector) initProspectorer(outlet channel.Outleter, states []file.Sta // Start starts the prospector func (p *Prospector) Start() { p.wg.Add(1) - logp.Info("Starting prospector of type: %v; id: %v ", p.config.InputType, p.ID()) + logp.Info("Starting prospector of type: %v; id: %v ", p.config.Type, p.ID()) onceWg := sync.WaitGroup{} if p.Once { diff --git a/filebeat/scripts/module/fileset/config/config.yml b/filebeat/scripts/module/fileset/config/config.yml index 56e3dfd95b82..0afd17317d4f 100644 --- a/filebeat/scripts/module/fileset/config/config.yml +++ b/filebeat/scripts/module/fileset/config/config.yml @@ -1,4 +1,4 @@ -input_type: log +type: log paths: {{ range $i, $path := .paths }} - {{$path}} diff --git a/filebeat/tests/load/filebeat.yml b/filebeat/tests/load/filebeat.yml index 6c48d42fd556..ae0e861577cb 100644 --- a/filebeat/tests/load/filebeat.yml +++ b/filebeat/tests/load/filebeat.yml @@ -1,6 +1,6 @@ filebeat: prospectors: - - input_type: log + - type: log paths: - /Users/ruflin/Dev/gopath/src/github.com/elastic/filebeat/tests/load/logs/* diff --git a/filebeat/tests/open-file-handlers/filebeat.yml b/filebeat/tests/open-file-handlers/filebeat.yml index f5d2c4228bee..996f9d6a55b2 100644 --- a/filebeat/tests/open-file-handlers/filebeat.yml +++ b/filebeat/tests/open-file-handlers/filebeat.yml @@ -1,7 +1,7 @@ filebeat.prospectors: # Reads logs generated by the generator containers -- input_type: log +- type: log paths: - /logfiles/* close_inactive: 1m @@ -9,12 +9,12 @@ filebeat.prospectors: clean_inactive: 5m # Reads docker container logs -#- input_type: log +#- type: log # paths: # - /var/lib/docker/containers/*/*-json.log # Reads system logs -#- input_type: log +#- type: log # paths: # - /var/log/* # diff --git a/filebeat/tests/system/config/filebeat.yml.j2 b/filebeat/tests/system/config/filebeat.yml.j2 index 22f8d195399b..1ad3312c48e0 100644 --- a/filebeat/tests/system/config/filebeat.yml.j2 +++ b/filebeat/tests/system/config/filebeat.yml.j2 @@ -5,7 +5,8 @@ filebeat.prospectors: {% set prospectors = true %} {% endif %} {% if prospectors %} -- input_type: {{input_type | default("log") }} +- type: {{type | default("log") }} + input_type: {{input_type_deprecated }} # Paths that should be crawled and fetched {% if path %}paths: - {{ path }}{% endif %} diff --git a/filebeat/tests/system/test_deprecated.py b/filebeat/tests/system/test_deprecated.py new file mode 100644 index 000000000000..0dd25fba926c --- /dev/null +++ b/filebeat/tests/system/test_deprecated.py @@ -0,0 +1,40 @@ +# coding=utf-8 + +from filebeat import BaseTest +import os +import codecs +import time + +""" +Test Harvesters +""" + + +class Test(BaseTest): + + def test_input_type_deprecated(self): + """ + Checks that harvesting works with deprecated input_type but message is outputted + """ + + self.render_config_template( + path=os.path.abspath(self.working_dir) + "/log/test.log", + input_type_deprecated="log", + scan_frequency="0.1s" + ) + os.mkdir(self.working_dir + "/log/") + + logfile = self.working_dir + "/log/test.log" + + with open(logfile, 'w') as f: + f.write("Hello world\n") + + filebeat = self.start_beat() + + # Let it read the file + self.wait_until( + lambda: self.output_has(lines=1), max_timeout=10) + + filebeat.check_kill_and_wait() + + assert self.log_contains("DEPRECATED: input_type prospector config is deprecated") diff --git a/filebeat/tests/system/test_prospector.py b/filebeat/tests/system/test_prospector.py index 75f0a552df97..6898af916242 100644 --- a/filebeat/tests/system/test_prospector.py +++ b/filebeat/tests/system/test_prospector.py @@ -79,7 +79,7 @@ def test_stdin(self): Test stdin input. Checks if reading is continued after the first read. """ self.render_config_template( - input_type="stdin" + type="stdin" ) proc = self.start_beat() @@ -115,7 +115,7 @@ def test_stdin_eof(self): Test that Filebeat works when stdin is closed. """ self.render_config_template( - input_type="stdin", + type="stdin", close_eof="true", ) diff --git a/filebeat/tests/system/test_reload.py b/filebeat/tests/system/test_reload.py index d6fdc3a7cea6..084d2a01eab5 100644 --- a/filebeat/tests/system/test_reload.py +++ b/filebeat/tests/system/test_reload.py @@ -7,7 +7,7 @@ prospectorConfigTemplate = """ -- input_type: log +- type: log paths: - {} scan_frequency: 1s diff --git a/filebeat/tests/system/test_shutdown.py b/filebeat/tests/system/test_shutdown.py index a87ac65be789..7eff12de8d66 100644 --- a/filebeat/tests/system/test_shutdown.py +++ b/filebeat/tests/system/test_shutdown.py @@ -153,7 +153,7 @@ def test_stopping_empty_path(self): """ prospector_raw = """ -- input_type: log +- type: log paths: [] """