diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 6c9a3c4513a..46c893e8cd1 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -15,6 +15,7 @@ https://github.com/elastic/beats/compare/v6.0.0-alpha1...master[Check the HEAD d *Affecting all Beats* *Filebeat* +- Rename `input_type` field to `prospector.type` {pull}4294[4294] *Heartbeat* @@ -66,6 +67,8 @@ https://github.com/elastic/beats/compare/v6.0.0-alpha1...master[Check the HEAD d *Affecting all Beats* *Filebeat* +- Deprecate `input_type` prospector config. Use `type` config option instead. {pull}4294[4294] + *Heartbeat* diff --git a/filebeat/_meta/common.full.p2.yml b/filebeat/_meta/common.full.p2.yml index 1a7b64c4658..6cccbd8f7e8 100644 --- a/filebeat/_meta/common.full.p2.yml +++ b/filebeat/_meta/common.full.p2.yml @@ -14,7 +14,7 @@ filebeat.prospectors: # * stdin: Reads the standard in #------------------------------ Log prospector -------------------------------- -- input_type: log +- type: log # Change to true to enable this prospector configuration. enabled: false @@ -208,7 +208,7 @@ filebeat.prospectors: #----------------------------- Stdin prospector ------------------------------- # Configuration to use stdin input -#- input_type: stdin +#- type: stdin #========================= Filebeat global options ============================ diff --git a/filebeat/_meta/common.p2.yml b/filebeat/_meta/common.p2.yml index 80d7d03cc69..6afa1b2d946 100644 --- a/filebeat/_meta/common.p2.yml +++ b/filebeat/_meta/common.p2.yml @@ -9,7 +9,7 @@ filebeat.prospectors: # you can use different prospectors for various configurations. # Below are the prospector specific configurations. -- input_type: log +- type: log # Change to true to enable this prospector configuration. enabled: false diff --git a/filebeat/_meta/fields.common.yml b/filebeat/_meta/fields.common.yml index 0c515594660..14258c62149 100644 --- a/filebeat/_meta/fields.common.yml +++ b/filebeat/_meta/fields.common.yml @@ -23,10 +23,10 @@ description: > The content of the line read from the log file. - - name: input_type + - name: prospector.type required: true description: > - The input type from which the event was generated. This field is set to the value specified for the `input_type` option in the prospector section of the Filebeat config file. + The prospector type from which the event was generated. This field is set to the value specified for the `type` option in the prospector section of the Filebeat config file. - name: read_timestamp description: > diff --git a/filebeat/config/config.go b/filebeat/config/config.go index c66be16db10..07cac05abc2 100644 --- a/filebeat/config/config.go +++ b/filebeat/config/config.go @@ -15,7 +15,7 @@ import ( // Defaults for config variables which are not set const ( - DefaultInputType = "log" + DefaultType = "log" ) type Config struct { @@ -39,15 +39,16 @@ var ( } ) +// Contains available prospector types const ( - LogInputType = "log" - StdinInputType = "stdin" + LogType = "log" + StdinType = "stdin" ) // List of valid input types -var ValidInputType = map[string]struct{}{ - StdinInputType: {}, - LogInputType: {}, +var ValidType = map[string]struct{}{ + StdinType: {}, + LogType: {}, } // getConfigFiles returns list of config files. diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index d8b6c07bf7d..14c5059373d 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -768,11 +768,11 @@ The content of the line read from the log file. [float] -=== input_type +=== prospector.type required: True -The input type from which the event was generated. This field is set to the value specified for the `input_type` option in the prospector section of the Filebeat config file. +The prospector type from which the event was generated. This field is set to the value specified for the `type` option in the prospector section of the Filebeat config file. [float] diff --git a/filebeat/docs/filebeat-filtering.asciidoc b/filebeat/docs/filebeat-filtering.asciidoc index 113e8c91c1c..00c34594829 100644 --- a/filebeat/docs/filebeat-filtering.asciidoc +++ b/filebeat/docs/filebeat-filtering.asciidoc @@ -93,7 +93,9 @@ The resulting output looks something like this: "inner": { "data": "value" }, - "input_type": "log", + "prospector": { + "type": "log", + }, "offset": 55, "outer": "value", "source": "input.json", diff --git a/filebeat/docs/getting-started.asciidoc b/filebeat/docs/getting-started.asciidoc index 3c8ba64ab69..7bf843d4953 100644 --- a/filebeat/docs/getting-started.asciidoc +++ b/filebeat/docs/getting-started.asciidoc @@ -137,7 +137,7 @@ started experience for common log formats. See <> to learn how to get started with modules. If you use Filebeat modules to get started, you can skip the content in this section, including the remaining getting started steps, and go directly to the <> -page. +page. To configure Filebeat manually, you edit the configuration file. For rpm and deb, you'll find the configuration file at `/etc/filebeat/filebeat.yml`. For mac and @@ -155,7 +155,7 @@ default values for most configuration options. [source,yaml] ------------------------------------------------------------------------------------- filebeat.prospectors: -- input_type: log +- type: log paths: - /var/log/*.log #- c:\programdata\elasticsearch\logs\* @@ -170,7 +170,7 @@ For the most basic Filebeat configuration, you can define a single prospector wi [source,yaml] ------------------------------------------------------------------------------------- filebeat.prospectors: -- input_type: log +- type: log paths: - /var/log/*.log ------------------------------------------------------------------------------------- diff --git a/filebeat/docs/how-filebeat-works.asciidoc b/filebeat/docs/how-filebeat-works.asciidoc index 3130b2c4077..1f7faea7739 100644 --- a/filebeat/docs/how-filebeat-works.asciidoc +++ b/filebeat/docs/how-filebeat-works.asciidoc @@ -1,9 +1,9 @@ [[how-filebeat-works]] == How Filebeat Works -In this topic, you learn about the key building blocks of Filebeat and how they work together. Understanding these concepts will help you make informed decisions about configuring Filebeat for specific use cases. +In this topic, you learn about the key building blocks of Filebeat and how they work together. Understanding these concepts will help you make informed decisions about configuring Filebeat for specific use cases. -Filebeat consists of two main components: <> and <>. These components work together to tail files and send event data to the output that you specify. +Filebeat consists of two main components: <> and <>. These components work together to tail files and send event data to the output that you specify. [float] @@ -26,14 +26,14 @@ To control when a harvester is closed, use the <> confi A prospector is responsible for managing the harvesters and finding all sources to read from. -If the input type is `log`, the prospector finds all files on the drive that match the defined glob paths and starts a harvester for each file. Each prospector runs in its own Go routine. +If the input type is `log`, the prospector finds all files on the drive that match the defined glob paths and starts a harvester for each file. Each prospector runs in its own Go routine. The following example configures Filebeat to harvest lines from all log files that match the specified glob patterns: [source,yaml] ------------------------------------------------------------------------------------- filebeat.prospectors: -- input_type: log +- type: log paths: - /var/log/*.log - /var/path2/*.log @@ -50,7 +50,7 @@ Filebeat keeps the state of each file and frequently flushes the state to disk i Each prospector keeps a state for each file it finds. Because files can be renamed or moved, the filename and path are not enough to identify a file. For each file, Filebeat stores unique identifiers to detect whether a file was harvested previously. -If your use case involves creating a large number of new files every day, you might find that the registry file grows to be too large. See <> for details about configuration options that you can set to resolve this issue. +If your use case involves creating a large number of new files every day, you might find that the registry file grows to be too large. See <> for details about configuration options that you can set to resolve this issue. [float] [[at-least-once-delivery]] @@ -58,11 +58,11 @@ If your use case involves creating a large number of new files every day, you mi Filebeat guarantees that events will be delivered to the configured output at least once and with no data loss. Filebeat is able to achieve this behavior -because it stores the delivery state of each event in the registry file. +because it stores the delivery state of each event in the registry file. In situations where the defined output is blocked and has not confirmed all events, Filebeat will keep trying to send events until the output acknowledges -that it has received the events. +that it has received the events. If Filebeat shuts down while it's in the process of sending events, it does not wait for the output to acknowledge all events before shutting down. Any events diff --git a/filebeat/docs/load-balancing.asciidoc b/filebeat/docs/load-balancing.asciidoc index d890562190c..c024b406c57 100644 --- a/filebeat/docs/load-balancing.asciidoc +++ b/filebeat/docs/load-balancing.asciidoc @@ -37,7 +37,7 @@ Example: [source,yaml] ------------------------------------------------------------------------------- filebeat.prospectors: -- input_type: log +- type: log paths: - /var/log/*.log output.logstash: @@ -63,7 +63,7 @@ Example: [source,yaml] ------------------------------------------------------------------------------- filebeat.prospectors: -- input_type: log +- type: log paths: - /var/log/*.log filebeat.spool_size: 4096 diff --git a/filebeat/docs/migration.asciidoc b/filebeat/docs/migration.asciidoc index 47ef518ae26..d765e5cf523 100644 --- a/filebeat/docs/migration.asciidoc +++ b/filebeat/docs/migration.asciidoc @@ -135,7 +135,7 @@ The equivalent `prospectors` section would look like this: [source,yaml] ------------------------------------------------------------------------------------- filebeat.prospectors: -- input_type: log +- type: log paths: - /var/log/messages - /var/log/*.log @@ -143,16 +143,16 @@ filebeat.prospectors: service: apache zone: us-east-1 fields_under_root: true -- input_type: stdin <1> -- input_type: log +- type: stdin <1> +- type: log paths: - /var/log/apache2/httpd-*.log ------------------------------------------------------------------------------------- -<1> The explicit `input_type` option was introduced to differentiate between normal files and +<1> The explicit `type` option was introduced to differentiate between normal files and stdin. In the future, additional types might be supported. -As you can see, apart from the new `input_type` options, +As you can see, apart from the new `type` options, which were before implicitly defined via the `type` custom field, the remaining options can be migrated mechanically. @@ -279,7 +279,7 @@ Filebeat configuration: [source,yaml] ------------------------------------------------------------------------------------- filebeat.prospectors: -- input_type: log +- type: log paths: - /var/log/*.log fields: @@ -366,7 +366,7 @@ you can use the <> configuration option: [source,yaml] ------------------------------------------------------------------------------------- filebeat.prospectors: -- input_type: log +- type: log paths: - /var/log/*.log fields: diff --git a/filebeat/docs/multiple-prospectors.asciidoc b/filebeat/docs/multiple-prospectors.asciidoc index 42b0760eb3d..57ae3dbbe51 100644 --- a/filebeat/docs/multiple-prospectors.asciidoc +++ b/filebeat/docs/multiple-prospectors.asciidoc @@ -4,10 +4,10 @@ When you need to collect lines from multiple files, you can simply configure a single prospector and specify multiple paths to start a harvester for each file. However, if you want to apply additional prospector-specific <> (such as `fields`, `include_lines`, `exclude_lines`, `multiline`, and so on) -to the lines harvested from specific files, you need to define multiple prospectors in the Filebeat config file. +to the lines harvested from specific files, you need to define multiple prospectors in the Filebeat config file. Within the config file, you can specify multiple prospectors, and each prospector can define multiple paths to crawl, as -shown in the following example. +shown in the following example. NOTE: Make sure a file is not defined more than once across all prospectors because this can lead to unexpected behaviour. @@ -15,11 +15,11 @@ to unexpected behaviour. [source,yaml] ------------------------------------------------------------------------------------- filebeat.prospectors: -- input_type: log +- type: log paths: - /var/log/system.log - /var/log/wifi.log -- input_type: log +- type: log paths: - "/var/log/apache2/*" fields: diff --git a/filebeat/docs/reference/configuration/filebeat-options.asciidoc b/filebeat/docs/reference/configuration/filebeat-options.asciidoc index b3c823cffea..0014000a3b2 100644 --- a/filebeat/docs/reference/configuration/filebeat-options.asciidoc +++ b/filebeat/docs/reference/configuration/filebeat-options.asciidoc @@ -11,11 +11,11 @@ Here is a sample configuration: [source,yaml] ------------------------------------------------------------------------------------- filebeat.prospectors: -- input_type: log +- type: log paths: - /var/log/apache/httpd-*.log -- input_type: log +- type: log paths: - /var/log/messages - /var/log/*.log @@ -23,14 +23,14 @@ filebeat.prospectors: ==== Options -===== input_type +===== type One of the following input types: * log: Reads every line of the log file (default) * stdin: Reads the standard in -The value that you specify here is used as the `input_type` for each event published to Logstash and Elasticsearch. +The value that you specify here is used as the `type` for each event published to Logstash and Elasticsearch. [[prospector-paths]] ===== paths diff --git a/filebeat/docs/reference/configuration/reload-configuration.asciidoc b/filebeat/docs/reference/configuration/reload-configuration.asciidoc index aa6f7560f91..ef7932a1108 100644 --- a/filebeat/docs/reference/configuration/reload-configuration.asciidoc +++ b/filebeat/docs/reference/configuration/reload-configuration.asciidoc @@ -4,7 +4,7 @@ beta[] You can configure Filebeat to dynamically reload prospector configuration files -when there are changes. To do this, you specify a path +when there are changes. To do this, you specify a path (https://golang.org/pkg/path/filepath/#Glob[Glob]) to watch for prospector configuration changes. When the files found by the Glob change, new prospectors are started/stopped according to changes in the configuration files. @@ -13,7 +13,7 @@ This feature is especially useful in container environments where one container is used to tail logs for services running in other containers on the same host. To enable dynamic config reloading, you specify the `path` and `reload` options -in the main `filebeat.yml` config file. For example: +in the main `filebeat.yml` config file. For example: [source,yaml] ------------------------------------------------------------------------------ @@ -23,24 +23,24 @@ filebeat.config.prospectors: reload.period: 10s ------------------------------------------------------------------------------ -`path`:: A Glob that defines the files to check for changes. -`reload.enabled`:: When set to `true`, enables dynamic config reload. +`path`:: A Glob that defines the files to check for changes. +`reload.enabled`:: When set to `true`, enables dynamic config reload. `reload.period`:: Specifies how often the files are checked for changes. Do not set the `period` to less than 1s because the modification time of files is often stored in seconds. Setting the `period` to less than 1s will result in unnecessary overhead. Each file found by the Glob must contain a list of one or more prospector -definitions. For example: +definitions. For example: [source,yaml] ------------------------------------------------------------------------------ -- input_type: log +- type: log paths: - /var/log/mysql.log scan_frequency: 10s -- input_type: log +- type: log paths: - /var/log/apache.log scan_frequency: 5s diff --git a/filebeat/docs/system-module-note.asciidoc b/filebeat/docs/system-module-note.asciidoc index 1774a041533..5d38e17a408 100644 --- a/filebeat/docs/system-module-note.asciidoc +++ b/filebeat/docs/system-module-note.asciidoc @@ -7,7 +7,7 @@ you plan to run the `system` module, edit the Filebeat configuration file, [source,yaml] ---------------------------------------------------------------------- -#- input_type: log +#- type: log #paths: #- /var/log/*.log ---------------------------------------------------------------------- @@ -16,4 +16,4 @@ For rpm and deb, you'll find the configuration file at `/etc/filebeat/filebeat.yml`. For mac and win, look in the archive that you extracted when you installed Filebeat. -=============================================================================== \ No newline at end of file +=============================================================================== diff --git a/filebeat/filebeat.full.yml b/filebeat/filebeat.full.yml index 06036fc779a..f20d047894a 100644 --- a/filebeat/filebeat.full.yml +++ b/filebeat/filebeat.full.yml @@ -183,7 +183,7 @@ filebeat.prospectors: # * stdin: Reads the standard in #------------------------------ Log prospector -------------------------------- -- input_type: log +- type: log # Change to true to enable this prospector configuration. enabled: false @@ -377,7 +377,7 @@ filebeat.prospectors: #----------------------------- Stdin prospector ------------------------------- # Configuration to use stdin input -#- input_type: stdin +#- type: stdin #========================= Filebeat global options ============================ diff --git a/filebeat/filebeat.yml b/filebeat/filebeat.yml index 790eee22a01..1adfa33ee23 100644 --- a/filebeat/filebeat.yml +++ b/filebeat/filebeat.yml @@ -105,7 +105,7 @@ filebeat.prospectors: # you can use different prospectors for various configurations. # Below are the prospector specific configurations. -- input_type: log +- type: log # Change to true to enable this prospector configuration. enabled: false diff --git a/filebeat/module/apache2/access/config/access.yml b/filebeat/module/apache2/access/config/access.yml index 56e3dfd95b8..0afd17317d4 100644 --- a/filebeat/module/apache2/access/config/access.yml +++ b/filebeat/module/apache2/access/config/access.yml @@ -1,4 +1,4 @@ -input_type: log +type: log paths: {{ range $i, $path := .paths }} - {{$path}} diff --git a/filebeat/module/apache2/access/test/test.log-expected.json b/filebeat/module/apache2/access/test/test.log-expected.json index 0997759acc7..7a7e78f4633 100644 --- a/filebeat/module/apache2/access/test/test.log-expected.json +++ b/filebeat/module/apache2/access/test/test.log-expected.json @@ -1,140 +1,143 @@ [ -{ - "_index": "filebeat-2016.12.27", - "_type": "log", - "_id": "AVlBCaYsqYg9cc5KQfcT", - "_score": null, - "_source": { - "@timestamp": "2016-12-26T14:16:29.000Z", - "offset": 73, - "apache2": { - "access": { - "response_code": "404", - "remote_ip": "::1", - "method": "GET", - "user_name": "-", - "http_version": "1.1", - "body_sent": { - "bytes": "209" + { + "_index": "filebeat-2016.12.27", + "_type": "log", + "_id": "AVlBCaYsqYg9cc5KQfcT", + "_score": null, + "_source": { + "@timestamp": "2016-12-26T14:16:29.000Z", + "offset": 73, + "apache2": { + "access": { + "response_code": "404", + "remote_ip": "::1", + "method": "GET", + "user_name": "-", + "http_version": "1.1", + "body_sent": { + "bytes": "209" + }, + "url": "/favicon.ico" + } + }, + "beat": { + "hostname": "192-168-0-7.rdsnet.ro", + "name": "192-168-0-7.rdsnet.ro", + "version": "6.0.0-alpha1" + }, + "read_timestamp": "2016-12-27T15:52:23.304Z", + "source": "module/apache2/access/test/test.log", + "fields": { + "pipeline_id": "apache2-access-with_plugins", + "source_type": "apache2-access" + }, + "prospector": { + "type": "log" + } }, - "url": "/favicon.ico" - } - }, - "beat": { - "hostname": "192-168-0-7.rdsnet.ro", - "name": "192-168-0-7.rdsnet.ro", - "version": "6.0.0-alpha1" - }, - "input_type": "log", - "read_timestamp": "2016-12-27T15:52:23.304Z", - "source": "module/apache2/access/test/test.log", - "fields": { - "pipeline_id": "apache2-access-with_plugins", - "source_type": "apache2-access" - }, - "type": "log" - }, - "fields": { - "@timestamp": [ - 1482761789000 - ] - }, - "sort": [ - 1482761789000 - ] -}, -{ - "_index": "filebeat-2016.12.27", - "_type": "log", - "_id": "AVlBCaYsqYg9cc5KQfcU", - "_score": null, - "_source": { - "@timestamp": "2016-12-26T16:22:13.000Z", - "offset": 238, - "apache2": { - "access": { - "referrer": "-", - "response_code": "404", - "remote_ip": "192.168.33.1", - "method": "GET", - "user_name": "-", - "http_version": "1.1", - "body_sent": { - "bytes": "499" + "fields": { + "@timestamp": [ + 1482761789000 + ] }, - "url": "/hello", - "user_agent": { - "major": "50", - "minor": "0", - "os": "Mac OS X 10.12", - "os_minor": "12", - "os_major": "10", - "name": "Firefox", - "os_name": "Mac OS X", - "device": "Other" - } - } - }, - "beat": { - "hostname": "192-168-0-7.rdsnet.ro", - "name": "192-168-0-7.rdsnet.ro", - "version": "6.0.0-alpha1" - }, - "input_type": "log", - "read_timestamp": "2016-12-27T15:52:23.304Z", - "source": "module/apache2/access/test/test.log", - "fields": { - "pipeline_id": "apache2-access-with_plugins", - "source_type": "apache2-access" - }, - "type": "log" - }, - "fields": { - "@timestamp": [ - 1482769333000 - ] - }, - "sort": [ - 1482769333000 - ] -}, -{ - "_index": "filebeat-2016.12.27", - "_type": "log", - "_id": "AVlBCaYsqYg9cc5KQfc-", - "_score": null, - "_source": { - "@timestamp": "2016-12-26T14:16:48.000Z", - "offset": 285, - "apache2": { - "access": { - "response_code": "408", - "remote_ip": "::1", - "user_name": "-" - } + "sort": [ + 1482761789000 + ] }, - "beat": { - "hostname": "192-168-0-7.rdsnet.ro", - "name": "192-168-0-7.rdsnet.ro", - "version": "6.0.0-alpha1" - }, - "input_type": "log", - "read_timestamp": "2016-12-27T16:04:58.319Z", - "source": "module/apache2/access/test/test.log", - "fields": { - "pipeline_id": "apache2-access-with_plugins", - "source_type": "apache2-access" + { + "_index": "filebeat-2016.12.27", + "_type": "log", + "_id": "AVlBCaYsqYg9cc5KQfcU", + "_score": null, + "_source": { + "@timestamp": "2016-12-26T16:22:13.000Z", + "offset": 238, + "apache2": { + "access": { + "referrer": "-", + "response_code": "404", + "remote_ip": "192.168.33.1", + "method": "GET", + "user_name": "-", + "http_version": "1.1", + "body_sent": { + "bytes": "499" + }, + "url": "/hello", + "user_agent": { + "major": "50", + "minor": "0", + "os": "Mac OS X 10.12", + "os_minor": "12", + "os_major": "10", + "name": "Firefox", + "os_name": "Mac OS X", + "device": "Other" + } + } + }, + "beat": { + "hostname": "192-168-0-7.rdsnet.ro", + "name": "192-168-0-7.rdsnet.ro", + "version": "6.0.0-alpha1" + }, + "read_timestamp": "2016-12-27T15:52:23.304Z", + "source": "module/apache2/access/test/test.log", + "fields": { + "pipeline_id": "apache2-access-with_plugins", + "source_type": "apache2-access" + }, + "prospector": { + "type": "log" + } + }, + "fields": { + "@timestamp": [ + 1482769333000 + ] + }, + "sort": [ + 1482769333000 + ] }, - "type": "log" - }, - "fields": { - "@timestamp": [ - 1482761808000 - ] - }, - "sort": [ - 1482761808000 - ] -} + { + "_index": "filebeat-2016.12.27", + "_type": "log", + "_id": "AVlBCaYsqYg9cc5KQfc-", + "_score": null, + "_source": { + "@timestamp": "2016-12-26T14:16:48.000Z", + "offset": 285, + "apache2": { + "access": { + "response_code": "408", + "remote_ip": "::1", + "user_name": "-" + } + }, + "beat": { + "hostname": "192-168-0-7.rdsnet.ro", + "name": "192-168-0-7.rdsnet.ro", + "version": "6.0.0-alpha1" + }, + "read_timestamp": "2016-12-27T16:04:58.319Z", + "source": "module/apache2/access/test/test.log", + "fields": { + "pipeline_id": "apache2-access-with_plugins", + "source_type": "apache2-access" + }, + "prospector": { + "type": "log" + } + }, + "fields": { + "@timestamp": [ + 1482761808000 + ] + }, + "sort": [ + 1482761808000 + ] + } ] diff --git a/filebeat/module/apache2/error/config/error.yml b/filebeat/module/apache2/error/config/error.yml index 56e3dfd95b8..0afd17317d4 100644 --- a/filebeat/module/apache2/error/config/error.yml +++ b/filebeat/module/apache2/error/config/error.yml @@ -1,4 +1,4 @@ -input_type: log +type: log paths: {{ range $i, $path := .paths }} - {{$path}} diff --git a/filebeat/module/apache2/error/test/test.log-expected.json b/filebeat/module/apache2/error/test/test.log-expected.json index 05008603595..d72695adc64 100644 --- a/filebeat/module/apache2/error/test/test.log-expected.json +++ b/filebeat/module/apache2/error/test/test.log-expected.json @@ -1,93 +1,96 @@ [ - { - "_index": "filebeat-2016.12.29", - "_type": "log", - "_id": "AVlGiYcNqYg9cc5KQfdz", - "_score": 1, - "_source": { - "@timestamp": "2016-12-26T16:15:55.103Z", - "offset": 229, - "apache2": { - "error": { - "level": "notice", - "module": "core", - "pid": "11379", - "message": "AH00094: Command line: '/usr/local/Cellar/httpd24/2.4.23_2/bin/httpd'" - } - }, - "beat": { - "hostname": "192-168-0-7.rdsnet.ro", - "name": "192-168-0-7.rdsnet.ro", - "version": "6.0.0-alpha1" - }, - "input_type": "log", - "source": "module/apache2/error/test/test.log", - "fields": { - "pipeline_id": "apache2-error-pipeline", - "source_type": "apache2-error" - }, - "type": "log" - } - }, - { - "_index": "filebeat-2016.12.29", - "_type": "log", - "_id": "AVlGiYcNqYg9cc5KQfd0", - "_score": 1, - "_source": { - "@timestamp": "2011-09-09T10:42:29.902Z", - "offset": 384, - "apache2": { - "error": { - "level": "error", - "module": "core", - "client": "72.15.99.187", - "pid": "35708", - "message": "File does not exist: /usr/local/apache2/htdocs/favicon.ico", - "tid": "4328636416" - } - }, - "beat": { - "hostname": "192-168-0-7.rdsnet.ro", - "name": "192-168-0-7.rdsnet.ro", - "version": "6.0.0-alpha1" - }, - "input_type": "log", - "source": "module/apache2/error/test/test.log", - "fields": { - "pipeline_id": "apache2-error-pipeline", - "source_type": "apache2-error" - }, - "type": "log" - } - }, - { - "_index": "filebeat-2016.12.29", - "_type": "log", - "_id": "AVlGiYcMqYg9cc5KQfdy", - "_score": 1, - "_source": { - "@timestamp": "2016-12-26T16:22:08.000Z", - "offset": 99, - "apache2": { - "error": { - "level": "error", - "client": "192.168.33.1", - "message": "File does not exist: /var/www/favicon.ico" - } - }, - "beat": { - "hostname": "192-168-0-7.rdsnet.ro", - "name": "192-168-0-7.rdsnet.ro", - "version": "6.0.0-alpha1" - }, - "input_type": "log", - "source": "module/apache2/error/test/test.log", - "fields": { - "pipeline_id": "apache2-error-pipeline", - "source_type": "apache2-error" - }, - "type": "log" - } - } + { + "_index": "filebeat-2016.12.29", + "_type": "log", + "_id": "AVlGiYcNqYg9cc5KQfdz", + "_score": 1, + "_source": { + "@timestamp": "2016-12-26T16:15:55.103Z", + "offset": 229, + "apache2": { + "error": { + "level": "notice", + "module": "core", + "pid": "11379", + "message": "AH00094: Command line: '/usr/local/Cellar/httpd24/2.4.23_2/bin/httpd'" + } + }, + "beat": { + "hostname": "192-168-0-7.rdsnet.ro", + "name": "192-168-0-7.rdsnet.ro", + "version": "6.0.0-alpha1" + }, + "prospector": { + "type": "log" + }, + "source": "module/apache2/error/test/test.log", + "fields": { + "pipeline_id": "apache2-error-pipeline", + "source_type": "apache2-error" + } + } + }, + { + "_index": "filebeat-2016.12.29", + "_type": "log", + "_id": "AVlGiYcNqYg9cc5KQfd0", + "_score": 1, + "_source": { + "@timestamp": "2011-09-09T10:42:29.902Z", + "offset": 384, + "apache2": { + "error": { + "level": "error", + "module": "core", + "client": "72.15.99.187", + "pid": "35708", + "message": "File does not exist: /usr/local/apache2/htdocs/favicon.ico", + "tid": "4328636416" + } + }, + "beat": { + "hostname": "192-168-0-7.rdsnet.ro", + "name": "192-168-0-7.rdsnet.ro", + "version": "6.0.0-alpha1" + }, + "prospector": { + "type": "log" + }, + "source": "module/apache2/error/test/test.log", + "fields": { + "pipeline_id": "apache2-error-pipeline", + "source_type": "apache2-error" + } + } + }, + { + "_index": "filebeat-2016.12.29", + "_type": "log", + "_id": "AVlGiYcMqYg9cc5KQfdy", + "_score": 1, + "_source": { + "@timestamp": "2016-12-26T16:22:08.000Z", + "offset": 99, + "apache2": { + "error": { + "level": "error", + "client": "192.168.33.1", + "message": "File does not exist: /var/www/favicon.ico" + } + }, + "beat": { + "hostname": "192-168-0-7.rdsnet.ro", + "name": "192-168-0-7.rdsnet.ro", + "version": "6.0.0-alpha1" + }, + "prospector": { + "type": "log" + }, + "source": "module/apache2/error/test/test.log", + "fields": { + "pipeline_id": "apache2-error-pipeline", + "source_type": "apache2-error" + } + } + } ] diff --git a/filebeat/module/auditd/log/config/log.yml b/filebeat/module/auditd/log/config/log.yml index 56e3dfd95b8..0afd17317d4 100644 --- a/filebeat/module/auditd/log/config/log.yml +++ b/filebeat/module/auditd/log/config/log.yml @@ -1,4 +1,4 @@ -input_type: log +type: log paths: {{ range $i, $path := .paths }} - {{$path}} diff --git a/filebeat/module/auditd/log/test/test.log-expected.json b/filebeat/module/auditd/log/test/test.log-expected.json index a0a661eb5d8..65590185ca2 100644 --- a/filebeat/module/auditd/log/test/test.log-expected.json +++ b/filebeat/module/auditd/log/test/test.log-expected.json @@ -7,13 +7,14 @@ "_score": null, "_source": { "offset": 172, - "input_type": "log", + "prospector": { + "type": "log" + }, "source": "/Users/me/go/src/github.com/elastic/beats/filebeat/module/auditd/log/test/test.log", "fileset": { "module": "auditd", "name": "log" }, - "type": "log", "error": "", "@timestamp": "2017-01-31T20:18:46.912Z", "auditd": { @@ -45,13 +46,14 @@ "_score": null, "_source": { "offset": 534, - "input_type": "log", + "prospector": { + "type": "log" + }, "source": "/Users/me/go/src/github.com/elastic/beats/filebeat/module/auditd/log/test/test.log", "fileset": { "module": "auditd", "name": "log" }, - "type": "log", "error": "", "@timestamp": "2017-01-31T20:18:46.912Z", "auditd": { diff --git a/filebeat/module/icinga/debug/config/debug.yml b/filebeat/module/icinga/debug/config/debug.yml index 8deb67e8131..98c684ef52f 100644 --- a/filebeat/module/icinga/debug/config/debug.yml +++ b/filebeat/module/icinga/debug/config/debug.yml @@ -1,4 +1,4 @@ -input_type: log +type: log paths: {{ range $i, $path := .paths }} - {{$path}} diff --git a/filebeat/module/icinga/debug/test/test.log-expected.json b/filebeat/module/icinga/debug/test/test.log-expected.json index d9636d71c83..95084f423c8 100644 --- a/filebeat/module/icinga/debug/test/test.log-expected.json +++ b/filebeat/module/icinga/debug/test/test.log-expected.json @@ -1,113 +1,116 @@ [ -{ - "_index": "filebeat-2017.04.04", - "_type": "log", - "_id": "AVs46val2BrA1a1-fCdy", - "_score": null, - "_source": { - "@timestamp": "2017-04-04T11:43:09.000Z", - "offset": 1763, - "beat": { - "hostname": "blerims-mbp.int.netways.de", - "name": "blerims-mbp.int.netways.de", - "version": "5.3.0" + { + "_index": "filebeat-2017.04.04", + "_type": "log", + "_id": "AVs46val2BrA1a1-fCdy", + "_score": null, + "_source": { + "@timestamp": "2017-04-04T11:43:09.000Z", + "offset": 1763, + "beat": { + "hostname": "blerims-mbp.int.netways.de", + "name": "blerims-mbp.int.netways.de", + "version": "5.3.0" + }, + "prospector": { + "type": "log" + }, + "icinga": { + "debug": { + "severity": "debug", + "message": "Query: UPDATE icinga_servicestatus SET acknowledgement_type = '0', active_checks_enabled = '1', check_command = 'mysql_health', check_source = 'demo', check_type = '0', current_check_attempt = '1', current_notification_number = '180', current_state = '2', endpoint_object_id = 242, event_handler = '', event_handler_enabled = '1', execution_time = '0.355594', flap_detection_enabled = '0', has_been_checked = '1', instance_id = 1, is_flapping = '0', is_reachable = '1', last_check = FROM_UNIXTIME(1491306189), last_hard_state = '2', last_hard_state_change = FROM_UNIXTIME(1491290599), last_notification = FROM_UNIXTIME(1491304989), last_state_change = FROM_UNIXTIME(1491290599), last_time_critical = FROM_UNIXTIME(1491306189), last_time_unknown = FROM_UNIXTIME(1491290589), latency = '0.001466', long_output = '', max_check_attempts = '5', next_check = FROM_UNIXTIME(1491306198), next_notification = FROM_UNIXTIME(1491306789), normal_check_interval = '0.166667', notifications_enabled = '1', original_attributes = 'null', output = 'CRITICAL - cannot connect to information_schema. Access denied for user \\'test1\\'@\\'blerims-mbp.int.netways.de\\' (using password: YES)', passive_checks_enabled = '1', percent_state_change = '0', perfdata = '', problem_has_been_acknowledged = '0', process_performance_data = '1', retry_check_interval = '0.166667', scheduled_downtime_depth = '0', service_object_id = 333, should_be_scheduled = '1', state_type = '1', status_update_time = FROM_UNIXTIME(1491306189) WHERE service_object_id = 333", + "facility": "IdoMysqlConnection" + } + }, + "source": "/usr/local/icinga2/var/log/icinga2/debug.log", + "fileset": { + "module": "icinga", + "name": "debug" + } + }, + "fields": { + "@timestamp": [ + 1491306189000 + ] + }, + "sort": [ + 1491306189000 + ] }, - "input_type": "log", - "icinga": { - "debug": { - "severity": "debug", - "message": "Query: UPDATE icinga_servicestatus SET acknowledgement_type = '0', active_checks_enabled = '1', check_command = 'mysql_health', check_source = 'demo', check_type = '0', current_check_attempt = '1', current_notification_number = '180', current_state = '2', endpoint_object_id = 242, event_handler = '', event_handler_enabled = '1', execution_time = '0.355594', flap_detection_enabled = '0', has_been_checked = '1', instance_id = 1, is_flapping = '0', is_reachable = '1', last_check = FROM_UNIXTIME(1491306189), last_hard_state = '2', last_hard_state_change = FROM_UNIXTIME(1491290599), last_notification = FROM_UNIXTIME(1491304989), last_state_change = FROM_UNIXTIME(1491290599), last_time_critical = FROM_UNIXTIME(1491306189), last_time_unknown = FROM_UNIXTIME(1491290589), latency = '0.001466', long_output = '', max_check_attempts = '5', next_check = FROM_UNIXTIME(1491306198), next_notification = FROM_UNIXTIME(1491306789), normal_check_interval = '0.166667', notifications_enabled = '1', original_attributes = 'null', output = 'CRITICAL - cannot connect to information_schema. Access denied for user \\'test1\\'@\\'blerims-mbp.int.netways.de\\' (using password: YES)', passive_checks_enabled = '1', percent_state_change = '0', perfdata = '', problem_has_been_acknowledged = '0', process_performance_data = '1', retry_check_interval = '0.166667', scheduled_downtime_depth = '0', service_object_id = 333, should_be_scheduled = '1', state_type = '1', status_update_time = FROM_UNIXTIME(1491306189) WHERE service_object_id = 333", - "facility": "IdoMysqlConnection" - } + { + "_index": "filebeat-2017.04.04", + "_type": "log", + "_id": "AVs46val2BrA1a1-fCdx", + "_score": null, + "_source": { + "@timestamp": "2017-04-04T11:43:09.000Z", + "offset": 141, + "beat": { + "hostname": "blerims-mbp.int.netways.de", + "name": "blerims-mbp.int.netways.de", + "version": "5.3.0" + }, + "prospector": { + "type": "log" + }, + "icinga": { + "debug": { + "severity": "debug", + "message": "Add to metric list:'icinga2.demo.services.procs.procs.perfdata.procs.warn 250 1491306189'.", + "facility": "GraphiteWriter" + } + }, + "source": "/usr/local/icinga2/var/log/icinga2/debug.log", + "fileset": { + "module": "icinga", + "name": "debug" + } + }, + "fields": { + "@timestamp": [ + 1491306189000 + ] + }, + "sort": [ + 1491306189000 + ] }, - "source": "/usr/local/icinga2/var/log/icinga2/debug.log", - "fileset": { - "module": "icinga", - "name": "debug" - }, - "type": "log" - }, - "fields": { - "@timestamp": [ - 1491306189000 - ] - }, - "sort": [ - 1491306189000 - ] -}, -{ - "_index": "filebeat-2017.04.04", - "_type": "log", - "_id": "AVs46val2BrA1a1-fCdx", - "_score": null, - "_source": { - "@timestamp": "2017-04-04T11:43:09.000Z", - "offset": 141, - "beat": { - "hostname": "blerims-mbp.int.netways.de", - "name": "blerims-mbp.int.netways.de", - "version": "5.3.0" - }, - "input_type": "log", - "icinga": { - "debug": { - "severity": "debug", - "message": "Add to metric list:'icinga2.demo.services.procs.procs.perfdata.procs.warn 250 1491306189'.", - "facility": "GraphiteWriter" - } - }, - "source": "/usr/local/icinga2/var/log/icinga2/debug.log", - "fileset": { - "module": "icinga", - "name": "debug" - }, - "type": "log" - }, - "fields": { - "@timestamp": [ - 1491306189000 - ] - }, - "sort": [ - 1491306189000 - ] -}, -{ - "_index": "filebeat-2017.04.04", - "_type": "log", - "_id": "AVs46wjD2BrA1a1-fCdz", - "_score": null, - "_source": { - "@timestamp": "2017-04-04T11:43:11.000Z", - "offset": 1927, - "beat": { - "hostname": "blerims-mbp.int.netways.de", - "name": "blerims-mbp.int.netways.de", - "version": "5.3.0" - }, - "input_type": "log", - "icinga": { - "debug": { - "severity": "notice", - "message": "Running command '/usr/lib/nagios/plugins/check_ping' '-H' 'mysql.icinga.com' '-c' '5000,100%' '-w' '3000,80%': PID 8288", - "facility": "Process" - } - }, - "source": "/usr/local/icinga2/var/log/icinga2/debug.log", - "fileset": { - "module": "icinga", - "name": "debug" - }, - "type": "log" - }, - "fields": { - "@timestamp": [ - 1491306191000 - ] - }, - "sort": [ - 1491306191000 - ] -} + { + "_index": "filebeat-2017.04.04", + "_type": "log", + "_id": "AVs46wjD2BrA1a1-fCdz", + "_score": null, + "_source": { + "@timestamp": "2017-04-04T11:43:11.000Z", + "offset": 1927, + "beat": { + "hostname": "blerims-mbp.int.netways.de", + "name": "blerims-mbp.int.netways.de", + "version": "5.3.0" + }, + "prospector": { + "type": "log" + }, + "icinga": { + "debug": { + "severity": "notice", + "message": "Running command '/usr/lib/nagios/plugins/check_ping' '-H' 'mysql.icinga.com' '-c' '5000,100%' '-w' '3000,80%': PID 8288", + "facility": "Process" + } + }, + "source": "/usr/local/icinga2/var/log/icinga2/debug.log", + "fileset": { + "module": "icinga", + "name": "debug" + }, + }, + "fields": { + "@timestamp": [ + 1491306191000 + ] + }, + "sort": [ + 1491306191000 + ] + } ] diff --git a/filebeat/module/icinga/main/config/main.yml b/filebeat/module/icinga/main/config/main.yml index 8deb67e8131..98c684ef52f 100644 --- a/filebeat/module/icinga/main/config/main.yml +++ b/filebeat/module/icinga/main/config/main.yml @@ -1,4 +1,4 @@ -input_type: log +type: log paths: {{ range $i, $path := .paths }} - {{$path}} diff --git a/filebeat/module/icinga/main/test/test.log-expected.json b/filebeat/module/icinga/main/test/test.log-expected.json index b5592b6c140..9b30dad5255 100644 --- a/filebeat/module/icinga/main/test/test.log-expected.json +++ b/filebeat/module/icinga/main/test/test.log-expected.json @@ -1,113 +1,116 @@ [ -{ - "_index": "filebeat-2017.04.04", - "_type": "log", - "_id": "AVs4ouOJ2BrA1a1-e_FX", - "_score": null, - "_source": { - "@timestamp": "2017-04-04T09:16:34.000Z", - "offset": 518, - "beat": { - "hostname": "blerims-mbp.int.netways.de", - "name": "blerims-mbp.int.netways.de", - "version": "5.3.0" + { + "_index": "filebeat-2017.04.04", + "_type": "log", + "_id": "AVs4ouOJ2BrA1a1-e_FX", + "_score": null, + "_source": { + "@timestamp": "2017-04-04T09:16:34.000Z", + "offset": 518, + "beat": { + "hostname": "blerims-mbp.int.netways.de", + "name": "blerims-mbp.int.netways.de", + "version": "5.3.0" + }, + "prospector": { + "type": "log" + }, + "icinga": { + "main": { + "severity": "warning", + "message": "Notification command for object 'demo!load' (PID: 19401, arguments: '/etc/icinga2/scripts/mail-service-notification.sh') terminated with exit code 127, output: /etc/icinga2/scripts/mail-service-notification.sh: 20: /etc/icinga2/scripts/mail-service-notification.sh: mail: not found\n/usr/bin/printf: write error: Broken pipe\n", + "facility": "PluginNotificationTask" + } + }, + "source": "/usr/local/icinga2/var/log/icinga2/icinga2.log", + "fileset": { + "module": "icinga", + "name": "main" + } + }, + "fields": { + "@timestamp": [ + 1491297394000 + ] + }, + "sort": [ + 1491297394000 + ] }, - "input_type": "log", - "icinga": { - "main": { - "severity": "warning", - "message": "Notification command for object 'demo!load' (PID: 19401, arguments: '/etc/icinga2/scripts/mail-service-notification.sh') terminated with exit code 127, output: /etc/icinga2/scripts/mail-service-notification.sh: 20: /etc/icinga2/scripts/mail-service-notification.sh: mail: not found\n/usr/bin/printf: write error: Broken pipe\n", - "facility": "PluginNotificationTask" - } + { + "_index": "filebeat-2017.04.04", + "_type": "log", + "_id": "AVs4ouOJ2BrA1a1-e_FW", + "_score": null, + "_source": { + "@timestamp": "2017-04-04T09:16:34.000Z", + "offset": 133, + "beat": { + "hostname": "blerims-mbp.int.netways.de", + "name": "blerims-mbp.int.netways.de", + "version": "5.3.0" + }, + "prospector": { + "type": "log" + }, + "icinga": { + "main": { + "severity": "information", + "message": "Sending 'Recovery' notification 'demo!load!mail-icingaadmin for user 'on-call'", + "facility": "Notification" + } + }, + "source": "/usr/local/icinga2/var/log/icinga2/icinga2.log", + "fileset": { + "module": "icinga", + "name": "main" + } + }, + "fields": { + "@timestamp": [ + 1491297394000 + ] + }, + "sort": [ + 1491297394000 + ] }, - "source": "/usr/local/icinga2/var/log/icinga2/icinga2.log", - "fileset": { - "module": "icinga", - "name": "main" - }, - "type": "log" - }, - "fields": { - "@timestamp": [ - 1491297394000 - ] - }, - "sort": [ - 1491297394000 - ] -}, -{ - "_index": "filebeat-2017.04.04", - "_type": "log", - "_id": "AVs4ouOJ2BrA1a1-e_FW", - "_score": null, - "_source": { - "@timestamp": "2017-04-04T09:16:34.000Z", - "offset": 133, - "beat": { - "hostname": "blerims-mbp.int.netways.de", - "name": "blerims-mbp.int.netways.de", - "version": "5.3.0" - }, - "input_type": "log", - "icinga": { - "main": { - "severity": "information", - "message": "Sending 'Recovery' notification 'demo!load!mail-icingaadmin for user 'on-call'", - "facility": "Notification" - } - }, - "source": "/usr/local/icinga2/var/log/icinga2/icinga2.log", - "fileset": { - "module": "icinga", - "name": "main" - }, - "type": "log" - }, - "fields": { - "@timestamp": [ - 1491297394000 - ] - }, - "sort": [ - 1491297394000 - ] -}, -{ - "_index": "filebeat-2017.04.04", - "_type": "log", - "_id": "AVs4ovY82BrA1a1-e_FY", - "_score": null, - "_source": { - "@timestamp": "2017-04-04T09:16:48.000Z", - "offset": 654, - "beat": { - "hostname": "blerims-mbp.int.netways.de", - "name": "blerims-mbp.int.netways.de", - "version": "5.3.0" - }, - "input_type": "log", - "icinga": { - "main": { - "severity": "information", - "message": "Query queue items: 0, query rate: 5.38333/s (323/min 1610/5min 4778/15min);", - "facility": "IdoMysqlConnection" - } - }, - "source": "/usr/local/icinga2/var/log/icinga2/icinga2.log", - "fileset": { - "module": "icinga", - "name": "main" - }, - "type": "log" - }, - "fields": { - "@timestamp": [ - 1491297408000 - ] - }, - "sort": [ - 1491297408000 - ] -} + { + "_index": "filebeat-2017.04.04", + "_type": "log", + "_id": "AVs4ovY82BrA1a1-e_FY", + "_score": null, + "_source": { + "@timestamp": "2017-04-04T09:16:48.000Z", + "offset": 654, + "beat": { + "hostname": "blerims-mbp.int.netways.de", + "name": "blerims-mbp.int.netways.de", + "version": "5.3.0" + }, + "prospector": { + "type": "log" + }, + "icinga": { + "main": { + "severity": "information", + "message": "Query queue items: 0, query rate: 5.38333/s (323/min 1610/5min 4778/15min);", + "facility": "IdoMysqlConnection" + } + }, + "source": "/usr/local/icinga2/var/log/icinga2/icinga2.log", + "fileset": { + "module": "icinga", + "name": "main" + } + }, + "fields": { + "@timestamp": [ + 1491297408000 + ] + }, + "sort": [ + 1491297408000 + ] + } ] diff --git a/filebeat/module/icinga/startup/config/startup.yml b/filebeat/module/icinga/startup/config/startup.yml index e9ef3077e6d..a86ec3f1f31 100644 --- a/filebeat/module/icinga/startup/config/startup.yml +++ b/filebeat/module/icinga/startup/config/startup.yml @@ -1,4 +1,4 @@ -input_type: log +type: log paths: {{ range $i, $path := .paths }} - {{$path}} diff --git a/filebeat/module/icinga/startup/test/test.log-expected.json b/filebeat/module/icinga/startup/test/test.log-expected.json index 0023201b0dd..567c1b03c8e 100644 --- a/filebeat/module/icinga/startup/test/test.log-expected.json +++ b/filebeat/module/icinga/startup/test/test.log-expected.json @@ -1,76 +1,78 @@ [ -{ - "_index": "filebeat-2017.04.04", - "_type": "log", - "_id": "AVs5obub2BrA1a1-fVkV", - "_score": null, - "_source": { - "@timestamp": "2017-04-04T15:42:27.138Z", - "offset": 63, - "beat": { - "hostname": "blerims-mbp.int.netways.de", - "name": "blerims-mbp.int.netways.de", - "version": "5.3.0" + { + "_index": "filebeat-2017.04.04", + "_type": "log", + "_id": "AVs5obub2BrA1a1-fVkV", + "_score": null, + "_source": { + "@timestamp": "2017-04-04T15:42:27.138Z", + "offset": 63, + "beat": { + "hostname": "blerims-mbp.int.netways.de", + "name": "blerims-mbp.int.netways.de", + "version": "5.3.0" + }, + "prospector": { + "type": "log" + }, + "icinga": { + "startup": { + "severity": "information", + "message": "Icinga application loader (version: r2.6.3-1)", + "facility": "cli" + } + }, + "source": "/usr/local/icinga2/var/log/icinga2/startup.log", + "fileset": { + "module": "icinga", + "name": "startup" + } + }, + "fields": { + "@timestamp": [ + 1491320547138 + ] + }, + "sort": [ + 1491320547138 + ] }, - "input_type": "log", - "icinga": { - "startup": { - "severity": "information", - "message": "Icinga application loader (version: r2.6.3-1)", - "facility": "cli" - } - }, - "source": "/usr/local/icinga2/var/log/icinga2/startup.log", - "fileset": { - "module": "icinga", - "name": "startup" - }, - "type": "log" - }, - "fields": { - "@timestamp": [ - 1491320547138 - ] - }, - "sort": [ - 1491320547138 - ] -}, -{ - "_index": "filebeat-2017.04.04", - "_type": "log", - "_id": "AVs5obub2BrA1a1-fVkW", - "_score": null, - "_source": { - "@timestamp": "2017-04-04T15:42:27.138Z", - "offset": 111, - "beat": { - "hostname": "blerims-mbp.int.netways.de", - "name": "blerims-mbp.int.netways.de", - "version": "5.3.0" - }, - "input_type": "log", - "icinga": { - "startup": { - "severity": "information", - "message": "Loading configuration file(s).", - "facility": "cli" - } - }, - "source": "/usr/local/icinga2/var/log/icinga2/startup.log", - "fileset": { - "module": "icinga", - "name": "startup" - }, - "type": "log" - }, - "fields": { - "@timestamp": [ - 1491320547138 - ] - }, - "sort": [ - 1491320547138 - ] -} + { + "_index": "filebeat-2017.04.04", + "_type": "log", + "_id": "AVs5obub2BrA1a1-fVkW", + "_score": null, + "_source": { + "@timestamp": "2017-04-04T15:42:27.138Z", + "offset": 111, + "beat": { + "hostname": "blerims-mbp.int.netways.de", + "name": "blerims-mbp.int.netways.de", + "version": "5.3.0" + }, + "prospector": { + "type": "log" + }, + "icinga": { + "startup": { + "severity": "information", + "message": "Loading configuration file(s).", + "facility": "cli" + } + }, + "source": "/usr/local/icinga2/var/log/icinga2/startup.log", + "fileset": { + "module": "icinga", + "name": "startup" + }, + }, + "fields": { + "@timestamp": [ + 1491320547138 + ] + }, + "sort": [ + 1491320547138 + ] + } ] diff --git a/filebeat/module/mysql/error/config/error.yml b/filebeat/module/mysql/error/config/error.yml index 56e3dfd95b8..0afd17317d4 100644 --- a/filebeat/module/mysql/error/config/error.yml +++ b/filebeat/module/mysql/error/config/error.yml @@ -1,4 +1,4 @@ -input_type: log +type: log paths: {{ range $i, $path := .paths }} - {{$path}} diff --git a/filebeat/module/mysql/slowlog/config/slowlog.yml b/filebeat/module/mysql/slowlog/config/slowlog.yml index 923f55654a6..ec998f5e646 100644 --- a/filebeat/module/mysql/slowlog/config/slowlog.yml +++ b/filebeat/module/mysql/slowlog/config/slowlog.yml @@ -1,4 +1,4 @@ -input_type: log +type: log paths: {{ range $i, $path := .paths }} - {{$path}} diff --git a/filebeat/module/nginx/access/config/nginx-access.yml b/filebeat/module/nginx/access/config/nginx-access.yml index 56e3dfd95b8..0afd17317d4 100644 --- a/filebeat/module/nginx/access/config/nginx-access.yml +++ b/filebeat/module/nginx/access/config/nginx-access.yml @@ -1,4 +1,4 @@ -input_type: log +type: log paths: {{ range $i, $path := .paths }} - {{$path}} diff --git a/filebeat/module/nginx/error/config/nginx-error.yml b/filebeat/module/nginx/error/config/nginx-error.yml index 56e3dfd95b8..0afd17317d4 100644 --- a/filebeat/module/nginx/error/config/nginx-error.yml +++ b/filebeat/module/nginx/error/config/nginx-error.yml @@ -1,4 +1,4 @@ -input_type: log +type: log paths: {{ range $i, $path := .paths }} - {{$path}} diff --git a/filebeat/module/system/auth/config/auth.yml b/filebeat/module/system/auth/config/auth.yml index 1af7dee5e13..003b5d06b3e 100644 --- a/filebeat/module/system/auth/config/auth.yml +++ b/filebeat/module/system/auth/config/auth.yml @@ -1,4 +1,4 @@ -input_type: log +type: log paths: {{ range $i, $path := .paths }} - {{$path}} diff --git a/filebeat/module/system/auth/test/test.log-expected.json b/filebeat/module/system/auth/test/test.log-expected.json index 9d24032c6b4..f601ddb6277 100644 --- a/filebeat/module/system/auth/test/test.log-expected.json +++ b/filebeat/module/system/auth/test/test.log-expected.json @@ -1,362 +1,372 @@ [ - { - "_index" : "test-filebeat-modules", - "_type" : "log", - "_id" : "AVpwYhxbd3MQAzSWoGjF", - "_score" : 1.0, - "_source" : { - "@timestamp" : "2017-02-23T00:13:35.000Z", - "system" : { - "auth" : { - "hostname" : "localhost", - "ssh" : { - "method" : "password", - "port" : "58803", - "ip" : "192.168.33.1", - "event" : "Accepted" - }, - "pid" : "7483", - "user" : "vagrant", - "timestamp" : "Feb 23 00:13:35" + { + "_index": "test-filebeat-modules", + "_type": "log", + "_id": "AVpwYhxbd3MQAzSWoGjF", + "_score": 1.0, + "_source": { + "@timestamp": "2017-02-23T00:13:35.000Z", + "system": { + "auth": { + "hostname": "localhost", + "ssh": { + "method": "password", + "port": "58803", + "ip": "192.168.33.1", + "event": "Accepted" + }, + "pid": "7483", + "user": "vagrant", + "timestamp": "Feb 23 00:13:35" + } + }, + "offset": 254, + "beat": { + "hostname": "a-mac-with-esc-key.local", + "name": "a-mac-with-esc-key.local", + "version": "6.0.0-alpha1" + }, + "prospector": { + "type": "log" + }, + "source": "/Users/tsg/src/github.com/elastic/beats/filebeat/module/system/auth/test/test.log", + "fileset": { + "module": "system", + "name": "auth" + } } - }, - "offset" : 254, - "beat" : { - "hostname" : "a-mac-with-esc-key.local", - "name" : "a-mac-with-esc-key.local", - "version" : "6.0.0-alpha1" - }, - "input_type" : "log", - "source" : "/Users/tsg/src/github.com/elastic/beats/filebeat/module/system/auth/test/test.log", - "fileset" : { - "module" : "system", - "name" : "auth" - }, - "type" : "log" - } - }, - { - "_index" : "test-filebeat-modules", - "_type" : "log", - "_id" : "AVpwYhxbd3MQAzSWoGjK", - "_score" : 1.0, - "_source" : { - "@timestamp" : "2017-02-23T00:08:48.000Z", - "system" : { - "auth" : { - "hostname" : "localhost", - "sudo" : { - "tty" : "pts/1", - "pwd" : "/home/vagrant", - "user" : "root", - "command" : "/bin/cat /var/log/secure" - }, - "user" : "vagrant", - "timestamp" : "Feb 23 00:08:48" + }, + { + "_index": "test-filebeat-modules", + "_type": "log", + "_id": "AVpwYhxbd3MQAzSWoGjK", + "_score": 1.0, + "_source": { + "@timestamp": "2017-02-23T00:08:48.000Z", + "system": { + "auth": { + "hostname": "localhost", + "sudo": { + "tty": "pts/1", + "pwd": "/home/vagrant", + "user": "root", + "command": "/bin/cat /var/log/secure" + }, + "user": "vagrant", + "timestamp": "Feb 23 00:08:48" + } + }, + "offset": 736, + "beat": { + "hostname": "a-mac-with-esc-key.local", + "name": "a-mac-with-esc-key.local", + "version": "6.0.0-alpha1" + }, + "prospector": { + "type": "log" + }, + "source": "/Users/tsg/src/github.com/elastic/beats/filebeat/module/system/auth/test/test.log", + "fileset": { + "module": "system", + "name": "auth" + } } - }, - "offset" : 736, - "beat" : { - "hostname" : "a-mac-with-esc-key.local", - "name" : "a-mac-with-esc-key.local", - "version" : "6.0.0-alpha1" - }, - "input_type" : "log", - "source" : "/Users/tsg/src/github.com/elastic/beats/filebeat/module/system/auth/test/test.log", - "fileset" : { - "module" : "system", - "name" : "auth" - }, - "type" : "log" - } - }, - { - "_index" : "test-filebeat-modules", - "_type" : "log", - "_id" : "AVpwYhxbd3MQAzSWoGjG", - "_score" : 1.0, - "_source" : { - "@timestamp" : "2017-02-21T21:56:12.000Z", - "system" : { - "auth" : { - "hostname" : "localhost", - "ssh" : { - "ip" : "10.0.2.2", - "event" : "Invalid" - }, - "pid" : "3430", - "user" : "test", - "timestamp" : "Feb 21 21:56:12" + }, + { + "_index": "test-filebeat-modules", + "_type": "log", + "_id": "AVpwYhxbd3MQAzSWoGjG", + "_score": 1.0, + "_source": { + "@timestamp": "2017-02-21T21:56:12.000Z", + "system": { + "auth": { + "hostname": "localhost", + "ssh": { + "ip": "10.0.2.2", + "event": "Invalid" + }, + "pid": "3430", + "user": "test", + "timestamp": "Feb 21 21:56:12" + } + }, + "offset": 324, + "beat": { + "hostname": "a-mac-with-esc-key.local", + "name": "a-mac-with-esc-key.local", + "version": "6.0.0-alpha1" + }, + "prospector": { + "type": "log" + }, + "source": "/Users/tsg/src/github.com/elastic/beats/filebeat/module/system/auth/test/test.log", + "fileset": { + "module": "system", + "name": "auth" + } } - }, - "offset" : 324, - "beat" : { - "hostname" : "a-mac-with-esc-key.local", - "name" : "a-mac-with-esc-key.local", - "version" : "6.0.0-alpha1" - }, - "input_type" : "log", - "source" : "/Users/tsg/src/github.com/elastic/beats/filebeat/module/system/auth/test/test.log", - "fileset" : { - "module" : "system", - "name" : "auth" - }, - "type" : "log" - } - }, - { - "_index" : "test-filebeat-modules", - "_type" : "log", - "_id" : "AVpwYhxbd3MQAzSWoGjJ", - "_score" : 1.0, - "_source" : { - "@timestamp" : "2017-02-19T15:30:04.000Z", - "system" : { - "auth" : { - "hostname" : "slave22", - "ssh" : { - "dropped_ip" : "123.57.245.163" - }, - "pid" : "18406", - "timestamp" : "Feb 19 15:30:04" + }, + { + "_index": "test-filebeat-modules", + "_type": "log", + "_id": "AVpwYhxbd3MQAzSWoGjJ", + "_score": 1.0, + "_source": { + "@timestamp": "2017-02-19T15:30:04.000Z", + "system": { + "auth": { + "hostname": "slave22", + "ssh": { + "dropped_ip": "123.57.245.163" + }, + "pid": "18406", + "timestamp": "Feb 19 15:30:04" + } + }, + "offset": 617, + "beat": { + "hostname": "a-mac-with-esc-key.local", + "name": "a-mac-with-esc-key.local", + "version": "6.0.0-alpha1" + }, + "prospector": { + "type": "log" + }, + "source": "/Users/tsg/src/github.com/elastic/beats/filebeat/module/system/auth/test/test.log", + "fileset": { + "module": "system", + "name": "auth" + } } - }, - "offset" : 617, - "beat" : { - "hostname" : "a-mac-with-esc-key.local", - "name" : "a-mac-with-esc-key.local", - "version" : "6.0.0-alpha1" - }, - "input_type" : "log", - "source" : "/Users/tsg/src/github.com/elastic/beats/filebeat/module/system/auth/test/test.log", - "fileset" : { - "module" : "system", - "name" : "auth" - }, - "type" : "log" - } - }, - { - "_index" : "test-filebeat-modules", - "_type" : "log", - "_id" : "AVpwYhxbd3MQAzSWoGjL", - "_score" : 1.0, - "_source" : { - "@timestamp" : "2017-02-24T00:13:02.000Z", - "system" : { - "auth" : { - "hostname" : "precise32", - "sudo" : { - "tty" : "pts/1", - "pwd" : "/home/vagrant", - "error" : "user NOT in sudoers", - "user" : "root", - "command" : "/bin/ls" - }, - "user" : "tsg", - "timestamp" : "Feb 24 00:13:02" + }, + { + "_index": "test-filebeat-modules", + "_type": "log", + "_id": "AVpwYhxbd3MQAzSWoGjL", + "_score": 1.0, + "_source": { + "@timestamp": "2017-02-24T00:13:02.000Z", + "system": { + "auth": { + "hostname": "precise32", + "sudo": { + "tty": "pts/1", + "pwd": "/home/vagrant", + "error": "user NOT in sudoers", + "user": "root", + "command": "/bin/ls" + }, + "user": "tsg", + "timestamp": "Feb 24 00:13:02" + } + }, + "offset": 861, + "beat": { + "hostname": "a-mac-with-esc-key.local", + "name": "a-mac-with-esc-key.local", + "version": "6.0.0-alpha1" + }, + "prospector": { + "type": "log" + }, + "source": "/Users/tsg/src/github.com/elastic/beats/filebeat/module/system/auth/test/test.log", + "fileset": { + "module": "system", + "name": "auth" + } } - }, - "offset" : 861, - "beat" : { - "hostname" : "a-mac-with-esc-key.local", - "name" : "a-mac-with-esc-key.local", - "version" : "6.0.0-alpha1" - }, - "input_type" : "log", - "source" : "/Users/tsg/src/github.com/elastic/beats/filebeat/module/system/auth/test/test.log", - "fileset" : { - "module" : "system", - "name" : "auth" - }, - "type" : "log" - } - }, - { - "_index" : "test-filebeat-modules", - "_type" : "log", - "_id" : "AVpwYhxbd3MQAzSWoGjM", - "_score" : 1.0, - "_source" : { - "@timestamp" : "2017-02-22T11:47:05.000Z", - "system" : { - "auth" : { - "hostname" : "localhost", - "pid" : "6991", - "groupadd" : { - "gid" : "48", - "name" : "apache" - }, - "timestamp" : "Feb 22 11:47:05" + }, + { + "_index": "test-filebeat-modules", + "_type": "log", + "_id": "AVpwYhxbd3MQAzSWoGjM", + "_score": 1.0, + "_source": { + "@timestamp": "2017-02-22T11:47:05.000Z", + "system": { + "auth": { + "hostname": "localhost", + "pid": "6991", + "groupadd": { + "gid": "48", + "name": "apache" + }, + "timestamp": "Feb 22 11:47:05" + } + }, + "offset": 934, + "beat": { + "hostname": "a-mac-with-esc-key.local", + "name": "a-mac-with-esc-key.local", + "version": "6.0.0-alpha1" + }, + "prospector": { + "type": "log" + }, + "source": "/Users/tsg/src/github.com/elastic/beats/filebeat/module/system/auth/test/test.log", + "fileset": { + "module": "system", + "name": "auth" + } } - }, - "offset" : 934, - "beat" : { - "hostname" : "a-mac-with-esc-key.local", - "name" : "a-mac-with-esc-key.local", - "version" : "6.0.0-alpha1" - }, - "input_type" : "log", - "source" : "/Users/tsg/src/github.com/elastic/beats/filebeat/module/system/auth/test/test.log", - "fileset" : { - "module" : "system", - "name" : "auth" - }, - "type" : "log" - } - }, - { - "_index" : "test-filebeat-modules", - "_type" : "log", - "_id" : "AVpwYhxbd3MQAzSWoGjN", - "_score" : 1.0, - "_source" : { - "@timestamp" : "2017-02-22T11:47:05.000Z", - "system" : { - "auth" : { - "hostname" : "localhost", - "pid" : "6995", - "useradd" : { - "uid" : "48", - "gid" : "48", - "shell" : "/sbin/nologin", - "name" : "apache", - "home" : "/usr/share/httpd" - }, - "timestamp" : "Feb 22 11:47:05" + }, + { + "_index": "test-filebeat-modules", + "_type": "log", + "_id": "AVpwYhxbd3MQAzSWoGjN", + "_score": 1.0, + "_source": { + "@timestamp": "2017-02-22T11:47:05.000Z", + "system": { + "auth": { + "hostname": "localhost", + "pid": "6995", + "useradd": { + "uid": "48", + "gid": "48", + "shell": "/sbin/nologin", + "name": "apache", + "home": "/usr/share/httpd" + }, + "timestamp": "Feb 22 11:47:05" + } + }, + "offset": 1057, + "beat": { + "hostname": "a-mac-with-esc-key.local", + "name": "a-mac-with-esc-key.local", + "version": "6.0.0-alpha1" + }, + "prospector": { + "type": "log" + }, + "source": "/Users/tsg/src/github.com/elastic/beats/filebeat/module/system/auth/test/test.log", + "fileset": { + "module": "system", + "name": "auth" + } } - }, - "offset" : 1057, - "beat" : { - "hostname" : "a-mac-with-esc-key.local", - "name" : "a-mac-with-esc-key.local", - "version" : "6.0.0-alpha1" - }, - "input_type" : "log", - "source" : "/Users/tsg/src/github.com/elastic/beats/filebeat/module/system/auth/test/test.log", - "fileset" : { - "module" : "system", - "name" : "auth" - }, - "type" : "log" - } - }, - { - "_index" : "test-filebeat-modules", - "_type" : "log", - "_id" : "AVpwYhxbd3MQAzSWoGjE", - "_score" : 1.0, - "_source" : { - "@timestamp" : "2017-02-21T21:54:44.000Z", - "system" : { - "auth" : { - "hostname" : "localhost", - "ssh" : { - "method" : "publickey", - "signature" : "RSA 39:33:99:e9:a0:dc:f2:33:a3:e5:72:3b:7c:3a:56:84", - "port" : "63673", - "ip" : "10.0.2.2", - "event" : "Accepted" - }, - "pid" : "3402", - "user" : "vagrant", - "timestamp" : "Feb 21 21:54:44" + }, + { + "_index": "test-filebeat-modules", + "_type": "log", + "_id": "AVpwYhxbd3MQAzSWoGjE", + "_score": 1.0, + "_source": { + "@timestamp": "2017-02-21T21:54:44.000Z", + "system": { + "auth": { + "hostname": "localhost", + "ssh": { + "method": "publickey", + "signature": "RSA 39:33:99:e9:a0:dc:f2:33:a3:e5:72:3b:7c:3a:56:84", + "port": "63673", + "ip": "10.0.2.2", + "event": "Accepted" + }, + "pid": "3402", + "user": "vagrant", + "timestamp": "Feb 21 21:54:44" + } + }, + "offset": 152, + "beat": { + "hostname": "a-mac-with-esc-key.local", + "name": "a-mac-with-esc-key.local", + "version": "6.0.0-alpha1" + }, + "prospector": { + "type": "log" + }, + "source": "/Users/tsg/src/github.com/elastic/beats/filebeat/module/system/auth/test/test.log", + "fileset": { + "module": "system", + "name": "auth" + } } - }, - "offset" : 152, - "beat" : { - "hostname" : "a-mac-with-esc-key.local", - "name" : "a-mac-with-esc-key.local", - "version" : "6.0.0-alpha1" - }, - "input_type" : "log", - "source" : "/Users/tsg/src/github.com/elastic/beats/filebeat/module/system/auth/test/test.log", - "fileset" : { - "module" : "system", - "name" : "auth" - }, - "type" : "log" - } - }, - { - "_index" : "test-filebeat-modules", - "_type" : "log", - "_id" : "AVpwYhxbd3MQAzSWoGjH", - "_score" : 1.0, - "_source" : { - "@timestamp" : "2017-02-20T08:35:22.000Z", - "system" : { - "auth" : { - "hostname" : "slave22", - "ssh" : { - "geoip" : { - "continent_name" : "Asia", - "city_name" : "Guangzhou", - "country_iso_code" : "CN", - "region_name" : "Guangdong", - "location" : { - "lon" : 113.25, - "lat" : 23.1167 - } + }, + { + "_index": "test-filebeat-modules", + "_type": "log", + "_id": "AVpwYhxbd3MQAzSWoGjH", + "_score": 1.0, + "_source": { + "@timestamp": "2017-02-20T08:35:22.000Z", + "system": { + "auth": { + "hostname": "slave22", + "ssh": { + "geoip": { + "continent_name": "Asia", + "city_name": "Guangzhou", + "country_iso_code": "CN", + "region_name": "Guangdong", + "location": { + "lon": 113.25, + "lat": 23.1167 + } + }, + "method": "password", + "port": "29160", + "ip": "116.31.116.24", + "event": "Failed" + }, + "pid": "5774", + "user": "root", + "timestamp": "Feb 20 08:35:22" + } + }, + "offset": 420, + "beat": { + "hostname": "a-mac-with-esc-key.local", + "name": "a-mac-with-esc-key.local", + "version": "6.0.0-alpha1" + }, + "prospector": { + "type": "log" }, - "method" : "password", - "port" : "29160", - "ip" : "116.31.116.24", - "event" : "Failed" - }, - "pid" : "5774", - "user" : "root", - "timestamp" : "Feb 20 08:35:22" + "source": "/Users/tsg/src/github.com/elastic/beats/filebeat/module/system/auth/test/test.log", + "fileset": { + "module": "system", + "name": "auth" + } } - }, - "offset" : 420, - "beat" : { - "hostname" : "a-mac-with-esc-key.local", - "name" : "a-mac-with-esc-key.local", - "version" : "6.0.0-alpha1" - }, - "input_type" : "log", - "source" : "/Users/tsg/src/github.com/elastic/beats/filebeat/module/system/auth/test/test.log", - "fileset" : { - "module" : "system", - "name" : "auth" - }, - "type" : "log" - } - }, - { - "_index" : "test-filebeat-modules", - "_type" : "log", - "_id" : "AVpwYhxbd3MQAzSWoGjI", - "_score" : 1.0, - "_source" : { - "@timestamp" : "2017-02-21T23:35:33.000Z", - "system" : { - "auth" : { - "hostname" : "localhost", - "sudo" : { - "tty" : "pts/0", - "pwd" : "/home/vagrant", - "user" : "root", - "command" : "/bin/ls" - }, - "user" : "vagrant", - "timestamp" : "Feb 21 23:35:33" + }, + { + "_index": "test-filebeat-modules", + "_type": "log", + "_id": "AVpwYhxbd3MQAzSWoGjI", + "_score": 1.0, + "_source": { + "@timestamp": "2017-02-21T23:35:33.000Z", + "system": { + "auth": { + "hostname": "localhost", + "sudo": { + "tty": "pts/0", + "pwd": "/home/vagrant", + "user": "root", + "command": "/bin/ls" + }, + "user": "vagrant", + "timestamp": "Feb 21 23:35:33" + } + }, + "offset": 522, + "beat": { + "hostname": "a-mac-with-esc-key.local", + "name": "a-mac-with-esc-key.local", + "version": "6.0.0-alpha1" + }, + "prospector": { + "type": "log" + }, + "source": "/Users/tsg/src/github.com/elastic/beats/filebeat/module/system/auth/test/test.log", + "fileset": { + "module": "system", + "name": "auth" + } } - }, - "offset" : 522, - "beat" : { - "hostname" : "a-mac-with-esc-key.local", - "name" : "a-mac-with-esc-key.local", - "version" : "6.0.0-alpha1" - }, - "input_type" : "log", - "source" : "/Users/tsg/src/github.com/elastic/beats/filebeat/module/system/auth/test/test.log", - "fileset" : { - "module" : "system", - "name" : "auth" - }, - "type" : "log" } - } ] diff --git a/filebeat/module/system/syslog/config/syslog.yml b/filebeat/module/system/syslog/config/syslog.yml index 1af7dee5e13..003b5d06b3e 100644 --- a/filebeat/module/system/syslog/config/syslog.yml +++ b/filebeat/module/system/syslog/config/syslog.yml @@ -1,4 +1,4 @@ -input_type: log +type: log paths: {{ range $i, $path := .paths }} - {{$path}} diff --git a/filebeat/module/system/syslog/test/darwin-syslog-sample.log-expected.json b/filebeat/module/system/syslog/test/darwin-syslog-sample.log-expected.json index cf6e1ae645f..fdf1f34aaa0 100644 --- a/filebeat/module/system/syslog/test/darwin-syslog-sample.log-expected.json +++ b/filebeat/module/system/syslog/test/darwin-syslog-sample.log-expected.json @@ -1,90 +1,93 @@ [ - { - "_index" : "filebeat-2017.01.27", - "_type" : "log", - "_id" : "AVngMBcpUmQau8zc_UuM", - "_score" : 1.0, - "_source" : { - "@timestamp" : "2017-12-13T11:35:28.000Z", - "system" : { - "syslog" : { - "hostname" : "a-mac-with-esc-key", - "pid" : "21412", - "program" : "GoogleSoftwareUpdateAgent", - "message" : "2016-12-13 11:35:28.420 GoogleSoftwareUpdateAgent[21412/0x700007399000] [lvl=2] -[KSAgentApp updateProductWithProductID:usingEngine:] Checking for updates for \"All Products\" using engine \n\t\t>>\n\t\tprocessor=\n\t\t\tisProcessing=NO actionsCompleted=0 progress=0.00\n\t\t\terrors=0 currentActionErrors=0\n\t\t\tevents=0 currentActionEvents=0\n\t\t\tactionQueue=( )\n\t\t>\n\t\tdelegate=(null)\n\t\tserverInfoStore=(null)\n\t\terrors=0\n\t>", - "timestamp" : "Dec 13 11:35:28" + { + "_index": "filebeat-2017.01.27", + "_type": "log", + "_id": "AVngMBcpUmQau8zc_UuM", + "_score": 1.0, + "_source": { + "@timestamp": "2017-12-13T11:35:28.000Z", + "system": { + "syslog": { + "hostname": "a-mac-with-esc-key", + "pid": "21412", + "program": "GoogleSoftwareUpdateAgent", + "message": "2016-12-13 11:35:28.420 GoogleSoftwareUpdateAgent[21412/0x700007399000] [lvl=2] -[KSAgentApp updateProductWithProductID:usingEngine:] Checking for updates for \"All Products\" using engine \n\t\t>>\n\t\tprocessor=\n\t\t\tisProcessing=NO actionsCompleted=0 progress=0.00\n\t\t\terrors=0 currentActionErrors=0\n\t\t\tevents=0 currentActionEvents=0\n\t\t\tactionQueue=( )\n\t\t>\n\t\tdelegate=(null)\n\t\tserverInfoStore=(null)\n\t\terrors=0\n\t>", + "timestamp": "Dec 13 11:35:28" + } + }, + "offset": 907, + "beat": { + "hostname": "a-mac-with-esc-key.local", + "name": "a-mac-with-esc-key.local", + "version": "6.0.0-alpha1" + }, + "prospector": { + "type": "log" + }, + "source": "module/system/syslog/test/darwin-syslog-sample.log", + "fields": { + "source_type": "system-syslog" + } } - }, - "offset" : 907, - "beat" : { - "hostname" : "a-mac-with-esc-key.local", - "name" : "a-mac-with-esc-key.local", - "version" : "6.0.0-alpha1" - }, - "input_type" : "log", - "source" : "module/system/syslog/test/darwin-syslog-sample.log", - "fields" : { - "source_type" : "system-syslog" - }, - "type" : "log" - } - }, - { - "_index" : "filebeat-2017.01.27", - "_type" : "log", - "_id" : "AVngMCdNUmQau8zc_UuN", - "_score" : 1.0, - "_source" : { - "@timestamp" : "2017-12-13T11:35:28.000Z", - "system" : { - "syslog" : { - "hostname" : "a-mac-with-esc-key", - "pid" : "21412", - "program" : "GoogleSoftwareUpdateAgent", - "message" : "2016-12-13 11:35:28.421 GoogleSoftwareUpdateAgent[21412/0x700007399000] [lvl=2] -[KSUpdateEngine updateAllExceptProduct:] KSUpdateEngine updating all installed products, except:'com.google.Keystone'.", - "timestamp" : "Dec 13 11:35:28" + }, + { + "_index": "filebeat-2017.01.27", + "_type": "log", + "_id": "AVngMCdNUmQau8zc_UuN", + "_score": 1.0, + "_source": { + "@timestamp": "2017-12-13T11:35:28.000Z", + "system": { + "syslog": { + "hostname": "a-mac-with-esc-key", + "pid": "21412", + "program": "GoogleSoftwareUpdateAgent", + "message": "2016-12-13 11:35:28.421 GoogleSoftwareUpdateAgent[21412/0x700007399000] [lvl=2] -[KSUpdateEngine updateAllExceptProduct:] KSUpdateEngine updating all installed products, except:'com.google.Keystone'.", + "timestamp": "Dec 13 11:35:28" + } + }, + "offset": 1176, + "beat": { + "hostname": "a-mac-with-esc-key.local", + "name": "a-mac-with-esc-key.local", + "version": "6.0.0-alpha1" + }, + "prospector": { + "type": "log" + }, + "source": "module/system/syslog/test/darwin-syslog-sample.log", + "fields": { + "source_type": "system-syslog" + } } - }, - "offset" : 1176, - "beat" : { - "hostname" : "a-mac-with-esc-key.local", - "name" : "a-mac-with-esc-key.local", - "version" : "6.0.0-alpha1" - }, - "input_type" : "log", - "source" : "module/system/syslog/test/darwin-syslog-sample.log", - "fields" : { - "source_type" : "system-syslog" - }, - "type" : "log" - } - }, - { - "_index": "test-filebeat-modules", - "_type": "doc", - "_id": "AVtFVFY1nbkdi71WgGxo", - "_score": 1, - "_source": { - "@timestamp": "2017-04-04T03:39:57.000Z", - "system": { - "syslog": { - "message": "--- last message repeated 1 time ---", - "timestamp": "Apr 4 03:39:57" + }, + { + "_index": "test-filebeat-modules", + "_type": "doc", + "_id": "AVtFVFY1nbkdi71WgGxo", + "_score": 1, + "_source": { + "@timestamp": "2017-04-04T03:39:57.000Z", + "system": { + "syslog": { + "message": "--- last message repeated 1 time ---", + "timestamp": "Apr 4 03:39:57" + } + }, + "offset": 1228, + "beat": { + "hostname": "a-mac-with-esc-key-2.local", + "name": "a-mac-with-esc-key-2.local", + "version": "6.0.0-alpha1" + }, + "prospector": { + "type": "log" + }, + "source": "/Users/tsg/src/github.com/elastic/beats/filebeat/module/system/syslog/test/darwin-syslog-sample.log", + "fileset": { + "module": "system", + "name": "syslog" + } } - }, - "offset": 1228, - "beat": { - "hostname": "a-mac-with-esc-key-2.local", - "name": "a-mac-with-esc-key-2.local", - "version": "6.0.0-alpha1" - }, - "input_type": "log", - "source": "/Users/tsg/src/github.com/elastic/beats/filebeat/module/system/syslog/test/darwin-syslog-sample.log", - "fileset": { - "module": "system", - "name": "syslog" - }, - "type": "log" } - } ] diff --git a/filebeat/prospector/config.go b/filebeat/prospector/config.go index 3658c77fa8e..99d8d2ad41b 100644 --- a/filebeat/prospector/config.go +++ b/filebeat/prospector/config.go @@ -4,16 +4,26 @@ import ( "time" cfg "github.com/elastic/beats/filebeat/config" + "github.com/elastic/beats/libbeat/logp" ) var ( defaultConfig = prospectorConfig{ ScanFrequency: 10 * time.Second, - InputType: cfg.DefaultInputType, + Type: cfg.DefaultType, } ) type prospectorConfig struct { ScanFrequency time.Duration `config:"scan_frequency" validate:"min=0,nonzero"` + Type string `config:"type"` InputType string `config:"input_type"` } + +func (c *prospectorConfig) Validate() error { + if c.InputType != "" { + logp.Deprecate("6.0.0", "input_type prospector config is deprecated. Use type instead.") + c.Type = c.InputType + } + return nil +} diff --git a/filebeat/prospector/log/config.go b/filebeat/prospector/log/config.go index f040f82ef46..cee66710bcb 100644 --- a/filebeat/prospector/log/config.go +++ b/filebeat/prospector/log/config.go @@ -15,7 +15,7 @@ import ( var ( defaultConfig = config{ // Common - InputType: cfg.DefaultInputType, + Type: cfg.DefaultType, CleanInactive: 0, // Prospector @@ -44,6 +44,7 @@ var ( type config struct { // Common + Type string `config:"type"` InputType string `config:"input_type"` CleanInactive time.Duration `config:"clean_inactive" validate:"min=0"` @@ -84,8 +85,13 @@ type config struct { func (c *config) Validate() error { + // DEPRECATED 6.0.0: warning is already outputted on propsector level + if c.InputType != "" { + c.Type = c.InputType + } + // Prospector - if c.InputType == cfg.LogInputType && len(c.Paths) == 0 { + if c.Type == cfg.LogType && len(c.Paths) == 0 { return fmt.Errorf("No paths were defined for prospector") } @@ -99,8 +105,8 @@ func (c *config) Validate() error { // Harvester // Check input type - if _, ok := cfg.ValidInputType[c.InputType]; !ok { - return fmt.Errorf("Invalid input type: %v", c.InputType) + if _, ok := cfg.ValidType[c.Type]; !ok { + return fmt.Errorf("Invalid input type: %v", c.Type) } if c.JSON != nil && len(c.JSON.MessageKey) == 0 && diff --git a/filebeat/prospector/log/config_test.go b/filebeat/prospector/log/config_test.go index c79a0d7361b..6537fbcecf3 100644 --- a/filebeat/prospector/log/config_test.go +++ b/filebeat/prospector/log/config_test.go @@ -47,7 +47,7 @@ func TestCleanOlderIgnoreOlder(t *testing.T) { config := config{ CleanInactive: 10*time.Hour + defaultConfig.ScanFrequency + 1*time.Second, IgnoreOlder: 10 * time.Hour, - InputType: "log", + Type: "log", Paths: []string{"hello"}, } diff --git a/filebeat/prospector/log/harvester.go b/filebeat/prospector/log/harvester.go index 6b61dab9ed4..854e72dd3a5 100644 --- a/filebeat/prospector/log/harvester.go +++ b/filebeat/prospector/log/harvester.go @@ -108,10 +108,10 @@ func NewHarvester( // open does open the file given under h.Path and assigns the file handler to h.file func (h *Harvester) open() error { - switch h.config.InputType { - case cfg.StdinInputType: + switch h.config.Type { + case cfg.StdinType: return h.openStdin() - case cfg.LogInputType: + case cfg.LogType: return h.openFile() default: return fmt.Errorf("Invalid harvester type: %+v", h.config) @@ -129,6 +129,7 @@ func (h *Harvester) forwardEvent(data *util.Data) error { if data.HasEvent() { data.Event[common.EventMetadataKey] = h.config.EventMetadata + data.Event.Put("prospector.type", h.config.Type) // run the filters before sending to spooler data.Event = h.processors.Run(data.Event) diff --git a/filebeat/prospector/log/log.go b/filebeat/prospector/log/log.go index 6e1fc81a182..2a4639dfd12 100644 --- a/filebeat/prospector/log/log.go +++ b/filebeat/prospector/log/log.go @@ -149,7 +149,6 @@ func (h *Harvester) Start() { "@timestamp": common.Time(message.Ts), "source": state.Source, "offset": state.Offset, // Offset here is the offset before the starting char. - "input_type": h.config.InputType, } data.Event.DeepUpdate(message.Fields) diff --git a/filebeat/prospector/log/prospector.go b/filebeat/prospector/log/prospector.go index 4aa913a8b05..944d7b95130 100644 --- a/filebeat/prospector/log/prospector.go +++ b/filebeat/prospector/log/prospector.go @@ -142,7 +142,7 @@ func (p *Prospector) Run() { } } else { // Check if existing source on disk and state are the same. Remove if not the case. - newState := file.NewState(stat, state.Source, p.config.InputType) + newState := file.NewState(stat, state.Source, p.config.Type) if !newState.FileStateOS.IsSame(state.FileStateOS) { p.removeState(state) logp.Debug("prospector", "Remove state for file as file removed or renamed: %s", state.Source) @@ -284,7 +284,7 @@ func (p *Prospector) scan() { logp.Debug("prospector", "Check file for harvesting: %s", path) // Create new state for comparison - newState := file.NewState(info, path, p.config.InputType) + newState := file.NewState(info, path, p.config.Type) // Load last state lastState := p.states.FindPrevious(newState) diff --git a/filebeat/prospector/prospector.go b/filebeat/prospector/prospector.go index c405ceeb759..8c9ecb2b982 100644 --- a/filebeat/prospector/prospector.go +++ b/filebeat/prospector/prospector.go @@ -69,13 +69,13 @@ func (p *Prospector) initProspectorer(outlet channel.Outleter, states []file.Sta var prospectorer Prospectorer var err error - switch p.config.InputType { - case cfg.StdinInputType: + switch p.config.Type { + case cfg.StdinType: prospectorer, err = stdin.NewProspector(config, outlet) - case cfg.LogInputType: + case cfg.LogType: prospectorer, err = log.NewProspector(config, states, outlet, p.done) default: - return fmt.Errorf("invalid prospector type: %v. Change input_type", p.config.InputType) + return fmt.Errorf("invalid prospector type: %v. Change type", p.config.Type) } if err != nil { @@ -90,7 +90,7 @@ func (p *Prospector) initProspectorer(outlet channel.Outleter, states []file.Sta // Start starts the prospector func (p *Prospector) Start() { p.wg.Add(1) - logp.Info("Starting prospector of type: %v; id: %v ", p.config.InputType, p.ID()) + logp.Info("Starting prospector of type: %v; id: %v ", p.config.Type, p.ID()) onceWg := sync.WaitGroup{} if p.Once { diff --git a/filebeat/scripts/module/fileset/config/config.yml b/filebeat/scripts/module/fileset/config/config.yml index 56e3dfd95b8..0afd17317d4 100644 --- a/filebeat/scripts/module/fileset/config/config.yml +++ b/filebeat/scripts/module/fileset/config/config.yml @@ -1,4 +1,4 @@ -input_type: log +type: log paths: {{ range $i, $path := .paths }} - {{$path}} diff --git a/filebeat/tests/load/filebeat.yml b/filebeat/tests/load/filebeat.yml index 6c48d42fd55..ae0e861577c 100644 --- a/filebeat/tests/load/filebeat.yml +++ b/filebeat/tests/load/filebeat.yml @@ -1,6 +1,6 @@ filebeat: prospectors: - - input_type: log + - type: log paths: - /Users/ruflin/Dev/gopath/src/github.com/elastic/filebeat/tests/load/logs/* diff --git a/filebeat/tests/open-file-handlers/filebeat.yml b/filebeat/tests/open-file-handlers/filebeat.yml index f5d2c4228be..996f9d6a55b 100644 --- a/filebeat/tests/open-file-handlers/filebeat.yml +++ b/filebeat/tests/open-file-handlers/filebeat.yml @@ -1,7 +1,7 @@ filebeat.prospectors: # Reads logs generated by the generator containers -- input_type: log +- type: log paths: - /logfiles/* close_inactive: 1m @@ -9,12 +9,12 @@ filebeat.prospectors: clean_inactive: 5m # Reads docker container logs -#- input_type: log +#- type: log # paths: # - /var/lib/docker/containers/*/*-json.log # Reads system logs -#- input_type: log +#- type: log # paths: # - /var/log/* # diff --git a/filebeat/tests/system/config/filebeat.yml.j2 b/filebeat/tests/system/config/filebeat.yml.j2 index 22f8d195399..1ad3312c48e 100644 --- a/filebeat/tests/system/config/filebeat.yml.j2 +++ b/filebeat/tests/system/config/filebeat.yml.j2 @@ -5,7 +5,8 @@ filebeat.prospectors: {% set prospectors = true %} {% endif %} {% if prospectors %} -- input_type: {{input_type | default("log") }} +- type: {{type | default("log") }} + input_type: {{input_type_deprecated }} # Paths that should be crawled and fetched {% if path %}paths: - {{ path }}{% endif %} diff --git a/filebeat/tests/system/test_deprecated.py b/filebeat/tests/system/test_deprecated.py new file mode 100644 index 00000000000..0dd25fba926 --- /dev/null +++ b/filebeat/tests/system/test_deprecated.py @@ -0,0 +1,40 @@ +# coding=utf-8 + +from filebeat import BaseTest +import os +import codecs +import time + +""" +Test Harvesters +""" + + +class Test(BaseTest): + + def test_input_type_deprecated(self): + """ + Checks that harvesting works with deprecated input_type but message is outputted + """ + + self.render_config_template( + path=os.path.abspath(self.working_dir) + "/log/test.log", + input_type_deprecated="log", + scan_frequency="0.1s" + ) + os.mkdir(self.working_dir + "/log/") + + logfile = self.working_dir + "/log/test.log" + + with open(logfile, 'w') as f: + f.write("Hello world\n") + + filebeat = self.start_beat() + + # Let it read the file + self.wait_until( + lambda: self.output_has(lines=1), max_timeout=10) + + filebeat.check_kill_and_wait() + + assert self.log_contains("DEPRECATED: input_type prospector config is deprecated") diff --git a/filebeat/tests/system/test_prospector.py b/filebeat/tests/system/test_prospector.py index 75f0a552df9..6898af91624 100644 --- a/filebeat/tests/system/test_prospector.py +++ b/filebeat/tests/system/test_prospector.py @@ -79,7 +79,7 @@ def test_stdin(self): Test stdin input. Checks if reading is continued after the first read. """ self.render_config_template( - input_type="stdin" + type="stdin" ) proc = self.start_beat() @@ -115,7 +115,7 @@ def test_stdin_eof(self): Test that Filebeat works when stdin is closed. """ self.render_config_template( - input_type="stdin", + type="stdin", close_eof="true", ) diff --git a/filebeat/tests/system/test_reload.py b/filebeat/tests/system/test_reload.py index d6fdc3a7cea..084d2a01eab 100644 --- a/filebeat/tests/system/test_reload.py +++ b/filebeat/tests/system/test_reload.py @@ -7,7 +7,7 @@ prospectorConfigTemplate = """ -- input_type: log +- type: log paths: - {} scan_frequency: 1s diff --git a/filebeat/tests/system/test_shutdown.py b/filebeat/tests/system/test_shutdown.py index a87ac65be78..7eff12de8d6 100644 --- a/filebeat/tests/system/test_shutdown.py +++ b/filebeat/tests/system/test_shutdown.py @@ -153,7 +153,7 @@ def test_stopping_empty_path(self): """ prospector_raw = """ -- input_type: log +- type: log paths: [] """