Merge remote-tracking branch 'upstream/master' into madv-rss-usage-kn…

…own-issue

CHANGELOG.next.asciidoc

@@ -100,6 +100,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Rename bad ECS field name tracing.trace.id to trace.id in aws elb fileset. {pull}22571[22571]
 - Fix parsing issues with nested JSON payloads in Elasticsearch audit log fileset. {pull}22975[22975]
 - Rename `network.direction` values in crowdstrike/falcon to `ingress`/`egress`. {pull}23041[23041]
+- Rename `s3` input to `aws-s3` input. {pull}23469[23469]
 
 *Heartbeat*
 - Adds negative body match. {pull}20728[20728]
@@ -231,6 +232,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix CPU usage metrics on VMs with dynamic CPU config {pull}23154[23154]
 - Add FAQ entry for madvdontneed variable {pull}23429[23429]
 - Fix panic due to unhandled DeletedFinalStateUnknown in k8s OnDelete {pull}23419[23419]
+- Fix error loop with runaway CPU use when the Kafka output encounters some connection errors {pull}23484[23484]
 
 *Auditbeat*
 
@@ -268,6 +270,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix integer overflow in S3 offsets when collecting very large files. {pull}22523[22523]
 - Fix various processing errors in the Suricata module. {pull}23236[23236]
 - Fix CredentialsJSON unpacking for `gcp-pubsub` and `httpjson` inputs. {pull}23277[23277]
+- Change the `event.created` in Netflow events to be the time the event was created by Filebeat
+  to be consistent with ECS. {pull}23094[23094]
 
 *Filebeat*
 
@@ -364,6 +368,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add support for organization and custom prefix in AWS/CloudTrail fileset. {issue}23109[23109] {pull}23126[23126]
 - Simplify regex for organization custom prefix in AWS/CloudTrail fileset. {issue}23203[23203] {pull}23204[23204]
 - Fix syslog header parsing in infoblox module. {issue}23272[23272] {pull}23273[23273]
+- Fix concurrent modification exception in Suricata ingest node pipeline. {pull}23534[23534]
 
 *Heartbeat*
 
@@ -480,6 +485,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Remove io.time from windows {pull}22237[22237]
 - Change vsphere.datastore.capacity.used.pct value to betweeen 0 and 1. {pull}23148[23148]
 - Update config in `windows.yml` file. {issue}23027[23027]{pull}23327[23327]
+- Add stack monitoring section to elasticsearch module documentation {pull}#23286[23286]
 
 *Packetbeat*
 
@@ -802,6 +808,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add parsing of tcp flags to AWS vpcflow fileset {issue}228020[22820] {pull}23157[23157]
 - Added `alternative_host` option to google pubsub input {pull}23215[23215]
 - Added `encode_as` and `decode_as` options to httpjson along with pluggable encoders/decoders {pull}23478[23478]
+- Added `application/x-ndjson` as decode option for httpjson input {pull}23521[23521]
+- Added `application/x-www-form-urlencoded` as encode option for httpjson input {pull}23521[23521]
 
 *Heartbeat*
 

Jenkinsfile

@@ -80,6 +80,7 @@ pipeline {
             whenTrue(env.ONLY_DOCS == 'false') {
               cmd(label: "make check-python", script: "make check-python")
               cmd(label: "make check-go", script: "make check-go")
+              cmd(label: "make notice", script: "make notice")
               cmd(label: "Check for changes", script: "make check-no-changes")
             }
           }

NOTICE.txt

@@ -5757,6 +5757,38 @@ SOFTWARE.
 <http://www.opensource.org/licenses/mit-license.php>
 
 
+--------------------------------------------------------------------------------
+Dependency : github.com/eapache/go-resiliency
+Version: v1.2.0
+Licence type (autodetected): MIT
+--------------------------------------------------------------------------------
+
+Contents of probable licence file $GOMODCACHE/github.com/eapache/go-resiliency@v1.2.0/LICENSE:
+
+The MIT License (MIT)
+
+Copyright (c) 2014 Evan Huus
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+
+
 --------------------------------------------------------------------------------
 Dependency : github.com/eclipse/paho.mqtt.golang
 Version: v1.2.1-0.20200121105743-0d940dd29fd2
@@ -13781,11 +13813,11 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 --------------------------------------------------------------------------------
 Dependency : github.com/shirou/gopsutil
-Version: v2.19.11+incompatible
+Version: v3.20.12+incompatible
 Licence type (autodetected): BSD-3-Clause
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/shirou/gopsutil@v2.19.11+incompatible/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/shirou/gopsutil@v3.20.12+incompatible/LICENSE:
 
 gopsutil is distributed under BSD license reproduced below.
 
@@ -26207,38 +26239,6 @@ IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 
---------------------------------------------------------------------------------
-Dependency : github.com/eapache/go-resiliency
-Version: v1.2.0
-Licence type (autodetected): MIT
---------------------------------------------------------------------------------
-
-Contents of probable licence file $GOMODCACHE/github.com/eapache/go-resiliency@v1.2.0/LICENSE:
-
-The MIT License (MIT)
-
-Copyright (c) 2014 Evan Huus
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
-
-
-
 --------------------------------------------------------------------------------
 Dependency : github.com/eapache/go-xerial-snappy
 Version: v0.0.0-20180814174437-776d5712da21

filebeat/docs/aws-credentials-examples.asciidoc

@@ -3,7 +3,7 @@
 [source,yaml]
 ----
 filebeat.inputs:
-- type: s3
+- type: aws-s3
   queue_url: https://sqs.us-east-1.amazonaws.com/123/test-queue
   access_key_id: '<access_key_id>'
   secret_access_key: '<secret_access_key>'
@@ -15,7 +15,7 @@ or
 [source,yaml]
 ----
 filebeat.inputs:
-- type: s3
+- type: aws-s3
   queue_url: https://sqs.us-east-1.amazonaws.com/123/test-queue
   access_key_id: '${AWS_ACCESS_KEY_ID:""}'
   secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}'
@@ -27,7 +27,7 @@ filebeat.inputs:
 [source,yaml]
 ----
 filebeat.inputs:
-- type: s3
+- type: aws-s3
   queue_url: https://sqs.us-east-1.amazonaws.com/123/test-queue
   role_arn: arn:aws:iam::123456789012:role/test-mb
 ----
@@ -37,7 +37,7 @@ filebeat.inputs:
 [source,yaml]
 ----
 filebeat.inputs:
-- type: s3
+- type: aws-s3
   queue_url: https://sqs.us-east-1.amazonaws.com/123/test-queue
   credential_profile_name: test-fb
 ----

filebeat/docs/filebeat-options.asciidoc

@@ -63,6 +63,7 @@ subdirectories of a directory.
 You can configure {beatname_uc} to use the following inputs:
 
 * <<{beatname_lc}-input-aws-cloudwatch>>
+* <<{beatname_lc}-input-aws-s3>>
 * <<{beatname_lc}-input-azure-eventhub>>
 * <<{beatname_lc}-input-cloudfoundry>>
 * <<{beatname_lc}-input-container>>
@@ -76,7 +77,6 @@ You can configure {beatname_uc} to use the following inputs:
 * <<{beatname_lc}-input-netflow>>
 * <<{beatname_lc}-input-o365audit>>
 * <<{beatname_lc}-input-redis>>
-* <<{beatname_lc}-input-s3>>
 * <<{beatname_lc}-input-stdin>>
 * <<{beatname_lc}-input-syslog>>
 * <<{beatname_lc}-input-tcp>>

filebeat/docs/modules/cisco.asciidoc

@@ -388,7 +388,7 @@ will be found under `rsa.raw`. The default is false.
 
 The Cisco Umbrella fileset primarily focuses on reading CSV files from an S3 bucket using the filebeat S3 input.
 
-To configure Cisco Umbrella to log to a self-managed S3 bucket please follow the https://docs.umbrella.com/deployment-umbrella/docs/log-management[Cisco Umbrella User Guide], and the link:filebeat-input-s3.html[S3 input documentation] to setup the necessary Amazon SQS queue.  Retrieving logs from a Cisco-managed S3 bucket is not currently supported.
+To configure Cisco Umbrella to log to a self-managed S3 bucket please follow the https://docs.umbrella.com/deployment-umbrella/docs/log-management[Cisco Umbrella User Guide], and the link:filebeat-input-aws-s3.html[AWS S3 input documentation] to setup the necessary Amazon SQS queue.  Retrieving logs from a Cisco-managed S3 bucket is not currently supported.
 
 This fileset supports all 4 log types:
 - Proxy
@@ -409,7 +409,7 @@ Example config:
 - module: cisco
   umbrella:
     enabled: true
-    var.input: s3
+    var.input: aws-s3
     var.queue_url: https://sqs.us-east-1.amazonaws.com/ID/CiscoQueue
     var.access_key_id: 123456
     var.secret_access_key: PASSWORD

filebeat/docs/modules/mssql.asciidoc

@@ -20,22 +20,22 @@ include::../include/gs-link.asciidoc[]
 include::../include/configuring-intro.asciidoc[]
 
 The following example shows how to set paths in the +modules.d/{modulename}.yml+
-file to override the default paths for Træfik logs:
+file to override the default paths for MSSQL logs:
 
 ["source","yaml",subs="attributes"]
 -----
 - module: mssql
   log:
     enabled: true
-    var.paths: ["/var/opt/mssql/log/error*"]
+    var.paths: ['C:\Program Files\Microsoft SQL Server\MSSQL.150\MSSQL\LOG\ERRORLOG*']
 -----
 
 
 To specify the same settings at the command line, you use:
 
 ["source","sh",subs="attributes"]
 -----
--M "mssql.log.var.paths=[/var/opt/mssql/log/error*]"
+-M "mssql.log.var.paths=['C:\Program Files\Microsoft SQL Server\MSSQL.150\MSSQL\LOG\ERRORLOG*']"
 -----
 
 //set the fileset name used in the included example

go.mod

@@ -57,6 +57,7 @@ require (
 	github.com/dop251/goja v0.0.0-20200831102558-9af81ddcf0e1
 	github.com/dop251/goja_nodejs v0.0.0-20171011081505-adff31b136e6
 	github.com/dustin/go-humanize v1.0.0
+	github.com/eapache/go-resiliency v1.2.0
 	github.com/eclipse/paho.mqtt.golang v1.2.1-0.20200121105743-0d940dd29fd2
 	github.com/elastic/ecs v1.6.0
 	github.com/elastic/elastic-agent-client/v7 v7.0.0-20200709172729-d43b7ad5833a