Merge remote-tracking branch 'upstream/master' into feature/2.0-skip-…

…ci-build-label-support * upstream/master: [JJBB] Set shallow cloning to 10 (elastic#21409) docs: add link to release notes for 7.9.2 (elastic#21405) (elastic#21419) docs: Prepare Changelog for 7.9.2 (elastic#21229) (elastic#21403) fix: mark flaky tests (elastic#21300) fix: use a fixed version of setuptools (elastic#21393) Move Kubernetes events metricset to its own block in reference config (elastic#21407) [libbeat] Enable WriteAheadLimit in the disk queue (elastic#21391) docs: fix apt/yum formatting (elastic#21362) Fix shutdown tracking in s3 input (elastic#21380) [libbeat] Fix position writing in the disk queue Add UBI 8 image to the dependencies report (elastic#21374) Fix debug message to show actual SQS message ID (elastic#20614) [Elastic Agent] Rename *ConfigChange to PolicyChange (elastic#20779) [Elastic Agent] Add install/uninstall sub-command (elastic#21206) [Filebeat][httpjson] Make httpjson use cursor input when using date cursor (elastic#20751) feat: prepare release pipelines (elastic#21238) Add IP validation to Security module (elastic#21325)
v1v · Sep 30, 2020 · cb2bd71 · cb2bd71
2 parents 2bf8051 + cadd5ff
commit cb2bd71
Show file tree

Hide file tree

Showing 93 changed files with 1,870 additions and 679 deletions.
diff --git a/.ci/jobs/apm-beats-update.yml b/.ci/jobs/apm-beats-update.yml
@@ -48,7 +48,7 @@
             before: true
         prune: true
         shallow-clone: true
-        depth: 3
+        depth: 10
         do-not-fetch-tags: true
         submodule:
             disable: false

diff --git a/.ci/jobs/beats-release-minor-major.yml b/.ci/jobs/beats-release-minor-major.yml
@@ -0,0 +1,20 @@
+---
+- job:
+    name: Beats/Release/beats-release-minor-major
+    display-name: 'Prepare Major/minor Release'
+    description: 'Automate the steps to prepare a new Release branch'
+    view: Beats
+    project-type: pipeline
+    pipeline-scm:
+        script-path: release_scripts/pipeline-release-minor-major.groovy
+        scm:
+        - git:
+            url: git@github.com:elastic/ingest-dev.git
+            refspec: +refs/heads/*:refs/remotes/origin/* +refs/pull/*/head:refs/remotes/origin/pr/*
+            wipe-workspace: 'True'
+            name: origin
+            shallow-clone: true
+            credentials-id: f6c7695a-671e-4f4f-a331-acdce44ff9ba
+            reference-repo: /var/lib/jenkins/.git-references/ingest-dev.git
+            branches:
+                - master
diff --git a/.ci/jobs/beats-release-patch.yml b/.ci/jobs/beats-release-patch.yml
@@ -0,0 +1,20 @@
+---
+- job:
+    name: Beats/Release/beats-release-patch
+    display-name: 'Prepare Patch Release'
+    description: 'Automate the steps to prepare a new Patch'
+    view: Beats
+    project-type: pipeline
+    pipeline-scm:
+        script-path: release_scripts/pipeline-release-patch.groovy
+        scm:
+        - git:
+            url: git@github.com:elastic/ingest-dev.git
+            refspec: +refs/heads/*:refs/remotes/origin/* +refs/pull/*/head:refs/remotes/origin/pr/*
+            wipe-workspace: 'True'
+            name: origin
+            shallow-clone: true
+            credentials-id: f6c7695a-671e-4f4f-a331-acdce44ff9ba
+            reference-repo: /var/lib/jenkins/.git-references/ingest-dev.git
+            branches:
+                - master
diff --git a/.ci/jobs/beats-tester.yml b/.ci/jobs/beats-tester.yml
@@ -44,7 +44,7 @@
               before: true
           prune: true
           shallow-clone: true
-          depth: 3
+          depth: 10
           do-not-fetch-tags: true
           submodule:
               disable: false

diff --git a/.ci/jobs/beats-windows-mbp.yml b/.ci/jobs/beats-windows-mbp.yml
@@ -44,7 +44,7 @@
             before: true
           prune: true
           shallow-clone: true
-          depth: 4
+          depth: 10
           do-not-fetch-tags: true
           submodule:
             disable: false

diff --git a/.ci/jobs/beats.yml b/.ci/jobs/beats.yml
@@ -46,7 +46,7 @@
             before: true
         prune: true
         shallow-clone: true
-        depth: 3
+        depth: 10
         do-not-fetch-tags: true
         submodule:
             disable: false

diff --git a/.ci/jobs/folders.yml b/.ci/jobs/folders.yml
@@ -4,3 +4,8 @@
     name: Beats
     description: Beats
     project-type: folder
+
+- job:
+    name: Beats/Release
+    description: Jobs for release preparation
+    project-type: folder
diff --git a/.ci/jobs/golang-crossbuild-mbp.yml b/.ci/jobs/golang-crossbuild-mbp.yml
@@ -31,7 +31,7 @@
             before: true
           prune: true
           shallow-clone: true
-          depth: 4
+          depth: 10
           do-not-fetch-tags: true
           submodule:
             disable: false

diff --git a/.ci/jobs/packaging.yml b/.ci/jobs/packaging.yml
@@ -44,7 +44,7 @@
               before: true
           prune: true
           shallow-clone: true
-          depth: 3
+          depth: 10
           do-not-fetch-tags: true
           submodule:
               disable: false

diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
@@ -3,6 +3,47 @@
 :issue: https://github.com/elastic/beats/issues/
 :pull: https://github.com/elastic/beats/pull/
 
+[[release-notes-7.9.2]]
+=== Beats version 7.9.2
+https://github.com/elastic/beats/compare/v7.9.1...v7.9.2[View commits]
+
+==== Breaking changes
+
+*Affecting all Beats*
+
+- Autodiscover doesn't generate any configuration when a variable is missing. Previously it generated an incomplete configuration. {pull}20898[20898]
+
+==== Bugfixes
+
+*Affecting all Beats*
+
+- Explicitly detect missing variables in autodiscover configuration, log them at the debug level. {issue}20568[20568] {pull}20898[20898]
+- Fix `libbeat.output.write.bytes` and `libbeat.output.read.bytes` metrics of the Elasticsearch output. {issue}20752[20752] {pull}21197[21197]
+
+*Filebeat*
+
+- Provide backwards compatibility for the `set` processor when Elasticsearch is less than 7.9.0. {pull}20908[20908]
+- Fix an error updating file size being logged when EOF is reached. {pull}21048[21048]
+- Fix error when processing AWS Cloudtrail Digest logs. {pull}21086[21086] {issue}20943[20943]
+
+*Metricbeat*
+
+- The Kibana collector applies backoff when errored at getting usage stats {pull}20772[20772]
+- The `elasticsearch/index` metricset only requests wildcard expansion for hidden indices if the monitored Elasticsearch cluster supports it. {pull}20938[20938]
+- Fix panic index out of range error when getting AWS account name. {pull}21101[21101] {issue}21095[21095]
+- Handle missing counters in the application_pool metricset. {pull}21071[21071]
+
+*Functionbeat*
+
+- Do not need Google credentials if not required for the operation. {issue}17329[17329] {pull}21072[21072]
+- Fix dependency issues of GCP functions. {issue}20830[20830] {pull}21070[21070]
+
+==== Added
+
+*Affecting all Beats*
+
+- Add container ECS fields in kubernetes metadata. {pull}20984[20984]
+
 [[release-notes-7.9.1]]
 === Beats version 7.9.1
 https://github.com/elastic/beats/compare/v7.9.0...v7.9.1[View commits]

diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -23,6 +23,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Autodiscover doesn't generate any configuration when a variable is missing. Previously it generated an incomplete configuration. {pull}20898[20898]
 - Remove redundant `cloudfoundry.*.timestamp` fields. This value is set in `@timestamp`. {pull}21175[21175]
 - Allow embedding of CAs, Certificate of private keys for anything that support TLS in ouputs and inputs https://github.com/elastic/beats/pull/21179
+- Update to Golang 1.12.1. {pull}11330[11330]
+- Disable Alibaba Cloud and Tencent Cloud metadata providers by default. {pull}13812[12812]
 
 *Auditbeat*
 
@@ -74,6 +76,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix PANW field spelling "veredict" to "verdict" on event.action {pull}18808[18808]
 - Removed experimental modules `citrix`, `kaspersky`, `rapid7` and `tenable`. {pull}20706[20706]
 - Add support for GMT timezone offsets in `decode_cef`. {pull}20993[20993]
+- Fix parsing of Elasticsearch node name by `elasticsearch/slowlog` fileset. {pull}14547[14547]
 
 *Heartbeat*
 
@@ -356,6 +359,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix timestamp handling in remote_write. {pull}21166[21166]
 - Fix remote_write flaky test. {pull}21173[21173]
 - Visualization title fixes in aws, azure and googlecloud compute dashboards. {pull}21098[21098]
+- Add a switch to the driver definition on SQL module to use pretty names {pull}17378[17378]
 
 *Packetbeat*
 
@@ -367,6 +371,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 
 - Fix invalid IP addresses in DNS query results from Sysmon data. {issue}18432[18432] {pull}18436[18436]
 - Fields from Winlogbeat modules were not being included in index templates and patterns. {pull}18983[18983]
+- Add source.ip validation for event ID 4778 in the Security module. {issue}19627[19627]
 
 *Functionbeat*
 
@@ -596,6 +601,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add type and sub_type to panw panos fileset {pull}20912[20912]
 - Always attempt community_id processor on zeek module {pull}21155[21155]
 - Add related.hosts ecs field to all modules {pull}21160[21160]
+- Keep cursor state between httpjson input restarts {pull}20751[20751]
 
 *Heartbeat*
 

diff --git a/NOTICE.txt b/NOTICE.txt
@@ -10907,6 +10907,36 @@ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
 WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 
+--------------------------------------------------------------------------------
+Dependency : github.com/blakerouse/service
+Version: v1.1.1-0.20200924160513-057808572ffa
+Licence type (autodetected): Zlib
+--------------------------------------------------------------------------------
+
+Contents of probable licence file $GOMODCACHE/github.com/blakerouse/service@v1.1.1-0.20200924160513-057808572ffa/LICENSE:
+
+Copyright (c) 2015 Daniel Theophanes
+
+This software is provided 'as-is', without any express or implied
+warranty. In no event will the authors be held liable for any damages
+arising from the use of this software.
+
+Permission is granted to anyone to use this software for any purpose,
+including commercial applications, and to alter it and redistribute it
+freely, subject to the following restrictions:
+
+   1. The origin of this software must not be misrepresented; you must not
+   claim that you wrote the original software. If you use this software
+   in a product, an acknowledgment in the product documentation would be
+   appreciated but is not required.
+
+   2. Altered source versions must be plainly marked as such, and must not be
+   misrepresented as being the original software.
+
+   3. This notice may not be removed or altered from any source
+   distribution.
+
+
 --------------------------------------------------------------------------------
 Dependency : github.com/lib/pq
 Version: v1.1.2-0.20190507191818-2ff3cb3adc01
@@ -11843,6 +11873,37 @@ Contents of probable licence file $GOMODCACHE/github.com/oklog/ulid@v1.3.1/LICEN
    limitations under the License.
 
 
+--------------------------------------------------------------------------------
+Dependency : github.com/otiai10/copy
+Version: v1.2.0
+Licence type (autodetected): MIT
+--------------------------------------------------------------------------------
+
+Contents of probable licence file $GOMODCACHE/github.com/otiai10/copy@v1.2.0/LICENSE:
+
+The MIT License (MIT)
+
+Copyright (c) 2018 otiai10
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+
+
 --------------------------------------------------------------------------------
 Dependency : github.com/pierrre/gotestcover
 Version: v0.0.0-20160517101806-924dca7d15f0
@@ -34009,6 +34070,54 @@ Contents of probable licence file $GOMODCACHE/github.com/opencontainers/runtime-
    limitations under the License.
 
 
+--------------------------------------------------------------------------------
+Dependency : github.com/otiai10/curr
+Version: v1.0.0
+Licence type (autodetected): MIT
+--------------------------------------------------------------------------------
+
+Contents of probable licence file $GOMODCACHE/github.com/otiai10/curr@v1.0.0/LICENSE:
+
+The MIT License (MIT)
+
+Copyright (c) 2020 Hiromu Ochiai
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+
+
+--------------------------------------------------------------------------------
+Dependency : github.com/otiai10/mint
+Version: v1.3.1
+Licence type (autodetected): MIT
+--------------------------------------------------------------------------------
+
+Contents of probable licence file $GOMODCACHE/github.com/otiai10/mint@v1.3.1/LICENSE:
+
+Copyright 2017 otiai10 (Hiromu OCHIAI)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+
 --------------------------------------------------------------------------------
 Dependency : github.com/oxtoacart/bpool
 Version: v0.0.0-20150712133111-4e1c5567d7c2

diff --git a/auditbeat/module/file_integrity/eventreader_test.go b/auditbeat/module/file_integrity/eventreader_test.go
@@ -41,6 +41,7 @@ func init() {
 const ErrorSharingViolation syscall.Errno = 32
 
 func TestEventReader(t *testing.T) {
+	t.Skip("Flaky test: about 1/10 of bulds fails https://github.com/elastic/beats/issues/21302")
 	// Make dir to monitor.
 	dir, err := ioutil.TempDir("", "audit")
 	if err != nil {
@@ -240,6 +241,7 @@ func TestEventReader(t *testing.T) {
 }
 
 func TestRaces(t *testing.T) {
+	t.Skip("Flaky test: about 1/20 of bulds fails https://github.com/elastic/beats/issues/21303")
 	const (
 		fileMode os.FileMode = 0640
 		N                    = 100

diff --git a/deploy/kubernetes/metricbeat-kubernetes.yaml b/deploy/kubernetes/metricbeat-kubernetes.yaml
@@ -36,8 +36,6 @@ data:
                     - state_cronjob
                     - state_resourcequota
                     - state_statefulset
-                    # Uncomment this to get k8s events:
-                    #- event
                 - module: kubernetes
                   metricsets:
                     - apiserver
@@ -46,6 +44,10 @@ data:
                   ssl.certificate_authorities:
                     - /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
                   period: 30s
+                # Uncomment this to get k8s events:
+                #- module: kubernetes
+                #  metricsets:
+                #    - event
         # To enable hints based autodiscover uncomment this:
         #- type: kubernetes
         #  node: ${NODE_NAME}

diff --git a/deploy/kubernetes/metricbeat/metricbeat-daemonset-configmap.yaml b/deploy/kubernetes/metricbeat/metricbeat-daemonset-configmap.yaml
@@ -36,8 +36,6 @@ data:
                     - state_cronjob
                     - state_resourcequota
                     - state_statefulset
-                    # Uncomment this to get k8s events:
-                    #- event
                 - module: kubernetes
                   metricsets:
                     - apiserver
@@ -46,6 +44,10 @@ data:
                   ssl.certificate_authorities:
                     - /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
                   period: 30s
+                # Uncomment this to get k8s events:
+                #- module: kubernetes
+                #  metricsets:
+                #    - event
         # To enable hints based autodiscover uncomment this:
         #- type: kubernetes
         #  node: ${NODE_NAME}

diff --git a/dev-tools/dependencies-report b/dev-tools/dependencies-report
@@ -41,3 +41,14 @@ go list -m -json all $@ | go run go.elastic.co/go-licence-detector \
 		-noticeTemplate "$SRCPATH/notice/dependencies.csv.tmpl" \
 		-noticeOut "$outfile" \
 		-depsOut ""
+
+
+# Fill-in required values for UBI images
+# Check headers in $SRCPATH/notice/dependencies.csv.tmpl:
+# name,url,version,revision,license
+ubi8url='https://catalog.redhat.com/software/containers/ubi8/ubi-minimal/5c359a62bed8bd75a2c3fba8'
+ubi8source='https://oss-dependencies.elastic.co/redhat/ubi/ubi-minimal-8-source.tar.gz'
+ubilicense='Custom;https://www.redhat.com/licenses/EULA_Red_Hat_Universal_Base_Image_English_20190422.pdf,https://oss-dependencies.elastic.co/redhat/ubi/ubi-minimal-8-source.tar.gz'
+cat <<EOF >> $outfile
+Red Hat Universal Base Image,$ubi8url,8,,$ubilicense,$ubi8source
+EOF