Merge remote-tracking branch 'elastic/master' into add-ecs-fields

.ci/packaging.groovy

@@ -212,6 +212,7 @@ pipeline {
                   'x-pack/filebeat',
                   'x-pack/heartbeat',
                   'x-pack/metricbeat',
+                  'x-pack/osquerybeat',
                   'x-pack/packetbeat'
                 )
               }

.ci/packer_cache.sh

@@ -45,6 +45,8 @@ function dockerPullImages() {
   docker.elastic.co/kibana/kibana:${SNAPSHOT}
   docker.elastic.co/logstash/logstash:${SNAPSHOT}
   docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-arm
+  docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-armhf
+  docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-armel
   docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-base-arm-debian9
   docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-darwin
   docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-main

.go-version

@@ -1 +1 @@
-1.15.12
+1.16.4

CHANGELOG-developer.next.asciidoc

@@ -54,6 +54,7 @@ The list below covers the major changes between 7.0.0-rc2 and master only.
 - Make implementing `Close` required for `reader.Reader` interfaces. {pull}20455[20455]
 - Remove `NumCPU` as clients should update the CPU count on the fly in case of config changes in a VM. {pull}23154[23154]
 - Remove Metricbeat EventFetcher and EventsFetcher interface. Use the reporter interface instead. {pull}25093[25093]
+- Update Darwin build image to a debian 10 base that increases the MacOS SDK and minimum supported version used in build to 10.14. {issue}24193[24193]
 
 ==== Bugfixes
 
@@ -113,4 +114,5 @@ The list below covers the major changes between 7.0.0-rc2 and master only.
 - Update Go version to 1.15.9. {pull}24442[24442]
 - Update Go version to 1.15.10. {pull}24606[24606]
 - Update Go version to 1.15.12. {pull}25629[25629]
+- Update Go version to 1.16.4. {issue}25346[25346] {pull}25671[25671]
 - Add sorting to array fields for generated data files (*-generated.json) {pull}25320[25320]

CHANGELOG.asciidoc

@@ -3,6 +3,21 @@
 :issue: https://github.com/elastic/beats/issues/
 :pull: https://github.com/elastic/beats/pull/
 
+[[release-notes-7.13.1]]
+=== Beats version 7.13.1
+https://github.com/elastic/beats/compare/v7.13.0...v7.13.1[View commits]
+
+==== Bugfixes
+
+*Auditbeat*
+
+- Mitigate deadlock is aws-s3 input when SQS visibility timeout is exceeded. {issue}25750[25750]
+- Fix httpjson cursor override with empty values by adding `ignore_empty_value` option. {pull}25802[25802]
+
+*Filebeat*
+
+- Improve inode reuse handling by removing state for removed files more eagerly from the internal state table in the logs inputs. {pull}25756[25756]
+
 [[release-notes-7.13.0]]
 === Beats version 7.13.0
 https://github.com/elastic/beats/compare/v7.12.1...v7.13.0[View commits]

CHANGELOG.next.asciidoc

@@ -32,6 +32,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Update to ECS 1.7.0. {pull}22571[22571]
 - Add support for SCRAM-SHA-512 and SCRAM-SHA-256 in Kafka output. {pull}12867[12867]
 - Remove id_field_data {pull}25239[25239]
+- Removed beats central management {pull}25696[25696], {issue}23908[23908]
+- MacOSX minimum supported version set to 10.14 {issue}24193{24193}
 
 *Auditbeat*
 
@@ -273,9 +275,6 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix integer overflow in S3 offsets when collecting very large files. {pull}22523[22523]
 - Fix CredentialsJSON unpacking for `gcp-pubsub` and `httpjson` inputs. {pull}23277[23277]
 - Fix issue with m365_defender, when parsing incidents that has no alerts attached: {pull}25421[25421]
-- Improve inode reuse handling by removing state for removed files more eagerly from the internal state table in the logs inputs. {pull}25756[25756]
-- Mitigate deadlock is aws-s3 input when SQS visibility timeout is exceeded. {issue}25750[25750]
-- Fix httpjson cursor override with empty values by adding `ignore_empty_value` option. {pull}25802[25802]
 
 *Filebeat*
 
@@ -807,6 +806,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - In Cisco Umbrella fileset add users from cisco.umbrella.identities to related.user. {pull}25776[25776]
 - Add fingerprint processor to generate fixed ids for `google_workspace` events. {pull}25841[25841]
 - Update PanOS module to parse HIP Match logs. {issue}24350[24350] {pull}25686[25686]
+- Enhance GCP module to populate orchestrator.* fields for GKE / K8S logs {pull}25368[25368]
 
 *Heartbeat*
 
@@ -937,6 +937,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Refactor state_* metricsets to share response from endpoint. {pull}25640[25640]
 - Add server id to zookeeper events. {pull}25550[25550]
 - Add additional network metrics to docker/network {pull}25354[25354]
+- Migrate ec2 metricsets to use cloudwatch input. {pull}25924[25924]
 - Reduce number of requests done by kubernetes metricsets to kubelet. {pull}25782[25782]
 
 *Packetbeat*

Jenkinsfile

@@ -204,8 +204,9 @@ def runLinting() {
   }
   mapParallelTasks['default'] = {
                                 cmd(label: "make check-python", script: "make check-python")
-                                cmd(label: "make check-go", script: "make check-go")
                                 cmd(label: "make notice", script: "make notice")
+                                // `make check-go` must follow `make notice` to ensure that the lint checks can be satisfied
+                                cmd(label: "make check-go", script: "make check-go")
                                 cmd(label: "Check for changes", script: "make check-no-changes")
                               }
 

Makefile

@@ -110,6 +110,7 @@ check-go:
 ## ccheck-no-changes : Check there is no local changes.
 .PHONY: check-no-changes
 check-no-changes:
+	@go mod tidy
 	@git diff | cat
 	@git update-index --refresh
 	@git diff-index --exit-code HEAD --

NOTICE.txt

@@ -37760,37 +37760,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 
---------------------------------------------------------------------------------
-Dependency : github.com/poy/eachers
-Version: v0.0.0-20181020210610-23942921fe77
-Licence type (autodetected): MIT
---------------------------------------------------------------------------------
-
-Contents of probable licence file $GOMODCACHE/github.com/poy/eachers@v0.0.0-20181020210610-23942921fe77/LICENSE.md:
-
-The MIT License (MIT)
-
-Copyright (c) 2016 Andrew Poydence
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
-
-
 --------------------------------------------------------------------------------
 Dependency : github.com/prometheus/client_golang
 Version: v1.1.1-0.20190913103102-20428fa0bffc

auditbeat/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.15.12
+FROM golang:1.16.4
 
 RUN \
     apt-get update \

auditbeat/cmd/root.go

@@ -35,7 +35,7 @@ const (
 	Name = "auditbeat"
 
 	// ecsVersion specifies the version of ECS that Auditbeat is implementing.
-	ecsVersion = "1.9.0"
+	ecsVersion = "1.10.0"
 )
 
 // RootCmd for running auditbeat.

auditbeat/docs/fields.asciidoc

@@ -3696,6 +3696,56 @@ example: docker
 
 --
 
+[float]
+=== data_stream
+
+The data_stream fields take part in defining the new data stream naming scheme.
+In the new data stream naming scheme the value of the data stream fields combine to the name of the actual data stream in the following manner: `{data_stream.type}-{data_stream.dataset}-{data_stream.namespace}`. This means the fields can only contain characters that are valid as part of names of data streams. More details about this can be found in this https://www.elastic.co/blog/an-introduction-to-the-elastic-data-stream-naming-scheme[blog post].
+An Elasticsearch data stream consists of one or more backing indices, and a data stream name forms part of the backing indices names. Due to this convention, data streams must also follow index naming restrictions. For example, data stream names cannot include `\`, `/`, `*`, `?`, `"`, `<`, `>`, `|`, ` ` (space character), `,`, or `#`. Please see the Elasticsearch reference for additional https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-create-index.html#indices-create-api-path-params[restrictions].
+
+
+*`data_stream.dataset`*::
++
+--
+The field can contain anything that makes sense to signify the source of the data.
+Examples include `nginx.access`, `prometheus`, `endpoint` etc. For data streams that otherwise fit, but that do not have dataset set we use the value "generic" for the dataset value. `event.dataset` should have the same value as `data_stream.dataset`.
+Beyond the Elasticsearch data stream naming criteria noted above, the `dataset` value has additional restrictions:
+  * Must not contain `-`
+  * No longer than 100 characters
+
+type: constant_keyword
+
+example: nginx.access
+
+--
+
+*`data_stream.namespace`*::
++
+--
+A user defined namespace. Namespaces are useful to allow grouping of data.
+Many users already organize their indices this way, and the data stream naming scheme now provides this best practice as a default. Many users will populate this field with `default`. If no value is used, it falls back to `default`.
+Beyond the Elasticsearch index naming criteria noted above, `namespace` value has the additional restrictions:
+  * Must not contain `-`
+  * No longer than 100 characters
+
+type: constant_keyword
+
+example: production
+
+--
+
+*`data_stream.type`*::
++
+--
+An overarching type for the data stream.
+Currently allowed values are "logs" and "metrics". We expect to also add "traces" and "synthetics" in the near future.
+
+type: constant_keyword
+
+example: logs
+
+--
+
 [float]
 === destination
 
@@ -7432,6 +7482,105 @@ type: keyword
 
 --
 
+[float]
+=== orchestrator
+
+Fields that describe the resources which container orchestrators manage or act upon.
+
+
+*`orchestrator.api_version`*::
++
+--
+API version being used to carry out the action
+
+type: keyword
+
+example: v1beta1
+
+--
+
+*`orchestrator.cluster.name`*::
++
+--
+Name of the cluster.
+
+type: keyword
+
+--
+
+*`orchestrator.cluster.url`*::
++
+--
+URL of the API used to manage the cluster.
+
+type: keyword
+
+--
+
+*`orchestrator.cluster.version`*::
++
+--
+The version of the cluster.
+
+type: keyword
+
+--
+
+*`orchestrator.namespace`*::
++
+--
+Namespace in which the action is taking place.
+
+type: keyword
+
+example: kube-system
+
+--
+
+*`orchestrator.organization`*::
++
+--
+Organization affected by the event (for multi-tenant orchestrator setups).
+
+type: keyword
+
+example: elastic
+
+--
+
+*`orchestrator.resource.name`*::
++
+--
+Name of the resource being acted upon.
+
+type: keyword
+
+example: test-pod-cdcws
+
+--
+
+*`orchestrator.resource.type`*::
++
+--
+Type of resource being acted upon.
+
+type: keyword
+
+example: service
+
+--
+
+*`orchestrator.type`*::
++
+--
+Orchestrator cluster type (e.g. kubernetes, nomad or cloudfoundry).
+
+type: keyword
+
+example: kubernetes
+
+--
+
 [float]
 === organization
 

dev-tools/mage/crossbuild.go

@@ -197,13 +197,22 @@ func crossBuildImage(platform string) (string, error) {
 	tagSuffix := "main"
 
 	switch {
-	case strings.HasPrefix(platform, "darwin"):
-		tagSuffix = "darwin"
-	case strings.HasPrefix(platform, "linux/arm"):
+	case platform == "darwin/amd64":
+		tagSuffix = "darwin-debian10"
+	case platform == "darwin/arm64":
+		tagSuffix = "darwin-arm64-debian10"
+	case platform == "linux/arm64":
 		tagSuffix = "arm"
+		// when it runs on a ARM64 host/worker.
 		if runtime.GOARCH == "arm64" {
 			tagSuffix = "base-arm-debian9"
 		}
+	case platform == "linux/armv5":
+		tagSuffix = "armel"
+	case platform == "linux/armv6":
+		tagSuffix = "armel"
+	case platform == "linux/armv7":
+		tagSuffix = "armhf"
 	case strings.HasPrefix(platform, "linux/mips"):
 		tagSuffix = "mips"
 	case strings.HasPrefix(platform, "linux/ppc"):

dev-tools/mage/gotest_test.go

@@ -364,7 +364,7 @@ func TestGoTest_Helper_WithWrongPanic(t *testing.T) {
 	t.Run("setup failing go-routine", func(t *testing.T) {
 		go func() {
 			time.Sleep(1 * time.Second)
-			t.Fatal("oops")
+			t.Error("oops")
 		}()
 	})
 

filebeat/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.15.12
+FROM golang:1.16.4
 
 RUN \
     apt-get update \